base

The base classes for text splitter.

Classes:

• TextSplitterBase –

  The base text splitter class.

TextSplitterBase

TextSplitterBase(
    splitter_name: str = '',
    chunk_size: int = 500,
    chunk_overlap: int = 100,
)

The base text splitter class.

Methods:

• split –

  Split a text into chunks.

Source code in src/rago/retrieval/text_splitter/base.py

def __init__(
    self,
    splitter_name: str = '',
    chunk_size: int = 500,
    chunk_overlap: int = 100,
) -> None:
    """Initialize the text splitter class."""
    self.chunk_size = chunk_size or self.default_chunk_size
    self.chunk_overlap = chunk_overlap or self.default_chunk_overlap
    self.splitter_name = splitter_name or self.default_splitter_name

    self._validate()
    self._setup()

split abstractmethod

split(text: str) -> Iterable[str]

Split a text into chunks.

Source code in src/rago/retrieval/text_splitter/base.py

@abstractmethod
def split(self, text: str) -> Iterable[str]:
    """Split a text into chunks."""
    return []