mirror of
https://github.com/crewAIInc/crewAI.git
synced 2026-01-11 00:58:30 +00:00
e16606672a
git-subtree-dir: packages/tools git-subtree-split: 78317b9c127f18bd040c1d77e3c0840cdc9a5b38
50 lines
2.1 KiB
Python
50 lines
2.1 KiB
Python
from crewai_tools.rag.chunkers.base_chunker import BaseChunker
|
|
from typing import List, Optional
|
|
|
|
|
|
class CsvChunker(BaseChunker):
|
|
def __init__(self, chunk_size: int = 1200, chunk_overlap: int = 100, separators: Optional[List[str]] = None, keep_separator: bool = True):
|
|
if separators is None:
|
|
separators = [
|
|
"\nRow ", # Row boundaries (from CSVLoader format)
|
|
"\n", # Line breaks
|
|
" | ", # Column separators
|
|
", ", # Comma separators
|
|
" ", # Word breaks
|
|
"", # Character level
|
|
]
|
|
super().__init__(chunk_size, chunk_overlap, separators, keep_separator)
|
|
|
|
|
|
class JsonChunker(BaseChunker):
|
|
def __init__(self, chunk_size: int = 2000, chunk_overlap: int = 200, separators: Optional[List[str]] = None, keep_separator: bool = True):
|
|
if separators is None:
|
|
separators = [
|
|
"\n\n", # Object/array boundaries
|
|
"\n", # Line breaks
|
|
"},", # Object endings
|
|
"],", # Array endings
|
|
", ", # Property separators
|
|
": ", # Key-value separators
|
|
" ", # Word breaks
|
|
"", # Character level
|
|
]
|
|
super().__init__(chunk_size, chunk_overlap, separators, keep_separator)
|
|
|
|
|
|
class XmlChunker(BaseChunker):
|
|
def __init__(self, chunk_size: int = 2500, chunk_overlap: int = 250, separators: Optional[List[str]] = None, keep_separator: bool = True):
|
|
if separators is None:
|
|
separators = [
|
|
"\n\n", # Element boundaries
|
|
"\n", # Line breaks
|
|
">", # Tag endings
|
|
". ", # Sentence endings (for text content)
|
|
"! ", # Exclamation endings
|
|
"? ", # Question endings
|
|
", ", # Comma separators
|
|
" ", # Word breaks
|
|
"", # Character level
|
|
]
|
|
super().__init__(chunk_size, chunk_overlap, separators, keep_separator)
|