mirror of
https://github.com/crewAIInc/crewAI.git
synced 2026-01-10 00:28:31 +00:00
21 lines
881 B
Python
21 lines
881 B
Python
from crewai_tools.rag.chunkers.base_chunker import BaseChunker
|
|
from typing import List, Optional
|
|
|
|
|
|
class WebsiteChunker(BaseChunker):
|
|
def __init__(self, chunk_size: int = 2500, chunk_overlap: int = 250, separators: Optional[List[str]] = None, keep_separator: bool = True):
|
|
if separators is None:
|
|
separators = [
|
|
"\n\n\n", # Major section breaks
|
|
"\n\n", # Paragraph breaks
|
|
"\n", # Line breaks
|
|
". ", # Sentence endings
|
|
"! ", # Exclamation endings
|
|
"? ", # Question endings
|
|
"; ", # Semicolon breaks
|
|
", ", # Comma breaks
|
|
" ", # Word breaks
|
|
"", # Character level
|
|
]
|
|
super().__init__(chunk_size, chunk_overlap, separators, keep_separator)
|