mirror of
https://github.com/crewAIInc/crewAI.git
synced 2026-01-09 16:18:30 +00:00
Address PR feedback: Improve documentation and add edge case tests
Co-Authored-By: Joe Moura <joao@crewai.com>
This commit is contained in:
@@ -144,10 +144,11 @@ class RAGStorage(BaseRAGStorage):
|
|||||||
Split text into chunks to avoid token limits.
|
Split text into chunks to avoid token limits.
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
text: Text to chunk
|
text: Input text to chunk.
|
||||||
|
|
||||||
Returns:
|
Returns:
|
||||||
List of text chunks
|
List[str]: A list of chunked text segments, adhering to defined size and overlap.
|
||||||
|
Empty list if input text is empty.
|
||||||
"""
|
"""
|
||||||
if not text:
|
if not text:
|
||||||
return []
|
return []
|
||||||
@@ -156,14 +157,25 @@ class RAGStorage(BaseRAGStorage):
|
|||||||
return [text]
|
return [text]
|
||||||
|
|
||||||
chunks = []
|
chunks = []
|
||||||
for i in range(0, len(text), MEMORY_CHUNK_SIZE - MEMORY_CHUNK_OVERLAP):
|
start_indices = range(0, len(text), MEMORY_CHUNK_SIZE - MEMORY_CHUNK_OVERLAP)
|
||||||
|
for i in start_indices:
|
||||||
chunk = text[i:i + MEMORY_CHUNK_SIZE]
|
chunk = text[i:i + MEMORY_CHUNK_SIZE]
|
||||||
if chunk: # Only add non-empty chunks
|
if chunk: # Only add non-empty chunks
|
||||||
chunks.append(chunk)
|
chunks.append(chunk)
|
||||||
|
|
||||||
return chunks
|
return chunks
|
||||||
|
|
||||||
def _generate_embedding(self, text: str, metadata: Dict[str, Any]) -> None: # type: ignore
|
def _generate_embedding(self, text: str, metadata: Optional[Dict[str, Any]] = None) -> Optional[None]:
|
||||||
|
"""
|
||||||
|
Generate embeddings for text and add to collection.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
text: Input text to generate embeddings for.
|
||||||
|
metadata: Optional metadata to associate with the embeddings.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
None if successful, None if text is empty.
|
||||||
|
"""
|
||||||
if not hasattr(self, "app") or not hasattr(self, "collection"):
|
if not hasattr(self, "app") or not hasattr(self, "collection"):
|
||||||
self._initialize_app()
|
self._initialize_app()
|
||||||
|
|
||||||
|
|||||||
@@ -50,3 +50,37 @@ def test_memory_with_large_input(short_term_memory):
|
|||||||
result = short_term_memory.search(large_input[:100], score_threshold=0.01)
|
result = short_term_memory.search(large_input[:100], score_threshold=0.01)
|
||||||
assert result[0]["context"] == large_input
|
assert result[0]["context"] == large_input
|
||||||
assert result[0]["metadata"]["agent"] == "test_agent"
|
assert result[0]["metadata"]["agent"] == "test_agent"
|
||||||
|
|
||||||
|
|
||||||
|
def test_memory_with_empty_input(short_term_memory):
|
||||||
|
"""Test that memory correctly handles empty input strings"""
|
||||||
|
empty_input = ""
|
||||||
|
|
||||||
|
with patch.object(
|
||||||
|
short_term_memory.storage, '_chunk_text',
|
||||||
|
return_value=[]
|
||||||
|
) as mock_chunk_text:
|
||||||
|
with patch.object(
|
||||||
|
short_term_memory.storage.collection, 'add'
|
||||||
|
) as mock_add:
|
||||||
|
short_term_memory.save(value=empty_input, agent="test_agent")
|
||||||
|
|
||||||
|
mock_chunk_text.assert_called_with(empty_input)
|
||||||
|
mock_add.assert_not_called()
|
||||||
|
|
||||||
|
|
||||||
|
def test_memory_with_exact_chunk_size_input(short_term_memory):
|
||||||
|
"""Test that memory correctly handles inputs that match chunk size exactly"""
|
||||||
|
exact_size_input = "x" * MEMORY_CHUNK_SIZE
|
||||||
|
|
||||||
|
with patch.object(
|
||||||
|
short_term_memory.storage, '_chunk_text',
|
||||||
|
return_value=[exact_size_input]
|
||||||
|
) as mock_chunk_text:
|
||||||
|
with patch.object(
|
||||||
|
short_term_memory.storage.collection, 'add'
|
||||||
|
) as mock_add:
|
||||||
|
short_term_memory.save(value=exact_size_input, agent="test_agent")
|
||||||
|
|
||||||
|
mock_chunk_text.assert_called_with(exact_size_input)
|
||||||
|
assert mock_add.call_count == 1
|
||||||
|
|||||||
Reference in New Issue
Block a user