From 7a21564743f6f9550e49f181bcdc7c02a0eacff6 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Tue, 4 Mar 2025 07:28:09 +0000 Subject: [PATCH] Fix #2271: Handle SQLite3 version check gracefully for ChromaDB Co-Authored-By: Joe Moura --- .../knowledge/storage/knowledge_storage.py | 45 ++++++++++------- src/crewai/memory/storage/rag_storage.py | 49 ++++++++++++------- .../utilities/embedding_configurator.py | 31 ++++++++++-- .../utilities/test_embedding_configurator.py | 32 ++++++++++++ 4 files changed, 119 insertions(+), 38 deletions(-) create mode 100644 tests/utilities/test_embedding_configurator.py diff --git a/src/crewai/knowledge/storage/knowledge_storage.py b/src/crewai/knowledge/storage/knowledge_storage.py index 72240e2b6..31c8cbd9f 100644 --- a/src/crewai/knowledge/storage/knowledge_storage.py +++ b/src/crewai/knowledge/storage/knowledge_storage.py @@ -83,28 +83,37 @@ class KnowledgeStorage(BaseKnowledgeStorage): raise Exception("Collection not initialized") def initialize_knowledge_storage(self): - base_path = os.path.join(db_storage_path(), "knowledge") - chroma_client = chromadb.PersistentClient( - path=base_path, - settings=Settings(allow_reset=True), - ) - - self.app = chroma_client - try: - collection_name = ( - f"knowledge_{self.collection_name}" - if self.collection_name - else "knowledge" + base_path = os.path.join(db_storage_path(), "knowledge") + chroma_client = chromadb.PersistentClient( + path=base_path, + settings=Settings(allow_reset=True), ) - if self.app: - self.collection = self.app.get_or_create_collection( - name=collection_name, embedding_function=self.embedder + + self.app = chroma_client + + try: + collection_name = ( + f"knowledge_{self.collection_name}" + if self.collection_name + else "knowledge" ) + if self.app: + self.collection = self.app.get_or_create_collection( + name=collection_name, embedding_function=self.embedder + ) + else: + raise Exception("Vector Database Client not initialized") + except Exception: + raise Exception("Failed to create or get collection") + except RuntimeError as e: + if "unsupported version of sqlite3" in str(e).lower(): + # Log a warning but continue without ChromaDB + logging.warning(f"ChromaDB requires SQLite3 >= 3.35.0. Current version is too old. Some features may be limited. Error: {e}") + self.app = None + self.collection = None else: - raise Exception("Vector Database Client not initialized") - except Exception: - raise Exception("Failed to create or get collection") + raise def reset(self): base_path = os.path.join(db_storage_path(), KNOWLEDGE_DIRECTORY) diff --git a/src/crewai/memory/storage/rag_storage.py b/src/crewai/memory/storage/rag_storage.py index fd4c77838..c74b96692 100644 --- a/src/crewai/memory/storage/rag_storage.py +++ b/src/crewai/memory/storage/rag_storage.py @@ -60,26 +60,41 @@ class RAGStorage(BaseRAGStorage): self.embedder_config = configurator.configure_embedder(self.embedder_config) def _initialize_app(self): - import chromadb - from chromadb.config import Settings - - self._set_embedder_config() - chroma_client = chromadb.PersistentClient( - path=self.path if self.path else self.storage_file_name, - settings=Settings(allow_reset=self.allow_reset), - ) - - self.app = chroma_client - try: - self.collection = self.app.get_collection( - name=self.type, embedding_function=self.embedder_config - ) - except Exception: - self.collection = self.app.create_collection( - name=self.type, embedding_function=self.embedder_config + import chromadb + from chromadb.config import Settings + + self._set_embedder_config() + if self.embedder_config is None: + # ChromaDB is not available, skip initialization + self.app = None + self.collection = None + return + + chroma_client = chromadb.PersistentClient( + path=self.path if self.path else self.storage_file_name, + settings=Settings(allow_reset=self.allow_reset), ) + self.app = chroma_client + + try: + self.collection = self.app.get_collection( + name=self.type, embedding_function=self.embedder_config + ) + except Exception: + self.collection = self.app.create_collection( + name=self.type, embedding_function=self.embedder_config + ) + except RuntimeError as e: + if "unsupported version of sqlite3" in str(e).lower(): + # Log a warning but continue without ChromaDB + logging.warning(f"ChromaDB requires SQLite3 >= 3.35.0. Current version is too old. Some features may be limited. Error: {e}") + self.app = None + self.collection = None + else: + raise + def _sanitize_role(self, role: str) -> str: """ Sanitizes agent roles to ensure valid directory names. diff --git a/src/crewai/utilities/embedding_configurator.py b/src/crewai/utilities/embedding_configurator.py index e523b60f0..6c5514739 100644 --- a/src/crewai/utilities/embedding_configurator.py +++ b/src/crewai/utilities/embedding_configurator.py @@ -1,12 +1,31 @@ import os +import logging from typing import Any, Dict, Optional, cast -from chromadb import Documents, EmbeddingFunction, Embeddings -from chromadb.api.types import validate_embedding_function +# Import chromadb conditionally to handle SQLite3 version errors +try: + from chromadb import Documents, EmbeddingFunction, Embeddings + from chromadb.api.types import validate_embedding_function + CHROMADB_AVAILABLE = True +except RuntimeError as e: + if "unsupported version of sqlite3" in str(e).lower(): + logging.warning(f"ChromaDB requires SQLite3 >= 3.35.0. Current version is too old. Some features may be limited. Error: {e}") + CHROMADB_AVAILABLE = False + # Define placeholder types for type hints + Documents = Any + EmbeddingFunction = Any + Embeddings = Any + validate_embedding_function = lambda x: x # noqa: E731 + else: + raise class EmbeddingConfigurator: def __init__(self): + if not CHROMADB_AVAILABLE: + self.embedding_functions = {} + return + self.embedding_functions = { "openai": self._configure_openai, "azure": self._configure_azure, @@ -24,8 +43,11 @@ class EmbeddingConfigurator: def configure_embedder( self, embedder_config: Optional[Dict[str, Any]] = None, - ) -> EmbeddingFunction: + ) -> Optional[EmbeddingFunction]: """Configures and returns an embedding function based on the provided config.""" + if not CHROMADB_AVAILABLE: + return None + if embedder_config is None: return self._create_default_embedding_function() @@ -47,6 +69,9 @@ class EmbeddingConfigurator: @staticmethod def _create_default_embedding_function(): + if not CHROMADB_AVAILABLE: + return None + from chromadb.utils.embedding_functions.openai_embedding_function import ( OpenAIEmbeddingFunction, ) diff --git a/tests/utilities/test_embedding_configurator.py b/tests/utilities/test_embedding_configurator.py new file mode 100644 index 000000000..434d91805 --- /dev/null +++ b/tests/utilities/test_embedding_configurator.py @@ -0,0 +1,32 @@ +import unittest +from unittest.mock import patch, MagicMock + +class TestEmbeddingConfigurator(unittest.TestCase): + @patch('crewai.utilities.embedding_configurator.CHROMADB_AVAILABLE', False) + def test_embedding_configurator_with_chromadb_unavailable(self): + from crewai.utilities.embedding_configurator import EmbeddingConfigurator + + # Create an instance of EmbeddingConfigurator + configurator = EmbeddingConfigurator() + + # Verify that embedding_functions is empty + self.assertEqual(configurator.embedding_functions, {}) + + # Verify that configure_embedder returns None + self.assertIsNone(configurator.configure_embedder()) + + @patch('crewai.utilities.embedding_configurator.CHROMADB_AVAILABLE', True) + def test_embedding_configurator_with_chromadb_available(self): + from crewai.utilities.embedding_configurator import EmbeddingConfigurator + + # Create an instance of EmbeddingConfigurator + configurator = EmbeddingConfigurator() + + # Verify that embedding_functions is not empty + self.assertNotEqual(configurator.embedding_functions, {}) + + # Mock the _create_default_embedding_function method + configurator._create_default_embedding_function = MagicMock(return_value="mock_embedding_function") + + # Verify that configure_embedder returns the mock embedding function + self.assertEqual(configurator.configure_embedder(), "mock_embedding_function")