andre/crewAI
1
0
Fork 0
mirror of https://github.com/crewAIInc/crewAI.git synced 2026-01-27 09:08:14 +00:00
Code Issues Actions 5 Packages Projects Releases Wiki Activity

Fix #2324: Make chromadb an optional dependency to support Alpine Linux

Browse Source 
Co-Authored-By: Joe Moura <joao@crewai.com>
This commit is contained in:
Devin AI
2025-03-11 05:44:05 +00:00
parent 41a670166a
commit 36e7236ffe
6 changed files with 275 additions and 118 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
README.md
View File

@@ -136,6 +136,14 @@ pip install 'crewai[tools]'
``` ```
The command above installs the basic package and also adds extra components which require more dependencies to function. The command above installs the basic package and also adds extra components which require more dependencies to function.
For vector storage and RAG capabilities, install with the chromadb extra:
```shell
pip install 'crewai[chromadb]'
```
Note: If you're using Alpine Linux or other environments where onnxruntime is not available, you can still use CrewAI without the chromadb dependency, but with limited vector storage functionality.
### Troubleshooting Dependencies ### Troubleshooting Dependencies
If you encounter issues during installation or usage, here are some common solutions: If you encounter issues during installation or usage, here are some common solutions:

4
pyproject.toml
View File

@@ -21,7 +21,6 @@ dependencies = [
"opentelemetry-sdk>=1.22.0", "opentelemetry-sdk>=1.22.0",
"opentelemetry-exporter-otlp-proto-http>=1.22.0", "opentelemetry-exporter-otlp-proto-http>=1.22.0",
# Data Handling # Data Handling
"chromadb>=0.5.23",
"openpyxl>=3.1.5", "openpyxl>=3.1.5",
"pyvis>=0.3.2", "pyvis>=0.3.2",
# Authentication and Security # Authentication and Security
@@ -64,6 +63,9 @@ mem0 = ["mem0ai>=0.1.29"]
docling = [ docling = [
"docling>=2.12.0", "docling>=2.12.0",
] ]
chromadb = [
"chromadb>=0.5.23",
]
[tool.uv] [tool.uv]
dev-dependencies = [ dev-dependencies = [

208
src/crewai/knowledge/storage/knowledge_storage.py
View File

@@ -6,11 +6,18 @@ import os
import shutil import shutil
from typing import Any, Dict, List, Optional, Union, cast from typing import Any, Dict, List, Optional, Union, cast
import chromadb try:
import chromadb.errors import chromadb
from chromadb.api import ClientAPI import chromadb.errors
from chromadb.api.types import OneOrMany from chromadb.api import ClientAPI
from chromadb.config import Settings from chromadb.api.types import OneOrMany
from chromadb.config import Settings
CHROMADB_AVAILABLE = True
except ImportError:
CHROMADB_AVAILABLE = False
# Define placeholder types for type checking
ClientAPI = Any
OneOrMany = Any
from crewai.knowledge.storage.base_knowledge_storage import BaseKnowledgeStorage from crewai.knowledge.storage.base_knowledge_storage import BaseKnowledgeStorage
from crewai.utilities import EmbeddingConfigurator from crewai.utilities import EmbeddingConfigurator
@@ -42,9 +49,9 @@ class KnowledgeStorage(BaseKnowledgeStorage):
search efficiency. search efficiency.
""" """
collection: Optional[chromadb.Collection] = None collection = None # Type will be chromadb.Collection when available
collection_name: Optional[str] = "knowledge" collection_name: Optional[str] = "knowledge"
app: Optional[ClientAPI] = None app = None # Type will be ClientAPI when available
def __init__( def __init__(
self, self,
@@ -61,63 +68,91 @@ class KnowledgeStorage(BaseKnowledgeStorage):
filter: Optional[dict] = None, filter: Optional[dict] = None,
score_threshold: float = 0.35, score_threshold: float = 0.35,
) -> List[Dict[str, Any]]: ) -> List[Dict[str, Any]]:
with suppress_logging(): try:
if self.collection: with suppress_logging():
fetched = self.collection.query( if self.collection:
query_texts=query, fetched = self.collection.query(
n_results=limit, query_texts=query,
where=filter, n_results=limit,
) where=filter,
results = [] )
for i in range(len(fetched["ids"][0])): # type: ignore results = []
result = { for i in range(len(fetched["ids"][0])): # type: ignore
"id": fetched["ids"][0][i], # type: ignore result = {
"metadata": fetched["metadatas"][0][i], # type: ignore "id": fetched["ids"][0][i], # type: ignore
"context": fetched["documents"][0][i], # type: ignore "metadata": fetched["metadatas"][0][i], # type: ignore
"score": fetched["distances"][0][i], # type: ignore "context": fetched["documents"][0][i], # type: ignore
} "score": fetched["distances"][0][i], # type: ignore
if result["score"] >= score_threshold: }
results.append(result) if result["score"] >= score_threshold:
return results results.append(result)
else: return results
raise Exception("Collection not initialized") else:
return []
except (ImportError, NameError, AttributeError, Exception):
# Return empty results if chromadb is not available or collection is not initialized
return []
def initialize_knowledge_storage(self): def initialize_knowledge_storage(self):
base_path = os.path.join(db_storage_path(), "knowledge") if not CHROMADB_AVAILABLE:
chroma_client = chromadb.PersistentClient( import logging
path=base_path, logging.warning(
settings=Settings(allow_reset=True), "ChromaDB is not installed. Knowledge storage functionality will be limited. "
) "Install with 'pip install crewai[chromadb]' to enable full functionality."
self.app = chroma_client
try:
collection_name = (
f"knowledge_{self.collection_name}"
if self.collection_name
else "knowledge"
) )
if self.app: self.app = None
self.collection = self.app.get_or_create_collection( self.collection = None
name=collection_name, embedding_function=self.embedder return
)
else: try:
raise Exception("Vector Database Client not initialized") base_path = os.path.join(db_storage_path(), "knowledge")
except Exception: chroma_client = chromadb.PersistentClient(
raise Exception("Failed to create or get collection")
def reset(self):
base_path = os.path.join(db_storage_path(), KNOWLEDGE_DIRECTORY)
if not self.app:
self.app = chromadb.PersistentClient(
path=base_path, path=base_path,
settings=Settings(allow_reset=True), settings=Settings(allow_reset=True),
) )
self.app.reset() self.app = chroma_client
shutil.rmtree(base_path)
self.app = None try:
self.collection = None collection_name = (
f"knowledge_{self.collection_name}"
if self.collection_name
else "knowledge"
)
if self.app:
self.collection = self.app.get_or_create_collection(
name=collection_name, embedding_function=self.embedder
)
else:
raise Exception("Vector Database Client not initialized")
except Exception:
raise Exception("Failed to create or get collection")
except Exception:
logging.warning(
"Error initializing ChromaDB. Knowledge storage functionality will be limited."
)
self.app = None
self.collection = None
def reset(self):
base_path = os.path.join(db_storage_path(), KNOWLEDGE_DIRECTORY)
try:
if not self.app:
self.app = chromadb.PersistentClient(
path=base_path,
settings=Settings(allow_reset=True),
)
self.app.reset()
shutil.rmtree(base_path)
self.app = None
self.collection = None
except (ImportError, NameError, AttributeError):
# Handle case when chromadb is not available
if os.path.exists(base_path):
shutil.rmtree(base_path)
self.app = None
self.collection = None
def save( def save(
self, self,
@@ -125,7 +160,8 @@ class KnowledgeStorage(BaseKnowledgeStorage):
metadata: Optional[Union[Dict[str, Any], List[Dict[str, Any]]]] = None, metadata: Optional[Union[Dict[str, Any], List[Dict[str, Any]]]] = None,
): ):
if not self.collection: if not self.collection:
raise Exception("Collection not initialized") # Just return silently if chromadb is not available
return
try: try:
# Create a dictionary to store unique documents # Create a dictionary to store unique documents
@@ -154,7 +190,7 @@ class KnowledgeStorage(BaseKnowledgeStorage):
filtered_ids.append(doc_id) filtered_ids.append(doc_id)
# If we have no metadata at all, set it to None # If we have no metadata at all, set it to None
final_metadata: Optional[OneOrMany[chromadb.Metadata]] = ( final_metadata = (
None if all(m is None for m in filtered_metadata) else filtered_metadata None if all(m is None for m in filtered_metadata) else filtered_metadata
) )
@@ -163,29 +199,33 @@ class KnowledgeStorage(BaseKnowledgeStorage):
metadatas=final_metadata, metadatas=final_metadata,
ids=filtered_ids, ids=filtered_ids,
) )
except chromadb.errors.InvalidDimensionException as e: except (ImportError, NameError, AttributeError) as e:
Logger(verbose=True).log( # Handle case when chromadb is not available
"error", return
"Embedding dimension mismatch. This usually happens when mixing different embedding models. Try resetting the collection using `crewai reset-memories -a`",
"red",
)
raise ValueError(
"Embedding dimension mismatch. Make sure you're using the same embedding model "
"across all operations with this collection."
"Try resetting the collection using `crewai reset-memories -a`"
) from e
except Exception as e: except Exception as e:
if "chromadb" in str(e.__class__):
# Handle chromadb-specific errors silently when chromadb might not be fully available
return
Logger(verbose=True).log("error", f"Failed to upsert documents: {e}", "red") Logger(verbose=True).log("error", f"Failed to upsert documents: {e}", "red")
raise # Don't raise the exception, just log it and continue
return
def _create_default_embedding_function(self): def _create_default_embedding_function(self):
from chromadb.utils.embedding_functions.openai_embedding_function import ( try:
OpenAIEmbeddingFunction, from chromadb.utils.embedding_functions.openai_embedding_function import (
) OpenAIEmbeddingFunction,
)
return OpenAIEmbeddingFunction( return OpenAIEmbeddingFunction(
api_key=os.getenv("OPENAI_API_KEY"), model_name="text-embedding-3-small" api_key=os.getenv("OPENAI_API_KEY"), model_name="text-embedding-3-small"
) )
except ImportError:
import logging
logging.warning(
"ChromaDB is not installed. Cannot create default embedding function. "
"Install with 'pip install crewai[chromadb]' to enable full functionality."
)
return None
def _set_embedder_config(self, embedder: Optional[Dict[str, Any]] = None) -> None: def _set_embedder_config(self, embedder: Optional[Dict[str, Any]] = None) -> None:
"""Set the embedding configuration for the knowledge storage. """Set the embedding configuration for the knowledge storage.
@@ -194,8 +234,12 @@ class KnowledgeStorage(BaseKnowledgeStorage):
embedder_config (Optional[Dict[str, Any]]): Configuration dictionary for the embedder. embedder_config (Optional[Dict[str, Any]]): Configuration dictionary for the embedder.
If None or empty, defaults to the default embedding function. If None or empty, defaults to the default embedding function.
""" """
self.embedder = ( try:
EmbeddingConfigurator().configure_embedder(embedder) self.embedder = (
if embedder EmbeddingConfigurator().configure_embedder(embedder)
else self._create_default_embedding_function() if embedder
) else self._create_default_embedding_function()
)
except (ImportError, NameError, AttributeError):
# Handle case when chromadb is not available
self.embedder = None

102
src/crewai/memory/storage/rag_storage.py
View File

@@ -4,9 +4,15 @@ import logging
import os import os
import shutil import shutil
import uuid import uuid
from typing import Any, Dict, List, Optional from typing import Any, Dict, List, Optional, cast
from chromadb.api import ClientAPI try:
from chromadb.api import ClientAPI
CHROMADB_AVAILABLE = True
except ImportError:
CHROMADB_AVAILABLE = False
# Define placeholder type for type checking
ClientAPI = Any
from crewai.memory.storage.base_rag_storage import BaseRAGStorage from crewai.memory.storage.base_rag_storage import BaseRAGStorage
from crewai.utilities import EmbeddingConfigurator from crewai.utilities import EmbeddingConfigurator
@@ -37,7 +43,7 @@ class RAGStorage(BaseRAGStorage):
search efficiency. search efficiency.
""" """
app: ClientAPI | None = None app = None # Type will be ClientAPI when available
def __init__( def __init__(
self, type, allow_reset=True, embedder_config=None, crew=None, path=None self, type, allow_reset=True, embedder_config=None, crew=None, path=None
@@ -60,25 +66,34 @@ class RAGStorage(BaseRAGStorage):
self.embedder_config = configurator.configure_embedder(self.embedder_config) self.embedder_config = configurator.configure_embedder(self.embedder_config)
def _initialize_app(self): def _initialize_app(self):
import chromadb
from chromadb.config import Settings
self._set_embedder_config()
chroma_client = chromadb.PersistentClient(
path=self.path if self.path else self.storage_file_name,
settings=Settings(allow_reset=self.allow_reset),
)
self.app = chroma_client
try: try:
self.collection = self.app.get_collection( import chromadb
name=self.type, embedding_function=self.embedder_config from chromadb.config import Settings
self._set_embedder_config()
chroma_client = chromadb.PersistentClient(
path=self.path if self.path else self.storage_file_name,
settings=Settings(allow_reset=self.allow_reset),
) )
except Exception:
self.collection = self.app.create_collection( self.app = chroma_client
name=self.type, embedding_function=self.embedder_config
try:
self.collection = self.app.get_collection(
name=self.type, embedding_function=self.embedder_config
)
except Exception:
self.collection = self.app.create_collection(
name=self.type, embedding_function=self.embedder_config
)
except ImportError:
import logging
logging.warning(
"ChromaDB is not installed. RAG storage functionality will be limited. "
"Install with 'pip install crewai[chromadb]' to enable full functionality."
) )
self.app = None
self.collection = None
def _sanitize_role(self, role: str) -> str: def _sanitize_role(self, role: str) -> str:
""" """
@@ -103,6 +118,10 @@ class RAGStorage(BaseRAGStorage):
def save(self, value: Any, metadata: Dict[str, Any]) -> None: def save(self, value: Any, metadata: Dict[str, Any]) -> None:
if not hasattr(self, "app") or not hasattr(self, "collection"): if not hasattr(self, "app") or not hasattr(self, "collection"):
self._initialize_app() self._initialize_app()
if not self.collection:
return
try: try:
self._generate_embedding(value, metadata) self._generate_embedding(value, metadata)
except Exception as e: except Exception as e:
@@ -117,6 +136,9 @@ class RAGStorage(BaseRAGStorage):
) -> List[Any]: ) -> List[Any]:
if not hasattr(self, "app"): if not hasattr(self, "app"):
self._initialize_app() self._initialize_app()
if not self.collection:
return []
try: try:
with suppress_logging(): with suppress_logging():
@@ -141,6 +163,9 @@ class RAGStorage(BaseRAGStorage):
def _generate_embedding(self, text: str, metadata: Dict[str, Any]) -> None: # type: ignore def _generate_embedding(self, text: str, metadata: Dict[str, Any]) -> None: # type: ignore
if not hasattr(self, "app") or not hasattr(self, "collection"): if not hasattr(self, "app") or not hasattr(self, "collection"):
self._initialize_app() self._initialize_app()
if not self.collection:
return
self.collection.add( self.collection.add(
documents=[text], documents=[text],
@@ -149,26 +174,37 @@ class RAGStorage(BaseRAGStorage):
) )
def reset(self) -> None: def reset(self) -> None:
if not self.app:
return
try: try:
if self.app: self.app.reset()
self.app.reset() path = f"{db_storage_path()}/{self.type}"
shutil.rmtree(f"{db_storage_path()}/{self.type}") if os.path.exists(path):
self.app = None shutil.rmtree(path)
self.collection = None self.app = None
self.collection = None
except Exception as e: except Exception as e:
if "attempt to write a readonly database" in str(e): if "attempt to write a readonly database" in str(e):
# Ignore this specific error # Ignore this specific error
pass pass
else: else:
raise Exception( logging.error(f"Error during {self.type} reset: {str(e)}")
f"An error occurred while resetting the {self.type} memory: {e}" # Don't raise the exception, just log it
)
def _create_default_embedding_function(self): def _create_default_embedding_function(self):
from chromadb.utils.embedding_functions.openai_embedding_function import ( try:
OpenAIEmbeddingFunction, from chromadb.utils.embedding_functions.openai_embedding_function import (
) OpenAIEmbeddingFunction,
)
return OpenAIEmbeddingFunction( return OpenAIEmbeddingFunction(
api_key=os.getenv("OPENAI_API_KEY"), model_name="text-embedding-3-small" api_key=os.getenv("OPENAI_API_KEY"), model_name="text-embedding-3-small"
) )
except ImportError:
import logging
logging.warning(
"ChromaDB is not installed. Cannot create default embedding function. "
"Install with 'pip install crewai[chromadb]' to enable full functionality."
)
return None

21
src/crewai/utilities/embedding_configurator.py
View File

@@ -1,8 +1,17 @@
import os import os
from typing import Any, Dict, Optional, cast from typing import Any, Dict, Optional, cast
from chromadb import Documents, EmbeddingFunction, Embeddings try:
from chromadb.api.types import validate_embedding_function from chromadb import Documents, EmbeddingFunction, Embeddings
from chromadb.api.types import validate_embedding_function
CHROMADB_AVAILABLE = True
except ImportError:
CHROMADB_AVAILABLE = False
# Define placeholder types for type checking
Documents = Any
EmbeddingFunction = Any
Embeddings = Any
def validate_embedding_function(func): return func
class EmbeddingConfigurator: class EmbeddingConfigurator:
@@ -47,6 +56,14 @@ class EmbeddingConfigurator:
@staticmethod @staticmethod
def _create_default_embedding_function(): def _create_default_embedding_function():
if not CHROMADB_AVAILABLE:
import logging
logging.warning(
"ChromaDB is not installed. Cannot create default embedding function. "
"Install with 'pip install crewai[chromadb]' to enable full functionality."
)
return None
from chromadb.utils.embedding_functions.openai_embedding_function import ( from chromadb.utils.embedding_functions.openai_embedding_function import (
OpenAIEmbeddingFunction, OpenAIEmbeddingFunction,
) )

50
tests/test_optional_dependencies.py Normal file
View File

@@ -0,0 +1,50 @@
import importlib
import sys
from unittest import mock
import pytest
def test_rag_storage_without_chromadb():
# Mock the import to simulate chromadb not being installed
with mock.patch.dict(sys.modules, {'chromadb': None}):
# Force reload to ensure our mock takes effect
if 'crewai.memory.storage.rag_storage' in sys.modules:
importlib.reload(sys.modules['crewai.memory.storage.rag_storage'])
# Now import and test
from crewai.memory.storage.rag_storage import RAGStorage
# Should not raise an exception
storage = RAGStorage(type="test", allow_reset=True)
# Methods should handle the case when chromadb is not available
assert storage.app is None
assert storage.collection is None
# These methods should not raise exceptions
storage.save("test", {})
results = storage.search("test")
assert results == []
storage.reset()
def test_knowledge_storage_without_chromadb():
# Mock the import to simulate chromadb not being installed
with mock.patch.dict(sys.modules, {'chromadb': None}):
# Force reload to ensure our mock takes effect
if 'crewai.knowledge.storage.knowledge_storage' in sys.modules:
importlib.reload(sys.modules['crewai.knowledge.storage.knowledge_storage'])
# Now import and test
from crewai.knowledge.storage.knowledge_storage import KnowledgeStorage
# Should not raise an exception
storage = KnowledgeStorage()
# Methods should handle the case when chromadb is not available
assert storage.app is None
assert storage.collection is None
# These methods should not raise exceptions
storage.initialize_knowledge_storage()
results = storage.search(["test"])
assert results == []
storage.reset()