andre/crewAI
1
0
Fork 0
mirror of https://github.com/crewAIInc/crewAI.git synced 2026-01-10 00:28:31 +00:00
Code Issues Actions 1 Packages Projects Releases Wiki Activity

Fix #2324: Make chromadb an optional dependency to support Alpine Linux

Browse Source 
Co-Authored-By: Joe Moura <joao@crewai.com>
This commit is contained in:
Devin AI
2025-03-11 05:44:05 +00:00
parent 41a670166a
commit 36e7236ffe
6 changed files with 275 additions and 118 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
README.md
View File

@@ -136,6 +136,14 @@ pip install 'crewai[tools]'
```
The command above installs the basic package and also adds extra components which require more dependencies to function.
For vector storage and RAG capabilities, install with the chromadb extra:
```shell
pip install 'crewai[chromadb]'
```
Note: If you're using Alpine Linux or other environments where onnxruntime is not available, you can still use CrewAI without the chromadb dependency, but with limited vector storage functionality.
### Troubleshooting Dependencies
If you encounter issues during installation or usage, here are some common solutions:

4
pyproject.toml
View File

@@ -21,7 +21,6 @@ dependencies = [
"opentelemetry-sdk>=1.22.0",
"opentelemetry-exporter-otlp-proto-http>=1.22.0",
# Data Handling
"chromadb>=0.5.23",
"openpyxl>=3.1.5",
"pyvis>=0.3.2",
# Authentication and Security
@@ -64,6 +63,9 @@ mem0 = ["mem0ai>=0.1.29"]
docling = [
"docling>=2.12.0",
]
chromadb = [
"chromadb>=0.5.23",
]
[tool.uv]
dev-dependencies = [

208
src/crewai/knowledge/storage/knowledge_storage.py
View File

@@ -6,11 +6,18 @@ import os
import shutil
from typing import Any, Dict, List, Optional, Union, cast
import chromadb
import chromadb.errors
from chromadb.api import ClientAPI
from chromadb.api.types import OneOrMany
from chromadb.config import Settings
try:
import chromadb
import chromadb.errors
from chromadb.api import ClientAPI
from chromadb.api.types import OneOrMany
from chromadb.config import Settings
CHROMADB_AVAILABLE = True
except ImportError:
CHROMADB_AVAILABLE = False
# Define placeholder types for type checking
ClientAPI = Any
OneOrMany = Any
from crewai.knowledge.storage.base_knowledge_storage import BaseKnowledgeStorage
from crewai.utilities import EmbeddingConfigurator
@@ -42,9 +49,9 @@ class KnowledgeStorage(BaseKnowledgeStorage):
search efficiency.
"""
collection: Optional[chromadb.Collection] = None
collection = None # Type will be chromadb.Collection when available
collection_name: Optional[str] = "knowledge"
app: Optional[ClientAPI] = None
app = None # Type will be ClientAPI when available
def __init__(
self,
@@ -61,63 +68,91 @@ class KnowledgeStorage(BaseKnowledgeStorage):
filter: Optional[dict] = None,
score_threshold: float = 0.35,
) -> List[Dict[str, Any]]:
with suppress_logging():
if self.collection:
fetched = self.collection.query(
query_texts=query,
n_results=limit,
where=filter,
)
results = []
for i in range(len(fetched["ids"][0])): # type: ignore
result = {
"id": fetched["ids"][0][i], # type: ignore
"metadata": fetched["metadatas"][0][i], # type: ignore
"context": fetched["documents"][0][i], # type: ignore
"score": fetched["distances"][0][i], # type: ignore
}
if result["score"] >= score_threshold:
results.append(result)
return results
else:
raise Exception("Collection not initialized")
try:
with suppress_logging():
if self.collection:
fetched = self.collection.query(
query_texts=query,
n_results=limit,
where=filter,
)
results = []
for i in range(len(fetched["ids"][0])): # type: ignore
result = {
"id": fetched["ids"][0][i], # type: ignore
"metadata": fetched["metadatas"][0][i], # type: ignore
"context": fetched["documents"][0][i], # type: ignore
"score": fetched["distances"][0][i], # type: ignore
}
if result["score"] >= score_threshold:
results.append(result)
return results
else:
return []
except (ImportError, NameError, AttributeError, Exception):
# Return empty results if chromadb is not available or collection is not initialized
return []
def initialize_knowledge_storage(self):
base_path = os.path.join(db_storage_path(), "knowledge")
chroma_client = chromadb.PersistentClient(
path=base_path,
settings=Settings(allow_reset=True),
)
self.app = chroma_client
try:
collection_name = (
f"knowledge_{self.collection_name}"
if self.collection_name
else "knowledge"
if not CHROMADB_AVAILABLE:
import logging
logging.warning(
"ChromaDB is not installed. Knowledge storage functionality will be limited. "
"Install with 'pip install crewai[chromadb]' to enable full functionality."
)
if self.app:
self.collection = self.app.get_or_create_collection(
name=collection_name, embedding_function=self.embedder
)
else:
raise Exception("Vector Database Client not initialized")
except Exception:
raise Exception("Failed to create or get collection")
def reset(self):
base_path = os.path.join(db_storage_path(), KNOWLEDGE_DIRECTORY)
if not self.app:
self.app = chromadb.PersistentClient(
self.app = None
self.collection = None
return
try:
base_path = os.path.join(db_storage_path(), "knowledge")
chroma_client = chromadb.PersistentClient(
path=base_path,
settings=Settings(allow_reset=True),
)
self.app.reset()
shutil.rmtree(base_path)
self.app = None
self.collection = None
self.app = chroma_client
try:
collection_name = (
f"knowledge_{self.collection_name}"
if self.collection_name
else "knowledge"
)
if self.app:
self.collection = self.app.get_or_create_collection(
name=collection_name, embedding_function=self.embedder
)
else:
raise Exception("Vector Database Client not initialized")
except Exception:
raise Exception("Failed to create or get collection")
except Exception:
logging.warning(
"Error initializing ChromaDB. Knowledge storage functionality will be limited."
)
self.app = None
self.collection = None
def reset(self):
base_path = os.path.join(db_storage_path(), KNOWLEDGE_DIRECTORY)
try:
if not self.app:
self.app = chromadb.PersistentClient(
path=base_path,
settings=Settings(allow_reset=True),
)
self.app.reset()
shutil.rmtree(base_path)
self.app = None
self.collection = None
except (ImportError, NameError, AttributeError):
# Handle case when chromadb is not available
if os.path.exists(base_path):
shutil.rmtree(base_path)
self.app = None
self.collection = None
def save(
self,
@@ -125,7 +160,8 @@ class KnowledgeStorage(BaseKnowledgeStorage):
metadata: Optional[Union[Dict[str, Any], List[Dict[str, Any]]]] = None,
):
if not self.collection:
raise Exception("Collection not initialized")
# Just return silently if chromadb is not available
return
try:
# Create a dictionary to store unique documents
@@ -154,7 +190,7 @@ class KnowledgeStorage(BaseKnowledgeStorage):
filtered_ids.append(doc_id)
# If we have no metadata at all, set it to None
final_metadata: Optional[OneOrMany[chromadb.Metadata]] = (
final_metadata = (
None if all(m is None for m in filtered_metadata) else filtered_metadata
)
@@ -163,29 +199,33 @@ class KnowledgeStorage(BaseKnowledgeStorage):
metadatas=final_metadata,
ids=filtered_ids,
)
except chromadb.errors.InvalidDimensionException as e:
Logger(verbose=True).log(
"error",
"Embedding dimension mismatch. This usually happens when mixing different embedding models. Try resetting the collection using `crewai reset-memories -a`",
"red",
)
raise ValueError(
"Embedding dimension mismatch. Make sure you're using the same embedding model "
"across all operations with this collection."
"Try resetting the collection using `crewai reset-memories -a`"
) from e
except (ImportError, NameError, AttributeError) as e:
# Handle case when chromadb is not available
return
except Exception as e:
if "chromadb" in str(e.__class__):
# Handle chromadb-specific errors silently when chromadb might not be fully available
return
Logger(verbose=True).log("error", f"Failed to upsert documents: {e}", "red")
raise
# Don't raise the exception, just log it and continue
return
def _create_default_embedding_function(self):
from chromadb.utils.embedding_functions.openai_embedding_function import (
OpenAIEmbeddingFunction,
)
try:
from chromadb.utils.embedding_functions.openai_embedding_function import (
OpenAIEmbeddingFunction,
)
return OpenAIEmbeddingFunction(
api_key=os.getenv("OPENAI_API_KEY"), model_name="text-embedding-3-small"
)
return OpenAIEmbeddingFunction(
api_key=os.getenv("OPENAI_API_KEY"), model_name="text-embedding-3-small"
)
except ImportError:
import logging
logging.warning(
"ChromaDB is not installed. Cannot create default embedding function. "
"Install with 'pip install crewai[chromadb]' to enable full functionality."
)
return None
def _set_embedder_config(self, embedder: Optional[Dict[str, Any]] = None) -> None:
"""Set the embedding configuration for the knowledge storage.
@@ -194,8 +234,12 @@ class KnowledgeStorage(BaseKnowledgeStorage):
embedder_config (Optional[Dict[str, Any]]): Configuration dictionary for the embedder.
If None or empty, defaults to the default embedding function.
"""
self.embedder = (
EmbeddingConfigurator().configure_embedder(embedder)
if embedder
else self._create_default_embedding_function()
)
try:
self.embedder = (
EmbeddingConfigurator().configure_embedder(embedder)
if embedder
else self._create_default_embedding_function()
)
except (ImportError, NameError, AttributeError):
# Handle case when chromadb is not available
self.embedder = None

102
src/crewai/memory/storage/rag_storage.py
View File

@@ -4,9 +4,15 @@ import logging
import os
import shutil
import uuid
from typing import Any, Dict, List, Optional
from typing import Any, Dict, List, Optional, cast
from chromadb.api import ClientAPI
try:
from chromadb.api import ClientAPI
CHROMADB_AVAILABLE = True
except ImportError:
CHROMADB_AVAILABLE = False
# Define placeholder type for type checking
ClientAPI = Any
from crewai.memory.storage.base_rag_storage import BaseRAGStorage
from crewai.utilities import EmbeddingConfigurator
@@ -37,7 +43,7 @@ class RAGStorage(BaseRAGStorage):
search efficiency.
"""
app: ClientAPI | None = None
app = None # Type will be ClientAPI when available
def __init__(
self, type, allow_reset=True, embedder_config=None, crew=None, path=None
@@ -60,25 +66,34 @@ class RAGStorage(BaseRAGStorage):
self.embedder_config = configurator.configure_embedder(self.embedder_config)
def _initialize_app(self):
import chromadb
from chromadb.config import Settings
self._set_embedder_config()
chroma_client = chromadb.PersistentClient(
path=self.path if self.path else self.storage_file_name,
settings=Settings(allow_reset=self.allow_reset),
)
self.app = chroma_client
try:
self.collection = self.app.get_collection(
name=self.type, embedding_function=self.embedder_config
import chromadb
from chromadb.config import Settings
self._set_embedder_config()
chroma_client = chromadb.PersistentClient(
path=self.path if self.path else self.storage_file_name,
settings=Settings(allow_reset=self.allow_reset),
)
except Exception:
self.collection = self.app.create_collection(
name=self.type, embedding_function=self.embedder_config
self.app = chroma_client
try:
self.collection = self.app.get_collection(
name=self.type, embedding_function=self.embedder_config
)
except Exception:
self.collection = self.app.create_collection(
name=self.type, embedding_function=self.embedder_config
)
except ImportError:
import logging
logging.warning(
"ChromaDB is not installed. RAG storage functionality will be limited. "
"Install with 'pip install crewai[chromadb]' to enable full functionality."
)
self.app = None
self.collection = None
def _sanitize_role(self, role: str) -> str:
"""
@@ -103,6 +118,10 @@ class RAGStorage(BaseRAGStorage):
def save(self, value: Any, metadata: Dict[str, Any]) -> None:
if not hasattr(self, "app") or not hasattr(self, "collection"):
self._initialize_app()
if not self.collection:
return
try:
self._generate_embedding(value, metadata)
except Exception as e:
@@ -117,6 +136,9 @@ class RAGStorage(BaseRAGStorage):
) -> List[Any]:
if not hasattr(self, "app"):
self._initialize_app()
if not self.collection:
return []
try:
with suppress_logging():
@@ -141,6 +163,9 @@ class RAGStorage(BaseRAGStorage):
def _generate_embedding(self, text: str, metadata: Dict[str, Any]) -> None: # type: ignore
if not hasattr(self, "app") or not hasattr(self, "collection"):
self._initialize_app()
if not self.collection:
return
self.collection.add(
documents=[text],
@@ -149,26 +174,37 @@ class RAGStorage(BaseRAGStorage):
)
def reset(self) -> None:
if not self.app:
return
try:
if self.app:
self.app.reset()
shutil.rmtree(f"{db_storage_path()}/{self.type}")
self.app = None
self.collection = None
self.app.reset()
path = f"{db_storage_path()}/{self.type}"
if os.path.exists(path):
shutil.rmtree(path)
self.app = None
self.collection = None
except Exception as e:
if "attempt to write a readonly database" in str(e):
# Ignore this specific error
pass
else:
raise Exception(
f"An error occurred while resetting the {self.type} memory: {e}"
)
logging.error(f"Error during {self.type} reset: {str(e)}")
# Don't raise the exception, just log it
def _create_default_embedding_function(self):
from chromadb.utils.embedding_functions.openai_embedding_function import (
OpenAIEmbeddingFunction,
)
try:
from chromadb.utils.embedding_functions.openai_embedding_function import (
OpenAIEmbeddingFunction,
)
return OpenAIEmbeddingFunction(
api_key=os.getenv("OPENAI_API_KEY"), model_name="text-embedding-3-small"
)
return OpenAIEmbeddingFunction(
api_key=os.getenv("OPENAI_API_KEY"), model_name="text-embedding-3-small"
)
except ImportError:
import logging
logging.warning(
"ChromaDB is not installed. Cannot create default embedding function. "
"Install with 'pip install crewai[chromadb]' to enable full functionality."
)
return None

21
src/crewai/utilities/embedding_configurator.py
View File

@@ -1,8 +1,17 @@
import os
from typing import Any, Dict, Optional, cast
from chromadb import Documents, EmbeddingFunction, Embeddings
from chromadb.api.types import validate_embedding_function
try:
from chromadb import Documents, EmbeddingFunction, Embeddings
from chromadb.api.types import validate_embedding_function
CHROMADB_AVAILABLE = True
except ImportError:
CHROMADB_AVAILABLE = False
# Define placeholder types for type checking
Documents = Any
EmbeddingFunction = Any
Embeddings = Any
def validate_embedding_function(func): return func
class EmbeddingConfigurator:
@@ -47,6 +56,14 @@ class EmbeddingConfigurator:
@staticmethod
def _create_default_embedding_function():
if not CHROMADB_AVAILABLE:
import logging
logging.warning(
"ChromaDB is not installed. Cannot create default embedding function. "
"Install with 'pip install crewai[chromadb]' to enable full functionality."
)
return None
from chromadb.utils.embedding_functions.openai_embedding_function import (
OpenAIEmbeddingFunction,
)

50
tests/test_optional_dependencies.py Normal file
View File

@@ -0,0 +1,50 @@
import importlib
import sys
from unittest import mock
import pytest
def test_rag_storage_without_chromadb():
# Mock the import to simulate chromadb not being installed
with mock.patch.dict(sys.modules, {'chromadb': None}):
# Force reload to ensure our mock takes effect
if 'crewai.memory.storage.rag_storage' in sys.modules:
importlib.reload(sys.modules['crewai.memory.storage.rag_storage'])
# Now import and test
from crewai.memory.storage.rag_storage import RAGStorage
# Should not raise an exception
storage = RAGStorage(type="test", allow_reset=True)
# Methods should handle the case when chromadb is not available
assert storage.app is None
assert storage.collection is None
# These methods should not raise exceptions
storage.save("test", {})
results = storage.search("test")
assert results == []
storage.reset()
def test_knowledge_storage_without_chromadb():
# Mock the import to simulate chromadb not being installed
with mock.patch.dict(sys.modules, {'chromadb': None}):
# Force reload to ensure our mock takes effect
if 'crewai.knowledge.storage.knowledge_storage' in sys.modules:
importlib.reload(sys.modules['crewai.knowledge.storage.knowledge_storage'])
# Now import and test
from crewai.knowledge.storage.knowledge_storage import KnowledgeStorage
# Should not raise an exception
storage = KnowledgeStorage()
# Methods should handle the case when chromadb is not available
assert storage.app is None
assert storage.collection is None
# These methods should not raise exceptions
storage.initialize_knowledge_storage()
results = storage.search(["test"])
assert results == []
storage.reset()