mirror of
https://github.com/crewAIInc/crewAI.git
synced 2026-01-11 00:58:30 +00:00
Fix #2919: Make chromadb an optional dependency to resolve package conflicts
Co-Authored-By: João <joao@crewai.com>
This commit is contained in:
@@ -21,7 +21,6 @@ dependencies = [
|
|||||||
"opentelemetry-sdk>=1.30.0",
|
"opentelemetry-sdk>=1.30.0",
|
||||||
"opentelemetry-exporter-otlp-proto-http>=1.30.0",
|
"opentelemetry-exporter-otlp-proto-http>=1.30.0",
|
||||||
# Data Handling
|
# Data Handling
|
||||||
"chromadb>=0.5.23",
|
|
||||||
"openpyxl>=3.1.5",
|
"openpyxl>=3.1.5",
|
||||||
"pyvis>=0.3.2",
|
"pyvis>=0.3.2",
|
||||||
# Authentication and Security
|
# Authentication and Security
|
||||||
@@ -49,6 +48,9 @@ tools = ["crewai-tools~=0.45.0"]
|
|||||||
embeddings = [
|
embeddings = [
|
||||||
"tiktoken~=0.7.0"
|
"tiktoken~=0.7.0"
|
||||||
]
|
]
|
||||||
|
storage = [
|
||||||
|
"chromadb>=0.5.23"
|
||||||
|
]
|
||||||
agentops = ["agentops>=0.3.0"]
|
agentops = ["agentops>=0.3.0"]
|
||||||
fastembed = ["fastembed>=0.4.1"]
|
fastembed = ["fastembed>=0.4.1"]
|
||||||
pdfplumber = [
|
pdfplumber = [
|
||||||
|
|||||||
@@ -6,11 +6,19 @@ import os
|
|||||||
import shutil
|
import shutil
|
||||||
from typing import Any, Dict, List, Optional, Union
|
from typing import Any, Dict, List, Optional, Union
|
||||||
|
|
||||||
import chromadb
|
try:
|
||||||
import chromadb.errors
|
import chromadb
|
||||||
from chromadb.api import ClientAPI
|
import chromadb.errors
|
||||||
from chromadb.api.types import OneOrMany
|
from chromadb.api import ClientAPI
|
||||||
from chromadb.config import Settings
|
from chromadb.api.types import OneOrMany
|
||||||
|
from chromadb.config import Settings
|
||||||
|
HAS_CHROMADB = True
|
||||||
|
except ImportError:
|
||||||
|
chromadb = None # type: ignore
|
||||||
|
ClientAPI = Any # type: ignore
|
||||||
|
OneOrMany = Any # type: ignore
|
||||||
|
Settings = Any # type: ignore
|
||||||
|
HAS_CHROMADB = False
|
||||||
|
|
||||||
from crewai.knowledge.storage.base_knowledge_storage import BaseKnowledgeStorage
|
from crewai.knowledge.storage.base_knowledge_storage import BaseKnowledgeStorage
|
||||||
from crewai.utilities import EmbeddingConfigurator
|
from crewai.utilities import EmbeddingConfigurator
|
||||||
@@ -62,6 +70,12 @@ class KnowledgeStorage(BaseKnowledgeStorage):
|
|||||||
filter: Optional[dict] = None,
|
filter: Optional[dict] = None,
|
||||||
score_threshold: float = 0.35,
|
score_threshold: float = 0.35,
|
||||||
) -> List[Dict[str, Any]]:
|
) -> List[Dict[str, Any]]:
|
||||||
|
if not HAS_CHROMADB:
|
||||||
|
raise ImportError(
|
||||||
|
"ChromaDB is required for knowledge storage features. "
|
||||||
|
"Please install it with 'pip install crewai[storage]'"
|
||||||
|
)
|
||||||
|
|
||||||
with suppress_logging():
|
with suppress_logging():
|
||||||
if self.collection:
|
if self.collection:
|
||||||
fetched = self.collection.query(
|
fetched = self.collection.query(
|
||||||
@@ -84,48 +98,78 @@ class KnowledgeStorage(BaseKnowledgeStorage):
|
|||||||
raise Exception("Collection not initialized")
|
raise Exception("Collection not initialized")
|
||||||
|
|
||||||
def initialize_knowledge_storage(self):
|
def initialize_knowledge_storage(self):
|
||||||
base_path = os.path.join(db_storage_path(), "knowledge")
|
if not HAS_CHROMADB:
|
||||||
chroma_client = chromadb.PersistentClient(
|
raise ImportError(
|
||||||
path=base_path,
|
"ChromaDB is required for knowledge storage features. "
|
||||||
settings=Settings(allow_reset=True),
|
"Please install it with 'pip install crewai[storage]'"
|
||||||
)
|
|
||||||
|
|
||||||
self.app = chroma_client
|
|
||||||
|
|
||||||
try:
|
|
||||||
collection_name = (
|
|
||||||
f"knowledge_{self.collection_name}"
|
|
||||||
if self.collection_name
|
|
||||||
else "knowledge"
|
|
||||||
)
|
)
|
||||||
if self.app:
|
|
||||||
self.collection = self.app.get_or_create_collection(
|
base_path = os.path.join(db_storage_path(), "knowledge")
|
||||||
name=sanitize_collection_name(collection_name),
|
try:
|
||||||
embedding_function=self.embedder,
|
chroma_client = chromadb.PersistentClient(
|
||||||
)
|
|
||||||
else:
|
|
||||||
raise Exception("Vector Database Client not initialized")
|
|
||||||
except Exception:
|
|
||||||
raise Exception("Failed to create or get collection")
|
|
||||||
|
|
||||||
def reset(self):
|
|
||||||
base_path = os.path.join(db_storage_path(), KNOWLEDGE_DIRECTORY)
|
|
||||||
if not self.app:
|
|
||||||
self.app = chromadb.PersistentClient(
|
|
||||||
path=base_path,
|
path=base_path,
|
||||||
settings=Settings(allow_reset=True),
|
settings=Settings(allow_reset=True),
|
||||||
)
|
)
|
||||||
|
|
||||||
self.app.reset()
|
self.app = chroma_client
|
||||||
shutil.rmtree(base_path)
|
|
||||||
self.app = None
|
try:
|
||||||
self.collection = None
|
collection_name = (
|
||||||
|
f"knowledge_{self.collection_name}"
|
||||||
|
if self.collection_name
|
||||||
|
else "knowledge"
|
||||||
|
)
|
||||||
|
if self.app:
|
||||||
|
self.collection = self.app.get_or_create_collection(
|
||||||
|
name=sanitize_collection_name(collection_name),
|
||||||
|
embedding_function=self.embedder,
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
raise Exception("Vector Database Client not initialized")
|
||||||
|
except Exception:
|
||||||
|
raise Exception("Failed to create or get collection")
|
||||||
|
except ImportError:
|
||||||
|
raise ImportError(
|
||||||
|
"ChromaDB is required for knowledge storage features. "
|
||||||
|
"Please install it with 'pip install crewai[storage]'"
|
||||||
|
)
|
||||||
|
|
||||||
|
def reset(self):
|
||||||
|
if not HAS_CHROMADB:
|
||||||
|
raise ImportError(
|
||||||
|
"ChromaDB is required for knowledge storage features. "
|
||||||
|
"Please install it with 'pip install crewai[storage]'"
|
||||||
|
)
|
||||||
|
|
||||||
|
base_path = os.path.join(db_storage_path(), KNOWLEDGE_DIRECTORY)
|
||||||
|
try:
|
||||||
|
if not self.app:
|
||||||
|
self.app = chromadb.PersistentClient(
|
||||||
|
path=base_path,
|
||||||
|
settings=Settings(allow_reset=True),
|
||||||
|
)
|
||||||
|
|
||||||
|
self.app.reset()
|
||||||
|
shutil.rmtree(base_path)
|
||||||
|
self.app = None
|
||||||
|
self.collection = None
|
||||||
|
except ImportError:
|
||||||
|
raise ImportError(
|
||||||
|
"ChromaDB is required for knowledge storage features. "
|
||||||
|
"Please install it with 'pip install crewai[storage]'"
|
||||||
|
)
|
||||||
|
|
||||||
def save(
|
def save(
|
||||||
self,
|
self,
|
||||||
documents: List[str],
|
documents: List[str],
|
||||||
metadata: Optional[Union[Dict[str, Any], List[Dict[str, Any]]]] = None,
|
metadata: Optional[Union[Dict[str, Any], List[Dict[str, Any]]]] = None,
|
||||||
):
|
):
|
||||||
|
if not HAS_CHROMADB:
|
||||||
|
raise ImportError(
|
||||||
|
"ChromaDB is required for knowledge storage features. "
|
||||||
|
"Please install it with 'pip install crewai[storage]'"
|
||||||
|
)
|
||||||
|
|
||||||
if not self.collection:
|
if not self.collection:
|
||||||
raise Exception("Collection not initialized")
|
raise Exception("Collection not initialized")
|
||||||
|
|
||||||
@@ -156,7 +200,7 @@ class KnowledgeStorage(BaseKnowledgeStorage):
|
|||||||
filtered_ids.append(doc_id)
|
filtered_ids.append(doc_id)
|
||||||
|
|
||||||
# If we have no metadata at all, set it to None
|
# If we have no metadata at all, set it to None
|
||||||
final_metadata: Optional[OneOrMany[chromadb.Metadata]] = (
|
final_metadata: Optional[OneOrMany[Any]] = (
|
||||||
None if all(m is None for m in filtered_metadata) else filtered_metadata
|
None if all(m is None for m in filtered_metadata) else filtered_metadata
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -165,29 +209,47 @@ class KnowledgeStorage(BaseKnowledgeStorage):
|
|||||||
metadatas=final_metadata,
|
metadatas=final_metadata,
|
||||||
ids=filtered_ids,
|
ids=filtered_ids,
|
||||||
)
|
)
|
||||||
except chromadb.errors.InvalidDimensionException as e:
|
except ImportError:
|
||||||
Logger(verbose=True).log(
|
raise ImportError(
|
||||||
"error",
|
"ChromaDB is required for knowledge storage features. "
|
||||||
"Embedding dimension mismatch. This usually happens when mixing different embedding models. Try resetting the collection using `crewai reset-memories -a`",
|
"Please install it with 'pip install crewai[storage]'"
|
||||||
"red",
|
|
||||||
)
|
)
|
||||||
raise ValueError(
|
|
||||||
"Embedding dimension mismatch. Make sure you're using the same embedding model "
|
|
||||||
"across all operations with this collection."
|
|
||||||
"Try resetting the collection using `crewai reset-memories -a`"
|
|
||||||
) from e
|
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
Logger(verbose=True).log("error", f"Failed to upsert documents: {e}", "red")
|
if HAS_CHROMADB and isinstance(e, chromadb.errors.InvalidDimensionException):
|
||||||
raise
|
Logger(verbose=True).log(
|
||||||
|
"error",
|
||||||
|
"Embedding dimension mismatch. This usually happens when mixing different embedding models. Try resetting the collection using `crewai reset-memories -a`",
|
||||||
|
"red",
|
||||||
|
)
|
||||||
|
raise ValueError(
|
||||||
|
"Embedding dimension mismatch. Make sure you're using the same embedding model "
|
||||||
|
"across all operations with this collection."
|
||||||
|
"Try resetting the collection using `crewai reset-memories -a`"
|
||||||
|
) from e
|
||||||
|
else:
|
||||||
|
Logger(verbose=True).log("error", f"Failed to upsert documents: {e}", "red")
|
||||||
|
raise
|
||||||
|
|
||||||
def _create_default_embedding_function(self):
|
def _create_default_embedding_function(self):
|
||||||
from chromadb.utils.embedding_functions.openai_embedding_function import (
|
if not HAS_CHROMADB:
|
||||||
OpenAIEmbeddingFunction,
|
raise ImportError(
|
||||||
)
|
"ChromaDB is required for knowledge storage features. "
|
||||||
|
"Please install it with 'pip install crewai[storage]'"
|
||||||
|
)
|
||||||
|
|
||||||
|
try:
|
||||||
|
from chromadb.utils.embedding_functions.openai_embedding_function import (
|
||||||
|
OpenAIEmbeddingFunction,
|
||||||
|
)
|
||||||
|
|
||||||
return OpenAIEmbeddingFunction(
|
return OpenAIEmbeddingFunction(
|
||||||
api_key=os.getenv("OPENAI_API_KEY"), model_name="text-embedding-3-small"
|
api_key=os.getenv("OPENAI_API_KEY"), model_name="text-embedding-3-small"
|
||||||
)
|
)
|
||||||
|
except ImportError:
|
||||||
|
raise ImportError(
|
||||||
|
"ChromaDB is required for knowledge storage features. "
|
||||||
|
"Please install it with 'pip install crewai[storage]'"
|
||||||
|
)
|
||||||
|
|
||||||
def _set_embedder_config(self, embedder: Optional[Dict[str, Any]] = None) -> None:
|
def _set_embedder_config(self, embedder: Optional[Dict[str, Any]] = None) -> None:
|
||||||
"""Set the embedding configuration for the knowledge storage.
|
"""Set the embedding configuration for the knowledge storage.
|
||||||
|
|||||||
@@ -4,9 +4,14 @@ import logging
|
|||||||
import os
|
import os
|
||||||
import shutil
|
import shutil
|
||||||
import uuid
|
import uuid
|
||||||
from typing import Any, Dict, List, Optional
|
from typing import Any, Dict, List, Optional, Union
|
||||||
|
|
||||||
from chromadb.api import ClientAPI
|
try:
|
||||||
|
from chromadb.api import ClientAPI
|
||||||
|
HAS_CHROMADB = True
|
||||||
|
except ImportError:
|
||||||
|
ClientAPI = Any # type: ignore
|
||||||
|
HAS_CHROMADB = False
|
||||||
|
|
||||||
from crewai.memory.storage.base_rag_storage import BaseRAGStorage
|
from crewai.memory.storage.base_rag_storage import BaseRAGStorage
|
||||||
from crewai.utilities import EmbeddingConfigurator
|
from crewai.utilities import EmbeddingConfigurator
|
||||||
@@ -60,24 +65,36 @@ class RAGStorage(BaseRAGStorage):
|
|||||||
self.embedder_config = configurator.configure_embedder(self.embedder_config)
|
self.embedder_config = configurator.configure_embedder(self.embedder_config)
|
||||||
|
|
||||||
def _initialize_app(self):
|
def _initialize_app(self):
|
||||||
import chromadb
|
if not HAS_CHROMADB:
|
||||||
from chromadb.config import Settings
|
raise ImportError(
|
||||||
|
"ChromaDB is required for memory storage features. "
|
||||||
self._set_embedder_config()
|
"Please install it with 'pip install crewai[storage]'"
|
||||||
chroma_client = chromadb.PersistentClient(
|
|
||||||
path=self.path if self.path else self.storage_file_name,
|
|
||||||
settings=Settings(allow_reset=self.allow_reset),
|
|
||||||
)
|
|
||||||
|
|
||||||
self.app = chroma_client
|
|
||||||
|
|
||||||
try:
|
|
||||||
self.collection = self.app.get_collection(
|
|
||||||
name=self.type, embedding_function=self.embedder_config
|
|
||||||
)
|
)
|
||||||
except Exception:
|
|
||||||
self.collection = self.app.create_collection(
|
try:
|
||||||
name=self.type, embedding_function=self.embedder_config
|
import chromadb
|
||||||
|
from chromadb.config import Settings
|
||||||
|
|
||||||
|
self._set_embedder_config()
|
||||||
|
chroma_client = chromadb.PersistentClient(
|
||||||
|
path=self.path if self.path else self.storage_file_name,
|
||||||
|
settings=Settings(allow_reset=self.allow_reset),
|
||||||
|
)
|
||||||
|
|
||||||
|
self.app = chroma_client
|
||||||
|
|
||||||
|
try:
|
||||||
|
self.collection = self.app.get_collection(
|
||||||
|
name=self.type, embedding_function=self.embedder_config
|
||||||
|
)
|
||||||
|
except Exception:
|
||||||
|
self.collection = self.app.create_collection(
|
||||||
|
name=self.type, embedding_function=self.embedder_config
|
||||||
|
)
|
||||||
|
except ImportError:
|
||||||
|
raise ImportError(
|
||||||
|
"ChromaDB is required for memory storage features. "
|
||||||
|
"Please install it with 'pip install crewai[storage]'"
|
||||||
)
|
)
|
||||||
|
|
||||||
def _sanitize_role(self, role: str) -> str:
|
def _sanitize_role(self, role: str) -> str:
|
||||||
@@ -165,10 +182,22 @@ class RAGStorage(BaseRAGStorage):
|
|||||||
)
|
)
|
||||||
|
|
||||||
def _create_default_embedding_function(self):
|
def _create_default_embedding_function(self):
|
||||||
from chromadb.utils.embedding_functions.openai_embedding_function import (
|
if not HAS_CHROMADB:
|
||||||
OpenAIEmbeddingFunction,
|
raise ImportError(
|
||||||
)
|
"ChromaDB is required for memory storage features. "
|
||||||
|
"Please install it with 'pip install crewai[storage]'"
|
||||||
|
)
|
||||||
|
|
||||||
|
try:
|
||||||
|
from chromadb.utils.embedding_functions.openai_embedding_function import (
|
||||||
|
OpenAIEmbeddingFunction,
|
||||||
|
)
|
||||||
|
|
||||||
return OpenAIEmbeddingFunction(
|
return OpenAIEmbeddingFunction(
|
||||||
api_key=os.getenv("OPENAI_API_KEY"), model_name="text-embedding-3-small"
|
api_key=os.getenv("OPENAI_API_KEY"), model_name="text-embedding-3-small"
|
||||||
)
|
)
|
||||||
|
except ImportError:
|
||||||
|
raise ImportError(
|
||||||
|
"ChromaDB is required for memory storage features. "
|
||||||
|
"Please install it with 'pip install crewai[storage]'"
|
||||||
|
)
|
||||||
|
|||||||
@@ -1,8 +1,21 @@
|
|||||||
import os
|
import os
|
||||||
from typing import Any, Dict, Optional, cast
|
from typing import Any, Dict, Optional, Union, cast, Protocol
|
||||||
|
|
||||||
from chromadb import Documents, EmbeddingFunction, Embeddings
|
try:
|
||||||
from chromadb.api.types import validate_embedding_function
|
from chromadb import Documents, EmbeddingFunction, Embeddings
|
||||||
|
from chromadb.api.types import validate_embedding_function
|
||||||
|
HAS_CHROMADB = True
|
||||||
|
except ImportError:
|
||||||
|
HAS_CHROMADB = False
|
||||||
|
|
||||||
|
class EmbeddingFunction(Protocol):
|
||||||
|
def __call__(self, input: Any) -> Any: ...
|
||||||
|
|
||||||
|
Documents = Any
|
||||||
|
Embeddings = Any
|
||||||
|
|
||||||
|
def validate_embedding_function(func: Any) -> None:
|
||||||
|
pass
|
||||||
|
|
||||||
|
|
||||||
class EmbeddingConfigurator:
|
class EmbeddingConfigurator:
|
||||||
@@ -26,6 +39,12 @@ class EmbeddingConfigurator:
|
|||||||
embedder_config: Optional[Dict[str, Any]] = None,
|
embedder_config: Optional[Dict[str, Any]] = None,
|
||||||
) -> EmbeddingFunction:
|
) -> EmbeddingFunction:
|
||||||
"""Configures and returns an embedding function based on the provided config."""
|
"""Configures and returns an embedding function based on the provided config."""
|
||||||
|
if not HAS_CHROMADB:
|
||||||
|
raise ImportError(
|
||||||
|
"ChromaDB is required for embedding functionality. "
|
||||||
|
"Please install it with 'pip install crewai[storage]'"
|
||||||
|
)
|
||||||
|
|
||||||
if embedder_config is None:
|
if embedder_config is None:
|
||||||
return self._create_default_embedding_function()
|
return self._create_default_embedding_function()
|
||||||
|
|
||||||
@@ -47,129 +66,249 @@ class EmbeddingConfigurator:
|
|||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def _create_default_embedding_function():
|
def _create_default_embedding_function():
|
||||||
from chromadb.utils.embedding_functions.openai_embedding_function import (
|
if not HAS_CHROMADB:
|
||||||
OpenAIEmbeddingFunction,
|
raise ImportError(
|
||||||
)
|
"ChromaDB is required for embedding functionality. "
|
||||||
|
"Please install it with 'pip install crewai[storage]'"
|
||||||
|
)
|
||||||
|
|
||||||
|
try:
|
||||||
|
from chromadb.utils.embedding_functions.openai_embedding_function import (
|
||||||
|
OpenAIEmbeddingFunction,
|
||||||
|
)
|
||||||
|
|
||||||
return OpenAIEmbeddingFunction(
|
return OpenAIEmbeddingFunction(
|
||||||
api_key=os.getenv("OPENAI_API_KEY"), model_name="text-embedding-3-small"
|
api_key=os.getenv("OPENAI_API_KEY"), model_name="text-embedding-3-small"
|
||||||
)
|
)
|
||||||
|
except ImportError:
|
||||||
|
raise ImportError(
|
||||||
|
"ChromaDB is required for embedding functionality. "
|
||||||
|
"Please install it with 'pip install crewai[storage]'"
|
||||||
|
)
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def _configure_openai(config, model_name):
|
def _configure_openai(config, model_name):
|
||||||
from chromadb.utils.embedding_functions.openai_embedding_function import (
|
if not HAS_CHROMADB:
|
||||||
OpenAIEmbeddingFunction,
|
raise ImportError(
|
||||||
)
|
"ChromaDB is required for embedding functionality. "
|
||||||
|
"Please install it with 'pip install crewai[storage]'"
|
||||||
|
)
|
||||||
|
|
||||||
|
try:
|
||||||
|
from chromadb.utils.embedding_functions.openai_embedding_function import (
|
||||||
|
OpenAIEmbeddingFunction,
|
||||||
|
)
|
||||||
|
|
||||||
return OpenAIEmbeddingFunction(
|
return OpenAIEmbeddingFunction(
|
||||||
api_key=config.get("api_key") or os.getenv("OPENAI_API_KEY"),
|
api_key=config.get("api_key") or os.getenv("OPENAI_API_KEY"),
|
||||||
model_name=model_name,
|
model_name=model_name,
|
||||||
api_base=config.get("api_base", None),
|
api_base=config.get("api_base", None),
|
||||||
api_type=config.get("api_type", None),
|
api_type=config.get("api_type", None),
|
||||||
api_version=config.get("api_version", None),
|
api_version=config.get("api_version", None),
|
||||||
default_headers=config.get("default_headers", None),
|
default_headers=config.get("default_headers", None),
|
||||||
dimensions=config.get("dimensions", None),
|
dimensions=config.get("dimensions", None),
|
||||||
deployment_id=config.get("deployment_id", None),
|
deployment_id=config.get("deployment_id", None),
|
||||||
organization_id=config.get("organization_id", None),
|
organization_id=config.get("organization_id", None),
|
||||||
)
|
)
|
||||||
|
except ImportError:
|
||||||
|
raise ImportError(
|
||||||
|
"ChromaDB is required for embedding functionality. "
|
||||||
|
"Please install it with 'pip install crewai[storage]'"
|
||||||
|
)
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def _configure_azure(config, model_name):
|
def _configure_azure(config, model_name):
|
||||||
from chromadb.utils.embedding_functions.openai_embedding_function import (
|
if not HAS_CHROMADB:
|
||||||
OpenAIEmbeddingFunction,
|
raise ImportError(
|
||||||
)
|
"ChromaDB is required for embedding functionality. "
|
||||||
|
"Please install it with 'pip install crewai[storage]'"
|
||||||
|
)
|
||||||
|
|
||||||
|
try:
|
||||||
|
from chromadb.utils.embedding_functions.openai_embedding_function import (
|
||||||
|
OpenAIEmbeddingFunction,
|
||||||
|
)
|
||||||
|
|
||||||
return OpenAIEmbeddingFunction(
|
return OpenAIEmbeddingFunction(
|
||||||
api_key=config.get("api_key"),
|
api_key=config.get("api_key"),
|
||||||
api_base=config.get("api_base"),
|
api_base=config.get("api_base"),
|
||||||
api_type=config.get("api_type", "azure"),
|
api_type=config.get("api_type", "azure"),
|
||||||
api_version=config.get("api_version"),
|
api_version=config.get("api_version"),
|
||||||
model_name=model_name,
|
model_name=model_name,
|
||||||
default_headers=config.get("default_headers"),
|
default_headers=config.get("default_headers"),
|
||||||
dimensions=config.get("dimensions"),
|
dimensions=config.get("dimensions"),
|
||||||
deployment_id=config.get("deployment_id"),
|
deployment_id=config.get("deployment_id"),
|
||||||
organization_id=config.get("organization_id"),
|
organization_id=config.get("organization_id"),
|
||||||
)
|
)
|
||||||
|
except ImportError:
|
||||||
|
raise ImportError(
|
||||||
|
"ChromaDB is required for embedding functionality. "
|
||||||
|
"Please install it with 'pip install crewai[storage]'"
|
||||||
|
)
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def _configure_ollama(config, model_name):
|
def _configure_ollama(config, model_name):
|
||||||
from chromadb.utils.embedding_functions.ollama_embedding_function import (
|
if not HAS_CHROMADB:
|
||||||
OllamaEmbeddingFunction,
|
raise ImportError(
|
||||||
)
|
"ChromaDB is required for embedding functionality. "
|
||||||
|
"Please install it with 'pip install crewai[storage]'"
|
||||||
|
)
|
||||||
|
|
||||||
|
try:
|
||||||
|
from chromadb.utils.embedding_functions.ollama_embedding_function import (
|
||||||
|
OllamaEmbeddingFunction,
|
||||||
|
)
|
||||||
|
|
||||||
return OllamaEmbeddingFunction(
|
return OllamaEmbeddingFunction(
|
||||||
url=config.get("url", "http://localhost:11434/api/embeddings"),
|
url=config.get("url", "http://localhost:11434/api/embeddings"),
|
||||||
model_name=model_name,
|
model_name=model_name,
|
||||||
)
|
)
|
||||||
|
except ImportError:
|
||||||
|
raise ImportError(
|
||||||
|
"ChromaDB is required for embedding functionality. "
|
||||||
|
"Please install it with 'pip install crewai[storage]'"
|
||||||
|
)
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def _configure_vertexai(config, model_name):
|
def _configure_vertexai(config, model_name):
|
||||||
from chromadb.utils.embedding_functions.google_embedding_function import (
|
if not HAS_CHROMADB:
|
||||||
GoogleVertexEmbeddingFunction,
|
raise ImportError(
|
||||||
)
|
"ChromaDB is required for embedding functionality. "
|
||||||
|
"Please install it with 'pip install crewai[storage]'"
|
||||||
|
)
|
||||||
|
|
||||||
|
try:
|
||||||
|
from chromadb.utils.embedding_functions.google_embedding_function import (
|
||||||
|
GoogleVertexEmbeddingFunction,
|
||||||
|
)
|
||||||
|
|
||||||
return GoogleVertexEmbeddingFunction(
|
return GoogleVertexEmbeddingFunction(
|
||||||
model_name=model_name,
|
model_name=model_name,
|
||||||
api_key=config.get("api_key"),
|
api_key=config.get("api_key"),
|
||||||
project_id=config.get("project_id"),
|
project_id=config.get("project_id"),
|
||||||
region=config.get("region"),
|
region=config.get("region"),
|
||||||
)
|
)
|
||||||
|
except ImportError:
|
||||||
|
raise ImportError(
|
||||||
|
"ChromaDB is required for embedding functionality. "
|
||||||
|
"Please install it with 'pip install crewai[storage]'"
|
||||||
|
)
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def _configure_google(config, model_name):
|
def _configure_google(config, model_name):
|
||||||
from chromadb.utils.embedding_functions.google_embedding_function import (
|
if not HAS_CHROMADB:
|
||||||
GoogleGenerativeAiEmbeddingFunction,
|
raise ImportError(
|
||||||
)
|
"ChromaDB is required for embedding functionality. "
|
||||||
|
"Please install it with 'pip install crewai[storage]'"
|
||||||
|
)
|
||||||
|
|
||||||
|
try:
|
||||||
|
from chromadb.utils.embedding_functions.google_embedding_function import (
|
||||||
|
GoogleGenerativeAiEmbeddingFunction,
|
||||||
|
)
|
||||||
|
|
||||||
return GoogleGenerativeAiEmbeddingFunction(
|
return GoogleGenerativeAiEmbeddingFunction(
|
||||||
model_name=model_name,
|
model_name=model_name,
|
||||||
api_key=config.get("api_key"),
|
api_key=config.get("api_key"),
|
||||||
task_type=config.get("task_type"),
|
task_type=config.get("task_type"),
|
||||||
)
|
)
|
||||||
|
except ImportError:
|
||||||
|
raise ImportError(
|
||||||
|
"ChromaDB is required for embedding functionality. "
|
||||||
|
"Please install it with 'pip install crewai[storage]'"
|
||||||
|
)
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def _configure_cohere(config, model_name):
|
def _configure_cohere(config, model_name):
|
||||||
from chromadb.utils.embedding_functions.cohere_embedding_function import (
|
if not HAS_CHROMADB:
|
||||||
CohereEmbeddingFunction,
|
raise ImportError(
|
||||||
)
|
"ChromaDB is required for embedding functionality. "
|
||||||
|
"Please install it with 'pip install crewai[storage]'"
|
||||||
|
)
|
||||||
|
|
||||||
|
try:
|
||||||
|
from chromadb.utils.embedding_functions.cohere_embedding_function import (
|
||||||
|
CohereEmbeddingFunction,
|
||||||
|
)
|
||||||
|
|
||||||
return CohereEmbeddingFunction(
|
return CohereEmbeddingFunction(
|
||||||
model_name=model_name,
|
model_name=model_name,
|
||||||
api_key=config.get("api_key"),
|
api_key=config.get("api_key"),
|
||||||
)
|
)
|
||||||
|
except ImportError:
|
||||||
|
raise ImportError(
|
||||||
|
"ChromaDB is required for embedding functionality. "
|
||||||
|
"Please install it with 'pip install crewai[storage]'"
|
||||||
|
)
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def _configure_voyageai(config, model_name):
|
def _configure_voyageai(config, model_name):
|
||||||
from chromadb.utils.embedding_functions.voyageai_embedding_function import (
|
if not HAS_CHROMADB:
|
||||||
VoyageAIEmbeddingFunction,
|
raise ImportError(
|
||||||
)
|
"ChromaDB is required for embedding functionality. "
|
||||||
|
"Please install it with 'pip install crewai[storage]'"
|
||||||
|
)
|
||||||
|
|
||||||
|
try:
|
||||||
|
from chromadb.utils.embedding_functions.voyageai_embedding_function import (
|
||||||
|
VoyageAIEmbeddingFunction,
|
||||||
|
)
|
||||||
|
|
||||||
return VoyageAIEmbeddingFunction(
|
return VoyageAIEmbeddingFunction(
|
||||||
model_name=model_name,
|
model_name=model_name,
|
||||||
api_key=config.get("api_key"),
|
api_key=config.get("api_key"),
|
||||||
)
|
)
|
||||||
|
except ImportError:
|
||||||
|
raise ImportError(
|
||||||
|
"ChromaDB is required for embedding functionality. "
|
||||||
|
"Please install it with 'pip install crewai[storage]'"
|
||||||
|
)
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def _configure_bedrock(config, model_name):
|
def _configure_bedrock(config, model_name):
|
||||||
from chromadb.utils.embedding_functions.amazon_bedrock_embedding_function import (
|
if not HAS_CHROMADB:
|
||||||
AmazonBedrockEmbeddingFunction,
|
raise ImportError(
|
||||||
)
|
"ChromaDB is required for embedding functionality. "
|
||||||
|
"Please install it with 'pip install crewai[storage]'"
|
||||||
|
)
|
||||||
|
|
||||||
|
try:
|
||||||
|
from chromadb.utils.embedding_functions.amazon_bedrock_embedding_function import (
|
||||||
|
AmazonBedrockEmbeddingFunction,
|
||||||
|
)
|
||||||
|
|
||||||
# Allow custom model_name override with backwards compatibility
|
# Allow custom model_name override with backwards compatibility
|
||||||
kwargs = {"session": config.get("session")}
|
kwargs = {"session": config.get("session")}
|
||||||
if model_name is not None:
|
if model_name is not None:
|
||||||
kwargs["model_name"] = model_name
|
kwargs["model_name"] = model_name
|
||||||
return AmazonBedrockEmbeddingFunction(**kwargs)
|
return AmazonBedrockEmbeddingFunction(**kwargs)
|
||||||
|
except ImportError:
|
||||||
|
raise ImportError(
|
||||||
|
"ChromaDB is required for embedding functionality. "
|
||||||
|
"Please install it with 'pip install crewai[storage]'"
|
||||||
|
)
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def _configure_huggingface(config, model_name):
|
def _configure_huggingface(config, model_name):
|
||||||
from chromadb.utils.embedding_functions.huggingface_embedding_function import (
|
if not HAS_CHROMADB:
|
||||||
HuggingFaceEmbeddingServer,
|
raise ImportError(
|
||||||
)
|
"ChromaDB is required for embedding functionality. "
|
||||||
|
"Please install it with 'pip install crewai[storage]'"
|
||||||
|
)
|
||||||
|
|
||||||
|
try:
|
||||||
|
from chromadb.utils.embedding_functions.huggingface_embedding_function import (
|
||||||
|
HuggingFaceEmbeddingServer,
|
||||||
|
)
|
||||||
|
|
||||||
return HuggingFaceEmbeddingServer(
|
return HuggingFaceEmbeddingServer(
|
||||||
url=config.get("api_url"),
|
url=config.get("api_url"),
|
||||||
)
|
)
|
||||||
|
except ImportError:
|
||||||
|
raise ImportError(
|
||||||
|
"ChromaDB is required for embedding functionality. "
|
||||||
|
"Please install it with 'pip install crewai[storage]'"
|
||||||
|
)
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def _configure_watson(config, model_name):
|
def _configure_watson(config, model_name):
|
||||||
@@ -212,6 +351,12 @@ class EmbeddingConfigurator:
|
|||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def _configure_custom(config):
|
def _configure_custom(config):
|
||||||
|
if not HAS_CHROMADB:
|
||||||
|
raise ImportError(
|
||||||
|
"ChromaDB is required for embedding functionality. "
|
||||||
|
"Please install it with 'pip install crewai[storage]'"
|
||||||
|
)
|
||||||
|
|
||||||
custom_embedder = config.get("embedder")
|
custom_embedder = config.get("embedder")
|
||||||
if isinstance(custom_embedder, EmbeddingFunction):
|
if isinstance(custom_embedder, EmbeddingFunction):
|
||||||
try:
|
try:
|
||||||
|
|||||||
74
tests/test_optional_dependencies.py
Normal file
74
tests/test_optional_dependencies.py
Normal file
@@ -0,0 +1,74 @@
|
|||||||
|
import pytest
|
||||||
|
import importlib
|
||||||
|
import sys
|
||||||
|
from unittest.mock import patch
|
||||||
|
|
||||||
|
|
||||||
|
def test_import_without_chromadb():
|
||||||
|
"""Test that crewai can be imported without chromadb."""
|
||||||
|
with patch.dict(sys.modules, {"chromadb": None}):
|
||||||
|
if "crewai.memory.storage.rag_storage" in sys.modules:
|
||||||
|
importlib.reload(sys.modules["crewai.memory.storage.rag_storage"])
|
||||||
|
if "crewai.knowledge.storage.knowledge_storage" in sys.modules:
|
||||||
|
importlib.reload(sys.modules["crewai.knowledge.storage.knowledge_storage"])
|
||||||
|
if "crewai.utilities.embedding_configurator" in sys.modules:
|
||||||
|
importlib.reload(sys.modules["crewai.utilities.embedding_configurator"])
|
||||||
|
|
||||||
|
import crewai
|
||||||
|
|
||||||
|
from crewai import Agent, Task, Crew, Process
|
||||||
|
|
||||||
|
agent = Agent(role="Test Agent", goal="Test Goal", backstory="Test Backstory")
|
||||||
|
task = Task(description="Test Task", agent=agent)
|
||||||
|
crew = Crew(agents=[agent], tasks=[task], process=Process.sequential)
|
||||||
|
|
||||||
|
|
||||||
|
def test_memory_storage_without_chromadb():
|
||||||
|
"""Test that memory storage raises appropriate error when chromadb is not available."""
|
||||||
|
with patch.dict(sys.modules, {"chromadb": None}):
|
||||||
|
if "crewai.memory.storage.rag_storage" in sys.modules:
|
||||||
|
importlib.reload(sys.modules["crewai.memory.storage.rag_storage"])
|
||||||
|
|
||||||
|
from crewai.memory.storage.rag_storage import RAGStorage, HAS_CHROMADB
|
||||||
|
|
||||||
|
assert not HAS_CHROMADB
|
||||||
|
|
||||||
|
with pytest.raises(ImportError) as excinfo:
|
||||||
|
storage = RAGStorage()
|
||||||
|
storage._initialize_app()
|
||||||
|
|
||||||
|
assert "ChromaDB is required for memory storage features" in str(excinfo.value)
|
||||||
|
|
||||||
|
|
||||||
|
def test_knowledge_storage_without_chromadb():
|
||||||
|
"""Test that knowledge storage raises appropriate error when chromadb is not available."""
|
||||||
|
with patch.dict(sys.modules, {"chromadb": None}):
|
||||||
|
if "crewai.knowledge.storage.knowledge_storage" in sys.modules:
|
||||||
|
importlib.reload(sys.modules["crewai.knowledge.storage.knowledge_storage"])
|
||||||
|
|
||||||
|
from crewai.knowledge.storage.knowledge_storage import KnowledgeStorage, HAS_CHROMADB
|
||||||
|
|
||||||
|
assert not HAS_CHROMADB
|
||||||
|
|
||||||
|
with pytest.raises(ImportError) as excinfo:
|
||||||
|
storage = KnowledgeStorage()
|
||||||
|
storage.initialize_knowledge_storage()
|
||||||
|
|
||||||
|
assert "ChromaDB is required for knowledge storage features" in str(excinfo.value)
|
||||||
|
|
||||||
|
|
||||||
|
def test_embedding_configurator_without_chromadb():
|
||||||
|
"""Test that embedding configurator raises appropriate error when chromadb is not available."""
|
||||||
|
with patch.dict(sys.modules, {"chromadb": None}):
|
||||||
|
if "crewai.utilities.embedding_configurator" in sys.modules:
|
||||||
|
importlib.reload(sys.modules["crewai.utilities.embedding_configurator"])
|
||||||
|
|
||||||
|
from crewai.utilities.embedding_configurator import EmbeddingConfigurator, HAS_CHROMADB
|
||||||
|
|
||||||
|
assert not HAS_CHROMADB
|
||||||
|
|
||||||
|
with pytest.raises(ImportError) as excinfo:
|
||||||
|
configurator = EmbeddingConfigurator()
|
||||||
|
configurator.configure_embedder()
|
||||||
|
|
||||||
|
assert "ChromaDB is required for embedding functionality" in str(excinfo.value)
|
||||||
Reference in New Issue
Block a user