mirror of
https://github.com/crewAIInc/crewAI.git
synced 2026-01-24 07:38:14 +00:00
feat: lazy import chromadb
This commit is contained in:
@@ -1,21 +1,24 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
import logging
|
import logging
|
||||||
import traceback
|
import traceback
|
||||||
from typing import Any, cast
|
from typing import TYPE_CHECKING, Any, cast
|
||||||
import warnings
|
import warnings
|
||||||
|
|
||||||
from crewai.knowledge.storage.base_knowledge_storage import BaseKnowledgeStorage
|
from crewai.knowledge.storage.base_knowledge_storage import BaseKnowledgeStorage
|
||||||
from crewai.rag.chromadb.config import ChromaDBConfig
|
|
||||||
from crewai.rag.chromadb.types import ChromaEmbeddingFunctionWrapper
|
|
||||||
from crewai.rag.config.utils import get_rag_client
|
from crewai.rag.config.utils import get_rag_client
|
||||||
from crewai.rag.core.base_client import BaseClient
|
from crewai.rag.core.base_client import BaseClient
|
||||||
from crewai.rag.core.base_embeddings_provider import BaseEmbeddingsProvider
|
from crewai.rag.core.base_embeddings_provider import BaseEmbeddingsProvider
|
||||||
from crewai.rag.embeddings.factory import build_embedder
|
from crewai.rag.embeddings.factory import build_embedder
|
||||||
from crewai.rag.embeddings.types import ProviderSpec
|
|
||||||
from crewai.rag.factory import create_client
|
from crewai.rag.factory import create_client
|
||||||
from crewai.rag.types import BaseRecord, SearchResult
|
from crewai.rag.types import BaseRecord, SearchResult
|
||||||
from crewai.utilities.logger import Logger
|
from crewai.utilities.logger import Logger
|
||||||
|
|
||||||
|
|
||||||
|
if TYPE_CHECKING:
|
||||||
|
from crewai.rag.embeddings.types import ProviderSpec
|
||||||
|
|
||||||
|
|
||||||
class KnowledgeStorage(BaseKnowledgeStorage):
|
class KnowledgeStorage(BaseKnowledgeStorage):
|
||||||
"""
|
"""
|
||||||
Extends Storage to handle embeddings for memory entries, improving
|
Extends Storage to handle embeddings for memory entries, improving
|
||||||
@@ -30,6 +33,9 @@ class KnowledgeStorage(BaseKnowledgeStorage):
|
|||||||
| None = None,
|
| None = None,
|
||||||
collection_name: str | None = None,
|
collection_name: str | None = None,
|
||||||
) -> None:
|
) -> None:
|
||||||
|
from crewai.rag.chromadb.config import ChromaDBConfig
|
||||||
|
from crewai.rag.chromadb.types import ChromaEmbeddingFunctionWrapper
|
||||||
|
|
||||||
self.collection_name = collection_name
|
self.collection_name = collection_name
|
||||||
self._client: BaseClient | None = None
|
self._client: BaseClient | None = None
|
||||||
|
|
||||||
|
|||||||
@@ -5,8 +5,6 @@ import traceback
|
|||||||
from typing import TYPE_CHECKING, Any, cast
|
from typing import TYPE_CHECKING, Any, cast
|
||||||
import warnings
|
import warnings
|
||||||
|
|
||||||
from crewai.rag.chromadb.config import ChromaDBConfig
|
|
||||||
from crewai.rag.chromadb.types import ChromaEmbeddingFunctionWrapper
|
|
||||||
from crewai.rag.config.utils import get_rag_client
|
from crewai.rag.config.utils import get_rag_client
|
||||||
from crewai.rag.embeddings.factory import build_embedder
|
from crewai.rag.embeddings.factory import build_embedder
|
||||||
from crewai.rag.factory import create_client
|
from crewai.rag.factory import create_client
|
||||||
@@ -37,6 +35,9 @@ class RAGStorage(BaseRAGStorage):
|
|||||||
crew: Crew | None = None,
|
crew: Crew | None = None,
|
||||||
path: str | None = None,
|
path: str | None = None,
|
||||||
) -> None:
|
) -> None:
|
||||||
|
from crewai.rag.chromadb.config import ChromaDBConfig
|
||||||
|
from crewai.rag.chromadb.types import ChromaEmbeddingFunctionWrapper
|
||||||
|
|
||||||
super().__init__(type, allow_reset, embedder_config, crew)
|
super().__init__(type, allow_reset, embedder_config, crew)
|
||||||
crew_agents = crew.agents if crew else []
|
crew_agents = crew.agents if crew else []
|
||||||
sanitized_roles = [self._sanitize_role(agent.role) for agent in crew_agents]
|
sanitized_roles = [self._sanitize_role(agent.role) for agent in crew_agents]
|
||||||
|
|||||||
@@ -1,11 +1,12 @@
|
|||||||
"""ChromaDB configuration model."""
|
"""ChromaDB configuration model."""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
from dataclasses import field
|
from dataclasses import field
|
||||||
import os
|
import os
|
||||||
from typing import Literal, cast
|
from typing import TYPE_CHECKING, Literal, cast
|
||||||
import warnings
|
import warnings
|
||||||
|
|
||||||
from chromadb.config import Settings
|
|
||||||
from pydantic.dataclasses import dataclass as pyd_dataclass
|
from pydantic.dataclasses import dataclass as pyd_dataclass
|
||||||
|
|
||||||
from crewai.rag.chromadb.constants import (
|
from crewai.rag.chromadb.constants import (
|
||||||
@@ -13,10 +14,15 @@ from crewai.rag.chromadb.constants import (
|
|||||||
DEFAULT_STORAGE_PATH,
|
DEFAULT_STORAGE_PATH,
|
||||||
DEFAULT_TENANT,
|
DEFAULT_TENANT,
|
||||||
)
|
)
|
||||||
from crewai.rag.chromadb.types import ChromaEmbeddingFunctionWrapper
|
|
||||||
from crewai.rag.config.base import BaseRagConfig
|
from crewai.rag.config.base import BaseRagConfig
|
||||||
|
|
||||||
|
|
||||||
|
if TYPE_CHECKING:
|
||||||
|
from chromadb.config import Settings
|
||||||
|
|
||||||
|
from crewai.rag.chromadb.types import ChromaEmbeddingFunctionWrapper
|
||||||
|
|
||||||
|
|
||||||
warnings.filterwarnings(
|
warnings.filterwarnings(
|
||||||
"ignore",
|
"ignore",
|
||||||
message=".*Mixing V1 models and V2 models.*",
|
message=".*Mixing V1 models and V2 models.*",
|
||||||
@@ -37,6 +43,8 @@ def _default_settings() -> Settings:
|
|||||||
Returns:
|
Returns:
|
||||||
Settings with persistent storage and reset enabled.
|
Settings with persistent storage and reset enabled.
|
||||||
"""
|
"""
|
||||||
|
from chromadb.config import Settings
|
||||||
|
|
||||||
return Settings(
|
return Settings(
|
||||||
persist_directory=DEFAULT_STORAGE_PATH,
|
persist_directory=DEFAULT_STORAGE_PATH,
|
||||||
allow_reset=True,
|
allow_reset=True,
|
||||||
@@ -54,6 +62,8 @@ def _default_embedding_function() -> ChromaEmbeddingFunctionWrapper:
|
|||||||
OpenAIEmbeddingFunction,
|
OpenAIEmbeddingFunction,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
from crewai.rag.chromadb.types import ChromaEmbeddingFunctionWrapper
|
||||||
|
|
||||||
return cast(
|
return cast(
|
||||||
ChromaEmbeddingFunctionWrapper,
|
ChromaEmbeddingFunctionWrapper,
|
||||||
OpenAIEmbeddingFunction(
|
OpenAIEmbeddingFunction(
|
||||||
|
|||||||
@@ -1,13 +1,18 @@
|
|||||||
"""Factory functions for creating ChromaDB clients."""
|
"""Factory functions for creating ChromaDB clients."""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
from hashlib import md5
|
from hashlib import md5
|
||||||
import os
|
import os
|
||||||
|
from typing import TYPE_CHECKING
|
||||||
|
|
||||||
from chromadb import PersistentClient
|
from chromadb import PersistentClient
|
||||||
import portalocker
|
import portalocker
|
||||||
|
|
||||||
from crewai.rag.chromadb.client import ChromaDBClient
|
|
||||||
from crewai.rag.chromadb.config import ChromaDBConfig
|
if TYPE_CHECKING:
|
||||||
|
from crewai.rag.chromadb.client import ChromaDBClient
|
||||||
|
from crewai.rag.chromadb.config import ChromaDBConfig
|
||||||
|
|
||||||
|
|
||||||
def create_client(config: ChromaDBConfig) -> ChromaDBClient:
|
def create_client(config: ChromaDBConfig) -> ChromaDBClient:
|
||||||
@@ -22,6 +27,7 @@ def create_client(config: ChromaDBConfig) -> ChromaDBClient:
|
|||||||
Notes:
|
Notes:
|
||||||
Need to update to use chromadb.Client to support more client types in the near future.
|
Need to update to use chromadb.Client to support more client types in the near future.
|
||||||
"""
|
"""
|
||||||
|
from crewai.rag.chromadb.client import ChromaDBClient
|
||||||
|
|
||||||
persist_dir = config.settings.persist_directory
|
persist_dir = config.settings.persist_directory
|
||||||
os.makedirs(persist_dir, exist_ok=True)
|
os.makedirs(persist_dir, exist_ok=True)
|
||||||
|
|||||||
Reference in New Issue
Block a user