mirror of
https://github.com/crewAIInc/crewAI.git
synced 2026-01-10 08:38:30 +00:00
Address lint failures and improve exception handling
- Remove unused imports (os from rag_storage.py, pytest from test_memory_fallback.py) - Add specific exception handling in fallback mechanism (ConnectionError, ImportError, ValueError) - Add comprehensive logging to track embedding provider selection and fallback attempts - Resolves CI lint failures and addresses PR review feedback Co-Authored-By: João <joao@crewai.com>
This commit is contained in:
@@ -1,7 +1,6 @@
|
|||||||
import contextlib
|
import contextlib
|
||||||
import io
|
import io
|
||||||
import logging
|
import logging
|
||||||
import os
|
|
||||||
import shutil
|
import shutil
|
||||||
import uuid
|
import uuid
|
||||||
from typing import Any, Dict, List, Optional
|
from typing import Any, Dict, List, Optional
|
||||||
|
|||||||
@@ -57,25 +57,36 @@ class EmbeddingConfigurator:
|
|||||||
|
|
||||||
def create_default_embedding_with_fallback(self) -> EmbeddingFunction:
|
def create_default_embedding_with_fallback(self) -> EmbeddingFunction:
|
||||||
"""Create an embedding function with fallback providers when OpenAI API key is not available."""
|
"""Create an embedding function with fallback providers when OpenAI API key is not available."""
|
||||||
|
import logging
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
if os.getenv("OPENAI_API_KEY"):
|
if os.getenv("OPENAI_API_KEY"):
|
||||||
|
logger.info("Using OpenAI embeddings")
|
||||||
return self._create_default_embedding_function()
|
return self._create_default_embedding_function()
|
||||||
|
|
||||||
|
logger.warning("OpenAI API key not found, attempting fallback providers")
|
||||||
|
|
||||||
try:
|
try:
|
||||||
|
logger.info("Attempting Ollama embedding provider")
|
||||||
return self.configure_embedder({
|
return self.configure_embedder({
|
||||||
"provider": "ollama",
|
"provider": "ollama",
|
||||||
"config": {"url": "http://localhost:11434/api/embeddings"},
|
"config": {"url": "http://localhost:11434/api/embeddings"},
|
||||||
"model": "nomic-embed-text"
|
"model": "nomic-embed-text"
|
||||||
})
|
})
|
||||||
except Exception:
|
except (ConnectionError, ImportError, ValueError) as e:
|
||||||
|
logger.warning(f"Ollama fallback failed: {str(e)}, trying HuggingFace")
|
||||||
try:
|
try:
|
||||||
|
logger.info("Attempting HuggingFace embedding provider")
|
||||||
return self.configure_embedder({
|
return self.configure_embedder({
|
||||||
"provider": "huggingface",
|
"provider": "huggingface",
|
||||||
"config": {"api_url": "https://api-inference.huggingface.co/pipeline/feature-extraction/sentence-transformers/all-MiniLM-L6-v2"}
|
"config": {"api_url": "https://api-inference.huggingface.co/pipeline/feature-extraction/sentence-transformers/all-MiniLM-L6-v2"}
|
||||||
})
|
})
|
||||||
except Exception:
|
except (ConnectionError, ImportError, ValueError) as e:
|
||||||
|
logger.warning(f"HuggingFace fallback failed: {str(e)}, using local SentenceTransformers")
|
||||||
from chromadb.utils.embedding_functions.sentence_transformer_embedding_function import (
|
from chromadb.utils.embedding_functions.sentence_transformer_embedding_function import (
|
||||||
SentenceTransformerEmbeddingFunction,
|
SentenceTransformerEmbeddingFunction,
|
||||||
)
|
)
|
||||||
|
logger.info("Using local SentenceTransformers embedding provider")
|
||||||
return SentenceTransformerEmbeddingFunction(model_name="all-MiniLM-L6-v2")
|
return SentenceTransformerEmbeddingFunction(model_name="all-MiniLM-L6-v2")
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
|
|||||||
@@ -1,5 +1,4 @@
|
|||||||
import os
|
import os
|
||||||
import pytest
|
|
||||||
from unittest.mock import patch
|
from unittest.mock import patch
|
||||||
|
|
||||||
from crewai import Agent, Task, Crew, Process
|
from crewai import Agent, Task, Crew, Process
|
||||||
|
|||||||
Reference in New Issue
Block a user