preparing new verison

* use copy to split testing and training on crews

* make tests handle new copy functionality on train and test

* fix last test

* fix test

docs/concepts/memory.mdx

@@ -105,9 +105,48 @@ my_crew = Crew(
     process=Process.sequential,
     memory=True,
     verbose=True,
-    embedder=embedding_functions.OpenAIEmbeddingFunction(
-            api_key=os.getenv("OPENAI_API_KEY"), model_name="text-embedding-3-small"
-        )
+    embedder={
+        "provider": "openai",
+        "config": {
+            "model": 'text-embedding-3-small'
+        }
+    }
+)
+```
+Alternatively, you can directly pass the OpenAIEmbeddingFunction to the embedder parameter.
+
+Example:
+```python Code
+from crewai import Crew, Agent, Task, Process
+from chromadb.utils.embedding_functions.openai_embedding_function import OpenAIEmbeddingFunction
+
+my_crew = Crew(
+    agents=[...],
+    tasks=[...],
+    process=Process.sequential,
+    memory=True,
+    verbose=True,
+    embedder=OpenAIEmbeddingFunction(api_key=os.getenv("OPENAI_API_KEY"), model_name="text-embedding-3-small"),
+)
+```
+
+### Using Ollama embeddings
+
+```python Code
+from crewai import Crew, Agent, Task, Process
+
+my_crew = Crew(
+    agents=[...],
+    tasks=[...],
+    process=Process.sequential,
+    memory=True,
+    verbose=True,
+    embedder={
+        "provider": "ollama",
+        "config": {
+            "model": "mxbai-embed-large"
+        }
+    }
 )
 ```
 
@@ -122,10 +161,13 @@ my_crew = Crew(
     process=Process.sequential,
     memory=True,
     verbose=True,
-    embedder=embedding_functions.OpenAIEmbeddingFunction(
-            api_key=os.getenv("OPENAI_API_KEY"),
-            model_name="text-embedding-ada-002"
-    )
+    embedder={
+        "provider": "google",
+        "config": {
+            "api_key": "<YOUR_API_KEY>",
+            "model_name": "<model_name>"
+        }
+    }
 )
 ```
 
@@ -181,10 +223,32 @@ my_crew = Crew(
     process=Process.sequential,
     memory=True,
     verbose=True,
-    embedder=embedding_functions.CohereEmbeddingFunction(
-        api_key=YOUR_API_KEY,
-        model_name="<model_name>"
-    )
+    embedder={
+        "provider": "cohere",
+        "config": {
+            "api_key": "YOUR_API_KEY",
+            "model_name": "<model_name>"
+        }
+    }
+)
+```
+### Using HuggingFace embeddings
+
+```python Code
+from crewai import Crew, Agent, Task, Process
+
+my_crew = Crew(
+    agents=[...],
+    tasks=[...],
+    process=Process.sequential,
+    memory=True,
+    verbose=True,
+    embedder={
+        "provider": "huggingface",
+        "config": {
+            "api_url": "<api_url>",
+        }
+    }
 )
 ```
 

pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "crewai"
-version = "0.74.2"
+version = "0.75.1"
 description = "Cutting-edge framework for orchestrating role-playing, autonomous AI agents. By fostering collaborative intelligence, CrewAI empowers agents to work together seamlessly, tackling complex tasks."
 readme = "README.md"
 requires-python = ">=3.10,<=3.13"

src/crewai/__init__.py

@@ -14,5 +14,5 @@ warnings.filterwarnings(
     category=UserWarning,
     module="pydantic.main",
 )
-__version__ = "0.74.2"
+__version__ = "0.75.1"
 __all__ = ["Agent", "Crew", "Process", "Task", "Pipeline", "Router", "LLM", "Flow"]

src/crewai/cli/cli.py

@@ -32,10 +32,11 @@ def crewai():
 @crewai.command()
 @click.argument("type", type=click.Choice(["crew", "pipeline", "flow"]))
 @click.argument("name")
-def create(type, name):
+@click.option("--provider", type=str, help="The provider to use for the crew")
+def create(type, name, provider):
     """Create a new crew, pipeline, or flow."""
     if type == "crew":
-        create_crew(name)
+        create_crew(name, provider)
     elif type == "pipeline":
         create_pipeline(name)
     elif type == "flow":

src/crewai/cli/create_crew.py

@@ -70,18 +70,19 @@ def copy_template_files(folder_path, name, class_name, parent_folder):
             copy_template(src_file, dst_file, name, class_name, folder_path.name)
 
 
-def create_crew(name, parent_folder=None):
+def create_crew(name, provider=None, parent_folder=None):
     folder_path, folder_name, class_name = create_folder_structure(name, parent_folder)
     env_vars = load_env_vars(folder_path)
 
-    provider_models = get_provider_data()
-    if not provider_models:
-        return
+    if not provider:
+        provider_models = get_provider_data()
+        if not provider_models:
+            return
 
-    selected_provider = select_provider(provider_models)
-    if not selected_provider:
-        return
-    provider = selected_provider
+        selected_provider = select_provider(provider_models)
+        if not selected_provider:
+            return
+        provider = selected_provider
 
     # selected_model = select_model(provider, provider_models)
     # if not selected_model:

src/crewai/cli/templates/crew/pyproject.toml

@@ -5,7 +5,7 @@ description = "{{name}} using crewAI"
 authors = [{ name = "Your Name", email = "you@example.com" }]
 requires-python = ">=3.10,<=3.13"
 dependencies = [
-    "crewai[tools]>=0.74.2,<1.0.0"
+    "crewai[tools]>=0.75.1,<1.0.0"
 ]
 
 [project.scripts]

src/crewai/cli/templates/flow/pyproject.toml

@@ -5,7 +5,7 @@ description = "{{name}} using crewAI"
 authors = [{ name = "Your Name", email = "you@example.com" }]
 requires-python = ">=3.10,<=3.13"
 dependencies = [
-    "crewai[tools]>=0.74.2,<1.0.0",
+    "crewai[tools]>=0.75.1,<1.0.0",
 ]
 
 [project.scripts]

src/crewai/cli/templates/pipeline/pyproject.toml

@@ -6,7 +6,7 @@ authors = ["Your Name <you@example.com>"]
 
 [tool.poetry.dependencies]
 python = ">=3.10,<=3.13"
-crewai = { extras = ["tools"], version = ">=0.74.2,<1.0.0" }
+crewai = { extras = ["tools"], version = ">=0.75.1,<1.0.0" }
 asyncio = "*"
 
 [tool.poetry.scripts]

src/crewai/cli/templates/pipeline_router/pyproject.toml

@@ -5,7 +5,7 @@ description = "{{name}} using crewAI"
 authors = ["Your Name <you@example.com>"]
 requires-python = ">=3.10,<=3.13"
 dependencies = [
-    "crewai[tools]>=0.74.2,<1.0.0"
+    "crewai[tools]>=0.75.1,<1.0.0"
 ]
 
 [project.scripts]

src/crewai/cli/templates/tool/pyproject.toml

@@ -5,6 +5,6 @@ description = "Power up your crews with {{folder_name}}"
 readme = "README.md"
 requires-python = ">=3.10,<=3.13"
 dependencies = [
-    "crewai[tools]>=0.74.2"
+    "crewai[tools]>=0.75.1"
 ]
 

src/crewai/crew.py

@@ -435,15 +435,16 @@ class Crew(BaseModel):
         self, n_iterations: int, filename: str, inputs: Optional[Dict[str, Any]] = {}
     ) -> None:
         """Trains the crew for a given number of iterations."""
-        self._setup_for_training(filename)
+        train_crew = self.copy()
+        train_crew._setup_for_training(filename)
 
         for n_iteration in range(n_iterations):
-            self._train_iteration = n_iteration
-            self.kickoff(inputs=inputs)
+            train_crew._train_iteration = n_iteration
+            train_crew.kickoff(inputs=inputs)
 
         training_data = CrewTrainingHandler(TRAINING_DATA_FILE).load()
 
-        for agent in self.agents:
+        for agent in train_crew.agents:
             result = TaskEvaluator(agent).evaluate_training_data(
                 training_data=training_data, agent_id=str(agent.id)
             )
@@ -987,17 +988,19 @@ class Crew(BaseModel):
         inputs: Optional[Dict[str, Any]] = None,
     ) -> None:
         """Test and evaluate the Crew with the given inputs for n iterations concurrently using concurrent.futures."""
-        self._test_execution_span = self._telemetry.test_execution_span(
-            self,
+        test_crew = self.copy()
+
+        self._test_execution_span = test_crew._telemetry.test_execution_span(
+            test_crew,
             n_iterations,
             inputs,
             openai_model_name,  # type: ignore[arg-type]
         )  # type: ignore[arg-type]
-        evaluator = CrewEvaluator(self, openai_model_name)  # type: ignore[arg-type]
+        evaluator = CrewEvaluator(test_crew, openai_model_name)  # type: ignore[arg-type]
 
         for i in range(1, n_iterations + 1):
             evaluator.set_iteration(i)
-            self.kickoff(inputs=inputs)
+            test_crew.kickoff(inputs=inputs)
 
         evaluator.print_crew_evaluation_result()
 

src/crewai/memory/entity/entity_memory.py

@@ -16,7 +16,7 @@ class EntityMemory(Memory):
             if storage
             else RAGStorage(
                 type="entities",
-                allow_reset=False,
+                allow_reset=True,
                 embedder_config=embedder_config,
                 crew=crew,
             )

src/crewai/memory/storage/rag_storage.py

@@ -8,6 +8,9 @@ from typing import Any, Dict, List, Optional
 from crewai.memory.storage.base_rag_storage import BaseRAGStorage
 from crewai.utilities.paths import db_storage_path
 from chromadb.api import ClientAPI
+from chromadb.api.types import validate_embedding_function
+from chromadb import Documents, EmbeddingFunction, Embeddings
+from typing import cast
 
 
 @contextlib.contextmanager
@@ -41,16 +44,93 @@ class RAGStorage(BaseRAGStorage):
         self.agents = agents
 
         self.type = type
-        self.embedder_config = embedder_config or self._create_embedding_function()
+
         self.allow_reset = allow_reset
         self._initialize_app()
 
+    def _set_embedder_config(self):
+        import chromadb.utils.embedding_functions as embedding_functions
+
+        if self.embedder_config is None:
+            self.embedder_config = self._create_default_embedding_function()
+
+        if isinstance(self.embedder_config, dict):
+            provider = self.embedder_config.get("provider")
+            config = self.embedder_config.get("config", {})
+            model_name = config.get("model")
+            if provider == "openai":
+                self.embedder_config = embedding_functions.OpenAIEmbeddingFunction(
+                    api_key=config.get("api_key") or os.getenv("OPENAI_API_KEY"),
+                    model_name=model_name,
+                )
+            elif provider == "azure":
+                self.embedder_config = embedding_functions.OpenAIEmbeddingFunction(
+                    api_key=config.get("api_key"),
+                    api_base=config.get("api_base"),
+                    api_type=config.get("api_type", "azure"),
+                    api_version=config.get("api_version"),
+                    model_name=model_name,
+                )
+            elif provider == "ollama":
+                from openai import OpenAI
+
+                class OllamaEmbeddingFunction(EmbeddingFunction):
+                    def __call__(self, input: Documents) -> Embeddings:
+                        client = OpenAI(
+                            base_url="http://localhost:11434/v1",
+                            api_key=config.get("api_key", "ollama"),
+                        )
+                        try:
+                            response = client.embeddings.create(
+                                input=input, model=model_name
+                            )
+                            embeddings = [item.embedding for item in response.data]
+                            return cast(Embeddings, embeddings)
+                        except Exception as e:
+                            raise e
+
+                self.embedder_config = OllamaEmbeddingFunction()
+            elif provider == "vertexai":
+                self.embedder_config = (
+                    embedding_functions.GoogleVertexEmbeddingFunction(
+                        model_name=model_name,
+                        api_key=config.get("api_key"),
+                    )
+                )
+            elif provider == "google":
+                self.embedder_config = (
+                    embedding_functions.GoogleGenerativeAiEmbeddingFunction(
+                        model_name=model_name,
+                        api_key=config.get("api_key"),
+                    )
+                )
+            elif provider == "cohere":
+                self.embedder_config = embedding_functions.CohereEmbeddingFunction(
+                    model_name=model_name,
+                    api_key=config.get("api_key"),
+                )
+            elif provider == "huggingface":
+                self.embedder_config = embedding_functions.HuggingFaceEmbeddingServer(
+                    url=config.get("api_url"),
+                )
+            else:
+                raise Exception(
+                    f"Unsupported embedding provider: {provider}, supported providers: [openai, azure, ollama, vertexai, google, cohere, huggingface]"
+                )
+        else:
+            validate_embedding_function(self.embedder_config)  # type: ignore # used for validating embedder_config if defined a embedding function/class
+            self.embedder_config = self.embedder_config
+
     def _initialize_app(self):
         import chromadb
+        from chromadb.config import Settings
 
+        self._set_embedder_config()
         chroma_client = chromadb.PersistentClient(
-            path=f"{db_storage_path()}/{self.type}/{self.agents}"
+            path=f"{db_storage_path()}/{self.type}/{self.agents}",
+            settings=Settings(allow_reset=self.allow_reset),
         )
+
         self.app = chroma_client
 
         try:
@@ -122,11 +202,15 @@ class RAGStorage(BaseRAGStorage):
             if self.app:
                 self.app.reset()
         except Exception as e:
-            raise Exception(
-                f"An error occurred while resetting the {self.type} memory: {e}"
-            )
+            if "attempt to write a readonly database" in str(e):
+                # Ignore this specific error
+                pass
+            else:
+                raise Exception(
+                    f"An error occurred while resetting the {self.type} memory: {e}"
+                )
 
-    def _create_embedding_function(self):
+    def _create_default_embedding_function(self):
         import chromadb.utils.embedding_functions as embedding_functions
 
         return embedding_functions.OpenAIEmbeddingFunction(

tests/crew_test.py

@@ -9,6 +9,7 @@ from unittest.mock import MagicMock, patch
 import instructor
 import pydantic_core
 import pytest
+
 from crewai.agent import Agent
 from crewai.agents.cache import CacheHandler
 from crewai.crew import Crew
@@ -497,6 +498,7 @@ def test_cache_hitting_between_agents():
 @pytest.mark.vcr(filter_headers=["authorization"])
 def test_api_calls_throttling(capsys):
     from unittest.mock import patch
+
     from crewai_tools import tool
 
     @tool
@@ -779,11 +781,14 @@ def test_async_task_execution_call_count():
     list_important_history.output = mock_task_output
     write_article.output = mock_task_output
 
-    with patch.object(
-        Task, "execute_sync", return_value=mock_task_output
-    ) as mock_execute_sync, patch.object(
-        Task, "execute_async", return_value=mock_future
-    ) as mock_execute_async:
+    with (
+        patch.object(
+            Task, "execute_sync", return_value=mock_task_output
+        ) as mock_execute_sync,
+        patch.object(
+            Task, "execute_async", return_value=mock_future
+        ) as mock_execute_async,
+    ):
         crew.kickoff()
 
         assert mock_execute_async.call_count == 2
@@ -1105,6 +1110,7 @@ def test_dont_set_agents_step_callback_if_already_set():
 @pytest.mark.vcr(filter_headers=["authorization"])
 def test_crew_function_calling_llm():
     from unittest.mock import patch
+
     from crewai_tools import tool
 
     llm = "gpt-4o"
@@ -1448,52 +1454,6 @@ def test_crew_does_not_interpolate_without_inputs():
             interpolate_task_inputs.assert_not_called()
 
 
-# def test_crew_partial_inputs():
-#     agent = Agent(
-#         role="{topic} Researcher",
-#         goal="Express hot takes on {topic}.",
-#         backstory="You have a lot of experience with {topic}.",
-#     )
-
-#     task = Task(
-#         description="Give me an analysis around {topic}.",
-#         expected_output="{points} bullet points about {topic}.",
-#     )
-
-#     crew = Crew(agents=[agent], tasks=[task], inputs={"topic": "AI"})
-#     inputs = {"topic": "AI"}
-#     crew._interpolate_inputs(inputs=inputs)  # Manual call for now
-
-#     assert crew.tasks[0].description == "Give me an analysis around AI."
-#     assert crew.tasks[0].expected_output == "{points} bullet points about AI."
-#     assert crew.agents[0].role == "AI Researcher"
-#     assert crew.agents[0].goal == "Express hot takes on AI."
-#     assert crew.agents[0].backstory == "You have a lot of experience with AI."
-
-
-# def test_crew_invalid_inputs():
-#     agent = Agent(
-#         role="{topic} Researcher",
-#         goal="Express hot takes on {topic}.",
-#         backstory="You have a lot of experience with {topic}.",
-#     )
-
-#     task = Task(
-#         description="Give me an analysis around {topic}.",
-#         expected_output="{points} bullet points about {topic}.",
-#     )
-
-#     crew = Crew(agents=[agent], tasks=[task], inputs={"subject": "AI"})
-#     inputs = {"subject": "AI"}
-#     crew._interpolate_inputs(inputs=inputs)  # Manual call for now
-
-#     assert crew.tasks[0].description == "Give me an analysis around {topic}."
-#     assert crew.tasks[0].expected_output == "{points} bullet points about {topic}."
-#     assert crew.agents[0].role == "{topic} Researcher"
-#     assert crew.agents[0].goal == "Express hot takes on {topic}."
-#     assert crew.agents[0].backstory == "You have a lot of experience with {topic}."
-
-
 def test_task_callback_on_crew():
     from unittest.mock import MagicMock, patch
 
@@ -1770,7 +1730,10 @@ def test_manager_agent_with_tools_raises_exception():
 @patch("crewai.crew.Crew.kickoff")
 @patch("crewai.crew.CrewTrainingHandler")
 @patch("crewai.crew.TaskEvaluator")
-def test_crew_train_success(task_evaluator, crew_training_handler, kickoff):
+@patch("crewai.crew.Crew.copy")
+def test_crew_train_success(
+    copy_mock, task_evaluator, crew_training_handler, kickoff_mock
+):
     task = Task(
         description="Come up with a list of 5 interesting ideas to explore for an article, then write one amazing paragraph highlight for each idea that showcases how good an article about this topic could be. Return the list of ideas with their paragraph and your notes.",
         expected_output="5 bullet points with a paragraph for each idea.",
@@ -1781,9 +1744,19 @@ def test_crew_train_success(task_evaluator, crew_training_handler, kickoff):
         agents=[researcher, writer],
         tasks=[task],
     )
+
+    # Create a mock for the copied crew
+    copy_mock.return_value = crew
+
     crew.train(
         n_iterations=2, inputs={"topic": "AI"}, filename="trained_agents_data.pkl"
     )
+
+    # Ensure kickoff is called on the copied crew
+    kickoff_mock.assert_has_calls(
+        [mock.call(inputs={"topic": "AI"}), mock.call(inputs={"topic": "AI"})]
+    )
+
     task_evaluator.assert_has_calls(
         [
             mock.call(researcher),
@@ -1822,10 +1795,6 @@ def test_crew_train_success(task_evaluator, crew_training_handler, kickoff):
         ]
     )
 
-    kickoff.assert_has_calls(
-        [mock.call(inputs={"topic": "AI"}), mock.call(inputs={"topic": "AI"})]
-    )
-
 
 def test_crew_train_error():
     task = Task(
@@ -1840,7 +1809,7 @@ def test_crew_train_error():
     )
 
     with pytest.raises(TypeError) as e:
-        crew.train()
+        crew.train()  # type: ignore purposefully throwing err
         assert "train() missing 1 required positional argument: 'n_iterations'" in str(
             e
         )
@@ -2536,8 +2505,9 @@ def test_conditional_should_execute():
 
 
 @mock.patch("crewai.crew.CrewEvaluator")
+@mock.patch("crewai.crew.Crew.copy")
 @mock.patch("crewai.crew.Crew.kickoff")
-def test_crew_testing_function(mock_kickoff, crew_evaluator):
+def test_crew_testing_function(kickoff_mock, copy_mock, crew_evaluator):
     task = Task(
         description="Come up with a list of 5 interesting ideas to explore for an article, then write one amazing paragraph highlight for each idea that showcases how good an article about this topic could be. Return the list of ideas with their paragraph and your notes.",
         expected_output="5 bullet points with a paragraph for each idea.",
@@ -2548,11 +2518,15 @@ def test_crew_testing_function(mock_kickoff, crew_evaluator):
         agents=[researcher],
         tasks=[task],
     )
+
+    # Create a mock for the copied crew
+    copy_mock.return_value = crew
+
     n_iterations = 2
     crew.test(n_iterations, openai_model_name="gpt-4o-mini", inputs={"topic": "AI"})
 
-    assert len(mock_kickoff.mock_calls) == n_iterations
-    mock_kickoff.assert_has_calls(
+    # Ensure kickoff is called on the copied crew
+    kickoff_mock.assert_has_calls(
         [mock.call(inputs={"topic": "AI"}), mock.call(inputs={"topic": "AI"})]
     )
 

uv.lock

@@ -627,7 +627,7 @@ wheels = [
 
 [[package]]
 name = "crewai"
-version = "0.74.2"
+version = "0.75.1"
 source = { editable = "." }
 dependencies = [
     { name = "appdirs" },