diff --git a/src/crewai/crew.py b/src/crewai/crew.py index d488783ea..b3d677291 100644 --- a/src/crewai/crew.py +++ b/src/crewai/crew.py @@ -16,6 +16,8 @@ from pydantic import ( field_validator, model_validator, ) + +from crewai.llm import LLM from pydantic_core import PydanticCustomError from crewai.agent import Agent @@ -1076,18 +1078,32 @@ class Crew(BaseModel): self, n_iterations: int, openai_model_name: Optional[str] = None, + llm: Optional[Union[str, LLM]] = None, inputs: Optional[Dict[str, Any]] = None, ) -> None: - """Test and evaluate the Crew with the given inputs for n iterations concurrently using concurrent.futures.""" + """Test and evaluate the Crew with the given inputs for n iterations concurrently using concurrent.futures. + + Args: + n_iterations: Number of iterations to run the test + openai_model_name: Name of OpenAI model to use (deprecated, use llm instead) + llm: LLM instance or model name to use for evaluation + inputs: Optional inputs to pass to the crew + """ test_crew = self.copy() + # Convert string to LLM instance if needed + if isinstance(llm, str): + llm = LLM(model=llm) + elif openai_model_name: + llm = LLM(model=openai_model_name) + self._test_execution_span = test_crew._telemetry.test_execution_span( test_crew, n_iterations, inputs, - openai_model_name, # type: ignore[arg-type] - ) # type: ignore[arg-type] - evaluator = CrewEvaluator(test_crew, openai_model_name) # type: ignore[arg-type] + getattr(llm, "model", openai_model_name), + ) + evaluator = CrewEvaluator(test_crew, llm) for i in range(1, n_iterations + 1): evaluator.set_iteration(i) diff --git a/src/crewai/utilities/evaluators/crew_evaluator_handler.py b/src/crewai/utilities/evaluators/crew_evaluator_handler.py index 3387d91b3..e01a8a6c3 100644 --- a/src/crewai/utilities/evaluators/crew_evaluator_handler.py +++ b/src/crewai/utilities/evaluators/crew_evaluator_handler.py @@ -1,4 +1,5 @@ from collections import defaultdict +from typing import Union from pydantic import BaseModel, Field from rich.box import HEAVY_EDGE @@ -6,6 +7,7 @@ from rich.console import Console from rich.table import Table from crewai.agent import Agent +from crewai.llm import LLM from crewai.task import Task from crewai.tasks.task_output import TaskOutput from crewai.telemetry import Telemetry @@ -32,9 +34,9 @@ class CrewEvaluator: run_execution_times: defaultdict = defaultdict(list) iteration: int = 0 - def __init__(self, crew, openai_model_name: str): + def __init__(self, crew, llm: Union[str, LLM]): self.crew = crew - self.openai_model_name = openai_model_name + self.llm = llm if isinstance(llm, LLM) else LLM(model=llm) self._telemetry = Telemetry() self._setup_for_evaluating() @@ -51,7 +53,7 @@ class CrewEvaluator: ), backstory="Evaluator agent for crew evaluation with precise capabilities to evaluate the performance of the agents in the crew based on the tasks they have performed", verbose=False, - llm=self.openai_model_name, + llm=self.llm, ) def _evaluation_task( diff --git a/tests/crew_test.py b/tests/crew_test.py index 2003ddada..fc327468a 100644 --- a/tests/crew_test.py +++ b/tests/crew_test.py @@ -14,6 +14,7 @@ from crewai.agent import Agent from crewai.agents.cache import CacheHandler from crewai.crew import Crew from crewai.crews.crew_output import CrewOutput +from crewai.llm import LLM from crewai.memory.contextual.contextual_memory import ContextualMemory from crewai.process import Process from crewai.task import Task @@ -1123,7 +1124,7 @@ def test_kickoff_for_each_empty_input(): assert results == [] -@pytest.mark.vcr(filter_headers=["authorization"]) +@pytest.mark.vcr(filter_headeruvs=["authorization"]) def test_kickoff_for_each_invalid_input(): """Tests if kickoff_for_each raises TypeError for invalid input types.""" @@ -2812,10 +2813,11 @@ def test_conditional_should_execute(): @mock.patch("crewai.crew.CrewEvaluator") @mock.patch("crewai.crew.Crew.copy") @mock.patch("crewai.crew.Crew.kickoff") -def test_crew_testing_function(kickoff_mock, copy_mock, crew_evaluator): +def test_crew_testing_function_with_openai_model_name(kickoff_mock, copy_mock, crew_evaluator): + """Test backward compatibility with openai_model_name parameter.""" task = Task( - description="Come up with a list of 5 interesting ideas to explore for an article, then write one amazing paragraph highlight for each idea that showcases how good an article about this topic could be. Return the list of ideas with their paragraph and your notes.", - expected_output="5 bullet points with a paragraph for each idea.", + description="Test task", + expected_output="Test output", agent=researcher, ) @@ -2824,20 +2826,87 @@ def test_crew_testing_function(kickoff_mock, copy_mock, crew_evaluator): tasks=[task], ) - # Create a mock for the copied crew copy_mock.return_value = crew n_iterations = 2 crew.test(n_iterations, openai_model_name="gpt-4o-mini", inputs={"topic": "AI"}) - # Ensure kickoff is called on the copied crew kickoff_mock.assert_has_calls( [mock.call(inputs={"topic": "AI"}), mock.call(inputs={"topic": "AI"})] ) crew_evaluator.assert_has_calls( [ - mock.call(crew, "gpt-4o-mini"), + mock.call(crew, mock.ANY), # ANY because we convert to LLM instance + mock.call().set_iteration(1), + mock.call().set_iteration(2), + mock.call().print_crew_evaluation_result(), + ] + ) + +@mock.patch("crewai.crew.CrewEvaluator") +@mock.patch("crewai.crew.Crew.copy") +@mock.patch("crewai.crew.Crew.kickoff") +def test_crew_testing_function_with_llm_instance(kickoff_mock, copy_mock, crew_evaluator): + """Test using LLM instance parameter.""" + task = Task( + description="Test task", + expected_output="Test output", + agent=researcher, + ) + + crew = Crew( + agents=[researcher], + tasks=[task], + ) + + copy_mock.return_value = crew + llm = LLM(model="gpt-4o-mini") + + n_iterations = 2 + crew.test(n_iterations, llm=llm, inputs={"topic": "AI"}) + + kickoff_mock.assert_has_calls( + [mock.call(inputs={"topic": "AI"}), mock.call(inputs={"topic": "AI"})] + ) + + crew_evaluator.assert_has_calls( + [ + mock.call(crew, llm), + mock.call().set_iteration(1), + mock.call().set_iteration(2), + mock.call().print_crew_evaluation_result(), + ] + ) + +@mock.patch("crewai.crew.CrewEvaluator") +@mock.patch("crewai.crew.Crew.copy") +@mock.patch("crewai.crew.Crew.kickoff") +def test_crew_testing_function_with_llm_string(kickoff_mock, copy_mock, crew_evaluator): + """Test using LLM string parameter.""" + task = Task( + description="Test task", + expected_output="Test output", + agent=researcher, + ) + + crew = Crew( + agents=[researcher], + tasks=[task], + ) + + copy_mock.return_value = crew + + n_iterations = 2 + crew.test(n_iterations, llm="gpt-4o-mini", inputs={"topic": "AI"}) + + kickoff_mock.assert_has_calls( + [mock.call(inputs={"topic": "AI"}), mock.call(inputs={"topic": "AI"})] + ) + + crew_evaluator.assert_has_calls( + [ + mock.call(crew, mock.ANY), # ANY because we don't care about the LLM instance details mock.call().set_iteration(1), mock.call().set_iteration(2), mock.call().print_crew_evaluation_result(), @@ -3125,4 +3194,4 @@ def test_multimodal_agent_live_image_analysis(): # Verify we got a meaningful response assert isinstance(result.raw, str) assert len(result.raw) > 100 # Expecting a detailed analysis - assert "error" not in result.raw.lower() # No error messages in response \ No newline at end of file + assert "error" not in result.raw.lower() # No error messages in response