fix: enable any llm to run test functionality

Co-Authored-By: Joe Moura <joao@crewai.com>

tests/crew_test.py

@@ -10,6 +10,7 @@ import instructor
 import pydantic_core
 import pytest
 
+from crewai.llm import LLM
 from crewai.agent import Agent
 from crewai.agents.cache import CacheHandler
 from crewai.crew import Crew
@@ -300,6 +301,35 @@ def test_hierarchical_process():
     )
 
 
+@pytest.mark.vcr(filter_headers=["authorization"])
+def test_crew_test_with_custom_llm():
+    """Test that Crew.test() works correctly with custom LLM instances."""
+    task = Task(
+        description="Test task",
+        expected_output="Test output",
+        agent=researcher,
+    )
+    custom_llm = LLM(model="gpt-4", temperature=0.5)
+    crew = Crew(agents=[researcher], tasks=[task], process=Process.sequential)
+    
+    with mock.patch('crewai.crew.CrewEvaluator') as mock_evaluator:
+        crew.test(n_iterations=1, openai_model_name=custom_llm)
+        mock_evaluator.assert_called_once_with(mock.ANY, custom_llm)
+
+@pytest.mark.vcr(filter_headers=["authorization"])
+def test_crew_test_backward_compatibility():
+    """Test that Crew.test() maintains backward compatibility with string model names."""
+    task = Task(
+        description="Test task",
+        expected_output="Test output",
+        agent=researcher,
+    )
+    crew = Crew(agents=[researcher], tasks=[task], process=Process.sequential)
+    
+    with mock.patch('crewai.crew.CrewEvaluator') as mock_evaluator:
+        crew.test(n_iterations=1, openai_model_name="gpt-4")
+        mock_evaluator.assert_called_once_with(mock.ANY, "gpt-4")
+
 def test_manager_llm_requirement_for_hierarchical_process():
     task = Task(
         description="Come up with a list of 5 interesting ideas to explore for an article, then write one amazing paragraph highlight for each idea that showcases how good an article about this topic could be. Return the list of ideas with their paragraph and your notes.",
@@ -1123,7 +1153,7 @@ def test_kickoff_for_each_empty_input():
     assert results == []
 
 
-@pytest.mark.vcr(filter_headers=["authorization"])
+@pytest.mark.vcr(filter_headeruvs=["authorization"])
 def test_kickoff_for_each_invalid_input():
     """Tests if kickoff_for_each raises TypeError for invalid input types."""
 
@@ -3125,4 +3155,4 @@ def test_multimodal_agent_live_image_analysis():
     # Verify we got a meaningful response
     assert isinstance(result.raw, str)
     assert len(result.raw) > 100  # Expecting a detailed analysis
-    assert "error" not in result.raw.lower()  # No error messages in response
+    assert "error" not in result.raw.lower()  # No error messages in response

tests/utilities/evaluators/test_crew_evaluator_handler.py

@@ -2,6 +2,7 @@ from unittest import mock
 
 import pytest
 
+from crewai.llm import LLM
 from crewai.agent import Agent
 from crewai.crew import Crew
 from crewai.task import Task
@@ -131,6 +132,19 @@ class TestCrewEvaluator:
         # Ensure the console prints the table
         console.assert_has_calls([mock.call(), mock.call().print(table())])
 
+    def test_evaluator_with_custom_llm(self, crew_planner):
+        """Test that CrewEvaluator correctly handles custom LLM instances."""
+        custom_llm = LLM(model="gpt-4", temperature=0.5)
+        evaluator = CrewEvaluator(crew_planner.crew, custom_llm)
+        assert evaluator.llm == custom_llm
+        assert evaluator.llm.temperature == 0.5
+
+    def test_evaluator_with_model_name(self, crew_planner):
+        """Test that CrewEvaluator correctly handles string model names."""
+        evaluator = CrewEvaluator(crew_planner.crew, "gpt-4")
+        assert isinstance(evaluator.llm, LLM)
+        assert evaluator.llm.model == "gpt-4"
+
     def test_evaluate(self, crew_planner):
         task_output = TaskOutput(
             description="Task 1", agent=str(crew_planner.crew.agents[0])