refactor: enhance LLM validation and error handling

Co-Authored-By: Joe Moura <joao@crewai.com>
test: update test cassettes and dependencies
2026-01-08 07:38:29 +00:00 · 2025-02-09 22:22:45 +00:00 · 2025-02-09 22:12:40 +00:00 · 2025-02-09 22:12:04 +00:00
9 changed files with 354 additions and 390 deletions
--- a/src/crewai/crew.py
+++ b/src/crewai/crew.py
@@ -1075,19 +1075,36 @@ class Crew(BaseModel):
    def test(
        self,
        n_iterations: int,
+        llm: Optional[Union[str, LLM]] = None,
        openai_model_name: Optional[str] = None,
        inputs: Optional[Dict[str, Any]] = None,
    ) -> None:
-        """Test and evaluate the Crew with the given inputs for n iterations concurrently using concurrent.futures."""
+        """Test and evaluate the Crew with the given inputs for n iterations.
+        
+        Args:
+            n_iterations: Number of iterations to run
+            llm: LLM instance or model name to use for evaluation
+            openai_model_name: (Deprecated) OpenAI model name for backward compatibility
+            inputs: Optional inputs for the crew
+            
+        Raises:
+            ValueError: If llm parameter is neither a string nor LLM instance
+        """
+        if llm and not isinstance(llm, (str, LLM)):
+            raise ValueError("llm parameter must be either a string model name or LLM instance")
        test_crew = self.copy()

+        # Handle backward compatibility
+        if openai_model_name:
+            llm = openai_model_name
+
        self._test_execution_span = test_crew._telemetry.test_execution_span(
            test_crew,
            n_iterations,
            inputs,
-            openai_model_name,  # type: ignore[arg-type]
-        )  # type: ignore[arg-type]
-        evaluator = CrewEvaluator(test_crew, openai_model_name)  # type: ignore[arg-type]
+            str(llm) if isinstance(llm, str) else (llm.model if llm else None),
+        )
+        evaluator = CrewEvaluator(test_crew, llm)

        for i in range(1, n_iterations + 1):
            evaluator.set_iteration(i)
--- a/src/crewai/patches/init.py
+++ b/src/crewai/patches/init.py
@@ -1,11 +0,0 @@
-"""
-Patches module for CrewAI.
-
-This module contains patches for external dependencies to fix known issues.
-
-Version: 1.0.0
-"""
-
-from crewai.patches.litellm_patch import apply_patches, patch_litellm_ollama_pt
-
-__all__ = ["apply_patches", "patch_litellm_ollama_pt"]
--- a/src/crewai/patches/litellm_patch.py
+++ b/src/crewai/patches/litellm_patch.py
@@ -1,186 +0,0 @@
-"""
-Patch for litellm to fix IndexError in ollama_pt function.
-
-This patch addresses issue #2744 in the crewAI repository, where an IndexError occurs
-in litellm's Ollama prompt template function when CrewAI Agent with Tools uses Ollama/Qwen models.
-
-Version: 1.0.0
-"""
-
-import json
-import logging
-from typing import Any, Dict, List, Optional, Tuple, Union
-
-# Set up logging
-logger = logging.getLogger(__name__)
-
-# Patch version
-PATCH_VERSION = "1.0.0"
-
-
-class PatchApplicationError(Exception):
-    """Exception raised when a patch fails to apply."""
-    pass
-
-
-def apply_patches() -> bool:
-    """
-    Apply all patches to fix known issues with dependencies.
-    
-    Returns:
-        bool: True if all patches were applied successfully, False otherwise.
-    """
-    success = patch_litellm_ollama_pt()
-    logger.info(f"LiteLLM ollama_pt patch applied: {success}")
-    return success
-
-
-def patch_litellm_ollama_pt() -> bool:
-    """
-    Patch the ollama_pt function in litellm to fix IndexError.
-    
-    The issue occurs when accessing messages[msg_i].get("tool_calls") without checking
-    if msg_i is within bounds of the messages list. This happens after tool execution
-    during the next LLM call.
-    
-    Returns:
-        bool: True if the patch was applied successfully, False otherwise.
-    
-    Raises:
-        PatchApplicationError: If there's an error during patch application.
-    """
-    try:
-        # Import the module containing the function to patch
-        import litellm.litellm_core_utils.prompt_templates.factory as factory
-        
-        # Define a patched version of the function
-        def patched_ollama_pt(model: str, messages: List[Dict]) -> Dict[str, Any]:
-            """
-            Patched version of ollama_pt that adds bounds checking.
-            
-            This fixes the IndexError that occurs when the assistant message is the last
-            message in the list and msg_i goes out of bounds.
-            
-            Args:
-                model: The model name.
-                messages: The list of messages to process.
-                
-            Returns:
-                Dict containing the prompt and images.
-            """
-            user_message_types = {"user", "tool", "function"}
-            msg_i = 0
-            images: List[str] = []
-            prompt = ""
-            
-            # Handle empty messages list
-            if not messages:
-                return {"prompt": prompt, "images": images}
-                
-            while msg_i < len(messages):
-                init_msg_i = msg_i
-                user_content_str = ""
-                ## MERGE CONSECUTIVE USER CONTENT ##
-                while msg_i < len(messages) and messages[msg_i]["role"] in user_message_types:
-                    msg_content = messages[msg_i].get("content")
-                    if msg_content:
-                        if isinstance(msg_content, list):
-                            for m in msg_content:
-                                if m.get("type", "") == "image_url":
-                                    if isinstance(m["image_url"], str):
-                                        images.append(m["image_url"])
-                                    elif isinstance(m["image_url"], dict):
-                                        images.append(m["image_url"]["url"])
-                                elif m.get("type", "") == "text":
-                                    user_content_str += m["text"]
-                        else:
-                            # Tool message content will always be a string
-                            user_content_str += msg_content
-
-                    msg_i += 1
-
-                if user_content_str:
-                    prompt += f"### User:\n{user_content_str}\n\n"
-
-                system_content_str, msg_i = factory._handle_ollama_system_message(
-                    messages, prompt, msg_i
-                )
-                if system_content_str:
-                    prompt += f"### System:\n{system_content_str}\n\n"
-
-                assistant_content_str = ""
-                ## MERGE CONSECUTIVE ASSISTANT CONTENT ##
-                while msg_i < len(messages) and messages[msg_i]["role"] == "assistant":
-                    assistant_content_str += factory.convert_content_list_to_str(messages[msg_i])
-                    msg_i += 1
-
-                    # Add bounds check before accessing messages[msg_i]
-                    # This is the key fix for the IndexError
-                    if msg_i < len(messages):
-                        tool_calls = messages[msg_i].get("tool_calls")
-                        ollama_tool_calls = []
-                        if tool_calls:
-                            for call in tool_calls:
-                                call_id = call["id"]
-                                function_name = call["function"]["name"]
-                                arguments = json.loads(call["function"]["arguments"])
-
-                                ollama_tool_calls.append(
-                                    {
-                                        "id": call_id,
-                                        "type": "function",
-                                        "function": {
-                                            "name": function_name,
-                                            "arguments": arguments,
-                                        },
-                                    }
-                                )
-
-                        if ollama_tool_calls:
-                            assistant_content_str += (
-                                f"Tool Calls: {json.dumps(ollama_tool_calls, indent=2)}"
-                            )
-
-                            msg_i += 1
-
-                if assistant_content_str:
-                    prompt += f"### Assistant:\n{assistant_content_str}\n\n"
-
-                if msg_i == init_msg_i:  # prevent infinite loops
-                    raise factory.litellm.BadRequestError(
-                        message=factory.BAD_MESSAGE_ERROR_STR + f"passed in {messages[msg_i]}",
-                        model=model,
-                        llm_provider="ollama",
-                    )
-
-            response_dict = {
-                "prompt": prompt,
-                "images": images,
-            }
-
-            return response_dict
-        
-        # Replace the original function with our patched version
-        factory.ollama_pt = patched_ollama_pt
-        
-        logger.info(f"Successfully applied litellm ollama_pt patch version {PATCH_VERSION}")
-        return True
-    except Exception as e:
-        error_msg = f"Failed to apply litellm ollama_pt patch: {e}"
-        logger.error(error_msg)
-        return False
-
-
-# For backwards compatibility
-def patch_litellm() -> bool:
-    """
-    Legacy function for backwards compatibility.
-    
-    Returns:
-        bool: True if the patch was applied successfully, False otherwise.
-    """
-    try:
-        return patch_litellm_ollama_pt()
-    except Exception as e:
-        logger.error(f"Failed to apply legacy litellm patch: {e}")
-        return False
--- a/src/crewai/utilities/evaluators/crew_evaluator_handler.py
+++ b/src/crewai/utilities/evaluators/crew_evaluator_handler.py
@@ -1,4 +1,6 @@
+import logging
 from collections import defaultdict
+from typing import Optional, Union

 from pydantic import BaseModel, Field
 from rich.box import HEAVY_EDGE
@@ -6,11 +8,22 @@ from rich.console import Console
 from rich.table import Table

 from crewai.agent import Agent
+from crewai.llm import LLM
 from crewai.task import Task
 from crewai.tasks.task_output import TaskOutput
 from crewai.telemetry import Telemetry


+class CrewEvaluationError(Exception):
+    """Raised when there is an error during crew evaluation."""
+    pass
+
+
+# Default values for evaluation metrics
+DEFAULT_TASK_SCORE = 9.0
+DEFAULT_EXECUTION_TIME = 60  # seconds
+
+
 class TaskEvaluationPydanticOutput(BaseModel):
    quality: float = Field(
        description="A score from 1 to 10 evaluating on completion, quality, and overall performance from the task_description and task_expected_output to the actual Task Output."
@@ -32,9 +45,27 @@ class CrewEvaluator:
    run_execution_times: defaultdict = defaultdict(list)
    iteration: int = 0

-    def __init__(self, crew, openai_model_name: str):
+    def __init__(self, crew, llm: Optional[Union[str, LLM]] = None):
+        """Initialize CrewEvaluator.
+        
+        Args:
+            crew: The crew to evaluate
+            llm: LLM instance or model name for evaluation
+        """
        self.crew = crew
-        self.openai_model_name = openai_model_name
+        logging.info(f"Initializing CrewEvaluator with LLM: {llm}")
+        
+        # Initialize tasks_scores with default values to avoid division by zero
+        self.tasks_scores = defaultdict(list)
+        for i in range(1, len(crew.tasks) + 1):
+            self.tasks_scores[i] = [DEFAULT_TASK_SCORE]
+        # Initialize run_execution_times with default values
+        self.run_execution_times = defaultdict(list)
+        for i in range(1, len(crew.tasks) + 1):
+            self.run_execution_times[i] = [DEFAULT_EXECUTION_TIME]
+        self.llm = llm if isinstance(llm, LLM) else (
+            LLM(model=llm) if isinstance(llm, str) else None
+        )
        self._telemetry = Telemetry()
        self._setup_for_evaluating()

@@ -51,7 +82,7 @@ class CrewEvaluator:
            ),
            backstory="Evaluator agent for crew evaluation with precise capabilities to evaluate the performance of the agents in the crew based on the tasks they have performed",
            verbose=False,
-            llm=self.openai_model_name,
+            llm=self.llm,
        )

    def _evaluation_task(
@@ -157,35 +188,57 @@ class CrewEvaluator:
        console.print(table)

    def evaluate(self, task_output: TaskOutput):
-        """Evaluates the performance of the agents in the crew based on the tasks they have performed."""
-        current_task = None
-        for task in self.crew.tasks:
-            if task.description == task_output.description:
-                current_task = task
-                break
+        """Evaluates the performance of the agents in the crew based on the tasks they have performed.
+        
+        Args:
+            task_output: The output from the task execution to evaluate
+            
+        Raises:
+            CrewEvaluationError: If evaluation fails or produces unexpected results
+            ValueError: If required inputs are missing or invalid
+        """
+        try:
+            # Find the matching task
+            current_task = None
+            for task in self.crew.tasks:
+                if task.description == task_output.description:
+                    current_task = task
+                    break

-        if not current_task or not task_output:
-            raise ValueError(
-                "Task to evaluate and task output are required for evaluation"
+            if not current_task or not task_output:
+                raise ValueError(
+                    "Task to evaluate and task output are required for evaluation"
+                )
+
+            # Create and execute evaluation task
+            evaluator_agent = self._evaluator_agent()
+            evaluation_task = self._evaluation_task(
+                evaluator_agent, current_task, task_output.raw
            )

-        evaluator_agent = self._evaluator_agent()
-        evaluation_task = self._evaluation_task(
-            evaluator_agent, current_task, task_output.raw
-        )
+            logging.info(f"Evaluating task: {current_task.description}")
+            evaluation_result = evaluation_task.execute_sync()

-        evaluation_result = evaluation_task.execute_sync()
+            # Process evaluation results
+            if isinstance(evaluation_result.pydantic, TaskEvaluationPydanticOutput):
+                self._test_result_span = self._telemetry.individual_test_result_span(
+                    self.crew,
+                    evaluation_result.pydantic.quality,
+                    current_task._execution_time,
+                    str(self.llm.model if self.llm else None),
+                )
+                self.tasks_scores[self.iteration].append(evaluation_result.pydantic.quality)
+                self.run_execution_times[self.iteration].append(
+                    current_task._execution_time
+                )
+                logging.info(f"Task evaluation completed with score: {evaluation_result.pydantic.quality}")
+            else:
+                raise CrewEvaluationError("Evaluation result is not in the expected format")

-        if isinstance(evaluation_result.pydantic, TaskEvaluationPydanticOutput):
-            self._test_result_span = self._telemetry.individual_test_result_span(
-                self.crew,
-                evaluation_result.pydantic.quality,
-                current_task._execution_time,
-                self.openai_model_name,
-            )
-            self.tasks_scores[self.iteration].append(evaluation_result.pydantic.quality)
-            self.run_execution_times[self.iteration].append(
-                current_task._execution_time
-            )
-        else:
-            raise ValueError("Evaluation result is not in the expected format")
+        except ValueError as e:
+            logging.error(f"Invalid input for task evaluation: {e}")
+            raise
+
+        except Exception as e:
+            logging.error(f"Error during task evaluation: {e}")
+            raise CrewEvaluationError(f"Failed to evaluate task: {e}")
--- a/tests/cassettes/test_agent_human_input.yaml
+++ b/tests/cassettes/test_agent_human_input.yaml
@@ -1,4 +1,87 @@
 interactions:
+- request:
+    body: !!binary |
+      CqcXCiQKIgoMc2VydmljZS5uYW1lEhIKEGNyZXdBSS10ZWxlbWV0cnkS/hYKEgoQY3Jld2FpLnRl
+      bGVtZXRyeRJ5ChBuJJtOdNaB05mOW/p3915eEgj2tkAd3rZcASoQVG9vbCBVc2FnZSBFcnJvcjAB
+      OYa7/URvKBUYQUpcFEVvKBUYShoKDmNyZXdhaV92ZXJzaW9uEggKBjAuODYuMEoPCgNsbG0SCAoG
+      Z3B0LTRvegIYAYUBAAEAABLJBwoQifhX01E5i+5laGdALAlZBBIIBuGM1aN+OPgqDENyZXcgQ3Jl
+      YXRlZDABORVGruBvKBUYQaipwOBvKBUYShoKDmNyZXdhaV92ZXJzaW9uEggKBjAuODYuMEoaCg5w
+      eXRob25fdmVyc2lvbhIICgYzLjEyLjdKLgoIY3Jld19rZXkSIgogN2U2NjA4OTg5ODU5YTY3ZWVj
+      ODhlZWY3ZmNlODUyMjVKMQoHY3Jld19pZBImCiRiOThiNWEwMC01YTI1LTQxMDctYjQwNS1hYmYz
+      MjBhOGYzYThKHAoMY3Jld19wcm9jZXNzEgwKCnNlcXVlbnRpYWxKEQoLY3Jld19tZW1vcnkSAhAA
+      ShoKFGNyZXdfbnVtYmVyX29mX3Rhc2tzEgIYAUobChVjcmV3X251bWJlcl9vZl9hZ2VudHMSAhgB
+      SuQCCgtjcmV3X2FnZW50cxLUAgrRAlt7ImtleSI6ICIyMmFjZDYxMWU0NGVmNWZhYzA1YjUzM2Q3
+      NWU4ODkzYiIsICJpZCI6ICJkNWIyMzM1YS0yMmIyLTQyZWEtYmYwNS03OTc3NmU3MmYzOTIiLCAi
+      cm9sZSI6ICJEYXRhIFNjaWVudGlzdCIsICJ2ZXJib3NlPyI6IGZhbHNlLCAibWF4X2l0ZXIiOiAy
+      MCwgIm1heF9ycG0iOiBudWxsLCAiZnVuY3Rpb25fY2FsbGluZ19sbG0iOiAiIiwgImxsbSI6ICJn
+      cHQtNG8tbWluaSIsICJkZWxlZ2F0aW9uX2VuYWJsZWQ/IjogZmFsc2UsICJhbGxvd19jb2RlX2V4
+      ZWN1dGlvbj8iOiBmYWxzZSwgIm1heF9yZXRyeV9saW1pdCI6IDIsICJ0b29sc19uYW1lcyI6IFsi
+      Z2V0IGdyZWV0aW5ncyJdfV1KkgIKCmNyZXdfdGFza3MSgwIKgAJbeyJrZXkiOiAiYTI3N2IzNGIy
+      YzE0NmYwYzU2YzVlMTM1NmU4ZjhhNTciLCAiaWQiOiAiMjJiZWMyMzEtY2QyMS00YzU4LTgyN2Ut
+      MDU4MWE4ZjBjMTExIiwgImFzeW5jX2V4ZWN1dGlvbj8iOiBmYWxzZSwgImh1bWFuX2lucHV0PyI6
+      IGZhbHNlLCAiYWdlbnRfcm9sZSI6ICJEYXRhIFNjaWVudGlzdCIsICJhZ2VudF9rZXkiOiAiMjJh
+      Y2Q2MTFlNDRlZjVmYWMwNWI1MzNkNzVlODg5M2IiLCAidG9vbHNfbmFtZXMiOiBbImdldCBncmVl
+      dGluZ3MiXX1degIYAYUBAAEAABKOAgoQ5WYoxRtTyPjge4BduhL0rRIIv2U6rvWALfwqDFRhc2sg
+      Q3JlYXRlZDABOX068uBvKBUYQZkv8+BvKBUYSi4KCGNyZXdfa2V5EiIKIDdlNjYwODk4OTg1OWE2
+      N2VlYzg4ZWVmN2ZjZTg1MjI1SjEKB2NyZXdfaWQSJgokYjk4YjVhMDAtNWEyNS00MTA3LWI0MDUt
+      YWJmMzIwYThmM2E4Si4KCHRhc2tfa2V5EiIKIGEyNzdiMzRiMmMxNDZmMGM1NmM1ZTEzNTZlOGY4
+      YTU3SjEKB3Rhc2tfaWQSJgokMjJiZWMyMzEtY2QyMS00YzU4LTgyN2UtMDU4MWE4ZjBjMTExegIY
+      AYUBAAEAABKQAQoQXyeDtJDFnyp2Fjk9YEGTpxIIaNE7gbhPNYcqClRvb2wgVXNhZ2UwATkaXTvj
+      bygVGEGvx0rjbygVGEoaCg5jcmV3YWlfdmVyc2lvbhIICgYwLjg2LjBKHAoJdG9vbF9uYW1lEg8K
+      DUdldCBHcmVldGluZ3NKDgoIYXR0ZW1wdHMSAhgBegIYAYUBAAEAABLVBwoQMWfznt0qwauEzl7T
+      UOQxRBII9q+pUS5EdLAqDENyZXcgQ3JlYXRlZDABORONPORvKBUYQSAoS+RvKBUYShoKDmNyZXdh
+      aV92ZXJzaW9uEggKBjAuODYuMEoaCg5weXRob25fdmVyc2lvbhIICgYzLjEyLjdKLgoIY3Jld19r
+      ZXkSIgogYzMwNzYwMDkzMjY3NjE0NDRkNTdjNzFkMWRhM2YyN2NKMQoHY3Jld19pZBImCiQ3OTQw
+      MTkyNS1iOGU5LTQ3MDgtODUzMC00NDhhZmEzYmY4YjBKHAoMY3Jld19wcm9jZXNzEgwKCnNlcXVl
+      bnRpYWxKEQoLY3Jld19tZW1vcnkSAhAAShoKFGNyZXdfbnVtYmVyX29mX3Rhc2tzEgIYAUobChVj
+      cmV3X251bWJlcl9vZl9hZ2VudHMSAhgBSuoCCgtjcmV3X2FnZW50cxLaAgrXAlt7ImtleSI6ICI5
+      OGYzYjFkNDdjZTk2OWNmMDU3NzI3Yjc4NDE0MjVjZCIsICJpZCI6ICI5OTJkZjYyZi1kY2FiLTQy
+      OTUtOTIwNi05MDBkNDExNGIxZTkiLCAicm9sZSI6ICJGcmllbmRseSBOZWlnaGJvciIsICJ2ZXJi
+      b3NlPyI6IGZhbHNlLCAibWF4X2l0ZXIiOiAyMCwgIm1heF9ycG0iOiBudWxsLCAiZnVuY3Rpb25f
+      Y2FsbGluZ19sbG0iOiAiIiwgImxsbSI6ICJncHQtNG8tbWluaSIsICJkZWxlZ2F0aW9uX2VuYWJs
+      ZWQ/IjogZmFsc2UsICJhbGxvd19jb2RlX2V4ZWN1dGlvbj8iOiBmYWxzZSwgIm1heF9yZXRyeV9s
+      aW1pdCI6IDIsICJ0b29sc19uYW1lcyI6IFsiZGVjaWRlIGdyZWV0aW5ncyJdfV1KmAIKCmNyZXdf
+      dGFza3MSiQIKhgJbeyJrZXkiOiAiODBkN2JjZDQ5MDk5MjkwMDgzODMyZjBlOTgzMzgwZGYiLCAi
+      aWQiOiAiMmZmNjE5N2UtYmEyNy00YjczLWI0YTctNGZhMDQ4ZTYyYjQ3IiwgImFzeW5jX2V4ZWN1
+      dGlvbj8iOiBmYWxzZSwgImh1bWFuX2lucHV0PyI6IGZhbHNlLCAiYWdlbnRfcm9sZSI6ICJGcmll
+      bmRseSBOZWlnaGJvciIsICJhZ2VudF9rZXkiOiAiOThmM2IxZDQ3Y2U5NjljZjA1NzcyN2I3ODQx
+      NDI1Y2QiLCAidG9vbHNfbmFtZXMiOiBbImRlY2lkZSBncmVldGluZ3MiXX1degIYAYUBAAEAABKO
+      AgoQnjTp5boK7/+DQxztYIpqihIIgGnMUkBtzHEqDFRhc2sgQ3JlYXRlZDABOcpYcuRvKBUYQalE
+      c+RvKBUYSi4KCGNyZXdfa2V5EiIKIGMzMDc2MDA5MzI2NzYxNDQ0ZDU3YzcxZDFkYTNmMjdjSjEK
+      B2NyZXdfaWQSJgokNzk0MDE5MjUtYjhlOS00NzA4LTg1MzAtNDQ4YWZhM2JmOGIwSi4KCHRhc2tf
+      a2V5EiIKIDgwZDdiY2Q0OTA5OTI5MDA4MzgzMmYwZTk4MzM4MGRmSjEKB3Rhc2tfaWQSJgokMmZm
+      NjE5N2UtYmEyNy00YjczLWI0YTctNGZhMDQ4ZTYyYjQ3egIYAYUBAAEAABKTAQoQ26H9pLUgswDN
+      p9XhJwwL6BIIx3bw7mAvPYwqClRvb2wgVXNhZ2UwATmy7NPlbygVGEEvb+HlbygVGEoaCg5jcmV3
+      YWlfdmVyc2lvbhIICgYwLjg2LjBKHwoJdG9vbF9uYW1lEhIKEERlY2lkZSBHcmVldGluZ3NKDgoI
+      YXR0ZW1wdHMSAhgBegIYAYUBAAEAAA==
+    headers:
+      Accept:
+      - '*/*'
+      Accept-Encoding:
+      - gzip, deflate
+      Connection:
+      - keep-alive
+      Content-Length:
+      - '2986'
+      Content-Type:
+      - application/x-protobuf
+      User-Agent:
+      - OTel-OTLP-Exporter-Python/1.27.0
+    method: POST
+    uri: https://telemetry.crewai.com:4319/v1/traces
+  response:
+    body:
+      string: "\n\0"
+    headers:
+      Content-Length:
+      - '2'
+      Content-Type:
+      - application/x-protobuf
+      Date:
+      - Fri, 27 Dec 2024 22:14:53 GMT
+    status:
+      code: 200
+      message: OK
 - request:
    body: '{"messages": [{"role": "system", "content": "You are test role. test backstory\nYour
      personal goal is: test goal\nTo give my best complete final answer to the task
@@ -22,18 +105,20 @@ interactions:
      - '824'
      content-type:
      - application/json
+      cookie:
+      - _cfuvid=ePJSDFdHag2D8lj21_ijAMWjoA6xfnPNxN4uekvC728-1727226247743-0.0.1.1-604800000
      host:
      - api.openai.com
      user-agent:
      - OpenAI/Python 1.52.1
      x-stainless-arch:
-      - arm64
+      - x64
      x-stainless-async:
      - 'false'
      x-stainless-lang:
      - python
      x-stainless-os:
-      - MacOS
+      - Linux
      x-stainless-package-version:
      - 1.52.1
      x-stainless-raw-response:
@@ -47,8 +132,8 @@ interactions:
    method: POST
    uri: https://api.openai.com/v1/chat/completions
  response:
-    content: "{\n  \"id\": \"chatcmpl-AaqIIsTxhvf75xvuu7gQScIlRSKbW\",\n  \"object\":
-      \"chat.completion\",\n  \"created\": 1733344190,\n  \"model\": \"gpt-4o-mini-2024-07-18\",\n
+    content: "{\n  \"id\": \"chatcmpl-AjCtZLLrWi8ZASpP9bz6HaCV7xBIn\",\n  \"object\":
+      \"chat.completion\",\n  \"created\": 1735337693,\n  \"model\": \"gpt-4o-mini-2024-07-18\",\n
      \ \"choices\": [\n    {\n      \"index\": 0,\n      \"message\": {\n        \"role\":
      \"assistant\",\n        \"content\": \"I now can give a great answer  \\nFinal
      Answer: Hi\",\n        \"refusal\": null\n      },\n      \"logprobs\": null,\n
@@ -57,12 +142,12 @@ interactions:
      {\n      \"cached_tokens\": 0,\n      \"audio_tokens\": 0\n    },\n    \"completion_tokens_details\":
      {\n      \"reasoning_tokens\": 0,\n      \"audio_tokens\": 0,\n      \"accepted_prediction_tokens\":
      0,\n      \"rejected_prediction_tokens\": 0\n    }\n  },\n  \"system_fingerprint\":
-      \"fp_0705bf87c0\"\n}\n"
+      \"fp_0aa8d3e20b\"\n}\n"
    headers:
      CF-Cache-Status:
      - DYNAMIC
      CF-RAY:
-      - 8ece8cfc3b1f4532-ATL
+      - 8f8caa83deca756b-SEA
      Connection:
      - keep-alive
      Content-Encoding:
@@ -70,14 +155,14 @@ interactions:
      Content-Type:
      - application/json
      Date:
-      - Wed, 04 Dec 2024 20:29:50 GMT
+      - Fri, 27 Dec 2024 22:14:53 GMT
      Server:
      - cloudflare
      Set-Cookie:
-      - __cf_bm=QJZZjZ6eqnVamqUkw.Bx0mj7oBi3a_vGEH1VODcUxlg-1733344190-1.0.1.1-xyN0ekA9xIrSwEhRBmTiWJ3Pt72UYLU5owKfkz5yihVmMTfsr_Qz.ssGPJ5cuft066v1xVjb4zOSTdFmesMSKg;
-        path=/; expires=Wed, 04-Dec-24 20:59:50 GMT; domain=.api.openai.com; HttpOnly;
+      - __cf_bm=wJkq_yLkzE3OdxE0aMJz.G0kce969.9JxRmZ0ratl4c-1735337693-1.0.1.1-OKpUoRrSPFGvWv5Hp5ET1PNZ7iZNHPKEAuakpcQUxxPSeisUIIR3qIOZ31MGmYugqB5.wkvidgbxOAagqJvmnw;
+        path=/; expires=Fri, 27-Dec-24 22:44:53 GMT; domain=.api.openai.com; HttpOnly;
        Secure; SameSite=None
-      - _cfuvid=eCIkP8GVPvpkg19eOhCquWFHm.RTQBQy4yHLGGEAH5c-1733344190334-0.0.1.1-604800000;
+      - _cfuvid=A_ASCLNAVfQoyucWOAIhecWtEpNotYoZr0bAFihgNxs-1735337693273-0.0.1.1-604800000;
        path=/; domain=.api.openai.com; HttpOnly; Secure; SameSite=None
      Transfer-Encoding:
      - chunked
@@ -90,7 +175,7 @@ interactions:
      openai-organization:
      - crewai-iuxna1
      openai-processing-ms:
-      - '313'
+      - '404'
      openai-version:
      - '2020-10-01'
      strict-transport-security:
@@ -108,7 +193,7 @@ interactions:
      x-ratelimit-reset-tokens:
      - 0s
      x-request-id:
-      - req_9fd9a8ee688045dcf7ac5f6fdf689372
+      - req_6ac84634bff9193743c4b0911c09b4a6
    http_version: HTTP/1.1
    status_code: 200
 - request:
@@ -131,20 +216,20 @@ interactions:
      content-type:
      - application/json
      cookie:
-      - __cf_bm=QJZZjZ6eqnVamqUkw.Bx0mj7oBi3a_vGEH1VODcUxlg-1733344190-1.0.1.1-xyN0ekA9xIrSwEhRBmTiWJ3Pt72UYLU5owKfkz5yihVmMTfsr_Qz.ssGPJ5cuft066v1xVjb4zOSTdFmesMSKg;
-        _cfuvid=eCIkP8GVPvpkg19eOhCquWFHm.RTQBQy4yHLGGEAH5c-1733344190334-0.0.1.1-604800000
+      - _cfuvid=A_ASCLNAVfQoyucWOAIhecWtEpNotYoZr0bAFihgNxs-1735337693273-0.0.1.1-604800000;
+        __cf_bm=wJkq_yLkzE3OdxE0aMJz.G0kce969.9JxRmZ0ratl4c-1735337693-1.0.1.1-OKpUoRrSPFGvWv5Hp5ET1PNZ7iZNHPKEAuakpcQUxxPSeisUIIR3qIOZ31MGmYugqB5.wkvidgbxOAagqJvmnw
      host:
      - api.openai.com
      user-agent:
      - OpenAI/Python 1.52.1
      x-stainless-arch:
-      - arm64
+      - x64
      x-stainless-async:
      - 'false'
      x-stainless-lang:
      - python
      x-stainless-os:
-      - MacOS
+      - Linux
      x-stainless-package-version:
      - 1.52.1
      x-stainless-raw-response:
@@ -158,8 +243,8 @@ interactions:
    method: POST
    uri: https://api.openai.com/v1/chat/completions
  response:
-    content: "{\n  \"id\": \"chatcmpl-AaqIIaQlLyoyPmk909PvAIfA2TmJL\",\n  \"object\":
-      \"chat.completion\",\n  \"created\": 1733344190,\n  \"model\": \"gpt-4o-mini-2024-07-18\",\n
+    content: "{\n  \"id\": \"chatcmpl-AjCtZNlWdrrPZhq0MJDqd16sMuQEJ\",\n  \"object\":
+      \"chat.completion\",\n  \"created\": 1735337693,\n  \"model\": \"gpt-4o-mini-2024-07-18\",\n
      \ \"choices\": [\n    {\n      \"index\": 0,\n      \"message\": {\n        \"role\":
      \"assistant\",\n        \"content\": \"True\",\n        \"refusal\": null\n
      \     },\n      \"logprobs\": null,\n      \"finish_reason\": \"stop\"\n    }\n
@@ -168,12 +253,12 @@ interactions:
      0,\n      \"audio_tokens\": 0\n    },\n    \"completion_tokens_details\": {\n
      \     \"reasoning_tokens\": 0,\n      \"audio_tokens\": 0,\n      \"accepted_prediction_tokens\":
      0,\n      \"rejected_prediction_tokens\": 0\n    }\n  },\n  \"system_fingerprint\":
-      \"fp_0705bf87c0\"\n}\n"
+      \"fp_0aa8d3e20b\"\n}\n"
    headers:
      CF-Cache-Status:
      - DYNAMIC
      CF-RAY:
-      - 8ece8d060b5e4532-ATL
+      - 8f8caa87094f756b-SEA
      Connection:
      - keep-alive
      Content-Encoding:
@@ -181,7 +266,7 @@ interactions:
      Content-Type:
      - application/json
      Date:
-      - Wed, 04 Dec 2024 20:29:50 GMT
+      - Fri, 27 Dec 2024 22:14:53 GMT
      Server:
      - cloudflare
      Transfer-Encoding:
@@ -195,7 +280,7 @@ interactions:
      openai-organization:
      - crewai-iuxna1
      openai-processing-ms:
-      - '375'
+      - '156'
      openai-version:
      - '2020-10-01'
      strict-transport-security:
@@ -213,7 +298,7 @@ interactions:
      x-ratelimit-reset-tokens:
      - 0s
      x-request-id:
-      - req_be7cb475e0859a82c37ee3f2871ea5ea
+      - req_ec74bef2a9ef7b2144c03fd7f7bbeab0
    http_version: HTTP/1.1
    status_code: 200
 - request:
@@ -242,20 +327,20 @@ interactions:
      content-type:
      - application/json
      cookie:
-      - __cf_bm=QJZZjZ6eqnVamqUkw.Bx0mj7oBi3a_vGEH1VODcUxlg-1733344190-1.0.1.1-xyN0ekA9xIrSwEhRBmTiWJ3Pt72UYLU5owKfkz5yihVmMTfsr_Qz.ssGPJ5cuft066v1xVjb4zOSTdFmesMSKg;
-        _cfuvid=eCIkP8GVPvpkg19eOhCquWFHm.RTQBQy4yHLGGEAH5c-1733344190334-0.0.1.1-604800000
+      - _cfuvid=A_ASCLNAVfQoyucWOAIhecWtEpNotYoZr0bAFihgNxs-1735337693273-0.0.1.1-604800000;
+        __cf_bm=wJkq_yLkzE3OdxE0aMJz.G0kce969.9JxRmZ0ratl4c-1735337693-1.0.1.1-OKpUoRrSPFGvWv5Hp5ET1PNZ7iZNHPKEAuakpcQUxxPSeisUIIR3qIOZ31MGmYugqB5.wkvidgbxOAagqJvmnw
      host:
      - api.openai.com
      user-agent:
      - OpenAI/Python 1.52.1
      x-stainless-arch:
-      - arm64
+      - x64
      x-stainless-async:
      - 'false'
      x-stainless-lang:
      - python
      x-stainless-os:
-      - MacOS
+      - Linux
      x-stainless-package-version:
      - 1.52.1
      x-stainless-raw-response:
@@ -269,22 +354,23 @@ interactions:
    method: POST
    uri: https://api.openai.com/v1/chat/completions
  response:
-    content: "{\n  \"id\": \"chatcmpl-AaqIJAAxpVfUOdrsgYKHwfRlHv4RS\",\n  \"object\":
-      \"chat.completion\",\n  \"created\": 1733344191,\n  \"model\": \"gpt-4o-mini-2024-07-18\",\n
+    content: "{\n  \"id\": \"chatcmpl-AjCtZGv4f3h7GDdhyOy9G0sB1lRgC\",\n  \"object\":
+      \"chat.completion\",\n  \"created\": 1735337693,\n  \"model\": \"gpt-4o-mini-2024-07-18\",\n
      \ \"choices\": [\n    {\n      \"index\": 0,\n      \"message\": {\n        \"role\":
-      \"assistant\",\n        \"content\": \"Thought: I now can give a great answer
-      \ \\nFinal Answer: Hello\",\n        \"refusal\": null\n      },\n      \"logprobs\":
-      null,\n      \"finish_reason\": \"stop\"\n    }\n  ],\n  \"usage\": {\n    \"prompt_tokens\":
-      188,\n    \"completion_tokens\": 14,\n    \"total_tokens\": 202,\n    \"prompt_tokens_details\":
-      {\n      \"cached_tokens\": 0,\n      \"audio_tokens\": 0\n    },\n    \"completion_tokens_details\":
-      {\n      \"reasoning_tokens\": 0,\n      \"audio_tokens\": 0,\n      \"accepted_prediction_tokens\":
+      \"assistant\",\n        \"content\": \"Thought: I understand the feedback and
+      will adjust my response accordingly.  \\nFinal Answer: Hello\",\n        \"refusal\":
+      null\n      },\n      \"logprobs\": null,\n      \"finish_reason\": \"stop\"\n
+      \   }\n  ],\n  \"usage\": {\n    \"prompt_tokens\": 188,\n    \"completion_tokens\":
+      18,\n    \"total_tokens\": 206,\n    \"prompt_tokens_details\": {\n      \"cached_tokens\":
+      0,\n      \"audio_tokens\": 0\n    },\n    \"completion_tokens_details\": {\n
+      \     \"reasoning_tokens\": 0,\n      \"audio_tokens\": 0,\n      \"accepted_prediction_tokens\":
      0,\n      \"rejected_prediction_tokens\": 0\n    }\n  },\n  \"system_fingerprint\":
-      \"fp_0705bf87c0\"\n}\n"
+      \"fp_0aa8d3e20b\"\n}\n"
    headers:
      CF-Cache-Status:
      - DYNAMIC
      CF-RAY:
-      - 8ece8d090fc34532-ATL
+      - 8f8caa88cac4756b-SEA
      Connection:
      - keep-alive
      Content-Encoding:
@@ -292,7 +378,7 @@ interactions:
      Content-Type:
      - application/json
      Date:
-      - Wed, 04 Dec 2024 20:29:51 GMT
+      - Fri, 27 Dec 2024 22:14:54 GMT
      Server:
      - cloudflare
      Transfer-Encoding:
@@ -306,7 +392,7 @@ interactions:
      openai-organization:
      - crewai-iuxna1
      openai-processing-ms:
-      - '484'
+      - '358'
      openai-version:
      - '2020-10-01'
      strict-transport-security:
@@ -324,7 +410,7 @@ interactions:
      x-ratelimit-reset-tokens:
      - 0s
      x-request-id:
-      - req_5bf4a565ad6c2567a1ed204ecac89134
+      - req_ae1ab6b206d28ded6fee3c83ed0c2ab7
    http_version: HTTP/1.1
    status_code: 200
 - request:
@@ -346,20 +432,20 @@ interactions:
      content-type:
      - application/json
      cookie:
-      - __cf_bm=QJZZjZ6eqnVamqUkw.Bx0mj7oBi3a_vGEH1VODcUxlg-1733344190-1.0.1.1-xyN0ekA9xIrSwEhRBmTiWJ3Pt72UYLU5owKfkz5yihVmMTfsr_Qz.ssGPJ5cuft066v1xVjb4zOSTdFmesMSKg;
-        _cfuvid=eCIkP8GVPvpkg19eOhCquWFHm.RTQBQy4yHLGGEAH5c-1733344190334-0.0.1.1-604800000
+      - _cfuvid=A_ASCLNAVfQoyucWOAIhecWtEpNotYoZr0bAFihgNxs-1735337693273-0.0.1.1-604800000;
+        __cf_bm=wJkq_yLkzE3OdxE0aMJz.G0kce969.9JxRmZ0ratl4c-1735337693-1.0.1.1-OKpUoRrSPFGvWv5Hp5ET1PNZ7iZNHPKEAuakpcQUxxPSeisUIIR3qIOZ31MGmYugqB5.wkvidgbxOAagqJvmnw
      host:
      - api.openai.com
      user-agent:
      - OpenAI/Python 1.52.1
      x-stainless-arch:
-      - arm64
+      - x64
      x-stainless-async:
      - 'false'
      x-stainless-lang:
      - python
      x-stainless-os:
-      - MacOS
+      - Linux
      x-stainless-package-version:
      - 1.52.1
      x-stainless-raw-response:
@@ -373,8 +459,8 @@ interactions:
    method: POST
    uri: https://api.openai.com/v1/chat/completions
  response:
-    content: "{\n  \"id\": \"chatcmpl-AaqIJqyG8vl9mxj2qDPZgaxyNLLIq\",\n  \"object\":
-      \"chat.completion\",\n  \"created\": 1733344191,\n  \"model\": \"gpt-4o-mini-2024-07-18\",\n
+    content: "{\n  \"id\": \"chatcmpl-AjCtaiHL4TY8Dssk0j2miqmjrzquy\",\n  \"object\":
+      \"chat.completion\",\n  \"created\": 1735337694,\n  \"model\": \"gpt-4o-mini-2024-07-18\",\n
      \ \"choices\": [\n    {\n      \"index\": 0,\n      \"message\": {\n        \"role\":
      \"assistant\",\n        \"content\": \"False\",\n        \"refusal\": null\n
      \     },\n      \"logprobs\": null,\n      \"finish_reason\": \"stop\"\n    }\n
@@ -383,12 +469,12 @@ interactions:
      0,\n      \"audio_tokens\": 0\n    },\n    \"completion_tokens_details\": {\n
      \     \"reasoning_tokens\": 0,\n      \"audio_tokens\": 0,\n      \"accepted_prediction_tokens\":
      0,\n      \"rejected_prediction_tokens\": 0\n    }\n  },\n  \"system_fingerprint\":
-      \"fp_0705bf87c0\"\n}\n"
+      \"fp_0aa8d3e20b\"\n}\n"
    headers:
      CF-Cache-Status:
      - DYNAMIC
      CF-RAY:
-      - 8ece8d0cfdeb4532-ATL
+      - 8f8caa8bdd26756b-SEA
      Connection:
      - keep-alive
      Content-Encoding:
@@ -396,7 +482,7 @@ interactions:
      Content-Type:
      - application/json
      Date:
-      - Wed, 04 Dec 2024 20:29:51 GMT
+      - Fri, 27 Dec 2024 22:14:54 GMT
      Server:
      - cloudflare
      Transfer-Encoding:
@@ -410,7 +496,7 @@ interactions:
      openai-organization:
      - crewai-iuxna1
      openai-processing-ms:
-      - '341'
+      - '184'
      openai-version:
      - '2020-10-01'
      strict-transport-security:
@@ -428,7 +514,7 @@ interactions:
      x-ratelimit-reset-tokens:
      - 0s
      x-request-id:
-      - req_5554bade8ceda00cf364b76a51b708ff
+      - req_652891f79c1104a7a8436275d78a69f1
    http_version: HTTP/1.1
    status_code: 200
 version: 1
--- a/tests/crew_test.py
+++ b/tests/crew_test.py
@@ -14,6 +14,9 @@ from crewai.agent import Agent
 from crewai.agents.cache import CacheHandler
 from crewai.crew import Crew
 from crewai.crews.crew_output import CrewOutput
+from collections import defaultdict
+from crewai.llm import LLM
+from crewai.utilities.evaluators.crew_evaluator_handler import CrewEvaluator
 from crewai.memory.contextual.contextual_memory import ContextualMemory
 from crewai.process import Process
 from crewai.task import Task
@@ -1123,7 +1126,7 @@ def test_kickoff_for_each_empty_input():
    assert results == []


-@pytest.mark.vcr(filter_headers=["authorization"])
+@pytest.mark.vcr(filter_headeruvs=["authorization"])
 def test_kickoff_for_each_invalid_input():
    """Tests if kickoff_for_each raises TypeError for invalid input types."""

@@ -2814,8 +2817,8 @@ def test_conditional_should_execute():
@mock.patch("crewai.crew.Crew.kickoff")
 def test_crew_testing_function(kickoff_mock, copy_mock, crew_evaluator):
    task = Task(
-        description="Come up with a list of 5 interesting ideas to explore for an article, then write one amazing paragraph highlight for each idea that showcases how good an article about this topic could be. Return the list of ideas with their paragraph and your notes.",
-        expected_output="5 bullet points with a paragraph for each idea.",
+        description="Test task description",
+        expected_output="Test output",
        agent=researcher,
    )

@@ -2837,7 +2840,7 @@ def test_crew_testing_function(kickoff_mock, copy_mock, crew_evaluator):

    crew_evaluator.assert_has_calls(
        [
-            mock.call(crew, "gpt-4o-mini"),
+            mock.call(crew, mock.ANY),
            mock.call().set_iteration(1),
            mock.call().set_iteration(2),
            mock.call().print_crew_evaluation_result(),
@@ -2845,6 +2848,73 @@ def test_crew_testing_function(kickoff_mock, copy_mock, crew_evaluator):
    )


+@mock.patch("crewai.crew.CrewEvaluator")
+@mock.patch("crewai.crew.Crew.copy")
+@mock.patch("crewai.crew.Crew.kickoff")
+def test_crew_testing_with_invalid_llm(kickoff_mock, copy_mock, crew_evaluator_mock):
+    """Test that Crew.test() properly validates LLM input."""
+    task = Task(
+        description="Test task",
+        expected_output="Test output",
+        agent=researcher,
+    )
+    crew = Crew(agents=[researcher], tasks=[task])
+    
+    with pytest.raises(ValueError, match="llm parameter must be either"):
+        crew.test(2, llm=123)  # Invalid type
+
+
+@mock.patch("crewai.crew.CrewEvaluator")
+@mock.patch("crewai.crew.Crew.copy")
+@mock.patch("crewai.crew.Crew.kickoff")
+def test_crew_testing_with_custom_llm(kickoff_mock, copy_mock, crew_evaluator_mock):
+    """Test that Crew.test() works with both string and LLM instance parameters."""
+    task = Task(
+        description="Test task",
+        expected_output="Test output",
+        agent=researcher,
+    )
+
+    crew = Crew(
+        agents=[researcher],
+        tasks=[task],
+    )
+
+    # Create a mock for the copied crew
+    copy_mock.return_value = crew
+
+    # Create a mock evaluator
+    mock_evaluator = mock.MagicMock()
+    mock_evaluator.print_crew_evaluation_result = mock.MagicMock()
+    mock_evaluator.set_iteration = mock.MagicMock()
+
+    # Mock the CrewEvaluator class
+    crew_evaluator_mock.return_value = mock_evaluator
+
+    # Test with string model name
+    crew.test(2, llm="gpt-4o-mini")
+    crew_evaluator_mock.assert_called_with(crew, "gpt-4o-mini")
+    mock_evaluator.set_iteration.assert_has_calls([mock.call(1), mock.call(2)])
+    mock_evaluator.print_crew_evaluation_result.assert_called_once()
+    crew_evaluator_mock.reset_mock()
+    mock_evaluator.reset_mock()
+
+    # Test with LLM instance
+    custom_llm = LLM(model="gpt-4o-mini")
+    crew.test(2, llm=custom_llm)
+    crew_evaluator_mock.assert_called_with(crew, custom_llm)
+    mock_evaluator.set_iteration.assert_has_calls([mock.call(1), mock.call(2)])
+    mock_evaluator.print_crew_evaluation_result.assert_called_once()
+    crew_evaluator_mock.reset_mock()
+    mock_evaluator.reset_mock()
+
+    # Test backward compatibility
+    crew.test(2, openai_model_name="gpt-4o-mini")
+    crew_evaluator_mock.assert_called_with(crew, "gpt-4o-mini")
+    mock_evaluator.set_iteration.assert_has_calls([mock.call(1), mock.call(2)])
+    mock_evaluator.print_crew_evaluation_result.assert_called_once()
+
+
@pytest.mark.vcr(filter_headers=["authorization"])
 def test_hierarchical_verbose_manager_agent():
    task = Task(
@@ -3125,4 +3195,4 @@ def test_multimodal_agent_live_image_analysis():
    # Verify we got a meaningful response
    assert isinstance(result.raw, str)
    assert len(result.raw) > 100  # Expecting a detailed analysis
-    assert "error" not in result.raw.lower()  # No error messages in response
+    assert "error" not in result.raw.lower()  # No error messages in response
--- a/tests/patches/test_litellm_patch.py
+++ b/tests/patches/test_litellm_patch.py
@@ -1,71 +0,0 @@
-"""
-Test for the litellm patch that fixes the IndexError in ollama_pt function.
-"""
-
-import sys
-import unittest
-from unittest.mock import MagicMock, patch
-
-import litellm
-import pytest
-from litellm.litellm_core_utils.prompt_templates.factory import ollama_pt
-
-from crewai.patches.litellm_patch import patch_litellm_ollama_pt
-
-
-class TestLitellmPatch(unittest.TestCase):
-    def test_ollama_pt_patch_fixes_index_error(self):
-        """Test that the patch fixes the IndexError in ollama_pt."""
-        # Create a message list where the assistant message is the last one
-        messages = [
-            {"role": "user", "content": "Hello"},
-            {"role": "assistant", "content": "Hi there"},
-        ]
-        
-        # Store the original function to restore it after the test
-        original_ollama_pt = litellm.litellm_core_utils.prompt_templates.factory.ollama_pt
-        
-        try:
-            # Apply the patch
-            success = patch_litellm_ollama_pt()
-            self.assertTrue(success, "Patch application failed")
-            
-            # Use the function from the module directly to ensure we're using the patched version
-            result = litellm.litellm_core_utils.prompt_templates.factory.ollama_pt("qwen3:4b", messages)
-            
-            # Verify the result is as expected
-            self.assertIn("prompt", result)
-            self.assertIn("images", result)
-            self.assertIn("### User:\nHello", result["prompt"])
-            self.assertIn("### Assistant:\nHi there", result["prompt"])
-        finally:
-            # Restore the original function to avoid affecting other tests
-            litellm.litellm_core_utils.prompt_templates.factory.ollama_pt = original_ollama_pt
-    
-    def test_ollama_pt_patch_with_empty_messages(self):
-        """Test that the patch handles empty message lists."""
-        messages = []
-        
-        # Store the original function to restore it after the test
-        original_ollama_pt = litellm.litellm_core_utils.prompt_templates.factory.ollama_pt
-        
-        try:
-            # Apply the patch
-            success = patch_litellm_ollama_pt()
-            self.assertTrue(success, "Patch application failed")
-            
-            # Use the function from the module directly to ensure we're using the patched version
-            result = litellm.litellm_core_utils.prompt_templates.factory.ollama_pt("qwen3:4b", messages)
-            
-            # Verify the result is as expected
-            self.assertIn("prompt", result)
-            self.assertIn("images", result)
-            self.assertEqual("", result["prompt"])
-            self.assertEqual([], result["images"])
-        finally:
-            # Restore the original function to avoid affecting other tests
-            litellm.litellm_core_utils.prompt_templates.factory.ollama_pt = original_ollama_pt
-
-
-if __name__ == "__main__":
-    unittest.main()
--- a/tests/utilities/evaluators/test_crew_evaluator_handler.py
+++ b/tests/utilities/evaluators/test_crew_evaluator_handler.py
@@ -23,7 +23,7 @@ class TestCrewEvaluator:
        )
        crew = Crew(agents=[agent], tasks=[task])

-        return CrewEvaluator(crew, openai_model_name="gpt-4o-mini")
+        return CrewEvaluator(crew, llm="gpt-4o-mini")

    def test_setup_for_evaluating(self, crew_planner):
        crew_planner._setup_for_evaluating()
--- a/uv.lock
+++ b/uv.lock
@@ -1,10 +1,18 @@
 version = 1
 requires-python = ">=3.10, <3.13"
 resolution-markers = [
-    "python_full_version < '3.11'",
-    "python_full_version == '3.11.*'",
-    "python_full_version >= '3.12' and python_full_version < '3.12.4'",
-    "python_full_version >= '3.12.4'",
+    "python_full_version < '3.11' and sys_platform == 'darwin'",
+    "python_full_version < '3.11' and platform_machine == 'aarch64' and sys_platform == 'linux'",
+    "(python_full_version < '3.11' and platform_machine != 'aarch64' and sys_platform == 'linux') or (python_full_version < '3.11' and sys_platform != 'darwin' and sys_platform != 'linux')",
+    "python_full_version == '3.11.*' and sys_platform == 'darwin'",
+    "python_full_version == '3.11.*' and platform_machine == 'aarch64' and sys_platform == 'linux'",
+    "(python_full_version == '3.11.*' and platform_machine != 'aarch64' and sys_platform == 'linux') or (python_full_version == '3.11.*' and sys_platform != 'darwin' and sys_platform != 'linux')",
+    "python_full_version >= '3.12' and python_full_version < '3.12.4' and sys_platform == 'darwin'",
+    "python_full_version >= '3.12' and python_full_version < '3.12.4' and platform_machine == 'aarch64' and sys_platform == 'linux'",
+    "(python_full_version >= '3.12' and python_full_version < '3.12.4' and platform_machine != 'aarch64' and sys_platform == 'linux') or (python_full_version >= '3.12' and python_full_version < '3.12.4' and sys_platform != 'darwin' and sys_platform != 'linux')",
+    "python_full_version >= '3.12.4' and sys_platform == 'darwin'",
+    "python_full_version >= '3.12.4' and platform_machine == 'aarch64' and sys_platform == 'linux'",
+    "(python_full_version >= '3.12.4' and platform_machine != 'aarch64' and sys_platform == 'linux') or (python_full_version >= '3.12.4' and sys_platform != 'darwin' and sys_platform != 'linux')",
 ]

 [[package]]
@@ -300,7 +308,7 @@ name = "build"
 version = "1.2.2.post1"
 source = { registry = "https://pypi.org/simple" }
 dependencies = [
-    { name = "colorama", marker = "os_name == 'nt'" },
+    { name = "colorama", marker = "(os_name == 'nt' and platform_machine != 'aarch64' and sys_platform == 'linux') or (os_name == 'nt' and sys_platform != 'darwin' and sys_platform != 'linux')" },
    { name = "importlib-metadata", marker = "python_full_version < '3.10.2'" },
    { name = "packaging" },
    { name = "pyproject-hooks" },
@@ -535,7 +543,7 @@ name = "click"
 version = "8.1.7"
 source = { registry = "https://pypi.org/simple" }
 dependencies = [
-    { name = "colorama", marker = "platform_system == 'Windows'" },
+    { name = "colorama", marker = "sys_platform == 'win32'" },
 ]
 sdist = { url = "https://files.pythonhosted.org/packages/96/d3/f04c7bfcf5c1862a2a5b845c6b2b360488cf47af55dfa79c98f6a6bf98b5/click-8.1.7.tar.gz", hash = "sha256:ca9853ad459e787e2192211578cc907e7594e294c7ccc834310722b41b9ca6de", size = 336121 }
 wheels = [
@@ -642,7 +650,6 @@ tools = [
 [package.dev-dependencies]
 dev = [
    { name = "cairosvg" },
-    { name = "crewai-tools" },
    { name = "mkdocs" },
    { name = "mkdocs-material" },
    { name = "mkdocs-material-extensions" },
@@ -696,7 +703,6 @@ requires-dist = [
 [package.metadata.requires-dev]
 dev = [
    { name = "cairosvg", specifier = ">=2.7.1" },
-    { name = "crewai-tools", specifier = ">=0.17.0" },
    { name = "mkdocs", specifier = ">=1.4.3" },
    { name = "mkdocs-material", specifier = ">=9.5.7" },
    { name = "mkdocs-material-extensions", specifier = ">=1.3.1" },
@@ -2462,7 +2468,7 @@ version = "1.6.1"
 source = { registry = "https://pypi.org/simple" }
 dependencies = [
    { name = "click" },
-    { name = "colorama", marker = "platform_system == 'Windows'" },
+    { name = "colorama", marker = "sys_platform == 'win32'" },
    { name = "ghp-import" },
    { name = "jinja2" },
    { name = "markdown" },
@@ -2643,7 +2649,7 @@ version = "2.10.2"
 source = { registry = "https://pypi.org/simple" }
 dependencies = [
    { name = "pygments" },
-    { name = "pywin32", marker = "platform_system == 'Windows'" },
+    { name = "pywin32", marker = "sys_platform == 'win32'" },
    { name = "tqdm" },
 ]
 sdist = { url = "https://files.pythonhosted.org/packages/3a/93/80ac75c20ce54c785648b4ed363c88f148bf22637e10c9863db4fbe73e74/mpire-2.10.2.tar.gz", hash = "sha256:f66a321e93fadff34585a4bfa05e95bd946cf714b442f51c529038eb45773d97", size = 271270 }
@@ -2890,7 +2896,7 @@ name = "nvidia-cudnn-cu12"
 version = "9.1.0.70"
 source = { registry = "https://pypi.org/simple" }
 dependencies = [
-    { name = "nvidia-cublas-cu12", marker = "(platform_machine != 'aarch64' and platform_system != 'Darwin') or (platform_system != 'Darwin' and platform_system != 'Linux')" },
+    { name = "nvidia-cublas-cu12", marker = "(platform_machine != 'aarch64' and sys_platform == 'linux') or (sys_platform != 'darwin' and sys_platform != 'linux')" },
 ]
 wheels = [
    { url = "https://files.pythonhosted.org/packages/9f/fd/713452cd72343f682b1c7b9321e23829f00b842ceaedcda96e742ea0b0b3/nvidia_cudnn_cu12-9.1.0.70-py3-none-manylinux2014_x86_64.whl", hash = "sha256:165764f44ef8c61fcdfdfdbe769d687e06374059fbb388b6c89ecb0e28793a6f", size = 664752741 },
@@ -2917,9 +2923,9 @@ name = "nvidia-cusolver-cu12"
 version = "11.4.5.107"
 source = { registry = "https://pypi.org/simple" }
 dependencies = [
-    { name = "nvidia-cublas-cu12", marker = "(platform_machine != 'aarch64' and platform_system != 'Darwin') or (platform_system != 'Darwin' and platform_system != 'Linux')" },
-    { name = "nvidia-cusparse-cu12", marker = "(platform_machine != 'aarch64' and platform_system != 'Darwin') or (platform_system != 'Darwin' and platform_system != 'Linux')" },
-    { name = "nvidia-nvjitlink-cu12", marker = "(platform_machine != 'aarch64' and platform_system != 'Darwin') or (platform_system != 'Darwin' and platform_system != 'Linux')" },
+    { name = "nvidia-cublas-cu12", marker = "(platform_machine != 'aarch64' and sys_platform == 'linux') or (sys_platform != 'darwin' and sys_platform != 'linux')" },
+    { name = "nvidia-cusparse-cu12", marker = "(platform_machine != 'aarch64' and sys_platform == 'linux') or (sys_platform != 'darwin' and sys_platform != 'linux')" },
+    { name = "nvidia-nvjitlink-cu12", marker = "(platform_machine != 'aarch64' and sys_platform == 'linux') or (sys_platform != 'darwin' and sys_platform != 'linux')" },
 ]
 wheels = [
    { url = "https://files.pythonhosted.org/packages/bc/1d/8de1e5c67099015c834315e333911273a8c6aaba78923dd1d1e25fc5f217/nvidia_cusolver_cu12-11.4.5.107-py3-none-manylinux1_x86_64.whl", hash = "sha256:8a7ec542f0412294b15072fa7dab71d31334014a69f953004ea7a118206fe0dd", size = 124161928 },
@@ -2930,7 +2936,7 @@ name = "nvidia-cusparse-cu12"
 version = "12.1.0.106"
 source = { registry = "https://pypi.org/simple" }
 dependencies = [
-    { name = "nvidia-nvjitlink-cu12", marker = "(platform_machine != 'aarch64' and platform_system != 'Darwin') or (platform_system != 'Darwin' and platform_system != 'Linux')" },
+    { name = "nvidia-nvjitlink-cu12", marker = "(platform_machine != 'aarch64' and sys_platform == 'linux') or (sys_platform != 'darwin' and sys_platform != 'linux')" },
 ]
 wheels = [
    { url = "https://files.pythonhosted.org/packages/65/5b/cfaeebf25cd9fdec14338ccb16f6b2c4c7fa9163aefcf057d86b9cc248bb/nvidia_cusparse_cu12-12.1.0.106-py3-none-manylinux1_x86_64.whl", hash = "sha256:f3b50f42cf363f86ab21f720998517a659a48131e8d538dc02f8768237bd884c", size = 195958278 },
@@ -3480,7 +3486,7 @@ name = "portalocker"
 version = "2.10.1"
 source = { registry = "https://pypi.org/simple" }
 dependencies = [
-    { name = "pywin32", marker = "platform_system == 'Windows'" },
+    { name = "pywin32", marker = "sys_platform == 'win32'" },
 ]
 sdist = { url = "https://files.pythonhosted.org/packages/ed/d3/c6c64067759e87af98cc668c1cc75171347d0f1577fab7ca3749134e3cd4/portalocker-2.10.1.tar.gz", hash = "sha256:ef1bf844e878ab08aee7e40184156e1151f228f103aa5c6bd0724cc330960f8f", size = 40891 }
 wheels = [
@@ -5022,19 +5028,19 @@ dependencies = [
    { name = "fsspec" },
    { name = "jinja2" },
    { name = "networkx" },
-    { name = "nvidia-cublas-cu12", marker = "platform_machine == 'x86_64' and platform_system == 'Linux'" },
-    { name = "nvidia-cuda-cupti-cu12", marker = "platform_machine == 'x86_64' and platform_system == 'Linux'" },
-    { name = "nvidia-cuda-nvrtc-cu12", marker = "platform_machine == 'x86_64' and platform_system == 'Linux'" },
-    { name = "nvidia-cuda-runtime-cu12", marker = "platform_machine == 'x86_64' and platform_system == 'Linux'" },
-    { name = "nvidia-cudnn-cu12", marker = "platform_machine == 'x86_64' and platform_system == 'Linux'" },
-    { name = "nvidia-cufft-cu12", marker = "platform_machine == 'x86_64' and platform_system == 'Linux'" },
-    { name = "nvidia-curand-cu12", marker = "platform_machine == 'x86_64' and platform_system == 'Linux'" },
-    { name = "nvidia-cusolver-cu12", marker = "platform_machine == 'x86_64' and platform_system == 'Linux'" },
-    { name = "nvidia-cusparse-cu12", marker = "platform_machine == 'x86_64' and platform_system == 'Linux'" },
-    { name = "nvidia-nccl-cu12", marker = "platform_machine == 'x86_64' and platform_system == 'Linux'" },
-    { name = "nvidia-nvtx-cu12", marker = "platform_machine == 'x86_64' and platform_system == 'Linux'" },
+    { name = "nvidia-cublas-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" },
+    { name = "nvidia-cuda-cupti-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" },
+    { name = "nvidia-cuda-nvrtc-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" },
+    { name = "nvidia-cuda-runtime-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" },
+    { name = "nvidia-cudnn-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" },
+    { name = "nvidia-cufft-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" },
+    { name = "nvidia-curand-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" },
+    { name = "nvidia-cusolver-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" },
+    { name = "nvidia-cusparse-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" },
+    { name = "nvidia-nccl-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" },
+    { name = "nvidia-nvtx-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" },
    { name = "sympy" },
-    { name = "triton", marker = "platform_machine == 'x86_64' and platform_system == 'Linux'" },
+    { name = "triton", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" },
    { name = "typing-extensions" },
 ]
 wheels = [
@@ -5081,7 +5087,7 @@ name = "tqdm"
 version = "4.66.5"
 source = { registry = "https://pypi.org/simple" }
 dependencies = [
-    { name = "colorama", marker = "platform_system == 'Windows'" },
+    { name = "colorama", marker = "sys_platform == 'win32'" },
 ]
 sdist = { url = "https://files.pythonhosted.org/packages/58/83/6ba9844a41128c62e810fddddd72473201f3eacde02046066142a2d96cc5/tqdm-4.66.5.tar.gz", hash = "sha256:e1020aef2e5096702d8a025ac7d16b1577279c9d63f8375b63083e9a5f0fcbad", size = 169504 }
 wheels = [
@@ -5124,7 +5130,7 @@ version = "0.27.0"
 source = { registry = "https://pypi.org/simple" }
 dependencies = [
    { name = "attrs" },
-    { name = "cffi", marker = "implementation_name != 'pypy' and os_name == 'nt'" },
+    { name = "cffi", marker = "(implementation_name != 'pypy' and os_name == 'nt' and platform_machine != 'aarch64' and sys_platform == 'linux') or (implementation_name != 'pypy' and os_name == 'nt' and sys_platform != 'darwin' and sys_platform != 'linux')" },
    { name = "exceptiongroup", marker = "python_full_version < '3.11'" },
    { name = "idna" },
    { name = "outcome" },
@@ -5155,7 +5161,7 @@ name = "triton"
 version = "3.0.0"
 source = { registry = "https://pypi.org/simple" }
 dependencies = [
-    { name = "filelock", marker = "(platform_machine != 'aarch64' and platform_system != 'Darwin') or (platform_system != 'Darwin' and platform_system != 'Linux')" },
+    { name = "filelock", marker = "(platform_machine != 'aarch64' and sys_platform == 'linux') or (sys_platform != 'darwin' and sys_platform != 'linux')" },
 ]
 wheels = [
    { url = "https://files.pythonhosted.org/packages/45/27/14cc3101409b9b4b9241d2ba7deaa93535a217a211c86c4cc7151fb12181/triton-3.0.0-1-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:e1efef76935b2febc365bfadf74bcb65a6f959a9872e5bddf44cc9e0adce1e1a", size = 209376304 },