From b2bda39e56528703dcc21b7b6253a4d635f18886 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Sat, 21 Jun 2025 16:33:22 +0000 Subject: [PATCH] Fix Pydantic validation error in LLMCallStartedEvent when TokenCalcHandler in tools list MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Add model_validator to sanitize tools list before validation - Filter out non-dict objects like TokenCalcHandler from tools list - Preserve dict tools while removing problematic objects - Add comprehensive test coverage for the fix and edge cases - Resolves issue #3043 Co-Authored-By: João --- src/crewai/utilities/events/llm_events.py | 11 +- .../events/test_llm_events_validation.py | 135 ++++++++++++++++++ 2 files changed, 145 insertions(+), 1 deletion(-) create mode 100644 tests/utilities/events/test_llm_events_validation.py diff --git a/src/crewai/utilities/events/llm_events.py b/src/crewai/utilities/events/llm_events.py index ca8d0367a..7df638e8b 100644 --- a/src/crewai/utilities/events/llm_events.py +++ b/src/crewai/utilities/events/llm_events.py @@ -1,7 +1,7 @@ from enum import Enum from typing import Any, Dict, List, Optional, Union -from pydantic import BaseModel +from pydantic import BaseModel, model_validator from crewai.utilities.events.base_events import BaseEvent @@ -27,6 +27,15 @@ class LLMCallStartedEvent(BaseEvent): callbacks: Optional[List[Any]] = None available_functions: Optional[Dict[str, Any]] = None + @model_validator(mode='before') + @classmethod + def sanitize_tools(cls, values): + """Sanitize tools list to only include dict objects, filtering out non-dict objects like TokenCalcHandler""" + if isinstance(values, dict) and 'tools' in values and values['tools'] is not None: + if isinstance(values['tools'], list): + values['tools'] = [tool for tool in values['tools'] if isinstance(tool, dict)] + return values + class LLMCallCompletedEvent(BaseEvent): """Event emitted when a LLM call completes""" diff --git a/tests/utilities/events/test_llm_events_validation.py b/tests/utilities/events/test_llm_events_validation.py new file mode 100644 index 000000000..ee16606d2 --- /dev/null +++ b/tests/utilities/events/test_llm_events_validation.py @@ -0,0 +1,135 @@ +import pytest +from crewai.utilities.events.llm_events import LLMCallStartedEvent +from crewai.utilities.token_counter_callback import TokenCalcHandler +from crewai.agents.agent_builder.utilities.base_token_process import TokenProcess + + +class TestLLMCallStartedEventValidation: + """Test cases for LLMCallStartedEvent validation and sanitization""" + + def test_normal_dict_tools_work(self): + """Test that normal dict tools work correctly""" + event = LLMCallStartedEvent( + messages=[{"role": "user", "content": "test message"}], + tools=[{"name": "tool1"}, {"name": "tool2"}], + callbacks=None + ) + assert event.tools == [{"name": "tool1"}, {"name": "tool2"}] + assert event.type == "llm_call_started" + + def test_token_calc_handler_in_tools_filtered_out(self): + """Test that TokenCalcHandler objects in tools list are filtered out""" + token_handler = TokenCalcHandler(TokenProcess()) + + event = LLMCallStartedEvent( + messages=[{"role": "user", "content": "test message"}], + tools=[{"name": "tool1"}, token_handler, {"name": "tool2"}], + callbacks=None + ) + + assert event.tools == [{"name": "tool1"}, {"name": "tool2"}] + assert len(event.tools) == 2 + + def test_mixed_objects_in_tools_only_dicts_preserved(self): + """Test that only dict objects are preserved when mixed types are in tools""" + token_handler = TokenCalcHandler(TokenProcess()) + + event = LLMCallStartedEvent( + messages=[{"role": "user", "content": "test message"}], + tools=[ + {"name": "tool1"}, + token_handler, + "string_tool", + {"name": "tool2"}, + 123, + {"name": "tool3"} + ], + callbacks=None + ) + + assert event.tools == [{"name": "tool1"}, {"name": "tool2"}, {"name": "tool3"}] + assert len(event.tools) == 3 + + def test_empty_tools_list_handled(self): + """Test that empty tools list is handled correctly""" + event = LLMCallStartedEvent( + messages=[{"role": "user", "content": "test message"}], + tools=[], + callbacks=None + ) + assert event.tools == [] + + def test_none_tools_handled(self): + """Test that None tools value is handled correctly""" + event = LLMCallStartedEvent( + messages=[{"role": "user", "content": "test message"}], + tools=None, + callbacks=None + ) + assert event.tools is None + + def test_all_non_dict_tools_results_in_empty_list(self): + """Test that when all tools are non-dict objects, result is empty list""" + token_handler = TokenCalcHandler(TokenProcess()) + + event = LLMCallStartedEvent( + messages=[{"role": "user", "content": "test message"}], + tools=[token_handler, "string_tool", 123, ["list_tool"]], + callbacks=None + ) + + assert event.tools == [] + + def test_reproduction_case_from_issue_3043(self): + """Test the exact reproduction case from GitHub issue #3043""" + token_handler = TokenCalcHandler(TokenProcess()) + + event = LLMCallStartedEvent( + messages=[{"role": "user", "content": "test message"}], + tools=[{"name": "tool1"}, token_handler], + callbacks=None + ) + + assert event.tools == [{"name": "tool1"}] + assert len(event.tools) == 1 + + def test_callbacks_with_token_handler_still_work(self): + """Test that TokenCalcHandler in callbacks still works normally""" + token_handler = TokenCalcHandler(TokenProcess()) + + event = LLMCallStartedEvent( + messages=[{"role": "user", "content": "test message"}], + tools=[{"name": "tool1"}], + callbacks=[token_handler] + ) + + assert event.tools == [{"name": "tool1"}] + assert event.callbacks == [token_handler] + + def test_string_messages_work(self): + """Test that string messages work with tool sanitization""" + token_handler = TokenCalcHandler(TokenProcess()) + + event = LLMCallStartedEvent( + messages="test message", + tools=[{"name": "tool1"}, token_handler], + callbacks=None + ) + + assert event.messages == "test message" + assert event.tools == [{"name": "tool1"}] + + def test_available_functions_preserved(self): + """Test that available_functions are preserved during sanitization""" + token_handler = TokenCalcHandler(TokenProcess()) + available_funcs = {"func1": lambda x: x} + + event = LLMCallStartedEvent( + messages=[{"role": "user", "content": "test message"}], + tools=[{"name": "tool1"}, token_handler], + callbacks=None, + available_functions=available_funcs + ) + + assert event.tools == [{"name": "tool1"}] + assert event.available_functions == available_funcs