From f7e8949e92ca8dcc10a8c344cfe1aaf2be968c6d Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Sun, 10 Aug 2025 04:45:13 +0000 Subject: [PATCH] Fix LiteAgent tool calling in LLMCallStartedEvent MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Fix hardcoded tools=None in LLMCallStartedEvent emission - Convert self._parsed_tools to proper dict format for event - Add comprehensive tests for tool calling functionality - Add reproduction test for GitHub issue #3302 Fixes #3302 Co-Authored-By: João --- src/crewai/lite_agent.py | 11 ++- tests/test_lite_agent.py | 188 ++++++++++++++++++++++++++++++++------- 2 files changed, 166 insertions(+), 33 deletions(-) diff --git a/src/crewai/lite_agent.py b/src/crewai/lite_agent.py index 2bd85fbe9..a1f13e738 100644 --- a/src/crewai/lite_agent.py +++ b/src/crewai/lite_agent.py @@ -525,7 +525,16 @@ class LiteAgent(FlowTrackable, BaseModel): self, event=LLMCallStartedEvent( messages=self._messages, - tools=None, + tools=[ + { + "name": tool.name, + "description": tool.description, + "args": tool.args, + } + for tool in self._parsed_tools + ] + if self._parsed_tools + else None, callbacks=self._callbacks, from_agent=self, model=model, diff --git a/tests/test_lite_agent.py b/tests/test_lite_agent.py index b495cb800..1fa4c526c 100644 --- a/tests/test_lite_agent.py +++ b/tests/test_lite_agent.py @@ -10,6 +10,7 @@ from crewai.flow import Flow, start from crewai.lite_agent import LiteAgent, LiteAgentOutput from crewai.tools import BaseTool from crewai.utilities.events import crewai_event_bus +from crewai.utilities.events.llm_events import LLMCallStartedEvent from crewai.utilities.events.agent_events import LiteAgentExecutionStartedEvent from crewai.utilities.events.tool_usage_events import ToolUsageStartedEvent from crewai.llms.base_llm import BaseLLM @@ -148,12 +149,12 @@ def test_lite_agent_with_tools(): "What is the population of Tokyo and how many people would that be per square kilometer if Tokyo's area is 2,194 square kilometers?" ) - assert ( - "21 million" in result.raw or "37 million" in result.raw - ), "Agent should find Tokyo's population" - assert ( - "per square kilometer" in result.raw - ), "Agent should calculate population density" + assert "21 million" in result.raw or "37 million" in result.raw, ( + "Agent should find Tokyo's population" + ) + assert "per square kilometer" in result.raw, ( + "Agent should calculate population density" + ) received_events = [] @@ -318,11 +319,17 @@ def test_sets_parent_flow_when_inside_flow(): flow.kickoff() assert captured_agent.parent_flow is flow + @pytest.mark.vcr(filter_headers=["authorization"]) def test_guardrail_is_called_using_string(): guardrail_events = defaultdict(list) - from crewai.utilities.events import LLMGuardrailCompletedEvent, LLMGuardrailStartedEvent + from crewai.utilities.events import ( + LLMGuardrailCompletedEvent, + LLMGuardrailStartedEvent, + ) + with crewai_event_bus.scoped_handlers(): + @crewai_event_bus.on(LLMGuardrailStartedEvent) def capture_guardrail_started(source, event): guardrail_events["started"].append(event) @@ -340,17 +347,26 @@ def test_guardrail_is_called_using_string(): result = agent.kickoff(messages="Top 10 best players in the world?") - assert len(guardrail_events['started']) == 2 - assert len(guardrail_events['completed']) == 2 - assert not guardrail_events['completed'][0].success - assert guardrail_events['completed'][1].success - assert "Here are the top 10 best soccer players in the world, focusing exclusively on Brazilian players" in result.raw + assert len(guardrail_events["started"]) == 2 + assert len(guardrail_events["completed"]) == 2 + assert not guardrail_events["completed"][0].success + assert guardrail_events["completed"][1].success + assert ( + "Here are the top 10 best soccer players in the world, focusing exclusively on Brazilian players" + in result.raw + ) + @pytest.mark.vcr(filter_headers=["authorization"]) def test_guardrail_is_called_using_callable(): guardrail_events = defaultdict(list) - from crewai.utilities.events import LLMGuardrailCompletedEvent, LLMGuardrailStartedEvent + from crewai.utilities.events import ( + LLMGuardrailCompletedEvent, + LLMGuardrailStartedEvent, + ) + with crewai_event_bus.scoped_handlers(): + @crewai_event_bus.on(LLMGuardrailStartedEvent) def capture_guardrail_started(source, event): guardrail_events["started"].append(event) @@ -368,16 +384,22 @@ def test_guardrail_is_called_using_callable(): result = agent.kickoff(messages="Top 1 best players in the world?") - assert len(guardrail_events['started']) == 1 - assert len(guardrail_events['completed']) == 1 - assert guardrail_events['completed'][0].success + assert len(guardrail_events["started"]) == 1 + assert len(guardrail_events["completed"]) == 1 + assert guardrail_events["completed"][0].success assert "Pelé - Santos, 1958" in result.raw + @pytest.mark.vcr(filter_headers=["authorization"]) def test_guardrail_reached_attempt_limit(): guardrail_events = defaultdict(list) - from crewai.utilities.events import LLMGuardrailCompletedEvent, LLMGuardrailStartedEvent + from crewai.utilities.events import ( + LLMGuardrailCompletedEvent, + LLMGuardrailStartedEvent, + ) + with crewai_event_bus.scoped_handlers(): + @crewai_event_bus.on(LLMGuardrailStartedEvent) def capture_guardrail_started(source, event): guardrail_events["started"].append(event) @@ -390,18 +412,23 @@ def test_guardrail_reached_attempt_limit(): role="Sports Analyst", goal="Gather information about the best soccer players", backstory="""You are an expert at gathering and organizing information. You carefully collect details and present them in a structured way.""", - guardrail=lambda output: (False, "You are not allowed to include Brazilian players"), + guardrail=lambda output: ( + False, + "You are not allowed to include Brazilian players", + ), guardrail_max_retries=2, ) - with pytest.raises(Exception, match="Agent's guardrail failed validation after 2 retries"): + with pytest.raises( + Exception, match="Agent's guardrail failed validation after 2 retries" + ): agent.kickoff(messages="Top 10 best players in the world?") - assert len(guardrail_events['started']) == 3 # 2 retries + 1 initial call - assert len(guardrail_events['completed']) == 3 # 2 retries + 1 initial call - assert not guardrail_events['completed'][0].success - assert not guardrail_events['completed'][1].success - assert not guardrail_events['completed'][2].success + assert len(guardrail_events["started"]) == 3 # 2 retries + 1 initial call + assert len(guardrail_events["completed"]) == 3 # 2 retries + 1 initial call + assert not guardrail_events["completed"][0].success + assert not guardrail_events["completed"][1].success + assert not guardrail_events["completed"][2].success @pytest.mark.vcr(filter_headers=["authorization"]) @@ -414,22 +441,35 @@ def test_agent_output_when_guardrail_returns_base_model(): role="Sports Analyst", goal="Gather information about the best soccer players", backstory="""You are an expert at gathering and organizing information. You carefully collect details and present them in a structured way.""", - guardrail=lambda output: (True, Player(name="Lionel Messi", country="Argentina")), + guardrail=lambda output: ( + True, + Player(name="Lionel Messi", country="Argentina"), + ), ) result = agent.kickoff(messages="Top 10 best players in the world?") assert result.pydantic == Player(name="Lionel Messi", country="Argentina") + def test_lite_agent_with_custom_llm_and_guardrails(): """Test that CustomLLM (inheriting from BaseLLM) works with guardrails.""" + class CustomLLM(BaseLLM): def __init__(self, response: str = "Custom response"): super().__init__(model="custom-model") self.response = response self.call_count = 0 - def call(self, messages, tools=None, callbacks=None, available_functions=None, from_task=None, from_agent=None) -> str: + def call( + self, + messages, + tools=None, + callbacks=None, + available_functions=None, + from_task=None, + from_agent=None, + ) -> str: self.call_count += 1 if "valid" in str(messages) and "feedback" in str(messages): @@ -456,7 +496,7 @@ def test_lite_agent_with_custom_llm_and_guardrails(): goal="Analyze soccer players", backstory="You analyze soccer players and their performance.", llm=custom_llm, - guardrail="Only include Brazilian players" + guardrail="Only include Brazilian players", ) result = agent.kickoff("Tell me about the best soccer players") @@ -474,7 +514,7 @@ def test_lite_agent_with_custom_llm_and_guardrails(): goal="Test goal", backstory="Test backstory", llm=custom_llm2, - guardrail=test_guardrail + guardrail=test_guardrail, ) result2 = agent2.kickoff("Test message") @@ -484,12 +524,96 @@ def test_lite_agent_with_custom_llm_and_guardrails(): @pytest.mark.vcr(filter_headers=["authorization"]) def test_lite_agent_with_invalid_llm(): """Test that LiteAgent raises proper error when create_llm returns None.""" - with patch('crewai.lite_agent.create_llm', return_value=None): + with patch("crewai.lite_agent.create_llm", return_value=None): with pytest.raises(ValueError) as exc_info: LiteAgent( role="Test Agent", - goal="Test goal", + goal="Test goal", backstory="Test backstory", - llm="invalid-model" + llm="invalid-model", ) - assert "Expected LLM instance of type BaseLLM" in str(exc_info.value) \ No newline at end of file + assert "Expected LLM instance of type BaseLLM" in str(exc_info.value) + + +def test_lite_agent_emits_llm_call_started_event_with_tools(): + """Test that LiteAgent emits LLMCallStartedEvent with correct tools information.""" + captured_events = [] + + @crewai_event_bus.on(LLMCallStartedEvent) + def capture_llm_event(source, event): + captured_events.append(event) + + agent = LiteAgent( + role="Test Agent", + goal="Test Goal", + backstory="Test Backstory", + tools=[WebSearchTool(), CalculatorTool()], + llm=Mock(spec=LLM), + ) + + agent.llm.call.return_value = "Final Answer: Test response" + + agent.kickoff("Test query") + + # Verify event was emitted with tools + assert len(captured_events) > 0 + event = captured_events[0] + assert event.tools is not None + assert len(event.tools) == 2 + + # Verify tool structure + tool_names = [tool["name"] for tool in event.tools] + assert "search_web" in tool_names + assert "calculate" in tool_names + + for tool in event.tools: + assert "name" in tool + assert "description" in tool + assert "args" in tool + + +def test_lite_agent_emits_llm_call_started_event_without_tools(): + """Test that LiteAgent emits LLMCallStartedEvent with tools=None when no tools provided.""" + captured_events = [] + + @crewai_event_bus.on(LLMCallStartedEvent) + def capture_llm_event(source, event): + captured_events.append(event) + + agent = LiteAgent( + role="Test Agent", + goal="Test Goal", + backstory="Test Backstory", + tools=[], # No tools + llm=Mock(spec=LLM), + ) + + agent.llm.call.return_value = "Final Answer: Test response" + + agent.kickoff("Test query") + + assert len(captured_events) > 0 + event = captured_events[0] + assert event.tools is None + + +@pytest.mark.vcr(filter_headers=["authorization"]) +def test_lite_agent_tool_calling_reproduction(): + """Test reproduction of GitHub issue #3302 - LiteAgent tool calling.""" + + agent = LiteAgent( + role="Research Assistant", + goal="Help with research tasks", + backstory="You are a helpful research assistant.", + tools=[WebSearchTool()], + llm=LLM(model="gpt-4o-mini"), + ) + + result = agent.kickoff("Search for information about Python programming") + + # Verify the agent produced a meaningful result + assert result.raw is not None + assert len(result.raw) > 0 + + # Verify tools were actually available during execution + assert len(agent._parsed_tools) > 0