Compare commits

...

1 Commits

Author SHA1 Message Date
Devin AI
f7e8949e92 Fix LiteAgent tool calling in LLMCallStartedEvent
- Fix hardcoded tools=None in LLMCallStartedEvent emission
- Convert self._parsed_tools to proper dict format for event
- Add comprehensive tests for tool calling functionality
- Add reproduction test for GitHub issue #3302

Fixes #3302

Co-Authored-By: João <joao@crewai.com>
2025-08-10 04:45:13 +00:00
2 changed files with 166 additions and 33 deletions

View File

@@ -525,7 +525,16 @@ class LiteAgent(FlowTrackable, BaseModel):
self,
event=LLMCallStartedEvent(
messages=self._messages,
tools=None,
tools=[
{
"name": tool.name,
"description": tool.description,
"args": tool.args,
}
for tool in self._parsed_tools
]
if self._parsed_tools
else None,
callbacks=self._callbacks,
from_agent=self,
model=model,

View File

@@ -10,6 +10,7 @@ from crewai.flow import Flow, start
from crewai.lite_agent import LiteAgent, LiteAgentOutput
from crewai.tools import BaseTool
from crewai.utilities.events import crewai_event_bus
from crewai.utilities.events.llm_events import LLMCallStartedEvent
from crewai.utilities.events.agent_events import LiteAgentExecutionStartedEvent
from crewai.utilities.events.tool_usage_events import ToolUsageStartedEvent
from crewai.llms.base_llm import BaseLLM
@@ -148,12 +149,12 @@ def test_lite_agent_with_tools():
"What is the population of Tokyo and how many people would that be per square kilometer if Tokyo's area is 2,194 square kilometers?"
)
assert (
"21 million" in result.raw or "37 million" in result.raw
), "Agent should find Tokyo's population"
assert (
"per square kilometer" in result.raw
), "Agent should calculate population density"
assert "21 million" in result.raw or "37 million" in result.raw, (
"Agent should find Tokyo's population"
)
assert "per square kilometer" in result.raw, (
"Agent should calculate population density"
)
received_events = []
@@ -318,11 +319,17 @@ def test_sets_parent_flow_when_inside_flow():
flow.kickoff()
assert captured_agent.parent_flow is flow
@pytest.mark.vcr(filter_headers=["authorization"])
def test_guardrail_is_called_using_string():
guardrail_events = defaultdict(list)
from crewai.utilities.events import LLMGuardrailCompletedEvent, LLMGuardrailStartedEvent
from crewai.utilities.events import (
LLMGuardrailCompletedEvent,
LLMGuardrailStartedEvent,
)
with crewai_event_bus.scoped_handlers():
@crewai_event_bus.on(LLMGuardrailStartedEvent)
def capture_guardrail_started(source, event):
guardrail_events["started"].append(event)
@@ -340,17 +347,26 @@ def test_guardrail_is_called_using_string():
result = agent.kickoff(messages="Top 10 best players in the world?")
assert len(guardrail_events['started']) == 2
assert len(guardrail_events['completed']) == 2
assert not guardrail_events['completed'][0].success
assert guardrail_events['completed'][1].success
assert "Here are the top 10 best soccer players in the world, focusing exclusively on Brazilian players" in result.raw
assert len(guardrail_events["started"]) == 2
assert len(guardrail_events["completed"]) == 2
assert not guardrail_events["completed"][0].success
assert guardrail_events["completed"][1].success
assert (
"Here are the top 10 best soccer players in the world, focusing exclusively on Brazilian players"
in result.raw
)
@pytest.mark.vcr(filter_headers=["authorization"])
def test_guardrail_is_called_using_callable():
guardrail_events = defaultdict(list)
from crewai.utilities.events import LLMGuardrailCompletedEvent, LLMGuardrailStartedEvent
from crewai.utilities.events import (
LLMGuardrailCompletedEvent,
LLMGuardrailStartedEvent,
)
with crewai_event_bus.scoped_handlers():
@crewai_event_bus.on(LLMGuardrailStartedEvent)
def capture_guardrail_started(source, event):
guardrail_events["started"].append(event)
@@ -368,16 +384,22 @@ def test_guardrail_is_called_using_callable():
result = agent.kickoff(messages="Top 1 best players in the world?")
assert len(guardrail_events['started']) == 1
assert len(guardrail_events['completed']) == 1
assert guardrail_events['completed'][0].success
assert len(guardrail_events["started"]) == 1
assert len(guardrail_events["completed"]) == 1
assert guardrail_events["completed"][0].success
assert "Pelé - Santos, 1958" in result.raw
@pytest.mark.vcr(filter_headers=["authorization"])
def test_guardrail_reached_attempt_limit():
guardrail_events = defaultdict(list)
from crewai.utilities.events import LLMGuardrailCompletedEvent, LLMGuardrailStartedEvent
from crewai.utilities.events import (
LLMGuardrailCompletedEvent,
LLMGuardrailStartedEvent,
)
with crewai_event_bus.scoped_handlers():
@crewai_event_bus.on(LLMGuardrailStartedEvent)
def capture_guardrail_started(source, event):
guardrail_events["started"].append(event)
@@ -390,18 +412,23 @@ def test_guardrail_reached_attempt_limit():
role="Sports Analyst",
goal="Gather information about the best soccer players",
backstory="""You are an expert at gathering and organizing information. You carefully collect details and present them in a structured way.""",
guardrail=lambda output: (False, "You are not allowed to include Brazilian players"),
guardrail=lambda output: (
False,
"You are not allowed to include Brazilian players",
),
guardrail_max_retries=2,
)
with pytest.raises(Exception, match="Agent's guardrail failed validation after 2 retries"):
with pytest.raises(
Exception, match="Agent's guardrail failed validation after 2 retries"
):
agent.kickoff(messages="Top 10 best players in the world?")
assert len(guardrail_events['started']) == 3 # 2 retries + 1 initial call
assert len(guardrail_events['completed']) == 3 # 2 retries + 1 initial call
assert not guardrail_events['completed'][0].success
assert not guardrail_events['completed'][1].success
assert not guardrail_events['completed'][2].success
assert len(guardrail_events["started"]) == 3 # 2 retries + 1 initial call
assert len(guardrail_events["completed"]) == 3 # 2 retries + 1 initial call
assert not guardrail_events["completed"][0].success
assert not guardrail_events["completed"][1].success
assert not guardrail_events["completed"][2].success
@pytest.mark.vcr(filter_headers=["authorization"])
@@ -414,22 +441,35 @@ def test_agent_output_when_guardrail_returns_base_model():
role="Sports Analyst",
goal="Gather information about the best soccer players",
backstory="""You are an expert at gathering and organizing information. You carefully collect details and present them in a structured way.""",
guardrail=lambda output: (True, Player(name="Lionel Messi", country="Argentina")),
guardrail=lambda output: (
True,
Player(name="Lionel Messi", country="Argentina"),
),
)
result = agent.kickoff(messages="Top 10 best players in the world?")
assert result.pydantic == Player(name="Lionel Messi", country="Argentina")
def test_lite_agent_with_custom_llm_and_guardrails():
"""Test that CustomLLM (inheriting from BaseLLM) works with guardrails."""
class CustomLLM(BaseLLM):
def __init__(self, response: str = "Custom response"):
super().__init__(model="custom-model")
self.response = response
self.call_count = 0
def call(self, messages, tools=None, callbacks=None, available_functions=None, from_task=None, from_agent=None) -> str:
def call(
self,
messages,
tools=None,
callbacks=None,
available_functions=None,
from_task=None,
from_agent=None,
) -> str:
self.call_count += 1
if "valid" in str(messages) and "feedback" in str(messages):
@@ -456,7 +496,7 @@ def test_lite_agent_with_custom_llm_and_guardrails():
goal="Analyze soccer players",
backstory="You analyze soccer players and their performance.",
llm=custom_llm,
guardrail="Only include Brazilian players"
guardrail="Only include Brazilian players",
)
result = agent.kickoff("Tell me about the best soccer players")
@@ -474,7 +514,7 @@ def test_lite_agent_with_custom_llm_and_guardrails():
goal="Test goal",
backstory="Test backstory",
llm=custom_llm2,
guardrail=test_guardrail
guardrail=test_guardrail,
)
result2 = agent2.kickoff("Test message")
@@ -484,12 +524,96 @@ def test_lite_agent_with_custom_llm_and_guardrails():
@pytest.mark.vcr(filter_headers=["authorization"])
def test_lite_agent_with_invalid_llm():
"""Test that LiteAgent raises proper error when create_llm returns None."""
with patch('crewai.lite_agent.create_llm', return_value=None):
with patch("crewai.lite_agent.create_llm", return_value=None):
with pytest.raises(ValueError) as exc_info:
LiteAgent(
role="Test Agent",
goal="Test goal",
goal="Test goal",
backstory="Test backstory",
llm="invalid-model"
llm="invalid-model",
)
assert "Expected LLM instance of type BaseLLM" in str(exc_info.value)
assert "Expected LLM instance of type BaseLLM" in str(exc_info.value)
def test_lite_agent_emits_llm_call_started_event_with_tools():
"""Test that LiteAgent emits LLMCallStartedEvent with correct tools information."""
captured_events = []
@crewai_event_bus.on(LLMCallStartedEvent)
def capture_llm_event(source, event):
captured_events.append(event)
agent = LiteAgent(
role="Test Agent",
goal="Test Goal",
backstory="Test Backstory",
tools=[WebSearchTool(), CalculatorTool()],
llm=Mock(spec=LLM),
)
agent.llm.call.return_value = "Final Answer: Test response"
agent.kickoff("Test query")
# Verify event was emitted with tools
assert len(captured_events) > 0
event = captured_events[0]
assert event.tools is not None
assert len(event.tools) == 2
# Verify tool structure
tool_names = [tool["name"] for tool in event.tools]
assert "search_web" in tool_names
assert "calculate" in tool_names
for tool in event.tools:
assert "name" in tool
assert "description" in tool
assert "args" in tool
def test_lite_agent_emits_llm_call_started_event_without_tools():
"""Test that LiteAgent emits LLMCallStartedEvent with tools=None when no tools provided."""
captured_events = []
@crewai_event_bus.on(LLMCallStartedEvent)
def capture_llm_event(source, event):
captured_events.append(event)
agent = LiteAgent(
role="Test Agent",
goal="Test Goal",
backstory="Test Backstory",
tools=[], # No tools
llm=Mock(spec=LLM),
)
agent.llm.call.return_value = "Final Answer: Test response"
agent.kickoff("Test query")
assert len(captured_events) > 0
event = captured_events[0]
assert event.tools is None
@pytest.mark.vcr(filter_headers=["authorization"])
def test_lite_agent_tool_calling_reproduction():
"""Test reproduction of GitHub issue #3302 - LiteAgent tool calling."""
agent = LiteAgent(
role="Research Assistant",
goal="Help with research tasks",
backstory="You are a helpful research assistant.",
tools=[WebSearchTool()],
llm=LLM(model="gpt-4o-mini"),
)
result = agent.kickoff("Search for information about Python programming")
# Verify the agent produced a meaningful result
assert result.raw is not None
assert len(result.raw) > 0
# Verify tools were actually available during execution
assert len(agent._parsed_tools) > 0