mirror of
https://github.com/crewAIInc/crewAI.git
synced 2026-01-11 00:58:30 +00:00
Fix LiteAgent tool calling in LLMCallStartedEvent
- Fix hardcoded tools=None in LLMCallStartedEvent emission - Convert self._parsed_tools to proper dict format for event - Add comprehensive tests for tool calling functionality - Add reproduction test for GitHub issue #3302 Fixes #3302 Co-Authored-By: João <joao@crewai.com>
This commit is contained in:
@@ -525,7 +525,16 @@ class LiteAgent(FlowTrackable, BaseModel):
|
|||||||
self,
|
self,
|
||||||
event=LLMCallStartedEvent(
|
event=LLMCallStartedEvent(
|
||||||
messages=self._messages,
|
messages=self._messages,
|
||||||
tools=None,
|
tools=[
|
||||||
|
{
|
||||||
|
"name": tool.name,
|
||||||
|
"description": tool.description,
|
||||||
|
"args": tool.args,
|
||||||
|
}
|
||||||
|
for tool in self._parsed_tools
|
||||||
|
]
|
||||||
|
if self._parsed_tools
|
||||||
|
else None,
|
||||||
callbacks=self._callbacks,
|
callbacks=self._callbacks,
|
||||||
from_agent=self,
|
from_agent=self,
|
||||||
model=model,
|
model=model,
|
||||||
|
|||||||
@@ -10,6 +10,7 @@ from crewai.flow import Flow, start
|
|||||||
from crewai.lite_agent import LiteAgent, LiteAgentOutput
|
from crewai.lite_agent import LiteAgent, LiteAgentOutput
|
||||||
from crewai.tools import BaseTool
|
from crewai.tools import BaseTool
|
||||||
from crewai.utilities.events import crewai_event_bus
|
from crewai.utilities.events import crewai_event_bus
|
||||||
|
from crewai.utilities.events.llm_events import LLMCallStartedEvent
|
||||||
from crewai.utilities.events.agent_events import LiteAgentExecutionStartedEvent
|
from crewai.utilities.events.agent_events import LiteAgentExecutionStartedEvent
|
||||||
from crewai.utilities.events.tool_usage_events import ToolUsageStartedEvent
|
from crewai.utilities.events.tool_usage_events import ToolUsageStartedEvent
|
||||||
from crewai.llms.base_llm import BaseLLM
|
from crewai.llms.base_llm import BaseLLM
|
||||||
@@ -148,12 +149,12 @@ def test_lite_agent_with_tools():
|
|||||||
"What is the population of Tokyo and how many people would that be per square kilometer if Tokyo's area is 2,194 square kilometers?"
|
"What is the population of Tokyo and how many people would that be per square kilometer if Tokyo's area is 2,194 square kilometers?"
|
||||||
)
|
)
|
||||||
|
|
||||||
assert (
|
assert "21 million" in result.raw or "37 million" in result.raw, (
|
||||||
"21 million" in result.raw or "37 million" in result.raw
|
"Agent should find Tokyo's population"
|
||||||
), "Agent should find Tokyo's population"
|
)
|
||||||
assert (
|
assert "per square kilometer" in result.raw, (
|
||||||
"per square kilometer" in result.raw
|
"Agent should calculate population density"
|
||||||
), "Agent should calculate population density"
|
)
|
||||||
|
|
||||||
received_events = []
|
received_events = []
|
||||||
|
|
||||||
@@ -318,11 +319,17 @@ def test_sets_parent_flow_when_inside_flow():
|
|||||||
flow.kickoff()
|
flow.kickoff()
|
||||||
assert captured_agent.parent_flow is flow
|
assert captured_agent.parent_flow is flow
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.vcr(filter_headers=["authorization"])
|
@pytest.mark.vcr(filter_headers=["authorization"])
|
||||||
def test_guardrail_is_called_using_string():
|
def test_guardrail_is_called_using_string():
|
||||||
guardrail_events = defaultdict(list)
|
guardrail_events = defaultdict(list)
|
||||||
from crewai.utilities.events import LLMGuardrailCompletedEvent, LLMGuardrailStartedEvent
|
from crewai.utilities.events import (
|
||||||
|
LLMGuardrailCompletedEvent,
|
||||||
|
LLMGuardrailStartedEvent,
|
||||||
|
)
|
||||||
|
|
||||||
with crewai_event_bus.scoped_handlers():
|
with crewai_event_bus.scoped_handlers():
|
||||||
|
|
||||||
@crewai_event_bus.on(LLMGuardrailStartedEvent)
|
@crewai_event_bus.on(LLMGuardrailStartedEvent)
|
||||||
def capture_guardrail_started(source, event):
|
def capture_guardrail_started(source, event):
|
||||||
guardrail_events["started"].append(event)
|
guardrail_events["started"].append(event)
|
||||||
@@ -340,17 +347,26 @@ def test_guardrail_is_called_using_string():
|
|||||||
|
|
||||||
result = agent.kickoff(messages="Top 10 best players in the world?")
|
result = agent.kickoff(messages="Top 10 best players in the world?")
|
||||||
|
|
||||||
assert len(guardrail_events['started']) == 2
|
assert len(guardrail_events["started"]) == 2
|
||||||
assert len(guardrail_events['completed']) == 2
|
assert len(guardrail_events["completed"]) == 2
|
||||||
assert not guardrail_events['completed'][0].success
|
assert not guardrail_events["completed"][0].success
|
||||||
assert guardrail_events['completed'][1].success
|
assert guardrail_events["completed"][1].success
|
||||||
assert "Here are the top 10 best soccer players in the world, focusing exclusively on Brazilian players" in result.raw
|
assert (
|
||||||
|
"Here are the top 10 best soccer players in the world, focusing exclusively on Brazilian players"
|
||||||
|
in result.raw
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.vcr(filter_headers=["authorization"])
|
@pytest.mark.vcr(filter_headers=["authorization"])
|
||||||
def test_guardrail_is_called_using_callable():
|
def test_guardrail_is_called_using_callable():
|
||||||
guardrail_events = defaultdict(list)
|
guardrail_events = defaultdict(list)
|
||||||
from crewai.utilities.events import LLMGuardrailCompletedEvent, LLMGuardrailStartedEvent
|
from crewai.utilities.events import (
|
||||||
|
LLMGuardrailCompletedEvent,
|
||||||
|
LLMGuardrailStartedEvent,
|
||||||
|
)
|
||||||
|
|
||||||
with crewai_event_bus.scoped_handlers():
|
with crewai_event_bus.scoped_handlers():
|
||||||
|
|
||||||
@crewai_event_bus.on(LLMGuardrailStartedEvent)
|
@crewai_event_bus.on(LLMGuardrailStartedEvent)
|
||||||
def capture_guardrail_started(source, event):
|
def capture_guardrail_started(source, event):
|
||||||
guardrail_events["started"].append(event)
|
guardrail_events["started"].append(event)
|
||||||
@@ -368,16 +384,22 @@ def test_guardrail_is_called_using_callable():
|
|||||||
|
|
||||||
result = agent.kickoff(messages="Top 1 best players in the world?")
|
result = agent.kickoff(messages="Top 1 best players in the world?")
|
||||||
|
|
||||||
assert len(guardrail_events['started']) == 1
|
assert len(guardrail_events["started"]) == 1
|
||||||
assert len(guardrail_events['completed']) == 1
|
assert len(guardrail_events["completed"]) == 1
|
||||||
assert guardrail_events['completed'][0].success
|
assert guardrail_events["completed"][0].success
|
||||||
assert "Pelé - Santos, 1958" in result.raw
|
assert "Pelé - Santos, 1958" in result.raw
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.vcr(filter_headers=["authorization"])
|
@pytest.mark.vcr(filter_headers=["authorization"])
|
||||||
def test_guardrail_reached_attempt_limit():
|
def test_guardrail_reached_attempt_limit():
|
||||||
guardrail_events = defaultdict(list)
|
guardrail_events = defaultdict(list)
|
||||||
from crewai.utilities.events import LLMGuardrailCompletedEvent, LLMGuardrailStartedEvent
|
from crewai.utilities.events import (
|
||||||
|
LLMGuardrailCompletedEvent,
|
||||||
|
LLMGuardrailStartedEvent,
|
||||||
|
)
|
||||||
|
|
||||||
with crewai_event_bus.scoped_handlers():
|
with crewai_event_bus.scoped_handlers():
|
||||||
|
|
||||||
@crewai_event_bus.on(LLMGuardrailStartedEvent)
|
@crewai_event_bus.on(LLMGuardrailStartedEvent)
|
||||||
def capture_guardrail_started(source, event):
|
def capture_guardrail_started(source, event):
|
||||||
guardrail_events["started"].append(event)
|
guardrail_events["started"].append(event)
|
||||||
@@ -390,18 +412,23 @@ def test_guardrail_reached_attempt_limit():
|
|||||||
role="Sports Analyst",
|
role="Sports Analyst",
|
||||||
goal="Gather information about the best soccer players",
|
goal="Gather information about the best soccer players",
|
||||||
backstory="""You are an expert at gathering and organizing information. You carefully collect details and present them in a structured way.""",
|
backstory="""You are an expert at gathering and organizing information. You carefully collect details and present them in a structured way.""",
|
||||||
guardrail=lambda output: (False, "You are not allowed to include Brazilian players"),
|
guardrail=lambda output: (
|
||||||
|
False,
|
||||||
|
"You are not allowed to include Brazilian players",
|
||||||
|
),
|
||||||
guardrail_max_retries=2,
|
guardrail_max_retries=2,
|
||||||
)
|
)
|
||||||
|
|
||||||
with pytest.raises(Exception, match="Agent's guardrail failed validation after 2 retries"):
|
with pytest.raises(
|
||||||
|
Exception, match="Agent's guardrail failed validation after 2 retries"
|
||||||
|
):
|
||||||
agent.kickoff(messages="Top 10 best players in the world?")
|
agent.kickoff(messages="Top 10 best players in the world?")
|
||||||
|
|
||||||
assert len(guardrail_events['started']) == 3 # 2 retries + 1 initial call
|
assert len(guardrail_events["started"]) == 3 # 2 retries + 1 initial call
|
||||||
assert len(guardrail_events['completed']) == 3 # 2 retries + 1 initial call
|
assert len(guardrail_events["completed"]) == 3 # 2 retries + 1 initial call
|
||||||
assert not guardrail_events['completed'][0].success
|
assert not guardrail_events["completed"][0].success
|
||||||
assert not guardrail_events['completed'][1].success
|
assert not guardrail_events["completed"][1].success
|
||||||
assert not guardrail_events['completed'][2].success
|
assert not guardrail_events["completed"][2].success
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.vcr(filter_headers=["authorization"])
|
@pytest.mark.vcr(filter_headers=["authorization"])
|
||||||
@@ -414,22 +441,35 @@ def test_agent_output_when_guardrail_returns_base_model():
|
|||||||
role="Sports Analyst",
|
role="Sports Analyst",
|
||||||
goal="Gather information about the best soccer players",
|
goal="Gather information about the best soccer players",
|
||||||
backstory="""You are an expert at gathering and organizing information. You carefully collect details and present them in a structured way.""",
|
backstory="""You are an expert at gathering and organizing information. You carefully collect details and present them in a structured way.""",
|
||||||
guardrail=lambda output: (True, Player(name="Lionel Messi", country="Argentina")),
|
guardrail=lambda output: (
|
||||||
|
True,
|
||||||
|
Player(name="Lionel Messi", country="Argentina"),
|
||||||
|
),
|
||||||
)
|
)
|
||||||
|
|
||||||
result = agent.kickoff(messages="Top 10 best players in the world?")
|
result = agent.kickoff(messages="Top 10 best players in the world?")
|
||||||
|
|
||||||
assert result.pydantic == Player(name="Lionel Messi", country="Argentina")
|
assert result.pydantic == Player(name="Lionel Messi", country="Argentina")
|
||||||
|
|
||||||
|
|
||||||
def test_lite_agent_with_custom_llm_and_guardrails():
|
def test_lite_agent_with_custom_llm_and_guardrails():
|
||||||
"""Test that CustomLLM (inheriting from BaseLLM) works with guardrails."""
|
"""Test that CustomLLM (inheriting from BaseLLM) works with guardrails."""
|
||||||
|
|
||||||
class CustomLLM(BaseLLM):
|
class CustomLLM(BaseLLM):
|
||||||
def __init__(self, response: str = "Custom response"):
|
def __init__(self, response: str = "Custom response"):
|
||||||
super().__init__(model="custom-model")
|
super().__init__(model="custom-model")
|
||||||
self.response = response
|
self.response = response
|
||||||
self.call_count = 0
|
self.call_count = 0
|
||||||
|
|
||||||
def call(self, messages, tools=None, callbacks=None, available_functions=None, from_task=None, from_agent=None) -> str:
|
def call(
|
||||||
|
self,
|
||||||
|
messages,
|
||||||
|
tools=None,
|
||||||
|
callbacks=None,
|
||||||
|
available_functions=None,
|
||||||
|
from_task=None,
|
||||||
|
from_agent=None,
|
||||||
|
) -> str:
|
||||||
self.call_count += 1
|
self.call_count += 1
|
||||||
|
|
||||||
if "valid" in str(messages) and "feedback" in str(messages):
|
if "valid" in str(messages) and "feedback" in str(messages):
|
||||||
@@ -456,7 +496,7 @@ def test_lite_agent_with_custom_llm_and_guardrails():
|
|||||||
goal="Analyze soccer players",
|
goal="Analyze soccer players",
|
||||||
backstory="You analyze soccer players and their performance.",
|
backstory="You analyze soccer players and their performance.",
|
||||||
llm=custom_llm,
|
llm=custom_llm,
|
||||||
guardrail="Only include Brazilian players"
|
guardrail="Only include Brazilian players",
|
||||||
)
|
)
|
||||||
|
|
||||||
result = agent.kickoff("Tell me about the best soccer players")
|
result = agent.kickoff("Tell me about the best soccer players")
|
||||||
@@ -474,7 +514,7 @@ def test_lite_agent_with_custom_llm_and_guardrails():
|
|||||||
goal="Test goal",
|
goal="Test goal",
|
||||||
backstory="Test backstory",
|
backstory="Test backstory",
|
||||||
llm=custom_llm2,
|
llm=custom_llm2,
|
||||||
guardrail=test_guardrail
|
guardrail=test_guardrail,
|
||||||
)
|
)
|
||||||
|
|
||||||
result2 = agent2.kickoff("Test message")
|
result2 = agent2.kickoff("Test message")
|
||||||
@@ -484,12 +524,96 @@ def test_lite_agent_with_custom_llm_and_guardrails():
|
|||||||
@pytest.mark.vcr(filter_headers=["authorization"])
|
@pytest.mark.vcr(filter_headers=["authorization"])
|
||||||
def test_lite_agent_with_invalid_llm():
|
def test_lite_agent_with_invalid_llm():
|
||||||
"""Test that LiteAgent raises proper error when create_llm returns None."""
|
"""Test that LiteAgent raises proper error when create_llm returns None."""
|
||||||
with patch('crewai.lite_agent.create_llm', return_value=None):
|
with patch("crewai.lite_agent.create_llm", return_value=None):
|
||||||
with pytest.raises(ValueError) as exc_info:
|
with pytest.raises(ValueError) as exc_info:
|
||||||
LiteAgent(
|
LiteAgent(
|
||||||
role="Test Agent",
|
role="Test Agent",
|
||||||
goal="Test goal",
|
goal="Test goal",
|
||||||
backstory="Test backstory",
|
backstory="Test backstory",
|
||||||
llm="invalid-model"
|
llm="invalid-model",
|
||||||
)
|
)
|
||||||
assert "Expected LLM instance of type BaseLLM" in str(exc_info.value)
|
assert "Expected LLM instance of type BaseLLM" in str(exc_info.value)
|
||||||
|
|
||||||
|
|
||||||
|
def test_lite_agent_emits_llm_call_started_event_with_tools():
|
||||||
|
"""Test that LiteAgent emits LLMCallStartedEvent with correct tools information."""
|
||||||
|
captured_events = []
|
||||||
|
|
||||||
|
@crewai_event_bus.on(LLMCallStartedEvent)
|
||||||
|
def capture_llm_event(source, event):
|
||||||
|
captured_events.append(event)
|
||||||
|
|
||||||
|
agent = LiteAgent(
|
||||||
|
role="Test Agent",
|
||||||
|
goal="Test Goal",
|
||||||
|
backstory="Test Backstory",
|
||||||
|
tools=[WebSearchTool(), CalculatorTool()],
|
||||||
|
llm=Mock(spec=LLM),
|
||||||
|
)
|
||||||
|
|
||||||
|
agent.llm.call.return_value = "Final Answer: Test response"
|
||||||
|
|
||||||
|
agent.kickoff("Test query")
|
||||||
|
|
||||||
|
# Verify event was emitted with tools
|
||||||
|
assert len(captured_events) > 0
|
||||||
|
event = captured_events[0]
|
||||||
|
assert event.tools is not None
|
||||||
|
assert len(event.tools) == 2
|
||||||
|
|
||||||
|
# Verify tool structure
|
||||||
|
tool_names = [tool["name"] for tool in event.tools]
|
||||||
|
assert "search_web" in tool_names
|
||||||
|
assert "calculate" in tool_names
|
||||||
|
|
||||||
|
for tool in event.tools:
|
||||||
|
assert "name" in tool
|
||||||
|
assert "description" in tool
|
||||||
|
assert "args" in tool
|
||||||
|
|
||||||
|
|
||||||
|
def test_lite_agent_emits_llm_call_started_event_without_tools():
|
||||||
|
"""Test that LiteAgent emits LLMCallStartedEvent with tools=None when no tools provided."""
|
||||||
|
captured_events = []
|
||||||
|
|
||||||
|
@crewai_event_bus.on(LLMCallStartedEvent)
|
||||||
|
def capture_llm_event(source, event):
|
||||||
|
captured_events.append(event)
|
||||||
|
|
||||||
|
agent = LiteAgent(
|
||||||
|
role="Test Agent",
|
||||||
|
goal="Test Goal",
|
||||||
|
backstory="Test Backstory",
|
||||||
|
tools=[], # No tools
|
||||||
|
llm=Mock(spec=LLM),
|
||||||
|
)
|
||||||
|
|
||||||
|
agent.llm.call.return_value = "Final Answer: Test response"
|
||||||
|
|
||||||
|
agent.kickoff("Test query")
|
||||||
|
|
||||||
|
assert len(captured_events) > 0
|
||||||
|
event = captured_events[0]
|
||||||
|
assert event.tools is None
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.vcr(filter_headers=["authorization"])
|
||||||
|
def test_lite_agent_tool_calling_reproduction():
|
||||||
|
"""Test reproduction of GitHub issue #3302 - LiteAgent tool calling."""
|
||||||
|
|
||||||
|
agent = LiteAgent(
|
||||||
|
role="Research Assistant",
|
||||||
|
goal="Help with research tasks",
|
||||||
|
backstory="You are a helpful research assistant.",
|
||||||
|
tools=[WebSearchTool()],
|
||||||
|
llm=LLM(model="gpt-4o-mini"),
|
||||||
|
)
|
||||||
|
|
||||||
|
result = agent.kickoff("Search for information about Python programming")
|
||||||
|
|
||||||
|
# Verify the agent produced a meaningful result
|
||||||
|
assert result.raw is not None
|
||||||
|
assert len(result.raw) > 0
|
||||||
|
|
||||||
|
# Verify tools were actually available during execution
|
||||||
|
assert len(agent._parsed_tools) > 0
|
||||||
|
|||||||
Reference in New Issue
Block a user