andre/crewAI
1
0
Fork 0
mirror of https://github.com/crewAIInc/crewAI.git synced 2026-01-10 16:48:30 +00:00
Code Issues Actions 4 Packages Projects Releases Wiki Activity

improving test

Browse Source
This commit is contained in:
João Moura
2025-05-27 01:27:25 -07:00
parent e636f1dc17
commit 88721788e9
14 changed files with 3606 additions and 68 deletions
Download Patch File Download Diff File Expand all files Collapse all files

126
tests/reasoning_interval_test.py
View File

@@ -8,62 +8,58 @@ from crewai.agents.crew_agent_executor import CrewAgentExecutor
from crewai.utilities.reasoning_handler import AgentReasoning
@pytest.fixture
def mock_llm_responses():
"""Fixture for mock LLM responses."""
return {
"initial_reasoning": "I'll solve this task step by step.\n\nREADY: I am ready to execute the task.\n\n",
"mid_execution_reasoning": "Based on progress so far, I'll adjust my approach.\n\nREADY: I am ready to continue executing the task.",
"execution_step": "I'm working on the task...",
"final_result": "Task completed successfully."
}
def test_agent_with_reasoning_interval():
"""Ensure that the agent triggers mid-execution reasoning based on the fixed interval."""
# Use a mock LLM to avoid real network calls
llm = MagicMock()
agent = Agent(
role="Test Agent",
goal="To test the reasoning interval feature",
backstory="I am a test agent created to verify the reasoning interval feature works correctly.",
llm=llm,
reasoning=True,
reasoning_interval=2, # Reason every 2 steps
verbose=True,
)
def test_agent_with_reasoning_interval(mock_llm_responses):
"""Test agent with reasoning interval."""
with patch('crewai.llm.LLM.call') as mock_llm_call:
mock_llm_call.return_value = mock_llm_responses["initial_reasoning"]
llm = MagicMock()
llm.call.return_value = mock_llm_responses["initial_reasoning"]
agent = Agent(
role="Test Agent",
goal="To test the reasoning interval feature",
backstory="I am a test agent created to verify the reasoning interval feature works correctly.",
llm=llm,
reasoning=True,
reasoning_interval=2, # Reason every 2 steps
verbose=True
)
task = Task(
description="Multi-step task that requires periodic reasoning.",
expected_output="The task should be completed with periodic reasoning.",
agent=agent
agent=agent,
)
with patch('crewai.agent.Agent.create_agent_executor') as mock_create_executor:
mock_executor = MagicMock()
mock_executor._handle_mid_execution_reasoning = MagicMock()
mock_executor.invoke.return_value = mock_llm_responses["final_result"]
mock_create_executor.return_value = mock_executor
# Create a mock executor that will be injected into the agent
mock_executor = MagicMock()
# Simulate the executor deciding to reason on the second step
mock_executor._should_trigger_reasoning = MagicMock(side_effect=[False, True])
mock_executor._handle_mid_execution_reasoning = MagicMock()
mock_executor.invoke.return_value = "Task completed successfully."
# Monkey-patch create_agent_executor so that it sets our mock_executor
def _fake_create_agent_executor(self, tools=None, task=None): # noqa: D401,E501
"""Replace the real executor with the mock while preserving behaviour."""
self.agent_executor = mock_executor
return mock_executor
with patch.object(Agent, "create_agent_executor", _fake_create_agent_executor):
result = agent.execute_task(task)
assert result == mock_llm_responses["final_result"]
mock_executor._handle_mid_execution_reasoning.assert_called()
# Validate results and that reasoning happened when expected
assert result == "Task completed successfully."
# _handle_mid_execution_reasoning should be called once (on the second step)
mock_executor._handle_mid_execution_reasoning.assert_called()
def test_agent_with_adaptive_reasoning(mock_llm_responses):
"""Test agent with adaptive reasoning."""
with patch('crewai.llm.LLM.call') as mock_llm_call:
mock_llm_call.return_value = mock_llm_responses["initial_reasoning"]
llm = MagicMock()
llm.call.return_value = mock_llm_responses["initial_reasoning"]
agent = Agent(
role="Test Agent",
goal="To test the adaptive reasoning feature",
@@ -73,26 +69,26 @@ def test_agent_with_adaptive_reasoning(mock_llm_responses):
adaptive_reasoning=True,
verbose=True
)
task = Task(
description="Complex task that requires adaptive reasoning.",
expected_output="The task should be completed with adaptive reasoning.",
agent=agent
)
with patch('crewai.agent.Agent.create_agent_executor') as mock_create_executor:
mock_executor = MagicMock()
mock_executor._should_adaptive_reason = MagicMock(return_value=True)
mock_executor._handle_mid_execution_reasoning = MagicMock()
mock_executor.invoke.return_value = mock_llm_responses["final_result"]
mock_create_executor.return_value = mock_executor
result = agent.execute_task(task)
assert result == mock_llm_responses["final_result"]
mock_executor._should_adaptive_reason.assert_called()
mock_executor._handle_mid_execution_reasoning.assert_called()
@@ -100,7 +96,7 @@ def test_mid_execution_reasoning_handler():
"""Test the mid-execution reasoning handler."""
llm = MagicMock()
llm.call.return_value = "Based on progress, I'll adjust my approach.\n\nREADY: I am ready to continue executing the task."
agent = Agent(
role="Test Agent",
goal="To test the mid-execution reasoning handler",
@@ -109,17 +105,17 @@ def test_mid_execution_reasoning_handler():
reasoning=True,
verbose=True
)
task = Task(
description="Task to test mid-execution reasoning handler.",
expected_output="The mid-execution reasoning handler should work correctly.",
agent=agent
)
agent.llm.call = MagicMock(return_value="Based on progress, I'll adjust my approach.\n\nREADY: I am ready to continue executing the task.")
reasoning_handler = AgentReasoning(task=task, agent=agent)
result = reasoning_handler.handle_mid_execution_reasoning(
current_steps=3,
tools_used=["search_tool", "calculator_tool"],
@@ -131,7 +127,7 @@ def test_mid_execution_reasoning_handler():
{"role": "system", "content": "Calculation result: 42"}
]
)
assert result is not None
assert hasattr(result, 'plan')
assert hasattr(result.plan, 'plan')
@@ -145,7 +141,7 @@ def test_should_trigger_reasoning_interval():
agent.reasoning = True
agent.reasoning_interval = 3
agent.adaptive_reasoning = False
executor = CrewAgentExecutor(
llm=MagicMock(),
task=MagicMock(),
@@ -159,16 +155,16 @@ def test_should_trigger_reasoning_interval():
tools_description="",
tools_handler=MagicMock()
)
executor.steps_since_reasoning = 0
assert executor._should_trigger_reasoning() is False
executor.steps_since_reasoning = 2
assert executor._should_trigger_reasoning() is False
executor.steps_since_reasoning = 3
assert executor._should_trigger_reasoning() is True
executor.steps_since_reasoning = 4
assert executor._should_trigger_reasoning() is True
@@ -179,7 +175,7 @@ def test_should_trigger_adaptive_reasoning():
agent.reasoning = True
agent.reasoning_interval = None
agent.adaptive_reasoning = True
executor = CrewAgentExecutor(
llm=MagicMock(),
task=MagicMock(),
@@ -193,14 +189,14 @@ def test_should_trigger_adaptive_reasoning():
tools_description="",
tools_handler=MagicMock()
)
executor.tools_used = ["tool1", "tool2", "tool3"]
assert executor._should_adaptive_reason() is True
executor.tools_used = ["tool1", "tool1", "tool1"]
executor.iterations = 6 # > max_iter // 2
assert executor._should_adaptive_reason() is True
executor.tools_used = ["tool1", "tool1", "tool1"]
executor.iterations = 2
executor.messages = [
@@ -209,7 +205,7 @@ def test_should_trigger_adaptive_reasoning():
{"role": "assistant", "content": "Let me try something else."}
]
assert executor._should_adaptive_reason() is True
executor.tools_used = ["tool1", "tool1", "tool1"]
executor.iterations = 2
executor.messages = [
@@ -233,7 +229,7 @@ def test_reasoning_interval_scenarios(interval, steps, should_reason):
agent.reasoning = True
agent.reasoning_interval = interval
agent.adaptive_reasoning = False
executor = CrewAgentExecutor(
llm=MagicMock(),
task=MagicMock(),
@@ -247,6 +243,6 @@ def test_reasoning_interval_scenarios(interval, steps, should_reason):
tools_description="",
tools_handler=MagicMock()
)
executor.steps_since_reasoning = steps
assert executor._should_trigger_reasoning() is should_reason