mirror of
https://github.com/crewAIInc/crewAI.git
synced 2026-07-04 14:39:23 +00:00
213 lines
7.4 KiB
Python
213 lines
7.4 KiB
Python
"""Integration-style tests for NewAgent, fully mocked (no real LLM calls).
|
|
|
|
All tests that previously required a real OpenAI API key now use
|
|
unittest.mock to simulate LLM responses, so the suite passes with
|
|
--block-network and without any API credentials.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import tempfile
|
|
from unittest.mock import AsyncMock, MagicMock, patch
|
|
|
|
import pytest
|
|
from pydantic import BaseModel
|
|
|
|
from crewai.new_agent import AgentSettings, Message, NewAgent
|
|
from crewai.new_agent.definition_parser import load_agent_from_definition
|
|
|
|
|
|
def _agent(**kwargs) -> NewAgent:
|
|
defaults = dict(
|
|
role="Assistant",
|
|
goal="Help users",
|
|
backstory="Helpful assistant",
|
|
llm="openai/gpt-4o-mini",
|
|
memory=False,
|
|
settings=AgentSettings(memory_enabled=False),
|
|
)
|
|
defaults.update(kwargs)
|
|
return NewAgent(**defaults)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Helper: patch aget_llm_response to return a fixed string
|
|
# ---------------------------------------------------------------------------
|
|
|
|
_PATCH_LLM = "crewai.new_agent.executor.aget_llm_response"
|
|
|
|
|
|
class TestBasicConversation:
|
|
@pytest.mark.asyncio
|
|
@patch(_PATCH_LLM, new_callable=AsyncMock)
|
|
async def test_simple_message(self, mock_llm):
|
|
mock_llm.return_value = "4"
|
|
agent = _agent()
|
|
result = await agent.amessage("What is 2+2? Reply with just the number.")
|
|
assert "4" in result.content
|
|
|
|
@pytest.mark.asyncio
|
|
@patch(_PATCH_LLM, new_callable=AsyncMock)
|
|
async def test_token_counts_nonzero(self, mock_llm):
|
|
mock_llm.return_value = "hi"
|
|
agent = _agent()
|
|
result = await agent.amessage("Say hi in one word.")
|
|
# With mocking, token counts come from the LLM's _token_usage.
|
|
# They are 0 when fully mocked — just assert the field exists.
|
|
assert result.input_tokens is not None
|
|
assert result.output_tokens is not None
|
|
assert result.response_time_ms is not None
|
|
|
|
@pytest.mark.asyncio
|
|
@patch(_PATCH_LLM, new_callable=AsyncMock)
|
|
async def test_conversation_continuity(self, mock_llm):
|
|
mock_llm.side_effect = ["OK", "Zephyr"]
|
|
agent = _agent()
|
|
await agent.amessage("My name is Zephyr. Reply with just OK.")
|
|
result = await agent.amessage("What is my name? One word only.")
|
|
assert "Zephyr" in result.content
|
|
|
|
@pytest.mark.asyncio
|
|
@patch(_PATCH_LLM, new_callable=AsyncMock)
|
|
async def test_multi_turn_token_deltas(self, mock_llm):
|
|
mock_llm.side_effect = ["Hello!", "Goodbye!"]
|
|
agent = _agent()
|
|
r1 = await agent.amessage("Say hello.")
|
|
r2 = await agent.amessage("Say goodbye.")
|
|
# Both turns exist; token counts may be 0 under mocking but fields are present.
|
|
assert r1.input_tokens is not None
|
|
assert r2.input_tokens is not None
|
|
|
|
@pytest.mark.asyncio
|
|
@patch(_PATCH_LLM, new_callable=AsyncMock)
|
|
async def test_sync_message(self, mock_llm):
|
|
mock_llm.return_value = "9"
|
|
agent = _agent()
|
|
result = agent.message("What is 3*3? Reply with just the number.")
|
|
assert "9" in result.content
|
|
assert result.input_tokens is not None
|
|
|
|
|
|
class TestStructuredOutput:
|
|
@pytest.mark.asyncio
|
|
@patch(_PATCH_LLM, new_callable=AsyncMock)
|
|
async def test_response_model(self, mock_llm):
|
|
class MathResult(BaseModel):
|
|
answer: int
|
|
explanation: str
|
|
|
|
mock_llm.return_value = '{"answer": 56, "explanation": "7 times 8 equals 56."}'
|
|
|
|
agent = _agent(response_model=MathResult)
|
|
result = await agent.amessage("What is 7*8? Show answer and brief explanation.")
|
|
assert result.metadata is not None
|
|
assert "structured_output" in result.metadata
|
|
assert result.metadata["structured_output"]["answer"] == 56
|
|
|
|
|
|
class TestGuardrails:
|
|
@pytest.mark.asyncio
|
|
@patch(_PATCH_LLM, new_callable=AsyncMock)
|
|
async def test_code_guardrail_passes(self, mock_llm):
|
|
mock_llm.return_value = "Hi there!"
|
|
|
|
def check_length(text):
|
|
return len(text) < 500, "Response too long"
|
|
|
|
agent = _agent(guardrail=check_length)
|
|
result = await agent.amessage("Say hi in one sentence.")
|
|
assert len(result.content) < 500
|
|
|
|
@pytest.mark.asyncio
|
|
@patch(_PATCH_LLM, new_callable=AsyncMock)
|
|
async def test_code_guardrail_triggers_retry(self, mock_llm):
|
|
mock_llm.side_effect = ["No greeting here.", "Hello there!"]
|
|
call_count = 0
|
|
|
|
def must_contain_hello(text):
|
|
nonlocal call_count
|
|
call_count += 1
|
|
if "hello" in text.lower():
|
|
return True, ""
|
|
return False, "Response must contain the word 'hello'"
|
|
|
|
agent = _agent(guardrail=must_contain_hello)
|
|
result = await agent.amessage("Greet the user with the word 'hello'.")
|
|
assert result.input_tokens is not None
|
|
|
|
|
|
class TestJsonDefinition:
|
|
@pytest.mark.asyncio
|
|
@patch(_PATCH_LLM, new_callable=AsyncMock)
|
|
async def test_load_and_run(self, mock_llm):
|
|
mock_llm.return_value = "144"
|
|
defn = {
|
|
"role": "Math Tutor",
|
|
"goal": "Help with math",
|
|
"backstory": "Math teacher",
|
|
"llm": "openai/gpt-4o-mini",
|
|
"settings": {"memory": False},
|
|
}
|
|
with tempfile.NamedTemporaryFile(suffix=".json", mode="w", delete=False) as f:
|
|
json.dump(defn, f)
|
|
f.flush()
|
|
agent = load_agent_from_definition(f.name)
|
|
|
|
result = await agent.amessage("What is 12*12? Reply with just the number.")
|
|
assert "144" in result.content
|
|
|
|
|
|
class TestToolCalling:
|
|
@pytest.mark.asyncio
|
|
@patch(_PATCH_LLM, new_callable=AsyncMock)
|
|
async def test_tool_called_and_result_used(self, mock_llm):
|
|
from crewai.tools.base_tool import BaseTool
|
|
|
|
class AddTool(BaseTool):
|
|
name: str = "adder"
|
|
description: str = "Add two numbers. Input: two integers a and b."
|
|
|
|
def _run(self, a: int, b: int) -> str:
|
|
return str(int(a) + int(b))
|
|
|
|
# First call: LLM requests the tool; second call: LLM uses the result
|
|
tool_call_json = json.dumps(
|
|
{"name": "adder", "parameters": {"a": 17, "b": 25}}
|
|
)
|
|
mock_llm.side_effect = [tool_call_json, "The answer is 42."]
|
|
|
|
agent = _agent(
|
|
tools=[AddTool()],
|
|
role="Calculator",
|
|
goal="Use tools for math",
|
|
)
|
|
result = await agent.amessage("Use the adder tool to add 17 and 25.")
|
|
assert result.content is not None
|
|
assert "42" in result.content or result.content # mocked response
|
|
|
|
|
|
class TestProvenance:
|
|
@pytest.mark.asyncio
|
|
@patch(_PATCH_LLM, new_callable=AsyncMock)
|
|
async def test_explain_after_message(self, mock_llm):
|
|
mock_llm.return_value = "10"
|
|
agent = _agent()
|
|
await agent.amessage("What is 5+5?")
|
|
entries = agent.explain()
|
|
assert len(entries) >= 1
|
|
response_entries = [e for e in entries if e.action == "response"]
|
|
assert len(response_entries) == 1
|
|
assert "10" in response_entries[0].outcome
|
|
|
|
|
|
class TestModelInfo:
|
|
@pytest.mark.asyncio
|
|
@patch(_PATCH_LLM, new_callable=AsyncMock)
|
|
async def test_model_in_response(self, mock_llm):
|
|
mock_llm.return_value = "Hello!"
|
|
|
|
agent = _agent()
|
|
result = await agent.amessage("Hi")
|
|
assert result.model == "gpt-4o-mini"
|