Files
crewAI/tests/tools/test_tool_usage.py
Greyson LaLonde 878c1a649a refactor: Move events module to crewai.events (#3425)
refactor(events): relocate events module & update imports

- Move events from utilities/ to top-level events/ with types/, listeners/, utils/ structure
- Update all source/tests/docs to new import paths
- Add backwards compatibility stubs in crewai.utilities.events with deprecation warnings
- Restore test mocks and fix related test imports
2025-09-02 10:06:42 -04:00

735 lines
22 KiB
Python

import datetime
import json
import random
import time
from unittest.mock import MagicMock, patch
import pytest
from pydantic import BaseModel, Field
from crewai import Agent, Task
from crewai.tools import BaseTool
from crewai.tools.tool_usage import ToolUsage
from crewai.events.event_bus import crewai_event_bus
from crewai.events.types.tool_usage_events import (
ToolSelectionErrorEvent,
ToolUsageFinishedEvent,
ToolValidateInputErrorEvent,
)
class RandomNumberToolInput(BaseModel):
min_value: int = Field(
..., description="The minimum value of the range (inclusive)"
)
max_value: int = Field(
..., description="The maximum value of the range (inclusive)"
)
class RandomNumberTool(BaseTool):
name: str = "Random Number Generator"
description: str = "Generates a random number within a specified range"
args_schema: type[BaseModel] = RandomNumberToolInput
def _run(self, min_value: int, max_value: int) -> int:
return random.randint(min_value, max_value)
# Example agent and task
example_agent = Agent(
role="Number Generator",
goal="Generate random numbers for various purposes",
backstory="You are an AI agent specialized in generating random numbers within specified ranges.",
tools=[RandomNumberTool()],
verbose=True,
)
example_task = Task(
description="Generate a random number between 1 and 100",
expected_output="A random number between 1 and 100",
agent=example_agent,
)
def test_random_number_tool_range():
tool = RandomNumberTool()
result = tool._run(1, 10)
assert 1 <= result <= 10
def test_random_number_tool_invalid_range():
tool = RandomNumberTool()
with pytest.raises(ValueError):
tool._run(10, 1) # min_value > max_value
def test_random_number_tool_schema():
tool = RandomNumberTool()
# Get the schema using model_json_schema()
schema = tool.args_schema.model_json_schema()
# Convert the schema to a string
schema_str = json.dumps(schema)
# Check if the schema string contains the expected fields
assert "min_value" in schema_str
assert "max_value" in schema_str
# Parse the schema string back to a dictionary
schema_dict = json.loads(schema_str)
# Check if the schema contains the correct field types
assert schema_dict["properties"]["min_value"]["type"] == "integer"
assert schema_dict["properties"]["max_value"]["type"] == "integer"
# Check if the schema contains the field descriptions
assert (
"minimum value" in schema_dict["properties"]["min_value"]["description"].lower()
)
assert (
"maximum value" in schema_dict["properties"]["max_value"]["description"].lower()
)
def test_tool_usage_render():
tool = RandomNumberTool()
tool_usage = ToolUsage(
tools_handler=MagicMock(),
tools=[tool],
task=MagicMock(),
function_calling_llm=MagicMock(),
agent=MagicMock(),
action=MagicMock(),
)
rendered = tool_usage._render()
# Updated checks to match the actual output
assert "Tool Name: Random Number Generator" in rendered
assert "Tool Arguments:" in rendered
assert (
"'min_value': {'description': 'The minimum value of the range (inclusive)', 'type': 'int'}"
in rendered
)
assert (
"'max_value': {'description': 'The maximum value of the range (inclusive)', 'type': 'int'}"
in rendered
)
assert (
"Tool Description: Generates a random number within a specified range"
in rendered
)
assert (
"Tool Name: Random Number Generator\nTool Arguments: {'min_value': {'description': 'The minimum value of the range (inclusive)', 'type': 'int'}, 'max_value': {'description': 'The maximum value of the range (inclusive)', 'type': 'int'}}\nTool Description: Generates a random number within a specified range"
in rendered
)
def test_validate_tool_input_booleans_and_none():
# Create a ToolUsage instance with mocks
tool_usage = ToolUsage(
tools_handler=MagicMock(),
tools=[],
task=MagicMock(),
function_calling_llm=MagicMock(),
agent=MagicMock(),
action=MagicMock(),
)
# Input with booleans and None
tool_input = '{"key1": True, "key2": False, "key3": None}'
expected_arguments = {"key1": True, "key2": False, "key3": None}
arguments = tool_usage._validate_tool_input(tool_input)
assert arguments == expected_arguments
def test_validate_tool_input_mixed_types():
# Create a ToolUsage instance with mocks
tool_usage = ToolUsage(
tools_handler=MagicMock(),
tools=[],
task=MagicMock(),
function_calling_llm=MagicMock(),
agent=MagicMock(),
action=MagicMock(),
)
# Input with mixed types
tool_input = '{"number": 123, "text": "Some text", "flag": True}'
expected_arguments = {"number": 123, "text": "Some text", "flag": True}
arguments = tool_usage._validate_tool_input(tool_input)
assert arguments == expected_arguments
def test_validate_tool_input_single_quotes():
# Create a ToolUsage instance with mocks
tool_usage = ToolUsage(
tools_handler=MagicMock(),
tools=[],
task=MagicMock(),
function_calling_llm=MagicMock(),
agent=MagicMock(),
action=MagicMock(),
)
# Input with single quotes instead of double quotes
tool_input = "{'key': 'value', 'flag': True}"
expected_arguments = {"key": "value", "flag": True}
arguments = tool_usage._validate_tool_input(tool_input)
assert arguments == expected_arguments
def test_validate_tool_input_invalid_json_repairable():
# Create a ToolUsage instance with mocks
tool_usage = ToolUsage(
tools_handler=MagicMock(),
tools=[],
task=MagicMock(),
function_calling_llm=MagicMock(),
agent=MagicMock(),
action=MagicMock(),
)
# Invalid JSON input that can be repaired
tool_input = '{"key": "value", "list": [1, 2, 3,]}'
expected_arguments = {"key": "value", "list": [1, 2, 3]}
arguments = tool_usage._validate_tool_input(tool_input)
assert arguments == expected_arguments
def test_validate_tool_input_with_special_characters():
# Create a ToolUsage instance with mocks
tool_usage = ToolUsage(
tools_handler=MagicMock(),
tools=[],
task=MagicMock(),
function_calling_llm=MagicMock(),
agent=MagicMock(),
action=MagicMock(),
)
# Input with special characters
tool_input = '{"message": "Hello, world! \u263a", "valid": True}'
expected_arguments = {"message": "Hello, world! ☺", "valid": True}
arguments = tool_usage._validate_tool_input(tool_input)
assert arguments == expected_arguments
def test_validate_tool_input_none_input():
tool_usage = ToolUsage(
tools_handler=MagicMock(),
tools=[],
task=MagicMock(),
function_calling_llm=None,
agent=MagicMock(),
action=MagicMock(),
)
arguments = tool_usage._validate_tool_input(None)
assert arguments == {}
def test_validate_tool_input_valid_json():
tool_usage = ToolUsage(
tools_handler=MagicMock(),
tools=[],
task=MagicMock(),
function_calling_llm=None,
agent=MagicMock(),
action=MagicMock(),
)
tool_input = '{"key": "value", "number": 42, "flag": true}'
expected_arguments = {"key": "value", "number": 42, "flag": True}
arguments = tool_usage._validate_tool_input(tool_input)
assert arguments == expected_arguments
def test_validate_tool_input_python_dict():
tool_usage = ToolUsage(
tools_handler=MagicMock(),
tools=[],
task=MagicMock(),
function_calling_llm=None,
agent=MagicMock(),
action=MagicMock(),
)
tool_input = "{'key': 'value', 'number': 42, 'flag': True}"
expected_arguments = {"key": "value", "number": 42, "flag": True}
arguments = tool_usage._validate_tool_input(tool_input)
assert arguments == expected_arguments
def test_validate_tool_input_json5_unquoted_keys():
tool_usage = ToolUsage(
tools_handler=MagicMock(),
tools=[],
task=MagicMock(),
function_calling_llm=None,
agent=MagicMock(),
action=MagicMock(),
)
tool_input = "{key: 'value', number: 42, flag: true}"
expected_arguments = {"key": "value", "number": 42, "flag": True}
arguments = tool_usage._validate_tool_input(tool_input)
assert arguments == expected_arguments
def test_validate_tool_input_with_trailing_commas():
tool_usage = ToolUsage(
tools_handler=MagicMock(),
tools=[],
task=MagicMock(),
function_calling_llm=None,
agent=MagicMock(),
action=MagicMock(),
)
tool_input = '{"key": "value", "number": 42, "flag": true,}'
expected_arguments = {"key": "value", "number": 42, "flag": True}
arguments = tool_usage._validate_tool_input(tool_input)
assert arguments == expected_arguments
def test_validate_tool_input_invalid_input():
# Create mock agent with proper string values
mock_agent = MagicMock()
mock_agent.key = "test_agent_key" # Must be a string
mock_agent.role = "test_agent_role" # Must be a string
mock_agent._original_role = "test_agent_role" # Must be a string
mock_agent.i18n = MagicMock()
mock_agent.verbose = False
# Create mock action with proper string value
mock_action = MagicMock()
mock_action.tool = "test_tool" # Must be a string
mock_action.tool_input = "test_input" # Must be a string
tool_usage = ToolUsage(
tools_handler=MagicMock(),
tools=[],
task=MagicMock(),
function_calling_llm=None,
agent=mock_agent,
action=mock_action,
)
invalid_inputs = [
"Just a string",
"['list', 'of', 'values']",
"12345",
"",
]
for invalid_input in invalid_inputs:
with pytest.raises(Exception) as e_info:
tool_usage._validate_tool_input(invalid_input)
assert (
"Tool input must be a valid dictionary in JSON or Python literal format"
in str(e_info.value)
)
# Test for None input separately
arguments = tool_usage._validate_tool_input(None)
assert arguments == {}
def test_validate_tool_input_complex_structure():
tool_usage = ToolUsage(
tools_handler=MagicMock(),
tools=[],
task=MagicMock(),
function_calling_llm=None,
agent=MagicMock(),
action=MagicMock(),
)
tool_input = """
{
"user": {
"name": "Alice",
"age": 30
},
"items": [
{"id": 1, "value": "Item1"},
{"id": 2, "value": "Item2",}
],
"active": true,
}
"""
expected_arguments = {
"user": {"name": "Alice", "age": 30},
"items": [
{"id": 1, "value": "Item1"},
{"id": 2, "value": "Item2"},
],
"active": True,
}
arguments = tool_usage._validate_tool_input(tool_input)
assert arguments == expected_arguments
def test_validate_tool_input_code_content():
tool_usage = ToolUsage(
tools_handler=MagicMock(),
tools=[],
task=MagicMock(),
function_calling_llm=None,
agent=MagicMock(),
action=MagicMock(),
)
tool_input = '{"filename": "script.py", "content": "def hello():\\n print(\'Hello, world!\')"}'
expected_arguments = {
"filename": "script.py",
"content": "def hello():\n print('Hello, world!')",
}
arguments = tool_usage._validate_tool_input(tool_input)
assert arguments == expected_arguments
def test_validate_tool_input_with_escaped_quotes():
tool_usage = ToolUsage(
tools_handler=MagicMock(),
tools=[],
task=MagicMock(),
function_calling_llm=None,
agent=MagicMock(),
action=MagicMock(),
)
tool_input = '{"text": "He said, \\"Hello, world!\\""}'
expected_arguments = {"text": 'He said, "Hello, world!"'}
arguments = tool_usage._validate_tool_input(tool_input)
assert arguments == expected_arguments
def test_validate_tool_input_large_json_content():
tool_usage = ToolUsage(
tools_handler=MagicMock(),
tools=[],
task=MagicMock(),
function_calling_llm=None,
agent=MagicMock(),
action=MagicMock(),
)
# Simulate a large JSON content
tool_input = (
'{"data": ' + json.dumps([{"id": i, "value": i * 2} for i in range(1000)]) + "}"
)
expected_arguments = {"data": [{"id": i, "value": i * 2} for i in range(1000)]}
arguments = tool_usage._validate_tool_input(tool_input)
assert arguments == expected_arguments
def test_tool_selection_error_event_direct():
"""Test tool selection error event emission directly from ToolUsage class."""
mock_agent = MagicMock()
mock_agent.key = "test_key"
mock_agent.role = "test_role"
mock_agent.i18n = MagicMock()
mock_agent.verbose = False
mock_task = MagicMock()
mock_tools_handler = MagicMock()
class TestTool(BaseTool):
name: str = "Test Tool"
description: str = "A test tool"
def _run(self, input: dict) -> str:
return "test result"
test_tool = TestTool()
tool_usage = ToolUsage(
tools_handler=mock_tools_handler,
tools=[test_tool],
task=mock_task,
function_calling_llm=None,
agent=mock_agent,
action=MagicMock(),
)
received_events = []
@crewai_event_bus.on(ToolSelectionErrorEvent)
def event_handler(source, event):
received_events.append(event)
with pytest.raises(Exception):
tool_usage._select_tool("Non Existent Tool")
assert len(received_events) == 1
event = received_events[0]
assert isinstance(event, ToolSelectionErrorEvent)
assert event.agent_key == "test_key"
assert event.agent_role == "test_role"
assert event.tool_name == "Non Existent Tool"
assert event.tool_args == {}
assert "Tool Name: Test Tool" in event.tool_class
assert "A test tool" in event.tool_class
assert "don't exist" in event.error
received_events.clear()
with pytest.raises(Exception):
tool_usage._select_tool("")
assert len(received_events) == 1
event = received_events[0]
assert isinstance(event, ToolSelectionErrorEvent)
assert event.agent_key == "test_key"
assert event.agent_role == "test_role"
assert event.tool_name == ""
assert event.tool_args == {}
assert "Test Tool" in event.tool_class
assert "forgot the Action name" in event.error
def test_tool_validate_input_error_event():
"""Test tool validation input error event emission from ToolUsage class."""
# Mock agent and required components
mock_agent = MagicMock()
mock_agent.key = "test_key"
mock_agent.role = "test_role"
mock_agent.verbose = False
mock_agent._original_role = "test_role"
# Mock i18n with error message
mock_i18n = MagicMock()
mock_i18n.errors.return_value = (
"Tool input must be a valid dictionary in JSON or Python literal format"
)
mock_agent.i18n = mock_i18n
# Mock task and tools handler
mock_task = MagicMock()
mock_tools_handler = MagicMock()
# Mock printer
mock_printer = MagicMock()
# Create test tool
class TestTool(BaseTool):
name: str = "Test Tool"
description: str = "A test tool"
def _run(self, input: dict) -> str:
return "test result"
test_tool = TestTool()
# Create ToolUsage instance
tool_usage = ToolUsage(
tools_handler=mock_tools_handler,
tools=[test_tool],
task=mock_task,
function_calling_llm=None,
agent=mock_agent,
action=MagicMock(tool="test_tool"),
)
tool_usage._printer = mock_printer
# Mock all parsing attempts to fail
with (
patch("json.loads", side_effect=json.JSONDecodeError("Test Error", "", 0)),
patch("ast.literal_eval", side_effect=ValueError),
patch("json5.loads", side_effect=json.JSONDecodeError("Test Error", "", 0)),
patch("json_repair.repair_json", side_effect=Exception("Failed to repair")),
):
received_events = []
@crewai_event_bus.on(ToolValidateInputErrorEvent)
def event_handler(source, event):
received_events.append(event)
# Test invalid input
invalid_input = "invalid json {[}"
with pytest.raises(Exception):
tool_usage._validate_tool_input(invalid_input)
# Verify event was emitted
assert len(received_events) == 1, "Expected one event to be emitted"
event = received_events[0]
assert isinstance(event, ToolValidateInputErrorEvent)
assert event.agent_key == "test_key"
assert event.agent_role == "test_role"
assert event.tool_name == "test_tool"
assert "must be a valid dictionary" in event.error
def test_tool_usage_finished_event_with_result():
"""Test that ToolUsageFinishedEvent is emitted with correct result attributes."""
# Create mock agent with proper string values
mock_agent = MagicMock()
mock_agent.key = "test_agent_key"
mock_agent.role = "test_agent_role"
mock_agent._original_role = "test_agent_role"
mock_agent.i18n = MagicMock()
mock_agent.verbose = False
# Create mock task
mock_task = MagicMock()
mock_task.delegations = 0
mock_task.name = "Test Task"
mock_task.description = "A test task for tool usage"
mock_task.id = "test-task-id"
# Create mock tool
class TestTool(BaseTool):
name: str = "Test Tool"
description: str = "A test tool"
def _run(self, input: dict) -> str:
return "test result"
test_tool = TestTool()
# Create mock tool calling
mock_tool_calling = MagicMock()
mock_tool_calling.arguments = {"arg1": "value1"}
# Create ToolUsage instance
tool_usage = ToolUsage(
tools_handler=MagicMock(),
tools=[test_tool],
task=mock_task,
function_calling_llm=None,
agent=mock_agent,
action=MagicMock(),
)
# Track received events
received_events = []
@crewai_event_bus.on(ToolUsageFinishedEvent)
def event_handler(source, event):
received_events.append(event)
# Call on_tool_use_finished with test data
started_at = time.time()
result = "test output result"
tool_usage.on_tool_use_finished(
tool=test_tool,
tool_calling=mock_tool_calling,
from_cache=False,
started_at=started_at,
result=result,
)
# Verify event was emitted
assert len(received_events) == 1, "Expected one event to be emitted"
event = received_events[0]
assert isinstance(event, ToolUsageFinishedEvent)
# Verify event attributes
assert event.agent_key == "test_agent_key"
assert event.agent_role == "test_agent_role"
assert event.tool_name == "Test Tool"
assert event.tool_args == {"arg1": "value1"}
assert event.tool_class == "TestTool"
assert event.run_attempts == 1 # Default value from ToolUsage
assert event.delegations == 0
assert event.from_cache is False
assert event.output == "test output result"
assert isinstance(event.started_at, datetime.datetime)
assert isinstance(event.finished_at, datetime.datetime)
assert event.type == "tool_usage_finished"
def test_tool_usage_finished_event_with_cached_result():
"""Test that ToolUsageFinishedEvent is emitted with correct result attributes when using cached result."""
# Create mock agent with proper string values
mock_agent = MagicMock()
mock_agent.key = "test_agent_key"
mock_agent.role = "test_agent_role"
mock_agent._original_role = "test_agent_role"
mock_agent.i18n = MagicMock()
mock_agent.verbose = False
# Create mock task
mock_task = MagicMock()
mock_task.delegations = 0
mock_task.name = "Test Task"
mock_task.description = "A test task for tool usage"
mock_task.id = "test-task-id"
# Create mock tool
class TestTool(BaseTool):
name: str = "Test Tool"
description: str = "A test tool"
def _run(self, input: dict) -> str:
return "test result"
test_tool = TestTool()
# Create mock tool calling
mock_tool_calling = MagicMock()
mock_tool_calling.arguments = {"arg1": "value1"}
# Create ToolUsage instance
tool_usage = ToolUsage(
tools_handler=MagicMock(),
tools=[test_tool],
task=mock_task,
function_calling_llm=None,
agent=mock_agent,
action=MagicMock(),
)
# Track received events
received_events = []
@crewai_event_bus.on(ToolUsageFinishedEvent)
def event_handler(source, event):
received_events.append(event)
# Call on_tool_use_finished with test data and from_cache=True
started_at = time.time()
result = "cached test output result"
tool_usage.on_tool_use_finished(
tool=test_tool,
tool_calling=mock_tool_calling,
from_cache=True,
started_at=started_at,
result=result,
)
# Verify event was emitted
assert len(received_events) == 1, "Expected one event to be emitted"
event = received_events[0]
assert isinstance(event, ToolUsageFinishedEvent)
# Verify event attributes
assert event.agent_key == "test_agent_key"
assert event.agent_role == "test_agent_role"
assert event.tool_name == "Test Tool"
assert event.tool_args == {"arg1": "value1"}
assert event.tool_class == "TestTool"
assert event.run_attempts == 1 # Default value from ToolUsage
assert event.delegations == 0
assert event.from_cache is True
assert event.output == "cached test output result"
assert isinstance(event.started_at, datetime.datetime)
assert isinstance(event.finished_at, datetime.datetime)
assert event.type == "tool_usage_finished"