Compare commits

...

2 Commits

Author SHA1 Message Date
Devin AI
14035a98c3 Address PR feedback: Add error handling and edge case tests
Co-Authored-By: Joe Moura <joao@crewai.com>
2025-05-13 17:57:49 +00:00
Devin AI
48f7721baf Fix prompt formatting biases affecting JSON output
Co-Authored-By: Joe Moura <joao@crewai.com>
2025-05-13 17:52:58 +00:00
5 changed files with 186 additions and 13 deletions

View File

@@ -135,15 +135,25 @@ class BaseTool(BaseModel, ABC):
)
def _generate_description(self):
args_schema = {
name: {
"description": field.description,
"type": BaseTool._get_arg_annotations(field.annotation),
}
for name, field in self.args_schema.model_fields.items()
}
import json
import logging
self.description = f"Tool Name: {self.name}\nTool Arguments: {args_schema}\nTool Description: {self.description}"
logger = logging.getLogger(__name__)
try:
args_schema = {
name: {
"description": field.description,
"type": BaseTool._get_arg_annotations(field.annotation),
}
for name, field in self.args_schema.model_fields.items()
}
args_json = json.dumps(args_schema)
except Exception as e:
logger.warning(f"Failed to serialize args schema: {e}")
args_json = str(args_schema)
self.description = f"Tool Name: {self.name}\nTool Arguments: {args_json}\nTool Description: {self.description}"
@staticmethod
def _get_arg_annotations(annotation: type[Any] | None) -> str:

View File

@@ -9,7 +9,7 @@
"task": "\nCurrent Task: {input}\n\nBegin! This is VERY important to you, use the tools available and give your best Final Answer, your job depends on it!\n\nThought:",
"memory": "\n\n# Useful context: \n{memory}",
"role_playing": "You are {role}. {backstory}\nYour personal goal is: {goal}",
"tools": "\nYou ONLY have access to the following tools, and should NEVER make up tools that are not listed here:\n\n{tools}\n\nUse the following format:\n\nThought: you should always think about what to do\nAction: the action to take, only one name of [{tool_names}], just the name, exactly as it's written.\nAction Input: the input to the action, just a simple python dictionary, enclosed in curly braces, using \" to wrap keys and values.\nObservation: the result of the action\n\nOnce all necessary information is gathered:\n\nThought: I now know the final answer\nFinal Answer: the final answer to the original input question\n",
"tools": "\nYou ONLY have access to the following tools, and should NEVER make up tools that are not listed here:\n\n{tools}\n\nUse the following format:\n\nThought: you should always think about what to do\nAction: the action to take, only one name of [{tool_names}], just the name, exactly as it's written.\nAction Input: the input to the action, just a simple python dictionary, enclosed in curly braces, using double quotes (\") to wrap keys and values.\nObservation: the result of the action\n\nOnce all necessary information is gathered:\n\nThought: I now know the final answer\nFinal Answer: the final answer to the original input question\n",
"no_tools": "\nTo give my best complete final answer to the task use the exact following format:\n\nThought: I now can give a great answer\nFinal Answer: Your final answer must be the great and the most complete as possible, it must be outcome described.\n\nI MUST use these formats, my job depends on it!",
"format": "I MUST either use a tool (use one at time) OR give my best final answer not both at the same time. To Use the following format:\n\nThought: you should always think about what to do\nAction: the action to take, should be one of [{tool_names}]\nAction Input: the input to the action, dictionary enclosed in curly braces\nObservation: the result of the action\n... (this Thought/Action/Action Input/Result can repeat N times)\nThought: I now can give a great answer\nFinal Answer: Your final answer must be the great and the most complete as possible, it must be outcome described\n\n",
"final_answer_format": "If you don't need to use any more tools, you must give your best complete final answer, make sure it satisfies the expected criteria, use the EXACT format below:\n\nThought: I now can give a great answer\nFinal Answer: my best complete final answer to the task.\n\n",

View File

@@ -0,0 +1,85 @@
import json
from unittest.mock import MagicMock
import pytest
from pydantic import BaseModel, Field
from crewai.tools import BaseTool
from crewai.tools.tool_usage import ToolUsage
class TestComplexInput(BaseModel):
special_chars: str = Field(
..., description="Parameter with special characters: \"'\\{}[]"
)
nested_dict: dict = Field(
..., description="A nested dictionary parameter"
)
unicode_text: str = Field(
..., description="Text with unicode characters: 你好, こんにちは, مرحبا"
)
class TestComplexTool(BaseTool):
name: str = "Complex JSON Tool"
description: str = "A tool for testing complex JSON formatting"
args_schema: type[BaseModel] = TestComplexInput
def _run(self, special_chars: str, nested_dict: dict, unicode_text: str) -> str:
return f"Processed complex input successfully"
def test_complex_json_formatting():
"""Test that complex JSON with special characters and nested structures is formatted correctly."""
tool = TestComplexTool()
assert "Tool Arguments:" in tool.description
description_parts = tool.description.split("Tool Arguments: ")
json_str = description_parts[1].split("\nTool Description:")[0]
parsed_json = json.loads(json_str)
assert "special_chars" in parsed_json
assert "nested_dict" in parsed_json
assert "unicode_text" in parsed_json
assert "\"'\\{}[]" in parsed_json["special_chars"]["description"]
assert "你好" in parsed_json["unicode_text"]["description"]
assert "こんにちは" in parsed_json["unicode_text"]["description"]
assert "مرحبا" in parsed_json["unicode_text"]["description"]
def test_complex_tool_usage_render():
"""Test that complex tool usage renders with proper JSON formatting."""
tool = TestComplexTool()
tool_usage = ToolUsage(
tools_handler=MagicMock(),
tools=[tool],
original_tools=[tool],
tools_description="Tool for testing complex JSON formatting",
tools_names="test_complex_tool",
task=MagicMock(),
function_calling_llm=MagicMock(),
agent=MagicMock(),
action=MagicMock(),
)
rendered = tool_usage._render()
rendered_parts = rendered.split("Tool Arguments: ")
if len(rendered_parts) > 1:
json_str = rendered_parts[1].split("\nTool Description:")[0]
try:
parsed_json = json.loads(json_str)
assert True # If we get here, JSON parsing succeeded
assert "special_chars" in parsed_json
assert "nested_dict" in parsed_json
assert "unicode_text" in parsed_json
except json.JSONDecodeError:
assert False, "The rendered tool arguments are not valid JSON"

View File

@@ -0,0 +1,78 @@
import json
from unittest.mock import MagicMock
import pytest
from pydantic import BaseModel, Field
from crewai.tools import BaseTool
from crewai.tools.tool_usage import ToolUsage
class TestJsonInput(BaseModel):
test_param: str = Field(
..., description="A test parameter"
)
another_param: int = Field(
..., description="Another test parameter"
)
class TestJsonTool(BaseTool):
name: str = "Test JSON Tool"
description: str = "A tool for testing JSON formatting"
args_schema: type[BaseModel] = TestJsonInput
def _run(self, test_param: str, another_param: int) -> str:
return f"Received {test_param} and {another_param}"
def test_tool_description_json_formatting():
"""Test that the tool description uses proper JSON formatting with double quotes."""
tool = TestJsonTool()
assert "Tool Arguments:" in tool.description
description_parts = tool.description.split("Tool Arguments: ")
json_str = description_parts[1].split("\nTool Description:")[0]
parsed_json = json.loads(json_str)
assert "test_param" in parsed_json
assert "another_param" in parsed_json
assert '"test_param"' in json_str
assert '"another_param"' in json_str
assert "'" not in json_str # No single quotes should be present
def test_tool_usage_json_formatting():
"""Test that the tool usage renders with proper JSON formatting."""
tool = TestJsonTool()
tool_usage = ToolUsage(
tools_handler=MagicMock(),
tools=[tool],
original_tools=[tool],
tools_description="Tool for testing JSON formatting",
tools_names="test_json_tool",
task=MagicMock(),
function_calling_llm=MagicMock(),
agent=MagicMock(),
action=MagicMock(),
)
rendered = tool_usage._render()
rendered_parts = rendered.split("Tool Arguments: ")
if len(rendered_parts) > 1:
json_str = rendered_parts[1].split("\nTool Description:")[0]
try:
parsed_json = json.loads(json_str)
assert True # If we get here, JSON parsing succeeded
except json.JSONDecodeError:
assert False, "The rendered tool arguments are not valid JSON"
assert '"test_param"' in json_str
assert '"another_param"' in json_str
assert "'" not in json_str # No single quotes should be present

View File

@@ -102,15 +102,15 @@ def test_tool_usage_render():
rendered = tool_usage._render()
# Updated checks to match the actual output
# Updated checks to match the actual output with JSON formatting
assert "Tool Name: Random Number Generator" in rendered
assert "Tool Arguments:" in rendered
assert (
"'min_value': {'description': 'The minimum value of the range (inclusive)', 'type': 'int'}"
'"min_value": {"description": "The minimum value of the range (inclusive)", "type": "int"}'
in rendered
)
assert (
"'max_value': {'description': 'The maximum value of the range (inclusive)', 'type': 'int'}"
'"max_value": {"description": "The maximum value of the range (inclusive)", "type": "int"}'
in rendered
)
assert (
@@ -118,6 +118,6 @@ def test_tool_usage_render():
in rendered
)
assert (
"Tool Name: Random Number Generator\nTool Arguments: {'min_value': {'description': 'The minimum value of the range (inclusive)', 'type': 'int'}, 'max_value': {'description': 'The maximum value of the range (inclusive)', 'type': 'int'}}\nTool Description: Generates a random number within a specified range"
'Tool Name: Random Number Generator\nTool Arguments: {"min_value": {"description": "The minimum value of the range (inclusive)", "type": "int"}, "max_value": {"description": "The maximum value of the range (inclusive)", "type": "int"}}\nTool Description: Generates a random number within a specified range'
in rendered
)