diff --git a/src/crewai/agent.py b/src/crewai/agent.py index 20f477aaf..f07408133 100644 --- a/src/crewai/agent.py +++ b/src/crewai/agent.py @@ -22,6 +22,7 @@ from crewai.utilities.converter import generate_model_description from crewai.utilities.events.agent_events import ( AgentExecutionCompletedEvent, AgentExecutionErrorEvent, + AgentExecutionStartedEvent, ) from crewai.utilities.events.crewai_event_bus import crewai_event_bus from crewai.utilities.llm_utils import create_llm @@ -231,6 +232,15 @@ class Agent(BaseAgent): task_prompt = self._use_trained_data(task_prompt=task_prompt) try: + crewai_event_bus.emit( + self, + event=AgentExecutionStartedEvent( + agent=self, + tools=self.tools, + task_prompt=task_prompt, + task=task, + ), + ) result = self.agent_executor.invoke( { "input": task_prompt, @@ -240,19 +250,27 @@ class Agent(BaseAgent): } )["output"] except Exception as e: - crewai_event_bus.emit( - self, - event=AgentExecutionErrorEvent( - agent=self, - task=task, - error=str(e), - ), - ) if e.__class__.__module__.startswith("litellm"): # Do not retry on litellm errors + crewai_event_bus.emit( + self, + event=AgentExecutionErrorEvent( + agent=self, + task=task, + error=str(e), + ), + ) raise e self._times_executed += 1 if self._times_executed > self.max_retry_limit: + crewai_event_bus.emit( + self, + event=AgentExecutionErrorEvent( + agent=self, + task=task, + error=str(e), + ), + ) raise e result = self.execute_task(task, context, tools) diff --git a/src/crewai/agents/crew_agent_executor.py b/src/crewai/agents/crew_agent_executor.py index 275a7aabf..6d34fea4e 100644 --- a/src/crewai/agents/crew_agent_executor.py +++ b/src/crewai/agents/crew_agent_executor.py @@ -19,10 +19,11 @@ from crewai.tools.tool_usage import ToolUsage, ToolUsageErrorException from crewai.utilities import I18N, Printer from crewai.utilities.constants import MAX_LLM_RETRY, TRAINING_DATA_FILE from crewai.utilities.events import ( - AgentExecutionErrorEvent, - AgentExecutionStartedEvent, + ToolUsageErrorEvent, + ToolUsageStartedEvent, + crewai_event_bus, ) -from crewai.utilities.events.crewai_event_bus import crewai_event_bus +from crewai.utilities.events.tool_usage_events import ToolUsageStartedEvent from crewai.utilities.exceptions.context_window_exceeding_exception import ( LLMContextLengthExceededException, ) @@ -90,16 +91,6 @@ class CrewAgentExecutor(CrewAgentExecutorMixin): self.llm.stop = list(set(self.llm.stop + self.stop)) def invoke(self, inputs: Dict[str, str]) -> Dict[str, Any]: - if self.agent and self.task: - crewai_event_bus.emit( - self, - event=AgentExecutionStartedEvent( - agent=self.agent, - tools=self.tools, - inputs=inputs, - task=self.task, - ), - ) if "system" in self.prompt: system_prompt = self._format_prompt(self.prompt.get("system", ""), inputs) user_prompt = self._format_prompt(self.prompt.get("user", ""), inputs) @@ -192,13 +183,6 @@ class CrewAgentExecutor(CrewAgentExecutorMixin): def _handle_unknown_error(self, exception: Exception) -> None: """Handle unknown errors by informing the user.""" - if self.agent: - crewai_event_bus.emit( - self, - event=AgentExecutionErrorEvent( - agent=self.agent, task=self.task, error=str(exception) - ), - ) self._printer.print( content="An unknown error occurred. Please check the details below.", color="red", @@ -371,40 +355,68 @@ class CrewAgentExecutor(CrewAgentExecutorMixin): ) def _execute_tool_and_check_finality(self, agent_action: AgentAction) -> ToolResult: - tool_usage = ToolUsage( - tools_handler=self.tools_handler, - tools=self.tools, - original_tools=self.original_tools, - tools_description=self.tools_description, - tools_names=self.tools_names, - function_calling_llm=self.function_calling_llm, - task=self.task, # type: ignore[arg-type] - agent=self.agent, - action=agent_action, - ) - tool_calling = tool_usage.parse_tool_calling(agent_action.text) - - if isinstance(tool_calling, ToolUsageErrorException): - tool_result = tool_calling.message - return ToolResult(result=tool_result, result_as_answer=False) - else: - if tool_calling.tool_name.casefold().strip() in [ - name.casefold().strip() for name in self.tool_name_to_tool_map - ] or tool_calling.tool_name.casefold().replace("_", " ") in [ - name.casefold().strip() for name in self.tool_name_to_tool_map - ]: - tool_result = tool_usage.use(tool_calling, agent_action.text) - tool = self.tool_name_to_tool_map.get(tool_calling.tool_name) - if tool: - return ToolResult( - result=tool_result, result_as_answer=tool.result_as_answer - ) - else: - tool_result = self._i18n.errors("wrong_tool_name").format( - tool=tool_calling.tool_name, - tools=", ".join([tool.name.casefold() for tool in self.tools]), + try: + if self.agent: + crewai_event_bus.emit( + self, + event=ToolUsageStartedEvent( + agent_key=self.agent.key, + agent_role=self.agent.role, + tool_name=agent_action.tool, + tool_args=agent_action.tool_input, + tool_class=agent_action.tool, + ), ) - return ToolResult(result=tool_result, result_as_answer=False) + tool_usage = ToolUsage( + tools_handler=self.tools_handler, + tools=self.tools, + original_tools=self.original_tools, + tools_description=self.tools_description, + tools_names=self.tools_names, + function_calling_llm=self.function_calling_llm, + task=self.task, # type: ignore[arg-type] + agent=self.agent, + action=agent_action, + ) + tool_calling = tool_usage.parse_tool_calling(agent_action.text) + + if isinstance(tool_calling, ToolUsageErrorException): + tool_result = tool_calling.message + return ToolResult(result=tool_result, result_as_answer=False) + else: + if tool_calling.tool_name.casefold().strip() in [ + name.casefold().strip() for name in self.tool_name_to_tool_map + ] or tool_calling.tool_name.casefold().replace("_", " ") in [ + name.casefold().strip() for name in self.tool_name_to_tool_map + ]: + tool_result = tool_usage.use(tool_calling, agent_action.text) + tool = self.tool_name_to_tool_map.get(tool_calling.tool_name) + if tool: + return ToolResult( + result=tool_result, result_as_answer=tool.result_as_answer + ) + else: + tool_result = self._i18n.errors("wrong_tool_name").format( + tool=tool_calling.tool_name, + tools=", ".join([tool.name.casefold() for tool in self.tools]), + ) + return ToolResult(result=tool_result, result_as_answer=False) + + except Exception as e: + # TODO: drop + if self.agent: + crewai_event_bus.emit( + self, + event=ToolUsageErrorEvent( # validation error + agent_key=self.agent.key, + agent_role=self.agent.role, + tool_name=agent_action.tool, + tool_args=agent_action.tool_input, + tool_class=agent_action.tool, + error=str(e), + ), + ) + raise e def _summarize_messages(self) -> None: messages_groups = [] diff --git a/src/crewai/llm.py b/src/crewai/llm.py index ada5c9bf3..4363d85ba 100644 --- a/src/crewai/llm.py +++ b/src/crewai/llm.py @@ -10,6 +10,8 @@ from typing import Any, Dict, List, Literal, Optional, Type, Union, cast from dotenv import load_dotenv from pydantic import BaseModel +from crewai.utilities.events.tool_usage_events import ToolExecutionErrorEvent + with warnings.catch_warnings(): warnings.simplefilter("ignore", UserWarning) import litellm @@ -18,6 +20,7 @@ with warnings.catch_warnings(): from litellm.utils import supports_response_schema +from crewai.utilities.events import crewai_event_bus from crewai.utilities.exceptions.context_window_exceeding_exception import ( LLMContextLengthExceededException, ) @@ -181,14 +184,14 @@ class LLM: def _is_anthropic_model(self, model: str) -> bool: """Determine if the model is from Anthropic provider. - + Args: model: The model identifier string. - + Returns: bool: True if the model is from Anthropic, False otherwise. """ - ANTHROPIC_PREFIXES = ('anthropic/', 'claude-', 'claude/') + ANTHROPIC_PREFIXES = ("anthropic/", "claude-", "claude/") return any(prefix in model.lower() for prefix in ANTHROPIC_PREFIXES) def call( @@ -199,7 +202,7 @@ class LLM: available_functions: Optional[Dict[str, Any]] = None, ) -> Union[str, Any]: """High-level LLM call method. - + Args: messages: Input messages for the LLM. Can be a string or list of message dictionaries. @@ -211,22 +214,22 @@ class LLM: during and after the LLM call. available_functions: Optional dict mapping function names to callables that can be invoked by the LLM. - + Returns: Union[str, Any]: Either a text response from the LLM (str) or the result of a tool function call (Any). - + Raises: TypeError: If messages format is invalid ValueError: If response format is not supported LLMContextLengthExceededException: If input exceeds model's context limit - + Examples: # Example 1: Simple string input >>> response = llm.call("Return the name of a random city.") >>> print(response) "Paris" - + # Example 2: Message list with system and user messages >>> messages = [ ... {"role": "system", "content": "You are a geography expert"}, @@ -315,7 +318,7 @@ class LLM: # --- 5) Handle the tool call tool_call = tool_calls[0] function_name = tool_call.function.name - + print("function_name", function_name) if function_name in available_functions: try: function_args = json.loads(tool_call.function.arguments) @@ -333,6 +336,15 @@ class LLM: logging.error( f"Error executing function '{function_name}': {e}" ) + crewai_event_bus.emit( + self, + event=ToolExecutionErrorEvent( + tool_name=function_name, + tool_args=function_args, + tool_class=fn, + error=str(e), + ), + ) return text_response else: @@ -348,36 +360,40 @@ class LLM: logging.error(f"LiteLLM call failed: {str(e)}") raise - def _format_messages_for_provider(self, messages: List[Dict[str, str]]) -> List[Dict[str, str]]: + def _format_messages_for_provider( + self, messages: List[Dict[str, str]] + ) -> List[Dict[str, str]]: """Format messages according to provider requirements. - + Args: messages: List of message dictionaries with 'role' and 'content' keys. Can be empty or None. - + Returns: List of formatted messages according to provider requirements. For Anthropic models, ensures first message has 'user' role. - + Raises: TypeError: If messages is None or contains invalid message format. """ if messages is None: raise TypeError("Messages cannot be None") - + # Validate message format first for msg in messages: if not isinstance(msg, dict) or "role" not in msg or "content" not in msg: - raise TypeError("Invalid message format. Each message must be a dict with 'role' and 'content' keys") - + raise TypeError( + "Invalid message format. Each message must be a dict with 'role' and 'content' keys" + ) + if not self.is_anthropic: return messages - + # Anthropic requires messages to start with 'user' role if not messages or messages[0]["role"] == "system": # If first message is system or empty, add a placeholder user message return [{"role": "user", "content": "."}, *messages] - + return messages def _get_custom_llm_provider(self) -> str: diff --git a/src/crewai/tools/tool_usage.py b/src/crewai/tools/tool_usage.py index 95519e16a..dde007312 100644 --- a/src/crewai/tools/tool_usage.py +++ b/src/crewai/tools/tool_usage.py @@ -17,12 +17,14 @@ from crewai.tools import BaseTool from crewai.tools.structured_tool import CrewStructuredTool from crewai.tools.tool_calling import InstructorToolCalling, ToolCalling from crewai.utilities import I18N, Converter, ConverterError, Printer -from crewai.utilities.events import ( +from crewai.utilities.events.crewai_event_bus import crewai_event_bus +from crewai.utilities.events.tool_usage_events import ( + ToolSelectionErrorEvent, ToolUsageErrorEvent, ToolUsageFinishedEvent, ToolUsageStartedEvent, + ToolValidateInputErrorEvent, ) -from crewai.utilities.events.crewai_event_bus import crewai_event_bus OPENAI_BIGGER_MODELS = [ "gpt-4", @@ -306,14 +308,33 @@ class ToolUsage: ): return tool self.task.increment_tools_errors() + tool_selection_data = { + "agent_key": self.agent.key, + "agent_role": self.agent.role, + "tool_name": tool_name, + "tool_args": {}, + "tool_class": self.tools_description, + } if tool_name and tool_name != "": - raise Exception( - f"Action '{tool_name}' don't exist, these are the only available Actions:\n{self.tools_description}" + error = f"Action '{tool_name}' don't exist, these are the only available Actions:\n{self.tools_description}" + crewai_event_bus.emit( + self, + ToolSelectionErrorEvent( + **tool_selection_data, + error=error, + ), ) + raise Exception(error) else: - raise Exception( - f"I forgot the Action name, these are the only available Actions: {self.tools_description}" + error = f"I forgot the Action name, these are the only available Actions: {self.tools_description}" + crewai_event_bus.emit( + self, + ToolSelectionErrorEvent( + **tool_selection_data, + error=error, + ), ) + raise Exception(error) def _render(self) -> str: """Render the tool name and description in plain text.""" @@ -449,18 +470,33 @@ class ToolUsage: if isinstance(arguments, dict): return arguments except Exception as e: - self._printer.print(content=f"Failed to repair JSON: {e}", color="red") + error = f"Failed to repair JSON: {e}" + self._printer.print(content=error, color="red") - # If all parsing attempts fail, raise an error - raise Exception( + error_message = ( "Tool input must be a valid dictionary in JSON or Python literal format" ) + self._emit_validate_input_error(error_message) + # If all parsing attempts fail, raise an error + raise Exception(error_message) + + def _emit_validate_input_error(self, final_error: str): + tool_selection_data = { + "agent_key": self.agent.key, + "agent_role": self.agent.role, + "tool_name": self.action.tool, + "tool_args": str(self.action.tool_input), + "tool_class": self.__class__.__name__, + } + + crewai_event_bus.emit( + self, + ToolValidateInputErrorEvent(**tool_selection_data, error=final_error), + ) def on_tool_error(self, tool: Any, tool_calling: ToolCalling, e: Exception) -> None: event_data = self._prepare_event_data(tool, tool_calling) - crewai_event_bus.emit( - self, event=ToolUsageErrorEvent(**{**event_data, "error": e}) - ) + crewai_event_bus.emit(self, ToolUsageErrorEvent(**{**event_data, "error": e})) def on_tool_use_finished( self, tool: Any, tool_calling: ToolCalling, from_cache: bool, started_at: float @@ -474,7 +510,7 @@ class ToolUsage: "from_cache": from_cache, } ) - crewai_event_bus.emit(self, event=ToolUsageFinishedEvent(**event_data)) + crewai_event_bus.emit(self, ToolUsageFinishedEvent(**event_data)) def _prepare_event_data(self, tool: Any, tool_calling: ToolCalling) -> dict: return { diff --git a/src/crewai/utilities/events/__init__.py b/src/crewai/utilities/events/__init__.py index b8b3b9cfb..7f3442360 100644 --- a/src/crewai/utilities/events/__init__.py +++ b/src/crewai/utilities/events/__init__.py @@ -25,7 +25,15 @@ from .flow_events import ( MethodExecutionFailedEvent, ) from .crewai_event_bus import CrewAIEventsBus, crewai_event_bus -from .tool_usage_events import ToolUsageFinishedEvent, ToolUsageErrorEvent, ToolUsageStartedEvent +from .tool_usage_events import ( + ToolUsageFinishedEvent, + ToolUsageErrorEvent, + ToolUsageStartedEvent, + ToolExecutionErrorEvent, + ToolSelectionErrorEvent, + ToolUsageEvent, + ToolValidateInputErrorEvent, +) # events from .event_listener import EventListener diff --git a/src/crewai/utilities/events/agent_events.py b/src/crewai/utilities/events/agent_events.py index 2f6fde522..4fe6baf99 100644 --- a/src/crewai/utilities/events/agent_events.py +++ b/src/crewai/utilities/events/agent_events.py @@ -16,7 +16,7 @@ class AgentExecutionStartedEvent(CrewEvent): agent: BaseAgent task: Any tools: Optional[Sequence[Union[BaseTool, CrewStructuredTool]]] - inputs: Dict[str, Any] + task_prompt: str type: str = "agent_execution_started" model_config = {"arbitrary_types_allowed": True} diff --git a/src/crewai/utilities/events/crewai_event_bus.py b/src/crewai/utilities/events/crewai_event_bus.py index e3cafa0e1..ffb931e0c 100644 --- a/src/crewai/utilities/events/crewai_event_bus.py +++ b/src/crewai/utilities/events/crewai_event_bus.py @@ -5,6 +5,7 @@ from typing import Any, Callable, Dict, List, Type, TypeVar, cast from blinker import Signal from crewai.utilities.events.crew_events import CrewEvent +from crewai.utilities.events.event_types import EventTypes EventT = TypeVar("EventT", bound=CrewEvent) @@ -29,9 +30,7 @@ class CrewAIEventsBus: def _initialize(self) -> None: """Initialize the event bus internal state""" self._signal = Signal("crewai_event_bus") - self._handlers: Dict[ - Type[CrewEvent], List[Callable[[Any, CrewEvent], None]] - ] = {} + self._handlers: Dict[Type[CrewEvent], List[Callable]] = {} def on( self, event_type: Type[EventT] @@ -54,7 +53,7 @@ class CrewAIEventsBus: if event_type not in self._handlers: self._handlers[event_type] = [] self._handlers[event_type].append( - cast(Callable[[Any, CrewEvent], None], handler) + cast(Callable[[Any, EventT], None], handler) ) return handler @@ -79,13 +78,13 @@ class CrewAIEventsBus: self._handlers.clear() def register_handler( - self, event_type: Type[EventT], handler: Callable[[Any, EventT], None] + self, event_type: Type[EventTypes], handler: Callable[[Any, EventTypes], None] ) -> None: """Register an event handler for a specific event type""" if event_type not in self._handlers: self._handlers[event_type] = [] self._handlers[event_type].append( - cast(Callable[[Any, CrewEvent], None], handler) + cast(Callable[[Any, EventTypes], None], handler) ) @contextmanager diff --git a/src/crewai/utilities/events/tool_usage_events.py b/src/crewai/utilities/events/tool_usage_events.py index 8348f1993..645238683 100644 --- a/src/crewai/utilities/events/tool_usage_events.py +++ b/src/crewai/utilities/events/tool_usage_events.py @@ -1,5 +1,5 @@ from datetime import datetime -from typing import Any, Dict +from typing import Any, Callable, Dict from .crew_events import CrewEvent @@ -10,7 +10,7 @@ class ToolUsageEvent(CrewEvent): agent_key: str agent_role: str tool_name: str - tool_args: Dict[str, Any] + tool_args: Dict[str, Any] | str tool_class: str run_attempts: int | None = None delegations: int | None = None @@ -38,3 +38,27 @@ class ToolUsageErrorEvent(ToolUsageEvent): error: Any type: str = "tool_usage_error" + + +class ToolValidateInputErrorEvent(ToolUsageEvent): + """Event emitted when a tool input validation encounters an error""" + + error: Any + type: str = "tool_validate_input_error" + + +class ToolSelectionErrorEvent(ToolUsageEvent): + """Event emitted when a tool selection encounters an error""" + + error: Any + type: str = "tool_selection_error" + + +class ToolExecutionErrorEvent(CrewEvent): + """Event emitted when a tool execution encounters an error""" + + error: Any + type: str = "tool_execution_error" + tool_name: str + tool_args: Dict[str, Any] + tool_class: Callable diff --git a/tests/cassettes/test_tool_execution_error_event.yaml b/tests/cassettes/test_tool_execution_error_event.yaml new file mode 100644 index 000000000..61583726a --- /dev/null +++ b/tests/cassettes/test_tool_execution_error_event.yaml @@ -0,0 +1,112 @@ +interactions: +- request: + body: '{"messages": [{"role": "user", "content": "Use the failing tool"}], "model": + "gpt-4o-mini", "stop": [], "tools": [{"type": "function", "function": {"name": + "failing_tool", "description": "This tool always fails.", "parameters": {"type": + "object", "properties": {"param": {"type": "string", "description": "A test + parameter"}}, "required": ["param"]}}}]}' + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate + connection: + - keep-alive + content-length: + - '353' + content-type: + - application/json + host: + - api.openai.com + user-agent: + - OpenAI/Python 1.61.0 + x-stainless-arch: + - arm64 + x-stainless-async: + - 'false' + x-stainless-lang: + - python + x-stainless-os: + - MacOS + x-stainless-package-version: + - 1.61.0 + x-stainless-raw-response: + - 'true' + x-stainless-retry-count: + - '0' + x-stainless-runtime: + - CPython + x-stainless-runtime-version: + - 3.12.8 + method: POST + uri: https://api.openai.com/v1/chat/completions + response: + content: "{\n \"id\": \"chatcmpl-B2P4zoJZuES7Aom8ugEq1modz5Vsl\",\n \"object\": + \"chat.completion\",\n \"created\": 1739912761,\n \"model\": \"gpt-4o-mini-2024-07-18\",\n + \ \"choices\": [\n {\n \"index\": 0,\n \"message\": {\n \"role\": + \"assistant\",\n \"content\": null,\n \"tool_calls\": [\n {\n + \ \"id\": \"call_F6fJxISpMKUBIGV6dd2vjRNG\",\n \"type\": + \"function\",\n \"function\": {\n \"name\": \"failing_tool\",\n + \ \"arguments\": \"{\\\"param\\\":\\\"test\\\"}\"\n }\n + \ }\n ],\n \"refusal\": null\n },\n \"logprobs\": + null,\n \"finish_reason\": \"tool_calls\"\n }\n ],\n \"usage\": {\n + \ \"prompt_tokens\": 51,\n \"completion_tokens\": 15,\n \"total_tokens\": + 66,\n \"prompt_tokens_details\": {\n \"cached_tokens\": 0,\n \"audio_tokens\": + 0\n },\n \"completion_tokens_details\": {\n \"reasoning_tokens\": + 0,\n \"audio_tokens\": 0,\n \"accepted_prediction_tokens\": 0,\n \"rejected_prediction_tokens\": + 0\n }\n },\n \"service_tier\": \"default\",\n \"system_fingerprint\": + \"fp_00428b782a\"\n}\n" + headers: + CF-RAY: + - 9140fa827f38eb1e-SJC + Connection: + - keep-alive + Content-Encoding: + - gzip + Content-Type: + - application/json + Date: + - Tue, 18 Feb 2025 21:06:02 GMT + Server: + - cloudflare + Set-Cookie: + - __cf_bm=xbuu3IQpCMh.43ZrqL1TRMECOc6QldgHV0hzOX1GrWI-1739912762-1.0.1.1-t7iyq5xMioPrwfeaHLvPT9rwRPp7Q9A9uIm69icH9dPxRD4xMA3cWqb1aXj1_e2IyAEQQWFe1UWjlmJ22aHh3Q; + path=/; expires=Tue, 18-Feb-25 21:36:02 GMT; domain=.api.openai.com; HttpOnly; + Secure; SameSite=None + - _cfuvid=x9l.Rhja8_wXDN.j8qcEU1PvvEqAwZp4Fd3s_aj4qwM-1739912762161-0.0.1.1-604800000; + path=/; domain=.api.openai.com; HttpOnly; Secure; SameSite=None + Transfer-Encoding: + - chunked + X-Content-Type-Options: + - nosniff + access-control-expose-headers: + - X-Request-ID + alt-svc: + - h3=":443"; ma=86400 + cf-cache-status: + - DYNAMIC + openai-organization: + - crewai-iuxna1 + openai-processing-ms: + - '861' + openai-version: + - '2020-10-01' + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + x-ratelimit-limit-requests: + - '30000' + x-ratelimit-limit-tokens: + - '150000000' + x-ratelimit-remaining-requests: + - '29999' + x-ratelimit-remaining-tokens: + - '149999978' + x-ratelimit-reset-requests: + - 2ms + x-ratelimit-reset-tokens: + - 0s + x-request-id: + - req_8666ec3aa6677cb346ba00993556051d + http_version: HTTP/1.1 + status_code: 200 +version: 1 diff --git a/tests/llm_test.py b/tests/llm_test.py index 2e5faf774..00bb69aa5 100644 --- a/tests/llm_test.py +++ b/tests/llm_test.py @@ -7,7 +7,8 @@ from pydantic import BaseModel from crewai.agents.agent_builder.utilities.base_token_process import TokenProcess from crewai.llm import LLM -from crewai.tools import tool +from crewai.utilities.events import crewai_event_bus +from crewai.utilities.events.tool_usage_events import ToolExecutionErrorEvent from crewai.utilities.token_counter_callback import TokenCalcHandler @@ -291,32 +292,36 @@ def anthropic_llm(): """Fixture providing an Anthropic LLM instance.""" return LLM(model="anthropic/claude-3-sonnet") + @pytest.fixture def system_message(): """Fixture providing a system message.""" return {"role": "system", "content": "test"} + @pytest.fixture def user_message(): """Fixture providing a user message.""" return {"role": "user", "content": "test"} + def test_anthropic_message_formatting_edge_cases(anthropic_llm): """Test edge cases for Anthropic message formatting.""" # Test None messages with pytest.raises(TypeError, match="Messages cannot be None"): anthropic_llm._format_messages_for_provider(None) - + # Test empty message list formatted = anthropic_llm._format_messages_for_provider([]) assert len(formatted) == 1 assert formatted[0]["role"] == "user" assert formatted[0]["content"] == "." - + # Test invalid message format with pytest.raises(TypeError, match="Invalid message format"): anthropic_llm._format_messages_for_provider([{"invalid": "message"}]) + def test_anthropic_model_detection(): """Test Anthropic model detection with various formats.""" models = [ @@ -327,11 +332,12 @@ def test_anthropic_model_detection(): ("", False), ("anthropomorphic", False), # Should not match partial words ] - + for model, expected in models: llm = LLM(model=model) assert llm.is_anthropic == expected, f"Failed for model: {model}" + def test_anthropic_message_formatting(anthropic_llm, system_message, user_message): """Test Anthropic message formatting with fixtures.""" # Test when first message is system @@ -371,3 +377,51 @@ def test_deepseek_r1_with_open_router(): result = llm.call("What is the capital of France?") assert isinstance(result, str) assert "Paris" in result + + +@pytest.mark.vcr(filter_headers=["authorization"]) +def test_tool_execution_error_event(): + llm = LLM(model="gpt-4o-mini") + + def failing_tool(param: str) -> str: + """This tool always fails.""" + raise Exception("Tool execution failed!") + + tool_schema = { + "type": "function", + "function": { + "name": "failing_tool", + "description": "This tool always fails.", + "parameters": { + "type": "object", + "properties": { + "param": {"type": "string", "description": "A test parameter"} + }, + "required": ["param"], + }, + }, + } + + received_events = [] + + @crewai_event_bus.on(ToolExecutionErrorEvent) + def event_handler(source, event): + received_events.append(event) + + available_functions = {"failing_tool": failing_tool} + + messages = [{"role": "user", "content": "Use the failing tool"}] + + llm.call( + messages, + tools=[tool_schema], + available_functions=available_functions, + ) + + assert len(received_events) == 1 + event = received_events[0] + assert isinstance(event, ToolExecutionErrorEvent) + assert event.tool_name == "failing_tool" + assert event.tool_args == {"param": "test"} + assert event.tool_class == failing_tool + assert "Tool execution failed!" in event.error diff --git a/tests/tools/test_tool_usage.py b/tests/tools/test_tool_usage.py index 7b2ccd416..941cb7c5f 100644 --- a/tests/tools/test_tool_usage.py +++ b/tests/tools/test_tool_usage.py @@ -1,6 +1,6 @@ import json import random -from unittest.mock import MagicMock +from unittest.mock import MagicMock, patch import pytest from pydantic import BaseModel, Field @@ -8,6 +8,11 @@ from pydantic import BaseModel, Field from crewai import Agent, Task from crewai.tools import BaseTool from crewai.tools.tool_usage import ToolUsage +from crewai.utilities.events import crewai_event_bus +from crewai.utilities.events.tool_usage_events import ( + ToolSelectionErrorEvent, + ToolValidateInputErrorEvent, +) class RandomNumberToolInput(BaseModel): @@ -226,7 +231,7 @@ def test_validate_tool_input_with_special_characters(): ) # Input with special characters - tool_input = '{"message": "Hello, world! \u263A", "valid": True}' + tool_input = '{"message": "Hello, world! \u263a", "valid": True}' expected_arguments = {"message": "Hello, world! ☺", "valid": True} arguments = tool_usage._validate_tool_input(tool_input) @@ -468,18 +473,141 @@ def test_validate_tool_input_large_json_content(): assert arguments == expected_arguments -def test_validate_tool_input_none_input(): +def test_tool_selection_error_event_direct(): + """Test tool selection error event emission directly from ToolUsage class.""" + mock_agent = MagicMock() + mock_agent.key = "test_key" + mock_agent.role = "test_role" + mock_agent.i18n = MagicMock() + mock_agent.verbose = False + + mock_task = MagicMock() + mock_tools_handler = MagicMock() + + class TestTool(BaseTool): + name: str = "Test Tool" + description: str = "A test tool" + + def _run(self, input: dict) -> str: + return "test result" + + test_tool = TestTool() + tool_usage = ToolUsage( - tools_handler=MagicMock(), - tools=[], - original_tools=[], - tools_description="", - tools_names="", - task=MagicMock(), + tools_handler=mock_tools_handler, + tools=[test_tool], + original_tools=[test_tool], + tools_description="Test Tool Description", + tools_names="Test Tool", + task=mock_task, function_calling_llm=None, - agent=MagicMock(), + agent=mock_agent, action=MagicMock(), ) - arguments = tool_usage._validate_tool_input(None) - assert arguments == {} # Expecting an empty dictionary + received_events = [] + + @crewai_event_bus.on(ToolSelectionErrorEvent) + def event_handler(source, event): + received_events.append(event) + + with pytest.raises(Exception) as exc_info: + tool_usage._select_tool("Non Existent Tool") + assert len(received_events) == 1 + event = received_events[0] + assert isinstance(event, ToolSelectionErrorEvent) + assert event.agent_key == "test_key" + assert event.agent_role == "test_role" + assert event.tool_name == "Non Existent Tool" + assert event.tool_args == {} + assert event.tool_class == "Test Tool Description" + assert "don't exist" in event.error + + received_events.clear() + with pytest.raises(Exception) as exc_info: + tool_usage._select_tool("") + + assert len(received_events) == 1 + event = received_events[0] + assert isinstance(event, ToolSelectionErrorEvent) + assert event.agent_key == "test_key" + assert event.agent_role == "test_role" + assert event.tool_name == "" + assert event.tool_args == {} + assert event.tool_class == "Test Tool Description" + assert "forgot the Action name" in event.error + + +def test_tool_validate_input_error_event(): + """Test tool validation input error event emission from ToolUsage class.""" + # Mock agent and required components + mock_agent = MagicMock() + mock_agent.key = "test_key" + mock_agent.role = "test_role" + mock_agent.verbose = False + mock_agent._original_role = "test_role" + + # Mock i18n with error message + mock_i18n = MagicMock() + mock_i18n.errors.return_value = ( + "Tool input must be a valid dictionary in JSON or Python literal format" + ) + mock_agent.i18n = mock_i18n + + # Mock task and tools handler + mock_task = MagicMock() + mock_tools_handler = MagicMock() + + # Mock printer + mock_printer = MagicMock() + + # Create test tool + class TestTool(BaseTool): + name: str = "Test Tool" + description: str = "A test tool" + + def _run(self, input: dict) -> str: + return "test result" + + test_tool = TestTool() + + # Create ToolUsage instance + tool_usage = ToolUsage( + tools_handler=mock_tools_handler, + tools=[test_tool], + original_tools=[test_tool], + tools_description="Test Tool Description", + tools_names="Test Tool", + task=mock_task, + function_calling_llm=None, + agent=mock_agent, + action=MagicMock(tool="test_tool"), + ) + tool_usage._printer = mock_printer + + # Mock all parsing attempts to fail + with ( + patch("json.loads", side_effect=json.JSONDecodeError("Test Error", "", 0)), + patch("ast.literal_eval", side_effect=ValueError), + patch("json5.loads", side_effect=json.JSONDecodeError("Test Error", "", 0)), + patch("json_repair.repair_json", side_effect=Exception("Failed to repair")), + ): + received_events = [] + + @crewai_event_bus.on(ToolValidateInputErrorEvent) + def event_handler(source, event): + received_events.append(event) + + # Test invalid input + invalid_input = "invalid json {[}" + with pytest.raises(Exception) as exc_info: + tool_usage._validate_tool_input(invalid_input) + + # Verify event was emitted + assert len(received_events) == 1, "Expected one event to be emitted" + event = received_events[0] + assert isinstance(event, ToolValidateInputErrorEvent) + assert event.agent_key == "test_key" + assert event.agent_role == "test_role" + assert event.tool_name == "test_tool" + assert "must be a valid dictionary" in event.error diff --git a/tests/utilities/test_events.py b/tests/utilities/test_events.py index 954edeefa..8a2142ca0 100644 --- a/tests/utilities/test_events.py +++ b/tests/utilities/test_events.py @@ -1,5 +1,6 @@ +import json from datetime import datetime -from unittest.mock import patch +from unittest.mock import MagicMock, patch import pytest from pydantic import Field @@ -10,6 +11,7 @@ from crewai.crew import Crew from crewai.flow.flow import Flow, listen, start from crewai.task import Task from crewai.tools.base_tool import BaseTool +from crewai.tools.tool_usage import ToolUsage from crewai.utilities.events.agent_events import ( AgentExecutionCompletedEvent, AgentExecutionErrorEvent, @@ -34,7 +36,9 @@ from crewai.utilities.events.task_events import ( TaskFailedEvent, TaskStartedEvent, ) -from crewai.utilities.events.tool_usage_events import ToolUsageErrorEvent +from crewai.utilities.events.tool_usage_events import ( + ToolUsageErrorEvent, +) base_agent = Agent( role="base_agent",