diff --git a/docs/edge/en/concepts/tools.mdx b/docs/edge/en/concepts/tools.mdx index da41da3b1..eb2f7426e 100644 --- a/docs/edge/en/concepts/tools.mdx +++ b/docs/edge/en/concepts/tools.mdx @@ -334,6 +334,118 @@ writer1 = Agent( #... ``` +## Reporting Tool Failures + +A tool can finish without raising and still fail to do what it was asked. Slack +answers `HTTP 200` with `{"ok": false, "error": "channel_not_found"}`; an MCP +server sets `isError`; a platform action returns an error payload. The tool call +"worked", so the error text reaches the agent as an ordinary result — the agent +narrates the problem in its final answer and the run is recorded as a success. + +Return a `ToolFailure` instead of an error string and the framework can tell the +difference: + +```python Code +from typing import Any + +from crewai.tools import BaseTool +from crewai.tools.tool_failure import ToolFailure + + +class SendSlackMessage(BaseTool): + name: str = "send_slack_message" + description: str = "Post a message to a Slack channel." + + def _run(self, channel: str, text: str) -> Any: + payload = slack.post(channel=channel, text=text) + if not payload["ok"]: + return ToolFailure( + message=f"Slack rejected the message: {payload['error']}", + code=payload["error"], + retryable=payload["error"] == "rate_limited", + ) + return payload +``` + +The agent still reads plain prose — `ToolFailure.as_agent_message()` renders the +message — so model behavior is unchanged. What changes is that the failure is now +visible to everything downstream. + +Detection is strictly declarative. CrewAI never guesses whether a string "looks +like" an error, so a tool that legitimately returns text about an error is never +misread as having failed. Failures are recorded when a tool returns a +`ToolFailure`, when a tool raises, when an MCP server sets `isError`, when a +tool's `max_usage_count` is spent, or when the agent calls a tool that does not exist. + +### Choosing a Failure Policy + +`tool_failure_policy` controls what happens next: + +| Policy | Behavior | +| :-- | :-- | +| `ignore` | Nothing is recorded, emitted, or acted on. | +| `warn` *(default)* | Records the failure, emits `ToolFailureDetectedEvent`, and continues. | +| `raise` | Records and emits, then aborts with `ToolExecutionFailedError`. | + +```python Code +from crewai import Agent, Task +from crewai.tools.tool_failure import ToolFailurePolicy + +agent = Agent( + role="Slack Messenger", + goal="Post the report to Slack", + backstory="...", + tools=[SendSlackMessage()], + tool_failure_policy=ToolFailurePolicy.WARN, +) + +# Tighten a single high-stakes task without changing the agent. +task = Task( + description="Post the final report to #engineering", + expected_output="Confirmation the message was posted", + agent=agent, + tool_failure_policy=ToolFailurePolicy.RAISE, +) +``` + +The most specific setting wins: tool, then task, then agent, then crew, then the +`warn` default. + +### Inspecting Failures + +Recorded failures are structured, so nothing downstream has to parse a string: + +```python Code +result = crew.kickoff() + +if result.has_tool_failures: + for record in result.tool_failures: + print(record.tool_name) # "send_slack_message" + print(record.failure.code) # "channel_not_found" + print(record.failure.reason) # ToolFailureReason.TOOL_REPORTED + print(record.summary()) +``` + +`tool_failures` is available on `TaskOutput`, `CrewOutput`, and +`LiteAgentOutput`. A crew can finish successfully with a non-empty list — check +it before treating `raw` as complete. + +To react as failures happen, subscribe to the event: + +```python Code +from crewai.events import ToolFailureDetectedEvent +from crewai.events.event_bus import crewai_event_bus + + +@crewai_event_bus.on(ToolFailureDetectedEvent) +def on_tool_failure(source, event): + print(f"{event.tool_name} failed: {event.failure.message} ({event.policy})") +``` + +The event is emitted before the `raise` policy aborts, so subscribers always +observe the failure. `ToolUsageFinishedEvent` also carries a `failure` field, letting +a trace UI mark the call as failed without correlating two events. + ## Conclusion Tools are pivotal in extending the capabilities of CrewAI agents, enabling them to undertake a broad spectrum of tasks and collaborate effectively. diff --git a/lib/crewai-tools/src/crewai_tools/tools/crewai_platform_tools/crewai_platform_action_tool.py b/lib/crewai-tools/src/crewai_tools/tools/crewai_platform_tools/crewai_platform_action_tool.py index 3a3ae3be9..4a6fdda59 100644 --- a/lib/crewai-tools/src/crewai_tools/tools/crewai_platform_tools/crewai_platform_action_tool.py +++ b/lib/crewai-tools/src/crewai_tools/tools/crewai_platform_tools/crewai_platform_action_tool.py @@ -5,6 +5,7 @@ import os from typing import Any from crewai.tools import BaseTool +from crewai.tools.tool_failure import ToolFailure from crewai.utilities.pydantic_schema_utils import create_model_from_schema from pydantic import Field, create_model import requests @@ -49,7 +50,7 @@ class CrewAIPlatformActionTool(BaseTool): self.action_name = action_name self.action_schema = action_schema - def _run(self, **kwargs: Any) -> str: + def _run(self, **kwargs: Any) -> Any: try: cleaned_kwargs = { key: value for key, value in kwargs.items() if value is not None @@ -85,9 +86,22 @@ class CrewAIPlatformActionTool(BaseTool): error_message = str(error_info) else: error_message = str(data) - return f"API request failed: {error_message}" + # The platform returns a non-2xx when the upstream app rejects + # the action -- e.g. Slack answering channel_not_found. That is + # the single most common way an agent "succeeds" at doing + # nothing, so report it as a failure rather than as prose. + return ToolFailure( + message=f"API request failed: {error_message}", + code=str(response.status_code), + retryable=response.status_code >= 500, + details={"action": self.action_name}, + ) return json.dumps(data, indent=2) except Exception as e: - return f"Error executing action {self.action_name}: {e!s}" + return ToolFailure( + message=f"Error executing action {self.action_name}: {e!s}", + code=e.__class__.__name__, + details={"action": self.action_name}, + ) diff --git a/lib/crewai/src/crewai/agent/core.py b/lib/crewai/src/crewai/agent/core.py index 64e74c02d..76cb93764 100644 --- a/lib/crewai/src/crewai/agent/core.py +++ b/lib/crewai/src/crewai/agent/core.py @@ -86,6 +86,7 @@ from crewai.skills.loader import load_skills from crewai.skills.models import INSTRUCTIONS, Skill as SkillModel from crewai.state.checkpoint_config import CheckpointConfig, apply_checkpoint from crewai.tools.agent_tools.agent_tools import AgentTools +from crewai.tools.tool_failure import ToolExecutionFailedError from crewai.types.callback import SerializableCallable from crewai.types.usage_metrics import UsageMetrics from crewai.utilities.agent_utils import ( @@ -131,7 +132,10 @@ if TYPE_CHECKING: from crewai.utilities.types import LLMMessage -_passthrough_exceptions: tuple[type[Exception], ...] = () +# Exceptions that must not be swallowed into the max_retry_limit loop. +# A tool_failure_policy="raise" abort is a deliberate stop, not a transient +# error worth re-running the whole task for. +_passthrough_exceptions: tuple[type[Exception], ...] = (ToolExecutionFailedError,) _EXECUTOR_CLASS_MAP: dict[str, type] = { "CrewAgentExecutor": CrewAgentExecutor, @@ -550,6 +554,8 @@ class Agent(BaseAgent): self._inject_date_to_task(task) + self.reset_tool_failures() + if self.tools_handler: self.tools_handler.last_used_tool = None diff --git a/lib/crewai/src/crewai/agents/agent_builder/base_agent.py b/lib/crewai/src/crewai/agents/agent_builder/base_agent.py index 4530f9527..188b58f14 100644 --- a/lib/crewai/src/crewai/agents/agent_builder/base_agent.py +++ b/lib/crewai/src/crewai/agents/agent_builder/base_agent.py @@ -44,6 +44,7 @@ from crewai.security.security_config import SecurityConfig from crewai.skills.models import Skill from crewai.state.checkpoint_config import CheckpointConfig, _coerce_checkpoint from crewai.tools.base_tool import BaseTool, Tool +from crewai.tools.tool_failure import ToolFailurePolicy, ToolFailureRecord from crewai.types.callback import SerializableCallable from crewai.utilities.config import process_config from crewai.utilities.i18n import I18N, get_i18n @@ -264,6 +265,7 @@ class BaseAgent(BaseModel, ABC, metaclass=AgentMeta): _original_backstory: str | None = PrivateAttr(default=None) _token_process: TokenProcess = PrivateAttr(default_factory=TokenProcess) _kickoff_event_id: str | None = PrivateAttr(default=None) + _tool_failures: list[ToolFailureRecord] = PrivateAttr(default_factory=list) id: UUID4 = Field(default_factory=uuid.uuid4, frozen=True) role: str = Field(description="Role of the agent") goal: str = Field(description="Objective of the agent") @@ -298,6 +300,18 @@ class BaseAgent(BaseModel, ABC, metaclass=AgentMeta): max_iter: int = Field( default=25, description="Maximum iterations for an agent to execute a task" ) + tool_failure_policy: ToolFailurePolicy = Field( + default=ToolFailurePolicy.WARN, + description=( + "How to react when a tool runs to completion but reports that it " + "failed (an upstream API rejecting the request, an MCP server " + "setting isError, a platform action returning an error payload). " + "'ignore' restores pre-1.16 behavior and records nothing; 'warn' " + "records the failure, emits ToolFailureDetectedEvent and keeps " + "going; 'raise' additionally aborts with ToolExecutionFailedError. " + "A Task or a tool may override this for a narrower scope." + ), + ) agent_executor: Annotated[ SerializeAsAny[BaseAgentExecutor] | None, BeforeValidator(_validate_executor_ref), @@ -652,6 +666,20 @@ class BaseAgent(BaseModel, ABC, metaclass=AgentMeta): ] return md5("|".join(source).encode(), usedforsecurity=False).hexdigest() + @property + def last_tool_failures(self) -> list[ToolFailureRecord]: + """Tool failures recorded during the most recent execution. + + Empty when nothing failed, or when ``tool_failure_policy`` is + ``ignore``. Reset at the start of each task execution, mirroring + ``last_messages``. + """ + return self._tool_failures + + def reset_tool_failures(self) -> None: + """Clear recorded tool failures before a new execution begins.""" + self._tool_failures = [] + @abstractmethod def execute_task( self, diff --git a/lib/crewai/src/crewai/agents/crew_agent_executor.py b/lib/crewai/src/crewai/agents/crew_agent_executor.py index 040520e1f..52998aacf 100644 --- a/lib/crewai/src/crewai/agents/crew_agent_executor.py +++ b/lib/crewai/src/crewai/agents/crew_agent_executor.py @@ -49,6 +49,14 @@ from crewai.hooks.tool_hooks import ( run_after_tool_call_hooks, run_before_tool_call_hooks, ) +from crewai.tools.tool_failure import ( + ToolExecutionFailedError, + ToolFailure, + ToolFailureReason, + detect_tool_failure, + failure_from_exception, + handle_tool_failure, +) from crewai.types.callback import SerializableCallable from crewai.utilities.agent_utils import ( _llm_stop_words_applied, @@ -431,6 +439,12 @@ class CrewAgentExecutor(BaseAgentExecutor): self._invoke_step_callback(formatted_answer) self._append_message(formatted_answer.text) + except ToolExecutionFailedError: + # tool_failure_policy="raise" asked for the run to stop; the + # generic handler below would otherwise feed it back to the + # LLM as a recoverable observation. + raise + except OutputParserError as e: formatted_answer = handle_output_parser_exception( # type: ignore[assignment] e=e, @@ -925,6 +939,7 @@ class CrewAgentExecutor(BaseAgentExecutor): from_cache = False result: str = "Tool not found" raw_tool_result: Any = result + tool_failure: ToolFailure | None = None input_str = json.dumps(args_dict) if args_dict else "" if self.tools_handler and self.tools_handler.cache and output_tool is not None: cached_result = self.tools_handler.cache.read( @@ -933,6 +948,7 @@ class CrewAgentExecutor(BaseAgentExecutor): if cached_result is not None: raw_tool_result = cached_result result = format_native_tool_output_for_agent(output_tool, cached_result) + tool_failure = detect_tool_failure(cached_result) from_cache = True agent_key = getattr(self.agent, "key", "unknown") if self.agent else "unknown" @@ -967,6 +983,9 @@ class CrewAgentExecutor(BaseAgentExecutor): elif max_usage_reached and original_tool: result = f"Tool '{func_name}' has reached its usage limit of {original_tool.max_usage_count} times and cannot be used anymore." raw_tool_result = result + tool_failure = ToolFailure( + message=result, reason=ToolFailureReason.USAGE_LIMIT + ) elif ( not from_cache and func_name in available_functions @@ -992,9 +1011,11 @@ class CrewAgentExecutor(BaseAgentExecutor): ) result = format_native_tool_output_for_agent(output_tool, raw_result) + tool_failure = detect_tool_failure(raw_result) except Exception as e: result = f"Error executing tool: {e}" raw_tool_result = result + tool_failure = failure_from_exception(e) if self.task: self.task.increment_tools_errors() crewai_event_bus.emit( @@ -1036,9 +1057,23 @@ class CrewAgentExecutor(BaseAgentExecutor): agent_key=agent_key, started_at=started_at, finished_at=datetime.now(), + failure=tool_failure, ), ) + # After the hooks and the finished event, so subscribers see the full + # lifecycle even when the policy is about to abort the run. + if tool_failure is not None: + handle_tool_failure( + tool_failure, + tool_name=func_name, + tool_args=args_dict, + tool=structured_tool, + agent=self.agent, + task=self.task, + crew=self.crew, + ) + return { "call_id": call_id, "func_name": func_name, @@ -1246,6 +1281,12 @@ class CrewAgentExecutor(BaseAgentExecutor): await self._ainvoke_step_callback(formatted_answer) self._append_message(formatted_answer.text) + except ToolExecutionFailedError: + # tool_failure_policy="raise" asked for the run to stop; the + # generic handler below would otherwise feed it back to the + # LLM as a recoverable observation. + raise + except OutputParserError as e: formatted_answer = handle_output_parser_exception( # type: ignore[assignment] e=e, diff --git a/lib/crewai/src/crewai/crews/crew_output.py b/lib/crewai/src/crewai/crews/crew_output.py index 13b431feb..0d993d0ff 100644 --- a/lib/crewai/src/crewai/crews/crew_output.py +++ b/lib/crewai/src/crewai/crews/crew_output.py @@ -7,6 +7,7 @@ from pydantic import BaseModel, Field from crewai.tasks.output_format import OutputFormat from crewai.tasks.task_output import TaskOutput +from crewai.tools.tool_failure import ToolFailureRecord from crewai.types.usage_metrics import UsageMetrics @@ -31,6 +32,21 @@ class CrewOutput(BaseModel): default_factory=UsageMetrics, ) + @property + def tool_failures(self) -> list[ToolFailureRecord]: + """Every tool failure recorded across all tasks, in task order. + + A crew can finish with a non-empty list: agents routinely narrate a + failed step in prose and carry on, which used to make the run look + entirely successful. Check this before treating ``raw`` as complete. + """ + return [failure for task in self.tasks_output for failure in task.tool_failures] + + @property + def has_tool_failures(self) -> bool: + """Whether any tool reported a failure during this crew run.""" + return any(task.tool_failures for task in self.tasks_output) + @property def usage_metrics(self) -> dict[str, Any]: """Token usage as a plain dict. diff --git a/lib/crewai/src/crewai/events/__init__.py b/lib/crewai/src/crewai/events/__init__.py index 8a31e5397..28cc717a9 100644 --- a/lib/crewai/src/crewai/events/__init__.py +++ b/lib/crewai/src/crewai/events/__init__.py @@ -147,6 +147,7 @@ if TYPE_CHECKING: ) from crewai.events.types.tool_usage_events import ( ToolExecutionErrorEvent, + ToolFailureDetectedEvent, ToolSelectionErrorEvent, ToolUsageErrorEvent, ToolUsageEvent, @@ -251,6 +252,7 @@ _LAZY_EVENT_MAPPING: dict[str, str] = { "TaskFailedEvent": "crewai.events.types.task_events", "TaskStartedEvent": "crewai.events.types.task_events", "ToolExecutionErrorEvent": "crewai.events.types.tool_usage_events", + "ToolFailureDetectedEvent": "crewai.events.types.tool_usage_events", "ToolSelectionErrorEvent": "crewai.events.types.tool_usage_events", "ToolUsageErrorEvent": "crewai.events.types.tool_usage_events", "ToolUsageEvent": "crewai.events.types.tool_usage_events", @@ -384,6 +386,7 @@ __all__ = [ "TaskFailedEvent", "TaskStartedEvent", "ToolExecutionErrorEvent", + "ToolFailureDetectedEvent", "ToolSelectionErrorEvent", "ToolUsageErrorEvent", "ToolUsageEvent", diff --git a/lib/crewai/src/crewai/events/event_listener.py b/lib/crewai/src/crewai/events/event_listener.py index a1a771f44..a1258235d 100644 --- a/lib/crewai/src/crewai/events/event_listener.py +++ b/lib/crewai/src/crewai/events/event_listener.py @@ -113,6 +113,7 @@ from crewai.events.types.task_events import ( TaskStartedEvent, ) from crewai.events.types.tool_usage_events import ( + ToolFailureDetectedEvent, ToolUsageErrorEvent, ToolUsageFinishedEvent, ToolUsageStartedEvent, @@ -439,6 +440,16 @@ class EventListener(BaseEventListener): event.run_attempts, ) + @crewai_event_bus.on(ToolFailureDetectedEvent) + def on_tool_failure_detected( + source: Any, event: ToolFailureDetectedEvent + ) -> None: + self.formatter.handle_tool_failure_detected( + event.tool_name, + event.failure, + event.policy, + ) + @crewai_event_bus.on(LLMCallStartedEvent) def on_llm_call_started(_: Any, event: LLMCallStartedEvent) -> None: self.text_stream = StringIO() diff --git a/lib/crewai/src/crewai/events/event_types.py b/lib/crewai/src/crewai/events/event_types.py index 8c589849a..655886413 100644 --- a/lib/crewai/src/crewai/events/event_types.py +++ b/lib/crewai/src/crewai/events/event_types.py @@ -117,6 +117,7 @@ from crewai.events.types.task_events import ( TaskStartedEvent, ) from crewai.events.types.tool_usage_events import ( + ToolFailureDetectedEvent, ToolUsageErrorEvent, ToolUsageFinishedEvent, ToolUsageStartedEvent, @@ -176,6 +177,7 @@ EventTypes = ( | AgentExecutionErrorEvent | ToolUsageFinishedEvent | ToolUsageErrorEvent + | ToolFailureDetectedEvent | ToolUsageStartedEvent | LLMCallStartedEvent | LLMCallCompletedEvent diff --git a/lib/crewai/src/crewai/events/listeners/tracing/trace_listener.py b/lib/crewai/src/crewai/events/listeners/tracing/trace_listener.py index 789e70e69..db7256617 100644 --- a/lib/crewai/src/crewai/events/listeners/tracing/trace_listener.py +++ b/lib/crewai/src/crewai/events/listeners/tracing/trace_listener.py @@ -126,6 +126,7 @@ from crewai.events.types.task_events import ( TaskStartedEvent, ) from crewai.events.types.tool_usage_events import ( + ToolFailureDetectedEvent, ToolUsageErrorEvent, ToolUsageFinishedEvent, ToolUsageStartedEvent, @@ -410,6 +411,12 @@ class TraceCollectionListener(BaseEventListener): def on_tool_error(source: Any, event: ToolUsageErrorEvent) -> None: self._handle_action_event("tool_usage_error", source, event) + @event_bus.on(ToolFailureDetectedEvent) + def on_tool_failure_detected( + source: Any, event: ToolFailureDetectedEvent + ) -> None: + self._handle_action_event("tool_failure_detected", source, event) + @event_bus.on(MemoryQueryStartedEvent) def on_memory_query_started( source: Any, event: MemoryQueryStartedEvent diff --git a/lib/crewai/src/crewai/events/types/tool_usage_events.py b/lib/crewai/src/crewai/events/types/tool_usage_events.py index 86e0a3087..1ddcb47df 100644 --- a/lib/crewai/src/crewai/events/types/tool_usage_events.py +++ b/lib/crewai/src/crewai/events/types/tool_usage_events.py @@ -5,6 +5,7 @@ from typing import Any, Literal from pydantic import ConfigDict from crewai.events.base_events import BaseEvent +from crewai.tools.tool_failure import ToolFailure, ToolFailurePolicy class ToolUsageEvent(BaseEvent): @@ -66,6 +67,12 @@ class ToolUsageFinishedEvent(ToolUsageEvent): finished_at: datetime from_cache: bool = False output: Any + failure: ToolFailure | None = None + """Set when the tool ran to completion but reported it did not succeed. + + Lets a trace UI render this call as failed without needing to correlate + a separate event. ``None`` for ordinary successful calls. + """ type: Literal["tool_usage_finished"] = "tool_usage_finished" @@ -76,6 +83,25 @@ class ToolUsageErrorEvent(ToolUsageEvent): type: Literal["tool_usage_error"] = "tool_usage_error" +class ToolFailureDetectedEvent(ToolUsageEvent): + """Event emitted when a tool completed but reported that it failed. + + Distinct from :class:`ToolUsageErrorEvent`, which covers a tool *raising*. + This one fires for the quieter case: the call returned normally and the + result says the work was not done -- an upstream API rejecting the + request, an MCP server setting ``isError``, a platform action coming back + with an error payload. + + Emitted for every policy except :attr:`ToolFailurePolicy.IGNORE`, and + emitted *before* the policy aborts execution, so subscribers observe the + failure even on a raising run. + """ + + failure: ToolFailure + policy: ToolFailurePolicy + type: Literal["tool_failure_detected"] = "tool_failure_detected" + + class ToolValidateInputErrorEvent(ToolUsageEvent): """Event emitted when a tool input validation encounters an error""" diff --git a/lib/crewai/src/crewai/events/utils/console_formatter.py b/lib/crewai/src/crewai/events/utils/console_formatter.py index 858dde0ac..dd5891207 100644 --- a/lib/crewai/src/crewai/events/utils/console_formatter.py +++ b/lib/crewai/src/crewai/events/utils/console_formatter.py @@ -492,6 +492,40 @@ To enable tracing, do any one of these: content, f"✅ Tool Execution Completed (#{iteration})", "green" ) + def handle_tool_failure_detected( + self, + tool_name: str, + failure: Any, + policy: Any, + ) -> None: + """Render a tool that ran to completion but reported it did not succeed. + + Distinct from :meth:`handle_tool_usage_error`, which covers a tool + raising. This is the quieter case that used to print as a green + "Tool Execution Completed" panel. + """ + if not self.verbose: + return + + with self._tool_counts_lock: + iteration = self.tool_usage_counts.get(tool_name, 1) + + content = Text() + content.append("Tool Reported Failure\n", style="red bold") + content.append("Tool: ", style="white") + content.append(f"{tool_name}\n", style="red bold") + content.append("Reason: ", style="white") + content.append(f"{getattr(failure, 'reason', 'unknown')}\n", style="red") + if getattr(failure, "code", None): + content.append("Code: ", style="white") + content.append(f"{failure.code}\n", style="red") + content.append("Message: ", style="white") + content.append(f"{getattr(failure, 'message', failure)}\n", style="red") + content.append("Policy: ", style="white") + content.append(f"{getattr(policy, 'value', policy)}\n", style="red") + + self.print_panel(content, f"⚠️ Tool Failure (#{iteration})", "red") + def handle_tool_usage_error( self, tool_name: str, diff --git a/lib/crewai/src/crewai/experimental/agent_executor.py b/lib/crewai/src/crewai/experimental/agent_executor.py index 96308f9c9..2cece31c9 100644 --- a/lib/crewai/src/crewai/experimental/agent_executor.py +++ b/lib/crewai/src/crewai/experimental/agent_executor.py @@ -73,6 +73,14 @@ from crewai.hooks.types import ( ) from crewai.tools.base_tool import BaseTool from crewai.tools.structured_tool import CrewStructuredTool +from crewai.tools.tool_failure import ( + ToolExecutionFailedError, + ToolFailure, + ToolFailureReason, + detect_tool_failure, + failure_from_exception, + handle_tool_failure, +) from crewai.utilities.agent_utils import ( _llm_stop_words_applied, build_text_tool_calling_fallback_message, @@ -1634,6 +1642,12 @@ class AgentExecutor(Flow[AgentExecutorState], BaseAgentExecutor): function_calling_llm=self.function_calling_llm, crew=self.crew, ) + except ToolExecutionFailedError: + # tool_failure_policy="raise" asked for the run to stop; the + # generic handler below would otherwise feed it back to the LLM + # as a recoverable observation. + raise + except Exception as e: if self.agent and self.agent.verbose: PRINTER.print(content=f"Error in tool execution: {e}", color="red") @@ -1949,6 +1963,7 @@ class AgentExecutor(Flow[AgentExecutorState], BaseAgentExecutor): from_cache = False result = "Tool not found" raw_tool_result: Any = result + tool_failure: ToolFailure | None = None input_str = json.dumps(args_dict) if args_dict else "" if self.tools_handler and self.tools_handler.cache and output_tool is not None: cached_result = self.tools_handler.cache.read( @@ -1957,6 +1972,7 @@ class AgentExecutor(Flow[AgentExecutorState], BaseAgentExecutor): if cached_result is not None: raw_tool_result = cached_result result = format_native_tool_output_for_agent(output_tool, cached_result) + tool_failure = detect_tool_failure(cached_result) from_cache = True # Emit tool usage started event @@ -2010,9 +2026,11 @@ class AgentExecutor(Flow[AgentExecutorState], BaseAgentExecutor): result = format_native_tool_output_for_agent( output_tool, raw_result ) + tool_failure = detect_tool_failure(raw_result) except Exception as e: result = f"Error executing tool: {e}" raw_tool_result = result + tool_failure = failure_from_exception(e) if self.task: self.task.increment_tools_errors() # Emit tool usage error event @@ -2035,6 +2053,9 @@ class AgentExecutor(Flow[AgentExecutorState], BaseAgentExecutor): else: result = f"Tool '{func_name}' has reached its maximum usage limit and cannot be used anymore." raw_tool_result = result + tool_failure = ToolFailure( + message=result, reason=ToolFailureReason.USAGE_LIMIT + ) # Execute after_tool_call hooks (even if blocked, to allow logging/monitoring) after_hook_context = ToolCallHookContext( @@ -2063,9 +2084,23 @@ class AgentExecutor(Flow[AgentExecutorState], BaseAgentExecutor): agent_key=agent_key, started_at=started_at, finished_at=datetime.now(), + failure=tool_failure, ), ) + # After the hooks and the finished event, so subscribers see the full + # lifecycle even when the policy is about to abort the run. + if tool_failure is not None: + handle_tool_failure( + tool_failure, + tool_name=func_name, + tool_args=args_dict, + tool=structured_tool, + agent=self.agent, + task=self.task, + crew=self.crew, + ) + return { "call_id": call_id, "func_name": func_name, diff --git a/lib/crewai/src/crewai/lite_agent.py b/lib/crewai/src/crewai/lite_agent.py index bc79c590c..941604198 100644 --- a/lib/crewai/src/crewai/lite_agent.py +++ b/lib/crewai/src/crewai/lite_agent.py @@ -72,6 +72,11 @@ from crewai.llm import LLM from crewai.llms.base_llm import BaseLLM from crewai.tools.base_tool import BaseTool from crewai.tools.structured_tool import CrewStructuredTool +from crewai.tools.tool_failure import ( + ToolExecutionFailedError, + ToolFailurePolicy, + ToolFailureRecord, +) from crewai.utilities.agent_utils import ( enforce_rpm_limit, format_message_for_llm, @@ -222,6 +227,13 @@ class LiteAgent(FlowTrackable, BaseModel): max_iterations: int = Field( default=15, description="Maximum number of iterations for tool usage" ) + tool_failure_policy: ToolFailurePolicy = Field( + default=ToolFailurePolicy.WARN, + description=( + "How to react when a tool runs to completion but reports that it " + "failed. See BaseAgent.tool_failure_policy." + ), + ) max_execution_time: int | None = Field( default=None, description=". Maximum execution time in seconds" ) @@ -289,6 +301,7 @@ class LiteAgent(FlowTrackable, BaseModel): _key: str = PrivateAttr(default_factory=lambda: str(uuid.uuid4())) _messages: list[LLMMessage] = PrivateAttr(default_factory=list) _iterations: int = PrivateAttr(default=0) + _tool_failures: list[ToolFailureRecord] = PrivateAttr(default_factory=list) _guardrail: GuardrailCallable | None = PrivateAttr(default=None) _guardrail_retry_count: int = PrivateAttr(default=0) _callbacks: list[TokenCalcHandler] = PrivateAttr(default_factory=list) @@ -519,6 +532,7 @@ class LiteAgent(FlowTrackable, BaseModel): try: self._iterations = 0 self.tools_results = [] + self._tool_failures = [] self._messages = self._format_messages( messages, response_format=response_format, input_files=input_files @@ -691,6 +705,7 @@ class LiteAgent(FlowTrackable, BaseModel): agent_role=self.role, usage_metrics=usage_metrics.model_dump() if usage_metrics else None, messages=self._messages, + tool_failures=list(self._tool_failures), ) if self._guardrail is not None: @@ -916,7 +931,10 @@ class LiteAgent(FlowTrackable, BaseModel): tools=self._parsed_tools, agent_key=self.key, agent_role=self.role, - agent=self.original_agent, + # Fall back to self so a standalone LiteAgent + # still resolves a tool_failure_policy and + # accumulates records for its output. + agent=self.original_agent or self, crew=None, ) except Exception as e: @@ -929,6 +947,10 @@ class LiteAgent(FlowTrackable, BaseModel): ) self._append_message(formatted_answer.text, role="assistant") + except ToolExecutionFailedError: + # tool_failure_policy="raise" asked for the run to stop. + raise + except OutputParserError as e: if self.verbose: PRINTER.print( diff --git a/lib/crewai/src/crewai/lite_agent_output.py b/lib/crewai/src/crewai/lite_agent_output.py index 4b4bec446..d0dbf7b5a 100644 --- a/lib/crewai/src/crewai/lite_agent_output.py +++ b/lib/crewai/src/crewai/lite_agent_output.py @@ -6,6 +6,7 @@ from typing import Any from pydantic import BaseModel, Field +from crewai.tools.tool_failure import ToolFailureRecord from crewai.types.usage_metrics import UsageMetrics from crewai.utilities.planning_types import TodoItem from crewai.utilities.types import LLMMessage @@ -50,6 +51,13 @@ class LiteAgentOutput(BaseModel): messages: list[LLMMessage] = Field( description="Messages of the agent", default_factory=list ) + tool_failures: list[ToolFailureRecord] = Field( + default_factory=list, + description=( + "Tools that ran during this execution but reported they did not " + "succeed. Always empty when tool_failure_policy is 'ignore'." + ), + ) plan: str | None = Field( default=None, description="The execution plan that was generated, if any" diff --git a/lib/crewai/src/crewai/mcp/client.py b/lib/crewai/src/crewai/mcp/client.py index adf1afb8c..90303ba47 100644 --- a/lib/crewai/src/crewai/mcp/client.py +++ b/lib/crewai/src/crewai/mcp/client.py @@ -430,7 +430,27 @@ class MCPClient: arguments: Tool arguments. Returns: - Tool execution result. + Tool execution result content. The ``isError`` flag is dropped; + use :meth:`call_tool_result` when the caller needs it. + """ + return (await self.call_tool_result(tool_name, arguments)).content + + async def call_tool_result( + self, tool_name: str, arguments: dict[str, Any] | None = None + ) -> _MCPToolResult: + """Call a tool and return its content together with the ``isError`` flag. + + MCP servers report a failed tool as a *successful* JSON-RPC response + carrying ``isError: true``. Callers that only take the content cannot + tell that apart from a normal result, which is how a failed step ends + up looking like a successful one. + + Args: + tool_name: Name of the tool to call. + arguments: Tool arguments. + + Returns: + The content string plus whether the server flagged it as an error. """ if not self.connected: await self.connect() @@ -492,7 +512,7 @@ class MCPClient: ), ) - return tool_result.content + return tool_result except Exception as e: failed_at = datetime.now() error_type = ( diff --git a/lib/crewai/src/crewai/task.py b/lib/crewai/src/crewai/task.py index b6d3b8acc..7a1168bf1 100644 --- a/lib/crewai/src/crewai/task.py +++ b/lib/crewai/src/crewai/task.py @@ -52,6 +52,7 @@ from crewai.security import Fingerprint, SecurityConfig from crewai.tasks.output_format import OutputFormat from crewai.tasks.task_output import TaskOutput from crewai.tools.base_tool import BaseTool +from crewai.tools.tool_failure import ToolFailurePolicy, collect_tool_failures from crewai.utilities.config import process_config from crewai.utilities.constants import NOT_SPECIFIED, _NotSpecified from crewai.utilities.converter import ( @@ -274,6 +275,14 @@ class Task(BaseModel): default=3, description="Maximum number of retries when guardrail fails" ) retry_count: int = Field(default=0, description="Current number of retries") + tool_failure_policy: ToolFailurePolicy | None = Field( + default=None, + description=( + "Overrides the executing agent's tool_failure_policy for this task " + "only. Leave None to inherit from the agent. Useful for tightening " + "a single high-stakes task to 'raise' without changing the agent." + ), + ) start_time: datetime.datetime | None = Field( default=None, description="Start time of the task execution" ) @@ -713,6 +722,7 @@ class Task(BaseModel): agent=agent.role, output_format=self._get_output_format(), messages=agent.last_messages, # type: ignore[attr-defined] + tool_failures=collect_tool_failures(agent), ) if self._guardrails: @@ -867,6 +877,7 @@ class Task(BaseModel): agent=agent.role, output_format=self._get_output_format(), messages=agent.last_messages, # type: ignore[attr-defined] + tool_failures=collect_tool_failures(agent), ) if self._guardrails: @@ -1405,6 +1416,7 @@ Follow these guidelines: agent=agent.role, output_format=self._get_output_format(), messages=agent.last_messages, # type: ignore[attr-defined] + tool_failures=collect_tool_failures(agent), ) return task_output @@ -1514,6 +1526,7 @@ Follow these guidelines: agent=agent.role, output_format=self._get_output_format(), messages=agent.last_messages, # type: ignore[attr-defined] + tool_failures=collect_tool_failures(agent), ) return task_output diff --git a/lib/crewai/src/crewai/tasks/task_output.py b/lib/crewai/src/crewai/tasks/task_output.py index 3bfd4d33d..5e24e930c 100644 --- a/lib/crewai/src/crewai/tasks/task_output.py +++ b/lib/crewai/src/crewai/tasks/task_output.py @@ -8,6 +8,7 @@ from typing import Any from pydantic import BaseModel, Field, model_validator from crewai.tasks.output_format import OutputFormat +from crewai.tools.tool_failure import ToolFailureRecord from crewai.utilities.types import LLMMessage @@ -46,6 +47,20 @@ class TaskOutput(BaseModel): messages: list[LLMMessage] = Field( description="Messages of the task", default_factory=list ) + tool_failures: list[ToolFailureRecord] = Field( + default_factory=list, + description=( + "Tools that ran during this task but reported they did not " + "succeed. Non-empty here means the task produced output despite " + "at least one step failing -- check it before trusting 'raw'. " + "Always empty when the agent's tool_failure_policy is 'ignore'." + ), + ) + + @property + def has_tool_failures(self) -> bool: + """Whether any tool reported a failure while producing this output.""" + return bool(self.tool_failures) @model_validator(mode="after") def set_summary(self) -> TaskOutput: diff --git a/lib/crewai/src/crewai/tools/__init__.py b/lib/crewai/src/crewai/tools/__init__.py index a2415b1b2..b77b9dcf5 100644 --- a/lib/crewai/src/crewai/tools/__init__.py +++ b/lib/crewai/src/crewai/tools/__init__.py @@ -1,8 +1,20 @@ from crewai.tools.base_tool import BaseTool, EnvVar, tool +from crewai.tools.tool_failure import ( + ToolExecutionFailedError, + ToolFailure, + ToolFailurePolicy, + ToolFailureReason, + ToolFailureRecord, +) __all__ = [ "BaseTool", "EnvVar", + "ToolExecutionFailedError", + "ToolFailure", + "ToolFailurePolicy", + "ToolFailureReason", + "ToolFailureRecord", "tool", ] diff --git a/lib/crewai/src/crewai/tools/mcp_native_tool.py b/lib/crewai/src/crewai/tools/mcp_native_tool.py index 94bff3993..101217a71 100644 --- a/lib/crewai/src/crewai/tools/mcp_native_tool.py +++ b/lib/crewai/src/crewai/tools/mcp_native_tool.py @@ -11,6 +11,7 @@ import contextvars from typing import Any from crewai.tools import BaseTool +from crewai.tools.tool_failure import ToolFailure, ToolFailureReason class MCPNativeTool(BaseTool): @@ -70,14 +71,15 @@ class MCPNativeTool(BaseTool): """Get the server name.""" return self._server_name - def _run(self, **kwargs: Any) -> str: + def _run(self, **kwargs: Any) -> Any: """Execute tool using the MCP client session. Args: **kwargs: Arguments to pass to the MCP tool. Returns: - Result from the MCP tool execution. + The tool's text result, or a :class:`ToolFailure` when the server + answered with ``isError: true``. """ try: try: @@ -98,7 +100,7 @@ class MCPNativeTool(BaseTool): f"Error executing MCP tool {self.original_tool_name}: {e!s}" ) from e - async def _run_async(self, **kwargs: Any) -> str: + async def _run_async(self, **kwargs: Any) -> Any: """Async implementation of tool execution. A fresh ``MCPClient`` is created for every invocation so that @@ -108,16 +110,37 @@ class MCPNativeTool(BaseTool): **kwargs: Arguments to pass to the MCP tool. Returns: - Result from the MCP tool execution. + The tool's text result, or a :class:`ToolFailure` when the server + answered with ``isError: true``. """ client = self._client_factory() await client.connect() try: - result = await client.call_tool(self.original_tool_name, kwargs) + tool_result = await client.call_tool_result(self.original_tool_name, kwargs) finally: await client.disconnect() + content = self._extract_content(tool_result.content) + + if tool_result.is_error: + # An MCP server signals a failed tool with isError on an otherwise + # successful response. Preserve that instead of handing the agent + # a plain string it cannot distinguish from a real result. + return ToolFailure( + message=content, + reason=ToolFailureReason.MCP_ERROR, + details={ + "server": self._server_name, + "tool": self._original_tool_name, + }, + ) + + return content + + @staticmethod + def _extract_content(result: Any) -> str: + """Flatten an MCP result payload into the text the agent sees.""" if isinstance(result, str): return result diff --git a/lib/crewai/src/crewai/tools/structured_tool.py b/lib/crewai/src/crewai/tools/structured_tool.py index e868cd7da..6e2e0debc 100644 --- a/lib/crewai/src/crewai/tools/structured_tool.py +++ b/lib/crewai/src/crewai/tools/structured_tool.py @@ -21,6 +21,7 @@ from pydantic import ( ) from typing_extensions import Self +from crewai.tools.tool_failure import ToolFailure from crewai.utilities.logger import Logger from crewai.utilities.pydantic_schema_utils import ( create_model_from_schema, @@ -56,6 +57,12 @@ def _infer_result_schema_from_callable( def _format_tool_output_for_agent(tool: Any, raw_result: Any) -> str: + # A declared failure is rendered as its message so the agent sees the + # same prose it would have seen from an error string. The structured + # object is consumed separately, by the failure-policy machinery. + if isinstance(raw_result, ToolFailure): + return raw_result.as_agent_message() + original_tool = getattr(tool, "_original_tool", None) if original_tool is not None: return cast(str, original_tool.format_output_for_agent(raw_result)) diff --git a/lib/crewai/src/crewai/tools/tool_failure.py b/lib/crewai/src/crewai/tools/tool_failure.py new file mode 100644 index 000000000..99eca4b88 --- /dev/null +++ b/lib/crewai/src/crewai/tools/tool_failure.py @@ -0,0 +1,306 @@ +"""Structured signalling for tools that run but do not succeed. + +A tool can complete without raising and still fail to do what it was asked +to do. An upstream API answers ``HTTP 200`` with +``{"ok": false, "error": "channel_not_found"}``; a CrewAI AMP action comes +back as ``API request failed: ...``; an MCP server sets ``isError``. In every +one of those cases the tool call itself "worked", so historically the result +reached the agent as an ordinary string and the run was recorded as a +success -- even when nothing actually happened. + +This module gives that outcome a type (:class:`ToolFailure`), a per-agent +reaction (:class:`ToolFailurePolicy`), and one place where both are applied +(:func:`handle_tool_failure`). + +A tool declares failure by returning a :class:`ToolFailure`:: + + class SendSlackMessage(BaseTool): + def _run(self, channel: str, text: str) -> Any: + payload = slack.post(channel=channel, text=text) + if not payload["ok"]: + return ToolFailure( + message=f"Slack rejected the message: {payload['error']}", + code=payload["error"], + ) + return payload + +Detection is strictly declarative -- nothing here guesses at whether a +string "looks like" an error. A failure is recorded only when something in +the call chain actually said so. +""" + +from __future__ import annotations + +from enum import Enum +import logging +from typing import TYPE_CHECKING, Any + +from pydantic import BaseModel, ConfigDict, Field + + +logger = logging.getLogger(__name__) + + +if TYPE_CHECKING: + from crewai.agents.agent_builder.base_agent import BaseAgent + from crewai.crew import Crew + from crewai.lite_agent import LiteAgent + from crewai.task import Task + + +class ToolFailureReason(str, Enum): + """Why a tool call is considered unsuccessful.""" + + TOOL_REPORTED = "tool_reported" + """The tool itself returned a :class:`ToolFailure`.""" + + EXCEPTION = "exception" + """The tool raised; the framework caught it and fed the text to the agent.""" + + MCP_ERROR = "mcp_error" + """An MCP server answered with ``isError: true``.""" + + USAGE_LIMIT = "usage_limit" + """The tool's ``max_usage_count`` was already spent.""" + + BLOCKED_BY_HOOK = "blocked_by_hook" + """A ``before_tool_call`` hook refused the call.""" + + UNKNOWN_TOOL = "unknown_tool" + """The agent asked for a tool that does not exist.""" + + INVALID_INPUT = "invalid_input" + """Arguments could not be parsed or validated into the tool's schema.""" + + +class ToolFailurePolicy(str, Enum): + """How an agent reacts when one of its tools reports a failure.""" + + IGNORE = "ignore" + """Pre-1.16 behavior: the failure is not recorded, emitted, or acted on.""" + + WARN = "warn" + """Record the failure, emit an event, and keep going. The default.""" + + RAISE = "raise" + """Record the failure, emit an event, then abort with + :class:`ToolExecutionFailedError`.""" + + +class ToolFailure(BaseModel): + """A tool's own report that it did not accomplish what it was asked to do. + + Return one from ``_run``/``_arun`` instead of an error string. The agent + still receives text (see :meth:`as_agent_message`), so model behavior is + unchanged -- but the framework now knows the call failed. + """ + + model_config = ConfigDict(frozen=True) + + message: str = Field( + description="Human and LLM readable explanation of what went wrong." + ) + reason: ToolFailureReason = Field( + default=ToolFailureReason.TOOL_REPORTED, + description="Category of failure, used for grouping and filtering.", + ) + code: str | None = Field( + default=None, + description=( + "Stable machine-readable identifier from the failing system, " + "e.g. 'channel_not_found' or 'rate_limited'." + ), + ) + retryable: bool = Field( + default=False, + description="Whether retrying the same call could plausibly succeed.", + ) + details: dict[str, Any] = Field( + default_factory=dict, + description="Any extra structured context the tool wants to preserve.", + ) + + def as_agent_message(self) -> str: + """Render the text the agent sees for this failure.""" + if self.code: + return f"{self.message} (code: {self.code})" + return self.message + + +class ToolFailureRecord(BaseModel): + """A :class:`ToolFailure` plus the context of the call that produced it. + + This is what lands on ``TaskOutput.tool_failures`` and on the event bus, + so consumers never have to parse a string to learn that a step failed. + """ + + model_config = ConfigDict(frozen=True) + + tool_name: str = Field(description="Name of the tool that failed.") + failure: ToolFailure = Field(description="The failure the tool reported.") + tool_args: dict[str, Any] | str | None = Field( + default=None, description="Arguments the tool was called with." + ) + agent_role: str | None = Field( + default=None, description="Role of the agent that made the call." + ) + task_name: str | None = Field( + default=None, description="Name or description of the task in flight." + ) + task_id: str | None = Field(default=None, description="Id of the task in flight.") + + @property + def message(self) -> str: + """Shorthand for the underlying failure message.""" + return self.failure.message + + def summary(self) -> str: + """One-line description suitable for logs and error messages.""" + where = f" during '{self.task_name}'" if self.task_name else "" + return ( + f"Tool '{self.tool_name}' failed{where}: {self.failure.as_agent_message()}" + ) + + +class ToolExecutionFailedError(Exception): + """Raised when a tool reports failure under :attr:`ToolFailurePolicy.RAISE`.""" + + def __init__(self, record: ToolFailureRecord) -> None: + self.record = record + super().__init__(record.summary()) + + +def detect_tool_failure(result: Any) -> ToolFailure | None: + """Return the failure a tool declared, if it declared one. + + Deliberately conservative: only an explicit :class:`ToolFailure` counts. + No string sniffing, so a tool legitimately returning text about an error + is never misread as having failed. + """ + if isinstance(result, ToolFailure): + return result + return None + + +def failure_from_exception( + error: BaseException, *, retryable: bool = False +) -> ToolFailure: + """Build a :class:`ToolFailure` from an exception a tool raised.""" + return ToolFailure( + message=str(error) or error.__class__.__name__, + reason=ToolFailureReason.EXCEPTION, + code=error.__class__.__name__, + retryable=retryable, + ) + + +def resolve_tool_failure_policy( + tool: Any = None, + agent: BaseAgent | LiteAgent | None = None, + task: Task | None = None, + crew: Crew | None = None, +) -> ToolFailurePolicy: + """Resolve the effective policy for one call. + + Most specific wins: tool, then task, then agent, then crew, then + :attr:`ToolFailurePolicy.WARN`. + """ + for source in (tool, task, agent, crew): + if source is None: + continue + policy = getattr(source, "tool_failure_policy", None) + if policy is None: + continue + try: + return ToolFailurePolicy(policy) + except ValueError: + # Never let a malformed policy take down a tool call: fall + # through to the next scope and ultimately to the default. + logger.warning( + "Ignoring invalid tool_failure_policy %r on %s; expected one of %s.", + policy, + type(source).__name__, + [member.value for member in ToolFailurePolicy], + ) + return ToolFailurePolicy.WARN + + +def collect_tool_failures(agent: Any) -> list[ToolFailureRecord]: + """Return the failures recorded on an agent, tolerating custom agents. + + Third-party ``BaseAgent`` implementations and test doubles are not + guaranteed to expose ``last_tool_failures`` as a list of records, and + building a task's output must never fail over telemetry. + """ + records = getattr(agent, "last_tool_failures", None) + if not isinstance(records, list): + return [] + return [record for record in records if isinstance(record, ToolFailureRecord)] + + +def _record_on_agent(agent: Any, record: ToolFailureRecord) -> None: + """Append to the agent's per-execution failure list when it has one.""" + failures = getattr(agent, "_tool_failures", None) + if isinstance(failures, list): + failures.append(record) + + +def handle_tool_failure( + failure: ToolFailure, + *, + tool_name: str, + tool_args: dict[str, Any] | str | None = None, + tool: Any = None, + agent: BaseAgent | LiteAgent | None = None, + task: Task | None = None, + crew: Crew | None = None, +) -> ToolFailureRecord | None: + """Apply the effective policy to a failure a tool just reported. + + Records it on the agent, emits :class:`ToolFailureDetectedEvent`, and + raises when the policy says to. Returns the record, or ``None`` when the + policy is :attr:`ToolFailurePolicy.IGNORE`. + + Raises: + ToolExecutionFailedError: When the effective policy is + :attr:`ToolFailurePolicy.RAISE`. + """ + policy = resolve_tool_failure_policy(tool=tool, agent=agent, task=task, crew=crew) + if policy is ToolFailurePolicy.IGNORE: + return None + + record = ToolFailureRecord( + tool_name=tool_name, + failure=failure, + tool_args=tool_args, + agent_role=getattr(agent, "role", None), + task_name=(task.name or task.description) if task else None, + task_id=str(task.id) if task else None, + ) + + _record_on_agent(agent, record) + + # Imported here: crewai.events pulls in the bus and its listeners, which + # import tool types back. A module-level import would cycle. + from crewai.events.event_bus import crewai_event_bus + from crewai.events.types.tool_usage_events import ToolFailureDetectedEvent + + crewai_event_bus.emit( + agent, + ToolFailureDetectedEvent( + tool_name=tool_name, + tool_args=tool_args if tool_args is not None else {}, + failure=failure, + policy=policy, + agent_role=record.agent_role, + agent_key=getattr(agent, "key", None), + agent=agent, + task_name=record.task_name, + task_id=record.task_id, + ), + ) + + if policy is ToolFailurePolicy.RAISE: + raise ToolExecutionFailedError(record) + + return record diff --git a/lib/crewai/src/crewai/tools/tool_usage.py b/lib/crewai/src/crewai/tools/tool_usage.py index 764f82bd9..c53b0fc89 100644 --- a/lib/crewai/src/crewai/tools/tool_usage.py +++ b/lib/crewai/src/crewai/tools/tool_usage.py @@ -24,6 +24,12 @@ from crewai.events.types.tool_usage_events import ( from crewai.telemetry.telemetry import Telemetry from crewai.tools.structured_tool import CrewStructuredTool from crewai.tools.tool_calling import InstructorToolCalling, ToolCalling +from crewai.tools.tool_failure import ( + ToolFailure, + ToolFailureReason, + detect_tool_failure, + failure_from_exception, +) from crewai.utilities.agent_utils import ( get_tool_names, render_text_description_and_args, @@ -110,6 +116,13 @@ class ToolUsage: self.function_calling_llm = function_calling_llm self.fingerprint_context = fingerprint_context or {} self.last_raw_result: Any = _RAW_RESULT_UNSET + self.last_failure: ToolFailure | None = None + """Failure reported by the most recent call, if it reported one. + + Set both for tools that return a :class:`ToolFailure` and for the + framework-generated failures (a raised exception that got stringified, + a spent usage limit) so callers do not have to re-derive them. + """ if ( self.function_calling_llm @@ -265,8 +278,11 @@ class ToolUsage: "run_attempts": self._run_attempts, } - if self.agent.fingerprint: # type: ignore - event_data.update(self.agent.fingerprint) # type: ignore + # Not every agent type carries a fingerprint (LiteAgent does not), + # so read it defensively rather than assuming the attribute exists. + agent_fingerprint = getattr(self.agent, "fingerprint", None) + if agent_fingerprint: + event_data.update(agent_fingerprint) if self.task: event_data["task_name"] = self.task.name or self.task.description event_data["task_id"] = str(self.task.id) @@ -309,6 +325,10 @@ class ToolUsage: if usage_limit_error: result = usage_limit_error self.last_raw_result = result + self.last_failure = ToolFailure( + message=usage_limit_error, + reason=ToolFailureReason.USAGE_LIMIT, + ) self._telemetry.tool_usage_error(llm=self.function_calling_llm) result = self._format_result(result=result) elif result is None: @@ -371,6 +391,7 @@ class ToolUsage: attempts=self._run_attempts, ) self.last_raw_result = result + self.last_failure = detect_tool_failure(result) result = self._format_result( result=tool.format_output_for_agent(result) ) @@ -436,6 +457,7 @@ class ToolUsage: f"\n{error_message}.\nMoving on then. {I18N_DEFAULT.slice('format').format(tool_names=self.tools_names)}" ).message self.last_raw_result = result + self.last_failure = failure_from_exception(e) if self.task: self.task.increment_tools_errors() if self.agent and self.agent.verbose: @@ -446,6 +468,7 @@ class ToolUsage: should_retry = True else: self.last_raw_result = result + self.last_failure = detect_tool_failure(result) result = self._format_result( result=tool.format_output_for_agent(result) ) @@ -504,9 +527,11 @@ class ToolUsage: "run_attempts": self._run_attempts, } - # TODO: Investigate fingerprint attribute availability on BaseAgent/LiteAgent - if self.agent.fingerprint: # type: ignore - event_data.update(self.agent.fingerprint) # type: ignore + # Not every agent type carries a fingerprint (LiteAgent does not), + # so read it defensively rather than assuming the attribute exists. + agent_fingerprint = getattr(self.agent, "fingerprint", None) + if agent_fingerprint: + event_data.update(agent_fingerprint) if self.task: event_data["task_name"] = self.task.name or self.task.description event_data["task_id"] = str(self.task.id) @@ -549,6 +574,10 @@ class ToolUsage: if usage_limit_error: result = usage_limit_error self.last_raw_result = result + self.last_failure = ToolFailure( + message=usage_limit_error, + reason=ToolFailureReason.USAGE_LIMIT, + ) self._telemetry.tool_usage_error(llm=self.function_calling_llm) result = self._format_result(result=result) elif result is None: @@ -611,6 +640,7 @@ class ToolUsage: attempts=self._run_attempts, ) self.last_raw_result = result + self.last_failure = detect_tool_failure(result) result = self._format_result( result=tool.format_output_for_agent(result) ) @@ -676,6 +706,7 @@ class ToolUsage: f"\n{error_message}.\nMoving on then. {I18N_DEFAULT.slice('format').format(tool_names=self.tools_names)}" ).message self.last_raw_result = result + self.last_failure = failure_from_exception(e) if self.task: self.task.increment_tools_errors() if self.agent and self.agent.verbose: @@ -686,6 +717,7 @@ class ToolUsage: should_retry = True else: self.last_raw_result = result + self.last_failure = detect_tool_failure(result) result = self._format_result( result=tool.format_output_for_agent(result) ) @@ -988,6 +1020,7 @@ class ToolUsage: "finished_at": datetime.datetime.fromtimestamp(finished_at), "from_cache": from_cache, "output": result, + "failure": self.last_failure, } ) if self.task: diff --git a/lib/crewai/src/crewai/utilities/agent_utils.py b/lib/crewai/src/crewai/utilities/agent_utils.py index 6a1d2fcf5..40b9b4f93 100644 --- a/lib/crewai/src/crewai/utilities/agent_utils.py +++ b/lib/crewai/src/crewai/utilities/agent_utils.py @@ -31,6 +31,12 @@ from crewai.tools.structured_tool import ( CrewStructuredTool, strip_composite_description_prefix, ) +from crewai.tools.tool_failure import ( + ToolFailure, + detect_tool_failure, + failure_from_exception, + handle_tool_failure, +) from crewai.tools.tool_types import ToolResult from crewai.utilities.errors import AgentRepositoryError from crewai.utilities.exceptions.context_window_exceeding_exception import ( @@ -1532,6 +1538,9 @@ class NativeToolCallResult: def format_native_tool_output_for_agent(tool: Any, raw_result: Any) -> str: """Format native tool output when a tool explicitly defines a formatter.""" + if isinstance(raw_result, ToolFailure): + return raw_result.as_agent_message() + formatter = inspect.getattr_static(tool, "format_output_for_agent", None) if formatter is None: return str(raw_result) @@ -1628,12 +1637,14 @@ def execute_single_native_tool_call( input_str = json.dumps(args_dict) if args_dict else "" result = "Tool not found" raw_tool_result: Any = result + tool_failure: ToolFailure | None = None if tools_handler and tools_handler.cache and output_tool is not None: cached_result = tools_handler.cache.read(tool=func_name, input=input_str) if cached_result is not None: raw_tool_result = cached_result result = format_native_tool_output_for_agent(output_tool, cached_result) + tool_failure = detect_tool_failure(cached_result) from_cache = True started_at = datetime.now() @@ -1685,9 +1696,11 @@ def execute_single_native_tool_call( ) result = format_native_tool_output_for_agent(output_tool, raw_result) + tool_failure = detect_tool_failure(raw_result) except Exception as e: result = f"Error executing tool: {e}" raw_tool_result = result + tool_failure = failure_from_exception(e) if task: task.increment_tools_errors() crewai_event_bus.emit( @@ -1733,9 +1746,23 @@ def execute_single_native_tool_call( plan_step_description=plan_step_description, started_at=started_at, finished_at=datetime.now(), + failure=tool_failure, ), ) + # After the hooks and the finished event, so subscribers see the full + # lifecycle even when the policy is about to abort the run. + if tool_failure is not None: + handle_tool_failure( + tool_failure, + tool_name=func_name, + tool_args=args_dict, + tool=structured_tool, + agent=agent, + task=task, + crew=crew, + ) + tool_message: LLMMessage = { "role": "tool", "tool_call_id": call_id, diff --git a/lib/crewai/src/crewai/utilities/tool_utils.py b/lib/crewai/src/crewai/utilities/tool_utils.py index 2d6e3c142..8eb3a1a3a 100644 --- a/lib/crewai/src/crewai/utilities/tool_utils.py +++ b/lib/crewai/src/crewai/utilities/tool_utils.py @@ -11,6 +11,11 @@ from crewai.hooks.tool_hooks import ( ) from crewai.security.fingerprint import Fingerprint from crewai.tools.structured_tool import CrewStructuredTool +from crewai.tools.tool_failure import ( + ToolFailure, + ToolFailureReason, + handle_tool_failure, +) from crewai.tools.tool_types import ToolResult from crewai.tools.tool_usage import ToolUsage, ToolUsageError from crewai.utilities.i18n import I18N_DEFAULT @@ -138,6 +143,19 @@ async def aexecute_tool_and_check_finality( modified_result = run_after_tool_call_hooks(after_hook_context) + # After the hooks, so a post_tool_call hook still gets to inspect or + # rewrite the result before the policy can abort the run. + if tool_usage.last_failure is not None: + handle_tool_failure( + tool_usage.last_failure, + tool_name=sanitized_tool_name, + tool_args=tool_input, + tool=tool, + agent=agent, + task=task, + crew=crew, + ) + return ToolResult( modified_result if modified_result is not None else tool_result, tool.result_as_answer, @@ -147,6 +165,18 @@ async def aexecute_tool_and_check_finality( tool=sanitized_tool_name, tools=", ".join(tool_name_to_tool_map.keys()), ) + handle_tool_failure( + ToolFailure( + message=tool_result, + reason=ToolFailureReason.UNKNOWN_TOOL, + code=sanitized_tool_name, + ), + tool_name=sanitized_tool_name, + tool_args=tool_calling.arguments, + agent=agent, + task=task, + crew=crew, + ) return ToolResult(result=tool_result, result_as_answer=False) @@ -260,6 +290,19 @@ def execute_tool_and_check_finality( modified_result = run_after_tool_call_hooks(after_hook_context) + # After the hooks, so a post_tool_call hook still gets to inspect or + # rewrite the result before the policy can abort the run. + if tool_usage.last_failure is not None: + handle_tool_failure( + tool_usage.last_failure, + tool_name=sanitized_tool_name, + tool_args=tool_input, + tool=tool, + agent=agent, + task=task, + crew=crew, + ) + return ToolResult( modified_result if modified_result is not None else tool_result, tool.result_as_answer, @@ -269,4 +312,16 @@ def execute_tool_and_check_finality( tool=sanitized_tool_name, tools=", ".join(tool_name_to_tool_map.keys()), ) + handle_tool_failure( + ToolFailure( + message=tool_result, + reason=ToolFailureReason.UNKNOWN_TOOL, + code=sanitized_tool_name, + ), + tool_name=sanitized_tool_name, + tool_args=tool_calling.arguments, + agent=agent, + task=task, + crew=crew, + ) return ToolResult(result=tool_result, result_as_answer=False) diff --git a/lib/crewai/tests/mcp/test_mcp_config.py b/lib/crewai/tests/mcp/test_mcp_config.py index ce123be6b..607d9d3e6 100644 --- a/lib/crewai/tests/mcp/test_mcp_config.py +++ b/lib/crewai/tests/mcp/test_mcp_config.py @@ -4,6 +4,7 @@ from unittest.mock import AsyncMock, patch import pytest from crewai.agent.core import Agent +from crewai.mcp.client import _MCPToolResult from crewai.mcp.config import MCPServerHTTP, MCPServerSSE, MCPServerStdio from crewai.tools.base_tool import BaseTool @@ -39,6 +40,9 @@ def _make_mock_client(tool_definitions): client.connect = AsyncMock() client.disconnect = AsyncMock() client.call_tool = AsyncMock(return_value="test result") + client.call_tool_result = AsyncMock( + return_value=_MCPToolResult("test result", False) + ) return client @@ -227,9 +231,9 @@ def test_parallel_mcp_tool_execution_same_tool(mock_tool_definitions): async def _call_tool(name, args): call_log.append(name) await asyncio.sleep(0.05) - return f"result-{name}" + return _MCPToolResult(f"result-{name}", False) - client.call_tool = AsyncMock(side_effect=_call_tool) + client.call_tool_result = AsyncMock(side_effect=_call_tool) return client with patch("crewai.mcp.tool_resolver.MCPClient", side_effect=_make_client): @@ -273,9 +277,9 @@ def test_parallel_mcp_tool_execution_different_tools(mock_tool_definitions): async def _call_tool(name, args): call_log.append(name) await asyncio.sleep(0.05) - return f"result-{name}" + return _MCPToolResult(f"result-{name}", False) - client.call_tool = AsyncMock(side_effect=_call_tool) + client.call_tool_result = AsyncMock(side_effect=_call_tool) return client with patch("crewai.mcp.tool_resolver.MCPClient", side_effect=_make_client): diff --git a/lib/crewai/tests/test_task.py b/lib/crewai/tests/test_task.py index a6525f8cb..583df5db0 100644 --- a/lib/crewai/tests/test_task.py +++ b/lib/crewai/tests/test_task.py @@ -162,6 +162,7 @@ def test_task_callback_returns_task_output(): "expected_output": "Bullet point list of 5 interesting ideas.", "output_format": OutputFormat.RAW, "messages": [], + "tool_failures": [], } assert output_dict == expected_output diff --git a/lib/crewai/tests/tools/test_tool_failure.py b/lib/crewai/tests/tools/test_tool_failure.py new file mode 100644 index 000000000..da0a703f3 --- /dev/null +++ b/lib/crewai/tests/tools/test_tool_failure.py @@ -0,0 +1,444 @@ +"""Tests for structured tool-failure signalling and the per-agent policy.""" + +from typing import Any + +import pytest + +from crewai import Agent, Crew, Task +from crewai.events.event_bus import crewai_event_bus +from crewai.events.types.tool_usage_events import ( + ToolFailureDetectedEvent, + ToolUsageFinishedEvent, +) +from crewai.llm import LLM +from crewai.tools import BaseTool +from crewai.tools.tool_failure import ( + ToolExecutionFailedError, + ToolFailure, + ToolFailurePolicy, + ToolFailureReason, + ToolFailureRecord, + detect_tool_failure, + failure_from_exception, + resolve_tool_failure_policy, +) + + +class SlackTool(BaseTool): + """Mirrors an upstream API that answers 200 with an error body.""" + + name: str = "slackbot_send_message" + description: str = "Post a message to a Slack channel." + + def _run(self, channel: str) -> Any: + return ToolFailure( + message=f"Slack rejected the message to {channel}", + code="channel_not_found", + ) + + +class WorkingTool(BaseTool): + name: str = "echo" + description: str = "Echo the input back." + + def _run(self, text: str) -> Any: + return f"echoed: {text}" + + +class ScriptedLLM(LLM): + """Emits a fixed sequence of ReAct steps without touching a provider.""" + + def __new__(cls, *args: Any, **kwargs: Any) -> "ScriptedLLM": + return object.__new__(cls) + + def __init__(self, steps: list[str]) -> None: + super().__init__(model="gpt-4o") + self._steps = steps + self._index = 0 + + def call(self, messages, tools=None, callbacks=None, available_functions=None, **kw): # noqa: ANN001, ANN003 + step = self._steps[min(self._index, len(self._steps) - 1)] + self._index += 1 + return step + + def supports_function_calling(self) -> bool: + return False + + +def _slack_steps() -> list[str]: + return [ + "Thought: posting\n" + "Action: slackbot_send_message\n" + 'Action Input: {"channel": "#joao-message"}', + "Thought: it failed\nFinal Answer: I could not post the message.", + ] + + +def _build_crew(policy: ToolFailurePolicy | None = None, **task_kwargs: Any): + agent_kwargs: dict[str, Any] = { + "role": "Slack Messenger", + "goal": "post a message", + "backstory": "b", + "llm": ScriptedLLM(_slack_steps()), + "tools": [SlackTool()], + } + if policy is not None: + agent_kwargs["tool_failure_policy"] = policy + agent = Agent(**agent_kwargs) + task = Task( + description="post to slack", + expected_output="confirmation", + agent=agent, + **task_kwargs, + ) + return Crew(agents=[agent], tasks=[task]), agent + + +class TestToolFailureModel: + def test_as_agent_message_includes_code(self) -> None: + failure = ToolFailure(message="nope", code="channel_not_found") + assert failure.as_agent_message() == "nope (code: channel_not_found)" + + def test_as_agent_message_without_code(self) -> None: + assert ToolFailure(message="nope").as_agent_message() == "nope" + + def test_default_reason_is_tool_reported(self) -> None: + assert ToolFailure(message="x").reason is ToolFailureReason.TOOL_REPORTED + + def test_detection_is_declarative_only(self) -> None: + """A string that merely looks like an error is not a failure.""" + assert detect_tool_failure("Error: something went wrong") is None + assert detect_tool_failure({"ok": False}) is None + assert detect_tool_failure(ToolFailure(message="x")) is not None + + def test_failure_from_exception(self) -> None: + failure = failure_from_exception(ValueError("bad input")) + assert failure.reason is ToolFailureReason.EXCEPTION + assert failure.code == "ValueError" + assert "bad input" in failure.message + + def test_record_summary_mentions_tool_and_task(self) -> None: + record = ToolFailureRecord( + tool_name="slackbot_send_message", + failure=ToolFailure(message="nope", code="channel_not_found"), + task_name="post to slack", + ) + summary = record.summary() + assert "slackbot_send_message" in summary + assert "post to slack" in summary + assert "channel_not_found" in summary + + +class TestPolicyResolution: + def test_defaults_to_warn(self) -> None: + assert resolve_tool_failure_policy() is ToolFailurePolicy.WARN + + def test_agent_policy_used_when_no_narrower_scope(self) -> None: + agent = Agent( + role="r", + goal="g", + backstory="b", + tool_failure_policy=ToolFailurePolicy.RAISE, + ) + assert resolve_tool_failure_policy(agent=agent) is ToolFailurePolicy.RAISE + + def test_task_overrides_agent(self) -> None: + agent = Agent( + role="r", + goal="g", + backstory="b", + tool_failure_policy=ToolFailurePolicy.WARN, + ) + task = Task( + description="d", + expected_output="e", + tool_failure_policy=ToolFailurePolicy.RAISE, + ) + resolved = resolve_tool_failure_policy(agent=agent, task=task) + assert resolved is ToolFailurePolicy.RAISE + + def test_unset_task_policy_falls_through_to_agent(self) -> None: + agent = Agent( + role="r", + goal="g", + backstory="b", + tool_failure_policy=ToolFailurePolicy.IGNORE, + ) + task = Task(description="d", expected_output="e") + resolved = resolve_tool_failure_policy(agent=agent, task=task) + assert resolved is ToolFailurePolicy.IGNORE + + def test_invalid_policy_is_ignored_rather_than_raising(self) -> None: + """A bad policy value must never take down a tool call.""" + + class Bogus: + tool_failure_policy = "not-a-policy" + + assert resolve_tool_failure_policy(agent=Bogus()) is ToolFailurePolicy.WARN + + def test_invalid_policy_falls_through_to_next_scope(self) -> None: + class Bogus: + tool_failure_policy = object() + + agent = Agent( + role="r", + goal="g", + backstory="b", + tool_failure_policy=ToolFailurePolicy.IGNORE, + ) + resolved = resolve_tool_failure_policy(tool=Bogus(), agent=agent) + assert resolved is ToolFailurePolicy.IGNORE + + def test_tool_overrides_everything(self) -> None: + class StrictTool(WorkingTool): + tool_failure_policy: ToolFailurePolicy = ToolFailurePolicy.RAISE + + agent = Agent( + role="r", + goal="g", + backstory="b", + tool_failure_policy=ToolFailurePolicy.IGNORE, + ) + resolved = resolve_tool_failure_policy(tool=StrictTool(), agent=agent) + assert resolved is ToolFailurePolicy.RAISE + + +class TestAgentDefault: + def test_agent_defaults_to_warn(self) -> None: + agent = Agent(role="r", goal="g", backstory="b") + assert agent.tool_failure_policy is ToolFailurePolicy.WARN + + def test_task_policy_defaults_to_none_so_it_inherits(self) -> None: + assert Task(description="d", expected_output="e").tool_failure_policy is None + + +class TestEndToEndPolicies: + def test_warn_records_and_emits_without_stopping(self) -> None: + crew, agent = _build_crew(ToolFailurePolicy.WARN) + events: list[ToolFailureDetectedEvent] = [] + + with crewai_event_bus.scoped_handlers(): + + @crewai_event_bus.on(ToolFailureDetectedEvent) + def _(source: Any, event: ToolFailureDetectedEvent) -> None: + events.append(event) + + result = crew.kickoff() + + assert len(events) == 1 + assert events[0].tool_name == "slackbot_send_message" + assert events[0].failure.code == "channel_not_found" + assert events[0].policy is ToolFailurePolicy.WARN + + assert result.has_tool_failures + assert len(result.tool_failures) == 1 + assert result.tool_failures[0].failure.code == "channel_not_found" + assert result.tasks_output[0].has_tool_failures + + def test_ignore_restores_previous_behaviour(self) -> None: + crew, _ = _build_crew(ToolFailurePolicy.IGNORE) + events: list[ToolFailureDetectedEvent] = [] + + with crewai_event_bus.scoped_handlers(): + + @crewai_event_bus.on(ToolFailureDetectedEvent) + def _(source: Any, event: ToolFailureDetectedEvent) -> None: + events.append(event) + + result = crew.kickoff() + + assert events == [] + assert not result.has_tool_failures + assert result.tool_failures == [] + + def test_raise_aborts_the_run(self) -> None: + crew, _ = _build_crew(ToolFailurePolicy.RAISE) + + with pytest.raises(ToolExecutionFailedError) as exc_info: + crew.kickoff() + + record = exc_info.value.record + assert record.tool_name == "slackbot_send_message" + assert record.failure.code == "channel_not_found" + + def test_event_is_emitted_before_raise(self) -> None: + """Subscribers must observe the failure even on an aborting run.""" + crew, _ = _build_crew(ToolFailurePolicy.RAISE) + events: list[ToolFailureDetectedEvent] = [] + + with crewai_event_bus.scoped_handlers(): + + @crewai_event_bus.on(ToolFailureDetectedEvent) + def _(source: Any, event: ToolFailureDetectedEvent) -> None: + events.append(event) + + with pytest.raises(ToolExecutionFailedError): + crew.kickoff() + + assert len(events) == 1 + + def test_task_policy_overrides_agent_end_to_end(self) -> None: + crew, _ = _build_crew( + ToolFailurePolicy.WARN, + tool_failure_policy=ToolFailurePolicy.RAISE, + ) + with pytest.raises(ToolExecutionFailedError): + crew.kickoff() + + def test_default_agent_warns(self) -> None: + """No explicit policy anywhere still records the failure.""" + crew, _ = _build_crew() + result = crew.kickoff() + assert result.has_tool_failures + + def test_finished_event_carries_the_failure(self) -> None: + crew, _ = _build_crew(ToolFailurePolicy.WARN) + finished: list[ToolUsageFinishedEvent] = [] + + with crewai_event_bus.scoped_handlers(): + + @crewai_event_bus.on(ToolUsageFinishedEvent) + def _(source: Any, event: ToolUsageFinishedEvent) -> None: + finished.append(event) + + crew.kickoff() + + slack_events = [e for e in finished if e.tool_name == "slackbot_send_message"] + assert slack_events + assert slack_events[0].failure is not None + assert slack_events[0].failure.code == "channel_not_found" + + def test_agent_sees_the_failure_message_as_plain_text(self) -> None: + """Model-facing behavior is unchanged: it still reads prose.""" + crew, _ = _build_crew(ToolFailurePolicy.WARN) + result = crew.kickoff() + tool_messages = [ + m + for m in result.tasks_output[0].messages + if "Slack rejected the message" in str(m.get("content", "")) + ] + assert tool_messages + + +class TestMCPIsErrorPlumbing: + """An MCP server flags a failed tool with isError on a 200 response.""" + + @staticmethod + def _tool(is_error: bool) -> Any: + from unittest.mock import AsyncMock + + from crewai.mcp.client import _MCPToolResult + from crewai.tools.mcp_native_tool import MCPNativeTool + + client = AsyncMock() + client.connect = AsyncMock() + client.disconnect = AsyncMock() + client.call_tool_result = AsyncMock( + return_value=_MCPToolResult("channel not found", is_error) + ) + return MCPNativeTool( + client_factory=lambda: client, + tool_name="post", + tool_schema={"description": "post a message"}, + server_name="slack", + ) + + def test_is_error_becomes_a_tool_failure(self) -> None: + result = self._tool(is_error=True).run() + assert isinstance(result, ToolFailure) + assert result.reason is ToolFailureReason.MCP_ERROR + assert result.message == "channel not found" + assert result.details["server"] == "slack" + + def test_successful_call_still_returns_plain_text(self) -> None: + assert self._tool(is_error=False).run() == "channel not found" + + +class TestPlatformActionTool: + """CrewAI AMP agentic-app actions -- the Slack case from the bug report.""" + + @staticmethod + def _tool() -> Any: + from crewai_tools.tools.crewai_platform_tools.crewai_platform_action_tool import ( # noqa: E501 + CrewAIPlatformActionTool, + ) + + return CrewAIPlatformActionTool( + description="Send a Slack message", + action_name="slackbot_send_message", + action_schema={ + "function": { + "name": "slackbot_send_message", + "parameters": { + "properties": {"channel": {"type": "string"}}, + "required": [], + }, + } + }, + ) + + def test_non_ok_response_becomes_a_tool_failure(self, monkeypatch) -> None: # noqa: ANN001 + from unittest.mock import Mock + + import crewai_tools.tools.crewai_platform_tools.crewai_platform_action_tool as mod + + response = Mock() + response.ok = False + response.status_code = 500 + response.json.return_value = { + "error": "Failed to execute action: Slack API error: channel_not_found" + } + monkeypatch.setattr(mod.requests, "post", Mock(return_value=response)) + monkeypatch.setenv("CREWAI_PLATFORM_INTEGRATION_TOKEN", "t") + + result = self._tool()._run(channel="#joao-message") + + assert isinstance(result, ToolFailure) + assert "channel_not_found" in result.message + assert result.retryable is True + + def test_ok_response_still_returns_json(self, monkeypatch) -> None: # noqa: ANN001 + from unittest.mock import Mock + + import crewai_tools.tools.crewai_platform_tools.crewai_platform_action_tool as mod + + response = Mock() + response.ok = True + response.json.return_value = {"ts": "1234.5678"} + monkeypatch.setattr(mod.requests, "post", Mock(return_value=response)) + monkeypatch.setenv("CREWAI_PLATFORM_INTEGRATION_TOKEN", "t") + + result = self._tool()._run(channel="#general") + + assert not isinstance(result, ToolFailure) + assert "1234.5678" in result + + +class TestSuccessfulToolsUnaffected: + def test_no_failure_recorded_for_a_working_tool(self) -> None: + agent = Agent( + role="Echoer", + goal="echo", + backstory="b", + llm=ScriptedLLM( + [ + 'Thought: echo\nAction: echo\nAction Input: {"text": "hi"}', + "Thought: done\nFinal Answer: echoed: hi", + ] + ), + tools=[WorkingTool()], + ) + task = Task(description="echo hi", expected_output="hi", agent=agent) + result = Crew(agents=[agent], tasks=[task]).kickoff() + + assert not result.has_tool_failures + assert result.tool_failures == [] + + def test_failures_reset_between_executions(self) -> None: + crew, agent = _build_crew(ToolFailurePolicy.WARN) + crew.kickoff() + assert len(agent.last_tool_failures) == 1 + + agent.llm = ScriptedLLM(_slack_steps()) + crew.kickoff() + assert len(agent.last_tool_failures) == 1, "records must not accumulate"