From 58fc692b0349088d32599c6082ef2b51260d7c66 Mon Sep 17 00:00:00 2001 From: Vinicius Brasil Date: Thu, 18 Jun 2026 21:20:30 -0700 Subject: [PATCH] Expose raw tool result to after_tool_call hooks Typed tools format their output to JSON before handing it to the agent, so `tool_result` reaches after hooks as a string and the original Python object is lost. Thread the unformatted result through as `raw_tool_result` so hooks can inspect the typed value. This covers all execution paths: native tool calls, the ReAct `ToolUsage` path, cached results, and error/blocked branches. --- .../src/crewai/agents/crew_agent_executor.py | 7 +++ lib/crewai/src/crewai/agents/tools_handler.py | 5 +- .../src/crewai/experimental/agent_executor.py | 9 ++- lib/crewai/src/crewai/hooks/tool_hooks.py | 5 ++ lib/crewai/src/crewai/tools/tool_usage.py | 11 ++++ .../src/crewai/utilities/agent_utils.py | 6 ++ lib/crewai/src/crewai/utilities/tool_utils.py | 4 ++ .../tests/agents/test_native_tool_calling.py | 43 ++++++++++++- lib/crewai/tests/hooks/test_tool_hooks.py | 4 ++ lib/crewai/tests/tools/test_tool_usage.py | 50 ++++++++++++++++ .../tests/utilities/test_agent_utils.py | 60 +++++++++++++++++-- 11 files changed, 195 insertions(+), 9 deletions(-) diff --git a/lib/crewai/src/crewai/agents/crew_agent_executor.py b/lib/crewai/src/crewai/agents/crew_agent_executor.py index 3a27a5ad5..80283080e 100644 --- a/lib/crewai/src/crewai/agents/crew_agent_executor.py +++ b/lib/crewai/src/crewai/agents/crew_agent_executor.py @@ -923,12 +923,14 @@ class CrewAgentExecutor(BaseAgentExecutor): from_cache = False result: str = "Tool not found" + raw_tool_result: Any = result input_str = json.dumps(args_dict) if args_dict else "" if self.tools_handler and self.tools_handler.cache and output_tool is not None: cached_result = self.tools_handler.cache.read( tool=func_name, input=input_str ) if cached_result is not None: + raw_tool_result = cached_result result = output_tool.format_output_for_agent(cached_result) from_cache = True @@ -973,8 +975,10 @@ class CrewAgentExecutor(BaseAgentExecutor): if hook_blocked: result = f"Tool execution blocked by hook. Tool: {func_name}" + raw_tool_result = result elif max_usage_reached and original_tool: result = f"Tool '{func_name}' has reached its usage limit of {original_tool.max_usage_count} times and cannot be used anymore." + raw_tool_result = result elif ( not from_cache and func_name in available_functions @@ -982,6 +986,7 @@ class CrewAgentExecutor(BaseAgentExecutor): ): try: raw_result = available_functions[func_name](**(args_dict or {})) + raw_tool_result = raw_result if self.tools_handler and self.tools_handler.cache: should_cache = True @@ -1001,6 +1006,7 @@ class CrewAgentExecutor(BaseAgentExecutor): result = output_tool.format_output_for_agent(raw_result) except Exception as e: result = f"Error executing tool: {e}" + raw_tool_result = result if self.task: self.task.increment_tools_errors() crewai_event_bus.emit( @@ -1024,6 +1030,7 @@ class CrewAgentExecutor(BaseAgentExecutor): task=self.task, crew=self.crew, tool_result=result, + raw_tool_result=raw_tool_result, ) after_hooks = get_after_tool_call_hooks() try: diff --git a/lib/crewai/src/crewai/agents/tools_handler.py b/lib/crewai/src/crewai/agents/tools_handler.py index 8ab759b85..c56226bc1 100644 --- a/lib/crewai/src/crewai/agents/tools_handler.py +++ b/lib/crewai/src/crewai/agents/tools_handler.py @@ -3,6 +3,7 @@ from __future__ import annotations import json +from typing import Any from pydantic import BaseModel, Field @@ -25,14 +26,14 @@ class ToolsHandler(BaseModel): def on_tool_use( self, calling: ToolCalling | InstructorToolCalling, - output: str, + output: Any, should_cache: bool = True, ) -> None: """Run when tool ends running. Args: calling: The tool calling instance. - output: The output from the tool execution. + output: The raw output from the tool execution. should_cache: Whether to cache the tool output. """ self.last_used_tool = calling diff --git a/lib/crewai/src/crewai/experimental/agent_executor.py b/lib/crewai/src/crewai/experimental/agent_executor.py index 26ffea215..b5e6884a9 100644 --- a/lib/crewai/src/crewai/experimental/agent_executor.py +++ b/lib/crewai/src/crewai/experimental/agent_executor.py @@ -1921,12 +1921,15 @@ class AgentExecutor(Flow[AgentExecutorState], BaseAgentExecutor): # Check cache before executing from_cache = False + result = "Tool not found" + raw_tool_result: Any = result input_str = json.dumps(args_dict) if args_dict else "" if self.tools_handler and self.tools_handler.cache and output_tool is not None: cached_result = self.tools_handler.cache.read( tool=func_name, input=input_str ) if cached_result is not None: + raw_tool_result = cached_result result = output_tool.format_output_for_agent(cached_result) from_cache = True @@ -1971,12 +1974,13 @@ class AgentExecutor(Flow[AgentExecutorState], BaseAgentExecutor): if hook_blocked: result = f"Tool execution blocked by hook. Tool: {func_name}" + raw_tool_result = result elif not from_cache and not max_usage_reached and output_tool is not None: - result = "Tool not found" if func_name in self._available_functions: try: tool_func = self._available_functions[func_name] raw_result = tool_func(**args_dict) + raw_tool_result = raw_result # Add to cache after successful execution (before string conversion) if self.tools_handler and self.tools_handler.cache: @@ -1993,6 +1997,7 @@ class AgentExecutor(Flow[AgentExecutorState], BaseAgentExecutor): result = output_tool.format_output_for_agent(raw_result) except Exception as e: result = f"Error executing tool: {e}" + raw_tool_result = result if self.task: self.task.increment_tools_errors() # Emit tool usage error event @@ -2014,6 +2019,7 @@ class AgentExecutor(Flow[AgentExecutorState], BaseAgentExecutor): result = f"Tool '{func_name}' has reached its usage limit of {original_tool.max_usage_count} times and cannot be used anymore." else: result = f"Tool '{func_name}' has reached its maximum usage limit and cannot be used anymore." + raw_tool_result = result # Execute after_tool_call hooks (even if blocked, to allow logging/monitoring) after_hook_context = ToolCallHookContext( @@ -2024,6 +2030,7 @@ class AgentExecutor(Flow[AgentExecutorState], BaseAgentExecutor): task=self.task, crew=self.crew, tool_result=result, + raw_tool_result=raw_tool_result, ) after_hooks = get_after_tool_call_hooks() try: diff --git a/lib/crewai/src/crewai/hooks/tool_hooks.py b/lib/crewai/src/crewai/hooks/tool_hooks.py index b54ca7b10..a860c01d4 100644 --- a/lib/crewai/src/crewai/hooks/tool_hooks.py +++ b/lib/crewai/src/crewai/hooks/tool_hooks.py @@ -40,6 +40,8 @@ class ToolCallHookContext: crew: Crew instance (may be None) tool_result: Tool execution result (only set for after_tool_call hooks). Can be modified by returning a new string from after_tool_call hook. + raw_tool_result: Raw Python tool execution result (only set for + after_tool_call hooks). This is not modified by after hooks. """ def __init__( @@ -51,6 +53,7 @@ class ToolCallHookContext: task: Task | None = None, crew: Crew | None = None, tool_result: str | None = None, + raw_tool_result: Any | None = None, ) -> None: """Initialize tool call hook context. @@ -62,6 +65,7 @@ class ToolCallHookContext: task: Optional current task crew: Optional crew instance tool_result: Optional tool result (for after hooks) + raw_tool_result: Optional raw tool result (for after hooks) """ self.tool_name = tool_name self.tool_input = tool_input @@ -70,6 +74,7 @@ class ToolCallHookContext: self.task = task self.crew = crew self.tool_result = tool_result + self.raw_tool_result = raw_tool_result def request_human_input( self, diff --git a/lib/crewai/src/crewai/tools/tool_usage.py b/lib/crewai/src/crewai/tools/tool_usage.py index d4c9bdbe2..32c734a29 100644 --- a/lib/crewai/src/crewai/tools/tool_usage.py +++ b/lib/crewai/src/crewai/tools/tool_usage.py @@ -106,6 +106,7 @@ class ToolUsage: self.action = action self.function_calling_llm = function_calling_llm self.fingerprint_context = fingerprint_context or {} + self.last_raw_result: Any = None if ( self.function_calling_llm @@ -231,6 +232,7 @@ class ToolUsage: result = I18N_DEFAULT.errors("task_repeated_usage").format( tool_names=self.tools_names ) + self.last_raw_result = result self._telemetry.tool_repeated_usage( llm=self.function_calling_llm, tool_name=sanitize_tool_name(tool.name), @@ -298,6 +300,7 @@ class ToolUsage: ) if usage_limit_error: result = usage_limit_error + self.last_raw_result = result self._telemetry.tool_usage_error(llm=self.function_calling_llm) result = self._format_result(result=result) elif result is None: @@ -359,6 +362,7 @@ class ToolUsage: tool_name=sanitize_tool_name(tool.name), attempts=self._run_attempts, ) + self.last_raw_result = result result = self._format_result( result=tool.format_output_for_agent(result) ) @@ -423,6 +427,7 @@ class ToolUsage: result = ToolUsageError( f"\n{error_message}.\nMoving on then. {I18N_DEFAULT.slice('format').format(tool_names=self.tools_names)}" ).message + self.last_raw_result = result if self.task: self.task.increment_tools_errors() if self.agent and self.agent.verbose: @@ -432,6 +437,7 @@ class ToolUsage: self.task.increment_tools_errors() should_retry = True else: + self.last_raw_result = result result = self._format_result( result=tool.format_output_for_agent(result) ) @@ -464,6 +470,7 @@ class ToolUsage: result = I18N_DEFAULT.errors("task_repeated_usage").format( tool_names=self.tools_names ) + self.last_raw_result = result self._telemetry.tool_repeated_usage( llm=self.function_calling_llm, tool_name=sanitize_tool_name(tool.name), @@ -533,6 +540,7 @@ class ToolUsage: ) if usage_limit_error: result = usage_limit_error + self.last_raw_result = result self._telemetry.tool_usage_error(llm=self.function_calling_llm) result = self._format_result(result=result) elif result is None: @@ -594,6 +602,7 @@ class ToolUsage: tool_name=sanitize_tool_name(tool.name), attempts=self._run_attempts, ) + self.last_raw_result = result result = self._format_result( result=tool.format_output_for_agent(result) ) @@ -658,6 +667,7 @@ class ToolUsage: result = ToolUsageError( f"\n{error_message}.\nMoving on then. {I18N_DEFAULT.slice('format').format(tool_names=self.tools_names)}" ).message + self.last_raw_result = result if self.task: self.task.increment_tools_errors() if self.agent and self.agent.verbose: @@ -667,6 +677,7 @@ class ToolUsage: self.task.increment_tools_errors() should_retry = True else: + self.last_raw_result = result result = self._format_result( result=tool.format_output_for_agent(result) ) diff --git a/lib/crewai/src/crewai/utilities/agent_utils.py b/lib/crewai/src/crewai/utilities/agent_utils.py index d66c43f83..5fffa01a9 100644 --- a/lib/crewai/src/crewai/utilities/agent_utils.py +++ b/lib/crewai/src/crewai/utilities/agent_utils.py @@ -1467,10 +1467,12 @@ def execute_single_native_tool_call( from_cache = False input_str = json.dumps(args_dict) if args_dict else "" result = "Tool not found" + raw_tool_result: Any = result if tools_handler and tools_handler.cache and output_tool is not None: cached_result = tools_handler.cache.read(tool=func_name, input=input_str) if cached_result is not None: + raw_tool_result = cached_result result = output_tool.format_output_for_agent(cached_result) from_cache = True @@ -1510,11 +1512,13 @@ def execute_single_native_tool_call( error_event_emitted = False if hook_blocked: result = f"Tool execution blocked by hook. Tool: {func_name}" + raw_tool_result = result elif not from_cache: if func_name in available_functions and output_tool is not None: try: tool_func = available_functions[func_name] raw_result = tool_func(**args_dict) + raw_tool_result = raw_result if tools_handler and tools_handler.cache: should_cache = True @@ -1530,6 +1534,7 @@ def execute_single_native_tool_call( result = output_tool.format_output_for_agent(raw_result) except Exception as e: result = f"Error executing tool: {e}" + raw_tool_result = result if task: task.increment_tools_errors() crewai_event_bus.emit( @@ -1555,6 +1560,7 @@ def execute_single_native_tool_call( task=task, crew=crew, tool_result=result, + raw_tool_result=raw_tool_result, ) try: for after_hook in get_after_tool_call_hooks(): diff --git a/lib/crewai/src/crewai/utilities/tool_utils.py b/lib/crewai/src/crewai/utilities/tool_utils.py index e19c3c81a..3fad2538b 100644 --- a/lib/crewai/src/crewai/utilities/tool_utils.py +++ b/lib/crewai/src/crewai/utilities/tool_utils.py @@ -116,6 +116,7 @@ async def aexecute_tool_and_check_finality( logger.log("error", f"Error in before_tool_call hook: {e}") tool_result = await tool_usage.ause(tool_calling, agent_action.text) + raw_tool_result = getattr(tool_usage, "last_raw_result", tool_result) after_hook_context = ToolCallHookContext( tool_name=sanitized_tool_name, @@ -125,6 +126,7 @@ async def aexecute_tool_and_check_finality( task=task, crew=crew, tool_result=tool_result, + raw_tool_result=raw_tool_result, ) after_hooks = get_after_tool_call_hooks() @@ -234,6 +236,7 @@ def execute_tool_and_check_finality( logger.log("error", f"Error in before_tool_call hook: {e}") tool_result = tool_usage.use(tool_calling, agent_action.text) + raw_tool_result = getattr(tool_usage, "last_raw_result", tool_result) after_hook_context = ToolCallHookContext( tool_name=sanitized_tool_name, @@ -243,6 +246,7 @@ def execute_tool_and_check_finality( task=task, crew=crew, tool_result=tool_result, + raw_tool_result=raw_tool_result, ) after_hooks = get_after_tool_call_hooks() diff --git a/lib/crewai/tests/agents/test_native_tool_calling.py b/lib/crewai/tests/agents/test_native_tool_calling.py index 8e32ad041..59f2cc7af 100644 --- a/lib/crewai/tests/agents/test_native_tool_calling.py +++ b/lib/crewai/tests/agents/test_native_tool_calling.py @@ -21,7 +21,7 @@ from crewai import Agent, Crew, Task from crewai.agents.parser import AgentFinish from crewai.events import crewai_event_bus from crewai.hooks import register_after_tool_call_hook, register_before_tool_call_hook -from crewai.hooks.tool_hooks import ToolCallHookContext +from crewai.hooks.tool_hooks import ToolCallHookContext, clear_after_tool_call_hooks from crewai.llm import LLM from crewai.tools.base_tool import BaseTool @@ -1227,6 +1227,47 @@ class TestNativeToolCallingJsonParseError: assert json.loads(result["result"]) == {"query": "crew", "score": 0.8} + def test_typed_output_after_hook_includes_raw_tool_result(self) -> None: + from crewai.utilities.agent_utils import convert_tools_to_openai_schema + + class SearchOutput(BaseModel): + query: str + score: float + + class TypedSearchTool(BaseTool): + name: str = "typed_search" + description: str = "Search for information" + output_schema: type[BaseModel] = SearchOutput + + def _run(self, query: str) -> SearchOutput: + return SearchOutput(query=query, score=0.8) + + seen_results: list[tuple[str | None, object]] = [] + + def after_hook(context: ToolCallHookContext) -> None: + seen_results.append((context.tool_result, context.raw_tool_result)) + + tool = TypedSearchTool() + executor = self._make_executor([tool]) + _, available_functions, _ = convert_tools_to_openai_schema([tool]) + + clear_after_tool_call_hooks() + register_after_tool_call_hook(after_hook) + try: + result = executor._execute_single_native_tool_call( + call_id="call_typed", + func_name="typed_search", + func_args='{"query": "crew"}', + available_functions=available_functions, + ) + finally: + clear_after_tool_call_hooks() + + assert json.loads(result["result"]) == {"query": "crew", "score": 0.8} + assert seen_results == [ + ('{"query":"crew","score":0.8}', SearchOutput(query="crew", score=0.8)) + ] + def test_native_tool_loop_falls_back_when_provider_rejects_tools(self) -> None: """Unsupported native tools errors should continue through ReAct.""" diff --git a/lib/crewai/tests/hooks/test_tool_hooks.py b/lib/crewai/tests/hooks/test_tool_hooks.py index 347eb56e5..14c6848a8 100644 --- a/lib/crewai/tests/hooks/test_tool_hooks.py +++ b/lib/crewai/tests/hooks/test_tool_hooks.py @@ -91,20 +91,24 @@ class TestToolCallHookContext: assert context.task == mock_task assert context.crew == mock_crew assert context.tool_result is None + assert context.raw_tool_result is None def test_context_with_result(self, mock_tool): """Test that context includes result when provided.""" tool_input = {"arg1": "value1"} tool_result = "Test tool result" + raw_tool_result = {"value": 42} context = ToolCallHookContext( tool_name="test_tool", tool_input=tool_input, tool=mock_tool, tool_result=tool_result, + raw_tool_result=raw_tool_result, ) assert context.tool_result == tool_result + assert context.raw_tool_result == raw_tool_result def test_tool_input_is_mutable_reference(self, mock_tool): """Test that modifying context.tool_input modifies the original dict.""" diff --git a/lib/crewai/tests/tools/test_tool_usage.py b/lib/crewai/tests/tools/test_tool_usage.py index 3e07bb670..0c75f20ed 100644 --- a/lib/crewai/tests/tools/test_tool_usage.py +++ b/lib/crewai/tests/tools/test_tool_usage.py @@ -7,6 +7,9 @@ import time from unittest.mock import MagicMock, patch from crewai import Agent, Task +from crewai.agents.cache.cache_handler import CacheHandler +from crewai.agents.parser import AgentAction +from crewai.agents.tools_handler import ToolsHandler from crewai.events.event_bus import crewai_event_bus from crewai.events.types.tool_usage_events import ( ToolSelectionErrorEvent, @@ -15,9 +18,15 @@ from crewai.events.types.tool_usage_events import ( ToolUsageStartedEvent, ToolValidateInputErrorEvent, ) +from crewai.hooks.tool_hooks import ( + ToolCallHookContext, + clear_after_tool_call_hooks, + register_after_tool_call_hook, +) from crewai.tools import BaseTool from crewai.tools.tool_calling import ToolCalling from crewai.tools.tool_usage import ToolUsage +from crewai.utilities.tool_utils import execute_tool_and_check_finality from pydantic import BaseModel, Field import pytest @@ -194,6 +203,47 @@ def test_tool_usage_cache_callback_receives_raw_typed_output(): ) +def test_react_tool_hooks_receive_agent_text_and_raw_cached_typed_output(): + structured_tool = TypedSearchTool().to_structured_tool() + tools_handler = ToolsHandler(cache=CacheHandler()) + seen_results: list[tuple[str | None, object]] = [] + + def after_hook(context: ToolCallHookContext) -> None: + seen_results.append((context.tool_result, context.raw_tool_result)) + + clear_after_tool_call_hooks() + register_after_tool_call_hook(after_hook) + + action = AgentAction( + thought="", + tool="typed_search", + tool_input='{"query": "crew"}', + text='Action: typed_search\nAction Input: {"query": "crew"}', + ) + + try: + first = execute_tool_and_check_finality( + agent_action=action, + tools=[structured_tool], + tools_handler=tools_handler, + ) + tools_handler.last_used_tool = None + second = execute_tool_and_check_finality( + agent_action=action, + tools=[structured_tool], + tools_handler=tools_handler, + ) + finally: + clear_after_tool_call_hooks() + + assert json.loads(first.result) == {"query": "crew", "score": 0.7} + assert json.loads(second.result) == {"query": "crew", "score": 0.7} + assert seen_results == [ + ('{"query":"crew","score":0.7}', SearchOutput(query="crew", score=0.7)), + ('{"query":"crew","score":0.7}', SearchOutput(query="crew", score=0.7)), + ] + + def test_validate_tool_input_booleans_and_none(): tool_usage = ToolUsage( tools_handler=MagicMock(), diff --git a/lib/crewai/tests/utilities/test_agent_utils.py b/lib/crewai/tests/utilities/test_agent_utils.py index 55606c2d4..e5d2855b5 100644 --- a/lib/crewai/tests/utilities/test_agent_utils.py +++ b/lib/crewai/tests/utilities/test_agent_utils.py @@ -10,6 +10,12 @@ from unittest.mock import AsyncMock, MagicMock, patch import pytest from pydantic import BaseModel, Field +from crewai.hooks.tool_hooks import ( + ToolCallHookContext, + clear_after_tool_call_hooks, + clear_before_tool_call_hooks, + register_after_tool_call_hook, +) from crewai.tools.base_tool import BaseTool from crewai.utilities.agent_utils import ( _asummarize_chunks, @@ -1032,11 +1038,6 @@ class TestExecuteSingleNativeToolCall: """Tests for execute_single_native_tool_call.""" def test_typed_tool_output_is_json_agent_text(self) -> None: - from crewai.hooks.tool_hooks import ( - clear_after_tool_call_hooks, - clear_before_tool_call_hooks, - ) - clear_before_tool_call_hooks() clear_after_tool_call_hooks() @@ -1078,6 +1079,55 @@ class TestExecuteSingleNativeToolCall: "score": 0.9, } + def test_after_hook_includes_raw_tool_result_for_typed_output(self) -> None: + clear_after_tool_call_hooks() + + class SearchOutput(BaseModel): + query: str + score: float + + class TypedSearchTool(BaseTool): + name: str = "typed_search" + description: str = "Search for a query" + output_schema: type[BaseModel] = SearchOutput + + def _run(self, query: str) -> SearchOutput: + return SearchOutput(query=query, score=0.9) + + seen_results: list[tuple[str | None, object]] = [] + + def after_hook(context: ToolCallHookContext) -> None: + seen_results.append((context.tool_result, context.raw_tool_result)) + + tool = TypedSearchTool() + tool_call = MagicMock() + tool_call.id = "call_1" + tool_call.function.name = "typed_search" + tool_call.function.arguments = '{"query": "crew"}' + + register_after_tool_call_hook(after_hook) + try: + result = execute_single_native_tool_call( + tool_call, + available_functions={"typed_search": tool._run}, + original_tools=[tool], + structured_tools=[tool.to_structured_tool()], + tools_handler=None, + agent=None, + task=None, + crew=None, + event_source=MagicMock(), + printer=None, + verbose=False, + ) + finally: + clear_after_tool_call_hooks() + + assert json.loads(result.result) == {"query": "crew", "score": 0.9} + assert seen_results == [ + ('{"query":"crew","score":0.9}', SearchOutput(query="crew", score=0.9)) + ] + def test_result_as_answer_false_on_tool_error(self) -> None: """When a tool with result_as_answer=True raises, result_as_answer must be False.