Fix: Add multimodal tools support in Agent.kickoff() and kickoff_async()

- Add _build_runtime_tools() helper method to combine tools from multiple sources
- Fix issue #3936 where multimodal=True was not adding AddImageTool in kickoff()
- Ensure kickoff_async() includes platform, MCP, and multimodal tools for consistency
- Prevent tool duplication by deduplicating tools by name
- Avoid mutating self.tools to prevent side effects across multiple kickoff calls
- Add comprehensive tests for multimodal tool handling in both sync and async kickoff methods

Co-Authored-By: João <joao@crewai.com>
This commit is contained in:
Devin AI
2025-11-18 11:27:00 +00:00
parent 9fcf55198f
commit 5715827ff7
2 changed files with 339 additions and 10 deletions

View File

@@ -1416,6 +1416,43 @@ class Agent(BaseAgent):
)
return None
def _build_runtime_tools(self) -> list[BaseTool]:
"""Build a list of tools for runtime execution without mutating self.tools.
This method combines tools from multiple sources:
- Agent's configured tools (self.tools)
- Platform tools (if self.apps is set)
- MCP tools (if self.mcps is set)
- Multimodal tools (if self.multimodal is True)
Returns:
A deduplicated list of tools ready for execution.
"""
runtime_tools: list[BaseTool] = list(self.tools or [])
if self.apps:
platform_tools = self.get_platform_tools(self.apps)
if platform_tools:
runtime_tools.extend(platform_tools)
if self.mcps:
mcp_tools = self.get_mcp_tools(self.mcps)
if mcp_tools:
runtime_tools.extend(mcp_tools)
if self.multimodal:
multimodal_tools = self.get_multimodal_tools()
runtime_tools.extend(multimodal_tools)
seen_names: set[str] = set()
deduplicated_tools: list[BaseTool] = []
for tool in runtime_tools:
if tool.name not in seen_names:
seen_names.add(tool.name)
deduplicated_tools.append(tool)
return deduplicated_tools
def kickoff(
self,
messages: str | list[LLMMessage],
@@ -1436,14 +1473,7 @@ class Agent(BaseAgent):
Returns:
LiteAgentOutput: The result of the agent execution.
"""
if self.apps:
platform_tools = self.get_platform_tools(self.apps)
if platform_tools:
self.tools.extend(platform_tools)
if self.mcps:
mcps = self.get_mcp_tools(self.mcps)
if mcps:
self.tools.extend(mcps)
runtime_tools = self._build_runtime_tools()
lite_agent = LiteAgent(
id=self.id,
@@ -1451,7 +1481,7 @@ class Agent(BaseAgent):
goal=self.goal,
backstory=self.backstory,
llm=self.llm,
tools=self.tools or [],
tools=runtime_tools,
max_iterations=self.max_iter,
max_execution_time=self.max_execution_time,
respect_context_window=self.respect_context_window,
@@ -1484,12 +1514,15 @@ class Agent(BaseAgent):
Returns:
LiteAgentOutput: The result of the agent execution.
"""
runtime_tools = self._build_runtime_tools()
lite_agent = LiteAgent(
id=self.id,
role=self.role,
goal=self.goal,
backstory=self.backstory,
llm=self.llm,
tools=self.tools or [],
tools=runtime_tools,
max_iterations=self.max_iter,
max_execution_time=self.max_execution_time,
respect_context_window=self.respect_context_window,

View File

@@ -0,0 +1,296 @@
"""Test Agent multimodal kickoff functionality."""
from unittest.mock import MagicMock, patch
import pytest
from crewai import Agent
from crewai.lite_agent import LiteAgent
from crewai.lite_agent_output import LiteAgentOutput
from crewai.tools.agent_tools.add_image_tool import AddImageTool
from crewai.tools.base_tool import BaseTool
@pytest.fixture
def mock_lite_agent():
"""Fixture to mock LiteAgent to avoid LLM calls."""
with patch("crewai.agent.core.LiteAgent") as mock_lite_agent_class:
mock_instance = MagicMock(spec=LiteAgent)
mock_output = LiteAgentOutput(
raw="Test output",
pydantic=None,
agent_role="test role",
usage_metrics=None,
messages=[],
)
mock_instance.kickoff.return_value = mock_output
mock_instance.kickoff_async.return_value = mock_output
mock_lite_agent_class.return_value = mock_instance
yield mock_lite_agent_class
def test_agent_kickoff_with_multimodal_true_adds_image_tool(mock_lite_agent):
"""Test that when multimodal=True, AddImageTool is added to the tools."""
agent = Agent(
role="test role",
goal="test goal",
backstory="test backstory",
multimodal=True,
)
agent.kickoff("Test message")
mock_lite_agent.assert_called_once()
call_kwargs = mock_lite_agent.call_args[1]
tools = call_kwargs["tools"]
assert any(isinstance(tool, AddImageTool) for tool in tools)
def test_agent_kickoff_with_multimodal_false_does_not_add_image_tool(mock_lite_agent):
"""Test that when multimodal=False, AddImageTool is not added to the tools."""
agent = Agent(
role="test role",
goal="test goal",
backstory="test backstory",
multimodal=False,
)
agent.kickoff("Test message")
mock_lite_agent.assert_called_once()
call_kwargs = mock_lite_agent.call_args[1]
tools = call_kwargs["tools"]
assert not any(isinstance(tool, AddImageTool) for tool in tools)
def test_agent_kickoff_does_not_mutate_self_tools(mock_lite_agent):
"""Test that calling kickoff does not mutate self.tools."""
class DummyTool(BaseTool):
name: str = "dummy_tool"
description: str = "A dummy tool"
def _run(self, **kwargs):
return "dummy result"
dummy_tool = DummyTool()
agent = Agent(
role="test role",
goal="test goal",
backstory="test backstory",
tools=[dummy_tool],
multimodal=True,
)
original_tools_count = len(agent.tools)
original_tools = list(agent.tools)
agent.kickoff("Test message")
assert len(agent.tools) == original_tools_count
assert agent.tools == original_tools
def test_agent_kickoff_multiple_calls_does_not_duplicate_tools(mock_lite_agent):
"""Test that calling kickoff multiple times does not duplicate tools."""
agent = Agent(
role="test role",
goal="test goal",
backstory="test backstory",
multimodal=True,
)
agent.kickoff("Test message 1")
first_call_tools = mock_lite_agent.call_args[1]["tools"]
first_call_image_tools = [
tool for tool in first_call_tools if isinstance(tool, AddImageTool)
]
agent.kickoff("Test message 2")
second_call_tools = mock_lite_agent.call_args[1]["tools"]
second_call_image_tools = [
tool for tool in second_call_tools if isinstance(tool, AddImageTool)
]
assert len(first_call_image_tools) == 1
assert len(second_call_image_tools) == 1
def test_agent_kickoff_async_with_multimodal_true_adds_image_tool(mock_lite_agent):
"""Test that when multimodal=True, AddImageTool is added in kickoff_async."""
agent = Agent(
role="test role",
goal="test goal",
backstory="test backstory",
multimodal=True,
)
import asyncio
asyncio.run(agent.kickoff_async("Test message"))
mock_lite_agent.assert_called_once()
call_kwargs = mock_lite_agent.call_args[1]
tools = call_kwargs["tools"]
assert any(isinstance(tool, AddImageTool) for tool in tools)
def test_agent_kickoff_async_with_multimodal_false_does_not_add_image_tool(
mock_lite_agent,
):
"""Test that when multimodal=False, AddImageTool is not added in kickoff_async."""
agent = Agent(
role="test role",
goal="test goal",
backstory="test backstory",
multimodal=False,
)
import asyncio
asyncio.run(agent.kickoff_async("Test message"))
mock_lite_agent.assert_called_once()
call_kwargs = mock_lite_agent.call_args[1]
tools = call_kwargs["tools"]
assert not any(isinstance(tool, AddImageTool) for tool in tools)
def test_agent_kickoff_async_does_not_mutate_self_tools(mock_lite_agent):
"""Test that calling kickoff_async does not mutate self.tools."""
class DummyTool(BaseTool):
name: str = "dummy_tool"
description: str = "A dummy tool"
def _run(self, **kwargs):
return "dummy result"
dummy_tool = DummyTool()
agent = Agent(
role="test role",
goal="test goal",
backstory="test backstory",
tools=[dummy_tool],
multimodal=True,
)
original_tools_count = len(agent.tools)
original_tools = list(agent.tools)
import asyncio
asyncio.run(agent.kickoff_async("Test message"))
assert len(agent.tools) == original_tools_count
assert agent.tools == original_tools
def test_agent_kickoff_with_existing_tools_and_multimodal(mock_lite_agent):
"""Test that multimodal tools are added alongside existing tools."""
class DummyTool(BaseTool):
name: str = "dummy_tool"
description: str = "A dummy tool"
def _run(self, **kwargs):
return "dummy result"
dummy_tool = DummyTool()
agent = Agent(
role="test role",
goal="test goal",
backstory="test backstory",
tools=[dummy_tool],
multimodal=True,
)
agent.kickoff("Test message")
mock_lite_agent.assert_called_once()
call_kwargs = mock_lite_agent.call_args[1]
tools = call_kwargs["tools"]
assert any(isinstance(tool, DummyTool) for tool in tools)
assert any(isinstance(tool, AddImageTool) for tool in tools)
assert len(tools) == 2
def test_agent_kickoff_deduplicates_tools_by_name(mock_lite_agent):
"""Test that tools with the same name are deduplicated."""
class DummyTool(BaseTool):
name: str = "dummy_tool"
description: str = "A dummy tool"
def _run(self, **kwargs):
return "dummy result"
dummy_tool1 = DummyTool()
dummy_tool2 = DummyTool()
agent = Agent(
role="test role",
goal="test goal",
backstory="test backstory",
tools=[dummy_tool1, dummy_tool2],
multimodal=False,
)
agent.kickoff("Test message")
mock_lite_agent.assert_called_once()
call_kwargs = mock_lite_agent.call_args[1]
tools = call_kwargs["tools"]
dummy_tools = [tool for tool in tools if isinstance(tool, DummyTool)]
assert len(dummy_tools) == 1
def test_agent_kickoff_async_includes_platform_and_mcp_tools(mock_lite_agent):
"""Test that kickoff_async includes platform and MCP tools like kickoff does."""
with patch.object(Agent, "get_platform_tools") as mock_platform_tools, patch.object(
Agent, "get_mcp_tools"
) as mock_mcp_tools:
class PlatformTool(BaseTool):
name: str = "platform_tool"
description: str = "A platform tool"
def _run(self, **kwargs):
return "platform result"
class MCPTool(BaseTool):
name: str = "mcp_tool"
description: str = "An MCP tool"
def _run(self, **kwargs):
return "mcp result"
mock_platform_tools.return_value = [PlatformTool()]
mock_mcp_tools.return_value = [MCPTool()]
agent = Agent(
role="test role",
goal="test goal",
backstory="test backstory",
apps=["test_app"],
mcps=["test_mcp"],
multimodal=True,
)
import asyncio
asyncio.run(agent.kickoff_async("Test message"))
mock_lite_agent.assert_called_once()
call_kwargs = mock_lite_agent.call_args[1]
tools = call_kwargs["tools"]
assert any(isinstance(tool, PlatformTool) for tool in tools)
assert any(isinstance(tool, MCPTool) for tool in tools)
assert any(isinstance(tool, AddImageTool) for tool in tools)