mirror of
https://github.com/crewAIInc/crewAI.git
synced 2025-12-16 04:18:35 +00:00
Fix: Add multimodal tools support in Agent.kickoff() and kickoff_async()
- Add _build_runtime_tools() helper method to combine tools from multiple sources - Fix issue #3936 where multimodal=True was not adding AddImageTool in kickoff() - Ensure kickoff_async() includes platform, MCP, and multimodal tools for consistency - Prevent tool duplication by deduplicating tools by name - Avoid mutating self.tools to prevent side effects across multiple kickoff calls - Add comprehensive tests for multimodal tool handling in both sync and async kickoff methods Co-Authored-By: João <joao@crewai.com>
This commit is contained in:
@@ -1416,6 +1416,43 @@ class Agent(BaseAgent):
|
||||
)
|
||||
return None
|
||||
|
||||
def _build_runtime_tools(self) -> list[BaseTool]:
|
||||
"""Build a list of tools for runtime execution without mutating self.tools.
|
||||
|
||||
This method combines tools from multiple sources:
|
||||
- Agent's configured tools (self.tools)
|
||||
- Platform tools (if self.apps is set)
|
||||
- MCP tools (if self.mcps is set)
|
||||
- Multimodal tools (if self.multimodal is True)
|
||||
|
||||
Returns:
|
||||
A deduplicated list of tools ready for execution.
|
||||
"""
|
||||
runtime_tools: list[BaseTool] = list(self.tools or [])
|
||||
|
||||
if self.apps:
|
||||
platform_tools = self.get_platform_tools(self.apps)
|
||||
if platform_tools:
|
||||
runtime_tools.extend(platform_tools)
|
||||
|
||||
if self.mcps:
|
||||
mcp_tools = self.get_mcp_tools(self.mcps)
|
||||
if mcp_tools:
|
||||
runtime_tools.extend(mcp_tools)
|
||||
|
||||
if self.multimodal:
|
||||
multimodal_tools = self.get_multimodal_tools()
|
||||
runtime_tools.extend(multimodal_tools)
|
||||
|
||||
seen_names: set[str] = set()
|
||||
deduplicated_tools: list[BaseTool] = []
|
||||
for tool in runtime_tools:
|
||||
if tool.name not in seen_names:
|
||||
seen_names.add(tool.name)
|
||||
deduplicated_tools.append(tool)
|
||||
|
||||
return deduplicated_tools
|
||||
|
||||
def kickoff(
|
||||
self,
|
||||
messages: str | list[LLMMessage],
|
||||
@@ -1436,14 +1473,7 @@ class Agent(BaseAgent):
|
||||
Returns:
|
||||
LiteAgentOutput: The result of the agent execution.
|
||||
"""
|
||||
if self.apps:
|
||||
platform_tools = self.get_platform_tools(self.apps)
|
||||
if platform_tools:
|
||||
self.tools.extend(platform_tools)
|
||||
if self.mcps:
|
||||
mcps = self.get_mcp_tools(self.mcps)
|
||||
if mcps:
|
||||
self.tools.extend(mcps)
|
||||
runtime_tools = self._build_runtime_tools()
|
||||
|
||||
lite_agent = LiteAgent(
|
||||
id=self.id,
|
||||
@@ -1451,7 +1481,7 @@ class Agent(BaseAgent):
|
||||
goal=self.goal,
|
||||
backstory=self.backstory,
|
||||
llm=self.llm,
|
||||
tools=self.tools or [],
|
||||
tools=runtime_tools,
|
||||
max_iterations=self.max_iter,
|
||||
max_execution_time=self.max_execution_time,
|
||||
respect_context_window=self.respect_context_window,
|
||||
@@ -1484,12 +1514,15 @@ class Agent(BaseAgent):
|
||||
Returns:
|
||||
LiteAgentOutput: The result of the agent execution.
|
||||
"""
|
||||
runtime_tools = self._build_runtime_tools()
|
||||
|
||||
lite_agent = LiteAgent(
|
||||
id=self.id,
|
||||
role=self.role,
|
||||
goal=self.goal,
|
||||
backstory=self.backstory,
|
||||
llm=self.llm,
|
||||
tools=self.tools or [],
|
||||
tools=runtime_tools,
|
||||
max_iterations=self.max_iter,
|
||||
max_execution_time=self.max_execution_time,
|
||||
respect_context_window=self.respect_context_window,
|
||||
|
||||
296
lib/crewai/tests/agents/test_agent_multimodal_kickoff.py
Normal file
296
lib/crewai/tests/agents/test_agent_multimodal_kickoff.py
Normal file
@@ -0,0 +1,296 @@
|
||||
"""Test Agent multimodal kickoff functionality."""
|
||||
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
|
||||
from crewai import Agent
|
||||
from crewai.lite_agent import LiteAgent
|
||||
from crewai.lite_agent_output import LiteAgentOutput
|
||||
from crewai.tools.agent_tools.add_image_tool import AddImageTool
|
||||
from crewai.tools.base_tool import BaseTool
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mock_lite_agent():
|
||||
"""Fixture to mock LiteAgent to avoid LLM calls."""
|
||||
with patch("crewai.agent.core.LiteAgent") as mock_lite_agent_class:
|
||||
mock_instance = MagicMock(spec=LiteAgent)
|
||||
mock_output = LiteAgentOutput(
|
||||
raw="Test output",
|
||||
pydantic=None,
|
||||
agent_role="test role",
|
||||
usage_metrics=None,
|
||||
messages=[],
|
||||
)
|
||||
mock_instance.kickoff.return_value = mock_output
|
||||
mock_instance.kickoff_async.return_value = mock_output
|
||||
mock_lite_agent_class.return_value = mock_instance
|
||||
yield mock_lite_agent_class
|
||||
|
||||
|
||||
def test_agent_kickoff_with_multimodal_true_adds_image_tool(mock_lite_agent):
|
||||
"""Test that when multimodal=True, AddImageTool is added to the tools."""
|
||||
agent = Agent(
|
||||
role="test role",
|
||||
goal="test goal",
|
||||
backstory="test backstory",
|
||||
multimodal=True,
|
||||
)
|
||||
|
||||
agent.kickoff("Test message")
|
||||
|
||||
mock_lite_agent.assert_called_once()
|
||||
call_kwargs = mock_lite_agent.call_args[1]
|
||||
tools = call_kwargs["tools"]
|
||||
|
||||
assert any(isinstance(tool, AddImageTool) for tool in tools)
|
||||
|
||||
|
||||
def test_agent_kickoff_with_multimodal_false_does_not_add_image_tool(mock_lite_agent):
|
||||
"""Test that when multimodal=False, AddImageTool is not added to the tools."""
|
||||
agent = Agent(
|
||||
role="test role",
|
||||
goal="test goal",
|
||||
backstory="test backstory",
|
||||
multimodal=False,
|
||||
)
|
||||
|
||||
agent.kickoff("Test message")
|
||||
|
||||
mock_lite_agent.assert_called_once()
|
||||
call_kwargs = mock_lite_agent.call_args[1]
|
||||
tools = call_kwargs["tools"]
|
||||
|
||||
assert not any(isinstance(tool, AddImageTool) for tool in tools)
|
||||
|
||||
|
||||
def test_agent_kickoff_does_not_mutate_self_tools(mock_lite_agent):
|
||||
"""Test that calling kickoff does not mutate self.tools."""
|
||||
|
||||
class DummyTool(BaseTool):
|
||||
name: str = "dummy_tool"
|
||||
description: str = "A dummy tool"
|
||||
|
||||
def _run(self, **kwargs):
|
||||
return "dummy result"
|
||||
|
||||
dummy_tool = DummyTool()
|
||||
agent = Agent(
|
||||
role="test role",
|
||||
goal="test goal",
|
||||
backstory="test backstory",
|
||||
tools=[dummy_tool],
|
||||
multimodal=True,
|
||||
)
|
||||
|
||||
original_tools_count = len(agent.tools)
|
||||
original_tools = list(agent.tools)
|
||||
|
||||
agent.kickoff("Test message")
|
||||
|
||||
assert len(agent.tools) == original_tools_count
|
||||
assert agent.tools == original_tools
|
||||
|
||||
|
||||
def test_agent_kickoff_multiple_calls_does_not_duplicate_tools(mock_lite_agent):
|
||||
"""Test that calling kickoff multiple times does not duplicate tools."""
|
||||
agent = Agent(
|
||||
role="test role",
|
||||
goal="test goal",
|
||||
backstory="test backstory",
|
||||
multimodal=True,
|
||||
)
|
||||
|
||||
agent.kickoff("Test message 1")
|
||||
first_call_tools = mock_lite_agent.call_args[1]["tools"]
|
||||
first_call_image_tools = [
|
||||
tool for tool in first_call_tools if isinstance(tool, AddImageTool)
|
||||
]
|
||||
|
||||
agent.kickoff("Test message 2")
|
||||
second_call_tools = mock_lite_agent.call_args[1]["tools"]
|
||||
second_call_image_tools = [
|
||||
tool for tool in second_call_tools if isinstance(tool, AddImageTool)
|
||||
]
|
||||
|
||||
assert len(first_call_image_tools) == 1
|
||||
assert len(second_call_image_tools) == 1
|
||||
|
||||
|
||||
def test_agent_kickoff_async_with_multimodal_true_adds_image_tool(mock_lite_agent):
|
||||
"""Test that when multimodal=True, AddImageTool is added in kickoff_async."""
|
||||
agent = Agent(
|
||||
role="test role",
|
||||
goal="test goal",
|
||||
backstory="test backstory",
|
||||
multimodal=True,
|
||||
)
|
||||
|
||||
import asyncio
|
||||
|
||||
asyncio.run(agent.kickoff_async("Test message"))
|
||||
|
||||
mock_lite_agent.assert_called_once()
|
||||
call_kwargs = mock_lite_agent.call_args[1]
|
||||
tools = call_kwargs["tools"]
|
||||
|
||||
assert any(isinstance(tool, AddImageTool) for tool in tools)
|
||||
|
||||
|
||||
def test_agent_kickoff_async_with_multimodal_false_does_not_add_image_tool(
|
||||
mock_lite_agent,
|
||||
):
|
||||
"""Test that when multimodal=False, AddImageTool is not added in kickoff_async."""
|
||||
agent = Agent(
|
||||
role="test role",
|
||||
goal="test goal",
|
||||
backstory="test backstory",
|
||||
multimodal=False,
|
||||
)
|
||||
|
||||
import asyncio
|
||||
|
||||
asyncio.run(agent.kickoff_async("Test message"))
|
||||
|
||||
mock_lite_agent.assert_called_once()
|
||||
call_kwargs = mock_lite_agent.call_args[1]
|
||||
tools = call_kwargs["tools"]
|
||||
|
||||
assert not any(isinstance(tool, AddImageTool) for tool in tools)
|
||||
|
||||
|
||||
def test_agent_kickoff_async_does_not_mutate_self_tools(mock_lite_agent):
|
||||
"""Test that calling kickoff_async does not mutate self.tools."""
|
||||
|
||||
class DummyTool(BaseTool):
|
||||
name: str = "dummy_tool"
|
||||
description: str = "A dummy tool"
|
||||
|
||||
def _run(self, **kwargs):
|
||||
return "dummy result"
|
||||
|
||||
dummy_tool = DummyTool()
|
||||
agent = Agent(
|
||||
role="test role",
|
||||
goal="test goal",
|
||||
backstory="test backstory",
|
||||
tools=[dummy_tool],
|
||||
multimodal=True,
|
||||
)
|
||||
|
||||
original_tools_count = len(agent.tools)
|
||||
original_tools = list(agent.tools)
|
||||
|
||||
import asyncio
|
||||
|
||||
asyncio.run(agent.kickoff_async("Test message"))
|
||||
|
||||
assert len(agent.tools) == original_tools_count
|
||||
assert agent.tools == original_tools
|
||||
|
||||
|
||||
def test_agent_kickoff_with_existing_tools_and_multimodal(mock_lite_agent):
|
||||
"""Test that multimodal tools are added alongside existing tools."""
|
||||
|
||||
class DummyTool(BaseTool):
|
||||
name: str = "dummy_tool"
|
||||
description: str = "A dummy tool"
|
||||
|
||||
def _run(self, **kwargs):
|
||||
return "dummy result"
|
||||
|
||||
dummy_tool = DummyTool()
|
||||
agent = Agent(
|
||||
role="test role",
|
||||
goal="test goal",
|
||||
backstory="test backstory",
|
||||
tools=[dummy_tool],
|
||||
multimodal=True,
|
||||
)
|
||||
|
||||
agent.kickoff("Test message")
|
||||
|
||||
mock_lite_agent.assert_called_once()
|
||||
call_kwargs = mock_lite_agent.call_args[1]
|
||||
tools = call_kwargs["tools"]
|
||||
|
||||
assert any(isinstance(tool, DummyTool) for tool in tools)
|
||||
assert any(isinstance(tool, AddImageTool) for tool in tools)
|
||||
assert len(tools) == 2
|
||||
|
||||
|
||||
def test_agent_kickoff_deduplicates_tools_by_name(mock_lite_agent):
|
||||
"""Test that tools with the same name are deduplicated."""
|
||||
|
||||
class DummyTool(BaseTool):
|
||||
name: str = "dummy_tool"
|
||||
description: str = "A dummy tool"
|
||||
|
||||
def _run(self, **kwargs):
|
||||
return "dummy result"
|
||||
|
||||
dummy_tool1 = DummyTool()
|
||||
dummy_tool2 = DummyTool()
|
||||
|
||||
agent = Agent(
|
||||
role="test role",
|
||||
goal="test goal",
|
||||
backstory="test backstory",
|
||||
tools=[dummy_tool1, dummy_tool2],
|
||||
multimodal=False,
|
||||
)
|
||||
|
||||
agent.kickoff("Test message")
|
||||
|
||||
mock_lite_agent.assert_called_once()
|
||||
call_kwargs = mock_lite_agent.call_args[1]
|
||||
tools = call_kwargs["tools"]
|
||||
|
||||
dummy_tools = [tool for tool in tools if isinstance(tool, DummyTool)]
|
||||
assert len(dummy_tools) == 1
|
||||
|
||||
|
||||
def test_agent_kickoff_async_includes_platform_and_mcp_tools(mock_lite_agent):
|
||||
"""Test that kickoff_async includes platform and MCP tools like kickoff does."""
|
||||
with patch.object(Agent, "get_platform_tools") as mock_platform_tools, patch.object(
|
||||
Agent, "get_mcp_tools"
|
||||
) as mock_mcp_tools:
|
||||
|
||||
class PlatformTool(BaseTool):
|
||||
name: str = "platform_tool"
|
||||
description: str = "A platform tool"
|
||||
|
||||
def _run(self, **kwargs):
|
||||
return "platform result"
|
||||
|
||||
class MCPTool(BaseTool):
|
||||
name: str = "mcp_tool"
|
||||
description: str = "An MCP tool"
|
||||
|
||||
def _run(self, **kwargs):
|
||||
return "mcp result"
|
||||
|
||||
mock_platform_tools.return_value = [PlatformTool()]
|
||||
mock_mcp_tools.return_value = [MCPTool()]
|
||||
|
||||
agent = Agent(
|
||||
role="test role",
|
||||
goal="test goal",
|
||||
backstory="test backstory",
|
||||
apps=["test_app"],
|
||||
mcps=["test_mcp"],
|
||||
multimodal=True,
|
||||
)
|
||||
|
||||
import asyncio
|
||||
|
||||
asyncio.run(agent.kickoff_async("Test message"))
|
||||
|
||||
mock_lite_agent.assert_called_once()
|
||||
call_kwargs = mock_lite_agent.call_args[1]
|
||||
tools = call_kwargs["tools"]
|
||||
|
||||
assert any(isinstance(tool, PlatformTool) for tool in tools)
|
||||
assert any(isinstance(tool, MCPTool) for tool in tools)
|
||||
assert any(isinstance(tool, AddImageTool) for tool in tools)
|
||||
Reference in New Issue
Block a user