mirror of
https://github.com/crewAIInc/crewAI.git
synced 2026-09-24 11:50:16 +00:00
fix(llm): collapse multimodal content with the shared helper (#7527)
For response_model calls the messages are flattened into one prompt for InternalInstructor with an f-string, so a multimodal content list reached the model as its Python repr. AGENTS.md's "Message Content" section says never to str() the content; use message_content_text() instead.
This commit is contained in:
@@ -1263,6 +1263,7 @@ class LLM(BaseLLM):
|
||||
"""
|
||||
if response_model and self.is_litellm:
|
||||
from crewai.hooks.llm_hooks import model_call_hooks_dispatched
|
||||
from crewai.utilities.agent_utils import message_content_text
|
||||
from crewai.utilities.internal_instructor import InternalInstructor
|
||||
|
||||
messages = params.get("messages", [])
|
||||
@@ -1270,7 +1271,8 @@ class LLM(BaseLLM):
|
||||
raise ValueError("Messages are required when using response_model")
|
||||
|
||||
combined_content = "\n\n".join(
|
||||
f"{msg['role'].upper()}: {msg['content']}" for msg in messages
|
||||
f"{msg['role'].upper()}: {message_content_text(msg)}"
|
||||
for msg in messages
|
||||
)
|
||||
|
||||
instructor_instance = InternalInstructor(
|
||||
@@ -1420,6 +1422,7 @@ class LLM(BaseLLM):
|
||||
"""
|
||||
if response_model and self.is_litellm:
|
||||
from crewai.hooks.llm_hooks import model_call_hooks_dispatched
|
||||
from crewai.utilities.agent_utils import message_content_text
|
||||
from crewai.utilities.internal_instructor import InternalInstructor
|
||||
|
||||
messages = params.get("messages", [])
|
||||
@@ -1427,7 +1430,8 @@ class LLM(BaseLLM):
|
||||
raise ValueError("Messages are required when using response_model")
|
||||
|
||||
combined_content = "\n\n".join(
|
||||
f"{msg['role'].upper()}: {msg['content']}" for msg in messages
|
||||
f"{msg['role'].upper()}: {message_content_text(msg)}"
|
||||
for msg in messages
|
||||
)
|
||||
|
||||
instructor_instance = InternalInstructor(
|
||||
|
||||
62
lib/crewai/tests/llms/test_structured_output_multimodal.py
Normal file
62
lib/crewai/tests/llms/test_structured_output_multimodal.py
Normal file
@@ -0,0 +1,62 @@
|
||||
"""Structured output must collapse multimodal content with the shared helper."""
|
||||
|
||||
from unittest.mock import patch
|
||||
|
||||
import pytest
|
||||
from pydantic import BaseModel
|
||||
|
||||
from crewai.llm import LLM
|
||||
|
||||
|
||||
class Answer(BaseModel):
|
||||
text: str
|
||||
|
||||
|
||||
MULTIMODAL = [
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "text", "text": "What is in this image?"},
|
||||
{"type": "image_url", "image_url": {"url": "http://example.com/x.png"}},
|
||||
],
|
||||
}
|
||||
]
|
||||
|
||||
|
||||
def _stub_llm():
|
||||
"""An LLM with the instructor call captured, no network involved."""
|
||||
llm = object.__new__(LLM)
|
||||
llm.is_litellm = True
|
||||
llm._handle_emit_call_events = lambda **kwargs: None
|
||||
captured: dict[str, str] = {}
|
||||
|
||||
class FakeInstructor:
|
||||
def __init__(self, *, content, model, llm):
|
||||
captured["content"] = content
|
||||
|
||||
def to_pydantic(self):
|
||||
return Answer(text="ok")
|
||||
|
||||
patcher = patch(
|
||||
"crewai.utilities.internal_instructor.InternalInstructor", FakeInstructor
|
||||
)
|
||||
return llm, captured, patcher
|
||||
|
||||
|
||||
def test_sync_structured_output_collapses_multimodal_content():
|
||||
llm, captured, patcher = _stub_llm()
|
||||
with patcher:
|
||||
llm._handle_non_streaming_response(
|
||||
params={"messages": MULTIMODAL}, response_model=Answer
|
||||
)
|
||||
assert captured["content"] == "USER: What is in this image?"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_async_structured_output_collapses_multimodal_content():
|
||||
llm, captured, patcher = _stub_llm()
|
||||
with patcher:
|
||||
await llm._ahandle_non_streaming_response(
|
||||
params={"messages": MULTIMODAL}, response_model=Answer
|
||||
)
|
||||
assert captured["content"] == "USER: What is in this image?"
|
||||
Reference in New Issue
Block a user