fix: support multimodal content in Bedrock message formatting

- Add format_text_content override for Bedrock's {"text": ...} format
- Handle pre-formatted list content in _format_messages_for_converse
- Update Bedrock tests to use Claude 3 Haiku for on-demand availability
- Add VCR cassettes for Bedrock multimodal tests
This commit is contained in:
Greyson LaLonde
2026-01-22 22:27:58 -05:00
parent 83bab3531b
commit 19d6a47d0c
4 changed files with 112 additions and 7 deletions

View File

@@ -1360,11 +1360,15 @@ class BedrockCompletion(BaseLLM):
)
else:
# Convert to Converse API format with proper content structure
# Ensure content is not None
text_content = content if content else ""
converse_messages.append(
{"role": role, "content": [{"text": text_content}]}
)
if isinstance(content, list):
# Already formatted as multimodal content blocks
converse_messages.append({"role": role, "content": content})
else:
# String content - wrap in text block
text_content = content if content else ""
converse_messages.append(
{"role": role, "content": [{"text": text_content}]}
)
# CRITICAL: Handle model-specific conversation requirements
# Cohere and some other models require conversation to end with user message
@@ -1703,3 +1707,16 @@ class BedrockCompletion(BaseLLM):
"video/3gpp": "three_gp",
}
return format_map.get(content_type)
def format_text_content(self, text: str) -> dict[str, Any]:
"""Format text as a Bedrock content block.
Bedrock uses {"text": "..."} format instead of {"type": "text", "text": "..."}.
Args:
text: The text content to format.
Returns:
A content block in Bedrock's expected format.
"""
return {"text": text}

View File

@@ -0,0 +1,43 @@
interactions:
- request:
body: '{"messages": [{"role": "user", "content": [{"text": "What type of document
is this? Answer in one word."}, {"document": {"name": "document", "format":
"pdf", "source": {"bytes": "JVBERi0xLjQKMSAwIG9iaiA8PCAvVHlwZSAvQ2F0YWxvZyAvUGFnZXMgMiAwIFIgPj4gZW5kb2JqCjIgMCBvYmogPDwgL1R5cGUgL1BhZ2VzIC9LaWRzIFszIDAgUl0gL0NvdW50IDEgPj4gZW5kb2JqCjMgMCBvYmogPDwgL1R5cGUgL1BhZ2UgL1BhcmVudCAyIDAgUiAvTWVkaWFCb3ggWzAgMCA2MTIgNzkyXSA+PiBlbmRvYmoKeHJlZgowIDQKMDAwMDAwMDAwMCA2NTUzNSBmCjAwMDAwMDAwMDkgMDAwMDAgbgowMDAwMDAwMDU4IDAwMDAwIG4KMDAwMDAwMDExNSAwMDAwMCBuCnRyYWlsZXIgPDwgL1NpemUgNCAvUm9vdCAxIDAgUiA+PgpzdGFydHhyZWYKMTk2CiUlRU9GCg=="}}}]}],
"inferenceConfig": {}}'
headers:
Content-Length:
- '646'
Content-Type:
- !!binary |
YXBwbGljYXRpb24vanNvbg==
User-Agent:
- X-USER-AGENT-XXX
amz-sdk-invocation-id:
- AMZ-SDK-INVOCATION-ID-XXX
amz-sdk-request:
- !!binary |
YXR0ZW1wdD0x
authorization:
- AUTHORIZATION-XXX
x-amz-date:
- X-AMZ-DATE-XXX
method: POST
uri: https://bedrock-runtime.us-west-2.amazonaws.com/model/anthropic.claude-3-haiku-20240307-v1%3A0/converse
response:
body:
string: '{"metrics":{"latencyMs":867},"output":{"message":{"content":[{"text":"PDF"}],"role":"assistant"}},"stopReason":"end_turn","usage":{"inputTokens":57,"outputTokens":4,"serverToolUsage":{},"totalTokens":61}}'
headers:
Connection:
- keep-alive
Content-Length:
- '204'
Content-Type:
- application/json
Date:
- Fri, 23 Jan 2026 03:26:35 GMT
x-amzn-RequestId:
- X-AMZN-REQUESTID-XXX
status:
code: 200
message: OK
version: 1

File diff suppressed because one or more lines are too long

View File

@@ -153,7 +153,7 @@ class TestBedrockMultimodalIntegration:
@pytest.mark.vcr()
def test_describe_image(self, test_image_bytes: bytes) -> None:
"""Test Bedrock Claude can describe an image."""
llm = LLM(model="bedrock/anthropic.claude-3-5-sonnet-20241022-v2:0")
llm = LLM(model="bedrock/anthropic.claude-3-haiku-20240307-v1:0")
files = {"image": ImageFile(source=test_image_bytes)}
messages = _build_multimodal_message(
@@ -171,7 +171,7 @@ class TestBedrockMultimodalIntegration:
@pytest.mark.vcr()
def test_analyze_pdf(self) -> None:
"""Test Bedrock Claude can analyze a PDF."""
llm = LLM(model="bedrock/anthropic.claude-3-5-sonnet-20241022-v2:0")
llm = LLM(model="bedrock/anthropic.claude-3-haiku-20240307-v1:0")
files = {"document": PDFFile(source=MINIMAL_PDF)}
messages = _build_multimodal_message(