mirror of
https://github.com/crewAIInc/crewAI.git
synced 2026-09-21 18:36:47 +00:00
* fix(files): return None instead of bare raise in get_uploader get_uploader is documented to return None for an unsupported provider, and every caller branches on `if uploader is None`. Two fallthrough paths ran a bare `raise` with no active exception, so an unknown provider and a Bedrock provider without a configured S3 bucket raised "RuntimeError: No active exception to reraise" instead of returning None. Return None in both paths and widen the return types to `... | None`. The Bedrock "not configured" guard now treats a falsy bucket_name (None or "") as unconfigured, not only an absent one. The except ImportError re-raises are unaffected. Fixes #7282 * fix(files): raise ValueError from get_uploader for unknown/unconfigured providers Per review, raise a ValueError with a concrete reason instead of returning None. Returning None let the resolver silently fall back to inline and hid the misconfiguration from the user, so the docstring no longer promises None and the return types drop `| None`. The Bedrock guard also treats a falsy bucket_name (None or "") as unconfigured. The ImportError re-raises are unchanged. cleanup skips providers it cannot build an uploader for, so it routes get_uploader through a local helper that treats the ValueError as "unavailable" and continues the pass. * refactor(files): surface get_uploader errors through the resolver Follow-up to review. get_uploader now raises ValueError, so _get_uploader no longer promises FileUploader | None: it returns the uploader and lets the error propagate through resolve() to the caller instead of swallowing it and falling back to inline. Drop the now-dead `if uploader is None` checks at the two upload call sites. Also make the unknown-provider ValueError list the supported providers, and add a happy-path test that a configured provider returns its uploader. * fix(files): surface uploader lookup errors in async batch resolution aresolve_files gathers with return_exceptions=True, which was silently dropping files when _get_uploader raised (a missing provider SDK, or an unknown or unconfigured provider). A batch shares one provider, so such a lookup failure applies to every file: re-raise ValueError and ImportError to surface it, matching the sync resolve_files path. Genuine per-file upload errors are still logged and skipped. * fix(files): only re-raise uploader config errors in async batch resolution The earlier fix re-raised any ValueError or ImportError from asyncio.gather(return_exceptions=True), so one unrelated per-file error (for example a stream that raises ValueError when read) aborted the whole batch instead of the intended log-and-skip. _get_uploader now translates the lookup failure into a dedicated UploaderConfigurationError, and aresolve_files re-raises only that, since it applies to every file for the provider. Ordinary per-file failures stay best-effort. Adds regression tests for the wrap, a provider-setup error surfacing from the batch, and an unrelated per-file error skipped while the rest resolve. * style(files): apply ruff import sort and formatting to resolver tests --------- Co-authored-by: Vidit Ostwal <110953813+Vidit-Ostwal@users.noreply.github.com>
236 lines
8.1 KiB
Python
236 lines
8.1 KiB
Python
"""Tests for FileResolver."""
|
|
|
|
from crewai_files import FileBytes, ImageFile
|
|
from crewai_files.cache.upload_cache import UploadCache
|
|
from crewai_files.core.resolved import InlineBase64, InlineBytes
|
|
from crewai_files.processing.exceptions import UploaderConfigurationError
|
|
from crewai_files.resolution.resolver import (
|
|
FileResolver,
|
|
FileResolverConfig,
|
|
create_resolver,
|
|
)
|
|
import pytest
|
|
|
|
|
|
# Minimal valid PNG
|
|
MINIMAL_PNG = (
|
|
b"\x89PNG\r\n\x1a\n\x00\x00\x00\rIHDR\x00\x00\x00\x08\x00\x00\x00\x08"
|
|
b"\x01\x00\x00\x00\x00\xf9Y\xab\xcd\x00\x00\x00\nIDATx\x9cc`\x00\x00"
|
|
b"\x00\x02\x00\x01\xe2!\xbc3\x00\x00\x00\x00IEND\xaeB`\x82"
|
|
)
|
|
|
|
|
|
class TestFileResolverConfig:
|
|
"""Tests for FileResolverConfig."""
|
|
|
|
def test_default_config(self):
|
|
"""Test default configuration values."""
|
|
config = FileResolverConfig()
|
|
|
|
assert config.prefer_upload is False
|
|
assert config.upload_threshold_bytes is None
|
|
assert config.use_bytes_for_bedrock is True
|
|
|
|
def test_custom_config(self):
|
|
"""Test custom configuration values."""
|
|
config = FileResolverConfig(
|
|
prefer_upload=True,
|
|
upload_threshold_bytes=1024 * 1024,
|
|
use_bytes_for_bedrock=False,
|
|
)
|
|
|
|
assert config.prefer_upload is True
|
|
assert config.upload_threshold_bytes == 1024 * 1024
|
|
assert config.use_bytes_for_bedrock is False
|
|
|
|
|
|
class TestFileResolver:
|
|
"""Tests for FileResolver class."""
|
|
|
|
def test_resolve_inline_base64(self):
|
|
"""Test resolving file as inline base64."""
|
|
resolver = FileResolver()
|
|
file = ImageFile(source=FileBytes(data=MINIMAL_PNG, filename="test.png"))
|
|
|
|
resolved = resolver.resolve(file, "openai")
|
|
|
|
assert isinstance(resolved, InlineBase64)
|
|
assert resolved.content_type == "image/png"
|
|
assert len(resolved.data) > 0
|
|
|
|
def test_resolve_inline_bytes_for_bedrock(self):
|
|
"""Test resolving file as inline bytes for Bedrock."""
|
|
config = FileResolverConfig(use_bytes_for_bedrock=True)
|
|
resolver = FileResolver(config=config)
|
|
file = ImageFile(source=FileBytes(data=MINIMAL_PNG, filename="test.png"))
|
|
|
|
resolved = resolver.resolve(file, "bedrock")
|
|
|
|
assert isinstance(resolved, InlineBytes)
|
|
assert resolved.content_type == "image/png"
|
|
assert resolved.data == MINIMAL_PNG
|
|
|
|
def test_resolve_files_multiple(self):
|
|
"""Test resolving multiple files."""
|
|
resolver = FileResolver()
|
|
files = {
|
|
"image1": ImageFile(
|
|
source=FileBytes(data=MINIMAL_PNG, filename="test1.png")
|
|
),
|
|
"image2": ImageFile(
|
|
source=FileBytes(data=MINIMAL_PNG, filename="test2.png")
|
|
),
|
|
}
|
|
|
|
resolved = resolver.resolve_files(files, "openai")
|
|
|
|
assert len(resolved) == 2
|
|
assert "image1" in resolved
|
|
assert "image2" in resolved
|
|
assert all(isinstance(r, InlineBase64) for r in resolved.values())
|
|
|
|
def test_resolve_with_cache(self):
|
|
"""Test resolver uses cache."""
|
|
cache = UploadCache()
|
|
resolver = FileResolver(upload_cache=cache)
|
|
file = ImageFile(source=FileBytes(data=MINIMAL_PNG, filename="test.png"))
|
|
|
|
resolved1 = resolver.resolve(file, "openai")
|
|
resolved2 = resolver.resolve(file, "openai")
|
|
|
|
assert isinstance(resolved1, InlineBase64)
|
|
assert isinstance(resolved2, InlineBase64)
|
|
assert resolved1.data == resolved2.data
|
|
|
|
def test_clear_cache(self):
|
|
"""Test clearing resolver cache."""
|
|
cache = UploadCache()
|
|
file = ImageFile(source=FileBytes(data=MINIMAL_PNG, filename="test.png"))
|
|
|
|
cache.set(file=file, provider="gemini", file_id="test")
|
|
|
|
resolver = FileResolver(upload_cache=cache)
|
|
resolver.clear_cache()
|
|
|
|
assert len(cache) == 0
|
|
|
|
def test_get_cached_uploads(self):
|
|
"""Test getting cached uploads from resolver."""
|
|
cache = UploadCache()
|
|
file = ImageFile(source=FileBytes(data=MINIMAL_PNG, filename="test.png"))
|
|
|
|
cache.set(file=file, provider="gemini", file_id="test-1")
|
|
cache.set(file=file, provider="anthropic", file_id="test-2")
|
|
|
|
resolver = FileResolver(upload_cache=cache)
|
|
|
|
gemini_uploads = resolver.get_cached_uploads("gemini")
|
|
anthropic_uploads = resolver.get_cached_uploads("anthropic")
|
|
|
|
assert len(gemini_uploads) == 1
|
|
assert len(anthropic_uploads) == 1
|
|
|
|
def test_get_cached_uploads_empty(self):
|
|
"""Test getting cached uploads when no cache."""
|
|
resolver = FileResolver() # No cache
|
|
|
|
uploads = resolver.get_cached_uploads("gemini")
|
|
|
|
assert uploads == []
|
|
|
|
|
|
class TestCreateResolver:
|
|
"""Tests for create_resolver factory function."""
|
|
|
|
def test_create_default_resolver(self):
|
|
"""Test creating resolver with default settings."""
|
|
resolver = create_resolver()
|
|
|
|
assert resolver.config.prefer_upload is False
|
|
assert resolver.upload_cache is not None
|
|
|
|
def test_create_resolver_with_options(self):
|
|
"""Test creating resolver with custom options."""
|
|
resolver = create_resolver(
|
|
prefer_upload=True,
|
|
upload_threshold_bytes=5 * 1024 * 1024,
|
|
enable_cache=False,
|
|
)
|
|
|
|
assert resolver.config.prefer_upload is True
|
|
assert resolver.config.upload_threshold_bytes == 5 * 1024 * 1024
|
|
assert resolver.upload_cache is None
|
|
|
|
def test_create_resolver_cache_enabled(self):
|
|
"""Test resolver has cache when enabled."""
|
|
resolver = create_resolver(enable_cache=True)
|
|
|
|
assert resolver.upload_cache is not None
|
|
|
|
def test_create_resolver_cache_disabled(self):
|
|
"""Test resolver has no cache when disabled."""
|
|
resolver = create_resolver(enable_cache=False)
|
|
|
|
assert resolver.upload_cache is None
|
|
|
|
|
|
class _NoUploaderResolver(FileResolver):
|
|
"""Resolver whose provider has no usable uploader, so every file fails setup."""
|
|
|
|
def _get_uploader(self, provider):
|
|
raise UploaderConfigurationError(
|
|
f"no file uploader available for provider {provider!r}"
|
|
)
|
|
|
|
|
|
class _OneBadFileResolver(FileResolver):
|
|
"""Resolver that fails one specific file with an ordinary per-file error."""
|
|
|
|
async def aresolve(self, file, provider):
|
|
if file.filename == "bad.png":
|
|
raise ValueError("corrupt image stream")
|
|
return await super().aresolve(file, provider)
|
|
|
|
|
|
class TestBatchUploaderErrors:
|
|
"""A provider setup failure must surface, an unrelated per-file error must not."""
|
|
|
|
def test_get_uploader_wraps_lookup_failure_as_configuration_error(
|
|
self, monkeypatch
|
|
):
|
|
"""Bedrock with no bucket configured raises UploaderConfigurationError, not a raw ValueError."""
|
|
monkeypatch.delenv("CREWAI_BEDROCK_S3_BUCKET", raising=False)
|
|
resolver = FileResolver()
|
|
|
|
with pytest.raises(
|
|
UploaderConfigurationError, match="CREWAI_BEDROCK_S3_BUCKET"
|
|
):
|
|
resolver._get_uploader("bedrock")
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_aresolve_files_surfaces_uploader_configuration_error(self):
|
|
"""A provider whose uploader cannot be built aborts the whole batch, since it affects every file."""
|
|
resolver = _NoUploaderResolver(config=FileResolverConfig(prefer_upload=True))
|
|
files = {
|
|
"image1": ImageFile(
|
|
source=FileBytes(data=MINIMAL_PNG, filename="test1.png")
|
|
)
|
|
}
|
|
|
|
with pytest.raises(UploaderConfigurationError):
|
|
await resolver.aresolve_files(files, "openai")
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_aresolve_files_skips_unrelated_per_file_errors(self):
|
|
"""One file failing with an ordinary error is logged and skipped; the rest still resolve."""
|
|
resolver = _OneBadFileResolver()
|
|
files = {
|
|
"good": ImageFile(source=FileBytes(data=MINIMAL_PNG, filename="good.png")),
|
|
"bad": ImageFile(source=FileBytes(data=MINIMAL_PNG, filename="bad.png")),
|
|
}
|
|
|
|
resolved = await resolver.aresolve_files(files, "openai")
|
|
|
|
assert set(resolved) == {"good"}
|
|
assert isinstance(resolved["good"], InlineBase64)
|