From b92e80be5344085c0d9d929a4a98a8fc379b57f2 Mon Sep 17 00:00:00 2001 From: Daniel Barreto Date: Wed, 9 Sep 2026 13:48:53 -0300 Subject: [PATCH] chore(tools): make the vision_tool more dynamic (#7350) * chore(tools): make the vision_tool more dynamic * tackle review comments * chore: update tool specifications * refactor(tools): simplify vision tool model selection --------- Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> Co-authored-by: ViditOstwal Co-authored-by: Vidit Ostwal <110953813+Vidit-Ostwal@users.noreply.github.com> --- docs/edge/ar/tools/ai-ml/visiontool.mdx | 16 +- docs/edge/en/tools/ai-ml/visiontool.mdx | 20 +- docs/edge/ko/tools/ai-ml/visiontool.mdx | 16 +- docs/edge/pt-BR/tools/ai-ml/visiontool.mdx | 16 +- .../tools/vision_tool/vision_tool.py | 88 ++++++-- .../tests/tools/test_vision_tool.py | 205 ++++++++++++++++++ lib/crewai-tools/tool.specs.json | 17 ++ 7 files changed, 347 insertions(+), 31 deletions(-) create mode 100644 lib/crewai-tools/tests/tools/test_vision_tool.py diff --git a/docs/edge/ar/tools/ai-ml/visiontool.mdx b/docs/edge/ar/tools/ai-ml/visiontool.mdx index ecd09a6a3..4dc0943d7 100644 --- a/docs/edge/ar/tools/ai-ml/visiontool.mdx +++ b/docs/edge/ar/tools/ai-ml/visiontool.mdx @@ -12,6 +12,16 @@ mode: "wide" تُستخدم هذه الأداة لاستخراج النص من الصور. عند تمريرها إلى الوكيل، ستستخرج النص من الصورة ثم تستخدمه لتوليد استجابة أو تقرير أو أي مخرج آخر. يجب تمرير عنوان URL أو مسار الصورة إلى الوكيل. +يمكنك أيضًا طرح استعلام `query` مخصص حول الصورة واختيار مستوى تعقيد `complexity_level` يقوم تلقائيًا باختيار النموذج الأنسب للطلب: + +| مستوى التعقيد | النموذج | +| :--------------- | :------------ | +| `easy` | `gpt-5.6-luna` | +| `medium` (افتراضي) | `gpt-5.6-terra` | +| `hard` | `gpt-5.6-sol` | + +عند تمرير `llm` أو `model` بشكل صريح إلى الأداة، فإنه يأخذ الأولوية على اختيار النموذج المستند إلى مستوى التعقيد. + ## التثبيت ثبّت حزمة crewai_tools @@ -43,8 +53,10 @@ def researcher(self) -> Agent: ## المعاملات -تتطلب VisionTool المعاملات التالية: +تقبل VisionTool المعاملات التالية: | المعامل | النوع | الوصف | | :----------------- | :------- | :------------------------------------------------------------------------------- | -| **image_path_url** | `string` | **إلزامي**. مسار ملف الصورة المراد استخراج النص منها. | +| **image_path_url** | `string` | **إلزامي**. مسار ملف الصورة (أو عنوان URL) المراد استخراج النص منها. | +| **query** | `string` | **اختياري**. السؤال أو التعليمات المراد طرحها على النموذج حول الصورة. القيمة الافتراضية هي `"What's in this image?"`. | +| **complexity_level** | `string` | **اختياري**. مستوى تعقيد الطلب، والذي يحدد النموذج: `easy` أو `medium` أو `hard`. القيمة الافتراضية هي `medium`. | diff --git a/docs/edge/en/tools/ai-ml/visiontool.mdx b/docs/edge/en/tools/ai-ml/visiontool.mdx index 887516a74..485e76c9b 100644 --- a/docs/edge/en/tools/ai-ml/visiontool.mdx +++ b/docs/edge/en/tools/ai-ml/visiontool.mdx @@ -12,6 +12,16 @@ mode: "wide" This tool is used to extract text from images. When passed to the agent it will extract the text from the image and then use it to generate a response, report or any other output. The URL or the PATH of the image should be passed to the Agent. +You can also ask a custom `query` about the image and pick a `complexity_level` that automatically selects the model best suited for the request: + +| Complexity level | Model | +| :--------------- | :------------ | +| `easy` | `gpt-5.6-luna` | +| `medium` (default) | `gpt-5.6-terra` | +| `hard` | `gpt-5.6-sol` | + +When an explicit `llm` or `model` is provided to the tool, it takes precedence over the complexity-based model selection. + ## Installation Install the crewai_tools package @@ -43,8 +53,10 @@ def researcher(self) -> Agent: ## Arguments -The VisionTool requires the following arguments: +The VisionTool accepts the following arguments: -| Argument | Type | Description | -| :----------------- | :------- | :------------------------------------------------------------------------------- | -| **image_path_url** | `string` | **Mandatory**. The path to the image file from which text needs to be extracted. | +| Argument | Type | Description | +| :------------------- | :------- | :----------------------------------------------------------------------------------------------------------- | +| **image_path_url** | `string` | **Mandatory**. The path to the image file (or URL) from which text needs to be extracted. | +| **query** | `string` | **Optional**. The question or instruction to ask the model about the image. Defaults to `"What's in this image?"`. | +| **complexity_level** | `string` | **Optional**. The complexity of the request, which selects the model: `easy`, `medium`, or `hard`. Defaults to `medium`. | diff --git a/docs/edge/ko/tools/ai-ml/visiontool.mdx b/docs/edge/ko/tools/ai-ml/visiontool.mdx index 166c89b2c..7e5ffaa01 100644 --- a/docs/edge/ko/tools/ai-ml/visiontool.mdx +++ b/docs/edge/ko/tools/ai-ml/visiontool.mdx @@ -12,6 +12,16 @@ mode: "wide" 이 도구는 이미지에서 텍스트를 추출하는 데 사용됩니다. 에이전트에 전달되면 이미지에서 텍스트를 추출한 후 이를 사용하여 응답, 보고서 또는 기타 출력을 생성합니다. 이미지의 URL 또는 경로(PATH)를 에이전트에 전달해야 합니다. +이미지에 대한 사용자 지정 `query`를 요청하고, 요청에 가장 적합한 모델을 자동으로 선택하는 `complexity_level`을 지정할 수도 있습니다: + +| 복잡도 수준 | 모델 | +| :--------------- | :------------ | +| `easy` | `gpt-5.6-luna` | +| `medium` (기본값) | `gpt-5.6-terra` | +| `hard` | `gpt-5.6-sol` | + +도구에 명시적인 `llm` 또는 `model`을 제공하면, 복잡도 기반 모델 선택보다 우선합니다. + ## 설치 crewai_tools 패키지를 설치하세요 @@ -43,8 +53,10 @@ def researcher(self) -> Agent: ## 인수 -VisionTool은 다음과 같은 인수가 필요합니다: +VisionTool은 다음과 같은 인수를 받습니다: | 인수 | 타입 | 설명 | | :------------------ | :------- | :-------------------------------------------------------------------------------- | -| **image_path_url** | `string` | **필수**. 텍스트를 추출해야 하는 이미지 파일의 경로입니다. | +| **image_path_url** | `string` | **필수**. 텍스트를 추출해야 하는 이미지 파일의 경로(또는 URL)입니다. | +| **query** | `string` | **선택**. 이미지에 대해 모델에 묻는 질문 또는 지시입니다. 기본값은 `"What's in this image?"`입니다. | +| **complexity_level** | `string` | **선택**. 모델을 선택하는 요청의 복잡도입니다: `easy`, `medium`, `hard`. 기본값은 `medium`입니다. | diff --git a/docs/edge/pt-BR/tools/ai-ml/visiontool.mdx b/docs/edge/pt-BR/tools/ai-ml/visiontool.mdx index 8d43da512..915610929 100644 --- a/docs/edge/pt-BR/tools/ai-ml/visiontool.mdx +++ b/docs/edge/pt-BR/tools/ai-ml/visiontool.mdx @@ -12,6 +12,16 @@ mode: "wide" Esta ferramenta é utilizada para extrair texto de imagens. Quando passada para o agente, ela extrai o texto da imagem e depois o utiliza para gerar uma resposta, relatório ou qualquer outra saída. A URL ou o CAMINHO da imagem deve ser passado para o Agente. +Você também pode fazer uma `query` personalizada sobre a imagem e escolher um `complexity_level` que seleciona automaticamente o modelo mais adequado para a solicitação: + +| Nível de complexidade | Modelo | +| :--------------- | :------------ | +| `easy` | `gpt-5.6-luna` | +| `medium` (padrão) | `gpt-5.6-terra` | +| `hard` | `gpt-5.6-sol` | + +Quando um `llm` ou `model` explícito é fornecido à ferramenta, ele tem precedência sobre a seleção de modelo baseada na complexidade. + ## Instalação Instale o pacote crewai_tools @@ -43,8 +53,10 @@ def researcher(self) -> Agent: ## Argumentos -O VisionTool requer os seguintes argumentos: +O VisionTool aceita os seguintes argumentos: | Argumento | Tipo | Descrição | | :------------------ | :------- | :------------------------------------------------------------------------------- | -| **image_path_url** | `string` | **Obrigatório**. O caminho para o arquivo de imagem do qual o texto será extraído. | \ No newline at end of file +| **image_path_url** | `string` | **Obrigatório**. O caminho para o arquivo de imagem (ou URL) do qual o texto será extraído. | +| **query** | `string` | **Opcional**. A pergunta ou instrução a ser feita ao modelo sobre a imagem. O padrão é `"What's in this image?"`. | +| **complexity_level** | `string` | **Opcional**. A complexidade da solicitação, que seleciona o modelo: `easy`, `medium` ou `hard`. O padrão é `medium`. | \ No newline at end of file diff --git a/lib/crewai-tools/src/crewai_tools/tools/vision_tool/vision_tool.py b/lib/crewai-tools/src/crewai_tools/tools/vision_tool/vision_tool.py index bc6d6b464..c3b76b68a 100644 --- a/lib/crewai-tools/src/crewai_tools/tools/vision_tool/vision_tool.py +++ b/lib/crewai-tools/src/crewai_tools/tools/vision_tool/vision_tool.py @@ -1,6 +1,6 @@ import base64 from pathlib import Path -from typing import Any +from typing import Any, Literal from crewai import LLM from crewai.tools import BaseTool, EnvVar @@ -10,10 +10,34 @@ from pydantic import BaseModel, Field, PrivateAttr, field_validator from crewai_tools.security.safe_path import validate_file_path +ComplexityLevel = Literal["easy", "medium", "hard"] + +# Maps a complexity level to the OpenAI model used to answer the request. +COMPLEXITY_MODEL_MAP: dict[ComplexityLevel, str] = { + "easy": "gpt-5.6-luna", + "medium": "gpt-5.6-terra", + "hard": "gpt-5.6-sol", +} + +# Model used when no complexity level or explicit model/LLM is provided. +DEFAULT_MODEL: str = COMPLEXITY_MODEL_MAP["medium"] + + class ImagePromptSchema(BaseModel): """Input for Vision Tool.""" image_path_url: str = "The image path or URL." + query: str = Field( + default="What's in this image?", + description="The question or instruction to ask the model about the image.", + ) + complexity_level: ComplexityLevel = Field( + default="medium", + description=( + "The complexity of the request, which selects the model: " + "'easy', 'medium', 'hard'." + ), + ) @field_validator("image_path_url") @classmethod @@ -57,41 +81,60 @@ class VisionTool(BaseTool): ] ) - _model: str = PrivateAttr(default="gpt-4o-mini") - _llm: LLM | None = PrivateAttr(default=None) + _explicit_llm: LLM | None = PrivateAttr(default=None) + _explicit_model: str | None = PrivateAttr(default=None) + _llms_by_model: dict[str, LLM] = PrivateAttr(default_factory=dict) def __init__( - self, llm: LLM | None = None, model: str = "gpt-4o-mini", **kwargs: Any + self, llm: LLM | None = None, model: str | None = None, **kwargs: Any ) -> None: """Initialize the vision tool. Args: - llm: Optional LLM instance to use - model: Model identifier to use if no LLM is provided + llm: Optional LLM instance to use. When set, it always takes + precedence over ``model`` and the complexity-based selection. + model: Model identifier to use if no LLM is provided. When set, it + takes precedence over the complexity-based model selection. **kwargs: Additional arguments for the base tool """ super().__init__(**kwargs) - self._model = model - self._llm = llm + self._explicit_llm = llm + self._explicit_model = model @property def model(self) -> str: - """Get the current model identifier.""" - return self._model + """Get the configured model identifier, or the default model.""" + return ( + self._explicit_model if self._explicit_model is not None else DEFAULT_MODEL + ) @model.setter def model(self, value: str) -> None: - """Set the model identifier and reset LLM if it was auto-created.""" - self._model = value - if self._llm is not None and getattr(self._llm, "model", None) != value: - self._llm = None + """Set the model override; an explicitly supplied LLM still takes precedence.""" + self._explicit_model = value @property def llm(self) -> LLM: - """Get the LLM instance, creating one if needed.""" - if self._llm is None: - self._llm = LLM(model=self._model, stop=["STOP", "END"]) - return self._llm + """Get the LLM for the default complexity, honoring explicit overrides.""" + return self._llm_for_complexity("medium") + + def _get_or_create_llm(self, model: str) -> LLM: + """Reuse one LLM instance per model.""" + if model not in self._llms_by_model: + self._llms_by_model[model] = LLM(model=model, stop=["STOP", "END"]) + return self._llms_by_model[model] + + def _llm_for_complexity(self, complexity_level: ComplexityLevel) -> LLM: + """Select an explicit LLM, explicit model, or complexity model, in that order.""" + if self._explicit_llm is not None: + return self._explicit_llm + + model = ( + self._explicit_model + if self._explicit_model is not None + else COMPLEXITY_MODEL_MAP[complexity_level] + ) + return self._get_or_create_llm(model) def _run(self, **kwargs: Any) -> str: try: @@ -99,7 +142,8 @@ class VisionTool(BaseTool): if not image_path_url: return "Image Path or URL is required." - ImagePromptSchema(image_path_url=image_path_url) + inputs = ImagePromptSchema(**kwargs) + image_path_url = inputs.image_path_url if image_path_url.startswith("http"): image_data = image_path_url @@ -114,7 +158,7 @@ class VisionTool(BaseTool): { "role": "user", "content": [ - {"type": "text", "text": "What's in this image?"}, + {"type": "text", "text": inputs.query}, { "type": "image_url", "image_url": {"url": image_data}, @@ -122,7 +166,9 @@ class VisionTool(BaseTool): ], }, ] - return self.llm.call(messages=messages) + return self._llm_for_complexity(inputs.complexity_level).call( + messages=messages + ) except Exception as e: return f"An error occurred: {e!s}" diff --git a/lib/crewai-tools/tests/tools/test_vision_tool.py b/lib/crewai-tools/tests/tools/test_vision_tool.py new file mode 100644 index 000000000..c15e2e6a3 --- /dev/null +++ b/lib/crewai-tools/tests/tools/test_vision_tool.py @@ -0,0 +1,205 @@ +from typing import Any +from unittest.mock import MagicMock, patch + +from crewai_tools.tools.vision_tool.vision_tool import ( + COMPLEXITY_MODEL_MAP, + DEFAULT_MODEL, + ComplexityLevel, + VisionTool, +) +import pytest + +IMAGE_URL = "http://example.com/image.png" + + +def _llm_mock(return_value: str = "described") -> MagicMock: + llm = MagicMock() + llm.call.return_value = return_value + return llm + + +def test_explicit_llm_is_used_over_complexity_map() -> None: + """An explicitly provided LLM takes precedence over the complexity map.""" + llm = _llm_mock("from explicit llm") + + tool = VisionTool(llm=llm) + result = tool._run(image_path_url=IMAGE_URL, complexity_level="hard") + + assert result == "from explicit llm" + llm.call.assert_called_once() + + +@patch("crewai_tools.tools.vision_tool.vision_tool.LLM") +def test_explicit_model_overrides_complexity_map(mock_llm_cls: MagicMock) -> None: + """VisionTool(model="custom") must use "custom", not the complexity map.""" + mock_llm_cls.return_value = _llm_mock() + + tool = VisionTool(model="custom-model") + tool._run(image_path_url=IMAGE_URL, complexity_level="hard") + + mock_llm_cls.assert_called_once_with(model="custom-model", stop=["STOP", "END"]) + + +@patch("crewai_tools.tools.vision_tool.vision_tool.LLM") +def test_model_setter_marks_model_explicit(mock_llm_cls: MagicMock) -> None: + """Assigning through the model setter also overrides the complexity map.""" + mock_llm_cls.return_value = _llm_mock() + + tool = VisionTool() + tool.model = "setter-model" + tool._run(image_path_url=IMAGE_URL, complexity_level="easy") + + mock_llm_cls.assert_called_once_with(model="setter-model", stop=["STOP", "END"]) + + +@pytest.mark.parametrize(("complexity_level", "expected_model"), COMPLEXITY_MODEL_MAP.items()) +@patch("crewai_tools.tools.vision_tool.vision_tool.LLM") +def test_complexity_level_selects_expected_model( + mock_llm_cls: MagicMock, complexity_level: ComplexityLevel, expected_model: str +) -> None: + """Each complexity level maps to its corresponding model when none is set.""" + mock_llm_cls.return_value = _llm_mock() + + tool = VisionTool() + tool._run(image_path_url=IMAGE_URL, complexity_level=complexity_level) + + mock_llm_cls.assert_called_once_with(model=expected_model, stop=["STOP", "END"]) + + +@patch("crewai_tools.tools.vision_tool.vision_tool.LLM") +def test_default_uses_medium_complexity_model(mock_llm_cls: MagicMock) -> None: + """Without a model, llm, or complexity level, the default (medium) is used.""" + mock_llm_cls.return_value = _llm_mock() + + tool = VisionTool() + tool._run(image_path_url=IMAGE_URL) + + mock_llm_cls.assert_called_once_with(model=DEFAULT_MODEL, stop=["STOP", "END"]) + + +@patch("crewai_tools.tools.vision_tool.vision_tool.LLM") +def test_reading_llm_property_does_not_override_complexity_level(mock_llm_cls: MagicMock) -> None: + """Reading the default LLM must not override later complexity selection.""" + mock_llm_cls.return_value = _llm_mock() + + tool = VisionTool() + _ = tool.llm # caches the medium-tier model as a side effect + mock_llm_cls.assert_called_once_with(model=DEFAULT_MODEL, stop=["STOP", "END"]) + + mock_llm_cls.reset_mock() + tool._run(image_path_url=IMAGE_URL, complexity_level="hard") + + mock_llm_cls.assert_called_once_with( + model=COMPLEXITY_MODEL_MAP["hard"], stop=["STOP", "END"] + ) + + +@patch("crewai_tools.tools.vision_tool.vision_tool.LLM") +def test_same_instance_resolves_model_per_call(mock_llm_cls: MagicMock) -> None: + """One instance across two calls resolves each call's model independently.""" + mock_llm_cls.side_effect = lambda **kwargs: _llm_mock() + + tool = VisionTool() + tool._run(image_path_url=IMAGE_URL, complexity_level="easy") + tool._run(image_path_url=IMAGE_URL, complexity_level="hard") + + used_models = [call.kwargs["model"] for call in mock_llm_cls.call_args_list] + assert used_models == [ + COMPLEXITY_MODEL_MAP["easy"], + COMPLEXITY_MODEL_MAP["hard"], + ] + + +@patch("crewai_tools.tools.vision_tool.vision_tool.LLM") +def test_complexity_llm_is_cached_per_level(mock_llm_cls: MagicMock) -> None: + """Repeated calls with the same complexity level reuse a single LLM.""" + mock_llm_cls.return_value = _llm_mock() + + tool = VisionTool() + tool._run(image_path_url=IMAGE_URL, complexity_level="hard") + tool._run(image_path_url=IMAGE_URL, complexity_level="hard") + + mock_llm_cls.assert_called_once_with( + model=COMPLEXITY_MODEL_MAP["hard"], stop=["STOP", "END"] + ) + + +def test_invalid_complexity_level_returns_error_string() -> None: + """An unexpected complexity level fails validation and is reported as an error. + + ``ImagePromptSchema``'s ``Literal`` validation raises, which ``_run`` catches + and surfaces as a generic error string rather than propagating. + """ + tool = VisionTool() + + result = tool._run(image_path_url=IMAGE_URL, complexity_level="very hard") + + assert result.startswith("An error occurred") + + +@pytest.mark.parametrize( + ("query_args", "expected_query"), + [({}, "What's in this image?"), ({"query": "Read the sign."}, "Read the sign.")], +) +def test_query_is_sent_with_image( + query_args: dict[str, Any], expected_query: str +) -> None: + llm = _llm_mock("image answer") + tool = VisionTool(llm=llm) + + assert tool._run(image_path_url=IMAGE_URL, **query_args) == "image answer" + llm.call.assert_called_once_with( + messages=[ + { + "role": "user", + "content": [ + {"type": "text", "text": expected_query}, + {"type": "image_url", "image_url": {"url": IMAGE_URL}}, + ], + } + ] + ) + + +@patch("crewai_tools.tools.vision_tool.vision_tool.LLM") +def test_llm_property_shares_default_model_cache(mock_llm_cls: MagicMock) -> None: + mock_llm_cls.return_value = _llm_mock("cached answer") + tool = VisionTool() + llm = tool.llm + + assert tool._run(image_path_url=IMAGE_URL) == "cached answer" + assert tool.llm is llm + llm.call.assert_called_once() + mock_llm_cls.assert_called_once_with(model=DEFAULT_MODEL, stop=["STOP", "END"]) + + +@patch("crewai_tools.tools.vision_tool.vision_tool.LLM") +def test_model_setter_preserves_explicit_llm(mock_llm_cls: MagicMock) -> None: + llm = _llm_mock("explicit answer") + llm.model = "original-model" + tool = VisionTool(llm=llm, model="constructor-model") + tool.model = "replacement-model" + + assert tool.model == "replacement-model" + assert tool.llm is llm + assert tool._run(image_path_url=IMAGE_URL, complexity_level="hard") == "explicit answer" + llm.call.assert_called_once() + mock_llm_cls.assert_not_called() + + +@patch("crewai_tools.tools.vision_tool.vision_tool.LLM") +def test_model_setter_switches_cached_model(mock_llm_cls: MagicMock) -> None: + default_llm = _llm_mock("default answer") + custom_llm = _llm_mock("custom answer") + mock_llm_cls.side_effect = [default_llm, custom_llm] + tool = VisionTool() + + assert tool.model == DEFAULT_MODEL + assert tool._run(image_path_url=IMAGE_URL) == "default answer" + tool.model = "custom-model" + assert tool._run(image_path_url=IMAGE_URL, complexity_level="hard") == "custom answer" + assert tool.llm is custom_llm + tool.model = DEFAULT_MODEL + assert tool._run(image_path_url=IMAGE_URL) == "default answer" + assert tool.llm is default_llm + assert mock_llm_cls.call_count == 2 diff --git a/lib/crewai-tools/tool.specs.json b/lib/crewai-tools/tool.specs.json index dfcb43d4a..674cc33c1 100644 --- a/lib/crewai-tools/tool.specs.json +++ b/lib/crewai-tools/tool.specs.json @@ -27110,10 +27110,27 @@ "run_params_schema": { "description": "Input for Vision Tool.", "properties": { + "complexity_level": { + "default": "medium", + "description": "The complexity of the request, which selects the model: 'easy', 'medium', 'hard'.", + "enum": [ + "easy", + "medium", + "hard" + ], + "title": "Complexity Level", + "type": "string" + }, "image_path_url": { "default": "The image path or URL.", "title": "Image Path Url", "type": "string" + }, + "query": { + "default": "What's in this image?", + "description": "The question or instruction to ask the model about the image.", + "title": "Query", + "type": "string" } }, "title": "ImagePromptSchema",