Merge branch 'main' into feat/trace-ui-execution

Merge pull request #2476 from crewAIInc/devin/1742990927-fix-issue-2475
Fix multimodal agent validation errors with image processing
2025-12-18 05:18:31 +00:00 · 2025-03-26 16:47:19 -03:00 · 2025-03-26 16:40:23 -03:00 · 2025-03-26 16:30:17 -03:00 · 2025-03-26 16:29:24 -03:00 · 2025-03-26 16:29:15 -03:00
7 changed files with 650 additions and 9 deletions
--- a/src/crewai/tools/tool_usage.py
+++ b/src/crewai/tools/tool_usage.py
@@ -117,7 +117,10 @@ class ToolUsage:
                self._printer.print(content=f"\n\n{error}\n", color="red")
            return error

-        if isinstance(tool, CrewStructuredTool) and tool.name == self._i18n.tools("add_image")["name"]:  # type: ignore
+        if (
+            isinstance(tool, CrewStructuredTool)
+            and tool.name == self._i18n.tools("add_image")["name"]  # type: ignore
+        ):
            try:
                result = self._use(tool_string=tool_string, tool=tool, calling=calling)
                return result
@@ -181,7 +184,9 @@ class ToolUsage:

                if calling.arguments:
                    try:
-                        acceptable_args = tool.args_schema.model_json_schema()["properties"].keys()  # type: ignore
+                        acceptable_args = tool.args_schema.model_json_schema()[
+                            "properties"
+                        ].keys()  # type: ignore
                        arguments = {
                            k: v
                            for k, v in calling.arguments.items()
@@ -202,7 +207,7 @@ class ToolUsage:
                        error=e, tool=tool.name, tool_inputs=tool.description
                    )
                    error = ToolUsageErrorException(
-                        f'\n{error_message}.\nMoving on then. {self._i18n.slice("format").format(tool_names=self.tools_names)}'
+                        f"\n{error_message}.\nMoving on then. {self._i18n.slice('format').format(tool_names=self.tools_names)}"
                    ).message
                    self.task.increment_tools_errors()
                    if self.agent.verbose:
@@ -244,6 +249,7 @@ class ToolUsage:
            tool_calling=calling,
            from_cache=from_cache,
            started_at=started_at,
+            result=result,
        )

        if (
@@ -380,7 +386,7 @@ class ToolUsage:
                raise
            else:
                return ToolUsageErrorException(
-                    f'{self._i18n.errors("tool_arguments_error")}'
+                    f"{self._i18n.errors('tool_arguments_error')}"
                )

        if not isinstance(arguments, dict):
@@ -388,7 +394,7 @@ class ToolUsage:
                raise
            else:
                return ToolUsageErrorException(
-                    f'{self._i18n.errors("tool_arguments_error")}'
+                    f"{self._i18n.errors('tool_arguments_error')}"
                )

        return ToolCalling(
@@ -416,7 +422,7 @@ class ToolUsage:
                if self.agent.verbose:
                    self._printer.print(content=f"\n\n{e}\n", color="red")
                return ToolUsageErrorException(  # type: ignore # Incompatible return value type (got "ToolUsageErrorException", expected "ToolCalling | InstructorToolCalling")
-                    f'{self._i18n.errors("tool_usage_error").format(error=e)}\nMoving on then. {self._i18n.slice("format").format(tool_names=self.tools_names)}'
+                    f"{self._i18n.errors('tool_usage_error').format(error=e)}\nMoving on then. {self._i18n.slice('format').format(tool_names=self.tools_names)}"
                )
            return self._tool_calling(tool_string)

@@ -492,7 +498,12 @@ class ToolUsage:
        crewai_event_bus.emit(self, ToolUsageErrorEvent(**{**event_data, "error": e}))

    def on_tool_use_finished(
-        self, tool: Any, tool_calling: ToolCalling, from_cache: bool, started_at: float
+        self,
+        tool: Any,
+        tool_calling: ToolCalling,
+        from_cache: bool,
+        started_at: float,
+        result: Any,
    ) -> None:
        finished_at = time.time()
        event_data = self._prepare_event_data(tool, tool_calling)
@@ -501,6 +512,7 @@ class ToolUsage:
                "started_at": datetime.datetime.fromtimestamp(started_at),
                "finished_at": datetime.datetime.fromtimestamp(finished_at),
                "from_cache": from_cache,
+                "output": result,
            }
        )
        crewai_event_bus.emit(self, ToolUsageFinishedEvent(**event_data))
--- a/src/crewai/utilities/events/llm_events.py
+++ b/src/crewai/utilities/events/llm_events.py
@@ -12,10 +12,15 @@ class LLMCallType(Enum):


 class LLMCallStartedEvent(CrewEvent):
-    """Event emitted when a LLM call starts"""
+    """Event emitted when a LLM call starts
+    
+    Attributes:
+        messages: Content can be either a string or a list of dictionaries that support
+            multimodal content (text, images, etc.)
+    """

    type: str = "llm_call_started"
-    messages: Union[str, List[Dict[str, str]]]
+    messages: Union[str, List[Dict[str, Any]]]
    tools: Optional[List[dict]] = None
    callbacks: Optional[List[Any]] = None
    available_functions: Optional[Dict[str, Any]] = None
--- a/src/crewai/utilities/events/tool_usage_events.py
+++ b/src/crewai/utilities/events/tool_usage_events.py
@@ -30,6 +30,7 @@ class ToolUsageFinishedEvent(ToolUsageEvent):
    started_at: datetime
    finished_at: datetime
    from_cache: bool = False
+    output: Any
    type: str = "tool_usage_finished"


--- a/tests/cassettes/test_multimodal_agent_describing_image_successfully.yaml
+++ b/tests/cassettes/test_multimodal_agent_describing_image_successfully.yaml
@@ -0,0 +1,378 @@
+interactions:
+- request:
+    body: !!binary |
+      CpIKCiQKIgoMc2VydmljZS5uYW1lEhIKEGNyZXdBSS10ZWxlbWV0cnkS6QkKEgoQY3Jld2FpLnRl
+      bGVtZXRyeRLBBwoQ08SlQ6w2FsCauTgZCqberRIITfOsgNi1qJkqDENyZXcgQ3JlYXRlZDABOdjG
+      6D/PcDAYQahPEkDPcDAYShsKDmNyZXdhaV92ZXJzaW9uEgkKBzAuMTA4LjBKGgoOcHl0aG9uX3Zl
+      cnNpb24SCAoGMy4xMi45Si4KCGNyZXdfa2V5EiIKIDkwNzMxMTU4MzVlMWNhZjJhNmUxNTIyZDA1
+      YTBiNTFkSjEKB2NyZXdfaWQSJgokMzdjOGM4NzgtN2NmZC00YjEyLWE4YzctYzIyZDZlOTIxODBk
+      ShwKDGNyZXdfcHJvY2VzcxIMCgpzZXF1ZW50aWFsShEKC2NyZXdfbWVtb3J5EgIQAEoaChRjcmV3
+      X251bWJlcl9vZl90YXNrcxICGAFKGwoVY3Jld19udW1iZXJfb2ZfYWdlbnRzEgIYAUrgAgoLY3Jl
+      d19hZ2VudHMS0AIKzQJbeyJrZXkiOiAiNzYyM2ZjNGY3ZDk0Y2YzZmRiZmNjMjlmYjBiMDIyYmIi
+      LCAiaWQiOiAiYmVjMjljMTAtOTljYi00MzQwLWIwYTItMWU1NTVkNGRmZGM0IiwgInJvbGUiOiAi
+      VmlzdWFsIFF1YWxpdHkgSW5zcGVjdG9yIiwgInZlcmJvc2U/IjogdHJ1ZSwgIm1heF9pdGVyIjog
+      MjUsICJtYXhfcnBtIjogbnVsbCwgImZ1bmN0aW9uX2NhbGxpbmdfbGxtIjogIiIsICJsbG0iOiAi
+      b3BlbmFpL2dwdC00byIsICJkZWxlZ2F0aW9uX2VuYWJsZWQ/IjogZmFsc2UsICJhbGxvd19jb2Rl
+      X2V4ZWN1dGlvbj8iOiBmYWxzZSwgIm1heF9yZXRyeV9saW1pdCI6IDIsICJ0b29sc19uYW1lcyI6
+      IFtdfV1KjQIKCmNyZXdfdGFza3MS/gEK+wFbeyJrZXkiOiAiMDExM2E5ZTg0N2M2NjI2ZDY0ZDZk
+      Yzk4M2IwNDA5MTgiLCAiaWQiOiAiZWQzYmY1YWUtZTBjMS00MjIxLWFhYTgtMThlNjVkYTMyZjc1
+      IiwgImFzeW5jX2V4ZWN1dGlvbj8iOiBmYWxzZSwgImh1bWFuX2lucHV0PyI6IGZhbHNlLCAiYWdl
+      bnRfcm9sZSI6ICJWaXN1YWwgUXVhbGl0eSBJbnNwZWN0b3IiLCAiYWdlbnRfa2V5IjogIjc2MjNm
+      YzRmN2Q5NGNmM2ZkYmZjYzI5ZmIwYjAyMmJiIiwgInRvb2xzX25hbWVzIjogW119XXoCGAGFAQAB
+      AAASjgIKECo77ESam8oLrZMmgLLaoksSCLE6x14/Kb1vKgxUYXNrIENyZWF0ZWQwATlI/chAz3Aw
+      GEEAgMpAz3AwGEouCghjcmV3X2tleRIiCiA5MDczMTE1ODM1ZTFjYWYyYTZlMTUyMmQwNWEwYjUx
+      ZEoxCgdjcmV3X2lkEiYKJDM3YzhjODc4LTdjZmQtNGIxMi1hOGM3LWMyMmQ2ZTkyMTgwZEouCgh0
+      YXNrX2tleRIiCiAwMTEzYTllODQ3YzY2MjZkNjRkNmRjOTgzYjA0MDkxOEoxCgd0YXNrX2lkEiYK
+      JGVkM2JmNWFlLWUwYzEtNDIyMS1hYWE4LTE4ZTY1ZGEzMmY3NXoCGAGFAQABAAA=
+    headers:
+      Accept:
+      - '*/*'
+      Accept-Encoding:
+      - gzip, deflate, zstd
+      Connection:
+      - keep-alive
+      Content-Length:
+      - '1301'
+      Content-Type:
+      - application/x-protobuf
+      User-Agent:
+      - OTel-OTLP-Exporter-Python/1.31.1
+    method: POST
+    uri: https://telemetry.crewai.com:4319/v1/traces
+  response:
+    body:
+      string: "\n\0"
+    headers:
+      Content-Length:
+      - '2'
+      Content-Type:
+      - application/x-protobuf
+      Date:
+      - Wed, 26 Mar 2025 19:24:52 GMT
+    status:
+      code: 200
+      message: OK
+- request:
+    body: '{"messages": [{"role": "system", "content": "You are Visual Quality Inspector.
+      Senior quality control expert with expertise in visual inspection\nYour personal
+      goal is: Perform detailed quality analysis of product images\nYou ONLY have
+      access to the following tools, and should NEVER make up tools that are not listed
+      here:\n\nTool Name: Add image to content\nTool Arguments: {''image_url'': {''description'':
+      ''The URL or path of the image to add'', ''type'': ''str''}, ''action'': {''description'':
+      ''Optional context or question about the image'', ''type'': ''Union[str, NoneType]''}}\nTool
+      Description: See image to understand its content, you can optionally ask a question
+      about the image\n\nIMPORTANT: Use the following format in your response:\n\n```\nThought:
+      you should always think about what to do\nAction: the action to take, only one
+      name of [Add image to content], just the name, exactly as it''s written.\nAction
+      Input: the input to the action, just a simple JSON object, enclosed in curly
+      braces, using \" to wrap keys and values.\nObservation: the result of the action\n```\n\nOnce
+      all necessary information is gathered, return the following format:\n\n```\nThought:
+      I now know the final answer\nFinal Answer: the final answer to the original
+      input question\n```"}, {"role": "user", "content": "\nCurrent Task: \n        Analyze
+      the product image at https://www.us.maguireshoes.com/cdn/shop/files/FW24-Edito-Lucena-Distressed-01_1920x.jpg?v=1736371244
+      with focus on:\n        1. Quality of materials\n        2. Manufacturing defects\n        3.
+      Compliance with standards\n        Provide a detailed report highlighting any
+      issues found.\n        \n\nThis is the expected criteria for your final answer:
+      A detailed report highlighting any issues found\nyou MUST return the actual
+      complete content as the final answer, not a summary.\n\nBegin! This is VERY
+      important to you, use the tools available and give your best Final Answer, your
+      job depends on it!\n\nThought:"}], "model": "gpt-4o", "stop": ["\nObservation:"],
+      "temperature": 0.7}'
+    headers:
+      accept:
+      - application/json
+      accept-encoding:
+      - gzip, deflate, zstd
+      connection:
+      - keep-alive
+      content-length:
+      - '2033'
+      content-type:
+      - application/json
+      host:
+      - api.openai.com
+      user-agent:
+      - OpenAI/Python 1.68.2
+      x-stainless-arch:
+      - x64
+      x-stainless-async:
+      - 'false'
+      x-stainless-lang:
+      - python
+      x-stainless-os:
+      - MacOS
+      x-stainless-package-version:
+      - 1.68.2
+      x-stainless-raw-response:
+      - 'true'
+      x-stainless-read-timeout:
+      - '600.0'
+      x-stainless-retry-count:
+      - '0'
+      x-stainless-runtime:
+      - CPython
+      x-stainless-runtime-version:
+      - 3.12.9
+    method: POST
+    uri: https://api.openai.com/v1/chat/completions
+  response:
+    content: "{\n  \"id\": \"chatcmpl-BFQepLwSYYzdKLylSFsgcJeg6GTqS\",\n  \"object\":
+      \"chat.completion\",\n  \"created\": 1743017091,\n  \"model\": \"gpt-4o-2024-08-06\",\n
+      \ \"choices\": [\n    {\n      \"index\": 0,\n      \"message\": {\n        \"role\":
+      \"assistant\",\n        \"content\": \"Thought: I need to examine the product
+      image to assess the quality of materials, look for any manufacturing defects,
+      and check compliance with standards.\\n\\nAction: Add image to content\\nAction
+      Input: {\\\"image_url\\\": \\\"https://www.us.maguireshoes.com/cdn/shop/files/FW24-Edito-Lucena-Distressed-01_1920x.jpg?v=1736371244\\\",
+      \\\"action\\\": \\\"Analyze the quality of materials, manufacturing defects,
+      and compliance with standards.\\\"}\",\n        \"refusal\": null,\n        \"annotations\":
+      []\n      },\n      \"logprobs\": null,\n      \"finish_reason\": \"stop\"\n
+      \   }\n  ],\n  \"usage\": {\n    \"prompt_tokens\": 413,\n    \"completion_tokens\":
+      101,\n    \"total_tokens\": 514,\n    \"prompt_tokens_details\": {\n      \"cached_tokens\":
+      0,\n      \"audio_tokens\": 0\n    },\n    \"completion_tokens_details\": {\n
+      \     \"reasoning_tokens\": 0,\n      \"audio_tokens\": 0,\n      \"accepted_prediction_tokens\":
+      0,\n      \"rejected_prediction_tokens\": 0\n    }\n  },\n  \"service_tier\":
+      \"default\",\n  \"system_fingerprint\": \"fp_7e8d90e604\"\n}\n"
+    headers:
+      CF-RAY:
+      - 926907d79dcff1e7-GRU
+      Connection:
+      - keep-alive
+      Content-Encoding:
+      - gzip
+      Content-Type:
+      - application/json
+      Date:
+      - Wed, 26 Mar 2025 19:24:53 GMT
+      Server:
+      - cloudflare
+      Set-Cookie:
+      - __cf_bm=WK433.4kW8cr9rwvOlk4EZ2SfRYK9lAPwXCBYEvLcmU-1743017093-1.0.1.1-kVZyUew5rUbMk.2koGJF_rmX.fTseqN241n2M40n8KvBGoKgy6KM6xBmvFbIVWxUs2Y5ZAz8mWy9CrGjaNKSfCzxmv4.pq78z_DGHr37PgI;
+        path=/; expires=Wed, 26-Mar-25 19:54:53 GMT; domain=.api.openai.com; HttpOnly;
+        Secure; SameSite=None
+      - _cfuvid=T77PMcuNYeyzK0tQyDOe7EScjVBVzW_7DpD3YQBqmUc-1743017093675-0.0.1.1-604800000;
+        path=/; domain=.api.openai.com; HttpOnly; Secure; SameSite=None
+      Transfer-Encoding:
+      - chunked
+      X-Content-Type-Options:
+      - nosniff
+      access-control-expose-headers:
+      - X-Request-ID
+      alt-svc:
+      - h3=":443"; ma=86400
+      cf-cache-status:
+      - DYNAMIC
+      openai-organization:
+      - crewai-iuxna1
+      openai-processing-ms:
+      - '1729'
+      openai-version:
+      - '2020-10-01'
+      strict-transport-security:
+      - max-age=31536000; includeSubDomains; preload
+      x-ratelimit-limit-requests:
+      - '50000'
+      x-ratelimit-limit-tokens:
+      - '150000000'
+      x-ratelimit-remaining-requests:
+      - '49999'
+      x-ratelimit-remaining-tokens:
+      - '149999534'
+      x-ratelimit-reset-requests:
+      - 1ms
+      x-ratelimit-reset-tokens:
+      - 0s
+      x-request-id:
+      - req_2399c3355adf16734907c73611a7d330
+    http_version: HTTP/1.1
+    status_code: 200
+- request:
+    body: !!binary |
+      CtgBCiQKIgoMc2VydmljZS5uYW1lEhIKEGNyZXdBSS10ZWxlbWV0cnkSrwEKEgoQY3Jld2FpLnRl
+      bGVtZXRyeRKYAQoQp2ACB2xRGve4HGtU2RdWCBIIlQcsbhK22ykqClRvb2wgVXNhZ2UwATlACEXG
+      z3AwGEHAjGPGz3AwGEobCg5jcmV3YWlfdmVyc2lvbhIJCgcwLjEwOC4wSiMKCXRvb2xfbmFtZRIW
+      ChRBZGQgaW1hZ2UgdG8gY29udGVudEoOCghhdHRlbXB0cxICGAF6AhgBhQEAAQAA
+    headers:
+      Accept:
+      - '*/*'
+      Accept-Encoding:
+      - gzip, deflate, zstd
+      Connection:
+      - keep-alive
+      Content-Length:
+      - '219'
+      Content-Type:
+      - application/x-protobuf
+      User-Agent:
+      - OTel-OTLP-Exporter-Python/1.31.1
+    method: POST
+    uri: https://telemetry.crewai.com:4319/v1/traces
+  response:
+    body:
+      string: "\n\0"
+    headers:
+      Content-Length:
+      - '2'
+      Content-Type:
+      - application/x-protobuf
+      Date:
+      - Wed, 26 Mar 2025 19:24:57 GMT
+    status:
+      code: 200
+      message: OK
+- request:
+    body: '{"messages": [{"role": "system", "content": "You are Visual Quality Inspector.
+      Senior quality control expert with expertise in visual inspection\nYour personal
+      goal is: Perform detailed quality analysis of product images\nYou ONLY have
+      access to the following tools, and should NEVER make up tools that are not listed
+      here:\n\nTool Name: Add image to content\nTool Arguments: {''image_url'': {''description'':
+      ''The URL or path of the image to add'', ''type'': ''str''}, ''action'': {''description'':
+      ''Optional context or question about the image'', ''type'': ''Union[str, NoneType]''}}\nTool
+      Description: See image to understand its content, you can optionally ask a question
+      about the image\n\nIMPORTANT: Use the following format in your response:\n\n```\nThought:
+      you should always think about what to do\nAction: the action to take, only one
+      name of [Add image to content], just the name, exactly as it''s written.\nAction
+      Input: the input to the action, just a simple JSON object, enclosed in curly
+      braces, using \" to wrap keys and values.\nObservation: the result of the action\n```\n\nOnce
+      all necessary information is gathered, return the following format:\n\n```\nThought:
+      I now know the final answer\nFinal Answer: the final answer to the original
+      input question\n```"}, {"role": "user", "content": "\nCurrent Task: \n        Analyze
+      the product image at https://www.us.maguireshoes.com/cdn/shop/files/FW24-Edito-Lucena-Distressed-01_1920x.jpg?v=1736371244
+      with focus on:\n        1. Quality of materials\n        2. Manufacturing defects\n        3.
+      Compliance with standards\n        Provide a detailed report highlighting any
+      issues found.\n        \n\nThis is the expected criteria for your final answer:
+      A detailed report highlighting any issues found\nyou MUST return the actual
+      complete content as the final answer, not a summary.\n\nBegin! This is VERY
+      important to you, use the tools available and give your best Final Answer, your
+      job depends on it!\n\nThought:"}, {"role": "user", "content": [{"type": "text",
+      "text": "Analyze the quality of materials, manufacturing defects, and compliance
+      with standards."}, {"type": "image_url", "image_url": {"url": "https://www.us.maguireshoes.com/cdn/shop/files/FW24-Edito-Lucena-Distressed-01_1920x.jpg?v=1736371244"}}]},
+      {"role": "assistant", "content": "Thought: I need to examine the product image
+      to assess the quality of materials, look for any manufacturing defects, and
+      check compliance with standards.\n\nAction: Add image to content\nAction Input:
+      {\"image_url\": \"https://www.us.maguireshoes.com/cdn/shop/files/FW24-Edito-Lucena-Distressed-01_1920x.jpg?v=1736371244\",
+      \"action\": \"Analyze the quality of materials, manufacturing defects, and compliance
+      with standards.\"}"}], "model": "gpt-4o", "stop": ["\nObservation:"], "temperature":
+      0.7}'
+    headers:
+      accept:
+      - application/json
+      accept-encoding:
+      - gzip, deflate, zstd
+      connection:
+      - keep-alive
+      content-length:
+      - '2797'
+      content-type:
+      - application/json
+      cookie:
+      - __cf_bm=WK433.4kW8cr9rwvOlk4EZ2SfRYK9lAPwXCBYEvLcmU-1743017093-1.0.1.1-kVZyUew5rUbMk.2koGJF_rmX.fTseqN241n2M40n8KvBGoKgy6KM6xBmvFbIVWxUs2Y5ZAz8mWy9CrGjaNKSfCzxmv4.pq78z_DGHr37PgI;
+        _cfuvid=T77PMcuNYeyzK0tQyDOe7EScjVBVzW_7DpD3YQBqmUc-1743017093675-0.0.1.1-604800000
+      host:
+      - api.openai.com
+      user-agent:
+      - OpenAI/Python 1.68.2
+      x-stainless-arch:
+      - x64
+      x-stainless-async:
+      - 'false'
+      x-stainless-lang:
+      - python
+      x-stainless-os:
+      - MacOS
+      x-stainless-package-version:
+      - 1.68.2
+      x-stainless-raw-response:
+      - 'true'
+      x-stainless-read-timeout:
+      - '600.0'
+      x-stainless-retry-count:
+      - '0'
+      x-stainless-runtime:
+      - CPython
+      x-stainless-runtime-version:
+      - 3.12.9
+    method: POST
+    uri: https://api.openai.com/v1/chat/completions
+  response:
+    content: "{\n  \"id\": \"chatcmpl-BFQetNNvmPgPxhzaKiHYsPqm8aN0i\",\n  \"object\":
+      \"chat.completion\",\n  \"created\": 1743017095,\n  \"model\": \"gpt-4o-2024-08-06\",\n
+      \ \"choices\": [\n    {\n      \"index\": 0,\n      \"message\": {\n        \"role\":
+      \"assistant\",\n        \"content\": \"Observation: The image displays a black
+      leather boot with a pointed toe and a low heel. \\n\\nQuality of Materials:\\n1.
+      The leather appears to be of good quality, displaying a consistent texture and
+      finish, which suggests durability.\\n2. The material has a slight sheen, indicating
+      a possible finishing treatment that enhances the appearance and may offer some
+      protection.\\n\\nManufacturing Defects:\\n1. There are no visible stitching
+      errors; the seams appear straight and clean.\\n2. No apparent glue marks or
+      uneven edges, which indicates good craftsmanship.\\n3. There is a slight distressed
+      effect, but it appears intentional as part of the design rather than a defect.\\n\\nCompliance
+      with Standards:\\n1. The shoe design seems to comply with typical fashion standards,
+      showing a balance of aesthetics and functionality.\\n2. The heel height and
+      shape appear to provide stability, aligning with safety standards for footwear.\\n\\nFinal
+      Answer: The analysis of the product image reveals that the black leather boot
+      is made of high-quality materials with no visible manufacturing defects. The
+      craftsmanship is precise, with clean seams and a well-executed design. The distressed
+      effect appears intentional and part of the aesthetic. The boot seems to comply
+      with fashion and safety standards, offering both style and functionality. No
+      significant issues were found.\",\n        \"refusal\": null,\n        \"annotations\":
+      []\n      },\n      \"logprobs\": null,\n      \"finish_reason\": \"stop\"\n
+      \   }\n  ],\n  \"usage\": {\n    \"prompt_tokens\": 1300,\n    \"completion_tokens\":
+      250,\n    \"total_tokens\": 1550,\n    \"prompt_tokens_details\": {\n      \"cached_tokens\":
+      0,\n      \"audio_tokens\": 0\n    },\n    \"completion_tokens_details\": {\n
+      \     \"reasoning_tokens\": 0,\n      \"audio_tokens\": 0,\n      \"accepted_prediction_tokens\":
+      0,\n      \"rejected_prediction_tokens\": 0\n    }\n  },\n  \"service_tier\":
+      \"default\",\n  \"system_fingerprint\": \"fp_3a5b33c01a\"\n}\n"
+    headers:
+      CF-RAY:
+      - 926907e45f33f1e7-GRU
+      Connection:
+      - keep-alive
+      Content-Encoding:
+      - gzip
+      Content-Type:
+      - application/json
+      Date:
+      - Wed, 26 Mar 2025 19:25:01 GMT
+      Server:
+      - cloudflare
+      Transfer-Encoding:
+      - chunked
+      X-Content-Type-Options:
+      - nosniff
+      access-control-expose-headers:
+      - X-Request-ID
+      alt-svc:
+      - h3=":443"; ma=86400
+      cf-cache-status:
+      - DYNAMIC
+      openai-organization:
+      - crewai-iuxna1
+      openai-processing-ms:
+      - '7242'
+      openai-version:
+      - '2020-10-01'
+      strict-transport-security:
+      - max-age=31536000; includeSubDomains; preload
+      x-ratelimit-limit-input-images:
+      - '250000'
+      x-ratelimit-limit-requests:
+      - '50000'
+      x-ratelimit-limit-tokens:
+      - '150000000'
+      x-ratelimit-remaining-input-images:
+      - '249999'
+      x-ratelimit-remaining-requests:
+      - '49999'
+      x-ratelimit-remaining-tokens:
+      - '149998641'
+      x-ratelimit-reset-input-images:
+      - 0s
+      x-ratelimit-reset-requests:
+      - 1ms
+      x-ratelimit-reset-tokens:
+      - 0s
+      x-request-id:
+      - req_c5dd144c8ac1bb3bd96ffbba40707b2d
+    http_version: HTTP/1.1
+    status_code: 200
+version: 1
--- a/tests/crew_test.py
+++ b/tests/crew_test.py
@@ -3731,6 +3731,44 @@ def test_multimodal_agent_image_tool_handling():
        assert result["content"][1]["type"] == "image_url"


+@pytest.mark.vcr(filter_headers=["authorization"])
+def test_multimodal_agent_describing_image_successfully():
+    """
+    Test that a multimodal agent can process images without validation errors.
+    This test reproduces the scenario from issue #2475.
+    """
+    llm = LLM(model="openai/gpt-4o", temperature=0.7)  # model with vision capabilities
+
+    expert_analyst = Agent(
+        role="Visual Quality Inspector",
+        goal="Perform detailed quality analysis of product images",
+        backstory="Senior quality control expert with expertise in visual inspection",
+        llm=llm,
+        verbose=True,
+        allow_delegation=False,
+        multimodal=True,
+    )
+
+    inspection_task = Task(
+        description="""
+        Analyze the product image at https://www.us.maguireshoes.com/cdn/shop/files/FW24-Edito-Lucena-Distressed-01_1920x.jpg?v=1736371244 with focus on:
+        1. Quality of materials
+        2. Manufacturing defects
+        3. Compliance with standards
+        Provide a detailed report highlighting any issues found.
+        """,
+        expected_output="A detailed report highlighting any issues found",
+        agent=expert_analyst,
+    )
+
+    crew = Crew(agents=[expert_analyst], tasks=[inspection_task])
+    result = crew.kickoff()
+
+    task_output = result.tasks_output[0]
+    assert isinstance(task_output, TaskOutput)
+    assert task_output.raw == result.raw
+
+
@pytest.mark.vcr(filter_headers=["authorization"])
 def test_multimodal_agent_live_image_analysis():
    """
--- a/tests/test_multimodal_validation.py
+++ b/tests/test_multimodal_validation.py
@@ -0,0 +1,46 @@
+import os
+
+import pytest
+
+from crewai import LLM, Agent, Crew, Task
+
+
+@pytest.mark.skip(reason="Only run manually with valid API keys")
+def test_multimodal_agent_with_image_url():
+    """
+    Test that a multimodal agent can process images without validation errors.
+    This test reproduces the scenario from issue #2475.
+    """
+    OPENAI_API_KEY = os.getenv("OPENAI_API_KEY")
+    if not OPENAI_API_KEY:
+        pytest.skip("OPENAI_API_KEY environment variable not set")
+
+    llm = LLM(
+        model="openai/gpt-4o",  # model with vision capabilities
+        api_key=OPENAI_API_KEY,
+        temperature=0.7
+    )
+
+    expert_analyst = Agent(
+        role="Visual Quality Inspector",
+        goal="Perform detailed quality analysis of product images",
+        backstory="Senior quality control expert with expertise in visual inspection",
+        llm=llm,
+        verbose=True,
+        allow_delegation=False,
+        multimodal=True
+    )
+
+    inspection_task = Task(
+        description="""
+        Analyze the product image at https://www.us.maguireshoes.com/collections/spring-25/products/lucena-black-boot with focus on:
+        1. Quality of materials
+        2. Manufacturing defects
+        3. Compliance with standards
+        Provide a detailed report highlighting any issues found.
+        """,
+        expected_output="A detailed report highlighting any issues found",
+        agent=expert_analyst
+    )
+
+    crew = Crew(agents=[expert_analyst], tasks=[inspection_task])
--- a/tests/tools/test_tool_usage.py
+++ b/tests/tools/test_tool_usage.py
@@ -1,5 +1,7 @@
+import datetime
 import json
 import random
+import time
 from unittest.mock import MagicMock, patch

 import pytest
@@ -11,6 +13,7 @@ from crewai.tools.tool_usage import ToolUsage
 from crewai.utilities.events import crewai_event_bus
 from crewai.utilities.events.tool_usage_events import (
    ToolSelectionErrorEvent,
+    ToolUsageFinishedEvent,
    ToolValidateInputErrorEvent,
 )

@@ -624,3 +627,161 @@ def test_tool_validate_input_error_event():
        assert event.agent_role == "test_role"
        assert event.tool_name == "test_tool"
        assert "must be a valid dictionary" in event.error
+
+
+def test_tool_usage_finished_event_with_result():
+    """Test that ToolUsageFinishedEvent is emitted with correct result attributes."""
+    # Create mock agent with proper string values
+    mock_agent = MagicMock()
+    mock_agent.key = "test_agent_key"
+    mock_agent.role = "test_agent_role"
+    mock_agent._original_role = "test_agent_role"
+    mock_agent.i18n = MagicMock()
+    mock_agent.verbose = False
+
+    # Create mock task
+    mock_task = MagicMock()
+    mock_task.delegations = 0
+
+    # Create mock tool
+    class TestTool(BaseTool):
+        name: str = "Test Tool"
+        description: str = "A test tool"
+
+        def _run(self, input: dict) -> str:
+            return "test result"
+
+    test_tool = TestTool()
+
+    # Create mock tool calling
+    mock_tool_calling = MagicMock()
+    mock_tool_calling.arguments = {"arg1": "value1"}
+
+    # Create ToolUsage instance
+    tool_usage = ToolUsage(
+        tools_handler=MagicMock(),
+        tools=[test_tool],
+        original_tools=[test_tool],
+        tools_description="Test Tool Description",
+        tools_names="Test Tool",
+        task=mock_task,
+        function_calling_llm=None,
+        agent=mock_agent,
+        action=MagicMock(),
+    )
+
+    # Track received events
+    received_events = []
+
+    @crewai_event_bus.on(ToolUsageFinishedEvent)
+    def event_handler(source, event):
+        received_events.append(event)
+
+    # Call on_tool_use_finished with test data
+    started_at = time.time()
+    result = "test output result"
+    tool_usage.on_tool_use_finished(
+        tool=test_tool,
+        tool_calling=mock_tool_calling,
+        from_cache=False,
+        started_at=started_at,
+        result=result,
+    )
+
+    # Verify event was emitted
+    assert len(received_events) == 1, "Expected one event to be emitted"
+    event = received_events[0]
+    assert isinstance(event, ToolUsageFinishedEvent)
+
+    # Verify event attributes
+    assert event.agent_key == "test_agent_key"
+    assert event.agent_role == "test_agent_role"
+    assert event.tool_name == "Test Tool"
+    assert event.tool_args == {"arg1": "value1"}
+    assert event.tool_class == "TestTool"
+    assert event.run_attempts == 1  # Default value from ToolUsage
+    assert event.delegations == 0
+    assert event.from_cache is False
+    assert event.output == "test output result"
+    assert isinstance(event.started_at, datetime.datetime)
+    assert isinstance(event.finished_at, datetime.datetime)
+    assert event.type == "tool_usage_finished"
+
+
+def test_tool_usage_finished_event_with_cached_result():
+    """Test that ToolUsageFinishedEvent is emitted with correct result attributes when using cached result."""
+    # Create mock agent with proper string values
+    mock_agent = MagicMock()
+    mock_agent.key = "test_agent_key"
+    mock_agent.role = "test_agent_role"
+    mock_agent._original_role = "test_agent_role"
+    mock_agent.i18n = MagicMock()
+    mock_agent.verbose = False
+
+    # Create mock task
+    mock_task = MagicMock()
+    mock_task.delegations = 0
+
+    # Create mock tool
+    class TestTool(BaseTool):
+        name: str = "Test Tool"
+        description: str = "A test tool"
+
+        def _run(self, input: dict) -> str:
+            return "test result"
+
+    test_tool = TestTool()
+
+    # Create mock tool calling
+    mock_tool_calling = MagicMock()
+    mock_tool_calling.arguments = {"arg1": "value1"}
+
+    # Create ToolUsage instance
+    tool_usage = ToolUsage(
+        tools_handler=MagicMock(),
+        tools=[test_tool],
+        original_tools=[test_tool],
+        tools_description="Test Tool Description",
+        tools_names="Test Tool",
+        task=mock_task,
+        function_calling_llm=None,
+        agent=mock_agent,
+        action=MagicMock(),
+    )
+
+    # Track received events
+    received_events = []
+
+    @crewai_event_bus.on(ToolUsageFinishedEvent)
+    def event_handler(source, event):
+        received_events.append(event)
+
+    # Call on_tool_use_finished with test data and from_cache=True
+    started_at = time.time()
+    result = "cached test output result"
+    tool_usage.on_tool_use_finished(
+        tool=test_tool,
+        tool_calling=mock_tool_calling,
+        from_cache=True,
+        started_at=started_at,
+        result=result,
+    )
+
+    # Verify event was emitted
+    assert len(received_events) == 1, "Expected one event to be emitted"
+    event = received_events[0]
+    assert isinstance(event, ToolUsageFinishedEvent)
+
+    # Verify event attributes
+    assert event.agent_key == "test_agent_key"
+    assert event.agent_role == "test_agent_role"
+    assert event.tool_name == "Test Tool"
+    assert event.tool_args == {"arg1": "value1"}
+    assert event.tool_class == "TestTool"
+    assert event.run_attempts == 1  # Default value from ToolUsage
+    assert event.delegations == 0
+    assert event.from_cache is True
+    assert event.output == "cached test output result"
+    assert isinstance(event.started_at, datetime.datetime)
+    assert isinstance(event.finished_at, datetime.datetime)
+    assert event.type == "tool_usage_finished"
Author	SHA1	Message	Date
Eduardo Chiarotti	c37a75797a	Merge branch 'main' into feat/trace-ui-execution	2025-03-26 16:47:19 -03:00
Lucas Gomide	73701fda1e	Merge pull request #2476 from crewAIInc/devin/1742990927-fix-issue-2475 Fix multimodal agent validation errors with image processing	2025-03-26 16:40:23 -03:00
lucasgomide	3deeba4cab	test: adding missing test to ensure multimodal content structures	2025-03-26 16:30:17 -03:00
Devin AI	e3dde17af0	docs: improve LLMCallStartedEvent docstring to clarify multimodal support Co-Authored-By: Joe Moura <joao@crewai.com>	2025-03-26 16:29:24 -03:00
Devin AI	49b8cc95ae	fix: update LLMCallStartedEvent message type to support multimodal content (#2475 ) fix: sort imports in test file to fix linting fix: properly sort imports with ruff Co-Authored-By: Joe Moura <joao@crewai.com>	2025-03-26 16:29:15 -03:00
Eduardo Chiarotti	f71aae97e0	feat: add tests	2025-03-26 13:52:17 -03:00
Eduardo Chiarotti	161f552c77	Merge branch 'main' into feat/trace-ui-execution	2025-03-26 13:39:06 -03:00
Tony Kipkemboi	0b58911153	Merge pull request #2482 from crewAIInc/docs/improve-observability docs: update theme to mint and modify opik observability doc	2025-03-26 11:40:45 -04:00
Eduardo Chiarotti	7c5160bc92	feat: add type ignore	2025-03-26 10:07:35 -03:00
Eduardo Chiarotti	fbd9d832ef	feat: add output to ToolUsageFinishedEvent	2025-03-26 09:58:37 -03:00