From bb4890a1718a17c7542b62f0e6f14bb763c01d8d Mon Sep 17 00:00:00 2001 From: lorenzejay Date: Tue, 26 May 2026 09:30:18 -0700 Subject: [PATCH] Update Anthropic test cases to use new model and tool structure - Changed the model from "claude-3-5-haiku-20241022" to "claude-sonnet-4-6" in the test setup. - Updated the request and response formats in the YAML test cassette to reflect the new tool structure and improved content formatting. - Adjusted the expected response body to match the new output format from the assistant, including changes in tool usage and response details. - Increased rate limit values in the response headers for better testing scenarios. --- ..._kickoff_structured_output_with_tools.yaml | 275 +++++++++++++++++- .../tests/llms/anthropic/test_anthropic.py | 2 +- 2 files changed, 260 insertions(+), 17 deletions(-) diff --git a/lib/crewai/tests/cassettes/llms/anthropic/test_anthropic_agent_kickoff_structured_output_with_tools.yaml b/lib/crewai/tests/cassettes/llms/anthropic/test_anthropic_agent_kickoff_structured_output_with_tools.yaml index 9eec54f1a..9b717aedf 100644 --- a/lib/crewai/tests/cassettes/llms/anthropic/test_anthropic_agent_kickoff_structured_output_with_tools.yaml +++ b/lib/crewai/tests/cassettes/llms/anthropic/test_anthropic_agent_kickoff_structured_output_with_tools.yaml @@ -1,14 +1,10 @@ interactions: - request: - body: '{"max_tokens":4096,"messages":[{"role":"user","content":"\nCurrent Task: - Calculate 15 + 27 using your add_numbers tool. Report the result."}],"model":"claude-3-5-haiku-20241022","stop_sequences":["\nObservation:"],"stream":false,"system":"You + body: '{"max_tokens":4096,"messages":[{"role":"user","content":[{"type":"text","text":"\nCurrent + Task: Calculate 15 + 27 using your add_numbers tool. Report the result.","cache_control":{"type":"ephemeral"}}]}],"model":"claude-sonnet-4-6","stop_sequences":["\nObservation:"],"stream":false,"system":[{"type":"text","text":"You are Calculator. You are a calculator assistant that uses tools to compute results.\nYour - personal goal is: Perform calculations using available tools","tool_choice":{"type":"tool","name":"structured_output"},"tools":[{"name":"structured_output","description":"Output - the structured response","input_schema":{"type":"object","description":"Structured - output for calculation results.","title":"CalculationResult","properties":{"operation":{"type":"string","description":"The - mathematical operation performed","title":"Operation"},"result":{"type":"integer","description":"The - result of the calculation","title":"Result"},"explanation":{"type":"string","description":"Brief - explanation of the calculation","title":"Explanation"}},"additionalProperties":false,"required":["operation","result","explanation"]}}]}' + personal goal is: Perform calculations using available tools","cache_control":{"type":"ephemeral"}}],"tools":[{"name":"add_numbers","description":"Add + two numbers together and return the sum.","input_schema":{"properties":{"a":{"type":"integer"},"b":{"type":"integer"}},"required":["a","b"],"type":"object","additionalProperties":false},"strict":true}]}' headers: User-Agent: - X-USER-AGENT-XXX @@ -21,7 +17,7 @@ interactions: connection: - keep-alive content-length: - - '1050' + - '762' content-type: - application/json host: @@ -50,8 +46,8 @@ interactions: uri: https://api.anthropic.com/v1/messages response: body: - string: '{"model":"claude-3-5-haiku-20241022","id":"msg_01MsoNSVoPuoMYGCcJLvfXS6","type":"message","role":"assistant","content":[{"type":"tool_use","id":"toolu_01TtAsqddWjE7C4GYmCKavdg","name":"structured_output","input":{"operation":"Addition","result":42,"explanation":"Added - 15 and 27 together"}}],"stop_reason":"tool_use","stop_sequence":null,"usage":{"input_tokens":573,"cache_creation_input_tokens":0,"cache_read_input_tokens":0,"cache_creation":{"ephemeral_5m_input_tokens":0,"ephemeral_1h_input_tokens":0},"output_tokens":75,"service_tier":"standard","inference_geo":"not_available"}}' + string: '{"model":"claude-sonnet-4-6","id":"msg_01Hi6qhbcDQJas5A7kqA7Xif","type":"message","role":"assistant","content":[{"type":"text","text":"I''ll + calculate 15 + 27 right away using the `add_numbers` tool!"},{"type":"tool_use","id":"toolu_0111pu1MrxY8aTqNzYSWAgQF","name":"add_numbers","input":{"a":15,"b":27},"caller":{"type":"direct"}}],"stop_reason":"tool_use","stop_sequence":null,"stop_details":null,"usage":{"input_tokens":633,"cache_creation_input_tokens":0,"cache_read_input_tokens":0,"cache_creation":{"ephemeral_5m_input_tokens":0,"ephemeral_1h_input_tokens":0},"output_tokens":92,"service_tier":"standard","inference_geo":"global"}}' headers: CF-RAY: - CF-RAY-XXX @@ -62,7 +58,7 @@ interactions: Content-Type: - application/json Date: - - Thu, 12 Feb 2026 22:11:20 GMT + - Tue, 26 May 2026 16:27:07 GMT Server: - cloudflare Transfer-Encoding: @@ -84,11 +80,123 @@ interactions: anthropic-ratelimit-output-tokens-reset: - ANTHROPIC-RATELIMIT-OUTPUT-TOKENS-RESET-XXX anthropic-ratelimit-requests-limit: - - '4000' + - '20000' anthropic-ratelimit-requests-remaining: - - '3999' + - '19999' anthropic-ratelimit-requests-reset: - - '2026-02-12T22:11:18Z' + - '2026-05-26T16:26:47Z' + anthropic-ratelimit-tokens-limit: + - ANTHROPIC-RATELIMIT-TOKENS-LIMIT-XXX + anthropic-ratelimit-tokens-remaining: + - ANTHROPIC-RATELIMIT-TOKENS-REMAINING-XXX + anthropic-ratelimit-tokens-reset: + - ANTHROPIC-RATELIMIT-TOKENS-RESET-XXX + cf-cache-status: + - DYNAMIC + request-id: + - REQUEST-ID-XXX + set-cookie: + - SET-COOKIE-XXX + strict-transport-security: + - STS-XXX + traceresponse: + - 00-0c532faf4d60e2359554c05e42d6c2b0-8ed2bd3616e95a40-01 + vary: + - Accept-Encoding + x-envoy-upstream-service-time: + - '19361' + status: + code: 200 + message: OK +- request: + body: '{"max_tokens":4096,"messages":[{"role":"user","content":[{"type":"text","text":"\nCurrent + Task: Calculate 15 + 27 using your add_numbers tool. Report the result.","cache_control":{"type":"ephemeral"}}]},{"role":"assistant","content":[{"type":"tool_use","id":"toolu_0111pu1MrxY8aTqNzYSWAgQF","name":"add_numbers","input":{"a":15,"b":27}}]},{"role":"user","content":[{"type":"tool_result","tool_use_id":"toolu_0111pu1MrxY8aTqNzYSWAgQF","content":"42"}]}],"model":"claude-sonnet-4-6","stop_sequences":["\nObservation:"],"stream":false,"system":[{"type":"text","text":"You + are Calculator. You are a calculator assistant that uses tools to compute results.\nYour + personal goal is: Perform calculations using available tools","cache_control":{"type":"ephemeral"}}],"tools":[{"name":"add_numbers","description":"Add + two numbers together and return the sum.","input_schema":{"properties":{"a":{"type":"integer"},"b":{"type":"integer"}},"required":["a","b"],"type":"object","additionalProperties":false},"strict":true}]}' + headers: + User-Agent: + - X-USER-AGENT-XXX + accept: + - application/json + accept-encoding: + - ACCEPT-ENCODING-XXX + anthropic-version: + - '2023-06-01' + connection: + - keep-alive + content-length: + - '1011' + content-type: + - application/json + cookie: + - COOKIE-XXX + host: + - api.anthropic.com + x-api-key: + - X-API-KEY-XXX + x-stainless-arch: + - X-STAINLESS-ARCH-XXX + x-stainless-async: + - 'false' + x-stainless-lang: + - python + x-stainless-os: + - X-STAINLESS-OS-XXX + x-stainless-package-version: + - 0.73.0 + x-stainless-retry-count: + - '0' + x-stainless-runtime: + - CPython + x-stainless-runtime-version: + - 3.13.3 + x-stainless-timeout: + - NOT_GIVEN + method: POST + uri: https://api.anthropic.com/v1/messages + response: + body: + string: "{\"model\":\"claude-sonnet-4-6\",\"id\":\"msg_01Eqs4XnbiSubo442YWNnn7G\",\"type\":\"message\",\"role\":\"assistant\",\"content\":[{\"type\":\"text\",\"text\":\"The + result of **15 + 27 = 42**! \U0001F9EE\\n\\nUsing the `add_numbers` tool, + I calculated that adding 15 and 27 together gives you a sum of **42**.\"}],\"stop_reason\":\"end_turn\",\"stop_sequence\":null,\"stop_details\":null,\"usage\":{\"input_tokens\":717,\"cache_creation_input_tokens\":0,\"cache_read_input_tokens\":0,\"cache_creation\":{\"ephemeral_5m_input_tokens\":0,\"ephemeral_1h_input_tokens\":0},\"output_tokens\":52,\"service_tier\":\"standard\",\"inference_geo\":\"global\"}}" + headers: + CF-RAY: + - CF-RAY-XXX + Connection: + - keep-alive + Content-Security-Policy: + - CSP-FILTERED + Content-Type: + - application/json + Date: + - Tue, 26 May 2026 16:27:09 GMT + Server: + - cloudflare + Transfer-Encoding: + - chunked + X-Robots-Tag: + - none + anthropic-organization-id: + - ANTHROPIC-ORGANIZATION-ID-XXX + anthropic-ratelimit-input-tokens-limit: + - ANTHROPIC-RATELIMIT-INPUT-TOKENS-LIMIT-XXX + anthropic-ratelimit-input-tokens-remaining: + - ANTHROPIC-RATELIMIT-INPUT-TOKENS-REMAINING-XXX + anthropic-ratelimit-input-tokens-reset: + - ANTHROPIC-RATELIMIT-INPUT-TOKENS-RESET-XXX + anthropic-ratelimit-output-tokens-limit: + - ANTHROPIC-RATELIMIT-OUTPUT-TOKENS-LIMIT-XXX + anthropic-ratelimit-output-tokens-remaining: + - ANTHROPIC-RATELIMIT-OUTPUT-TOKENS-REMAINING-XXX + anthropic-ratelimit-output-tokens-reset: + - ANTHROPIC-RATELIMIT-OUTPUT-TOKENS-RESET-XXX + anthropic-ratelimit-requests-limit: + - '20000' + anthropic-ratelimit-requests-remaining: + - '19999' + anthropic-ratelimit-requests-reset: + - '2026-05-26T16:27:07Z' anthropic-ratelimit-tokens-limit: - ANTHROPIC-RATELIMIT-TOKENS-LIMIT-XXX anthropic-ratelimit-tokens-remaining: @@ -101,8 +209,143 @@ interactions: - REQUEST-ID-XXX strict-transport-security: - STS-XXX + traceresponse: + - 00-92d206559118bcf8b9b39b7ecd0c6e0e-68dd5efb3bbaff0e-01 + vary: + - Accept-Encoding x-envoy-upstream-service-time: - - '1234' + - '2021' + status: + code: 200 + message: OK +- request: + body: "{\"max_tokens\":4096,\"messages\":[{\"role\":\"user\",\"content\":\"The + result of **15 + 27 = 42**! \U0001F9EE\\n\\nUsing the `add_numbers` tool, I + calculated that adding 15 and 27 together gives you a sum of **42**.\"}],\"model\":\"claude-sonnet-4-6\",\"stream\":false,\"system\":\"Format + your final answer according to the following OpenAPI schema: {\\n \\\"type\\\": + \\\"json_schema\\\",\\n \\\"json_schema\\\": {\\n \\\"name\\\": \\\"CalculationResult\\\",\\n + \ \\\"strict\\\": true,\\n \\\"schema\\\": {\\n \\\"description\\\": + \\\"Structured output for calculation results.\\\",\\n \\\"properties\\\": + {\\n \\\"operation\\\": {\\n \\\"description\\\": \\\"The mathematical + operation performed\\\",\\n \\\"title\\\": \\\"Operation\\\",\\n \\\"type\\\": + \\\"string\\\"\\n },\\n \\\"result\\\": {\\n \\\"description\\\": + \\\"The result of the calculation\\\",\\n \\\"title\\\": \\\"Result\\\",\\n + \ \\\"type\\\": \\\"integer\\\"\\n },\\n \\\"explanation\\\": + {\\n \\\"description\\\": \\\"Brief explanation of the calculation\\\",\\n + \ \\\"title\\\": \\\"Explanation\\\",\\n \\\"type\\\": \\\"string\\\"\\n + \ }\\n },\\n \\\"required\\\": [\\n \\\"operation\\\",\\n + \ \\\"result\\\",\\n \\\"explanation\\\"\\n ],\\n \\\"title\\\": + \\\"CalculationResult\\\",\\n \\\"type\\\": \\\"object\\\",\\n \\\"additionalProperties\\\": + false\\n }\\n }\\n}\\n\\nIMPORTANT: Preserve the original content exactly + as-is. Do NOT rewrite, paraphrase, or modify the meaning of the content. Only + structure it to match the schema format.\\n\\nDo not include the OpenAPI schema + in the final output. Ensure the final output does not include any code block + markers like ```json or ```python.\",\"tool_choice\":{\"type\":\"tool\",\"name\":\"structured_output\"},\"tools\":[{\"name\":\"structured_output\",\"description\":\"Output + the structured response\",\"input_schema\":{\"type\":\"object\",\"description\":\"Structured + output for calculation results.\",\"title\":\"CalculationResult\",\"properties\":{\"operation\":{\"type\":\"string\",\"description\":\"The + mathematical operation performed\",\"title\":\"Operation\"},\"result\":{\"type\":\"integer\",\"description\":\"The + result of the calculation\",\"title\":\"Result\"},\"explanation\":{\"type\":\"string\",\"description\":\"Brief + explanation of the calculation\",\"title\":\"Explanation\"}},\"additionalProperties\":false,\"required\":[\"operation\",\"result\",\"explanation\"]}}]}" + headers: + User-Agent: + - X-USER-AGENT-XXX + accept: + - application/json + accept-encoding: + - ACCEPT-ENCODING-XXX + anthropic-version: + - '2023-06-01' + connection: + - keep-alive + content-length: + - '2289' + content-type: + - application/json + cookie: + - COOKIE-XXX + host: + - api.anthropic.com + x-api-key: + - X-API-KEY-XXX + x-stainless-arch: + - X-STAINLESS-ARCH-XXX + x-stainless-async: + - 'false' + x-stainless-lang: + - python + x-stainless-os: + - X-STAINLESS-OS-XXX + x-stainless-package-version: + - 0.73.0 + x-stainless-retry-count: + - '0' + x-stainless-runtime: + - CPython + x-stainless-runtime-version: + - 3.13.3 + x-stainless-timeout: + - NOT_GIVEN + method: POST + uri: https://api.anthropic.com/v1/messages + response: + body: + string: '{"model":"claude-sonnet-4-6","id":"msg_012b6HHB7T6qZuKcXB5jkhpx","type":"message","role":"assistant","content":[{"type":"tool_use","id":"toolu_01UX2REqVJnQ3qNjHzVV1oNp","name":"structured_output","input":{"operation":"add_numbers","result":42,"explanation":"Adding + 15 and 27 together gives you a sum of 42."},"caller":{"type":"direct"}}],"stop_reason":"tool_use","stop_sequence":null,"stop_details":null,"usage":{"input_tokens":1123,"cache_creation_input_tokens":0,"cache_read_input_tokens":0,"cache_creation":{"ephemeral_5m_input_tokens":0,"ephemeral_1h_input_tokens":0},"output_tokens":84,"service_tier":"standard","inference_geo":"global"}}' + headers: + CF-RAY: + - CF-RAY-XXX + Connection: + - keep-alive + Content-Security-Policy: + - CSP-FILTERED + Content-Type: + - application/json + Date: + - Tue, 26 May 2026 16:27:11 GMT + Server: + - cloudflare + Transfer-Encoding: + - chunked + X-Robots-Tag: + - none + anthropic-organization-id: + - ANTHROPIC-ORGANIZATION-ID-XXX + anthropic-ratelimit-input-tokens-limit: + - ANTHROPIC-RATELIMIT-INPUT-TOKENS-LIMIT-XXX + anthropic-ratelimit-input-tokens-remaining: + - ANTHROPIC-RATELIMIT-INPUT-TOKENS-REMAINING-XXX + anthropic-ratelimit-input-tokens-reset: + - ANTHROPIC-RATELIMIT-INPUT-TOKENS-RESET-XXX + anthropic-ratelimit-output-tokens-limit: + - ANTHROPIC-RATELIMIT-OUTPUT-TOKENS-LIMIT-XXX + anthropic-ratelimit-output-tokens-remaining: + - ANTHROPIC-RATELIMIT-OUTPUT-TOKENS-REMAINING-XXX + anthropic-ratelimit-output-tokens-reset: + - ANTHROPIC-RATELIMIT-OUTPUT-TOKENS-RESET-XXX + anthropic-ratelimit-requests-limit: + - '20000' + anthropic-ratelimit-requests-remaining: + - '19999' + anthropic-ratelimit-requests-reset: + - '2026-05-26T16:27:09Z' + anthropic-ratelimit-tokens-limit: + - ANTHROPIC-RATELIMIT-TOKENS-LIMIT-XXX + anthropic-ratelimit-tokens-remaining: + - ANTHROPIC-RATELIMIT-TOKENS-REMAINING-XXX + anthropic-ratelimit-tokens-reset: + - ANTHROPIC-RATELIMIT-TOKENS-RESET-XXX + cf-cache-status: + - DYNAMIC + request-id: + - REQUEST-ID-XXX + strict-transport-security: + - STS-XXX + traceresponse: + - 00-c97065bea414b95b086529ea43832d89-d5c4bad6e383ad9f-01 + vary: + - Accept-Encoding + x-envoy-upstream-service-time: + - '2133' status: code: 200 message: OK diff --git a/lib/crewai/tests/llms/anthropic/test_anthropic.py b/lib/crewai/tests/llms/anthropic/test_anthropic.py index 81a51c8d6..8bb60d912 100644 --- a/lib/crewai/tests/llms/anthropic/test_anthropic.py +++ b/lib/crewai/tests/llms/anthropic/test_anthropic.py @@ -975,7 +975,7 @@ def test_anthropic_agent_kickoff_structured_output_with_tools(): role="Calculator", goal="Perform calculations using available tools", backstory="You are a calculator assistant that uses tools to compute results.", - llm=LLM(model="anthropic/claude-3-5-haiku-20241022"), + llm=LLM(model="anthropic/claude-sonnet-4-6"), tools=[add_numbers], verbose=True, )