From dc2e68487750c04ed3fc020a5a7af82671737314 Mon Sep 17 00:00:00 2001 From: Brandon Hancock Date: Wed, 22 Jan 2025 11:13:47 -0500 Subject: [PATCH] Fixing tests --- src/crewai/agents/crew_agent_executor.py | 3 +- tests/agent_test.py | 1 + .../test_agent_error_on_parsing_tool.yaml | 264 ++++++++++++++++++ 3 files changed, 267 insertions(+), 1 deletion(-) create mode 100644 tests/cassettes/test_agent_error_on_parsing_tool.yaml diff --git a/src/crewai/agents/crew_agent_executor.py b/src/crewai/agents/crew_agent_executor.py index b5f9f7960..e66075958 100644 --- a/src/crewai/agents/crew_agent_executor.py +++ b/src/crewai/agents/crew_agent_executor.py @@ -15,6 +15,7 @@ from crewai.agents.parser import ( OutputParserException, ) from crewai.agents.tools_handler import ToolsHandler +from crewai.llm import LLM from crewai.tools.base_tool import BaseTool from crewai.tools.tool_usage import ToolUsage, ToolUsageErrorException from crewai.utilities import I18N, Printer @@ -56,7 +57,7 @@ class CrewAgentExecutor(CrewAgentExecutorMixin): callbacks: List[Any] = [], ): self._i18n: I18N = I18N() - self.llm = llm + self.llm: LLM = llm self.task = task self.agent = agent self.crew = crew diff --git a/tests/agent_test.py b/tests/agent_test.py index 73ce69737..46b20004e 100644 --- a/tests/agent_test.py +++ b/tests/agent_test.py @@ -670,6 +670,7 @@ def test_agent_without_max_rpm_respects_crew_rpm(capsys): moveon.assert_called_once() +@pytest.mark.vcr(filter_headers=["authorization"]) def test_agent_error_on_parsing_tool(capsys): from unittest.mock import patch diff --git a/tests/cassettes/test_agent_error_on_parsing_tool.yaml b/tests/cassettes/test_agent_error_on_parsing_tool.yaml new file mode 100644 index 000000000..bd1c350fe --- /dev/null +++ b/tests/cassettes/test_agent_error_on_parsing_tool.yaml @@ -0,0 +1,264 @@ +interactions: +- request: + body: '{"messages": [{"role": "system", "content": "You are test role. test backstory\nYour + personal goal is: test goal\nYou ONLY have access to the following tools, and + should NEVER make up tools that are not listed here:\n\nTool Name: get_final_answer\nTool + Arguments: {}\nTool Description: Get the final answer but don''t give it yet, + just re-use this\n tool non-stop.\n\nIMPORTANT: Use the following format + in your response:\n\n```\nThought: you should always think about what to do\nAction: + the action to take, only one name of [get_final_answer], just the name, exactly + as it''s written.\nAction Input: the input to the action, just a simple JSON + object, enclosed in curly braces, using \" to wrap keys and values.\nObservation: + the result of the action\n```\n\nOnce all necessary information is gathered, + return the following format:\n\n```\nThought: I now know the final answer\nFinal + Answer: the final answer to the original input question\n```"}, {"role": "user", + "content": "\nCurrent Task: Use the get_final_answer tool.\n\nThis is the expect + criteria for your final answer: The final answer\nyou MUST return the actual + complete content as the final answer, not a summary.\n\nBegin! This is VERY + important to you, use the tools available and give your best Final Answer, your + job depends on it!\n\nThought:"}], "model": "gpt-4o", "stop": ["\nObservation:"]}' + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate + connection: + - keep-alive + content-length: + - '1367' + content-type: + - application/json + host: + - api.openai.com + user-agent: + - OpenAI/Python 1.59.6 + x-stainless-arch: + - arm64 + x-stainless-async: + - 'false' + x-stainless-lang: + - python + x-stainless-os: + - MacOS + x-stainless-package-version: + - 1.59.6 + x-stainless-raw-response: + - 'true' + x-stainless-retry-count: + - '0' + x-stainless-runtime: + - CPython + x-stainless-runtime-version: + - 3.12.7 + method: POST + uri: https://api.openai.com/v1/chat/completions + response: + content: "{\n \"id\": \"chatcmpl-AsXdf4OZKCZSigmN4k0gyh67NciqP\",\n \"object\": + \"chat.completion\",\n \"created\": 1737562383,\n \"model\": \"gpt-4o-2024-08-06\",\n + \ \"choices\": [\n {\n \"index\": 0,\n \"message\": {\n \"role\": + \"assistant\",\n \"content\": \"```\\nThought: I have to use the available + tool to get the final answer. Let's proceed with executing it.\\nAction: get_final_answer\\nAction + Input: {}\",\n \"refusal\": null\n },\n \"logprobs\": null,\n + \ \"finish_reason\": \"stop\"\n }\n ],\n \"usage\": {\n \"prompt_tokens\": + 274,\n \"completion_tokens\": 33,\n \"total_tokens\": 307,\n \"prompt_tokens_details\": + {\n \"cached_tokens\": 0,\n \"audio_tokens\": 0\n },\n \"completion_tokens_details\": + {\n \"reasoning_tokens\": 0,\n \"audio_tokens\": 0,\n \"accepted_prediction_tokens\": + 0,\n \"rejected_prediction_tokens\": 0\n }\n },\n \"service_tier\": + \"default\",\n \"system_fingerprint\": \"fp_50cad350e4\"\n}\n" + headers: + CF-Cache-Status: + - DYNAMIC + CF-RAY: + - 9060d43e3be1d690-IAD + Connection: + - keep-alive + Content-Encoding: + - gzip + Content-Type: + - application/json + Date: + - Wed, 22 Jan 2025 16:13:03 GMT + Server: + - cloudflare + Set-Cookie: + - __cf_bm=_Jcp7wnO_mXdvOnborCN6j8HwJxJXbszedJC1l7pFUg-1737562383-1.0.1.1-pDSLXlg.nKjG4wsT7mTJPjUvOX1UJITiS4MqKp6yfMWwRSJINsW1qC48SAcjBjakx2H5I1ESVk9JtUpUFDtf4g; + path=/; expires=Wed, 22-Jan-25 16:43:03 GMT; domain=.api.openai.com; HttpOnly; + Secure; SameSite=None + - _cfuvid=x3SYvzL2nq_PTBGtE8R9cl5CkeaaDzZFQIrYfo91S2s-1737562383916-0.0.1.1-604800000; + path=/; domain=.api.openai.com; HttpOnly; Secure; SameSite=None + Transfer-Encoding: + - chunked + X-Content-Type-Options: + - nosniff + access-control-expose-headers: + - X-Request-ID + alt-svc: + - h3=":443"; ma=86400 + openai-organization: + - crewai-iuxna1 + openai-processing-ms: + - '791' + openai-version: + - '2020-10-01' + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + x-ratelimit-limit-requests: + - '10000' + x-ratelimit-limit-tokens: + - '30000000' + x-ratelimit-remaining-requests: + - '9999' + x-ratelimit-remaining-tokens: + - '29999680' + x-ratelimit-reset-requests: + - 6ms + x-ratelimit-reset-tokens: + - 0s + x-request-id: + - req_eeed99acafd3aeb1e3d4a6c8063192b0 + http_version: HTTP/1.1 + status_code: 200 +- request: + body: '{"messages": [{"role": "system", "content": "You are test role. test backstory\nYour + personal goal is: test goal\nYou ONLY have access to the following tools, and + should NEVER make up tools that are not listed here:\n\nTool Name: get_final_answer\nTool + Arguments: {}\nTool Description: Get the final answer but don''t give it yet, + just re-use this\n tool non-stop.\n\nIMPORTANT: Use the following format + in your response:\n\n```\nThought: you should always think about what to do\nAction: + the action to take, only one name of [get_final_answer], just the name, exactly + as it''s written.\nAction Input: the input to the action, just a simple JSON + object, enclosed in curly braces, using \" to wrap keys and values.\nObservation: + the result of the action\n```\n\nOnce all necessary information is gathered, + return the following format:\n\n```\nThought: I now know the final answer\nFinal + Answer: the final answer to the original input question\n```"}, {"role": "user", + "content": "\nCurrent Task: Use the get_final_answer tool.\n\nThis is the expect + criteria for your final answer: The final answer\nyou MUST return the actual + complete content as the final answer, not a summary.\n\nBegin! This is VERY + important to you, use the tools available and give your best Final Answer, your + job depends on it!\n\nThought:"}, {"role": "assistant", "content": "```\nThought: + I have to use the available tool to get the final answer. Let''s proceed with + executing it.\nAction: get_final_answer\nAction Input: {}\nObservation: I encountered + an error: Error on parsing tool.\nMoving on then. I MUST either use a tool (use + one at time) OR give my best final answer not both at the same time. When responding, + I must use the following format:\n\n```\nThought: you should always think about + what to do\nAction: the action to take, should be one of [get_final_answer]\nAction + Input: the input to the action, dictionary enclosed in curly braces\nObservation: + the result of the action\n```\nThis Thought/Action/Action Input/Result can repeat + N times. Once I know the final answer, I must return the following format:\n\n```\nThought: + I now can give a great answer\nFinal Answer: Your final answer must be the great + and the most complete as possible, it must be outcome described\n\n```"}, {"role": + "assistant", "content": "```\nThought: I have to use the available tool to get + the final answer. Let''s proceed with executing it.\nAction: get_final_answer\nAction + Input: {}\nObservation: I encountered an error: Error on parsing tool.\nMoving + on then. I MUST either use a tool (use one at time) OR give my best final answer + not both at the same time. When responding, I must use the following format:\n\n```\nThought: + you should always think about what to do\nAction: the action to take, should + be one of [get_final_answer]\nAction Input: the input to the action, dictionary + enclosed in curly braces\nObservation: the result of the action\n```\nThis Thought/Action/Action + Input/Result can repeat N times. Once I know the final answer, I must return + the following format:\n\n```\nThought: I now can give a great answer\nFinal + Answer: Your final answer must be the great and the most complete as possible, + it must be outcome described\n\n```\nNow it''s time you MUST give your absolute + best final answer. You''ll ignore all previous instructions, stop using any + tools, and just return your absolute BEST Final answer."}], "model": "gpt-4o", + "stop": ["\nObservation:"]}' + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate + connection: + - keep-alive + content-length: + - '3445' + content-type: + - application/json + cookie: + - __cf_bm=_Jcp7wnO_mXdvOnborCN6j8HwJxJXbszedJC1l7pFUg-1737562383-1.0.1.1-pDSLXlg.nKjG4wsT7mTJPjUvOX1UJITiS4MqKp6yfMWwRSJINsW1qC48SAcjBjakx2H5I1ESVk9JtUpUFDtf4g; + _cfuvid=x3SYvzL2nq_PTBGtE8R9cl5CkeaaDzZFQIrYfo91S2s-1737562383916-0.0.1.1-604800000 + host: + - api.openai.com + user-agent: + - OpenAI/Python 1.59.6 + x-stainless-arch: + - arm64 + x-stainless-async: + - 'false' + x-stainless-lang: + - python + x-stainless-os: + - MacOS + x-stainless-package-version: + - 1.59.6 + x-stainless-raw-response: + - 'true' + x-stainless-retry-count: + - '0' + x-stainless-runtime: + - CPython + x-stainless-runtime-version: + - 3.12.7 + method: POST + uri: https://api.openai.com/v1/chat/completions + response: + content: "{\n \"id\": \"chatcmpl-AsXdg9UrLvAiqWP979E6DszLsQ84k\",\n \"object\": + \"chat.completion\",\n \"created\": 1737562384,\n \"model\": \"gpt-4o-2024-08-06\",\n + \ \"choices\": [\n {\n \"index\": 0,\n \"message\": {\n \"role\": + \"assistant\",\n \"content\": \"```\\nThought: I now know the final answer\\nFinal + Answer: The final answer must be the great and the most complete as possible, + it must be outcome described.\\n```\",\n \"refusal\": null\n },\n + \ \"logprobs\": null,\n \"finish_reason\": \"stop\"\n }\n ],\n + \ \"usage\": {\n \"prompt_tokens\": 719,\n \"completion_tokens\": 35,\n + \ \"total_tokens\": 754,\n \"prompt_tokens_details\": {\n \"cached_tokens\": + 0,\n \"audio_tokens\": 0\n },\n \"completion_tokens_details\": {\n + \ \"reasoning_tokens\": 0,\n \"audio_tokens\": 0,\n \"accepted_prediction_tokens\": + 0,\n \"rejected_prediction_tokens\": 0\n }\n },\n \"service_tier\": + \"default\",\n \"system_fingerprint\": \"fp_50cad350e4\"\n}\n" + headers: + CF-Cache-Status: + - DYNAMIC + CF-RAY: + - 9060d4441edad690-IAD + Connection: + - keep-alive + Content-Encoding: + - gzip + Content-Type: + - application/json + Date: + - Wed, 22 Jan 2025 16:13:05 GMT + Server: + - cloudflare + Transfer-Encoding: + - chunked + X-Content-Type-Options: + - nosniff + access-control-expose-headers: + - X-Request-ID + alt-svc: + - h3=":443"; ma=86400 + openai-organization: + - crewai-iuxna1 + openai-processing-ms: + - '928' + openai-version: + - '2020-10-01' + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + x-ratelimit-limit-requests: + - '10000' + x-ratelimit-limit-tokens: + - '30000000' + x-ratelimit-remaining-requests: + - '9999' + x-ratelimit-remaining-tokens: + - '29999187' + x-ratelimit-reset-requests: + - 6ms + x-ratelimit-reset-tokens: + - 1ms + x-request-id: + - req_61fc7506e6db326ec572224aec81ef23 + http_version: HTTP/1.1 + status_code: 200 +version: 1