From edaa966e2688596184529daeee546271ceed2e51 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Sun, 29 Dec 2024 05:07:27 +0000 Subject: [PATCH] test: add test_kickoff_for_each_output_file_interpolation Co-Authored-By: Joe Moura --- ...ff_for_each_output_file_interpolation.yaml | 236 ++++++++++++++++++ tests/crew_test.py | 47 ++++ 2 files changed, 283 insertions(+) create mode 100644 tests/cassettes/test_kickoff_for_each_output_file_interpolation.yaml diff --git a/tests/cassettes/test_kickoff_for_each_output_file_interpolation.yaml b/tests/cassettes/test_kickoff_for_each_output_file_interpolation.yaml new file mode 100644 index 000000000..b3180c833 --- /dev/null +++ b/tests/cassettes/test_kickoff_for_each_output_file_interpolation.yaml @@ -0,0 +1,236 @@ +interactions: +- request: + body: '{"messages": [{"role": "system", "content": "You are TestAgent. Testing + interpolation logic\nYour personal goal is: Validate output_file interpolation + with kickoff_for_each\nTo give my best complete final answer to the task use + the exact following format:\n\nThought: I now can give a great answer\nFinal + Answer: Your final answer must be the great and the most complete as possible, + it must be outcome described.\n\nI MUST use these formats, my job depends on + it!"}, {"role": "user", "content": "\nCurrent Task: Create a brief summary for + the ID: alpha\n\nThis is the expect criteria for your final answer: Should write + a file with the ID included in the path\nyou MUST return the actual complete + content as the final answer, not a summary.\n\nBegin! This is VERY important + to you, use the tools available and give your best Final Answer, your job depends + on it!\n\nThought:"}], "model": "gpt-4o-mini", "stop": ["\nObservation:"], "stream": + false}' + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate + connection: + - keep-alive + content-length: + - '948' + content-type: + - application/json + host: + - api.openai.com + user-agent: + - OpenAI/Python 1.52.1 + x-stainless-arch: + - x64 + x-stainless-async: + - 'false' + x-stainless-lang: + - python + x-stainless-os: + - Linux + x-stainless-package-version: + - 1.52.1 + x-stainless-raw-response: + - 'true' + x-stainless-retry-count: + - '0' + x-stainless-runtime: + - CPython + x-stainless-runtime-version: + - 3.12.7 + method: POST + uri: https://api.openai.com/v1/chat/completions + response: + content: "{\n \"id\": \"chatcmpl-AjflQm9Vq8QvtY5ORMahjQVL061VL\",\n \"object\": + \"chat.completion\",\n \"created\": 1735448664,\n \"model\": \"gpt-4o-mini-2024-07-18\",\n + \ \"choices\": [\n {\n \"index\": 0,\n \"message\": {\n \"role\": + \"assistant\",\n \"content\": \"I now can give a great answer \\nFinal + Answer: The content for the file with ID 'alpha' is as follows:\\n\\n```\\nThis + is the content specific to ID: alpha. This file serves as a reference for the + alpha ID and may include various information relevant to its usage. The ID aids + in categorization and organization of data. \\n```\\n\\nThe file should be + created with the path including the ID, for example: `./output/alpha_file.txt`.\",\n + \ \"refusal\": null\n },\n \"logprobs\": null,\n \"finish_reason\": + \"stop\"\n }\n ],\n \"usage\": {\n \"prompt_tokens\": 175,\n \"completion_tokens\": + 94,\n \"total_tokens\": 269,\n \"prompt_tokens_details\": {\n \"cached_tokens\": + 0,\n \"audio_tokens\": 0\n },\n \"completion_tokens_details\": {\n + \ \"reasoning_tokens\": 0,\n \"audio_tokens\": 0,\n \"accepted_prediction_tokens\": + 0,\n \"rejected_prediction_tokens\": 0\n }\n },\n \"system_fingerprint\": + \"fp_0aa8d3e20b\"\n}\n" + headers: + CF-Cache-Status: + - DYNAMIC + CF-RAY: + - 8f973fc8a8e176cd-SEA + Connection: + - keep-alive + Content-Encoding: + - gzip + Content-Type: + - application/json + Date: + - Sun, 29 Dec 2024 05:04:25 GMT + Server: + - cloudflare + Set-Cookie: + - __cf_bm=Td9pau1aHgSMMxQpKkO8mpGk5DWBDjUinUYlfJPsgcw-1735448665-1.0.1.1-wH0kiPhZtNwT_fIqYCb3obZgU2VIxoPL.QrruewXdLVLT9FeI4b6BU.sHqsBjMODx6Tvtuzz9GQRvoY1nWs6Fw; + path=/; expires=Sun, 29-Dec-24 05:34:25 GMT; domain=.api.openai.com; HttpOnly; + Secure; SameSite=None + - _cfuvid=P0VQx4Uj_TpVJBRs2gSXmq3srUR2DW7oj5Vmq54U47M-1735448665828-0.0.1.1-604800000; + path=/; domain=.api.openai.com; HttpOnly; Secure; SameSite=None + Transfer-Encoding: + - chunked + X-Content-Type-Options: + - nosniff + access-control-expose-headers: + - X-Request-ID + alt-svc: + - h3=":443"; ma=86400 + openai-organization: + - crewai-iuxna1 + openai-processing-ms: + - '1288' + openai-version: + - '2020-10-01' + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + x-ratelimit-limit-requests: + - '30000' + x-ratelimit-limit-tokens: + - '150000000' + x-ratelimit-remaining-requests: + - '29999' + x-ratelimit-remaining-tokens: + - '149999786' + x-ratelimit-reset-requests: + - 2ms + x-ratelimit-reset-tokens: + - 0s + x-request-id: + - req_9dd7c0e64935fade58b5b8359fcc9c02 + http_version: HTTP/1.1 + status_code: 200 +- request: + body: '{"messages": [{"role": "system", "content": "You are TestAgent. Testing + interpolation logic\nYour personal goal is: Validate output_file interpolation + with kickoff_for_each\nTo give my best complete final answer to the task use + the exact following format:\n\nThought: I now can give a great answer\nFinal + Answer: Your final answer must be the great and the most complete as possible, + it must be outcome described.\n\nI MUST use these formats, my job depends on + it!"}, {"role": "user", "content": "\nCurrent Task: Create a brief summary for + the ID: beta\n\nThis is the expect criteria for your final answer: Should write + a file with the ID included in the path\nyou MUST return the actual complete + content as the final answer, not a summary.\n\nBegin! This is VERY important + to you, use the tools available and give your best Final Answer, your job depends + on it!\n\nThought:"}], "model": "gpt-4o-mini", "stop": ["\nObservation:"], "stream": + false}' + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate + connection: + - keep-alive + content-length: + - '947' + content-type: + - application/json + cookie: + - __cf_bm=Td9pau1aHgSMMxQpKkO8mpGk5DWBDjUinUYlfJPsgcw-1735448665-1.0.1.1-wH0kiPhZtNwT_fIqYCb3obZgU2VIxoPL.QrruewXdLVLT9FeI4b6BU.sHqsBjMODx6Tvtuzz9GQRvoY1nWs6Fw; + _cfuvid=P0VQx4Uj_TpVJBRs2gSXmq3srUR2DW7oj5Vmq54U47M-1735448665828-0.0.1.1-604800000 + host: + - api.openai.com + user-agent: + - OpenAI/Python 1.52.1 + x-stainless-arch: + - x64 + x-stainless-async: + - 'false' + x-stainless-lang: + - python + x-stainless-os: + - Linux + x-stainless-package-version: + - 1.52.1 + x-stainless-raw-response: + - 'true' + x-stainless-retry-count: + - '0' + x-stainless-runtime: + - CPython + x-stainless-runtime-version: + - 3.12.7 + method: POST + uri: https://api.openai.com/v1/chat/completions + response: + content: "{\n \"id\": \"chatcmpl-AjflSvzNtfBXfJSerUuWwQM47tE4N\",\n \"object\": + \"chat.completion\",\n \"created\": 1735448666,\n \"model\": \"gpt-4o-mini-2024-07-18\",\n + \ \"choices\": [\n {\n \"index\": 0,\n \"message\": {\n \"role\": + \"assistant\",\n \"content\": \"I now can give a great answer \\nFinal + Answer: Here is the content to be written to the file for ID: beta\\n\\n```\\nThis + is the content specifically related to the ID: beta.\\n```\\n\\nThe file will + be created at the following path: `/path/to/directory/beta.txt` \\nThank you + for the opportunity to complete this task!\",\n \"refusal\": null\n },\n + \ \"logprobs\": null,\n \"finish_reason\": \"stop\"\n }\n ],\n + \ \"usage\": {\n \"prompt_tokens\": 175,\n \"completion_tokens\": 72,\n + \ \"total_tokens\": 247,\n \"prompt_tokens_details\": {\n \"cached_tokens\": + 0,\n \"audio_tokens\": 0\n },\n \"completion_tokens_details\": {\n + \ \"reasoning_tokens\": 0,\n \"audio_tokens\": 0,\n \"accepted_prediction_tokens\": + 0,\n \"rejected_prediction_tokens\": 0\n }\n },\n \"system_fingerprint\": + \"fp_0aa8d3e20b\"\n}\n" + headers: + CF-Cache-Status: + - DYNAMIC + CF-RAY: + - 8f973fd1af7576cd-SEA + Connection: + - keep-alive + Content-Encoding: + - gzip + Content-Type: + - application/json + Date: + - Sun, 29 Dec 2024 05:04:27 GMT + Server: + - cloudflare + Transfer-Encoding: + - chunked + X-Content-Type-Options: + - nosniff + access-control-expose-headers: + - X-Request-ID + alt-svc: + - h3=":443"; ma=86400 + openai-organization: + - crewai-iuxna1 + openai-processing-ms: + - '1475' + openai-version: + - '2020-10-01' + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + x-ratelimit-limit-requests: + - '30000' + x-ratelimit-limit-tokens: + - '150000000' + x-ratelimit-remaining-requests: + - '29999' + x-ratelimit-remaining-tokens: + - '149999785' + x-ratelimit-reset-requests: + - 2ms + x-ratelimit-reset-tokens: + - 0s + x-request-id: + - req_dc9a842ed35e213e52c2d6fe9f11c36a + http_version: HTTP/1.1 + status_code: 200 +version: 1 diff --git a/tests/crew_test.py b/tests/crew_test.py index 74bcf08d3..80320af25 100644 --- a/tests/crew_test.py +++ b/tests/crew_test.py @@ -1183,6 +1183,53 @@ def test_kickoff_for_each_error_handling(): crew.kickoff_for_each(inputs=inputs) +@pytest.mark.vcr(filter_headers=["authorization"]) +def test_kickoff_for_each_output_file_interpolation(tmp_path): + """Test that output_file paths are correctly interpolated when using kickoff_for_each.""" + # Create test directories + output_dir = tmp_path / "output_files" + output_dir.mkdir() + + agent = Agent( + role="TestAgent", + goal="Validate output_file interpolation with kickoff_for_each", + backstory="Testing interpolation logic", + allow_code_execution=False, + ) + + task = Task( + description="Create a brief summary for the ID: {id_var}", + expected_output="Should write a file with the ID included in the path", + agent=agent, + output_file=str(output_dir / "summary_{id_var}.md") + ) + + crew = Crew(agents=[agent], tasks=[task], verbose=True) + + inputs = [ + {"id_var": "alpha"}, + {"id_var": "beta"}, + ] + + # Use kickoff_for_each + results = crew.kickoff_for_each(inputs=inputs) + + # Verify each iteration's output_file was properly interpolated + for input_data, result in zip(inputs, results): + expected_path = output_dir / f"summary_{input_data['id_var']}.md" + assert expected_path.exists(), f"Expected output file {expected_path} was not created" + + # Verify the output file was created with the correct name + assert expected_path.exists(), ( + f"Expected output file {expected_path} was not created" + ) + + # Verify file contains some content + assert expected_path.stat().st_size > 0, ( + f"Output file {expected_path} is empty" + ) + + @pytest.mark.vcr(filter_headers=["authorization"]) @pytest.mark.asyncio async def test_kickoff_async_basic_functionality_and_output():