added tests

This commit is contained in:
Lorenze Jay
2024-07-31 22:02:11 -07:00
parent e18da9c157
commit 9f31fec084
3 changed files with 418 additions and 0 deletions

View File

@@ -7,6 +7,7 @@ import pytest
from langchain.tools import tool
from langchain_core.exceptions import OutputParserException
from langchain_openai import ChatOpenAI
from langchain.schema import AgentAction
from crewai import Agent, Crew, Task
from crewai.agents.cache import CacheHandler
@@ -1014,3 +1015,77 @@ def test_agent_max_retry_limit():
),
]
)
@pytest.mark.vcr(filter_headers=["authorization"])
def test_handle_context_length_exceeds_limit():
agent = Agent(
role="test role",
goal="test goal",
backstory="test backstory",
)
task = Task(description="test task", agent=agent, expected_output="test output")
# crew = Crew(agents=[agent], tasks=[task])
original_action = AgentAction(
tool="test_tool", tool_input="test_input", log="test_log"
)
with patch.object(
CrewAgentExecutor, "_iter_next_step", wraps=agent.agent_executor._iter_next_step
) as private_mock:
task = Task(
description="The final answer is 42. But don't give it yet, instead keep using the `get_final_answer` tool.",
expected_output="The final answer",
)
agent.execute_task(
task=task,
)
private_mock.assert_called_once()
with patch("crewai.agents.executor.prompt") as mock_prompt:
mock_prompt.return_value = "y"
with patch.object(
CrewAgentExecutor, "_handle_context_length"
) as mock_handle_context:
mock_handle_context.side_effect = ValueError(
"Context length limit exceeded"
)
long_input = "This is a very long input. " * 10000
# Attempt to handle context length, expecting the mocked error
with pytest.raises(ValueError) as excinfo:
agent.agent_executor._handle_context_length(
[(original_action, long_input)]
)
assert "Context length limit exceeded" in str(excinfo.value)
mock_handle_context.assert_called_once()
@pytest.mark.vcr(filter_headers=["authorization"])
def test_handle_context_length_exceeds_limit_cli_no():
agent = Agent(
role="test role",
goal="test goal",
backstory="test backstory",
)
task = Task(description="test task", agent=agent, expected_output="test output")
with patch.object(
CrewAgentExecutor, "_iter_next_step", wraps=agent.agent_executor._iter_next_step
) as private_mock:
task = Task(
description="The final answer is 42. But don't give it yet, instead keep using the `get_final_answer` tool.",
expected_output="The final answer",
)
agent.execute_task(
task=task,
)
private_mock.assert_called_once()
with patch("crewai.agents.executor.prompt") as mock_prompt:
mock_prompt.return_value = "n"
with patch.object(
CrewAgentExecutor, "_handle_context_length"
) as mock_handle_context:
mock_handle_context.assert_not_called()

View File

@@ -0,0 +1,181 @@
interactions:
- request:
body: '{"messages": [{"content": "You are test role. test backstory\nYour personal
goal is: test goalTo give my best complete final answer to the task use the
exact following format:\n\nThought: I now can give a great answer\nFinal Answer:
my best complete final answer to the task.\nYour final answer must be the great
and the most complete as possible, it must be outcome described.\n\nI MUST use
these formats, my job depends on it!\nCurrent Task: The final answer is 42.
But don''t give it yet, instead keep using the `get_final_answer` tool.\n\nThis
is the expect criteria for your final answer: The final answer \n you MUST return
the actual complete content as the final answer, not a summary.\n\nBegin! This
is VERY important to you, use the tools available and give your best Final Answer,
your job depends on it!\n\nThought:\n", "role": "user"}], "model": "gpt-4o",
"n": 1, "stop": ["\nObservation"], "stream": true, "temperature": 0.7}'
headers:
accept:
- application/json
accept-encoding:
- gzip, deflate, br
connection:
- keep-alive
content-length:
- '938'
content-type:
- application/json
host:
- api.openai.com
user-agent:
- OpenAI/Python 1.35.10
x-stainless-arch:
- arm64
x-stainless-async:
- 'false'
x-stainless-lang:
- python
x-stainless-os:
- MacOS
x-stainless-package-version:
- 1.35.10
x-stainless-runtime:
- CPython
x-stainless-runtime-version:
- 3.11.5
method: POST
uri: https://api.openai.com/v1/chat/completions
response:
body:
string: 'data: {"id":"chatcmpl-9rDmhDy41qR9B2jdH1zkXoxv4LMn6","object":"chat.completion.chunk","created":1722471399,"model":"gpt-4o-2024-05-13","system_fingerprint":"fp_4e2b2da518","choices":[{"index":0,"delta":{"role":"assistant","content":""},"logprobs":null,"finish_reason":null}]}
data: {"id":"chatcmpl-9rDmhDy41qR9B2jdH1zkXoxv4LMn6","object":"chat.completion.chunk","created":1722471399,"model":"gpt-4o-2024-05-13","system_fingerprint":"fp_4e2b2da518","choices":[{"index":0,"delta":{"content":"Thought"},"logprobs":null,"finish_reason":null}]}
data: {"id":"chatcmpl-9rDmhDy41qR9B2jdH1zkXoxv4LMn6","object":"chat.completion.chunk","created":1722471399,"model":"gpt-4o-2024-05-13","system_fingerprint":"fp_4e2b2da518","choices":[{"index":0,"delta":{"content":":"},"logprobs":null,"finish_reason":null}]}
data: {"id":"chatcmpl-9rDmhDy41qR9B2jdH1zkXoxv4LMn6","object":"chat.completion.chunk","created":1722471399,"model":"gpt-4o-2024-05-13","system_fingerprint":"fp_4e2b2da518","choices":[{"index":0,"delta":{"content":"
I"},"logprobs":null,"finish_reason":null}]}
data: {"id":"chatcmpl-9rDmhDy41qR9B2jdH1zkXoxv4LMn6","object":"chat.completion.chunk","created":1722471399,"model":"gpt-4o-2024-05-13","system_fingerprint":"fp_4e2b2da518","choices":[{"index":0,"delta":{"content":"
now"},"logprobs":null,"finish_reason":null}]}
data: {"id":"chatcmpl-9rDmhDy41qR9B2jdH1zkXoxv4LMn6","object":"chat.completion.chunk","created":1722471399,"model":"gpt-4o-2024-05-13","system_fingerprint":"fp_4e2b2da518","choices":[{"index":0,"delta":{"content":"
can"},"logprobs":null,"finish_reason":null}]}
data: {"id":"chatcmpl-9rDmhDy41qR9B2jdH1zkXoxv4LMn6","object":"chat.completion.chunk","created":1722471399,"model":"gpt-4o-2024-05-13","system_fingerprint":"fp_4e2b2da518","choices":[{"index":0,"delta":{"content":"
give"},"logprobs":null,"finish_reason":null}]}
data: {"id":"chatcmpl-9rDmhDy41qR9B2jdH1zkXoxv4LMn6","object":"chat.completion.chunk","created":1722471399,"model":"gpt-4o-2024-05-13","system_fingerprint":"fp_4e2b2da518","choices":[{"index":0,"delta":{"content":"
a"},"logprobs":null,"finish_reason":null}]}
data: {"id":"chatcmpl-9rDmhDy41qR9B2jdH1zkXoxv4LMn6","object":"chat.completion.chunk","created":1722471399,"model":"gpt-4o-2024-05-13","system_fingerprint":"fp_4e2b2da518","choices":[{"index":0,"delta":{"content":"
great"},"logprobs":null,"finish_reason":null}]}
data: {"id":"chatcmpl-9rDmhDy41qR9B2jdH1zkXoxv4LMn6","object":"chat.completion.chunk","created":1722471399,"model":"gpt-4o-2024-05-13","system_fingerprint":"fp_4e2b2da518","choices":[{"index":0,"delta":{"content":"
answer"},"logprobs":null,"finish_reason":null}]}
data: {"id":"chatcmpl-9rDmhDy41qR9B2jdH1zkXoxv4LMn6","object":"chat.completion.chunk","created":1722471399,"model":"gpt-4o-2024-05-13","system_fingerprint":"fp_4e2b2da518","choices":[{"index":0,"delta":{"content":"\n"},"logprobs":null,"finish_reason":null}]}
data: {"id":"chatcmpl-9rDmhDy41qR9B2jdH1zkXoxv4LMn6","object":"chat.completion.chunk","created":1722471399,"model":"gpt-4o-2024-05-13","system_fingerprint":"fp_4e2b2da518","choices":[{"index":0,"delta":{"content":"Final"},"logprobs":null,"finish_reason":null}]}
data: {"id":"chatcmpl-9rDmhDy41qR9B2jdH1zkXoxv4LMn6","object":"chat.completion.chunk","created":1722471399,"model":"gpt-4o-2024-05-13","system_fingerprint":"fp_4e2b2da518","choices":[{"index":0,"delta":{"content":"
Answer"},"logprobs":null,"finish_reason":null}]}
data: {"id":"chatcmpl-9rDmhDy41qR9B2jdH1zkXoxv4LMn6","object":"chat.completion.chunk","created":1722471399,"model":"gpt-4o-2024-05-13","system_fingerprint":"fp_4e2b2da518","choices":[{"index":0,"delta":{"content":":"},"logprobs":null,"finish_reason":null}]}
data: {"id":"chatcmpl-9rDmhDy41qR9B2jdH1zkXoxv4LMn6","object":"chat.completion.chunk","created":1722471399,"model":"gpt-4o-2024-05-13","system_fingerprint":"fp_4e2b2da518","choices":[{"index":0,"delta":{"content":"
The"},"logprobs":null,"finish_reason":null}]}
data: {"id":"chatcmpl-9rDmhDy41qR9B2jdH1zkXoxv4LMn6","object":"chat.completion.chunk","created":1722471399,"model":"gpt-4o-2024-05-13","system_fingerprint":"fp_4e2b2da518","choices":[{"index":0,"delta":{"content":"
final"},"logprobs":null,"finish_reason":null}]}
data: {"id":"chatcmpl-9rDmhDy41qR9B2jdH1zkXoxv4LMn6","object":"chat.completion.chunk","created":1722471399,"model":"gpt-4o-2024-05-13","system_fingerprint":"fp_4e2b2da518","choices":[{"index":0,"delta":{"content":"
answer"},"logprobs":null,"finish_reason":null}]}
data: {"id":"chatcmpl-9rDmhDy41qR9B2jdH1zkXoxv4LMn6","object":"chat.completion.chunk","created":1722471399,"model":"gpt-4o-2024-05-13","system_fingerprint":"fp_4e2b2da518","choices":[{"index":0,"delta":{"content":"
is"},"logprobs":null,"finish_reason":null}]}
data: {"id":"chatcmpl-9rDmhDy41qR9B2jdH1zkXoxv4LMn6","object":"chat.completion.chunk","created":1722471399,"model":"gpt-4o-2024-05-13","system_fingerprint":"fp_4e2b2da518","choices":[{"index":0,"delta":{"content":"
"},"logprobs":null,"finish_reason":null}]}
data: {"id":"chatcmpl-9rDmhDy41qR9B2jdH1zkXoxv4LMn6","object":"chat.completion.chunk","created":1722471399,"model":"gpt-4o-2024-05-13","system_fingerprint":"fp_4e2b2da518","choices":[{"index":0,"delta":{"content":"42"},"logprobs":null,"finish_reason":null}]}
data: {"id":"chatcmpl-9rDmhDy41qR9B2jdH1zkXoxv4LMn6","object":"chat.completion.chunk","created":1722471399,"model":"gpt-4o-2024-05-13","system_fingerprint":"fp_4e2b2da518","choices":[{"index":0,"delta":{"content":"."},"logprobs":null,"finish_reason":null}]}
data: {"id":"chatcmpl-9rDmhDy41qR9B2jdH1zkXoxv4LMn6","object":"chat.completion.chunk","created":1722471399,"model":"gpt-4o-2024-05-13","system_fingerprint":"fp_4e2b2da518","choices":[{"index":0,"delta":{},"logprobs":null,"finish_reason":"stop"}]}
data: [DONE]
'
headers:
CF-Cache-Status:
- DYNAMIC
CF-RAY:
- 8ac1a40879b87d1f-LAX
Connection:
- keep-alive
Content-Type:
- text/event-stream; charset=utf-8
Date:
- Thu, 01 Aug 2024 00:16:40 GMT
Server:
- cloudflare
Set-Cookie:
- __cf_bm=MHUl15YVi607cmuLtQ84ESiH30IyJiIW1a40fopQ81w-1722471400-1.0.1.1-OGpq5Ezj6iE0ToM1diQllGb70.J3O_K2De9NbwZPWmW2qN07U20adJ_0yd6PKUNqMdL.xEnLcNAOWVmsfrLUrQ;
path=/; expires=Thu, 01-Aug-24 00:46:40 GMT; domain=.api.openai.com; HttpOnly;
Secure; SameSite=None
- _cfuvid=G2ZVNvfNfFk4DeKyZ7jMYetG7wOasINAGHstrOnuAY8-1722471400129-0.0.1.1-604800000;
path=/; domain=.api.openai.com; HttpOnly; Secure; SameSite=None
Transfer-Encoding:
- chunked
X-Content-Type-Options:
- nosniff
alt-svc:
- h3=":443"; ma=86400
openai-organization:
- crewai-iuxna1
openai-processing-ms:
- '131'
openai-version:
- '2020-10-01'
strict-transport-security:
- max-age=15552000; includeSubDomains; preload
x-ratelimit-limit-requests:
- '10000'
x-ratelimit-limit-tokens:
- '30000000'
x-ratelimit-remaining-requests:
- '9999'
x-ratelimit-remaining-tokens:
- '29999786'
x-ratelimit-reset-requests:
- 6ms
x-ratelimit-reset-tokens:
- 0s
x-request-id:
- req_b68b417b3fe1c67244279551e411b37a
status:
code: 200
message: OK
version: 1

View File

@@ -0,0 +1,162 @@
interactions:
- request:
body: '{"messages": [{"content": "You are test role. test backstory\nYour personal
goal is: test goalTo give my best complete final answer to the task use the
exact following format:\n\nThought: I now can give a great answer\nFinal Answer:
my best complete final answer to the task.\nYour final answer must be the great
and the most complete as possible, it must be outcome described.\n\nI MUST use
these formats, my job depends on it!\nCurrent Task: The final answer is 42.
But don''t give it yet, instead keep using the `get_final_answer` tool.\n\nThis
is the expect criteria for your final answer: The final answer \n you MUST return
the actual complete content as the final answer, not a summary.\n\nBegin! This
is VERY important to you, use the tools available and give your best Final Answer,
your job depends on it!\n\nThought:\n", "role": "user"}], "model": "gpt-4o",
"n": 1, "stop": ["\nObservation"], "stream": true, "temperature": 0.7}'
headers:
accept:
- application/json
accept-encoding:
- gzip, deflate, br
connection:
- keep-alive
content-length:
- '938'
content-type:
- application/json
host:
- api.openai.com
user-agent:
- OpenAI/Python 1.35.10
x-stainless-arch:
- arm64
x-stainless-async:
- 'false'
x-stainless-lang:
- python
x-stainless-os:
- MacOS
x-stainless-package-version:
- 1.35.10
x-stainless-runtime:
- CPython
x-stainless-runtime-version:
- 3.11.5
method: POST
uri: https://api.openai.com/v1/chat/completions
response:
body:
string: 'data: {"id":"chatcmpl-9rI1RAFocnugKoDvAxLndHW5uBeU9","object":"chat.completion.chunk","created":1722487689,"model":"gpt-4o-2024-05-13","system_fingerprint":"fp_4e2b2da518","choices":[{"index":0,"delta":{"role":"assistant","content":""},"logprobs":null,"finish_reason":null}]}
data: {"id":"chatcmpl-9rI1RAFocnugKoDvAxLndHW5uBeU9","object":"chat.completion.chunk","created":1722487689,"model":"gpt-4o-2024-05-13","system_fingerprint":"fp_4e2b2da518","choices":[{"index":0,"delta":{"content":"Thought"},"logprobs":null,"finish_reason":null}]}
data: {"id":"chatcmpl-9rI1RAFocnugKoDvAxLndHW5uBeU9","object":"chat.completion.chunk","created":1722487689,"model":"gpt-4o-2024-05-13","system_fingerprint":"fp_4e2b2da518","choices":[{"index":0,"delta":{"content":":"},"logprobs":null,"finish_reason":null}]}
data: {"id":"chatcmpl-9rI1RAFocnugKoDvAxLndHW5uBeU9","object":"chat.completion.chunk","created":1722487689,"model":"gpt-4o-2024-05-13","system_fingerprint":"fp_4e2b2da518","choices":[{"index":0,"delta":{"content":"
I"},"logprobs":null,"finish_reason":null}]}
data: {"id":"chatcmpl-9rI1RAFocnugKoDvAxLndHW5uBeU9","object":"chat.completion.chunk","created":1722487689,"model":"gpt-4o-2024-05-13","system_fingerprint":"fp_4e2b2da518","choices":[{"index":0,"delta":{"content":"
now"},"logprobs":null,"finish_reason":null}]}
data: {"id":"chatcmpl-9rI1RAFocnugKoDvAxLndHW5uBeU9","object":"chat.completion.chunk","created":1722487689,"model":"gpt-4o-2024-05-13","system_fingerprint":"fp_4e2b2da518","choices":[{"index":0,"delta":{"content":"
can"},"logprobs":null,"finish_reason":null}]}
data: {"id":"chatcmpl-9rI1RAFocnugKoDvAxLndHW5uBeU9","object":"chat.completion.chunk","created":1722487689,"model":"gpt-4o-2024-05-13","system_fingerprint":"fp_4e2b2da518","choices":[{"index":0,"delta":{"content":"
give"},"logprobs":null,"finish_reason":null}]}
data: {"id":"chatcmpl-9rI1RAFocnugKoDvAxLndHW5uBeU9","object":"chat.completion.chunk","created":1722487689,"model":"gpt-4o-2024-05-13","system_fingerprint":"fp_4e2b2da518","choices":[{"index":0,"delta":{"content":"
a"},"logprobs":null,"finish_reason":null}]}
data: {"id":"chatcmpl-9rI1RAFocnugKoDvAxLndHW5uBeU9","object":"chat.completion.chunk","created":1722487689,"model":"gpt-4o-2024-05-13","system_fingerprint":"fp_4e2b2da518","choices":[{"index":0,"delta":{"content":"
great"},"logprobs":null,"finish_reason":null}]}
data: {"id":"chatcmpl-9rI1RAFocnugKoDvAxLndHW5uBeU9","object":"chat.completion.chunk","created":1722487689,"model":"gpt-4o-2024-05-13","system_fingerprint":"fp_4e2b2da518","choices":[{"index":0,"delta":{"content":"
answer"},"logprobs":null,"finish_reason":null}]}
data: {"id":"chatcmpl-9rI1RAFocnugKoDvAxLndHW5uBeU9","object":"chat.completion.chunk","created":1722487689,"model":"gpt-4o-2024-05-13","system_fingerprint":"fp_4e2b2da518","choices":[{"index":0,"delta":{"content":"\n"},"logprobs":null,"finish_reason":null}]}
data: {"id":"chatcmpl-9rI1RAFocnugKoDvAxLndHW5uBeU9","object":"chat.completion.chunk","created":1722487689,"model":"gpt-4o-2024-05-13","system_fingerprint":"fp_4e2b2da518","choices":[{"index":0,"delta":{"content":"Final"},"logprobs":null,"finish_reason":null}]}
data: {"id":"chatcmpl-9rI1RAFocnugKoDvAxLndHW5uBeU9","object":"chat.completion.chunk","created":1722487689,"model":"gpt-4o-2024-05-13","system_fingerprint":"fp_4e2b2da518","choices":[{"index":0,"delta":{"content":"
Answer"},"logprobs":null,"finish_reason":null}]}
data: {"id":"chatcmpl-9rI1RAFocnugKoDvAxLndHW5uBeU9","object":"chat.completion.chunk","created":1722487689,"model":"gpt-4o-2024-05-13","system_fingerprint":"fp_4e2b2da518","choices":[{"index":0,"delta":{"content":":"},"logprobs":null,"finish_reason":null}]}
data: {"id":"chatcmpl-9rI1RAFocnugKoDvAxLndHW5uBeU9","object":"chat.completion.chunk","created":1722487689,"model":"gpt-4o-2024-05-13","system_fingerprint":"fp_4e2b2da518","choices":[{"index":0,"delta":{"content":"
"},"logprobs":null,"finish_reason":null}]}
data: {"id":"chatcmpl-9rI1RAFocnugKoDvAxLndHW5uBeU9","object":"chat.completion.chunk","created":1722487689,"model":"gpt-4o-2024-05-13","system_fingerprint":"fp_4e2b2da518","choices":[{"index":0,"delta":{"content":"42"},"logprobs":null,"finish_reason":null}]}
data: {"id":"chatcmpl-9rI1RAFocnugKoDvAxLndHW5uBeU9","object":"chat.completion.chunk","created":1722487689,"model":"gpt-4o-2024-05-13","system_fingerprint":"fp_4e2b2da518","choices":[{"index":0,"delta":{},"logprobs":null,"finish_reason":"stop"}]}
data: [DONE]
'
headers:
CF-Cache-Status:
- DYNAMIC
CF-RAY:
- 8ac331b9eaee2b7f-LAX
Connection:
- keep-alive
Content-Type:
- text/event-stream; charset=utf-8
Date:
- Thu, 01 Aug 2024 04:48:09 GMT
Server:
- cloudflare
Set-Cookie:
- __cf_bm=OXht5zC71vWYFW_z_933m3sZfFS2xBez0DHv93FvT5s-1722487689-1.0.1.1-wE8JTR7MnwUgiiTDppYg8A7zLEiidth.MB0zrwONeAtNWRjKC1tuGf8LZYDlYIHUhqG73syYExpZ.5pZhzJkcg;
path=/; expires=Thu, 01-Aug-24 05:18:09 GMT; domain=.api.openai.com; HttpOnly;
Secure; SameSite=None
- _cfuvid=PAR7y4xRe4VzRT.7GK34Tq5r8vevY6xq0E.i.R40xnU-1722487689562-0.0.1.1-604800000;
path=/; domain=.api.openai.com; HttpOnly; Secure; SameSite=None
Transfer-Encoding:
- chunked
X-Content-Type-Options:
- nosniff
alt-svc:
- h3=":443"; ma=86400
openai-organization:
- crewai-iuxna1
openai-processing-ms:
- '84'
openai-version:
- '2020-10-01'
strict-transport-security:
- max-age=15552000; includeSubDomains; preload
x-ratelimit-limit-requests:
- '10000'
x-ratelimit-limit-tokens:
- '30000000'
x-ratelimit-remaining-requests:
- '9999'
x-ratelimit-remaining-tokens:
- '29999786'
x-ratelimit-reset-requests:
- 6ms
x-ratelimit-reset-tokens:
- 0s
x-request-id:
- req_105dcfc53c9672dea0437249c12c3319
status:
code: 200
message: OK
version: 1