mirror of
https://github.com/crewAIInc/crewAI.git
synced 2026-01-08 15:48:29 +00:00
Supporting no-code Guardrail creation (#2636)
* feat: support to define a guardrail task no-code * feat: add auto-discovery for Guardrail code execution mode * feat: handle malformed or invalid response from CodeInterpreterTool * feat: allow to set unsafe_mode from Guardrail task * feat: renaming GuardrailTask to TaskGuardrail * feat: ensure guardrail is callable while initializing Task * feat: remove Docker availability check from TaskGuardrail The CodeInterpreterTool already ensures compliance with this requirement. * refactor: replace if/raise with assert For this use case `assert` is more appropriate choice * test: remove useless or duplicated test * fix: attempt to fix type-checker * feat: support to define a task guardrail using YAML config * refactor: simplify TaskGuardrail to use LLM for validation, no code generation * docs: update TaskGuardrail doc strings * refactor: drop task paramenter from TaskGuardrail This parameter was used to get the model from the `task.agent` which is a quite bit redudant since we could propagate the llm directly
This commit is contained in:
Lucas Gomide
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
713
tests/cassettes/test_guardrail_emits_events.yaml
Normal file
713
tests/cassettes/test_guardrail_emits_events.yaml
Normal file
@@ -0,0 +1,713 @@
|
||||
interactions:
|
||||
- request:
|
||||
body: '{"messages": [{"role": "system", "content": "You are Test Agent. Test Backstory\nYour
|
||||
personal goal is: Test Goal\nTo give my best complete final answer to the task
|
||||
respond using the exact following format:\n\nThought: I now can give a great
|
||||
answer\nFinal Answer: Your final answer must be the great and the most complete
|
||||
as possible, it must be outcome described.\n\nI MUST use these formats, my job
|
||||
depends on it!"}, {"role": "user", "content": "\nCurrent Task: Gather information
|
||||
about available books on the First World War\n\nThis is the expected criteria
|
||||
for your final answer: A list of available books on the First World War\nyou
|
||||
MUST return the actual complete content as the final answer, not a summary.\n\nBegin!
|
||||
This is VERY important to you, use the tools available and give your best Final
|
||||
Answer, your job depends on it!\n\nThought:"}], "model": "gpt-4o-mini", "stop":
|
||||
["\nObservation:"]}'
|
||||
headers:
|
||||
accept:
|
||||
- application/json
|
||||
accept-encoding:
|
||||
- gzip, deflate
|
||||
connection:
|
||||
- keep-alive
|
||||
content-length:
|
||||
- '903'
|
||||
content-type:
|
||||
- application/json
|
||||
host:
|
||||
- api.openai.com
|
||||
user-agent:
|
||||
- OpenAI/Python 1.68.2
|
||||
x-stainless-arch:
|
||||
- arm64
|
||||
x-stainless-async:
|
||||
- 'false'
|
||||
x-stainless-lang:
|
||||
- python
|
||||
x-stainless-os:
|
||||
- MacOS
|
||||
x-stainless-package-version:
|
||||
- 1.68.2
|
||||
x-stainless-raw-response:
|
||||
- 'true'
|
||||
x-stainless-read-timeout:
|
||||
- '600.0'
|
||||
x-stainless-retry-count:
|
||||
- '0'
|
||||
x-stainless-runtime:
|
||||
- CPython
|
||||
x-stainless-runtime-version:
|
||||
- 3.12.9
|
||||
method: POST
|
||||
uri: https://api.openai.com/v1/chat/completions
|
||||
response:
|
||||
body:
|
||||
string: !!binary |
|
||||
H4sIAAAAAAAAAwAAAP//jFbbbhs3EH33Vwz2pUAgGXbqxInenJtzgZs0dmKgdWHMcmd3J+ZytkOu
|
||||
ZLUI0N/o7/VLCpK7kpwL2hfL0pDDM2fOXP7cAyi4KhZQmBaD6Xo7f/Ke3rz75G+ffXz04celD+ef
|
||||
yifL8hc5P/145opZvCHlJzJhurVvpOstBZbRbJQwUPR6eHz04PH9xw+Oj5Ohk4psvNb0YX4k844d
|
||||
z+8f3D+aHxzPDx+Nt1thQ75YwK97AAB/pr8Rp6votljAwWz6pSPvsaFisTkEUKjY+EuB3rMP6EIx
|
||||
2xqNuEAuQX8FTlZg0EHDSwKEJsIGdH5FCnDlXrBDCyfp+wJekhKwBwTLPoDU4CRgaQlKkRsP4iC0
|
||||
BC9YfYBLUVvBJeoMyEV6IhjXwBKVZfDQk/qeTOAleUBXxasd+cWVu3KH+3Dv3lVxYi38PDCFyfMl
|
||||
+UDq4IWKC1cFlGt4rmxaOENlhPfUof4+0L17MIcTcLIkCxX1bEJ8OXqg256UyRnyEf8paYcOvNiK
|
||||
ST1Ug8aTG+zwagYtN63lpt34aEVVNN0PSs60sEKtUWk/Yr+fsV98TUQG/FpaB2+IGnQjzkiOUkvO
|
||||
xyTIknTJtEruW4q+Z4AO7fqPCICDB4ODJz+DDj+JQokh2Pg1kmjEefp9SAEmOD+OVI4wPrhKHC0g
|
||||
wjsPouvpmdOU+ZSuw8eHRxAkfj7KmE/3X+/DGa1JJ2pRFWPqoGWf3IQWA5gI3meiXWCliKe2bAKs
|
||||
OLSAUIsZklBuaA20JBdy8mtuBh0hH20ZPB1c4vlkaAY/ZvwJaomKcLkPF4Npu4lHB2iMRe6ogpXo
|
||||
DVQUkO0m8/kxS1jFn4Y+hjgSPOkPCNWuwQdssj6MdCWGGVDXt+j5j8lZRYY9SwYXXZJm7A8y9lOR
|
||||
qlxTfCKK+KLFEfx7KUkDnCouyW9wD0FKlkaxbycmsQ+RkC9E+7XmoFbpACcJ/3CnsGaAVlyTyY+e
|
||||
gqLzHNtUhNaLD/MYvuU6i/dhhn8Zz78dFJ6gufETUZdo7QI+skkJj5Q9ozqqht2OVp7hkis4D5Fv
|
||||
L9vcRAV79pPe6tRZ1oR6R+gyBMsu0lzKiLljywE1pkUxUMPbfgFeDMu8F8uBDVpIve02+Eyib7Gn
|
||||
fFCGYKTLQR7v6GtS/QJO4GlKNrwcFT3C+mYRv6NACi9RQ4rvomWfeiD0KkuuIsKxLuflep7/i/KU
|
||||
wYUZDKkRxjxJJKFi1Cmo8UzivNRJbiuMiR0VHGSbr0fbUGK3CgLPXZUkd4nqc5mfdKQce/zZROTz
|
||||
rZ7Y7Ta7saVWK9QKzvbhqdT1Tn3RbW8l1r24VJSTY3ZLsUvqyCUtbJJZsTdD7vrfSyJ3PZqQa+dx
|
||||
DubtkjT5uJA+puU8K/ufv/728IzxPzJzoqEdFJ53Pa3HZlXHc+0Ou9+oopVyCOSiB3TbyJZiBxeI
|
||||
ojLrmlJCEBRX01jJTOQ2sTtcUkCHB9+dBTGwn2gFr6wdMiXVJLyx5YpW4uCpqPJ2VMRhZ9fAO7eS
|
||||
7JLceyWfGyqYVsWJlSZVRXtX0XmiGEPecxzetSho7mHxCFoLJZqbRmVwVY4kCTwN/R2Bx0lOIbll
|
||||
5+OEzDndDAd/p5elt/OedMshNxDPjeM6Uj32tq+Z2t/dXZTqwWPcn9xg7Y4BXVxFYj7S1vTbaPm8
|
||||
2ZOsNL1K6b+4WtTs2LfXSujFxZ3IB+mLZP28B/Bb2seGOytW0at0fbgOckPpucPjcR8rtmvg1nqU
|
||||
Nz+AIkhAuzU8PJoMdxxe57nld1a6wqBpqdpe3e5/OFQsO4a9nbC/hvMt3zl0ds3/cb81GEN9oOq6
|
||||
V6pyJXzrmFJck793bENzAlz4uPkYug5MGlNRUY2Dzctr4dc+UHdds2tIe+W8wdb9dVWiwYcHVX1Q
|
||||
7H3e+xcAAP//AwDV5F0jzwsAAA==
|
||||
headers:
|
||||
CF-RAY:
|
||||
- 937ec970ab837df5-GRU
|
||||
Connection:
|
||||
- keep-alive
|
||||
Content-Encoding:
|
||||
- gzip
|
||||
Content-Type:
|
||||
- application/json
|
||||
Date:
|
||||
- Tue, 29 Apr 2025 12:26:28 GMT
|
||||
Server:
|
||||
- cloudflare
|
||||
Set-Cookie:
|
||||
- __cf_bm=lSQwOucDEe7UVr7Rj6WZvsTkBkgGT9q7hgVX9AzK42s-1745929588-1.0.1.1-6m0TeDAKI0Hgbl6.GmWHMBMkIpmnfhOu3jQKfjmcvWLHqWUWoE1O4xa9VCtZYXv6_9poUVQq_oCNtzy8eL1XDc8_J3aRMOG3LCvOyvqCawk;
|
||||
path=/; expires=Tue, 29-Apr-25 12:56:28 GMT; domain=.api.openai.com; HttpOnly;
|
||||
Secure; SameSite=None
|
||||
- _cfuvid=TeUS2y0sO.6zigvKZd5.0zRS7sujoYCd9.wMQJboxxo-1745929588265-0.0.1.1-604800000;
|
||||
path=/; domain=.api.openai.com; HttpOnly; Secure; SameSite=None
|
||||
Transfer-Encoding:
|
||||
- chunked
|
||||
X-Content-Type-Options:
|
||||
- nosniff
|
||||
access-control-expose-headers:
|
||||
- X-Request-ID
|
||||
alt-svc:
|
||||
- h3=":443"; ma=86400
|
||||
cf-cache-status:
|
||||
- DYNAMIC
|
||||
openai-organization:
|
||||
- crewai-iuxna1
|
||||
openai-processing-ms:
|
||||
- '10782'
|
||||
openai-version:
|
||||
- '2020-10-01'
|
||||
strict-transport-security:
|
||||
- max-age=31536000; includeSubDomains; preload
|
||||
x-ratelimit-limit-requests:
|
||||
- '30000'
|
||||
x-ratelimit-limit-tokens:
|
||||
- '150000000'
|
||||
x-ratelimit-remaining-requests:
|
||||
- '29999'
|
||||
x-ratelimit-remaining-tokens:
|
||||
- '149999808'
|
||||
x-ratelimit-reset-requests:
|
||||
- 2ms
|
||||
x-ratelimit-reset-tokens:
|
||||
- 0s
|
||||
x-request-id:
|
||||
- req_cbf6ad64d8b470c6d9eff91852f85e1c
|
||||
status:
|
||||
code: 200
|
||||
message: OK
|
||||
- request:
|
||||
body: '{"messages": [{"role": "system", "content": "You are Guardrail Agent. You
|
||||
are a expert at validating the output of a task. By providing effective feedback
|
||||
if the output is not valid.\nYour personal goal is: Validate the output of the
|
||||
task\n\nTo give my best complete final answer to the task respond using the
|
||||
exact following format:\n\nThought: I now can give a great answer\nFinal Answer:
|
||||
Your final answer must be the great and the most complete as possible, it must
|
||||
be outcome described.\n\nI MUST use these formats, my job depends on it!\nIMPORTANT:
|
||||
Your final answer MUST contain all the information requested in the following
|
||||
format: {\n \"valid\": bool,\n \"feedback\": str | None\n}\n\nIMPORTANT: Ensure
|
||||
the final output does not include any code block markers like ```json or ```python."},
|
||||
{"role": "user", "content": "\n Ensure the following task result complies
|
||||
with the given guardrail.\n\n Task result:\n Here is a list of
|
||||
notable books on the First World War, encompassing various perspectives and
|
||||
themes:\n\n1. **\"All Quiet on the Western Front\" by Erich Maria Remarque**
|
||||
- A novel depicting the experiences of German soldiers during World War I, highlighting
|
||||
the horrors of trench warfare.\n\n2. **\"The First World War\" by John Keegan**
|
||||
- A comprehensive overview of the war, analyzing its causes, major battles,
|
||||
and consequences.\n\n3. **\"A World Undone: The Story of the Great War, 1914
|
||||
to 1918\" by G.J. Meyer** - A narrative history that covers the entire conflict
|
||||
with a focus on key events and figures.\n\n4. **\"The Guns of August\" by Barbara
|
||||
W. Tuchman** - An acclaimed work detailing the events leading up to the war
|
||||
and the early stages of combat, emphasizing the decisions of leaders.\n\n5.
|
||||
**\"Goodbye to All That\" by Robert Graves** - An autobiography that captures
|
||||
the experience of trench warfare from a soldier''s perspective, along with the
|
||||
transition to post-war life.\n\n6. **\"With Our Backs to the Wall: Victory and
|
||||
Defeat in 1918\" by David Stevenson** - An analysis of the final year of the
|
||||
war, outlining both the military strategies and the socio-political contexts
|
||||
that shaped the outcome.\n\n7. **\"The Great War: A Combat History of the First
|
||||
World War\" by Peter Hart** - This book provides a battle-by-battle account,
|
||||
using personal diaries and accounts to bring the war''s events to life.\n\n8.
|
||||
**\"The War to End All Wars: The American Military Experience in World War I\"
|
||||
by Edward M. Coffman** - An exploration of American involvement in the war,
|
||||
discussing military strategies and impacts.\n\n9. **\"Over the Top: A Soldier\u2019s
|
||||
Diary of the First World War\" by Arthur Empey** - A firsthand account of trench
|
||||
warfare written by an American volunteer, offering a raw depiction of combat
|
||||
experiences.\n\n10. **\"The First World War: A New Illustrated History\" by
|
||||
Gordon Corrigan** - A richly illustrated book that presents a chronological
|
||||
history of the war, accessible for readers of all backgrounds.\n\nThis list
|
||||
provides a variety of insights and narratives that capture the complexity and
|
||||
significance of the First World War.\n\n Guardrail:\n Ensure the
|
||||
authors are from Italy\n \n Your task:\n - Confirm if the
|
||||
Task result complies with the guardrail.\n - If not, provide clear feedback
|
||||
explaining what is wrong (e.g., by how much it violates the rule, or what specific
|
||||
part fails).\n - Focus only on identifying issues \u2014 do not propose
|
||||
corrections.\n - If the Task result complies with the guardrail, saying
|
||||
that is valid\n "}], "model": "gpt-4o-mini", "stop": ["\nObservation:"]}'
|
||||
headers:
|
||||
accept:
|
||||
- application/json
|
||||
accept-encoding:
|
||||
- gzip, deflate
|
||||
connection:
|
||||
- keep-alive
|
||||
content-length:
|
||||
- '3636'
|
||||
content-type:
|
||||
- application/json
|
||||
host:
|
||||
- api.openai.com
|
||||
user-agent:
|
||||
- OpenAI/Python 1.68.2
|
||||
x-stainless-arch:
|
||||
- arm64
|
||||
x-stainless-async:
|
||||
- 'false'
|
||||
x-stainless-lang:
|
||||
- python
|
||||
x-stainless-os:
|
||||
- MacOS
|
||||
x-stainless-package-version:
|
||||
- 1.68.2
|
||||
x-stainless-raw-response:
|
||||
- 'true'
|
||||
x-stainless-read-timeout:
|
||||
- '600.0'
|
||||
x-stainless-retry-count:
|
||||
- '0'
|
||||
x-stainless-runtime:
|
||||
- CPython
|
||||
x-stainless-runtime-version:
|
||||
- 3.12.9
|
||||
method: POST
|
||||
uri: https://api.openai.com/v1/chat/completions
|
||||
response:
|
||||
body:
|
||||
string: !!binary |
|
||||
H4sIAAAAAAAAAwAAAP//jFPJbhsxDL37Kwidx4HjOrDjW4qiRZBb4m6oA4OWODNqNNREouwYgf+9
|
||||
0HhNF6CXEYaPyyP5+NoDUNaoKShdo+imdf3393SH+qv9/vLlefXw0MZw/222nn1No9sPQRU5wi9/
|
||||
kpZD1IX2TetIrOcdrAOhUM56OR5dXQ+vryaTDmi8IZfDqlb6I99vLNv+cDAc9Qfj/uVkH117qymq
|
||||
KfzoAQC8dt/Mkw29qCkMioOloRixIjU9OgGo4F22KIzRRkEWVZxA7VmIO+qz2qeqlincAvs1aGSo
|
||||
7IoAocr8ATmuKQDM+aNldHDT/U/hdc4Ac7VCZ81cTaFEF6nYGUsis0T9lO1zNasJBOMTBIrJCRhP
|
||||
EdgLdAPbwNpKDVITVAmDCWgdLEljigTsmcCXHepsFDKASWofImAgKINv4FbQbS7gxrkj1hDnNWTv
|
||||
g5exZUmBOFdNLMFSLMCydslYruAThQZ5U3SVPt/t34ebApANeKkpxALWtdU1rKx3KBQ7n0DPyQbK
|
||||
FUHqPK8DhxQFltTRs8gXczXn7fkOApUpYtYBJ+fOAGT2grmBbvuPe2R73LfzVRv8Mv4WqkrLNtaL
|
||||
QBg9591G8a3q0G0P4LHTVXojFdUG37SyEP9EXbnxaK8rdZLzGTreg+IF3ck+uTwAb/ItDAlaF8+U
|
||||
qTTqmswp9CRjTMb6M6B31vWfbP6We9e55ep/0p8ArakVMos2kLH6bccnt0D52v/ldpxyR1hFCiur
|
||||
aSGWQt6EoRKT292gipso1CxKyxWFNtjdIZbtYvDuejgZDgfXA9Xb9n4BAAD//wMAG/c/OJYEAAA=
|
||||
headers:
|
||||
CF-RAY:
|
||||
- 937ec9b72e717e0f-GRU
|
||||
Connection:
|
||||
- keep-alive
|
||||
Content-Encoding:
|
||||
- gzip
|
||||
Content-Type:
|
||||
- application/json
|
||||
Date:
|
||||
- Tue, 29 Apr 2025 12:26:30 GMT
|
||||
Server:
|
||||
- cloudflare
|
||||
Set-Cookie:
|
||||
- __cf_bm=2Kv7JW0NnxXgvGH0KQnR1VcNF47nEYxhEB4bYlFvRfA-1745929590-1.0.1.1-gmQvNlZR_WoNPIkf.07YchEy.a13FU2sP7wZropzue.84PKf6DVY_clzo6DqmBOk8PtKtM3gV952vhWHuE0ygk7096hW9jQTST91FDi30Jc;
|
||||
path=/; expires=Tue, 29-Apr-25 12:56:30 GMT; domain=.api.openai.com; HttpOnly;
|
||||
Secure; SameSite=None
|
||||
- _cfuvid=HHxY6s22oZkfAHAh8HGGIeDkKOuavigZ5DkKL6.WyGc-1745929590485-0.0.1.1-604800000;
|
||||
path=/; domain=.api.openai.com; HttpOnly; Secure; SameSite=None
|
||||
Transfer-Encoding:
|
||||
- chunked
|
||||
X-Content-Type-Options:
|
||||
- nosniff
|
||||
access-control-expose-headers:
|
||||
- X-Request-ID
|
||||
alt-svc:
|
||||
- h3=":443"; ma=86400
|
||||
cf-cache-status:
|
||||
- DYNAMIC
|
||||
openai-organization:
|
||||
- crewai-iuxna1
|
||||
openai-processing-ms:
|
||||
- '1744'
|
||||
openai-version:
|
||||
- '2020-10-01'
|
||||
strict-transport-security:
|
||||
- max-age=31536000; includeSubDomains; preload
|
||||
x-ratelimit-limit-requests:
|
||||
- '30000'
|
||||
x-ratelimit-limit-tokens:
|
||||
- '150000000'
|
||||
x-ratelimit-remaining-requests:
|
||||
- '29999'
|
||||
x-ratelimit-remaining-tokens:
|
||||
- '149999140'
|
||||
x-ratelimit-reset-requests:
|
||||
- 2ms
|
||||
x-ratelimit-reset-tokens:
|
||||
- 0s
|
||||
x-request-id:
|
||||
- req_74247d0bd52dcebe834bd1cdb4b38470
|
||||
status:
|
||||
code: 200
|
||||
message: OK
|
||||
- request:
|
||||
body: '{"messages": [{"role": "system", "content": "You are Test Agent. Test Backstory\nYour
|
||||
personal goal is: Test Goal\nTo give my best complete final answer to the task
|
||||
respond using the exact following format:\n\nThought: I now can give a great
|
||||
answer\nFinal Answer: Your final answer must be the great and the most complete
|
||||
as possible, it must be outcome described.\n\nI MUST use these formats, my job
|
||||
depends on it!"}, {"role": "user", "content": "\nCurrent Task: Gather information
|
||||
about available books on the First World War\n\nThis is the expected criteria
|
||||
for your final answer: A list of available books on the First World War\nyou
|
||||
MUST return the actual complete content as the final answer, not a summary.\n\nThis
|
||||
is the context you''re working with:\n### Previous attempt failed validation:
|
||||
The task result does not comply with the guardrail because none of the listed
|
||||
authors are from Italy. All authors mentioned are from different countries,
|
||||
including Germany, the UK, the USA, and others, which violates the requirement
|
||||
that authors must be Italian.\n\n\n### Previous result:\nHere is a list of notable
|
||||
books on the First World War, encompassing various perspectives and themes:\n\n1.
|
||||
**\"All Quiet on the Western Front\" by Erich Maria Remarque** - A novel depicting
|
||||
the experiences of German soldiers during World War I, highlighting the horrors
|
||||
of trench warfare.\n\n2. **\"The First World War\" by John Keegan** - A comprehensive
|
||||
overview of the war, analyzing its causes, major battles, and consequences.\n\n3.
|
||||
**\"A World Undone: The Story of the Great War, 1914 to 1918\" by G.J. Meyer**
|
||||
- A narrative history that covers the entire conflict with a focus on key events
|
||||
and figures.\n\n4. **\"The Guns of August\" by Barbara W. Tuchman** - An acclaimed
|
||||
work detailing the events leading up to the war and the early stages of combat,
|
||||
emphasizing the decisions of leaders.\n\n5. **\"Goodbye to All That\" by Robert
|
||||
Graves** - An autobiography that captures the experience of trench warfare from
|
||||
a soldier''s perspective, along with the transition to post-war life.\n\n6.
|
||||
**\"With Our Backs to the Wall: Victory and Defeat in 1918\" by David Stevenson**
|
||||
- An analysis of the final year of the war, outlining both the military strategies
|
||||
and the socio-political contexts that shaped the outcome.\n\n7. **\"The Great
|
||||
War: A Combat History of the First World War\" by Peter Hart** - This book provides
|
||||
a battle-by-battle account, using personal diaries and accounts to bring the
|
||||
war''s events to life.\n\n8. **\"The War to End All Wars: The American Military
|
||||
Experience in World War I\" by Edward M. Coffman** - An exploration of American
|
||||
involvement in the war, discussing military strategies and impacts.\n\n9. **\"Over
|
||||
the Top: A Soldier\u2019s Diary of the First World War\" by Arthur Empey** -
|
||||
A firsthand account of trench warfare written by an American volunteer, offering
|
||||
a raw depiction of combat experiences.\n\n10. **\"The First World War: A New
|
||||
Illustrated History\" by Gordon Corrigan** - A richly illustrated book that
|
||||
presents a chronological history of the war, accessible for readers of all backgrounds.\n\nThis
|
||||
list provides a variety of insights and narratives that capture the complexity
|
||||
and significance of the First World War.\n\n\nTry again, making sure to address
|
||||
the validation error.\n\nBegin! This is VERY important to you, use the tools
|
||||
available and give your best Final Answer, your job depends on it!\n\nThought:"}],
|
||||
"model": "gpt-4o-mini", "stop": ["\nObservation:"]}'
|
||||
headers:
|
||||
accept:
|
||||
- application/json
|
||||
accept-encoding:
|
||||
- gzip, deflate
|
||||
connection:
|
||||
- keep-alive
|
||||
content-length:
|
||||
- '3509'
|
||||
content-type:
|
||||
- application/json
|
||||
cookie:
|
||||
- __cf_bm=2Kv7JW0NnxXgvGH0KQnR1VcNF47nEYxhEB4bYlFvRfA-1745929590-1.0.1.1-gmQvNlZR_WoNPIkf.07YchEy.a13FU2sP7wZropzue.84PKf6DVY_clzo6DqmBOk8PtKtM3gV952vhWHuE0ygk7096hW9jQTST91FDi30Jc;
|
||||
_cfuvid=HHxY6s22oZkfAHAh8HGGIeDkKOuavigZ5DkKL6.WyGc-1745929590485-0.0.1.1-604800000
|
||||
host:
|
||||
- api.openai.com
|
||||
user-agent:
|
||||
- OpenAI/Python 1.68.2
|
||||
x-stainless-arch:
|
||||
- arm64
|
||||
x-stainless-async:
|
||||
- 'false'
|
||||
x-stainless-lang:
|
||||
- python
|
||||
x-stainless-os:
|
||||
- MacOS
|
||||
x-stainless-package-version:
|
||||
- 1.68.2
|
||||
x-stainless-raw-response:
|
||||
- 'true'
|
||||
x-stainless-read-timeout:
|
||||
- '600.0'
|
||||
x-stainless-retry-count:
|
||||
- '0'
|
||||
x-stainless-runtime:
|
||||
- CPython
|
||||
x-stainless-runtime-version:
|
||||
- 3.12.9
|
||||
method: POST
|
||||
uri: https://api.openai.com/v1/chat/completions
|
||||
response:
|
||||
body:
|
||||
string: !!binary |
|
||||
H4sIAAAAAAAAAwAAAP//jFZtbtxGEv2vUxT4ZwFjRtAocmTpn7yBHCW24cTGGkFkCDXdRbLsZhVT
|
||||
3ZzxKDCw19gb5B65yZ5k0U3OhxUF2D8zIJtdXa/eq9f1+xFAxb66hMq1mFzXh/nzn+lHt/rl5Q+v
|
||||
vVu8ufrB/xTf6Q3+/Ozdx9BXs7xDlx/Jpe2uY6ddHyixyrjsjDBRjro4P3t6cXrx9OKkLHTqKeRt
|
||||
TZ/mZzrvWHh+enJ6Nj85ny+eTbtbZUexuoRfjwAAfi+/OU/x9Lm6hBKrvOkoRmyoutx9BFCZhvym
|
||||
whg5JpRUzfaLTiWRlNRvQHQNDgUaXhEgNDltQIlrMoBbuWbBAFfl+RK+JyPgCAiBYwKtQTThMhAs
|
||||
VT9FUIHUElyzxQTv1YKH92iwNk6JBJYbuEkYGAVwSK1anAGLU+vVMLE0sEJjHSL0ZLEnl3hFEVB8
|
||||
DttRvLyVW1kcw5Mnt9VNgI4VRDuCP/8AoRgH0dsqn/JdDgPX+uQJzOEKBuHfBgJBy8esCFgSWVqz
|
||||
5DPzWZpRxqTG03ktlyeHAWp0KQIudUgFHXd9eaP1AcSbjH0LLqpjSpvjnO3pmO1LhGYgMwRPDKIx
|
||||
Wf4T4THjVyVjXGGTI2jJ4RWaU3iDoWMyLljetRxLraE3XbHP2YKnhBzIAzqngxRecqL0uSdjEkcl
|
||||
2X12wTNZBD9Yxv8IYzOo1Q0xr46UskGniVeYBT5WKCYbmiZQLDC/GWG+HUIgSIbOEXgKAeGFoXiC
|
||||
FwX9CPYtWcMKbxOybDlqDPuWHYiuKEBqMcEykPgIaIljYpfxGMXIKrDm1B5ytKc2KXjq2Y1cBa4p
|
||||
Y99h5lGhyUhcO9YlP78osn+PVsCcTZz9o+gER2HghOeNcYcTHHil4hkDjbhe8BCp7wm+I3g5JOY4
|
||||
gcvmYNSSxJyirshWTOsRJX3ugxrFkkdD2mvgVEA5HCLFGXT4UQ2WmFLIj7n4QaWZJ7IOqK5pUmMO
|
||||
sEbbKnFTvlzSRsUXWE93UvyKk1yTa+MhTFJ8gxoU/okpoQ339/hAeNT1LUa+nzI2arh0z9gWD7ti
|
||||
F3sGLTdt4KZNW9FFboRrdihpi27b6w+1G7QUhFecJTwJcOK0YPt2wkYpZY/yDMOodEyTJ1yFbJTi
|
||||
TeH6GJ4bNyiJdgSFkO1GpZxWosRD38LtcYdds86dMjZifoV7K8n5sSTuMNGhmW0d0ggzyyM4p90S
|
||||
U4FxPsK4ur8fzLio1Vgc4aFL3EQhadqtI9DEy2fsWCZW9uV7tPOnLqhNJUHIux7reDeENBjNoFPD
|
||||
QLMdObFFIw8OOzT0+SDATnNRTLUfCXn2F99jeIUp0UTHtaG4LDPzLDzRcOi6PNJReqTWEHSdvW6j
|
||||
gzQ7QB3Kf//9nwgfdTChDaTWdGjaPTla11ToYolZehl60rFE2dB2ZPVx41oN2pTDDyzeqdSB3UjP
|
||||
xQhqcbF4Ol9cLJ4dw8tHDe5KvBHC62xJW2wxx0cLG1irfRpxoWDYbDspJsNEzfYGSuRa2WWEfoXi
|
||||
qCNJMbdg0A35wwu1Vsu98veeXgAsTnaXZ8NqopOprTgVjxvT/1eWnNznu+eQHYf9eAVIM7kWbjum
|
||||
qKmMCrnixeYiBMLSF9uCiz9wqdkk2PxBq9kLaZOF1ZMf3asQKzhxxJ4k8XSnFjMqE8jOU7I2DKX5
|
||||
Su/TkAHrVkclgOcVWTwYBcZa96a1DqVnH8qkG0LiGh0l8l+5UrFhatDx/hp5WPLDqcuoHiLmyU+G
|
||||
EA4WUPIQVS7WPO99mFa+7Ca8oE1vuowPtlY1C8f2zgijSp7mYtK+KqtfjgA+lEly+Go4rHrTrk93
|
||||
ST9ROe58cTbGq/YD7H717OJ8Wk2aMOwXFqeLxeyRiHfjLBIPptHKoWvJ7/fuR1ccPOvBwtEB7r/m
|
||||
81jsETtL8/+E3y84R30if9cb+dFnHvvMKE/4f/fZrs4l4SrmO93RXWKyzIWnGocwzt1V3MRE3V3N
|
||||
0pD1xuPwXfd3fokOvz3x9Ul19OXofwAAAP//AwDXPLUkigwAAA==
|
||||
headers:
|
||||
CF-RAY:
|
||||
- 937ec9c4b8307e0f-GRU
|
||||
Connection:
|
||||
- keep-alive
|
||||
Content-Encoding:
|
||||
- gzip
|
||||
Content-Type:
|
||||
- application/json
|
||||
Date:
|
||||
- Tue, 29 Apr 2025 12:26:41 GMT
|
||||
Server:
|
||||
- cloudflare
|
||||
Transfer-Encoding:
|
||||
- chunked
|
||||
X-Content-Type-Options:
|
||||
- nosniff
|
||||
access-control-expose-headers:
|
||||
- X-Request-ID
|
||||
alt-svc:
|
||||
- h3=":443"; ma=86400
|
||||
cf-cache-status:
|
||||
- DYNAMIC
|
||||
openai-organization:
|
||||
- crewai-iuxna1
|
||||
openai-processing-ms:
|
||||
- '10386'
|
||||
openai-version:
|
||||
- '2020-10-01'
|
||||
strict-transport-security:
|
||||
- max-age=31536000; includeSubDomains; preload
|
||||
x-ratelimit-limit-requests:
|
||||
- '30000'
|
||||
x-ratelimit-limit-tokens:
|
||||
- '150000000'
|
||||
x-ratelimit-remaining-requests:
|
||||
- '29999'
|
||||
x-ratelimit-remaining-tokens:
|
||||
- '149999169'
|
||||
x-ratelimit-reset-requests:
|
||||
- 2ms
|
||||
x-ratelimit-reset-tokens:
|
||||
- 0s
|
||||
x-request-id:
|
||||
- req_7e9c58909cf39af106866bae0c0cf7e1
|
||||
status:
|
||||
code: 200
|
||||
message: OK
|
||||
- request:
|
||||
body: '{"messages": [{"role": "system", "content": "You are Guardrail Agent. You
|
||||
are a expert at validating the output of a task. By providing effective feedback
|
||||
if the output is not valid.\nYour personal goal is: Validate the output of the
|
||||
task\n\nTo give my best complete final answer to the task respond using the
|
||||
exact following format:\n\nThought: I now can give a great answer\nFinal Answer:
|
||||
Your final answer must be the great and the most complete as possible, it must
|
||||
be outcome described.\n\nI MUST use these formats, my job depends on it!\nIMPORTANT:
|
||||
Your final answer MUST contain all the information requested in the following
|
||||
format: {\n \"valid\": bool,\n \"feedback\": str | None\n}\n\nIMPORTANT: Ensure
|
||||
the final output does not include any code block markers like ```json or ```python."},
|
||||
{"role": "user", "content": "\n Ensure the following task result complies
|
||||
with the given guardrail.\n\n Task result:\n Here is a list of
|
||||
notable books on the First World War written by Italian authors, incorporating
|
||||
various perspectives and themes:\n\n1. **\"Il mio nome \u00e8 nessuno\" by Dario
|
||||
Fo** - A unique narrative intertwining personal stories and historical facts
|
||||
about the impacts of World War I on Italian society.\n\n2. **\"La guerra dei
|
||||
nostri nonni\" by Mario avagliano and Marco Palmieri** - This book provides
|
||||
a detailed account of the experiences of Italian soldiers during the First World
|
||||
War, focusing on their motivations and struggles.\n\n3. **\"Sulle tracce della
|
||||
Grande Guerra\" by Sergio Staino** - A graphic novel that blends artistic expression
|
||||
with historical narrative to depict the life of soldiers in the trenches of
|
||||
the Great War.\n\n4. **\"L''intera storia della Prima Guerra Mondiale\" by Giuseppe
|
||||
De Lutiis** - A comprehensive overview that explores the geopolitical causes,
|
||||
major battles, and long-term effects of the war on Italy and beyond.\n\n5. **\"La
|
||||
Grande Guerra in Friuli\" by Paolo Cattaruzza** - This book emphasizes the regional
|
||||
impact of World War I in Friuli, highlighting the significant battles and the
|
||||
experiences of local civilians and soldiers.\n\n6. **\"Lettere di un soldato\"
|
||||
by Alessandro F. Brigante** - A collection of letters written by a soldier during
|
||||
the war, providing a personal and intimate perspective on the realities of combat.\n\n7.
|
||||
**\"Azzurri in trincea\" by Mario Isnenghi** - The book examines the experience
|
||||
of Italian soldiers in the front lines, focusing on their culture, morale, and
|
||||
the shared camaraderie among troops.\n\n8. **\"La guerra di Matteo\" by Franco
|
||||
Cardini** - A historical fiction that follows a young Italian man\u2019s journey
|
||||
through the war, offering insights into the emotional and psychological impacts
|
||||
of conflict.\n\n9. **\"1915-1918. La Grande Guerra\" by Andrea Nativi** - A
|
||||
scholarly work that analyzes the strategies and technological advancements employed
|
||||
by Italian forces during the First World War.\n\n10. **\"Il giorno della vittoria\"
|
||||
by Vincenzo Pardini** - A captivating exploration of the final offensives leading
|
||||
to the end of the war, examining how they shaped Italy\u2019s national identity.\n\nThis
|
||||
list highlights a range of Italian authors who offer diverse narratives and
|
||||
profound insights into the multifaceted experiences and legacies of the First
|
||||
World War.\n\n Guardrail:\n Ensure the authors are from Italy\n \n Your
|
||||
task:\n - Confirm if the Task result complies with the guardrail.\n -
|
||||
If not, provide clear feedback explaining what is wrong (e.g., by how much it
|
||||
violates the rule, or what specific part fails).\n - Focus only on identifying
|
||||
issues \u2014 do not propose corrections.\n - If the Task result complies
|
||||
with the guardrail, saying that is valid\n "}], "model": "gpt-4o-mini",
|
||||
"stop": ["\nObservation:"]}'
|
||||
headers:
|
||||
accept:
|
||||
- application/json
|
||||
accept-encoding:
|
||||
- gzip, deflate
|
||||
connection:
|
||||
- keep-alive
|
||||
content-length:
|
||||
- '3829'
|
||||
content-type:
|
||||
- application/json
|
||||
cookie:
|
||||
- __cf_bm=2Kv7JW0NnxXgvGH0KQnR1VcNF47nEYxhEB4bYlFvRfA-1745929590-1.0.1.1-gmQvNlZR_WoNPIkf.07YchEy.a13FU2sP7wZropzue.84PKf6DVY_clzo6DqmBOk8PtKtM3gV952vhWHuE0ygk7096hW9jQTST91FDi30Jc;
|
||||
_cfuvid=HHxY6s22oZkfAHAh8HGGIeDkKOuavigZ5DkKL6.WyGc-1745929590485-0.0.1.1-604800000
|
||||
host:
|
||||
- api.openai.com
|
||||
user-agent:
|
||||
- OpenAI/Python 1.68.2
|
||||
x-stainless-arch:
|
||||
- arm64
|
||||
x-stainless-async:
|
||||
- 'false'
|
||||
x-stainless-lang:
|
||||
- python
|
||||
x-stainless-os:
|
||||
- MacOS
|
||||
x-stainless-package-version:
|
||||
- 1.68.2
|
||||
x-stainless-raw-response:
|
||||
- 'true'
|
||||
x-stainless-read-timeout:
|
||||
- '600.0'
|
||||
x-stainless-retry-count:
|
||||
- '0'
|
||||
x-stainless-runtime:
|
||||
- CPython
|
||||
x-stainless-runtime-version:
|
||||
- 3.12.9
|
||||
method: POST
|
||||
uri: https://api.openai.com/v1/chat/completions
|
||||
response:
|
||||
body:
|
||||
string: !!binary |
|
||||
H4sIAAAAAAAAAwAAAP//jJJNb9swDIbv/hWEzvHgeFmT+NYNKFbstGLADnNhKBJtq5EpQZLTFUH+
|
||||
+yDHiZ19ALsYMB++FF+SxwSAKckKYKLlQXRWpx+f8Av5dXj8vH16kG+f9j6Xuxf5vf56nx3YIirM
|
||||
7gVFuKjeCdNZjUEZOmPhkAeMVZfr1Ydtvr3LlgPojEQdZY0N6cqknSKV5lm+SrN1utyM6tYogZ4V
|
||||
8CMBADgO39gnSfzJCsgWl0iH3vMGWXFNAmDO6Bhh3HvlA6fAFhMUhgLS0Pq31vRNGwp4BDKvIDhB
|
||||
ow4IHJrYP3Dyr+gASnpQxDXcD/8FHEsCKNmBayVLVkBwPS7OsRpR7rjYxzD1Wpd0mj/usO491yOc
|
||||
AU5kAo8DHGw/j+R0NapNY53Z+d+krFakfFs55N5QNOWDsWygpwTgeRhofzMjZp3pbKiC2ePw3Ppu
|
||||
HCib9jjRfDPCYALXM9XmAm7qVRIDV9rPVsIEFy3KSTrtj/dSmRlIZq7/7OZvtc/OFTX/U34CQqAN
|
||||
KCvrUCpx63hKcxjP/F9p1ykPDTOP7qAEVkGhi5uQWPNen4+P+TcfsKtqRQ0669T5AmtbZe+3+SbP
|
||||
s23GklPyCwAA//8DANS97/6PAwAA
|
||||
headers:
|
||||
CF-RAY:
|
||||
- 937eca0769737e0f-GRU
|
||||
Connection:
|
||||
- keep-alive
|
||||
Content-Encoding:
|
||||
- gzip
|
||||
Content-Type:
|
||||
- application/json
|
||||
Date:
|
||||
- Tue, 29 Apr 2025 12:26:42 GMT
|
||||
Server:
|
||||
- cloudflare
|
||||
Transfer-Encoding:
|
||||
- chunked
|
||||
X-Content-Type-Options:
|
||||
- nosniff
|
||||
access-control-expose-headers:
|
||||
- X-Request-ID
|
||||
alt-svc:
|
||||
- h3=":443"; ma=86400
|
||||
cf-cache-status:
|
||||
- DYNAMIC
|
||||
openai-organization:
|
||||
- crewai-iuxna1
|
||||
openai-processing-ms:
|
||||
- '924'
|
||||
openai-version:
|
||||
- '2020-10-01'
|
||||
strict-transport-security:
|
||||
- max-age=31536000; includeSubDomains; preload
|
||||
x-ratelimit-limit-requests:
|
||||
- '30000'
|
||||
x-ratelimit-limit-tokens:
|
||||
- '150000000'
|
||||
x-ratelimit-remaining-requests:
|
||||
- '29999'
|
||||
x-ratelimit-remaining-tokens:
|
||||
- '149999094'
|
||||
x-ratelimit-reset-requests:
|
||||
- 2ms
|
||||
x-ratelimit-reset-tokens:
|
||||
- 0s
|
||||
x-request-id:
|
||||
- req_242f5797b2a0bf5867b01ab6027921fa
|
||||
status:
|
||||
code: 200
|
||||
message: OK
|
||||
- request:
|
||||
body: '{"messages": [{"role": "system", "content": "You are Test Agent. Test Backstory\nYour
|
||||
personal goal is: Test Goal\nTo give my best complete final answer to the task
|
||||
respond using the exact following format:\n\nThought: I now can give a great
|
||||
answer\nFinal Answer: Your final answer must be the great and the most complete
|
||||
as possible, it must be outcome described.\n\nI MUST use these formats, my job
|
||||
depends on it!"}, {"role": "user", "content": "\nCurrent Task: Test task\n\nThis
|
||||
is the expected criteria for your final answer: Output\nyou MUST return the
|
||||
actual complete content as the final answer, not a summary.\n\nBegin! This is
|
||||
VERY important to you, use the tools available and give your best Final Answer,
|
||||
your job depends on it!\n\nThought:"}], "model": "gpt-4o-mini", "stop": ["\nObservation:"]}'
|
||||
headers:
|
||||
accept:
|
||||
- application/json
|
||||
accept-encoding:
|
||||
- gzip, deflate
|
||||
connection:
|
||||
- keep-alive
|
||||
content-length:
|
||||
- '807'
|
||||
content-type:
|
||||
- application/json
|
||||
cookie:
|
||||
- __cf_bm=2Kv7JW0NnxXgvGH0KQnR1VcNF47nEYxhEB4bYlFvRfA-1745929590-1.0.1.1-gmQvNlZR_WoNPIkf.07YchEy.a13FU2sP7wZropzue.84PKf6DVY_clzo6DqmBOk8PtKtM3gV952vhWHuE0ygk7096hW9jQTST91FDi30Jc;
|
||||
_cfuvid=HHxY6s22oZkfAHAh8HGGIeDkKOuavigZ5DkKL6.WyGc-1745929590485-0.0.1.1-604800000
|
||||
host:
|
||||
- api.openai.com
|
||||
user-agent:
|
||||
- OpenAI/Python 1.68.2
|
||||
x-stainless-arch:
|
||||
- arm64
|
||||
x-stainless-async:
|
||||
- 'false'
|
||||
x-stainless-lang:
|
||||
- python
|
||||
x-stainless-os:
|
||||
- MacOS
|
||||
x-stainless-package-version:
|
||||
- 1.68.2
|
||||
x-stainless-raw-response:
|
||||
- 'true'
|
||||
x-stainless-read-timeout:
|
||||
- '600.0'
|
||||
x-stainless-retry-count:
|
||||
- '0'
|
||||
x-stainless-runtime:
|
||||
- CPython
|
||||
x-stainless-runtime-version:
|
||||
- 3.12.9
|
||||
method: POST
|
||||
uri: https://api.openai.com/v1/chat/completions
|
||||
response:
|
||||
body:
|
||||
string: !!binary |
|
||||
H4sIAAAAAAAAAwAAAP//jFNNTxsxEL3nV4x8TqIQICm5tYeqiAMSqoSqFq0ce3Z3wOvZ2rMbAuK/
|
||||
V/YGNrQcell5583He+Pn5wmAIqs2oEytxTStm325wSuOVw+if3wz14Iu7tZP29vLm+vb05Wapgre
|
||||
3qOR16q54aZ1KMR+gE1ALZi6nqzPzi+WF6vFMgMNW3SprGpldsazhjzNlovl2Wyxnp18OlTXTAaj
|
||||
2sDPCQDAc/4mnt7io9rAYvoaaTBGXaHavCUBqMAuRZSOkaJoL2o6goa9oM/UL8HzDoz2UFGPoKFK
|
||||
tEH7uMMA8Mt/Ja8dfM7/G/heI5guBPQCouMDkO/Z9RihDdyTJV+BBqk5cFfVoL0Fi6LJoYWAsWUf
|
||||
EbY6ogX2IDUCPrZoBC2YQIKBNDTo0xbRzuESduQctAFjnpimD2tOhywCdiQ1dwKxaxod6Il8NQX0
|
||||
sQuJjNRaAHsMe9AxTQKKYLjHgBaEoUEc+gb83VHANDwCliUaoR7dfp41cyeGG3ylk6QiGKcDyT6r
|
||||
TLQC1uhjWiP5kkOjk4yBQdm5kpyLedRgnJTHZQ6kTc6PLyhg2UWdTOI7544A7T1L7putcXdAXt7M
|
||||
4LhqA2/jX6WqJE+xLgLqyD5dfBRuVUZfJgB32XTdOx+pNnDTSiH8gHncyfnp0E+NXh/R1foACot2
|
||||
Y3y5PFj1fb9i8EU8sq0y2tRox9LR47qzxEfA5Ej1v2w+6j0oJ1/9T/sRMAZbQVu0AS2Z94rHtID3
|
||||
2cQfp71tORNWEUNPBgshDOkmLJa6c8MDVXEfBZuiJF9haAMNr7RsC7vVRq8WtlyoycvkDwAAAP//
|
||||
AwC4yvtmswQAAA==
|
||||
headers:
|
||||
CF-RAY:
|
||||
- 937eca0f18177e0f-GRU
|
||||
Connection:
|
||||
- keep-alive
|
||||
Content-Encoding:
|
||||
- gzip
|
||||
Content-Type:
|
||||
- application/json
|
||||
Date:
|
||||
- Tue, 29 Apr 2025 12:26:44 GMT
|
||||
Server:
|
||||
- cloudflare
|
||||
Transfer-Encoding:
|
||||
- chunked
|
||||
X-Content-Type-Options:
|
||||
- nosniff
|
||||
access-control-expose-headers:
|
||||
- X-Request-ID
|
||||
alt-svc:
|
||||
- h3=":443"; ma=86400
|
||||
cf-cache-status:
|
||||
- DYNAMIC
|
||||
openai-organization:
|
||||
- crewai-iuxna1
|
||||
openai-processing-ms:
|
||||
- '1988'
|
||||
openai-version:
|
||||
- '2020-10-01'
|
||||
strict-transport-security:
|
||||
- max-age=31536000; includeSubDomains; preload
|
||||
x-ratelimit-limit-requests:
|
||||
- '30000'
|
||||
x-ratelimit-limit-tokens:
|
||||
- '150000000'
|
||||
x-ratelimit-remaining-requests:
|
||||
- '29999'
|
||||
x-ratelimit-remaining-tokens:
|
||||
- '149999832'
|
||||
x-ratelimit-reset-requests:
|
||||
- 2ms
|
||||
x-ratelimit-reset-tokens:
|
||||
- 0s
|
||||
x-request-id:
|
||||
- req_99b127ab59c932c0b46dbecc738df8ba
|
||||
status:
|
||||
code: 200
|
||||
message: OK
|
||||
version: 1
|
||||
288
tests/cassettes/test_guardrail_when_an_error_occurs.yaml
Normal file
288
tests/cassettes/test_guardrail_when_an_error_occurs.yaml
Normal file
@@ -0,0 +1,288 @@
|
||||
interactions:
|
||||
- request:
|
||||
body: '{"messages": [{"role": "system", "content": "You are Test Agent. Test Backstory\nYour
|
||||
personal goal is: Test Goal\nTo give my best complete final answer to the task
|
||||
respond using the exact following format:\n\nThought: I now can give a great
|
||||
answer\nFinal Answer: Your final answer must be the great and the most complete
|
||||
as possible, it must be outcome described.\n\nI MUST use these formats, my job
|
||||
depends on it!"}, {"role": "user", "content": "\nCurrent Task: Gather information
|
||||
about available books on the First World War\n\nThis is the expected criteria
|
||||
for your final answer: A list of available books on the First World War\nyou
|
||||
MUST return the actual complete content as the final answer, not a summary.\n\nBegin!
|
||||
This is VERY important to you, use the tools available and give your best Final
|
||||
Answer, your job depends on it!\n\nThought:"}], "model": "gpt-4o-mini", "stop":
|
||||
["\nObservation:"]}'
|
||||
headers:
|
||||
accept:
|
||||
- application/json
|
||||
accept-encoding:
|
||||
- gzip, deflate
|
||||
connection:
|
||||
- keep-alive
|
||||
content-length:
|
||||
- '903'
|
||||
content-type:
|
||||
- application/json
|
||||
host:
|
||||
- api.openai.com
|
||||
user-agent:
|
||||
- OpenAI/Python 1.68.2
|
||||
x-stainless-arch:
|
||||
- arm64
|
||||
x-stainless-async:
|
||||
- 'false'
|
||||
x-stainless-lang:
|
||||
- python
|
||||
x-stainless-os:
|
||||
- MacOS
|
||||
x-stainless-package-version:
|
||||
- 1.68.2
|
||||
x-stainless-raw-response:
|
||||
- 'true'
|
||||
x-stainless-read-timeout:
|
||||
- '600.0'
|
||||
x-stainless-retry-count:
|
||||
- '0'
|
||||
x-stainless-runtime:
|
||||
- CPython
|
||||
x-stainless-runtime-version:
|
||||
- 3.12.9
|
||||
method: POST
|
||||
uri: https://api.openai.com/v1/chat/completions
|
||||
response:
|
||||
body:
|
||||
string: !!binary |
|
||||
H4sIAAAAAAAAAwAAAP//jFdhb9w2Ev3uXzHQlwCBbewmThzvN1+Q2G3PQM+X1mjPhTFLjqSpKY46
|
||||
pHazV+S/H0hK2t00Be6LYZEi9d68x8fZP08AKrbVCirTYjRd787+cU/3P7+9tfTPq+1PTffDtr7+
|
||||
tf759+HG9tdSnaYVsv6dTJxWnRvpekeRxZdpo4SR0q7Ly4s3V68Xi+VlnujEkkvLmj6eXchZx57P
|
||||
Xi1eXZwtLs+W78bVrbChUK3gPycAAH/mvwmnt/S5WsHidBrpKARsqFrNLwFUKi6NVBgCh4g+Vqf7
|
||||
SSM+ks/QvwMvWzDooeENAUKTYAP6sCUFePQf2aOD6/y8gltSAg6AkOgqteRDWuc4RJAacIPscO0I
|
||||
1iLPAcRDbAk+soYID6LOwgPq6tE/+uU5vHz5WH1qCf7tiPotumfSsIJb2cKHQaUneCAfIUpaA+xh
|
||||
ebW8eKxgvYP3rXKI0rek8N6hPr98mcACwKeWQ/44WHIbCsA+SgZRBPoMNZooGiC2GMGRhXFehrhW
|
||||
wufEIz1vUU9B6pqUfQPsAzdtPNivF8eRDTpAbyGIYXRgdx47NiFtQqhuB68WsQVDPg66G3mdJ/6v
|
||||
Cv/rsSw/eSueVpDrEUV3E4ybLMhDApP4J7TLq+W7Uoeb8+/P4Y52pHMB8tOLAPS5xyKOR1WM6T8j
|
||||
G8rMCchH1kwSthxbQKjFDFmxtcQWOnYcUXcQomKkhilknmltO3To0wdImbyhAOTtoGQTpCDOcvpK
|
||||
etvwhh2jD4COnwvx1yNx5+BfA1OcTPJAIZJ6+KjiY6H3Qdm0cIfKCPfUof4x0Ez0GnrhxqOP4GVD
|
||||
rghqsI+DUiGpFNhliAW74tDhAfAM+IY00Zlx2yELPrsVvjuFNQayE1AcYiupxLI9qkJmd7G39Veu
|
||||
L5S+l9bDD0QN+plJeYReZcM21RksReTkzVmGNvl974pszpowDsfuTPJhjI7C6YFyp7N0yQDoHHDX
|
||||
o8mlb5ysk4frGlkLhzeFw42IXe8oOS6J9anFUZZ7WZNGuFHcUDg+eThEWbM0in27AyUjg4/huGyH
|
||||
xsHEdiz9VPm/Hj6EnjSIH8+a0fHgpdGeTPb2KI4RXzs2ce/WXkI8Sz4nxUzvbaH3HiOGqClE8sla
|
||||
TalzI8k+stfsDj/DLYbIvtnTnQbAtCqejRs9RxtKjB2hTdCHPm+1N9OMq8TDrFaR/ID/i5Q1HFOq
|
||||
7NUqELNdDVPk0XWXe9eljySp4IO3ZOFHQjPGyr2gHePjYiKmDSpFuENzx84deHIeSXI5mU7UcegV
|
||||
U+bHo1RNF4zyeojfTtcsLnV9i4H/OzG2ZDiw+AAdWkroUglzkBiVEA6z893fHrIVXMP7chMT3JYz
|
||||
M3ON7OGGXTLvsWvNtGI6ZRGfKYCOAGKrMjTtQW7Grw6iSgccAxgcQjFPesI6pmyJ7SkMofh4y5ZA
|
||||
0TeUNggy6BQcV4VTcWJi9guhlmDM5LKchcqPODi4xe7Ai12Jb5qv3J43EtHBLu0idbk7jq1XPGZp
|
||||
gyFialxSTKDbZUkSfie+OXPF50B1TSbO+28TqAx8uSjIk/FuRQON9WbTIjm4E+0HbWQGO5a8ZWeV
|
||||
/IswpTc5Vzw2Bx1Cm/abrHYUHF9nRctN61IGTqNr8RbWFLdEHtBzh65kQr69AmDHNsSSGS1KEeED
|
||||
mnZUNsw9TEqhAIPnPwY6jJyyXb6iyuENgGsZ4rdanlPo8HmE1sEG3ZC7JKXRAlCLAvqdeEodBmm6
|
||||
DG1qemJiPqmZ+EseHp16ftjXKdVDwNRb+sG5gwn0XorEuaP8bZz5MveQTppeZR2+WlrV7Dm0T0oY
|
||||
xKd+MTVdVZ79cgLwW+5Vh6P2s+pVuj4+RXmm/Lnl5dirVvsWeT/75vXFOBsTw/3E5WKaONrwaSz1
|
||||
QbtbGTQt2f3SfW+Mg2U5mDg5oP1XON/au1Bn3/w/2+8njKE+kn3qlSybY8r715TST4i/e20ucwZc
|
||||
BdING3qKTJqksFTj4EpjX4VdiNQ91ewb0l65dPd1/2TXaPDtwtaL6uTLyf8AAAD//wMA10Lu/OsM
|
||||
AAA=
|
||||
headers:
|
||||
CF-RAY:
|
||||
- 937ed42dee2e621f-GRU
|
||||
Connection:
|
||||
- keep-alive
|
||||
Content-Encoding:
|
||||
- gzip
|
||||
Content-Type:
|
||||
- application/json
|
||||
Date:
|
||||
- Tue, 29 Apr 2025 12:33:48 GMT
|
||||
Server:
|
||||
- cloudflare
|
||||
Set-Cookie:
|
||||
- __cf_bm=mLRCnpdB3n_6medIZWHnUu8MNRGZsD6riaRhN47PK74-1745930028-1.0.1.1-M2lDM1_V9hNCK0MZrBnFalF3lndC3JkS8zhDOGww_LmOrgdpU9fZLpNZUmyinCQOnlCjDjDYJUECM82ffT1anqBiO1NoDeNp91EPKiK7s.8;
|
||||
path=/; expires=Tue, 29-Apr-25 13:03:48 GMT; domain=.api.openai.com; HttpOnly;
|
||||
Secure; SameSite=None
|
||||
- _cfuvid=eTrj_ZhCx2XuylS5vYROwUlPrJBwOyrbS2Ki.msl45E-1745930028010-0.0.1.1-604800000;
|
||||
path=/; domain=.api.openai.com; HttpOnly; Secure; SameSite=None
|
||||
Transfer-Encoding:
|
||||
- chunked
|
||||
X-Content-Type-Options:
|
||||
- nosniff
|
||||
access-control-expose-headers:
|
||||
- X-Request-ID
|
||||
alt-svc:
|
||||
- h3=":443"; ma=86400
|
||||
cf-cache-status:
|
||||
- DYNAMIC
|
||||
openai-organization:
|
||||
- crewai-iuxna1
|
||||
openai-processing-ms:
|
||||
- '10856'
|
||||
openai-version:
|
||||
- '2020-10-01'
|
||||
strict-transport-security:
|
||||
- max-age=31536000; includeSubDomains; preload
|
||||
x-ratelimit-limit-requests:
|
||||
- '30000'
|
||||
x-ratelimit-limit-tokens:
|
||||
- '150000000'
|
||||
x-ratelimit-remaining-requests:
|
||||
- '29999'
|
||||
x-ratelimit-remaining-tokens:
|
||||
- '149999807'
|
||||
x-ratelimit-reset-requests:
|
||||
- 2ms
|
||||
x-ratelimit-reset-tokens:
|
||||
- 0s
|
||||
x-request-id:
|
||||
- req_bc2d62d8325b2bdd3e98544a66389132
|
||||
status:
|
||||
code: 200
|
||||
message: OK
|
||||
- request:
|
||||
body: '{"messages": [{"role": "system", "content": "You are Guardrail Agent. You
|
||||
are a expert at validating the output of a task. By providing effective feedback
|
||||
if the output is not valid.\nYour personal goal is: Validate the output of the
|
||||
task\n\nTo give my best complete final answer to the task respond using the
|
||||
exact following format:\n\nThought: I now can give a great answer\nFinal Answer:
|
||||
Your final answer must be the great and the most complete as possible, it must
|
||||
be outcome described.\n\nI MUST use these formats, my job depends on it!\nIMPORTANT:
|
||||
Your final answer MUST contain all the information requested in the following
|
||||
format: {\n \"valid\": bool,\n \"feedback\": str | None\n}\n\nIMPORTANT: Ensure
|
||||
the final output does not include any code block markers like ```json or ```python."},
|
||||
{"role": "user", "content": "\n Ensure the following task result complies
|
||||
with the given guardrail.\n\n Task result:\n Here is a comprehensive
|
||||
list of available books on the First World War:\n\n1. **\"The Sleepwalkers:
|
||||
How Europe Went to War in 1914\" by Christopher Clark** \n This book delves
|
||||
into the complex factors that led to the outbreak of the war, offering insights
|
||||
into the political and social dynamics of early 20th century Europe.\n\n2. **\"A
|
||||
World Undone: The Story of the Great War, 1914 to 1918\" by G.J. Meyer** \n Meyer''s
|
||||
expansive narrative covers the entire war with a focus on both military strategies
|
||||
and the human experiences endured by soldiers and civilians alike.\n\n3. **\"All
|
||||
Quiet on the Western Front\" by Erich Maria Remarque** \n A poignant novel
|
||||
that captures the resilience and trauma experienced by German soldiers during
|
||||
World War I, based on the author''s own experiences.\n\n4. **\"The First World
|
||||
War\" by John Keegan** \n Keegan provides a detailed military history of
|
||||
the war, featuring insights on battles, strategies, and the overall impact on
|
||||
global affairs.\n\n5. **\"Goodbye to All That\" by Robert Graves** \n This
|
||||
autobiography recounts the author''s experiences as a soldier during the war,
|
||||
offering a personal and critical perspective on the conflicts and the post-war
|
||||
era.\n\n6. **\"Catastrophe 1914: Europe Goes to War\" by Max Hastings** \n Hastings
|
||||
chronicles the events leading up to World War I and the early battles, detailing
|
||||
the war''s initial impact on European societies.\n\n7. **\"The War That Ended
|
||||
Peace: The Road to 1914\" by Margaret MacMillan** \n MacMillan explores the
|
||||
political and historical factors that contributed to the outbreak of war, emphasizing
|
||||
the decisions made by leaders across Europe.\n\n8. **\"The First World War:
|
||||
A Complete History\" by Martin Gilbert** \n This complete history takes readers
|
||||
through the entirety of the war, from its causes to its aftermath, using a wide
|
||||
range of sources.\n\n9. **\"1914: The Year the World Ended\" by Paul Ham** \n Ham
|
||||
focuses on the pivotal year of 1914 and the early war''s devastation, analyzing
|
||||
its long-lasting effects on the world.\n\n10. **\"War Horse\" by Michael Morpurgo** \n This
|
||||
children''s novel tells the story of a horse and his experiences during the
|
||||
war, highlighting the bond between animals and humans amidst the chaos.\n\nEach
|
||||
of these books offers unique perspectives and rich details about the First World
|
||||
War, making them valuable resources for anyone interested in this pivotal period
|
||||
in history.\n\n Guardrail:\n Ensure the authors are from Italy\n \n Your
|
||||
task:\n - Confirm if the Task result complies with the guardrail.\n -
|
||||
If not, provide clear feedback explaining what is wrong (e.g., by how much it
|
||||
violates the rule, or what specific part fails).\n - Focus only on identifying
|
||||
issues \u2014 do not propose corrections.\n - If the Task result complies
|
||||
with the guardrail, saying that is valid\n "}], "model": "gpt-4o-mini",
|
||||
"stop": ["\nObservation:"]}'
|
||||
headers:
|
||||
accept:
|
||||
- application/json
|
||||
accept-encoding:
|
||||
- gzip, deflate
|
||||
connection:
|
||||
- keep-alive
|
||||
content-length:
|
||||
- '3917'
|
||||
content-type:
|
||||
- application/json
|
||||
cookie:
|
||||
- __cf_bm=mLRCnpdB3n_6medIZWHnUu8MNRGZsD6riaRhN47PK74-1745930028-1.0.1.1-M2lDM1_V9hNCK0MZrBnFalF3lndC3JkS8zhDOGww_LmOrgdpU9fZLpNZUmyinCQOnlCjDjDYJUECM82ffT1anqBiO1NoDeNp91EPKiK7s.8;
|
||||
_cfuvid=eTrj_ZhCx2XuylS5vYROwUlPrJBwOyrbS2Ki.msl45E-1745930028010-0.0.1.1-604800000
|
||||
host:
|
||||
- api.openai.com
|
||||
user-agent:
|
||||
- OpenAI/Python 1.68.2
|
||||
x-stainless-arch:
|
||||
- arm64
|
||||
x-stainless-async:
|
||||
- 'false'
|
||||
x-stainless-lang:
|
||||
- python
|
||||
x-stainless-os:
|
||||
- MacOS
|
||||
x-stainless-package-version:
|
||||
- 1.68.2
|
||||
x-stainless-raw-response:
|
||||
- 'true'
|
||||
x-stainless-read-timeout:
|
||||
- '600.0'
|
||||
x-stainless-retry-count:
|
||||
- '0'
|
||||
x-stainless-runtime:
|
||||
- CPython
|
||||
x-stainless-runtime-version:
|
||||
- 3.12.9
|
||||
method: POST
|
||||
uri: https://api.openai.com/v1/chat/completions
|
||||
response:
|
||||
body:
|
||||
string: !!binary |
|
||||
H4sIAAAAAAAAAwAAAP//jFPLbtswELz7KxY8y4HsJHWsW9wiQVq0hzx6aBUIa3IlMaFIlVw5NQL/
|
||||
e0Epiew+gF4EamZnuC8+TwCEViIDIWtk2bRmurqm29X1+/PNzYfL06+fFV8svz2sttuF+XHzUSRR
|
||||
4dYPJPlVdSRd0xpi7exAS0/IFF1ni5PT5XE6m896onGKTJRVLU9P3LTRVk/n6fxkmi6ms7MXde20
|
||||
pCAy+D4BAHjuvzFPq+inyCBNXpGGQsCKRPYWBCC8MxERGIIOjJZFMpLSWSbbp35bu66qOYMrsO4J
|
||||
JFqo9IYAoYr5A9rwRB4gtxfaooHz/j+D59wC5GKDRqtcZFCiCZQMYEmk1igfI56LL84SuBK4JsCO
|
||||
a+cDGB2YFGjbo4zhETyFzjCgJyi9a+CK0WyP4NyYA2VDNraY1BjpuCYP0nWWvaaQQOhkDRjgknyD
|
||||
dpv0BnefEkCrhvPNUS5yu9vviaeyCxjnYjtj9gi01jHGS/tp3L8wu7f+G1e13q3Db1JRaqtDXXjC
|
||||
4GzsdWDXip7dTQDu+zl3B6MTrXdNywW7R+qvWywXg58Y12tk370sgWDHaEb87PRVdeBXKGLUJuxt
|
||||
ipAoa1KjdFwr7JR2e8Rkr+o/s/mb91C5ttX/2I+ElNQyqaL1pLQ8rHgM8xRf37/C3rrcJywC+Y2W
|
||||
VLAmHyehqMTODG9ChG1gaopS24p86/XwMMq2SI+X87P5PF2mYrKb/AIAAP//AwD77a3iJgQAAA==
|
||||
headers:
|
||||
CF-RAY:
|
||||
- 937ed6bd68faa435-GRU
|
||||
Connection:
|
||||
- keep-alive
|
||||
Content-Encoding:
|
||||
- gzip
|
||||
Content-Type:
|
||||
- application/json
|
||||
Date:
|
||||
- Tue, 29 Apr 2025 12:35:23 GMT
|
||||
Server:
|
||||
- cloudflare
|
||||
Transfer-Encoding:
|
||||
- chunked
|
||||
X-Content-Type-Options:
|
||||
- nosniff
|
||||
access-control-expose-headers:
|
||||
- X-Request-ID
|
||||
alt-svc:
|
||||
- h3=":443"; ma=86400
|
||||
cf-cache-status:
|
||||
- DYNAMIC
|
||||
openai-organization:
|
||||
- crewai-iuxna1
|
||||
openai-processing-ms:
|
||||
- '1138'
|
||||
openai-version:
|
||||
- '2020-10-01'
|
||||
strict-transport-security:
|
||||
- max-age=31536000; includeSubDomains; preload
|
||||
x-ratelimit-limit-requests:
|
||||
- '30000'
|
||||
x-ratelimit-limit-tokens:
|
||||
- '150000000'
|
||||
x-ratelimit-remaining-requests:
|
||||
- '29999'
|
||||
x-ratelimit-remaining-tokens:
|
||||
- '149999072'
|
||||
x-ratelimit-reset-requests:
|
||||
- 2ms
|
||||
x-ratelimit-reset-tokens:
|
||||
- 0s
|
||||
x-request-id:
|
||||
- req_2ba1be014a5974ba354aff564e26516a
|
||||
status:
|
||||
code: 200
|
||||
message: OK
|
||||
version: 1
|
||||
File diff suppressed because it is too large
Load Diff
250
tests/cassettes/test_task_guardrail_process_output.yaml
Normal file
250
tests/cassettes/test_task_guardrail_process_output.yaml
Normal file
@@ -0,0 +1,250 @@
|
||||
interactions:
|
||||
- request:
|
||||
body: '{"messages": [{"role": "system", "content": "You are Guardrail Agent. You
|
||||
are a expert at validating the output of a task. By providing effective feedback
|
||||
if the output is not valid.\nYour personal goal is: Validate the output of the
|
||||
task\n\nTo give my best complete final answer to the task respond using the
|
||||
exact following format:\n\nThought: I now can give a great answer\nFinal Answer:
|
||||
Your final answer must be the great and the most complete as possible, it must
|
||||
be outcome described.\n\nI MUST use these formats, my job depends on it!\nIMPORTANT:
|
||||
Your final answer MUST contain all the information requested in the following
|
||||
format: {\n \"valid\": bool,\n \"feedback\": str | None\n}\n\nIMPORTANT: Ensure
|
||||
the final output does not include any code block markers like ```json or ```python."},
|
||||
{"role": "user", "content": "\n Ensure the following task result complies
|
||||
with the given guardrail.\n\n Task result:\n \n Lorem Ipsum
|
||||
is simply dummy text of the printing and typesetting industry. Lorem Ipsum has
|
||||
been the industry''s standard dummy text ever\n \n\n Guardrail:\n Ensure
|
||||
the result has less than 10 words\n \n Your task:\n - Confirm
|
||||
if the Task result complies with the guardrail.\n - If not, provide clear
|
||||
feedback explaining what is wrong (e.g., by how much it violates the rule, or
|
||||
what specific part fails).\n - Focus only on identifying issues \u2014
|
||||
do not propose corrections.\n - If the Task result complies with the
|
||||
guardrail, saying that is valid\n "}], "model": "gpt-4o-mini", "stop":
|
||||
["\nObservation:"]}'
|
||||
headers:
|
||||
accept:
|
||||
- application/json
|
||||
accept-encoding:
|
||||
- gzip, deflate
|
||||
connection:
|
||||
- keep-alive
|
||||
content-length:
|
||||
- '1629'
|
||||
content-type:
|
||||
- application/json
|
||||
host:
|
||||
- api.openai.com
|
||||
user-agent:
|
||||
- OpenAI/Python 1.68.2
|
||||
x-stainless-arch:
|
||||
- arm64
|
||||
x-stainless-async:
|
||||
- 'false'
|
||||
x-stainless-lang:
|
||||
- python
|
||||
x-stainless-os:
|
||||
- MacOS
|
||||
x-stainless-package-version:
|
||||
- 1.68.2
|
||||
x-stainless-raw-response:
|
||||
- 'true'
|
||||
x-stainless-read-timeout:
|
||||
- '600.0'
|
||||
x-stainless-retry-count:
|
||||
- '0'
|
||||
x-stainless-runtime:
|
||||
- CPython
|
||||
x-stainless-runtime-version:
|
||||
- 3.12.9
|
||||
method: POST
|
||||
uri: https://api.openai.com/v1/chat/completions
|
||||
response:
|
||||
body:
|
||||
string: !!binary |
|
||||
H4sIAAAAAAAAAwAAAP//jFPLbtswELz7KxY824GkxLGtW4KiQB+XBmkRtAqENbmSmFAkQVJ2UsP/
|
||||
HlByLKdNgV4IcGdnOPvgbgLApGA5MN5g4K1Vs+ub718rm324+5z+/CLt1dXD5fU3s1jd3Wx//GbT
|
||||
yDDrB+LhlXXGTWsVBWn0AHNHGCiqpouL+XKVZum8B1ojSEVabcPswsxaqeUsS7KLWbKYpcsDuzGS
|
||||
k2c5/JoAAOz6M/rUgp5YDsn0NdKS91gTy49JAMwZFSMMvZc+oA5sOoLc6EC6t37bmK5uQg6fQJst
|
||||
cNRQyw0BQh39A2q/JQdQ6I9So4Kr/p7DrtAABdugkqJgOVSoPE2HYEUk1sgfY7xgtw1BQP8Ijnyn
|
||||
AsTHUWoP6SVsjRN+CvTEiYTUNYSGoO7QCYdSgZKtDGAqqCiaCA1qSJOBBetnOAicFazQ+9MCHVWd
|
||||
x9hk3Sl1AqDWJmAcUt/a+wOyPzZTmdo6s/Z/UFkltfRN6Qi90bFxPhjLenQ/Abjvh9a9mQOzzrQ2
|
||||
lME8Uv/ceTIf9Ni4KyM6Tw9gMAHVCWt+OX1HrxQUUCp/MnbGkTckRuq4I9gJaU6AyUnVf7t5T3uo
|
||||
XOr6f+RHgHOygURpHQnJ31Y8pjmKX+lfaccu94aZJ7eRnMogycVJCKqwU8OCM//sA7VlJXVNzjo5
|
||||
bHlly+R8lS2zLFklbLKfvAAAAP//AwCHe/Jh8wMAAA==
|
||||
headers:
|
||||
CF-RAY:
|
||||
- 937b20ddf9607def-GRU
|
||||
Connection:
|
||||
- keep-alive
|
||||
Content-Encoding:
|
||||
- gzip
|
||||
Content-Type:
|
||||
- application/json
|
||||
Date:
|
||||
- Tue, 29 Apr 2025 01:46:56 GMT
|
||||
Server:
|
||||
- cloudflare
|
||||
Set-Cookie:
|
||||
- __cf_bm=nHa2kVJI_yO1RIsmZcEednJ1e9UVy1liv_sjBNtSj7Q-1745891216-1.0.1.1-jUH9kFawVBjnbq8sIL2.MQx.p7JvBZWUhqlkNKRlStWSgQxT0eZMPcgq9TCQoJAjuyNwhqfpK4HuX6x5n8UbQgAb6JrWJEG823e6GpGROEA;
|
||||
path=/; expires=Tue, 29-Apr-25 02:16:56 GMT; domain=.api.openai.com; HttpOnly;
|
||||
Secure; SameSite=None
|
||||
- _cfuvid=gg2UeahMCOOR8YhitRtzDwENMOnTOuQdyTMVJVHG0Mg-1745891216085-0.0.1.1-604800000;
|
||||
path=/; domain=.api.openai.com; HttpOnly; Secure; SameSite=None
|
||||
Transfer-Encoding:
|
||||
- chunked
|
||||
X-Content-Type-Options:
|
||||
- nosniff
|
||||
access-control-expose-headers:
|
||||
- X-Request-ID
|
||||
alt-svc:
|
||||
- h3=":443"; ma=86400
|
||||
cf-cache-status:
|
||||
- DYNAMIC
|
||||
openai-organization:
|
||||
- crewai-iuxna1
|
||||
openai-processing-ms:
|
||||
- '896'
|
||||
openai-version:
|
||||
- '2020-10-01'
|
||||
strict-transport-security:
|
||||
- max-age=31536000; includeSubDomains; preload
|
||||
x-ratelimit-limit-requests:
|
||||
- '30000'
|
||||
x-ratelimit-limit-tokens:
|
||||
- '150000000'
|
||||
x-ratelimit-remaining-requests:
|
||||
- '29999'
|
||||
x-ratelimit-remaining-tokens:
|
||||
- '149999631'
|
||||
x-ratelimit-reset-requests:
|
||||
- 2ms
|
||||
x-ratelimit-reset-tokens:
|
||||
- 0s
|
||||
x-request-id:
|
||||
- req_859221ed1aedb26cc9d335004ccf183e
|
||||
status:
|
||||
code: 200
|
||||
message: OK
|
||||
- request:
|
||||
body: '{"messages": [{"role": "system", "content": "You are Guardrail Agent. You
|
||||
are a expert at validating the output of a task. By providing effective feedback
|
||||
if the output is not valid.\nYour personal goal is: Validate the output of the
|
||||
task\n\nTo give my best complete final answer to the task respond using the
|
||||
exact following format:\n\nThought: I now can give a great answer\nFinal Answer:
|
||||
Your final answer must be the great and the most complete as possible, it must
|
||||
be outcome described.\n\nI MUST use these formats, my job depends on it!\nIMPORTANT:
|
||||
Your final answer MUST contain all the information requested in the following
|
||||
format: {\n \"valid\": bool,\n \"feedback\": str | None\n}\n\nIMPORTANT: Ensure
|
||||
the final output does not include any code block markers like ```json or ```python."},
|
||||
{"role": "user", "content": "\n Ensure the following task result complies
|
||||
with the given guardrail.\n\n Task result:\n \n Lorem Ipsum
|
||||
is simply dummy text of the printing and typesetting industry. Lorem Ipsum has
|
||||
been the industry''s standard dummy text ever\n \n\n Guardrail:\n Ensure
|
||||
the result has less than 500 words\n \n Your task:\n -
|
||||
Confirm if the Task result complies with the guardrail.\n - If not, provide
|
||||
clear feedback explaining what is wrong (e.g., by how much it violates the rule,
|
||||
or what specific part fails).\n - Focus only on identifying issues \u2014
|
||||
do not propose corrections.\n - If the Task result complies with the
|
||||
guardrail, saying that is valid\n "}], "model": "gpt-4o-mini", "stop":
|
||||
["\nObservation:"]}'
|
||||
headers:
|
||||
accept:
|
||||
- application/json
|
||||
accept-encoding:
|
||||
- gzip, deflate
|
||||
connection:
|
||||
- keep-alive
|
||||
content-length:
|
||||
- '1630'
|
||||
content-type:
|
||||
- application/json
|
||||
cookie:
|
||||
- __cf_bm=nHa2kVJI_yO1RIsmZcEednJ1e9UVy1liv_sjBNtSj7Q-1745891216-1.0.1.1-jUH9kFawVBjnbq8sIL2.MQx.p7JvBZWUhqlkNKRlStWSgQxT0eZMPcgq9TCQoJAjuyNwhqfpK4HuX6x5n8UbQgAb6JrWJEG823e6GpGROEA;
|
||||
_cfuvid=gg2UeahMCOOR8YhitRtzDwENMOnTOuQdyTMVJVHG0Mg-1745891216085-0.0.1.1-604800000
|
||||
host:
|
||||
- api.openai.com
|
||||
user-agent:
|
||||
- OpenAI/Python 1.68.2
|
||||
x-stainless-arch:
|
||||
- arm64
|
||||
x-stainless-async:
|
||||
- 'false'
|
||||
x-stainless-lang:
|
||||
- python
|
||||
x-stainless-os:
|
||||
- MacOS
|
||||
x-stainless-package-version:
|
||||
- 1.68.2
|
||||
x-stainless-raw-response:
|
||||
- 'true'
|
||||
x-stainless-read-timeout:
|
||||
- '600.0'
|
||||
x-stainless-retry-count:
|
||||
- '0'
|
||||
x-stainless-runtime:
|
||||
- CPython
|
||||
x-stainless-runtime-version:
|
||||
- 3.12.9
|
||||
method: POST
|
||||
uri: https://api.openai.com/v1/chat/completions
|
||||
response:
|
||||
body:
|
||||
string: !!binary |
|
||||
H4sIAAAAAAAAAwAAAP//jJJNb9swDIbv/hWEzvHgfHRpfesOG3opsGE7LYXBSLStRZY0iU43BPnv
|
||||
g5wPu10H7GLAfPhSfEkeMgChlShByBZZdt7kH758e1wzbnfbO6o/f1osV3T/+BO7UNNDIWZJ4bY/
|
||||
SPJF9U66zhti7ewJy0DIlKrO16ub27v5srgZQOcUmSRrPOcrl3fa6nxRLFZ5sc7nt2d167SkKEr4
|
||||
ngEAHIZv6tMq+iVKKGaXSEcxYkOivCYBiOBMigiMUUdGy2I2Qukskx1a/9q6vmm5hAew7hkkWmj0
|
||||
ngChSf0D2vhMAWBjP2qLBu6H/xIOGwuwEXs0Wm1ECRx6mp1iNZHaotylsO2N2djj9PFAdR/RnOEE
|
||||
oLWOMQ1wsP10JserUeMaH9w2vpKKWlsd2yoQRmeTqcjOi4EeM4CnYaD9ixkJH1znuWK3o+G583KG
|
||||
4Vz2ONLF7RmyYzQT1XI5e6NepYhRmzhZiZAoW1KjdNwf9kq7Ccgmrv/u5q3aJ+faNv9TfgRSkmdS
|
||||
lQ+ktHzpeEwLlM78X2nXKQ8Ni0hhryVVrCmkTSiqsTen4xPxd2TqqlrbhoIP+nSBta/SueD7QtWF
|
||||
yI7ZHwAAAP//AwAiLXhqjwMAAA==
|
||||
headers:
|
||||
CF-RAY:
|
||||
- 937b2311ee091b1b-GRU
|
||||
Connection:
|
||||
- keep-alive
|
||||
Content-Encoding:
|
||||
- gzip
|
||||
Content-Type:
|
||||
- application/json
|
||||
Date:
|
||||
- Tue, 29 Apr 2025 01:48:26 GMT
|
||||
Server:
|
||||
- cloudflare
|
||||
Transfer-Encoding:
|
||||
- chunked
|
||||
X-Content-Type-Options:
|
||||
- nosniff
|
||||
access-control-expose-headers:
|
||||
- X-Request-ID
|
||||
alt-svc:
|
||||
- h3=":443"; ma=86400
|
||||
cf-cache-status:
|
||||
- DYNAMIC
|
||||
openai-organization:
|
||||
- crewai-iuxna1
|
||||
openai-processing-ms:
|
||||
- '610'
|
||||
openai-version:
|
||||
- '2020-10-01'
|
||||
strict-transport-security:
|
||||
- max-age=31536000; includeSubDomains; preload
|
||||
x-ratelimit-limit-requests:
|
||||
- '30000'
|
||||
x-ratelimit-limit-tokens:
|
||||
- '150000000'
|
||||
x-ratelimit-remaining-requests:
|
||||
- '29999'
|
||||
x-ratelimit-remaining-tokens:
|
||||
- '149999631'
|
||||
x-ratelimit-reset-requests:
|
||||
- 2ms
|
||||
x-ratelimit-reset-tokens:
|
||||
- 0s
|
||||
x-request-id:
|
||||
- req_c136835c16be6bc1e4d820f239c4b620
|
||||
status:
|
||||
code: 200
|
||||
message: OK
|
||||
version: 1
|
||||
@@ -6,6 +6,7 @@ research_task:
|
||||
expected_output: >
|
||||
A list with 10 bullet points of the most relevant information about {topic}
|
||||
agent: researcher
|
||||
guardrail: ensure each bullet contains its source
|
||||
|
||||
reporting_task:
|
||||
description: >
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
from typing import List
|
||||
from unittest.mock import patch
|
||||
|
||||
import pytest
|
||||
|
||||
@@ -142,6 +143,15 @@ def test_agent_function_calling_llm():
|
||||
), "agent's function_calling_llm is incorrect"
|
||||
|
||||
|
||||
def test_task_guardrail():
|
||||
crew = InternalCrew()
|
||||
research_task = crew.research_task()
|
||||
assert research_task.guardrail == "ensure each bullet contains its source"
|
||||
|
||||
reporting_task = crew.reporting_task()
|
||||
assert reporting_task.guardrail is None
|
||||
|
||||
|
||||
@pytest.mark.vcr(filter_headers=["authorization"])
|
||||
def test_before_kickoff_modification():
|
||||
crew = InternalCrew()
|
||||
|
||||
@@ -1,11 +1,16 @@
|
||||
"""Tests for task guardrails functionality."""
|
||||
|
||||
from unittest.mock import Mock
|
||||
from unittest.mock import ANY, Mock, patch
|
||||
|
||||
import pytest
|
||||
|
||||
from crewai.task import Task
|
||||
from crewai import Agent, Task
|
||||
from crewai.llm import LLM
|
||||
from crewai.tasks.task_guardrail import TaskGuardrail
|
||||
from crewai.tasks.task_output import TaskOutput
|
||||
from crewai.utilities.events import (
|
||||
TaskGuardrailCompletedEvent,
|
||||
TaskGuardrailStartedEvent,
|
||||
)
|
||||
from crewai.utilities.events.crewai_event_bus import crewai_event_bus
|
||||
|
||||
|
||||
def test_task_without_guardrail():
|
||||
@@ -22,7 +27,7 @@ def test_task_without_guardrail():
|
||||
assert result.raw == "test result"
|
||||
|
||||
|
||||
def test_task_with_successful_guardrail():
|
||||
def test_task_with_successful_guardrail_func():
|
||||
"""Test that successful guardrail validation passes transformed result."""
|
||||
|
||||
def guardrail(result: TaskOutput):
|
||||
@@ -127,3 +132,138 @@ def test_guardrail_error_in_context():
|
||||
|
||||
assert "Task failed guardrail validation" in str(exc_info.value)
|
||||
assert "Expected JSON, got string" in str(exc_info.value)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def sample_agent():
|
||||
return Agent(role="Test Agent", goal="Test Goal", backstory="Test Backstory")
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def task_output():
|
||||
return TaskOutput(
|
||||
raw="""
|
||||
Lorem Ipsum is simply dummy text of the printing and typesetting industry. Lorem Ipsum has been the industry's standard dummy text ever
|
||||
""",
|
||||
description="Test task",
|
||||
expected_output="Output",
|
||||
agent="Test Agent",
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.vcr(filter_headers=["authorization"])
|
||||
def test_task_guardrail_process_output(task_output):
|
||||
guardrail = TaskGuardrail(
|
||||
description="Ensure the result has less than 10 words", llm=LLM(model="gpt-4o")
|
||||
)
|
||||
|
||||
result = guardrail(task_output)
|
||||
assert result[0] is False
|
||||
|
||||
assert "exceeding the guardrail limit of fewer than" in result[1].lower()
|
||||
|
||||
guardrail = TaskGuardrail(
|
||||
description="Ensure the result has less than 500 words", llm=LLM(model="gpt-4o")
|
||||
)
|
||||
|
||||
result = guardrail(task_output)
|
||||
assert result[0] is True
|
||||
assert result[1] == task_output.raw
|
||||
|
||||
|
||||
@pytest.mark.vcr(filter_headers=["authorization"])
|
||||
def test_guardrail_emits_events(sample_agent):
|
||||
started_guardrail = []
|
||||
completed_guardrail = []
|
||||
|
||||
with crewai_event_bus.scoped_handlers():
|
||||
|
||||
@crewai_event_bus.on(TaskGuardrailStartedEvent)
|
||||
def handle_guardrail_started(source, event):
|
||||
started_guardrail.append(
|
||||
{"guardrail": event.guardrail, "retry_count": event.retry_count}
|
||||
)
|
||||
|
||||
@crewai_event_bus.on(TaskGuardrailCompletedEvent)
|
||||
def handle_guardrail_completed(source, event):
|
||||
completed_guardrail.append(
|
||||
{
|
||||
"success": event.success,
|
||||
"result": event.result,
|
||||
"error": event.error,
|
||||
"retry_count": event.retry_count,
|
||||
}
|
||||
)
|
||||
|
||||
task = Task(
|
||||
description="Gather information about available books on the First World War",
|
||||
agent=sample_agent,
|
||||
expected_output="A list of available books on the First World War",
|
||||
guardrail="Ensure the authors are from Italy",
|
||||
)
|
||||
|
||||
result = task.execute_sync(agent=sample_agent)
|
||||
|
||||
def custom_guardrail(result: TaskOutput):
|
||||
return (True, "good result from callable function")
|
||||
|
||||
task = Task(
|
||||
description="Test task",
|
||||
expected_output="Output",
|
||||
guardrail=custom_guardrail,
|
||||
)
|
||||
|
||||
task.execute_sync(agent=sample_agent)
|
||||
|
||||
expected_started_events = [
|
||||
{"guardrail": "Ensure the authors are from Italy", "retry_count": 0},
|
||||
{"guardrail": "Ensure the authors are from Italy", "retry_count": 1},
|
||||
{
|
||||
"guardrail": """def custom_guardrail(result: TaskOutput):
|
||||
return (True, "good result from callable function")""",
|
||||
"retry_count": 0,
|
||||
},
|
||||
]
|
||||
|
||||
expected_completed_events = [
|
||||
{
|
||||
"success": False,
|
||||
"result": None,
|
||||
"error": "The task result does not comply with the guardrail because none of "
|
||||
"the listed authors are from Italy. All authors mentioned are from "
|
||||
"different countries, including Germany, the UK, the USA, and others, "
|
||||
"which violates the requirement that authors must be Italian.",
|
||||
"retry_count": 0,
|
||||
},
|
||||
{"success": True, "result": result.raw, "error": None, "retry_count": 1},
|
||||
{
|
||||
"success": True,
|
||||
"result": "good result from callable function",
|
||||
"error": None,
|
||||
"retry_count": 0,
|
||||
},
|
||||
]
|
||||
assert started_guardrail == expected_started_events
|
||||
assert completed_guardrail == expected_completed_events
|
||||
|
||||
|
||||
@pytest.mark.vcr(filter_headers=["authorization"])
|
||||
def test_guardrail_when_an_error_occurs(sample_agent, task_output):
|
||||
with (
|
||||
patch(
|
||||
"crewai.Agent.kickoff",
|
||||
side_effect=Exception("Unexpected error"),
|
||||
),
|
||||
pytest.raises(
|
||||
Exception,
|
||||
match="Error while validating the task output: Unexpected error",
|
||||
),
|
||||
):
|
||||
task = Task(
|
||||
description="Gather information about available books on the First World War",
|
||||
agent=sample_agent,
|
||||
expected_output="A list of available books on the First World War",
|
||||
guardrail="Ensure the authors are from Italy",
|
||||
max_retries=0,
|
||||
)
|
||||
task.execute_sync(agent=sample_agent)
|
||||
|
||||
Reference in New Issue
Block a user