Merge branch 'main' into brandon/cre-509-hitl-multiple-rounds-of-followup

src/crewai/agents/agent_builder/base_agent_executor_mixin.py

@@ -3,16 +3,15 @@ from typing import TYPE_CHECKING, Optional
 
 from crewai.memory.entity.entity_memory_item import EntityMemoryItem
 from crewai.memory.long_term.long_term_memory_item import LongTermMemoryItem
+from crewai.utilities import I18N
 from crewai.utilities.converter import ConverterError
 from crewai.utilities.evaluators.task_evaluator import TaskEvaluator
-from crewai.utilities import I18N
 from crewai.utilities.printer import Printer
 
-
 if TYPE_CHECKING:
+    from crewai.agents.agent_builder.base_agent import BaseAgent
     from crewai.crew import Crew
     from crewai.task import Task
-    from crewai.agents.agent_builder.base_agent import BaseAgent
 
 
 class CrewAgentExecutorMixin:
@@ -100,14 +99,19 @@ class CrewAgentExecutorMixin:
                 print(f"Failed to add to long term memory: {e}")
                 pass
 
-    def _ask_human_input(self, final_answer: dict) -> str:
+    def _ask_human_input(self, final_answer: str) -> str:
         """Prompt human input for final decision making."""
         self._printer.print(
             content=f"\033[1m\033[95m ## Final Result:\033[00m \033[92m{final_answer}\033[00m"
         )
 
         self._printer.print(
-            content="\n\n=====\n## Please provide feedback on the Final Result and the Agent's actions:",
+            content=(
+                "\n\n=====\n"
+                "## Please provide feedback on the Final Result and the Agent's actions. "
+                "Respond with 'looks good' or a similar phrase when you're satisfied.\n"
+                "=====\n"
+            ),
             color="bold_yellow",
         )
         return input()

src/crewai/agents/crew_agent_executor.py

@@ -16,7 +16,7 @@ from crewai.agents.tools_handler import ToolsHandler
 from crewai.tools.base_tool import BaseTool
 from crewai.tools.tool_usage import ToolUsage, ToolUsageErrorException
 from crewai.utilities import I18N, Printer
-from crewai.utilities.constants import TRAINING_DATA_FILE
+from crewai.utilities.constants import MAX_LLM_RETRY, TRAINING_DATA_FILE
 from crewai.utilities.exceptions.context_window_exceeding_exception import (
     LLMContextLengthExceededException,
 )
@@ -90,7 +90,6 @@ class CrewAgentExecutor(CrewAgentExecutorMixin):
         if "system" in self.prompt:
             system_prompt = self._format_prompt(self.prompt.get("system", ""), inputs)
             user_prompt = self._format_prompt(self.prompt.get("user", ""), inputs)
-
             self.messages.append(self._format_msg(system_prompt, role="system"))
             self.messages.append(self._format_msg(user_prompt))
         else:
@@ -103,17 +102,8 @@ class CrewAgentExecutor(CrewAgentExecutorMixin):
         formatted_answer = self._invoke_loop()
 
         if self.ask_for_human_input:
-            human_feedback = self._ask_human_input(formatted_answer.output)
-            if self.crew and self.crew._train:
-                self._handle_crew_training_output(formatted_answer, human_feedback)
+            formatted_answer = self._handle_human_feedback(formatted_answer)
 
-            # Making sure we only ask for it once, so disabling for the next thought loop
-            self.ask_for_human_input = False
-            self.messages.append(self._format_msg(f"Feedback: {human_feedback}"))
-            formatted_answer = self._invoke_loop()
-
-            if self.crew and self.crew._train:
-                self._handle_crew_training_output(formatted_answer)
         self._create_short_term_memory(formatted_answer)
         self._create_long_term_memory(formatted_answer)
         return {"output": formatted_answer.output}
@@ -326,16 +316,14 @@ class CrewAgentExecutor(CrewAgentExecutorMixin):
 
     def _handle_context_length(self) -> None:
         if self.respect_context_window:
-            self._logger.log(
-                "debug",
-                "Context length exceeded. Summarizing content to fit the model context window.",
+            self._printer.print(
+                content="Context length exceeded. Summarizing content to fit the model context window.",
                 color="yellow",
             )
             self._summarize_messages()
         else:
-            self._logger.log(
-                "debug",
-                "Context length exceeded. Consider using smaller text or RAG tools from crewai_tools.",
+            self._printer.print(
+                content="Context length exceeded. Consider using smaller text or RAG tools from crewai_tools.",
                 color="red",
             )
             raise SystemExit(
@@ -362,15 +350,13 @@ class CrewAgentExecutor(CrewAgentExecutorMixin):
                     ] = result.output
                     training_handler.save(training_data)
                 else:
-                    self._logger.log(
-                        "error",
-                        "Invalid train iteration type or agent_id not in training data.",
+                    self._printer.print(
+                        content="Invalid train iteration type or agent_id not in training data.",
                         color="red",
                     )
             else:
-                self._logger.log(
-                    "error",
-                    "Crew is None or does not have _train_iteration attribute.",
+                self._printer.print(
+                    content="Crew is None or does not have _train_iteration attribute.",
                     color="red",
                 )
 
@@ -388,15 +374,13 @@ class CrewAgentExecutor(CrewAgentExecutorMixin):
                         train_iteration, agent_id, training_data
                     )
                 else:
-                    self._logger.log(
-                        "error",
-                        "Invalid train iteration type. Expected int.",
+                    self._printer.print(
+                        content="Invalid train iteration type. Expected int.",
                         color="red",
                     )
             else:
-                self._logger.log(
-                    "error",
-                    "Crew is None or does not have _train_iteration attribute.",
+                self._printer.print(
+                    content="Crew is None or does not have _train_iteration attribute.",
                     color="red",
                 )
 
@@ -412,3 +396,82 @@ class CrewAgentExecutor(CrewAgentExecutorMixin):
     def _format_msg(self, prompt: str, role: str = "user") -> Dict[str, str]:
         prompt = prompt.rstrip()
         return {"role": role, "content": prompt}
+
+    def _handle_human_feedback(self, formatted_answer: AgentFinish) -> AgentFinish:
+        """
+        Handles the human feedback loop, allowing the user to provide feedback
+        on the agent's output and determining if additional iterations are needed.
+
+        Parameters:
+            formatted_answer (AgentFinish): The initial output from the agent.
+
+        Returns:
+            AgentFinish: The final output after incorporating human feedback.
+        """
+        while self.ask_for_human_input:
+            human_feedback = self._ask_human_input(formatted_answer.output)
+            print("Human feedback: ", human_feedback)
+
+            if self.crew and self.crew._train:
+                self._handle_crew_training_output(formatted_answer, human_feedback)
+
+            # Make an LLM call to verify if additional changes are requested based on human feedback
+            additional_changes_prompt = self._i18n.slice(
+                "human_feedback_classification"
+            ).format(feedback=human_feedback)
+
+            retry_count = 0
+            llm_call_successful = False
+            additional_changes_response = None
+
+            while retry_count < MAX_LLM_RETRY and not llm_call_successful:
+                try:
+                    additional_changes_response = (
+                        self.llm.call(
+                            [
+                                self._format_msg(
+                                    additional_changes_prompt, role="system"
+                                )
+                            ],
+                            callbacks=self.callbacks,
+                        )
+                        .strip()
+                        .lower()
+                    )
+                    llm_call_successful = True
+                except Exception as e:
+                    retry_count += 1
+
+                    self._printer.print(
+                        content=f"Error during LLM call to classify human feedback: {e}. Retrying... ({retry_count}/{MAX_LLM_RETRY})",
+                        color="red",
+                    )
+
+            if not llm_call_successful:
+                self._printer.print(
+                    content="Error processing feedback after multiple attempts.",
+                    color="red",
+                )
+                self.ask_for_human_input = False
+                break
+
+            if additional_changes_response == "false":
+                self.ask_for_human_input = False
+            elif additional_changes_response == "true":
+                self.ask_for_human_input = True
+                # Add human feedback to messages
+                self.messages.append(self._format_msg(f"Feedback: {human_feedback}"))
+                # Invoke the loop again with updated messages
+                formatted_answer = self._invoke_loop()
+
+                if self.crew and self.crew._train:
+                    self._handle_crew_training_output(formatted_answer)
+            else:
+                # Unexpected response
+                self._printer.print(
+                    content=f"Unexpected response from LLM: '{additional_changes_response}'. Assuming no additional changes requested.",
+                    color="red",
+                )
+                self.ask_for_human_input = False
+
+        return formatted_answer

src/crewai/translations/en.json

@@ -22,7 +22,8 @@
     "sumamrize_instruction": "Summarize the following text, make sure to include all the important information: {group}",
     "summary": "This is a summary of our conversation so far:\n{merged_summary}",
     "manager_request": "Your best answer to your coworker asking you this, accounting for the context shared.",
-    "formatted_task_instructions": "Ensure your final answer contains only the content in the following format: {output_format}\n\nEnsure the final output does not include any code block markers like ```json or ```python."
+    "formatted_task_instructions": "Ensure your final answer contains only the content in the following format: {output_format}\n\nEnsure the final output does not include any code block markers like ```json or ```python.",
+    "human_feedback_classification": "Determine if the following feedback indicates that the user is satisfied or if further changes are needed. Respond with 'True' if further changes are needed, or 'False' if the user is satisfied. **Important** Do not include any additional commentary outside of your 'True' or 'False' response.\n\nFeedback: \"{feedback}\""
   },
   "errors": {
     "force_final_answer_error": "You can't keep going, this was the best you could do.\n {formatted_answer.text}",

src/crewai/utilities/constants.py

@@ -2,3 +2,4 @@ TRAINING_DATA_FILE = "training_data.pkl"
 TRAINED_AGENTS_DATA_FILE = "trained_agents_data.pkl"
 DEFAULT_SCORE_THRESHOLD = 0.35
 KNOWLEDGE_DIRECTORY = "knowledge"
+MAX_LLM_RETRY = 3

tests/agent_test.py

@@ -3,19 +3,20 @@
 import os
 from unittest import mock
 from unittest.mock import patch
+
 import pytest
 
 from crewai import Agent, Crew, Task
 from crewai.agents.cache import CacheHandler
 from crewai.agents.crew_agent_executor import CrewAgentExecutor
 from crewai.agents.parser import AgentAction, CrewAgentParser, OutputParserException
+from crewai.knowledge.source.string_knowledge_source import StringKnowledgeSource
 from crewai.llm import LLM
 from crewai.tools import tool
 from crewai.tools.tool_calling import InstructorToolCalling
 from crewai.tools.tool_usage import ToolUsage
 from crewai.tools.tool_usage_events import ToolUsageFinished
 from crewai.utilities import RPMController
-from crewai.knowledge.source.string_knowledge_source import StringKnowledgeSource
 from crewai.utilities.events import Emitter
 
 
@@ -985,8 +986,7 @@ def test_agent_definition_based_on_dict():
 # test for human input
 @pytest.mark.vcr(filter_headers=["authorization"])
 def test_agent_human_input():
-    from unittest.mock import patch
-
+    # Agent configuration
     config = {
         "role": "test role",
         "goal": "test goal",
@@ -995,6 +995,7 @@ def test_agent_human_input():
 
     agent = Agent(**config)
 
+    # Task configuration with human input enabled
     task = Task(
         agent=agent,
         description="Say the word: Hi",
@@ -1002,11 +1003,26 @@ def test_agent_human_input():
         human_input=True,
     )
 
-    with patch.object(CrewAgentExecutor, "_ask_human_input") as mock_human_input:
-        mock_human_input.return_value = "Don't say hi, say Hello instead!"
+    # Side effect function for _ask_human_input to simulate multiple feedback iterations
+    feedback_responses = iter(
+        [
+            "Don't say hi, say Hello instead!",  # First feedback
+            "looks good",  # Second feedback to exit loop
+        ]
+    )
+
+    def ask_human_input_side_effect(*args, **kwargs):
+        return next(feedback_responses)
+
+    with patch.object(
+        CrewAgentExecutor, "_ask_human_input", side_effect=ask_human_input_side_effect
+    ) as mock_human_input:
+        # Execute the task
         output = agent.execute_task(task)
-        mock_human_input.assert_called_once()
-        assert output == "Hello"
+
+        # Assertions to ensure the agent behaves correctly
+        assert mock_human_input.call_count == 2  # Should have asked for feedback twice
+        assert output.strip().lower() == "hello"  # Final output should be 'Hello'
 
 
 def test_interpolate_inputs():

tests/cassettes/test_agent_human_input.yaml

@@ -9,7 +9,8 @@ interactions:
       is the expect criteria for your final answer: The word: Hi\nyou MUST return
       the actual complete content as the final answer, not a summary.\n\nBegin! This
       is VERY important to you, use the tools available and give your best Final Answer,
-      your job depends on it!\n\nThought:"}], "model": "gpt-4o"}'
+      your job depends on it!\n\nThought:"}], "model": "gpt-4o-mini", "stop": ["\nObservation:"],
+      "stream": false}'
     headers:
       accept:
       - application/json
@@ -18,16 +19,13 @@ interactions:
       connection:
       - keep-alive
       content-length:
-      - '774'
+      - '824'
       content-type:
       - application/json
-      cookie:
-      - __cf_bm=rb61BZH2ejzD5YPmLaEJqI7km71QqyNJGTVdNxBq6qk-1727213194-1.0.1.1-pJ49onmgX9IugEMuYQMralzD7oj_6W.CHbSu4Su1z3NyjTGYg.rhgJZWng8feFYah._oSnoYlkTjpK1Wd2C9FA;
-        _cfuvid=lbRdAddVWV6W3f5Dm9SaOPWDUOxqtZBSPr_fTW26nEA-1727213194587-0.0.1.1-604800000
       host:
       - api.openai.com
       user-agent:
-      - OpenAI/Python 1.47.0
+      - OpenAI/Python 1.52.1
       x-stainless-arch:
       - arm64
       x-stainless-async:
@@ -37,29 +35,34 @@ interactions:
       x-stainless-os:
       - MacOS
       x-stainless-package-version:
-      - 1.47.0
+      - 1.52.1
       x-stainless-raw-response:
       - 'true'
+      x-stainless-retry-count:
+      - '0'
       x-stainless-runtime:
       - CPython
       x-stainless-runtime-version:
-      - 3.11.7
+      - 3.12.7
     method: POST
     uri: https://api.openai.com/v1/chat/completions
   response:
-    content: "{\n  \"id\": \"chatcmpl-AB7WMYMmqACvaemh26N6a62wxlxvx\",\n  \"object\":
-      \"chat.completion\",\n  \"created\": 1727213882,\n  \"model\": \"gpt-4o-2024-05-13\",\n
+    content: "{\n  \"id\": \"chatcmpl-AaqIIsTxhvf75xvuu7gQScIlRSKbW\",\n  \"object\":
+      \"chat.completion\",\n  \"created\": 1733344190,\n  \"model\": \"gpt-4o-mini-2024-07-18\",\n
       \ \"choices\": [\n    {\n      \"index\": 0,\n      \"message\": {\n        \"role\":
-      \"assistant\",\n        \"content\": \"Thought: I now can give a great answer\\nFinal
+      \"assistant\",\n        \"content\": \"I now can give a great answer  \\nFinal
       Answer: Hi\",\n        \"refusal\": null\n      },\n      \"logprobs\": null,\n
       \     \"finish_reason\": \"stop\"\n    }\n  ],\n  \"usage\": {\n    \"prompt_tokens\":
-      158,\n    \"completion_tokens\": 14,\n    \"total_tokens\": 172,\n    \"completion_tokens_details\":
-      {\n      \"reasoning_tokens\": 0\n    }\n  },\n  \"system_fingerprint\": \"fp_e375328146\"\n}\n"
+      158,\n    \"completion_tokens\": 12,\n    \"total_tokens\": 170,\n    \"prompt_tokens_details\":
+      {\n      \"cached_tokens\": 0,\n      \"audio_tokens\": 0\n    },\n    \"completion_tokens_details\":
+      {\n      \"reasoning_tokens\": 0,\n      \"audio_tokens\": 0,\n      \"accepted_prediction_tokens\":
+      0,\n      \"rejected_prediction_tokens\": 0\n    }\n  },\n  \"system_fingerprint\":
+      \"fp_0705bf87c0\"\n}\n"
     headers:
       CF-Cache-Status:
       - DYNAMIC
       CF-RAY:
-      - 8c85eb4f58751cf3-GRU
+      - 8ece8cfc3b1f4532-ATL
       Connection:
       - keep-alive
       Content-Encoding:
@@ -67,7 +70,118 @@ interactions:
       Content-Type:
       - application/json
       Date:
-      - Tue, 24 Sep 2024 21:38:03 GMT
+      - Wed, 04 Dec 2024 20:29:50 GMT
+      Server:
+      - cloudflare
+      Set-Cookie:
+      - __cf_bm=QJZZjZ6eqnVamqUkw.Bx0mj7oBi3a_vGEH1VODcUxlg-1733344190-1.0.1.1-xyN0ekA9xIrSwEhRBmTiWJ3Pt72UYLU5owKfkz5yihVmMTfsr_Qz.ssGPJ5cuft066v1xVjb4zOSTdFmesMSKg;
+        path=/; expires=Wed, 04-Dec-24 20:59:50 GMT; domain=.api.openai.com; HttpOnly;
+        Secure; SameSite=None
+      - _cfuvid=eCIkP8GVPvpkg19eOhCquWFHm.RTQBQy4yHLGGEAH5c-1733344190334-0.0.1.1-604800000;
+        path=/; domain=.api.openai.com; HttpOnly; Secure; SameSite=None
+      Transfer-Encoding:
+      - chunked
+      X-Content-Type-Options:
+      - nosniff
+      access-control-expose-headers:
+      - X-Request-ID
+      alt-svc:
+      - h3=":443"; ma=86400
+      openai-organization:
+      - crewai-iuxna1
+      openai-processing-ms:
+      - '313'
+      openai-version:
+      - '2020-10-01'
+      strict-transport-security:
+      - max-age=31536000; includeSubDomains; preload
+      x-ratelimit-limit-requests:
+      - '30000'
+      x-ratelimit-limit-tokens:
+      - '150000000'
+      x-ratelimit-remaining-requests:
+      - '29999'
+      x-ratelimit-remaining-tokens:
+      - '149999816'
+      x-ratelimit-reset-requests:
+      - 2ms
+      x-ratelimit-reset-tokens:
+      - 0s
+      x-request-id:
+      - req_9fd9a8ee688045dcf7ac5f6fdf689372
+    http_version: HTTP/1.1
+    status_code: 200
+- request:
+    body: '{"messages": [{"role": "system", "content": "Determine if the following
+      feedback indicates that the user is satisfied or if further changes are needed.
+      Respond with ''True'' if further changes are needed, or ''False'' if the user
+      is satisfied. **Important** Do not include any additional commentary outside
+      of your ''True'' or ''False'' response.\n\nFeedback: \"Don''t say hi, say Hello
+      instead!\""}], "model": "gpt-4o-mini", "stop": ["\nObservation:"], "stream":
+      false}'
+    headers:
+      accept:
+      - application/json
+      accept-encoding:
+      - gzip, deflate
+      connection:
+      - keep-alive
+      content-length:
+      - '461'
+      content-type:
+      - application/json
+      cookie:
+      - __cf_bm=QJZZjZ6eqnVamqUkw.Bx0mj7oBi3a_vGEH1VODcUxlg-1733344190-1.0.1.1-xyN0ekA9xIrSwEhRBmTiWJ3Pt72UYLU5owKfkz5yihVmMTfsr_Qz.ssGPJ5cuft066v1xVjb4zOSTdFmesMSKg;
+        _cfuvid=eCIkP8GVPvpkg19eOhCquWFHm.RTQBQy4yHLGGEAH5c-1733344190334-0.0.1.1-604800000
+      host:
+      - api.openai.com
+      user-agent:
+      - OpenAI/Python 1.52.1
+      x-stainless-arch:
+      - arm64
+      x-stainless-async:
+      - 'false'
+      x-stainless-lang:
+      - python
+      x-stainless-os:
+      - MacOS
+      x-stainless-package-version:
+      - 1.52.1
+      x-stainless-raw-response:
+      - 'true'
+      x-stainless-retry-count:
+      - '0'
+      x-stainless-runtime:
+      - CPython
+      x-stainless-runtime-version:
+      - 3.12.7
+    method: POST
+    uri: https://api.openai.com/v1/chat/completions
+  response:
+    content: "{\n  \"id\": \"chatcmpl-AaqIIaQlLyoyPmk909PvAIfA2TmJL\",\n  \"object\":
+      \"chat.completion\",\n  \"created\": 1733344190,\n  \"model\": \"gpt-4o-mini-2024-07-18\",\n
+      \ \"choices\": [\n    {\n      \"index\": 0,\n      \"message\": {\n        \"role\":
+      \"assistant\",\n        \"content\": \"True\",\n        \"refusal\": null\n
+      \     },\n      \"logprobs\": null,\n      \"finish_reason\": \"stop\"\n    }\n
+      \ ],\n  \"usage\": {\n    \"prompt_tokens\": 78,\n    \"completion_tokens\":
+      1,\n    \"total_tokens\": 79,\n    \"prompt_tokens_details\": {\n      \"cached_tokens\":
+      0,\n      \"audio_tokens\": 0\n    },\n    \"completion_tokens_details\": {\n
+      \     \"reasoning_tokens\": 0,\n      \"audio_tokens\": 0,\n      \"accepted_prediction_tokens\":
+      0,\n      \"rejected_prediction_tokens\": 0\n    }\n  },\n  \"system_fingerprint\":
+      \"fp_0705bf87c0\"\n}\n"
+    headers:
+      CF-Cache-Status:
+      - DYNAMIC
+      CF-RAY:
+      - 8ece8d060b5e4532-ATL
+      Connection:
+      - keep-alive
+      Content-Encoding:
+      - gzip
+      Content-Type:
+      - application/json
+      Date:
+      - Wed, 04 Dec 2024 20:29:50 GMT
       Server:
       - cloudflare
       Transfer-Encoding:
@@ -76,28 +190,30 @@ interactions:
       - nosniff
       access-control-expose-headers:
       - X-Request-ID
+      alt-svc:
+      - h3=":443"; ma=86400
       openai-organization:
       - crewai-iuxna1
       openai-processing-ms:
-      - '262'
+      - '375'
       openai-version:
       - '2020-10-01'
       strict-transport-security:
       - max-age=31536000; includeSubDomains; preload
       x-ratelimit-limit-requests:
-      - '10000'
+      - '30000'
       x-ratelimit-limit-tokens:
-      - '30000000'
+      - '150000000'
       x-ratelimit-remaining-requests:
-      - '9999'
+      - '29999'
       x-ratelimit-remaining-tokens:
-      - '29999817'
+      - '149999898'
       x-ratelimit-reset-requests:
-      - 6ms
+      - 2ms
       x-ratelimit-reset-tokens:
       - 0s
       x-request-id:
-      - req_69b1deae1cc3cbf488cee975cd3b04df
+      - req_be7cb475e0859a82c37ee3f2871ea5ea
     http_version: HTTP/1.1
     status_code: 200
 - request:
@@ -110,8 +226,10 @@ interactions:
       is the expect criteria for your final answer: The word: Hi\nyou MUST return
       the actual complete content as the final answer, not a summary.\n\nBegin! This
       is VERY important to you, use the tools available and give your best Final Answer,
-      your job depends on it!\n\nThought:"}, {"role": "user", "content": "Feedback:
-      Don''t say hi, say Hello instead!"}], "model": "gpt-4o"}'
+      your job depends on it!\n\nThought:"}, {"role": "assistant", "content": "I now
+      can give a great answer  \nFinal Answer: Hi"}, {"role": "user", "content": "Feedback:
+      Don''t say hi, say Hello instead!"}], "model": "gpt-4o-mini", "stop": ["\nObservation:"],
+      "stream": false}'
     headers:
       accept:
       - application/json
@@ -120,16 +238,16 @@ interactions:
       connection:
       - keep-alive
       content-length:
-      - '849'
+      - '986'
       content-type:
       - application/json
       cookie:
-      - __cf_bm=rb61BZH2ejzD5YPmLaEJqI7km71QqyNJGTVdNxBq6qk-1727213194-1.0.1.1-pJ49onmgX9IugEMuYQMralzD7oj_6W.CHbSu4Su1z3NyjTGYg.rhgJZWng8feFYah._oSnoYlkTjpK1Wd2C9FA;
-        _cfuvid=lbRdAddVWV6W3f5Dm9SaOPWDUOxqtZBSPr_fTW26nEA-1727213194587-0.0.1.1-604800000
+      - __cf_bm=QJZZjZ6eqnVamqUkw.Bx0mj7oBi3a_vGEH1VODcUxlg-1733344190-1.0.1.1-xyN0ekA9xIrSwEhRBmTiWJ3Pt72UYLU5owKfkz5yihVmMTfsr_Qz.ssGPJ5cuft066v1xVjb4zOSTdFmesMSKg;
+        _cfuvid=eCIkP8GVPvpkg19eOhCquWFHm.RTQBQy4yHLGGEAH5c-1733344190334-0.0.1.1-604800000
       host:
       - api.openai.com
       user-agent:
-      - OpenAI/Python 1.47.0
+      - OpenAI/Python 1.52.1
       x-stainless-arch:
       - arm64
       x-stainless-async:
@@ -139,29 +257,34 @@ interactions:
       x-stainless-os:
       - MacOS
       x-stainless-package-version:
-      - 1.47.0
+      - 1.52.1
       x-stainless-raw-response:
       - 'true'
+      x-stainless-retry-count:
+      - '0'
       x-stainless-runtime:
       - CPython
       x-stainless-runtime-version:
-      - 3.11.7
+      - 3.12.7
     method: POST
     uri: https://api.openai.com/v1/chat/completions
   response:
-    content: "{\n  \"id\": \"chatcmpl-AB7WNec1Ohw0pEU91kuCTuts2hXWM\",\n  \"object\":
-      \"chat.completion\",\n  \"created\": 1727213883,\n  \"model\": \"gpt-4o-2024-05-13\",\n
+    content: "{\n  \"id\": \"chatcmpl-AaqIJAAxpVfUOdrsgYKHwfRlHv4RS\",\n  \"object\":
+      \"chat.completion\",\n  \"created\": 1733344191,\n  \"model\": \"gpt-4o-mini-2024-07-18\",\n
       \ \"choices\": [\n    {\n      \"index\": 0,\n      \"message\": {\n        \"role\":
-      \"assistant\",\n        \"content\": \"Thought: I now can give a great answer\\nFinal
-      Answer: Hello\",\n        \"refusal\": null\n      },\n      \"logprobs\": null,\n
-      \     \"finish_reason\": \"stop\"\n    }\n  ],\n  \"usage\": {\n    \"prompt_tokens\":
-      172,\n    \"completion_tokens\": 14,\n    \"total_tokens\": 186,\n    \"completion_tokens_details\":
-      {\n      \"reasoning_tokens\": 0\n    }\n  },\n  \"system_fingerprint\": \"fp_e375328146\"\n}\n"
+      \"assistant\",\n        \"content\": \"Thought: I now can give a great answer
+      \ \\nFinal Answer: Hello\",\n        \"refusal\": null\n      },\n      \"logprobs\":
+      null,\n      \"finish_reason\": \"stop\"\n    }\n  ],\n  \"usage\": {\n    \"prompt_tokens\":
+      188,\n    \"completion_tokens\": 14,\n    \"total_tokens\": 202,\n    \"prompt_tokens_details\":
+      {\n      \"cached_tokens\": 0,\n      \"audio_tokens\": 0\n    },\n    \"completion_tokens_details\":
+      {\n      \"reasoning_tokens\": 0,\n      \"audio_tokens\": 0,\n      \"accepted_prediction_tokens\":
+      0,\n      \"rejected_prediction_tokens\": 0\n    }\n  },\n  \"system_fingerprint\":
+      \"fp_0705bf87c0\"\n}\n"
     headers:
       CF-Cache-Status:
       - DYNAMIC
       CF-RAY:
-      - 8c85eb52cd7c1cf3-GRU
+      - 8ece8d090fc34532-ATL
       Connection:
       - keep-alive
       Content-Encoding:
@@ -169,7 +292,7 @@ interactions:
       Content-Type:
       - application/json
       Date:
-      - Tue, 24 Sep 2024 21:38:03 GMT
+      - Wed, 04 Dec 2024 20:29:51 GMT
       Server:
       - cloudflare
       Transfer-Encoding:
@@ -178,28 +301,134 @@ interactions:
       - nosniff
       access-control-expose-headers:
       - X-Request-ID
+      alt-svc:
+      - h3=":443"; ma=86400
       openai-organization:
       - crewai-iuxna1
       openai-processing-ms:
-      - '261'
+      - '484'
       openai-version:
       - '2020-10-01'
       strict-transport-security:
       - max-age=31536000; includeSubDomains; preload
       x-ratelimit-limit-requests:
-      - '10000'
+      - '30000'
       x-ratelimit-limit-tokens:
-      - '30000000'
+      - '150000000'
       x-ratelimit-remaining-requests:
-      - '9999'
+      - '29999'
       x-ratelimit-remaining-tokens:
-      - '29999806'
+      - '149999793'
       x-ratelimit-reset-requests:
-      - 6ms
+      - 2ms
       x-ratelimit-reset-tokens:
       - 0s
       x-request-id:
-      - req_11a316792b5f54af94cce0c702aec290
+      - req_5bf4a565ad6c2567a1ed204ecac89134
+    http_version: HTTP/1.1
+    status_code: 200
+- request:
+    body: '{"messages": [{"role": "system", "content": "Determine if the following
+      feedback indicates that the user is satisfied or if further changes are needed.
+      Respond with ''True'' if further changes are needed, or ''False'' if the user
+      is satisfied. **Important** Do not include any additional commentary outside
+      of your ''True'' or ''False'' response.\n\nFeedback: \"looks good\""}], "model":
+      "gpt-4o-mini", "stop": ["\nObservation:"], "stream": false}'
+    headers:
+      accept:
+      - application/json
+      accept-encoding:
+      - gzip, deflate
+      connection:
+      - keep-alive
+      content-length:
+      - '439'
+      content-type:
+      - application/json
+      cookie:
+      - __cf_bm=QJZZjZ6eqnVamqUkw.Bx0mj7oBi3a_vGEH1VODcUxlg-1733344190-1.0.1.1-xyN0ekA9xIrSwEhRBmTiWJ3Pt72UYLU5owKfkz5yihVmMTfsr_Qz.ssGPJ5cuft066v1xVjb4zOSTdFmesMSKg;
+        _cfuvid=eCIkP8GVPvpkg19eOhCquWFHm.RTQBQy4yHLGGEAH5c-1733344190334-0.0.1.1-604800000
+      host:
+      - api.openai.com
+      user-agent:
+      - OpenAI/Python 1.52.1
+      x-stainless-arch:
+      - arm64
+      x-stainless-async:
+      - 'false'
+      x-stainless-lang:
+      - python
+      x-stainless-os:
+      - MacOS
+      x-stainless-package-version:
+      - 1.52.1
+      x-stainless-raw-response:
+      - 'true'
+      x-stainless-retry-count:
+      - '0'
+      x-stainless-runtime:
+      - CPython
+      x-stainless-runtime-version:
+      - 3.12.7
+    method: POST
+    uri: https://api.openai.com/v1/chat/completions
+  response:
+    content: "{\n  \"id\": \"chatcmpl-AaqIJqyG8vl9mxj2qDPZgaxyNLLIq\",\n  \"object\":
+      \"chat.completion\",\n  \"created\": 1733344191,\n  \"model\": \"gpt-4o-mini-2024-07-18\",\n
+      \ \"choices\": [\n    {\n      \"index\": 0,\n      \"message\": {\n        \"role\":
+      \"assistant\",\n        \"content\": \"False\",\n        \"refusal\": null\n
+      \     },\n      \"logprobs\": null,\n      \"finish_reason\": \"stop\"\n    }\n
+      \ ],\n  \"usage\": {\n    \"prompt_tokens\": 73,\n    \"completion_tokens\":
+      1,\n    \"total_tokens\": 74,\n    \"prompt_tokens_details\": {\n      \"cached_tokens\":
+      0,\n      \"audio_tokens\": 0\n    },\n    \"completion_tokens_details\": {\n
+      \     \"reasoning_tokens\": 0,\n      \"audio_tokens\": 0,\n      \"accepted_prediction_tokens\":
+      0,\n      \"rejected_prediction_tokens\": 0\n    }\n  },\n  \"system_fingerprint\":
+      \"fp_0705bf87c0\"\n}\n"
+    headers:
+      CF-Cache-Status:
+      - DYNAMIC
+      CF-RAY:
+      - 8ece8d0cfdeb4532-ATL
+      Connection:
+      - keep-alive
+      Content-Encoding:
+      - gzip
+      Content-Type:
+      - application/json
+      Date:
+      - Wed, 04 Dec 2024 20:29:51 GMT
+      Server:
+      - cloudflare
+      Transfer-Encoding:
+      - chunked
+      X-Content-Type-Options:
+      - nosniff
+      access-control-expose-headers:
+      - X-Request-ID
+      alt-svc:
+      - h3=":443"; ma=86400
+      openai-organization:
+      - crewai-iuxna1
+      openai-processing-ms:
+      - '341'
+      openai-version:
+      - '2020-10-01'
+      strict-transport-security:
+      - max-age=31536000; includeSubDomains; preload
+      x-ratelimit-limit-requests:
+      - '30000'
+      x-ratelimit-limit-tokens:
+      - '150000000'
+      x-ratelimit-remaining-requests:
+      - '29999'
+      x-ratelimit-remaining-tokens:
+      - '149999902'
+      x-ratelimit-reset-requests:
+      - 2ms
+      x-ratelimit-reset-tokens:
+      - 0s
+      x-request-id:
+      - req_5554bade8ceda00cf364b76a51b708ff
     http_version: HTTP/1.1
     status_code: 200
 version: 1