Adding support for Crew throttling using RPM (#124)

* Add translations
* fixing translations
* Adding support for Crew throttling with RPM

src/crewai/agent.py

@@ -12,6 +12,7 @@ from pydantic import (
     ConfigDict,
     Field,
     InstanceOf,
+    PrivateAttr,
     field_validator,
     model_validator,
 )
@@ -46,6 +47,8 @@ class Agent(BaseModel):
     """
 
     __hash__ = object.__hash__
+    _request_within_rpm_limit: Any = PrivateAttr(default=None)
+
     model_config = ConfigDict(arbitrary_types_allowed=True)
     id: UUID4 = Field(
         default_factory=uuid.uuid4,
@@ -139,6 +142,10 @@ class Agent(BaseModel):
         self.tools_handler = ToolsHandler(cache=self.cache_handler)
         self.__create_agent_executor()
 
+    def set_request_within_rpm_limit(self, ensure_function) -> None:
+        self._request_within_rpm_limit = ensure_function
+        self.__create_agent_executor()
+
     def __create_agent_executor(self) -> CrewAgentExecutor:
         """Create an agent executor for the agent.
 
@@ -157,11 +164,12 @@ class Agent(BaseModel):
             "verbose": self.verbose,
             "handle_parsing_errors": True,
             "max_iterations": self.max_iter,
+            "request_within_rpm_limit": self._request_within_rpm_limit,
         }
 
         if self.memory:
             summary_memory = ConversationSummaryMemory(
-                llm=self.llm, memory_key="chat_history", input_key="input"
+                llm=self.llm, input_key="input", memory_key="chat_history"
             )
             executor_args["memory"] = summary_memory
             agent_args["chat_history"] = lambda x: x["chat_history"]

src/crewai/agents/executor.py

@@ -19,6 +19,7 @@ from crewai.tools.cache_tools import CacheTools
 class CrewAgentExecutor(AgentExecutor):
     i18n: I18N = I18N()
     iterations: int = 0
+    request_within_rpm_limit: Any = None
     max_iterations: Optional[int] = 15
     force_answer_max_iterations: Optional[int] = None
 
@@ -54,29 +55,30 @@ class CrewAgentExecutor(AgentExecutor):
         start_time = time.time()
         # We now enter the agent loop (until it returns something).
         while self._should_continue(self.iterations, time_elapsed):
-            next_step_output = self._take_next_step(
+            if not self.request_within_rpm_limit or self.request_within_rpm_limit():
-                name_to_tool_map,
+                next_step_output = self._take_next_step(
-                color_mapping,
+                    name_to_tool_map,
-                inputs,
+                    color_mapping,
-                intermediate_steps,
+                    inputs,
-                run_manager=run_manager,
+                    intermediate_steps,
-            )
+                    run_manager=run_manager,
-            if isinstance(next_step_output, AgentFinish):
-                return self._return(
-                    next_step_output, intermediate_steps, run_manager=run_manager
                 )
-
+                if isinstance(next_step_output, AgentFinish):
-            intermediate_steps.extend(next_step_output)
-            if len(next_step_output) == 1:
-                next_step_action = next_step_output[0]
-                # See if tool should return directly
-                tool_return = self._get_tool_return(next_step_action)
-                if tool_return is not None:
                     return self._return(
-                        tool_return, intermediate_steps, run_manager=run_manager
+                        next_step_output, intermediate_steps, run_manager=run_manager
                     )
-            self.iterations += 1
+
-            time_elapsed = time.time() - start_time
+                intermediate_steps.extend(next_step_output)
+                if len(next_step_output) == 1:
+                    next_step_action = next_step_output[0]
+                    # See if tool should return directly
+                    tool_return = self._get_tool_return(next_step_action)
+                    if tool_return is not None:
+                        return self._return(
+                            tool_return, intermediate_steps, run_manager=run_manager
+                        )
+                self.iterations += 1
+                time_elapsed = time.time() - start_time
         output = self.agent.return_stopped_response(
             self.early_stopping_method, intermediate_steps, **inputs
         )

src/crewai/crew.py

@@ -1,6 +1,8 @@
 import json
+import threading
+import time
 import uuid
-from typing import Any, Dict, List, Optional, Union
+from typing import Any, ClassVar, Dict, List, Optional, Union
 
 from pydantic import (
     UUID4,
@@ -9,6 +11,7 @@ from pydantic import (
     Field,
     InstanceOf,
     Json,
+    PrivateAttr,
     field_validator,
     model_validator,
 )
@@ -33,10 +36,16 @@ class Crew(BaseModel):
         verbose: Indicates the verbosity level for logging during execution.
         config: Configuration settings for the crew.
         cache_handler: Handles caching for the crew's operations.
+        max_rpm: Maximum number of requests per minute for the crew execution to be respected.
+        rpm: Current number of requests per minute for the crew execution.
         id: A unique identifier for the crew instance.
     """
 
     __hash__ = object.__hash__
+    _timer: Optional[threading.Timer] = PrivateAttr(default=None)
+    lock: ClassVar[threading.Lock] = threading.Lock()
+    rpm: ClassVar[int] = 0
+    max_rpm: Optional[int] = Field(default=None)
     model_config = ConfigDict(arbitrary_types_allowed=True)
     tasks: List[Task] = Field(default_factory=list)
     agents: List[Agent] = Field(default_factory=list)
@@ -64,6 +73,12 @@ class Crew(BaseModel):
     def check_config_type(cls, v: Union[Json, Dict[str, Any]]):
         return json.loads(v) if isinstance(v, Json) else v
 
+    @model_validator(mode="after")
+    def set_reset_counter(self):
+        if self.max_rpm:
+            self._reset_request_count()
+        return self
+
     @model_validator(mode="after")
     def check_config(self):
         """Validates that the crew is properly configured with agents and tasks."""
@@ -80,6 +95,7 @@ class Crew(BaseModel):
         if self.agents:
             for agent in self.agents:
                 agent.set_cache_handler(self.cache_handler)
+                agent.set_request_within_rpm_limit(self.ensure_request_within_rpm_limit)
         return self
 
     def _setup_from_config(self):
@@ -100,6 +116,24 @@ class Crew(BaseModel):
         del task_config["agent"]
         return Task(**task_config, agent=task_agent)
 
+    def ensure_request_within_rpm_limit(self):
+        if not self.max_rpm:
+            return True
+
+        with Crew.lock:
+            if Crew.rpm < self.max_rpm:
+                Crew.rpm += 1
+                return True
+            self._log("info", "Max RPM reached, waiting for next minute to start.")
+
+        return self._wait_for_next_minute()
+
+    def _wait_for_next_minute(self):
+        time.sleep(60)
+        with Crew.lock:
+            Crew.rpm = 0
+            return True
+
     def kickoff(self) -> str:
         """Starts the crew to work on its assigned tasks."""
         for agent in self.agents:
@@ -115,9 +149,8 @@ class Crew(BaseModel):
         for task in self.tasks:
             self._prepare_and_execute_task(task)
             task_output = task.execute(task_output)
-            self._log(
+            self._log("debug", f"\n[{task.agent.role}] Task output: {task_output}\n\n")
-                "debug", f"\n\n[{task.agent.role}] Task output: {task_output}\n\n"
+        self._stop_timer()
-            )
         return task_output
 
     def _prepare_and_execute_task(self, task):
@@ -135,4 +168,14 @@ class Crew(BaseModel):
             2 if isinstance(self.verbose, bool) and self.verbose else self.verbose
         )
         if verbose_level and level_map[level] <= verbose_level:
-            print(message)
+            print(f"\n{message}")
+
+    def _stop_timer(self):
+        if self._timer:
+            self._timer.cancel()
+
+    def _reset_request_count(self):
+        self._stop_timer()
+        self._timer = threading.Timer(60.0, self._reset_request_count)
+        self._timer.start()
+        Crew.rpm = 0

tests/cassettes/test_api_calls_throttling.yaml

@@ -0,0 +1,666 @@
+interactions:
+- request:
+    body: '{"messages": [{"role": "user", "content": "You are test role.\ntest backstory\n\nYour
+      personal goal is: test goalTOOLS:\n------\nYou have access to the following
+      tools:\n\nget_final_answer: get_final_answer(numbers) -> float - Get the final
+      answer but don''t give it yet, just re-use this\n        tool non-stop.\n\nTo
+      use a tool, please use the exact following format:\n\n```\nThought: Do I need
+      to use a tool? Yes\nAction: the action to take, should be one of [get_final_answer],
+      just the name.\nAction Input: the input to the action\nObservation: the result
+      of the action\n```\n\nWhen you have a response for your task, or if you do not
+      need to use a tool, you MUST use the format:\n\n```\nThought: Do I need to use
+      a tool? No\nFinal Answer: [your response here]This is the summary of your work
+      so far:\nBegin! This is VERY important to you, your job depends on it!\n\nCurrent
+      Task: Don''t give a Final Answer, instead keep using the `get_final_answer`
+      tool.\n"}], "model": "gpt-4", "n": 1, "stop": ["\nObservation"], "stream": false,
+      "temperature": 0.7}'
+    headers:
+      accept:
+      - application/json
+      accept-encoding:
+      - gzip, deflate
+      connection:
+      - keep-alive
+      content-length:
+      - '1055'
+      content-type:
+      - application/json
+      host:
+      - api.openai.com
+      user-agent:
+      - OpenAI/Python 1.7.1
+      x-stainless-arch:
+      - arm64
+      x-stainless-async:
+      - 'false'
+      x-stainless-lang:
+      - python
+      x-stainless-os:
+      - MacOS
+      x-stainless-package-version:
+      - 1.7.1
+      x-stainless-runtime:
+      - CPython
+      x-stainless-runtime-version:
+      - 3.11.7
+    method: POST
+    uri: https://api.openai.com/v1/chat/completions
+  response:
+    content: "{\n  \"id\": \"chatcmpl-8gQL7bRtT6PceuMWF91CSHDk1cziN\",\n  \"object\":
+      \"chat.completion\",\n  \"created\": 1705121717,\n  \"model\": \"gpt-4-0613\",\n
+      \ \"choices\": [\n    {\n      \"index\": 0,\n      \"message\": {\n        \"role\":
+      \"assistant\",\n        \"content\": \"Thought: Do I need to use a tool? Yes\\nAction:
+      get_final_answer\\nAction Input: [1, 2, 3]\"\n      },\n      \"logprobs\":
+      null,\n      \"finish_reason\": \"stop\"\n    }\n  ],\n  \"usage\": {\n    \"prompt_tokens\":
+      226,\n    \"completion_tokens\": 30,\n    \"total_tokens\": 256\n  },\n  \"system_fingerprint\":
+      null\n}\n"
+    headers:
+      CF-Cache-Status:
+      - DYNAMIC
+      CF-RAY:
+      - 844b0bccfc300301-GRU
+      Cache-Control:
+      - no-cache, must-revalidate
+      Connection:
+      - keep-alive
+      Content-Encoding:
+      - gzip
+      Content-Type:
+      - application/json
+      Date:
+      - Sat, 13 Jan 2024 04:55:19 GMT
+      Server:
+      - cloudflare
+      Set-Cookie:
+      - __cf_bm=LaiWRSzgH0KTl1zCkIyak7.Xo2gxpZU1B.b9dgz2TzI-1705121719-1-Ab9fhdn6OHTc5/5ibPLP1pc+XN4Z3ORV0Nku/+S5BqdXcS84TsVDiACZUN0Xxvew6yfZ6ApEBQOBsEhJNyVkqCM=;
+        path=/; expires=Sat, 13-Jan-24 05:25:19 GMT; domain=.api.openai.com; HttpOnly;
+        Secure; SameSite=None
+      - _cfuvid=6J6x2HNoj7l3E6T3yHnLcNZb273IuGETTaD1BBeY_OQ-1705121719455-0-604800000;
+        path=/; domain=.api.openai.com; HttpOnly; Secure; SameSite=None
+      Transfer-Encoding:
+      - chunked
+      access-control-allow-origin:
+      - '*'
+      alt-svc:
+      - h3=":443"; ma=86400
+      openai-model:
+      - gpt-4-0613
+      openai-organization:
+      - user-z7g4wmlazxqvc5wjyaaaocfz
+      openai-processing-ms:
+      - '1751'
+      openai-version:
+      - '2020-10-01'
+      strict-transport-security:
+      - max-age=15724800; includeSubDomains
+      x-ratelimit-limit-requests:
+      - '10000'
+      x-ratelimit-limit-tokens:
+      - '300000'
+      x-ratelimit-remaining-requests:
+      - '9999'
+      x-ratelimit-remaining-tokens:
+      - '299760'
+      x-ratelimit-reset-requests:
+      - 6ms
+      x-ratelimit-reset-tokens:
+      - 48ms
+      x-request-id:
+      - ae0de27cb07da02e7bce2d30958b7e14
+    http_version: HTTP/1.1
+    status_code: 200
+- request:
+    body: '{"messages": [{"role": "user", "content": "You are test role.\ntest backstory\n\nYour
+      personal goal is: test goalTOOLS:\n------\nYou have access to the following
+      tools:\n\nget_final_answer: get_final_answer(numbers) -> float - Get the final
+      answer but don''t give it yet, just re-use this\n        tool non-stop.\n\nTo
+      use a tool, please use the exact following format:\n\n```\nThought: Do I need
+      to use a tool? Yes\nAction: the action to take, should be one of [get_final_answer],
+      just the name.\nAction Input: the input to the action\nObservation: the result
+      of the action\n```\n\nWhen you have a response for your task, or if you do not
+      need to use a tool, you MUST use the format:\n\n```\nThought: Do I need to use
+      a tool? No\nFinal Answer: [your response here]This is the summary of your work
+      so far:\nBegin! This is VERY important to you, your job depends on it!\n\nCurrent
+      Task: Don''t give a Final Answer, instead keep using the `get_final_answer`
+      tool.\nThought: Do I need to use a tool? Yes\nAction: get_final_answer\nAction
+      Input: [1, 2, 3]\nObservation: 42\nThought: "}], "model": "gpt-4", "n": 1, "stop":
+      ["\nObservation"], "stream": false, "temperature": 0.7}'
+    headers:
+      accept:
+      - application/json
+      accept-encoding:
+      - gzip, deflate
+      connection:
+      - keep-alive
+      content-length:
+      - '1171'
+      content-type:
+      - application/json
+      cookie:
+      - __cf_bm=LaiWRSzgH0KTl1zCkIyak7.Xo2gxpZU1B.b9dgz2TzI-1705121719-1-Ab9fhdn6OHTc5/5ibPLP1pc+XN4Z3ORV0Nku/+S5BqdXcS84TsVDiACZUN0Xxvew6yfZ6ApEBQOBsEhJNyVkqCM=;
+        _cfuvid=6J6x2HNoj7l3E6T3yHnLcNZb273IuGETTaD1BBeY_OQ-1705121719455-0-604800000
+      host:
+      - api.openai.com
+      user-agent:
+      - OpenAI/Python 1.7.1
+      x-stainless-arch:
+      - arm64
+      x-stainless-async:
+      - 'false'
+      x-stainless-lang:
+      - python
+      x-stainless-os:
+      - MacOS
+      x-stainless-package-version:
+      - 1.7.1
+      x-stainless-runtime:
+      - CPython
+      x-stainless-runtime-version:
+      - 3.11.7
+    method: POST
+    uri: https://api.openai.com/v1/chat/completions
+  response:
+    content: "{\n  \"id\": \"chatcmpl-8gQL9CyNZOIEDKgfmDn68aH26tLrn\",\n  \"object\":
+      \"chat.completion\",\n  \"created\": 1705121719,\n  \"model\": \"gpt-4-0613\",\n
+      \ \"choices\": [\n    {\n      \"index\": 0,\n      \"message\": {\n        \"role\":
+      \"assistant\",\n        \"content\": \"Do I need to use a tool? Yes\\nAction:
+      get_final_answer\\nAction Input: [4, 5, 6]\"\n      },\n      \"logprobs\":
+      null,\n      \"finish_reason\": \"stop\"\n    }\n  ],\n  \"usage\": {\n    \"prompt_tokens\":
+      265,\n    \"completion_tokens\": 28,\n    \"total_tokens\": 293\n  },\n  \"system_fingerprint\":
+      null\n}\n"
+    headers:
+      CF-Cache-Status:
+      - DYNAMIC
+      CF-RAY:
+      - 844b0bdaee800301-GRU
+      Cache-Control:
+      - no-cache, must-revalidate
+      Connection:
+      - keep-alive
+      Content-Encoding:
+      - gzip
+      Content-Type:
+      - application/json
+      Date:
+      - Sat, 13 Jan 2024 04:55:22 GMT
+      Server:
+      - cloudflare
+      Transfer-Encoding:
+      - chunked
+      access-control-allow-origin:
+      - '*'
+      alt-svc:
+      - h3=":443"; ma=86400
+      openai-model:
+      - gpt-4-0613
+      openai-organization:
+      - user-z7g4wmlazxqvc5wjyaaaocfz
+      openai-processing-ms:
+      - '2615'
+      openai-version:
+      - '2020-10-01'
+      strict-transport-security:
+      - max-age=15724800; includeSubDomains
+      x-ratelimit-limit-requests:
+      - '10000'
+      x-ratelimit-limit-tokens:
+      - '300000'
+      x-ratelimit-remaining-requests:
+      - '9999'
+      x-ratelimit-remaining-tokens:
+      - '299731'
+      x-ratelimit-reset-requests:
+      - 6ms
+      x-ratelimit-reset-tokens:
+      - 53ms
+      x-request-id:
+      - 9d85a22aa63ee4cea3ad01a3b68316b4
+    http_version: HTTP/1.1
+    status_code: 200
+- request:
+    body: '{"messages": [{"role": "user", "content": "You are test role.\ntest backstory\n\nYour
+      personal goal is: test goalTOOLS:\n------\nYou have access to the following
+      tools:\n\nget_final_answer: get_final_answer(numbers) -> float - Get the final
+      answer but don''t give it yet, just re-use this\n        tool non-stop.\n\nTo
+      use a tool, please use the exact following format:\n\n```\nThought: Do I need
+      to use a tool? Yes\nAction: the action to take, should be one of [get_final_answer],
+      just the name.\nAction Input: the input to the action\nObservation: the result
+      of the action\n```\n\nWhen you have a response for your task, or if you do not
+      need to use a tool, you MUST use the format:\n\n```\nThought: Do I need to use
+      a tool? No\nFinal Answer: [your response here]This is the summary of your work
+      so far:\nBegin! This is VERY important to you, your job depends on it!\n\nCurrent
+      Task: Don''t give a Final Answer, instead keep using the `get_final_answer`
+      tool.\nThought: Do I need to use a tool? Yes\nAction: get_final_answer\nAction
+      Input: [1, 2, 3]\nObservation: 42\nThought: Do I need to use a tool? Yes\nAction:
+      get_final_answer\nAction Input: [4, 5, 6]\nObservation: 42\nThought: "}], "model":
+      "gpt-4", "n": 1, "stop": ["\nObservation"], "stream": false, "temperature":
+      0.7}'
+    headers:
+      accept:
+      - application/json
+      accept-encoding:
+      - gzip, deflate
+      connection:
+      - keep-alive
+      content-length:
+      - '1278'
+      content-type:
+      - application/json
+      cookie:
+      - __cf_bm=LaiWRSzgH0KTl1zCkIyak7.Xo2gxpZU1B.b9dgz2TzI-1705121719-1-Ab9fhdn6OHTc5/5ibPLP1pc+XN4Z3ORV0Nku/+S5BqdXcS84TsVDiACZUN0Xxvew6yfZ6ApEBQOBsEhJNyVkqCM=;
+        _cfuvid=6J6x2HNoj7l3E6T3yHnLcNZb273IuGETTaD1BBeY_OQ-1705121719455-0-604800000
+      host:
+      - api.openai.com
+      user-agent:
+      - OpenAI/Python 1.7.1
+      x-stainless-arch:
+      - arm64
+      x-stainless-async:
+      - 'false'
+      x-stainless-lang:
+      - python
+      x-stainless-os:
+      - MacOS
+      x-stainless-package-version:
+      - 1.7.1
+      x-stainless-runtime:
+      - CPython
+      x-stainless-runtime-version:
+      - 3.11.7
+    method: POST
+    uri: https://api.openai.com/v1/chat/completions
+  response:
+    content: "{\n  \"id\": \"chatcmpl-8gQMARI9zwx2Z1Uel5us1xzxPAuGJ\",\n  \"object\":
+      \"chat.completion\",\n  \"created\": 1705121782,\n  \"model\": \"gpt-4-0613\",\n
+      \ \"choices\": [\n    {\n      \"index\": 0,\n      \"message\": {\n        \"role\":
+      \"assistant\",\n        \"content\": \"Do I need to use a tool? Yes\\nAction:
+      get_final_answer\\nAction Input: [7, 8, 9]\"\n      },\n      \"logprobs\":
+      null,\n      \"finish_reason\": \"stop\"\n    }\n  ],\n  \"usage\": {\n    \"prompt_tokens\":
+      301,\n    \"completion_tokens\": 28,\n    \"total_tokens\": 329\n  },\n  \"system_fingerprint\":
+      null\n}\n"
+    headers:
+      CF-Cache-Status:
+      - DYNAMIC
+      CF-RAY:
+      - 844b0d63fa7a02ee-GRU
+      Cache-Control:
+      - no-cache, must-revalidate
+      Connection:
+      - keep-alive
+      Content-Encoding:
+      - gzip
+      Content-Type:
+      - application/json
+      Date:
+      - Sat, 13 Jan 2024 04:56:24 GMT
+      Server:
+      - cloudflare
+      Transfer-Encoding:
+      - chunked
+      access-control-allow-origin:
+      - '*'
+      alt-svc:
+      - h3=":443"; ma=86400
+      openai-model:
+      - gpt-4-0613
+      openai-organization:
+      - user-z7g4wmlazxqvc5wjyaaaocfz
+      openai-processing-ms:
+      - '1878'
+      openai-version:
+      - '2020-10-01'
+      strict-transport-security:
+      - max-age=15724800; includeSubDomains
+      x-ratelimit-limit-requests:
+      - '10000'
+      x-ratelimit-limit-tokens:
+      - '300000'
+      x-ratelimit-remaining-requests:
+      - '9999'
+      x-ratelimit-remaining-tokens:
+      - '299705'
+      x-ratelimit-reset-requests:
+      - 6ms
+      x-ratelimit-reset-tokens:
+      - 58ms
+      x-request-id:
+      - d98dffd5ba9915b41051dcad49109eba
+    http_version: HTTP/1.1
+    status_code: 200
+- request:
+    body: '{"messages": [{"role": "user", "content": "You are test role.\ntest backstory\n\nYour
+      personal goal is: test goalTOOLS:\n------\nYou have access to the following
+      tools:\n\nget_final_answer: get_final_answer(numbers) -> float - Get the final
+      answer but don''t give it yet, just re-use this\n        tool non-stop.\n\nTo
+      use a tool, please use the exact following format:\n\n```\nThought: Do I need
+      to use a tool? Yes\nAction: the action to take, should be one of [get_final_answer],
+      just the name.\nAction Input: the input to the action\nObservation: the result
+      of the action\n```\n\nWhen you have a response for your task, or if you do not
+      need to use a tool, you MUST use the format:\n\n```\nThought: Do I need to use
+      a tool? No\nFinal Answer: [your response here]This is the summary of your work
+      so far:\nBegin! This is VERY important to you, your job depends on it!\n\nCurrent
+      Task: Don''t give a Final Answer, instead keep using the `get_final_answer`
+      tool.\nThought: Do I need to use a tool? Yes\nAction: get_final_answer\nAction
+      Input: [1, 2, 3]\nObservation: 42\nThought: Do I need to use a tool? Yes\nAction:
+      get_final_answer\nAction Input: [4, 5, 6]\nObservation: 42\nThought: Do I need
+      to use a tool? Yes\nAction: get_final_answer\nAction Input: [7, 8, 9]\nObservation:
+      42\nThought: "}], "model": "gpt-4", "n": 1, "stop": ["\nObservation"], "stream":
+      false, "temperature": 0.7}'
+    headers:
+      accept:
+      - application/json
+      accept-encoding:
+      - gzip, deflate
+      connection:
+      - keep-alive
+      content-length:
+      - '1385'
+      content-type:
+      - application/json
+      cookie:
+      - __cf_bm=LaiWRSzgH0KTl1zCkIyak7.Xo2gxpZU1B.b9dgz2TzI-1705121719-1-Ab9fhdn6OHTc5/5ibPLP1pc+XN4Z3ORV0Nku/+S5BqdXcS84TsVDiACZUN0Xxvew6yfZ6ApEBQOBsEhJNyVkqCM=;
+        _cfuvid=6J6x2HNoj7l3E6T3yHnLcNZb273IuGETTaD1BBeY_OQ-1705121719455-0-604800000
+      host:
+      - api.openai.com
+      user-agent:
+      - OpenAI/Python 1.7.1
+      x-stainless-arch:
+      - arm64
+      x-stainless-async:
+      - 'false'
+      x-stainless-lang:
+      - python
+      x-stainless-os:
+      - MacOS
+      x-stainless-package-version:
+      - 1.7.1
+      x-stainless-runtime:
+      - CPython
+      x-stainless-runtime-version:
+      - 3.11.7
+    method: POST
+    uri: https://api.openai.com/v1/chat/completions
+  response:
+    content: "{\n  \"id\": \"chatcmpl-8gQMC51udijWPWSLCk1Cx36OVtDUX\",\n  \"object\":
+      \"chat.completion\",\n  \"created\": 1705121784,\n  \"model\": \"gpt-4-0613\",\n
+      \ \"choices\": [\n    {\n      \"index\": 0,\n      \"message\": {\n        \"role\":
+      \"assistant\",\n        \"content\": \"Do I need to use a tool? Yes\\nAction:
+      get_final_answer\\nAction Input: [10, 11, 12]\"\n      },\n      \"logprobs\":
+      null,\n      \"finish_reason\": \"stop\"\n    }\n  ],\n  \"usage\": {\n    \"prompt_tokens\":
+      337,\n    \"completion_tokens\": 28,\n    \"total_tokens\": 365\n  },\n  \"system_fingerprint\":
+      null\n}\n"
+    headers:
+      CF-Cache-Status:
+      - DYNAMIC
+      CF-RAY:
+      - 844b0d72cbbd02ee-GRU
+      Cache-Control:
+      - no-cache, must-revalidate
+      Connection:
+      - keep-alive
+      Content-Encoding:
+      - gzip
+      Content-Type:
+      - application/json
+      Date:
+      - Sat, 13 Jan 2024 04:56:27 GMT
+      Server:
+      - cloudflare
+      Transfer-Encoding:
+      - chunked
+      access-control-allow-origin:
+      - '*'
+      alt-svc:
+      - h3=":443"; ma=86400
+      openai-model:
+      - gpt-4-0613
+      openai-organization:
+      - user-z7g4wmlazxqvc5wjyaaaocfz
+      openai-processing-ms:
+      - '2939'
+      openai-version:
+      - '2020-10-01'
+      strict-transport-security:
+      - max-age=15724800; includeSubDomains
+      x-ratelimit-limit-requests:
+      - '10000'
+      x-ratelimit-limit-tokens:
+      - '300000'
+      x-ratelimit-remaining-requests:
+      - '9999'
+      x-ratelimit-remaining-tokens:
+      - '299680'
+      x-ratelimit-reset-requests:
+      - 6ms
+      x-ratelimit-reset-tokens:
+      - 64ms
+      x-request-id:
+      - 528f38b4e5dd09a1fd25893edb27e4b4
+    http_version: HTTP/1.1
+    status_code: 200
+- request:
+    body: '{"messages": [{"role": "user", "content": "You are test role.\ntest backstory\n\nYour
+      personal goal is: test goalTOOLS:\n------\nYou have access to the following
+      tools:\n\nget_final_answer: get_final_answer(numbers) -> float - Get the final
+      answer but don''t give it yet, just re-use this\n        tool non-stop.\n\nTo
+      use a tool, please use the exact following format:\n\n```\nThought: Do I need
+      to use a tool? Yes\nAction: the action to take, should be one of [get_final_answer],
+      just the name.\nAction Input: the input to the action\nObservation: the result
+      of the action\n```\n\nWhen you have a response for your task, or if you do not
+      need to use a tool, you MUST use the format:\n\n```\nThought: Do I need to use
+      a tool? No\nFinal Answer: [your response here]This is the summary of your work
+      so far:\nBegin! This is VERY important to you, your job depends on it!\n\nCurrent
+      Task: Don''t give a Final Answer, instead keep using the `get_final_answer`
+      tool.\nThought: Do I need to use a tool? Yes\nAction: get_final_answer\nAction
+      Input: [1, 2, 3]\nObservation: 42\nThought: Do I need to use a tool? Yes\nAction:
+      get_final_answer\nAction Input: [4, 5, 6]\nObservation: 42\nThought: Do I need
+      to use a tool? Yes\nAction: get_final_answer\nAction Input: [7, 8, 9]\nObservation:
+      42\nThought: Do I need to use a tool? Yes\nAction: get_final_answer\nAction
+      Input: [10, 11, 12]\nObservation: I''ve used too many tools for this task. I''m
+      going to give you my absolute BEST Final answer now and not use any more tools.\nThought:
+      "}], "model": "gpt-4", "n": 1, "stop": ["\nObservation"], "stream": false, "temperature":
+      0.7}'
+    headers:
+      accept:
+      - application/json
+      accept-encoding:
+      - gzip, deflate
+      connection:
+      - keep-alive
+      content-length:
+      - '1616'
+      content-type:
+      - application/json
+      cookie:
+      - __cf_bm=LaiWRSzgH0KTl1zCkIyak7.Xo2gxpZU1B.b9dgz2TzI-1705121719-1-Ab9fhdn6OHTc5/5ibPLP1pc+XN4Z3ORV0Nku/+S5BqdXcS84TsVDiACZUN0Xxvew6yfZ6ApEBQOBsEhJNyVkqCM=;
+        _cfuvid=6J6x2HNoj7l3E6T3yHnLcNZb273IuGETTaD1BBeY_OQ-1705121719455-0-604800000
+      host:
+      - api.openai.com
+      user-agent:
+      - OpenAI/Python 1.7.1
+      x-stainless-arch:
+      - arm64
+      x-stainless-async:
+      - 'false'
+      x-stainless-lang:
+      - python
+      x-stainless-os:
+      - MacOS
+      x-stainless-package-version:
+      - 1.7.1
+      x-stainless-runtime:
+      - CPython
+      x-stainless-runtime-version:
+      - 3.11.7
+    method: POST
+    uri: https://api.openai.com/v1/chat/completions
+  response:
+    content: "{\n  \"id\": \"chatcmpl-8gQMGpMHplpQ8ApiUZmhbX4z9gVuK\",\n  \"object\":
+      \"chat.completion\",\n  \"created\": 1705121788,\n  \"model\": \"gpt-4-0613\",\n
+      \ \"choices\": [\n    {\n      \"index\": 0,\n      \"message\": {\n        \"role\":
+      \"assistant\",\n        \"content\": \"Do I need to use a tool? No\\nFinal Answer:
+      I've used the tool 'get_final_answer' multiple times and have observed the result.
+      However, as per my task, I am to keep using this tool and not provide a final
+      answer.\"\n      },\n      \"logprobs\": null,\n      \"finish_reason\": \"stop\"\n
+      \   }\n  ],\n  \"usage\": {\n    \"prompt_tokens\": 399,\n    \"completion_tokens\":
+      52,\n    \"total_tokens\": 451\n  },\n  \"system_fingerprint\": null\n}\n"
+    headers:
+      CF-Cache-Status:
+      - DYNAMIC
+      CF-RAY:
+      - 844b0d86a87f02ee-GRU
+      Cache-Control:
+      - no-cache, must-revalidate
+      Connection:
+      - keep-alive
+      Content-Encoding:
+      - gzip
+      Content-Type:
+      - application/json
+      Date:
+      - Sat, 13 Jan 2024 04:56:31 GMT
+      Server:
+      - cloudflare
+      Transfer-Encoding:
+      - chunked
+      access-control-allow-origin:
+      - '*'
+      alt-svc:
+      - h3=":443"; ma=86400
+      openai-model:
+      - gpt-4-0613
+      openai-organization:
+      - user-z7g4wmlazxqvc5wjyaaaocfz
+      openai-processing-ms:
+      - '3227'
+      openai-version:
+      - '2020-10-01'
+      strict-transport-security:
+      - max-age=15724800; includeSubDomains
+      x-ratelimit-limit-requests:
+      - '10000'
+      x-ratelimit-limit-tokens:
+      - '300000'
+      x-ratelimit-remaining-requests:
+      - '9999'
+      x-ratelimit-remaining-tokens:
+      - '299624'
+      x-ratelimit-reset-requests:
+      - 6ms
+      x-ratelimit-reset-tokens:
+      - 75ms
+      x-request-id:
+      - 7a91d2e523c43f5fc9a1191437491d68
+    http_version: HTTP/1.1
+    status_code: 200
+- request:
+    body: '{"messages": [{"role": "user", "content": "Progressively summarize the
+      lines of conversation provided, adding onto the previous summary returning a
+      new summary.\n\nEXAMPLE\nCurrent summary:\nThe human asks what the AI thinks
+      of artificial intelligence. The AI thinks artificial intelligence is a force
+      for good.\n\nNew lines of conversation:\nHuman: Why do you think artificial
+      intelligence is a force for good?\nAI: Because artificial intelligence will
+      help humans reach their full potential.\n\nNew summary:\nThe human asks what
+      the AI thinks of artificial intelligence. The AI thinks artificial intelligence
+      is a force for good because it will help humans reach their full potential.\nEND
+      OF EXAMPLE\n\nCurrent summary:\n\n\nNew lines of conversation:\nHuman: Don''t
+      give a Final Answer, instead keep using the `get_final_answer` tool.\nAI: I''ve
+      used the tool ''get_final_answer'' multiple times and have observed the result.
+      However, as per my task, I am to keep using this tool and not provide a final
+      answer.\n\nNew summary:"}], "model": "gpt-4", "n": 1, "stream": false, "temperature":
+      0.7}'
+    headers:
+      accept:
+      - application/json
+      accept-encoding:
+      - gzip, deflate
+      connection:
+      - keep-alive
+      content-length:
+      - '1094'
+      content-type:
+      - application/json
+      cookie:
+      - __cf_bm=LaiWRSzgH0KTl1zCkIyak7.Xo2gxpZU1B.b9dgz2TzI-1705121719-1-Ab9fhdn6OHTc5/5ibPLP1pc+XN4Z3ORV0Nku/+S5BqdXcS84TsVDiACZUN0Xxvew6yfZ6ApEBQOBsEhJNyVkqCM=;
+        _cfuvid=6J6x2HNoj7l3E6T3yHnLcNZb273IuGETTaD1BBeY_OQ-1705121719455-0-604800000
+      host:
+      - api.openai.com
+      user-agent:
+      - OpenAI/Python 1.7.1
+      x-stainless-arch:
+      - arm64
+      x-stainless-async:
+      - 'false'
+      x-stainless-lang:
+      - python
+      x-stainless-os:
+      - MacOS
+      x-stainless-package-version:
+      - 1.7.1
+      x-stainless-runtime:
+      - CPython
+      x-stainless-runtime-version:
+      - 3.11.7
+    method: POST
+    uri: https://api.openai.com/v1/chat/completions
+  response:
+    content: "{\n  \"id\": \"chatcmpl-8gQMJYMA8HO4tGJr8wUqEIJYn1xUa\",\n  \"object\":
+      \"chat.completion\",\n  \"created\": 1705121791,\n  \"model\": \"gpt-4-0613\",\n
+      \ \"choices\": [\n    {\n      \"index\": 0,\n      \"message\": {\n        \"role\":
+      \"assistant\",\n        \"content\": \"The human instructs the AI not to give
+      a final answer, but to continue using the 'get_final_answer' tool. The AI acknowledges
+      that it has used the tool multiple times and understands the directive to continue
+      using it without providing a final answer.\"\n      },\n      \"logprobs\":
+      null,\n      \"finish_reason\": \"stop\"\n    }\n  ],\n  \"usage\": {\n    \"prompt_tokens\":
+      195,\n    \"completion_tokens\": 50,\n    \"total_tokens\": 245\n  },\n  \"system_fingerprint\":
+      null\n}\n"
+    headers:
+      CF-Cache-Status:
+      - DYNAMIC
+      CF-RAY:
+      - 844b0d9dafa902ee-GRU
+      Cache-Control:
+      - no-cache, must-revalidate
+      Connection:
+      - keep-alive
+      Content-Encoding:
+      - gzip
+      Content-Type:
+      - application/json
+      Date:
+      - Sat, 13 Jan 2024 04:56:34 GMT
+      Server:
+      - cloudflare
+      Transfer-Encoding:
+      - chunked
+      access-control-allow-origin:
+      - '*'
+      alt-svc:
+      - h3=":443"; ma=86400
+      openai-model:
+      - gpt-4-0613
+      openai-organization:
+      - user-z7g4wmlazxqvc5wjyaaaocfz
+      openai-processing-ms:
+      - '2830'
+      openai-version:
+      - '2020-10-01'
+      strict-transport-security:
+      - max-age=15724800; includeSubDomains
+      x-ratelimit-limit-requests:
+      - '10000'
+      x-ratelimit-limit-tokens:
+      - '300000'
+      x-ratelimit-remaining-requests:
+      - '9999'
+      x-ratelimit-remaining-tokens:
+      - '299742'
+      x-ratelimit-reset-requests:
+      - 6ms
+      x-ratelimit-reset-tokens:
+      - 51ms
+      x-request-id:
+      - 0192af0bde71c3bb536287175e1a4458
+    http_version: HTTP/1.1
+    status_code: 200
+version: 1

tests/crew_test.py

@@ -266,3 +266,42 @@ def test_cache_hitting_between_agents():
         read.return_value = "12"
         crew.kickoff()
         read.assert_called_with("multiplier", "2,6")
+
+
+@pytest.mark.vcr(filter_headers=["authorization"])
+def test_api_calls_throttling(capsys):
+    from unittest.mock import patch
+
+    from langchain.tools import tool
+
+    @tool
+    def get_final_answer(numbers) -> float:
+        """Get the final answer but don't give it yet, just re-use this
+        tool non-stop."""
+        return 42
+
+    agent = Agent(
+        role="test role",
+        goal="test goal",
+        backstory="test backstory",
+        max_iter=5,
+        allow_delegation=False,
+        verbose=True,
+    )
+
+    task = Task(
+        description="Don't give a Final Answer, instead keep using the `get_final_answer` tool.",
+        tools=[get_final_answer],
+        agent=agent,
+    )
+
+    tasks = task
+
+    crew = Crew(agents=[agent], tasks=[tasks], max_rpm=2, verbose=2)
+
+    with patch.object(Crew, "_wait_for_next_minute") as moveon:
+        moveon.return_value = True
+        crew.kickoff()
+        captured = capsys.readouterr()
+        assert "Max RPM reached, waiting for next minute to start." in captured.out
+        moveon.assert_called()