From 4f455a351149d66e13bdfcb2623aff0116e81a41 Mon Sep 17 00:00:00 2001 From: alex-clawd Date: Sun, 26 Jul 2026 02:12:29 -0700 Subject: [PATCH] fix(openai): recover from tools + reasoning_effort 400 at runtime Any tool-using agent on gpt-5.4 or newer fails with a hard 400 on the default endpoint: Function tools with reasoning_effort are not supported for gpt-5.5 in /v1/chat/completions. To use function tools, use /v1/responses or set reasoning_effort to 'none'. `_prepare_completion_params` gated `reasoning_effort` behind o1, but the value still reaches the payload for any model through `params.update(self.additional_params)`, which bypasses the typed field. Tools are then attached unconditionally. Probed against the live endpoint rather than inferred from the report: model tools + 'high' tools + 'none' gpt-5.6/-sol/... rejected accepted gpt-5.5 rejected accepted gpt-5.4/-mini/-nano rejected accepted gpt-5.2 and earlier accepted accepted o1 / o3 / o4-mini accepted rejected The boundary is gpt-5.4, not 5.6. Note the last row: "none" is not a safe blanket fallback either. The 400 is the source of truth: catch it, drop reasoning_effort, retry once, and remember the model so the round trip is paid once per process. The known families are a fast path only, so OpenAI extending the restriction doesn't need a release here. Detection matches the structured `param` field plus the message, so the unsupported-value 400 that o1/o3 return for "none" doesn't trigger a useless retry. Logged at warning, not debug -- silently downgrading a requested reasoning effort should be visible. Composed with the responses-only fallback from #6656 in a single handler in _call_completions / _acall_completions, covering sync, async, streaming and non-streaming. Verified both recoveries still work live after rebasing: gpt-5.5 + tools + reasoning_effort -> tool call returned (was 400) gpt-5-pro -> 'PONG', learned {'gpt-5-pro'} Tests: 28 cases covering the fast path, error classification, sync + async retry, no-retry on unrelated 400s, and a guard against the drop tests passing vacuously (the typed reasoning_effort field is gated behind is_o1_model, so additional_params is the path that actually leaks). tests/llms/openai -> 149 passed, 1 skipped. --- .../llms/providers/openai/completion.py | 258 ++++++++++++---- .../test_gpt56_tools_reasoning_effort.py | 285 ++++++++++++++++++ 2 files changed, 486 insertions(+), 57 deletions(-) create mode 100644 lib/crewai/tests/llms/openai/test_gpt56_tools_reasoning_effort.py diff --git a/lib/crewai/src/crewai/llms/providers/openai/completion.py b/lib/crewai/src/crewai/llms/providers/openai/completion.py index 2cd07e863..33afe7c9f 100644 --- a/lib/crewai/src/crewai/llms/providers/openai/completion.py +++ b/lib/crewai/src/crewai/llms/providers/openai/completion.py @@ -8,7 +8,14 @@ import os from typing import TYPE_CHECKING, Any, ClassVar, Literal, TypedDict import httpx -from openai import APIConnectionError, AsyncOpenAI, NotFoundError, OpenAI, Stream +from openai import ( + APIConnectionError, + AsyncOpenAI, + BadRequestError, + NotFoundError, + OpenAI, + Stream, +) from openai.lib.streaming.chat import ChatCompletionStream from openai.types.chat import ( ChatCompletion, @@ -57,6 +64,27 @@ if TYPE_CHECKING: # per process rather than on every call. _LEARNED_RESPONSES_ONLY_MODELS: set[str] = set() +# Known model families that reject `reasoning_effort` alongside function tools on +# /v1/chat/completions. This is only a fast path to avoid a wasted round trip -- +# the authoritative check is `_is_tools_reasoning_effort_error`, which catches +# families we haven't measured yet. +# +# Measured against /v1/chat/completions: gpt-5.4, gpt-5.5 and gpt-5.6 (including +# -mini, -nano, dated snapshots and the 5.6 sol/terra/luna variants) all reject +# the combination. gpt-5.2 and earlier, o1, o3 and o4-mini accept it. +# Matched as substrings against the lowercased model name so provider prefixes +# ("openai/gpt-5.5") and suffixed variants ("gpt-5.6-sol") are covered too. +_TOOLS_REASONING_EFFORT_INCOMPATIBLE: tuple[str, ...] = ( + "gpt-5.4", + "gpt-5.5", + "gpt-5.6", +) + +# Models learned at runtime from a 400, keyed by the model string as configured. +# Populated by `_remember_reasoning_effort_conflict` so the retry is paid once +# per model per process rather than on every call. +_LEARNED_TOOLS_REASONING_EFFORT_CONFLICTS: set[str] = set() + class WebSearchResult(TypedDict, total=False): """Result from web search built-in tool.""" @@ -498,50 +526,67 @@ class OpenAICompletion(BaseLLM): ) -> str | Any: """Call OpenAI Chat Completions API. - Falls back to the Responses API when the model turns out not to be served - by chat completions, which OpenAI reports as a 404 distinct from a genuine - unknown model. + Two failures are recovered rather than surfaced, both driven by what the API + actually returns rather than a hardcoded model list: + + - 404 for a model that exists but isn't served here: retry on the Responses + API. + - 400 for ``reasoning_effort`` alongside function tools: retry without the + parameter. + + Each is remembered so the wasted round trip is paid once per model. """ completion_params = self._prepare_completion_params( messages=messages, tools=tools ) - try: + def dispatch(params: dict[str, Any]) -> str | Any: if self._effective_stream(): return self._handle_streaming_completion( - params=completion_params, + params=params, + available_functions=available_functions, + from_task=from_task, + from_agent=from_agent, + response_model=response_model, + ) + return self._handle_completion( + params=params, + available_functions=available_functions, + from_task=from_task, + from_agent=from_agent, + response_model=response_model, + ) + + try: + return dispatch(completion_params) + except Exception as e: + cause = e.__cause__ or e + + if self._is_responses_only_error(cause) and not self.custom_openai: + self._remember_responses_only_model() + logging.debug( + "Retrying %r on the Responses API: not served by " + '/v1/chat/completions. Set api="responses" to skip this.', + self.model, + ) + return self._call_responses( + messages=messages, + tools=tools, available_functions=available_functions, from_task=from_task, from_agent=from_agent, response_model=response_model, ) - return self._handle_completion( - params=completion_params, - available_functions=available_functions, - from_task=from_task, - from_agent=from_agent, - response_model=response_model, - ) - except Exception as e: - if self.custom_openai or not self._is_responses_only_error( - e.__cause__ or e - ): - raise - self._remember_responses_only_model() - logging.debug( - "Retrying %r on the Responses API: not served by " - '/v1/chat/completions. Set api="responses" to skip this.', - self.model, - ) - return self._call_responses( - messages=messages, - tools=tools, - available_functions=available_functions, - from_task=from_task, - from_agent=from_agent, - response_model=response_model, - ) + if self._is_tools_reasoning_effort_error(cause): + retry_params = self._retry_params_without_reasoning_effort( + completion_params + ) + if retry_params is not None: + self._remember_reasoning_effort_conflict() + return dispatch(retry_params) + + raise async def acall( self, @@ -618,44 +663,54 @@ class OpenAICompletion(BaseLLM): ) -> str | Any: """Async call to OpenAI Chat Completions API. - Falls back to the Responses API for models not served by chat completions, - mirroring :meth:`_call_completions`. + Recovers from the same two failures as :meth:`_call_completions`. """ completion_params = self._prepare_completion_params( messages=messages, tools=tools ) - try: + async def dispatch(params: dict[str, Any]) -> str | Any: if self._effective_stream(): return await self._ahandle_streaming_completion( - params=completion_params, + params=params, + available_functions=available_functions, + from_task=from_task, + from_agent=from_agent, + response_model=response_model, + ) + return await self._ahandle_completion( + params=params, + available_functions=available_functions, + from_task=from_task, + from_agent=from_agent, + response_model=response_model, + ) + + try: + return await dispatch(completion_params) + except Exception as e: + cause = e.__cause__ or e + + if self._is_responses_only_error(cause) and not self.custom_openai: + self._remember_responses_only_model() + return await self._acall_responses( + messages=messages, + tools=tools, available_functions=available_functions, from_task=from_task, from_agent=from_agent, response_model=response_model, ) - return await self._ahandle_completion( - params=completion_params, - available_functions=available_functions, - from_task=from_task, - from_agent=from_agent, - response_model=response_model, - ) - except Exception as e: - if self.custom_openai or not self._is_responses_only_error( - e.__cause__ or e - ): - raise - self._remember_responses_only_model() - return await self._acall_responses( - messages=messages, - tools=tools, - available_functions=available_functions, - from_task=from_task, - from_agent=from_agent, - response_model=response_model, - ) + if self._is_tools_reasoning_effort_error(cause): + retry_params = self._retry_params_without_reasoning_effort( + completion_params + ) + if retry_params is not None: + self._remember_reasoning_effort_conflict() + return await dispatch(retry_params) + + raise def _call_responses( self, @@ -1648,6 +1703,74 @@ class OpenAICompletion(BaseLLM): ) return f"Model {self.model} not found: {error}" + def _supports_reasoning_effort_with_tools(self, model: str) -> bool: + """Whether a model accepts `reasoning_effort` together with function tools. + + On /v1/chat/completions, newer reasoning models reject the combination: + + "Function tools with reasoning_effort are not supported for gpt-5.5 + in /v1/chat/completions. To use function tools, use /v1/responses or + set reasoning_effort to 'none'." + + Known families are listed in `_TOOLS_REASONING_EFFORT_INCOMPATIBLE` purely + to skip a doomed request. Models discovered through a 400 at runtime are + remembered in `_LEARNED_TOOLS_REASONING_EFFORT_CONFLICTS`, so OpenAI can + extend the restriction to new families without a release here. + + The Responses API has no such restriction (it takes + `reasoning: {"effort": ...}`), so this only constrains the completions path. + """ + if model in _LEARNED_TOOLS_REASONING_EFFORT_CONFLICTS: + return False + model_lower = model.lower() + return not any( + family in model_lower for family in _TOOLS_REASONING_EFFORT_INCOMPATIBLE + ) + + @staticmethod + def _is_tools_reasoning_effort_error(error: BaseException) -> bool: + """Whether a 400 is OpenAI rejecting `reasoning_effort` next to tools. + + Matches on the structured fields the API returns rather than the prose, + which OpenAI has already reworded once between model generations: + + {"error": {"message": "Function tools with reasoning_effort are not + supported for gpt-5.5 in /v1/chat/completions. ...", + "type": "invalid_request_error", "param": "reasoning_effort"}} + """ + if not isinstance(error, BadRequestError): + return False + param = None + body = getattr(error, "body", None) + if isinstance(body, dict): + inner = body.get("error") + if isinstance(inner, dict): + param = inner.get("param") + if param != "reasoning_effort": + return False + message = str(getattr(error, "message", "") or error).lower() + return "function tools" in message and "reasoning_effort" in message + + def _remember_reasoning_effort_conflict(self) -> None: + """Record that this model can't combine `reasoning_effort` with tools.""" + _LEARNED_TOOLS_REASONING_EFFORT_CONFLICTS.add(self.model) + + def _retry_params_without_reasoning_effort( + self, params: dict[str, Any] + ) -> dict[str, Any] | None: + """Strip `reasoning_effort` for a retry, or None if there's nothing to strip.""" + if params.get("reasoning_effort") is None: + return None + retry_params = {k: v for k, v in params.items() if k != "reasoning_effort"} + logging.warning( + "Dropping reasoning_effort=%r and retrying: %r rejects it alongside " + 'function tools on /v1/chat/completions. Use api="responses" to keep ' + "reasoning effort with tools.", + params["reasoning_effort"], + self.model, + ) + return retry_params + def _effective_api(self) -> str: """Which OpenAI API to actually use for this model. @@ -1718,6 +1841,27 @@ class OpenAICompletion(BaseLLM): params["tools"] = self._convert_tools_for_interference(tools) params["tool_choice"] = "auto" + # gpt-5.4+ reject `reasoning_effort` on the completions endpoint as soon + # as function tools are present. Anything other than "none" is a hard 400, + # so drop it up front for models we already know about rather than burn a + # round trip. Families we haven't measured are caught by the 400 handler + # in `_call_completions` instead. + # Checked after the tools block on purpose: `reasoning_effort` can also + # arrive through `additional_params`, which bypasses the typed field. + if ( + params.get("tools") + and params.get("reasoning_effort") not in (None, "none") + and not self._supports_reasoning_effort_with_tools(self.model) + ): + dropped = params.pop("reasoning_effort") + logging.debug( + "Dropping reasoning_effort=%r for model %r: function tools and " + "reasoning_effort cannot be combined on /v1/chat/completions. " + 'Use api="responses" to keep reasoning effort with tools.', + dropped, + self.model, + ) + crewai_specific_params = { "callbacks", "available_functions", diff --git a/lib/crewai/tests/llms/openai/test_gpt56_tools_reasoning_effort.py b/lib/crewai/tests/llms/openai/test_gpt56_tools_reasoning_effort.py new file mode 100644 index 000000000..c7b699e09 --- /dev/null +++ b/lib/crewai/tests/llms/openai/test_gpt56_tools_reasoning_effort.py @@ -0,0 +1,285 @@ +"""Regression tests for models rejecting `reasoning_effort` alongside tools. + +On /v1/chat/completions, gpt-5.4 and newer return a hard 400 when function tools +and a non-"none" `reasoning_effort` are sent together: + + Function tools with reasoning_effort are not supported for gpt-5.5 in + /v1/chat/completions. To use function tools, use /v1/responses or set + reasoning_effort to 'none'. + +Measured against the live endpoint (2026-07): + + model tools + "high" tools + "none" + gpt-5.6[-sol/...] rejected accepted + gpt-5.5 rejected accepted + gpt-5.4[-mini/nano] rejected accepted + gpt-5.2 accepted accepted + o1 / o3 / o4-mini accepted rejected + +Known families are dropped before the request as a fast path. Anything else is +caught from the 400 and retried without the parameter, so a newly-restricted +model doesn't need a release here. The Responses API accepts +`reasoning: {"effort": ...}` with tools and is untouched. +""" + +import httpx +import pytest +from openai import BadRequestError + +from crewai.llms.providers.openai import completion as completion_module +from crewai.llms.providers.openai.completion import OpenAICompletion + + +MESSAGES = [{"role": "user", "content": "hi"}] + +# Pre-converted OpenAI tool schema — passed through by +# _convert_tools_for_interference, keeping these tests focused on param handling. +TOOLS = [ + { + "type": "function", + "function": { + "name": "web_search", + "description": "Search the web", + "parameters": {"type": "object", "properties": {}}, + }, + } +] + + +def build(model: str, **kwargs) -> OpenAICompletion: + return OpenAICompletion(model=model, api_key="sk-test", **kwargs) + + +def make_bad_request( + model: str = "gpt-5.5", param: str = "reasoning_effort", message: str | None = None +) -> BadRequestError: + """Build the 400 OpenAI returns for the tools + reasoning_effort combination.""" + if message is None: + message = ( + f"Function tools with reasoning_effort are not supported for {model} " + "in /v1/chat/completions. To use function tools, use /v1/responses " + "or set reasoning_effort to 'none'." + ) + body = {"error": {"message": message, "type": "invalid_request_error", "param": param}} + response = httpx.Response( + status_code=400, + json=body, + request=httpx.Request("POST", "https://api.openai.com/v1/chat/completions"), + ) + return BadRequestError(message, response=response, body=body) + + +@pytest.fixture(autouse=True) +def _clear_learned_conflicts(): + """Keep the process-wide learned set from leaking between tests.""" + completion_module._LEARNED_TOOLS_REASONING_EFFORT_CONFLICTS.clear() + yield + completion_module._LEARNED_TOOLS_REASONING_EFFORT_CONFLICTS.clear() + + +class TestKnownFamiliesFastPath: + """Models we've measured are stripped before the request goes out.""" + + @pytest.mark.parametrize( + "model", + [ + "gpt-5.6", + "openai/gpt-5.6", + "gpt-5.6-sol", + "GPT-5.6-Terra", + "gpt-5.6-luna", + "gpt-5.5", + "gpt-5.5-2026-04-23", + "gpt-5.4", + "gpt-5.4-mini", + "gpt-5.4-nano", + ], + ) + def test_drops_reasoning_effort_for_incompatible_models(self, model: str): + llm = build(model, additional_params={"reasoning_effort": "high"}) + + params = llm._prepare_completion_params(MESSAGES, tools=TOOLS) + + assert "reasoning_effort" not in params + # Tools must survive — they are the reason the call exists. + assert params["tools"] + assert params["tool_choice"] == "auto" + + def test_drops_reasoning_effort_supplied_via_additional_params(self): + """additional_params bypasses the typed field, so it must be checked too.""" + llm = build("gpt-5.5", additional_params={"reasoning_effort": "medium"}) + + params = llm._prepare_completion_params(MESSAGES, tools=TOOLS) + + assert "reasoning_effort" not in params + + def test_keeps_reasoning_effort_without_tools(self): + """Without tools the combination is legal, so nothing should change.""" + llm = build("gpt-5.5", additional_params={"reasoning_effort": "high"}) + + params = llm._prepare_completion_params(MESSAGES, tools=None) + + assert params["reasoning_effort"] == "high" + + def test_keeps_explicit_none_effort_with_tools(self): + """OpenAI explicitly allows reasoning_effort='none' with tools here.""" + llm = build("gpt-5.6", additional_params={"reasoning_effort": "none"}) + + params = llm._prepare_completion_params(MESSAGES, tools=TOOLS) + + assert params["reasoning_effort"] == "none" + + @pytest.mark.parametrize("model", ["o1", "o3-mini", "gpt-5.2", "gpt-5"]) + def test_unaffected_models_keep_reasoning_effort_with_tools(self, model: str): + """Measured as accepting the combination — must not be stripped.""" + llm = build(model, additional_params={"reasoning_effort": "high"}) + + params = llm._prepare_completion_params(MESSAGES, tools=TOOLS) + + assert params["reasoning_effort"] == "high" + + def test_fast_path_actually_has_something_to_drop(self): + """Guard against the drop tests passing vacuously. + + The typed `reasoning_effort` field is gated behind `is_o1_model`, so it + never reaches the payload for gpt-5.x. If a test set it that way there + would be nothing to strip and the assertion would pass for the wrong + reason — `additional_params` is the path that actually leaks. + """ + leaked = build( + "gpt-5.5", additional_params={"reasoning_effort": "high"} + )._prepare_completion_params(MESSAGES, tools=None) + assert leaked["reasoning_effort"] == "high" + + typed = build( + "gpt-5.5", reasoning_effort="high" + )._prepare_completion_params(MESSAGES, tools=None) + assert "reasoning_effort" not in typed + + +class TestErrorClassification: + """Only the specific tools + reasoning_effort 400 should trigger a retry.""" + + def test_matches_the_reported_error(self): + assert OpenAICompletion._is_tools_reasoning_effort_error(make_bad_request()) + + def test_ignores_other_bad_requests(self): + error = make_bad_request( + param="max_tokens", + message="Unsupported parameter: 'max_tokens' is not supported.", + ) + assert not OpenAICompletion._is_tools_reasoning_effort_error(error) + + def test_ignores_unsupported_effort_value(self): + """o1/o3 reject reasoning_effort='none'; that is a different failure.""" + error = make_bad_request( + message="Unsupported value: 'reasoning_effort' does not support 'none'." + ) + assert not OpenAICompletion._is_tools_reasoning_effort_error(error) + + def test_ignores_unrelated_exceptions(self): + assert not OpenAICompletion._is_tools_reasoning_effort_error( + RuntimeError("boom") + ) + + +class TestRuntimeRetry: + """Unmeasured models are recovered from the 400 rather than failing the run.""" + + def test_retries_without_reasoning_effort_and_succeeds(self, monkeypatch): + # gpt-5.9 is deliberately not in the known-families tuple. + llm = build("gpt-5.9", additional_params={"reasoning_effort": "high"}) + seen: list[dict] = [] + + def fake_handle(params, **kwargs): + seen.append(params) + if "reasoning_effort" in params: + raise make_bad_request(model="gpt-5.9") + return "ok" + + monkeypatch.setattr(llm, "_handle_completion", fake_handle) + + result = llm._call_completions(MESSAGES, tools=TOOLS) + + assert result == "ok" + assert len(seen) == 2, "expected one failed call and one retry" + assert seen[0]["reasoning_effort"] == "high" + assert "reasoning_effort" not in seen[1] + # Tools have to survive the retry. + assert seen[1]["tools"] + + def test_remembers_the_conflict_for_later_calls(self, monkeypatch): + llm = build("gpt-5.9", additional_params={"reasoning_effort": "high"}) + + def fake_handle(params, **kwargs): + if "reasoning_effort" in params: + raise make_bad_request(model="gpt-5.9") + return "ok" + + monkeypatch.setattr(llm, "_handle_completion", fake_handle) + llm._call_completions(MESSAGES, tools=TOOLS) + + # Second call must strip up front instead of paying the 400 again. + params = llm._prepare_completion_params(MESSAGES, tools=TOOLS) + assert "reasoning_effort" not in params + assert not llm._supports_reasoning_effort_with_tools("gpt-5.9") + + def test_does_not_retry_unrelated_bad_request(self, monkeypatch): + llm = build("gpt-5.9", additional_params={"reasoning_effort": "high"}) + calls: list[dict] = [] + + def fake_handle(params, **kwargs): + calls.append(params) + raise make_bad_request(param="max_tokens", message="Unsupported parameter") + + monkeypatch.setattr(llm, "_handle_completion", fake_handle) + + with pytest.raises(BadRequestError): + llm._call_completions(MESSAGES, tools=TOOLS) + + assert len(calls) == 1, "must not retry errors it doesn't understand" + + def test_does_not_retry_when_nothing_to_strip(self, monkeypatch): + """No reasoning_effort to remove means the retry can't help.""" + llm = build("gpt-5.9") + calls: list[dict] = [] + + def fake_handle(params, **kwargs): + calls.append(params) + raise make_bad_request(model="gpt-5.9") + + monkeypatch.setattr(llm, "_handle_completion", fake_handle) + + with pytest.raises(BadRequestError): + llm._call_completions(MESSAGES, tools=TOOLS) + + assert len(calls) == 1 + + @pytest.mark.asyncio + async def test_async_path_retries_too(self, monkeypatch): + llm = build("gpt-5.9", additional_params={"reasoning_effort": "high"}) + seen: list[dict] = [] + + async def fake_handle(params, **kwargs): + seen.append(params) + if "reasoning_effort" in params: + raise make_bad_request(model="gpt-5.9") + return "ok" + + monkeypatch.setattr(llm, "_ahandle_completion", fake_handle) + + result = await llm._acall_completions(MESSAGES, tools=TOOLS) + + assert result == "ok" + assert len(seen) == 2 + assert "reasoning_effort" not in seen[1] + + +class TestResponsesApiUntouched: + def test_responses_api_keeps_effort_with_tools(self): + """The Responses API has no such restriction — it takes reasoning.effort.""" + llm = build("gpt-5.6", api="responses", reasoning_effort="high") + + params = llm._prepare_responses_params(MESSAGES, tools=TOOLS) + + assert params["reasoning"] == {"effort": "high"}