fix(openai): recover from tools + reasoning_effort 400 at runtime

Any tool-using agent on gpt-5.4 or newer fails with a hard 400 on the default
endpoint:

    Function tools with reasoning_effort are not supported for gpt-5.5 in
    /v1/chat/completions. To use function tools, use /v1/responses or set
    reasoning_effort to 'none'.

`_prepare_completion_params` gated `reasoning_effort` behind o1, but the value
still reaches the payload for any model through `params.update(self.additional_params)`,
which bypasses the typed field. Tools are then attached unconditionally.

Probed against the live endpoint rather than inferred from the report:

    model                tools + 'high'   tools + 'none'
    gpt-5.6/-sol/...     rejected         accepted
    gpt-5.5              rejected         accepted
    gpt-5.4/-mini/-nano  rejected         accepted
    gpt-5.2 and earlier  accepted         accepted
    o1 / o3 / o4-mini    accepted         rejected

The boundary is gpt-5.4, not 5.6. Note the last row: "none" is not a safe blanket
fallback either.

The 400 is the source of truth: catch it, drop reasoning_effort, retry once, and
remember the model so the round trip is paid once per process. The known families
are a fast path only, so OpenAI extending the restriction doesn't need a release
here. Detection matches the structured `param` field plus the message, so the
unsupported-value 400 that o1/o3 return for "none" doesn't trigger a useless retry.

Logged at warning, not debug -- silently downgrading a requested reasoning effort
should be visible.

Composed with the responses-only fallback from #6656 in a single handler in
_call_completions / _acall_completions, covering sync, async, streaming and
non-streaming. Verified both recoveries still work live after rebasing:

    gpt-5.5 + tools + reasoning_effort -> tool call returned (was 400)
    gpt-5-pro                          -> 'PONG', learned {'gpt-5-pro'}

Tests: 28 cases covering the fast path, error classification, sync + async retry,
no-retry on unrelated 400s, and a guard against the drop tests passing vacuously
(the typed reasoning_effort field is gated behind is_o1_model, so additional_params
is the path that actually leaks). tests/llms/openai -> 149 passed, 1 skipped.
This commit is contained in:
alex-clawd
2026-07-26 02:12:29 -07:00
parent b64c92c87b
commit 4f455a3511
2 changed files with 486 additions and 57 deletions

View File

@@ -8,7 +8,14 @@ import os
from typing import TYPE_CHECKING, Any, ClassVar, Literal, TypedDict
import httpx
from openai import APIConnectionError, AsyncOpenAI, NotFoundError, OpenAI, Stream
from openai import (
APIConnectionError,
AsyncOpenAI,
BadRequestError,
NotFoundError,
OpenAI,
Stream,
)
from openai.lib.streaming.chat import ChatCompletionStream
from openai.types.chat import (
ChatCompletion,
@@ -57,6 +64,27 @@ if TYPE_CHECKING:
# per process rather than on every call.
_LEARNED_RESPONSES_ONLY_MODELS: set[str] = set()
# Known model families that reject `reasoning_effort` alongside function tools on
# /v1/chat/completions. This is only a fast path to avoid a wasted round trip --
# the authoritative check is `_is_tools_reasoning_effort_error`, which catches
# families we haven't measured yet.
#
# Measured against /v1/chat/completions: gpt-5.4, gpt-5.5 and gpt-5.6 (including
# -mini, -nano, dated snapshots and the 5.6 sol/terra/luna variants) all reject
# the combination. gpt-5.2 and earlier, o1, o3 and o4-mini accept it.
# Matched as substrings against the lowercased model name so provider prefixes
# ("openai/gpt-5.5") and suffixed variants ("gpt-5.6-sol") are covered too.
_TOOLS_REASONING_EFFORT_INCOMPATIBLE: tuple[str, ...] = (
"gpt-5.4",
"gpt-5.5",
"gpt-5.6",
)
# Models learned at runtime from a 400, keyed by the model string as configured.
# Populated by `_remember_reasoning_effort_conflict` so the retry is paid once
# per model per process rather than on every call.
_LEARNED_TOOLS_REASONING_EFFORT_CONFLICTS: set[str] = set()
class WebSearchResult(TypedDict, total=False):
"""Result from web search built-in tool."""
@@ -498,50 +526,67 @@ class OpenAICompletion(BaseLLM):
) -> str | Any:
"""Call OpenAI Chat Completions API.
Falls back to the Responses API when the model turns out not to be served
by chat completions, which OpenAI reports as a 404 distinct from a genuine
unknown model.
Two failures are recovered rather than surfaced, both driven by what the API
actually returns rather than a hardcoded model list:
- 404 for a model that exists but isn't served here: retry on the Responses
API.
- 400 for ``reasoning_effort`` alongside function tools: retry without the
parameter.
Each is remembered so the wasted round trip is paid once per model.
"""
completion_params = self._prepare_completion_params(
messages=messages, tools=tools
)
try:
def dispatch(params: dict[str, Any]) -> str | Any:
if self._effective_stream():
return self._handle_streaming_completion(
params=completion_params,
params=params,
available_functions=available_functions,
from_task=from_task,
from_agent=from_agent,
response_model=response_model,
)
return self._handle_completion(
params=params,
available_functions=available_functions,
from_task=from_task,
from_agent=from_agent,
response_model=response_model,
)
try:
return dispatch(completion_params)
except Exception as e:
cause = e.__cause__ or e
if self._is_responses_only_error(cause) and not self.custom_openai:
self._remember_responses_only_model()
logging.debug(
"Retrying %r on the Responses API: not served by "
'/v1/chat/completions. Set api="responses" to skip this.',
self.model,
)
return self._call_responses(
messages=messages,
tools=tools,
available_functions=available_functions,
from_task=from_task,
from_agent=from_agent,
response_model=response_model,
)
return self._handle_completion(
params=completion_params,
available_functions=available_functions,
from_task=from_task,
from_agent=from_agent,
response_model=response_model,
)
except Exception as e:
if self.custom_openai or not self._is_responses_only_error(
e.__cause__ or e
):
raise
self._remember_responses_only_model()
logging.debug(
"Retrying %r on the Responses API: not served by "
'/v1/chat/completions. Set api="responses" to skip this.',
self.model,
)
return self._call_responses(
messages=messages,
tools=tools,
available_functions=available_functions,
from_task=from_task,
from_agent=from_agent,
response_model=response_model,
)
if self._is_tools_reasoning_effort_error(cause):
retry_params = self._retry_params_without_reasoning_effort(
completion_params
)
if retry_params is not None:
self._remember_reasoning_effort_conflict()
return dispatch(retry_params)
raise
async def acall(
self,
@@ -618,44 +663,54 @@ class OpenAICompletion(BaseLLM):
) -> str | Any:
"""Async call to OpenAI Chat Completions API.
Falls back to the Responses API for models not served by chat completions,
mirroring :meth:`_call_completions`.
Recovers from the same two failures as :meth:`_call_completions`.
"""
completion_params = self._prepare_completion_params(
messages=messages, tools=tools
)
try:
async def dispatch(params: dict[str, Any]) -> str | Any:
if self._effective_stream():
return await self._ahandle_streaming_completion(
params=completion_params,
params=params,
available_functions=available_functions,
from_task=from_task,
from_agent=from_agent,
response_model=response_model,
)
return await self._ahandle_completion(
params=params,
available_functions=available_functions,
from_task=from_task,
from_agent=from_agent,
response_model=response_model,
)
try:
return await dispatch(completion_params)
except Exception as e:
cause = e.__cause__ or e
if self._is_responses_only_error(cause) and not self.custom_openai:
self._remember_responses_only_model()
return await self._acall_responses(
messages=messages,
tools=tools,
available_functions=available_functions,
from_task=from_task,
from_agent=from_agent,
response_model=response_model,
)
return await self._ahandle_completion(
params=completion_params,
available_functions=available_functions,
from_task=from_task,
from_agent=from_agent,
response_model=response_model,
)
except Exception as e:
if self.custom_openai or not self._is_responses_only_error(
e.__cause__ or e
):
raise
self._remember_responses_only_model()
return await self._acall_responses(
messages=messages,
tools=tools,
available_functions=available_functions,
from_task=from_task,
from_agent=from_agent,
response_model=response_model,
)
if self._is_tools_reasoning_effort_error(cause):
retry_params = self._retry_params_without_reasoning_effort(
completion_params
)
if retry_params is not None:
self._remember_reasoning_effort_conflict()
return await dispatch(retry_params)
raise
def _call_responses(
self,
@@ -1648,6 +1703,74 @@ class OpenAICompletion(BaseLLM):
)
return f"Model {self.model} not found: {error}"
def _supports_reasoning_effort_with_tools(self, model: str) -> bool:
"""Whether a model accepts `reasoning_effort` together with function tools.
On /v1/chat/completions, newer reasoning models reject the combination:
"Function tools with reasoning_effort are not supported for gpt-5.5
in /v1/chat/completions. To use function tools, use /v1/responses or
set reasoning_effort to 'none'."
Known families are listed in `_TOOLS_REASONING_EFFORT_INCOMPATIBLE` purely
to skip a doomed request. Models discovered through a 400 at runtime are
remembered in `_LEARNED_TOOLS_REASONING_EFFORT_CONFLICTS`, so OpenAI can
extend the restriction to new families without a release here.
The Responses API has no such restriction (it takes
`reasoning: {"effort": ...}`), so this only constrains the completions path.
"""
if model in _LEARNED_TOOLS_REASONING_EFFORT_CONFLICTS:
return False
model_lower = model.lower()
return not any(
family in model_lower for family in _TOOLS_REASONING_EFFORT_INCOMPATIBLE
)
@staticmethod
def _is_tools_reasoning_effort_error(error: BaseException) -> bool:
"""Whether a 400 is OpenAI rejecting `reasoning_effort` next to tools.
Matches on the structured fields the API returns rather than the prose,
which OpenAI has already reworded once between model generations:
{"error": {"message": "Function tools with reasoning_effort are not
supported for gpt-5.5 in /v1/chat/completions. ...",
"type": "invalid_request_error", "param": "reasoning_effort"}}
"""
if not isinstance(error, BadRequestError):
return False
param = None
body = getattr(error, "body", None)
if isinstance(body, dict):
inner = body.get("error")
if isinstance(inner, dict):
param = inner.get("param")
if param != "reasoning_effort":
return False
message = str(getattr(error, "message", "") or error).lower()
return "function tools" in message and "reasoning_effort" in message
def _remember_reasoning_effort_conflict(self) -> None:
"""Record that this model can't combine `reasoning_effort` with tools."""
_LEARNED_TOOLS_REASONING_EFFORT_CONFLICTS.add(self.model)
def _retry_params_without_reasoning_effort(
self, params: dict[str, Any]
) -> dict[str, Any] | None:
"""Strip `reasoning_effort` for a retry, or None if there's nothing to strip."""
if params.get("reasoning_effort") is None:
return None
retry_params = {k: v for k, v in params.items() if k != "reasoning_effort"}
logging.warning(
"Dropping reasoning_effort=%r and retrying: %r rejects it alongside "
'function tools on /v1/chat/completions. Use api="responses" to keep '
"reasoning effort with tools.",
params["reasoning_effort"],
self.model,
)
return retry_params
def _effective_api(self) -> str:
"""Which OpenAI API to actually use for this model.
@@ -1718,6 +1841,27 @@ class OpenAICompletion(BaseLLM):
params["tools"] = self._convert_tools_for_interference(tools)
params["tool_choice"] = "auto"
# gpt-5.4+ reject `reasoning_effort` on the completions endpoint as soon
# as function tools are present. Anything other than "none" is a hard 400,
# so drop it up front for models we already know about rather than burn a
# round trip. Families we haven't measured are caught by the 400 handler
# in `_call_completions` instead.
# Checked after the tools block on purpose: `reasoning_effort` can also
# arrive through `additional_params`, which bypasses the typed field.
if (
params.get("tools")
and params.get("reasoning_effort") not in (None, "none")
and not self._supports_reasoning_effort_with_tools(self.model)
):
dropped = params.pop("reasoning_effort")
logging.debug(
"Dropping reasoning_effort=%r for model %r: function tools and "
"reasoning_effort cannot be combined on /v1/chat/completions. "
'Use api="responses" to keep reasoning effort with tools.',
dropped,
self.model,
)
crewai_specific_params = {
"callbacks",
"available_functions",

View File

@@ -0,0 +1,285 @@
"""Regression tests for models rejecting `reasoning_effort` alongside tools.
On /v1/chat/completions, gpt-5.4 and newer return a hard 400 when function tools
and a non-"none" `reasoning_effort` are sent together:
Function tools with reasoning_effort are not supported for gpt-5.5 in
/v1/chat/completions. To use function tools, use /v1/responses or set
reasoning_effort to 'none'.
Measured against the live endpoint (2026-07):
model tools + "high" tools + "none"
gpt-5.6[-sol/...] rejected accepted
gpt-5.5 rejected accepted
gpt-5.4[-mini/nano] rejected accepted
gpt-5.2 accepted accepted
o1 / o3 / o4-mini accepted rejected
Known families are dropped before the request as a fast path. Anything else is
caught from the 400 and retried without the parameter, so a newly-restricted
model doesn't need a release here. The Responses API accepts
`reasoning: {"effort": ...}` with tools and is untouched.
"""
import httpx
import pytest
from openai import BadRequestError
from crewai.llms.providers.openai import completion as completion_module
from crewai.llms.providers.openai.completion import OpenAICompletion
MESSAGES = [{"role": "user", "content": "hi"}]
# Pre-converted OpenAI tool schema — passed through by
# _convert_tools_for_interference, keeping these tests focused on param handling.
TOOLS = [
{
"type": "function",
"function": {
"name": "web_search",
"description": "Search the web",
"parameters": {"type": "object", "properties": {}},
},
}
]
def build(model: str, **kwargs) -> OpenAICompletion:
return OpenAICompletion(model=model, api_key="sk-test", **kwargs)
def make_bad_request(
model: str = "gpt-5.5", param: str = "reasoning_effort", message: str | None = None
) -> BadRequestError:
"""Build the 400 OpenAI returns for the tools + reasoning_effort combination."""
if message is None:
message = (
f"Function tools with reasoning_effort are not supported for {model} "
"in /v1/chat/completions. To use function tools, use /v1/responses "
"or set reasoning_effort to 'none'."
)
body = {"error": {"message": message, "type": "invalid_request_error", "param": param}}
response = httpx.Response(
status_code=400,
json=body,
request=httpx.Request("POST", "https://api.openai.com/v1/chat/completions"),
)
return BadRequestError(message, response=response, body=body)
@pytest.fixture(autouse=True)
def _clear_learned_conflicts():
"""Keep the process-wide learned set from leaking between tests."""
completion_module._LEARNED_TOOLS_REASONING_EFFORT_CONFLICTS.clear()
yield
completion_module._LEARNED_TOOLS_REASONING_EFFORT_CONFLICTS.clear()
class TestKnownFamiliesFastPath:
"""Models we've measured are stripped before the request goes out."""
@pytest.mark.parametrize(
"model",
[
"gpt-5.6",
"openai/gpt-5.6",
"gpt-5.6-sol",
"GPT-5.6-Terra",
"gpt-5.6-luna",
"gpt-5.5",
"gpt-5.5-2026-04-23",
"gpt-5.4",
"gpt-5.4-mini",
"gpt-5.4-nano",
],
)
def test_drops_reasoning_effort_for_incompatible_models(self, model: str):
llm = build(model, additional_params={"reasoning_effort": "high"})
params = llm._prepare_completion_params(MESSAGES, tools=TOOLS)
assert "reasoning_effort" not in params
# Tools must survive — they are the reason the call exists.
assert params["tools"]
assert params["tool_choice"] == "auto"
def test_drops_reasoning_effort_supplied_via_additional_params(self):
"""additional_params bypasses the typed field, so it must be checked too."""
llm = build("gpt-5.5", additional_params={"reasoning_effort": "medium"})
params = llm._prepare_completion_params(MESSAGES, tools=TOOLS)
assert "reasoning_effort" not in params
def test_keeps_reasoning_effort_without_tools(self):
"""Without tools the combination is legal, so nothing should change."""
llm = build("gpt-5.5", additional_params={"reasoning_effort": "high"})
params = llm._prepare_completion_params(MESSAGES, tools=None)
assert params["reasoning_effort"] == "high"
def test_keeps_explicit_none_effort_with_tools(self):
"""OpenAI explicitly allows reasoning_effort='none' with tools here."""
llm = build("gpt-5.6", additional_params={"reasoning_effort": "none"})
params = llm._prepare_completion_params(MESSAGES, tools=TOOLS)
assert params["reasoning_effort"] == "none"
@pytest.mark.parametrize("model", ["o1", "o3-mini", "gpt-5.2", "gpt-5"])
def test_unaffected_models_keep_reasoning_effort_with_tools(self, model: str):
"""Measured as accepting the combination — must not be stripped."""
llm = build(model, additional_params={"reasoning_effort": "high"})
params = llm._prepare_completion_params(MESSAGES, tools=TOOLS)
assert params["reasoning_effort"] == "high"
def test_fast_path_actually_has_something_to_drop(self):
"""Guard against the drop tests passing vacuously.
The typed `reasoning_effort` field is gated behind `is_o1_model`, so it
never reaches the payload for gpt-5.x. If a test set it that way there
would be nothing to strip and the assertion would pass for the wrong
reason — `additional_params` is the path that actually leaks.
"""
leaked = build(
"gpt-5.5", additional_params={"reasoning_effort": "high"}
)._prepare_completion_params(MESSAGES, tools=None)
assert leaked["reasoning_effort"] == "high"
typed = build(
"gpt-5.5", reasoning_effort="high"
)._prepare_completion_params(MESSAGES, tools=None)
assert "reasoning_effort" not in typed
class TestErrorClassification:
"""Only the specific tools + reasoning_effort 400 should trigger a retry."""
def test_matches_the_reported_error(self):
assert OpenAICompletion._is_tools_reasoning_effort_error(make_bad_request())
def test_ignores_other_bad_requests(self):
error = make_bad_request(
param="max_tokens",
message="Unsupported parameter: 'max_tokens' is not supported.",
)
assert not OpenAICompletion._is_tools_reasoning_effort_error(error)
def test_ignores_unsupported_effort_value(self):
"""o1/o3 reject reasoning_effort='none'; that is a different failure."""
error = make_bad_request(
message="Unsupported value: 'reasoning_effort' does not support 'none'."
)
assert not OpenAICompletion._is_tools_reasoning_effort_error(error)
def test_ignores_unrelated_exceptions(self):
assert not OpenAICompletion._is_tools_reasoning_effort_error(
RuntimeError("boom")
)
class TestRuntimeRetry:
"""Unmeasured models are recovered from the 400 rather than failing the run."""
def test_retries_without_reasoning_effort_and_succeeds(self, monkeypatch):
# gpt-5.9 is deliberately not in the known-families tuple.
llm = build("gpt-5.9", additional_params={"reasoning_effort": "high"})
seen: list[dict] = []
def fake_handle(params, **kwargs):
seen.append(params)
if "reasoning_effort" in params:
raise make_bad_request(model="gpt-5.9")
return "ok"
monkeypatch.setattr(llm, "_handle_completion", fake_handle)
result = llm._call_completions(MESSAGES, tools=TOOLS)
assert result == "ok"
assert len(seen) == 2, "expected one failed call and one retry"
assert seen[0]["reasoning_effort"] == "high"
assert "reasoning_effort" not in seen[1]
# Tools have to survive the retry.
assert seen[1]["tools"]
def test_remembers_the_conflict_for_later_calls(self, monkeypatch):
llm = build("gpt-5.9", additional_params={"reasoning_effort": "high"})
def fake_handle(params, **kwargs):
if "reasoning_effort" in params:
raise make_bad_request(model="gpt-5.9")
return "ok"
monkeypatch.setattr(llm, "_handle_completion", fake_handle)
llm._call_completions(MESSAGES, tools=TOOLS)
# Second call must strip up front instead of paying the 400 again.
params = llm._prepare_completion_params(MESSAGES, tools=TOOLS)
assert "reasoning_effort" not in params
assert not llm._supports_reasoning_effort_with_tools("gpt-5.9")
def test_does_not_retry_unrelated_bad_request(self, monkeypatch):
llm = build("gpt-5.9", additional_params={"reasoning_effort": "high"})
calls: list[dict] = []
def fake_handle(params, **kwargs):
calls.append(params)
raise make_bad_request(param="max_tokens", message="Unsupported parameter")
monkeypatch.setattr(llm, "_handle_completion", fake_handle)
with pytest.raises(BadRequestError):
llm._call_completions(MESSAGES, tools=TOOLS)
assert len(calls) == 1, "must not retry errors it doesn't understand"
def test_does_not_retry_when_nothing_to_strip(self, monkeypatch):
"""No reasoning_effort to remove means the retry can't help."""
llm = build("gpt-5.9")
calls: list[dict] = []
def fake_handle(params, **kwargs):
calls.append(params)
raise make_bad_request(model="gpt-5.9")
monkeypatch.setattr(llm, "_handle_completion", fake_handle)
with pytest.raises(BadRequestError):
llm._call_completions(MESSAGES, tools=TOOLS)
assert len(calls) == 1
@pytest.mark.asyncio
async def test_async_path_retries_too(self, monkeypatch):
llm = build("gpt-5.9", additional_params={"reasoning_effort": "high"})
seen: list[dict] = []
async def fake_handle(params, **kwargs):
seen.append(params)
if "reasoning_effort" in params:
raise make_bad_request(model="gpt-5.9")
return "ok"
monkeypatch.setattr(llm, "_ahandle_completion", fake_handle)
result = await llm._acall_completions(MESSAGES, tools=TOOLS)
assert result == "ok"
assert len(seen) == 2
assert "reasoning_effort" not in seen[1]
class TestResponsesApiUntouched:
def test_responses_api_keeps_effort_with_tools(self):
"""The Responses API has no such restriction — it takes reasoning.effort."""
llm = build("gpt-5.6", api="responses", reasoning_effort="high")
params = llm._prepare_responses_params(MESSAGES, tools=TOOLS)
assert params["reasoning"] == {"effort": "high"}