mirror of
https://github.com/crewAIInc/crewAI.git
synced 2026-07-29 10:39:23 +00:00
Compare commits
6 Commits
feat/file-
...
fix/gpt56-
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
4625305a30 | ||
|
|
155ba11c4a | ||
|
|
6c64ffb03f | ||
|
|
ff32483763 | ||
|
|
52ab798410 | ||
|
|
4f455a3511 |
@@ -8,7 +8,14 @@ import os
|
||||
from typing import TYPE_CHECKING, Any, ClassVar, Literal, TypedDict
|
||||
|
||||
import httpx
|
||||
from openai import APIConnectionError, AsyncOpenAI, NotFoundError, OpenAI, Stream
|
||||
from openai import (
|
||||
APIConnectionError,
|
||||
AsyncOpenAI,
|
||||
BadRequestError,
|
||||
NotFoundError,
|
||||
OpenAI,
|
||||
Stream,
|
||||
)
|
||||
from openai.lib.streaming.chat import ChatCompletionStream
|
||||
from openai.types.chat import (
|
||||
ChatCompletion,
|
||||
@@ -55,7 +62,16 @@ if TYPE_CHECKING:
|
||||
# the model string as configured. Populated from the 404 by
|
||||
# `_remember_responses_only_model` so the wasted round trip is paid once per model
|
||||
# per process rather than on every call.
|
||||
_LEARNED_RESPONSES_ONLY_MODELS: set[str] = set()
|
||||
# Keyed by (endpoint, model) rather than model alone: the same model name can
|
||||
# behave differently on api.openai.com and on an OpenAI-compatible proxy, so a
|
||||
# conflict learned against one endpoint must not leak to another.
|
||||
_LEARNED_RESPONSES_ONLY_MODELS: set[tuple[str, str]] = set()
|
||||
|
||||
# Models learned at runtime to reject `reasoning_effort` alongside function tools,
|
||||
# keyed by (endpoint, model). Populated from the 400 by
|
||||
# `_remember_reasoning_effort_conflict` so the wasted round trip is paid once per
|
||||
# model per process rather than on every call.
|
||||
_LEARNED_TOOLS_REASONING_EFFORT_CONFLICTS: set[tuple[str, str]] = set()
|
||||
|
||||
|
||||
class WebSearchResult(TypedDict, total=False):
|
||||
@@ -400,11 +416,7 @@ class OpenAICompletion(BaseLLM):
|
||||
"api_key": self.api_key,
|
||||
"organization": self.organization,
|
||||
"project": self.project,
|
||||
"base_url": self.base_url
|
||||
or self.api_base
|
||||
or os.getenv("OPENAI_BASE_URL")
|
||||
or os.getenv("OPENAI_API_BASE")
|
||||
or None,
|
||||
"base_url": self._resolved_base_url(),
|
||||
"timeout": self.timeout,
|
||||
"max_retries": self.max_retries,
|
||||
"default_headers": self.default_headers,
|
||||
@@ -498,50 +510,67 @@ class OpenAICompletion(BaseLLM):
|
||||
) -> str | Any:
|
||||
"""Call OpenAI Chat Completions API.
|
||||
|
||||
Falls back to the Responses API when the model turns out not to be served
|
||||
by chat completions, which OpenAI reports as a 404 distinct from a genuine
|
||||
unknown model.
|
||||
Two failures are recovered rather than surfaced, both driven by what the API
|
||||
actually returns rather than a hardcoded model list:
|
||||
|
||||
- 404 for a model that exists but isn't served here: retry on the Responses
|
||||
API.
|
||||
- 400 for ``reasoning_effort`` alongside function tools: retry without the
|
||||
parameter.
|
||||
|
||||
Each is remembered so the wasted round trip is paid once per model.
|
||||
"""
|
||||
completion_params = self._prepare_completion_params(
|
||||
messages=messages, tools=tools
|
||||
)
|
||||
|
||||
try:
|
||||
def dispatch(params: dict[str, Any]) -> str | Any:
|
||||
if self._effective_stream():
|
||||
return self._handle_streaming_completion(
|
||||
params=completion_params,
|
||||
params=params,
|
||||
available_functions=available_functions,
|
||||
from_task=from_task,
|
||||
from_agent=from_agent,
|
||||
response_model=response_model,
|
||||
)
|
||||
return self._handle_completion(
|
||||
params=params,
|
||||
available_functions=available_functions,
|
||||
from_task=from_task,
|
||||
from_agent=from_agent,
|
||||
response_model=response_model,
|
||||
)
|
||||
|
||||
try:
|
||||
return dispatch(completion_params)
|
||||
except Exception as e:
|
||||
cause = e.__cause__ or e
|
||||
|
||||
if self._is_responses_only_error(cause) and not self.custom_openai:
|
||||
self._remember_responses_only_model()
|
||||
logging.debug(
|
||||
"Retrying %r on the Responses API: not served by "
|
||||
'/v1/chat/completions. Set api="responses" to skip this.',
|
||||
self.model,
|
||||
)
|
||||
return self._call_responses(
|
||||
messages=messages,
|
||||
tools=tools,
|
||||
available_functions=available_functions,
|
||||
from_task=from_task,
|
||||
from_agent=from_agent,
|
||||
response_model=response_model,
|
||||
)
|
||||
|
||||
return self._handle_completion(
|
||||
params=completion_params,
|
||||
available_functions=available_functions,
|
||||
from_task=from_task,
|
||||
from_agent=from_agent,
|
||||
response_model=response_model,
|
||||
)
|
||||
except Exception as e:
|
||||
if self.custom_openai or not self._is_responses_only_error(
|
||||
e.__cause__ or e
|
||||
):
|
||||
raise
|
||||
self._remember_responses_only_model()
|
||||
logging.debug(
|
||||
"Retrying %r on the Responses API: not served by "
|
||||
'/v1/chat/completions. Set api="responses" to skip this.',
|
||||
self.model,
|
||||
)
|
||||
return self._call_responses(
|
||||
messages=messages,
|
||||
tools=tools,
|
||||
available_functions=available_functions,
|
||||
from_task=from_task,
|
||||
from_agent=from_agent,
|
||||
response_model=response_model,
|
||||
)
|
||||
if self._is_tools_reasoning_effort_error(cause):
|
||||
retry_params = self._retry_params_without_reasoning_effort(
|
||||
completion_params
|
||||
)
|
||||
if retry_params is not None:
|
||||
self._remember_reasoning_effort_conflict()
|
||||
return dispatch(retry_params)
|
||||
|
||||
raise
|
||||
|
||||
async def acall(
|
||||
self,
|
||||
@@ -618,44 +647,54 @@ class OpenAICompletion(BaseLLM):
|
||||
) -> str | Any:
|
||||
"""Async call to OpenAI Chat Completions API.
|
||||
|
||||
Falls back to the Responses API for models not served by chat completions,
|
||||
mirroring :meth:`_call_completions`.
|
||||
Recovers from the same two failures as :meth:`_call_completions`.
|
||||
"""
|
||||
completion_params = self._prepare_completion_params(
|
||||
messages=messages, tools=tools
|
||||
)
|
||||
|
||||
try:
|
||||
async def dispatch(params: dict[str, Any]) -> str | Any:
|
||||
if self._effective_stream():
|
||||
return await self._ahandle_streaming_completion(
|
||||
params=completion_params,
|
||||
params=params,
|
||||
available_functions=available_functions,
|
||||
from_task=from_task,
|
||||
from_agent=from_agent,
|
||||
response_model=response_model,
|
||||
)
|
||||
return await self._ahandle_completion(
|
||||
params=params,
|
||||
available_functions=available_functions,
|
||||
from_task=from_task,
|
||||
from_agent=from_agent,
|
||||
response_model=response_model,
|
||||
)
|
||||
|
||||
try:
|
||||
return await dispatch(completion_params)
|
||||
except Exception as e:
|
||||
cause = e.__cause__ or e
|
||||
|
||||
if self._is_responses_only_error(cause) and not self.custom_openai:
|
||||
self._remember_responses_only_model()
|
||||
return await self._acall_responses(
|
||||
messages=messages,
|
||||
tools=tools,
|
||||
available_functions=available_functions,
|
||||
from_task=from_task,
|
||||
from_agent=from_agent,
|
||||
response_model=response_model,
|
||||
)
|
||||
|
||||
return await self._ahandle_completion(
|
||||
params=completion_params,
|
||||
available_functions=available_functions,
|
||||
from_task=from_task,
|
||||
from_agent=from_agent,
|
||||
response_model=response_model,
|
||||
)
|
||||
except Exception as e:
|
||||
if self.custom_openai or not self._is_responses_only_error(
|
||||
e.__cause__ or e
|
||||
):
|
||||
raise
|
||||
self._remember_responses_only_model()
|
||||
return await self._acall_responses(
|
||||
messages=messages,
|
||||
tools=tools,
|
||||
available_functions=available_functions,
|
||||
from_task=from_task,
|
||||
from_agent=from_agent,
|
||||
response_model=response_model,
|
||||
)
|
||||
if self._is_tools_reasoning_effort_error(cause):
|
||||
retry_params = self._retry_params_without_reasoning_effort(
|
||||
completion_params
|
||||
)
|
||||
if retry_params is not None:
|
||||
self._remember_reasoning_effort_conflict()
|
||||
return await dispatch(retry_params)
|
||||
|
||||
raise
|
||||
|
||||
def _call_responses(
|
||||
self,
|
||||
@@ -1678,7 +1717,7 @@ class OpenAICompletion(BaseLLM):
|
||||
|
||||
def _remember_responses_only_model(self) -> None:
|
||||
"""Record that this model must use the Responses API."""
|
||||
_LEARNED_RESPONSES_ONLY_MODELS.add(self.model)
|
||||
_LEARNED_RESPONSES_ONLY_MODELS.add(self._model_cache_key())
|
||||
|
||||
def _model_not_found_message(self, error: Exception) -> str:
|
||||
"""Build a 404 message that says what to do about it.
|
||||
@@ -1694,6 +1733,126 @@ class OpenAICompletion(BaseLLM):
|
||||
)
|
||||
return f"Model {self.model} not found: {error}"
|
||||
|
||||
def _supports_reasoning_effort_with_tools(self, model: str) -> bool:
|
||||
"""Whether this model is already known to reject the combination.
|
||||
|
||||
Nothing is assumed up front -- no model list. A model is only known once a
|
||||
400 has been seen for it, which `_call_completions` recovers from:
|
||||
|
||||
"Function tools with reasoning_effort are not supported for gpt-5.5
|
||||
in /v1/chat/completions. To use function tools, use /v1/responses or
|
||||
set reasoning_effort to 'none'."
|
||||
|
||||
So OpenAI can extend the restriction to new families without a release here.
|
||||
|
||||
The Responses API has no such restriction (it takes
|
||||
`reasoning: {"effort": ...}`), so this only constrains the completions path.
|
||||
"""
|
||||
return (
|
||||
self._model_cache_key(model)
|
||||
not in _LEARNED_TOOLS_REASONING_EFFORT_CONFLICTS
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _is_tools_reasoning_effort_error(error: BaseException) -> bool:
|
||||
"""Whether a 400 is OpenAI rejecting `reasoning_effort` next to tools.
|
||||
|
||||
Matches on the structured fields the API returns rather than the prose,
|
||||
which OpenAI has already reworded once between model generations:
|
||||
|
||||
{"error": {"message": "Function tools with reasoning_effort are not
|
||||
supported for gpt-5.5 in /v1/chat/completions. ...",
|
||||
"type": "invalid_request_error", "param": "reasoning_effort"}}
|
||||
"""
|
||||
if not isinstance(error, BadRequestError):
|
||||
return False
|
||||
param, body_message = OpenAICompletion._error_param_and_message(error)
|
||||
if param != "reasoning_effort":
|
||||
return False
|
||||
return "function tools" in body_message and "reasoning_effort" in body_message
|
||||
|
||||
@staticmethod
|
||||
def _error_param_and_message(error: BaseException) -> tuple[str | None, str]:
|
||||
"""Pull ``param`` and the API message out of an OpenAI error body.
|
||||
|
||||
The SDK populates ``body`` either flat (``{"message": ..., "param": ...}``)
|
||||
or nested under an ``"error"`` key depending on how the response was
|
||||
parsed, so both are handled. Falls back to the exception text.
|
||||
"""
|
||||
body = getattr(error, "body", None)
|
||||
source: dict[str, Any] | None = None
|
||||
if isinstance(body, dict):
|
||||
inner = body.get("error")
|
||||
source = inner if isinstance(inner, dict) else body
|
||||
param = source.get("param") if source else None
|
||||
message = str((source or {}).get("message") or "") or str(
|
||||
getattr(error, "message", "") or error
|
||||
)
|
||||
return param, message.lower()
|
||||
|
||||
def _resolved_base_url(self) -> str | None:
|
||||
"""The endpoint this instance actually talks to, or None for OpenAI's own.
|
||||
|
||||
Environment variables count: an instance can be pointed at a proxy without
|
||||
either field being set. Single source of truth for both the client config
|
||||
and the learned-behaviour cache keys, which must agree on what "endpoint"
|
||||
means or a proxy ends up sharing api.openai.com's cache.
|
||||
"""
|
||||
return (
|
||||
self.base_url
|
||||
or self.api_base
|
||||
or os.getenv("OPENAI_BASE_URL")
|
||||
or os.getenv("OPENAI_API_BASE")
|
||||
or None
|
||||
)
|
||||
|
||||
def _model_cache_key(self, model: str | None = None) -> tuple[str, str]:
|
||||
"""Identity for the learned-behaviour caches: the endpoint plus the model.
|
||||
|
||||
The same model string can be served by api.openai.com and by an
|
||||
OpenAI-compatible proxy with different capabilities, so a conflict learned
|
||||
against one endpoint must not silently apply to the other.
|
||||
"""
|
||||
return (
|
||||
self._resolved_base_url() or "https://api.openai.com/v1",
|
||||
model if model is not None else self.model,
|
||||
)
|
||||
|
||||
def _is_recoverable_completion_error(self, error: BaseException) -> bool:
|
||||
"""Whether ``_call_completions`` will retry instead of surfacing this error.
|
||||
|
||||
Kept next to the retry logic so the handlers that suppress failure
|
||||
reporting and the handler that performs the retry can't drift apart.
|
||||
"""
|
||||
cause = error.__cause__ or error
|
||||
if self._is_tools_reasoning_effort_error(cause):
|
||||
return True
|
||||
return self._is_responses_only_error(cause) and not self.custom_openai
|
||||
|
||||
def _remember_reasoning_effort_conflict(self) -> None:
|
||||
"""Record that this model can't combine `reasoning_effort` with tools."""
|
||||
_LEARNED_TOOLS_REASONING_EFFORT_CONFLICTS.add(self._model_cache_key())
|
||||
|
||||
def _retry_params_without_reasoning_effort(
|
||||
self, params: dict[str, Any]
|
||||
) -> dict[str, Any] | None:
|
||||
"""Force `reasoning_effort="none"` for a retry, or None if already there.
|
||||
|
||||
Omitting the parameter is not enough. Some models (gpt-5.6-*) apply a
|
||||
server-side default, so a request with no `reasoning_effort` at all still
|
||||
fails the same way -- only an explicit "none" is accepted alongside tools.
|
||||
"""
|
||||
if params.get("reasoning_effort") == "none":
|
||||
return None
|
||||
retry_params = {**params, "reasoning_effort": "none"}
|
||||
logging.warning(
|
||||
'Retrying with reasoning_effort="none": %r rejects reasoning effort '
|
||||
'alongside function tools on /v1/chat/completions. Use api="responses" '
|
||||
"to keep reasoning effort with tools.",
|
||||
self.model,
|
||||
)
|
||||
return retry_params
|
||||
|
||||
def _effective_api(self) -> str:
|
||||
"""Which OpenAI API to actually use for this model.
|
||||
|
||||
@@ -1708,7 +1867,7 @@ class OpenAICompletion(BaseLLM):
|
||||
if (
|
||||
self.api == "completions"
|
||||
and not self.custom_openai
|
||||
and self.model in _LEARNED_RESPONSES_ONLY_MODELS
|
||||
and self._model_cache_key() in _LEARNED_RESPONSES_ONLY_MODELS
|
||||
):
|
||||
return "responses"
|
||||
return self.api
|
||||
@@ -1764,6 +1923,30 @@ class OpenAICompletion(BaseLLM):
|
||||
params["tools"] = self._convert_tools_for_interference(tools)
|
||||
params["tool_choice"] = "auto"
|
||||
|
||||
# Some models reject `reasoning_effort` on the completions endpoint as
|
||||
# soon as function tools are present. Once a 400 has taught us that about
|
||||
# this model, skip straight to "none" instead of repeating the doomed
|
||||
# request; the first occurrence is recovered in `_call_completions`.
|
||||
# Checked after the tools block on purpose: `reasoning_effort` can also
|
||||
# arrive through `additional_params`, which bypasses the typed field.
|
||||
# An absent `reasoning_effort` is not safe either: these models apply a
|
||||
# server-side default, so sending nothing is rejected exactly like sending
|
||||
# "high". Only an explicit "none" is accepted alongside tools.
|
||||
if (
|
||||
params.get("tools")
|
||||
and params.get("reasoning_effort") != "none"
|
||||
and not self._supports_reasoning_effort_with_tools(self.model)
|
||||
):
|
||||
requested = params.get("reasoning_effort")
|
||||
params["reasoning_effort"] = "none"
|
||||
logging.debug(
|
||||
'Forcing reasoning_effort="none" (requested %r) for model %r: '
|
||||
"function tools and reasoning effort cannot be combined on "
|
||||
'/v1/chat/completions. Use api="responses" to keep both.',
|
||||
requested,
|
||||
self.model,
|
||||
)
|
||||
|
||||
crewai_specific_params = {
|
||||
"callbacks",
|
||||
"available_functions",
|
||||
@@ -1942,6 +2125,12 @@ class OpenAICompletion(BaseLLM):
|
||||
params["messages"], content, from_agent
|
||||
)
|
||||
except NotFoundError as e:
|
||||
# `_call_completions` retries a responses-only model on the Responses
|
||||
# API. Reporting a failed call here would surface an error the user
|
||||
# never experiences, and the ValueError wrapper would also hide the
|
||||
# NotFoundError that the retry needs in order to recognize it.
|
||||
if self._is_recoverable_completion_error(e):
|
||||
raise
|
||||
error_msg = self._model_not_found_message(e)
|
||||
logging.error(error_msg)
|
||||
self._emit_call_failed_event(
|
||||
@@ -1960,6 +2149,12 @@ class OpenAICompletion(BaseLLM):
|
||||
logging.error(f"Context window exceeded: {e}")
|
||||
raise LLMContextLengthExceededError(str(e)) from e
|
||||
|
||||
# Recoverable by the caller: `_call_completions` retries these on the
|
||||
# Responses API or without `reasoning_effort`. Reporting a failed call
|
||||
# here would surface an error the user never actually experiences.
|
||||
if self._is_recoverable_completion_error(e):
|
||||
raise
|
||||
|
||||
error_msg = f"OpenAI API call failed: {e!s}"
|
||||
logging.error(error_msg)
|
||||
self._emit_call_failed_event(
|
||||
@@ -2365,6 +2560,12 @@ class OpenAICompletion(BaseLLM):
|
||||
if usage.get("total_tokens", 0) > 0:
|
||||
logging.info(f"OpenAI API usage: {usage}")
|
||||
except NotFoundError as e:
|
||||
# `_call_completions` retries a responses-only model on the Responses
|
||||
# API. Reporting a failed call here would surface an error the user
|
||||
# never experiences, and the ValueError wrapper would also hide the
|
||||
# NotFoundError that the retry needs in order to recognize it.
|
||||
if self._is_recoverable_completion_error(e):
|
||||
raise
|
||||
error_msg = self._model_not_found_message(e)
|
||||
logging.error(error_msg)
|
||||
self._emit_call_failed_event(
|
||||
@@ -2383,6 +2584,12 @@ class OpenAICompletion(BaseLLM):
|
||||
logging.error(f"Context window exceeded: {e}")
|
||||
raise LLMContextLengthExceededError(str(e)) from e
|
||||
|
||||
# Recoverable by the caller: `_call_completions` retries these on the
|
||||
# Responses API or without `reasoning_effort`. Reporting a failed call
|
||||
# here would surface an error the user never actually experiences.
|
||||
if self._is_recoverable_completion_error(e):
|
||||
raise
|
||||
|
||||
error_msg = f"OpenAI API call failed: {e!s}"
|
||||
logging.error(error_msg)
|
||||
self._emit_call_failed_event(
|
||||
|
||||
@@ -0,0 +1,340 @@
|
||||
"""Regression tests for models rejecting `reasoning_effort` alongside tools.
|
||||
|
||||
On /v1/chat/completions, gpt-5.4 and newer return a hard 400 when function tools
|
||||
and a non-"none" `reasoning_effort` are sent together:
|
||||
|
||||
Function tools with reasoning_effort are not supported for gpt-5.5 in
|
||||
/v1/chat/completions. To use function tools, use /v1/responses or set
|
||||
reasoning_effort to 'none'.
|
||||
|
||||
Measured against the live endpoint (2026-07):
|
||||
|
||||
model tools + "high" tools + "none"
|
||||
gpt-5.6[-sol/...] rejected accepted
|
||||
gpt-5.5 rejected accepted
|
||||
gpt-5.4[-mini/nano] rejected accepted
|
||||
gpt-5.2 accepted accepted
|
||||
o1 / o3 / o4-mini accepted rejected
|
||||
|
||||
Known families are dropped before the request as a fast path. Anything else is
|
||||
caught from the 400 and retried without the parameter, so a newly-restricted
|
||||
model doesn't need a release here. The Responses API accepts
|
||||
`reasoning: {"effort": ...}` with tools and is untouched.
|
||||
"""
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
from openai import BadRequestError
|
||||
|
||||
from crewai.llms.providers.openai import completion as completion_module
|
||||
from crewai.llms.providers.openai.completion import OpenAICompletion
|
||||
|
||||
|
||||
MESSAGES = [{"role": "user", "content": "hi"}]
|
||||
|
||||
# Pre-converted OpenAI tool schema — passed through by
|
||||
# _convert_tools_for_interference, keeping these tests focused on param handling.
|
||||
TOOLS = [
|
||||
{
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "web_search",
|
||||
"description": "Search the web",
|
||||
"parameters": {"type": "object", "properties": {}},
|
||||
},
|
||||
}
|
||||
]
|
||||
|
||||
|
||||
def build(model: str, **kwargs) -> OpenAICompletion:
|
||||
return OpenAICompletion(model=model, api_key="sk-test", **kwargs)
|
||||
|
||||
|
||||
def make_bad_request(
|
||||
model: str = "gpt-5.5", param: str = "reasoning_effort", message: str | None = None
|
||||
) -> BadRequestError:
|
||||
"""Build the 400 OpenAI returns for the tools + reasoning_effort combination."""
|
||||
if message is None:
|
||||
message = (
|
||||
f"Function tools with reasoning_effort are not supported for {model} "
|
||||
"in /v1/chat/completions. To use function tools, use /v1/responses "
|
||||
"or set reasoning_effort to 'none'."
|
||||
)
|
||||
body = {"error": {"message": message, "type": "invalid_request_error", "param": param}}
|
||||
response = httpx.Response(
|
||||
status_code=400,
|
||||
json=body,
|
||||
request=httpx.Request("POST", "https://api.openai.com/v1/chat/completions"),
|
||||
)
|
||||
return BadRequestError(message, response=response, body=body)
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _clear_learned_conflicts():
|
||||
"""Keep the process-wide learned set from leaking between tests."""
|
||||
completion_module._LEARNED_TOOLS_REASONING_EFFORT_CONFLICTS.clear()
|
||||
yield
|
||||
completion_module._LEARNED_TOOLS_REASONING_EFFORT_CONFLICTS.clear()
|
||||
|
||||
|
||||
class TestLearnedFastPath:
|
||||
"""Nothing is assumed up front; a 400 is what teaches us about a model."""
|
||||
|
||||
def test_sends_reasoning_effort_for_an_unknown_model(self):
|
||||
"""No model list, so the first request goes out as the caller asked.
|
||||
|
||||
The 400 it may come back with is recovered in `_call_completions`.
|
||||
"""
|
||||
llm = build("gpt-5.6", additional_params={"reasoning_effort": "high"})
|
||||
|
||||
params = llm._prepare_completion_params(MESSAGES, tools=TOOLS)
|
||||
|
||||
assert params["reasoning_effort"] == "high"
|
||||
|
||||
def test_forces_none_once_the_model_is_known(self):
|
||||
"""After a 400, skip the doomed request and go straight to "none".
|
||||
|
||||
Explicit "none" rather than removing the key: gpt-5.6-* apply a
|
||||
server-side default, so omitting it is rejected the same way.
|
||||
"""
|
||||
llm = build("gpt-5.6", additional_params={"reasoning_effort": "high"})
|
||||
llm._remember_reasoning_effort_conflict()
|
||||
|
||||
params = llm._prepare_completion_params(MESSAGES, tools=TOOLS)
|
||||
|
||||
assert "reasoning_effort" in params, "the key must be sent, not dropped"
|
||||
assert params["reasoning_effort"] == "none"
|
||||
# Tools must survive — they are the reason the call exists.
|
||||
assert params["tools"]
|
||||
assert params["tool_choice"] == "auto"
|
||||
|
||||
def test_absent_reasoning_effort_is_also_forced_to_none(self):
|
||||
"""Sending nothing is rejected exactly like sending "high".
|
||||
|
||||
These models apply a server-side default, so a learned model with no
|
||||
`reasoning_effort` set would 400 on every call and never benefit from the
|
||||
cache.
|
||||
"""
|
||||
llm = build("gpt-5.6-sol")
|
||||
llm._remember_reasoning_effort_conflict()
|
||||
|
||||
params = llm._prepare_completion_params(MESSAGES, tools=TOOLS)
|
||||
|
||||
assert params["reasoning_effort"] == "none"
|
||||
|
||||
def test_learning_applies_to_the_additional_params_leak(self):
|
||||
"""additional_params bypasses the typed field, so it must be checked too."""
|
||||
llm = build("gpt-5.5", additional_params={"reasoning_effort": "medium"})
|
||||
llm._remember_reasoning_effort_conflict()
|
||||
|
||||
params = llm._prepare_completion_params(MESSAGES, tools=TOOLS)
|
||||
|
||||
assert params["reasoning_effort"] == "none"
|
||||
|
||||
def test_keeps_reasoning_effort_without_tools(self):
|
||||
"""Without tools the combination is legal, so nothing should change."""
|
||||
llm = build("gpt-5.5", additional_params={"reasoning_effort": "high"})
|
||||
llm._remember_reasoning_effort_conflict()
|
||||
|
||||
params = llm._prepare_completion_params(MESSAGES, tools=None)
|
||||
|
||||
assert params["reasoning_effort"] == "high"
|
||||
|
||||
def test_learning_does_not_touch_other_models(self):
|
||||
"""A conflict is remembered per model, not globally."""
|
||||
build("gpt-5.5")._remember_reasoning_effort_conflict()
|
||||
other = build("gpt-5.2", additional_params={"reasoning_effort": "high"})
|
||||
|
||||
params = other._prepare_completion_params(MESSAGES, tools=TOOLS)
|
||||
|
||||
assert params["reasoning_effort"] == "high"
|
||||
|
||||
def test_fast_path_actually_has_something_to_drop(self):
|
||||
"""Guard against the assertions above passing vacuously.
|
||||
|
||||
The typed `reasoning_effort` field is gated behind `is_o1_model`, so it
|
||||
never reaches the payload for gpt-5.x. If a test set it that way there
|
||||
would be nothing to rewrite and the assertion would pass for the wrong
|
||||
reason — `additional_params` is the path that actually leaks.
|
||||
"""
|
||||
leaked = build(
|
||||
"gpt-5.5", additional_params={"reasoning_effort": "high"}
|
||||
)._prepare_completion_params(MESSAGES, tools=None)
|
||||
assert leaked["reasoning_effort"] == "high"
|
||||
|
||||
typed = build("gpt-5.5", reasoning_effort="high")._prepare_completion_params(
|
||||
MESSAGES, tools=None
|
||||
)
|
||||
assert "reasoning_effort" not in typed
|
||||
|
||||
|
||||
class TestErrorClassification:
|
||||
"""Only the specific tools + reasoning_effort 400 should trigger a retry."""
|
||||
|
||||
def test_matches_the_reported_error(self):
|
||||
assert OpenAICompletion._is_tools_reasoning_effort_error(make_bad_request())
|
||||
|
||||
def test_ignores_other_bad_requests(self):
|
||||
error = make_bad_request(
|
||||
param="max_tokens",
|
||||
message="Unsupported parameter: 'max_tokens' is not supported.",
|
||||
)
|
||||
assert not OpenAICompletion._is_tools_reasoning_effort_error(error)
|
||||
|
||||
def test_ignores_unsupported_effort_value(self):
|
||||
"""o1/o3 reject reasoning_effort='none'; that is a different failure."""
|
||||
error = make_bad_request(
|
||||
message="Unsupported value: 'reasoning_effort' does not support 'none'."
|
||||
)
|
||||
assert not OpenAICompletion._is_tools_reasoning_effort_error(error)
|
||||
|
||||
def test_ignores_unrelated_exceptions(self):
|
||||
assert not OpenAICompletion._is_tools_reasoning_effort_error(
|
||||
RuntimeError("boom")
|
||||
)
|
||||
|
||||
|
||||
class TestRuntimeRetry:
|
||||
"""Unmeasured models are recovered from the 400 rather than failing the run."""
|
||||
|
||||
def test_retries_without_reasoning_effort_and_succeeds(self, monkeypatch):
|
||||
# gpt-5.9 is deliberately not in the known-families tuple.
|
||||
llm = build("gpt-5.9", additional_params={"reasoning_effort": "high"})
|
||||
seen: list[dict] = []
|
||||
|
||||
def fake_handle(params, **kwargs):
|
||||
seen.append(params)
|
||||
if params.get("reasoning_effort") not in (None, "none"):
|
||||
raise make_bad_request(model="gpt-5.9")
|
||||
return "ok"
|
||||
|
||||
monkeypatch.setattr(llm, "_handle_completion", fake_handle)
|
||||
|
||||
result = llm._call_completions(MESSAGES, tools=TOOLS)
|
||||
|
||||
assert result == "ok"
|
||||
assert len(seen) == 2, "expected one failed call and one retry"
|
||||
assert seen[0]["reasoning_effort"] == "high"
|
||||
assert seen[1]["reasoning_effort"] == "none"
|
||||
# Tools have to survive the retry.
|
||||
assert seen[1]["tools"]
|
||||
|
||||
def test_remembers_the_conflict_for_later_calls(self, monkeypatch):
|
||||
llm = build("gpt-5.9", additional_params={"reasoning_effort": "high"})
|
||||
|
||||
def fake_handle(params, **kwargs):
|
||||
if params.get("reasoning_effort") not in (None, "none"):
|
||||
raise make_bad_request(model="gpt-5.9")
|
||||
return "ok"
|
||||
|
||||
monkeypatch.setattr(llm, "_handle_completion", fake_handle)
|
||||
llm._call_completions(MESSAGES, tools=TOOLS)
|
||||
|
||||
# Second call must strip up front instead of paying the 400 again.
|
||||
params = llm._prepare_completion_params(MESSAGES, tools=TOOLS)
|
||||
assert params["reasoning_effort"] == "none"
|
||||
assert not llm._supports_reasoning_effort_with_tools("gpt-5.9")
|
||||
|
||||
def test_does_not_retry_unrelated_bad_request(self, monkeypatch):
|
||||
llm = build("gpt-5.9", additional_params={"reasoning_effort": "high"})
|
||||
calls: list[dict] = []
|
||||
|
||||
def fake_handle(params, **kwargs):
|
||||
calls.append(params)
|
||||
raise make_bad_request(param="max_tokens", message="Unsupported parameter")
|
||||
|
||||
monkeypatch.setattr(llm, "_handle_completion", fake_handle)
|
||||
|
||||
with pytest.raises(BadRequestError):
|
||||
llm._call_completions(MESSAGES, tools=TOOLS)
|
||||
|
||||
assert len(calls) == 1, "must not retry errors it doesn't understand"
|
||||
|
||||
def test_learning_is_scoped_to_the_endpoint(self, monkeypatch):
|
||||
"""A conflict learned against one endpoint must not leak to another.
|
||||
|
||||
An OpenAI-compatible proxy may accept the combination that api.openai.com
|
||||
rejects, so stripping the parameter there would silently degrade it.
|
||||
"""
|
||||
real = build("gpt-5.9", additional_params={"reasoning_effort": "high"})
|
||||
proxy = build(
|
||||
"gpt-5.9",
|
||||
base_url="https://proxy.internal/v1",
|
||||
additional_params={"reasoning_effort": "high"},
|
||||
)
|
||||
|
||||
real._remember_reasoning_effort_conflict()
|
||||
|
||||
assert (
|
||||
real._prepare_completion_params(MESSAGES, tools=TOOLS)["reasoning_effort"]
|
||||
== "none"
|
||||
)
|
||||
assert (
|
||||
proxy._prepare_completion_params(MESSAGES, tools=TOOLS)[
|
||||
"reasoning_effort"
|
||||
]
|
||||
== "high"
|
||||
)
|
||||
|
||||
def test_recoverable_error_is_not_reported_as_a_failure(self):
|
||||
"""The retry must not surface a failure the user never experiences.
|
||||
|
||||
`_handle_completion` logs "OpenAI API call failed" and emits
|
||||
LLMCallFailedEvent for anything it catches. For an error the caller is about
|
||||
to recover from, that produces a user-visible error panel for a call that
|
||||
ultimately succeeds.
|
||||
"""
|
||||
llm = build("gpt-5.9")
|
||||
|
||||
assert llm._is_recoverable_completion_error(
|
||||
ValueError("wrapped").with_traceback(None)
|
||||
) is False
|
||||
|
||||
recoverable = ValueError("wrapped")
|
||||
recoverable.__cause__ = make_bad_request()
|
||||
assert llm._is_recoverable_completion_error(recoverable)
|
||||
|
||||
def test_does_not_retry_twice_when_already_none(self, monkeypatch):
|
||||
"""Once reasoning_effort is "none" there is nothing left to try."""
|
||||
llm = build("gpt-5.9", additional_params={"reasoning_effort": "none"})
|
||||
calls: list[dict] = []
|
||||
|
||||
def fake_handle(params, **kwargs):
|
||||
calls.append(params)
|
||||
raise make_bad_request(model="gpt-5.9")
|
||||
|
||||
monkeypatch.setattr(llm, "_handle_completion", fake_handle)
|
||||
|
||||
with pytest.raises(BadRequestError):
|
||||
llm._call_completions(MESSAGES, tools=TOOLS)
|
||||
|
||||
assert len(calls) == 1, "must not retry when already at 'none'"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_async_path_retries_too(self, monkeypatch):
|
||||
llm = build("gpt-5.9", additional_params={"reasoning_effort": "high"})
|
||||
seen: list[dict] = []
|
||||
|
||||
async def fake_handle(params, **kwargs):
|
||||
seen.append(params)
|
||||
if params.get("reasoning_effort") not in (None, "none"):
|
||||
raise make_bad_request(model="gpt-5.9")
|
||||
return "ok"
|
||||
|
||||
monkeypatch.setattr(llm, "_ahandle_completion", fake_handle)
|
||||
|
||||
result = await llm._acall_completions(MESSAGES, tools=TOOLS)
|
||||
|
||||
assert result == "ok"
|
||||
assert len(seen) == 2
|
||||
assert seen[1]["reasoning_effort"] == "none"
|
||||
|
||||
|
||||
class TestResponsesApiUntouched:
|
||||
def test_responses_api_keeps_effort_with_tools(self):
|
||||
"""The Responses API has no such restriction — it takes reasoning.effort."""
|
||||
llm = build("gpt-5.6", api="responses", reasoning_effort="high")
|
||||
|
||||
params = llm._prepare_responses_params(MESSAGES, tools=TOOLS)
|
||||
|
||||
assert params["reasoning"] == {"effort": "high"}
|
||||
@@ -14,6 +14,8 @@ retried on the Responses API, then the model is remembered so the wasted round t
|
||||
is paid once per process.
|
||||
"""
|
||||
|
||||
from types import SimpleNamespace
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
from openai import NotFoundError
|
||||
@@ -187,10 +189,103 @@ class TestEffectiveApi:
|
||||
llm = build(
|
||||
"gpt-5-pro", custom_openai=True, base_url="https://my-vllm.internal/v1"
|
||||
)
|
||||
completion_module._LEARNED_RESPONSES_ONLY_MODELS.add("gpt-5-pro")
|
||||
completion_module._LEARNED_RESPONSES_ONLY_MODELS.add(llm._model_cache_key())
|
||||
|
||||
assert llm._effective_api() == "completions"
|
||||
|
||||
def test_learning_is_scoped_to_the_endpoint(self):
|
||||
"""A conflict learned against one endpoint must not leak to another.
|
||||
|
||||
The same model name can be served by api.openai.com and by an
|
||||
OpenAI-compatible proxy with different capabilities.
|
||||
"""
|
||||
real = build("gpt-5-pro")
|
||||
proxy = build("gpt-5-pro", base_url="https://proxy.internal/v1")
|
||||
|
||||
real._remember_responses_only_model()
|
||||
|
||||
assert real._effective_api() == "responses"
|
||||
assert proxy._effective_api() == "completions"
|
||||
|
||||
|
||||
class TestEndpointResolution:
|
||||
"""The cache key must agree with the endpoint the client actually uses."""
|
||||
|
||||
def test_env_configured_proxy_gets_its_own_key(self, monkeypatch):
|
||||
"""An instance can be pointed at a proxy purely through the environment.
|
||||
|
||||
`_get_client_params` honours OPENAI_BASE_URL, so a key that stopped at the
|
||||
base_url/api_base fields would file a proxy under api.openai.com -- the
|
||||
exact leakage the tuple key exists to prevent.
|
||||
"""
|
||||
monkeypatch.setenv("OPENAI_BASE_URL", "https://proxy.internal/v1")
|
||||
llm = build("gpt-5-pro")
|
||||
|
||||
assert llm._model_cache_key() == ("https://proxy.internal/v1", "gpt-5-pro")
|
||||
assert llm._get_client_params()["base_url"] == "https://proxy.internal/v1"
|
||||
|
||||
def test_explicit_base_url_wins_over_env(self, monkeypatch):
|
||||
monkeypatch.setenv("OPENAI_BASE_URL", "https://proxy.internal/v1")
|
||||
llm = build("gpt-5-pro", base_url="https://explicit.internal/v1")
|
||||
|
||||
assert llm._model_cache_key()[0] == "https://explicit.internal/v1"
|
||||
|
||||
def test_defaults_to_openai(self, monkeypatch):
|
||||
monkeypatch.delenv("OPENAI_BASE_URL", raising=False)
|
||||
monkeypatch.delenv("OPENAI_API_BASE", raising=False)
|
||||
|
||||
assert build("gpt-5-pro")._model_cache_key() == (
|
||||
"https://api.openai.com/v1",
|
||||
"gpt-5-pro",
|
||||
)
|
||||
|
||||
def test_env_proxy_does_not_inherit_openai_learning(self, monkeypatch):
|
||||
monkeypatch.delenv("OPENAI_BASE_URL", raising=False)
|
||||
monkeypatch.delenv("OPENAI_API_BASE", raising=False)
|
||||
real = build("gpt-5-pro")
|
||||
real._remember_responses_only_model()
|
||||
assert real._effective_api() == "responses"
|
||||
|
||||
# Same model string, different endpoint -> must not inherit the learning.
|
||||
monkeypatch.setenv("OPENAI_BASE_URL", "https://proxy.internal/v1")
|
||||
assert build("gpt-5-pro")._effective_api() == "completions"
|
||||
|
||||
|
||||
class TestFailureReportingSuppressed:
|
||||
"""A recovered 404 must not surface as a failed call."""
|
||||
|
||||
def test_not_found_handler_defers_to_the_retry(self, monkeypatch):
|
||||
"""NotFoundError is caught before the generic handler.
|
||||
|
||||
Without a guard there it logs, emits LLMCallFailedEvent, and re-raises as
|
||||
ValueError -- which both reports a failure the user never sees and hides
|
||||
the NotFoundError the retry needs to recognize.
|
||||
"""
|
||||
llm = build("gpt-5-pro")
|
||||
emitted: list[str] = []
|
||||
monkeypatch.setattr(
|
||||
llm,
|
||||
"_emit_call_failed_event",
|
||||
lambda **kwargs: emitted.append(kwargs.get("error", "")),
|
||||
)
|
||||
error = make_not_found(RESPONSES_ONLY_MESSAGES[0])
|
||||
|
||||
class FakeCompletions:
|
||||
def create(self, **kwargs):
|
||||
raise error
|
||||
|
||||
monkeypatch.setattr(
|
||||
llm,
|
||||
"_get_sync_client",
|
||||
lambda: SimpleNamespace(chat=SimpleNamespace(completions=FakeCompletions())),
|
||||
)
|
||||
|
||||
# The original NotFoundError must survive, not a ValueError wrapper.
|
||||
with pytest.raises(NotFoundError):
|
||||
llm._handle_completion({"model": "gpt-5-pro", "messages": MESSAGES})
|
||||
|
||||
assert emitted == []
|
||||
|
||||
|
||||
class TestNotFoundMessage:
|
||||
@pytest.mark.parametrize("message", RESPONSES_ONLY_MESSAGES)
|
||||
|
||||
Reference in New Issue
Block a user