From 7baf8f9ba1a03e34ec360b38ae62ef33d245e708 Mon Sep 17 00:00:00 2001 From: Lorenze Jay <63378463+lorenzejay@users.noreply.github.com> Date: Thu, 9 Jul 2026 15:30:16 -0700 Subject: [PATCH] improving custom OpenAI urls (#6490) * Support legacy OpenAI base URL env var * Add custom OpenAI-compatible endpoint support * Refactor OpenAI completion module test to restore original module state - Added logic to save and restore the original OpenAI completion module during the test to prevent issues with class re-imports affecting subsequent tests. - Ensured that the test checks for the presence of the module and its attributes only after the module is properly reloaded. - Improved test reliability by avoiding potential failures due to module state changes across tests. * addressing comments --- docs/edge/en/concepts/llms.mdx | 12 ++ docs/edge/en/learn/litellm-removal-guide.mdx | 97 +++++++++- lib/crewai/src/crewai/llm.py | 54 +++++- .../llms/providers/openai/completion.py | 25 +++ lib/crewai/tests/llms/openai/test_openai.py | 168 +++++++++++++++++- 5 files changed, 341 insertions(+), 15 deletions(-) diff --git a/docs/edge/en/concepts/llms.mdx b/docs/edge/en/concepts/llms.mdx index 85921b6ea..3d6e52ad7 100644 --- a/docs/edge/en/concepts/llms.mdx +++ b/docs/edge/en/concepts/llms.mdx @@ -144,6 +144,18 @@ In this section, you'll find detailed examples that help you select, configure, ) ``` + **Custom OpenAI-Compatible Endpoint:** + ```python Code + from crewai import LLM + + llm = LLM( + model="anthropic/claude-sonnet-4-6", + custom_openai=True, + base_url="https://your-gateway.example.com/v1", + api_key="your-gateway-api-key", + ) + ``` + **Advanced Configuration:** ```python Code from crewai import LLM diff --git a/docs/edge/en/learn/litellm-removal-guide.mdx b/docs/edge/en/learn/litellm-removal-guide.mdx index 4580f7b32..a4fe1de1e 100644 --- a/docs/edge/en/learn/litellm-removal-guide.mdx +++ b/docs/edge/en/learn/litellm-removal-guide.mdx @@ -240,14 +240,15 @@ from crewai import LLM # After (OpenAI-compatible mode, no LiteLLM needed): llm = LLM( - model="openai/llama3", + model="llama3", + custom_openai=True, base_url="http://localhost:11434/v1", api_key="ollama" # Ollama doesn't require a real API key ) ``` - Many local inference servers (Ollama, vLLM, LM Studio, llama.cpp) expose an OpenAI-compatible API. You can use the `openai/` prefix with a custom `base_url` to connect to any of them natively. + Many local inference servers (Ollama, vLLM, LM Studio, llama.cpp) expose an OpenAI-compatible API. You can use `custom_openai=True` with a custom `base_url` to connect to any of them natively while keeping the model ID your gateway expects. ### Step 4: Update your YAML configs @@ -295,6 +296,92 @@ crewai run uv run pytest ``` +## Custom OpenAI-Compatible Endpoints + +Many providers and local servers (Ollama, vLLM, LM Studio, llama.cpp, LiteLLM proxies, and hosted gateways) expose an **OpenAI-compatible** API. Instead of routing these through LiteLLM, you can talk to them directly with CrewAI's native OpenAI integration by setting `custom_openai=True`. + +This is the recommended replacement for any LiteLLM provider that offers an OpenAI-compatible endpoint. + +### How it works + +- `custom_openai=True` forces CrewAI to use the native OpenAI SDK, regardless of the model name. +- The model ID is passed to the endpoint without validation against OpenAI's known-model list. This lets you use arbitrary model IDs your gateway expects (for example, `anthropic/claude-sonnet-4-6` served behind an OpenAI-compatible proxy). An optional leading `openai/` routing prefix is stripped. +- A base URL is **required**. CrewAI resolves it, in order, from: + 1. `base_url=...` + 2. `api_base=...` + 3. `OPENAI_BASE_URL` environment variable + 4. `OPENAI_API_BASE` environment variable (legacy) + + If none are set, CrewAI raises a `ValueError` so misconfiguration fails fast instead of silently hitting `api.openai.com`. + +```python +from crewai import LLM + +llm = LLM( + model="anthropic/claude-sonnet-4-6", # passed through as-is + custom_openai=True, + base_url="https://your-gateway.example/v1", + api_key="your-key", +) +``` + +### Connect to common servers + + + + ```python + from crewai import LLM + + llm = LLM( + model="llama3.2:latest", + custom_openai=True, + base_url="http://localhost:11434/v1", + api_key="ollama", # Ollama ignores it, but the client requires a value + ) + ``` + + + ```python + from crewai import LLM + + llm = LLM( + model="meta-llama/Meta-Llama-3.1-8B-Instruct", + custom_openai=True, + base_url="http://localhost:8000/v1", + api_key="not-needed", + ) + ``` + + + ```python + from crewai import LLM + + llm = LLM( + model="your-loaded-model", + custom_openai=True, + base_url="http://localhost:1234/v1", + api_key="lm-studio", + ) + ``` + + + ```bash + export OPENAI_BASE_URL="https://your-gateway.example/v1" + export OPENAI_API_KEY="your-key" + ``` + ```python + from crewai import LLM + + # base_url is picked up from OPENAI_BASE_URL / OPENAI_API_BASE + llm = LLM(model="anthropic/claude-sonnet-4-6", custom_openai=True) + ``` + + + + + If you use the `openai/` prefix with a model that isn't a known OpenAI model and pass `base_url` or `api_base` directly, CrewAI automatically treats it as a custom OpenAI-compatible endpoint. Environment variables alone do not enable automatic routing for unknown models; set `custom_openai=True` when configuring the endpoint through `OPENAI_BASE_URL` or `OPENAI_API_BASE`. + + ## Quick Reference: Model String Mapping Here are common migration paths from LiteLLM-dependent providers to native ones: @@ -321,7 +408,8 @@ llm = LLM(model="anthropic/claude-sonnet-4-20250514") # High quality # Ollama → OpenAI-compatible (keep using local models) # llm = LLM(model="ollama/llama3") llm = LLM( - model="openai/llama3", + model="llama3", + custom_openai=True, base_url="http://localhost:11434/v1", api_key="ollama" ) @@ -349,6 +437,9 @@ llm = LLM( Native providers use the same environment variables you're already familiar with. No changes needed for `OPENAI_API_KEY`, `ANTHROPIC_API_KEY`, `GEMINI_API_KEY`, etc. + + Most of these providers expose an OpenAI-compatible API. Use `custom_openai=True` with their base URL and API key — see [Custom OpenAI-Compatible Endpoints](#custom-openai-compatible-endpoints). For example, Groq: `LLM(model="llama-3.1-70b-versatile", custom_openai=True, base_url="https://api.groq.com/openai/v1", api_key="...")`. The model ID is passed through untouched, so use whatever ID the provider expects. + ## Related Resources diff --git a/lib/crewai/src/crewai/llm.py b/lib/crewai/src/crewai/llm.py index 5db0d300b..b0b5cd3a1 100644 --- a/lib/crewai/src/crewai/llm.py +++ b/lib/crewai/src/crewai/llm.py @@ -394,19 +394,35 @@ class LLM(BaseLLM): """Factory method that routes to native SDK or falls back to LiteLLM. Routing priority: - 1. If 'provider' kwarg is present, use that provider with constants - 2. If only 'model' kwarg, use constants to infer provider - 3. If "/" in model name: + 1. If ``custom_openai=True``, force the native OpenAI provider, + overriding any explicit provider. A custom endpoint is required. + 2. If ``provider`` is present, use that provider. + 3. If "/" is in the model name: - Check if prefix is a native provider (openai/anthropic/azure/bedrock/gemini) - If yes, validate model against constants - If valid, route to native SDK; otherwise route to LiteLLM + 4. Otherwise, infer the provider from the model name. """ if not model or not isinstance(model, str): raise ValueError("Model must be a non-empty string") + custom_openai = bool(kwargs.pop("custom_openai", False)) + custom_openai_route = custom_openai explicit_provider = kwargs.get("provider") - if explicit_provider: + if custom_openai: + if not cls._has_custom_openai_endpoint(kwargs): + raise ValueError( + "custom_openai=True requires base_url, api_base, " + "OPENAI_BASE_URL, or OPENAI_API_BASE" + ) + provider = "openai" + use_native = True + prefix, separator, model_part = model.partition("/") + model_string = ( + model_part if separator and prefix.lower() == "openai" else model + ) + elif explicit_provider: provider = explicit_provider use_native = True model_string = model @@ -435,9 +451,17 @@ class LLM(BaseLLM): canonical_provider = provider_mapping.get(prefix.lower()) - if canonical_provider and cls._validate_model_in_constants( - model_part, canonical_provider - ): + valid_native_model = bool( + canonical_provider + and cls._validate_model_in_constants(model_part, canonical_provider) + ) + custom_openai_route = bool( + canonical_provider == "openai" + and not valid_native_model + and cls._has_custom_openai_base_url(kwargs) + ) + + if canonical_provider and (valid_native_model or custom_openai_route): provider = canonical_provider use_native = True model_string = model_part @@ -455,6 +479,8 @@ class LLM(BaseLLM): try: # Remove 'provider' from kwargs if it exists to avoid duplicate keyword argument kwargs_copy = {k: v for k, v in kwargs.items() if k != "provider"} + if custom_openai_route: + kwargs_copy["custom_openai"] = True return cast( Self, native_class(model=model_string, provider=provider, **kwargs_copy), @@ -590,6 +616,20 @@ class LLM(BaseLLM): return cls._matches_provider_pattern(model, provider) + @staticmethod + def _has_custom_openai_base_url(kwargs: dict[str, Any]) -> bool: + """Return whether this call explicitly configures a custom endpoint.""" + return bool(kwargs.get("base_url") or kwargs.get("api_base")) + + @classmethod + def _has_custom_openai_endpoint(cls, kwargs: dict[str, Any]) -> bool: + """Return whether a custom endpoint is configured explicitly or by env.""" + return bool( + cls._has_custom_openai_base_url(kwargs) + or os.getenv("OPENAI_BASE_URL") + or os.getenv("OPENAI_API_BASE") + ) + @classmethod def _infer_provider_from_model(cls, model: str) -> str: """Infer the provider from the model name. diff --git a/lib/crewai/src/crewai/llms/providers/openai/completion.py b/lib/crewai/src/crewai/llms/providers/openai/completion.py index 77d2bbbdd..78e8d23e8 100644 --- a/lib/crewai/src/crewai/llms/providers/openai/completion.py +++ b/lib/crewai/src/crewai/llms/providers/openai/completion.py @@ -232,6 +232,7 @@ class OpenAICompletion(BaseLLM): auto_chain: bool = False auto_chain_reasoning: bool = False api_base: str | None = None + custom_openai: bool = False is_o1_model: bool = False is_gpt4_model: bool = False @@ -245,6 +246,20 @@ class OpenAICompletion(BaseLLM): def _normalize_openai_fields(cls, data: Any) -> Any: if not isinstance(data, dict): return data + if data.get("custom_openai"): + custom_base_url = ( + data.get("base_url") + or data.get("api_base") + or os.getenv("OPENAI_BASE_URL") + or os.getenv("OPENAI_API_BASE") + ) + if not custom_base_url: + raise ValueError( + "custom_openai=True requires base_url, api_base, " + "OPENAI_BASE_URL, or OPENAI_API_BASE" + ) + if not data.get("base_url") and not data.get("api_base"): + data["base_url"] = custom_base_url if not data.get("provider"): data["provider"] = "openai" data["api_key"] = data.get("api_key") or os.getenv("OPENAI_API_KEY") @@ -355,6 +370,15 @@ class OpenAICompletion(BaseLLM): config["seed"] = self.seed if self.reasoning_effort is not None: config["reasoning_effort"] = self.reasoning_effort + if self.custom_openai: + config["model"] = self.model + config["custom_openai"] = True + config["base_url"] = ( + self.base_url + or self.api_base + or os.getenv("OPENAI_BASE_URL") + or os.getenv("OPENAI_API_BASE") + ) return config def _get_client_params(self) -> dict[str, Any]: @@ -372,6 +396,7 @@ class OpenAICompletion(BaseLLM): "base_url": self.base_url or self.api_base or os.getenv("OPENAI_BASE_URL") + or os.getenv("OPENAI_API_BASE") or None, "timeout": self.timeout, "max_retries": self.max_retries, diff --git a/lib/crewai/tests/llms/openai/test_openai.py b/lib/crewai/tests/llms/openai/test_openai.py index 836abe838..d5bc797d8 100644 --- a/lib/crewai/tests/llms/openai/test_openai.py +++ b/lib/crewai/tests/llms/openai/test_openai.py @@ -30,10 +30,156 @@ def test_openai_completion_is_used_when_no_provider_prefix(): llm = LLM(model="gpt-4o") from crewai.llms.providers.openai.completion import OpenAICompletion - assert isinstance(llm, OpenAICompletion) + assert llm.__class__.__name__ == "OpenAICompletion" assert llm.provider == "openai" assert llm.model == "gpt-4o" + +def test_custom_openai_flag_uses_native_openai_without_provider_prefix(): + """Custom OpenAI-compatible endpoints can serve arbitrary model ids.""" + with patch.dict(os.environ, {"OPENAI_API_KEY": "test-key"}, clear=False): + llm = LLM( + model="anthropic/claude-sonnet-4-6", + custom_openai=True, + base_url="https://gateway.example/v1", + is_litellm=False, + ) + + assert llm.__class__.__name__ == "OpenAICompletion" + assert llm.is_litellm is False + assert llm.provider == "openai" + assert llm.model == "anthropic/claude-sonnet-4-6" + assert llm.base_url == "https://gateway.example/v1" + assert llm.custom_openai is True + assert "custom_openai" not in llm.additional_params + + config = llm.to_config_dict() + assert config["model"] == "anthropic/claude-sonnet-4-6" + assert config["custom_openai"] is True + assert config["base_url"] == "https://gateway.example/v1" + + rebuilt = LLM(**config) + assert isinstance(rebuilt, OpenAICompletion) + assert rebuilt.model == "anthropic/claude-sonnet-4-6" + assert rebuilt.base_url == "https://gateway.example/v1" + + +def test_custom_openai_flag_requires_custom_base_url(): + """Avoid routing arbitrary custom model ids to api.openai.com by mistake.""" + with patch.dict(os.environ, {"OPENAI_API_KEY": "test-key"}, clear=True): + with pytest.raises(ValueError, match="custom_openai=True requires"): + LLM( + model="anthropic/claude-sonnet-4-6", + custom_openai=True, + is_litellm=False, + ) + + +def test_direct_custom_openai_completion_requires_custom_base_url(): + """Direct construction must not silently fall back to api.openai.com.""" + with patch.dict(os.environ, {"OPENAI_API_KEY": "test-key"}, clear=True): + with pytest.raises(ValueError, match="custom_openai=True requires"): + OpenAICompletion( + model="anthropic/claude-sonnet-4-6", + custom_openai=True, + ) + + +def test_custom_openai_flag_strips_openai_routing_prefix(): + """The openai/ routing prefix is not part of the gateway's model id.""" + with patch.dict(os.environ, {"OPENAI_API_KEY": "test-key"}, clear=False): + llm = LLM( + model="openai/anthropic/claude-sonnet-4-6", + custom_openai=True, + base_url="https://gateway.example/v1", + is_litellm=False, + ) + + assert isinstance(llm, OpenAICompletion) + assert llm.model == "anthropic/claude-sonnet-4-6" + + +def test_openai_prefixed_custom_endpoint_uses_native_sdk_for_nested_model_id(): + """Custom OpenAI-compatible endpoints may serve non-OpenAI model ids.""" + with patch.dict(os.environ, {"OPENAI_API_KEY": "test-key"}, clear=False): + llm = LLM( + model="openai/anthropic/claude-sonnet-4-6", + base_url="https://gateway.example/v1", + is_litellm=False, + ) + + assert llm.__class__.__name__ == "OpenAICompletion" + assert llm.is_litellm is False + assert llm.provider == "openai" + assert llm.model == "anthropic/claude-sonnet-4-6" + assert llm.custom_openai is True + assert llm.base_url == "https://gateway.example/v1" + +def test_explicit_custom_openai_uses_legacy_api_base_env_var(): + """Explicit custom routing supports the legacy endpoint environment variable.""" + with patch.dict( + os.environ, + { + "OPENAI_API_KEY": "test-key", + "OPENAI_API_BASE": "https://gateway.example/v1", + }, + clear=False, + ): + os.environ.pop("OPENAI_BASE_URL", None) + llm = LLM( + model="openai/anthropic/claude-sonnet-4-6", + custom_openai=True, + is_litellm=False, + ) + + assert isinstance(llm, OpenAICompletion) + assert llm.is_litellm is False + assert llm.provider == "openai" + assert llm.model == "anthropic/claude-sonnet-4-6" + assert llm.custom_openai is True + + +def test_openai_prefixed_unknown_model_ignores_ambient_base_url_for_routing(): + """Ambient OpenAI configuration must not opt unknown models into native routing.""" + with patch.dict( + os.environ, + { + "OPENAI_API_KEY": "test-key", + "OPENAI_BASE_URL": "https://gateway.example/v1", + }, + clear=True, + ): + with ( + patch("crewai.llm._ensure_litellm", return_value=False), + pytest.raises(ImportError, match="LiteLLM fallback package"), + ): + LLM(model="openai/not-a-real-openai-model") + + +@pytest.mark.parametrize("endpoint_field", ["api_base", "env"]) +def test_custom_openai_config_preserves_resolved_endpoint(endpoint_field): + """Serialized custom OpenAI configs can reconstruct the same endpoint.""" + kwargs = {} + env = {"OPENAI_API_KEY": "test-key"} + if endpoint_field == "api_base": + kwargs["api_base"] = "https://gateway.example/v1" + else: + env["OPENAI_API_BASE"] = "https://gateway.example/v1" + + with patch.dict(os.environ, env, clear=True): + llm = LLM( + model="anthropic/claude-sonnet-4-6", + custom_openai=True, + **kwargs, + ) + + config = llm.to_config_dict() + assert config["base_url"] == "https://gateway.example/v1" + rebuilt = LLM(**config) + assert isinstance(rebuilt, OpenAICompletion) + assert rebuilt.base_url == "https://gateway.example/v1" + + @pytest.mark.vcr() def test_openai_is_default_provider_without_explicit_llm_set_on_agent(): """ @@ -60,14 +206,13 @@ def test_openai_is_default_provider_without_explicit_llm_set_on_agent(): -def test_openai_completion_module_is_imported(): +def test_openai_completion_module_is_imported(monkeypatch): """ Test that the completion module is properly imported when using OpenAI provider """ module_name = "crewai.llms.providers.openai.completion" - if module_name in sys.modules: - del sys.modules[module_name] + monkeypatch.delitem(sys.modules, module_name, raising=False) LLM(model="gpt-4o") @@ -421,12 +566,25 @@ def test_openai_get_client_params_with_env_var(): client_params = llm._get_client_params() assert client_params["base_url"] == "https://env.openai.com/v1" +def test_openai_get_client_params_with_legacy_api_base_env_var(): + """ + Test that _get_client_params uses OPENAI_API_BASE when OPENAI_BASE_URL is absent. + """ + with patch.dict(os.environ, { + "OPENAI_API_BASE": "https://legacy-env.openai.com/v1", + }, clear=False): + os.environ.pop("OPENAI_BASE_URL", None) + llm = OpenAICompletion(model="gpt-4o") + client_params = llm._get_client_params() + assert client_params["base_url"] == "https://legacy-env.openai.com/v1" + def test_openai_get_client_params_priority_order(): """ - Test the priority order: base_url > api_base > OPENAI_BASE_URL env var + Test the priority order: base_url > api_base > OPENAI_BASE_URL > OPENAI_API_BASE """ with patch.dict(os.environ, { "OPENAI_BASE_URL": "https://env.openai.com/v1", + "OPENAI_API_BASE": "https://legacy-env.openai.com/v1", }): llm1 = OpenAICompletion( model="gpt-4o",