From 8307d4ba14277cfe9e164974475ed400b208ce68 Mon Sep 17 00:00:00 2001 From: lorenzejay Date: Wed, 8 Jul 2026 16:09:42 -0700 Subject: [PATCH] Add custom OpenAI-compatible endpoint support --- docs/edge/en/concepts/llms.mdx | 12 ++++++ docs/edge/en/learn/litellm-removal-guide.mdx | 5 ++- lib/crewai/src/crewai/llm.py | 33 +++++++++++---- .../llms/providers/openai/completion.py | 4 ++ lib/crewai/tests/llms/openai/test_openai.py | 42 ++++++++++++++++++- 5 files changed, 84 insertions(+), 12 deletions(-) diff --git a/docs/edge/en/concepts/llms.mdx b/docs/edge/en/concepts/llms.mdx index 85921b6ea..3d6e52ad7 100644 --- a/docs/edge/en/concepts/llms.mdx +++ b/docs/edge/en/concepts/llms.mdx @@ -144,6 +144,18 @@ In this section, you'll find detailed examples that help you select, configure, ) ``` + **Custom OpenAI-Compatible Endpoint:** + ```python Code + from crewai import LLM + + llm = LLM( + model="anthropic/claude-sonnet-4-6", + custom_openai=True, + base_url="https://your-gateway.example.com/v1", + api_key="your-gateway-api-key", + ) + ``` + **Advanced Configuration:** ```python Code from crewai import LLM diff --git a/docs/edge/en/learn/litellm-removal-guide.mdx b/docs/edge/en/learn/litellm-removal-guide.mdx index 4580f7b32..c1f0a498e 100644 --- a/docs/edge/en/learn/litellm-removal-guide.mdx +++ b/docs/edge/en/learn/litellm-removal-guide.mdx @@ -240,14 +240,15 @@ from crewai import LLM # After (OpenAI-compatible mode, no LiteLLM needed): llm = LLM( - model="openai/llama3", + model="llama3", + custom_openai=True, base_url="http://localhost:11434/v1", api_key="ollama" # Ollama doesn't require a real API key ) ``` - Many local inference servers (Ollama, vLLM, LM Studio, llama.cpp) expose an OpenAI-compatible API. You can use the `openai/` prefix with a custom `base_url` to connect to any of them natively. + Many local inference servers (Ollama, vLLM, LM Studio, llama.cpp) expose an OpenAI-compatible API. You can use `custom_openai=True` with a custom `base_url` to connect to any of them natively while keeping the model ID your gateway expects. ### Step 4: Update your YAML configs diff --git a/lib/crewai/src/crewai/llm.py b/lib/crewai/src/crewai/llm.py index 9a90d7690..014ed2f9c 100644 --- a/lib/crewai/src/crewai/llm.py +++ b/lib/crewai/src/crewai/llm.py @@ -404,9 +404,20 @@ class LLM(BaseLLM): if not model or not isinstance(model, str): raise ValueError("Model must be a non-empty string") + custom_openai = bool(kwargs.pop("custom_openai", False)) + custom_openai_route = custom_openai explicit_provider = kwargs.get("provider") - if explicit_provider: + if custom_openai: + if not cls._has_custom_openai_base_url(kwargs): + raise ValueError( + "custom_openai=True requires base_url, api_base, " + "OPENAI_BASE_URL, or OPENAI_API_BASE" + ) + provider = "openai" + use_native = True + model_string = model + elif explicit_provider: provider = explicit_provider use_native = True model_string = model @@ -435,13 +446,17 @@ class LLM(BaseLLM): canonical_provider = provider_mapping.get(prefix.lower()) - if canonical_provider and ( - cls._validate_model_in_constants(model_part, canonical_provider) - or ( - canonical_provider == "openai" - and cls._has_custom_openai_base_url(kwargs) - ) - ): + valid_native_model = bool( + canonical_provider + and cls._validate_model_in_constants(model_part, canonical_provider) + ) + custom_openai_route = bool( + canonical_provider == "openai" + and not valid_native_model + and cls._has_custom_openai_base_url(kwargs) + ) + + if canonical_provider and (valid_native_model or custom_openai_route): provider = canonical_provider use_native = True model_string = model_part @@ -459,6 +474,8 @@ class LLM(BaseLLM): try: # Remove 'provider' from kwargs if it exists to avoid duplicate keyword argument kwargs_copy = {k: v for k, v in kwargs.items() if k != "provider"} + if custom_openai_route: + kwargs_copy["custom_openai"] = True return cast( Self, native_class(model=model_string, provider=provider, **kwargs_copy), diff --git a/lib/crewai/src/crewai/llms/providers/openai/completion.py b/lib/crewai/src/crewai/llms/providers/openai/completion.py index 164a43453..a3dea99e6 100644 --- a/lib/crewai/src/crewai/llms/providers/openai/completion.py +++ b/lib/crewai/src/crewai/llms/providers/openai/completion.py @@ -232,6 +232,7 @@ class OpenAICompletion(BaseLLM): auto_chain: bool = False auto_chain_reasoning: bool = False api_base: str | None = None + custom_openai: bool = False is_o1_model: bool = False is_gpt4_model: bool = False @@ -355,6 +356,9 @@ class OpenAICompletion(BaseLLM): config["seed"] = self.seed if self.reasoning_effort is not None: config["reasoning_effort"] = self.reasoning_effort + if self.custom_openai: + config["model"] = self.model + config["custom_openai"] = True return config def _get_client_params(self) -> dict[str, Any]: diff --git a/lib/crewai/tests/llms/openai/test_openai.py b/lib/crewai/tests/llms/openai/test_openai.py index 415aaf1a2..fd1b48e39 100644 --- a/lib/crewai/tests/llms/openai/test_openai.py +++ b/lib/crewai/tests/llms/openai/test_openai.py @@ -30,10 +30,46 @@ def test_openai_completion_is_used_when_no_provider_prefix(): llm = LLM(model="gpt-4o") from crewai.llms.providers.openai.completion import OpenAICompletion - assert isinstance(llm, OpenAICompletion) + assert llm.__class__.__name__ == "OpenAICompletion" assert llm.provider == "openai" assert llm.model == "gpt-4o" + +def test_custom_openai_flag_uses_native_openai_without_provider_prefix(): + """Custom OpenAI-compatible endpoints can serve arbitrary model ids.""" + with patch.dict(os.environ, {"OPENAI_API_KEY": "test-key"}, clear=False): + llm = LLM( + model="anthropic/claude-sonnet-4-6", + custom_openai=True, + base_url="https://asimov.example/v1", + is_litellm=False, + ) + + assert llm.__class__.__name__ == "OpenAICompletion" + assert llm.is_litellm is False + assert llm.provider == "openai" + assert llm.model == "anthropic/claude-sonnet-4-6" + assert llm.base_url == "https://asimov.example/v1" + assert llm.custom_openai is True + assert "custom_openai" not in llm.additional_params + + config = llm.to_config_dict() + assert config["model"] == "anthropic/claude-sonnet-4-6" + assert config["custom_openai"] is True + assert config["base_url"] == "https://asimov.example/v1" + + +def test_custom_openai_flag_requires_custom_base_url(): + """Avoid routing arbitrary custom model ids to api.openai.com by mistake.""" + with patch.dict(os.environ, {"OPENAI_API_KEY": "test-key"}, clear=True): + with pytest.raises(ValueError, match="custom_openai=True requires"): + LLM( + model="anthropic/claude-sonnet-4-6", + custom_openai=True, + is_litellm=False, + ) + + def test_openai_prefixed_custom_endpoint_uses_native_sdk_for_nested_model_id(): """Custom OpenAI-compatible endpoints may serve non-OpenAI model ids.""" with patch.dict(os.environ, {"OPENAI_API_KEY": "test-key"}, clear=False): @@ -43,10 +79,11 @@ def test_openai_prefixed_custom_endpoint_uses_native_sdk_for_nested_model_id(): is_litellm=False, ) - assert isinstance(llm, OpenAICompletion) + assert llm.__class__.__name__ == "OpenAICompletion" assert llm.is_litellm is False assert llm.provider == "openai" assert llm.model == "anthropic/claude-sonnet-4-6" + assert llm.custom_openai is True assert llm.base_url == "https://asimov.example/v1" def test_openai_prefixed_custom_endpoint_uses_legacy_api_base_env_var(): @@ -69,6 +106,7 @@ def test_openai_prefixed_custom_endpoint_uses_legacy_api_base_env_var(): assert llm.is_litellm is False assert llm.provider == "openai" assert llm.model == "anthropic/claude-sonnet-4-6" + assert llm.custom_openai is True @pytest.mark.vcr() def test_openai_is_default_provider_without_explicit_llm_set_on_agent():