fix: detect Azure OpenAI models by endpoint, not just deployment name

Fixes #4478. The is_openai_model flag was only set based on model name
prefixes (gpt-, o1-, text-), but Azure deployment names can be anything
(e.g. gpt5nano). Now is_openai_model is also True when the endpoint is
an Azure OpenAI endpoint (openai.azure.com/openai/deployments/), ensuring
response_model, tool calling, and structured output work regardless of
the deployment name.

Co-Authored-By: João <joao@crewai.com>
This commit is contained in:
Devin AI
2026-02-13 12:31:38 +00:00
parent f7e3b4dbe0
commit 33ed0a2ad5
2 changed files with 221 additions and 4 deletions

View File

@@ -171,15 +171,16 @@ class AzureCompletion(BaseLLM):
self.stream = stream
self.response_format = response_format
self.is_openai_model = any(
prefix in model.lower() for prefix in ["gpt-", "o1-", "text-"]
)
self.is_azure_openai_endpoint = (
"openai.azure.com" in self.endpoint
and "/openai/deployments/" in self.endpoint
)
self.is_openai_model = (
any(prefix in model.lower() for prefix in ["gpt-", "o1-", "text-"])
or self.is_azure_openai_endpoint
)
@staticmethod
def _validate_and_fix_endpoint(endpoint: str, model: str) -> str:
"""Validate and fix Azure endpoint URL format.

View File

@@ -1403,3 +1403,219 @@ def test_azure_stop_words_still_applied_to_regular_responses():
assert "Observation:" not in result
assert "Found results" not in result
assert "I need to search for more information" in result
# =============================================================================
# Tests for issue #4478: Custom deployment names with Azure OpenAI endpoints
# =============================================================================
def test_azure_custom_deployment_name_detected_as_openai_on_azure_openai_endpoint():
"""
Test that custom deployment names (e.g. 'gpt5nano') are correctly detected
as OpenAI models when using an Azure OpenAI endpoint.
Regression test for https://github.com/crewAIInc/crewAI/issues/4478
"""
from crewai.llms.providers.azure.completion import AzureCompletion
llm = AzureCompletion(
model="gpt5nano",
api_key="test-key",
endpoint="https://test.openai.azure.com/openai/deployments/gpt5nano",
)
assert llm.is_azure_openai_endpoint is True
assert llm.is_openai_model is True
assert llm.supports_function_calling() is True
def test_azure_custom_deployment_name_response_model_respected():
"""
Test that response_model is respected for custom Azure deployment names.
This is the core bug from issue #4478: LLM call does not adhere to
pydantic response_model for custom deployment names like 'gpt5nano'.
"""
from pydantic import BaseModel, Field
from crewai.llms.providers.azure.completion import AzureCompletion
class SimpleResponse(BaseModel):
message: str = Field(description="A primary message")
reason: str = Field(description="Reasoning for the response")
llm = AzureCompletion(
model="gpt5nano",
api_key="test-key",
endpoint="https://test.openai.azure.com/openai/deployments/gpt5nano",
)
json_response = '{"message": "Paris", "reason": "It is the capital of France"}'
with patch.object(llm.client, 'complete') as mock_complete:
mock_message = MagicMock()
mock_message.content = json_response
mock_message.tool_calls = None
mock_choice = MagicMock()
mock_choice.message = mock_message
mock_response = MagicMock()
mock_response.choices = [mock_choice]
mock_response.usage = MagicMock(
prompt_tokens=50, completion_tokens=30, total_tokens=80
)
mock_complete.return_value = mock_response
result = llm.call(
messages=[{"role": "user", "content": "What is the capital of France?"}],
response_model=SimpleResponse,
)
assert isinstance(result, SimpleResponse)
assert result.message == "Paris"
assert result.reason == "It is the capital of France"
def test_azure_custom_deployment_name_includes_response_format_in_params():
"""
Test that _prepare_completion_params includes response_format for custom
deployment names on Azure OpenAI endpoints.
"""
from pydantic import BaseModel, Field
from crewai.llms.providers.azure.completion import AzureCompletion
class MyModel(BaseModel):
answer: str = Field(description="The answer")
llm = AzureCompletion(
model="my-custom-gpt4o",
api_key="test-key",
endpoint="https://test.openai.azure.com/openai/deployments/my-custom-gpt4o",
)
params = llm._prepare_completion_params(
messages=[{"role": "user", "content": "test"}],
response_model=MyModel,
)
assert "response_format" in params
def test_azure_custom_deployment_name_includes_tools_in_params():
"""
Test that _prepare_completion_params includes tools for custom deployment
names on Azure OpenAI endpoints.
"""
from crewai.llms.providers.azure.completion import AzureCompletion
llm = AzureCompletion(
model="my-custom-gpt4o",
api_key="test-key",
endpoint="https://test.openai.azure.com/openai/deployments/my-custom-gpt4o",
)
tools = [
{
"type": "function",
"function": {
"name": "get_weather",
"description": "Get weather info",
"parameters": {
"type": "object",
"properties": {"location": {"type": "string"}},
},
},
}
]
params = llm._prepare_completion_params(
messages=[{"role": "user", "content": "test"}],
tools=tools,
)
assert "tools" in params
assert "tool_choice" in params
def test_azure_non_openai_endpoint_custom_name_not_detected_as_openai():
"""
Test that custom model names on non-Azure-OpenAI endpoints (like Azure AI
Inference) are still correctly NOT detected as OpenAI models.
"""
from crewai.llms.providers.azure.completion import AzureCompletion
llm = AzureCompletion(
model="my-custom-model",
api_key="test-key",
endpoint="https://models.inference.ai.azure.com",
)
assert llm.is_azure_openai_endpoint is False
assert llm.is_openai_model is False
assert llm.supports_function_calling() is False
def test_azure_custom_deployment_name_streaming_response_model():
"""
Test that streaming responses with custom deployment names also respect
response_model via the _finalize_streaming_response path.
"""
from pydantic import BaseModel, Field
from crewai.llms.providers.azure.completion import AzureCompletion
class StreamResult(BaseModel):
answer: str = Field(description="The answer")
llm = AzureCompletion(
model="gpt5nano",
api_key="test-key",
endpoint="https://test.openai.azure.com/openai/deployments/gpt5nano",
stream=True,
)
json_content = '{"answer": "42"}'
params = llm._prepare_completion_params(
messages=[{"role": "user", "content": "test"}],
response_model=StreamResult,
)
result = llm._finalize_streaming_response(
full_response=json_content,
tool_calls={},
usage_data={"total_tokens": 10},
params=params,
response_model=StreamResult,
)
assert isinstance(result, StreamResult)
assert result.answer == "42"
def test_azure_various_custom_deployment_names_on_openai_endpoint():
"""
Test multiple custom deployment name patterns that should all be detected
as OpenAI models when on an Azure OpenAI endpoint.
"""
from crewai.llms.providers.azure.completion import AzureCompletion
custom_names = [
"gpt5nano",
"my-gpt4-deployment",
"production-model-v2",
"team-a-llm",
"chatbot-engine",
]
for name in custom_names:
llm = AzureCompletion(
model=name,
api_key="test-key",
endpoint=f"https://test.openai.azure.com/openai/deployments/{name}",
)
assert llm.is_openai_model is True, (
f"Custom deployment '{name}' on Azure OpenAI endpoint should be detected as OpenAI model"
)
assert llm.supports_function_calling() is True, (
f"Custom deployment '{name}' on Azure OpenAI endpoint should support function calling"
)