diff --git a/reproduce_issue_3165.py b/reproduce_issue_3165.py new file mode 100644 index 000000000..4faf9c931 --- /dev/null +++ b/reproduce_issue_3165.py @@ -0,0 +1,108 @@ +#!/usr/bin/env python3 +""" +Reproduction script for issue #3165: LLM Failed with Custom OpenAI-Compatible Endpoint + +This script reproduces the bug where CrewAI shows generic "LLM Failed" errors +instead of propagating specific error details from custom endpoints. +""" + +import os +import sys +from crewai import Agent, Task, Crew +from crewai.llm import LLM + +def test_custom_endpoint_error_handling(): + """Test error handling with a custom OpenAI-compatible endpoint.""" + + print("Testing custom endpoint error handling...") + + custom_llm = LLM( + model="gpt-3.5-turbo", + base_url="https://non-existent-endpoint.example.com/v1", + api_key="fake-api-key-for-testing" + ) + + agent = Agent( + role="Test Agent", + goal="Test custom endpoint error handling", + backstory="A test agent for reproducing issue #3165", + llm=custom_llm, + verbose=True + ) + + task = Task( + description="Say hello world", + expected_output="A simple greeting", + agent=agent + ) + + crew = Crew( + agents=[agent], + tasks=[task], + verbose=True + ) + + try: + print("\nAttempting to run crew with custom endpoint...") + result = crew.kickoff() + print(f"Unexpected success: {result}") + except Exception as e: + print(f"\nCaught exception: {type(e).__name__}") + print(f"Exception message: {str(e)}") + + if "LLM Failed" in str(e) and "connection" not in str(e).lower(): + print("\n❌ BUG CONFIRMED: Generic 'LLM Failed' error without specific details") + print("Expected: Specific connection/authentication error details") + return False + else: + print("\n✅ Good: Specific error details preserved") + return True + +def test_direct_llm_call(): + """Test direct LLM call with custom endpoint.""" + + print("\n" + "="*60) + print("Testing direct LLM call with custom endpoint...") + + custom_llm = LLM( + model="gpt-3.5-turbo", + base_url="https://non-existent-endpoint.example.com/v1", + api_key="fake-api-key-for-testing" + ) + + try: + print("Attempting direct LLM call...") + response = custom_llm.call("Hello world") + print(f"Unexpected success: {response}") + except Exception as e: + print(f"\nCaught exception: {type(e).__name__}") + print(f"Exception message: {str(e)}") + + error_msg = str(e).lower() + if any(keyword in error_msg for keyword in ["connection", "resolve", "network", "timeout", "unreachable"]): + print("\n✅ Good: Specific connection error details preserved") + return True + else: + print("\n❌ BUG CONFIRMED: Generic error without connection details") + print("Expected: Specific connection error details") + return False + +if __name__ == "__main__": + print("Reproducing issue #3165: LLM Failed with Custom OpenAI-Compatible Endpoint") + print("="*80) + + crew_test_passed = test_custom_endpoint_error_handling() + direct_test_passed = test_direct_llm_call() + + print("\n" + "="*80) + print("SUMMARY:") + print(f"Crew-level test: {'PASSED' if crew_test_passed else 'FAILED (bug confirmed)'}") + print(f"Direct LLM test: {'PASSED' if direct_test_passed else 'FAILED (bug confirmed)'}") + + if not crew_test_passed or not direct_test_passed: + print("\n❌ Issue #3165 reproduced successfully") + print("CrewAI is showing generic errors instead of specific endpoint error details") + sys.exit(1) + else: + print("\n✅ Issue #3165 appears to be fixed") + sys.exit(0) diff --git a/src/crewai/llm.py b/src/crewai/llm.py index d6f40a09a..c475cd4a1 100644 --- a/src/crewai/llm.py +++ b/src/crewai/llm.py @@ -984,10 +984,27 @@ class LLM(BaseLLM): # whether to summarize the content or abort based on the respect_context_window flag raise except Exception as e: + error_info = { + "error_type": type(e).__name__, + "original_error": str(e), + "endpoint_info": { + "base_url": self.base_url, + "model": self.model, + "api_base": self.api_base, + } if self.base_url or self.api_base else None + } + assert hasattr(crewai_event_bus, "emit") crewai_event_bus.emit( self, - event=LLMCallFailedEvent(error=str(e), from_task=from_task, from_agent=from_agent), + event=LLMCallFailedEvent( + error=str(e), + error_type=error_info["error_type"], + original_error=error_info["original_error"], + endpoint_info=error_info["endpoint_info"], + from_task=from_task, + from_agent=from_agent + ), ) logging.error(f"LiteLLM call failed: {str(e)}") raise diff --git a/src/crewai/utilities/events/llm_events.py b/src/crewai/utilities/events/llm_events.py index 74101a041..b4d880a72 100644 --- a/src/crewai/utilities/events/llm_events.py +++ b/src/crewai/utilities/events/llm_events.py @@ -67,6 +67,9 @@ class LLMCallFailedEvent(LLMEventBase): error: str type: str = "llm_call_failed" + error_type: Optional[str] = None + original_error: Optional[str] = None + endpoint_info: Optional[Dict[str, Any]] = None class FunctionCall(BaseModel): diff --git a/src/crewai/utilities/events/utils/console_formatter.py b/src/crewai/utilities/events/utils/console_formatter.py index 24f92e386..f54b4e5d7 100644 --- a/src/crewai/utilities/events/utils/console_formatter.py +++ b/src/crewai/utilities/events/utils/console_formatter.py @@ -721,7 +721,7 @@ class ConsoleFormatter: self.print() def handle_llm_call_failed( - self, tool_branch: Optional[Tree], error: str, crew_tree: Optional[Tree] + self, tool_branch: Optional[Tree], error: str, crew_tree: Optional[Tree], event: Optional[Any] = None ) -> None: """Handle LLM call failed event.""" if not self.verbose: @@ -764,9 +764,19 @@ class ConsoleFormatter: self.print(tree_to_use) self.print() - # Show error panel + # Show detailed error panel error_content = Text() error_content.append("❌ LLM Call Failed\n", style="red bold") + + if event and hasattr(event, 'error_type') and event.error_type: + error_content.append(f"Error Type: {event.error_type}\n", style="yellow") + + if event and hasattr(event, 'endpoint_info') and event.endpoint_info: + endpoint = event.endpoint_info.get('base_url') or event.endpoint_info.get('api_base') + if endpoint: + error_content.append(f"Endpoint: {endpoint}\n", style="cyan") + error_content.append(f"Model: {event.endpoint_info.get('model', 'unknown')}\n", style="cyan") + error_content.append("Error: ", style="white") error_content.append(str(error), style="red") diff --git a/tests/test_custom_endpoint_error_handling.py b/tests/test_custom_endpoint_error_handling.py new file mode 100644 index 000000000..cd9a62714 --- /dev/null +++ b/tests/test_custom_endpoint_error_handling.py @@ -0,0 +1,185 @@ +""" +Tests for custom endpoint error handling (issue #3165). + +These tests verify that CrewAI properly propagates specific error details +from custom OpenAI-compatible endpoints instead of showing generic "LLM Failed" errors. +""" + +import pytest +from unittest.mock import patch, MagicMock +from crewai.llm import LLM +from crewai.utilities.events.llm_events import LLMCallFailedEvent +from crewai.utilities.events.utils.console_formatter import ConsoleFormatter +import requests + + +class TestCustomEndpointErrorHandling: + """Test error handling for custom OpenAI-compatible endpoints.""" + + def test_connection_error_preserves_details(self): + """Test that connection errors preserve specific error details.""" + custom_llm = LLM( + model="gpt-3.5-turbo", + base_url="https://non-existent-endpoint.example.com/v1", + api_key="fake-api-key" + ) + + with patch('litellm.completion') as mock_completion: + mock_completion.side_effect = requests.exceptions.ConnectionError( + "Failed to establish a new connection: [Errno -2] Name or service not known" + ) + + with pytest.raises(requests.exceptions.ConnectionError) as exc_info: + custom_llm.call("Hello world") + + assert "Name or service not known" in str(exc_info.value) + + def test_authentication_error_preserves_details(self): + """Test that authentication errors preserve specific error details.""" + custom_llm = LLM( + model="gpt-3.5-turbo", + base_url="https://api.openai.com/v1", + api_key="invalid-api-key" + ) + + with patch('litellm.completion') as mock_completion: + mock_completion.side_effect = Exception( + "AuthenticationError: Incorrect API key provided" + ) + + with pytest.raises(Exception) as exc_info: + custom_llm.call("Hello world") + + assert "AuthenticationError" in str(exc_info.value) + assert "Incorrect API key" in str(exc_info.value) + + def test_llm_call_failed_event_enhanced_fields(self): + """Test that LLMCallFailedEvent includes enhanced error information.""" + custom_llm = LLM( + model="gpt-3.5-turbo", + base_url="https://custom-endpoint.example.com/v1", + api_key="test-key" + ) + + captured_events = [] + + def capture_event(sender, event): + captured_events.append(event) + + with patch('crewai.utilities.events.crewai_event_bus.crewai_event_bus.emit', side_effect=capture_event): + with patch('litellm.completion') as mock_completion: + mock_completion.side_effect = requests.exceptions.ConnectionError( + "Connection failed" + ) + + with pytest.raises(requests.exceptions.ConnectionError): + custom_llm.call("Hello world") + + assert len(captured_events) == 2 # Started and Failed events + failed_event = captured_events[1] + assert isinstance(failed_event, LLMCallFailedEvent) + assert failed_event.error_type == "ConnectionError" + assert failed_event.original_error == "Connection failed" + assert failed_event.endpoint_info is not None + assert failed_event.endpoint_info["base_url"] == "https://custom-endpoint.example.com/v1" + assert failed_event.endpoint_info["model"] == "gpt-3.5-turbo" + + def test_console_formatter_displays_enhanced_error_info(self): + """Test that console formatter displays enhanced error information.""" + formatter = ConsoleFormatter(verbose=True) + + mock_event = MagicMock() + mock_event.error_type = "ConnectionError" + mock_event.endpoint_info = { + "base_url": "https://custom-endpoint.example.com/v1", + "model": "gpt-3.5-turbo" + } + + captured_output = [] + + def mock_print_panel(content, title, style): + captured_output.append(str(content)) + + formatter.print_panel = mock_print_panel + + formatter.handle_llm_call_failed( + tool_branch=None, + error="Connection failed", + crew_tree=None, + event=mock_event + ) + + output = captured_output[0] + assert "Error Type: ConnectionError" in output + assert "Endpoint: https://custom-endpoint.example.com/v1" in output + assert "Model: gpt-3.5-turbo" in output + assert "Connection failed" in output + + def test_backward_compatibility_without_enhanced_fields(self): + """Test that console formatter works without enhanced fields for backward compatibility.""" + formatter = ConsoleFormatter(verbose=True) + + captured_output = [] + + def mock_print_panel(content, title, style): + captured_output.append(str(content)) + + formatter.print_panel = mock_print_panel + + formatter.handle_llm_call_failed( + tool_branch=None, + error="Generic error message", + crew_tree=None, + event=None + ) + + output = captured_output[0] + assert "❌ LLM Call Failed" in output + assert "Generic error message" in output + assert "Error Type:" not in output + assert "Endpoint:" not in output + + def test_streaming_response_error_handling(self): + """Test that streaming responses also preserve error details.""" + custom_llm = LLM( + model="gpt-3.5-turbo", + base_url="https://custom-endpoint.example.com/v1", + api_key="test-key", + stream=True + ) + + with patch('litellm.completion') as mock_completion: + mock_completion.side_effect = requests.exceptions.ConnectionError( + "Streaming connection failed" + ) + + with pytest.raises(Exception) as exc_info: + custom_llm.call("Hello world") + + assert "Streaming connection failed" in str(exc_info.value) + + def test_non_custom_endpoint_error_handling(self): + """Test that standard OpenAI endpoint errors are handled normally.""" + standard_llm = LLM( + model="gpt-3.5-turbo", + api_key="test-key" + ) + + captured_events = [] + + def capture_event(sender, event): + captured_events.append(event) + + with patch('crewai.utilities.events.crewai_event_bus.crewai_event_bus.emit', side_effect=capture_event): + with patch('litellm.completion') as mock_completion: + mock_completion.side_effect = Exception("Standard API error") + + with pytest.raises(Exception): + standard_llm.call("Hello world") + + assert len(captured_events) == 2 # Started and Failed events + failed_event = captured_events[1] + assert isinstance(failed_event, LLMCallFailedEvent) + assert failed_event.error_type == "Exception" + assert failed_event.original_error == "Standard API error" + assert failed_event.endpoint_info is None # No custom endpoint info