Fix LLMGuardrailResult JSON parsing with trailing characters

- Extract robust JSON cleaning logic into shared clean_json_from_text() function
- Update LiteAgent to use clean_json_from_text() before model_validate_json()
- Add comprehensive test cases for JSON with trailing characters, markdown formatting, and prefixes
- Fixes GitHub issue #3191 where valid JSON failed to parse due to trailing text
- Maintains backward compatibility with existing JSON parsing behavior

Co-Authored-By: Jo\u00E3o <joao@crewai.com>
This commit is contained in:
Devin AI
2025-07-19 22:56:22 +00:00
parent 942014962e
commit b7cb0186bd
5 changed files with 292 additions and 19 deletions

View File

@@ -492,4 +492,102 @@ def test_lite_agent_with_invalid_llm():
backstory="Test backstory",
llm="invalid-model"
)
assert "Expected LLM instance of type BaseLLM" in str(exc_info.value)
assert "Expected LLM instance of type BaseLLM" in str(exc_info.value)
def test_lite_agent_structured_output_with_trailing_characters():
"""Test that LiteAgent can handle JSON responses with trailing characters."""
from unittest.mock import patch
class SimpleOutput(BaseModel):
summary: str = Field(description="A brief summary")
confidence: int = Field(description="Confidence level from 1-100")
mock_response_with_trailing = '''{"summary": "Test summary", "confidence": 85}
Additional text after JSON that should be ignored.
Final Answer: This text should also be ignored.'''
with patch('crewai.lite_agent.get_llm_response') as mock_llm:
mock_llm.return_value = mock_response_with_trailing
agent = LiteAgent(
role="Test Agent",
goal="Test goal",
backstory="Test backstory",
llm=LLM(model="gpt-4o-mini"),
)
result = agent.kickoff(
"Test message",
response_format=SimpleOutput
)
assert result.pydantic is not None
assert isinstance(result.pydantic, SimpleOutput)
assert result.pydantic.summary == "Test summary"
assert result.pydantic.confidence == 85
def test_lite_agent_structured_output_with_markdown():
"""Test that LiteAgent can handle JSON responses wrapped in markdown."""
from unittest.mock import patch
class SimpleOutput(BaseModel):
summary: str = Field(description="A brief summary")
confidence: int = Field(description="Confidence level from 1-100")
mock_response_with_markdown = '''```json
{"summary": "Test summary with markdown", "confidence": 90}
```'''
with patch('crewai.lite_agent.get_llm_response') as mock_llm:
mock_llm.return_value = mock_response_with_markdown
agent = LiteAgent(
role="Test Agent",
goal="Test goal",
backstory="Test backstory",
llm=LLM(model="gpt-4o-mini"),
)
result = agent.kickoff(
"Test message",
response_format=SimpleOutput
)
assert result.pydantic is not None
assert isinstance(result.pydantic, SimpleOutput)
assert result.pydantic.summary == "Test summary with markdown"
assert result.pydantic.confidence == 90
def test_lite_agent_structured_output_with_final_answer_prefix():
"""Test that LiteAgent can handle JSON responses with Final Answer prefix."""
from unittest.mock import patch
class SimpleOutput(BaseModel):
summary: str = Field(description="A brief summary")
confidence: int = Field(description="Confidence level from 1-100")
mock_response_with_prefix = '''Final Answer: {"summary": "Test summary with prefix", "confidence": 95}'''
with patch('crewai.lite_agent.get_llm_response') as mock_llm:
mock_llm.return_value = mock_response_with_prefix
agent = LiteAgent(
role="Test Agent",
goal="Test goal",
backstory="Test backstory",
llm=LLM(model="gpt-4o-mini"),
)
result = agent.kickoff(
"Test message",
response_format=SimpleOutput
)
assert result.pydantic is not None
assert isinstance(result.pydantic, SimpleOutput)
assert result.pydantic.summary == "Test summary with prefix"
assert result.pydantic.confidence == 95