mirror of
https://github.com/crewAIInc/crewAI.git
synced 2026-01-26 16:48:13 +00:00
Fix LLMGuardrailResult JSON parsing with trailing characters
- Extract robust JSON cleaning logic into shared clean_json_from_text() function - Update LiteAgent to use clean_json_from_text() before model_validate_json() - Add comprehensive test cases for JSON with trailing characters, markdown formatting, and prefixes - Fixes GitHub issue #3191 where valid JSON failed to parse due to trailing text - Maintains backward compatibility with existing JSON parsing behavior Co-Authored-By: Jo\u00E3o <joao@crewai.com>
This commit is contained in:
@@ -302,3 +302,78 @@ def test_hallucination_guardrail_description_in_events():
|
||||
|
||||
event = LLMGuardrailStartedEvent(guardrail=guardrail, retry_count=0)
|
||||
assert event.guardrail == "HallucinationGuardrail (no-op)"
|
||||
|
||||
|
||||
def test_llm_guardrail_with_trailing_characters():
|
||||
"""Test that LLMGuardrail can handle responses with trailing characters."""
|
||||
from unittest.mock import patch
|
||||
|
||||
mock_response_with_trailing = '''{"valid": true, "feedback": null}
|
||||
|
||||
Some additional text that should be ignored.
|
||||
More trailing content.'''
|
||||
|
||||
with patch('crewai.Agent.kickoff') as mock_kickoff:
|
||||
from crewai.agent import LiteAgentOutput
|
||||
from crewai.tasks.llm_guardrail import LLMGuardrailResult
|
||||
|
||||
mock_output = LiteAgentOutput(
|
||||
raw=mock_response_with_trailing,
|
||||
pydantic=LLMGuardrailResult(valid=True, feedback=None),
|
||||
agent_role="Guardrail Agent",
|
||||
usage_metrics=None
|
||||
)
|
||||
mock_kickoff.return_value = mock_output
|
||||
|
||||
guardrail = LLMGuardrail(
|
||||
description="Test guardrail",
|
||||
llm=LLM(model="gpt-4o-mini")
|
||||
)
|
||||
|
||||
task_output = TaskOutput(
|
||||
raw="Test task output",
|
||||
description="Test task",
|
||||
expected_output="Output",
|
||||
agent="Test Agent",
|
||||
)
|
||||
|
||||
result = guardrail(task_output)
|
||||
assert result[0] is True
|
||||
assert result[1] == "Test task output"
|
||||
|
||||
|
||||
def test_llm_guardrail_with_markdown_formatting():
|
||||
"""Test that LLMGuardrail can handle responses with markdown formatting."""
|
||||
from unittest.mock import patch
|
||||
|
||||
mock_response_with_markdown = '''```json
|
||||
{"valid": false, "feedback": "The output does not meet the requirements"}
|
||||
```'''
|
||||
|
||||
with patch('crewai.Agent.kickoff') as mock_kickoff:
|
||||
from crewai.agent import LiteAgentOutput
|
||||
from crewai.tasks.llm_guardrail import LLMGuardrailResult
|
||||
|
||||
mock_output = LiteAgentOutput(
|
||||
raw=mock_response_with_markdown,
|
||||
pydantic=LLMGuardrailResult(valid=False, feedback="The output does not meet the requirements"),
|
||||
agent_role="Guardrail Agent",
|
||||
usage_metrics=None
|
||||
)
|
||||
mock_kickoff.return_value = mock_output
|
||||
|
||||
guardrail = LLMGuardrail(
|
||||
description="Test guardrail",
|
||||
llm=LLM(model="gpt-4o-mini")
|
||||
)
|
||||
|
||||
task_output = TaskOutput(
|
||||
raw="Test task output",
|
||||
description="Test task",
|
||||
expected_output="Output",
|
||||
agent="Test Agent",
|
||||
)
|
||||
|
||||
result = guardrail(task_output)
|
||||
assert result[0] is False
|
||||
assert result[1] == "The output does not meet the requirements"
|
||||
|
||||
Reference in New Issue
Block a user