mirror of
https://github.com/crewAIInc/crewAI.git
synced 2026-01-11 17:18:29 +00:00
08fa3797ca
* feat: add exchanged messages in LLMCallCompletedEvent * feat: add GoalAlignment metric for Agent evaluation * feat: add SemanticQuality metric for Agent evaluation * feat: add Tool Metrics for Agent evaluation * feat: add Reasoning Metrics for Agent evaluation, still in progress * feat: add AgentEvaluator class This class will evaluate Agent' results and report to user * fix: do not evaluate Agent by default This is a experimental feature we still need refine it further * test: add Agent eval tests * fix: render all feedback per iteration * style: resolve linter issues * style: fix mypy issues * fix: allow messages be empty on LLMCallCompletedEvent
31 lines
854 B
Python
31 lines
854 B
Python
"""Robust JSON parsing utilities for evaluation responses."""
|
|
|
|
import json
|
|
import re
|
|
from typing import Any
|
|
|
|
|
|
def extract_json_from_llm_response(text: str) -> dict[str, Any]:
|
|
try:
|
|
return json.loads(text)
|
|
except json.JSONDecodeError:
|
|
pass
|
|
|
|
json_patterns = [
|
|
# Standard markdown code blocks with json
|
|
r'```json\s*([\s\S]*?)\s*```',
|
|
# Code blocks without language specifier
|
|
r'```\s*([\s\S]*?)\s*```',
|
|
# Inline code with JSON
|
|
r'`([{\\[].*[}\]])`',
|
|
]
|
|
|
|
for pattern in json_patterns:
|
|
matches = re.findall(pattern, text, re.IGNORECASE | re.DOTALL)
|
|
for match in matches:
|
|
try:
|
|
return json.loads(match.strip())
|
|
except json.JSONDecodeError:
|
|
continue
|
|
raise ValueError("No valid JSON found in the response")
|