mirror of
https://github.com/crewAIInc/crewAI.git
synced 2025-12-27 09:48:30 +00:00
Compare commits
3 Commits
1.6.1
...
devin/1757
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
42e93984c3 | ||
|
|
41ad22a573 | ||
|
|
c84bdac4a6 |
@@ -25,3 +25,16 @@ ACTION_REGEX: Final[re.Pattern[str]] = re.compile(
|
|||||||
ACTION_INPUT_ONLY_REGEX: Final[re.Pattern[str]] = re.compile(
|
ACTION_INPUT_ONLY_REGEX: Final[re.Pattern[str]] = re.compile(
|
||||||
r"\s*Action\s*\d*\s*Input\s*\d*\s*:\s*(.*)", re.DOTALL
|
r"\s*Action\s*\d*\s*Input\s*\d*\s*:\s*(.*)", re.DOTALL
|
||||||
)
|
)
|
||||||
|
|
||||||
|
HARMONY_START_PATTERN: Final[re.Pattern[str]] = re.compile(
|
||||||
|
r"<\|start\|>assistant<\|channel\|>(\w+)(?:\s+to=(\w+))?<\|message\|>(.*?)<\|(?:end|call)\|>",
|
||||||
|
re.DOTALL
|
||||||
|
)
|
||||||
|
HARMONY_ANALYSIS_CHANNEL: Final[str] = "analysis"
|
||||||
|
HARMONY_COMMENTARY_CHANNEL: Final[str] = "commentary"
|
||||||
|
HARMONY_FINAL_ANSWER_ERROR_MESSAGE: Final[str] = (
|
||||||
|
"I did it wrong. Invalid Harmony Format: I need to use proper channel structure."
|
||||||
|
)
|
||||||
|
HARMONY_MISSING_CONTENT_ERROR_MESSAGE: Final[str] = (
|
||||||
|
"I did it wrong. Invalid Harmony Format: Missing content in message section."
|
||||||
|
)
|
||||||
|
|||||||
@@ -1,19 +1,26 @@
|
|||||||
"""Agent output parsing module for ReAct-style LLM responses.
|
"""Agent output parsing module for multiple LLM response formats.
|
||||||
|
|
||||||
This module provides parsing functionality for agent outputs that follow
|
This module provides parsing functionality for agent outputs that follow
|
||||||
the ReAct (Reasoning and Acting) format, converting them into structured
|
different formats (ReAct, OpenAI Harmony, etc.), converting them into structured
|
||||||
AgentAction or AgentFinish objects.
|
AgentAction or AgentFinish objects with automatic format detection.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
|
import re
|
||||||
|
from abc import ABC, abstractmethod
|
||||||
from dataclasses import dataclass
|
from dataclasses import dataclass
|
||||||
|
|
||||||
from json_repair import repair_json
|
from json_repair import repair_json
|
||||||
|
|
||||||
from crewai.agents.constants import (
|
from crewai.agents.constants import (
|
||||||
|
ACTION_INPUT_ONLY_REGEX,
|
||||||
ACTION_INPUT_REGEX,
|
ACTION_INPUT_REGEX,
|
||||||
ACTION_REGEX,
|
ACTION_REGEX,
|
||||||
ACTION_INPUT_ONLY_REGEX,
|
|
||||||
FINAL_ANSWER_ACTION,
|
FINAL_ANSWER_ACTION,
|
||||||
|
HARMONY_ANALYSIS_CHANNEL,
|
||||||
|
HARMONY_COMMENTARY_CHANNEL,
|
||||||
|
HARMONY_FINAL_ANSWER_ERROR_MESSAGE,
|
||||||
|
HARMONY_MISSING_CONTENT_ERROR_MESSAGE,
|
||||||
|
HARMONY_START_PATTERN,
|
||||||
MISSING_ACTION_AFTER_THOUGHT_ERROR_MESSAGE,
|
MISSING_ACTION_AFTER_THOUGHT_ERROR_MESSAGE,
|
||||||
MISSING_ACTION_INPUT_AFTER_ACTION_ERROR_MESSAGE,
|
MISSING_ACTION_INPUT_AFTER_ACTION_ERROR_MESSAGE,
|
||||||
UNABLE_TO_REPAIR_JSON_RESULTS,
|
UNABLE_TO_REPAIR_JSON_RESULTS,
|
||||||
@@ -60,26 +67,166 @@ class OutputParserException(Exception):
|
|||||||
super().__init__(error)
|
super().__init__(error)
|
||||||
|
|
||||||
|
|
||||||
|
class BaseOutputParser(ABC):
|
||||||
|
"""Abstract base class for output parsers."""
|
||||||
|
|
||||||
|
@abstractmethod
|
||||||
|
def can_parse(self, text: str) -> bool:
|
||||||
|
"""Check if this parser can handle the given text format."""
|
||||||
|
|
||||||
|
@abstractmethod
|
||||||
|
def parse_text(self, text: str) -> AgentAction | AgentFinish:
|
||||||
|
"""Parse the text into AgentAction or AgentFinish."""
|
||||||
|
|
||||||
|
|
||||||
|
class OutputFormatRegistry:
|
||||||
|
"""Registry for managing different output format parsers."""
|
||||||
|
|
||||||
|
def __init__(self):
|
||||||
|
self._parsers: dict[str, BaseOutputParser] = {}
|
||||||
|
|
||||||
|
def register(self, name: str, parser: BaseOutputParser) -> None:
|
||||||
|
"""Register a parser for a specific format."""
|
||||||
|
self._parsers[name] = parser
|
||||||
|
|
||||||
|
def detect_and_parse(self, text: str) -> AgentAction | AgentFinish:
|
||||||
|
"""Automatically detect format and parse with appropriate parser."""
|
||||||
|
for parser in self._parsers.values():
|
||||||
|
if parser.can_parse(text):
|
||||||
|
return parser.parse_text(text)
|
||||||
|
|
||||||
|
return self._parsers.get('react', ReActParser()).parse_text(text)
|
||||||
|
|
||||||
|
|
||||||
|
class ReActParser(BaseOutputParser):
|
||||||
|
"""Parser for ReAct format outputs."""
|
||||||
|
|
||||||
|
def can_parse(self, text: str) -> bool:
|
||||||
|
"""Check if text follows ReAct format."""
|
||||||
|
return (
|
||||||
|
FINAL_ANSWER_ACTION in text or
|
||||||
|
ACTION_INPUT_REGEX.search(text) is not None or
|
||||||
|
ACTION_REGEX.search(text) is not None
|
||||||
|
)
|
||||||
|
|
||||||
|
def parse_text(self, text: str) -> AgentAction | AgentFinish:
|
||||||
|
"""Parse ReAct format text."""
|
||||||
|
thought = _extract_thought(text)
|
||||||
|
includes_answer = FINAL_ANSWER_ACTION in text
|
||||||
|
action_match = ACTION_INPUT_REGEX.search(text)
|
||||||
|
|
||||||
|
if includes_answer:
|
||||||
|
final_answer = text.split(FINAL_ANSWER_ACTION)[-1].strip()
|
||||||
|
if final_answer.endswith("```"):
|
||||||
|
count = final_answer.count("```")
|
||||||
|
if count % 2 != 0:
|
||||||
|
final_answer = final_answer[:-3].rstrip()
|
||||||
|
return AgentFinish(thought=thought, output=final_answer, text=text)
|
||||||
|
|
||||||
|
if action_match:
|
||||||
|
action = action_match.group(1)
|
||||||
|
clean_action = _clean_action(action)
|
||||||
|
|
||||||
|
action_input = action_match.group(2).strip()
|
||||||
|
|
||||||
|
tool_input = action_input.strip(" ").strip('"')
|
||||||
|
safe_tool_input = _safe_repair_json(tool_input)
|
||||||
|
|
||||||
|
return AgentAction(
|
||||||
|
thought=thought, tool=clean_action, tool_input=safe_tool_input, text=text
|
||||||
|
)
|
||||||
|
|
||||||
|
if not ACTION_REGEX.search(text):
|
||||||
|
raise OutputParserException(
|
||||||
|
f"{MISSING_ACTION_AFTER_THOUGHT_ERROR_MESSAGE}\n{_I18N.slice('final_answer_format')}",
|
||||||
|
)
|
||||||
|
if not ACTION_INPUT_ONLY_REGEX.search(text):
|
||||||
|
raise OutputParserException(
|
||||||
|
MISSING_ACTION_INPUT_AFTER_ACTION_ERROR_MESSAGE,
|
||||||
|
)
|
||||||
|
err_format = _I18N.slice("format_without_tools")
|
||||||
|
error = f"{err_format}"
|
||||||
|
raise OutputParserException(error)
|
||||||
|
|
||||||
|
|
||||||
|
class HarmonyParser(BaseOutputParser):
|
||||||
|
"""Parser for OpenAI Harmony format outputs."""
|
||||||
|
|
||||||
|
def can_parse(self, text: str) -> bool:
|
||||||
|
"""Check if text follows OpenAI Harmony format."""
|
||||||
|
return HARMONY_START_PATTERN.search(text) is not None
|
||||||
|
|
||||||
|
def parse_text(self, text: str) -> AgentAction | AgentFinish:
|
||||||
|
"""Parse OpenAI Harmony format text."""
|
||||||
|
matches = HARMONY_START_PATTERN.findall(text)
|
||||||
|
|
||||||
|
if not matches:
|
||||||
|
raise OutputParserException(HARMONY_MISSING_CONTENT_ERROR_MESSAGE)
|
||||||
|
|
||||||
|
channel, tool_name, content = matches[-1]
|
||||||
|
content = content.strip()
|
||||||
|
|
||||||
|
if channel == HARMONY_ANALYSIS_CHANNEL:
|
||||||
|
return AgentFinish(
|
||||||
|
thought=f"Analysis: {content}",
|
||||||
|
output=content,
|
||||||
|
text=text
|
||||||
|
)
|
||||||
|
|
||||||
|
if channel == HARMONY_COMMENTARY_CHANNEL and tool_name:
|
||||||
|
thought_content = content
|
||||||
|
tool_input = content
|
||||||
|
|
||||||
|
try:
|
||||||
|
json_match = re.search(r'\{.*\}', content, re.DOTALL)
|
||||||
|
if json_match:
|
||||||
|
tool_input = json_match.group(0)
|
||||||
|
thought_content = content[:json_match.start()].strip()
|
||||||
|
if not thought_content:
|
||||||
|
thought_content = f"Using tool {tool_name}"
|
||||||
|
except Exception:
|
||||||
|
tool_input = content
|
||||||
|
|
||||||
|
safe_tool_input = _safe_repair_json(tool_input)
|
||||||
|
|
||||||
|
return AgentAction(
|
||||||
|
thought=thought_content,
|
||||||
|
tool=tool_name,
|
||||||
|
tool_input=safe_tool_input,
|
||||||
|
text=text
|
||||||
|
)
|
||||||
|
|
||||||
|
raise OutputParserException(HARMONY_FINAL_ANSWER_ERROR_MESSAGE)
|
||||||
|
|
||||||
|
|
||||||
|
_format_registry = OutputFormatRegistry()
|
||||||
|
_format_registry.register('react', ReActParser())
|
||||||
|
_format_registry.register('harmony', HarmonyParser())
|
||||||
|
|
||||||
|
|
||||||
def parse(text: str) -> AgentAction | AgentFinish:
|
def parse(text: str) -> AgentAction | AgentFinish:
|
||||||
"""Parse agent output text into AgentAction or AgentFinish.
|
"""Parse agent output text into AgentAction or AgentFinish.
|
||||||
|
|
||||||
Expects output to be in one of two formats.
|
Automatically detects the format (ReAct, OpenAI Harmony, etc.) and uses
|
||||||
|
the appropriate parser. Maintains backward compatibility with existing ReAct format.
|
||||||
|
|
||||||
If the output signals that an action should be taken,
|
Supports multiple formats:
|
||||||
should be in the below format. This will result in an AgentAction
|
|
||||||
being returned.
|
|
||||||
|
|
||||||
|
ReAct format:
|
||||||
Thought: agent thought here
|
Thought: agent thought here
|
||||||
Action: search
|
Action: search
|
||||||
Action Input: what is the temperature in SF?
|
Action Input: what is the temperature in SF?
|
||||||
|
|
||||||
If the output signals that a final answer should be given,
|
Or for final answers:
|
||||||
should be in the below format. This will result in an AgentFinish
|
|
||||||
being returned.
|
|
||||||
|
|
||||||
Thought: agent thought here
|
Thought: agent thought here
|
||||||
Final Answer: The temperature is 100 degrees
|
Final Answer: The temperature is 100 degrees
|
||||||
|
|
||||||
|
OpenAI Harmony format:
|
||||||
|
<|start|>assistant<|channel|>analysis<|message|>The temperature is 100 degrees<|end|>
|
||||||
|
|
||||||
|
Or for tool actions:
|
||||||
|
<|start|>assistant<|channel|>commentary to=search<|message|>{"query": "temperature in SF"}<|call|>
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
text: The agent output text to parse.
|
text: The agent output text to parse.
|
||||||
|
|
||||||
@@ -87,50 +234,9 @@ def parse(text: str) -> AgentAction | AgentFinish:
|
|||||||
AgentAction or AgentFinish based on the content.
|
AgentAction or AgentFinish based on the content.
|
||||||
|
|
||||||
Raises:
|
Raises:
|
||||||
OutputParserException: If the text format is invalid.
|
OutputParserException: If the text format is invalid or unsupported.
|
||||||
"""
|
"""
|
||||||
thought = _extract_thought(text)
|
return _format_registry.detect_and_parse(text)
|
||||||
includes_answer = FINAL_ANSWER_ACTION in text
|
|
||||||
action_match = ACTION_INPUT_REGEX.search(text)
|
|
||||||
|
|
||||||
if includes_answer:
|
|
||||||
final_answer = text.split(FINAL_ANSWER_ACTION)[-1].strip()
|
|
||||||
# Check whether the final answer ends with triple backticks.
|
|
||||||
if final_answer.endswith("```"):
|
|
||||||
# Count occurrences of triple backticks in the final answer.
|
|
||||||
count = final_answer.count("```")
|
|
||||||
# If count is odd then it's an unmatched trailing set; remove it.
|
|
||||||
if count % 2 != 0:
|
|
||||||
final_answer = final_answer[:-3].rstrip()
|
|
||||||
return AgentFinish(thought=thought, output=final_answer, text=text)
|
|
||||||
|
|
||||||
elif action_match:
|
|
||||||
action = action_match.group(1)
|
|
||||||
clean_action = _clean_action(action)
|
|
||||||
|
|
||||||
action_input = action_match.group(2).strip()
|
|
||||||
|
|
||||||
tool_input = action_input.strip(" ").strip('"')
|
|
||||||
safe_tool_input = _safe_repair_json(tool_input)
|
|
||||||
|
|
||||||
return AgentAction(
|
|
||||||
thought=thought, tool=clean_action, tool_input=safe_tool_input, text=text
|
|
||||||
)
|
|
||||||
|
|
||||||
if not ACTION_REGEX.search(text):
|
|
||||||
raise OutputParserException(
|
|
||||||
f"{MISSING_ACTION_AFTER_THOUGHT_ERROR_MESSAGE}\n{_I18N.slice('final_answer_format')}",
|
|
||||||
)
|
|
||||||
elif not ACTION_INPUT_ONLY_REGEX.search(text):
|
|
||||||
raise OutputParserException(
|
|
||||||
MISSING_ACTION_INPUT_AFTER_ACTION_ERROR_MESSAGE,
|
|
||||||
)
|
|
||||||
else:
|
|
||||||
err_format = _I18N.slice("format_without_tools")
|
|
||||||
error = f"{err_format}"
|
|
||||||
raise OutputParserException(
|
|
||||||
error,
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def _extract_thought(text: str) -> str:
|
def _extract_thought(text: str) -> str:
|
||||||
@@ -149,8 +255,7 @@ def _extract_thought(text: str) -> str:
|
|||||||
return ""
|
return ""
|
||||||
thought = text[:thought_index].strip()
|
thought = text[:thought_index].strip()
|
||||||
# Remove any triple backticks from the thought string
|
# Remove any triple backticks from the thought string
|
||||||
thought = thought.replace("```", "").strip()
|
return thought.replace("```", "").strip()
|
||||||
return thought
|
|
||||||
|
|
||||||
|
|
||||||
def _clean_action(text: str) -> str:
|
def _clean_action(text: str) -> str:
|
||||||
|
|||||||
@@ -1,11 +1,11 @@
|
|||||||
import pytest
|
import pytest
|
||||||
|
|
||||||
|
from crewai.agents import parser
|
||||||
from crewai.agents.crew_agent_executor import (
|
from crewai.agents.crew_agent_executor import (
|
||||||
AgentAction,
|
AgentAction,
|
||||||
AgentFinish,
|
AgentFinish,
|
||||||
OutputParserException,
|
OutputParserException,
|
||||||
)
|
)
|
||||||
from crewai.agents import parser
|
|
||||||
|
|
||||||
|
|
||||||
def test_valid_action_parsing_special_characters():
|
def test_valid_action_parsing_special_characters():
|
||||||
@@ -345,12 +345,16 @@ def test_integration_valid_and_invalid():
|
|||||||
"""
|
"""
|
||||||
parts = text.strip().split("\n\n")
|
parts = text.strip().split("\n\n")
|
||||||
results = []
|
results = []
|
||||||
for part in parts:
|
|
||||||
|
def parse_part(part_text):
|
||||||
try:
|
try:
|
||||||
result = parser.parse(part.strip())
|
return parser.parse(part_text.strip())
|
||||||
results.append(result)
|
|
||||||
except OutputParserException as e:
|
except OutputParserException as e:
|
||||||
results.append(e)
|
return e
|
||||||
|
|
||||||
|
for part in parts:
|
||||||
|
result = parse_part(part)
|
||||||
|
results.append(result)
|
||||||
|
|
||||||
assert isinstance(results[0], AgentAction)
|
assert isinstance(results[0], AgentAction)
|
||||||
assert isinstance(results[1], AgentFinish)
|
assert isinstance(results[1], AgentFinish)
|
||||||
@@ -359,3 +363,96 @@ def test_integration_valid_and_invalid():
|
|||||||
|
|
||||||
|
|
||||||
# TODO: ADD TEST TO MAKE SURE ** REMOVAL DOESN'T MESS UP ANYTHING
|
# TODO: ADD TEST TO MAKE SURE ** REMOVAL DOESN'T MESS UP ANYTHING
|
||||||
|
|
||||||
|
|
||||||
|
def test_harmony_analysis_channel_parsing():
|
||||||
|
"""Test parsing OpenAI Harmony analysis channel (final answer)."""
|
||||||
|
text = "<|start|>assistant<|channel|>analysis<|message|>The temperature in SF is 72°F<|end|>"
|
||||||
|
result = parser.parse(text)
|
||||||
|
assert isinstance(result, parser.AgentFinish)
|
||||||
|
assert result.output == "The temperature in SF is 72°F"
|
||||||
|
assert "Analysis:" in result.thought
|
||||||
|
|
||||||
|
|
||||||
|
def test_harmony_commentary_channel_parsing():
|
||||||
|
"""Test parsing OpenAI Harmony commentary channel (tool action)."""
|
||||||
|
text = '<|start|>assistant<|channel|>commentary to=search<|message|>{"query": "temperature in SF"}<|call|>'
|
||||||
|
result = parser.parse(text)
|
||||||
|
assert isinstance(result, parser.AgentAction)
|
||||||
|
assert result.tool == "search"
|
||||||
|
assert result.tool_input == '{"query": "temperature in SF"}'
|
||||||
|
|
||||||
|
|
||||||
|
def test_harmony_commentary_with_thought():
|
||||||
|
"""Test Harmony commentary with reasoning before JSON."""
|
||||||
|
text = '<|start|>assistant<|channel|>commentary to=search<|message|>I need to find the temperature {"query": "SF weather"}<|call|>'
|
||||||
|
result = parser.parse(text)
|
||||||
|
assert isinstance(result, parser.AgentAction)
|
||||||
|
assert result.tool == "search"
|
||||||
|
assert result.thought == "I need to find the temperature"
|
||||||
|
assert result.tool_input == '{"query": "SF weather"}'
|
||||||
|
|
||||||
|
|
||||||
|
def test_harmony_multiple_blocks():
|
||||||
|
"""Test parsing multiple Harmony blocks (uses last one)."""
|
||||||
|
text = '''<|start|>assistant<|channel|>analysis<|message|>Thinking about this<|end|>
|
||||||
|
<|start|>assistant<|channel|>commentary to=search<|message|>{"query": "test"}<|call|>'''
|
||||||
|
result = parser.parse(text)
|
||||||
|
assert isinstance(result, parser.AgentAction)
|
||||||
|
assert result.tool == "search"
|
||||||
|
|
||||||
|
|
||||||
|
def test_harmony_format_detection():
|
||||||
|
"""Test that Harmony format is properly detected."""
|
||||||
|
harmony_text = "<|start|>assistant<|channel|>analysis<|message|>result<|end|>"
|
||||||
|
react_text = "Thought: test\nFinal Answer: result"
|
||||||
|
|
||||||
|
harmony_result = parser.parse(harmony_text)
|
||||||
|
react_result = parser.parse(react_text)
|
||||||
|
|
||||||
|
assert isinstance(harmony_result, parser.AgentFinish)
|
||||||
|
assert isinstance(react_result, parser.AgentFinish)
|
||||||
|
assert harmony_result.output == "result"
|
||||||
|
assert react_result.output == "result"
|
||||||
|
|
||||||
|
|
||||||
|
def test_harmony_invalid_format_error():
|
||||||
|
"""Test error handling for invalid Harmony format."""
|
||||||
|
text = "<|start|>assistant<|channel|>unknown<|message|>content<|end|>"
|
||||||
|
with pytest.raises(parser.OutputParserException) as exc_info:
|
||||||
|
parser.parse(text)
|
||||||
|
assert "Invalid Harmony Format" in str(exc_info.value)
|
||||||
|
|
||||||
|
|
||||||
|
def test_automatic_format_detection():
|
||||||
|
"""Test that the parser automatically detects different formats."""
|
||||||
|
react_action = "Thought: Let's search\nAction: search\nAction Input: query"
|
||||||
|
react_finish = "Thought: Done\nFinal Answer: result"
|
||||||
|
|
||||||
|
harmony_action = '<|start|>assistant<|channel|>commentary to=tool<|message|>{"input": "test"}<|call|>'
|
||||||
|
harmony_finish = "<|start|>assistant<|channel|>analysis<|message|>final result<|end|>"
|
||||||
|
|
||||||
|
assert isinstance(parser.parse(react_action), parser.AgentAction)
|
||||||
|
assert isinstance(parser.parse(react_finish), parser.AgentFinish)
|
||||||
|
assert isinstance(parser.parse(harmony_action), parser.AgentAction)
|
||||||
|
assert isinstance(parser.parse(harmony_finish), parser.AgentFinish)
|
||||||
|
|
||||||
|
|
||||||
|
def test_format_registry():
|
||||||
|
"""Test the format registry functionality."""
|
||||||
|
from crewai.agents.parser import _format_registry
|
||||||
|
|
||||||
|
assert 'react' in _format_registry._parsers
|
||||||
|
assert 'harmony' in _format_registry._parsers
|
||||||
|
|
||||||
|
react_text = "Thought: test\nAction: search\nAction Input: query"
|
||||||
|
harmony_text = "<|start|>assistant<|channel|>analysis<|message|>result<|end|>"
|
||||||
|
|
||||||
|
assert _format_registry._parsers['react'].can_parse(react_text)
|
||||||
|
assert _format_registry._parsers['harmony'].can_parse(harmony_text)
|
||||||
|
assert not _format_registry._parsers['react'].can_parse(harmony_text)
|
||||||
|
assert not _format_registry._parsers['harmony'].can_parse(react_text)
|
||||||
|
|
||||||
|
|
||||||
|
def test_backward_compatibility():
|
||||||
|
"""Test that all existing ReAct format tests still pass."""
|
||||||
|
|||||||
Reference in New Issue
Block a user