Feature/use converter instead of manually trimming (#894)

* Exploring output being passed to tool selector to see if we can better format data

* WIP. Adding JSON repair functionality

* Almost done implementing JSON repair. Testing fixes vs current base case.

* More action cleanup with additional tests

* WIP. Trying to figure out what is going on with tool descriptions

* Update tool description generation

* WIP. Trying to find out what is causing the tools to duplicate

* Replacing tools properly instead of duplicating them accidentally

* Fixing issues for MR

* Update dependencies for JSON_REPAIR

* More cleaning up pull request

* preppering for call

* Fix type-checking issues

---------

Co-authored-by: João Moura <joaomdmoura@gmail.com>

src/crewai/agents/agent_builder/utilities/base_agent_tool.py

@@ -24,6 +24,7 @@ class BaseAgentTools(BaseModel, ABC):
             is_list = coworker.startswith("[") and coworker.endswith("]")
             if is_list:
                 coworker = coworker[1:-1].split(",")[0]
+
         return coworker
 
     def delegate_work(
@@ -40,11 +41,13 @@ class BaseAgentTools(BaseModel, ABC):
         coworker = self._get_coworker(coworker, **kwargs)
         return self._execute(coworker, question, context)
 
-    def _execute(self, agent: Union[str, None], task: str, context: Union[str, None]):
+    def _execute(
+        self, agent_name: Union[str, None], task: str, context: Union[str, None]
+    ):
         """Execute the command."""
         try:
-            if agent is None:
-                agent = ""
+            if agent_name is None:
+                agent_name = ""
 
             # It is important to remove the quotes from the agent name.
             # The reason we have to do this is because less-powerful LLM's
@@ -53,7 +56,7 @@ class BaseAgentTools(BaseModel, ABC):
             # {"task": "....", "coworker": "....
             # when it should look like this:
             # {"task": "....", "coworker": "...."}
-            agent_name = agent.casefold().replace('"', "").replace("\n", "")
+            agent_name = agent_name.casefold().replace('"', "").replace("\n", "")
 
             agent = [  # type: ignore # Incompatible types in assignment (expression has type "list[BaseAgent]", variable has type "str | None")
                 available_agent
@@ -75,9 +78,9 @@ class BaseAgentTools(BaseModel, ABC):
             )
 
         agent = agent[0]
-        task = Task(  # type: ignore # Incompatible types in assignment (expression has type "Task", variable has type "str")
+        task_with_assigned_agent = Task(  # type: ignore # Incompatible types in assignment (expression has type "Task", variable has type "str")
             description=task,
             agent=agent,
             expected_output="Your best answer to your coworker asking you this, accounting for the context shared.",
         )
-        return agent.execute_task(task, context)  # type: ignore # "str" has no attribute "execute_task"
+        return agent.execute_task(task_with_assigned_agent, context)

src/crewai/agents/agent_builder/utilities/base_output_converter.py

@@ -1,7 +1,7 @@
 from abc import ABC, abstractmethod
 from typing import Any, Optional
 
-from pydantic import BaseModel, Field, PrivateAttr
+from pydantic import BaseModel, Field
 
 
 class OutputConverter(BaseModel, ABC):
@@ -21,7 +21,6 @@ class OutputConverter(BaseModel, ABC):
         max_attempts (int): Maximum number of conversion attempts (default: 3).
     """
 
-    _is_gpt: bool = PrivateAttr(default=True)
     text: str = Field(description="Text to be converted.")
     llm: Any = Field(description="The language model to be used to convert the text.")
     model: Any = Field(description="The model to be used to convert the text.")
@@ -41,7 +40,8 @@ class OutputConverter(BaseModel, ABC):
         """Convert text to json."""
         pass
 
-    @abstractmethod  # type: ignore #  Name "_is_gpt" already defined on line 25
-    def _is_gpt(self, llm):  # type: ignore #  Name "_is_gpt" already defined on line 25
+    @property
+    @abstractmethod
+    def is_gpt(self) -> bool:
         """Return if llm provided is of gpt from openai."""
         pass

src/crewai/agents/executor.py

@@ -1,14 +1,6 @@
 import threading
 import time
-from typing import (
-    Any,
-    Dict,
-    Iterator,
-    List,
-    Optional,
-    Tuple,
-    Union,
-)
+from typing import Any, Dict, Iterator, List, Optional, Tuple, Union
 
 from langchain.agents import AgentExecutor
 from langchain.agents.agent import ExceptionTool
@@ -19,9 +11,7 @@ from langchain_core.tools import BaseTool
 from langchain_core.utils.input import get_color_mapping
 from pydantic import InstanceOf
 
-from crewai.agents.agent_builder.base_agent_executor_mixin import (
-    CrewAgentExecutorMixin,
-)
+from crewai.agents.agent_builder.base_agent_executor_mixin import CrewAgentExecutorMixin
 from crewai.agents.tools_handler import ToolsHandler
 from crewai.tools.tool_usage import ToolUsage, ToolUsageErrorException
 from crewai.utilities import I18N

src/crewai/agents/parser.py

@@ -1,6 +1,7 @@
 import re
 from typing import Any, Union
 
+from json_repair import repair_json
 from langchain.agents.output_parsers import ReActSingleInputOutputParser
 from langchain_core.agents import AgentAction, AgentFinish
 from langchain_core.exceptions import OutputParserException
@@ -48,11 +49,15 @@ class CrewAgentParser(ReActSingleInputOutputParser):
                 raise OutputParserException(
                     f"{FINAL_ANSWER_AND_PARSABLE_ACTION_ERROR_MESSAGE}: {text}"
                 )
-            action = action_match.group(1).strip()
-            action_input = action_match.group(2)
-            tool_input = action_input.strip(" ")
-            tool_input = tool_input.strip('"')
-            return AgentAction(action, tool_input, text)
+            action = action_match.group(1)
+            clean_action = self._clean_action(action)
+
+            action_input = action_match.group(2).strip()
+
+            tool_input = action_input.strip(" ").strip('"')
+            safe_tool_input = self._safe_repair_json(tool_input)
+
+            return AgentAction(clean_action, safe_tool_input, text)
 
         elif includes_answer:
             return AgentFinish(
@@ -87,3 +92,30 @@ class CrewAgentParser(ReActSingleInputOutputParser):
                 llm_output=text,
                 send_to_llm=True,
             )
+
+    def _clean_action(self, text: str) -> str:
+        """Clean action string by removing non-essential formatting characters."""
+        return re.sub(r"^\s*\*+\s*|\s*\*+\s*$", "", text).strip()
+
+    def _safe_repair_json(self, tool_input: str) -> str:
+        UNABLE_TO_REPAIR_JSON_RESULTS = ['""', "{}"]
+
+        # Skip repair if the input starts and ends with square brackets
+        # Explanation: The JSON parser has issues handling inputs that are enclosed in square brackets ('[]').
+        # These are typically valid JSON arrays or strings that do not require repair. Attempting to repair such inputs
+        # might lead to unintended alterations, such as wrapping the entire input in additional layers or modifying
+        # the structure in a way that changes its meaning. By skipping the repair for inputs that start and end with
+        # square brackets, we preserve the integrity of these valid JSON structures and avoid unnecessary modifications.
+        if tool_input.startswith("[") and tool_input.endswith("]"):
+            return tool_input
+
+        # Before repair, handle common LLM issues:
+        # 1. Replace """ with " to avoid JSON parser errors
+
+        tool_input = tool_input.replace('"""', '"')
+
+        result = repair_json(tool_input)
+        if result in UNABLE_TO_REPAIR_JSON_RESULTS:
+            return tool_input
+
+        return str(result)