feat: Add Train feature for Crews (#686)

* feat: add training logic to agent and crew

* feat: add training logic to agent executor

* feat: add input parameter  to cli command

* feat: add utilities for the training logic

* feat: polish code, logic and add private variables

* feat: add docstring and type hinting to executor

* feat: add constant file, add constant to code

* feat: fix name of training handler function

* feat: remove unused var

* feat: change file handler file name

* feat: Add training handler file, class and change on the code

* feat: fix name error from file

* fix: change import to adapt to logic

* feat: add training handler test

* feat: add tests for file and training_handler

* feat: add test for task evaluator function

* feat: change text to fit in-screen

* feat: add test for train function

* feat: add test for agent training_handler function

* feat: add test for agent._use_trained_data

src/crewai/utilities/__init__.py

@@ -1,9 +1,22 @@
 from .converter import Converter, ConverterError
+from .file_handler import FileHandler
 from .i18n import I18N
 from .instructor import Instructor
 from .logger import Logger
+from .parser import YamlParser
 from .printer import Printer
 from .prompts import Prompts
 from .rpm_controller import RPMController
-from .fileHandler import FileHandler
-from .parser import YamlParser
+
+__all__ = [
+    "Converter",
+    "ConverterError",
+    "FileHandler",
+    "I18N",
+    "Instructor",
+    "Logger",
+    "Printer",
+    "Prompts",
+    "RPMController",
+    "YamlParser",
+]

src/crewai/utilities/constants.py

@@ -0,0 +1,2 @@
+TRAINING_DATA_FILE = "training_data.pkl"
+TRAINED_AGENTS_DATA_FILE = "trained_agents_data.pkl"

src/crewai/utilities/evaluators/task_evaluator.py

@@ -26,6 +26,18 @@ class TaskEvaluation(BaseModel):
     )
 
 
+class TrainingTaskEvaluation(BaseModel):
+    suggestions: List[str] = Field(
+        description="Based on the Human Feedbacks and the comparison between Initial Outputs and Improved outputs provide action items based on human_feedback for future tasks."
+    )
+    quality: float = Field(
+        description="A score from 0 to 10 evaluating on completion, quality, and overall performance from the improved output to the initial output based on the human feedback."
+    )
+    final_summary: str = Field(
+        description="A step by step action items to improve the next Agent based on the human-feedback and improved output."
+    )
+
+
 class TaskEvaluator:
     def __init__(self, original_agent):
         self.llm = original_agent.llm
@@ -59,3 +71,49 @@ class TaskEvaluator:
 
     def _is_gpt(self, llm) -> bool:
         return isinstance(llm, ChatOpenAI) and llm.openai_api_base is None
+
+    def evaluate_training_data(
+        self, training_data: dict, agent_id: str
+    ) -> TrainingTaskEvaluation:
+        """
+        Evaluate the training data based on the llm output, human feedback, and improved output.
+
+        Parameters:
+            - training_data (dict): The training data to be evaluated.
+            - agent_id (str): The ID of the agent.
+        """
+
+        output_training_data = training_data[agent_id]
+
+        final_aggregated_data = ""
+        for _, data in output_training_data.items():
+            final_aggregated_data += (
+                f"Initial Output:\n{data['initial_output']}\n\n"
+                f"Human Feedback:\n{data['human_feedback']}\n\n"
+                f"Improved Output:\n{data['improved_output']}\n\n"
+            )
+
+        evaluation_query = (
+            "Assess the quality of the training data based on the llm output, human feedback , and llm output improved result.\n\n"
+            f"{final_aggregated_data}"
+            "Please provide:\n"
+            "- Based on the Human Feedbacks and the comparison between Initial Outputs and Improved outputs provide action items based on human_feedback for future tasks\n"
+            "- A score from 0 to 10 evaluating on completion, quality, and overall performance from the improved output to the initial output based on the human feedback\n"
+        )
+        instructions = "I'm gonna convert this raw text into valid JSON."
+
+        if not self._is_gpt(self.llm):
+            model_schema = PydanticSchemaParser(
+                model=TrainingTaskEvaluation
+            ).get_schema()
+            instructions = f"{instructions}\n\nThe json should have the following structure, with the following keys:\n{model_schema}"
+
+        converter = Converter(
+            llm=self.llm,
+            text=evaluation_query,
+            model=TrainingTaskEvaluation,
+            instructions=instructions,
+        )
+
+        pydantic_result = converter.to_pydantic()
+        return pydantic_result

src/crewai/utilities/fileHandler.py

@@ -1,20 +0,0 @@
-import os
-from datetime import datetime
-
-
-class FileHandler:
-    """take care of file operations, currently it only logs messages to a file"""
-
-    def __init__(self, file_path):
-        if isinstance(file_path, bool):
-            self._path = os.path.join(os.curdir, "logs.txt")
-        elif isinstance(file_path, str):
-            self._path = file_path
-        else:
-            raise ValueError("file_path must be either a boolean or a string.")
-
-    def log(self, **kwargs):
-        now = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
-        message = f"{now}: ".join([f"{key}={value}" for key, value in kwargs.items()])
-        with open(self._path, "a", encoding = 'utf-8') as file:
-            file.write(message + "\n")

src/crewai/utilities/file_handler.py

@@ -0,0 +1,69 @@
+import os
+import pickle
+from datetime import datetime
+
+
+class FileHandler:
+    """take care of file operations, currently it only logs messages to a file"""
+
+    def __init__(self, file_path):
+        if isinstance(file_path, bool):
+            self._path = os.path.join(os.curdir, "logs.txt")
+        elif isinstance(file_path, str):
+            self._path = file_path
+        else:
+            raise ValueError("file_path must be either a boolean or a string.")
+
+    def log(self, **kwargs):
+        now = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
+        message = f"{now}: ".join([f"{key}={value}" for key, value in kwargs.items()])
+        with open(self._path, "a", encoding="utf-8") as file:
+            file.write(message + "\n")
+
+
+class PickleHandler:
+    def __init__(self, file_name: str) -> None:
+        """
+        Initialize the PickleHandler with the name of the file where data will be stored.
+        The file will be saved in the current directory.
+
+        Parameters:
+        - file_name (str): The name of the file for saving and loading data.
+        """
+        self.file_path = os.path.join(os.getcwd(), file_name)
+        self._initialize_file()
+
+    def _initialize_file(self) -> None:
+        """
+        Initialize the file with an empty dictionary if it does not exist or is empty.
+        """
+        if not os.path.exists(self.file_path) or os.path.getsize(self.file_path) == 0:
+            self.save({})  # Save an empty dictionary to initialize the file
+
+    def save(self, data) -> None:
+        """
+        Save the data to the specified file using pickle.
+
+        Parameters:
+        - data (object): The data to be saved.
+        """
+        with open(self.file_path, "wb") as file:
+            pickle.dump(data, file)
+
+    def load(self) -> dict:
+        """
+        Load the data from the specified file using pickle.
+
+        Returns:
+        - dict: The data loaded from the file.
+        """
+        if not os.path.exists(self.file_path) or os.path.getsize(self.file_path) == 0:
+            return {}  # Return an empty dictionary if the file does not exist or is empty
+
+        with open(self.file_path, "rb") as file:
+            try:
+                return pickle.load(file)
+            except EOFError:
+                return {}  # Return an empty dictionary if the file is empty or corrupted
+            except Exception:
+                raise  # Raise any other exceptions that occur during loading

src/crewai/utilities/training_handler.py

@@ -0,0 +1,31 @@
+from crewai.utilities.file_handler import PickleHandler
+
+
+class CrewTrainingHandler(PickleHandler):
+    def save_trained_data(self, agent_id: str, trained_data: dict) -> None:
+        """
+        Save the trained data for a specific agent.
+
+        Parameters:
+        - agent_id (str): The ID of the agent.
+        - trained_data (dict): The trained data to be saved.
+        """
+        data = self.load()
+        data[agent_id] = trained_data
+        self.save(data)
+
+    def append(self, train_iteration: int, agent_id: str, new_data) -> None:
+        """
+        Append new data to the existing pickle file.
+
+        Parameters:
+        - new_data (object): The new data to be appended.
+        """
+        data = self.load()
+
+        if agent_id in data:
+            data[agent_id][train_iteration] = new_data
+        else:
+            data[agent_id] = {train_iteration: new_data}
+
+        self.save(data)