From 3f8b15f34c86fc3f6b5ea995257704d94f30574a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Moura?= Date: Thu, 26 Sep 2024 14:17:15 -0300 Subject: [PATCH] Fixing trainign feature --- src/crewai/agent.py | 10 ++++++---- src/crewai/agents/crew_agent_executor.py | 19 +++++++------------ .../utilities/evaluators/task_evaluator.py | 10 +++++----- 3 files changed, 18 insertions(+), 21 deletions(-) diff --git a/src/crewai/agent.py b/src/crewai/agent.py index dbeb955b8..af7d245b9 100644 --- a/src/crewai/agent.py +++ b/src/crewai/agent.py @@ -344,8 +344,9 @@ class Agent(BaseAgent): human_feedbacks = [ i["human_feedback"] for i in data.get(agent_id, {}).values() ] - task_prompt += "You MUST follow these feedbacks: \n " + "\n - ".join( - human_feedbacks + task_prompt += ( + "\n\nYou MUST follow these instructions: \n " + + "\n - ".join(human_feedbacks) ) return task_prompt @@ -354,8 +355,9 @@ class Agent(BaseAgent): """Use trained data for the agent task prompt to improve output.""" if data := CrewTrainingHandler(TRAINED_AGENTS_DATA_FILE).load(): if trained_data_output := data.get(self.role): - task_prompt += "You MUST follow these feedbacks: \n " + "\n - ".join( - trained_data_output["suggestions"] + task_prompt += ( + "\n\nYou MUST follow these instructions: \n - " + + "\n - ".join(trained_data_output["suggestions"]) ) return task_prompt diff --git a/src/crewai/agents/crew_agent_executor.py b/src/crewai/agents/crew_agent_executor.py index 81aa2a740..5d78bf5e7 100644 --- a/src/crewai/agents/crew_agent_executor.py +++ b/src/crewai/agents/crew_agent_executor.py @@ -96,6 +96,9 @@ class CrewAgentExecutor(CrewAgentExecutorMixin): self.messages.append(self._format_msg(f"Feedback: {human_feedback}")) formatted_answer = self._invoke_loop() + if self.crew and self.crew._train: + self._handle_crew_training_output(formatted_answer) + return {"output": formatted_answer.output} def _invoke_loop(self, formatted_answer=None): @@ -294,24 +297,16 @@ class CrewAgentExecutor(CrewAgentExecutorMixin): ) -> None: """Function to handle the process of the training data.""" agent_id = str(self.agent.id) - if ( CrewTrainingHandler(TRAINING_DATA_FILE).load() and not self.ask_for_human_input ): training_data = CrewTrainingHandler(TRAINING_DATA_FILE).load() if training_data.get(agent_id): - if self.crew is not None and hasattr(self.crew, "_train_iteration"): - training_data[agent_id][self.crew._train_iteration][ - "improved_output" - ] = result.output - CrewTrainingHandler(TRAINING_DATA_FILE).save(training_data) - else: - self._logger.log( - "error", - "Invalid crew or missing _train_iteration attribute.", - color="red", - ) + training_data[agent_id][self.crew._train_iteration][ + "improved_output" + ] = result.output + CrewTrainingHandler(TRAINING_DATA_FILE).save(training_data) if self.ask_for_human_input and human_feedback is not None: training_data = { diff --git a/src/crewai/utilities/evaluators/task_evaluator.py b/src/crewai/utilities/evaluators/task_evaluator.py index 112ed3d8a..6246a16e4 100644 --- a/src/crewai/utilities/evaluators/task_evaluator.py +++ b/src/crewai/utilities/evaluators/task_evaluator.py @@ -49,7 +49,7 @@ class TaskEvaluation(BaseModel): class TrainingTaskEvaluation(BaseModel): suggestions: List[str] = Field( - description="Based on the Human Feedbacks and the comparison between Initial Outputs and Improved outputs provide action items based on human_feedback for future tasks." + description="List of clear, actionable instructions derived from the Human Feedbacks to enhance the Agent's performance. Analyze the differences between Initial Outputs and Improved Outputs to generate specific action items for future tasks. Ensure all key and specific points from the human feedback are incorporated into these instructions." ) quality: float = Field( description="A score from 0 to 10 evaluating on completion, quality, and overall performance from the improved output to the initial output based on the human feedback." @@ -107,16 +107,16 @@ class TaskEvaluator: final_aggregated_data = "" for _, data in output_training_data.items(): final_aggregated_data += ( - f"Initial Output:\n{data.get('initial_output', '')}\n\n" - f"Human Feedback:\n{data.get('human_feedback', '')}\n\n" - f"Improved Output:\n{data.get('improved_output', '')}\n\n" + f"Initial Output:\n{data['initial_output']}\n\n" + f"Human Feedback:\n{data['human_feedback']}\n\n" + f"Improved Output:\n{data['improved_output']}\n\n" ) evaluation_query = ( "Assess the quality of the training data based on the llm output, human feedback , and llm output improved result.\n\n" f"{final_aggregated_data}" "Please provide:\n" - "- Based on the Human Feedbacks and the comparison between Initial Outputs and Improved outputs provide action items based on human_feedback for future tasks\n" + "- Provide a list of clear, actionable instructions derived from the Human Feedbacks to enhance the Agent's performance. Analyze the differences between Initial Outputs and Improved Outputs to generate specific action items for future tasks. Ensure all key and specificpoints from the human feedback are incorporated into these instructions.\n" "- A score from 0 to 10 evaluating on completion, quality, and overall performance from the improved output to the initial output based on the human feedback\n" ) instructions = "I'm gonna convert this raw text into valid JSON."