From 4469461b38f58e785826766ffd7a3f991d5b5b24 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Tue, 31 Dec 2024 01:56:38 -0300 Subject: [PATCH] fix: Include agent knowledge in planning process (#1818) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * test: Add test demonstrating knowledge not included in planning process Issue #1703: Add test to verify that agent knowledge sources are not currently included in the planning process. This test will help validate the fix once implemented. - Creates agent with knowledge sources - Verifies knowledge context missing from planning - Checks other expected components are present Co-Authored-By: Joe Moura * fix: Include agent knowledge in planning process Issue #1703: Integrate agent knowledge sources into planning summaries - Add agent_knowledge field to task summaries in planning_handler - Update test to verify knowledge inclusion - Ensure knowledge context is available during planning phase The planning agent now has access to agent knowledge when creating task execution plans, allowing for better informed planning decisions. Co-Authored-By: Joe Moura * style: Fix import sorting in test_knowledge_planning.py - Reorganize imports according to ruff linting rules - Fix I001 linting error Co-Authored-By: Joe Moura * test: Update task summary assertions to include knowledge field Co-Authored-By: Joe Moura * fix: Update ChromaDB mock path and fix knowledge string formatting Co-Authored-By: Joe Moura * fix: Improve knowledge integration in planning process with error handling Co-Authored-By: Joe Moura * fix: Update task summary format for empty tools and knowledge - Change empty tools message to 'agent has no tools' - Remove agent_knowledge field when empty - Update test assertions to match new format - Improve test messages for clarity Co-Authored-By: Joe Moura * fix: Update string formatting for agent tools in task summary Co-Authored-By: Joe Moura * fix: Update string formatting for agent tools in task summary Co-Authored-By: Joe Moura * fix: Update string formatting for agent tools and knowledge in task summary Co-Authored-By: Joe Moura * fix: Update knowledge field formatting in task summary Co-Authored-By: Joe Moura * style: Fix import sorting in test_planning_handler.py Co-Authored-By: Joe Moura * style: Fix import sorting order in test_planning_handler.py Co-Authored-By: Joe Moura * test: Add ChromaDB mocking to test_create_tasks_summary_with_knowledge_and_tools Co-Authored-By: Joe Moura --------- Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Co-authored-by: Joe Moura Co-authored-by: João Moura --- src/crewai/utilities/planning_handler.py | 34 +++++++-- tests/utilities/test_knowledge_planning.py | 84 ++++++++++++++++++++++ tests/utilities/test_planning_handler.py | 73 ++++++++++++++++++- 3 files changed, 184 insertions(+), 7 deletions(-) create mode 100644 tests/utilities/test_knowledge_planning.py diff --git a/src/crewai/utilities/planning_handler.py b/src/crewai/utilities/planning_handler.py index 590f42389..21ee093a1 100644 --- a/src/crewai/utilities/planning_handler.py +++ b/src/crewai/utilities/planning_handler.py @@ -1,3 +1,5 @@ +import json +import logging from typing import Any, List, Optional from pydantic import BaseModel, Field @@ -5,6 +7,8 @@ from pydantic import BaseModel, Field from crewai.agent import Agent from crewai.task import Task +logger = logging.getLogger(__name__) + class PlanPerTask(BaseModel): task: str = Field(..., description="The task for which the plan is created") @@ -68,19 +72,39 @@ class CrewPlanner: output_pydantic=PlannerTaskPydanticOutput, ) + def _get_agent_knowledge(self, task: Task) -> List[str]: + """ + Safely retrieve knowledge source content from the task's agent. + + Args: + task: The task containing an agent with potential knowledge sources + + Returns: + List[str]: A list of knowledge source strings + """ + try: + if task.agent and task.agent.knowledge_sources: + return [source.content for source in task.agent.knowledge_sources] + except AttributeError: + logger.warning("Error accessing agent knowledge sources") + return [] + def _create_tasks_summary(self) -> str: """Creates a summary of all tasks.""" tasks_summary = [] for idx, task in enumerate(self.tasks): - tasks_summary.append( - f""" + knowledge_list = self._get_agent_knowledge(task) + task_summary = f""" Task Number {idx + 1} - {task.description} "task_description": {task.description} "task_expected_output": {task.expected_output} "agent": {task.agent.role if task.agent else "None"} "agent_goal": {task.agent.goal if task.agent else "None"} "task_tools": {task.tools} - "agent_tools": {task.agent.tools if task.agent else "None"} - """ - ) + "agent_tools": %s%s""" % ( + f"[{', '.join(str(tool) for tool in task.agent.tools)}]" if task.agent and task.agent.tools else '"agent has no tools"', + f',\n "agent_knowledge": "[\\"{knowledge_list[0]}\\"]"' if knowledge_list and str(knowledge_list) != "None" else "" + ) + + tasks_summary.append(task_summary) return " ".join(tasks_summary) diff --git a/tests/utilities/test_knowledge_planning.py b/tests/utilities/test_knowledge_planning.py new file mode 100644 index 000000000..37b6df69f --- /dev/null +++ b/tests/utilities/test_knowledge_planning.py @@ -0,0 +1,84 @@ +""" +Tests for verifying the integration of knowledge sources in the planning process. +This module ensures that agent knowledge is properly included during task planning. +""" + +from unittest.mock import patch + +import pytest + +from crewai.agent import Agent +from crewai.knowledge.source.string_knowledge_source import StringKnowledgeSource +from crewai.task import Task +from crewai.utilities.planning_handler import CrewPlanner + + +@pytest.fixture +def mock_knowledge_source(): + """ + Create a mock knowledge source with test content. + Returns: + StringKnowledgeSource: + A knowledge source containing AI-related test content + """ + content = """ + Important context about AI: + 1. AI systems use machine learning algorithms + 2. Neural networks are a key component + 3. Training data is essential for good performance + """ + return StringKnowledgeSource(content=content) + +@patch('crewai.knowledge.storage.knowledge_storage.chromadb') +def test_knowledge_included_in_planning(mock_chroma): + """Test that verifies knowledge sources are properly included in planning.""" + # Mock ChromaDB collection + mock_collection = mock_chroma.return_value.get_or_create_collection.return_value + mock_collection.add.return_value = None + + # Create an agent with knowledge + agent = Agent( + role="AI Researcher", + goal="Research and explain AI concepts", + backstory="Expert in artificial intelligence", + knowledge_sources=[ + StringKnowledgeSource( + content="AI systems require careful training and validation." + ) + ] + ) + + # Create a task for the agent + task = Task( + description="Explain the basics of AI systems", + expected_output="A clear explanation of AI fundamentals", + agent=agent + ) + + # Create a crew planner + planner = CrewPlanner([task], None) + + # Get the task summary + task_summary = planner._create_tasks_summary() + + # Verify that knowledge is included in planning when present + assert "AI systems require careful training" in task_summary, \ + "Knowledge content should be present in task summary when knowledge exists" + assert '"agent_knowledge"' in task_summary, \ + "agent_knowledge field should be present in task summary when knowledge exists" + + # Verify that knowledge is properly formatted + assert isinstance(task.agent.knowledge_sources, list), \ + "Knowledge sources should be stored in a list" + assert len(task.agent.knowledge_sources) > 0, \ + "At least one knowledge source should be present" + assert task.agent.knowledge_sources[0].content in task_summary, \ + "Knowledge source content should be included in task summary" + + # Verify that other expected components are still present + assert task.description in task_summary, \ + "Task description should be present in task summary" + assert task.expected_output in task_summary, \ + "Expected output should be present in task summary" + assert agent.role in task_summary, \ + "Agent role should be present in task summary" diff --git a/tests/utilities/test_planning_handler.py b/tests/utilities/test_planning_handler.py index 85101606d..e15877c9f 100644 --- a/tests/utilities/test_planning_handler.py +++ b/tests/utilities/test_planning_handler.py @@ -1,10 +1,14 @@ -from unittest.mock import patch +from typing import Optional +from unittest.mock import MagicMock, patch import pytest +from pydantic import BaseModel from crewai.agent import Agent +from crewai.knowledge.source.string_knowledge_source import StringKnowledgeSource from crewai.task import Task from crewai.tasks.task_output import TaskOutput +from crewai.tools.base_tool import BaseTool from crewai.utilities.planning_handler import ( CrewPlanner, PlannerTaskPydanticOutput, @@ -92,7 +96,72 @@ class TestCrewPlanner: tasks_summary = crew_planner._create_tasks_summary() assert isinstance(tasks_summary, str) assert tasks_summary.startswith("\n Task Number 1 - Task 1") - assert tasks_summary.endswith('"agent_tools": []\n ') + assert '"agent_tools": "agent has no tools"' in tasks_summary + # Knowledge field should not be present when empty + assert '"agent_knowledge"' not in tasks_summary + + @patch('crewai.knowledge.storage.knowledge_storage.chromadb') + def test_create_tasks_summary_with_knowledge_and_tools(self, mock_chroma): + """Test task summary generation with both knowledge and tools present.""" + # Mock ChromaDB collection + mock_collection = mock_chroma.return_value.get_or_create_collection.return_value + mock_collection.add.return_value = None + + # Create mock tools with proper string descriptions and structured tool support + class MockTool(BaseTool): + name: str + description: str + + def __init__(self, name: str, description: str): + tool_data = {"name": name, "description": description} + super().__init__(**tool_data) + + def __str__(self): + return self.name + + def __repr__(self): + return self.name + + def to_structured_tool(self): + return self + + def _run(self, *args, **kwargs): + pass + + def _generate_description(self) -> str: + """Override _generate_description to avoid args_schema handling.""" + return self.description + + tool1 = MockTool("tool1", "Tool 1 description") + tool2 = MockTool("tool2", "Tool 2 description") + + # Create a task with knowledge and tools + task = Task( + description="Task with knowledge and tools", + expected_output="Expected output", + agent=Agent( + role="Test Agent", + goal="Test Goal", + backstory="Test Backstory", + tools=[tool1, tool2], + knowledge_sources=[ + StringKnowledgeSource(content="Test knowledge content") + ] + ) + ) + + # Create planner with the new task + planner = CrewPlanner([task], None) + tasks_summary = planner._create_tasks_summary() + + # Verify task summary content + assert isinstance(tasks_summary, str) + assert task.description in tasks_summary + assert task.expected_output in tasks_summary + assert '"agent_tools": [tool1, tool2]' in tasks_summary + assert '"agent_knowledge": "[\\"Test knowledge content\\"]"' in tasks_summary + assert task.agent.role in tasks_summary + assert task.agent.goal in tasks_summary def test_handle_crew_planning_different_llm(self, crew_planner_different_llm): with patch.object(Task, "execute_sync") as execute: