feat: Support list of YAML files for agents_config and tasks_config in CrewBase

Co-Authored-By: Joe Moura <joao@crewai.com>

src/crewai/project/crew_base.py

@@ -25,11 +25,22 @@ def CrewBase(cls: T) -> T:
         def __init__(self, *args, **kwargs):
             super().__init__(*args, **kwargs)
 
-            agents_config_path = self.base_directory / self.original_agents_config_path
-            tasks_config_path = self.base_directory / self.original_tasks_config_path
+            agents_config_paths = []
+            tasks_config_paths = []
+            
+            if isinstance(self.original_agents_config_path, list):
+                agents_config_paths = [self.base_directory / path for path in self.original_agents_config_path]
+            else:
+                agents_config_paths = self.base_directory / self.original_agents_config_path
+                
+            if isinstance(self.original_tasks_config_path, list):
+                tasks_config_paths = [self.base_directory / path for path in self.original_tasks_config_path]
+            else:
+                tasks_config_paths = self.base_directory / self.original_tasks_config_path
 
-            self.agents_config = self.load_yaml(agents_config_path)
-            self.tasks_config = self.load_yaml(tasks_config_path)
+            # Load and merge configurations
+            self.agents_config = self.load_and_merge_yaml_configs(agents_config_paths)
+            self.tasks_config = self.load_and_merge_yaml_configs(tasks_config_paths)
 
             self.map_all_agent_variables()
             self.map_all_task_variables()
@@ -75,6 +86,31 @@ def CrewBase(cls: T) -> T:
             except FileNotFoundError:
                 print(f"File not found: {config_path}")
                 raise
+                
+        def load_and_merge_yaml_configs(self, config_paths: list[Path] | Path) -> dict:
+            """
+            Load and merge configurations from multiple YAML files or a single file.
+            Later files in the list will override earlier ones for duplicate keys.
+            
+            Args:
+                config_paths: A Path object or list of Path objects pointing to YAML files
+                
+            Returns:
+                A dictionary with merged configurations
+            """
+            if isinstance(config_paths, Path):
+                return self.load_yaml(config_paths)
+                
+            result = {}
+            for path in config_paths:
+                config = self.load_yaml(path)
+                if config:
+                    for key, value in config.items():
+                        if key in result and isinstance(result[key], dict) and isinstance(value, dict):
+                            result[key].update(value)
+                        else:
+                            result[key] = value
+            return result
 
         def _get_all_functions(self):
             return {

tests/config/multi/agents1.yaml

@@ -0,0 +1,5 @@
+test_agent1:
+  role: Test Agent 1
+  goal: Test Goal 1
+  backstory: Test Backstory 1
+  verbose: true

tests/config/multi/agents2.yaml

@@ -0,0 +1,8 @@
+test_agent1:
+  role: Updated Test Agent 1
+  goal: Updated Test Goal 1
+test_agent2:
+  role: Test Agent 2
+  goal: Test Goal 2
+  backstory: Test Backstory 2
+  verbose: true

tests/config/multi/tasks1.yaml

@@ -0,0 +1,4 @@
+test_task1:
+  description: Test Description 1
+  expected_output: Test Output 1
+  agent: test_agent1

tests/config/multi/tasks2.yaml

@@ -0,0 +1,6 @@
+test_task1:
+  description: Updated Test Description 1
+test_task2:
+  description: Test Description 2
+  expected_output: Test Output 2
+  agent: test_agent2

tests/project_test.py

@@ -184,3 +184,68 @@ def test_multiple_before_after_kickoff():
     assert "plants" in result.raw, "First before_kickoff not executed"
     assert "processed first" in result.raw, "First after_kickoff not executed"
     assert "processed second" in result.raw, "Second after_kickoff not executed"
+
+
+@pytest.mark.vcr(filter_headers=["authorization"])
+def test_multiple_yaml_configs():
+    @CrewBase
+    class MultiConfigCrew:
+        agents_config = ["config/multi/agents1.yaml", "config/multi/agents2.yaml"]
+        tasks_config = ["config/multi/tasks1.yaml", "config/multi/tasks2.yaml"]
+
+        @agent
+        def test_agent1(self):
+            return Agent(config=self.agents_config["test_agent1"])
+
+        @agent
+        def test_agent2(self):
+            return Agent(config=self.agents_config["test_agent2"])
+
+        @task
+        def test_task1(self):
+            task_config = self.tasks_config["test_task1"].copy()
+            if isinstance(task_config.get("agent"), str):
+                agent_name = task_config.pop("agent")
+                if hasattr(self, agent_name):
+                    task_config["agent"] = getattr(self, agent_name)()
+            return Task(config=task_config)
+
+        @task
+        def test_task2(self):
+            task_config = self.tasks_config["test_task2"].copy()
+            if isinstance(task_config.get("agent"), str):
+                agent_name = task_config.pop("agent")
+                if hasattr(self, agent_name):
+                    task_config["agent"] = getattr(self, agent_name)()
+            return Task(config=task_config)
+
+        @crew
+        def crew(self):
+            return Crew(agents=self.agents, tasks=self.tasks, verbose=True)
+
+    crew = MultiConfigCrew()
+    
+    assert "test_agent1" in crew.agents_config
+    assert "test_agent2" in crew.agents_config
+    
+    assert crew.agents_config["test_agent1"]["role"] == "Updated Test Agent 1"
+    assert crew.agents_config["test_agent1"]["goal"] == "Updated Test Goal 1"
+    assert crew.agents_config["test_agent1"]["backstory"] == "Test Backstory 1"
+    assert crew.agents_config["test_agent1"]["verbose"] is True
+    
+    assert "test_task1" in crew.tasks_config
+    assert "test_task2" in crew.tasks_config
+    
+    assert crew.tasks_config["test_task1"]["description"] == "Updated Test Description 1"
+    assert crew.tasks_config["test_task1"]["expected_output"] == "Test Output 1"
+    assert crew.tasks_config["test_task1"]["agent"].role == "Updated Test Agent 1"
+
+    agent1 = crew.test_agent1()
+    agent2 = crew.test_agent2()
+    task1 = crew.test_task1()
+    task2 = crew.test_task2()
+    
+    assert agent1.role == "Updated Test Agent 1"
+    assert agent2.role == "Test Agent 2"
+    assert task1.description == "Updated Test Description 1"
+    assert task2.description == "Test Description 2"