From c64c0698c550bfd08e5cbe7d45e1f7bc224f2112 Mon Sep 17 00:00:00 2001 From: Lorenze Jay Date: Tue, 18 Feb 2025 14:29:39 -0800 Subject: [PATCH] Refactor event system and improve crew testing - Extract base CrewEvent class to a new base_events.py module - Update event imports across multiple event-related files - Modify CrewTestStartedEvent to use eval_llm instead of openai_model_name - Add LLM creation validation in crew testing method - Improve type handling and event consistency --- src/crewai/crew.py | 6 +++++- src/crewai/utilities/events/agent_events.py | 2 +- src/crewai/utilities/events/base_events.py | 10 ++++++++++ src/crewai/utilities/events/crew_events.py | 14 ++++---------- src/crewai/utilities/events/crewai_event_bus.py | 2 +- src/crewai/utilities/events/event_listener.py | 2 +- src/crewai/utilities/events/flow_events.py | 2 +- src/crewai/utilities/events/tool_usage_events.py | 2 +- 8 files changed, 24 insertions(+), 16 deletions(-) create mode 100644 src/crewai/utilities/events/base_events.py diff --git a/src/crewai/crew.py b/src/crewai/crew.py index 7bb11321a..802402a55 100644 --- a/src/crewai/crew.py +++ b/src/crewai/crew.py @@ -1201,6 +1201,10 @@ class Crew(BaseModel): ) -> None: """Test and evaluate the Crew with the given inputs for n iterations concurrently using concurrent.futures.""" try: + eval_llm = create_llm(eval_llm) + if not eval_llm: + raise ValueError("Failed to create LLM instance.") + crewai_event_bus.emit( self, CrewTestStartedEvent( @@ -1211,7 +1215,7 @@ class Crew(BaseModel): ), ) test_crew = self.copy() - evaluator = CrewEvaluator(test_crew, eval_llm) + evaluator = CrewEvaluator(test_crew, eval_llm) # type: ignore[arg-type] for i in range(1, n_iterations + 1): evaluator.set_iteration(i) diff --git a/src/crewai/utilities/events/agent_events.py b/src/crewai/utilities/events/agent_events.py index 4fe6baf99..ed0480957 100644 --- a/src/crewai/utilities/events/agent_events.py +++ b/src/crewai/utilities/events/agent_events.py @@ -4,7 +4,7 @@ from crewai.agents.agent_builder.base_agent import BaseAgent from crewai.tools.base_tool import BaseTool from crewai.tools.structured_tool import CrewStructuredTool -from .crew_events import CrewEvent +from .base_events import CrewEvent if TYPE_CHECKING: from crewai.agents.agent_builder.base_agent import BaseAgent diff --git a/src/crewai/utilities/events/base_events.py b/src/crewai/utilities/events/base_events.py new file mode 100644 index 000000000..b29ae6fb6 --- /dev/null +++ b/src/crewai/utilities/events/base_events.py @@ -0,0 +1,10 @@ +from datetime import datetime + +from pydantic import BaseModel, Field + + +class CrewEvent(BaseModel): + """Base class for all crew events""" + + timestamp: datetime = Field(default_factory=datetime.now) + type: str diff --git a/src/crewai/utilities/events/crew_events.py b/src/crewai/utilities/events/crew_events.py index b92de856d..13dfd8e34 100644 --- a/src/crewai/utilities/events/crew_events.py +++ b/src/crewai/utilities/events/crew_events.py @@ -1,14 +1,8 @@ -from datetime import datetime -from typing import Any, Dict, Optional +from typing import Any, Dict, Optional, Union -from pydantic import BaseModel, Field +from pydantic import InstanceOf - -class CrewEvent(BaseModel): - """Base class for all crew events""" - - timestamp: datetime = Field(default_factory=datetime.now) - type: str +from crewai.utilities.events.base_events import CrewEvent class CrewKickoffStartedEvent(CrewEvent): @@ -67,7 +61,7 @@ class CrewTestStartedEvent(CrewEvent): crew_name: Optional[str] n_iterations: int - openai_model_name: Optional[str] + eval_llm: Optional[Union[str, Any]] inputs: Optional[Dict[str, Any]] type: str = "crew_test_started" diff --git a/src/crewai/utilities/events/crewai_event_bus.py b/src/crewai/utilities/events/crewai_event_bus.py index ffb931e0c..c0cf50908 100644 --- a/src/crewai/utilities/events/crewai_event_bus.py +++ b/src/crewai/utilities/events/crewai_event_bus.py @@ -4,7 +4,7 @@ from typing import Any, Callable, Dict, List, Type, TypeVar, cast from blinker import Signal -from crewai.utilities.events.crew_events import CrewEvent +from crewai.utilities.events.base_events import CrewEvent from crewai.utilities.events.event_types import EventTypes EventT = TypeVar("EventT", bound=CrewEvent) diff --git a/src/crewai/utilities/events/event_listener.py b/src/crewai/utilities/events/event_listener.py index ada20147c..7f461429d 100644 --- a/src/crewai/utilities/events/event_listener.py +++ b/src/crewai/utilities/events/event_listener.py @@ -85,7 +85,7 @@ class EventListener(BaseEventListener): cloned_crew, event.n_iterations, event.inputs, - event.openai_model_name, + event.eval_llm, ) self.logger.log( f"🚀 Crew '{event.crew_name}' started test", diff --git a/src/crewai/utilities/events/flow_events.py b/src/crewai/utilities/events/flow_events.py index b3daf889f..435d64214 100644 --- a/src/crewai/utilities/events/flow_events.py +++ b/src/crewai/utilities/events/flow_events.py @@ -2,7 +2,7 @@ from typing import Any, Dict, Optional, Union from pydantic import BaseModel -from .crew_events import CrewEvent +from .base_events import CrewEvent class FlowEvent(CrewEvent): diff --git a/src/crewai/utilities/events/tool_usage_events.py b/src/crewai/utilities/events/tool_usage_events.py index 645238683..aa375dcd7 100644 --- a/src/crewai/utilities/events/tool_usage_events.py +++ b/src/crewai/utilities/events/tool_usage_events.py @@ -1,7 +1,7 @@ from datetime import datetime from typing import Any, Callable, Dict -from .crew_events import CrewEvent +from .base_events import CrewEvent class ToolUsageEvent(CrewEvent):