andre/crewAI
1
0
Fork 0
mirror of https://github.com/crewAIInc/crewAI.git synced 2026-01-11 00:58:30 +00:00
Code Issues Actions 5 Packages Projects Releases Wiki Activity

telemetry initialization and enhance event handling (#2853)

Browse Source 
* Refactor Crew class memory initialization and enhance event handling

- Simplified the initialization of the external memory attribute in the Crew class.
- Updated memory system retrieval logic for consistency in key usage.
- Introduced a singleton pattern for the Telemetry class to ensure a single instance.
- Replaced telemetry usage in CrewEvaluator with event bus emissions for test results.
- Added new CrewTestResultEvent to handle crew test results more effectively.
- Updated event listener to process CrewTestResultEvent and log telemetry data accordingly.
- Enhanced tests to validate the singleton pattern in Telemetry and the new event handling logic.

* linted

* Remove unused telemetry attribute from Crew class memory initialization

* fix ordering of test

* Implement thread-safe singleton pattern in Telemetry class

- Introduced a threading lock to ensure safe instantiation of the Telemetry singleton.
- Updated the __new__ method to utilize double-checked locking for instance creation.
This commit is contained in:
Lorenze Jay
2025-05-21 10:32:03 -07:00
committed by GitHub
parent 169d3233e8
commit 31ffa90075
8 changed files with 302 additions and 108 deletions
Download Patch File Download Diff File Expand all files Collapse all files

98
src/crewai/crew.py
View File

@@ -315,9 +315,7 @@ class Crew(FlowTrackable, BaseModel):
"""Initialize private memory attributes."""
self._external_memory = (
# External memory doesn’t support a default value since it was designed to be managed entirely externally
self.external_memory.set_crew(self)
if self.external_memory
else None
self.external_memory.set_crew(self) if self.external_memory else None
)
self._long_term_memory = self.long_term_memory
@@ -1204,7 +1202,6 @@ class Crew(FlowTrackable, BaseModel):
"_long_term_memory",
"_entity_memory",
"_external_memory",
"_telemetry",
"agents",
"tasks",
"knowledge_sources",
@@ -1397,10 +1394,10 @@ class Crew(FlowTrackable, BaseModel):
memory_systems = self._get_memory_systems()
for memory_type, config in memory_systems.items():
if (system := config.get('system')) is not None:
name = config.get('name')
if (system := config.get("system")) is not None:
name = config.get("name")
try:
reset_fn: Callable = cast(Callable, config.get('reset'))
reset_fn: Callable = cast(Callable, config.get("reset"))
reset_fn(system)
self._logger.log(
"info",
@@ -1422,14 +1419,14 @@ class Crew(FlowTrackable, BaseModel):
"""
memory_systems = self._get_memory_systems()
config = memory_systems[memory_type]
system = config.get('system')
name = config.get('name')
system = config.get("system")
name = config.get("name")
if system is None:
raise RuntimeError(f"{name} memory system is not initialized")
try:
reset_fn: Callable = cast(Callable, config.get('reset'))
reset_fn: Callable = cast(Callable, config.get("reset"))
reset_fn(system)
self._logger.log(
"info",
@@ -1442,58 +1439,67 @@ class Crew(FlowTrackable, BaseModel):
def _get_memory_systems(self):
"""Get all available memory systems with their configuration.
Returns:
Dict containing all memory systems with their reset functions and display names.
"""
def default_reset(memory):
return memory.reset()
def knowledge_reset(memory):
return self.reset_knowledge(memory)
# Get knowledge for agents
agent_knowledges = [getattr(agent, "knowledge", None) for agent in self.agents
if getattr(agent, "knowledge", None) is not None]
# Get knowledge for agents
agent_knowledges = [
getattr(agent, "knowledge", None)
for agent in self.agents
if getattr(agent, "knowledge", None) is not None
]
# Get knowledge for crew and agents
crew_knowledge = getattr(self, "knowledge", None)
crew_and_agent_knowledges = ([crew_knowledge] if crew_knowledge is not None else []) + agent_knowledges
crew_and_agent_knowledges = (
[crew_knowledge] if crew_knowledge is not None else []
) + agent_knowledges
return {
'short': {
'system': getattr(self, "_short_term_memory", None),
'reset': default_reset,
'name': 'Short Term'
"short": {
"system": getattr(self, "_short_term_memory", None),
"reset": default_reset,
"name": "Short Term",
},
'entity': {
'system': getattr(self, "_entity_memory", None),
'reset': default_reset,
'name': 'Entity'
"entity": {
"system": getattr(self, "_entity_memory", None),
"reset": default_reset,
"name": "Entity",
},
'external': {
'system': getattr(self, "_external_memory", None),
'reset': default_reset,
'name': 'External'
"external": {
"system": getattr(self, "_external_memory", None),
"reset": default_reset,
"name": "External",
},
'long': {
'system': getattr(self, "_long_term_memory", None),
'reset': default_reset,
'name': 'Long Term'
"long": {
"system": getattr(self, "_long_term_memory", None),
"reset": default_reset,
"name": "Long Term",
},
'kickoff_outputs': {
'system': getattr(self, "_task_output_handler", None),
'reset': default_reset,
'name': 'Task Output'
"kickoff_outputs": {
"system": getattr(self, "_task_output_handler", None),
"reset": default_reset,
"name": "Task Output",
},
'knowledge': {
'system': crew_and_agent_knowledges if crew_and_agent_knowledges else None,
'reset': knowledge_reset,
'name': 'Crew Knowledge and Agent Knowledge'
"knowledge": {
"system": crew_and_agent_knowledges
if crew_and_agent_knowledges
else None,
"reset": knowledge_reset,
"name": "Crew Knowledge and Agent Knowledge",
},
"agent_knowledge": {
"system": agent_knowledges if agent_knowledges else None,
"reset": knowledge_reset,
"name": "Agent Knowledge",
},
'agent_knowledge': {
'system': agent_knowledges if agent_knowledges else None,
'reset': knowledge_reset,
'name': 'Agent Knowledge'
}
}
def reset_knowledge(self, knowledges: List[Knowledge]) -> None:

11
src/crewai/telemetry/telemetry.py
View File

@@ -9,6 +9,7 @@ import warnings
from contextlib import contextmanager
from importlib.metadata import version
from typing import TYPE_CHECKING, Any, Optional
import threading
from opentelemetry import trace
from opentelemetry.exporter.otlp.proto.http.trace_exporter import (
@@ -64,6 +65,16 @@ class Telemetry:
attribute in the Crew class.
"""
_instance = None
_lock = threading.Lock()
def __new__(cls):
if cls._instance is None:
with cls._lock:
if cls._instance is None:
cls._instance = super(Telemetry, cls).__new__(cls)
return cls._instance
def __init__(self) -> None:
self.ready: bool = False
self.trace_set: bool = False

16
src/crewai/utilities/evaluators/crew_evaluator_handler.py
View File

@@ -9,7 +9,8 @@ from crewai.agent import Agent
from crewai.llm import BaseLLM
from crewai.task import Task
from crewai.tasks.task_output import TaskOutput
from crewai.telemetry import Telemetry
from crewai.utilities.events import crewai_event_bus
from crewai.utilities.events.crew_events import CrewTestResultEvent
class TaskEvaluationPydanticOutput(BaseModel):
@@ -36,7 +37,6 @@ class CrewEvaluator:
def __init__(self, crew, eval_llm: InstanceOf[BaseLLM]):
self.crew = crew
self.llm = eval_llm
self._telemetry = Telemetry()
self._setup_for_evaluating()
def _setup_for_evaluating(self) -> None:
@@ -178,11 +178,15 @@ class CrewEvaluator:
evaluation_result = evaluation_task.execute_sync()
if isinstance(evaluation_result.pydantic, TaskEvaluationPydanticOutput):
self._test_result_span = self._telemetry.individual_test_result_span(
crewai_event_bus.emit(
self.crew,
evaluation_result.pydantic.quality,
current_task.execution_duration,
self.llm.model,
CrewTestResultEvent(
quality=evaluation_result.pydantic.quality,
execution_duration=current_task.execution_duration,
model=self.llm.model,
crew_name=self.crew.name,
crew=self.crew,
),
)
self.tasks_scores[self.iteration].append(evaluation_result.pydantic.quality)
self.run_execution_times[self.iteration].append(

9
src/crewai/utilities/events/crew_events.py
View File

@@ -100,3 +100,12 @@ class CrewTestFailedEvent(CrewBaseEvent):
error: str
type: str = "crew_test_failed"
class CrewTestResultEvent(CrewBaseEvent):
"""Event emitted when a crew test result is available"""
quality: float
execution_duration: float
model: str
type: str = "crew_test_result"

10
src/crewai/utilities/events/event_listener.py
View File

@@ -37,6 +37,7 @@ from .crew_events import (
CrewKickoffStartedEvent,
CrewTestCompletedEvent,
CrewTestFailedEvent,
CrewTestResultEvent,
CrewTestStartedEvent,
CrewTrainCompletedEvent,
CrewTrainFailedEvent,
@@ -134,6 +135,15 @@ class EventListener(BaseEventListener):
def on_crew_train_failed(source, event: CrewTrainFailedEvent):
self.formatter.handle_crew_train_failed(event.crew_name or "Crew")
@crewai_event_bus.on(CrewTestResultEvent)
def on_crew_test_result(source, event: CrewTestResultEvent):
self._telemetry.individual_test_result_span(
source.crew,
event.quality,
int(event.execution_duration),
event.model,
)
# ----------- TASK EVENTS -----------
@crewai_event_bus.on(TaskStartedEvent)