mirror of
https://github.com/crewAIInc/crewAI.git
synced 2026-05-03 00:02:36 +00:00
telemetry initialization and enhance event handling (#2853)
* Refactor Crew class memory initialization and enhance event handling - Simplified the initialization of the external memory attribute in the Crew class. - Updated memory system retrieval logic for consistency in key usage. - Introduced a singleton pattern for the Telemetry class to ensure a single instance. - Replaced telemetry usage in CrewEvaluator with event bus emissions for test results. - Added new CrewTestResultEvent to handle crew test results more effectively. - Updated event listener to process CrewTestResultEvent and log telemetry data accordingly. - Enhanced tests to validate the singleton pattern in Telemetry and the new event handling logic. * linted * Remove unused telemetry attribute from Crew class memory initialization * fix ordering of test * Implement thread-safe singleton pattern in Telemetry class - Introduced a threading lock to ensure safe instantiation of the Telemetry singleton. - Updated the __new__ method to utilize double-checked locking for instance creation.
This commit is contained in:
@@ -9,7 +9,8 @@ from crewai.agent import Agent
|
||||
from crewai.llm import BaseLLM
|
||||
from crewai.task import Task
|
||||
from crewai.tasks.task_output import TaskOutput
|
||||
from crewai.telemetry import Telemetry
|
||||
from crewai.utilities.events import crewai_event_bus
|
||||
from crewai.utilities.events.crew_events import CrewTestResultEvent
|
||||
|
||||
|
||||
class TaskEvaluationPydanticOutput(BaseModel):
|
||||
@@ -36,7 +37,6 @@ class CrewEvaluator:
|
||||
def __init__(self, crew, eval_llm: InstanceOf[BaseLLM]):
|
||||
self.crew = crew
|
||||
self.llm = eval_llm
|
||||
self._telemetry = Telemetry()
|
||||
self._setup_for_evaluating()
|
||||
|
||||
def _setup_for_evaluating(self) -> None:
|
||||
@@ -178,11 +178,15 @@ class CrewEvaluator:
|
||||
evaluation_result = evaluation_task.execute_sync()
|
||||
|
||||
if isinstance(evaluation_result.pydantic, TaskEvaluationPydanticOutput):
|
||||
self._test_result_span = self._telemetry.individual_test_result_span(
|
||||
crewai_event_bus.emit(
|
||||
self.crew,
|
||||
evaluation_result.pydantic.quality,
|
||||
current_task.execution_duration,
|
||||
self.llm.model,
|
||||
CrewTestResultEvent(
|
||||
quality=evaluation_result.pydantic.quality,
|
||||
execution_duration=current_task.execution_duration,
|
||||
model=self.llm.model,
|
||||
crew_name=self.crew.name,
|
||||
crew=self.crew,
|
||||
),
|
||||
)
|
||||
self.tasks_scores[self.iteration].append(evaluation_result.pydantic.quality)
|
||||
self.run_execution_times[self.iteration].append(
|
||||
|
||||
Reference in New Issue
Block a user