telemetry initialization and enhance event handling (#2853)

* Refactor Crew class memory initialization and enhance event handling

- Simplified the initialization of the external memory attribute in the Crew class.
- Updated memory system retrieval logic for consistency in key usage.
- Introduced a singleton pattern for the Telemetry class to ensure a single instance.
- Replaced telemetry usage in CrewEvaluator with event bus emissions for test results.
- Added new CrewTestResultEvent to handle crew test results more effectively.
- Updated event listener to process CrewTestResultEvent and log telemetry data accordingly.
- Enhanced tests to validate the singleton pattern in Telemetry and the new event handling logic.

* linted

* Remove unused telemetry attribute from Crew class memory initialization

* fix ordering of test

* Implement thread-safe singleton pattern in Telemetry class

- Introduced a threading lock to ensure safe instantiation of the Telemetry singleton.
- Updated the __new__ method to utilize double-checked locking for instance creation.
This commit is contained in:
Lorenze Jay
2025-05-21 10:32:03 -07:00
committed by GitHub
parent 169d3233e8
commit 31ffa90075
8 changed files with 302 additions and 108 deletions

File diff suppressed because one or more lines are too long

View File

@@ -1,4 +1,3 @@
import os
from datetime import datetime
from unittest.mock import Mock, patch
@@ -22,6 +21,7 @@ from crewai.utilities.events.crew_events import (
CrewKickoffFailedEvent,
CrewKickoffStartedEvent,
CrewTestCompletedEvent,
CrewTestResultEvent,
CrewTestStartedEvent,
)
from crewai.utilities.events.crewai_event_bus import crewai_event_bus
@@ -38,7 +38,6 @@ from crewai.utilities.events.llm_events import (
LLMCallCompletedEvent,
LLMCallFailedEvent,
LLMCallStartedEvent,
LLMCallType,
LLMStreamChunkEvent,
)
from crewai.utilities.events.task_events import (
@@ -132,6 +131,10 @@ def test_crew_emits_test_kickoff_type_event():
def handle_crew_test_end(source, event):
received_events.append(event)
@crewai_event_bus.on(CrewTestResultEvent)
def handle_crew_test_result(source, event):
received_events.append(event)
eval_llm = LLM(model="gpt-4o-mini")
with (
patch.object(
@@ -149,13 +152,16 @@ def test_crew_emits_test_kickoff_type_event():
assert args[2] is None
assert args[3] == eval_llm
assert len(received_events) == 2
assert len(received_events) == 3
assert received_events[0].crew_name == "TestCrew"
assert isinstance(received_events[0].timestamp, datetime)
assert received_events[0].type == "crew_test_started"
assert received_events[1].crew_name == "TestCrew"
assert isinstance(received_events[1].timestamp, datetime)
assert received_events[1].type == "crew_test_completed"
assert received_events[1].type == "crew_test_result"
assert received_events[2].crew_name == "TestCrew"
assert isinstance(received_events[2].timestamp, datetime)
assert received_events[2].type == "crew_test_completed"
@pytest.mark.vcr(filter_headers=["authorization"])
@@ -309,7 +315,7 @@ def test_agent_emits_execution_error_event():
) as invoke_mock:
invoke_mock.side_effect = Exception(error_message)
with pytest.raises(Exception) as e:
with pytest.raises(Exception):
base_agent.execute_task(
task=base_task,
)