telemetry initialization and enhance event handling (#2853)

* Refactor Crew class memory initialization and enhance event handling

- Simplified the initialization of the external memory attribute in the Crew class.
- Updated memory system retrieval logic for consistency in key usage.
- Introduced a singleton pattern for the Telemetry class to ensure a single instance.
- Replaced telemetry usage in CrewEvaluator with event bus emissions for test results.
- Added new CrewTestResultEvent to handle crew test results more effectively.
- Updated event listener to process CrewTestResultEvent and log telemetry data accordingly.
- Enhanced tests to validate the singleton pattern in Telemetry and the new event handling logic.

* linted

* Remove unused telemetry attribute from Crew class memory initialization

* fix ordering of test

* Implement thread-safe singleton pattern in Telemetry class

- Introduced a threading lock to ensure safe instantiation of the Telemetry singleton.
- Updated the __new__ method to utilize double-checked locking for instance creation.

tests/telemetry/test_telemetry.py

@@ -6,6 +6,8 @@ import pytest
 from crewai import Agent, Crew, Task
 from crewai.telemetry import Telemetry
 
+from opentelemetry import trace
+
 
 @pytest.mark.parametrize(
     "env_var,value,expected_ready",
@@ -34,9 +36,6 @@ def test_telemetry_enabled_by_default():
             assert telemetry.ready is True
 
 
-from opentelemetry import trace
-
-
 @patch("crewai.telemetry.telemetry.logger.error")
 @patch(
     "opentelemetry.exporter.otlp.proto.http.trace_exporter.OTLPSpanExporter.export",
@@ -67,3 +66,32 @@ def test_telemetry_fails_due_connect_timeout(export_mock, logger_mock):
 
     export_mock.assert_called_once()
     logger_mock.assert_called_once_with(error)
+
+
+def test_telemetry_singleton_pattern():
+    """Test that Telemetry uses the singleton pattern correctly."""
+    Telemetry._instance = None
+
+    telemetry1 = Telemetry()
+    telemetry2 = Telemetry()
+
+    assert telemetry1 is telemetry2
+
+    setattr(telemetry1, "test_attribute", "test_value")
+    assert hasattr(telemetry2, "test_attribute")
+    assert getattr(telemetry2, "test_attribute") == "test_value"
+
+    import threading
+
+    instances = []
+
+    def create_instance():
+        instances.append(Telemetry())
+
+    threads = [threading.Thread(target=create_instance) for _ in range(5)]
+    for thread in threads:
+        thread.start()
+    for thread in threads:
+        thread.join()
+
+    assert all(instance is telemetry1 for instance in instances)

tests/utilities/test_events.py

@@ -1,4 +1,3 @@
-import os
 from datetime import datetime
 from unittest.mock import Mock, patch
 
@@ -22,6 +21,7 @@ from crewai.utilities.events.crew_events import (
     CrewKickoffFailedEvent,
     CrewKickoffStartedEvent,
     CrewTestCompletedEvent,
+    CrewTestResultEvent,
     CrewTestStartedEvent,
 )
 from crewai.utilities.events.crewai_event_bus import crewai_event_bus
@@ -38,7 +38,6 @@ from crewai.utilities.events.llm_events import (
     LLMCallCompletedEvent,
     LLMCallFailedEvent,
     LLMCallStartedEvent,
-    LLMCallType,
     LLMStreamChunkEvent,
 )
 from crewai.utilities.events.task_events import (
@@ -132,6 +131,10 @@ def test_crew_emits_test_kickoff_type_event():
     def handle_crew_test_end(source, event):
         received_events.append(event)
 
+    @crewai_event_bus.on(CrewTestResultEvent)
+    def handle_crew_test_result(source, event):
+        received_events.append(event)
+
     eval_llm = LLM(model="gpt-4o-mini")
     with (
         patch.object(
@@ -149,13 +152,16 @@ def test_crew_emits_test_kickoff_type_event():
         assert args[2] is None
         assert args[3] == eval_llm
 
-    assert len(received_events) == 2
+    assert len(received_events) == 3
     assert received_events[0].crew_name == "TestCrew"
     assert isinstance(received_events[0].timestamp, datetime)
     assert received_events[0].type == "crew_test_started"
     assert received_events[1].crew_name == "TestCrew"
     assert isinstance(received_events[1].timestamp, datetime)
-    assert received_events[1].type == "crew_test_completed"
+    assert received_events[1].type == "crew_test_result"
+    assert received_events[2].crew_name == "TestCrew"
+    assert isinstance(received_events[2].timestamp, datetime)
+    assert received_events[2].type == "crew_test_completed"
 
 
 @pytest.mark.vcr(filter_headers=["authorization"])
@@ -309,7 +315,7 @@ def test_agent_emits_execution_error_event():
     ) as invoke_mock:
         invoke_mock.side_effect = Exception(error_message)
 
-        with pytest.raises(Exception) as e:
+        with pytest.raises(Exception):
             base_agent.execute_task(
                 task=base_task,
             )