From 4b7db05f727dc99ab7e6fd962edb795da8a6ca77 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Thu, 13 Nov 2025 12:53:20 +0000 Subject: [PATCH] Fix: Respect CREWAI_DISABLE_TELEMETRY for tracing requests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Add is_tracking_disabled() helper function to check CREWAI_DISABLE_TELEMETRY and CREWAI_DISABLE_TRACKING - Add guards in TraceBatchManager to prevent network calls when tracking is disabled - Add guards in TraceCollectionListener to prevent listener registration when tracking is disabled - Add comprehensive tests covering both disabled and enabled scenarios - Fixes issue #3907 where telemetry requests were still being made despite CREWAI_DISABLE_TELEMETRY=true Co-Authored-By: João --- .../listeners/tracing/trace_batch_manager.py | 17 +- .../listeners/tracing/trace_listener.py | 15 +- .../crewai/events/listeners/tracing/utils.py | 13 + .../tests/tracing/test_tracing_disable.py | 300 ++++++++++++++++++ 4 files changed, 342 insertions(+), 3 deletions(-) create mode 100644 lib/crewai/tests/tracing/test_tracing_disable.py diff --git a/lib/crewai/src/crewai/events/listeners/tracing/trace_batch_manager.py b/lib/crewai/src/crewai/events/listeners/tracing/trace_batch_manager.py index 3571e45ab..da78f3760 100644 --- a/lib/crewai/src/crewai/events/listeners/tracing/trace_batch_manager.py +++ b/lib/crewai/src/crewai/events/listeners/tracing/trace_batch_manager.py @@ -12,7 +12,10 @@ from crewai.cli.authentication.token import AuthError, get_auth_token from crewai.cli.plus_api import PlusAPI from crewai.cli.version import get_crewai_version from crewai.events.listeners.tracing.types import TraceEvent -from crewai.events.listeners.tracing.utils import should_auto_collect_first_time_traces +from crewai.events.listeners.tracing.utils import ( + is_tracking_disabled, + should_auto_collect_first_time_traces, +) from crewai.utilities.constants import CREWAI_BASE_URL @@ -107,6 +110,9 @@ class TraceBatchManager: ): """Send batch initialization to backend""" + if is_tracking_disabled(): + return + if not self.plus_api or not self.current_batch: return @@ -204,6 +210,9 @@ class TraceBatchManager: def _send_events_to_backend(self) -> int: """Send buffered events to backend with graceful failure handling""" + if is_tracking_disabled(): + return 200 + if not self.plus_api or not self.trace_batch_id or not self.event_buffer: return 500 try: @@ -243,6 +252,9 @@ class TraceBatchManager: def finalize_batch(self) -> TraceBatch | None: """Finalize batch and return it for sending""" + if is_tracking_disabled(): + return None + if not self.current_batch: return None @@ -299,6 +311,9 @@ class TraceBatchManager: Args: events_count: Number of events that were successfully sent """ + if is_tracking_disabled(): + return + if not self.plus_api or not self.trace_batch_id: return diff --git a/lib/crewai/src/crewai/events/listeners/tracing/trace_listener.py b/lib/crewai/src/crewai/events/listeners/tracing/trace_listener.py index 462671141..71bc50d27 100644 --- a/lib/crewai/src/crewai/events/listeners/tracing/trace_listener.py +++ b/lib/crewai/src/crewai/events/listeners/tracing/trace_listener.py @@ -10,13 +10,15 @@ from crewai.cli.authentication.token import AuthError, get_auth_token from crewai.cli.version import get_crewai_version from crewai.events.base_event_listener import BaseEventListener from crewai.events.event_bus import CrewAIEventsBus -from crewai.events.utils.console_formatter import ConsoleFormatter from crewai.events.listeners.tracing.first_time_trace_handler import ( FirstTimeTraceHandler, ) from crewai.events.listeners.tracing.trace_batch_manager import TraceBatchManager from crewai.events.listeners.tracing.types import TraceEvent -from crewai.events.listeners.tracing.utils import safe_serialize_to_dict +from crewai.events.listeners.tracing.utils import ( + is_tracking_disabled, + safe_serialize_to_dict, +) from crewai.events.types.agent_events import ( AgentExecutionCompletedEvent, AgentExecutionErrorEvent, @@ -80,6 +82,7 @@ from crewai.events.types.tool_usage_events import ( ToolUsageFinishedEvent, ToolUsageStartedEvent, ) +from crewai.events.utils.console_formatter import ConsoleFormatter class TraceCollectionListener(BaseEventListener): @@ -118,6 +121,10 @@ class TraceCollectionListener(BaseEventListener): if self._initialized: return + if is_tracking_disabled(): + self._initialized = True + return + super().__init__() self.batch_manager = batch_manager or TraceBatchManager() self._initialized = True @@ -154,6 +161,10 @@ class TraceCollectionListener(BaseEventListener): if self._listeners_setup: return + if is_tracking_disabled(): + self._listeners_setup = True + return + self._register_flow_event_handlers(crewai_event_bus) self._register_context_event_handlers(crewai_event_bus) self._register_action_event_handlers(crewai_event_bus) diff --git a/lib/crewai/src/crewai/events/listeners/tracing/utils.py b/lib/crewai/src/crewai/events/listeners/tracing/utils.py index 9c5a30a05..dba450ac6 100644 --- a/lib/crewai/src/crewai/events/listeners/tracing/utils.py +++ b/lib/crewai/src/crewai/events/listeners/tracing/utils.py @@ -27,6 +27,19 @@ def is_tracing_enabled() -> bool: return os.getenv("CREWAI_TRACING_ENABLED", "false").lower() == "true" +def is_tracking_disabled() -> bool: + """Check if tracking/tracing should be disabled. + + This acts as a master kill switch for all outbound telemetry and tracing. + Returns True if either CREWAI_DISABLE_TELEMETRY or CREWAI_DISABLE_TRACKING + environment variables are set to 'true'. + """ + return ( + os.getenv("CREWAI_DISABLE_TELEMETRY", "false").lower() == "true" + or os.getenv("CREWAI_DISABLE_TRACKING", "false").lower() == "true" + ) + + def on_first_execution_tracing_confirmation() -> bool: if _is_test_environment(): return False diff --git a/lib/crewai/tests/tracing/test_tracing_disable.py b/lib/crewai/tests/tracing/test_tracing_disable.py new file mode 100644 index 000000000..b1cdbec52 --- /dev/null +++ b/lib/crewai/tests/tracing/test_tracing_disable.py @@ -0,0 +1,300 @@ +"""Tests for CREWAI_DISABLE_TELEMETRY affecting tracing system.""" + +import os +from unittest.mock import MagicMock, Mock, patch + +import pytest + +from crewai.events.listeners.tracing.trace_batch_manager import TraceBatchManager +from crewai.events.listeners.tracing.trace_listener import TraceCollectionListener +from crewai.events.listeners.tracing.utils import is_tracking_disabled + + +@pytest.fixture(autouse=True) +def reset_singleton(): + """Reset TraceCollectionListener singleton between tests.""" + TraceCollectionListener._instance = None + TraceCollectionListener._initialized = False + TraceCollectionListener._listeners_setup = False + yield + TraceCollectionListener._instance = None + TraceCollectionListener._initialized = False + TraceCollectionListener._listeners_setup = False + + +@pytest.fixture +def mock_plus_api(): + """Mock PlusAPI to prevent actual network calls.""" + with patch("crewai.events.listeners.tracing.trace_batch_manager.PlusAPI") as mock: + api_instance = MagicMock() + mock.return_value = api_instance + yield api_instance + + +@pytest.mark.parametrize( + "env_var,value,expected_disabled", + [ + ("CREWAI_DISABLE_TELEMETRY", "true", True), + ("CREWAI_DISABLE_TELEMETRY", "TRUE", True), + ("CREWAI_DISABLE_TELEMETRY", "True", True), + ("CREWAI_DISABLE_TRACKING", "true", True), + ("CREWAI_DISABLE_TRACKING", "TRUE", True), + ("CREWAI_DISABLE_TELEMETRY", "false", False), + ("CREWAI_DISABLE_TRACKING", "false", False), + ], +) +def test_is_tracking_disabled_env_vars(env_var, value, expected_disabled): + """Test is_tracking_disabled() with different environment variables.""" + with patch.dict(os.environ, {env_var: value}, clear=True): + assert is_tracking_disabled() == expected_disabled + + +def test_is_tracking_disabled_default(): + """Test is_tracking_disabled() returns False by default.""" + with patch.dict(os.environ, {}, clear=True): + assert is_tracking_disabled() is False + + +def test_trace_batch_manager_initialize_backend_batch_disabled(mock_plus_api): + """Test that _initialize_backend_batch does not make network calls when disabled.""" + with patch.dict(os.environ, {"CREWAI_DISABLE_TELEMETRY": "true"}): + with patch( + "crewai.events.listeners.tracing.trace_batch_manager.get_auth_token" + ) as mock_auth: + mock_auth.return_value = "test_token" + manager = TraceBatchManager() + manager.current_batch = MagicMock() + manager.current_batch.batch_id = "test_batch_id" + + manager._initialize_backend_batch( + user_context={"user_id": "test"}, + execution_metadata={"execution_type": "crew"}, + use_ephemeral=False, + ) + + mock_plus_api.initialize_trace_batch.assert_not_called() + mock_plus_api.initialize_ephemeral_trace_batch.assert_not_called() + + +def test_trace_batch_manager_initialize_backend_batch_ephemeral_disabled( + mock_plus_api, +): + """Test that ephemeral batch initialization does not make network calls when disabled.""" + with patch.dict(os.environ, {"CREWAI_DISABLE_TELEMETRY": "true"}): + with patch( + "crewai.events.listeners.tracing.trace_batch_manager.get_auth_token" + ) as mock_auth: + mock_auth.return_value = "test_token" + manager = TraceBatchManager() + manager.current_batch = MagicMock() + manager.current_batch.batch_id = "test_batch_id" + + manager._initialize_backend_batch( + user_context={"user_id": "test"}, + execution_metadata={"execution_type": "crew"}, + use_ephemeral=True, + ) + + mock_plus_api.initialize_trace_batch.assert_not_called() + mock_plus_api.initialize_ephemeral_trace_batch.assert_not_called() + + +def test_trace_batch_manager_send_events_disabled(mock_plus_api): + """Test that _send_events_to_backend returns success without making calls when disabled.""" + with patch.dict(os.environ, {"CREWAI_DISABLE_TELEMETRY": "true"}): + with patch( + "crewai.events.listeners.tracing.trace_batch_manager.get_auth_token" + ) as mock_auth: + mock_auth.return_value = "test_token" + manager = TraceBatchManager() + manager.trace_batch_id = "test_batch_id" + manager.event_buffer = [MagicMock()] + + result = manager._send_events_to_backend() + + assert result == 200 + mock_plus_api.send_trace_events.assert_not_called() + mock_plus_api.send_ephemeral_trace_events.assert_not_called() + + +def test_trace_batch_manager_finalize_batch_disabled(mock_plus_api): + """Test that finalize_batch returns None without making calls when disabled.""" + with patch.dict(os.environ, {"CREWAI_DISABLE_TELEMETRY": "true"}): + with patch( + "crewai.events.listeners.tracing.trace_batch_manager.get_auth_token" + ) as mock_auth: + mock_auth.return_value = "test_token" + manager = TraceBatchManager() + manager.current_batch = MagicMock() + + result = manager.finalize_batch() + + assert result is None + mock_plus_api.finalize_trace_batch.assert_not_called() + mock_plus_api.finalize_ephemeral_trace_batch.assert_not_called() + mock_plus_api.mark_trace_batch_as_failed.assert_not_called() + + +def test_trace_batch_manager_finalize_backend_batch_disabled(mock_plus_api): + """Test that _finalize_backend_batch does not make network calls when disabled.""" + with patch.dict(os.environ, {"CREWAI_DISABLE_TELEMETRY": "true"}): + with patch( + "crewai.events.listeners.tracing.trace_batch_manager.get_auth_token" + ) as mock_auth: + mock_auth.return_value = "test_token" + manager = TraceBatchManager() + manager.trace_batch_id = "test_batch_id" + + manager._finalize_backend_batch(events_count=5) + + mock_plus_api.finalize_trace_batch.assert_not_called() + mock_plus_api.finalize_ephemeral_trace_batch.assert_not_called() + + +def test_trace_collection_listener_init_disabled(): + """Test that TraceCollectionListener initialization is skipped when disabled.""" + with patch.dict(os.environ, {"CREWAI_DISABLE_TELEMETRY": "true"}): + listener = TraceCollectionListener() + + assert listener._initialized is True + assert not hasattr(listener, "batch_manager") + assert not hasattr(listener, "first_time_handler") + + +def test_trace_collection_listener_setup_listeners_disabled(): + """Test that setup_listeners does not register handlers when disabled.""" + with patch.dict(os.environ, {"CREWAI_DISABLE_TELEMETRY": "true"}): + listener = TraceCollectionListener() + mock_event_bus = MagicMock() + + listener.setup_listeners(mock_event_bus) + + assert listener._listeners_setup is True + mock_event_bus.on.assert_not_called() + + +def test_trace_batch_manager_enabled_makes_calls(mock_plus_api): + """Test that network calls ARE made when tracking is enabled (negative test).""" + with patch.dict(os.environ, {"CREWAI_DISABLE_TELEMETRY": "false"}): + with patch( + "crewai.events.listeners.tracing.trace_batch_manager.get_auth_token" + ) as mock_auth: + mock_auth.return_value = "test_token" + with patch( + "crewai.events.listeners.tracing.trace_batch_manager.should_auto_collect_first_time_traces" + ) as mock_first_time: + mock_first_time.return_value = False + + mock_response = Mock() + mock_response.status_code = 201 + mock_response.json.return_value = {"trace_id": "test_trace_id"} + mock_plus_api.initialize_trace_batch.return_value = mock_response + + manager = TraceBatchManager() + manager.current_batch = MagicMock() + manager.current_batch.batch_id = "test_batch_id" + manager.current_batch.version = "1.0.0" + + manager._initialize_backend_batch( + user_context={"user_id": "test"}, + execution_metadata={"execution_type": "crew"}, + use_ephemeral=False, + ) + + mock_plus_api.initialize_trace_batch.assert_called_once() + + +def test_trace_batch_manager_enabled_ephemeral_makes_calls(mock_plus_api): + """Test that ephemeral network calls ARE made when tracking is enabled (negative test).""" + with patch.dict(os.environ, {"CREWAI_DISABLE_TELEMETRY": "false"}): + with patch( + "crewai.events.listeners.tracing.trace_batch_manager.get_auth_token" + ) as mock_auth: + mock_auth.return_value = "test_token" + with patch( + "crewai.events.listeners.tracing.trace_batch_manager.should_auto_collect_first_time_traces" + ) as mock_first_time: + mock_first_time.return_value = False + + mock_response = Mock() + mock_response.status_code = 201 + mock_response.json.return_value = { + "ephemeral_trace_id": "test_ephemeral_id" + } + mock_plus_api.initialize_ephemeral_trace_batch.return_value = ( + mock_response + ) + + manager = TraceBatchManager() + manager.current_batch = MagicMock() + manager.current_batch.batch_id = "test_batch_id" + manager.current_batch.version = "1.0.0" + + manager._initialize_backend_batch( + user_context={"user_id": "test"}, + execution_metadata={"execution_type": "crew"}, + use_ephemeral=True, + ) + + mock_plus_api.initialize_ephemeral_trace_batch.assert_called_once() + + +def test_trace_collection_listener_enabled_registers_handlers(): + """Test that handlers ARE registered when tracking is enabled (negative test).""" + with patch.dict(os.environ, {"CREWAI_DISABLE_TELEMETRY": "false"}): + with patch( + "crewai.events.listeners.tracing.trace_batch_manager.get_auth_token" + ): + with patch( + "crewai.events.listeners.tracing.trace_batch_manager.PlusAPI" + ): + listener = TraceCollectionListener() + + assert hasattr(listener, "batch_manager") + assert hasattr(listener, "first_time_handler") + + listener._listeners_setup = False + + with patch.object( + listener, "_register_flow_event_handlers" + ) as mock_flow: + with patch.object( + listener, "_register_context_event_handlers" + ) as mock_context: + with patch.object( + listener, "_register_action_event_handlers" + ) as mock_action: + mock_event_bus = MagicMock() + listener.setup_listeners(mock_event_bus) + + mock_flow.assert_called_once_with(mock_event_bus) + mock_context.assert_called_once_with(mock_event_bus) + mock_action.assert_called_once_with(mock_event_bus) + assert listener._listeners_setup is True + + +def test_crewai_disable_tracking_also_works(): + """Test that CREWAI_DISABLE_TRACKING also disables tracing.""" + with patch.dict(os.environ, {"CREWAI_DISABLE_TRACKING": "true"}): + with patch( + "crewai.events.listeners.tracing.trace_batch_manager.get_auth_token" + ) as mock_auth: + mock_auth.return_value = "test_token" + with patch( + "crewai.events.listeners.tracing.trace_batch_manager.PlusAPI" + ) as mock_plus_api: + api_instance = MagicMock() + mock_plus_api.return_value = api_instance + + manager = TraceBatchManager() + manager.current_batch = MagicMock() + manager.current_batch.batch_id = "test_batch_id" + + manager._initialize_backend_batch( + user_context={"user_id": "test"}, + execution_metadata={"execution_type": "crew"}, + use_ephemeral=False, + ) + + api_instance.initialize_trace_batch.assert_not_called() + api_instance.initialize_ephemeral_trace_batch.assert_not_called()