From d0d22038f5f9bececcea1739e62a77d1ec89b9a9 Mon Sep 17 00:00:00 2001 From: Greyson LaLonde Date: Fri, 7 Nov 2025 17:28:32 -0500 Subject: [PATCH] chore: remove telemetry mocks --- lib/crewai/tests/conftest.py | 173 +----------------- lib/crewai/tests/telemetry/test_telemetry.py | 4 - .../tests/telemetry/test_telemetry_disable.py | 5 - 3 files changed, 1 insertion(+), 181 deletions(-) diff --git a/lib/crewai/tests/conftest.py b/lib/crewai/tests/conftest.py index 0b05f4e1c..fab3e55f0 100644 --- a/lib/crewai/tests/conftest.py +++ b/lib/crewai/tests/conftest.py @@ -3,7 +3,6 @@ import os import tempfile from pathlib import Path from typing import Any -from unittest.mock import Mock, patch import pytest from dotenv import load_dotenv @@ -11,7 +10,7 @@ from dotenv import load_dotenv load_result = load_dotenv(override=True) -@pytest.fixture(autouse=True) +@pytest.fixture(autouse=True, scope="function") def setup_test_environment(): """Set up test environment with a temporary directory for SQLite storage.""" with tempfile.TemporaryDirectory() as temp_dir: @@ -44,176 +43,6 @@ def setup_test_environment(): # Cleanup is handled automatically when tempfile context exits -def pytest_configure(config): - config.addinivalue_line( - "markers", "telemetry: mark test as a telemetry test (don't mock telemetry)" - ) - - -@pytest.fixture(autouse=True) -def auto_mock_telemetry(request): - if request.node.get_closest_marker("telemetry"): - telemetry_env = { - key: value - for key, value in os.environ.items() - if key not in ["CREWAI_DISABLE_TELEMETRY", "OTEL_SDK_DISABLED"] - } - with patch.dict(os.environ, telemetry_env, clear=True): - yield - return - - if "telemetry" in str(request.fspath): - telemetry_env = { - key: value - for key, value in os.environ.items() - if key not in ["CREWAI_DISABLE_TELEMETRY", "OTEL_SDK_DISABLED"] - } - with patch.dict(os.environ, telemetry_env, clear=True): - yield - return - - with patch.dict( - os.environ, {"CREWAI_DISABLE_TELEMETRY": "true", "OTEL_SDK_DISABLED": "true"} - ): - with patch("crewai.telemetry.Telemetry") as mock_telemetry_class: - mock_instance = create_mock_telemetry_instance() - mock_telemetry_class.return_value = mock_instance - - # Create mock for TraceBatchManager - mock_trace_manager = Mock() - mock_trace_manager.add_trace = Mock() - mock_trace_manager.send_batch = Mock() - mock_trace_manager.stop = Mock() - - # Create mock for BatchSpanProcessor to prevent OpenTelemetry background threads - mock_batch_processor = Mock() - mock_batch_processor.shutdown = Mock() - mock_batch_processor.force_flush = Mock() - - with ( - patch( - "crewai.events.event_listener.Telemetry", - mock_telemetry_class, - ), - patch("crewai.tools.tool_usage.Telemetry", mock_telemetry_class), - patch("crewai.cli.command.Telemetry", mock_telemetry_class), - patch("crewai.cli.create_flow.Telemetry", mock_telemetry_class), - patch( - "crewai.events.listeners.tracing.trace_batch_manager.TraceBatchManager", - return_value=mock_trace_manager, - ), - patch( - "crewai.events.listeners.tracing.trace_listener.TraceBatchManager", - return_value=mock_trace_manager, - ), - patch( - "crewai.events.listeners.tracing.first_time_trace_handler.TraceBatchManager", - return_value=mock_trace_manager, - ), - patch( - "opentelemetry.sdk.trace.export.BatchSpanProcessor", - return_value=mock_batch_processor, - ), - ): - yield mock_instance - - -def create_mock_telemetry_instance(): - mock_instance = Mock() - - mock_instance.ready = False - mock_instance.trace_set = False - mock_instance._initialized = True - - mock_instance._is_telemetry_disabled.return_value = True - mock_instance._should_execute_telemetry.return_value = False - - telemetry_methods = [ - "set_tracer", - "crew_creation", - "task_started", - "task_ended", - "tool_usage", - "tool_repeated_usage", - "tool_usage_error", - "crew_execution_span", - "end_crew", - "flow_creation_span", - "flow_execution_span", - "individual_test_result_span", - "test_execution_span", - "deploy_signup_error_span", - "start_deployment_span", - "create_crew_deployment_span", - "get_crew_logs_span", - "remove_crew_span", - "flow_plotting_span", - "_add_attribute", - "_safe_telemetry_operation", - ] - - for method in telemetry_methods: - setattr(mock_instance, method, Mock(return_value=None)) - - mock_instance.task_started.return_value = None - - return mock_instance - - -@pytest.fixture -def mock_opentelemetry_components(): - with ( - patch("opentelemetry.trace.get_tracer") as mock_get_tracer, - patch("opentelemetry.trace.set_tracer_provider") as mock_set_provider, - patch("opentelemetry.baggage.set_baggage") as mock_set_baggage, - patch("opentelemetry.baggage.get_baggage") as mock_get_baggage, - patch("opentelemetry.context.attach") as mock_attach, - patch("opentelemetry.context.detach") as mock_detach, - ): - mock_tracer = Mock() - mock_span = Mock() - mock_tracer.start_span.return_value = mock_span - mock_get_tracer.return_value = mock_tracer - - yield { - "get_tracer": mock_get_tracer, - "set_tracer_provider": mock_set_provider, - "tracer": mock_tracer, - "span": mock_span, - "set_baggage": mock_set_baggage, - "get_baggage": mock_get_baggage, - "attach": mock_attach, - "detach": mock_detach, - } - - -@pytest.fixture(autouse=True) -def clear_event_bus_handlers(setup_test_environment): - """Clear event bus handlers after each test for isolation. - - Handlers registered during the test are allowed to run, then cleaned up - after the test completes. - - Depends on setup_test_environment to ensure cleanup happens in correct order. - """ - from crewai.events.event_bus import crewai_event_bus - from crewai.experimental.evaluation.evaluation_listener import ( - EvaluationTraceCallback, - ) - - yield - - # Shutdown event bus without waiting to avoid hanging on blocked threads - crewai_event_bus.shutdown(wait=False) - crewai_event_bus._initialize() - - callback = EvaluationTraceCallback() - callback.traces.clear() - callback.current_agent_id = None - callback.current_task_id = None - - - HEADERS_TO_FILTER = { "authorization": "AUTHORIZATION-XXX", "content-security-policy": "CSP-FILTERED", diff --git a/lib/crewai/tests/telemetry/test_telemetry.py b/lib/crewai/tests/telemetry/test_telemetry.py index b7df3bc25..35c86e994 100644 --- a/lib/crewai/tests/telemetry/test_telemetry.py +++ b/lib/crewai/tests/telemetry/test_telemetry.py @@ -19,7 +19,6 @@ def cleanup_telemetry(): Telemetry._lock = threading.Lock() -@pytest.mark.telemetry @pytest.mark.parametrize( "env_var,value,expected_ready", [ @@ -38,8 +37,6 @@ def test_telemetry_environment_variables(env_var, value, expected_ready): telemetry = Telemetry() assert telemetry.ready is expected_ready - -@pytest.mark.telemetry def test_telemetry_enabled_by_default(): """Test that telemetry is enabled by default.""" with patch.dict(os.environ, {}, clear=True): @@ -83,7 +80,6 @@ def test_telemetry_fails_due_connect_timeout(export_mock, logger_mock): assert call[0][0] == error -@pytest.mark.telemetry def test_telemetry_singleton_pattern(): """Test that Telemetry uses the singleton pattern correctly.""" Telemetry._instance = None diff --git a/lib/crewai/tests/telemetry/test_telemetry_disable.py b/lib/crewai/tests/telemetry/test_telemetry_disable.py index 5e4e9d3c1..7979c32d7 100644 --- a/lib/crewai/tests/telemetry/test_telemetry_disable.py +++ b/lib/crewai/tests/telemetry/test_telemetry_disable.py @@ -13,7 +13,6 @@ def cleanup_telemetry(): Telemetry._instance = None -@pytest.mark.telemetry @pytest.mark.parametrize( "env_var,value,expected_ready", [ @@ -33,7 +32,6 @@ def test_telemetry_environment_variables(env_var, value, expected_ready): assert telemetry.ready is expected_ready -@pytest.mark.telemetry def test_telemetry_enabled_by_default(): """Test that telemetry is enabled by default.""" with patch.dict(os.environ, {}, clear=True): @@ -42,7 +40,6 @@ def test_telemetry_enabled_by_default(): assert telemetry.ready is True -@pytest.mark.telemetry def test_telemetry_disable_after_singleton_creation(): """Test that telemetry operations are disabled when env var is set after singleton creation.""" with patch.dict(os.environ, {}, clear=True): @@ -62,7 +59,6 @@ def test_telemetry_disable_after_singleton_creation(): mock_operation.assert_not_called() -@pytest.mark.telemetry def test_telemetry_disable_with_multiple_instances(): """Test that multiple telemetry instances respect dynamically changed env vars.""" with patch.dict(os.environ, {}, clear=True): @@ -81,7 +77,6 @@ def test_telemetry_disable_with_multiple_instances(): mock_operation.assert_not_called() -@pytest.mark.telemetry def test_telemetry_otel_sdk_disabled_after_creation(): """Test that OTEL_SDK_DISABLED also works when set after singleton creation.""" with patch.dict(os.environ, {}, clear=True):