andre/crewAI
1
0
Fork 0
mirror of https://github.com/crewAIInc/crewAI.git synced 2026-01-26 00:28:13 +00:00
Code Issues Actions 4 Packages Projects Releases Wiki Activity

Mock telemetry in pytest tests (#3340)

Browse Source 
* Add telemetry mocking for pytest tests

- Mock telemetry by default for all tests except telemetry-specific tests
- Add @pytest.mark.telemetry marker for real telemetry tests
- Reduce test overhead and improve isolation

* Fix telemetry test isolation

- Properly isolate telemetry tests from mocking environment
- Preserve API keys and other necessary environment variables
- Ensure telemetry tests can run with real telemetry instances
This commit is contained in:
Greyson LaLonde
2025-08-18 11:55:30 -04:00
committed by GitHub
parent 0d3e462791
commit a17b93a7f8
4 changed files with 161 additions and 23 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
tests/telemetry/test_telemetry.py
View File

@@ -1,4 +1,5 @@
import os
import threading
from unittest.mock import patch
import pytest
@@ -11,12 +12,16 @@ from opentelemetry import trace
@pytest.fixture(autouse=True)
def cleanup_telemetry():
"""Automatically clean up Telemetry singleton between tests."""
Telemetry._instance = None
if hasattr(Telemetry, "_lock"):
Telemetry._lock = threading.Lock()
yield
Telemetry._instance = None
if hasattr(Telemetry, "_lock"):
Telemetry._lock = threading.Lock()
@pytest.mark.telemetry
@pytest.mark.parametrize(
"env_var,value,expected_ready",
[
@@ -36,6 +41,7 @@ def test_telemetry_environment_variables(env_var, value, expected_ready):
assert telemetry.ready is expected_ready
@pytest.mark.telemetry
def test_telemetry_enabled_by_default():
"""Test that telemetry is enabled by default."""
with patch.dict(os.environ, {}, clear=True):
@@ -44,6 +50,7 @@ def test_telemetry_enabled_by_default():
assert telemetry.ready is True
@pytest.mark.telemetry
@patch("crewai.telemetry.telemetry.logger.error")
@patch(
"opentelemetry.exporter.otlp.proto.http.trace_exporter.OTLPSpanExporter.export",
@@ -76,6 +83,7 @@ def test_telemetry_fails_due_connect_timeout(export_mock, logger_mock):
logger_mock.assert_called_once_with(error)
@pytest.mark.telemetry
def test_telemetry_singleton_pattern():
"""Test that Telemetry uses the singleton pattern correctly."""
Telemetry._instance = None

52
tests/telemetry/test_telemetry_disable.py
View File

@@ -14,14 +14,18 @@ def cleanup_telemetry():
Telemetry._instance = None
@pytest.mark.parametrize("env_var,value,expected_ready", [
("OTEL_SDK_DISABLED", "true", False),
("OTEL_SDK_DISABLED", "TRUE", False),
("CREWAI_DISABLE_TELEMETRY", "true", False),
("CREWAI_DISABLE_TELEMETRY", "TRUE", False),
("OTEL_SDK_DISABLED", "false", True),
("CREWAI_DISABLE_TELEMETRY", "false", True),
])
@pytest.mark.telemetry
@pytest.mark.parametrize(
"env_var,value,expected_ready",
[
("OTEL_SDK_DISABLED", "true", False),
("OTEL_SDK_DISABLED", "TRUE", False),
("CREWAI_DISABLE_TELEMETRY", "true", False),
("CREWAI_DISABLE_TELEMETRY", "TRUE", False),
("OTEL_SDK_DISABLED", "false", True),
("CREWAI_DISABLE_TELEMETRY", "false", True),
],
)
def test_telemetry_environment_variables(env_var, value, expected_ready):
"""Test telemetry state with different environment variable configurations."""
with patch.dict(os.environ, {env_var: value}):
@@ -30,6 +34,7 @@ def test_telemetry_environment_variables(env_var, value, expected_ready):
assert telemetry.ready is expected_ready
@pytest.mark.telemetry
def test_telemetry_enabled_by_default():
"""Test that telemetry is enabled by default."""
with patch.dict(os.environ, {}, clear=True):
@@ -38,57 +43,60 @@ def test_telemetry_enabled_by_default():
assert telemetry.ready is True
@pytest.mark.telemetry
def test_telemetry_disable_after_singleton_creation():
"""Test that telemetry operations are disabled when env var is set after singleton creation."""
with patch.dict(os.environ, {}, clear=True):
with patch("crewai.telemetry.telemetry.TracerProvider"):
telemetry = Telemetry()
assert telemetry.ready is True
mock_operation = MagicMock()
telemetry._safe_telemetry_operation(mock_operation)
mock_operation.assert_called_once()
mock_operation.reset_mock()
os.environ['CREWAI_DISABLE_TELEMETRY'] = 'true'
os.environ["CREWAI_DISABLE_TELEMETRY"] = "true"
telemetry._safe_telemetry_operation(mock_operation)
mock_operation.assert_not_called()
@pytest.mark.telemetry
def test_telemetry_disable_with_multiple_instances():
"""Test that multiple telemetry instances respect dynamically changed env vars."""
with patch.dict(os.environ, {}, clear=True):
with patch("crewai.telemetry.telemetry.TracerProvider"):
telemetry1 = Telemetry()
assert telemetry1.ready is True
os.environ['CREWAI_DISABLE_TELEMETRY'] = 'true'
os.environ["CREWAI_DISABLE_TELEMETRY"] = "true"
telemetry2 = Telemetry()
assert telemetry2 is telemetry1
assert telemetry2.ready is True
mock_operation = MagicMock()
telemetry2._safe_telemetry_operation(mock_operation)
mock_operation.assert_not_called()
@pytest.mark.telemetry
def test_telemetry_otel_sdk_disabled_after_creation():
"""Test that OTEL_SDK_DISABLED also works when set after singleton creation."""
with patch.dict(os.environ, {}, clear=True):
with patch("crewai.telemetry.telemetry.TracerProvider"):
telemetry = Telemetry()
assert telemetry.ready is True
mock_operation = MagicMock()
telemetry._safe_telemetry_operation(mock_operation)
mock_operation.assert_called_once()
mock_operation.reset_mock()
os.environ['OTEL_SDK_DISABLED'] = 'true'
os.environ["OTEL_SDK_DISABLED"] = "true"
telemetry._safe_telemetry_operation(mock_operation)
mock_operation.assert_not_called()