diff --git a/lib/crewai/src/crewai/telemetry/telemetry.py b/lib/crewai/src/crewai/telemetry/telemetry.py index ddc5bcfd3..5b91c5af9 100644 --- a/lib/crewai/src/crewai/telemetry/telemetry.py +++ b/lib/crewai/src/crewai/telemetry/telemetry.py @@ -21,6 +21,7 @@ import threading from typing import TYPE_CHECKING, Any import weakref +from crewai_core.project import get_project_id from opentelemetry import trace from opentelemetry.context import Context from opentelemetry.exporter.otlp.proto.http.trace_exporter import ( @@ -54,6 +55,7 @@ from crewai.telemetry.utils import ( add_crew_attributes, close_span, detect_coding_agent, + detect_runtime_context, ) from crewai.utilities.i18n import I18N_DEFAULT from crewai.utilities.logger_utils import suppress_warnings @@ -172,6 +174,7 @@ class Telemetry: # Weak so instrumented apps' providers are not kept alive by telemetry. self._common_attributes_providers: weakref.WeakSet[Any] = weakref.WeakSet() self._common_attributes_lock = threading.Lock() + self._common_attributes: dict[str, str] | None = None if self._is_telemetry_disabled(): return @@ -203,6 +206,39 @@ class Telemetry: raise self.ready = False + def _common_span_attributes(self) -> dict[str, str]: + """Build the attributes every span carries, once per process. + + Memoized because it is computed per provider and reads the project's + ``pyproject.toml``, and because a process cannot change which + assistant, runtime, or project it belongs to partway through. + + Returns: + Attributes to stamp on every span. ``project_id`` is omitted for + projects that do not declare one. + """ + if self._common_attributes is not None: + return self._common_attributes + + attributes = { + "coding_agent": detect_coding_agent(), + "runtime_context": detect_runtime_context(), + } + + try: + # Read-only: minting an id belongs to the CLI commands a user + # invoked, not to a library call during execution. + project_id = get_project_id() + except Exception as e: # Telemetry must never break execution. + logger.debug(f"Failed to read project id: {e}") + project_id = None + + if project_id: + attributes["project_id"] = project_id + + self._common_attributes = attributes + return attributes + def _attach_common_attributes(self, provider: Any) -> None: """Attach process-wide attributes to every span a provider emits. @@ -230,9 +266,7 @@ class Telemetry: if provider in self._common_attributes_providers: return add_span_processor( - CommonAttributesSpanProcessor( - {"coding_agent": detect_coding_agent()} - ) + CommonAttributesSpanProcessor(self._common_span_attributes()) ) self._common_attributes_providers.add(provider) except Exception as e: # Telemetry must never break execution. diff --git a/lib/crewai/src/crewai/telemetry/utils.py b/lib/crewai/src/crewai/telemetry/utils.py index 36c96a385..32805a55c 100644 --- a/lib/crewai/src/crewai/telemetry/utils.py +++ b/lib/crewai/src/crewai/telemetry/utils.py @@ -12,7 +12,10 @@ from typing import TYPE_CHECKING, Any, Final from opentelemetry.trace import Span, Status, StatusCode -from crewai.utilities.constants import CODING_AGENT_ENV_MARKERS +from crewai.utilities.constants import ( + CODING_AGENT_ENV_MARKERS, + RUNTIME_CONTEXT_ENV_MARKERS, +) if TYPE_CHECKING: @@ -20,23 +23,32 @@ if TYPE_CHECKING: from crewai.task import Task -# Editors whose integrated terminal implies a human is likely present. Used only -# as a weaker fallback when no explicit coding-agent marker is found. +# Editors whose integrated terminal a process can be launched from. Matched on +# an exact value rather than presence, since these variables name the terminal. _EDITOR_TERM_MARKERS: Final[tuple[tuple[str, str, str], ...]] = ( ("TERM_PROGRAM", "vscode", "vscode_terminal"), ("TERMINAL_EMULATOR", "JetBrains-JediTerm", "jetbrains_terminal"), ) -_FALLBACK_AGENT_NAMES: Final[tuple[str, ...]] = ("non_interactive", "unknown") +# Marks a process running inside a container when no more specific runtime +# marker is present. +_DOCKER_ENV_PATH: Final[str] = "/.dockerenv" + +_UNKNOWN: Final[str] = "unknown" # The complete set of values detect_coding_agent() can ever return. Every value # is a literal from CODING_AGENT_ENV_MARKERS or this module, which is what makes # the function structurally incapable of emitting PII: no environment value, # path, hostname, or user-supplied string can reach the return value. KNOWN_CODING_AGENTS: Final[frozenset[str]] = frozenset( - [name for name, _ in CODING_AGENT_ENV_MARKERS] + [name for name, _ in CODING_AGENT_ENV_MARKERS] + [_UNKNOWN] +) + +# The same guarantee for detect_runtime_context(): a closed set of literals. +KNOWN_RUNTIME_CONTEXTS: Final[frozenset[str]] = frozenset( + [name for name, _ in RUNTIME_CONTEXT_ENV_MARKERS] + [name for _, _, name in _EDITOR_TERM_MARKERS] - + list(_FALLBACK_AGENT_NAMES) + + ["interactive", "non_interactive", _UNKNOWN] ) @@ -57,27 +69,60 @@ def detect_coding_agent() -> str: integrated terminal, so a result names the environment the process is running *under*, not proof that an agent authored the code. + Answers only *which assistant*. Where the process runs is + :func:`detect_runtime_context`, so an "unknown" here is a genuine gap in + the marker table rather than a run that never had an assistant to find. + Returns: - A normalized assistant name (e.g. "claude_code", "cursor", "codex"), - an editor terminal hint (e.g. "vscode_terminal"), "non_interactive" - when no marker is found and there is no TTY, or "unknown" otherwise. - The result is always a member of KNOWN_CODING_AGENTS. + A normalized assistant name (e.g. "claude_code", "cursor", "codex"), or + "unknown" when no marker matches. The result is always a member of + KNOWN_CODING_AGENTS. """ for agent_name, env_vars in CODING_AGENT_ENV_MARKERS: if any(os.environ.get(env_var) for env_var in env_vars): return agent_name - for env_var, expected, agent_name in _EDITOR_TERM_MARKERS: + return _UNKNOWN + + +def detect_runtime_context() -> str: + """Best-effort detection of where this process is running. + + Separate from :func:`detect_coding_agent` because the two answer different + questions. Automated runs -- CI, containers, serverless -- have no + assistant to detect, and reporting them in the assistant field made an + unrecognized assistant indistinguishable from a run that could never have + had one. + + Precedence runs most specific first: CI and hosted IDEs typically run + inside containers, so a bare container match only applies once the more + specific markers have been ruled out. + + Returns: + One of the runtime names (e.g. "ci", "serverless", "container", + "notebook", "hosted_ide"), an editor terminal (e.g. "vscode_terminal"), + "interactive" or "non_interactive" when only a TTY check applies, or + "unknown" when that check cannot be made. The result is always a member + of KNOWN_RUNTIME_CONTEXTS. + """ + for context_name, env_vars in RUNTIME_CONTEXT_ENV_MARKERS: + if any(os.environ.get(env_var) for env_var in env_vars): + return context_name + + for env_var, expected, context_name in _EDITOR_TERM_MARKERS: if os.environ.get(env_var) == expected: - return agent_name + return context_name try: - if not sys.stdout.isatty(): - return "non_interactive" - except (AttributeError, ValueError, OSError): - return "unknown" + if os.path.exists(_DOCKER_ENV_PATH): + return "container" + except OSError: + pass - return "unknown" + try: + return "interactive" if sys.stdout.isatty() else "non_interactive" + except (AttributeError, ValueError, OSError): + return _UNKNOWN def add_agent_fingerprint_to_span( diff --git a/lib/crewai/src/crewai/utilities/constants.py b/lib/crewai/src/crewai/utilities/constants.py index bf1c43964..9a1c6551a 100644 --- a/lib/crewai/src/crewai/utilities/constants.py +++ b/lib/crewai/src/crewai/utilities/constants.py @@ -13,14 +13,20 @@ from pydantic_core import CoreSchema __all__ = [ "CC_ENV_VAR", + "CI_ENV_VARS", "CODEX_ENV_VARS", "CODING_AGENT_ENV_MARKERS", + "CONTAINER_ENV_VARS", "CREWAI_TRAINED_AGENTS_FILE_ENV", "CURSOR_ENV_VARS", "EMITTER_COLOR", + "HOSTED_IDE_ENV_VARS", "KNOWLEDGE_DIRECTORY", "MAX_FILE_NAME_LENGTH", + "NOTEBOOK_ENV_VARS", "NOT_SPECIFIED", + "RUNTIME_CONTEXT_ENV_MARKERS", + "SERVERLESS_ENV_VARS", "TRAINED_AGENTS_DATA_FILE", "TRAINING_DATA_FILE", ] @@ -69,6 +75,59 @@ CODING_AGENT_ENV_MARKERS: Final[tuple[tuple[str, tuple[str, ...]], ...]] = ( ("cursor", CURSOR_ENV_VARS), ) +# Markers for *where* a process runs, kept separate from which assistant is +# driving it. The two answer different questions: a scheduled container run has +# no assistant to detect, and folding it into the assistant field made "no +# marker found" and "no assistant possible" indistinguishable. +# +# These are published platform contracts rather than per-tool observations, so +# they do not need the case-by-case verification the assistant table requires. +# Presence is all that is checked; no value is ever read. +CI_ENV_VARS: Final[tuple[str, ...]] = ( + "APPVEYOR", + "BITBUCKET_BUILD_NUMBER", + "BUILDKITE", + "CI", + "CIRCLECI", + "DRONE", + "GITHUB_ACTIONS", + "GITLAB_CI", + "JENKINS_URL", + "TEAMCITY_VERSION", + "TF_BUILD", + "TRAVIS", +) +SERVERLESS_ENV_VARS: Final[tuple[str, ...]] = ( + "AWS_EXECUTION_ENV", + "AWS_LAMBDA_FUNCTION_NAME", + "DYNO", + "FUNCTION_TARGET", + "K_SERVICE", + "VERCEL", + "WEBSITE_INSTANCE_ID", +) +HOSTED_IDE_ENV_VARS: Final[tuple[str, ...]] = ( + "CODESPACES", + "GITPOD_WORKSPACE_ID", + "REPL_ID", +) +NOTEBOOK_ENV_VARS: Final[tuple[str, ...]] = ( + "COLAB_RELEASE_TAG", + "JPY_PARENT_PID", +) +CONTAINER_ENV_VARS: Final[tuple[str, ...]] = ("KUBERNETES_SERVICE_HOST",) + +# Ordered most specific first. CI jobs and hosted IDEs usually run inside +# containers, so a bare container match is only meaningful once the others have +# been ruled out. +RUNTIME_CONTEXT_ENV_MARKERS: Final[tuple[tuple[str, tuple[str, ...]], ...]] = ( + ("ci", CI_ENV_VARS), + ("serverless", SERVERLESS_ENV_VARS), + ("hosted_ide", HOSTED_IDE_ENV_VARS), + ("notebook", NOTEBOOK_ENV_VARS), + ("container", CONTAINER_ENV_VARS), +) + class _NotSpecified: """Sentinel class to detect when no value has been explicitly provided. diff --git a/lib/crewai/tests/telemetry/test_coding_agent_detection.py b/lib/crewai/tests/telemetry/test_coding_agent_detection.py index 087ad2920..a7b81923a 100644 --- a/lib/crewai/tests/telemetry/test_coding_agent_detection.py +++ b/lib/crewai/tests/telemetry/test_coding_agent_detection.py @@ -5,20 +5,38 @@ from unittest.mock import patch import pytest -from crewai.telemetry.utils import KNOWN_CODING_AGENTS, detect_coding_agent +from crewai.telemetry.utils import ( + KNOWN_CODING_AGENTS, + KNOWN_RUNTIME_CONTEXTS, + detect_coding_agent, + detect_runtime_context, +) from crewai.utilities.constants import ( CC_ENV_VAR, CODEX_ENV_VARS, CODING_AGENT_ENV_MARKERS, CURSOR_ENV_VARS, + RUNTIME_CONTEXT_ENV_MARKERS, ) -# Derived from the shared table rather than restated, so adding an assistant -# there cannot leave these tests silently checking a stale marker set. -ALL_MARKERS = tuple( - var for _, env_vars in CODING_AGENT_ENV_MARKERS for var in env_vars -) + ("TERM_PROGRAM", "TERMINAL_EMULATOR") +# Derived from the shared tables rather than restated, so adding an assistant +# or runtime there cannot leave these tests silently checking a stale set. +RUNTIME_MARKERS = tuple( + var for _, env_vars in RUNTIME_CONTEXT_ENV_MARKERS for var in env_vars +) + +ALL_MARKERS = ( + tuple(var for _, env_vars in CODING_AGENT_ENV_MARKERS for var in env_vars) + + RUNTIME_MARKERS + + ("TERM_PROGRAM", "TERMINAL_EMULATOR") +) + +EVERY_RUNTIME_CASE = [ + (var, context) + for context, env_vars in RUNTIME_CONTEXT_ENV_MARKERS + for var in env_vars +] EVERY_MARKER_CASE = [ (var, agent) for agent, env_vars in CODING_AGENT_ENV_MARKERS for var in env_vars @@ -216,16 +234,16 @@ def test_concurrent_attach_registers_the_processor_once(isolated_telemetry, clea def test_editor_terminal_requires_exact_value(clean_env): clean_env.setenv("TERM_PROGRAM", "vscode") - assert detect_coding_agent() == "vscode_terminal" + assert detect_runtime_context() == "vscode_terminal" clean_env.setenv("TERM_PROGRAM", "iTerm.app") - assert detect_coding_agent() != "vscode_terminal" + assert detect_runtime_context() != "vscode_terminal" -def test_explicit_agent_marker_wins_over_editor_terminal(clean_env): +def test_editor_terminal_is_not_reported_as_an_assistant(clean_env): + """An editor's terminal says where a process runs, not who drove it.""" clean_env.setenv("TERM_PROGRAM", "vscode") - clean_env.setenv("CLAUDECODE", "1") - assert detect_coding_agent() == "claude_code" + assert detect_coding_agent() == "unknown" def test_empty_marker_value_is_ignored(clean_env): @@ -233,12 +251,61 @@ def test_empty_marker_value_is_ignored(clean_env): assert detect_coding_agent() != "claude_code" +@pytest.mark.parametrize(("env_var", "expected"), EVERY_RUNTIME_CASE) +def test_detects_every_runtime_marker(clean_env, env_var, expected): + """Every runtime marker must map to its context.""" + clean_env.setenv(env_var, "1") + assert detect_runtime_context() == expected + + +def test_runtime_precedence_prefers_the_most_specific(clean_env): + """CI and hosted IDEs usually run in containers; the specific one wins.""" + clean_env.setenv("KUBERNETES_SERVICE_HOST", "10.0.0.1") + assert detect_runtime_context() == "container" + + clean_env.setenv("GITHUB_ACTIONS", "true") + assert detect_runtime_context() == "ci" + + +def test_an_automated_run_still_reports_an_unknown_assistant(clean_env): + """The split must keep the two fields independent. + + A CI run has no assistant to find, which is different from failing to + recognize one - the reason they no longer share a field. + """ + clean_env.setenv("CI", "true") + assert detect_runtime_context() == "ci" + assert detect_coding_agent() == "unknown" + + +def test_assistant_and_runtime_are_reported_together(clean_env): + """An assistant inside CI must not mask either signal.""" + clean_env.setenv("CI", "true") + clean_env.setenv("CLAUDECODE", "1") + assert detect_coding_agent() == "claude_code" + assert detect_runtime_context() == "ci" + + def test_falls_back_to_non_interactive_without_tty(clean_env, monkeypatch): + monkeypatch.setattr("os.path.exists", lambda path: False) monkeypatch.setattr("sys.stdout", type("S", (), {"isatty": lambda self: False})()) - assert detect_coding_agent() == "non_interactive" + assert detect_runtime_context() == "non_interactive" -def test_falls_back_to_unknown_with_tty(clean_env, monkeypatch): +def test_falls_back_to_interactive_with_tty(clean_env, monkeypatch): + monkeypatch.setattr("os.path.exists", lambda path: False) + monkeypatch.setattr("sys.stdout", type("S", (), {"isatty": lambda self: True})()) + assert detect_runtime_context() == "interactive" + + +def test_dockerenv_marks_a_container(clean_env, monkeypatch): + """The container check is the last resort before the TTY fallback.""" + monkeypatch.setattr("os.path.exists", lambda path: path == "/.dockerenv") + assert detect_runtime_context() == "container" + + +def test_unmatched_assistant_is_unknown(clean_env, monkeypatch): + """No marker means a gap in the table, reported as unknown.""" monkeypatch.setattr("sys.stdout", type("S", (), {"isatty": lambda self: True})()) assert detect_coding_agent() == "unknown" @@ -255,8 +322,9 @@ def test_handles_broken_stdout(clean_env, monkeypatch): def isatty(self): raise ValueError("detached") + monkeypatch.setattr("os.path.exists", lambda path: False) monkeypatch.setattr("sys.stdout", BrokenStdout()) - assert detect_coding_agent() == "unknown" + assert detect_runtime_context() == "unknown" def test_result_is_always_a_known_literal(clean_env): @@ -269,15 +337,18 @@ def test_result_is_always_a_known_literal(clean_env): for var in ALL_MARKERS: clean_env.setenv(var, sensitive) - result = detect_coding_agent() - assert result in KNOWN_CODING_AGENTS - assert sensitive not in result + agent = detect_coding_agent() + context = detect_runtime_context() + assert agent in KNOWN_CODING_AGENTS + assert context in KNOWN_RUNTIME_CONTEXTS + assert sensitive not in agent + assert sensitive not in context clean_env.delenv(var, raising=False) def test_known_agents_contains_no_pii_shaped_values(): """Every possible emitted value is a short, opaque identifier.""" - for name in KNOWN_CODING_AGENTS: + for name in KNOWN_CODING_AGENTS | KNOWN_RUNTIME_CONTEXTS: assert name.replace("_", "").isalnum(), name assert len(name) <= 32, name @@ -402,3 +473,107 @@ def test_attaching_to_a_provider_without_processors_is_safe(isolated_telemetry): telemetry = isolated_telemetry() telemetry._attach_common_attributes(object()) + + +def _common_attributes(monkeypatch, project_id=None): + """Build the process-wide span attributes with a stubbed project id.""" + from crewai.telemetry.telemetry import Telemetry + + monkeypatch.setattr( + "crewai.telemetry.telemetry.get_project_id", lambda *a, **k: project_id + ) + telemetry = Telemetry.__new__(Telemetry) + telemetry._common_attributes = None + return telemetry._common_span_attributes() + + +def test_common_attributes_carry_agent_and_runtime(clean_env, monkeypatch): + """Both fields ride on every span, independently of each other.""" + clean_env.setenv("CLAUDECODE", "1") + clean_env.setenv("GITHUB_ACTIONS", "true") + + attributes = _common_attributes(monkeypatch) + + assert attributes["coding_agent"] == "claude_code" + assert attributes["runtime_context"] == "ci" + + +def test_common_attributes_include_project_id_when_declared(clean_env, monkeypatch): + attributes = _common_attributes(monkeypatch, project_id="proj-123") + + assert attributes["project_id"] == "proj-123" + + +def test_project_id_is_omitted_when_absent(clean_env, monkeypatch): + """Projects without an id must not report a placeholder.""" + attributes = _common_attributes(monkeypatch, project_id=None) + + assert "project_id" not in attributes + + +def test_project_id_lookup_never_breaks_telemetry(clean_env, monkeypatch): + """A failed lookup degrades to omitting the attribute.""" + from crewai.telemetry.telemetry import Telemetry + + def boom(*args, **kwargs): + raise OSError("unreadable") + + monkeypatch.setattr("crewai.telemetry.telemetry.get_project_id", boom) + telemetry = Telemetry.__new__(Telemetry) + telemetry._common_attributes = None + + attributes = telemetry._common_span_attributes() + + assert "project_id" not in attributes + assert "coding_agent" in attributes + + +def test_common_attributes_are_computed_once(clean_env, monkeypatch): + """The project file must not be re-read for each provider.""" + from crewai.telemetry.telemetry import Telemetry + + calls = [] + + def counting_get_project_id(*args, **kwargs): + calls.append(1) + return "proj-123" + + monkeypatch.setattr( + "crewai.telemetry.telemetry.get_project_id", counting_get_project_id + ) + telemetry = Telemetry.__new__(Telemetry) + telemetry._common_attributes = None + + first = telemetry._common_span_attributes() + second = telemetry._common_span_attributes() + + assert first is second + assert len(calls) == 1 + + +def test_all_common_attributes_land_on_exported_spans(clean_env, monkeypatch): + """End-to-end: every common attribute survives onto arbitrary spans.""" + from opentelemetry.sdk.trace import TracerProvider + from opentelemetry.sdk.trace.export import SimpleSpanProcessor + from opentelemetry.sdk.trace.export.in_memory_span_exporter import ( + InMemorySpanExporter, + ) + + from crewai.telemetry.telemetry import CommonAttributesSpanProcessor + + clean_env.setenv("CLAUDECODE", "1") + clean_env.setenv("CI", "true") + attributes = _common_attributes(monkeypatch, project_id="proj-123") + + exporter = InMemorySpanExporter() + provider = TracerProvider() + provider.add_span_processor(CommonAttributesSpanProcessor(attributes)) + provider.add_span_processor(SimpleSpanProcessor(exporter)) + + provider.get_tracer("test").start_span("Feature Usage").end() + provider.force_flush() + + exported = dict(exporter.get_finished_spans()[0].attributes) + assert exported["coding_agent"] == "claude_code" + assert exported["runtime_context"] == "ci" + assert exported["project_id"] == "proj-123"