andre/crewAI
1
0
Fork 0
mirror of https://github.com/crewAIInc/crewAI.git synced 2026-08-10 08:21:54 +00:00
Code Issues Actions 6 Packages Projects Releases Wiki Activity

feat(telemetry): split runtime context from coding agent, add project id

Browse Source 
The coding-agent field answered two questions at once. A run with no TTY
reported "non_interactive" and an editor's integrated terminal reported
"vscode_terminal", both in the same field as the assistant name, so a run
that never had an assistant to detect was indistinguishable from one
whose assistant we failed to recognize. Together those two values were
the majority of what the field reported.

detect_coding_agent now answers only which assistant, returning "unknown"
when no marker matches. detect_runtime_context answers where the process
runs: ci, serverless, hosted_ide, notebook, container, the editor
terminals, and the interactive/non_interactive fallback. Both ride on
every span, so an assistant running inside CI reports both rather than
one masking the other.

The runtime markers are published platform contracts - CI providers,
container and serverless runtimes, hosted IDEs - so unlike the assistant
table they need no per-tool verification step. Presence is checked; no
value is read. The assistant table is unchanged: its entries still
require a confirmed, session-scoped variable, and the existing guard test
still enforces that.

Spans also carry project_id when the project declares one. It is read
through the read-only accessor, since minting an id belongs to the CLI
commands a user invoked rather than to a library call during execution,
and it is omitted entirely for projects without one. The attributes are
computed once per process and memoized, so the project file is not
re-read for each provider.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Joao Moura
2026-08-07 15:12:51 -07:00
parent 5fa010450b
commit 9a67ecf63b
4 changed files with 351 additions and 38 deletions
Download Patch File Download Diff File Expand all files Collapse all files

40
lib/crewai/src/crewai/telemetry/telemetry.py
View File

@@ -21,6 +21,7 @@ import threading
from typing import TYPE_CHECKING, Any
import weakref
from crewai_core.project import get_project_id
from opentelemetry import trace
from opentelemetry.context import Context
from opentelemetry.exporter.otlp.proto.http.trace_exporter import (
@@ -54,6 +55,7 @@ from crewai.telemetry.utils import (
add_crew_attributes,
close_span,
detect_coding_agent,
detect_runtime_context,
)
from crewai.utilities.i18n import I18N_DEFAULT
from crewai.utilities.logger_utils import suppress_warnings
@@ -172,6 +174,7 @@ class Telemetry:
# Weak so instrumented apps' providers are not kept alive by telemetry.
self._common_attributes_providers: weakref.WeakSet[Any] = weakref.WeakSet()
self._common_attributes_lock = threading.Lock()
self._common_attributes: dict[str, str] | None = None
if self._is_telemetry_disabled():
return
@@ -203,6 +206,39 @@ class Telemetry:
raise
self.ready = False
def _common_span_attributes(self) -> dict[str, str]:
"""Build the attributes every span carries, once per process.
Memoized because it is computed per provider and reads the project's
``pyproject.toml``, and because a process cannot change which
assistant, runtime, or project it belongs to partway through.
Returns:
Attributes to stamp on every span. ``project_id`` is omitted for
projects that do not declare one.
"""
if self._common_attributes is not None:
return self._common_attributes
attributes = {
"coding_agent": detect_coding_agent(),
"runtime_context": detect_runtime_context(),
}
try:
# Read-only: minting an id belongs to the CLI commands a user
# invoked, not to a library call during execution.
project_id = get_project_id()
except Exception as e: # Telemetry must never break execution.
logger.debug(f"Failed to read project id: {e}")
project_id = None
if project_id:
attributes["project_id"] = project_id
self._common_attributes = attributes
return attributes
def _attach_common_attributes(self, provider: Any) -> None:
"""Attach process-wide attributes to every span a provider emits.
@@ -230,9 +266,7 @@ class Telemetry:
if provider in self._common_attributes_providers:
return
add_span_processor(
CommonAttributesSpanProcessor(
{"coding_agent": detect_coding_agent()}
)
CommonAttributesSpanProcessor(self._common_span_attributes())
)
self._common_attributes_providers.add(provider)
except Exception as e: # Telemetry must never break execution.

79
lib/crewai/src/crewai/telemetry/utils.py
View File

@@ -12,7 +12,10 @@ from typing import TYPE_CHECKING, Any, Final
from opentelemetry.trace import Span, Status, StatusCode
from crewai.utilities.constants import CODING_AGENT_ENV_MARKERS
from crewai.utilities.constants import (
CODING_AGENT_ENV_MARKERS,
RUNTIME_CONTEXT_ENV_MARKERS,
)
if TYPE_CHECKING:
@@ -20,23 +23,32 @@ if TYPE_CHECKING:
from crewai.task import Task
# Editors whose integrated terminal implies a human is likely present. Used only
# as a weaker fallback when no explicit coding-agent marker is found.
# Editors whose integrated terminal a process can be launched from. Matched on
# an exact value rather than presence, since these variables name the terminal.
_EDITOR_TERM_MARKERS: Final[tuple[tuple[str, str, str], ...]] = (
("TERM_PROGRAM", "vscode", "vscode_terminal"),
("TERMINAL_EMULATOR", "JetBrains-JediTerm", "jetbrains_terminal"),
)
_FALLBACK_AGENT_NAMES: Final[tuple[str, ...]] = ("non_interactive", "unknown")
# Marks a process running inside a container when no more specific runtime
# marker is present.
_DOCKER_ENV_PATH: Final[str] = "/.dockerenv"
_UNKNOWN: Final[str] = "unknown"
# The complete set of values detect_coding_agent() can ever return. Every value
# is a literal from CODING_AGENT_ENV_MARKERS or this module, which is what makes
# the function structurally incapable of emitting PII: no environment value,
# path, hostname, or user-supplied string can reach the return value.
KNOWN_CODING_AGENTS: Final[frozenset[str]] = frozenset(
[name for name, _ in CODING_AGENT_ENV_MARKERS]
[name for name, _ in CODING_AGENT_ENV_MARKERS] + [_UNKNOWN]
)
# The same guarantee for detect_runtime_context(): a closed set of literals.
KNOWN_RUNTIME_CONTEXTS: Final[frozenset[str]] = frozenset(
[name for name, _ in RUNTIME_CONTEXT_ENV_MARKERS]
+ [name for _, _, name in _EDITOR_TERM_MARKERS]
+ list(_FALLBACK_AGENT_NAMES)
+ ["interactive", "non_interactive", _UNKNOWN]
)
@@ -57,27 +69,60 @@ def detect_coding_agent() -> str:
integrated terminal, so a result names the environment the process is
running *under*, not proof that an agent authored the code.
Answers only *which assistant*. Where the process runs is
:func:`detect_runtime_context`, so an "unknown" here is a genuine gap in
the marker table rather than a run that never had an assistant to find.
Returns:
A normalized assistant name (e.g. "claude_code", "cursor", "codex"),
an editor terminal hint (e.g. "vscode_terminal"), "non_interactive"
when no marker is found and there is no TTY, or "unknown" otherwise.
The result is always a member of KNOWN_CODING_AGENTS.
A normalized assistant name (e.g. "claude_code", "cursor", "codex"), or
"unknown" when no marker matches. The result is always a member of
KNOWN_CODING_AGENTS.
"""
for agent_name, env_vars in CODING_AGENT_ENV_MARKERS:
if any(os.environ.get(env_var) for env_var in env_vars):
return agent_name
for env_var, expected, agent_name in _EDITOR_TERM_MARKERS:
return _UNKNOWN
def detect_runtime_context() -> str:
"""Best-effort detection of where this process is running.
Separate from :func:`detect_coding_agent` because the two answer different
questions. Automated runs -- CI, containers, serverless -- have no
assistant to detect, and reporting them in the assistant field made an
unrecognized assistant indistinguishable from a run that could never have
had one.
Precedence runs most specific first: CI and hosted IDEs typically run
inside containers, so a bare container match only applies once the more
specific markers have been ruled out.
Returns:
One of the runtime names (e.g. "ci", "serverless", "container",
"notebook", "hosted_ide"), an editor terminal (e.g. "vscode_terminal"),
"interactive" or "non_interactive" when only a TTY check applies, or
"unknown" when that check cannot be made. The result is always a member
of KNOWN_RUNTIME_CONTEXTS.
"""
for context_name, env_vars in RUNTIME_CONTEXT_ENV_MARKERS:
if any(os.environ.get(env_var) for env_var in env_vars):
return context_name
for env_var, expected, context_name in _EDITOR_TERM_MARKERS:
if os.environ.get(env_var) == expected:
return agent_name
return context_name
try:
if not sys.stdout.isatty():
return "non_interactive"
except (AttributeError, ValueError, OSError):
return "unknown"
if os.path.exists(_DOCKER_ENV_PATH):
return "container"
except OSError:
pass
return "unknown"
try:
return "interactive" if sys.stdout.isatty() else "non_interactive"
except (AttributeError, ValueError, OSError):
return _UNKNOWN
def add_agent_fingerprint_to_span(

59
lib/crewai/src/crewai/utilities/constants.py
View File

@@ -13,14 +13,20 @@ from pydantic_core import CoreSchema
__all__ = [
"CC_ENV_VAR",
"CI_ENV_VARS",
"CODEX_ENV_VARS",
"CODING_AGENT_ENV_MARKERS",
"CONTAINER_ENV_VARS",
"CREWAI_TRAINED_AGENTS_FILE_ENV",
"CURSOR_ENV_VARS",
"EMITTER_COLOR",
"HOSTED_IDE_ENV_VARS",
"KNOWLEDGE_DIRECTORY",
"MAX_FILE_NAME_LENGTH",
"NOTEBOOK_ENV_VARS",
"NOT_SPECIFIED",
"RUNTIME_CONTEXT_ENV_MARKERS",
"SERVERLESS_ENV_VARS",
"TRAINED_AGENTS_DATA_FILE",
"TRAINING_DATA_FILE",
]
@@ -69,6 +75,59 @@ CODING_AGENT_ENV_MARKERS: Final[tuple[tuple[str, tuple[str, ...]], ...]] = (
("cursor", CURSOR_ENV_VARS),
)
# Markers for *where* a process runs, kept separate from which assistant is
# driving it. The two answer different questions: a scheduled container run has
# no assistant to detect, and folding it into the assistant field made "no
# marker found" and "no assistant possible" indistinguishable.
#
# These are published platform contracts rather than per-tool observations, so
# they do not need the case-by-case verification the assistant table requires.
# Presence is all that is checked; no value is ever read.
CI_ENV_VARS: Final[tuple[str, ...]] = (
"APPVEYOR",
"BITBUCKET_BUILD_NUMBER",
"BUILDKITE",
"CI",
"CIRCLECI",
"DRONE",
"GITHUB_ACTIONS",
"GITLAB_CI",
"JENKINS_URL",
"TEAMCITY_VERSION",
"TF_BUILD",
"TRAVIS",
)
SERVERLESS_ENV_VARS: Final[tuple[str, ...]] = (
"AWS_EXECUTION_ENV",
"AWS_LAMBDA_FUNCTION_NAME",
"DYNO",
"FUNCTION_TARGET",
"K_SERVICE",
"VERCEL",
"WEBSITE_INSTANCE_ID",
)
HOSTED_IDE_ENV_VARS: Final[tuple[str, ...]] = (
"CODESPACES",
"GITPOD_WORKSPACE_ID",
"REPL_ID",
)
NOTEBOOK_ENV_VARS: Final[tuple[str, ...]] = (
"COLAB_RELEASE_TAG",
"JPY_PARENT_PID",
)
CONTAINER_ENV_VARS: Final[tuple[str, ...]] = ("KUBERNETES_SERVICE_HOST",)
# Ordered most specific first. CI jobs and hosted IDEs usually run inside
# containers, so a bare container match is only meaningful once the others have
# been ruled out.
RUNTIME_CONTEXT_ENV_MARKERS: Final[tuple[tuple[str, tuple[str, ...]], ...]] = (
("ci", CI_ENV_VARS),
("serverless", SERVERLESS_ENV_VARS),
("hosted_ide", HOSTED_IDE_ENV_VARS),
("notebook", NOTEBOOK_ENV_VARS),
("container", CONTAINER_ENV_VARS),
)
class _NotSpecified:
"""Sentinel class to detect when no value has been explicitly provided.

211
lib/crewai/tests/telemetry/test_coding_agent_detection.py
View File

@@ -5,20 +5,38 @@ from unittest.mock import patch
import pytest
from crewai.telemetry.utils import KNOWN_CODING_AGENTS, detect_coding_agent
from crewai.telemetry.utils import (
KNOWN_CODING_AGENTS,
KNOWN_RUNTIME_CONTEXTS,
detect_coding_agent,
detect_runtime_context,
)
from crewai.utilities.constants import (
CC_ENV_VAR,
CODEX_ENV_VARS,
CODING_AGENT_ENV_MARKERS,
CURSOR_ENV_VARS,
RUNTIME_CONTEXT_ENV_MARKERS,
)
# Derived from the shared table rather than restated, so adding an assistant
# there cannot leave these tests silently checking a stale marker set.
ALL_MARKERS = tuple(
var for _, env_vars in CODING_AGENT_ENV_MARKERS for var in env_vars
) + ("TERM_PROGRAM", "TERMINAL_EMULATOR")
# Derived from the shared tables rather than restated, so adding an assistant
# or runtime there cannot leave these tests silently checking a stale set.
RUNTIME_MARKERS = tuple(
var for _, env_vars in RUNTIME_CONTEXT_ENV_MARKERS for var in env_vars
)
ALL_MARKERS = (
tuple(var for _, env_vars in CODING_AGENT_ENV_MARKERS for var in env_vars)
+ RUNTIME_MARKERS
+ ("TERM_PROGRAM", "TERMINAL_EMULATOR")
)
EVERY_RUNTIME_CASE = [
(var, context)
for context, env_vars in RUNTIME_CONTEXT_ENV_MARKERS
for var in env_vars
]
EVERY_MARKER_CASE = [
(var, agent) for agent, env_vars in CODING_AGENT_ENV_MARKERS for var in env_vars
@@ -216,16 +234,16 @@ def test_concurrent_attach_registers_the_processor_once(isolated_telemetry, clea
def test_editor_terminal_requires_exact_value(clean_env):
clean_env.setenv("TERM_PROGRAM", "vscode")
assert detect_coding_agent() == "vscode_terminal"
assert detect_runtime_context() == "vscode_terminal"
clean_env.setenv("TERM_PROGRAM", "iTerm.app")
assert detect_coding_agent() != "vscode_terminal"
assert detect_runtime_context() != "vscode_terminal"
def test_explicit_agent_marker_wins_over_editor_terminal(clean_env):
def test_editor_terminal_is_not_reported_as_an_assistant(clean_env):
"""An editor's terminal says where a process runs, not who drove it."""
clean_env.setenv("TERM_PROGRAM", "vscode")
clean_env.setenv("CLAUDECODE", "1")
assert detect_coding_agent() == "claude_code"
assert detect_coding_agent() == "unknown"
def test_empty_marker_value_is_ignored(clean_env):
@@ -233,12 +251,61 @@ def test_empty_marker_value_is_ignored(clean_env):
assert detect_coding_agent() != "claude_code"
@pytest.mark.parametrize(("env_var", "expected"), EVERY_RUNTIME_CASE)
def test_detects_every_runtime_marker(clean_env, env_var, expected):
"""Every runtime marker must map to its context."""
clean_env.setenv(env_var, "1")
assert detect_runtime_context() == expected
def test_runtime_precedence_prefers_the_most_specific(clean_env):
"""CI and hosted IDEs usually run in containers; the specific one wins."""
clean_env.setenv("KUBERNETES_SERVICE_HOST", "10.0.0.1")
assert detect_runtime_context() == "container"
clean_env.setenv("GITHUB_ACTIONS", "true")
assert detect_runtime_context() == "ci"
def test_an_automated_run_still_reports_an_unknown_assistant(clean_env):
"""The split must keep the two fields independent.
A CI run has no assistant to find, which is different from failing to
recognize one - the reason they no longer share a field.
"""
clean_env.setenv("CI", "true")
assert detect_runtime_context() == "ci"
assert detect_coding_agent() == "unknown"
def test_assistant_and_runtime_are_reported_together(clean_env):
"""An assistant inside CI must not mask either signal."""
clean_env.setenv("CI", "true")
clean_env.setenv("CLAUDECODE", "1")
assert detect_coding_agent() == "claude_code"
assert detect_runtime_context() == "ci"
def test_falls_back_to_non_interactive_without_tty(clean_env, monkeypatch):
monkeypatch.setattr("os.path.exists", lambda path: False)
monkeypatch.setattr("sys.stdout", type("S", (), {"isatty": lambda self: False})())
assert detect_coding_agent() == "non_interactive"
assert detect_runtime_context() == "non_interactive"
def test_falls_back_to_unknown_with_tty(clean_env, monkeypatch):
def test_falls_back_to_interactive_with_tty(clean_env, monkeypatch):
monkeypatch.setattr("os.path.exists", lambda path: False)
monkeypatch.setattr("sys.stdout", type("S", (), {"isatty": lambda self: True})())
assert detect_runtime_context() == "interactive"
def test_dockerenv_marks_a_container(clean_env, monkeypatch):
"""The container check is the last resort before the TTY fallback."""
monkeypatch.setattr("os.path.exists", lambda path: path == "/.dockerenv")
assert detect_runtime_context() == "container"
def test_unmatched_assistant_is_unknown(clean_env, monkeypatch):
"""No marker means a gap in the table, reported as unknown."""
monkeypatch.setattr("sys.stdout", type("S", (), {"isatty": lambda self: True})())
assert detect_coding_agent() == "unknown"
@@ -255,8 +322,9 @@ def test_handles_broken_stdout(clean_env, monkeypatch):
def isatty(self):
raise ValueError("detached")
monkeypatch.setattr("os.path.exists", lambda path: False)
monkeypatch.setattr("sys.stdout", BrokenStdout())
assert detect_coding_agent() == "unknown"
assert detect_runtime_context() == "unknown"
def test_result_is_always_a_known_literal(clean_env):
@@ -269,15 +337,18 @@ def test_result_is_always_a_known_literal(clean_env):
for var in ALL_MARKERS:
clean_env.setenv(var, sensitive)
result = detect_coding_agent()
assert result in KNOWN_CODING_AGENTS
assert sensitive not in result
agent = detect_coding_agent()
context = detect_runtime_context()
assert agent in KNOWN_CODING_AGENTS
assert context in KNOWN_RUNTIME_CONTEXTS
assert sensitive not in agent
assert sensitive not in context
clean_env.delenv(var, raising=False)
def test_known_agents_contains_no_pii_shaped_values():
"""Every possible emitted value is a short, opaque identifier."""
for name in KNOWN_CODING_AGENTS:
for name in KNOWN_CODING_AGENTS | KNOWN_RUNTIME_CONTEXTS:
assert name.replace("_", "").isalnum(), name
assert len(name) <= 32, name
@@ -402,3 +473,107 @@ def test_attaching_to_a_provider_without_processors_is_safe(isolated_telemetry):
telemetry = isolated_telemetry()
telemetry._attach_common_attributes(object())
def _common_attributes(monkeypatch, project_id=None):
"""Build the process-wide span attributes with a stubbed project id."""
from crewai.telemetry.telemetry import Telemetry
monkeypatch.setattr(
"crewai.telemetry.telemetry.get_project_id", lambda *a, **k: project_id
)
telemetry = Telemetry.__new__(Telemetry)
telemetry._common_attributes = None
return telemetry._common_span_attributes()
def test_common_attributes_carry_agent_and_runtime(clean_env, monkeypatch):
"""Both fields ride on every span, independently of each other."""
clean_env.setenv("CLAUDECODE", "1")
clean_env.setenv("GITHUB_ACTIONS", "true")
attributes = _common_attributes(monkeypatch)
assert attributes["coding_agent"] == "claude_code"
assert attributes["runtime_context"] == "ci"
def test_common_attributes_include_project_id_when_declared(clean_env, monkeypatch):
attributes = _common_attributes(monkeypatch, project_id="proj-123")
assert attributes["project_id"] == "proj-123"
def test_project_id_is_omitted_when_absent(clean_env, monkeypatch):
"""Projects without an id must not report a placeholder."""
attributes = _common_attributes(monkeypatch, project_id=None)
assert "project_id" not in attributes
def test_project_id_lookup_never_breaks_telemetry(clean_env, monkeypatch):
"""A failed lookup degrades to omitting the attribute."""
from crewai.telemetry.telemetry import Telemetry
def boom(*args, **kwargs):
raise OSError("unreadable")
monkeypatch.setattr("crewai.telemetry.telemetry.get_project_id", boom)
telemetry = Telemetry.__new__(Telemetry)
telemetry._common_attributes = None
attributes = telemetry._common_span_attributes()
assert "project_id" not in attributes
assert "coding_agent" in attributes
def test_common_attributes_are_computed_once(clean_env, monkeypatch):
"""The project file must not be re-read for each provider."""
from crewai.telemetry.telemetry import Telemetry
calls = []
def counting_get_project_id(*args, **kwargs):
calls.append(1)
return "proj-123"
monkeypatch.setattr(
"crewai.telemetry.telemetry.get_project_id", counting_get_project_id
)
telemetry = Telemetry.__new__(Telemetry)
telemetry._common_attributes = None
first = telemetry._common_span_attributes()
second = telemetry._common_span_attributes()
assert first is second
assert len(calls) == 1
def test_all_common_attributes_land_on_exported_spans(clean_env, monkeypatch):
"""End-to-end: every common attribute survives onto arbitrary spans."""
from opentelemetry.sdk.trace import TracerProvider
from opentelemetry.sdk.trace.export import SimpleSpanProcessor
from opentelemetry.sdk.trace.export.in_memory_span_exporter import (
InMemorySpanExporter,
)
from crewai.telemetry.telemetry import CommonAttributesSpanProcessor
clean_env.setenv("CLAUDECODE", "1")
clean_env.setenv("CI", "true")
attributes = _common_attributes(monkeypatch, project_id="proj-123")
exporter = InMemorySpanExporter()
provider = TracerProvider()
provider.add_span_processor(CommonAttributesSpanProcessor(attributes))
provider.add_span_processor(SimpleSpanProcessor(exporter))
provider.get_tracer("test").start_span("Feature Usage").end()
provider.force_flush()
exported = dict(exporter.get_finished_spans()[0].attributes)
assert exported["coding_agent"] == "claude_code"
assert exported["runtime_context"] == "ci"
assert exported["project_id"] == "proj-123"