mirror of
https://github.com/crewAIInc/crewAI.git
synced 2026-09-24 03:41:36 +00:00
feat(tracing): record the last traced run for crewai eval instead of printing it
After a traced run, crewAI printed a panel with the execution id and a viewer link. The id is internal, and the panel exposed it for no purpose a user has. Nothing is printed now. Instead, once the run's spans have reached Wharf, crewAI writes `.crewai/last_run.json` in the project: the execution id, when the run started and ended, whether it was traced anonymously or under an account, and the AMP base url. `crewai eval` reads it back, so the user evaluates their last run without pasting anything. GrantSpanExporter.record_export replaces show_trace_summary at the two finish points (an authenticated run's shutdown, an anonymous run's share). Only a run whose every export succeeded is recorded — a partial export would name a run the grader could not read whole. Recording is silent in the TUI and under message suppression too, off under the test suite, and inert inside a deployment, where the platform binds the execution before crewAI's own tracing starts. The record is written atomically and a write failure never fails the run. The crew, flow and json_crew scaffolds now ignore `.crewai/` like the declarative flow scaffold already did. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
@@ -1,3 +1,4 @@
|
||||
.env
|
||||
__pycache__/
|
||||
.DS_Store
|
||||
.crewai/
|
||||
|
||||
@@ -2,3 +2,4 @@
|
||||
__pycache__/
|
||||
lib/
|
||||
.DS_Store
|
||||
.crewai/
|
||||
|
||||
@@ -2,3 +2,4 @@
|
||||
__pycache__/
|
||||
.DS_Store
|
||||
report.md
|
||||
.crewai/
|
||||
|
||||
17
lib/cli/tests/test_scaffold_gitignore.py
Normal file
17
lib/cli/tests/test_scaffold_gitignore.py
Normal file
@@ -0,0 +1,17 @@
|
||||
"""Every runnable scaffold ignores `.crewai/`, where crewAI records the last run."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
TEMPLATES = Path(__file__).resolve().parents[1] / "src" / "crewai_cli" / "templates"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("template", ["crew", "flow", "json_crew", "declarative_flow"])
|
||||
def test_the_scaffold_gitignore_covers_the_crewai_directory(template):
|
||||
lines = (TEMPLATES / template / ".gitignore").read_text(encoding="utf-8").splitlines()
|
||||
assert ".crewai/" in lines
|
||||
assert ".env" in lines
|
||||
@@ -164,7 +164,7 @@ def _start_tracing(execution_uuid: str, tracing: bool | None) -> None:
|
||||
|
||||
def finish_authenticated_trace() -> None:
|
||||
if session.shutdown():
|
||||
exporter.show_trace_summary()
|
||||
exporter.record_export()
|
||||
|
||||
stack.callback(finish_authenticated_trace)
|
||||
_activate_tracing(ExecutionTrace(session, stack))
|
||||
|
||||
@@ -143,7 +143,7 @@ class EphemeralSpanBuffer(SpanExporter):
|
||||
logger.warning("Ephemeral trace export failed; buffer discarded")
|
||||
finally:
|
||||
exporter.shutdown()
|
||||
exporter.show_trace_summary()
|
||||
exporter.record_export()
|
||||
except TraceGrantError as error:
|
||||
logger.warning(
|
||||
"Ephemeral trace grant failed (HTTP %s); buffer discarded",
|
||||
|
||||
@@ -16,16 +16,10 @@ from crewai_core.plus_api import PlusAPI
|
||||
from opentelemetry.exporter.otlp.proto.common.trace_encoder import encode_spans
|
||||
from opentelemetry.sdk.trace import ReadableSpan
|
||||
from opentelemetry.sdk.trace.export import SpanExportResult, SpanExporter
|
||||
from rich.style import Style
|
||||
from rich.text import Text
|
||||
|
||||
from crewai.auth.token import AuthError, get_auth_token
|
||||
from crewai.context import get_platform_integration_token
|
||||
from crewai.events.listeners.tracing.utils import (
|
||||
is_tui_mode,
|
||||
should_suppress_tracing_messages,
|
||||
)
|
||||
from crewai.events.utils.console_formatter import ConsoleFormatter
|
||||
from crewai.telemetry.tracing import last_run
|
||||
from crewai.telemetry.tracing.session import MAX_EXPORT_BATCH_SIZE, otlp_exporter
|
||||
|
||||
|
||||
@@ -172,18 +166,24 @@ class TraceGrantClient:
|
||||
|
||||
|
||||
class GrantSpanExporter(SpanExporter):
|
||||
"""Refresh a grant before exporting; delegate OTLP transport to the shared path."""
|
||||
"""Refresh a grant before exporting; delegate OTLP transport to the shared path.
|
||||
|
||||
Nothing about the export is printed. Once the run's spans have reached
|
||||
Wharf, ``record_export`` writes the project's ``.crewai/last_run.json``
|
||||
(``last_run``) so ``crewai eval`` can find the run without an id being
|
||||
shown to anyone.
|
||||
"""
|
||||
|
||||
def __init__(self, client: TraceGrantClient, grant: TraceGrant):
|
||||
self._client = client
|
||||
self._grant = grant
|
||||
self._lock = Lock()
|
||||
self._delegate = self._exporter(grant)
|
||||
self._trace_url = grant.trace_url
|
||||
self._exported = False
|
||||
self._export_failed = False
|
||||
self._summary_shown = False
|
||||
self._suppress_output = should_suppress_tracing_messages() or is_tui_mode()
|
||||
self._recorded = False
|
||||
self._first_start_ns: int | None = None
|
||||
self._last_end_ns: int | None = None
|
||||
|
||||
@staticmethod
|
||||
def _exporter(grant: TraceGrant) -> SpanExporter:
|
||||
@@ -200,9 +200,28 @@ class GrantSpanExporter(SpanExporter):
|
||||
self._export_failed = True
|
||||
raise
|
||||
self._export_failed |= result != SpanExportResult.SUCCESS
|
||||
self._exported |= bool(spans) and result == SpanExportResult.SUCCESS
|
||||
if spans and result == SpanExportResult.SUCCESS:
|
||||
self._exported = True
|
||||
self._note_span_times(spans)
|
||||
return result
|
||||
|
||||
def _note_span_times(self, spans: Sequence[ReadableSpan]) -> None:
|
||||
"""The run's first start and last end, across every exported batch."""
|
||||
starts = [s.start_time for s in spans if s.start_time is not None]
|
||||
ends = [s.end_time for s in spans if s.end_time is not None]
|
||||
if starts:
|
||||
first = min(starts)
|
||||
self._first_start_ns = (
|
||||
first
|
||||
if self._first_start_ns is None
|
||||
else min(self._first_start_ns, first)
|
||||
)
|
||||
if ends:
|
||||
last = max(ends)
|
||||
self._last_end_ns = (
|
||||
last if self._last_end_ns is None else max(self._last_end_ns, last)
|
||||
)
|
||||
|
||||
def _export(self, spans: Sequence[ReadableSpan]) -> SpanExportResult:
|
||||
result = SpanExportResult.SUCCESS
|
||||
pending = [
|
||||
@@ -240,7 +259,6 @@ class GrantSpanExporter(SpanExporter):
|
||||
exporter = self._exporter(grant)
|
||||
self._delegate.shutdown()
|
||||
self._delegate, self._grant = exporter, grant
|
||||
self._trace_url = grant.trace_url or self._trace_url
|
||||
if self._delegate.export(batch) != SpanExportResult.SUCCESS:
|
||||
return SpanExportResult.FAILURE
|
||||
logger.info(
|
||||
@@ -250,32 +268,27 @@ class GrantSpanExporter(SpanExporter):
|
||||
)
|
||||
return result
|
||||
|
||||
def show_trace_summary(self) -> None:
|
||||
"""Display the exported execution UUID and AMP's optional viewer link once."""
|
||||
def record_export(self) -> None:
|
||||
"""Record the run for `crewai eval` once its spans have reached Wharf.
|
||||
|
||||
Silent: nothing is printed. Only a run whose every export succeeded is
|
||||
recorded — a partial export would name a run the grader cannot read
|
||||
whole.
|
||||
"""
|
||||
with self._lock:
|
||||
if (
|
||||
not self._exported
|
||||
or self._export_failed
|
||||
or self._summary_shown
|
||||
or self._suppress_output
|
||||
or should_suppress_tracing_messages()
|
||||
or is_tui_mode()
|
||||
):
|
||||
if not self._exported or self._export_failed or self._recorded:
|
||||
return
|
||||
self._summary_shown = True
|
||||
content = Text()
|
||||
content.append("Traces exported\n", style="green bold")
|
||||
content.append("Execution trace ID: ", style="white")
|
||||
content.append(self._grant.execution_uuid, style="green")
|
||||
if self._trace_url:
|
||||
content.append("\n\nView traces:\n", style="white bold")
|
||||
content.append(
|
||||
self._trace_url,
|
||||
style=Style(color="cyan", underline=True, link=self._trace_url),
|
||||
)
|
||||
ConsoleFormatter(verbose=True).print_panel(
|
||||
content, "🔗 Execution Traces", "green"
|
||||
self._recorded = True
|
||||
execution_uuid = self._grant.execution_uuid
|
||||
api = getattr(self._client, "_api", None)
|
||||
last_run.record_last_run(
|
||||
execution_id=execution_uuid,
|
||||
tier=getattr(self._client, "_tier", None),
|
||||
started_at_ns=self._first_start_ns,
|
||||
finished_at_ns=self._last_end_ns,
|
||||
amp_base_url=getattr(api, "base_url", None),
|
||||
)
|
||||
logger.debug("Traces exported for execution %s", execution_uuid)
|
||||
|
||||
def shutdown(self) -> None:
|
||||
with self._lock:
|
||||
|
||||
92
lib/crewai/src/crewai/telemetry/tracing/last_run.py
Normal file
92
lib/crewai/src/crewai/telemetry/tracing/last_run.py
Normal file
@@ -0,0 +1,92 @@
|
||||
"""The last traced run of a project, recorded for `crewai eval`.
|
||||
|
||||
Once a run's spans have reached Wharf, crewAI writes ``.crewai/last_run.json``
|
||||
in the project directory: the execution id, when the run started and ended,
|
||||
and whether it was traced anonymously or under an account. Nothing is
|
||||
printed — the id is internal — and ``crewai eval`` reads it back to evaluate
|
||||
the run without the user pasting anything.
|
||||
|
||||
Inside a deployment nothing is recorded: the platform binds the execution
|
||||
before crewAI would start its own tracing, so this code never runs there.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime, timezone
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
LAST_RUN_DIR = ".crewai"
|
||||
LAST_RUN_FILE = "last_run.json"
|
||||
|
||||
|
||||
def project_dir() -> Path:
|
||||
"""The run's project: the working directory, as every project-local path in crewAI."""
|
||||
return Path.cwd()
|
||||
|
||||
|
||||
def recording_enabled() -> bool:
|
||||
"""Off under the test suite, so kickoff-level tests leave no file behind."""
|
||||
return os.environ.get("CREWAI_TESTING", "").lower() != "true"
|
||||
|
||||
|
||||
def last_run_path(directory: Path | None = None) -> Path:
|
||||
return (directory or project_dir()) / LAST_RUN_DIR / LAST_RUN_FILE
|
||||
|
||||
|
||||
def _iso(nanoseconds: int | None) -> str | None:
|
||||
if nanoseconds is None:
|
||||
return None
|
||||
return datetime.fromtimestamp(nanoseconds / 1e9, tz=timezone.utc).isoformat(
|
||||
timespec="milliseconds"
|
||||
)
|
||||
|
||||
|
||||
def record_last_run(
|
||||
*,
|
||||
execution_id: str,
|
||||
tier: str | None,
|
||||
started_at_ns: int | None,
|
||||
finished_at_ns: int | None,
|
||||
amp_base_url: str | None,
|
||||
) -> Path | None:
|
||||
"""Write the record atomically; the path, or None when recording is off
|
||||
or the write failed. Never raises — a run is never failed by this."""
|
||||
if not recording_enabled():
|
||||
return None
|
||||
record: dict[str, Any] = {
|
||||
"execution_id": execution_id,
|
||||
"tier": tier,
|
||||
"started_at": _iso(started_at_ns),
|
||||
"finished_at": _iso(finished_at_ns),
|
||||
"recorded_at": datetime.now(timezone.utc).isoformat(timespec="milliseconds"),
|
||||
"amp_base_url": amp_base_url,
|
||||
}
|
||||
path = last_run_path()
|
||||
try:
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
temporary = path.with_name(path.name + ".tmp")
|
||||
temporary.write_text(json.dumps(record, indent=2) + "\n", encoding="utf-8")
|
||||
os.replace(temporary, path)
|
||||
except OSError as error:
|
||||
logger.debug("Could not record the last run in %s: %s", path, error)
|
||||
return None
|
||||
return path
|
||||
|
||||
|
||||
def read_last_run(directory: Path | None = None) -> dict[str, Any] | None:
|
||||
"""The record, or None when there is none or it cannot be read."""
|
||||
path = last_run_path(directory)
|
||||
try:
|
||||
loaded = json.loads(path.read_text(encoding="utf-8"))
|
||||
except (OSError, ValueError):
|
||||
return None
|
||||
if not isinstance(loaded, dict) or not loaded.get("execution_id"):
|
||||
return None
|
||||
return loaded
|
||||
65
lib/crewai/tests/telemetry/test_last_run.py
Normal file
65
lib/crewai/tests/telemetry/test_last_run.py
Normal file
@@ -0,0 +1,65 @@
|
||||
"""`.crewai/last_run.json`: the run `crewai eval` evaluates by default."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
|
||||
import pytest
|
||||
|
||||
from crewai.telemetry.tracing import last_run
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def project(monkeypatch, tmp_path):
|
||||
monkeypatch.setattr(last_run, "project_dir", lambda: tmp_path)
|
||||
monkeypatch.setattr(last_run, "recording_enabled", lambda: True)
|
||||
return tmp_path
|
||||
|
||||
|
||||
def test_a_run_is_recorded_atomically_and_read_back(project):
|
||||
path = last_run.record_last_run(
|
||||
execution_id="6f31fe1a-20bd-4bfe-a011-25d6b9341f62",
|
||||
tier="ephemeral",
|
||||
started_at_ns=1_758_240_000_000_000_000,
|
||||
finished_at_ns=1_758_240_009_500_000_000,
|
||||
amp_base_url="https://app.crewai.com",
|
||||
)
|
||||
assert path == project / ".crewai" / "last_run.json"
|
||||
assert not path.with_name("last_run.json.tmp").exists()
|
||||
written = json.loads(path.read_text(encoding="utf-8"))
|
||||
assert written["execution_id"] == "6f31fe1a-20bd-4bfe-a011-25d6b9341f62"
|
||||
assert written["tier"] == "ephemeral"
|
||||
assert written["started_at"] == "2025-09-19T00:00:00.000+00:00"
|
||||
assert written["finished_at"] == "2025-09-19T00:00:09.500+00:00"
|
||||
assert written["amp_base_url"] == "https://app.crewai.com"
|
||||
assert written["recorded_at"]
|
||||
assert last_run.read_last_run(project) == written
|
||||
|
||||
|
||||
def test_the_newest_run_replaces_the_previous_one(project):
|
||||
last_run.record_last_run(execution_id="first", tier=None, started_at_ns=None, finished_at_ns=None, amp_base_url=None)
|
||||
last_run.record_last_run(execution_id="second", tier="authenticated", started_at_ns=None, finished_at_ns=None, amp_base_url=None)
|
||||
written = last_run.read_last_run(project)
|
||||
assert written is not None and written["execution_id"] == "second"
|
||||
assert written["started_at"] is None and written["finished_at"] is None
|
||||
|
||||
|
||||
def test_nothing_is_recorded_under_the_test_suite(monkeypatch, tmp_path):
|
||||
monkeypatch.setattr(last_run, "project_dir", lambda: tmp_path)
|
||||
monkeypatch.setenv("CREWAI_TESTING", "true")
|
||||
assert last_run.record_last_run(execution_id="x", tier=None, started_at_ns=None, finished_at_ns=None, amp_base_url=None) is None
|
||||
assert not (tmp_path / ".crewai").exists()
|
||||
|
||||
|
||||
def test_a_missing_or_broken_record_reads_as_none(project):
|
||||
assert last_run.read_last_run(project) is None
|
||||
(project / ".crewai").mkdir()
|
||||
(project / ".crewai" / "last_run.json").write_text("not json", encoding="utf-8")
|
||||
assert last_run.read_last_run(project) is None
|
||||
(project / ".crewai" / "last_run.json").write_text(json.dumps({"tier": "ephemeral"}), encoding="utf-8")
|
||||
assert last_run.read_last_run(project) is None # no execution id: no run
|
||||
|
||||
|
||||
def test_a_write_failure_never_raises(project, monkeypatch):
|
||||
(project / ".crewai").write_text("a file where the directory should be", encoding="utf-8")
|
||||
assert last_run.record_last_run(execution_id="x", tier=None, started_at_ns=None, finished_at_ns=None, amp_base_url=None) is None
|
||||
@@ -131,6 +131,177 @@ def record(session, name="execute flow"):
|
||||
).end()
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def recorded_runs(monkeypatch, tmp_path):
|
||||
"""Recording on, into tmp_path (the suite's CREWAI_TESTING turns it off)."""
|
||||
from crewai.telemetry.tracing import last_run
|
||||
|
||||
monkeypatch.setattr(last_run, "project_dir", lambda: tmp_path)
|
||||
monkeypatch.setattr(last_run, "recording_enabled", lambda: True)
|
||||
return tmp_path
|
||||
|
||||
|
||||
def _last_run(directory):
|
||||
from crewai.telemetry.tracing import last_run
|
||||
|
||||
return last_run.read_last_run(directory)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("authenticated", "approved", "status", "suppression", "recorded"),
|
||||
[
|
||||
(True, True, 200, None, True),
|
||||
(False, True, 200, None, True),
|
||||
(False, False, 200, None, False),
|
||||
(True, True, 401, None, False),
|
||||
(False, True, 401, None, False),
|
||||
(True, True, 200, "messages", True),
|
||||
(False, True, 200, "messages", True),
|
||||
(True, True, 200, "tui", True),
|
||||
(False, True, 200, "tui", True),
|
||||
],
|
||||
)
|
||||
def test_nothing_is_printed_after_an_export_and_a_successful_one_is_recorded(
|
||||
collector,
|
||||
recorded_runs,
|
||||
monkeypatch,
|
||||
capsys,
|
||||
authenticated,
|
||||
approved,
|
||||
status,
|
||||
suppression,
|
||||
recorded,
|
||||
):
|
||||
from crewai.events.listeners.tracing.utils import (
|
||||
set_suppress_tracing_messages,
|
||||
set_tui_mode,
|
||||
)
|
||||
from crewai.execution import begin_execution, end_execution, get_execution_uuid
|
||||
from crewai.telemetry.tracing.context import get_trace_session
|
||||
|
||||
url = collector.url + "/crewai_plus/otel_traces/run?access_code=secret&x=[value]"
|
||||
collector.grant_override = {"trace_url": url}
|
||||
collector.export_status = status
|
||||
if authenticated:
|
||||
monkeypatch.setenv("CREWAI_USER_PAT", "synthetic-pat")
|
||||
seen = {}
|
||||
|
||||
def run():
|
||||
if suppression == "messages":
|
||||
set_suppress_tracing_messages(True)
|
||||
elif suppression == "tui":
|
||||
set_tui_mode(True)
|
||||
with trace_consent(lambda: approved):
|
||||
token = begin_execution(tracing=True)
|
||||
try:
|
||||
seen["uuid"] = get_execution_uuid()
|
||||
session = get_trace_session()
|
||||
record(session)
|
||||
nested = begin_execution(tracing=True)
|
||||
end_execution(nested)
|
||||
finally:
|
||||
end_execution(token)
|
||||
|
||||
copy_context().run(run)
|
||||
output = capsys.readouterr().out
|
||||
# The id and the viewer link are internal: nothing about tracing is printed.
|
||||
assert url not in output and "View traces:" not in output
|
||||
assert "Execution trace ID:" not in output and "Traces exported" not in output
|
||||
assert len(collector.batches) == int(authenticated or approved)
|
||||
# A run whose spans reached Wharf is recorded for `crewai eval`, silently,
|
||||
# in the TUI and under message suppression too; a failed export is not.
|
||||
written = _last_run(recorded_runs)
|
||||
if recorded:
|
||||
assert written is not None
|
||||
assert written["execution_id"] == seen["uuid"]
|
||||
assert written["tier"] == ("authenticated" if authenticated else "ephemeral")
|
||||
assert written["started_at"] and written["finished_at"] and written["recorded_at"]
|
||||
assert written["amp_base_url"] == collector.url
|
||||
else:
|
||||
assert written is None
|
||||
|
||||
|
||||
@pytest.mark.parametrize("first_status", [200, 401])
|
||||
def test_a_deferred_run_is_recorded_once_at_finalization_and_a_failed_export_never(
|
||||
collector, recorded_runs, monkeypatch, capsys, first_status
|
||||
):
|
||||
from crewai.execution import begin_execution, end_execution
|
||||
from crewai.telemetry.tracing.context import get_trace_session
|
||||
|
||||
monkeypatch.setenv("CREWAI_USER_PAT", "synthetic-pat")
|
||||
collector.export_status = first_status
|
||||
token = begin_execution(tracing=True)
|
||||
try:
|
||||
session = get_trace_session()
|
||||
record(session)
|
||||
session.flush()
|
||||
finally:
|
||||
lifetime = end_execution(token, defer=True)
|
||||
assert _last_run(recorded_runs) is None # not finished yet
|
||||
|
||||
collector.export_status = 200
|
||||
token = begin_execution(tracing=True, trace_session=lifetime)
|
||||
try:
|
||||
record(get_trace_session())
|
||||
finally:
|
||||
end_execution(token)
|
||||
lifetime.finish()
|
||||
assert capsys.readouterr().out == ""
|
||||
# A run one of whose exports failed is never recorded: the grader could not read it whole.
|
||||
assert (_last_run(recorded_runs) is not None) == (first_status == 200)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("credential", [None, "pat"])
|
||||
def test_a_refreshed_grant_still_records_the_run_once(collector, recorded_runs, capsys, credential):
|
||||
collector.grant_override = {"trace_url": collector.url + "/crewai_plus/otel_traces/original"}
|
||||
client = TraceGrantClient(credential)
|
||||
grant = replace(
|
||||
client.create(str(uuid4())),
|
||||
expires_at=datetime.now(timezone.utc) + timedelta(seconds=1),
|
||||
)
|
||||
collector.grant_override = {}
|
||||
exporter = GrantSpanExporter(client, grant)
|
||||
session = TraceSession(grant.execution_uuid, [exporter])
|
||||
try:
|
||||
record(session)
|
||||
finally:
|
||||
session.shutdown()
|
||||
exporter.record_export()
|
||||
first = recorded_runs.joinpath(".crewai", "last_run.json").stat().st_mtime_ns
|
||||
exporter.record_export()
|
||||
assert recorded_runs.joinpath(".crewai", "last_run.json").stat().st_mtime_ns == first
|
||||
assert capsys.readouterr().out == ""
|
||||
written = _last_run(recorded_runs)
|
||||
assert written is not None and written["execution_id"] == grant.execution_uuid
|
||||
assert written["tier"] == ("authenticated" if credential else "ephemeral")
|
||||
assert len(collector.grants) == 2
|
||||
assert all(
|
||||
payload["include_trace_url"] is True for _, _, payload in collector.grants
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("authenticated", [True, False])
|
||||
def test_an_export_without_a_viewer_url_is_recorded_all_the_same(
|
||||
collector, recorded_runs, monkeypatch, capsys, authenticated
|
||||
):
|
||||
from crewai.execution import begin_execution, end_execution, get_execution_uuid
|
||||
from crewai.telemetry.tracing.context import get_trace_session
|
||||
|
||||
if authenticated:
|
||||
monkeypatch.setenv("CREWAI_USER_PAT", "synthetic-pat")
|
||||
with trace_consent(lambda: True):
|
||||
token = begin_execution(tracing=True)
|
||||
try:
|
||||
execution_uuid = get_execution_uuid()
|
||||
record(get_trace_session())
|
||||
finally:
|
||||
end_execution(token)
|
||||
assert capsys.readouterr().out == ""
|
||||
written = _last_run(recorded_runs)
|
||||
assert written is not None and written["execution_id"] == execution_uuid
|
||||
assert len(collector.batches) == 1
|
||||
|
||||
|
||||
def test_credential_precedence_and_missing_login(monkeypatch):
|
||||
monkeypatch.setenv("CREWAI_USER_PAT", "pat")
|
||||
monkeypatch.setenv("CREWAI_PLATFORM_INTEGRATION_TOKEN", "integration")
|
||||
@@ -214,151 +385,6 @@ def test_unusable_optional_viewer_url_does_not_break_grant(collector, value):
|
||||
assert TraceGrantClient("pat").create(str(uuid4())).trace_url is None
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("authenticated", "approved", "status", "suppression", "shows_link"),
|
||||
[
|
||||
(True, True, 200, None, True),
|
||||
(False, True, 200, None, True),
|
||||
(False, False, 200, None, False),
|
||||
(True, True, 401, None, False),
|
||||
(False, True, 401, None, False),
|
||||
(True, True, 200, "messages", False),
|
||||
(False, True, 200, "messages", False),
|
||||
(True, True, 200, "tui", False),
|
||||
(False, True, 200, "tui", False),
|
||||
],
|
||||
)
|
||||
def test_viewer_link_is_printed_once_after_successful_execution_export(
|
||||
collector,
|
||||
monkeypatch,
|
||||
capsys,
|
||||
authenticated,
|
||||
approved,
|
||||
status,
|
||||
suppression,
|
||||
shows_link,
|
||||
):
|
||||
from crewai.events.listeners.tracing.utils import (
|
||||
set_suppress_tracing_messages,
|
||||
set_tui_mode,
|
||||
)
|
||||
from crewai.execution import begin_execution, end_execution
|
||||
from crewai.telemetry.tracing.context import get_trace_session
|
||||
|
||||
url = collector.url + "/crewai_plus/otel_traces/run?access_code=secret&x=[value]"
|
||||
collector.grant_override = {"trace_url": url}
|
||||
collector.export_status = status
|
||||
if authenticated:
|
||||
monkeypatch.setenv("CREWAI_USER_PAT", "synthetic-pat")
|
||||
|
||||
def run():
|
||||
if suppression == "messages":
|
||||
set_suppress_tracing_messages(True)
|
||||
elif suppression == "tui":
|
||||
set_tui_mode(True)
|
||||
with trace_consent(lambda: approved):
|
||||
token = begin_execution(tracing=True)
|
||||
try:
|
||||
session = get_trace_session()
|
||||
record(session)
|
||||
nested = begin_execution(tracing=True)
|
||||
end_execution(nested)
|
||||
assert "View traces:" not in capsys.readouterr().out
|
||||
finally:
|
||||
end_execution(token)
|
||||
|
||||
copy_context().run(run)
|
||||
output = capsys.readouterr().out
|
||||
assert output.count(url) == int(shows_link)
|
||||
assert output.count("View traces:") == int(shows_link)
|
||||
assert output.count("Execution trace ID:") == int(shows_link)
|
||||
assert len(collector.batches) == int(authenticated or approved)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("first_status", [200, 401])
|
||||
def test_deferred_trace_link_waits_for_finalization_and_remembers_export_failures(
|
||||
collector, monkeypatch, capsys, first_status
|
||||
):
|
||||
from crewai.execution import begin_execution, end_execution
|
||||
from crewai.telemetry.tracing.context import get_trace_session
|
||||
|
||||
monkeypatch.setenv("CREWAI_USER_PAT", "synthetic-pat")
|
||||
url = collector.url + "/crewai_plus/otel_traces/run"
|
||||
collector.grant_override = {"trace_url": url}
|
||||
collector.export_status = first_status
|
||||
token = begin_execution(tracing=True)
|
||||
try:
|
||||
session = get_trace_session()
|
||||
record(session)
|
||||
session.flush()
|
||||
finally:
|
||||
lifetime = end_execution(token, defer=True)
|
||||
assert url not in capsys.readouterr().out
|
||||
|
||||
collector.export_status = 200
|
||||
token = begin_execution(tracing=True, trace_session=lifetime)
|
||||
try:
|
||||
record(get_trace_session())
|
||||
finally:
|
||||
end_execution(token)
|
||||
lifetime.finish()
|
||||
assert capsys.readouterr().out.count(url) == int(first_status == 200)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("updated_url", [False, True])
|
||||
@pytest.mark.parametrize("credential", [None, "pat"])
|
||||
def test_refreshed_grant_preserves_or_updates_viewer_url(
|
||||
collector, capsys, updated_url, credential
|
||||
):
|
||||
original = collector.url + "/crewai_plus/otel_traces/original"
|
||||
renewed = collector.url + "/crewai_plus/otel_traces/renewed"
|
||||
collector.grant_override = {"trace_url": original}
|
||||
client = TraceGrantClient(credential)
|
||||
grant = replace(
|
||||
client.create(str(uuid4())),
|
||||
expires_at=datetime.now(timezone.utc) + timedelta(seconds=1),
|
||||
)
|
||||
collector.grant_override = {"trace_url": renewed} if updated_url else {}
|
||||
exporter = GrantSpanExporter(client, grant)
|
||||
session = TraceSession(grant.execution_uuid, [exporter])
|
||||
try:
|
||||
record(session)
|
||||
finally:
|
||||
session.shutdown()
|
||||
exporter.show_trace_summary()
|
||||
exporter.show_trace_summary()
|
||||
output = capsys.readouterr().out
|
||||
assert output.count("View traces:") == 1
|
||||
assert (renewed if updated_url else original) in output
|
||||
assert len(collector.grants) == 2
|
||||
assert all(
|
||||
payload["include_trace_url"] is True for _, _, payload in collector.grants
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("authenticated", [True, False])
|
||||
def test_export_without_viewer_url_still_shows_execution_uuid(
|
||||
collector, monkeypatch, capsys, authenticated
|
||||
):
|
||||
from crewai.execution import begin_execution, end_execution, get_execution_uuid
|
||||
from crewai.telemetry.tracing.context import get_trace_session
|
||||
|
||||
if authenticated:
|
||||
monkeypatch.setenv("CREWAI_USER_PAT", "synthetic-pat")
|
||||
with trace_consent(lambda: True):
|
||||
token = begin_execution(tracing=True)
|
||||
try:
|
||||
execution_uuid = get_execution_uuid()
|
||||
record(get_trace_session())
|
||||
assert "Execution trace ID:" not in capsys.readouterr().out
|
||||
finally:
|
||||
end_execution(token)
|
||||
output = capsys.readouterr().out
|
||||
assert output.count(f"Execution trace ID: {execution_uuid}") == 1
|
||||
assert "View traces:" not in output
|
||||
assert len(collector.batches) == 1
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"override",
|
||||
[
|
||||
|
||||
Reference in New Issue
Block a user