feat(tracing): record the last traced run for crewai eval instead of printing it

After a traced run, crewAI printed a panel with the execution id and a
viewer link. The id is internal, and the panel exposed it for no purpose a
user has. Nothing is printed now. Instead, once the run's spans have
reached Wharf, crewAI writes `.crewai/last_run.json` in the project: the
execution id, when the run started and ended, whether it was traced
anonymously or under an account, and the AMP base url. `crewai eval` reads
it back, so the user evaluates their last run without pasting anything.

GrantSpanExporter.record_export replaces show_trace_summary at the two
finish points (an authenticated run's shutdown, an anonymous run's share).
Only a run whose every export succeeded is recorded — a partial export
would name a run the grader could not read whole. Recording is silent in
the TUI and under message suppression too, off under the test suite, and
inert inside a deployment, where the platform binds the execution before
crewAI's own tracing starts. The record is written atomically and a write
failure never fails the run.

The crew, flow and json_crew scaffolds now ignore `.crewai/` like the
declarative flow scaffold already did.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
Joao Moura
2026-09-20 00:48:05 -07:00
parent 0374c63129
commit 96d91872c0
10 changed files with 399 additions and 183 deletions

View File

@@ -1,3 +1,4 @@
.env
__pycache__/
.DS_Store
.crewai/

View File

@@ -2,3 +2,4 @@
__pycache__/
lib/
.DS_Store
.crewai/

View File

@@ -2,3 +2,4 @@
__pycache__/
.DS_Store
report.md
.crewai/

View File

@@ -0,0 +1,17 @@
"""Every runnable scaffold ignores `.crewai/`, where crewAI records the last run."""
from __future__ import annotations
from pathlib import Path
import pytest
TEMPLATES = Path(__file__).resolve().parents[1] / "src" / "crewai_cli" / "templates"
@pytest.mark.parametrize("template", ["crew", "flow", "json_crew", "declarative_flow"])
def test_the_scaffold_gitignore_covers_the_crewai_directory(template):
lines = (TEMPLATES / template / ".gitignore").read_text(encoding="utf-8").splitlines()
assert ".crewai/" in lines
assert ".env" in lines

View File

@@ -164,7 +164,7 @@ def _start_tracing(execution_uuid: str, tracing: bool | None) -> None:
def finish_authenticated_trace() -> None:
if session.shutdown():
exporter.show_trace_summary()
exporter.record_export()
stack.callback(finish_authenticated_trace)
_activate_tracing(ExecutionTrace(session, stack))

View File

@@ -143,7 +143,7 @@ class EphemeralSpanBuffer(SpanExporter):
logger.warning("Ephemeral trace export failed; buffer discarded")
finally:
exporter.shutdown()
exporter.show_trace_summary()
exporter.record_export()
except TraceGrantError as error:
logger.warning(
"Ephemeral trace grant failed (HTTP %s); buffer discarded",

View File

@@ -16,16 +16,10 @@ from crewai_core.plus_api import PlusAPI
from opentelemetry.exporter.otlp.proto.common.trace_encoder import encode_spans
from opentelemetry.sdk.trace import ReadableSpan
from opentelemetry.sdk.trace.export import SpanExportResult, SpanExporter
from rich.style import Style
from rich.text import Text
from crewai.auth.token import AuthError, get_auth_token
from crewai.context import get_platform_integration_token
from crewai.events.listeners.tracing.utils import (
is_tui_mode,
should_suppress_tracing_messages,
)
from crewai.events.utils.console_formatter import ConsoleFormatter
from crewai.telemetry.tracing import last_run
from crewai.telemetry.tracing.session import MAX_EXPORT_BATCH_SIZE, otlp_exporter
@@ -172,18 +166,24 @@ class TraceGrantClient:
class GrantSpanExporter(SpanExporter):
"""Refresh a grant before exporting; delegate OTLP transport to the shared path."""
"""Refresh a grant before exporting; delegate OTLP transport to the shared path.
Nothing about the export is printed. Once the run's spans have reached
Wharf, ``record_export`` writes the project's ``.crewai/last_run.json``
(``last_run``) so ``crewai eval`` can find the run without an id being
shown to anyone.
"""
def __init__(self, client: TraceGrantClient, grant: TraceGrant):
self._client = client
self._grant = grant
self._lock = Lock()
self._delegate = self._exporter(grant)
self._trace_url = grant.trace_url
self._exported = False
self._export_failed = False
self._summary_shown = False
self._suppress_output = should_suppress_tracing_messages() or is_tui_mode()
self._recorded = False
self._first_start_ns: int | None = None
self._last_end_ns: int | None = None
@staticmethod
def _exporter(grant: TraceGrant) -> SpanExporter:
@@ -200,9 +200,28 @@ class GrantSpanExporter(SpanExporter):
self._export_failed = True
raise
self._export_failed |= result != SpanExportResult.SUCCESS
self._exported |= bool(spans) and result == SpanExportResult.SUCCESS
if spans and result == SpanExportResult.SUCCESS:
self._exported = True
self._note_span_times(spans)
return result
def _note_span_times(self, spans: Sequence[ReadableSpan]) -> None:
"""The run's first start and last end, across every exported batch."""
starts = [s.start_time for s in spans if s.start_time is not None]
ends = [s.end_time for s in spans if s.end_time is not None]
if starts:
first = min(starts)
self._first_start_ns = (
first
if self._first_start_ns is None
else min(self._first_start_ns, first)
)
if ends:
last = max(ends)
self._last_end_ns = (
last if self._last_end_ns is None else max(self._last_end_ns, last)
)
def _export(self, spans: Sequence[ReadableSpan]) -> SpanExportResult:
result = SpanExportResult.SUCCESS
pending = [
@@ -240,7 +259,6 @@ class GrantSpanExporter(SpanExporter):
exporter = self._exporter(grant)
self._delegate.shutdown()
self._delegate, self._grant = exporter, grant
self._trace_url = grant.trace_url or self._trace_url
if self._delegate.export(batch) != SpanExportResult.SUCCESS:
return SpanExportResult.FAILURE
logger.info(
@@ -250,32 +268,27 @@ class GrantSpanExporter(SpanExporter):
)
return result
def show_trace_summary(self) -> None:
"""Display the exported execution UUID and AMP's optional viewer link once."""
def record_export(self) -> None:
"""Record the run for `crewai eval` once its spans have reached Wharf.
Silent: nothing is printed. Only a run whose every export succeeded is
recorded — a partial export would name a run the grader cannot read
whole.
"""
with self._lock:
if (
not self._exported
or self._export_failed
or self._summary_shown
or self._suppress_output
or should_suppress_tracing_messages()
or is_tui_mode()
):
if not self._exported or self._export_failed or self._recorded:
return
self._summary_shown = True
content = Text()
content.append("Traces exported\n", style="green bold")
content.append("Execution trace ID: ", style="white")
content.append(self._grant.execution_uuid, style="green")
if self._trace_url:
content.append("\n\nView traces:\n", style="white bold")
content.append(
self._trace_url,
style=Style(color="cyan", underline=True, link=self._trace_url),
)
ConsoleFormatter(verbose=True).print_panel(
content, "🔗 Execution Traces", "green"
self._recorded = True
execution_uuid = self._grant.execution_uuid
api = getattr(self._client, "_api", None)
last_run.record_last_run(
execution_id=execution_uuid,
tier=getattr(self._client, "_tier", None),
started_at_ns=self._first_start_ns,
finished_at_ns=self._last_end_ns,
amp_base_url=getattr(api, "base_url", None),
)
logger.debug("Traces exported for execution %s", execution_uuid)
def shutdown(self) -> None:
with self._lock:

View File

@@ -0,0 +1,92 @@
"""The last traced run of a project, recorded for `crewai eval`.
Once a run's spans have reached Wharf, crewAI writes ``.crewai/last_run.json``
in the project directory: the execution id, when the run started and ended,
and whether it was traced anonymously or under an account. Nothing is
printed — the id is internal — and ``crewai eval`` reads it back to evaluate
the run without the user pasting anything.
Inside a deployment nothing is recorded: the platform binds the execution
before crewAI would start its own tracing, so this code never runs there.
"""
from __future__ import annotations
from datetime import datetime, timezone
import json
import logging
import os
from pathlib import Path
from typing import Any
logger = logging.getLogger(__name__)
LAST_RUN_DIR = ".crewai"
LAST_RUN_FILE = "last_run.json"
def project_dir() -> Path:
"""The run's project: the working directory, as every project-local path in crewAI."""
return Path.cwd()
def recording_enabled() -> bool:
"""Off under the test suite, so kickoff-level tests leave no file behind."""
return os.environ.get("CREWAI_TESTING", "").lower() != "true"
def last_run_path(directory: Path | None = None) -> Path:
return (directory or project_dir()) / LAST_RUN_DIR / LAST_RUN_FILE
def _iso(nanoseconds: int | None) -> str | None:
if nanoseconds is None:
return None
return datetime.fromtimestamp(nanoseconds / 1e9, tz=timezone.utc).isoformat(
timespec="milliseconds"
)
def record_last_run(
*,
execution_id: str,
tier: str | None,
started_at_ns: int | None,
finished_at_ns: int | None,
amp_base_url: str | None,
) -> Path | None:
"""Write the record atomically; the path, or None when recording is off
or the write failed. Never raises — a run is never failed by this."""
if not recording_enabled():
return None
record: dict[str, Any] = {
"execution_id": execution_id,
"tier": tier,
"started_at": _iso(started_at_ns),
"finished_at": _iso(finished_at_ns),
"recorded_at": datetime.now(timezone.utc).isoformat(timespec="milliseconds"),
"amp_base_url": amp_base_url,
}
path = last_run_path()
try:
path.parent.mkdir(parents=True, exist_ok=True)
temporary = path.with_name(path.name + ".tmp")
temporary.write_text(json.dumps(record, indent=2) + "\n", encoding="utf-8")
os.replace(temporary, path)
except OSError as error:
logger.debug("Could not record the last run in %s: %s", path, error)
return None
return path
def read_last_run(directory: Path | None = None) -> dict[str, Any] | None:
"""The record, or None when there is none or it cannot be read."""
path = last_run_path(directory)
try:
loaded = json.loads(path.read_text(encoding="utf-8"))
except (OSError, ValueError):
return None
if not isinstance(loaded, dict) or not loaded.get("execution_id"):
return None
return loaded

View File

@@ -0,0 +1,65 @@
"""`.crewai/last_run.json`: the run `crewai eval` evaluates by default."""
from __future__ import annotations
import json
import pytest
from crewai.telemetry.tracing import last_run
@pytest.fixture
def project(monkeypatch, tmp_path):
monkeypatch.setattr(last_run, "project_dir", lambda: tmp_path)
monkeypatch.setattr(last_run, "recording_enabled", lambda: True)
return tmp_path
def test_a_run_is_recorded_atomically_and_read_back(project):
path = last_run.record_last_run(
execution_id="6f31fe1a-20bd-4bfe-a011-25d6b9341f62",
tier="ephemeral",
started_at_ns=1_758_240_000_000_000_000,
finished_at_ns=1_758_240_009_500_000_000,
amp_base_url="https://app.crewai.com",
)
assert path == project / ".crewai" / "last_run.json"
assert not path.with_name("last_run.json.tmp").exists()
written = json.loads(path.read_text(encoding="utf-8"))
assert written["execution_id"] == "6f31fe1a-20bd-4bfe-a011-25d6b9341f62"
assert written["tier"] == "ephemeral"
assert written["started_at"] == "2025-09-19T00:00:00.000+00:00"
assert written["finished_at"] == "2025-09-19T00:00:09.500+00:00"
assert written["amp_base_url"] == "https://app.crewai.com"
assert written["recorded_at"]
assert last_run.read_last_run(project) == written
def test_the_newest_run_replaces_the_previous_one(project):
last_run.record_last_run(execution_id="first", tier=None, started_at_ns=None, finished_at_ns=None, amp_base_url=None)
last_run.record_last_run(execution_id="second", tier="authenticated", started_at_ns=None, finished_at_ns=None, amp_base_url=None)
written = last_run.read_last_run(project)
assert written is not None and written["execution_id"] == "second"
assert written["started_at"] is None and written["finished_at"] is None
def test_nothing_is_recorded_under_the_test_suite(monkeypatch, tmp_path):
monkeypatch.setattr(last_run, "project_dir", lambda: tmp_path)
monkeypatch.setenv("CREWAI_TESTING", "true")
assert last_run.record_last_run(execution_id="x", tier=None, started_at_ns=None, finished_at_ns=None, amp_base_url=None) is None
assert not (tmp_path / ".crewai").exists()
def test_a_missing_or_broken_record_reads_as_none(project):
assert last_run.read_last_run(project) is None
(project / ".crewai").mkdir()
(project / ".crewai" / "last_run.json").write_text("not json", encoding="utf-8")
assert last_run.read_last_run(project) is None
(project / ".crewai" / "last_run.json").write_text(json.dumps({"tier": "ephemeral"}), encoding="utf-8")
assert last_run.read_last_run(project) is None # no execution id: no run
def test_a_write_failure_never_raises(project, monkeypatch):
(project / ".crewai").write_text("a file where the directory should be", encoding="utf-8")
assert last_run.record_last_run(execution_id="x", tier=None, started_at_ns=None, finished_at_ns=None, amp_base_url=None) is None

View File

@@ -131,6 +131,177 @@ def record(session, name="execute flow"):
).end()
@pytest.fixture
def recorded_runs(monkeypatch, tmp_path):
"""Recording on, into tmp_path (the suite's CREWAI_TESTING turns it off)."""
from crewai.telemetry.tracing import last_run
monkeypatch.setattr(last_run, "project_dir", lambda: tmp_path)
monkeypatch.setattr(last_run, "recording_enabled", lambda: True)
return tmp_path
def _last_run(directory):
from crewai.telemetry.tracing import last_run
return last_run.read_last_run(directory)
@pytest.mark.parametrize(
("authenticated", "approved", "status", "suppression", "recorded"),
[
(True, True, 200, None, True),
(False, True, 200, None, True),
(False, False, 200, None, False),
(True, True, 401, None, False),
(False, True, 401, None, False),
(True, True, 200, "messages", True),
(False, True, 200, "messages", True),
(True, True, 200, "tui", True),
(False, True, 200, "tui", True),
],
)
def test_nothing_is_printed_after_an_export_and_a_successful_one_is_recorded(
collector,
recorded_runs,
monkeypatch,
capsys,
authenticated,
approved,
status,
suppression,
recorded,
):
from crewai.events.listeners.tracing.utils import (
set_suppress_tracing_messages,
set_tui_mode,
)
from crewai.execution import begin_execution, end_execution, get_execution_uuid
from crewai.telemetry.tracing.context import get_trace_session
url = collector.url + "/crewai_plus/otel_traces/run?access_code=secret&x=[value]"
collector.grant_override = {"trace_url": url}
collector.export_status = status
if authenticated:
monkeypatch.setenv("CREWAI_USER_PAT", "synthetic-pat")
seen = {}
def run():
if suppression == "messages":
set_suppress_tracing_messages(True)
elif suppression == "tui":
set_tui_mode(True)
with trace_consent(lambda: approved):
token = begin_execution(tracing=True)
try:
seen["uuid"] = get_execution_uuid()
session = get_trace_session()
record(session)
nested = begin_execution(tracing=True)
end_execution(nested)
finally:
end_execution(token)
copy_context().run(run)
output = capsys.readouterr().out
# The id and the viewer link are internal: nothing about tracing is printed.
assert url not in output and "View traces:" not in output
assert "Execution trace ID:" not in output and "Traces exported" not in output
assert len(collector.batches) == int(authenticated or approved)
# A run whose spans reached Wharf is recorded for `crewai eval`, silently,
# in the TUI and under message suppression too; a failed export is not.
written = _last_run(recorded_runs)
if recorded:
assert written is not None
assert written["execution_id"] == seen["uuid"]
assert written["tier"] == ("authenticated" if authenticated else "ephemeral")
assert written["started_at"] and written["finished_at"] and written["recorded_at"]
assert written["amp_base_url"] == collector.url
else:
assert written is None
@pytest.mark.parametrize("first_status", [200, 401])
def test_a_deferred_run_is_recorded_once_at_finalization_and_a_failed_export_never(
collector, recorded_runs, monkeypatch, capsys, first_status
):
from crewai.execution import begin_execution, end_execution
from crewai.telemetry.tracing.context import get_trace_session
monkeypatch.setenv("CREWAI_USER_PAT", "synthetic-pat")
collector.export_status = first_status
token = begin_execution(tracing=True)
try:
session = get_trace_session()
record(session)
session.flush()
finally:
lifetime = end_execution(token, defer=True)
assert _last_run(recorded_runs) is None # not finished yet
collector.export_status = 200
token = begin_execution(tracing=True, trace_session=lifetime)
try:
record(get_trace_session())
finally:
end_execution(token)
lifetime.finish()
assert capsys.readouterr().out == ""
# A run one of whose exports failed is never recorded: the grader could not read it whole.
assert (_last_run(recorded_runs) is not None) == (first_status == 200)
@pytest.mark.parametrize("credential", [None, "pat"])
def test_a_refreshed_grant_still_records_the_run_once(collector, recorded_runs, capsys, credential):
collector.grant_override = {"trace_url": collector.url + "/crewai_plus/otel_traces/original"}
client = TraceGrantClient(credential)
grant = replace(
client.create(str(uuid4())),
expires_at=datetime.now(timezone.utc) + timedelta(seconds=1),
)
collector.grant_override = {}
exporter = GrantSpanExporter(client, grant)
session = TraceSession(grant.execution_uuid, [exporter])
try:
record(session)
finally:
session.shutdown()
exporter.record_export()
first = recorded_runs.joinpath(".crewai", "last_run.json").stat().st_mtime_ns
exporter.record_export()
assert recorded_runs.joinpath(".crewai", "last_run.json").stat().st_mtime_ns == first
assert capsys.readouterr().out == ""
written = _last_run(recorded_runs)
assert written is not None and written["execution_id"] == grant.execution_uuid
assert written["tier"] == ("authenticated" if credential else "ephemeral")
assert len(collector.grants) == 2
assert all(
payload["include_trace_url"] is True for _, _, payload in collector.grants
)
@pytest.mark.parametrize("authenticated", [True, False])
def test_an_export_without_a_viewer_url_is_recorded_all_the_same(
collector, recorded_runs, monkeypatch, capsys, authenticated
):
from crewai.execution import begin_execution, end_execution, get_execution_uuid
from crewai.telemetry.tracing.context import get_trace_session
if authenticated:
monkeypatch.setenv("CREWAI_USER_PAT", "synthetic-pat")
with trace_consent(lambda: True):
token = begin_execution(tracing=True)
try:
execution_uuid = get_execution_uuid()
record(get_trace_session())
finally:
end_execution(token)
assert capsys.readouterr().out == ""
written = _last_run(recorded_runs)
assert written is not None and written["execution_id"] == execution_uuid
assert len(collector.batches) == 1
def test_credential_precedence_and_missing_login(monkeypatch):
monkeypatch.setenv("CREWAI_USER_PAT", "pat")
monkeypatch.setenv("CREWAI_PLATFORM_INTEGRATION_TOKEN", "integration")
@@ -214,151 +385,6 @@ def test_unusable_optional_viewer_url_does_not_break_grant(collector, value):
assert TraceGrantClient("pat").create(str(uuid4())).trace_url is None
@pytest.mark.parametrize(
("authenticated", "approved", "status", "suppression", "shows_link"),
[
(True, True, 200, None, True),
(False, True, 200, None, True),
(False, False, 200, None, False),
(True, True, 401, None, False),
(False, True, 401, None, False),
(True, True, 200, "messages", False),
(False, True, 200, "messages", False),
(True, True, 200, "tui", False),
(False, True, 200, "tui", False),
],
)
def test_viewer_link_is_printed_once_after_successful_execution_export(
collector,
monkeypatch,
capsys,
authenticated,
approved,
status,
suppression,
shows_link,
):
from crewai.events.listeners.tracing.utils import (
set_suppress_tracing_messages,
set_tui_mode,
)
from crewai.execution import begin_execution, end_execution
from crewai.telemetry.tracing.context import get_trace_session
url = collector.url + "/crewai_plus/otel_traces/run?access_code=secret&x=[value]"
collector.grant_override = {"trace_url": url}
collector.export_status = status
if authenticated:
monkeypatch.setenv("CREWAI_USER_PAT", "synthetic-pat")
def run():
if suppression == "messages":
set_suppress_tracing_messages(True)
elif suppression == "tui":
set_tui_mode(True)
with trace_consent(lambda: approved):
token = begin_execution(tracing=True)
try:
session = get_trace_session()
record(session)
nested = begin_execution(tracing=True)
end_execution(nested)
assert "View traces:" not in capsys.readouterr().out
finally:
end_execution(token)
copy_context().run(run)
output = capsys.readouterr().out
assert output.count(url) == int(shows_link)
assert output.count("View traces:") == int(shows_link)
assert output.count("Execution trace ID:") == int(shows_link)
assert len(collector.batches) == int(authenticated or approved)
@pytest.mark.parametrize("first_status", [200, 401])
def test_deferred_trace_link_waits_for_finalization_and_remembers_export_failures(
collector, monkeypatch, capsys, first_status
):
from crewai.execution import begin_execution, end_execution
from crewai.telemetry.tracing.context import get_trace_session
monkeypatch.setenv("CREWAI_USER_PAT", "synthetic-pat")
url = collector.url + "/crewai_plus/otel_traces/run"
collector.grant_override = {"trace_url": url}
collector.export_status = first_status
token = begin_execution(tracing=True)
try:
session = get_trace_session()
record(session)
session.flush()
finally:
lifetime = end_execution(token, defer=True)
assert url not in capsys.readouterr().out
collector.export_status = 200
token = begin_execution(tracing=True, trace_session=lifetime)
try:
record(get_trace_session())
finally:
end_execution(token)
lifetime.finish()
assert capsys.readouterr().out.count(url) == int(first_status == 200)
@pytest.mark.parametrize("updated_url", [False, True])
@pytest.mark.parametrize("credential", [None, "pat"])
def test_refreshed_grant_preserves_or_updates_viewer_url(
collector, capsys, updated_url, credential
):
original = collector.url + "/crewai_plus/otel_traces/original"
renewed = collector.url + "/crewai_plus/otel_traces/renewed"
collector.grant_override = {"trace_url": original}
client = TraceGrantClient(credential)
grant = replace(
client.create(str(uuid4())),
expires_at=datetime.now(timezone.utc) + timedelta(seconds=1),
)
collector.grant_override = {"trace_url": renewed} if updated_url else {}
exporter = GrantSpanExporter(client, grant)
session = TraceSession(grant.execution_uuid, [exporter])
try:
record(session)
finally:
session.shutdown()
exporter.show_trace_summary()
exporter.show_trace_summary()
output = capsys.readouterr().out
assert output.count("View traces:") == 1
assert (renewed if updated_url else original) in output
assert len(collector.grants) == 2
assert all(
payload["include_trace_url"] is True for _, _, payload in collector.grants
)
@pytest.mark.parametrize("authenticated", [True, False])
def test_export_without_viewer_url_still_shows_execution_uuid(
collector, monkeypatch, capsys, authenticated
):
from crewai.execution import begin_execution, end_execution, get_execution_uuid
from crewai.telemetry.tracing.context import get_trace_session
if authenticated:
monkeypatch.setenv("CREWAI_USER_PAT", "synthetic-pat")
with trace_consent(lambda: True):
token = begin_execution(tracing=True)
try:
execution_uuid = get_execution_uuid()
record(get_trace_session())
assert "Execution trace ID:" not in capsys.readouterr().out
finally:
end_execution(token)
output = capsys.readouterr().out
assert output.count(f"Execution trace ID: {execution_uuid}") == 1
assert "View traces:" not in output
assert len(collector.batches) == 1
@pytest.mark.parametrize(
"override",
[