From c7e50bdaa2d3f8c67df5e73fbfe19f2d0c5a125b Mon Sep 17 00:00:00 2001 From: Joao Moura Date: Sun, 20 Sep 2026 01:54:52 -0700 Subject: [PATCH] fix(tracing): never record a truncated ephemeral run; resolve the record path inside the guard; pin the window and the once MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - An ephemeral buffer that dropped spans at its cap still uploaded the rest and recorded the run. The grader could not read that run whole, so it is no longer recorded (the upload still happens). Test added. - last_run_path() ran before the try in record_last_run; a cwd that vanished mid-run would have raised out of a function documented never to raise. It is inside now, with the same debug log and None. - Tests: the "recorded once" check counted an unchanged mtime, which proves nothing on a coarse filesystem — it now counts calls to record_last_run; the first-start/last-end fold across export batches had no value test — batches now arrive out of order and the ISO window is asserted. - Docs: the conversational-flows comment said "one trace link" for output that no longer exists; tracing.mdx now names .crewai/last_run.json, what it holds, who reads it, and when it is not written. en, ar, ko, pt-BR. 85 passed (test_session_trace_export + test_last_run); ruff and mypy clean. Co-Authored-By: Claude Fable 5.1 --- .../ar/guides/flows/conversational-flows.mdx | 2 +- docs/edge/ar/observability/tracing.mdx | 2 + .../en/guides/flows/conversational-flows.mdx | 2 +- docs/edge/en/observability/tracing.mdx | 2 + docs/edge/ko/observability/tracing.mdx | 2 + .../guides/flows/conversational-flows.mdx | 2 +- docs/edge/pt-BR/observability/tracing.mdx | 2 + .../src/crewai/telemetry/tracing/ephemeral.py | 5 +- .../src/crewai/telemetry/tracing/last_run.py | 11 +++- .../telemetry/test_session_trace_export.py | 61 ++++++++++++++++++- 10 files changed, 81 insertions(+), 10 deletions(-) diff --git a/docs/edge/ar/guides/flows/conversational-flows.mdx b/docs/edge/ar/guides/flows/conversational-flows.mdx index e7d4253b0..e58933b92 100644 --- a/docs/edge/ar/guides/flows/conversational-flows.mdx +++ b/docs/edge/ar/guides/flows/conversational-flows.mdx @@ -82,7 +82,7 @@ try: flow.handle_turn("Where is my order?", session_id=session_id) flow.handle_turn("What about returns?", session_id=session_id) finally: - flow.finalize_session_traces() # one trace link for the whole chat + flow.finalize_session_traces() # one trace for the whole chat ``` ## بث جولة diff --git a/docs/edge/ar/observability/tracing.mdx b/docs/edge/ar/observability/tracing.mdx index adf4f0853..549570545 100644 --- a/docs/edge/ar/observability/tracing.mdx +++ b/docs/edge/ar/observability/tracing.mdx @@ -154,6 +154,8 @@ result = flow.kickoff() للتتبعات المرتبطة بحسابك، افتح [علامة تبويب التتبعات في لوحة تحكم CrewAI AMP](https://app.crewai.com/crewai_plus/trace_batches) لعرض تفاعلات الوكلاء واستخدام الأدوات واستدعاءات LLM. ![واجهة تتبع CrewAI](/images/view-traces.png) +بعد تشغيلٍ وصلت نطاقاته (spans) إلى CrewAI AMP، يكتب crewAI الملف `.crewai/last_run.json` في مجلد المشروع: معرّف التنفيذ، وما إذا كان التتبع مجهولاً أو تحت حسابك، ووقت البدء والانتهاء، وعنوان AMP الذي أُرسل إليه. يقرأه الأمر `crewai eval` لتقييم آخر تشغيل دون نسخ أي معرّف. يتضمن `.gitignore` في القالب المجلد `.crewai/`؛ ولا يُسجَّل إلا التشغيل الذي صُدِّر كاملاً، ولا يُكتب شيء عند `CREWAI_TESTING=true`. + ### البديل: إعداد متغير البيئة يمكنك أيضاً تفعيل التتبع عالمياً عبر تعيين متغير بيئة: diff --git a/docs/edge/en/guides/flows/conversational-flows.mdx b/docs/edge/en/guides/flows/conversational-flows.mdx index c0ae83d74..67b53b701 100644 --- a/docs/edge/en/guides/flows/conversational-flows.mdx +++ b/docs/edge/en/guides/flows/conversational-flows.mdx @@ -82,7 +82,7 @@ try: flow.handle_turn("Where is my order?", session_id=session_id) flow.handle_turn("What about returns?", session_id=session_id) finally: - flow.finalize_session_traces() # one trace link for the whole chat + flow.finalize_session_traces() # one trace for the whole chat ``` ## Streaming a turn diff --git a/docs/edge/en/observability/tracing.mdx b/docs/edge/en/observability/tracing.mdx index 6aeb46156..252474f5b 100644 --- a/docs/edge/en/observability/tracing.mdx +++ b/docs/edge/en/observability/tracing.mdx @@ -154,6 +154,8 @@ Traces are uploaded only after a successful authenticated export or an explicitl For traces associated with your account, open the [Traces tab in the CrewAI AMP dashboard](https://app.crewai.com/crewai_plus/trace_batches) to view agent interactions, tool usage, and LLM calls. ![CrewAI Tracing Interface](/images/view-traces.png) +After a run whose spans reached CrewAI AMP, crewAI writes `.crewai/last_run.json` in the project directory: the run's execution id, whether it was traced anonymously or under your account, when it started and finished, and the AMP it was sent to. `crewai eval` reads it to evaluate the last run without you copying an id. The scaffold's `.gitignore` covers `.crewai/`; only a run exported whole is recorded, and nothing is written under `CREWAI_TESTING=true`. + ### Alternative: Environment Variable Configuration You can also enable tracing globally by setting an environment variable: diff --git a/docs/edge/ko/observability/tracing.mdx b/docs/edge/ko/observability/tracing.mdx index e7b3ad90e..3da5b0414 100644 --- a/docs/edge/ko/observability/tracing.mdx +++ b/docs/edge/ko/observability/tracing.mdx @@ -154,6 +154,8 @@ result = flow.kickoff() 계정에 연결된 추적은 [CrewAI AMP 대시보드의 Traces 탭](https://app.crewai.com/crewai_plus/trace_batches)에서 agent 상호 작용, 도구 사용 및 LLM 호출을 확인하세요. ![CrewAI Tracing Interface](/images/view-traces.png) +스팬이 CrewAI AMP에 도달한 실행이 끝나면 crewAI는 프로젝트 디렉터리에 `.crewai/last_run.json`을 기록합니다. 실행 ID, 익명 또는 계정으로 추적되었는지, 시작·종료 시각, 전송된 AMP 주소가 담깁니다. `crewai eval`은 이 파일을 읽어 ID를 복사하지 않고도 마지막 실행을 평가합니다. 스캐폴드의 `.gitignore`는 `.crewai/`를 포함하며, 전체가 내보내진 실행만 기록되고 `CREWAI_TESTING=true`에서는 아무것도 기록되지 않습니다. + ### 대안: 환경 변수 구성 환경 변수를 설정하여 전역적으로 추적을 활성화할 수도 있습니다: diff --git a/docs/edge/pt-BR/guides/flows/conversational-flows.mdx b/docs/edge/pt-BR/guides/flows/conversational-flows.mdx index 38bf02d4c..257f7c3a0 100644 --- a/docs/edge/pt-BR/guides/flows/conversational-flows.mdx +++ b/docs/edge/pt-BR/guides/flows/conversational-flows.mdx @@ -82,7 +82,7 @@ try: flow.handle_turn("Where is my order?", session_id=session_id) flow.handle_turn("What about returns?", session_id=session_id) finally: - flow.finalize_session_traces() # one trace link for the whole chat + flow.finalize_session_traces() # one trace for the whole chat ``` ## Streaming de um turno diff --git a/docs/edge/pt-BR/observability/tracing.mdx b/docs/edge/pt-BR/observability/tracing.mdx index c21fcf439..c31439978 100644 --- a/docs/edge/pt-BR/observability/tracing.mdx +++ b/docs/edge/pt-BR/observability/tracing.mdx @@ -154,6 +154,8 @@ Os rastreamentos são enviados somente após uma exportação autenticada bem-su Para rastreamentos associados à sua conta, abra a [aba Traces no painel CrewAI AMP](https://app.crewai.com/crewai_plus/trace_batches) para visualizar interações dos agentes, uso de ferramentas e chamadas de LLM. ![CrewAI Tracing Interface](/images/view-traces.png) +Após uma execução cujos spans chegaram ao CrewAI AMP, o crewAI grava `.crewai/last_run.json` no diretório do projeto: o id da execução, se foi rastreada anonimamente ou na sua conta, quando começou e terminou, e o AMP para onde foi enviada. O `crewai eval` lê esse arquivo para avaliar a última execução sem você copiar um id. O `.gitignore` do scaffold já cobre `.crewai/`; só uma execução exportada por inteiro é registrada, e nada é gravado com `CREWAI_TESTING=true`. + ### Alternativa: Configuração de Variável de Ambiente Você também pode habilitar o rastreamento globalmente definindo uma variável de ambiente: diff --git a/lib/crewai/src/crewai/telemetry/tracing/ephemeral.py b/lib/crewai/src/crewai/telemetry/tracing/ephemeral.py index 905c59034..0c160a895 100644 --- a/lib/crewai/src/crewai/telemetry/tracing/ephemeral.py +++ b/lib/crewai/src/crewai/telemetry/tracing/ephemeral.py @@ -143,7 +143,10 @@ class EphemeralSpanBuffer(SpanExporter): logger.warning("Ephemeral trace export failed; buffer discarded") finally: exporter.shutdown() - exporter.record_export() + if ( + not self._dropped + ): # a truncated trace is not recorded: the grader could not read it whole + exporter.record_export() except TraceGrantError as error: logger.warning( "Ephemeral trace grant failed (HTTP %s); buffer discarded", diff --git a/lib/crewai/src/crewai/telemetry/tracing/last_run.py b/lib/crewai/src/crewai/telemetry/tracing/last_run.py index 8e1f50075..31e2230b1 100644 --- a/lib/crewai/src/crewai/telemetry/tracing/last_run.py +++ b/lib/crewai/src/crewai/telemetry/tracing/last_run.py @@ -68,14 +68,19 @@ def record_last_run( "recorded_at": datetime.now(timezone.utc).isoformat(timespec="milliseconds"), "amp_base_url": amp_base_url, } - path = last_run_path() + path: Path | None = None try: + path = last_run_path() path.parent.mkdir(parents=True, exist_ok=True) temporary = path.with_name(path.name + ".tmp") temporary.write_text(json.dumps(record, indent=2) + "\n", encoding="utf-8") os.replace(temporary, path) - except OSError as error: - logger.debug("Could not record the last run in %s: %s", path, error) + except OSError as error: # a vanished cwd fails last_run_path() too; the run is never failed by this + logger.debug( + "Could not record the last run in %s: %s", + path or ".crewai/last_run.json", + error, + ) return None return path diff --git a/lib/crewai/tests/telemetry/test_session_trace_export.py b/lib/crewai/tests/telemetry/test_session_trace_export.py index 509fa473e..839011ce7 100644 --- a/lib/crewai/tests/telemetry/test_session_trace_export.py +++ b/lib/crewai/tests/telemetry/test_session_trace_export.py @@ -252,7 +252,18 @@ def test_a_deferred_run_is_recorded_once_at_finalization_and_a_failed_export_nev @pytest.mark.parametrize("credential", [None, "pat"]) -def test_a_refreshed_grant_still_records_the_run_once(collector, recorded_runs, capsys, credential): +def test_a_refreshed_grant_still_records_the_run_once( + collector, recorded_runs, monkeypatch, capsys, credential +): + from crewai.telemetry.tracing import last_run + + writes = [] + real_record = last_run.record_last_run + monkeypatch.setattr( + last_run, + "record_last_run", + lambda **fields: (writes.append(fields["execution_id"]), real_record(**fields))[1], + ) collector.grant_override = {"trace_url": collector.url + "/crewai_plus/otel_traces/original"} client = TraceGrantClient(credential) grant = replace( @@ -267,9 +278,8 @@ def test_a_refreshed_grant_still_records_the_run_once(collector, recorded_runs, finally: session.shutdown() exporter.record_export() - first = recorded_runs.joinpath(".crewai", "last_run.json").stat().st_mtime_ns exporter.record_export() - assert recorded_runs.joinpath(".crewai", "last_run.json").stat().st_mtime_ns == first + assert writes == [grant.execution_uuid] # written once, however often the exporter is told the run ended assert capsys.readouterr().out == "" written = _last_run(recorded_runs) assert written is not None and written["execution_id"] == grant.execution_uuid @@ -280,6 +290,51 @@ def test_a_refreshed_grant_still_records_the_run_once(collector, recorded_runs, ) +def test_the_recorded_window_is_the_first_start_and_last_end_across_batches( + collector, recorded_runs +): + """Batches reach the exporter in any order; the record spans them all.""" + from crewai.telemetry.tracing import last_run + + memory = InMemorySpanExporter() + provider = TracerProvider() + provider.add_span_processor(SimpleSpanProcessor(memory)) + tracer = provider.get_tracer("window") + second = 1_000_000_000 + base = 1_700_000_000 * second + for name, start, end in (("late", 10, 12), ("early", 0, 5), ("middle", 3, 9)): + tracer.start_span(name, start_time=base + start * second).end(end_time=base + end * second) + provider.shutdown() + by_name = {span.name: span for span in memory.get_finished_spans()} + client = TraceGrantClient(None) + exporter = GrantSpanExporter(client, client.create(str(uuid4()))) + try: + assert exporter.export([by_name["late"]]) == SpanExportResult.SUCCESS + assert exporter.export([by_name["early"], by_name["middle"]]) == SpanExportResult.SUCCESS + finally: + exporter.shutdown() + exporter.record_export() + written = _last_run(recorded_runs) + assert written is not None + assert written["started_at"] == last_run._iso(base) + assert written["finished_at"] == last_run._iso(base + 12 * second) + + +def test_a_truncated_ephemeral_trace_is_uploaded_but_not_recorded( + collector, recorded_runs, monkeypatch +): + """The buffer dropped spans at its cap: the grader could not read the run whole.""" + monkeypatch.setenv("CREWAI_EPHEMERAL_TRACE_MAX_SPANS", "1") + buffer = EphemeralSpanBuffer() + monkeypatch.setattr(ephemeral, "EphemeralSpanBuffer", lambda: buffer) + with trace_consent(lambda: True), ephemeral_tracing(str(uuid4())) as session: + record(session, "first") + record(session, "second") + assert buffer._dropped == 1 + assert len(collector.batches) == 1 # what survived was still shared + assert _last_run(recorded_runs) is None + + @pytest.mark.parametrize("authenticated", [True, False]) def test_an_export_without_a_viewer_url_is_recorded_all_the_same( collector, recorded_runs, monkeypatch, capsys, authenticated