mirror of
https://github.com/crewAIInc/crewAI.git
synced 2026-09-22 19:06:25 +00:00
feat(tracing): port trace events sessions to OSS (#7464)
Some checks failed
CodeQL Advanced / Analyze (actions) (push) Has been cancelled
CodeQL Advanced / Analyze (python) (push) Has been cancelled
Check Documentation Broken Links / Check broken links (push) Has been cancelled
Vulnerability Scan / Detect changes (push) Has been cancelled
Vulnerability Scan / pip-audit (push) Has been cancelled
Build uv cache / build-cache (3.10) (push) Has been cancelled
Build uv cache / build-cache (3.11) (push) Has been cancelled
Build uv cache / build-cache (3.12) (push) Has been cancelled
Build uv cache / build-cache (3.13) (push) Has been cancelled
Nightly Canary Release / Check for new commits (push) Has been cancelled
Nightly Canary Release / Build nightly packages (push) Has been cancelled
Nightly Canary Release / Publish nightly to PyPI (push) Has been cancelled
Some checks failed
CodeQL Advanced / Analyze (actions) (push) Has been cancelled
CodeQL Advanced / Analyze (python) (push) Has been cancelled
Check Documentation Broken Links / Check broken links (push) Has been cancelled
Vulnerability Scan / Detect changes (push) Has been cancelled
Vulnerability Scan / pip-audit (push) Has been cancelled
Build uv cache / build-cache (3.10) (push) Has been cancelled
Build uv cache / build-cache (3.11) (push) Has been cancelled
Build uv cache / build-cache (3.12) (push) Has been cancelled
Build uv cache / build-cache (3.13) (push) Has been cancelled
Nightly Canary Release / Check for new commits (push) Has been cancelled
Nightly Canary Release / Build nightly packages (push) Has been cancelled
Nightly Canary Release / Publish nightly to PyPI (push) Has been cancelled
* feat(tracing): port enterprise event sessions to OSS * fix(tracing): address review findings and verify concurrent exports * fix(tracing): keep redactor ownership in enterprise integrations * test(tracing): isolate intentional failures from cleanup assertions
This commit is contained in:
@@ -4,11 +4,12 @@ Two-column layout: left sidebar (tasks/agents/tokens) + main content
|
||||
(task header, plan checklist, activity timeline, streaming output).
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import json as _json
|
||||
import re
|
||||
import threading
|
||||
import time
|
||||
from typing import Any, ClassVar, cast
|
||||
from typing import Any, ClassVar
|
||||
|
||||
from crewai_core.telemetry import Telemetry
|
||||
from rich.text import Text
|
||||
@@ -56,27 +57,6 @@ def _truncate_log_text(value: Any, limit: int) -> str | None:
|
||||
return f"{text[: max(0, limit - len(suffix))]}{suffix}"
|
||||
|
||||
|
||||
def _enable_tracing_in_dotenv() -> None:
|
||||
"""Append CREWAI_TRACING_ENABLED=true to .env if not already set."""
|
||||
from pathlib import Path
|
||||
|
||||
env_file = Path.cwd() / ".env"
|
||||
key = "CREWAI_TRACING_ENABLED"
|
||||
try:
|
||||
if env_file.exists():
|
||||
content = env_file.read_text()
|
||||
if key in content:
|
||||
return
|
||||
sep = "" if content.endswith("\n") or not content else "\n"
|
||||
env_file.write_text(f"{content}{sep}{key}=true\n")
|
||||
else:
|
||||
env_file.write_text(f"{key}=true\n")
|
||||
except OSError:
|
||||
# Persisting the tracing flag is best-effort; an unwritable .env
|
||||
# must not block the run (tracing stays enabled for this session).
|
||||
pass
|
||||
|
||||
|
||||
def _unescape_text(s: str) -> str:
|
||||
"""Replace literal backslash-n sequences with real newlines."""
|
||||
return s.replace("\\n", "\n").replace("\\t", " ")
|
||||
@@ -227,8 +207,10 @@ class TraceConsentScreen(ModalScreen[bool]):
|
||||
}
|
||||
#consent-dialog {
|
||||
width: 50;
|
||||
max-width: 95%;
|
||||
height: auto;
|
||||
max-height: 16;
|
||||
max-height: 90%;
|
||||
overflow-y: auto;
|
||||
background: #1c1c1c;
|
||||
border: tall #333333;
|
||||
padding: 1 2 2 2;
|
||||
@@ -275,61 +257,35 @@ class TraceConsentScreen(ModalScreen[bool]):
|
||||
Binding("escape", "consent_no", "Cancel", show=False),
|
||||
]
|
||||
|
||||
def __init__(self) -> None:
|
||||
super().__init__()
|
||||
self._sending = False
|
||||
self._frame = 0
|
||||
self._spin_timer: Any = None
|
||||
|
||||
def compose(self) -> ComposeResult:
|
||||
with Vertical(id="consent-dialog"):
|
||||
yield Static(self._build_content(), id="consent-text")
|
||||
with Horizontal(id="consent-buttons"):
|
||||
yield Button("View Traces", id="btn-consent-yes", classes="consent-btn")
|
||||
yield Button(
|
||||
"Share Trace",
|
||||
id="btn-consent-yes",
|
||||
classes="consent-btn",
|
||||
)
|
||||
yield Button("Cancel", id="btn-consent-no", classes="consent-btn")
|
||||
|
||||
def _build_content(self) -> Text:
|
||||
t = Text()
|
||||
t.append(" View execution traces on CrewAI AMP\n\n", style=f"bold {_C_TEXT}")
|
||||
t.append(" Sends agent decisions, tool calls, and\n", style=_C_DIM)
|
||||
t.append(" timing data. Link expires in 24h.\n\n", style=_C_DIM)
|
||||
t.append(" Traces will be enabled for future runs.\n", style=_C_MUTED)
|
||||
t.append(
|
||||
" Share this execution trace with CrewAI?\n\n", style=f"bold {_C_TEXT}"
|
||||
)
|
||||
t.append(" The trace is stored locally and may include\n", style=_C_DIM)
|
||||
t.append(" prompts, inputs, outputs, and tool calls.\n\n", style=_C_DIM)
|
||||
t.append(" Sharing uploads it. Cancel or wait 20 seconds\n", style=_C_MUTED)
|
||||
t.append(" to discard it without uploading.\n", style=_C_MUTED)
|
||||
return t
|
||||
|
||||
def _start_sending(self) -> None:
|
||||
self._sending = True
|
||||
btn_yes = self.query_one("#btn-consent-yes", Button)
|
||||
btn_no = self.query_one("#btn-consent-no", Button)
|
||||
btn_yes.disabled = True
|
||||
btn_yes.label = f"{_SPINNER[0]} Loading…"
|
||||
btn_no.display = False
|
||||
self._spin_timer = self.set_interval(1 / 8, self._spin_tick)
|
||||
cast("CrewRunApp", self.app)._on_trace_consent_accepted()
|
||||
|
||||
def _spin_tick(self) -> None:
|
||||
self._frame += 1
|
||||
try:
|
||||
btn = self.query_one("#btn-consent-yes", Button)
|
||||
btn.label = f"{_SPINNER[self._frame % len(_SPINNER)]} Loading…"
|
||||
except Exception: # noqa: S110
|
||||
pass
|
||||
|
||||
def on_button_pressed(self, event: Button.Pressed) -> None:
|
||||
if self._sending:
|
||||
return
|
||||
if event.button.id == "btn-consent-yes":
|
||||
self._start_sending()
|
||||
else:
|
||||
self.dismiss(False)
|
||||
self.dismiss(event.button.id == "btn-consent-yes")
|
||||
|
||||
def action_consent_yes(self) -> None:
|
||||
if self._sending:
|
||||
return
|
||||
self._start_sending()
|
||||
self.dismiss(True)
|
||||
|
||||
def action_consent_no(self) -> None:
|
||||
if self._sending:
|
||||
return
|
||||
self.dismiss(False)
|
||||
|
||||
|
||||
@@ -474,15 +430,6 @@ FooterKey .footer-key--key {
|
||||
background: #444444;
|
||||
}
|
||||
|
||||
#btn-traces-done {
|
||||
background: #1a3a3a;
|
||||
color: #1F7982;
|
||||
border: none;
|
||||
}
|
||||
#btn-traces-done:hover {
|
||||
background: #1F7982;
|
||||
color: #e0e0e0;
|
||||
}
|
||||
"""
|
||||
|
||||
BINDINGS: ClassVar[list[BindingType]] = [
|
||||
@@ -582,8 +529,9 @@ FooterKey .footer-key--key {
|
||||
self._current_method: str | None = None
|
||||
self._elapsed_frozen: float | None = None
|
||||
self._want_deploy: bool = False
|
||||
self._trace_url: str | None = None
|
||||
self._consent_screen: TraceConsentScreen | None = None
|
||||
self._trace_consent_pending: threading.Event | None = None
|
||||
self._discard_trace_on_exit = False
|
||||
self._telemetry: Telemetry | None = None
|
||||
|
||||
@property
|
||||
@@ -701,13 +649,18 @@ FooterKey .footer-key--key {
|
||||
set_tui_mode(True)
|
||||
set_suppress_tracing_messages(True)
|
||||
try:
|
||||
result = self._crew.kickoff(inputs=self._default_inputs)
|
||||
from crewai.telemetry.tracing.ephemeral import trace_consent
|
||||
|
||||
with trace_consent(self._request_trace_consent):
|
||||
result = self._crew.kickoff(inputs=self._default_inputs)
|
||||
output = result.raw if result and hasattr(result, "raw") else None
|
||||
with self._lock:
|
||||
self._crew_result = result
|
||||
self.call_from_thread(self._on_crew_done, output)
|
||||
if not self._discard_trace_on_exit:
|
||||
self.call_from_thread(self._on_crew_done, output)
|
||||
except Exception as e:
|
||||
self.call_from_thread(self._on_crew_failed, str(e))
|
||||
if not self._discard_trace_on_exit:
|
||||
self.call_from_thread(self._on_crew_failed, str(e))
|
||||
|
||||
@work(thread=True, exclusive=True, group="flow")
|
||||
def _run_flow_worker(self) -> None:
|
||||
@@ -721,13 +674,18 @@ FooterKey .footer-key--key {
|
||||
try:
|
||||
# A declarative flow returns either a CrewOutput (has ``.raw``) or a
|
||||
# bare value (str/dict/pydantic); _stringify_output handles both.
|
||||
result = self._flow.kickoff(inputs=self._flow_inputs)
|
||||
from crewai.telemetry.tracing.ephemeral import trace_consent
|
||||
|
||||
with trace_consent(self._request_trace_consent):
|
||||
result = self._flow.kickoff(inputs=self._flow_inputs)
|
||||
output = self._stringify_output(result)
|
||||
with self._lock:
|
||||
self._crew_result = result
|
||||
self.call_from_thread(self._on_crew_done, output)
|
||||
if not self._discard_trace_on_exit:
|
||||
self.call_from_thread(self._on_crew_done, output)
|
||||
except Exception as e:
|
||||
self.call_from_thread(self._on_crew_failed, str(e))
|
||||
if not self._discard_trace_on_exit:
|
||||
self.call_from_thread(self._on_crew_failed, str(e))
|
||||
|
||||
def _set_flow_step_status(self, name: str, status: str) -> None:
|
||||
"""Update a flow method step's status. Caller must hold ``self._lock``."""
|
||||
@@ -778,27 +736,8 @@ FooterKey .footer-key--key {
|
||||
f"No result received before {self._run_noun} completed"
|
||||
)
|
||||
entry["duration"] = now - entry["start_time"]
|
||||
try:
|
||||
from crewai.events.listeners.tracing.trace_listener import (
|
||||
TraceCollectionListener,
|
||||
)
|
||||
|
||||
listener: TraceCollectionListener | None = getattr(
|
||||
TraceCollectionListener, "_instance", None
|
||||
)
|
||||
if listener and listener.batch_manager:
|
||||
bm = listener.batch_manager
|
||||
self._trace_url = (
|
||||
getattr(bm, "trace_url", None) or bm.ephemeral_trace_url
|
||||
)
|
||||
except Exception: # noqa: S110
|
||||
pass
|
||||
try:
|
||||
self.query_one("#sidebar-actions").display = True
|
||||
if self._trace_url:
|
||||
btn = self.query_one("#btn-traces", Button)
|
||||
btn.label = "✔ Open Traces"
|
||||
btn.id = "btn-traces-done"
|
||||
except Exception: # noqa: S110
|
||||
pass
|
||||
self._tick()
|
||||
@@ -860,11 +799,17 @@ FooterKey .footer-key--key {
|
||||
except Exception: # noqa: S110
|
||||
pass
|
||||
|
||||
def _finalize_conversational_session(self) -> None:
|
||||
def _finalize_conversational_session(self, *, discard: bool = False) -> None:
|
||||
if not (self._is_conversational and self._flow):
|
||||
return
|
||||
try:
|
||||
self._flow.finalize_session_traces()
|
||||
from crewai.telemetry.tracing.ephemeral import trace_consent
|
||||
|
||||
with trace_consent(self._request_trace_consent):
|
||||
if discard:
|
||||
self._flow.finalize_session_traces(discard=True)
|
||||
else:
|
||||
self._flow.finalize_session_traces()
|
||||
except Exception: # noqa: S110
|
||||
pass
|
||||
previous = self._conversation_previous_defer_trace_finalization
|
||||
@@ -885,9 +830,7 @@ FooterKey .footer-key--key {
|
||||
if not message:
|
||||
return
|
||||
if message.lower() in self._conversation_exit_commands:
|
||||
self._finalize_conversational_session()
|
||||
self._unsubscribe()
|
||||
self.exit(self._crew_result)
|
||||
self.run_worker(self.action_quit())
|
||||
return
|
||||
if self._conversation_turn_in_progress:
|
||||
return
|
||||
@@ -916,14 +859,24 @@ FooterKey .footer-key--key {
|
||||
set_tui_mode(True)
|
||||
set_suppress_tracing_messages(True)
|
||||
try:
|
||||
result = self._flow.handle_turn(message)
|
||||
from crewai.telemetry.tracing.ephemeral import trace_consent
|
||||
|
||||
with trace_consent(self._request_trace_consent):
|
||||
result = self._flow.handle_turn(message)
|
||||
if hasattr(result, "get_full_text") and hasattr(result, "result"):
|
||||
for _chunk in result:
|
||||
pass
|
||||
result = result.result
|
||||
self.call_from_thread(self._on_conversation_turn_done, result)
|
||||
if not self._discard_trace_on_exit:
|
||||
self.call_from_thread(self._on_conversation_turn_done, result)
|
||||
except Exception as e:
|
||||
self.call_from_thread(self._on_conversation_turn_failed, str(e))
|
||||
if not self._discard_trace_on_exit:
|
||||
self.call_from_thread(self._on_conversation_turn_failed, str(e))
|
||||
finally:
|
||||
if self._discard_trace_on_exit:
|
||||
# A first turn only stores its deferred trace when it returns.
|
||||
# Release it here even after the UI has already closed.
|
||||
self._finalize_conversational_session(discard=True)
|
||||
|
||||
def _on_conversation_turn_done(self, result: Any) -> None:
|
||||
with self._lock:
|
||||
@@ -1026,92 +979,65 @@ FooterKey .footer-key--key {
|
||||
self._refresh_log_panel()
|
||||
|
||||
async def action_quit(self) -> None:
|
||||
self._finalize_conversational_session()
|
||||
if (
|
||||
not self._is_conversational
|
||||
or self._conversation_turn_in_progress
|
||||
or self._trace_consent_pending is not None
|
||||
):
|
||||
self._discard_trace_on_exit = True
|
||||
if self._trace_consent_pending is not None:
|
||||
self._trace_consent_pending.set()
|
||||
if not self._conversation_turn_in_progress:
|
||||
await asyncio.to_thread(
|
||||
self._finalize_conversational_session,
|
||||
discard=self._discard_trace_on_exit,
|
||||
)
|
||||
self._unsubscribe()
|
||||
self.exit(self._crew_result)
|
||||
|
||||
def _request_trace_consent(self) -> bool:
|
||||
"""Wait in the execution worker while the UI asks for upload consent."""
|
||||
if self._discard_trace_on_exit:
|
||||
return False
|
||||
done = threading.Event()
|
||||
decision: list[bool] = []
|
||||
self._trace_consent_pending = done
|
||||
|
||||
def accepted(value: bool | None) -> None:
|
||||
decision.append(value is True)
|
||||
done.set()
|
||||
|
||||
def show() -> None:
|
||||
if self._discard_trace_on_exit:
|
||||
done.set()
|
||||
return
|
||||
self._consent_screen = TraceConsentScreen()
|
||||
self.push_screen(self._consent_screen, accepted)
|
||||
|
||||
try:
|
||||
self.call_from_thread(show)
|
||||
return (
|
||||
done.wait(timeout=20)
|
||||
and not self._discard_trace_on_exit
|
||||
and bool(decision)
|
||||
and decision[0]
|
||||
)
|
||||
finally:
|
||||
self._trace_consent_pending = None
|
||||
if not self._discard_trace_on_exit:
|
||||
self.call_from_thread(self._dismiss_consent_modal)
|
||||
|
||||
def action_view_traces(self) -> None:
|
||||
if self._status != "completed":
|
||||
return
|
||||
# Recorded here rather than in on_button_pressed so the `t` key binding
|
||||
# is counted too, and only once the action can actually do something.
|
||||
self._record_tui_button_click("view_traces")
|
||||
if self._trace_url:
|
||||
import webbrowser
|
||||
|
||||
try:
|
||||
webbrowser.open(self._trace_url)
|
||||
except Exception: # noqa: S110
|
||||
pass
|
||||
return
|
||||
self._consent_screen = TraceConsentScreen()
|
||||
self.push_screen(self._consent_screen)
|
||||
|
||||
def _on_trace_consent_accepted(self) -> None:
|
||||
self._send_traces_worker()
|
||||
|
||||
@work(thread=True)
|
||||
def _send_traces_worker(self) -> None:
|
||||
import webbrowser
|
||||
|
||||
try:
|
||||
from crewai.events.listeners.tracing.utils import (
|
||||
set_suppress_tracing_messages,
|
||||
set_tui_mode,
|
||||
)
|
||||
|
||||
set_tui_mode(True)
|
||||
set_suppress_tracing_messages(True)
|
||||
|
||||
from crewai.events.listeners.tracing.trace_listener import (
|
||||
TraceCollectionListener,
|
||||
)
|
||||
from crewai.events.listeners.tracing.utils import (
|
||||
mark_first_execution_completed,
|
||||
)
|
||||
|
||||
listener: TraceCollectionListener | None = getattr(
|
||||
TraceCollectionListener, "_instance", None
|
||||
)
|
||||
if not listener:
|
||||
self.call_from_thread(self._dismiss_consent_modal)
|
||||
return
|
||||
|
||||
bm = listener.batch_manager
|
||||
url = getattr(bm, "trace_url", None) or bm.ephemeral_trace_url
|
||||
|
||||
if not url:
|
||||
handler = listener.first_time_handler
|
||||
handler.set_batch_manager(bm)
|
||||
handler._initialize_backend_and_send_events()
|
||||
url = handler.ephemeral_url or bm.ephemeral_trace_url
|
||||
|
||||
if listener.first_time_handler.is_first_time:
|
||||
mark_first_execution_completed(user_consented=True)
|
||||
|
||||
_enable_tracing_in_dotenv()
|
||||
|
||||
if url:
|
||||
self._trace_url = url
|
||||
|
||||
def _done() -> None:
|
||||
self._dismiss_consent_modal()
|
||||
try:
|
||||
btn = self.query_one("#btn-traces", Button)
|
||||
btn.label = "✔ Open Traces"
|
||||
btn.id = "btn-traces-done"
|
||||
except Exception: # noqa: S110
|
||||
pass
|
||||
|
||||
self.call_from_thread(_done)
|
||||
try:
|
||||
webbrowser.open(url)
|
||||
except Exception: # noqa: S110
|
||||
pass
|
||||
else:
|
||||
self.call_from_thread(self._dismiss_consent_modal)
|
||||
except Exception:
|
||||
self.call_from_thread(self._dismiss_consent_modal)
|
||||
self.notify(
|
||||
"Trace sharing is requested when the execution finishes. "
|
||||
"A trace link is not available for this run.",
|
||||
title="Execution traces",
|
||||
)
|
||||
|
||||
def _dismiss_consent_modal(self) -> None:
|
||||
try:
|
||||
@@ -1141,7 +1067,7 @@ FooterKey .footer-key--key {
|
||||
pass
|
||||
|
||||
def on_button_pressed(self, event: Button.Pressed) -> None:
|
||||
if event.button.id in ("btn-traces", "btn-traces-done"):
|
||||
if event.button.id == "btn-traces":
|
||||
self.action_view_traces()
|
||||
elif event.button.id == "btn-deploy":
|
||||
self.action_deploy_crew()
|
||||
|
||||
@@ -3,8 +3,6 @@ import time
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import Mock
|
||||
|
||||
import pytest
|
||||
|
||||
from crewai.events.event_bus import crewai_event_bus
|
||||
from crewai.events.types.crew_events import CrewKickoffStartedEvent
|
||||
from crewai.events.types.flow_events import (
|
||||
@@ -34,16 +32,17 @@ from crewai.events.types.tool_usage_events import (
|
||||
ToolUsageFinishedEvent,
|
||||
ToolUsageStartedEvent,
|
||||
)
|
||||
from crewai_cli.command import AuthenticationRequiredError
|
||||
from crewai_cli import run_crew
|
||||
from crewai_cli.command import AuthenticationRequiredError
|
||||
from crewai_cli.crew_run_tui import (
|
||||
CrewRunApp,
|
||||
_LOG_ARGS_TEXT_LIMIT,
|
||||
_LOG_RESULT_TEXT_LIMIT,
|
||||
_LOG_TRUNCATION_SUFFIX,
|
||||
CrewRunApp,
|
||||
_format_json_in_text,
|
||||
_try_parse_structured,
|
||||
)
|
||||
import pytest
|
||||
|
||||
|
||||
def _app_with_plan() -> CrewRunApp:
|
||||
@@ -141,16 +140,18 @@ def test_chain_deploy_does_not_login_for_deploy_exit(monkeypatch, capsys) -> Non
|
||||
def test_view_traces_button_click_records_telemetry(monkeypatch) -> None:
|
||||
app = CrewRunApp()
|
||||
app._status = "completed"
|
||||
app._trace_url = "https://app.crewai.com/traces/test"
|
||||
app._telemetry = Mock()
|
||||
opened_urls: list[str] = []
|
||||
|
||||
monkeypatch.setattr("webbrowser.open", lambda url: opened_urls.append(url))
|
||||
notice = Mock()
|
||||
monkeypatch.setattr(app, "notify", notice)
|
||||
|
||||
app.on_button_pressed(SimpleNamespace(button=SimpleNamespace(id="btn-traces")))
|
||||
|
||||
app._telemetry.feature_usage_span.assert_called_once_with("cli_usage:view_traces")
|
||||
assert opened_urls == ["https://app.crewai.com/traces/test"]
|
||||
notice.assert_called_once_with(
|
||||
"Trace sharing is requested when the execution finishes. "
|
||||
"A trace link is not available for this run.",
|
||||
title="Execution traces",
|
||||
)
|
||||
|
||||
|
||||
def test_deploy_button_click_records_telemetry() -> None:
|
||||
@@ -1710,7 +1711,9 @@ async def test_declarative_flow_runs_on_tui() -> None:
|
||||
app._flow = FakeFlow()
|
||||
app._flow_inputs = {"topic": "AI"}
|
||||
# A step left active (no Finished event) must be swept to done by _on_crew_done.
|
||||
app._flow_steps = [{"name": "compute", "call_type": "expression", "status": "active"}]
|
||||
app._flow_steps = [
|
||||
{"name": "compute", "call_type": "expression", "status": "active"}
|
||||
]
|
||||
|
||||
async with app.run_test() as pilot:
|
||||
for _ in range(100):
|
||||
@@ -1729,16 +1732,14 @@ def test_view_traces_keybinding_records_telemetry(monkeypatch) -> None:
|
||||
"""The `t` binding reaches the action directly, never on_button_pressed."""
|
||||
app = CrewRunApp()
|
||||
app._status = "completed"
|
||||
app._trace_url = "https://app.crewai.com/traces/test"
|
||||
app._telemetry = Mock()
|
||||
opened_urls: list[str] = []
|
||||
|
||||
monkeypatch.setattr("webbrowser.open", lambda url: opened_urls.append(url))
|
||||
notice = Mock()
|
||||
monkeypatch.setattr(app, "notify", notice)
|
||||
|
||||
app.action_view_traces()
|
||||
|
||||
app._telemetry.feature_usage_span.assert_called_once_with("cli_usage:view_traces")
|
||||
assert opened_urls == ["https://app.crewai.com/traces/test"]
|
||||
notice.assert_called_once()
|
||||
|
||||
|
||||
def test_deploy_keybinding_records_telemetry() -> None:
|
||||
@@ -1784,30 +1785,14 @@ def test_button_press_records_exactly_once(monkeypatch) -> None:
|
||||
"""Recording moved into the action; the button must not double-count."""
|
||||
app = CrewRunApp()
|
||||
app._status = "completed"
|
||||
app._trace_url = "https://app.crewai.com/traces/test"
|
||||
app._telemetry = Mock()
|
||||
|
||||
monkeypatch.setattr("webbrowser.open", lambda url: None)
|
||||
monkeypatch.setattr(app, "notify", Mock())
|
||||
|
||||
app.on_button_pressed(SimpleNamespace(button=SimpleNamespace(id="btn-traces")))
|
||||
|
||||
assert app._telemetry.feature_usage_span.call_count == 1
|
||||
|
||||
|
||||
def test_finished_traces_button_still_records(monkeypatch) -> None:
|
||||
"""The button's id is swapped to btn-traces-done once a trace URL exists."""
|
||||
app = CrewRunApp()
|
||||
app._status = "completed"
|
||||
app._trace_url = "https://app.crewai.com/traces/test"
|
||||
app._telemetry = Mock()
|
||||
|
||||
monkeypatch.setattr("webbrowser.open", lambda url: None)
|
||||
|
||||
app.on_button_pressed(SimpleNamespace(button=SimpleNamespace(id="btn-traces-done")))
|
||||
|
||||
app._telemetry.feature_usage_span.assert_called_once_with("cli_usage:view_traces")
|
||||
|
||||
|
||||
def test_try_parse_structured_rejects_non_serializable_literals() -> None:
|
||||
"""ast.literal_eval("[...]") is a valid [Ellipsis] list but cannot be JSON-encoded."""
|
||||
assert _try_parse_structured("[...]") is None
|
||||
@@ -1840,5 +1825,5 @@ def test_format_json_in_text_survives_deep_nesting() -> None:
|
||||
|
||||
def test_format_json_in_text_still_pretty_prints_valid_json() -> None:
|
||||
assert _format_json_in_text('data: {"a": 1} and [...]') == (
|
||||
'data: ' + '{\n "a": 1\n}' + ' and [...]'
|
||||
"data: " + '{\n "a": 1\n}' + " and [...]"
|
||||
)
|
||||
|
||||
234
lib/cli/tests/test_trace_consent_session.py
Normal file
234
lib/cli/tests/test_trace_consent_session.py
Normal file
@@ -0,0 +1,234 @@
|
||||
"""The terminal UI resolves local-session consent without the legacy uploader."""
|
||||
|
||||
import asyncio
|
||||
import threading
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import Mock
|
||||
|
||||
from crewai_cli.crew_run_tui import CrewRunApp, TraceConsentScreen
|
||||
import pytest
|
||||
from textual.events import Mount
|
||||
from textual.widgets import Button
|
||||
|
||||
|
||||
class ConsentApp(CrewRunApp):
|
||||
def on_mount(self, event: Mount | None = None) -> None:
|
||||
"""Show the real UI without starting a crew or a refresh worker."""
|
||||
if event is not None:
|
||||
event.prevent_default()
|
||||
|
||||
|
||||
async def wait_for_consent(app, pilot):
|
||||
for _ in range(50):
|
||||
await pilot.pause(0.01)
|
||||
if isinstance(app.screen, TraceConsentScreen):
|
||||
return app.screen
|
||||
raise AssertionError("The execution worker did not open the consent prompt")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
("action", "approved"),
|
||||
[
|
||||
("y", True),
|
||||
("n", False),
|
||||
("escape", False),
|
||||
("click-yes", True),
|
||||
("click-no", False),
|
||||
],
|
||||
)
|
||||
async def test_session_consent_buttons_and_keys_resolve_execution_worker(
|
||||
monkeypatch, action, approved
|
||||
):
|
||||
app = ConsentApp()
|
||||
legacy = Mock(side_effect=AssertionError("Session consent invoked legacy upload"))
|
||||
monkeypatch.setattr(app, "_send_traces_worker", legacy, raising=False)
|
||||
async with app.run_test(size=(100, 40)) as pilot:
|
||||
result = asyncio.create_task(asyncio.to_thread(app._request_trace_consent))
|
||||
screen = await wait_for_consent(app, pilot)
|
||||
assert str(screen.query_one("#btn-consent-yes", Button).label) == "Share Trace"
|
||||
content = screen._build_content().plain
|
||||
assert "stored locally" in content and "Sharing uploads it" in content
|
||||
assert "prompts, inputs, outputs, and tool calls" in content
|
||||
assert "20 seconds" in content
|
||||
assert not result.done()
|
||||
if action.startswith("click-"):
|
||||
assert await pilot.click(f"#btn-consent-{action.removeprefix('click-')}")
|
||||
else:
|
||||
await pilot.press(action)
|
||||
assert await asyncio.wait_for(result, timeout=5) is approved
|
||||
await pilot.pause()
|
||||
assert not isinstance(app.screen, TraceConsentScreen)
|
||||
legacy.assert_not_called()
|
||||
assert app._trace_consent_pending is None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_consent_timeout_returns_false_and_dismisses_modal(monkeypatch):
|
||||
app = ConsentApp()
|
||||
waits = []
|
||||
event = threading.Event()
|
||||
|
||||
class ExpiredEvent:
|
||||
def set(self):
|
||||
event.set()
|
||||
|
||||
def wait(self, *, timeout):
|
||||
waits.append(timeout)
|
||||
return False
|
||||
|
||||
# Replace only this module's event factory, leaving Textual's threads alone.
|
||||
monkeypatch.setattr(
|
||||
"crewai_cli.crew_run_tui.threading", SimpleNamespace(Event=ExpiredEvent)
|
||||
)
|
||||
legacy = Mock(side_effect=AssertionError("Timeout invoked legacy upload"))
|
||||
monkeypatch.setattr(app, "_send_traces_worker", legacy, raising=False)
|
||||
async with app.run_test(size=(100, 40)) as pilot:
|
||||
result = await asyncio.wait_for(
|
||||
asyncio.to_thread(app._request_trace_consent), timeout=5
|
||||
)
|
||||
assert result is False
|
||||
await pilot.pause()
|
||||
assert not isinstance(app.screen, TraceConsentScreen)
|
||||
assert waits == [20]
|
||||
assert app._trace_consent_pending is None
|
||||
legacy.assert_not_called()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_quit_rejects_pending_consent_and_releases_worker(monkeypatch):
|
||||
app = ConsentApp()
|
||||
legacy = Mock(side_effect=AssertionError("Quit invoked legacy upload"))
|
||||
monkeypatch.setattr(app, "_send_traces_worker", legacy, raising=False)
|
||||
async with app.run_test(size=(100, 40)) as pilot:
|
||||
result = asyncio.create_task(asyncio.to_thread(app._request_trace_consent))
|
||||
await wait_for_consent(app, pilot)
|
||||
await app.action_quit()
|
||||
assert await asyncio.wait_for(result, timeout=5) is False
|
||||
assert app._trace_consent_pending is None
|
||||
legacy.assert_not_called()
|
||||
|
||||
|
||||
def test_consent_callback_refuses_nonboolean_screen_results(monkeypatch):
|
||||
app = CrewRunApp()
|
||||
monkeypatch.setattr(app, "call_from_thread", lambda callback: callback())
|
||||
monkeypatch.setattr(app, "push_screen", lambda screen, callback: callback("yes"))
|
||||
monkeypatch.setattr(app, "_dismiss_consent_modal", lambda: None)
|
||||
assert app._request_trace_consent() is False
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_conversation_quit_can_ask_for_deferred_session_consent(monkeypatch):
|
||||
app = ConsentApp(conversational=True)
|
||||
decisions = []
|
||||
|
||||
class DeferredFlow:
|
||||
defer_trace_finalization = True
|
||||
|
||||
def finalize_session_traces(self):
|
||||
decisions.append(app._request_trace_consent())
|
||||
|
||||
app._flow = DeferredFlow()
|
||||
app._conversation_previous_defer_trace_finalization = False
|
||||
legacy = Mock(side_effect=AssertionError("Conversation quit invoked legacy upload"))
|
||||
monkeypatch.setattr(app, "_send_traces_worker", legacy, raising=False)
|
||||
async with app.run_test(size=(100, 40)) as pilot:
|
||||
quit_task = asyncio.create_task(app.action_quit())
|
||||
await wait_for_consent(app, pilot)
|
||||
await pilot.press("n")
|
||||
await asyncio.wait_for(quit_task, timeout=5)
|
||||
assert decisions == [False]
|
||||
assert app._flow.defer_trace_finalization is False
|
||||
assert app._trace_consent_pending is None
|
||||
legacy.assert_not_called()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("prior_turn", [False, True])
|
||||
async def test_quit_during_turn_discards_trace_after_worker_finishes(
|
||||
monkeypatch, prior_turn
|
||||
):
|
||||
entered, release, finalized = (
|
||||
threading.Event(),
|
||||
threading.Event(),
|
||||
threading.Event(),
|
||||
)
|
||||
app = ConsentApp(conversational=True)
|
||||
completions = []
|
||||
|
||||
class RunningFlow:
|
||||
defer_trace_finalization = True
|
||||
trace = "prior turn" if prior_turn else None
|
||||
|
||||
def handle_turn(self, message):
|
||||
entered.set()
|
||||
release.wait(timeout=5)
|
||||
# A first kickoff stores its deferred lifetime just before returning.
|
||||
self.trace = "completed turn"
|
||||
return "done"
|
||||
|
||||
def finalize_session_traces(self, *, discard=False):
|
||||
completions.append(discard)
|
||||
self.trace = None
|
||||
finalized.set()
|
||||
|
||||
app._flow = RunningFlow()
|
||||
app._conversation_previous_defer_trace_finalization = False
|
||||
app._conversation_turn_in_progress = True
|
||||
app._status = "working"
|
||||
prompt = Mock(side_effect=AssertionError("Cancelled execution asked for consent"))
|
||||
monkeypatch.setattr(app, "push_screen", prompt)
|
||||
async with app.run_test(size=(100, 40)):
|
||||
app._run_conversation_turn_worker("hello")
|
||||
assert await asyncio.to_thread(entered.wait, 5)
|
||||
try:
|
||||
await app.action_quit()
|
||||
assert completions == []
|
||||
assert app._request_trace_consent() is False
|
||||
finally:
|
||||
release.set()
|
||||
assert await asyncio.to_thread(finalized.wait, 5)
|
||||
assert completions == [True]
|
||||
assert app._flow.trace is None
|
||||
assert app._flow.defer_trace_finalization is False
|
||||
prompt.assert_not_called()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("kind", ["crew", "flow"])
|
||||
async def test_quit_during_execution_rejects_late_consent(monkeypatch, kind):
|
||||
entered, release = threading.Event(), threading.Event()
|
||||
app = ConsentApp()
|
||||
decisions = []
|
||||
|
||||
class RunningExecution:
|
||||
def kickoff(self, inputs=None):
|
||||
entered.set()
|
||||
release.wait(timeout=5)
|
||||
decisions.append(app._request_trace_consent())
|
||||
return "done"
|
||||
|
||||
setattr(app, f"_{kind}", RunningExecution())
|
||||
app._status = "working"
|
||||
prompt = Mock(side_effect=AssertionError("Cancelled execution asked for consent"))
|
||||
completed = Mock()
|
||||
failed = Mock()
|
||||
monkeypatch.setattr(app, "push_screen", prompt)
|
||||
monkeypatch.setattr(app, "_on_crew_done", completed)
|
||||
monkeypatch.setattr(app, "_on_crew_failed", failed)
|
||||
async with app.run_test(size=(100, 40)):
|
||||
# Own the worker thread so the test awaits its body even after Textual
|
||||
# cancels its worker wrappers on exit (running threads are not stopped).
|
||||
run_worker = getattr(app, f"_run_{kind}_worker").__wrapped__
|
||||
worker = asyncio.create_task(asyncio.to_thread(run_worker, app))
|
||||
assert await asyncio.to_thread(entered.wait, 5)
|
||||
try:
|
||||
await app.action_quit()
|
||||
finally:
|
||||
release.set()
|
||||
await asyncio.wait_for(worker, timeout=5)
|
||||
assert decisions == [False]
|
||||
assert app._trace_consent_pending is None
|
||||
prompt.assert_not_called()
|
||||
completed.assert_not_called()
|
||||
failed.assert_not_called()
|
||||
Reference in New Issue
Block a user