From c66d9586135596f6f15c6857ae9c4ca10f5c1829 Mon Sep 17 00:00:00 2001 From: Joao Moura Date: Thu, 30 Jul 2026 00:16:08 -0700 Subject: [PATCH] feat(tracing): collect skill usage events PR #6652 added SkillUsedEvent but deliberately shipped no listener wiring, so the event reached no collector. The trace listener subscribed to the five setup events -- discovery, load, activation, failure -- and none of them can answer the question skills observability is for: activation is idempotent and fires once at setup, so an agent using a skill across twenty turns produces exactly one event. SkillUsedEvent is the only runtime signal and the only one that re-fires per execution. Subscribing to it lets a trace attribute skill usage to an agent and a task. Co-Authored-By: Claude Opus 5 (1M context) --- .../listeners/tracing/trace_listener.py | 8 +++ .../tests/tracing/test_skill_used_tracing.py | 69 +++++++++++++++++++ 2 files changed, 77 insertions(+) create mode 100644 lib/crewai/tests/tracing/test_skill_used_tracing.py diff --git a/lib/crewai/src/crewai/events/listeners/tracing/trace_listener.py b/lib/crewai/src/crewai/events/listeners/tracing/trace_listener.py index e1fb2024c..be3285cf3 100644 --- a/lib/crewai/src/crewai/events/listeners/tracing/trace_listener.py +++ b/lib/crewai/src/crewai/events/listeners/tracing/trace_listener.py @@ -119,6 +119,7 @@ from crewai.events.types.skill_events import ( SkillDiscoveryStartedEvent, SkillLoadFailedEvent, SkillLoadedEvent, + SkillUsedEvent, ) from crewai.events.types.system_events import SignalEvent, on_signal from crewai.events.types.task_events import ( @@ -609,6 +610,13 @@ class TraceCollectionListener(BaseEventListener): def on_skill_load_failed(source: Any, event: SkillLoadFailedEvent) -> None: self._handle_action_event("skill_load_failed", source, event) + @event_bus.on(SkillUsedEvent) + def on_skill_used(source: Any, event: SkillUsedEvent) -> None: + # The other five describe setup; this is the only one that says a + # skill was actually used, and the only one that re-fires per + # execution. Without it a trace cannot attribute usage to a task. + self._handle_action_event("skill_used", source, event) + def _register_a2a_event_handlers(self, event_bus: CrewAIEventsBus) -> None: """Register handlers for A2A (Agent-to-Agent) events.""" diff --git a/lib/crewai/tests/tracing/test_skill_used_tracing.py b/lib/crewai/tests/tracing/test_skill_used_tracing.py new file mode 100644 index 000000000..6a57370db --- /dev/null +++ b/lib/crewai/tests/tracing/test_skill_used_tracing.py @@ -0,0 +1,69 @@ +"""Skill usage must reach the trace collector. + +The five setup events (discovery, load, activation, failure) describe how an +agent was configured and fire once. ``SkillUsedEvent`` is the only runtime +signal -- it re-fires on every execution -- so without it a trace cannot say +which skills an agent actually used, on which task, or how often. +""" + +from pathlib import Path +from unittest.mock import patch + +from crewai.events.event_bus import CrewAIEventsBus +from crewai.events.listeners.tracing.trace_listener import TraceCollectionListener +from crewai.events.types.skill_events import ( + SkillActivatedEvent, + SkillUsedEvent, +) +import pytest + + +@pytest.fixture +def registered_listener(): + """A listener wired to an isolated bus, with event handling captured.""" + bus = CrewAIEventsBus() + listener = TraceCollectionListener.__new__(TraceCollectionListener) + + with patch.object(TraceCollectionListener, "_handle_action_event") as handled: + listener._register_action_event_handlers(bus) + yield bus, handled + + +def _event_types(handled) -> list[str]: + return [call.args[0] for call in handled.call_args_list] + + +class TestSkillUsedIsCollected: + def test_skill_used_reaches_the_collector(self, registered_listener): + bus, handled = registered_listener + + bus.emit( + None, + SkillUsedEvent( + skill_name="pdf-processing", + skill_path=Path("/skills/pdf-processing"), + ), + ) + bus.flush() + + assert "skill_used" in _event_types(handled), ( + "SkillUsedEvent was emitted but the trace listener ignored it" + ) + + def test_every_use_is_collected(self, registered_listener): + """Activation is idempotent; usage is not. One event per use.""" + bus, handled = registered_listener + + for _ in range(3): + bus.emit(None, SkillUsedEvent(skill_name="pdf-processing")) + bus.flush() + + assert _event_types(handled).count("skill_used") == 3 + + def test_setup_events_are_still_collected(self, registered_listener): + bus, handled = registered_listener + + bus.emit(None, SkillActivatedEvent(skill_name="pdf-processing")) + bus.flush() + + assert "skill_activated" in _event_types(handled)