Compare commits

..

1 Commits

Author SHA1 Message Date
Joao Moura
c66d958613 feat(tracing): collect skill usage events
PR #6652 added SkillUsedEvent but deliberately shipped no listener wiring,
so the event reached no collector. The trace listener subscribed to the five
setup events -- discovery, load, activation, failure -- and none of them can
answer the question skills observability is for: activation is idempotent and
fires once at setup, so an agent using a skill across twenty turns produces
exactly one event.

SkillUsedEvent is the only runtime signal and the only one that re-fires per
execution. Subscribing to it lets a trace attribute skill usage to an agent
and a task.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
2026-07-30 00:16:08 -07:00
3 changed files with 78 additions and 1 deletions

2
.github/security.md vendored
View File

@@ -12,4 +12,4 @@ Please submit reports through one of the following channels:
- **Please do not** disclose vulnerabilities via public GitHub issues, pull requests,
or social media
- Reports submitted via channels other than the methods above will not be reviewed and will be dismissed
- Reports submitted via channels other than this Bugcrowd submission email will not be reviewed and will be dismissed

View File

@@ -119,6 +119,7 @@ from crewai.events.types.skill_events import (
SkillDiscoveryStartedEvent,
SkillLoadFailedEvent,
SkillLoadedEvent,
SkillUsedEvent,
)
from crewai.events.types.system_events import SignalEvent, on_signal
from crewai.events.types.task_events import (
@@ -609,6 +610,13 @@ class TraceCollectionListener(BaseEventListener):
def on_skill_load_failed(source: Any, event: SkillLoadFailedEvent) -> None:
self._handle_action_event("skill_load_failed", source, event)
@event_bus.on(SkillUsedEvent)
def on_skill_used(source: Any, event: SkillUsedEvent) -> None:
# The other five describe setup; this is the only one that says a
# skill was actually used, and the only one that re-fires per
# execution. Without it a trace cannot attribute usage to a task.
self._handle_action_event("skill_used", source, event)
def _register_a2a_event_handlers(self, event_bus: CrewAIEventsBus) -> None:
"""Register handlers for A2A (Agent-to-Agent) events."""

View File

@@ -0,0 +1,69 @@
"""Skill usage must reach the trace collector.
The five setup events (discovery, load, activation, failure) describe how an
agent was configured and fire once. ``SkillUsedEvent`` is the only runtime
signal -- it re-fires on every execution -- so without it a trace cannot say
which skills an agent actually used, on which task, or how often.
"""
from pathlib import Path
from unittest.mock import patch
from crewai.events.event_bus import CrewAIEventsBus
from crewai.events.listeners.tracing.trace_listener import TraceCollectionListener
from crewai.events.types.skill_events import (
SkillActivatedEvent,
SkillUsedEvent,
)
import pytest
@pytest.fixture
def registered_listener():
"""A listener wired to an isolated bus, with event handling captured."""
bus = CrewAIEventsBus()
listener = TraceCollectionListener.__new__(TraceCollectionListener)
with patch.object(TraceCollectionListener, "_handle_action_event") as handled:
listener._register_action_event_handlers(bus)
yield bus, handled
def _event_types(handled) -> list[str]:
return [call.args[0] for call in handled.call_args_list]
class TestSkillUsedIsCollected:
def test_skill_used_reaches_the_collector(self, registered_listener):
bus, handled = registered_listener
bus.emit(
None,
SkillUsedEvent(
skill_name="pdf-processing",
skill_path=Path("/skills/pdf-processing"),
),
)
bus.flush()
assert "skill_used" in _event_types(handled), (
"SkillUsedEvent was emitted but the trace listener ignored it"
)
def test_every_use_is_collected(self, registered_listener):
"""Activation is idempotent; usage is not. One event per use."""
bus, handled = registered_listener
for _ in range(3):
bus.emit(None, SkillUsedEvent(skill_name="pdf-processing"))
bus.flush()
assert _event_types(handled).count("skill_used") == 3
def test_setup_events_are_still_collected(self, registered_listener):
bus, handled = registered_listener
bus.emit(None, SkillActivatedEvent(skill_name="pdf-processing"))
bus.flush()
assert "skill_activated" in _event_types(handled)