mirror of
https://github.com/crewAIInc/crewAI.git
synced 2026-08-10 16:32:28 +00:00
feat(tracing): collect skill usage events
PR #6652 added SkillUsedEvent but deliberately shipped no listener wiring, so the event reached no collector. The trace listener subscribed to the five setup events -- discovery, load, activation, failure -- and none of them can answer the question skills observability is for: activation is idempotent and fires once at setup, so an agent using a skill across twenty turns produces exactly one event. SkillUsedEvent is the only runtime signal and the only one that re-fires per execution. Subscribing to it lets a trace attribute skill usage to an agent and a task. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -119,6 +119,7 @@ from crewai.events.types.skill_events import (
|
||||
SkillDiscoveryStartedEvent,
|
||||
SkillLoadFailedEvent,
|
||||
SkillLoadedEvent,
|
||||
SkillUsedEvent,
|
||||
)
|
||||
from crewai.events.types.system_events import SignalEvent, on_signal
|
||||
from crewai.events.types.task_events import (
|
||||
@@ -609,6 +610,13 @@ class TraceCollectionListener(BaseEventListener):
|
||||
def on_skill_load_failed(source: Any, event: SkillLoadFailedEvent) -> None:
|
||||
self._handle_action_event("skill_load_failed", source, event)
|
||||
|
||||
@event_bus.on(SkillUsedEvent)
|
||||
def on_skill_used(source: Any, event: SkillUsedEvent) -> None:
|
||||
# The other five describe setup; this is the only one that says a
|
||||
# skill was actually used, and the only one that re-fires per
|
||||
# execution. Without it a trace cannot attribute usage to a task.
|
||||
self._handle_action_event("skill_used", source, event)
|
||||
|
||||
def _register_a2a_event_handlers(self, event_bus: CrewAIEventsBus) -> None:
|
||||
"""Register handlers for A2A (Agent-to-Agent) events."""
|
||||
|
||||
|
||||
69
lib/crewai/tests/tracing/test_skill_used_tracing.py
Normal file
69
lib/crewai/tests/tracing/test_skill_used_tracing.py
Normal file
@@ -0,0 +1,69 @@
|
||||
"""Skill usage must reach the trace collector.
|
||||
|
||||
The five setup events (discovery, load, activation, failure) describe how an
|
||||
agent was configured and fire once. ``SkillUsedEvent`` is the only runtime
|
||||
signal -- it re-fires on every execution -- so without it a trace cannot say
|
||||
which skills an agent actually used, on which task, or how often.
|
||||
"""
|
||||
|
||||
from pathlib import Path
|
||||
from unittest.mock import patch
|
||||
|
||||
from crewai.events.event_bus import CrewAIEventsBus
|
||||
from crewai.events.listeners.tracing.trace_listener import TraceCollectionListener
|
||||
from crewai.events.types.skill_events import (
|
||||
SkillActivatedEvent,
|
||||
SkillUsedEvent,
|
||||
)
|
||||
import pytest
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def registered_listener():
|
||||
"""A listener wired to an isolated bus, with event handling captured."""
|
||||
bus = CrewAIEventsBus()
|
||||
listener = TraceCollectionListener.__new__(TraceCollectionListener)
|
||||
|
||||
with patch.object(TraceCollectionListener, "_handle_action_event") as handled:
|
||||
listener._register_action_event_handlers(bus)
|
||||
yield bus, handled
|
||||
|
||||
|
||||
def _event_types(handled) -> list[str]:
|
||||
return [call.args[0] for call in handled.call_args_list]
|
||||
|
||||
|
||||
class TestSkillUsedIsCollected:
|
||||
def test_skill_used_reaches_the_collector(self, registered_listener):
|
||||
bus, handled = registered_listener
|
||||
|
||||
bus.emit(
|
||||
None,
|
||||
SkillUsedEvent(
|
||||
skill_name="pdf-processing",
|
||||
skill_path=Path("/skills/pdf-processing"),
|
||||
),
|
||||
)
|
||||
bus.flush()
|
||||
|
||||
assert "skill_used" in _event_types(handled), (
|
||||
"SkillUsedEvent was emitted but the trace listener ignored it"
|
||||
)
|
||||
|
||||
def test_every_use_is_collected(self, registered_listener):
|
||||
"""Activation is idempotent; usage is not. One event per use."""
|
||||
bus, handled = registered_listener
|
||||
|
||||
for _ in range(3):
|
||||
bus.emit(None, SkillUsedEvent(skill_name="pdf-processing"))
|
||||
bus.flush()
|
||||
|
||||
assert _event_types(handled).count("skill_used") == 3
|
||||
|
||||
def test_setup_events_are_still_collected(self, registered_listener):
|
||||
bus, handled = registered_listener
|
||||
|
||||
bus.emit(None, SkillActivatedEvent(skill_name="pdf-processing"))
|
||||
bus.flush()
|
||||
|
||||
assert "skill_activated" in _event_types(handled)
|
||||
Reference in New Issue
Block a user