mirror of
https://github.com/crewAIInc/crewAI.git
synced 2026-08-13 01:38:41 +00:00
65a4b7cede
* fix: emit FlowStartedEvent when a boundary hook aborts the flow A HookAborted at EXECUTION_START or INPUT propagated before `FlowStartedEvent` was emitted, so a policy deny left logs but no record of the execution. On abort, stamp the state id and open the flow scope before re-raising: the deny surfaces as a started -> failed execution while normal runs keep the existing ordering — the started event carries hook-resolved inputs and `id` rewrites keep redirecting persistence restoration. * docs: translate execution-boundary-hooks page to ar, ko, and pt-BR The English page updated on this branch had never been localized. Translate it into the three supported locales following `DOCS_TRANSLATIONS.md` and register the page in each locale's navigation in `docs/docs.json`. Untranslated link targets (the step-hooks page and the aborting-an-operation anchor) are omitted rather than pointed at English, matching the locale navigation convention.
599 lines
18 KiB
Python
599 lines
18 KiB
Python
"""Conformance suite for the framework-native interception points.
|
|
|
|
For each wired point this suite asserts the shared contract: the probe hook
|
|
sees a well-shaped payload, an in-place/returned modification is honored, and a
|
|
:class:`HookAborted` interrupts the step.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import asyncio
|
|
from unittest.mock import patch
|
|
|
|
from crewai.agent import Agent
|
|
from crewai.crew import Crew
|
|
from crewai.events.event_bus import crewai_event_bus
|
|
from crewai.events.types.crew_events import CrewKickoffCompletedEvent
|
|
from crewai.events.types.flow_events import (
|
|
FlowFailedEvent,
|
|
FlowFinishedEvent,
|
|
FlowStartedEvent,
|
|
)
|
|
from crewai.flow.flow import Flow, listen, start
|
|
from crewai.hooks.dispatch import (
|
|
HookAborted,
|
|
InterceptionPoint,
|
|
clear_all,
|
|
on,
|
|
)
|
|
from crewai.task import Task
|
|
import pytest
|
|
|
|
|
|
@pytest.fixture(autouse=True)
|
|
def clear_dispatch_registry():
|
|
clear_all()
|
|
yield
|
|
clear_all()
|
|
|
|
|
|
class _SimpleFlow(Flow):
|
|
@start()
|
|
def begin(self):
|
|
return "begin"
|
|
|
|
@listen(begin)
|
|
def finish(self, _):
|
|
return "flow-result"
|
|
|
|
|
|
class _FailingFlow(Flow):
|
|
@start()
|
|
def begin(self):
|
|
raise RuntimeError("flow boom")
|
|
|
|
|
|
class _ReentrantFailingFlow(Flow):
|
|
"""Kicks itself off once from inside a method, then fails in the outer run."""
|
|
|
|
@start()
|
|
async def begin(self):
|
|
if getattr(self, "_reentered", False):
|
|
return "inner-ok"
|
|
self._reentered = True
|
|
await self.kickoff_async()
|
|
raise RuntimeError("outer boom")
|
|
|
|
|
|
class TestFlowExecutionBoundaries:
|
|
"""execution_start / input / output / execution_end on a flow."""
|
|
|
|
def test_all_boundary_points_fire_once(self):
|
|
fired: list[str] = []
|
|
|
|
for point in (
|
|
InterceptionPoint.EXECUTION_START,
|
|
InterceptionPoint.INPUT,
|
|
InterceptionPoint.OUTPUT,
|
|
InterceptionPoint.EXECUTION_END,
|
|
):
|
|
|
|
@on(point)
|
|
def _probe(ctx, _point=point):
|
|
fired.append(_point.value)
|
|
|
|
_SimpleFlow().kickoff(inputs={"seed": 1})
|
|
|
|
assert fired == [
|
|
"execution_start",
|
|
"input",
|
|
"output",
|
|
"execution_end",
|
|
]
|
|
|
|
def test_output_modification_is_honored(self):
|
|
@on(InterceptionPoint.OUTPUT)
|
|
def rewrite(ctx):
|
|
return "intercepted"
|
|
|
|
result = _SimpleFlow().kickoff()
|
|
assert result == "intercepted"
|
|
|
|
def test_input_payload_carries_inputs(self):
|
|
seen: dict = {}
|
|
|
|
@on(InterceptionPoint.INPUT)
|
|
def capture(ctx):
|
|
seen.update(ctx.payload or {})
|
|
|
|
_SimpleFlow().kickoff(inputs={"seed": 42})
|
|
assert seen == {"seed": 42}
|
|
|
|
def test_input_hook_rewrite_lands_in_flow_state(self):
|
|
@on(InterceptionPoint.INPUT)
|
|
def inject(ctx):
|
|
return {**(ctx.payload or {}), "injected": "by-hook"}
|
|
|
|
class _StateReader(Flow):
|
|
@start()
|
|
def begin(self):
|
|
return self.state["injected"]
|
|
|
|
assert _StateReader().kickoff(inputs={"seed": 1}) == "by-hook"
|
|
|
|
def test_abort_at_execution_start_interrupts(self):
|
|
@on(InterceptionPoint.EXECUTION_START)
|
|
def block(ctx):
|
|
raise HookAborted(reason="not allowed", source="policy")
|
|
|
|
with pytest.raises(HookAborted) as exc:
|
|
_SimpleFlow().kickoff()
|
|
assert exc.value.reason == "not allowed"
|
|
|
|
|
|
class TestFlowStepPoints:
|
|
"""pre_step / post_step for flow methods (kind=flow_method)."""
|
|
|
|
def test_pre_and_post_step_fire_per_method(self):
|
|
kinds: list[tuple[str, str | None]] = []
|
|
|
|
@on(InterceptionPoint.PRE_STEP)
|
|
def pre(ctx):
|
|
kinds.append(("pre", ctx.step_name))
|
|
|
|
@on(InterceptionPoint.POST_STEP)
|
|
def post(ctx):
|
|
kinds.append(("post", ctx.step_name))
|
|
|
|
_SimpleFlow().kickoff()
|
|
|
|
assert ("pre", "begin") in kinds
|
|
assert ("post", "begin") in kinds
|
|
assert ("pre", "finish") in kinds
|
|
assert ("post", "finish") in kinds
|
|
|
|
def test_post_step_can_rewrite_method_output(self):
|
|
@on(InterceptionPoint.POST_STEP)
|
|
def rewrite(ctx):
|
|
if ctx.step_name == "finish":
|
|
return "rewritten"
|
|
return None
|
|
|
|
assert _SimpleFlow().kickoff() == "rewritten"
|
|
|
|
|
|
class TestTaskStepPoints:
|
|
"""pre_step / post_step for task execution (kind=task)."""
|
|
|
|
def test_post_step_rewrite_is_persisted_to_output_file(
|
|
self, tmp_path, monkeypatch
|
|
):
|
|
@on(InterceptionPoint.POST_STEP)
|
|
def sanitize(ctx):
|
|
return ctx.payload.model_copy(update={"raw": "sanitized output"})
|
|
|
|
monkeypatch.chdir(tmp_path)
|
|
agent = Agent(role="Writer", goal="Write", backstory="Writes things.")
|
|
task = Task(
|
|
description="Write something",
|
|
expected_output="Some text",
|
|
output_file="output.txt",
|
|
agent=agent,
|
|
)
|
|
|
|
with patch.object(Agent, "execute_task", return_value="original output"):
|
|
result = task.execute_sync(agent=agent)
|
|
|
|
assert result.raw == "sanitized output"
|
|
assert (tmp_path / "output.txt").read_text() == "sanitized output"
|
|
|
|
|
|
class TestExecutionEndOnFailure:
|
|
"""execution_end fires exactly once, on success and on failure alike."""
|
|
|
|
@staticmethod
|
|
def _crew() -> Crew:
|
|
agent = Agent(role="Writer", goal="Write", backstory="Writes things.")
|
|
task = Task(
|
|
description="Write something",
|
|
expected_output="Some text",
|
|
agent=agent,
|
|
)
|
|
return Crew(agents=[agent], tasks=[task], verbose=False)
|
|
|
|
def test_crew_success_fires_completed_once(self):
|
|
seen: list[tuple[str, BaseException | None]] = []
|
|
|
|
@on(InterceptionPoint.EXECUTION_END)
|
|
def capture(ctx):
|
|
seen.append((ctx.status, ctx.error))
|
|
|
|
with patch.object(Agent, "execute_task", return_value="fine"):
|
|
self._crew().kickoff()
|
|
|
|
assert seen == [("completed", None)]
|
|
|
|
def test_crew_failure_fires_failed_once_and_reraises(self):
|
|
seen = []
|
|
|
|
@on(InterceptionPoint.EXECUTION_END)
|
|
def capture(ctx):
|
|
seen.append(ctx)
|
|
|
|
error = RuntimeError("crew boom")
|
|
with patch.object(Agent, "execute_task", side_effect=error):
|
|
with pytest.raises(RuntimeError, match="crew boom"):
|
|
self._crew().kickoff()
|
|
|
|
assert len(seen) == 1
|
|
assert seen[0].status == "failed"
|
|
assert seen[0].error is error
|
|
assert seen[0].output is None
|
|
|
|
def test_crew_kickoff_async_failure_fires_failed_once(self):
|
|
seen = []
|
|
|
|
@on(InterceptionPoint.EXECUTION_END)
|
|
def capture(ctx):
|
|
seen.append(ctx)
|
|
|
|
with patch.object(
|
|
Agent, "execute_task", side_effect=RuntimeError("crew boom")
|
|
):
|
|
with pytest.raises(RuntimeError, match="crew boom"):
|
|
asyncio.run(self._crew().kickoff_async())
|
|
|
|
assert len(seen) == 1
|
|
assert seen[0].status == "failed"
|
|
|
|
def test_flow_success_fires_completed_once(self):
|
|
seen: list[tuple[str, BaseException | None]] = []
|
|
|
|
@on(InterceptionPoint.EXECUTION_END)
|
|
def capture(ctx):
|
|
seen.append((ctx.status, ctx.error))
|
|
|
|
_SimpleFlow().kickoff()
|
|
|
|
assert seen == [("completed", None)]
|
|
|
|
def test_flow_failure_fires_failed_once_and_reraises(self):
|
|
seen = []
|
|
|
|
@on(InterceptionPoint.EXECUTION_END)
|
|
def capture(ctx):
|
|
seen.append(ctx)
|
|
|
|
with pytest.raises(RuntimeError, match="flow boom"):
|
|
_FailingFlow().kickoff()
|
|
|
|
assert len(seen) == 1
|
|
assert seen[0].status == "failed"
|
|
assert isinstance(seen[0].error, RuntimeError)
|
|
assert seen[0].output is None
|
|
|
|
def test_reentrant_flow_kickoff_pairs_ends_per_invocation(self):
|
|
seen: list[tuple[str, str | None]] = []
|
|
|
|
@on(InterceptionPoint.EXECUTION_START)
|
|
def capture_start(ctx):
|
|
seen.append(("start", None))
|
|
|
|
@on(InterceptionPoint.EXECUTION_END)
|
|
def capture_end(ctx):
|
|
seen.append(("end", ctx.status))
|
|
|
|
with pytest.raises(RuntimeError, match="outer boom"):
|
|
_ReentrantFailingFlow().kickoff()
|
|
|
|
assert seen == [
|
|
("start", None),
|
|
("start", None),
|
|
("end", "completed"),
|
|
("end", "failed"),
|
|
]
|
|
|
|
def test_no_execution_end_when_execution_start_aborts(self):
|
|
seen = []
|
|
|
|
@on(InterceptionPoint.EXECUTION_START)
|
|
def block(ctx):
|
|
raise HookAborted(reason="blocked")
|
|
|
|
@on(InterceptionPoint.EXECUTION_END)
|
|
def capture(ctx):
|
|
seen.append(ctx)
|
|
|
|
with pytest.raises(HookAborted):
|
|
_SimpleFlow().kickoff()
|
|
with pytest.raises(HookAborted):
|
|
self._crew().kickoff()
|
|
|
|
assert seen == []
|
|
|
|
def test_aborting_execution_end_hook_fires_once_for_flow(self):
|
|
calls: list[str] = []
|
|
|
|
@on(InterceptionPoint.EXECUTION_END)
|
|
def abort_end(ctx):
|
|
calls.append(ctx.status)
|
|
raise HookAborted(reason="no")
|
|
|
|
with pytest.raises(HookAborted):
|
|
_SimpleFlow().kickoff()
|
|
|
|
assert calls == ["completed"]
|
|
|
|
def test_aborting_execution_end_hook_fires_once_for_crew(self):
|
|
calls: list[str] = []
|
|
|
|
@on(InterceptionPoint.EXECUTION_END)
|
|
def abort_end(ctx):
|
|
calls.append(ctx.status)
|
|
raise HookAborted(reason="no")
|
|
|
|
with patch.object(Agent, "execute_task", return_value="fine"):
|
|
with pytest.raises(HookAborted):
|
|
self._crew().kickoff()
|
|
|
|
assert calls == ["completed"]
|
|
|
|
|
|
class TestCrewOutput:
|
|
def test_output_modification_reaches_kickoff_completed_event(self):
|
|
@on(InterceptionPoint.OUTPUT)
|
|
def append_notice(ctx):
|
|
if hasattr(ctx.payload, "raw") and isinstance(ctx.payload.raw, str):
|
|
ctx.payload.raw += "\nchanged by hook"
|
|
return None
|
|
|
|
completed_raw: list[str] = []
|
|
|
|
@crewai_event_bus.on(CrewKickoffCompletedEvent)
|
|
def capture_completed(_source, event: CrewKickoffCompletedEvent):
|
|
completed_raw.append(event.output.raw)
|
|
|
|
agent = Agent(role="Writer", goal="Write", backstory="Writes things.")
|
|
task = Task(
|
|
description="Write something",
|
|
expected_output="Some text",
|
|
agent=agent,
|
|
)
|
|
crew = Crew(agents=[agent], tasks=[task], verbose=False)
|
|
|
|
with patch.object(Agent, "execute_task", return_value="original output"):
|
|
result = crew.kickoff()
|
|
crewai_event_bus.flush()
|
|
|
|
assert result.raw.endswith("changed by hook")
|
|
assert completed_raw
|
|
assert completed_raw[-1].endswith("changed by hook")
|
|
|
|
|
|
class TestFlowLifecycleEventOrdering:
|
|
"""Boundary hooks resolve the inputs before FlowStartedEvent snapshots
|
|
them; an abort at EXECUTION_START still surfaces as started -> failed."""
|
|
|
|
def test_abort_at_execution_start_emits_started_then_failed(self):
|
|
lifecycle: list[str] = []
|
|
|
|
with crewai_event_bus.scoped_handlers():
|
|
|
|
@crewai_event_bus.on(FlowStartedEvent)
|
|
def on_started(_source, _event):
|
|
lifecycle.append("flow_started")
|
|
|
|
@crewai_event_bus.on(FlowFailedEvent)
|
|
def on_failed(_source, _event):
|
|
lifecycle.append("flow_failed")
|
|
|
|
@crewai_event_bus.on(FlowFinishedEvent)
|
|
def on_finished(_source, _event):
|
|
lifecycle.append("flow_finished")
|
|
|
|
@on(InterceptionPoint.EXECUTION_START)
|
|
def block(ctx):
|
|
raise HookAborted(reason="denied", source="policy")
|
|
|
|
with pytest.raises(HookAborted, match="denied"):
|
|
_SimpleFlow().kickoff()
|
|
crewai_event_bus.flush()
|
|
|
|
assert lifecycle == ["flow_started", "flow_failed"]
|
|
|
|
def test_boundary_id_rewrite_redirects_state_identity(self):
|
|
# Listeners key execution records off flow_id at emission time.
|
|
seen_ids: list[str] = []
|
|
rewritten_id = "0b8ee866-77b3-4dc6-9de4-90a92cbc9fcf"
|
|
|
|
with crewai_event_bus.scoped_handlers():
|
|
|
|
@crewai_event_bus.on(FlowStartedEvent)
|
|
def capture(source, _event):
|
|
seen_ids.append(source.flow_id)
|
|
|
|
@on(InterceptionPoint.EXECUTION_START)
|
|
def rewrite(ctx):
|
|
return {**(ctx.payload or {}), "id": rewritten_id}
|
|
|
|
_SimpleFlow().kickoff(
|
|
inputs={"id": "2f9deb0a-41c5-4f4e-9be9-3a5a34c4dc94"}
|
|
)
|
|
crewai_event_bus.flush()
|
|
|
|
assert seen_ids == [rewritten_id]
|
|
|
|
def test_abort_keeps_state_id_from_inputs(self):
|
|
# Error-by-id correlation reads the state id stamped at abort time.
|
|
seen_ids: list[str] = []
|
|
pinned_id = "0b8ee866-77b3-4dc6-9de4-90a92cbc9fcf"
|
|
|
|
with crewai_event_bus.scoped_handlers():
|
|
|
|
@crewai_event_bus.on(FlowStartedEvent)
|
|
def capture(source, _event):
|
|
seen_ids.append(source.flow_id)
|
|
|
|
@on(InterceptionPoint.EXECUTION_START)
|
|
def block(ctx):
|
|
raise HookAborted(reason="denied")
|
|
|
|
with pytest.raises(HookAborted):
|
|
_SimpleFlow().kickoff(inputs={"id": pinned_id})
|
|
crewai_event_bus.flush()
|
|
|
|
assert seen_ids == [pinned_id]
|
|
|
|
def test_input_redaction_shields_started_event_inputs(self):
|
|
captured: list[dict | None] = []
|
|
|
|
with crewai_event_bus.scoped_handlers():
|
|
|
|
@crewai_event_bus.on(FlowStartedEvent)
|
|
def capture(_source, event):
|
|
captured.append(event.inputs)
|
|
|
|
@on(InterceptionPoint.INPUT)
|
|
def redact(ctx):
|
|
return {**(ctx.payload or {}), "api_key": "[REDACTED]"}
|
|
|
|
_SimpleFlow().kickoff(inputs={"api_key": "sk-live-secret"})
|
|
crewai_event_bus.flush()
|
|
|
|
assert captured == [{"api_key": "[REDACTED]"}]
|
|
|
|
def test_abort_preserves_chained_hook_rewrite(self):
|
|
# A replacement returned by an earlier hook lands on ctx.payload as
|
|
# each hook runs; a later abort must not fall back to pre-hook inputs.
|
|
seen_ids: list[str] = []
|
|
captured: list[dict | None] = []
|
|
rewritten_id = "0b8ee866-77b3-4dc6-9de4-90a92cbc9fcf"
|
|
|
|
with crewai_event_bus.scoped_handlers():
|
|
|
|
@crewai_event_bus.on(FlowStartedEvent)
|
|
def capture(source, event):
|
|
seen_ids.append(source.flow_id)
|
|
captured.append(event.inputs)
|
|
|
|
@on(InterceptionPoint.EXECUTION_START)
|
|
def rewrite(ctx):
|
|
return {**(ctx.payload or {}), "id": rewritten_id}
|
|
|
|
@on(InterceptionPoint.EXECUTION_START)
|
|
def block(_ctx):
|
|
raise HookAborted(reason="denied")
|
|
|
|
with pytest.raises(HookAborted):
|
|
_SimpleFlow().kickoff(
|
|
inputs={"id": "2f9deb0a-41c5-4f4e-9be9-3a5a34c4dc94"}
|
|
)
|
|
crewai_event_bus.flush()
|
|
|
|
assert seen_ids == [rewritten_id]
|
|
assert captured == [{"id": rewritten_id}]
|
|
|
|
def test_abort_at_input_preserves_execution_start_rewrite(self):
|
|
captured: list[dict | None] = []
|
|
|
|
with crewai_event_bus.scoped_handlers():
|
|
|
|
@crewai_event_bus.on(FlowStartedEvent)
|
|
def capture(_source, event):
|
|
captured.append(event.inputs)
|
|
|
|
@on(InterceptionPoint.EXECUTION_START)
|
|
def redact(ctx):
|
|
return {**(ctx.payload or {}), "api_key": "[REDACTED]"}
|
|
|
|
@on(InterceptionPoint.INPUT)
|
|
def block(_ctx):
|
|
raise HookAborted(reason="denied")
|
|
|
|
with pytest.raises(HookAborted):
|
|
_SimpleFlow().kickoff(inputs={"api_key": "sk-live-secret"})
|
|
crewai_event_bus.flush()
|
|
|
|
assert captured == [{"api_key": "[REDACTED]"}]
|
|
|
|
def test_kickoff_emits_started_exactly_once(self):
|
|
started: list[str] = []
|
|
|
|
with crewai_event_bus.scoped_handlers():
|
|
|
|
@crewai_event_bus.on(FlowStartedEvent)
|
|
def capture(_source, event):
|
|
started.append(event.flow_name)
|
|
|
|
_SimpleFlow().kickoff()
|
|
crewai_event_bus.flush()
|
|
|
|
assert started == ["_SimpleFlow"]
|
|
|
|
def test_reentrant_kickoff_emits_started_once_per_invocation(self):
|
|
started: list[str] = []
|
|
|
|
with crewai_event_bus.scoped_handlers():
|
|
|
|
@crewai_event_bus.on(FlowStartedEvent)
|
|
def capture(_source, event):
|
|
started.append(event.flow_name)
|
|
|
|
with pytest.raises(RuntimeError, match="outer boom"):
|
|
_ReentrantFailingFlow().kickoff()
|
|
crewai_event_bus.flush()
|
|
|
|
# One started per kickoff invocation: outer run + its nested kickoff.
|
|
assert started == ["_ReentrantFailingFlow", "_ReentrantFailingFlow"]
|
|
|
|
def test_deferred_session_deny_at_execution_start_keeps_session_open(self):
|
|
started: list[FlowStartedEvent] = []
|
|
failed: list[FlowFailedEvent] = []
|
|
finished: list[FlowFinishedEvent] = []
|
|
|
|
class _DeferredFlow(Flow):
|
|
defer_trace_finalization = True
|
|
|
|
@start()
|
|
def begin(self):
|
|
return "turn-result"
|
|
|
|
with crewai_event_bus.scoped_handlers():
|
|
|
|
@crewai_event_bus.on(FlowStartedEvent)
|
|
def on_started(_source, event):
|
|
started.append(event)
|
|
|
|
@crewai_event_bus.on(FlowFailedEvent)
|
|
def on_failed(_source, event):
|
|
failed.append(event)
|
|
|
|
@crewai_event_bus.on(FlowFinishedEvent)
|
|
def on_finished(_source, event):
|
|
finished.append(event)
|
|
|
|
@on(InterceptionPoint.EXECUTION_START)
|
|
def block(ctx):
|
|
raise HookAborted(reason="denied")
|
|
|
|
flow = _DeferredFlow()
|
|
with pytest.raises(HookAborted):
|
|
flow.kickoff()
|
|
crewai_event_bus.flush()
|
|
|
|
# Deferred sessions suppress per-turn terminal events; the
|
|
# session stays open under the already-emitted started event.
|
|
assert len(started) == 1
|
|
assert failed == []
|
|
assert finished == []
|
|
|
|
clear_all()
|
|
assert flow.kickoff() == "turn-result"
|
|
flow.finalize_session_traces()
|
|
crewai_event_bus.flush()
|
|
|
|
assert len(started) == 1
|
|
assert failed == []
|
|
assert len(finished) == 1
|
|
assert finished[0].started_event_id == started[0].event_id
|