mirror of
https://github.com/crewAIInc/crewAI.git
synced 2025-12-16 04:18:35 +00:00
feat: add console logging for LLM guardrail events (#3105)
* feat: add console logging for memory events * fix: emit guardrail events in correct order and handle exceptions * fix: remove unreachable elif conditions in guardrail event listener * fix: resolve mypy type errors in guardrail event handler
This commit is contained in:
@@ -97,7 +97,7 @@ class Task(BaseModel):
|
|||||||
)
|
)
|
||||||
context: Union[List["Task"], None, _NotSpecified] = Field(
|
context: Union[List["Task"], None, _NotSpecified] = Field(
|
||||||
description="Other tasks that will have their output used as context for this task.",
|
description="Other tasks that will have their output used as context for this task.",
|
||||||
default=NOT_SPECIFIED
|
default=NOT_SPECIFIED,
|
||||||
)
|
)
|
||||||
async_execution: Optional[bool] = Field(
|
async_execution: Optional[bool] = Field(
|
||||||
description="Whether the task should be executed asynchronously or not.",
|
description="Whether the task should be executed asynchronously or not.",
|
||||||
@@ -158,9 +158,7 @@ class Task(BaseModel):
|
|||||||
end_time: Optional[datetime.datetime] = Field(
|
end_time: Optional[datetime.datetime] = Field(
|
||||||
default=None, description="End time of the task execution"
|
default=None, description="End time of the task execution"
|
||||||
)
|
)
|
||||||
model_config = {
|
model_config = {"arbitrary_types_allowed": True}
|
||||||
"arbitrary_types_allowed": True
|
|
||||||
}
|
|
||||||
|
|
||||||
@field_validator("guardrail")
|
@field_validator("guardrail")
|
||||||
@classmethod
|
@classmethod
|
||||||
@@ -204,7 +202,6 @@ class Task(BaseModel):
|
|||||||
# Check return annotation if present, but don't require it
|
# Check return annotation if present, but don't require it
|
||||||
return_annotation = sig.return_annotation
|
return_annotation = sig.return_annotation
|
||||||
if return_annotation != inspect.Signature.empty:
|
if return_annotation != inspect.Signature.empty:
|
||||||
|
|
||||||
return_annotation_args = get_args(return_annotation)
|
return_annotation_args = get_args(return_annotation)
|
||||||
if not (
|
if not (
|
||||||
get_origin(return_annotation) is tuple
|
get_origin(return_annotation) is tuple
|
||||||
@@ -437,7 +434,7 @@ class Task(BaseModel):
|
|||||||
guardrail_result = process_guardrail(
|
guardrail_result = process_guardrail(
|
||||||
output=task_output,
|
output=task_output,
|
||||||
guardrail=self._guardrail,
|
guardrail=self._guardrail,
|
||||||
retry_count=self.retry_count
|
retry_count=self.retry_count,
|
||||||
)
|
)
|
||||||
if not guardrail_result.success:
|
if not guardrail_result.success:
|
||||||
if self.retry_count >= self.max_retries:
|
if self.retry_count >= self.max_retries:
|
||||||
@@ -510,8 +507,6 @@ class Task(BaseModel):
|
|||||||
)
|
)
|
||||||
from crewai.utilities.events.crewai_event_bus import crewai_event_bus
|
from crewai.utilities.events.crewai_event_bus import crewai_event_bus
|
||||||
|
|
||||||
result = self._guardrail(task_output)
|
|
||||||
|
|
||||||
crewai_event_bus.emit(
|
crewai_event_bus.emit(
|
||||||
self,
|
self,
|
||||||
LLMGuardrailStartedEvent(
|
LLMGuardrailStartedEvent(
|
||||||
@@ -519,7 +514,13 @@ class Task(BaseModel):
|
|||||||
),
|
),
|
||||||
)
|
)
|
||||||
|
|
||||||
guardrail_result = GuardrailResult.from_tuple(result)
|
try:
|
||||||
|
result = self._guardrail(task_output)
|
||||||
|
guardrail_result = GuardrailResult.from_tuple(result)
|
||||||
|
except Exception as e:
|
||||||
|
guardrail_result = GuardrailResult(
|
||||||
|
success=False, result=None, error=f"Guardrail execution error: {str(e)}"
|
||||||
|
)
|
||||||
|
|
||||||
crewai_event_bus.emit(
|
crewai_event_bus.emit(
|
||||||
self,
|
self,
|
||||||
|
|||||||
@@ -22,6 +22,10 @@ from crewai.utilities.events.llm_events import (
|
|||||||
LLMCallStartedEvent,
|
LLMCallStartedEvent,
|
||||||
LLMStreamChunkEvent,
|
LLMStreamChunkEvent,
|
||||||
)
|
)
|
||||||
|
from crewai.utilities.events.llm_guardrail_events import (
|
||||||
|
LLMGuardrailStartedEvent,
|
||||||
|
LLMGuardrailCompletedEvent,
|
||||||
|
)
|
||||||
from crewai.utilities.events.utils.console_formatter import ConsoleFormatter
|
from crewai.utilities.events.utils.console_formatter import ConsoleFormatter
|
||||||
|
|
||||||
from .agent_events import (
|
from .agent_events import (
|
||||||
@@ -370,6 +374,23 @@ class EventListener(BaseEventListener):
|
|||||||
print(content, end="", flush=True)
|
print(content, end="", flush=True)
|
||||||
self.next_chunk = self.text_stream.tell()
|
self.next_chunk = self.text_stream.tell()
|
||||||
|
|
||||||
|
# ----------- LLM GUARDRAIL EVENTS -----------
|
||||||
|
|
||||||
|
@crewai_event_bus.on(LLMGuardrailStartedEvent)
|
||||||
|
def on_llm_guardrail_started(source, event: LLMGuardrailStartedEvent):
|
||||||
|
guardrail_str = str(event.guardrail)
|
||||||
|
guardrail_name = (
|
||||||
|
guardrail_str[:50] + "..." if len(guardrail_str) > 50 else guardrail_str
|
||||||
|
)
|
||||||
|
|
||||||
|
self.formatter.handle_guardrail_started(guardrail_name, event.retry_count)
|
||||||
|
|
||||||
|
@crewai_event_bus.on(LLMGuardrailCompletedEvent)
|
||||||
|
def on_llm_guardrail_completed(source, event: LLMGuardrailCompletedEvent):
|
||||||
|
self.formatter.handle_guardrail_completed(
|
||||||
|
event.success, event.error, event.retry_count
|
||||||
|
)
|
||||||
|
|
||||||
@crewai_event_bus.on(CrewTestStartedEvent)
|
@crewai_event_bus.on(CrewTestStartedEvent)
|
||||||
def on_crew_test_started(source, event: CrewTestStartedEvent):
|
def on_crew_test_started(source, event: CrewTestStartedEvent):
|
||||||
cloned_crew = source.copy()
|
cloned_crew = source.copy()
|
||||||
|
|||||||
@@ -1473,9 +1473,7 @@ class ConsoleFormatter:
|
|||||||
return None
|
return None
|
||||||
|
|
||||||
memory_branch = branch_to_use.add("")
|
memory_branch = branch_to_use.add("")
|
||||||
self.update_tree_label(
|
self.update_tree_label(memory_branch, "🧠", "Memory Retrieval Started", "blue")
|
||||||
memory_branch, "🧠", "Memory Retrieval Started", "blue"
|
|
||||||
)
|
|
||||||
|
|
||||||
self.print(tree_to_use)
|
self.print(tree_to_use)
|
||||||
self.print()
|
self.print()
|
||||||
@@ -1549,7 +1547,6 @@ class ConsoleFormatter:
|
|||||||
if memory_content:
|
if memory_content:
|
||||||
add_panel()
|
add_panel()
|
||||||
|
|
||||||
|
|
||||||
def handle_memory_query_completed(
|
def handle_memory_query_completed(
|
||||||
self,
|
self,
|
||||||
agent_branch: Optional[Tree],
|
agent_branch: Optional[Tree],
|
||||||
@@ -1616,11 +1613,8 @@ class ConsoleFormatter:
|
|||||||
sources_branch.add(f"❌ {memory_type} - Error: {error}")
|
sources_branch.add(f"❌ {memory_type} - Error: {error}")
|
||||||
break
|
break
|
||||||
|
|
||||||
|
|
||||||
def handle_memory_save_started(
|
def handle_memory_save_started(
|
||||||
self,
|
self, agent_branch: Optional[Tree], crew_tree: Optional[Tree]
|
||||||
agent_branch: Optional[Tree],
|
|
||||||
crew_tree: Optional[Tree]
|
|
||||||
) -> None:
|
) -> None:
|
||||||
if not self.verbose:
|
if not self.verbose:
|
||||||
return None
|
return None
|
||||||
@@ -1633,7 +1627,7 @@ class ConsoleFormatter:
|
|||||||
|
|
||||||
for child in tree_to_use.children:
|
for child in tree_to_use.children:
|
||||||
if "Memory Update" in str(child.label):
|
if "Memory Update" in str(child.label):
|
||||||
break
|
break
|
||||||
else:
|
else:
|
||||||
memory_branch = tree_to_use.add("")
|
memory_branch = tree_to_use.add("")
|
||||||
self.update_tree_label(
|
self.update_tree_label(
|
||||||
@@ -1700,4 +1694,62 @@ class ConsoleFormatter:
|
|||||||
memory_branch.add(content)
|
memory_branch.add(content)
|
||||||
|
|
||||||
self.print(tree_to_use)
|
self.print(tree_to_use)
|
||||||
self.print()
|
self.print()
|
||||||
|
|
||||||
|
def handle_guardrail_started(
|
||||||
|
self,
|
||||||
|
guardrail_name: str,
|
||||||
|
retry_count: int,
|
||||||
|
) -> None:
|
||||||
|
"""Display guardrail evaluation started status.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
guardrail_name: Name/description of the guardrail being evaluated.
|
||||||
|
retry_count: Zero-based retry count (0 = first attempt).
|
||||||
|
"""
|
||||||
|
if not self.verbose:
|
||||||
|
return
|
||||||
|
|
||||||
|
content = self.create_status_content(
|
||||||
|
"Guardrail Evaluation Started",
|
||||||
|
guardrail_name,
|
||||||
|
"yellow",
|
||||||
|
Status="🔄 Evaluating",
|
||||||
|
Attempt=f"{retry_count + 1}",
|
||||||
|
)
|
||||||
|
self.print_panel(content, "🛡️ Guardrail Check", "yellow")
|
||||||
|
|
||||||
|
def handle_guardrail_completed(
|
||||||
|
self,
|
||||||
|
success: bool,
|
||||||
|
error: Optional[str],
|
||||||
|
retry_count: int,
|
||||||
|
) -> None:
|
||||||
|
"""Display guardrail evaluation result.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
success: Whether validation passed.
|
||||||
|
error: Error message if validation failed.
|
||||||
|
retry_count: Zero-based retry count.
|
||||||
|
"""
|
||||||
|
if not self.verbose:
|
||||||
|
return
|
||||||
|
|
||||||
|
if success:
|
||||||
|
content = self.create_status_content(
|
||||||
|
"Guardrail Passed",
|
||||||
|
"Validation Successful",
|
||||||
|
"green",
|
||||||
|
Status="✅ Validated",
|
||||||
|
Attempts=f"{retry_count + 1}",
|
||||||
|
)
|
||||||
|
self.print_panel(content, "🛡️ Guardrail Success", "green")
|
||||||
|
else:
|
||||||
|
content = self.create_status_content(
|
||||||
|
"Guardrail Failed",
|
||||||
|
"Validation Error",
|
||||||
|
"red",
|
||||||
|
Error=str(error) if error else "Unknown error",
|
||||||
|
Attempts=f"{retry_count + 1}",
|
||||||
|
)
|
||||||
|
self.print_panel(content, "🛡️ Guardrail Failed", "red")
|
||||||
|
|||||||
Reference in New Issue
Block a user