Compare commits

...

2 Commits

Author SHA1 Message Date
Greyson LaLonde
eda4430d6f Merge branch 'main' into gl/fix/hitl-flow-plot 2026-02-02 13:37:12 -05:00
Greyson LaLonde
3f264b4cc8 fix: add human_feedback metadata and visualization 2026-02-02 13:18:22 -05:00
6 changed files with 295 additions and 21 deletions

View File

@@ -513,11 +513,17 @@ class FlowMeta(type):
and attr_value.__is_router__
):
routers.add(attr_name)
possible_returns = get_possible_return_constants(attr_value)
if possible_returns:
router_paths[attr_name] = possible_returns
if (
hasattr(attr_value, "__router_paths__")
and attr_value.__router_paths__
):
router_paths[attr_name] = attr_value.__router_paths__
else:
router_paths[attr_name] = []
possible_returns = get_possible_return_constants(attr_value)
if possible_returns:
router_paths[attr_name] = possible_returns
else:
router_paths[attr_name] = []
# Handle start methods that are also routers (e.g., @human_feedback with emit)
if (

View File

@@ -1025,7 +1025,7 @@ class TriggeredByHighlighter {
const isAndOrRouter = edge.dashes || edge.label === "AND";
const highlightColor = isAndOrRouter
? "{{ CREWAI_ORANGE }}"
? (edge.color?.color || "{{ CREWAI_ORANGE }}")
: getComputedStyle(document.documentElement).getPropertyValue('--edge-or-color').trim();
const updateData = {
@@ -1080,7 +1080,7 @@ class TriggeredByHighlighter {
// Keep the original edge color instead of turning gray
const isAndOrRouter = edge.dashes || edge.label === "AND";
const baseColor = isAndOrRouter
? "{{ CREWAI_ORANGE }}"
? (edge.color?.color || "{{ CREWAI_ORANGE }}")
: getComputedStyle(document.documentElement).getPropertyValue('--edge-or-color').trim();
// Convert color to rgba with opacity for vis.js
@@ -1142,7 +1142,7 @@ class TriggeredByHighlighter {
const defaultColor =
edge.dashes || edge.label === "AND"
? "{{ CREWAI_ORANGE }}"
? (edge.color?.color || "{{ CREWAI_ORANGE }}")
: getComputedStyle(document.documentElement).getPropertyValue('--edge-or-color').trim();
const currentOpacity = edge.opacity !== undefined ? edge.opacity : 1.0;
const currentWidth =
@@ -1253,7 +1253,7 @@ class TriggeredByHighlighter {
const defaultColor =
edge.dashes || edge.label === "AND"
? "{{ CREWAI_ORANGE }}"
? (edge.color?.color || "{{ CREWAI_ORANGE }}")
: getComputedStyle(document.documentElement).getPropertyValue('--edge-or-color').trim();
const currentOpacity = edge.opacity !== undefined ? edge.opacity : 1.0;
const currentWidth =
@@ -2370,7 +2370,7 @@ class NetworkManager {
this.edges.forEach((edge) => {
let edgeColor;
if (edge.dashes || edge.label === "AND") {
edgeColor = "{{ CREWAI_ORANGE }}";
edgeColor = edge.color?.color || "{{ CREWAI_ORANGE }}";
} else {
edgeColor = orEdgeColor;
}

View File

@@ -129,7 +129,7 @@ def _create_edges_from_condition(
edges: list[StructureEdge] = []
if isinstance(condition, str):
if condition in nodes:
if condition in nodes and condition != target:
edges.append(
StructureEdge(
source=condition,
@@ -140,7 +140,7 @@ def _create_edges_from_condition(
)
elif callable(condition) and hasattr(condition, "__name__"):
method_name = condition.__name__
if method_name in nodes:
if method_name in nodes and method_name != target:
edges.append(
StructureEdge(
source=method_name,
@@ -163,7 +163,7 @@ def _create_edges_from_condition(
is_router_path=False,
)
for trigger in triggers
if trigger in nodes
if trigger in nodes and trigger != target
)
else:
for sub_cond in conditions_list:
@@ -196,9 +196,34 @@ def build_flow_structure(flow: Flow[Any]) -> FlowStructure:
node_metadata["type"] = "start"
start_methods.append(method_name)
if (
hasattr(method, "__human_feedback_config__")
and method.__human_feedback_config__
):
config = method.__human_feedback_config__
node_metadata["is_human_feedback"] = True
node_metadata["human_feedback_message"] = config.message
if config.emit:
node_metadata["human_feedback_emit"] = list(config.emit)
if config.llm:
llm_str = (
config.llm
if isinstance(config.llm, str)
else str(type(config.llm).__name__)
)
node_metadata["human_feedback_llm"] = llm_str
if config.default_outcome:
node_metadata["human_feedback_default_outcome"] = config.default_outcome
if hasattr(method, "__is_router__") and method.__is_router__:
node_metadata["is_router"] = True
node_metadata["type"] = "router"
if "is_human_feedback" not in node_metadata:
node_metadata["type"] = "router"
else:
node_metadata["type"] = "human_feedback"
router_methods.append(method_name)
if method_name in flow._router_paths:
@@ -317,7 +342,7 @@ def build_flow_structure(flow: Flow[Any]) -> FlowStructure:
is_router_path=False,
)
for trigger_method in methods
if str(trigger_method) in nodes
if str(trigger_method) in nodes and str(trigger_method) != listener_name
)
elif is_flow_condition_dict(condition_data):
edges.extend(

View File

@@ -81,6 +81,7 @@ class JSExtension(Extension):
CREWAI_ORANGE = "#FF5A50"
HITL_BLUE = "#4A90E2"
DARK_GRAY = "#333333"
WHITE = "#FFFFFF"
GRAY = "#666666"
@@ -225,6 +226,7 @@ def render_interactive(
nodes_list: list[dict[str, Any]] = []
for name, metadata in dag["nodes"].items():
node_type: str = metadata.get("type", "listen")
is_human_feedback: bool = metadata.get("is_human_feedback", False)
color_config: dict[str, Any]
font_color: str
@@ -241,6 +243,17 @@ def render_interactive(
}
font_color = "var(--node-text-color)"
border_width = 3
elif node_type == "human_feedback":
color_config = {
"background": "var(--node-bg-router)",
"border": HITL_BLUE,
"highlight": {
"background": "var(--node-bg-router)",
"border": HITL_BLUE,
},
}
font_color = "var(--node-text-color)"
border_width = 3
elif node_type == "router":
color_config = {
"background": "var(--node-bg-router)",
@@ -266,16 +279,57 @@ def render_interactive(
title_parts: list[str] = []
type_badge_bg: str = (
CREWAI_ORANGE if node_type in ["start", "router"] else DARK_GRAY
)
display_type = node_type
type_badge_bg: str
if node_type == "human_feedback":
type_badge_bg = HITL_BLUE
display_type = "HITL"
elif node_type in ["start", "router"]:
type_badge_bg = CREWAI_ORANGE
else:
type_badge_bg = DARK_GRAY
title_parts.append(f"""
<div style="border-bottom: 1px solid rgba(102,102,102,0.15); padding-bottom: 8px; margin-bottom: 10px;">
<div style="font-size: 13px; font-weight: 700; color: {DARK_GRAY}; margin-bottom: 6px;">{name}</div>
<span style="display: inline-block; background: {type_badge_bg}; color: white; padding: 2px 8px; border-radius: 4px; font-size: 10px; font-weight: 600; text-transform: uppercase; letter-spacing: 0.5px;">{node_type}</span>
<span style="display: inline-block; background: {type_badge_bg}; color: white; padding: 2px 8px; border-radius: 4px; font-size: 10px; font-weight: 600; text-transform: uppercase; letter-spacing: 0.5px;">{display_type}</span>
</div>
""")
if is_human_feedback:
feedback_msg = metadata.get("human_feedback_message", "")
if feedback_msg:
title_parts.append(f"""
<div style="margin-bottom: 8px;">
<div style="font-size: 10px; text-transform: uppercase; color: {GRAY}; letter-spacing: 0.5px; margin-bottom: 4px; font-weight: 600;">👤 Human Feedback</div>
<div style="background: rgba(74,144,226,0.08); padding: 6px 8px; border-radius: 4px; font-size: 11px; color: {DARK_GRAY}; border: 1px solid rgba(74,144,226,0.2); line-height: 1.4;">{feedback_msg}</div>
</div>
""")
if metadata.get("human_feedback_emit"):
emit_options = metadata["human_feedback_emit"]
emit_items = "".join(
[
f'<li style="margin: 3px 0;"><code style="background: rgba(74,144,226,0.08); padding: 2px 6px; border-radius: 3px; font-size: 10px; color: {HITL_BLUE}; border: 1px solid rgba(74,144,226,0.2); font-weight: 600;">{opt}</code></li>'
for opt in emit_options
]
)
title_parts.append(f"""
<div style="margin-bottom: 8px;">
<div style="font-size: 10px; text-transform: uppercase; color: {GRAY}; letter-spacing: 0.5px; margin-bottom: 4px; font-weight: 600;">Outcomes</div>
<ul style="list-style: none; padding: 0; margin: 0;">{emit_items}</ul>
</div>
""")
if metadata.get("human_feedback_llm"):
llm_model = metadata["human_feedback_llm"]
title_parts.append(f"""
<div style="margin-bottom: 8px;">
<div style="font-size: 10px; text-transform: uppercase; color: {GRAY}; letter-spacing: 0.5px; margin-bottom: 3px; font-weight: 600;">LLM</div>
<span style="display: inline-block; background: rgba(102,102,102,0.08); padding: 3px 8px; border-radius: 4px; font-size: 10px; color: {DARK_GRAY}; border: 1px solid rgba(102,102,102,0.12);">{llm_model}</span>
</div>
""")
if metadata.get("condition_type"):
condition = metadata["condition_type"]
if condition == "AND":
@@ -309,7 +363,7 @@ def render_interactive(
</div>
""")
if metadata.get("router_paths"):
if metadata.get("router_paths") and not is_human_feedback:
paths = metadata["router_paths"]
paths_items = "".join(
[
@@ -365,7 +419,11 @@ def render_interactive(
edge_dashes: bool | list[int] = False
if edge["is_router_path"]:
edge_color = CREWAI_ORANGE
source_node = dag["nodes"].get(edge["source"], {})
if source_node.get("is_human_feedback", False):
edge_color = HITL_BLUE
else:
edge_color = CREWAI_ORANGE
edge_dashes = [15, 10]
if "router_path_label" in edge:
edge_label = edge["router_path_label"]
@@ -417,6 +475,7 @@ def render_interactive(
css_content = css_content.replace("'{{ DARK_GRAY }}'", DARK_GRAY)
css_content = css_content.replace("'{{ GRAY }}'", GRAY)
css_content = css_content.replace("'{{ CREWAI_ORANGE }}'", CREWAI_ORANGE)
css_content = css_content.replace("'{{ HITL_BLUE }}'", HITL_BLUE)
css_output_path.write_text(css_content, encoding="utf-8")
@@ -430,6 +489,7 @@ def render_interactive(
js_content = js_content.replace("{{ DARK_GRAY }}", DARK_GRAY)
js_content = js_content.replace("{{ GRAY }}", GRAY)
js_content = js_content.replace("{{ CREWAI_ORANGE }}", CREWAI_ORANGE)
js_content = js_content.replace("{{ HITL_BLUE }}", HITL_BLUE)
js_content = js_content.replace("'{{ nodeData }}'", dag_nodes_json)
js_content = js_content.replace("'{{ dagData }}'", dag_full_json)
js_content = js_content.replace("'{{ nodes_list_json }}'", json.dumps(nodes_list))
@@ -441,6 +501,7 @@ def render_interactive(
html_content = template.render(
CREWAI_ORANGE=CREWAI_ORANGE,
HITL_BLUE=HITL_BLUE,
DARK_GRAY=DARK_GRAY,
WHITE=WHITE,
GRAY=GRAY,

View File

@@ -21,6 +21,11 @@ class NodeMetadata(TypedDict, total=False):
class_signature: str
class_name: str
class_line_number: int
is_human_feedback: bool
human_feedback_message: str
human_feedback_emit: list[str]
human_feedback_llm: str
human_feedback_default_outcome: str
class StructureEdge(TypedDict, total=False):

View File

@@ -8,6 +8,7 @@ from pathlib import Path
import pytest
from crewai.flow.flow import Flow, and_, listen, or_, router, start
from crewai.flow.human_feedback import human_feedback
from crewai.flow.visualization import (
build_flow_structure,
visualize_flow_structure,
@@ -667,4 +668,180 @@ def test_no_warning_for_properly_typed_router(caplog):
# No warnings should be logged
warning_messages = [r.message for r in caplog.records if r.levelno >= logging.WARNING]
assert not any("Could not determine return paths" in msg for msg in warning_messages)
assert not any("Found listeners waiting for triggers" in msg for msg in warning_messages)
assert not any("Found listeners waiting for triggers" in msg for msg in warning_messages)
def test_human_feedback_node_metadata():
"""Test that human feedback nodes have correct metadata."""
from typing import Literal
class HITLFlow(Flow):
"""Flow with human-in-the-loop feedback."""
@start()
@human_feedback(
message="Please review the output:",
emit=["approved", "rejected"],
llm="gpt-4o-mini",
)
def review_content(self) -> Literal["approved", "rejected"]:
return "approved"
@listen("approved")
def on_approved(self):
return "published"
@listen("rejected")
def on_rejected(self):
return "discarded"
flow = HITLFlow()
structure = build_flow_structure(flow)
review_node = structure["nodes"]["review_content"]
assert review_node["is_human_feedback"] is True
assert review_node["type"] == "human_feedback"
assert review_node["human_feedback_message"] == "Please review the output:"
assert review_node["human_feedback_emit"] == ["approved", "rejected"]
assert review_node["human_feedback_llm"] == "gpt-4o-mini"
def test_human_feedback_visualization_includes_hitl_data():
"""Test that visualization includes human feedback data in HTML."""
from typing import Literal
class HITLFlow(Flow):
"""Flow with human-in-the-loop feedback."""
@start()
@human_feedback(
message="Please review the output:",
emit=["approved", "rejected"],
llm="gpt-4o-mini",
)
def review_content(self) -> Literal["approved", "rejected"]:
return "approved"
@listen("approved")
def on_approved(self):
return "published"
flow = HITLFlow()
structure = build_flow_structure(flow)
html_file = visualize_flow_structure(structure, "test_hitl.html", show=False)
html_path = Path(html_file)
js_file = html_path.parent / f"{html_path.stem}_script.js"
js_content = js_file.read_text(encoding="utf-8")
assert "HITL" in js_content
assert "Please review the output:" in js_content
assert "approved" in js_content
assert "rejected" in js_content
assert "#4A90E2" in js_content
def test_human_feedback_without_emit_metadata():
"""Test that human feedback without emit has correct metadata."""
class HITLSimpleFlow(Flow):
"""Flow with simple human feedback (no routing)."""
@start()
@human_feedback(message="Please provide feedback:")
def review_step(self):
return "content"
@listen(review_step)
def next_step(self):
return "done"
flow = HITLSimpleFlow()
structure = build_flow_structure(flow)
review_node = structure["nodes"]["review_step"]
assert review_node["is_human_feedback"] is True
assert "is_router" not in review_node or review_node["is_router"] is False
assert review_node["type"] == "start"
assert review_node["human_feedback_message"] == "Please provide feedback:"
def test_human_feedback_with_default_outcome():
"""Test that human feedback with default outcome includes it in metadata."""
from typing import Literal
class HITLDefaultFlow(Flow):
"""Flow with human feedback that has a default outcome."""
@start()
@human_feedback(
message="Review this:",
emit=["approved", "needs_work"],
llm="gpt-4o-mini",
default_outcome="needs_work",
)
def review(self) -> Literal["approved", "needs_work"]:
return "approved"
@listen("approved")
def on_approved(self):
return "published"
@listen("needs_work")
def on_needs_work(self):
return "revised"
flow = HITLDefaultFlow()
structure = build_flow_structure(flow)
review_node = structure["nodes"]["review"]
assert review_node["is_human_feedback"] is True
assert review_node["human_feedback_default_outcome"] == "needs_work"
def test_mixed_router_and_human_feedback():
"""Test flow with both regular routers and human feedback routers."""
from typing import Literal
class MixedFlow(Flow):
"""Flow with both regular routers and HITL."""
@start()
def init(self):
return "initialized"
@router(init)
def auto_decision(self) -> Literal["path_a", "path_b"]:
return "path_a"
@listen("path_a")
@human_feedback(
message="Review this step:",
emit=["continue", "stop"],
llm="gpt-4o-mini",
)
def human_review(self) -> Literal["continue", "stop"]:
return "continue"
@listen("continue")
def proceed(self):
return "done"
@listen("stop")
def halt(self):
return "halted"
flow = MixedFlow()
structure = build_flow_structure(flow)
auto_node = structure["nodes"]["auto_decision"]
assert auto_node["type"] == "router"
assert auto_node["is_router"] is True
assert "is_human_feedback" not in auto_node or auto_node["is_human_feedback"] is False
human_node = structure["nodes"]["human_review"]
assert human_node["type"] == "human_feedback"
assert human_node["is_router"] is True
assert human_node["is_human_feedback"] is True
assert human_node["human_feedback_message"] == "Review this step:"