Merge branch 'main' into feat/agents-md-v11

This commit is contained in:
João Moura authored and GitHub committed 2026-10-20 10:53:14 -07:00
commit 4bb2b466a6
8 files changed
+415 -24

No files matched your search

+2 -2
View File
@@ -7,7 +7,7 @@ mode: "wide"
# Arize Phoenix Integration
This guide demonstrates how to integrate **Arize Phoenix** with **CrewAI** using OpenTelemetry via the [OpenInference](https://github.com/openinference/openinference) SDK. By the end of this guide, you will be able to trace your CrewAI agents and debug agent behavior.
This guide demonstrates how to integrate **Arize Phoenix** with **CrewAI** using OpenTelemetry via the [OpenInference](https://github.com/Arize-ai/openinference) SDK. By the end of this guide, you will be able to trace your CrewAI agents and debug agent behavior.
> **What is Arize Phoenix?** [Arize Phoenix](https://arize.com/phoenix/) is the open-source observability and evaluation option from [Arize AI](https://arize.com/?utm_source=crewai-docs&utm_medium=partner&utm_campaign=partner-docs&utm_content=observability-arize-phoenix). Use Phoenix when you want to run locally or self-host. Use [Arize AX](https://arize.com/products/ax/) for a managed cloud or enterprise self-hosted platform for production AI systems.
@@ -152,4 +152,4 @@ Open your Phoenix project and navigate to the project you specified in the `proj
- [Arize LLM evaluation guide](https://arize.com/resources/llm-evaluation/) - Methods and metrics for evaluating LLM applications.
- [CrewAI Documentation](https://docs.crewai.com/) - Overview of the CrewAI framework.
- [OpenTelemetry Docs](https://opentelemetry.io/docs/) - OpenTelemetry guide
- [OpenInference GitHub](https://github.com/openinference/openinference) - Source code for OpenInference SDK.
- [OpenInference GitHub](https://github.com/Arize-ai/openinference) - Source code for OpenInference SDK.
+1 -1
View File
@@ -234,4 +234,4 @@ See the [Braintrust Eval SDK guide](https://www.braintrust.dev/docs/start/eval-s
- [Braintrust Eval SDK](https://www.braintrust.dev/docs/start/eval-sdk) - Run experiments via the SDK
- [CrewAI Documentation](https://docs.crewai.com/) - Overview of the CrewAI framework
- [OpenTelemetry Docs](https://opentelemetry.io/docs/) - OpenTelemetry guide
- [Braintrust GitHub](https://github.com/braintrustdata/braintrust) - Source code for Braintrust SDK
- [Braintrust GitHub](https://github.com/braintrustdata/braintrust-sdk-python) - Source code for Braintrust SDK
@@ -7,7 +7,7 @@ mode: "wide"
# Arize Phoenix Integration
This guide demonstrates how to integrate **Arize Phoenix** with **CrewAI** using OpenTelemetry via the [OpenInference](https://github.com/openinference/openinference) SDK. By the end of this guide, you will be able to trace your CrewAI agents and debug agent behavior.
This guide demonstrates how to integrate **Arize Phoenix** with **CrewAI** using OpenTelemetry via the [OpenInference](https://github.com/Arize-ai/openinference) SDK. By the end of this guide, you will be able to trace your CrewAI agents and debug agent behavior.
> **What is Arize Phoenix?** [Arize Phoenix](https://arize.com/phoenix/) is the open-source observability and evaluation option from [Arize AI](https://arize.com/?utm_source=crewai-docs&utm_medium=partner&utm_campaign=partner-docs&utm_content=observability-arize-phoenix). Use Phoenix when you want to run locally or self-host. Use [Arize AX](https://arize.com/products/ax/) for a managed cloud or enterprise self-hosted platform for production AI systems.
@@ -152,4 +152,4 @@ Open your Phoenix project and navigate to the project you specified in the `proj
- [Arize LLM evaluation guide](https://arize.com/resources/llm-evaluation/) - Methods and metrics for evaluating LLM applications.
- [CrewAI Documentation](https://docs.crewai.com/) - Overview of the CrewAI framework.
- [OpenTelemetry Docs](https://opentelemetry.io/docs/) - OpenTelemetry guide
- [OpenInference GitHub](https://github.com/openinference/openinference) - Source code for OpenInference SDK.
- [OpenInference GitHub](https://github.com/Arize-ai/openinference) - Source code for OpenInference SDK.
@@ -234,4 +234,4 @@ See the [Braintrust Eval SDK guide](https://www.braintrust.dev/docs/start/eval-s
- [Braintrust Eval SDK](https://www.braintrust.dev/docs/start/eval-sdk) - Run experiments via the SDK
- [CrewAI Documentation](https://docs.crewai.com/) - Overview of the CrewAI framework
- [OpenTelemetry Docs](https://opentelemetry.io/docs/) - OpenTelemetry guide
- [Braintrust GitHub](https://github.com/braintrustdata/braintrust) - Source code for Braintrust SDK
- [Braintrust GitHub](https://github.com/braintrustdata/braintrust-sdk-python) - Source code for Braintrust SDK
@@ -43,7 +43,7 @@ import re
import stat
import sys
import time
from typing import Any, Literal, NoReturn, cast
from typing import Any, BinaryIO, Literal, NoReturn, cast
from urllib.parse import urlparse
import uuid
import webbrowser
@@ -161,6 +161,13 @@ NOT_TRACED = (
f"({LAST_RUN_FILE})."
)
# With no terminal, what the crew prints while it runs (its panels, the final
# answer, every tool call) goes to this file in the project's `.crewai/`, not to
# the coding agent's screen: in one measured run it was 341 of 378 lines, all
# before the verdict, and an agent pays for every line or has the end cut off.
RUN_LOG_FILE = "last_run.log"
RUN_LOG_TAIL_LINES = 40
# Why `crewai eval` ran the crew before evaluating, a word from a closed list:
# nothing traced yet, the project changed since its last traced run, or the run
# app's Evaluate pressed on a run that was not traced.
@@ -1002,11 +1009,23 @@ def _run_the_crew_to_evaluate(
# The handshake (`crewai_core.eval_handshake`) carries what this command
# decided before the project's `.env` was loaded: the AMP origins its saved
# login may go to, and whether it just turned tracing on.
# Whoever reads the output decides, as for the brief: stdout, not stdin.
run_log = _run_log_path() if nobody_watching() else None
if run_log is not None:
console.print(
f"The crew's output goes to {_shown_path(run_log)} while it runs.",
style="dim",
)
began = datetime.now(timezone.utc)
with evaluating_after_run(
trusted_amp_origins=trusted or (), tracing_turned_on=turned_on_now
) as watched:
started = time.monotonic()
with (
evaluating_after_run(
trusted_amp_origins=trusted or (), tracing_turned_on=turned_on_now
) as watched,
_run_output_to(run_log) as run_log,
):
run_crew(**run_args)
seconds = time.monotonic() - started
# The app says what it did with the run it watched, and its word settles
# this: it evaluated it (nothing left to do), or it had nothing to evaluate
@@ -1018,7 +1037,8 @@ def _run_the_crew_to_evaluate(
remember_project_state(str(marker["execution_id"]))
return None
record_stop("untraced_after_run")
console.print(NOT_TRACED, style="bold red")
_say_what_the_run_left(run_log, seconds, traced=False)
console.print(_not_traced(run_log), style="bold red")
raise SystemExit(1)
# The project's record, only when it was written after this run began: a
@@ -1029,13 +1049,157 @@ def _run_the_crew_to_evaluate(
record.get("execution_id") if _recorded_since(record, began) else None
)
if traced:
_say_what_the_run_left(run_log, seconds, traced=True)
return str(traced)
record_stop("untraced_after_run")
console.print(NOT_TRACED, style="bold red")
_say_what_the_run_left(run_log, seconds, traced=False)
console.print(_not_traced(run_log), style="bold red")
raise SystemExit(1)
def _not_traced(run_log: Path | None) -> str:
"""NOT_TRACED, pointing at where the run's output went."""
if run_log is None:
return NOT_TRACED
return NOT_TRACED.replace("(its output is above)", "(its last lines are above)")
def _run_log_path() -> Path | None:
"""Where a run with no terminal sends its output, or None when the
project's `.crewai/` cannot be written (the output then stays on screen)."""
from crewai.telemetry.tracing.last_run import LAST_RUN_DIR, project_dir
try:
path = project_dir() / LAST_RUN_DIR / RUN_LOG_FILE
path.parent.mkdir(parents=True, exist_ok=True)
except OSError:
return None
return path
@contextlib.contextmanager
def _run_output_to(path: Path | None) -> Iterator[Path | None]:
"""Send everything printed inside the block to PATH, and yield it; with no
PATH, yield None and change nothing.
It moves this process's own stdout and stderr (file descriptors 1 and 2),
which every child the run starts inherits, so each way a project runs —
`uv run`, the JSON crew runner, a declarative flow — is covered without
touching any of them. Anything that keeps them from moving leaves the
output where it was: a quiet log is a nicety, never a failed run. Both are
always put back after, one by one, whatever failed: a stream left on the
log would swallow the verdict, and the run's own error is never hidden by
a failure to put them back."""
log = _open_run_log(path) if path is not None else None
if log is None:
yield None
return
with log:
_flush_std_streams()
saved: dict[int, int] = {}
moved: list[int] = []
try:
for fd in (1, 2):
saved[fd] = os.dup(fd)
for fd in (1, 2):
os.dup2(log.fileno(), fd)
moved.append(fd)
except OSError:
_put_back(saved, moved)
yield None
return
_point_windows_std_handles_at(1, 2)
try:
yield path
finally:
_flush_std_streams()
_put_back(saved, moved)
_point_windows_std_handles_at(1, 2)
def _open_run_log(path: Path) -> BinaryIO | None:
"""The run log, opened for writing owner-only and never through a link:
it holds final answers, tool output and tracebacks that can carry a
credential. A log an earlier run left is emptied and its mode tightened.
None when it cannot be opened that way."""
flags = os.O_WRONLY | os.O_CREAT | os.O_TRUNC | getattr(os, "O_NOFOLLOW", 0)
try:
if path.is_symlink():
return None # a link someone planted does not choose where it goes
fd = os.open(path, flags, 0o600)
except OSError:
return None
with contextlib.suppress(OSError, AttributeError):
os.fchmod(fd, 0o600)
return os.fdopen(fd, "wb")
def _flush_std_streams() -> None:
for stream in (sys.stdout, sys.stderr):
with contextlib.suppress(Exception):
stream.flush()
def _put_back(saved: dict[int, int], moved: list[int]) -> None:
"""Point every MOVED descriptor at its SAVED original again, and close
every saved copy: each on its own, so one failure stops none of the rest."""
for fd in moved:
with contextlib.suppress(OSError):
os.dup2(saved[fd], fd)
for copy in saved.values():
_close_quietly(copy)
def _close_quietly(fd: int) -> None:
with contextlib.suppress(OSError):
os.close(fd)
def _point_windows_std_handles_at(out_fd: int, err_fd: int) -> None:
"""On Windows a child takes the process's standard handles, not its file
descriptors: point them where descriptors 1 and 2 now go."""
if os.name != "nt":
return
with contextlib.suppress(Exception):
import ctypes
import msvcrt
kernel32 = ctypes.windll.kernel32 # type: ignore[attr-defined]
handle_of = msvcrt.get_osfhandle # type: ignore[attr-defined]
kernel32.SetStdHandle(-11, handle_of(out_fd)) # STD_OUTPUT_HANDLE
kernel32.SetStdHandle(-12, handle_of(err_fd)) # STD_ERROR_HANDLE
def _say_what_the_run_left(log: Path | None, seconds: float, traced: bool) -> None:
"""After a quiet run: where its output is, and — when it left nothing to
evaluate — its last lines, which usually hold the error."""
if log is None:
return
where = _shown_path(log)
if traced:
console.print(
f"The crew ran in {seconds:.0f}s; its output is in {where}.", style="dim"
)
return
with contextlib.suppress(OSError):
lines = log.read_text(encoding="utf-8", errors="replace").splitlines()
tail = lines[-RUN_LOG_TAIL_LINES:]
if tail:
console.print(
f"The crew's output is in {where}; its last {len(tail)} lines:",
style="bold",
)
console.print(Text("\n".join(tail)))
def _shown_path(path: Path) -> str:
try:
return str(path.relative_to(Path.cwd()))
except ValueError:
return str(path)
def _recorded_since(record: dict[str, Any], began: datetime) -> bool:
"""Was RECORD written after BEGAN? A record without a readable, zoned stamp
is not assumed to be this run's."""
@@ -3458,3 +3458,197 @@ def test_a_project_that_turned_tracing_off_is_not_asked_to_trace_toward_an_untru
assert exited.value.code == 1
assert usage == ["cli_usage:eval_stopped:untrusted_amp"]
assert "CREWAI_TRACING_ENABLED=false" in (directory / ".env").read_text()
# ── with no terminal, the run's own output goes to a log, not the agent ────
def _a_tracing_project(directory: Path) -> None:
(directory / "pyproject.toml").write_text("[project]\nname = 'demo'\n")
(directory / ".env").write_text("CREWAI_TRACING_ENABLED=true\n")
def test_with_no_terminal_the_runs_own_output_goes_to_a_log_not_the_agent(
project, monkeypatch, capfd
):
"""A coding agent reads every line `crewai eval` prints, and pays for it:
the crew's panels, its final answer and its tool calls go to
`.crewai/last_run.log`, and the agent gets one line saying so, then the
verdict."""
directory, _ = project
_a_tracing_project(directory)
monkeypatch.setattr(eval_module.sys.stdin, "isatty", lambda: False)
monkeypatch.setattr(eval_module, "nobody_watching", lambda: True)
def fake_run_crew() -> None:
# what the run's child processes print: straight to the descriptors
os.write(1, b"Crew Execution Started panel\n")
os.write(2, b"a warning on stderr\n")
record_last_run(directory, "the-new-run", recorded_at=_now())
monkeypatch.setattr(run_crew_module, "run_crew", fake_run_crew)
install(monkeypatch, FakeAMP(statuses=[done()]))
eval_module.eval_crew()
out, err = capfd.readouterr()
log = (directory / ".crewai" / "last_run.log").read_text()
assert "Crew Execution Started panel" in log and "a warning on stderr" in log
assert "Crew Execution Started panel" not in out + err
assert "a warning on stderr" not in out + err
said = " ".join(out.split())
assert "The crew's output goes to .crewai/last_run.log while it runs." in said
assert "its output is in .crewai/last_run.log." in said
assert "Goal gate: PASSED" in said
def test_with_no_terminal_a_run_that_left_no_trace_shows_its_last_lines(
project, monkeypatch, capfd, usage
):
"""The error a failed run printed is what the agent needs to fix it: its
last lines are shown, and the sentence says where the rest is."""
directory, _ = project
_a_tracing_project(directory)
monkeypatch.setattr(eval_module.sys.stdin, "isatty", lambda: False)
monkeypatch.setattr(eval_module, "nobody_watching", lambda: True)
def fake_run_crew() -> None:
os.write(1, b"".join(f"line {n}\n".encode() for n in range(100)))
os.write(2, b"KeyError: 'topic'\n") # and no trace recorded
monkeypatch.setattr(run_crew_module, "run_crew", fake_run_crew)
amp = install(monkeypatch, FakeAMP())
with pytest.raises(SystemExit) as exited:
eval_module.eval_crew()
assert exited.value.code == 1
said = " ".join(capfd.readouterr().out.split())
assert "its last 40 lines:" in said
assert "KeyError: 'topic'" in said and "line 99" in said
assert "line 50 " not in said # only the tail
assert "(its last lines are above)" in said
assert amp.calls == [] and usage[-1] == "cli_usage:eval_stopped:untraced_after_run"
def test_at_a_terminal_the_run_prints_where_it_always_did(project, monkeypatch, capfd):
directory, _ = project
_a_tracing_project(directory)
monkeypatch.setattr(eval_module.sys.stdin, "isatty", lambda: True)
monkeypatch.setattr(eval_module, "nobody_watching", lambda: False) # a person reads stdout
def fake_run_crew() -> None:
os.write(1, b"Crew Execution Started panel\n")
record_last_run(directory, "the-new-run", recorded_at=_now())
monkeypatch.setattr(run_crew_module, "run_crew", fake_run_crew)
install(monkeypatch, FakeAMP(statuses=[done()]))
eval_module.eval_crew()
out, _ = capfd.readouterr()
assert "Crew Execution Started panel" in out
assert "last_run.log" not in out
assert not (directory / ".crewai" / "last_run.log").exists()
def test_what_decides_is_whether_anyone_reads_stdout_not_stdin(project, monkeypatch, capfd):
"""A coding agent can hand the command a terminal for stdin and still read
a captured stdout: the transcript stays out of what it reads."""
directory, _ = project
_a_tracing_project(directory)
monkeypatch.setattr(eval_module.sys.stdin, "isatty", lambda: True)
monkeypatch.setattr(eval_module, "nobody_watching", lambda: True)
def fake_run_crew() -> None:
os.write(1, b"Crew Execution Started panel\n")
record_last_run(directory, "the-new-run", recorded_at=_now())
monkeypatch.setattr(run_crew_module, "run_crew", fake_run_crew)
install(monkeypatch, FakeAMP(statuses=[done()]))
eval_module.eval_crew()
assert "Crew Execution Started panel" not in capfd.readouterr().out
assert "Crew Execution Started panel" in (directory / ".crewai" / "last_run.log").read_text()
@pytest.mark.skipif(os.name == "nt", reason="POSIX file modes")
def test_the_run_log_is_the_owners_alone_even_one_left_by_an_earlier_run(tmp_path):
log = tmp_path / "last_run.log"
log.write_text("an earlier run\n")
log.chmod(0o644)
with eval_module._run_output_to(log) as where:
assert where == log
os.write(1, b"this run\n")
assert log.read_text() == "this run\n" # emptied, then written
assert log.stat().st_mode & 0o777 == 0o600
@pytest.mark.skipif(os.name == "nt", reason="symbolic links need privileges on Windows")
def test_a_link_at_the_logs_place_leaves_the_output_on_screen(tmp_path, capfd):
elsewhere = tmp_path / "elsewhere.txt"
elsewhere.write_text("")
log = tmp_path / "last_run.log"
log.symlink_to(elsewhere)
with eval_module._run_output_to(log) as where:
assert where is None
os.write(1, b"on screen\n")
assert "on screen" in capfd.readouterr().out
assert elsewhere.read_text() == ""
def test_a_move_that_fails_halfway_puts_back_what_it_moved(tmp_path, monkeypatch, capfd):
"""Descriptor 1 moved, 2 refused: 1 comes back before the block runs, so
the verdict printed after it still reaches the screen."""
real_dup2 = os.dup2
targets: list[int] = []
def dup2(src: int, dst: int, inheritable: bool = True) -> int:
targets.append(dst)
if dst == 2 and len(targets) == 2:
raise OSError("no room for stderr")
return real_dup2(src, dst, inheritable)
with monkeypatch.context() as patched:
patched.setattr(eval_module.os, "dup2", dup2)
with eval_module._run_output_to(tmp_path / "last_run.log") as where:
assert where is None
os.write(1, b"the verdict\n")
assert "the verdict" in capfd.readouterr().out
assert (tmp_path / "last_run.log").read_text() == ""
def test_putting_back_goes_on_past_a_failure_and_keeps_the_runs_own_error(tmp_path, monkeypatch, capfd):
"""Putting stdout back fails; stderr is put back anyway, every saved copy
is closed, and the error the run raised is the one that surfaces."""
real_dup2 = os.dup2
stdout_copy = os.dup(1)
calls: list[int] = []
def dup2(src: int, dst: int, inheritable: bool = True) -> int:
calls.append(dst)
if len(calls) == 3: # the first put-back: stdout
raise OSError("cannot put stdout back")
return real_dup2(src, dst, inheritable)
def the_run() -> None:
raise ValueError("the run's own error")
try:
with monkeypatch.context() as patched:
patched.setattr(eval_module.os, "dup2", dup2)
with pytest.raises(ValueError, match="the run's own error"):
with eval_module._run_output_to(tmp_path / "last_run.log"):
the_run()
assert calls == [1, 2, 1, 2] # stderr was put back after stdout failed
os.write(2, b"stderr is back\n")
assert "stderr is back" in capfd.readouterr().err
finally:
real_dup2(stdout_copy, 1) # the test puts stdout back itself
os.close(stdout_copy)
@@ -625,7 +625,8 @@ class AnthropicCompletion(BaseLLM):
system_message: Extracted system message
tools: Tool definitions
available_functions: Available functions for tool calling. When provided
with a single tool, tool_choice is automatically set to force tool use.
with a single tool, tool_choice is "auto" so the model may call
that tool or answer in text.
Returns:
Parameters dictionary for Anthropic API
@@ -652,8 +653,7 @@ class AnthropicCompletion(BaseLLM):
# When tool_search is enabled and there are 2+ regular tools,
# inject the search tool and mark regular tools with defer_loading.
# With only 1 tool there's nothing to search — skip tool search
# entirely so the normal forced tool_choice optimisation still works.
# With only 1 tool there's nothing to search, so skip tool search.
regular_tools = [
t
for t in converted_tools
@@ -664,10 +664,11 @@ class AnthropicCompletion(BaseLLM):
params["tools"] = converted_tools
# available_functions means the tool can be executed, not that the
# model must call it. Named tool_choice is also rejected by newer
# Claude models (Sonnet 5.5, Opus 5.5, Fable 5.1, Mythos 5.1).
if available_functions and len(regular_tools) == 1:
tool_name = regular_tools[0].get("name")
if tool_name and tool_name in available_functions:
params["tool_choice"] = {"type": "tool", "name": tool_name}
params["tool_choice"] = {"type": "auto"}
if self.thinking:
if isinstance(self.thinking, AnthropicThinkingConfig):
@@ -1470,9 +1470,9 @@ def test_tool_search_passthrough_preserves_tool_search_type():
assert "input_schema" in converted[1]
def test_tool_search_single_tool_skips_search_and_forces_choice():
"""With only 1 tool, tool_search is skipped (nothing to search) and the
normal forced tool_choice optimisation still applies."""
def test_tool_search_single_tool_skips_search_and_uses_auto_choice():
"""With only 1 tool, tool_search is skipped (nothing to search) and
tool_choice stays auto."""
llm = LLM(model="anthropic/claude-sonnet-4-5", tool_search=True)
crewai_tools = [
@@ -1500,9 +1500,8 @@ def test_tool_search_single_tool_skips_search_and_forces_choice():
available_functions={"test_tool": lambda q: "result"},
)
# Single tool — tool_search skipped, tool_choice forced as normal
assert "tool_choice" in params
assert params["tool_choice"]["name"] == "test_tool"
# Single tool — tool_search skipped, model chooses whether to call it
assert params["tool_choice"] == {"type": "auto"}
# No tool search tool should be injected
tool_types = [t.get("type", "") for t in params["tools"]]
@@ -1513,6 +1512,39 @@ def test_tool_search_single_tool_skips_search_and_forces_choice():
assert "defer_loading" not in params["tools"][0]
def test_single_tool_with_available_functions_uses_auto_choice():
"""One executable tool is offered, not forced.
Newer Claude models reject tool_choice {"type": "tool"}. Auto lets the
model call the tool or answer in text.
"""
llm = LLM(model="anthropic/claude-sonnet-5-5")
tool = {
"type": "function",
"function": {
"name": "get_weather",
"description": "Get the weather for a city.",
"parameters": {
"type": "object",
"properties": {"city": {"type": "string"}},
"required": ["city"],
},
},
}
formatted_messages, system_message = llm._format_messages_for_anthropic(
[{"role": "user", "content": "What's the weather in Paris?"}]
)
params = llm._prepare_completion_params(
formatted_messages,
system_message,
[tool],
available_functions={"get_weather": lambda city: "sunny"},
)
assert params["tool_choice"] == {"type": "auto"}
assert [t["name"] for t in params["tools"]] == ["get_weather"]
def test_tool_search_via_llm_class():
"""Verify tool_search param passes through LLM class correctly."""
from crewai.llms.providers.anthropic.completion import (