mirror of
https://github.com/crewAIInc/crewAI.git
synced 2026-10-22 01:43:13 +02:00
Merge branch 'main' into feat/agents-md-v11
This commit is contained in:
8 files changed
+415
-24
No files matched your search
@@ -7,7 +7,7 @@ mode: "wide"
|
||||
|
||||
# Arize Phoenix Integration
|
||||
|
||||
This guide demonstrates how to integrate **Arize Phoenix** with **CrewAI** using OpenTelemetry via the [OpenInference](https://github.com/openinference/openinference) SDK. By the end of this guide, you will be able to trace your CrewAI agents and debug agent behavior.
|
||||
This guide demonstrates how to integrate **Arize Phoenix** with **CrewAI** using OpenTelemetry via the [OpenInference](https://github.com/Arize-ai/openinference) SDK. By the end of this guide, you will be able to trace your CrewAI agents and debug agent behavior.
|
||||
|
||||
> **What is Arize Phoenix?** [Arize Phoenix](https://arize.com/phoenix/) is the open-source observability and evaluation option from [Arize AI](https://arize.com/?utm_source=crewai-docs&utm_medium=partner&utm_campaign=partner-docs&utm_content=observability-arize-phoenix). Use Phoenix when you want to run locally or self-host. Use [Arize AX](https://arize.com/products/ax/) for a managed cloud or enterprise self-hosted platform for production AI systems.
|
||||
|
||||
@@ -152,4 +152,4 @@ Open your Phoenix project and navigate to the project you specified in the `proj
|
||||
- [Arize LLM evaluation guide](https://arize.com/resources/llm-evaluation/) - Methods and metrics for evaluating LLM applications.
|
||||
- [CrewAI Documentation](https://docs.crewai.com/) - Overview of the CrewAI framework.
|
||||
- [OpenTelemetry Docs](https://opentelemetry.io/docs/) - OpenTelemetry guide
|
||||
- [OpenInference GitHub](https://github.com/openinference/openinference) - Source code for OpenInference SDK.
|
||||
- [OpenInference GitHub](https://github.com/Arize-ai/openinference) - Source code for OpenInference SDK.
|
||||
@@ -234,4 +234,4 @@ See the [Braintrust Eval SDK guide](https://www.braintrust.dev/docs/start/eval-s
|
||||
- [Braintrust Eval SDK](https://www.braintrust.dev/docs/start/eval-sdk) - Run experiments via the SDK
|
||||
- [CrewAI Documentation](https://docs.crewai.com/) - Overview of the CrewAI framework
|
||||
- [OpenTelemetry Docs](https://opentelemetry.io/docs/) - OpenTelemetry guide
|
||||
- [Braintrust GitHub](https://github.com/braintrustdata/braintrust) - Source code for Braintrust SDK
|
||||
- [Braintrust GitHub](https://github.com/braintrustdata/braintrust-sdk-python) - Source code for Braintrust SDK
|
||||
@@ -7,7 +7,7 @@ mode: "wide"
|
||||
|
||||
# Arize Phoenix Integration
|
||||
|
||||
This guide demonstrates how to integrate **Arize Phoenix** with **CrewAI** using OpenTelemetry via the [OpenInference](https://github.com/openinference/openinference) SDK. By the end of this guide, you will be able to trace your CrewAI agents and debug agent behavior.
|
||||
This guide demonstrates how to integrate **Arize Phoenix** with **CrewAI** using OpenTelemetry via the [OpenInference](https://github.com/Arize-ai/openinference) SDK. By the end of this guide, you will be able to trace your CrewAI agents and debug agent behavior.
|
||||
|
||||
> **What is Arize Phoenix?** [Arize Phoenix](https://arize.com/phoenix/) is the open-source observability and evaluation option from [Arize AI](https://arize.com/?utm_source=crewai-docs&utm_medium=partner&utm_campaign=partner-docs&utm_content=observability-arize-phoenix). Use Phoenix when you want to run locally or self-host. Use [Arize AX](https://arize.com/products/ax/) for a managed cloud or enterprise self-hosted platform for production AI systems.
|
||||
|
||||
@@ -152,4 +152,4 @@ Open your Phoenix project and navigate to the project you specified in the `proj
|
||||
- [Arize LLM evaluation guide](https://arize.com/resources/llm-evaluation/) - Methods and metrics for evaluating LLM applications.
|
||||
- [CrewAI Documentation](https://docs.crewai.com/) - Overview of the CrewAI framework.
|
||||
- [OpenTelemetry Docs](https://opentelemetry.io/docs/) - OpenTelemetry guide
|
||||
- [OpenInference GitHub](https://github.com/openinference/openinference) - Source code for OpenInference SDK.
|
||||
- [OpenInference GitHub](https://github.com/Arize-ai/openinference) - Source code for OpenInference SDK.
|
||||
@@ -234,4 +234,4 @@ See the [Braintrust Eval SDK guide](https://www.braintrust.dev/docs/start/eval-s
|
||||
- [Braintrust Eval SDK](https://www.braintrust.dev/docs/start/eval-sdk) - Run experiments via the SDK
|
||||
- [CrewAI Documentation](https://docs.crewai.com/) - Overview of the CrewAI framework
|
||||
- [OpenTelemetry Docs](https://opentelemetry.io/docs/) - OpenTelemetry guide
|
||||
- [Braintrust GitHub](https://github.com/braintrustdata/braintrust) - Source code for Braintrust SDK
|
||||
- [Braintrust GitHub](https://github.com/braintrustdata/braintrust-sdk-python) - Source code for Braintrust SDK
|
||||
@@ -43,7 +43,7 @@ import re
|
||||
import stat
|
||||
import sys
|
||||
import time
|
||||
from typing import Any, Literal, NoReturn, cast
|
||||
from typing import Any, BinaryIO, Literal, NoReturn, cast
|
||||
from urllib.parse import urlparse
|
||||
import uuid
|
||||
import webbrowser
|
||||
@@ -161,6 +161,13 @@ NOT_TRACED = (
|
||||
f"({LAST_RUN_FILE})."
|
||||
)
|
||||
|
||||
# With no terminal, what the crew prints while it runs (its panels, the final
|
||||
# answer, every tool call) goes to this file in the project's `.crewai/`, not to
|
||||
# the coding agent's screen: in one measured run it was 341 of 378 lines, all
|
||||
# before the verdict, and an agent pays for every line or has the end cut off.
|
||||
RUN_LOG_FILE = "last_run.log"
|
||||
RUN_LOG_TAIL_LINES = 40
|
||||
|
||||
# Why `crewai eval` ran the crew before evaluating, a word from a closed list:
|
||||
# nothing traced yet, the project changed since its last traced run, or the run
|
||||
# app's Evaluate pressed on a run that was not traced.
|
||||
@@ -1002,11 +1009,23 @@ def _run_the_crew_to_evaluate(
|
||||
# The handshake (`crewai_core.eval_handshake`) carries what this command
|
||||
# decided before the project's `.env` was loaded: the AMP origins its saved
|
||||
# login may go to, and whether it just turned tracing on.
|
||||
# Whoever reads the output decides, as for the brief: stdout, not stdin.
|
||||
run_log = _run_log_path() if nobody_watching() else None
|
||||
if run_log is not None:
|
||||
console.print(
|
||||
f"The crew's output goes to {_shown_path(run_log)} while it runs.",
|
||||
style="dim",
|
||||
)
|
||||
began = datetime.now(timezone.utc)
|
||||
with evaluating_after_run(
|
||||
trusted_amp_origins=trusted or (), tracing_turned_on=turned_on_now
|
||||
) as watched:
|
||||
started = time.monotonic()
|
||||
with (
|
||||
evaluating_after_run(
|
||||
trusted_amp_origins=trusted or (), tracing_turned_on=turned_on_now
|
||||
) as watched,
|
||||
_run_output_to(run_log) as run_log,
|
||||
):
|
||||
run_crew(**run_args)
|
||||
seconds = time.monotonic() - started
|
||||
|
||||
# The app says what it did with the run it watched, and its word settles
|
||||
# this: it evaluated it (nothing left to do), or it had nothing to evaluate
|
||||
@@ -1018,7 +1037,8 @@ def _run_the_crew_to_evaluate(
|
||||
remember_project_state(str(marker["execution_id"]))
|
||||
return None
|
||||
record_stop("untraced_after_run")
|
||||
console.print(NOT_TRACED, style="bold red")
|
||||
_say_what_the_run_left(run_log, seconds, traced=False)
|
||||
console.print(_not_traced(run_log), style="bold red")
|
||||
raise SystemExit(1)
|
||||
|
||||
# The project's record, only when it was written after this run began: a
|
||||
@@ -1029,13 +1049,157 @@ def _run_the_crew_to_evaluate(
|
||||
record.get("execution_id") if _recorded_since(record, began) else None
|
||||
)
|
||||
if traced:
|
||||
_say_what_the_run_left(run_log, seconds, traced=True)
|
||||
return str(traced)
|
||||
|
||||
record_stop("untraced_after_run")
|
||||
console.print(NOT_TRACED, style="bold red")
|
||||
_say_what_the_run_left(run_log, seconds, traced=False)
|
||||
console.print(_not_traced(run_log), style="bold red")
|
||||
raise SystemExit(1)
|
||||
|
||||
|
||||
def _not_traced(run_log: Path | None) -> str:
|
||||
"""NOT_TRACED, pointing at where the run's output went."""
|
||||
if run_log is None:
|
||||
return NOT_TRACED
|
||||
return NOT_TRACED.replace("(its output is above)", "(its last lines are above)")
|
||||
|
||||
|
||||
def _run_log_path() -> Path | None:
|
||||
"""Where a run with no terminal sends its output, or None when the
|
||||
project's `.crewai/` cannot be written (the output then stays on screen)."""
|
||||
from crewai.telemetry.tracing.last_run import LAST_RUN_DIR, project_dir
|
||||
|
||||
try:
|
||||
path = project_dir() / LAST_RUN_DIR / RUN_LOG_FILE
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
except OSError:
|
||||
return None
|
||||
return path
|
||||
|
||||
|
||||
@contextlib.contextmanager
|
||||
def _run_output_to(path: Path | None) -> Iterator[Path | None]:
|
||||
"""Send everything printed inside the block to PATH, and yield it; with no
|
||||
PATH, yield None and change nothing.
|
||||
|
||||
It moves this process's own stdout and stderr (file descriptors 1 and 2),
|
||||
which every child the run starts inherits, so each way a project runs —
|
||||
`uv run`, the JSON crew runner, a declarative flow — is covered without
|
||||
touching any of them. Anything that keeps them from moving leaves the
|
||||
output where it was: a quiet log is a nicety, never a failed run. Both are
|
||||
always put back after, one by one, whatever failed: a stream left on the
|
||||
log would swallow the verdict, and the run's own error is never hidden by
|
||||
a failure to put them back."""
|
||||
log = _open_run_log(path) if path is not None else None
|
||||
if log is None:
|
||||
yield None
|
||||
return
|
||||
with log:
|
||||
_flush_std_streams()
|
||||
saved: dict[int, int] = {}
|
||||
moved: list[int] = []
|
||||
try:
|
||||
for fd in (1, 2):
|
||||
saved[fd] = os.dup(fd)
|
||||
for fd in (1, 2):
|
||||
os.dup2(log.fileno(), fd)
|
||||
moved.append(fd)
|
||||
except OSError:
|
||||
_put_back(saved, moved)
|
||||
yield None
|
||||
return
|
||||
_point_windows_std_handles_at(1, 2)
|
||||
try:
|
||||
yield path
|
||||
finally:
|
||||
_flush_std_streams()
|
||||
_put_back(saved, moved)
|
||||
_point_windows_std_handles_at(1, 2)
|
||||
|
||||
|
||||
def _open_run_log(path: Path) -> BinaryIO | None:
|
||||
"""The run log, opened for writing owner-only and never through a link:
|
||||
it holds final answers, tool output and tracebacks that can carry a
|
||||
credential. A log an earlier run left is emptied and its mode tightened.
|
||||
None when it cannot be opened that way."""
|
||||
flags = os.O_WRONLY | os.O_CREAT | os.O_TRUNC | getattr(os, "O_NOFOLLOW", 0)
|
||||
try:
|
||||
if path.is_symlink():
|
||||
return None # a link someone planted does not choose where it goes
|
||||
fd = os.open(path, flags, 0o600)
|
||||
except OSError:
|
||||
return None
|
||||
with contextlib.suppress(OSError, AttributeError):
|
||||
os.fchmod(fd, 0o600)
|
||||
return os.fdopen(fd, "wb")
|
||||
|
||||
|
||||
def _flush_std_streams() -> None:
|
||||
for stream in (sys.stdout, sys.stderr):
|
||||
with contextlib.suppress(Exception):
|
||||
stream.flush()
|
||||
|
||||
|
||||
def _put_back(saved: dict[int, int], moved: list[int]) -> None:
|
||||
"""Point every MOVED descriptor at its SAVED original again, and close
|
||||
every saved copy: each on its own, so one failure stops none of the rest."""
|
||||
for fd in moved:
|
||||
with contextlib.suppress(OSError):
|
||||
os.dup2(saved[fd], fd)
|
||||
for copy in saved.values():
|
||||
_close_quietly(copy)
|
||||
|
||||
|
||||
def _close_quietly(fd: int) -> None:
|
||||
with contextlib.suppress(OSError):
|
||||
os.close(fd)
|
||||
|
||||
|
||||
def _point_windows_std_handles_at(out_fd: int, err_fd: int) -> None:
|
||||
"""On Windows a child takes the process's standard handles, not its file
|
||||
descriptors: point them where descriptors 1 and 2 now go."""
|
||||
if os.name != "nt":
|
||||
return
|
||||
with contextlib.suppress(Exception):
|
||||
import ctypes
|
||||
import msvcrt
|
||||
|
||||
kernel32 = ctypes.windll.kernel32 # type: ignore[attr-defined]
|
||||
handle_of = msvcrt.get_osfhandle # type: ignore[attr-defined]
|
||||
kernel32.SetStdHandle(-11, handle_of(out_fd)) # STD_OUTPUT_HANDLE
|
||||
kernel32.SetStdHandle(-12, handle_of(err_fd)) # STD_ERROR_HANDLE
|
||||
|
||||
|
||||
def _say_what_the_run_left(log: Path | None, seconds: float, traced: bool) -> None:
|
||||
"""After a quiet run: where its output is, and — when it left nothing to
|
||||
evaluate — its last lines, which usually hold the error."""
|
||||
if log is None:
|
||||
return
|
||||
where = _shown_path(log)
|
||||
if traced:
|
||||
console.print(
|
||||
f"The crew ran in {seconds:.0f}s; its output is in {where}.", style="dim"
|
||||
)
|
||||
return
|
||||
with contextlib.suppress(OSError):
|
||||
lines = log.read_text(encoding="utf-8", errors="replace").splitlines()
|
||||
tail = lines[-RUN_LOG_TAIL_LINES:]
|
||||
if tail:
|
||||
console.print(
|
||||
f"The crew's output is in {where}; its last {len(tail)} lines:",
|
||||
style="bold",
|
||||
)
|
||||
console.print(Text("\n".join(tail)))
|
||||
|
||||
|
||||
def _shown_path(path: Path) -> str:
|
||||
try:
|
||||
return str(path.relative_to(Path.cwd()))
|
||||
except ValueError:
|
||||
return str(path)
|
||||
|
||||
|
||||
def _recorded_since(record: dict[str, Any], began: datetime) -> bool:
|
||||
"""Was RECORD written after BEGAN? A record without a readable, zoned stamp
|
||||
is not assumed to be this run's."""
|
||||
|
||||
@@ -3458,3 +3458,197 @@ def test_a_project_that_turned_tracing_off_is_not_asked_to_trace_toward_an_untru
|
||||
assert exited.value.code == 1
|
||||
assert usage == ["cli_usage:eval_stopped:untrusted_amp"]
|
||||
assert "CREWAI_TRACING_ENABLED=false" in (directory / ".env").read_text()
|
||||
|
||||
|
||||
# ── with no terminal, the run's own output goes to a log, not the agent ────
|
||||
|
||||
|
||||
def _a_tracing_project(directory: Path) -> None:
|
||||
(directory / "pyproject.toml").write_text("[project]\nname = 'demo'\n")
|
||||
(directory / ".env").write_text("CREWAI_TRACING_ENABLED=true\n")
|
||||
|
||||
|
||||
def test_with_no_terminal_the_runs_own_output_goes_to_a_log_not_the_agent(
|
||||
project, monkeypatch, capfd
|
||||
):
|
||||
"""A coding agent reads every line `crewai eval` prints, and pays for it:
|
||||
the crew's panels, its final answer and its tool calls go to
|
||||
`.crewai/last_run.log`, and the agent gets one line saying so, then the
|
||||
verdict."""
|
||||
directory, _ = project
|
||||
_a_tracing_project(directory)
|
||||
monkeypatch.setattr(eval_module.sys.stdin, "isatty", lambda: False)
|
||||
monkeypatch.setattr(eval_module, "nobody_watching", lambda: True)
|
||||
|
||||
def fake_run_crew() -> None:
|
||||
# what the run's child processes print: straight to the descriptors
|
||||
os.write(1, b"Crew Execution Started panel\n")
|
||||
os.write(2, b"a warning on stderr\n")
|
||||
record_last_run(directory, "the-new-run", recorded_at=_now())
|
||||
|
||||
monkeypatch.setattr(run_crew_module, "run_crew", fake_run_crew)
|
||||
install(monkeypatch, FakeAMP(statuses=[done()]))
|
||||
|
||||
eval_module.eval_crew()
|
||||
|
||||
out, err = capfd.readouterr()
|
||||
log = (directory / ".crewai" / "last_run.log").read_text()
|
||||
assert "Crew Execution Started panel" in log and "a warning on stderr" in log
|
||||
assert "Crew Execution Started panel" not in out + err
|
||||
assert "a warning on stderr" not in out + err
|
||||
said = " ".join(out.split())
|
||||
assert "The crew's output goes to .crewai/last_run.log while it runs." in said
|
||||
assert "its output is in .crewai/last_run.log." in said
|
||||
assert "Goal gate: PASSED" in said
|
||||
|
||||
|
||||
def test_with_no_terminal_a_run_that_left_no_trace_shows_its_last_lines(
|
||||
project, monkeypatch, capfd, usage
|
||||
):
|
||||
"""The error a failed run printed is what the agent needs to fix it: its
|
||||
last lines are shown, and the sentence says where the rest is."""
|
||||
directory, _ = project
|
||||
_a_tracing_project(directory)
|
||||
monkeypatch.setattr(eval_module.sys.stdin, "isatty", lambda: False)
|
||||
monkeypatch.setattr(eval_module, "nobody_watching", lambda: True)
|
||||
|
||||
def fake_run_crew() -> None:
|
||||
os.write(1, b"".join(f"line {n}\n".encode() for n in range(100)))
|
||||
os.write(2, b"KeyError: 'topic'\n") # and no trace recorded
|
||||
|
||||
monkeypatch.setattr(run_crew_module, "run_crew", fake_run_crew)
|
||||
amp = install(monkeypatch, FakeAMP())
|
||||
|
||||
with pytest.raises(SystemExit) as exited:
|
||||
eval_module.eval_crew()
|
||||
|
||||
assert exited.value.code == 1
|
||||
said = " ".join(capfd.readouterr().out.split())
|
||||
assert "its last 40 lines:" in said
|
||||
assert "KeyError: 'topic'" in said and "line 99" in said
|
||||
assert "line 50 " not in said # only the tail
|
||||
assert "(its last lines are above)" in said
|
||||
assert amp.calls == [] and usage[-1] == "cli_usage:eval_stopped:untraced_after_run"
|
||||
|
||||
|
||||
def test_at_a_terminal_the_run_prints_where_it_always_did(project, monkeypatch, capfd):
|
||||
directory, _ = project
|
||||
_a_tracing_project(directory)
|
||||
monkeypatch.setattr(eval_module.sys.stdin, "isatty", lambda: True)
|
||||
monkeypatch.setattr(eval_module, "nobody_watching", lambda: False) # a person reads stdout
|
||||
|
||||
def fake_run_crew() -> None:
|
||||
os.write(1, b"Crew Execution Started panel\n")
|
||||
record_last_run(directory, "the-new-run", recorded_at=_now())
|
||||
|
||||
monkeypatch.setattr(run_crew_module, "run_crew", fake_run_crew)
|
||||
install(monkeypatch, FakeAMP(statuses=[done()]))
|
||||
|
||||
eval_module.eval_crew()
|
||||
|
||||
out, _ = capfd.readouterr()
|
||||
assert "Crew Execution Started panel" in out
|
||||
assert "last_run.log" not in out
|
||||
assert not (directory / ".crewai" / "last_run.log").exists()
|
||||
|
||||
|
||||
def test_what_decides_is_whether_anyone_reads_stdout_not_stdin(project, monkeypatch, capfd):
|
||||
"""A coding agent can hand the command a terminal for stdin and still read
|
||||
a captured stdout: the transcript stays out of what it reads."""
|
||||
directory, _ = project
|
||||
_a_tracing_project(directory)
|
||||
monkeypatch.setattr(eval_module.sys.stdin, "isatty", lambda: True)
|
||||
monkeypatch.setattr(eval_module, "nobody_watching", lambda: True)
|
||||
|
||||
def fake_run_crew() -> None:
|
||||
os.write(1, b"Crew Execution Started panel\n")
|
||||
record_last_run(directory, "the-new-run", recorded_at=_now())
|
||||
|
||||
monkeypatch.setattr(run_crew_module, "run_crew", fake_run_crew)
|
||||
install(monkeypatch, FakeAMP(statuses=[done()]))
|
||||
|
||||
eval_module.eval_crew()
|
||||
|
||||
assert "Crew Execution Started panel" not in capfd.readouterr().out
|
||||
assert "Crew Execution Started panel" in (directory / ".crewai" / "last_run.log").read_text()
|
||||
|
||||
|
||||
@pytest.mark.skipif(os.name == "nt", reason="POSIX file modes")
|
||||
def test_the_run_log_is_the_owners_alone_even_one_left_by_an_earlier_run(tmp_path):
|
||||
log = tmp_path / "last_run.log"
|
||||
log.write_text("an earlier run\n")
|
||||
log.chmod(0o644)
|
||||
|
||||
with eval_module._run_output_to(log) as where:
|
||||
assert where == log
|
||||
os.write(1, b"this run\n")
|
||||
|
||||
assert log.read_text() == "this run\n" # emptied, then written
|
||||
assert log.stat().st_mode & 0o777 == 0o600
|
||||
|
||||
|
||||
@pytest.mark.skipif(os.name == "nt", reason="symbolic links need privileges on Windows")
|
||||
def test_a_link_at_the_logs_place_leaves_the_output_on_screen(tmp_path, capfd):
|
||||
elsewhere = tmp_path / "elsewhere.txt"
|
||||
elsewhere.write_text("")
|
||||
log = tmp_path / "last_run.log"
|
||||
log.symlink_to(elsewhere)
|
||||
|
||||
with eval_module._run_output_to(log) as where:
|
||||
assert where is None
|
||||
os.write(1, b"on screen\n")
|
||||
|
||||
assert "on screen" in capfd.readouterr().out
|
||||
assert elsewhere.read_text() == ""
|
||||
|
||||
|
||||
def test_a_move_that_fails_halfway_puts_back_what_it_moved(tmp_path, monkeypatch, capfd):
|
||||
"""Descriptor 1 moved, 2 refused: 1 comes back before the block runs, so
|
||||
the verdict printed after it still reaches the screen."""
|
||||
real_dup2 = os.dup2
|
||||
targets: list[int] = []
|
||||
|
||||
def dup2(src: int, dst: int, inheritable: bool = True) -> int:
|
||||
targets.append(dst)
|
||||
if dst == 2 and len(targets) == 2:
|
||||
raise OSError("no room for stderr")
|
||||
return real_dup2(src, dst, inheritable)
|
||||
|
||||
with monkeypatch.context() as patched:
|
||||
patched.setattr(eval_module.os, "dup2", dup2)
|
||||
with eval_module._run_output_to(tmp_path / "last_run.log") as where:
|
||||
assert where is None
|
||||
os.write(1, b"the verdict\n")
|
||||
|
||||
assert "the verdict" in capfd.readouterr().out
|
||||
assert (tmp_path / "last_run.log").read_text() == ""
|
||||
|
||||
|
||||
def test_putting_back_goes_on_past_a_failure_and_keeps_the_runs_own_error(tmp_path, monkeypatch, capfd):
|
||||
"""Putting stdout back fails; stderr is put back anyway, every saved copy
|
||||
is closed, and the error the run raised is the one that surfaces."""
|
||||
real_dup2 = os.dup2
|
||||
stdout_copy = os.dup(1)
|
||||
calls: list[int] = []
|
||||
|
||||
def dup2(src: int, dst: int, inheritable: bool = True) -> int:
|
||||
calls.append(dst)
|
||||
if len(calls) == 3: # the first put-back: stdout
|
||||
raise OSError("cannot put stdout back")
|
||||
return real_dup2(src, dst, inheritable)
|
||||
|
||||
def the_run() -> None:
|
||||
raise ValueError("the run's own error")
|
||||
|
||||
try:
|
||||
with monkeypatch.context() as patched:
|
||||
patched.setattr(eval_module.os, "dup2", dup2)
|
||||
with pytest.raises(ValueError, match="the run's own error"):
|
||||
with eval_module._run_output_to(tmp_path / "last_run.log"):
|
||||
the_run()
|
||||
assert calls == [1, 2, 1, 2] # stderr was put back after stdout failed
|
||||
os.write(2, b"stderr is back\n")
|
||||
assert "stderr is back" in capfd.readouterr().err
|
||||
finally:
|
||||
real_dup2(stdout_copy, 1) # the test puts stdout back itself
|
||||
os.close(stdout_copy)
|
||||
@@ -625,7 +625,8 @@ class AnthropicCompletion(BaseLLM):
|
||||
system_message: Extracted system message
|
||||
tools: Tool definitions
|
||||
available_functions: Available functions for tool calling. When provided
|
||||
with a single tool, tool_choice is automatically set to force tool use.
|
||||
with a single tool, tool_choice is "auto" so the model may call
|
||||
that tool or answer in text.
|
||||
|
||||
Returns:
|
||||
Parameters dictionary for Anthropic API
|
||||
@@ -652,8 +653,7 @@ class AnthropicCompletion(BaseLLM):
|
||||
|
||||
# When tool_search is enabled and there are 2+ regular tools,
|
||||
# inject the search tool and mark regular tools with defer_loading.
|
||||
# With only 1 tool there's nothing to search — skip tool search
|
||||
# entirely so the normal forced tool_choice optimisation still works.
|
||||
# With only 1 tool there's nothing to search, so skip tool search.
|
||||
regular_tools = [
|
||||
t
|
||||
for t in converted_tools
|
||||
@@ -664,10 +664,11 @@ class AnthropicCompletion(BaseLLM):
|
||||
|
||||
params["tools"] = converted_tools
|
||||
|
||||
# available_functions means the tool can be executed, not that the
|
||||
# model must call it. Named tool_choice is also rejected by newer
|
||||
# Claude models (Sonnet 5.5, Opus 5.5, Fable 5.1, Mythos 5.1).
|
||||
if available_functions and len(regular_tools) == 1:
|
||||
tool_name = regular_tools[0].get("name")
|
||||
if tool_name and tool_name in available_functions:
|
||||
params["tool_choice"] = {"type": "tool", "name": tool_name}
|
||||
params["tool_choice"] = {"type": "auto"}
|
||||
|
||||
if self.thinking:
|
||||
if isinstance(self.thinking, AnthropicThinkingConfig):
|
||||
|
||||
@@ -1470,9 +1470,9 @@ def test_tool_search_passthrough_preserves_tool_search_type():
|
||||
assert "input_schema" in converted[1]
|
||||
|
||||
|
||||
def test_tool_search_single_tool_skips_search_and_forces_choice():
|
||||
"""With only 1 tool, tool_search is skipped (nothing to search) and the
|
||||
normal forced tool_choice optimisation still applies."""
|
||||
def test_tool_search_single_tool_skips_search_and_uses_auto_choice():
|
||||
"""With only 1 tool, tool_search is skipped (nothing to search) and
|
||||
tool_choice stays auto."""
|
||||
llm = LLM(model="anthropic/claude-sonnet-4-5", tool_search=True)
|
||||
|
||||
crewai_tools = [
|
||||
@@ -1500,9 +1500,8 @@ def test_tool_search_single_tool_skips_search_and_forces_choice():
|
||||
available_functions={"test_tool": lambda q: "result"},
|
||||
)
|
||||
|
||||
# Single tool — tool_search skipped, tool_choice forced as normal
|
||||
assert "tool_choice" in params
|
||||
assert params["tool_choice"]["name"] == "test_tool"
|
||||
# Single tool — tool_search skipped, model chooses whether to call it
|
||||
assert params["tool_choice"] == {"type": "auto"}
|
||||
|
||||
# No tool search tool should be injected
|
||||
tool_types = [t.get("type", "") for t in params["tools"]]
|
||||
@@ -1513,6 +1512,39 @@ def test_tool_search_single_tool_skips_search_and_forces_choice():
|
||||
assert "defer_loading" not in params["tools"][0]
|
||||
|
||||
|
||||
def test_single_tool_with_available_functions_uses_auto_choice():
|
||||
"""One executable tool is offered, not forced.
|
||||
|
||||
Newer Claude models reject tool_choice {"type": "tool"}. Auto lets the
|
||||
model call the tool or answer in text.
|
||||
"""
|
||||
llm = LLM(model="anthropic/claude-sonnet-5-5")
|
||||
tool = {
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "get_weather",
|
||||
"description": "Get the weather for a city.",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {"city": {"type": "string"}},
|
||||
"required": ["city"],
|
||||
},
|
||||
},
|
||||
}
|
||||
formatted_messages, system_message = llm._format_messages_for_anthropic(
|
||||
[{"role": "user", "content": "What's the weather in Paris?"}]
|
||||
)
|
||||
params = llm._prepare_completion_params(
|
||||
formatted_messages,
|
||||
system_message,
|
||||
[tool],
|
||||
available_functions={"get_weather": lambda city: "sunny"},
|
||||
)
|
||||
|
||||
assert params["tool_choice"] == {"type": "auto"}
|
||||
assert [t["name"] for t in params["tools"]] == ["get_weather"]
|
||||
|
||||
|
||||
def test_tool_search_via_llm_class():
|
||||
"""Verify tool_search param passes through LLM class correctly."""
|
||||
from crewai.llms.providers.anthropic.completion import (
|
||||
|
||||
Reference in new issue
Block a user