diff --git a/docs/edge/en/observability/arize-phoenix.mdx b/docs/edge/en/observability/arize-phoenix.mdx index cdbe5b822..d92f3cc60 100644 --- a/docs/edge/en/observability/arize-phoenix.mdx +++ b/docs/edge/en/observability/arize-phoenix.mdx @@ -7,7 +7,7 @@ mode: "wide" # Arize Phoenix Integration -This guide demonstrates how to integrate **Arize Phoenix** with **CrewAI** using OpenTelemetry via the [OpenInference](https://github.com/openinference/openinference) SDK. By the end of this guide, you will be able to trace your CrewAI agents and debug agent behavior. +This guide demonstrates how to integrate **Arize Phoenix** with **CrewAI** using OpenTelemetry via the [OpenInference](https://github.com/Arize-ai/openinference) SDK. By the end of this guide, you will be able to trace your CrewAI agents and debug agent behavior. > **What is Arize Phoenix?** [Arize Phoenix](https://arize.com/phoenix/) is the open-source observability and evaluation option from [Arize AI](https://arize.com/?utm_source=crewai-docs&utm_medium=partner&utm_campaign=partner-docs&utm_content=observability-arize-phoenix). Use Phoenix when you want to run locally or self-host. Use [Arize AX](https://arize.com/products/ax/) for a managed cloud or enterprise self-hosted platform for production AI systems. @@ -152,4 +152,4 @@ Open your Phoenix project and navigate to the project you specified in the `proj - [Arize LLM evaluation guide](https://arize.com/resources/llm-evaluation/) - Methods and metrics for evaluating LLM applications. - [CrewAI Documentation](https://docs.crewai.com/) - Overview of the CrewAI framework. - [OpenTelemetry Docs](https://opentelemetry.io/docs/) - OpenTelemetry guide -- [OpenInference GitHub](https://github.com/openinference/openinference) - Source code for OpenInference SDK. +- [OpenInference GitHub](https://github.com/Arize-ai/openinference) - Source code for OpenInference SDK. diff --git a/docs/edge/en/observability/braintrust.mdx b/docs/edge/en/observability/braintrust.mdx index 21ab52c85..c2ca9c5f3 100644 --- a/docs/edge/en/observability/braintrust.mdx +++ b/docs/edge/en/observability/braintrust.mdx @@ -234,4 +234,4 @@ See the [Braintrust Eval SDK guide](https://www.braintrust.dev/docs/start/eval-s - [Braintrust Eval SDK](https://www.braintrust.dev/docs/start/eval-sdk) - Run experiments via the SDK - [CrewAI Documentation](https://docs.crewai.com/) - Overview of the CrewAI framework - [OpenTelemetry Docs](https://opentelemetry.io/docs/) - OpenTelemetry guide -- [Braintrust GitHub](https://github.com/braintrustdata/braintrust) - Source code for Braintrust SDK +- [Braintrust GitHub](https://github.com/braintrustdata/braintrust-sdk-python) - Source code for Braintrust SDK diff --git a/docs/v1.15.28/en/observability/arize-phoenix.mdx b/docs/v1.15.28/en/observability/arize-phoenix.mdx index cdbe5b822..d92f3cc60 100644 --- a/docs/v1.15.28/en/observability/arize-phoenix.mdx +++ b/docs/v1.15.28/en/observability/arize-phoenix.mdx @@ -7,7 +7,7 @@ mode: "wide" # Arize Phoenix Integration -This guide demonstrates how to integrate **Arize Phoenix** with **CrewAI** using OpenTelemetry via the [OpenInference](https://github.com/openinference/openinference) SDK. By the end of this guide, you will be able to trace your CrewAI agents and debug agent behavior. +This guide demonstrates how to integrate **Arize Phoenix** with **CrewAI** using OpenTelemetry via the [OpenInference](https://github.com/Arize-ai/openinference) SDK. By the end of this guide, you will be able to trace your CrewAI agents and debug agent behavior. > **What is Arize Phoenix?** [Arize Phoenix](https://arize.com/phoenix/) is the open-source observability and evaluation option from [Arize AI](https://arize.com/?utm_source=crewai-docs&utm_medium=partner&utm_campaign=partner-docs&utm_content=observability-arize-phoenix). Use Phoenix when you want to run locally or self-host. Use [Arize AX](https://arize.com/products/ax/) for a managed cloud or enterprise self-hosted platform for production AI systems. @@ -152,4 +152,4 @@ Open your Phoenix project and navigate to the project you specified in the `proj - [Arize LLM evaluation guide](https://arize.com/resources/llm-evaluation/) - Methods and metrics for evaluating LLM applications. - [CrewAI Documentation](https://docs.crewai.com/) - Overview of the CrewAI framework. - [OpenTelemetry Docs](https://opentelemetry.io/docs/) - OpenTelemetry guide -- [OpenInference GitHub](https://github.com/openinference/openinference) - Source code for OpenInference SDK. +- [OpenInference GitHub](https://github.com/Arize-ai/openinference) - Source code for OpenInference SDK. diff --git a/docs/v1.15.28/en/observability/braintrust.mdx b/docs/v1.15.28/en/observability/braintrust.mdx index 21ab52c85..c2ca9c5f3 100644 --- a/docs/v1.15.28/en/observability/braintrust.mdx +++ b/docs/v1.15.28/en/observability/braintrust.mdx @@ -234,4 +234,4 @@ See the [Braintrust Eval SDK guide](https://www.braintrust.dev/docs/start/eval-s - [Braintrust Eval SDK](https://www.braintrust.dev/docs/start/eval-sdk) - Run experiments via the SDK - [CrewAI Documentation](https://docs.crewai.com/) - Overview of the CrewAI framework - [OpenTelemetry Docs](https://opentelemetry.io/docs/) - OpenTelemetry guide -- [Braintrust GitHub](https://github.com/braintrustdata/braintrust) - Source code for Braintrust SDK +- [Braintrust GitHub](https://github.com/braintrustdata/braintrust-sdk-python) - Source code for Braintrust SDK diff --git a/lib/cli/src/crewai_cli/experimental/eval_crew.py b/lib/cli/src/crewai_cli/experimental/eval_crew.py index 87718918c..b7abe2cff 100644 --- a/lib/cli/src/crewai_cli/experimental/eval_crew.py +++ b/lib/cli/src/crewai_cli/experimental/eval_crew.py @@ -43,7 +43,7 @@ import re import stat import sys import time -from typing import Any, Literal, NoReturn, cast +from typing import Any, BinaryIO, Literal, NoReturn, cast from urllib.parse import urlparse import uuid import webbrowser @@ -161,6 +161,13 @@ NOT_TRACED = ( f"({LAST_RUN_FILE})." ) +# With no terminal, what the crew prints while it runs (its panels, the final +# answer, every tool call) goes to this file in the project's `.crewai/`, not to +# the coding agent's screen: in one measured run it was 341 of 378 lines, all +# before the verdict, and an agent pays for every line or has the end cut off. +RUN_LOG_FILE = "last_run.log" +RUN_LOG_TAIL_LINES = 40 + # Why `crewai eval` ran the crew before evaluating, a word from a closed list: # nothing traced yet, the project changed since its last traced run, or the run # app's Evaluate pressed on a run that was not traced. @@ -1002,11 +1009,23 @@ def _run_the_crew_to_evaluate( # The handshake (`crewai_core.eval_handshake`) carries what this command # decided before the project's `.env` was loaded: the AMP origins its saved # login may go to, and whether it just turned tracing on. + # Whoever reads the output decides, as for the brief: stdout, not stdin. + run_log = _run_log_path() if nobody_watching() else None + if run_log is not None: + console.print( + f"The crew's output goes to {_shown_path(run_log)} while it runs.", + style="dim", + ) began = datetime.now(timezone.utc) - with evaluating_after_run( - trusted_amp_origins=trusted or (), tracing_turned_on=turned_on_now - ) as watched: + started = time.monotonic() + with ( + evaluating_after_run( + trusted_amp_origins=trusted or (), tracing_turned_on=turned_on_now + ) as watched, + _run_output_to(run_log) as run_log, + ): run_crew(**run_args) + seconds = time.monotonic() - started # The app says what it did with the run it watched, and its word settles # this: it evaluated it (nothing left to do), or it had nothing to evaluate @@ -1018,7 +1037,8 @@ def _run_the_crew_to_evaluate( remember_project_state(str(marker["execution_id"])) return None record_stop("untraced_after_run") - console.print(NOT_TRACED, style="bold red") + _say_what_the_run_left(run_log, seconds, traced=False) + console.print(_not_traced(run_log), style="bold red") raise SystemExit(1) # The project's record, only when it was written after this run began: a @@ -1029,13 +1049,157 @@ def _run_the_crew_to_evaluate( record.get("execution_id") if _recorded_since(record, began) else None ) if traced: + _say_what_the_run_left(run_log, seconds, traced=True) return str(traced) record_stop("untraced_after_run") - console.print(NOT_TRACED, style="bold red") + _say_what_the_run_left(run_log, seconds, traced=False) + console.print(_not_traced(run_log), style="bold red") raise SystemExit(1) +def _not_traced(run_log: Path | None) -> str: + """NOT_TRACED, pointing at where the run's output went.""" + if run_log is None: + return NOT_TRACED + return NOT_TRACED.replace("(its output is above)", "(its last lines are above)") + + +def _run_log_path() -> Path | None: + """Where a run with no terminal sends its output, or None when the + project's `.crewai/` cannot be written (the output then stays on screen).""" + from crewai.telemetry.tracing.last_run import LAST_RUN_DIR, project_dir + + try: + path = project_dir() / LAST_RUN_DIR / RUN_LOG_FILE + path.parent.mkdir(parents=True, exist_ok=True) + except OSError: + return None + return path + + +@contextlib.contextmanager +def _run_output_to(path: Path | None) -> Iterator[Path | None]: + """Send everything printed inside the block to PATH, and yield it; with no + PATH, yield None and change nothing. + + It moves this process's own stdout and stderr (file descriptors 1 and 2), + which every child the run starts inherits, so each way a project runs — + `uv run`, the JSON crew runner, a declarative flow — is covered without + touching any of them. Anything that keeps them from moving leaves the + output where it was: a quiet log is a nicety, never a failed run. Both are + always put back after, one by one, whatever failed: a stream left on the + log would swallow the verdict, and the run's own error is never hidden by + a failure to put them back.""" + log = _open_run_log(path) if path is not None else None + if log is None: + yield None + return + with log: + _flush_std_streams() + saved: dict[int, int] = {} + moved: list[int] = [] + try: + for fd in (1, 2): + saved[fd] = os.dup(fd) + for fd in (1, 2): + os.dup2(log.fileno(), fd) + moved.append(fd) + except OSError: + _put_back(saved, moved) + yield None + return + _point_windows_std_handles_at(1, 2) + try: + yield path + finally: + _flush_std_streams() + _put_back(saved, moved) + _point_windows_std_handles_at(1, 2) + + +def _open_run_log(path: Path) -> BinaryIO | None: + """The run log, opened for writing owner-only and never through a link: + it holds final answers, tool output and tracebacks that can carry a + credential. A log an earlier run left is emptied and its mode tightened. + None when it cannot be opened that way.""" + flags = os.O_WRONLY | os.O_CREAT | os.O_TRUNC | getattr(os, "O_NOFOLLOW", 0) + try: + if path.is_symlink(): + return None # a link someone planted does not choose where it goes + fd = os.open(path, flags, 0o600) + except OSError: + return None + with contextlib.suppress(OSError, AttributeError): + os.fchmod(fd, 0o600) + return os.fdopen(fd, "wb") + + +def _flush_std_streams() -> None: + for stream in (sys.stdout, sys.stderr): + with contextlib.suppress(Exception): + stream.flush() + + +def _put_back(saved: dict[int, int], moved: list[int]) -> None: + """Point every MOVED descriptor at its SAVED original again, and close + every saved copy: each on its own, so one failure stops none of the rest.""" + for fd in moved: + with contextlib.suppress(OSError): + os.dup2(saved[fd], fd) + for copy in saved.values(): + _close_quietly(copy) + + +def _close_quietly(fd: int) -> None: + with contextlib.suppress(OSError): + os.close(fd) + + +def _point_windows_std_handles_at(out_fd: int, err_fd: int) -> None: + """On Windows a child takes the process's standard handles, not its file + descriptors: point them where descriptors 1 and 2 now go.""" + if os.name != "nt": + return + with contextlib.suppress(Exception): + import ctypes + import msvcrt + + kernel32 = ctypes.windll.kernel32 # type: ignore[attr-defined] + handle_of = msvcrt.get_osfhandle # type: ignore[attr-defined] + kernel32.SetStdHandle(-11, handle_of(out_fd)) # STD_OUTPUT_HANDLE + kernel32.SetStdHandle(-12, handle_of(err_fd)) # STD_ERROR_HANDLE + + +def _say_what_the_run_left(log: Path | None, seconds: float, traced: bool) -> None: + """After a quiet run: where its output is, and — when it left nothing to + evaluate — its last lines, which usually hold the error.""" + if log is None: + return + where = _shown_path(log) + if traced: + console.print( + f"The crew ran in {seconds:.0f}s; its output is in {where}.", style="dim" + ) + return + with contextlib.suppress(OSError): + lines = log.read_text(encoding="utf-8", errors="replace").splitlines() + tail = lines[-RUN_LOG_TAIL_LINES:] + if tail: + console.print( + f"The crew's output is in {where}; its last {len(tail)} lines:", + style="bold", + ) + console.print(Text("\n".join(tail))) + + +def _shown_path(path: Path) -> str: + try: + return str(path.relative_to(Path.cwd())) + except ValueError: + return str(path) + + def _recorded_since(record: dict[str, Any], began: datetime) -> bool: """Was RECORD written after BEGAN? A record without a readable, zoned stamp is not assumed to be this run's.""" diff --git a/lib/cli/tests/experimental/test_eval_crew.py b/lib/cli/tests/experimental/test_eval_crew.py index 4d90d7d6b..dd6768440 100644 --- a/lib/cli/tests/experimental/test_eval_crew.py +++ b/lib/cli/tests/experimental/test_eval_crew.py @@ -3458,3 +3458,197 @@ def test_a_project_that_turned_tracing_off_is_not_asked_to_trace_toward_an_untru assert exited.value.code == 1 assert usage == ["cli_usage:eval_stopped:untrusted_amp"] assert "CREWAI_TRACING_ENABLED=false" in (directory / ".env").read_text() + + +# ── with no terminal, the run's own output goes to a log, not the agent ──── + + +def _a_tracing_project(directory: Path) -> None: + (directory / "pyproject.toml").write_text("[project]\nname = 'demo'\n") + (directory / ".env").write_text("CREWAI_TRACING_ENABLED=true\n") + + +def test_with_no_terminal_the_runs_own_output_goes_to_a_log_not_the_agent( + project, monkeypatch, capfd +): + """A coding agent reads every line `crewai eval` prints, and pays for it: + the crew's panels, its final answer and its tool calls go to + `.crewai/last_run.log`, and the agent gets one line saying so, then the + verdict.""" + directory, _ = project + _a_tracing_project(directory) + monkeypatch.setattr(eval_module.sys.stdin, "isatty", lambda: False) + monkeypatch.setattr(eval_module, "nobody_watching", lambda: True) + + def fake_run_crew() -> None: + # what the run's child processes print: straight to the descriptors + os.write(1, b"Crew Execution Started panel\n") + os.write(2, b"a warning on stderr\n") + record_last_run(directory, "the-new-run", recorded_at=_now()) + + monkeypatch.setattr(run_crew_module, "run_crew", fake_run_crew) + install(monkeypatch, FakeAMP(statuses=[done()])) + + eval_module.eval_crew() + + out, err = capfd.readouterr() + log = (directory / ".crewai" / "last_run.log").read_text() + assert "Crew Execution Started panel" in log and "a warning on stderr" in log + assert "Crew Execution Started panel" not in out + err + assert "a warning on stderr" not in out + err + said = " ".join(out.split()) + assert "The crew's output goes to .crewai/last_run.log while it runs." in said + assert "its output is in .crewai/last_run.log." in said + assert "Goal gate: PASSED" in said + + +def test_with_no_terminal_a_run_that_left_no_trace_shows_its_last_lines( + project, monkeypatch, capfd, usage +): + """The error a failed run printed is what the agent needs to fix it: its + last lines are shown, and the sentence says where the rest is.""" + directory, _ = project + _a_tracing_project(directory) + monkeypatch.setattr(eval_module.sys.stdin, "isatty", lambda: False) + monkeypatch.setattr(eval_module, "nobody_watching", lambda: True) + + def fake_run_crew() -> None: + os.write(1, b"".join(f"line {n}\n".encode() for n in range(100))) + os.write(2, b"KeyError: 'topic'\n") # and no trace recorded + + monkeypatch.setattr(run_crew_module, "run_crew", fake_run_crew) + amp = install(monkeypatch, FakeAMP()) + + with pytest.raises(SystemExit) as exited: + eval_module.eval_crew() + + assert exited.value.code == 1 + said = " ".join(capfd.readouterr().out.split()) + assert "its last 40 lines:" in said + assert "KeyError: 'topic'" in said and "line 99" in said + assert "line 50 " not in said # only the tail + assert "(its last lines are above)" in said + assert amp.calls == [] and usage[-1] == "cli_usage:eval_stopped:untraced_after_run" + + +def test_at_a_terminal_the_run_prints_where_it_always_did(project, monkeypatch, capfd): + directory, _ = project + _a_tracing_project(directory) + monkeypatch.setattr(eval_module.sys.stdin, "isatty", lambda: True) + monkeypatch.setattr(eval_module, "nobody_watching", lambda: False) # a person reads stdout + + def fake_run_crew() -> None: + os.write(1, b"Crew Execution Started panel\n") + record_last_run(directory, "the-new-run", recorded_at=_now()) + + monkeypatch.setattr(run_crew_module, "run_crew", fake_run_crew) + install(monkeypatch, FakeAMP(statuses=[done()])) + + eval_module.eval_crew() + + out, _ = capfd.readouterr() + assert "Crew Execution Started panel" in out + assert "last_run.log" not in out + assert not (directory / ".crewai" / "last_run.log").exists() + + +def test_what_decides_is_whether_anyone_reads_stdout_not_stdin(project, monkeypatch, capfd): + """A coding agent can hand the command a terminal for stdin and still read + a captured stdout: the transcript stays out of what it reads.""" + directory, _ = project + _a_tracing_project(directory) + monkeypatch.setattr(eval_module.sys.stdin, "isatty", lambda: True) + monkeypatch.setattr(eval_module, "nobody_watching", lambda: True) + + def fake_run_crew() -> None: + os.write(1, b"Crew Execution Started panel\n") + record_last_run(directory, "the-new-run", recorded_at=_now()) + + monkeypatch.setattr(run_crew_module, "run_crew", fake_run_crew) + install(monkeypatch, FakeAMP(statuses=[done()])) + + eval_module.eval_crew() + + assert "Crew Execution Started panel" not in capfd.readouterr().out + assert "Crew Execution Started panel" in (directory / ".crewai" / "last_run.log").read_text() + + +@pytest.mark.skipif(os.name == "nt", reason="POSIX file modes") +def test_the_run_log_is_the_owners_alone_even_one_left_by_an_earlier_run(tmp_path): + log = tmp_path / "last_run.log" + log.write_text("an earlier run\n") + log.chmod(0o644) + + with eval_module._run_output_to(log) as where: + assert where == log + os.write(1, b"this run\n") + + assert log.read_text() == "this run\n" # emptied, then written + assert log.stat().st_mode & 0o777 == 0o600 + + +@pytest.mark.skipif(os.name == "nt", reason="symbolic links need privileges on Windows") +def test_a_link_at_the_logs_place_leaves_the_output_on_screen(tmp_path, capfd): + elsewhere = tmp_path / "elsewhere.txt" + elsewhere.write_text("") + log = tmp_path / "last_run.log" + log.symlink_to(elsewhere) + + with eval_module._run_output_to(log) as where: + assert where is None + os.write(1, b"on screen\n") + + assert "on screen" in capfd.readouterr().out + assert elsewhere.read_text() == "" + + +def test_a_move_that_fails_halfway_puts_back_what_it_moved(tmp_path, monkeypatch, capfd): + """Descriptor 1 moved, 2 refused: 1 comes back before the block runs, so + the verdict printed after it still reaches the screen.""" + real_dup2 = os.dup2 + targets: list[int] = [] + + def dup2(src: int, dst: int, inheritable: bool = True) -> int: + targets.append(dst) + if dst == 2 and len(targets) == 2: + raise OSError("no room for stderr") + return real_dup2(src, dst, inheritable) + + with monkeypatch.context() as patched: + patched.setattr(eval_module.os, "dup2", dup2) + with eval_module._run_output_to(tmp_path / "last_run.log") as where: + assert where is None + os.write(1, b"the verdict\n") + + assert "the verdict" in capfd.readouterr().out + assert (tmp_path / "last_run.log").read_text() == "" + + +def test_putting_back_goes_on_past_a_failure_and_keeps_the_runs_own_error(tmp_path, monkeypatch, capfd): + """Putting stdout back fails; stderr is put back anyway, every saved copy + is closed, and the error the run raised is the one that surfaces.""" + real_dup2 = os.dup2 + stdout_copy = os.dup(1) + calls: list[int] = [] + + def dup2(src: int, dst: int, inheritable: bool = True) -> int: + calls.append(dst) + if len(calls) == 3: # the first put-back: stdout + raise OSError("cannot put stdout back") + return real_dup2(src, dst, inheritable) + + def the_run() -> None: + raise ValueError("the run's own error") + + try: + with monkeypatch.context() as patched: + patched.setattr(eval_module.os, "dup2", dup2) + with pytest.raises(ValueError, match="the run's own error"): + with eval_module._run_output_to(tmp_path / "last_run.log"): + the_run() + assert calls == [1, 2, 1, 2] # stderr was put back after stdout failed + os.write(2, b"stderr is back\n") + assert "stderr is back" in capfd.readouterr().err + finally: + real_dup2(stdout_copy, 1) # the test puts stdout back itself + os.close(stdout_copy) diff --git a/lib/crewai/src/crewai/llms/providers/anthropic/completion.py b/lib/crewai/src/crewai/llms/providers/anthropic/completion.py index a3d8d7833..8c582db94 100644 --- a/lib/crewai/src/crewai/llms/providers/anthropic/completion.py +++ b/lib/crewai/src/crewai/llms/providers/anthropic/completion.py @@ -625,7 +625,8 @@ class AnthropicCompletion(BaseLLM): system_message: Extracted system message tools: Tool definitions available_functions: Available functions for tool calling. When provided - with a single tool, tool_choice is automatically set to force tool use. + with a single tool, tool_choice is "auto" so the model may call + that tool or answer in text. Returns: Parameters dictionary for Anthropic API @@ -652,8 +653,7 @@ class AnthropicCompletion(BaseLLM): # When tool_search is enabled and there are 2+ regular tools, # inject the search tool and mark regular tools with defer_loading. - # With only 1 tool there's nothing to search — skip tool search - # entirely so the normal forced tool_choice optimisation still works. + # With only 1 tool there's nothing to search, so skip tool search. regular_tools = [ t for t in converted_tools @@ -664,10 +664,11 @@ class AnthropicCompletion(BaseLLM): params["tools"] = converted_tools + # available_functions means the tool can be executed, not that the + # model must call it. Named tool_choice is also rejected by newer + # Claude models (Sonnet 5.5, Opus 5.5, Fable 5.1, Mythos 5.1). if available_functions and len(regular_tools) == 1: - tool_name = regular_tools[0].get("name") - if tool_name and tool_name in available_functions: - params["tool_choice"] = {"type": "tool", "name": tool_name} + params["tool_choice"] = {"type": "auto"} if self.thinking: if isinstance(self.thinking, AnthropicThinkingConfig): diff --git a/lib/crewai/tests/llms/anthropic/test_anthropic.py b/lib/crewai/tests/llms/anthropic/test_anthropic.py index 059627ba9..8fce27066 100644 --- a/lib/crewai/tests/llms/anthropic/test_anthropic.py +++ b/lib/crewai/tests/llms/anthropic/test_anthropic.py @@ -1470,9 +1470,9 @@ def test_tool_search_passthrough_preserves_tool_search_type(): assert "input_schema" in converted[1] -def test_tool_search_single_tool_skips_search_and_forces_choice(): - """With only 1 tool, tool_search is skipped (nothing to search) and the - normal forced tool_choice optimisation still applies.""" +def test_tool_search_single_tool_skips_search_and_uses_auto_choice(): + """With only 1 tool, tool_search is skipped (nothing to search) and + tool_choice stays auto.""" llm = LLM(model="anthropic/claude-sonnet-4-5", tool_search=True) crewai_tools = [ @@ -1500,9 +1500,8 @@ def test_tool_search_single_tool_skips_search_and_forces_choice(): available_functions={"test_tool": lambda q: "result"}, ) - # Single tool — tool_search skipped, tool_choice forced as normal - assert "tool_choice" in params - assert params["tool_choice"]["name"] == "test_tool" + # Single tool — tool_search skipped, model chooses whether to call it + assert params["tool_choice"] == {"type": "auto"} # No tool search tool should be injected tool_types = [t.get("type", "") for t in params["tools"]] @@ -1513,6 +1512,39 @@ def test_tool_search_single_tool_skips_search_and_forces_choice(): assert "defer_loading" not in params["tools"][0] +def test_single_tool_with_available_functions_uses_auto_choice(): + """One executable tool is offered, not forced. + + Newer Claude models reject tool_choice {"type": "tool"}. Auto lets the + model call the tool or answer in text. + """ + llm = LLM(model="anthropic/claude-sonnet-5-5") + tool = { + "type": "function", + "function": { + "name": "get_weather", + "description": "Get the weather for a city.", + "parameters": { + "type": "object", + "properties": {"city": {"type": "string"}}, + "required": ["city"], + }, + }, + } + formatted_messages, system_message = llm._format_messages_for_anthropic( + [{"role": "user", "content": "What's the weather in Paris?"}] + ) + params = llm._prepare_completion_params( + formatted_messages, + system_message, + [tool], + available_functions={"get_weather": lambda city: "sunny"}, + ) + + assert params["tool_choice"] == {"type": "auto"} + assert [t["name"] for t in params["tools"]] == ["get_weather"] + + def test_tool_search_via_llm_class(): """Verify tool_search param passes through LLM class correctly.""" from crewai.llms.providers.anthropic.completion import (