fix(llm): model keys read a provider's other names; a LiteLLM choice survives a missing SDK

- The router's provider aliases are now one table, `crewai.llm.PROVIDER_ALIASES`,
  and `llm_overlay` compares model names through it: `google/x` and
  `gemini/x` are one model whether the llm is built in the block or was
  built before it (a prebuilt instance records `gemini`). An exact key still
  wins, and an aggregator's route is still compared whole.
- When the declared model's SDK is missing, a caller's explicit
  `is_litellm=True` is kept for the mapped model instead of dropping to a
  native route.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>

lib/cli/src/crewai_cli/cli.py

@@ -54,6 +54,12 @@ def eval_crew(*args: Any, **kwargs: Any) -> Any:
     return _eval_crew(*args, **kwargs)
 
 
+def eval_models(*args: Any, **kwargs: Any) -> Any:
+    from crewai_cli.experimental.eval_crew import eval_models as _eval_models
+
+    return _eval_models(*args, **kwargs)
+
+
 if TYPE_CHECKING:
     # mypy sees the real classes; at runtime the shims below defer the
     # heavy imports until a command actually instantiates them.
@@ -692,12 +698,52 @@ def run(
         "crewAI recorded for the run."
     ),
 )
-def eval_command(run_id: str | None) -> None:
-    """Evaluate the last traced run through CrewAI AMP.
+@click.option(
+    "--models",
+    "models",
+    type=str,
+    default=None,
+    metavar="LIST",
+    help=(
+        "Compare models on this project's deployment instead: ONE comma-separated "
+        'list of provider/model, e.g. "openai/gpt-4o-mini,anthropic/claude-haiku-4-5". '
+        "The deployment runs once as deployed and once per model; needs `crewai login`."
+    ),
+)
+@click.option(
+    "--deployment",
+    "deployment_id",
+    type=str,
+    default=None,
+    metavar="UUID",
+    help=(
+        "With --models: the deployment to run, when AMP cannot tell it from the "
+        "project id."
+    ),
+)
+def eval_command(
+    run_id: str | None, models: str | None, deployment_id: str | None
+) -> None:
+    """Evaluate the last traced run through CrewAI AMP, or compare models on the
+    project's deployment (--models).
 
-    Exits 0 only when the goal gate PASSED, and 1 otherwise — a failed gate, no
-    verdict, or an evaluation that could not run — so a CI job can gate on it.
+    A run's evaluation exits 0 only when the goal gate PASSED, and 1 otherwise — a
+    failed gate, no verdict, or an evaluation that could not run — so a CI job can
+    gate on it. A comparison exits 0 when it finished, 1 when it failed or could
+    not start.
     """
+    if models is not None:
+        if run_id is not None:
+            raise click.UsageError(
+                "--run grades a run that already happened; --models runs the "
+                "deployment again. Give one of them."
+            )
+        eval_models(models, deployment_id=deployment_id)
+        return
+    if deployment_id is not None:
+        raise click.UsageError(
+            "--deployment names the deployment --models runs; add --models LIST."
+        )
     eval_crew(run_id=run_id)
 
 

lib/cli/src/crewai_cli/deploy/main.py

@@ -15,7 +15,11 @@ from crewai_cli.command import BaseCommand, PlusAPIMixin
 from crewai_cli.constants import DEFAULT_CREWAI_ENTERPRISE_URL
 from crewai_cli.deploy.archive import ArchiveError, create_project_zip
 from crewai_cli.deploy.validate import DeployValidator, Severity, render_report
-from crewai_cli.utils import fetch_and_json_env_file, get_project_name
+from crewai_cli.utils import (
+    fetch_and_json_env_file,
+    get_project_id,
+    get_project_name,
+)
 
 
 console = Console()
@@ -232,6 +236,14 @@ class DeployCommand(BaseCommand, PlusAPIMixin):
         BaseCommand.__init__(self)
         PlusAPIMixin.__init__(self, telemetry=self._telemetry)
         self.project_name = get_project_name(require=True)
+        # Sent with every create and push, so AMP knows which project a
+        # deployment runs — `crewai eval --models` finds the deployment by it.
+        # Read, never minted: a deploy does not rewrite pyproject.toml.
+        self.project_id = get_project_id()
+
+    def _project(self) -> dict[str, str]:
+        """`project_id=` for a create or push request, when the project has one."""
+        return {"project_id": self.project_id} if self.project_id else {}
 
     def _standard_no_param_error_message(self) -> None:
         """
@@ -424,10 +436,10 @@ class DeployCommand(BaseCommand, PlusAPIMixin):
             env_vars = fetch_and_json_env_file()
             return self._update_crew_from_zip(deployment_uuid, repository, env_vars)
         if uuid:
-            return self.plus_api_client.deploy_by_uuid(uuid)
+            return self.plus_api_client.deploy_by_uuid(uuid, **self._project())
         if not project_name:
             raise ValueError("project_name is required to deploy by name")
-        return self.plus_api_client.deploy_by_name(project_name)
+        return self.plus_api_client.deploy_by_name(project_name, **self._project())
 
     def _deploy_from_local_source(
         self,
@@ -438,9 +450,9 @@ class DeployCommand(BaseCommand, PlusAPIMixin):
     ) -> Any | None:
         """Deploy using local origin, as before AMP zip_deployment existed."""
         if remote_repo_url and uuid:
-            return self.plus_api_client.deploy_by_uuid(uuid)
+            return self.plus_api_client.deploy_by_uuid(uuid, **self._project())
         if remote_repo_url and project_name:
-            return self.plus_api_client.deploy_by_name(project_name)
+            return self.plus_api_client.deploy_by_name(project_name, **self._project())
         if uuid:
             _display_git_remote_help()
             env_vars = fetch_and_json_env_file()
@@ -627,6 +639,7 @@ class DeployCommand(BaseCommand, PlusAPIMixin):
                 zip_file_path,
                 name=self.project_name,
                 env=env_vars,
+                **self._project(),
             )
         finally:
             zip_file_path.unlink(missing_ok=True)
@@ -649,6 +662,7 @@ class DeployCommand(BaseCommand, PlusAPIMixin):
                 uuid,
                 zip_file_path,
                 env=env_vars,
+                **self._project(),
             )
         finally:
             zip_file_path.unlink(missing_ok=True)
@@ -692,13 +706,17 @@ class DeployCommand(BaseCommand, PlusAPIMixin):
         """
         if not self.project_name:
             raise ValueError("project_name is required to create a deployment payload")
-        return {
+        payload: CreateCrewPayload = {
             "deploy": {
                 "name": self.project_name,
                 "repo_clone_url": remote_repo_url,
                 "env": env_vars,
             }
         }
+        if self.project_id:
+            payload["deploy"]["project_id"] = self.project_id
+            payload["project_id"] = self.project_id
+        return payload
 
     def _display_creation_success(self, json_response: dict[str, Any]) -> None:
         """

lib/cli/src/crewai_cli/experimental/eval_crew.py

@@ -10,6 +10,13 @@ Who may evaluate what is AMP's decision: an anonymous run once without an
 account, then it needs one; a run traced while logged in for that
 organization's members; a deployment execution for members who may see its
 traces. The command sends the saved `crewai login` when there is one.
+
+`crewai eval --models "a,b"` compares models instead: AMP finds the project's
+deployment by `[tool.crewai].project_id` (or `--deployment`), which runs once as
+deployed and once per model; each run is graded, and the comparison — the four
+grades, cost and time per model, and what would make the agents do better — is
+printed when it is done. It always needs the login: it runs the deployment.
+Every evaluation, of either kind, is filed under the project id.
 """
 
 from __future__ import annotations
@@ -23,8 +30,9 @@ import os
 from pathlib import Path
 import sys
 import time
-from typing import Any
+from typing import Any, NoReturn, cast
 from urllib.parse import urlparse
+import uuid
 import webbrowser
 
 import click
@@ -33,11 +41,16 @@ from crewai_core.settings import Settings
 from dotenv import load_dotenv, set_key
 import httpx
 from rich.console import Console
+from rich.table import Table
 from rich.text import Text
 
 from crewai_cli.authentication.token import AuthError, get_auth_token
 from crewai_cli.plus_api import PlusAPI
-from crewai_cli.utils import get_or_create_project_id, is_dmn_mode_enabled
+from crewai_cli.utils import (
+    get_or_create_project_id,
+    get_project_id,
+    is_dmn_mode_enabled,
+)
 
 
 console = Console()
@@ -107,7 +120,7 @@ def _note(text: str, style: str = "dim") -> None:
 
 def eval_crew(run_id: str | None = None) -> None:
     """Evaluate the last traced run of this project, or the run RUN_ID."""
-    get_or_create_project_id()
+    project_id = get_or_create_project_id()
     # Read before the project's .env is loaded, so a project cannot add itself.
     trusted = _trusted_amp_origins()
     _load_project_env()
@@ -140,6 +153,7 @@ def eval_crew(run_id: str | None = None) -> None:
             client,
             execution_id,
             wait_for_spans=run_id is None and _ran_just_now(record),
+            project_id=project_id,
         )
     except EvaluationStoppedError as stopped:
         _fail(str(stopped))
@@ -334,7 +348,13 @@ def evaluate_run(
     for them is always on here.
     """
     client = _amp_client(_machine_amp_origins(), note=note)
-    started = _start_evaluation(client, execution_id, wait_for_spans=True, note=note)
+    started = _start_evaluation(
+        client,
+        execution_id,
+        wait_for_spans=True,
+        note=note,
+        project_id=get_project_id(),
+    )
     # After the start, exactly as the command counts it: `cli_usage:eval` is the
     # count of evaluations that began, and an evaluation the app runs is one.
     _record_usage(logged_in=client.api_key is not None)
@@ -622,6 +642,7 @@ def _start_evaluation(
     *,
     wait_for_spans: bool = False,
     note: Callable[[str], None] = _note,
+    project_id: str | None = None,
 ) -> dict[str, Any]:
     deadline = time.monotonic() + SPANS_WAIT_SECONDS if wait_for_spans else 0.0
     said = False
@@ -630,7 +651,9 @@ def _start_evaluation(
     eval_config = project_eval_config(note=note)
     while True:
         try:
-            response = client.create_evaluation(execution_id, eval_config=eval_config)
+            response = client.create_evaluation(
+                execution_id, eval_config=eval_config, project_id=project_id
+            )
         except httpx.HTTPError as error:
             raise EvaluationStoppedError(
                 f"Could not reach AMP to start the evaluation: {error}"
@@ -647,6 +670,17 @@ def _start_evaluation(
             time.sleep(SPANS_POLL_SECONDS)
             continue
         break
+    return _accepted(response, f"run {execution_id}", note=note)
+
+
+def _accepted(
+    response: httpx.Response,
+    subject: str,
+    *,
+    note: Callable[[str], None] = _note,
+    about_a_deployment: bool = False,
+) -> dict[str, Any]:
+    """AMP's answer to a start: the evaluation it opened, or its refusal as a sentence."""
     if response.status_code in (200, 202):
         payload = _payload(response)
         # The id is what every later call is made with, so a missing or
@@ -666,7 +700,9 @@ def _start_evaluation(
         raise EvaluationStoppedError(
             f"AMP answered without an evaluation id ({response.status_code})."
         )
-    raise EvaluationStoppedError(_refusal_message(response, f"run {execution_id}"))
+    raise EvaluationStoppedError(
+        _refusal_message(response, subject, about_a_deployment=about_a_deployment)
+    )
 
 
 def _report_url(value: Any) -> str | None:
@@ -705,12 +741,16 @@ def _wait(
     url: str | None,
     *,
     on_status: Callable[[dict[str, Any]], None] | None = None,
+    answer: str = "verdict",
 ) -> dict[str, Any]:
     """Poll until the evaluation is done or failed.
 
     Every unfinished answer goes to ON_STATUS, so a caller that has somewhere to
-    show progress can show it; the caller decides what a Ctrl-C means.
+    show progress can show it; the caller decides what a Ctrl-C means. A done
+    answer must carry a well-formed ANSWER: a run's `verdict`, or a models
+    evaluation's `comparison`.
     """
+    well_formed = _WELL_FORMED[answer]
     where = f" at {url}" if url else ""
     subject = f"evaluation {evaluation_id}"
     misses = (
@@ -738,9 +778,9 @@ def _wait(
         misses = 0
         payload = _payload(response) or {}
         status = payload.get("status")
-        if status == "done" and not _well_formed_verdict(payload.get("verdict")):
+        if status == "done" and not well_formed(payload):
             raise EvaluationStoppedError(
-                f"AMP answered done without a verdict (protocol error); follow it{where or ' on AMP'}."
+                f"AMP answered done without a {answer} (protocol error); follow it{where or ' on AMP'}."
             )
         if status in FINISHED:
             return payload
@@ -753,6 +793,10 @@ def _wait(
         time.sleep(POLL_SECONDS)
 
 
+def _carries_a_verdict(payload: dict[str, Any]) -> bool:
+    return _well_formed_verdict(payload.get("verdict"))
+
+
 def _well_formed_verdict(verdict: Any) -> bool:
     """`{"gate": "<word>", "grades": {area: 1..5 | null}}` — anything else is a
     protocol error. A grade is an exact integer in range: `True` is an `int` to
@@ -769,6 +813,403 @@ def _a_grade(grade: Any) -> bool:
     return grade is None or (type(grade) is int and 1 <= grade <= 5)
 
 
+def _carries_a_comparison(payload: dict[str, Any]) -> bool:
+    """`{"comparison": {"models": [{...}, ...]}}` — the rows are read cell by
+    cell when printed, and a cell that is not what it should be prints as "—";
+    a comparison with no rows to print is a protocol error."""
+    comparison = payload.get("comparison")
+    return (
+        isinstance(comparison, dict)
+        and isinstance(comparison.get("models"), list)
+        and bool(comparison["models"])
+        and all(isinstance(row, dict) for row in comparison["models"])
+    )
+
+
+_WELL_FORMED: dict[str, Callable[[dict[str, Any]], bool]] = {
+    "verdict": _carries_a_verdict,
+    "comparison": _carries_a_comparison,
+}
+
+
+# ── crewai eval --models ─────────────────────────────────────────────────────
+
+MAX_MODELS = 5
+MAX_MODEL_CHARS = 200
+MODELS_EXAMPLE = '--models "openai/gpt-4o-mini,anthropic/claude-haiku-4-5"'
+LOGIN_REQUIRED = (
+    "Comparing models runs your deployment, so it needs your CrewAI AMP account: "
+    "log in with `crewai login` and run it again."
+)
+GRADE_COLUMNS = ("goal", "tasks", "agents", "tools")
+TOP_SUGGESTIONS = 3
+
+
+def parse_models(text: str | None) -> list[str]:
+    """The models to compare, from ONE comma-separated list.
+
+    Items are stripped and a repeat is dropped. Each must name its provider
+    (`provider/model`) — the deployment builds the model from exactly that
+    string, and a bare `gpt-4o` would be a guess about which provider bills it.
+    """
+    models: list[str] = []
+    for raw in (text or "").split(","):
+        item = raw.strip()
+        if not item or item in models:
+            continue
+        provider, separator, name = item.partition("/")
+        if not (separator and provider.strip() and name.strip()):
+            raise EvaluationStoppedError(
+                f"{item!r} names no provider: write each model as provider/model, "
+                f"e.g. {MODELS_EXAMPLE}."
+            )
+        if len(item) > MAX_MODEL_CHARS:
+            raise EvaluationStoppedError(
+                f"{item[:40]!r}… is longer than {MAX_MODEL_CHARS} characters; "
+                "that is not a model name."
+            )
+        models.append(item)
+    if not models:
+        raise EvaluationStoppedError(
+            f"--models names no model: give one to {MAX_MODELS}, e.g. {MODELS_EXAMPLE}."
+        )
+    if len(models) > MAX_MODELS:
+        raise EvaluationStoppedError(
+            f"--models names {len(models)} models; compare at most {MAX_MODELS} at once."
+        )
+    return models
+
+
+def _deployment_id(value: str | None) -> str | None:
+    """`--deployment` as the UUID AMP knows the deployment by, or None."""
+    if value is None:
+        return None
+    try:
+        return str(uuid.UUID(value.strip()))
+    except ValueError:
+        raise EvaluationStoppedError(
+            f"--deployment {value!r} is not a deployment id; it is the UUID AMP "
+            "shows for the deployment."
+        ) from None
+
+
+# Model names that are the same for everyone who types them are public. What
+# else a `--models` item can carry is a customer's: a fine-tune id
+# (`openai/ft:gpt-4o-mini:acme-corp::abc`), an Azure deployment name, a
+# self-hosted model or host. Those are sent as `<provider>/other`.
+OTHER_MODEL = "other"
+# The vendor segment of an aggregator id (`openrouter/openai/gpt-4o-mini`)
+# that is itself public.
+_PUBLIC_VENDORS = frozenset(
+    {"openai", "anthropic", "google", "meta-llama", "mistralai", "deepseek", "qwen"}
+)
+
+
+def telemetry_model_name(model: str) -> str:
+    """MODEL as the usage stats may carry it: as typed when it is a model crewAI
+    knows, else `<provider>/other` — and `other/other` for a provider crewAI does
+    not know, since a provider string can name a host too.
+
+    Known means an exact entry of crewAI's model catalog (the context-window
+    tables every provider resolves against), never a prefix of one: a fine-tune
+    or a deployment named after a public model is still the customer's.
+    """
+    catalog, providers = _known_models_and_providers()
+    provider, _, name = model.partition("/")
+    if provider not in providers:
+        return f"{OTHER_MODEL}/{OTHER_MODEL}"
+    vendor, nested, tail = name.partition("/")
+    public = (
+        name in catalog
+        or model in catalog
+        or (nested and vendor in _PUBLIC_VENDORS and tail in catalog)
+    )
+    if not public or any(part.startswith("ft:") for part in model.split("/")):
+        return f"{provider}/{OTHER_MODEL}"
+    return model
+
+
+def _known_models_and_providers() -> tuple[frozenset[str], frozenset[str]]:
+    """crewAI's catalog of models and the providers it routes; empty when this
+    environment's crewai cannot say, so every model is then sent as "other"."""
+    from crewai_cli.constants import PROVIDERS
+
+    try:
+        from crewai.llm import SUPPORTED_NATIVE_PROVIDERS
+        from crewai.llms.context_window import LLM_CONTEXT_WINDOW_SIZES
+    except Exception:
+        return frozenset(), frozenset()
+    return (
+        frozenset(LLM_CONTEXT_WINDOW_SIZES),
+        frozenset(SUPPORTED_NATIVE_PROVIDERS) | frozenset(PROVIDERS),
+    )
+
+
+def _record_models_usage(models: list[str]) -> None:
+    """Count a comparison that is actually starting, and which models it compares.
+
+    The models go through `telemetry_model_name`: a public model by name, any
+    other as `<provider>/other`. Nothing names the run, the deployment or the
+    organization. Always logged in: a comparison cannot start without the
+    account.
+    """
+    try:
+        from crewai_core.telemetry import Telemetry
+
+        telemetry = Telemetry()
+        telemetry.set_tracer()
+        telemetry.feature_usage_span(
+            "cli_usage:eval_models",
+            {
+                "authenticated": "true",
+                "models": ",".join(telemetry_model_name(m) for m in models),
+                "models_count": str(len(models)),
+            },
+        )
+    except Exception:  # noqa: S110 - telemetry must never break a command
+        pass
+
+
+def eval_models(models_text: str, deployment_id: str | None = None) -> None:
+    """Run this project's deployment once as deployed and once per model, and compare.
+
+    The deployment is AMP's to find, by the project id, unless DEPLOYMENT_ID
+    names one; its own models are the baseline, read off the deployment rather
+    than this checkout, which may differ from what was deployed.
+    """
+    try:
+        models = parse_models(models_text)
+        deployment = _deployment_id(deployment_id)
+    except EvaluationStoppedError as stopped:
+        _fail(str(stopped))
+    if not Path("pyproject.toml").is_file():
+        _fail(
+            "No crewAI project here (no pyproject.toml). Run `crewai eval --models` "
+            "from the directory of the project you deployed."
+        )
+    project_id = get_or_create_project_id()
+    if not project_id:
+        _fail(
+            "Could not read or write [tool.crewai].project_id in pyproject.toml, which "
+            "is how AMP finds this project's deployment."
+        )
+    # Read before the project's .env is loaded, so a project cannot add itself.
+    trusted = _trusted_amp_origins()
+    _load_project_env()
+    try:
+        client = _amp_client(trusted)
+    except EvaluationStoppedError as stopped:
+        _fail(str(stopped))
+    if client.api_key is None:
+        _fail(LOGIN_REQUIRED)
+
+    try:
+        response = client.create_models_evaluation(
+            models,
+            project_id=project_id,
+            eval_config=project_eval_config(),
+            deployment_id=deployment,
+        )
+    except httpx.HTTPError as error:
+        _fail(f"Could not reach AMP to start the comparison: {error}")
+    try:
+        started = _accepted(response, "the comparison", about_a_deployment=True)
+    except EvaluationStoppedError as stopped:
+        _fail(str(stopped))
+    # After, not before, as `cli_usage:eval` is counted: a refused request —
+    # no deployment, one this account may not run — is not a comparison.
+    _record_models_usage(models)
+    url = started.get("url")
+    console.print(
+        Text("Comparing ")
+        .append(", ".join(models), style="bold")
+        .append(" with the deployed models")
+    )
+    if url:
+        # Appended, never interpolated — the same reason as `eval_crew`'s link.
+        console.print(Text("Follow it at ").append(url, style="cyan underline"))
+        _open(url)
+
+    console.print("Waiting for the comparison…", style="dim")
+    shown: list[str] = []
+
+    def show_progress(payload: dict[str, Any]) -> None:
+        line = _progress_line(payload)
+        if line and (not shown or shown[-1] != line):
+            shown.append(line)
+            _note(line)
+
+    try:
+        finished = _wait(
+            client, started["id"], url, on_status=show_progress, answer="comparison"
+        )
+    except EvaluationStoppedError as stopped:
+        _fail(str(stopped))
+    except KeyboardInterrupt:
+        console.print(
+            Text(f"\nStill running{f' at {url}' if url else ''}."), style="yellow"
+        )
+        raise SystemExit(130) from None
+    _print_comparison(finished, url)
+    # The criteria the comparison was graded on, for the project to edit — only
+    # when it has none, exactly as after a Mode 1 evaluation.
+    _say_where_the_criteria_live(write_eval_config(finished))
+    if finished.get("status") != "done":
+        raise SystemExit(1)
+
+
+def _progress_line(payload: dict[str, Any]) -> str | None:
+    """One line for where the comparison is, from the poll's `progress` (or the
+    last of its `events`): which model is running, "model 2 of 3", and the
+    subject being judged. What is not there is left out, never guessed."""
+    progress = payload.get("progress")
+    events = payload.get("events")
+    if progress is None and isinstance(events, list) and events:
+        progress = events[-1]
+    if isinstance(progress, str):
+        return progress.strip() or None
+    if not isinstance(progress, dict):
+        return None
+    if isinstance(progress.get("message"), str) and progress["message"].strip():
+        return str(progress["message"]).strip()
+    event = progress.get("event")
+    detail = progress.get("payload")
+    if not isinstance(detail, dict):
+        detail = progress
+    index, total = detail.get("index"), detail.get("total")
+    position = (
+        f"model {index + 1} of {total}"
+        if type(index) is int and type(total) is int and 0 <= index < total
+        else None
+    )
+    name = next(
+        (
+            str(detail[key])
+            for key in ("label", "key")
+            if isinstance(detail.get(key), str) and detail[key]
+        ),
+        None,
+    )
+    subject = detail.get("subject")
+    if event == "judging" or isinstance(subject, str):
+        parts = [f"judging {subject}" if isinstance(subject, str) else "judging"]
+    elif event == "configuration_done":
+        parts = ["graded"]
+    else:
+        parts = ["running"]
+    parts += [part for part in (position, name) if part]
+    known = event is not None or isinstance(subject, str) or len(parts) > 1
+    return " · ".join(parts) if known else None
+
+
+def _print_comparison(finished: dict[str, Any], url: str | None) -> None:
+    """The models side by side, then what would make them better.
+
+    Every cell came over the wire, so each is a `Text`: a label such as
+    `Writer: [red]x[/red]` prints as written. The baseline — the deployment as
+    it is — is marked, and the best value in each column is starred: the highest
+    grade, the lowest cost and time. A column where every model is the same, or
+    only one has a value, stars nothing: there is no "best" to point at.
+    """
+    if finished.get("status") != "done":
+        console.print(
+            Text(f"Comparison failed: {finished.get('error') or 'no reason given'}"),
+            style="bold red",
+        )
+        if url:
+            console.print(Text(f"Report: {url}"))
+        return
+    comparison = finished["comparison"]  # _wait let only a well-formed one through
+    rows: list[dict[str, Any]] = comparison["models"]
+
+    grades = {area: [_grade_of(row, area) for row in rows] for area in GRADE_COLUMNS}
+    costs = [_number(row.get("cost_usd")) for row in rows]
+    seconds = [_number(row.get("seconds")) for row in rows]
+    best = {area: _best(values, max) for area, values in grades.items()}
+    cheapest, fastest = _best(costs, min), _best(seconds, min)
+
+    table = Table(show_edge=False, pad_edge=False)
+    for header in ("model", *GRADE_COLUMNS, "cost", "time"):
+        table.add_column(header, justify="left" if header == "model" else "right")
+    for n, row in enumerate(rows):
+        label = Text(str(row.get("label") or row.get("key") or "?"))
+        if row.get("baseline") is True:
+            label.append(" (deployed)", style="dim")
+        cells = [label]
+        for area in GRADE_COLUMNS:
+            grade = grades[area][n]
+            cells.append(
+                _starred(f"{grade}/5" if grade is not None else "—", grade, best[area])
+            )
+        cost, took = costs[n], seconds[n]
+        cells.append(
+            _starred(f"${cost:.4f}" if cost is not None else "—", cost, cheapest)
+        )
+        cells.append(
+            _starred(f"{took:.1f}s" if took is not None else "—", took, fastest)
+        )
+        table.add_row(*cells)
+    console.print(table)
+
+    suggestions = _top_suggestions(comparison.get("suggestions"))
+    if suggestions:
+        console.print(Text("What would make it better", style="bold"))
+    for n, item in enumerate(suggestions, 1):
+        where = " — ".join(
+            str(item[key])
+            for key in ("subject", "field")
+            if isinstance(item.get(key), str)
+        )
+        line = Text(f"{n}. ")
+        if where:
+            line.append(where, style="bold").append(": ")
+        line.append(str(item.get("problem") or ""))
+        if item.get("shared") is True:
+            line.append(" (every model)", style="dim")
+        console.print(line)
+        if isinstance(item.get("change"), str) and item["change"].strip():
+            console.print(Text(f"   change: {item['change'].strip()}"))
+    if url:
+        console.print(Text(f"Full report: {url}"))
+
+
+def _grade_of(row: dict[str, Any], area: str) -> int | None:
+    grades = row.get("grades")
+    grade = grades.get(area) if isinstance(grades, dict) else None
+    return grade if grade is not None and _a_grade(grade) else None
+
+
+def _number(value: Any) -> float | None:
+    """A non-negative number, or None: `True` is an `int` to Python and not a cost."""
+    if isinstance(value, bool) or not isinstance(value, (int, float)):
+        return None
+    return float(value) if value >= 0 else None
+
+
+def _best(values: list[Any], pick: Callable[[list[Any]], Any]) -> float | int | None:
+    known = [value for value in values if value is not None]
+    if len(known) < 2 or len(set(known)) == 1:
+        return None
+    return cast(float | int, pick(known))
+
+
+def _starred(text: str, value: Any, best: Any) -> Text:
+    cell = Text(text)
+    if best is not None and value == best:
+        cell.append(" ★", style="yellow")
+    return cell
+
+
+def _top_suggestions(value: Any) -> list[dict[str, Any]]:
+    """The first few suggestions, the ones every model needed first — a problem
+    every model had points at the prompt, not at a model."""
+    if not isinstance(value, list):
+        return []
+    items = [item for item in value if isinstance(item, dict) and item.get("problem")]
+    items.sort(key=lambda item: item.get("shared") is not True)
+    return items[:TOP_SUGGESTIONS]
+
+
 def _print_verdict(finished: dict[str, Any], url: str | None) -> None:
     """Everything printed here came over the wire, so it is composed as `Text`
     and never as markup: a `Console` parses square brackets, and an area named
@@ -817,15 +1258,26 @@ def _payload(response: httpx.Response) -> dict[str, Any] | None:
     return loaded if isinstance(loaded, dict) else None
 
 
-def _refusal_message(response: httpx.Response, subject: str) -> str:
+def _refusal_message(
+    response: httpx.Response, subject: str, *, about_a_deployment: bool = False
+) -> str:
     """AMP's own words when it sent them. SUBJECT is "run <id>" or "evaluation <id>".
 
     A sentence, not a print: the terminal and the run app both show it, and only
-    one of them shows it by printing.
+    one of them shows it by printing. ABOUT_A_DEPLOYMENT: a 403 that is not about
+    the credential — this account may not run that deployment — is AMP's
+    sentence alone, because logging in again changes nothing there.
     """
     payload = _payload(response) or {}
     message = str(payload.get("message") or "").strip()
     error = str(payload.get("error") or "")
+    if (
+        about_a_deployment
+        and response.status_code == 403
+        and error not in {"bad_credentials", "account_required"}
+        and message
+    ):
+        return message
     if response.status_code in (401, 403):
         if error == "account_required" and message:
             return message
@@ -843,7 +1295,7 @@ def _refused(response: httpx.Response, subject: str) -> None:
     _fail(_refusal_message(response, subject))
 
 
-def _fail(message: str) -> None:
+def _fail(message: str) -> NoReturn:
     # `Text`, because most of what reaches here is AMP's own sentence and a
     # `Console` parses square brackets. No caller relies on markup; the colour
     # comes from `style`.

lib/cli/src/crewai_cli/plus_api.py

@@ -28,17 +28,56 @@ class PlusAPI(_CorePlusAPI):
     EVALUATION_POLL_TIMEOUT = 30.0
 
     def create_evaluation(
-        self, execution_id: str, *, eval_config: str | None = None
+        self,
+        execution_id: str,
+        *,
+        eval_config: str | None = None,
+        project_id: str | None = None,
     ) -> httpx.Response:
         """Ask AMP to evaluate the traced run EXECUTION_ID (crewai eval).
 
         EVAL_CONFIG is the project's own `eval.jsonc` when it has one: what
         good means for this crew, in its own words. Sent as it was written,
-        comments and all, and read by the grader rather than here.
+        comments and all, and read by the grader rather than here. PROJECT_ID
+        is `[tool.crewai].project_id`, so the evaluation is filed under the
+        project it is about.
         """
         body: dict[str, str] = {"execution_id": execution_id}
         if eval_config:
             body["eval_config"] = eval_config
+        if project_id:
+            body["project_id"] = project_id
+        return self._make_request(
+            "POST",
+            self.EVALUATIONS_RESOURCE,
+            json=body,
+            timeout=self.EVALUATION_START_TIMEOUT,
+        )
+
+    def create_models_evaluation(
+        self,
+        models: list[str],
+        *,
+        project_id: str,
+        eval_config: str | None = None,
+        deployment_id: str | None = None,
+    ) -> httpx.Response:
+        """Ask AMP to run the project's deployment once per model and compare them.
+
+        AMP finds the deployment by PROJECT_ID (`[tool.crewai].project_id`)
+        unless DEPLOYMENT_ID names one. MODELS are `provider/model` strings;
+        the deployment's own models are the baseline, so they are not sent.
+        EVAL_CONFIG is the project's `eval.jsonc`, as `create_evaluation` sends it.
+        """
+        body: dict[str, Any] = {
+            "kind": "models",
+            "project_id": project_id,
+            "models": list(models),
+        }
+        if eval_config:
+            body["eval_config"] = eval_config
+        if deployment_id:
+            body["deployment_id"] = deployment_id
         return self._make_request(
             "POST",
             self.EVALUATIONS_RESOURCE,
@@ -90,11 +129,18 @@ class PlusAPI(_CorePlusAPI):
         *,
         name: str | None = None,
         env: dict[str, str] | None = None,
+        project_id: str | None = None,
     ) -> httpx.Response:
-        """Create a crew deployment from a local project ZIP archive."""
+        """Create a crew deployment from a local project ZIP archive.
+
+        PROJECT_ID is `[tool.crewai].project_id`, so AMP knows which project the
+        deployment runs; sent only when the project has one.
+        """
         data: dict[str, str] = {}
         if name:
             data["name"] = name
+        if project_id:
+            data["project_id"] = project_id
         if env:
             data.update({f"env[{key}]": value for key, value in env.items()})
         return self._make_multipart_request(
@@ -111,9 +157,15 @@ class PlusAPI(_CorePlusAPI):
         zip_file_path: str | Path,
         *,
         env: dict[str, str] | None = None,
+        project_id: str | None = None,
     ) -> httpx.Response:
-        """Update an existing crew deployment from a local project ZIP archive."""
+        """Update an existing crew deployment from a local project ZIP archive.
+
+        PROJECT_ID as for `create_crew_from_zip`.
+        """
         data: dict[str, str] = {}
+        if project_id:
+            data["project_id"] = project_id
         if env:
             data.update({f"env[{key}]": value for key, value in env.items()})
         return self._make_multipart_request(

lib/cli/tests/deploy/test_deploy_main.py

@@ -1316,3 +1316,127 @@ class TestDeployCommand(unittest.TestCase):
         from crewai_cli.version import get_crewai_version
 
         assert isinstance(get_crewai_version(), str)
+
+
+PROJECT_ID = "1f0c6a3e-8d0b-4c4e-9a52-6a9f6d1d2b7e"
+
+
+class TestDeploySendsTheProjectId(unittest.TestCase):
+    """Every create and push carries `[tool.crewai].project_id` when the
+    project has one, so AMP knows which project a deployment runs."""
+
+    @patch("crewai_cli.command.get_auth_token", return_value="test_token")
+    @patch("crewai_cli.deploy.main.get_project_id", return_value=PROJECT_ID)
+    @patch("crewai_cli.deploy.main.get_project_name", return_value="test_project")
+    @patch("crewai_cli.command.PlusAPI")
+    def setUp(self, _plus_api, _name, _project_id, _token):
+        self.deploy_command = deploy_main.DeployCommand()
+        self.mock_client = self.deploy_command.plus_api_client
+        self.mock_client.base_url = "https://app.crewai.com"
+        browser = patch("crewai_cli.deploy.main.webbrowser.open")
+        browser.start()
+        self.addCleanup(browser.stop)
+
+    def _created(self, uuid: str = "new-uuid") -> MagicMock:
+        response = MagicMock()
+        response.status_code = 201
+        response.is_success = True
+        response.json.return_value = {"uuid": uuid, "status": "created"}
+        return response
+
+    def _status(self, zip_deployment: bool) -> MagicMock:
+        response = MagicMock()
+        response.is_success = True
+        response.json.return_value = {"uuid": "test-uuid", "zip_deployment": zip_deployment}
+        return response
+
+    def test_the_command_reads_the_project_id(self):
+        self.assertEqual(self.deploy_command.project_id, PROJECT_ID)
+
+    @patch("crewai_cli.deploy.main.fetch_and_json_env_file", return_value={})
+    @patch("crewai_cli.deploy.main.git.Repository")
+    def test_create_from_git_sends_it(self, mock_repository, _env):
+        mock_repository.return_value.origin_url.return_value = "https://github.com/t/r.git"
+        mock_repository.return_value.create_initial_commit_if_needed.return_value = False
+        self.mock_client.create_crew.return_value = self._created()
+
+        with patch("sys.stdout", new=StringIO()):
+            self.deploy_command.create_crew(confirm=True, skip_validate=True)
+
+        payload = self.mock_client.create_crew.call_args.args[0]
+        self.assertEqual(payload["deploy"]["project_id"], PROJECT_ID)
+        self.assertEqual(payload["project_id"], PROJECT_ID)
+
+    @patch("crewai_cli.deploy.main.create_project_zip", return_value=Path("/tmp/p.zip"))
+    @patch("crewai_cli.deploy.main.fetch_and_json_env_file", return_value={})
+    @patch("crewai_cli.deploy.main.git.Repository")
+    def test_create_from_a_zip_sends_it(self, mock_repository, _env, _zip):
+        mock_repository.return_value.origin_url.return_value = None
+        mock_repository.return_value.create_initial_commit_if_needed.return_value = False
+        self.mock_client.create_crew_from_zip.return_value = self._created()
+
+        with patch("sys.stdout", new=StringIO()):
+            self.deploy_command.create_crew(confirm=True, skip_validate=True)
+
+        self.mock_client.create_crew_from_zip.assert_called_once_with(
+            Path("/tmp/p.zip"), name="test_project", env={}, project_id=PROJECT_ID
+        )
+
+    @patch("crewai_cli.deploy.main.git.Repository")
+    @patch("crewai_cli.deploy.main.DeployCommand._display_deployment_info")
+    def test_a_push_from_git_sends_it(self, _display, mock_repository):
+        mock_repository.return_value.origin_url.return_value = "https://github.com/t/r.git"
+        mock_repository.return_value.create_initial_commit_if_needed.return_value = False
+        self.mock_client.crew_status_by_uuid.return_value = self._status(False)
+        self.mock_client.crew_status_by_name.return_value = self._status(False)
+
+        self.deploy_command.deploy(uuid="test-uuid", skip_validate=True)
+        self.deploy_command.deploy(skip_validate=True)
+
+        self.mock_client.deploy_by_uuid.assert_called_once_with(
+            "test-uuid", project_id=PROJECT_ID
+        )
+        self.mock_client.deploy_by_name.assert_called_once_with(
+            "test_project", project_id=PROJECT_ID
+        )
+
+    @patch("crewai_cli.deploy.main.create_project_zip", return_value=Path("/tmp/p.zip"))
+    @patch("crewai_cli.deploy.main.fetch_and_json_env_file", return_value={})
+    @patch("crewai_cli.deploy.main.git.Repository")
+    @patch("crewai_cli.deploy.main.DeployCommand._display_deployment_info")
+    def test_a_push_from_a_zip_sends_it(self, _display, mock_repository, _env, _zip):
+        mock_repository.return_value.origin_url.return_value = None
+        mock_repository.return_value.create_initial_commit_if_needed.return_value = False
+        self.mock_client.crew_status_by_uuid.return_value = self._status(True)
+
+        self.deploy_command.deploy(uuid="test-uuid", skip_validate=True)
+
+        self.mock_client.update_crew_from_zip.assert_called_once_with(
+            "test-uuid", Path("/tmp/p.zip"), env={}, project_id=PROJECT_ID
+        )
+
+
+def test_a_project_without_an_id_sends_none(monkeypatch, tmp_path: Path):
+    """Read, never minted: a deploy does not rewrite pyproject.toml."""
+    monkeypatch.chdir(tmp_path)
+    (tmp_path / "pyproject.toml").write_text("[project]\nname = 'demo'\n")
+    monkeypatch.setattr("crewai_cli.command.get_auth_token", lambda: "t")
+    monkeypatch.setattr("crewai_cli.command.PlusAPI", MagicMock())
+    monkeypatch.setattr(deploy_main, "get_project_name", lambda require=False: "demo")
+
+    command = deploy_main.DeployCommand()
+
+    assert command.project_id is None and command._project() == {}
+    assert "project_id" not in (tmp_path / "pyproject.toml").read_text()
+
+
+def test_the_project_id_is_read_from_pyproject(monkeypatch, tmp_path: Path):
+    monkeypatch.chdir(tmp_path)
+    (tmp_path / "pyproject.toml").write_text(
+        f"[project]\nname = 'demo'\n\n[tool.crewai]\nproject_id = \"{PROJECT_ID}\"\n"
+    )
+    monkeypatch.setattr("crewai_cli.command.get_auth_token", lambda: "t")
+    monkeypatch.setattr("crewai_cli.command.PlusAPI", MagicMock())
+    monkeypatch.setattr(deploy_main, "get_project_name", lambda require=False: "demo")
+
+    assert deploy_main.DeployCommand()._project() == {"project_id": PROJECT_ID}

lib/cli/tests/experimental/test_eval_crew.py

@@ -44,9 +44,10 @@ class FakeAMP:
         self.api_key = None
         self.headers: dict[str, str] = {"X-Crewai-Organization-Id": "org-42"}
 
-    def create_evaluation(self, execution_id, *, eval_config=None):
+    def create_evaluation(self, execution_id, *, eval_config=None, project_id=None):
         self.calls.append(("create", execution_id))
         self.sent_config = eval_config
+        self.sent_project_id = project_id
         if isinstance(self.create, list):
             return self.create.pop(0) if len(self.create) > 1 else self.create[0]
         return self.create
@@ -679,7 +680,7 @@ def test_an_oversized_config_is_said_once_and_through_the_callers_note(
     answers = iter([httpx.Response(404, json={}), httpx.Response(404, json={}), QUEUED])
 
     class Client:
-        def create_evaluation(self, execution_id, *, eval_config=None):
+        def create_evaluation(self, execution_id, *, eval_config=None, project_id=None):
             sent.append(eval_config)
             return next(answers)
 
@@ -744,7 +745,7 @@ def test_only_a_web_address_is_printed_and_opened(url, opened):
 
 def test_a_report_url_that_is_not_a_web_address_is_dropped_with_a_note(monkeypatch):
     class Client:
-        def create_evaluation(self, execution_id, *, eval_config=None):
+        def create_evaluation(self, execution_id, *, eval_config=None, project_id=None):
             return httpx.Response(
                 202, json={"id": "ev-1", "url": "file:///etc/passwd", "status": "queued"}
             )
@@ -1207,3 +1208,395 @@ def test_an_unreadable_login_is_the_reason_given_when_nothing_was_traced(
     assert "[/Users/me/.config/crewai]" in out
     assert "add CREWAI_TRACING_ENABLED=true" not in out
 
+
+# ── Mode 1 files the evaluation under the project ───────────────────────────
+
+
+def test_the_evaluation_is_filed_under_the_project(project, monkeypatch):
+    directory, _ = project
+    record_last_run(directory)
+    monkeypatch.setattr(eval_module, "get_or_create_project_id", lambda: "proj-1")
+    amp = install(monkeypatch, FakeAMP(statuses=[done()]))
+
+    eval_module.eval_crew()
+
+    assert amp.sent_project_id == "proj-1"
+
+
+def test_the_app_files_its_evaluation_under_the_project_too(project, monkeypatch):
+    """The run app's door reads the id without minting one: it runs inside a
+    crew's process, where rewriting pyproject.toml is not its business."""
+    monkeypatch.setattr(eval_module, "get_project_id", lambda: "proj-2")
+    amp = install(monkeypatch, FakeAMP(statuses=[done()]))
+
+    eval_module.evaluate_run(EXECUTION_ID, on_started=lambda started: None)
+
+    assert amp.sent_project_id == "proj-2"
+
+
+# ── crewai eval --models ─────────────────────────────────────────────────────
+
+MODELS = "openai/gpt-4o-mini, openrouter/meta-llama/llama-4-maverick"
+DEPLOYMENT = "9b0a4a53-3f7c-4a8e-9a55-2f1c1e0f8e11"
+
+
+def comparison(**overrides):
+    body = {
+        "models": [
+            {
+                "key": "base", "label": "Poem composer: openai/gpt-5.6-sol", "baseline": True,
+                "grades": {"goal": 5, "tasks": 4, "agents": 3, "tools": None},
+                "gate": "passed", "cost_usd": 0.0040, "seconds": 9.5, "tokens": 900,
+            },
+            {
+                "key": "mini", "label": "Poem composer: openai/gpt-4o-mini", "baseline": False,
+                "grades": {"goal": 5, "tasks": 5, "agents": 2, "tools": None},
+                "gate": "passed", "cost_usd": 0.0012, "seconds": 7.6, "tokens": 736,
+            },
+        ],
+        "suggestions": [
+            {"subject": "agent:Poem composer", "field": "backstory", "problem": "only one model had it",
+             "evidence": "x", "change": "only-one change", "models_affected": ["mini"], "shared": False},
+            {"subject": "agent:Poem composer", "field": "goal", "problem": "the poem ignores the topic",
+             "evidence": "y", "change": "Write a poem about {topic}.", "models_affected": ["base", "mini"],
+             "shared": True},
+            {"subject": "task:write", "field": "expected_output", "problem": "no length", "change": "4 lines"},
+            {"subject": "agent:Poem composer", "field": "tools", "problem": "a fourth", "change": "never shown"},
+        ],
+    }
+    body.update(overrides)
+    return body
+
+
+def compared(**overrides):
+    return httpx.Response(200, json={"id": "ev-9", "status": "done", "url": URL, "comparison": comparison(**overrides)})
+
+
+class FakeModelsAMP(FakeAMP):
+    def __init__(self, create=None, statuses=None):
+        super().__init__(create=create if create is not None else httpx.Response(
+            202, json={"id": "ev-9", "url": URL, "status": "queued"}
+        ), statuses=statuses)
+        self.sent: dict | None = None
+
+    def create_models_evaluation(self, models, *, project_id, eval_config=None, deployment_id=None):
+        self.calls.append(("create_models", tuple(models)))
+        self.sent = {"models": models, "project_id": project_id, "eval_config": eval_config,
+                     "deployment_id": deployment_id}
+        return self.create
+
+
+@pytest.fixture
+def deployed(project, monkeypatch):
+    directory, opened = project
+    (directory / "pyproject.toml").write_text('[tool.crewai]\nproject_id = "proj-1"\n')
+    monkeypatch.setattr(eval_module, "get_or_create_project_id", lambda: "proj-1")
+    return directory, opened
+
+
+def test_models_are_compared_on_the_deployment_and_the_table_printed(deployed, monkeypatch, capsys):
+    directory, opened = deployed
+    (directory / "eval.jsonc").write_text('{"dataset": []}')
+    running = httpx.Response(200, json={"id": "ev-9", "status": "running", "progress": {
+        "event": "configuration_started", "payload": {"index": 1, "total": 3, "key": "mini"}}})
+    judging = httpx.Response(200, json={"id": "ev-9", "status": "running", "progress": {
+        "event": "judging", "payload": {"subject": "agent 'Poem composer'", "index": 1, "total": 3}}})
+    amp = install(monkeypatch, FakeModelsAMP(statuses=[running, running, judging, compared()]))
+
+    eval_module.eval_models(MODELS)
+
+    out = capsys.readouterr().out
+    assert amp.api_key == "login-token"
+    assert amp.sent == {
+        "models": ["openai/gpt-4o-mini", "openrouter/meta-llama/llama-4-maverick"],
+        "project_id": "proj-1", "eval_config": '{"dataset": []}', "deployment_id": None,
+    }
+    assert opened == [URL] and URL in out
+    # Progress: said once per change, never once per poll.
+    assert out.count("running · model 2 of 3 · mini") == 1
+    assert "judging agent 'Poem composer' · model 2 of 3" in out
+    lines = out.splitlines()
+    base = next(line for line in lines if "gpt-5.6-sol" in line)
+    mini = next(line for line in lines if "Poem composer: openai/gpt-4o-mini" in line)
+    assert "(deployed)" in base and "(deployed)" not in mini
+    # The best per column: tasks and agents differ, goal is a tie, tools nothing.
+    assert "5/5 ★" in mini and "4/5" in base and "4/5 ★" not in base
+    assert "3/5 ★" in base and "2/5 ★" not in mini
+    assert "$0.0012 ★" in mini and "7.6s ★" in mini and "9.5s ★" not in base
+    assert "5/5 ★" not in base  # goal: both 5, nothing to point at
+    # The top three suggestions, the one every model needed first.
+    assert "What would make it better" in out
+    assert out.index("the poem ignores the topic") < out.index("only one model had it")
+    assert "change: Write a poem about {topic}." in out
+    assert "a fourth" not in out
+
+
+def test_the_comparison_is_counted_with_the_models_and_nothing_that_names_it(deployed, monkeypatch):
+    spans: list[tuple[str, dict[str, str]]] = []
+
+    class FakeTelemetry:
+        def set_tracer(self) -> None:
+            pass
+
+        def feature_usage_span(self, feature, attributes=None) -> None:
+            spans.append((feature, attributes or {}))
+
+    monkeypatch.setattr("crewai_core.telemetry.Telemetry", FakeTelemetry)
+    install(monkeypatch, FakeModelsAMP(statuses=[compared()]))
+
+    eval_module.eval_models(MODELS)
+
+    assert spans == [("cli_usage:eval_models", {
+        "authenticated": "true",
+        # A public model by name; one crewAI's catalog does not list, by provider.
+        "models": "openai/gpt-4o-mini,openrouter/other",
+        "models_count": "2",
+    })]
+
+
+def test_a_refused_comparison_is_not_counted(deployed, monkeypatch, capsys):
+    spans: list[str] = []
+
+    class FakeTelemetry:
+        def set_tracer(self) -> None:
+            pass
+
+        def feature_usage_span(self, feature, attributes=None) -> None:
+            spans.append(feature)
+
+    monkeypatch.setattr("crewai_core.telemetry.Telemetry", FakeTelemetry)
+    refused = httpx.Response(404, json={"error": "deployment_not_found", "message": (
+        "No deployment of this project; deploy it with `crewai deploy create`, or pass --deployment.")})
+    install(monkeypatch, FakeModelsAMP(create=refused))
+
+    with pytest.raises(SystemExit) as stopped:
+        eval_module.eval_models(MODELS)
+
+    assert stopped.value.code == 1 and spans == []
+    assert "deploy it with `crewai deploy create`, or pass --deployment." in capsys.readouterr().out
+
+
+def test_the_deployment_named_is_sent(deployed, monkeypatch):
+    amp = install(monkeypatch, FakeModelsAMP(statuses=[compared()]))
+
+    eval_module.eval_models("openai/gpt-4o-mini", deployment_id=f" {DEPLOYMENT.upper()} ")
+
+    assert amp.sent["deployment_id"] == DEPLOYMENT
+
+
+def test_a_deployment_that_is_not_a_uuid_is_refused_before_anything_is_sent(deployed, monkeypatch, capsys):
+    amp = install(monkeypatch, FakeModelsAMP())
+
+    with pytest.raises(SystemExit) as stopped:
+        eval_module.eval_models("openai/gpt-4o-mini", deployment_id="poem_creator")
+
+    assert stopped.value.code == 1 and amp.calls == []
+    assert "is not a deployment id" in capsys.readouterr().out
+
+
+def test_comparing_models_needs_the_login(deployed, monkeypatch, capsys):
+    monkeypatch.setattr(eval_module, "saved_login", lambda: None)
+    amp = install(monkeypatch, FakeModelsAMP())
+
+    with pytest.raises(SystemExit) as stopped:
+        eval_module.eval_models(MODELS)
+
+    assert stopped.value.code == 1 and amp.calls == []
+    assert "log in with `crewai login`" in capsys.readouterr().out
+
+
+def test_comparing_models_happens_from_the_project(project, monkeypatch, capsys):
+    amp = install(monkeypatch, FakeModelsAMP())
+
+    with pytest.raises(SystemExit) as stopped:
+        eval_module.eval_models(MODELS)
+
+    assert stopped.value.code == 1 and amp.calls == []
+    assert "No crewAI project here" in capsys.readouterr().out
+
+
+@pytest.mark.parametrize(
+    ("text", "expected"),
+    [
+        ("openai/gpt-4o-mini", ["openai/gpt-4o-mini"]),
+        (" openai/gpt-4o-mini , openai/gpt-4o-mini,anthropic/claude-haiku-4-5, ",
+         ["openai/gpt-4o-mini", "anthropic/claude-haiku-4-5"]),
+        ("openrouter/meta-llama/llama-4-maverick", ["openrouter/meta-llama/llama-4-maverick"]),
+    ],
+)
+def test_the_models_are_one_list_stripped_with_repeats_dropped(text, expected):
+    assert eval_module.parse_models(text) == expected
+
+
+@pytest.mark.parametrize(
+    ("text", "said"),
+    [
+        ("gpt-4o-mini", "names no provider"),
+        ("openai/", "names no provider"),
+        ("/gpt-4o", "names no provider"),
+        (" , ", "names no model"),
+        (",".join(f"openai/m{n}" for n in range(6)), "at most 5"),
+        ("openai/" + "m" * 200, "longer than 200"),
+    ],
+)
+def test_a_list_amp_would_refuse_is_refused_here_with_an_example(text, said):
+    with pytest.raises(eval_module.EvaluationStoppedError, match=said) as refused:
+        eval_module.parse_models(text)
+    if "provider" in said or "names no model" in said:
+        assert "openai/gpt-4o-mini" in str(refused.value)
+
+
+def test_a_model_without_its_provider_exits_one_before_anything_is_sent(deployed, monkeypatch, capsys):
+    amp = install(monkeypatch, FakeModelsAMP())
+
+    with pytest.raises(SystemExit) as stopped:
+        eval_module.eval_models("gpt-4o-mini")
+
+    assert stopped.value.code == 1 and amp.calls == []
+    assert "provider/model" in capsys.readouterr().out
+
+
+def test_a_failed_comparison_exits_one_with_the_reason(deployed, monkeypatch, capsys):
+    failed = httpx.Response(200, json={"id": "ev-9", "status": "failed", "error": "the deployment answered 500"})
+    install(monkeypatch, FakeModelsAMP(statuses=[failed]))
+
+    with pytest.raises(SystemExit) as stopped:
+        eval_module.eval_models(MODELS)
+
+    assert stopped.value.code == 1
+    assert "Comparison failed: the deployment answered 500" in capsys.readouterr().out
+
+
+def test_done_without_a_comparison_is_a_protocol_error(deployed, monkeypatch, capsys):
+    install(monkeypatch, FakeModelsAMP(statuses=[httpx.Response(200, json={"id": "ev-9", "status": "done"})]))
+
+    with pytest.raises(SystemExit) as stopped:
+        eval_module.eval_models(MODELS)
+
+    assert stopped.value.code == 1
+    assert "done without a comparison (protocol error)" in capsys.readouterr().out
+
+
+def test_a_deployment_this_account_may_not_run_is_amps_sentence_alone(deployed, monkeypatch, capsys):
+    """Logging in again changes nothing about a deployment one may not run."""
+    refused = httpx.Response(403, json={"error": "not_allowed", "message": "You may not run poem_creator."})
+    install(monkeypatch, FakeModelsAMP(create=refused))
+
+    with pytest.raises(SystemExit):
+        eval_module.eval_models(MODELS)
+
+    out = capsys.readouterr().out
+    assert "You may not run poem_creator." in out and "crewai login" not in out
+
+
+def test_everything_from_the_wire_prints_literally(deployed, monkeypatch, capsys):
+    hostile = comparison(
+        models=[{"key": "k", "label": "[red]Writer[/red]: [link=https://evil.test]x[/link]", "baseline": True,
+                 "grades": {"goal": 9, "tasks": True, "agents": "5", "tools": None}, "cost_usd": True,
+                 "seconds": -1}],
+        suggestions=[{"subject": "[bold]agent[/bold]", "field": "goal", "problem": "[red]p[/red]",
+                      "change": "[link=https://evil.test]c[/link]"}],
+    )
+    install(monkeypatch, FakeModelsAMP(statuses=[httpx.Response(
+        200, json={"id": "ev-9", "status": "done", "url": URL, "comparison": hostile})]))
+
+    eval_module.eval_models(MODELS)
+
+    out = capsys.readouterr().out
+    assert "[red]Writer[/red]: [link=https://evil.test]x[/link]" in out
+    assert "[red]p[/red]" in out and "[link=https://evil.test]c[/link]" in out
+    # Not a grade, a cost or a time: each prints as "—", never as a value.
+    row = next(line for line in out.splitlines() if "Writer" in line)
+    assert "9/5" not in row and "True" not in row and "-1" not in row and "5/5" not in row
+    assert row.count("—") == 6
+
+
+@pytest.mark.parametrize(
+    ("progress", "line"),
+    [
+        ("Setting up the deployment", "Setting up the deployment"),
+        ({"message": "model 2 of 3: mini"}, "model 2 of 3: mini"),
+        ({"event": "configuration_done", "payload": {"index": 0, "total": 2, "key": "base"}},
+         "graded · model 1 of 2 · base"),
+        ({"subject": "the final output"}, "judging the final output"),
+        ({"unknown": "shape"}, None),
+        (["not", "a", "dict"], None),
+    ],
+)
+def test_the_progress_line_says_what_is_there_and_guesses_nothing(progress, line):
+    assert eval_module._progress_line({"progress": progress}) == line
+
+
+def test_the_last_event_stands_in_for_progress():
+    events = [{"event": "configuration_started", "payload": {"index": 0, "total": 2, "key": "base"}},
+              {"event": "judging", "payload": {"subject": "task 'write'"}}]
+
+    assert eval_module._progress_line({"events": events}) == "judging task 'write'"
+
+
+def test_the_cli_maps_models_and_deployment_to_the_comparison(monkeypatch):
+    calls = []
+    monkeypatch.setattr("crewai_cli.cli.eval_models", lambda *args, **kwargs: calls.append((args, kwargs)))
+    monkeypatch.setattr("crewai_cli.cli.eval_crew", lambda **kwargs: calls.append(("run", kwargs)))
+    runner = CliRunner()
+
+    assert runner.invoke(eval_command, ["--models", MODELS]).exit_code == 0
+    assert runner.invoke(eval_command, ["--models", "openai/gpt-4o", "--deployment", DEPLOYMENT]).exit_code == 0
+    assert calls == [((MODELS,), {"deployment_id": None}), (("openai/gpt-4o",), {"deployment_id": DEPLOYMENT})]
+
+    both = runner.invoke(eval_command, ["--models", MODELS, "--run", EXECUTION_ID])
+    alone = runner.invoke(eval_command, ["--deployment", DEPLOYMENT])
+    assert both.exit_code == 2 and "Give one of them" in both.output
+    assert alone.exit_code == 2 and "add --models LIST" in alone.output
+    assert len(calls) == 2
+    assert "--models" in runner.invoke(eval_command, ["--help"]).output
+
+
+def test_a_comparison_leaves_its_criteria_behind_when_the_project_has_none(deployed, monkeypatch, capsys):
+    directory, _ = deployed
+    answer = compared()
+    body = {**answer.json(), "eval_config": '// the criteria\n{"dataset": []}\n'}
+    install(monkeypatch, FakeModelsAMP(statuses=[httpx.Response(200, json=body)]))
+
+    eval_module.eval_models(MODELS)
+
+    assert (directory / "eval.jsonc").read_text() == '// the criteria\n{"dataset": []}\n'
+    assert "Wrote eval.jsonc" in capsys.readouterr().out
+
+
+def test_a_comparison_never_overwrites_the_projects_criteria(deployed, monkeypatch, capsys):
+    directory, _ = deployed
+    (directory / "eval.jsonc").write_text("// ours\n{}\n")
+    body = {**compared().json(), "eval_config": "// theirs\n{}\n"}
+    install(monkeypatch, FakeModelsAMP(statuses=[httpx.Response(200, json=body)]))
+
+    eval_module.eval_models(MODELS)
+
+    assert (directory / "eval.jsonc").read_text() == "// ours\n{}\n"
+    assert "Wrote eval.jsonc" not in capsys.readouterr().out
+
+
+@pytest.mark.parametrize(
+    ("typed", "sent"),
+    [
+        ("openai/gpt-4o-mini", "openai/gpt-4o-mini"),
+        ("anthropic/claude-haiku-4-5", "anthropic/claude-haiku-4-5"),
+        ("openrouter/openai/gpt-4o-mini", "openrouter/openai/gpt-4o-mini"),
+        # A customer's own: a fine-tune, a deployment name, a model named after
+        # a public one, an unknown name — by provider only.
+        ("openai/ft:gpt-4o-mini:acme-corp::abc", "openai/other"),
+        ("azure/my-deployment", "azure/other"),
+        ("openai/gpt-4o-acme", "openai/other"),
+        ("ollama/acme-llama", "ollama/other"),
+        # A provider crewAI does not know can name a host.
+        ("llm.acme.internal/llama-3", "other/other"),
+    ],
+)
+def test_usage_stats_name_a_model_only_when_it_is_a_public_one(typed, sent):
+    assert eval_module.telemetry_model_name(typed) == sent
+
+
+def test_without_crewais_catalog_every_model_is_other(monkeypatch):
+    monkeypatch.setattr(eval_module, "_known_models_and_providers", lambda: (frozenset(), frozenset()))
+
+    assert eval_module.telemetry_model_name("openai/gpt-4o-mini") == "other/other"

lib/cli/tests/test_plus_api.py

@@ -33,6 +33,60 @@ class TestPlusAPI(unittest.TestCase):
         )
         self.assertEqual(response, mock_response)
 
+    @patch("crewai_core.plus_api.PlusAPI._make_request")
+    def test_create_evaluation_files_it_under_the_project(self, mock_make_request):
+        self.api.create_evaluation("run-1", eval_config="{}", project_id="proj-1")
+
+        mock_make_request.assert_called_once_with(
+            "POST",
+            "/crewai_plus/api/v1/tracing/evaluations",
+            json={"execution_id": "run-1", "eval_config": "{}", "project_id": "proj-1"},
+            timeout=120.0,
+        )
+
+    @patch("crewai_core.plus_api.PlusAPI._make_request")
+    def test_create_models_evaluation(self, mock_make_request):
+        mock_response = MagicMock()
+        mock_make_request.return_value = mock_response
+
+        response = self.api.create_models_evaluation(
+            ["openai/gpt-4o-mini", "anthropic/claude-haiku-4-5"],
+            project_id="proj-1",
+            eval_config="// ours\n{}",
+            deployment_id="dep-1",
+        )
+
+        mock_make_request.assert_called_once_with(
+            "POST",
+            "/crewai_plus/api/v1/tracing/evaluations",
+            json={
+                "kind": "models",
+                "project_id": "proj-1",
+                "models": ["openai/gpt-4o-mini", "anthropic/claude-haiku-4-5"],
+                "eval_config": "// ours\n{}",
+                "deployment_id": "dep-1",
+            },
+            timeout=120.0,
+        )
+        self.assertEqual(response, mock_response)
+
+    @patch("crewai_core.plus_api.PlusAPI._make_request")
+    def test_create_models_evaluation_sends_only_what_it_was_given(
+        self, mock_make_request
+    ):
+        self.api.create_models_evaluation(["openai/gpt-4o-mini"], project_id="proj-1")
+
+        mock_make_request.assert_called_once_with(
+            "POST",
+            "/crewai_plus/api/v1/tracing/evaluations",
+            json={
+                "kind": "models",
+                "project_id": "proj-1",
+                "models": ["openai/gpt-4o-mini"],
+            },
+            timeout=120.0,
+        )
+
     @patch("crewai_core.plus_api.PlusAPI._make_request")
     def test_get_evaluation(self, mock_make_request):
         mock_response = MagicMock()
@@ -252,6 +306,42 @@ class TestPlusAPI(unittest.TestCase):
             "POST", "/crewai_plus/api/v1/crews/test_uuid/deploy"
         )
 
+    @patch("crewai_core.plus_api.PlusAPI._make_request")
+    def test_deploy_sends_the_project_id_when_there_is_one(self, mock_make_request):
+        self.api.deploy_by_name("test_project", project_id="proj-1")
+        mock_make_request.assert_called_with(
+            "POST",
+            "/crewai_plus/api/v1/crews/by-name/test_project/deploy",
+            json={"project_id": "proj-1"},
+        )
+        self.api.deploy_by_uuid("test_uuid", project_id="proj-1")
+        mock_make_request.assert_called_with(
+            "POST",
+            "/crewai_plus/api/v1/crews/test_uuid/deploy",
+            json={"project_id": "proj-1"},
+        )
+
+    @patch("crewai_cli.plus_api.PlusAPI._make_multipart_request")
+    def test_zip_deploys_send_the_project_id_when_there_is_one(
+        self, mock_make_multipart_request
+    ):
+        self.api.create_crew_from_zip("/tmp/test.zip", name="c", project_id="proj-1")
+        mock_make_multipart_request.assert_called_with(
+            "POST",
+            "/crewai_plus/api/v1/crews/zip",
+            zip_file_path="/tmp/test.zip",
+            data={"name": "c", "project_id": "proj-1"},
+            timeout=300,
+        )
+        self.api.update_crew_from_zip("u", "/tmp/test.zip", project_id="proj-1")
+        mock_make_multipart_request.assert_called_with(
+            "POST",
+            "/crewai_plus/api/v1/crews/u/zip_update",
+            zip_file_path="/tmp/test.zip",
+            data={"project_id": "proj-1"},
+            timeout=300,
+        )
+
     @patch("crewai_core.plus_api.PlusAPI._make_request")
     def test_crew_status_by_name(self, mock_make_request):
         self.api.crew_status_by_name("test_project")

lib/crewai-core/src/crewai_core/plus_api.py

@@ -58,10 +58,14 @@ class CrewDeploymentSpec(TypedDict):
     name: str
     repo_clone_url: str
     env: dict[str, str]
+    project_id: NotRequired[str]
 
 
 class CreateCrewPayload(TypedDict):
     deploy: CrewDeploymentSpec
+    # ``[tool.crewai].project_id``, beside the deployment's own fields and at
+    # the top level as every other deploy request sends it; only when set.
+    project_id: NotRequired[str]
 
 
 class _WithUserIdentifier(TypedDict):
@@ -320,13 +324,23 @@ class PlusAPI:
             params=params or None,
         )
 
-    def deploy_by_name(self, project_name: str) -> httpx.Response:
-        return self._make_request(
-            "POST", f"{self.CREWS_RESOURCE}/by-name/{project_name}/deploy"
-        )
+    def deploy_by_name(
+        self, project_name: str, *, project_id: str | None = None
+    ) -> httpx.Response:
+        """Redeploy by name, with PROJECT_ID (`[tool.crewai].project_id`) when set."""
+        endpoint = f"{self.CREWS_RESOURCE}/by-name/{project_name}/deploy"
+        if project_id:
+            return self._make_request("POST", endpoint, json={"project_id": project_id})
+        return self._make_request("POST", endpoint)
 
-    def deploy_by_uuid(self, uuid: str) -> httpx.Response:
-        return self._make_request("POST", f"{self.CREWS_RESOURCE}/{uuid}/deploy")
+    def deploy_by_uuid(
+        self, uuid: str, *, project_id: str | None = None
+    ) -> httpx.Response:
+        """Redeploy by uuid, with PROJECT_ID as for `deploy_by_name`."""
+        endpoint = f"{self.CREWS_RESOURCE}/{uuid}/deploy"
+        if project_id:
+            return self._make_request("POST", endpoint, json={"project_id": project_id})
+        return self._make_request("POST", endpoint)
 
     def crew_status_by_name(self, project_name: str) -> httpx.Response:
         return self._make_request(
@@ -368,11 +382,18 @@ class PlusAPI:
         *,
         name: str | None = None,
         env: dict[str, str] | None = None,
+        project_id: str | None = None,
     ) -> httpx.Response:
-        """Create a crew deployment from a local project ZIP archive."""
+        """Create a crew deployment from a local project ZIP archive.
+
+        PROJECT_ID is `[tool.crewai].project_id`, so AMP knows which project the
+        deployment runs; sent only when the project has one.
+        """
         data: dict[str, str] = {}
         if name:
             data["name"] = name
+        if project_id:
+            data["project_id"] = project_id
         if env:
             data.update({f"env[{key}]": value for key, value in env.items()})
         return self._make_multipart_request(
@@ -389,9 +410,15 @@ class PlusAPI:
         zip_file_path: str | Path,
         *,
         env: dict[str, str] | None = None,
+        project_id: str | None = None,
     ) -> httpx.Response:
-        """Update an existing crew deployment from a local project ZIP archive."""
+        """Update an existing crew deployment from a local project ZIP archive.
+
+        PROJECT_ID as for `create_crew_from_zip`.
+        """
         data: dict[str, str] = {}
+        if project_id:
+            data["project_id"] = project_id
         if env:
             data.update({f"env[{key}]": value for key, value in env.items()})
         return self._make_multipart_request(

lib/crewai-core/src/crewai_core/telemetry.py

@@ -580,6 +580,11 @@ class Telemetry:
     # its use under somebody else's name.
     FEATURE_ATTRIBUTES: ClassVar[dict[str, frozenset[str]]] = {
         "cli_usage:eval": frozenset({"authenticated"}),
+        # The models compared: a model in crewAI's own catalog by name (anyone
+        # can read it off a price list), any other — a fine-tune, a deployment
+        # name, a self-hosted model — as `<provider>/other`; the CLI decides
+        # (`telemetry_model_name`). Never a run, an output or an organization.
+        "cli_usage:eval_models": frozenset({"authenticated", "models", "models_count"}),
     }
 
     def feature_usage_span(

lib/crewai-core/tests/test_smoke.py

@@ -311,6 +311,45 @@ def test_core_feature_span_keeps_only_what_the_feature_may_send(
     assert "authenticated" not in sent and sent["feature"] == "cli_usage:deploy"
 
 
+def test_core_models_span_carries_the_models_compared_and_nothing_else(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    """`cli_usage:eval_models` names the models and how many — provider/model
+    names, not a run — and still drops an id or run content."""
+    from crewai_core.telemetry import Telemetry
+
+    Telemetry._instance = None
+    monkeypatch.delenv("OTEL_SDK_DISABLED", raising=False)
+    monkeypatch.delenv("CREWAI_DISABLE_TELEMETRY", raising=False)
+    monkeypatch.delenv("CREWAI_DISABLE_TRACKING", raising=False)
+
+    tracer = Mock()
+    span = Mock()
+    tracer.start_span.return_value = span
+    monkeypatch.setattr(
+        "crewai_core.telemetry.TracerProvider",
+        lambda **_kwargs: Mock(get_tracer=Mock(return_value=tracer)),
+    )
+
+    Telemetry().feature_usage_span(
+        "cli_usage:eval_models",
+        {
+            "authenticated": "true",
+            "models": "openai/gpt-4o-mini,anthropic/claude-haiku-4-5",
+            "models_count": "2",
+            "evaluation_id": "ev-1",
+            "organization_id": "org-1",
+        },
+    )
+
+    sent = {call.args[0]: call.args[1] for call in span.set_attribute.call_args_list}
+    assert sent["feature"] == "cli_usage:eval_models"
+    assert sent["authenticated"] == "true"
+    assert sent["models"] == "openai/gpt-4o-mini,anthropic/claude-haiku-4-5"
+    assert sent["models_count"] == "2"
+    assert "evaluation_id" not in sent and "organization_id" not in sent
+
+
 def test_core_telemetry_records_flow_creation_version(
     monkeypatch: pytest.MonkeyPatch,
 ) -> None:

lib/crewai/src/crewai/agent/core.py

@@ -79,7 +79,7 @@ from crewai.hooks.dispatch import HookAborted
 from crewai.knowledge.knowledge import Knowledge
 from crewai.knowledge.source.base_knowledge_source import BaseKnowledgeSource
 from crewai.lite_agent_output import LiteAgentOutput
-from crewai.llm_overlay import active as overlay_active, overlay_model_for
+from crewai.llm_overlay import active as overlay_active
 from crewai.llms.base_llm import BaseLLM
 from crewai.mcp.config import MCPServerConfig
 from crewai.rag.embeddings.types import EmbedderConfig
@@ -114,7 +114,7 @@ from crewai.utilities.env import get_env_context
 from crewai.utilities.guardrail import process_guardrail, serialize_guardrail_for_json
 from crewai.utilities.guardrail_types import GuardrailCallable, GuardrailType
 from crewai.utilities.i18n import I18N_DEFAULT
-from crewai.utilities.llm_utils import create_llm, create_llm_like
+from crewai.utilities.llm_utils import create_llm, overlay_llm_for
 from crewai.utilities.prompts import Prompts, StandardPromptResult, SystemPromptResult
 from crewai.utilities.pydantic_schema_utils import generate_model_description
 from crewai.utilities.string_utils import sanitize_tool_name
@@ -420,13 +420,7 @@ class Agent(BaseAgent):
         else:
             declared = create_llm(self.llm)
             self._declared_llm = declared
-            overlay_model = overlay_model_for(self.role)
-            if overlay_model:
-                self.llm = self._overlay_built = create_llm_like(
-                    overlay_model, declared
-                )
-            else:
-                self.llm = declared
+            self.llm = self._resolved_overlay(declared)
             self._overlay_read = True
         if self.function_calling_llm and not isinstance(
             self.function_calling_llm, BaseLLM
@@ -523,15 +517,22 @@ class Agent(BaseAgent):
         if self.role == role_before or overlay_active.get() is None:
             return
         previous = self.llm
-        declared = self._declared_now()
-        overlay_model = overlay_model_for(self.role)
-        if overlay_model:
-            self.llm = self._overlay_built = create_llm_like(overlay_model, declared)
-        else:
-            self.llm = declared
+        self.llm = self._resolved_overlay(self._declared_now())
         if isinstance(previous, BaseLLM) and isinstance(self.llm, BaseLLM):
             self.llm.stream = previous.stream
 
+    def _resolved_overlay(self, declared: BaseLLM | None) -> BaseLLM | None:
+        """The llm ``llm_overlay`` puts this agent on, given its ``declared`` one.
+
+        The role's key wins over the model keys; with neither, ``declared``.
+        What the overlay built is remembered, so a later miss can tell it from
+        an llm the caller assigned.
+        """
+        resolved = overlay_llm_for(self.role, declared)
+        if resolved is not declared:
+            self._overlay_built = resolved
+        return resolved
+
     def _declared_now(self) -> BaseLLM | None:
         """The llm this agent is declared with, as of now.
 

lib/crewai/src/crewai/lite_agent.py

@@ -69,7 +69,6 @@ from crewai.hooks.types import (
 )
 from crewai.lite_agent_output import LiteAgentOutput
 from crewai.llm import LLM
-from crewai.llm_overlay import overlay_model_for
 from crewai.llms.base_llm import BaseLLM
 from crewai.tools.base_tool import BaseTool
 from crewai.tools.structured_tool import CrewStructuredTool
@@ -102,7 +101,7 @@ from crewai.utilities.converter import (
 from crewai.utilities.guardrail import process_guardrail, serialize_guardrail_for_json
 from crewai.utilities.guardrail_types import GuardrailCallable, GuardrailType
 from crewai.utilities.i18n import I18N_DEFAULT
-from crewai.utilities.llm_utils import create_llm, create_llm_like
+from crewai.utilities.llm_utils import create_llm, overlay_llm_for
 from crewai.utilities.pydantic_schema_utils import (
     generate_model_description,
     serialize_model_class,
@@ -320,11 +319,7 @@ class LiteAgent(FlowTrackable, BaseModel):
     @model_validator(mode="after")
     def setup_llm(self) -> Self:
         """Set up the LLM and other components after initialization."""
-        declared = create_llm(self.llm)
-        overlay_model = overlay_model_for(self.role)
-        self.llm = (
-            create_llm_like(overlay_model, declared) if overlay_model else declared
-        )
+        self.llm = overlay_llm_for(self.role, create_llm(self.llm))
         if not isinstance(self.llm, BaseLLM):
             raise ValueError(
                 f"Expected LLM instance of type BaseLLM, got {type(self.llm).__name__}"

lib/crewai/src/crewai/llm.py

@@ -2,6 +2,7 @@ from __future__ import annotations
 
 from collections import defaultdict
 from collections.abc import Callable, Mapping
+from contextvars import ContextVar
 from datetime import datetime
 import json
 import logging
@@ -33,6 +34,7 @@ from crewai.events.types.tool_usage_events import (
     ToolUsageStartedEvent,
 )
 from crewai.hooks.dispatch import HookAborted
+from crewai.llm_overlay import overlay_model_for_model
 from crewai.llms._finish_reason_utils import extract_choices_finish_reason_and_id
 from crewai.llms.base_llm import (
     BaseLLM,
@@ -88,6 +90,10 @@ logger = logging.getLogger(__name__)
 _litellm_loaded = False
 LITELLM_AVAILABLE = False
 
+# The instance ``LLM.__new__`` just built on a model ``llm_overlay`` mapped to,
+# whose ``__init__`` call with the caller's arguments must be skipped.
+_overlay_built: ContextVar[LLM | None] = ContextVar("llm_overlay_built", default=None)
+
 if TYPE_CHECKING:
     import litellm
     from litellm.litellm_core_utils.get_supported_openai_params import (
@@ -194,6 +200,30 @@ SUPPORTED_NATIVE_PROVIDERS: Final[list[str]] = [
 ]
 
 
+# The provider a ``<prefix>/<model>`` string routes to, by prefix: the names
+# crewAI accepts for each native provider. ``llm_overlay`` compares model
+# names through the same table, so ``google/x`` and ``gemini/x`` are one model.
+PROVIDER_ALIASES: Final[dict[str, str]] = {
+    "openai": "openai",
+    "anthropic": "anthropic",
+    "claude": "anthropic",
+    "azure": "azure",
+    "azure_openai": "azure",
+    "google": "gemini",
+    "gemini": "gemini",
+    "bedrock": "bedrock",
+    "aws": "bedrock",
+    "openrouter": "openrouter",
+    "deepseek": "deepseek",
+    "ollama": "ollama",
+    "ollama_chat": "ollama_chat",
+    "hosted_vllm": "hosted_vllm",
+    "cerebras": "cerebras",
+    "dashscope": "dashscope",
+    "snowflake": "snowflake",
+}
+
+
 class Delta(TypedDict):
     content: str | None
     role: str | None
@@ -266,6 +296,11 @@ class LLM(BaseLLM):
         if not model or not isinstance(model, str):
             raise ValueError("Model must be a non-empty string")
 
+        if cls is LLM:
+            mapped = cls._overlay_mapped(model, is_litellm, kwargs)
+            if mapped is not None:
+                return mapped
+
         route = cls._resolve_route(model, kwargs)
         if route.native_class is not None and not is_litellm:
             try:
@@ -310,6 +345,39 @@ class LLM(BaseLLM):
 
         return object.__new__(cls)
 
+    def __init__(self, model: str, is_litellm: bool = False, **kwargs: Any) -> None:
+        # Python calls ``__init__`` on whatever ``__new__`` returns when it is an
+        # ``LLM``; an instance ``llm_overlay`` built on a mapped model is already
+        # initialized, and initializing it again with the caller's arguments
+        # would put the declared model back.
+        if _overlay_built.get() is self:
+            _overlay_built.set(None)
+            return
+        super().__init__(model=model, is_litellm=is_litellm, **kwargs)
+
+    @classmethod
+    def _overlay_mapped(
+        cls, model: str, is_litellm: bool, kwargs: dict[str, Any]
+    ) -> LLM | None:
+        """``LLM(model, **kwargs)`` built on the model an ``llm_overlay`` key maps it to.
+
+        ``None`` when no model key of the active overlay matches ``model``. The
+        provider a bare model routes to is put beside it, so a key written
+        ``model:openai/gpt-4o`` matches ``LLM(model="gpt-4o")``.
+        """
+        provider = kwargs.get("provider")
+        if not provider and "/" not in model:
+            provider = cls._infer_provider_from_model(model)
+        mapped_model = overlay_model_for_model(model, provider)
+        if mapped_model is None:
+            return None
+        from crewai.utilities.llm_utils import create_llm_from_kwargs_like
+
+        built = create_llm_from_kwargs_like(mapped_model, model, kwargs, is_litellm)
+        if isinstance(built, LLM):
+            _overlay_built.set(built)
+        return cast(LLM, built)
+
     @classmethod
     def _matches_provider_pattern(cls, model: str, provider: str) -> bool:
         """Check if a model name matches provider-specific patterns.
@@ -416,27 +484,7 @@ class LLM(BaseLLM):
         elif "/" in model:
             prefix, _, model_part = model.partition("/")
 
-            provider_mapping = {
-                "openai": "openai",
-                "anthropic": "anthropic",
-                "claude": "anthropic",
-                "azure": "azure",
-                "azure_openai": "azure",
-                "google": "gemini",
-                "gemini": "gemini",
-                "bedrock": "bedrock",
-                "aws": "bedrock",
-                "openrouter": "openrouter",
-                "deepseek": "deepseek",
-                "ollama": "ollama",
-                "ollama_chat": "ollama_chat",
-                "hosted_vllm": "hosted_vllm",
-                "cerebras": "cerebras",
-                "dashscope": "dashscope",
-                "snowflake": "snowflake",
-            }
-
-            canonical_provider = provider_mapping.get(prefix.lower())
+            canonical_provider = PROVIDER_ALIASES.get(prefix.lower())
 
             valid_native_model = bool(
                 canonical_provider

lib/crewai/src/crewai/llm_overlay.py

@@ -1,10 +1,37 @@
-"""Process-context ``role -> model`` overlay for agents.
+"""Process-context ``role -> model`` and ``model -> model`` overlay.
 
-``llm_overlay`` lets a per-run caller make specific agents use a different
-model without editing the code that builds them. While the block is active,
-an ``Agent`` or ``LiteAgent`` whose ``role`` is a key of the mapping is built
-with the mapped model instead of its declared ``llm``. Roles that are not
-keys, and every agent built outside the block, keep their own model.
+``llm_overlay`` lets a per-run caller make specific agents, or specific
+models, use a different model without editing the code that builds them.
+While the block is active, an ``Agent`` or ``LiteAgent`` whose ``role`` is a
+key of the mapping is built with the mapped model instead of its declared
+``llm``. Roles that are not keys, and every agent built outside the block,
+keep their own model.
+
+A key that starts with :data:`MODEL_KEY_PREFIX` names a model instead of a
+role: ``"model:openai/gpt-4o"`` maps every LLM built from that model string
+inside the block, and ``"model:*"`` every LLM built from any model string.
+That is what reaches an LLM no role names — a flow step's own
+``LLM(model="openai/gpt-4o").call(...)`` — and the simplest way to say "this
+whole run on model X". A model key is read wherever an LLM is built from a
+model string: ``LLM(model=...)`` (and so ``create_llm`` and an agent's
+``llm="..."``, which go through it), and an agent's declared ``llm`` instance
+when it was built outside the block. A model is compared as the caller wrote
+it and without the provider prefix native providers strip, so
+``"model:openai/gpt-4o"`` matches ``LLM(model="gpt-4o")`` and an instance
+whose ``model`` reads ``"gpt-4o"``, and ``"model:gpt-4o"`` matches
+``LLM(model="openai/gpt-4o")``; an exact key wins over a stripped one, and
+both over ``"model:*"``. Only the model's own provider prefix is ever
+stripped: an aggregator's route such as ``openrouter/openai/gpt-4o`` is that
+aggregator's model and matches only a key naming it whole. The mapped model is built once, never looked up
+again: ``{"model:a": "b", "model:b": "c"}`` puts an ``a`` on ``b``. For an
+agent a role key wins over every model key, so ``{"Researcher": x,
+"model:*": y}`` runs the Researcher on ``x`` and everything else on ``y`` —
+and ``x`` is built like the llm the caller declared, even when a model key
+already mapped that llm as it was built inside the block.
+The settings the caller passed to ``LLM(...)`` are carried to the mapped model
+by the same rule as a declared ``llm``'s (below). Only ``LLM`` itself is
+mapped: a subclass of it, or a provider class built directly, is the caller's
+own choice of class and keeps its model.
 
 Roles are matched exactly, by the text they have when the overlay is read;
 only whitespace around a role or a key is ignored, so a role a YAML file
@@ -46,6 +73,9 @@ registers an agent when it first emits) does not read it again, so an agent
 built outside a block keeps its model through a kickoff inside one.
 
 Example:
+    >>> with llm_overlay({"model:*": "openai/gpt-4o-mini"}):
+    ...     flow.kickoff()  # every LLM built in the run is on gpt-4o-mini
+
     >>> with llm_overlay({"Researcher": "openai/gpt-4o"}):
     ...     crew = build_crew()  # the Researcher agent is built on gpt-4o
     ...     crew.kickoff()
@@ -60,24 +90,57 @@ from __future__ import annotations
 from collections.abc import Iterator
 from contextlib import contextmanager
 from contextvars import ContextVar
+from typing import Any, Final, TypeVar
+import weakref
+
+
+MODEL_KEY_PREFIX: Final[str] = "model:"
+"""A key starting with this names a model (``"model:openai/gpt-4o"``), not a role."""
+
+_T = TypeVar("_T")
+
+ANY_MODEL: Final[str] = "*"
+"""The model part of the key that maps every LLM built from a model string."""
 
 
 active: ContextVar[dict[str, str] | None] = ContextVar(
     "llm_overlay_active", default=None
 )
 
+# Set while the overlay builds the model it mapped to, so that build is not
+# mapped again (a chain of model keys, or ``model:*`` under a role's model).
+_building_mapped: ContextVar[bool] = ContextVar(
+    "llm_overlay_building_mapped", default=False
+)
+
+# The LLMs a model key already mapped, by id. An agent reading its declared
+# ``llm`` skips one of these: its model is the mapped one, never a key again.
+_mapped_llms: weakref.WeakValueDictionary[int, Any] = weakref.WeakValueDictionary()
+
+# What the caller declared for an LLM the overlay built: the declared instance,
+# or ``LLM(model, **kwargs)``'s ``(model, kwargs, is_litellm)``. Dropped with
+# the LLM it describes.
+Declared = Any
+_declared_before: dict[int, Declared] = {}
+
 
 @contextmanager
 def llm_overlay(mapping: dict[str, str] | None) -> Iterator[None]:
-    """Route agent roles to models for the duration of the block.
+    """Route agent roles and models to models for the duration of the block.
 
     Args:
-        mapping: ``{role: model}`` for the block; ``None`` clears any active
-            overlay for the block. Whitespace around each role is dropped from
-            the copy the block uses, so a key matches a role that differs from
-            it only by its surrounding whitespace; the mapping passed in is
-            left as it is. The previous value is always restored on exit,
-            including when the block raises.
+        mapping: ``{role: model}`` and ``{"model:<model>": model}`` (or
+            ``{"model:*": model}``) for the block; ``None`` clears any active
+            overlay for the block. Whitespace around each role, and around the
+            model a model key names, is dropped from the copy the block uses,
+            so a key matches a role that differs from it only by its
+            surrounding whitespace; the mapping passed in is left as it is. The
+            previous value is always restored on exit, including when the
+            block raises.
+
+    Raises:
+        ValueError: Two keys name one role or one model with different
+            models, or a model key names no model.
     """
     token = active.set(_stripped(mapping))
     try:
@@ -90,7 +153,8 @@ def overlay_model_for(role: str | None) -> str | None:
     """The model the active overlay assigns to ``role``.
 
     Whitespace around ``role`` is ignored, as it was around the keys when the
-    overlay was set. An empty or ``None`` role matches nothing.
+    overlay was set. An empty or ``None`` role matches nothing, and a role is
+    never matched against a model key.
 
     Returns:
         The mapped model string, or ``None`` when no overlay is active or
@@ -100,22 +164,166 @@ def overlay_model_for(role: str | None) -> str | None:
     if not mapping or role is None:
         return None
     role = role.strip()
-    return mapping.get(role) if role else None
+    if not role or role.startswith(MODEL_KEY_PREFIX):
+        return None
+    return mapping.get(role)
+
+
+def overlay_model_for_model(
+    model: str | None, provider: str | None = None
+) -> str | None:
+    """The model the active overlay's model keys assign to an LLM on ``model``.
+
+    ``model`` is compared as written and, when a native provider strips its own
+    prefix, without it: ``openai/gpt-4o`` also as ``gpt-4o``, and ``gpt-4o``
+    routed to ``provider`` also as ``openai/gpt-4o``. A prefix is only ever the
+    model's OWN provider: in an aggregator's route (``openrouter/openai/gpt-4o``)
+    the rest is that aggregator's model id, never the model a key for
+    ``openai/gpt-4o`` or ``gpt-4o`` names, so it is compared whole. An existing
+    instance of such a route records ``model="openai/gpt-4o"`` under
+    ``provider="openrouter"``, and is compared as ``openrouter/openai/gpt-4o``.
+    ``model:*`` matches every model. Nothing matches while the overlay is
+    building a model it mapped.
+
+    Args:
+        model: The model string an LLM is being built from, or the ``model``
+            of an existing instance.
+        provider: The provider the LLM routes to, when ``model`` does not say.
+
+    Returns:
+        The mapped model string, or ``None`` when no overlay is active, it has
+        no model key for ``model``, or a mapped model is being built.
+    """
+    mapping = active.get()
+    if not mapping or not model or _building_mapped.get():
+        return None
+    forms = _model_forms(model.strip(), provider)
+    for form in forms:
+        mapped = mapping.get(MODEL_KEY_PREFIX + form)
+        if mapped is not None:
+            return mapped
+    # The same model under another of its provider's names: a key written
+    # `google/x` for an llm that routed `gemini/x`, or the other way round.
+    aliases = _aliases()
+    canonical = {_canonical(form, aliases) for form in forms}
+    for key, mapped in mapping.items():
+        named = (
+            key[len(MODEL_KEY_PREFIX) :] if key.startswith(MODEL_KEY_PREFIX) else None
+        )
+        if named and named != ANY_MODEL and _canonical(named, aliases) in canonical:
+            return mapped
+    return mapping.get(MODEL_KEY_PREFIX + ANY_MODEL)
+
+
+def _model_forms(model: str, provider: str | None) -> list[str]:
+    """``model`` in the forms a key may name it by, the one as written first."""
+    if provider and not model.startswith(f"{provider}/"):
+        routed = f"{provider}/{model}"
+        # A bare name is the native one with its prefix stripped; a name that
+        # still has a slash is the provider's own route id and only means
+        # something beside the provider.
+        forms = [routed] if "/" in model else [model, routed]
+    else:
+        forms = [model]
+    prefix, separator, rest = forms[-1].partition("/")
+    if separator and "/" not in rest and prefix.lower() in _aliases():
+        if rest not in forms:
+            forms.append(rest)
+    return forms
+
+
+def _aliases() -> dict[str, str]:
+    """The router's provider aliases (``crewai.llm.PROVIDER_ALIASES``): every
+    prefix ``LLM`` strips from a model string it routes natively, to the
+    provider it names."""
+    from crewai.llm import PROVIDER_ALIASES
+
+    return PROVIDER_ALIASES
+
+
+def _canonical(form: str, aliases: dict[str, str]) -> str:
+    """``form`` with its provider prefix under the provider's one name."""
+    prefix, separator, rest = form.partition("/")
+    provider = aliases.get(prefix.lower()) if separator else None
+    return f"{provider}/{rest}" if provider else form
+
+
+def overlay_model_for_llm(llm: Any) -> str | None:
+    """The model the active overlay's model keys assign to an existing ``llm``.
+
+    An agent's declared ``llm`` built outside the block is looked up by its
+    ``model`` and ``provider``; one a model key already mapped when it was
+    built is not looked up again.
+    """
+    if llm is None or _mapped_llms.get(id(llm)) is llm:
+        return None
+    return overlay_model_for_model(
+        getattr(llm, "model", None), getattr(llm, "provider", None)
+    )
+
+
+def declared_before_overlay(llm: Any) -> Declared | None:
+    """What the caller declared for ``llm`` before the overlay mapped it, or None."""
+    if llm is None or _mapped_llms.get(id(llm)) is not llm:
+        return None
+    return _declared_before.get(id(llm))
+
+
+@contextmanager
+def building_mapped_model() -> Iterator[None]:
+    """Build the model the overlay mapped to, without mapping it again."""
+    token = _building_mapped.set(True)
+    try:
+        yield
+    finally:
+        _building_mapped.reset(token)
+
+
+def mark_mapped(llm: _T, declared: Declared | None = None) -> _T:
+    """Record ``llm`` as built by the overlay, so no model key maps it again.
+
+    ``declared`` is what the caller declared before the overlay mapped it —
+    the declared instance, or ``LLM(model, **kwargs)``'s arguments as
+    ``(model, kwargs, is_litellm)`` — so a role key can still be built from
+    the caller's own declaration (:func:`declared_before_overlay`).
+    """
+    try:
+        _mapped_llms[id(llm)] = llm
+        if declared is not None:
+            _declared_before[id(llm)] = declared
+            weakref.finalize(llm, _declared_before.pop, id(llm), None)
+    except TypeError:
+        # Only a weakly-referenceable object can be remembered (every
+        # ``BaseLLM`` is). Anything else is returned as built: the one cost is
+        # that an agent declared with it looks its model up again, as it would
+        # for an llm built outside the block — never a failed build.
+        pass
+    return llm
 
 
 def _stripped(mapping: dict[str, str] | None) -> dict[str, str] | None:
     """A copy of ``mapping`` with the whitespace around each role dropped.
 
-    Two keys that differ only by whitespace name one role. When they name the
-    same model the copy holds it once; when they name different models the
-    mapping is ambiguous and is refused, so the model an agent runs on never
-    depends on dictionary order.
+    Two keys that differ only by whitespace name one role (or, after the
+    :data:`MODEL_KEY_PREFIX`, one model). When they name the same model the
+    copy holds it once; when they name different models the mapping is
+    ambiguous and is refused, so the model an agent runs on never depends on
+    dictionary order.
     """
     if mapping is None:
         return None
     stripped: dict[str, str] = {}
     for role, model in mapping.items():
         key = role.strip()
+        if key.startswith(MODEL_KEY_PREFIX):
+            named = key[len(MODEL_KEY_PREFIX) :].strip()
+            if not named:
+                raise ValueError(
+                    f"llm_overlay: the key {role!r} names no model; write "
+                    f"'{MODEL_KEY_PREFIX}<provider/model>' or "
+                    f"'{MODEL_KEY_PREFIX}{ANY_MODEL}'"
+                )
+            key = MODEL_KEY_PREFIX + named
         if key in stripped and stripped[key] != model:
             raise ValueError(
                 f"llm_overlay: role {key!r} is mapped twice with different models "

lib/crewai/src/crewai/telemetry/telemetry.py

@@ -1293,6 +1293,11 @@ class Telemetry:
     # its use under somebody else's name.
     FEATURE_ATTRIBUTES: ClassVar[dict[str, frozenset[str]]] = {
         "cli_usage:eval": frozenset({"authenticated"}),
+        # The models compared: a model in crewAI's own catalog by name (anyone
+        # can read it off a price list), any other — a fine-tune, a deployment
+        # name, a self-hosted model — as `<provider>/other`; the CLI decides
+        # (`telemetry_model_name`). Never a run, an output or an organization.
+        "cli_usage:eval_models": frozenset({"authenticated", "models", "models_count"}),
     }
 
     def feature_usage_span(

lib/crewai/src/crewai/utilities/llm_utils.py

@@ -6,6 +6,13 @@ from pydantic import ValidationError
 
 from crewai.constants import DEFAULT_LLM_MODEL, ENV_VARS, LITELLM_PARAMS
 from crewai.llm import LLM
+from crewai.llm_overlay import (
+    building_mapped_model,
+    declared_before_overlay,
+    mark_mapped,
+    overlay_model_for,
+    overlay_model_for_llm,
+)
 from crewai.llms.base_llm import BaseLLM
 
 
@@ -208,22 +215,114 @@ def create_llm_like(model: str, base: BaseLLM | None) -> BaseLLM:
     Returns:
         A new instance for ``model``; never ``base`` itself.
     """
-    if not isinstance(base, BaseLLM):
-        return LLM(model=model)
+    with building_mapped_model():
+        if not isinstance(base, BaseLLM):
+            return mark_mapped(LLM(model=model))
 
-    # The declared endpoint decides where an unknown ``openai/`` model routes (a
-    # custom OpenAI-compatible endpoint), so the same-provider question is asked
-    # with it: a self-hosted model mapped to another model on the same endpoint
-    # must keep that endpoint.
-    route = LLM._resolve_route(model, _configured_settings(base, _ENDPOINT_NAMES))
-    carried = _configured_settings(base, GENERATION_SETTINGS)
-    if _same_provider(base, route):
-        carried.update(_configured_settings(base, PROVIDER_SETTINGS))
-    target = LLM._resolve_route(model, carried).native_class or LLM
-    if type(base) is LLM and base.is_litellm:
+        # The declared endpoint decides where an unknown ``openai/`` model routes (a
+        # custom OpenAI-compatible endpoint), so the same-provider question is asked
+        # with it: a self-hosted model mapped to another model on the same endpoint
+        # must keep that endpoint.
+        route = LLM._resolve_route(model, _configured_settings(base, _ENDPOINT_NAMES))
+        carried = _configured_settings(base, GENERATION_SETTINGS)
+        if _same_provider(base, route):
+            carried.update(_configured_settings(base, PROVIDER_SETTINGS))
         # The declared llm runs through LiteLLM — by the caller's choice or
         # because no native class knew its model; either way that is the
         # environment its callbacks and extra kwargs were written for.
+        through_litellm = type(base) is LLM and base.is_litellm
+        return mark_mapped(
+            _build_like(model, carried, type(base), through_litellm), base
+        )
+
+
+def overlay_llm_for(role: str | None, declared: BaseLLM | None) -> BaseLLM | None:
+    """The llm an agent with ``role`` and ``declared`` llm runs on under ``llm_overlay``.
+
+    A role key wins, and its model is built from the caller's declaration even
+    when a model key already mapped ``declared`` on its way in; else a model
+    key matching ``declared``'s model (one a model key already mapped when it
+    was built is not looked up again); else ``declared`` itself. A mapped model
+    is built like ``declared`` (:func:`create_llm_like`). Outside any block
+    this is ``declared``.
+    """
+    role_model = overlay_model_for(role)
+    if role_model:
+        # The role wins, built from what the caller declared: when a model key
+        # already mapped the declared llm, its declaration, not the mapping.
+        before = declared_before_overlay(declared)
+        if isinstance(before, tuple):
+            declared_model, kwargs, is_litellm = before
+            return create_llm_from_kwargs_like(
+                role_model, declared_model, kwargs, is_litellm
+            )
+        base = before if isinstance(before, BaseLLM) else declared
+        return create_llm_like(role_model, base)
+    model = overlay_model_for_llm(declared)
+    return create_llm_like(model, declared) if model else declared
+
+
+def create_llm_from_kwargs_like(
+    model: str, declared_model: str, kwargs: dict[str, Any], is_litellm: bool
+) -> BaseLLM:
+    """Build ``model`` configured like ``LLM(declared_model, **kwargs)`` would be.
+
+    What ``llm_overlay``'s model keys build when ``LLM(model=...)`` is called
+    inside the block: the settings the caller passed are carried by the rule of
+    :func:`create_llm_like` — generation settings to any class that has them,
+    credentials and endpoints only to the same provider — without building the
+    declared model first (its provider's key may not be in this environment).
+    Everything the caller passed is theirs, so nothing counts as derived.
+    """
+    declaration = (declared_model, dict(kwargs), is_litellm)
+    with building_mapped_model():
+        settings = {
+            k: v
+            for k, v in kwargs.items()
+            if v is not None and not (isinstance(v, (list, dict)) and not v)
+        }
+        if isinstance(settings.get("endpoint"), str):
+            settings["endpoint"] = settings["endpoint"].split(_AZURE_DEPLOYMENT_PATH)[0]
+        endpoint = {k: settings[k] for k in _ENDPOINT_NAMES if k in settings}
+        route = LLM._resolve_route(model, endpoint)
+        carried = {k: settings[k] for k in GENERATION_SETTINGS if k in settings}
+        try:
+            declared = LLM._resolve_route(declared_model, kwargs)
+        except ImportError:
+            # The declared model's SDK is not installed here. Nothing is built
+            # on it, so that is no reason to fail; its credentials, though,
+            # cannot be matched to the new provider, so none are carried. A
+            # caller who chose LiteLLM keeps it.
+            return mark_mapped(
+                _build_like(model, carried, None, is_litellm), declaration
+            )
+        declared_class = (
+            LLM
+            if is_litellm or declared.native_class is None
+            else declared.native_class
+        )
+        if _same_route_provider(declared_class, declared.provider, route):
+            carried.update({k: settings[k] for k in PROVIDER_SETTINGS if k in settings})
+        return mark_mapped(
+            _build_like(model, carried, declared_class, declared_class is LLM),
+            declaration,
+        )
+
+
+def _build_like(
+    model: str,
+    carried: dict[str, Any],
+    base_class: type[BaseLLM] | None,
+    through_litellm: bool,
+) -> BaseLLM:
+    """Build ``model`` with the ``carried`` settings its class accepts.
+
+    ``base_class`` is the declared llm's class, ``None`` when it is not known;
+    ``additional_params`` travel only within one class.
+    """
+    route = LLM._resolve_route(model, carried)
+    target = route.native_class or LLM
+    if through_litellm:
         target = LLM
         carried["is_litellm"] = True
     accepted = {
@@ -231,7 +330,7 @@ def create_llm_like(model: str, base: BaseLLM | None) -> BaseLLM:
         for k, v in carried.items()
         if k in target.model_fields or k == "is_litellm"
     }
-    if type(base) is not target:
+    if base_class is not target:
         accepted.pop("additional_params", None)
     _carry_cap_under_the_targets_name(carried, accepted, target)
     if route.provider == "anthropic" and "temperature" in accepted:
@@ -239,6 +338,34 @@ def create_llm_like(model: str, base: BaseLLM | None) -> BaseLLM:
     return _build(model, accepted)
 
 
+def _same_route_provider(
+    declared_class: type[BaseLLM], declared_provider: str, route: Any
+) -> bool:
+    """:func:`_same_provider` for a declared llm known by its route, not an instance.
+
+    The same rule, case for case: a native declared class compares by class, a
+    LiteLLM-routed ``LLM`` by its provider — to a native route through the class
+    that provider names, so ``LLM(model="openai/gpt-4o", is_litellm=True)``
+    mapped to a native OpenAI model keeps its key and endpoint.
+    """
+    if route.native_class is None:
+        return declared_class is LLM and route.provider == declared_provider
+    if declared_class is route.native_class:
+        return not _serves_several_providers(route.native_class) or (
+            route.provider == declared_provider
+        )
+    if declared_class is not LLM:
+        return False
+    try:
+        named = LLM._get_native_provider(declared_provider or "")
+    except ImportError:
+        return False
+    return named is route.native_class and (
+        not _serves_several_providers(route.native_class)
+        or route.provider == declared_provider
+    )
+
+
 def _same_provider(base: BaseLLM, route: Any) -> bool:
     """Whether ``route`` lands where ``base``'s credentials and endpoint belong.
 

lib/crewai/tests/llms/openai/test_openai.py

@@ -214,6 +214,15 @@ def test_openai_completion_module_is_imported(monkeypatch):
     """
     module_name = "crewai.llms.providers.openai.completion"
 
+    # Re-importing binds the fresh module on its package too, and monkeypatch
+    # only restores sys.modules: without also restoring the attribute, every
+    # later test in this worker sees two module objects for one name, and a
+    # `patch("crewai.llms.providers.openai.completion.X")` patches the one that
+    # `from ... import X` does not read.
+    import crewai.llms.providers.openai as openai_package
+
+    if hasattr(openai_package, "completion"):
+        monkeypatch.setattr(openai_package, "completion", openai_package.completion)
     monkeypatch.delitem(sys.modules, module_name, raising=False)
 
     LLM(model="gpt-4o")

lib/crewai/tests/telemetry/test_tracer_isolation.py

@@ -118,6 +118,32 @@ def test_a_feature_span_keeps_only_what_the_feature_may_send(telemetry_with_expo
     assert deploy_span.attributes["feature"] == "cli_usage:deploy"
 
 
+def test_the_models_span_carries_the_models_compared_and_nothing_else(
+    telemetry_with_exporter,
+):
+    telemetry, exporter = telemetry_with_exporter
+    telemetry.set_tracer()
+
+    telemetry.feature_usage_span(
+        "cli_usage:eval_models",
+        {
+            "authenticated": "true",
+            "models": "openai/gpt-4o-mini,anthropic/claude-haiku-4-5",
+            "models_count": "2",
+            "evaluation_id": "ev-1",
+            "organization_id": "org-1",
+        },
+    )
+
+    (span,) = exporter.get_finished_spans()
+    assert span.attributes["feature"] == "cli_usage:eval_models"
+    assert span.attributes["authenticated"] == "true"
+    assert span.attributes["models"] == "openai/gpt-4o-mini,anthropic/claude-haiku-4-5"
+    assert span.attributes["models_count"] == "2"
+    assert "evaluation_id" not in span.attributes
+    assert "organization_id" not in span.attributes
+
+
 def test_our_spans_are_unaffected_by_an_application_provider(telemetry_with_exporter):
     """An app that installs its own provider must not divert our telemetry.
 

lib/crewai/tests/test_llm_overlay.py

@@ -1,10 +1,13 @@
-"""`llm_overlay` swaps an agent's model by role, for the calling context only.
+"""`llm_overlay` swaps an agent's model by role, and any LLM's by model, for
+the calling context only.
 
-The overlay is read in exactly three places: the validators where `Agent` and
-`LiteAgent` resolve their `llm`, and `Agent.interpolate_inputs`, which looks the
+The overlay is read in exactly four places: the validators where `Agent` and
+`LiteAgent` resolve their `llm`, `Agent.interpolate_inputs`, which looks the
 interpolated role up again because a templated role only becomes a key once a
-kickoff fills its placeholders in. So these tests build agents, interpolate
-them, and look at the model they end up with. No LLM is ever called.
+kickoff fills its placeholders in, and `LLM.__new__`, where a `model:` key maps
+an LLM built from a model string. So these tests build agents and LLMs,
+interpolate them, and look at the model they end up with. No LLM is ever called
+over the network.
 
 `create_llm("openai/gpt-4o")` returns the native OpenAI provider, which strips
 the `openai/` prefix, so the resolved model reads `"gpt-4o"`.
@@ -19,8 +22,14 @@ from typing import Any
 from crewai import Agent, Crew, Task
 from crewai.lite_agent import LiteAgent
 from crewai.llm import LLM
-from crewai.llm_overlay import active, llm_overlay, overlay_model_for
+from crewai.llm_overlay import (
+    MODEL_KEY_PREFIX,
+    active,
+    llm_overlay,
+    overlay_model_for,
+)
 from crewai.llms.base_llm import BaseLLM
+from crewai.utilities.llm_utils import create_llm
 import pytest
 
 
@@ -567,3 +576,315 @@ def test_re_validation_keeps_the_llm_a_kickoff_time_swap_set() -> None:
         RuntimeState(root=[agent])
 
     assert agent.llm is swapped
+
+
+# ── model keys: `model:<provider/model>` and `model:*` ───────────────────────
+
+
+def test_the_model_key_prefix_is_a_public_constant() -> None:
+    """Another package feature-detects the model-for-model form on it."""
+    assert MODEL_KEY_PREFIX == "model:"
+
+
+def test_a_bare_llm_call_in_a_flow_step_runs_on_the_mapped_model(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    """The case roles cannot reach: a flow step's own `LLM(...).call()`."""
+    from crewai.flow.flow import Flow, start
+
+    openai_class = type(LLM(model="openai/gpt-4o-mini"))
+    monkeypatch.setattr(
+        openai_class,
+        "call",
+        lambda self, messages, *args, **kwargs: f"answered by {self.model}",
+    )
+
+    class Poem(Flow):  # type: ignore[type-arg]
+        @start()
+        def write(self) -> str:
+            return LLM(model="openai/gpt-5.6-sol").call("a poem")
+
+    with llm_overlay({"model:openai/gpt-5.6-sol": "openai/gpt-4o-mini"}):
+        assert Poem().kickoff() == "answered by gpt-4o-mini"
+
+    assert Poem().kickoff() == "answered by gpt-5.6-sol"
+
+
+def test_a_model_key_matches_with_and_without_the_provider_prefix() -> None:
+    """Native providers strip `openai/`; a key written either way matches both."""
+    with llm_overlay({"model:openai/gpt-4o": "openai/gpt-4.1"}):
+        assert LLM(model="gpt-4o").model == "gpt-4.1"
+        assert LLM(model="openai/gpt-4o").model == "gpt-4.1"
+        assert LLM(model="openai/gpt-4o-mini").model == "gpt-4o-mini"
+    with llm_overlay({"model:gpt-4o": "openai/gpt-4.1"}):
+        assert LLM(model="openai/gpt-4o").model == "gpt-4.1"
+
+
+def test_an_exact_model_key_wins_over_a_stripped_one_and_over_the_wildcard() -> (
+    None
+):
+    with llm_overlay(
+        {
+            "model:gpt-4o": "openai/gpt-4.1-nano",
+            "model:openai/gpt-4o": "openai/gpt-4.1",
+            "model:*": "openai/gpt-4o-mini",
+        }
+    ):
+        assert LLM(model="openai/gpt-4o").model == "gpt-4.1"
+        assert LLM(model="gpt-4o").model == "gpt-4.1-nano"
+        assert LLM(model="openai/o3-mini").model == "gpt-4o-mini"
+
+
+def test_the_wildcard_maps_every_llm_built_from_a_model_string() -> None:
+    with llm_overlay({"model:*": "openai/gpt-4o-mini"}):
+        built = [
+            LLM(model="openai/gpt-4o"),
+            create_llm("anthropic/claude-haiku-4-5"),
+            LLM(model="gpt-5.6-sol"),
+        ]
+
+    assert [(type(b).__name__, b.model) for b in built] == [
+        ("OpenAICompletion", "gpt-4o-mini")
+    ] * 3
+    assert LLM(model="openai/gpt-4o").model == "gpt-4o"
+
+
+def test_a_mapped_model_is_not_mapped_again() -> None:
+    """A chain of keys is one step: the model a key maps to is built as it is."""
+    with llm_overlay(
+        {"model:openai/gpt-4o": "openai/gpt-4.1", "model:openai/gpt-4.1": "openai/o3"}
+    ):
+        assert LLM(model="openai/gpt-4o").model == "gpt-4.1"
+
+
+def test_the_caller_settings_follow_the_mapped_model_by_the_declared_llm_rule() -> (
+    None
+):
+    """Generation settings go to any provider; a key and an endpoint only to
+    the provider they were issued for."""
+    with llm_overlay({"model:*": "openai/gpt-4o"}):
+        same = LLM(model="openai/gpt-4o-mini", **CONFIGURATION)
+    with llm_overlay({"model:*": "anthropic/claude-haiku-4-5"}):
+        other = LLM(model="openai/gpt-4o-mini", **CONFIGURATION)
+
+    assert type(same).__name__ == "OpenAICompletion" and same.model == "gpt-4o"
+    assert _configuration_of(same) == CONFIGURATION
+    assert type(other).__name__ == "AnthropicCompletion"
+    assert other.model == "claude-haiku-4-5"
+    assert other.timeout == 42 and other.temperature == 0.1
+    assert other.max_tokens == 77
+    assert other.api_key != "k" and other.base_url is None
+
+
+def test_a_model_mapped_onto_litellm_is_not_initialized_again() -> None:
+    """`LLM.__new__` returning an `LLM` makes Python call `__init__` with the
+    caller's arguments; the mapped instance must keep the mapped model."""
+    pytest.importorskip("litellm")
+    with llm_overlay({"model:*": "groq/llama-3.1-8b-instant"}):
+        built = LLM(model="openai/gpt-4o-mini", temperature=0.3, api_key="k")
+
+    assert type(built).__name__ == "LLM" and built.is_litellm
+    assert built.model == "groq/llama-3.1-8b-instant"
+    assert built.temperature == 0.3 and built.api_key != "k"
+
+
+def test_an_agent_declared_with_a_model_string_runs_on_the_mapped_model() -> None:
+    with llm_overlay({"model:openai/gpt-4o-mini": "openai/gpt-4o"}):
+        agent = _agent("Writer")
+
+    assert agent.llm.model == "gpt-4o"
+
+
+def test_an_agent_whose_llm_was_built_outside_the_block_is_mapped_by_its_model() -> (
+    None
+):
+    """The declared instance is looked up by its model, with the configuration
+    carried like a role's swap."""
+    declared = _configured_llm()
+    with llm_overlay({"model:openai/gpt-4o-mini": "openai/gpt-4o"}):
+        agent = Agent(role="Writer", goal="g", backstory="b", llm=declared)
+        lite = LiteAgent(role="Writer", goal="g", backstory="b", llm=declared)
+
+    for built in (agent.llm, lite.llm):
+        assert built is not declared and built.model == "gpt-4o"
+        assert _configuration_of(built) == CONFIGURATION
+
+
+def test_a_role_key_wins_over_model_keys_for_that_agent() -> None:
+    with llm_overlay(
+        {
+            "Researcher": "openai/gpt-4o",
+            "model:*": "openai/gpt-4.1-nano",
+        }
+    ):
+        researcher = _agent("Researcher")
+        writer = _agent("Writer")
+        outside = Agent(
+            role="Researcher", goal="g", backstory="b", llm=_configured_llm()
+        )
+
+    assert researcher.llm.model == "gpt-4o"
+    assert writer.llm.model == "gpt-4.1-nano"
+    assert outside.llm.model == "gpt-4o"
+
+
+def test_a_role_that_looks_like_a_model_key_is_never_one() -> None:
+    with llm_overlay({"model:*": "openai/gpt-4o"}):
+        assert overlay_model_for("model:*") is None
+
+
+def test_a_model_key_that_names_no_model_is_refused() -> None:
+    with pytest.raises(ValueError, match="names no model"):
+        with llm_overlay({"model: ": "openai/gpt-4o"}):
+            pass
+
+
+def test_a_subclass_of_llm_keeps_its_model() -> None:
+    """Only `LLM` itself routes; a subclass is the caller's own choice of class."""
+    pytest.importorskip("litellm")
+
+    class Mine(LLM):
+        pass
+
+    with llm_overlay({"model:*": "openai/gpt-4o"}):
+        mine = Mine(model="groq/llama-3.1-8b-instant")
+
+    assert type(mine) is Mine and mine.model == "groq/llama-3.1-8b-instant"
+
+
+def test_a_declared_model_whose_sdk_is_missing_still_maps(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    """Nothing is built on the declared model, so its SDK need not be here; its
+    key cannot be matched to the new provider, so it stays behind."""
+    native = LLM._get_native_provider.__func__  # type: ignore[attr-defined]
+
+    def without_anthropic(cls: type[LLM], provider: str) -> Any:
+        if provider in ("anthropic", "claude"):
+            raise ImportError("Anthropic native provider not available")
+        return native(cls, provider)
+
+    monkeypatch.setattr(LLM, "_get_native_provider", classmethod(without_anthropic))
+    with llm_overlay({"model:*": "openai/gpt-4o-mini"}):
+        built = LLM(model="anthropic/claude-haiku-4-5", api_key="k", temperature=0.2)
+
+    assert type(built).__name__ == "OpenAICompletion" and built.model == "gpt-4o-mini"
+    assert built.temperature == 0.2 and built.api_key != "k"
+
+
+def test_a_litellm_routed_llm_keeps_its_key_on_the_same_provider_only() -> None:
+    """Same provider is a question about the provider, not the class: an
+    `LLM(...)` the caller routed through LiteLLM keeps its key and endpoint
+    when mapped to another model of that provider, and another provider's
+    model never gets them."""
+    pytest.importorskip("litellm")
+    declared = {"api_key": "k", "base_url": "http://localhost:9999/v1"}
+    with llm_overlay({"model:*": "openai/gpt-4o-mini"}):
+        same = LLM(model="openai/gpt-4o", is_litellm=True, **declared)
+    with llm_overlay({"model:*": "anthropic/claude-haiku-4-5"}):
+        other = LLM(model="openai/gpt-4o", is_litellm=True, **declared)
+
+    assert "gpt-4o-mini" in same.model
+    assert same.api_key == "k" and same.base_url == "http://localhost:9999/v1"
+    assert "claude-haiku-4-5" in other.model
+    assert other.api_key != "k" and other.base_url != "http://localhost:9999/v1"
+
+
+@pytest.mark.parametrize(
+    ("built", "key"),
+    [
+        ("openrouter/openai/gpt-4o", "model:openai/gpt-4o"),
+        ("openrouter/openai/gpt-4o", "model:gpt-4o"),
+    ],
+)
+def test_an_aggregators_route_is_never_the_native_model_it_names(
+    built: str, key: str
+) -> None:
+    """`openrouter/openai/gpt-4o` is OpenRouter's model, not OpenAI's: a key
+    written for the native model leaves it alone."""
+    with llm_overlay({key: "openai/gpt-4.1"}):
+        llm = LLM(model=built)
+
+    assert type(llm).__name__ == "OpenAICompatibleCompletion"
+    assert llm.model == "openai/gpt-4o"
+
+
+def test_an_aggregators_route_is_matched_by_its_own_full_name() -> None:
+    declared = LLM(model="openrouter/openai/gpt-4o")
+    with llm_overlay({"model:openrouter/openai/gpt-4o": "openai/gpt-4.1"}):
+        built = LLM(model="openrouter/openai/gpt-4o")
+        agent = Agent(role="Writer", goal="g", backstory="b", llm=declared)
+    with llm_overlay({"model:openai/gpt-4o": "openai/gpt-4.1"}):
+        untouched = Agent(role="Writer", goal="g", backstory="b", llm=declared)
+
+    assert built.model == "gpt-4.1" and agent.llm.model == "gpt-4.1"
+    assert untouched.llm is declared
+
+
+def test_a_native_provider_prefix_is_still_its_own() -> None:
+    with llm_overlay({"model:llama3": "ollama/qwen3"}):
+        assert LLM(model="ollama/llama3").model == "qwen3"
+
+
+def test_a_role_key_is_built_from_the_declaration_a_model_key_mapped() -> None:
+    """Role wins, with the caller's declared settings: the llm a model key
+    swapped on the way in is not what the role's model is built like."""
+    overlay = {"Researcher": "openai/gpt-4o", "model:*": "anthropic/claude-haiku-4-5"}
+    with llm_overlay(overlay):
+        declared = _configured_llm()  # mapped to Anthropic, without the key
+        researcher = Agent(role="Researcher", goal="g", backstory="b", llm=declared)
+        by_string = Agent(
+            role="Researcher", goal="g", backstory="b", llm="openai/gpt-4o-mini"
+        )
+        writer = Agent(role="Writer", goal="g", backstory="b", llm=declared)
+
+    assert type(declared).__name__ == "AnthropicCompletion"
+    assert type(researcher.llm).__name__ == "OpenAICompletion"
+    assert researcher.llm.model == "gpt-4o"
+    assert _configuration_of(researcher.llm) == CONFIGURATION
+    assert by_string.llm.model == "gpt-4o"
+    assert writer.llm is declared
+
+
+def test_a_providers_other_name_is_the_same_model_on_both_paths(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    """`google/x` routes as `gemini/x`, so a key written either way matches an
+    llm built in the block and one built before it, whichever name it used."""
+    pytest.importorskip("google.genai")
+    monkeypatch.setenv("GEMINI_API_KEY", "g")
+    outside = LLM(model="google/gemini-2.5-pro")  # records provider "gemini"
+    for key in ("model:google/gemini-2.5-pro", "model:gemini/gemini-2.5-pro"):
+        with llm_overlay({key: "openai/gpt-4o-mini"}):
+            built = [LLM(model="google/gemini-2.5-pro"), LLM(model="gemini/gemini-2.5-pro")]
+            agent = Agent(role="Writer", goal="g", backstory="b", llm=outside)
+
+        assert [b.model for b in built] == ["gpt-4o-mini", "gpt-4o-mini"], key
+        assert agent.llm.model == "gpt-4o-mini", key
+
+
+def test_an_aggregator_route_under_an_alias_still_is_not_the_native_model() -> None:
+    with llm_overlay({"model:google/gemini-2.5-pro": "openai/gpt-4o-mini"}):
+        llm = LLM(model="openrouter/google/gemini-2.5-pro")
+
+    assert llm.model == "google/gemini-2.5-pro"
+
+
+def test_a_caller_who_chose_litellm_keeps_it_when_the_declared_sdk_is_missing(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    pytest.importorskip("litellm")
+    native = LLM._get_native_provider.__func__  # type: ignore[attr-defined]
+
+    def without_anthropic(cls: type[LLM], provider: str) -> Any:
+        if provider in ("anthropic", "claude"):
+            raise ImportError("Anthropic native provider not available")
+        return native(cls, provider)
+
+    monkeypatch.setattr(LLM, "_get_native_provider", classmethod(without_anthropic))
+    with llm_overlay({"model:*": "openai/gpt-4o-mini"}):
+        chosen = LLM(model="anthropic/claude-haiku-4-5", is_litellm=True)
+        default = LLM(model="anthropic/claude-haiku-4-5")
+
+    assert type(chosen).__name__ == "LLM" and chosen.is_litellm
+    assert type(default).__name__ == "OpenAICompletion"