diff --git a/lib/cli/src/crewai_cli/eval_crew.py b/lib/cli/src/crewai_cli/eval_crew.py index 308def5d6..17b1ce900 100644 --- a/lib/cli/src/crewai_cli/eval_crew.py +++ b/lib/cli/src/crewai_cli/eval_crew.py @@ -49,7 +49,9 @@ STATUSES = {"queued", "running"} | FINISHED def eval_crew(run_id: str | None = None) -> None: """Evaluate the last traced run of this project, or the run RUN_ID.""" - get_or_create_project_id() + # The project's own id, minted into pyproject.toml and committed, so a + # project's evaluations can be shown together later. None outside a project. + project_id = get_or_create_project_id() # Read before the project's .env is loaded, so a project cannot add itself. trusted = _trusted_amp_origins() _load_project_env() @@ -66,7 +68,7 @@ def eval_crew(run_id: str | None = None) -> None: f"The run was traced to {recorded_amp}; evaluating at the configured AMP {client.base_url}.", style="yellow", ) - started = _start_evaluation(client, execution_id) + started = _start_evaluation(client, execution_id, project_id) url = started.get("url") console.print(f"Evaluating run [bold]{execution_id}[/bold]") if url: @@ -237,9 +239,11 @@ def _enable_tracing() -> None: ) -def _start_evaluation(client: PlusAPI, execution_id: str) -> dict[str, Any]: +def _start_evaluation( + client: PlusAPI, execution_id: str, project_id: str | None = None +) -> dict[str, Any]: try: - response = client.create_evaluation(execution_id) + response = client.create_evaluation(execution_id, project_id) except httpx.HTTPError as error: _fail(f"Could not reach AMP to start the evaluation: {error}") if response.status_code in (200, 202): diff --git a/lib/cli/src/crewai_cli/plus_api.py b/lib/cli/src/crewai_cli/plus_api.py index f03661e5b..effb6eda1 100644 --- a/lib/cli/src/crewai_cli/plus_api.py +++ b/lib/cli/src/crewai_cli/plus_api.py @@ -27,12 +27,22 @@ class PlusAPI(_CorePlusAPI): EVALUATION_START_TIMEOUT = 120.0 EVALUATION_POLL_TIMEOUT = 30.0 - def create_evaluation(self, execution_id: str) -> httpx.Response: - """Ask AMP to evaluate the traced run EXECUTION_ID (crewai eval).""" + def create_evaluation( + self, execution_id: str, project_id: str | None = None + ) -> httpx.Response: + """Ask AMP to evaluate the traced run EXECUTION_ID (crewai eval). + + PROJECT_ID is the id crewAI keeps in the project's pyproject.toml, sent + so a project's evaluations can be shown together; omitted when this is + not a crewAI project. + """ + body: dict[str, str] = {"execution_id": execution_id} + if project_id: + body["project_id"] = project_id return self._make_request( "POST", self.EVALUATIONS_RESOURCE, - json={"execution_id": execution_id}, + json=body, timeout=self.EVALUATION_START_TIMEOUT, ) diff --git a/lib/cli/tests/test_eval_crew.py b/lib/cli/tests/test_eval_crew.py index 4c91e3fe3..4014f987d 100644 --- a/lib/cli/tests/test_eval_crew.py +++ b/lib/cli/tests/test_eval_crew.py @@ -17,6 +17,7 @@ from crewai_cli.cli import eval_command EXECUTION_ID = "6f31fe1a-20bd-4bfe-a011-25d6b9341f62" +PROJECT_ID = "3e0f4b5a-1111-2222-3333-444455556666" URL = "https://evolve.crewai.test/e/ev-1" @@ -31,8 +32,8 @@ class FakeAMP: self.calls: list[tuple] = [] self.api_key = None - def create_evaluation(self, execution_id): - self.calls.append(("create", execution_id)) + def create_evaluation(self, execution_id, project_id=None): + self.calls.append(("create", execution_id, project_id)) return self.create def get_evaluation(self, evaluation_id): @@ -51,7 +52,7 @@ def done(gate="passed", grades=None): def project(tmp_path, monkeypatch): monkeypatch.chdir(tmp_path) monkeypatch.setattr(eval_module, "console", Console(width=240)) # one sentence per line in the captured output - monkeypatch.setattr(eval_module, "get_or_create_project_id", lambda: None) + monkeypatch.setattr(eval_module, "get_or_create_project_id", lambda: PROJECT_ID) monkeypatch.setattr(eval_module, "saved_login", lambda: "login-token") monkeypatch.setattr(eval_module.time, "sleep", lambda seconds: None) monkeypatch.setattr(eval_module, "is_dmn_mode_enabled", lambda: False) @@ -89,7 +90,7 @@ def test_the_last_run_is_evaluated_the_url_opened_and_the_verdict_printed(projec out = capsys.readouterr().out assert amp.api_key == "login-token" - assert amp.calls == [("create", EXECUTION_ID), ("get", "ev-1"), ("get", "ev-1")] + assert amp.calls == [("create", EXECUTION_ID, PROJECT_ID), ("get", "ev-1"), ("get", "ev-1")] assert opened == [URL] assert EXECUTION_ID in out and URL in out assert "Goal gate: PASSED" in out and "goal 5/5" in out and "cost not measured" in out @@ -104,7 +105,7 @@ def test_run_names_another_execution_and_an_anonymous_caller_sends_no_token(proj eval_module.eval_crew(run_id="other-run") assert amp.api_key is None - assert amp.calls[0] == ("create", "other-run") + assert amp.calls[0] == ("create", "other-run", PROJECT_ID) assert "Goal gate: FAILED" in capsys.readouterr().out @@ -292,7 +293,7 @@ def test_amp_unreachable_at_the_start_is_a_sentence_not_a_traceback(project, mon directory, _ = project record_last_run(directory) amp = install(monkeypatch, FakeAMP()) - monkeypatch.setattr(amp, "create_evaluation", lambda execution_id: (_ for _ in ()).throw(httpx.ConnectError("connection refused"))) + monkeypatch.setattr(amp, "create_evaluation", lambda execution_id, project_id=None: (_ for _ in ()).throw(httpx.ConnectError("connection refused"))) with pytest.raises(SystemExit) as exit_: eval_module.eval_crew() @@ -373,7 +374,7 @@ def test_run_skips_the_offer_when_nothing_is_recorded(project, monkeypatch, caps eval_module.eval_crew(run_id="named-run") - assert amp.calls[0] == ("create", "named-run") + assert amp.calls[0] == ("create", "named-run", PROJECT_ID) def test_outside_a_crewai_project_nothing_is_written_and_it_says_so(project, monkeypatch, capsys): @@ -429,7 +430,7 @@ def test_without_a_traced_run_it_offers_to_turn_tracing_on_and_run_the_crew(proj text, kwargs = prompts[0] assert "CREWAI_TRACING_ENABLED=true stays in .env" in text and kwargs == {"default": True} # Enter is yes (João's call); the prompt names both effects assert "CREWAI_TRACING_ENABLED=true" in (directory / ".env").read_text() - assert amp.calls[0] == ("create", "fresh-run") + assert amp.calls[0] == ("create", "fresh-run", PROJECT_ID) assert "Tracing is on for this project" in capsys.readouterr().out @@ -493,6 +494,24 @@ def test_only_a_missing_login_reads_as_anonymous(monkeypatch, capsys): assert "Could not read the saved login" in out and type(broken).__name__ in out and "crewai login" in out +def test_the_projects_own_id_rides_along_so_its_runs_can_be_shown_together(project, monkeypatch, capsys): + """crewAI keeps the id in pyproject.toml, committed — so it is the same id on + every machine, in CI, and for a teammate.""" + directory, _ = project + record_last_run(directory) + amp = install(monkeypatch, FakeAMP(statuses=[done()])) + + eval_module.eval_crew() + + assert amp.calls[0] == ("create", EXECUTION_ID, PROJECT_ID) + + # outside a crewAI project there is no id, and the request simply omits it + monkeypatch.setattr(eval_module, "get_or_create_project_id", lambda: None) + amp = install(monkeypatch, FakeAMP(statuses=[done()])) + eval_module.eval_crew() + assert amp.calls[0] == ("create", EXECUTION_ID, None) + + def test_read_last_run_reads_the_record_crewai_writes(tmp_path): assert eval_module.read_last_run(tmp_path) is None (tmp_path / ".crewai").mkdir() diff --git a/lib/cli/tests/test_plus_api.py b/lib/cli/tests/test_plus_api.py index e40cfdbc2..0bfc63504 100644 --- a/lib/cli/tests/test_plus_api.py +++ b/lib/cli/tests/test_plus_api.py @@ -31,7 +31,22 @@ class TestPlusAPI(unittest.TestCase): json={"execution_id": "6f31fe1a-20bd-4bfe-a011-25d6b9341f62"}, timeout=120.0, # AMP reads the run's spans inside this request ) - self.assertEqual(response, mock_response) + + @patch("crewai_core.plus_api.PlusAPI._make_request") + def test_create_evaluation_with_a_project_id(self, mock_make_request): + mock_make_request.return_value = MagicMock() + + self.api.create_evaluation("6f31fe1a-20bd-4bfe-a011-25d6b9341f62", "proj-1") + + mock_make_request.assert_called_once_with( + "POST", + "/crewai_plus/api/v1/tracing/evaluations", + json={ + "execution_id": "6f31fe1a-20bd-4bfe-a011-25d6b9341f62", + "project_id": "proj-1", + }, + timeout=120.0, + ) @patch("crewai_core.plus_api.PlusAPI._make_request") def test_get_evaluation(self, mock_make_request):