mirror of
https://github.com/crewAIInc/crewAI.git
synced 2026-10-22 01:43:13 +02:00
João MouraandClaude Opus 5.5
00e2ff98e9
* feat(cli): crewai eval prints a markdown brief when run by an agent When stdout is not a terminal (or CREWAI_DMN is on), `crewai eval` and `crewai eval --models` print the poll response's `brief_markdown` after the verdict and the report link — written straight to stdout, never through Rich markup — or "Brief (markdown): <brief_url>" when only the link came. An older AMP sends neither and nothing extra is printed. In a terminal nothing changes. Exit codes unchanged. `crewai eval --help` and the `--models` help say so, and the AGENTS.md the crew, flow and declarative-flow templates scaffold gains an "Evaluating" section telling a coding agent to run `crewai eval` after a change and act on the brief. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com> * docs(cli): an agent re-runs before it re-evaluates; each command's exit rule; the brief may be a link Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com> * fix(cli): keep the eval guidance out of tool projects; the brief is conditional in --help `crewai create tool` copies the shared AGENTS.md too, but a tool project has no `crewai run` and no traced run for `crewai eval`. The evaluation blocks are marked in the template and dropped for a tool (`evaluation=False`); crew, flow and JSON-crew projects keep them, markers stripped. --help now says the brief prints when AMP provides it, else its link, else nothing more. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com> --------- Co-authored-by: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
504 lines
18 KiB
Python
504 lines
18 KiB
Python
import os
|
|
import tempfile
|
|
import unittest
|
|
import unittest.mock
|
|
from contextlib import contextmanager
|
|
from datetime import datetime, timedelta
|
|
from pathlib import Path
|
|
from unittest import mock
|
|
from unittest.mock import MagicMock, patch
|
|
|
|
import pytest
|
|
from crewai_cli.shared.token_manager import TokenManager
|
|
from crewai_cli.tools.main import ToolCommand
|
|
from pytest import raises
|
|
|
|
|
|
@contextmanager
|
|
def in_temp_dir():
|
|
original_dir = os.getcwd()
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
os.chdir(temp_dir)
|
|
try:
|
|
yield temp_dir
|
|
finally:
|
|
os.chdir(original_dir)
|
|
|
|
|
|
@pytest.fixture
|
|
def tool_command():
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
with patch.object(
|
|
TokenManager, "_get_secure_storage_path", return_value=Path(temp_dir)
|
|
):
|
|
TokenManager().save_tokens(
|
|
"test-token", (datetime.now() + timedelta(seconds=36000)).timestamp()
|
|
)
|
|
tool_command = ToolCommand()
|
|
with patch.object(tool_command, "login"):
|
|
yield tool_command
|
|
|
|
|
|
@patch("crewai_cli.tools.main.subprocess.run")
|
|
def test_create_success(mock_subprocess, capsys, tool_command):
|
|
with in_temp_dir():
|
|
tool_command.create("test-tool")
|
|
output = capsys.readouterr().out
|
|
assert "Creating custom tool test_tool..." in output
|
|
|
|
assert os.path.isdir("test_tool")
|
|
assert os.path.isfile(os.path.join("test_tool", "README.md"))
|
|
assert os.path.isfile(os.path.join("test_tool", "pyproject.toml"))
|
|
assert os.path.isfile(
|
|
os.path.join("test_tool", "src", "test_tool", "__init__.py")
|
|
)
|
|
assert os.path.isfile(os.path.join("test_tool", "src", "test_tool", "tool.py"))
|
|
|
|
with open(os.path.join("test_tool", "pyproject.toml"), "r") as f:
|
|
content = f.read()
|
|
assert '"crewai[tools]>=1.15.0,<2.0.0"' in content
|
|
|
|
with open(os.path.join("test_tool", "src", "test_tool", "tool.py"), "r") as f:
|
|
content = f.read()
|
|
assert "class TestTool" in content
|
|
|
|
mock_subprocess.assert_called_once_with(["git", "init"], check=True)
|
|
|
|
|
|
@patch("crewai_cli.tools.main.subprocess.run")
|
|
def test_create_scaffolds_assistant_instructions(mock_subprocess, tool_command):
|
|
with in_temp_dir():
|
|
tool_command.create("test-tool")
|
|
|
|
agents_md = Path("test_tool", "AGENTS.md").read_text(encoding="utf-8")
|
|
assert "CrewAI Reference for AI Coding Assistants" in agents_md
|
|
assert "Never disable, block, or silence CrewAI's built-in observability" in agents_md
|
|
claude_md = Path("test_tool", "CLAUDE.md").read_text(encoding="utf-8")
|
|
assert "@AGENTS.md" in claude_md.splitlines()
|
|
cursor_md = Path("test_tool", "CURSOR.md").read_text(encoding="utf-8")
|
|
assert "@AGENTS.md" in cursor_md.splitlines()
|
|
gemini_md = Path("test_tool", "GEMINI.md").read_text(encoding="utf-8")
|
|
assert "@./AGENTS.md" in gemini_md.splitlines()
|
|
|
|
|
|
@patch("crewai_cli.tools.main.subprocess.run")
|
|
def test_a_tool_project_gets_no_evaluation_guidance(mock_subprocess, tool_command):
|
|
# A tool project has no `crewai run` and no traced run for `crewai eval`.
|
|
with in_temp_dir():
|
|
tool_command.create("test-tool")
|
|
|
|
agents_md = Path("test_tool", "AGENTS.md").read_text(encoding="utf-8")
|
|
assert "Evaluating this" not in agents_md
|
|
assert "crewai eval" not in agents_md
|
|
assert "crewai-eval:" not in agents_md
|
|
assert "## Quick Reference" in agents_md
|
|
|
|
|
|
@patch("crewai_cli.tools.main.subprocess.run")
|
|
@patch("crewai_cli.plus_api.PlusAPI.get_tool")
|
|
@patch("crewai_cli.tools.main.ToolCommand._print_current_organization")
|
|
def test_install_success(
|
|
mock_print_org, mock_get, mock_subprocess_run, capsys, tool_command
|
|
):
|
|
mock_get_response = MagicMock()
|
|
mock_get_response.status_code = 200
|
|
mock_get_response.json.return_value = {
|
|
"handle": "sample-tool",
|
|
"repository": {"handle": "sample-repo", "url": "https://example.com/repo"},
|
|
}
|
|
mock_get.return_value = mock_get_response
|
|
mock_subprocess_run.return_value = MagicMock(stderr=None)
|
|
|
|
tool_command.install("sample-tool")
|
|
output = capsys.readouterr().out
|
|
assert "Successfully installed sample-tool" in output
|
|
|
|
mock_get.assert_has_calls([mock.call("sample-tool"), mock.call().json()])
|
|
mock_subprocess_run.assert_any_call(
|
|
[
|
|
"uv",
|
|
"add",
|
|
"--index",
|
|
"sample-repo=https://example.com/repo",
|
|
"sample-tool",
|
|
],
|
|
capture_output=False,
|
|
text=True,
|
|
check=True,
|
|
env=unittest.mock.ANY,
|
|
)
|
|
|
|
mock_print_org.assert_called_once()
|
|
|
|
|
|
@patch("crewai_cli.tools.main.subprocess.run")
|
|
@patch("crewai_cli.plus_api.PlusAPI.get_tool")
|
|
def test_install_success_from_pypi(mock_get, mock_subprocess_run, capsys, tool_command):
|
|
mock_get_response = MagicMock()
|
|
mock_get_response.status_code = 200
|
|
mock_get_response.json.return_value = {
|
|
"handle": "sample-tool",
|
|
"repository": {"handle": "sample-repo", "url": "https://example.com/repo"},
|
|
"source": "pypi",
|
|
}
|
|
mock_get.return_value = mock_get_response
|
|
mock_subprocess_run.return_value = MagicMock(stderr=None)
|
|
|
|
tool_command.install("sample-tool")
|
|
output = capsys.readouterr().out
|
|
assert "Successfully installed sample-tool" in output
|
|
|
|
mock_get.assert_has_calls([mock.call("sample-tool"), mock.call().json()])
|
|
mock_subprocess_run.assert_any_call(
|
|
[
|
|
"uv",
|
|
"add",
|
|
"sample-tool",
|
|
],
|
|
capture_output=False,
|
|
text=True,
|
|
check=True,
|
|
env=unittest.mock.ANY,
|
|
)
|
|
|
|
|
|
@patch("crewai_cli.plus_api.PlusAPI.get_tool")
|
|
def test_install_tool_not_found(mock_get, capsys, tool_command):
|
|
mock_get_response = MagicMock()
|
|
mock_get_response.status_code = 404
|
|
mock_get.return_value = mock_get_response
|
|
|
|
with raises(SystemExit):
|
|
tool_command.install("non-existent-tool")
|
|
output = capsys.readouterr().out
|
|
assert "No tool found with this name" in output
|
|
|
|
mock_get.assert_called_once_with("non-existent-tool")
|
|
|
|
|
|
@patch("crewai_cli.plus_api.PlusAPI.get_tool")
|
|
def test_install_api_error(mock_get, capsys, tool_command):
|
|
mock_get_response = MagicMock()
|
|
mock_get_response.status_code = 500
|
|
mock_get.return_value = mock_get_response
|
|
|
|
with raises(SystemExit):
|
|
tool_command.install("error-tool")
|
|
output = capsys.readouterr().out
|
|
assert "Failed to get tool details" in output
|
|
|
|
mock_get.assert_called_once_with("error-tool")
|
|
|
|
|
|
@patch("crewai_cli.tools.main.git.Repository")
|
|
def test_publish_when_not_in_sync(mock_repository, capsys, tool_command):
|
|
mock_repository.return_value.is_synced.return_value = False
|
|
|
|
with raises(SystemExit):
|
|
tool_command.publish(is_public=True)
|
|
|
|
output = capsys.readouterr().out
|
|
assert "Local changes need to be resolved before publishing" in output
|
|
mock_repository.return_value.is_synced.assert_called_once_with()
|
|
|
|
|
|
@patch("crewai_cli.tools.main.get_project_name", return_value="sample-tool")
|
|
@patch("crewai_cli.tools.main.get_project_version", return_value="1.0.0")
|
|
@patch("crewai_cli.tools.main.get_project_description", return_value="A sample tool")
|
|
@patch("crewai_cli.tools.main.subprocess.run")
|
|
@patch("crewai_cli.tools.main.os.listdir", return_value=["sample-tool-1.0.0.tar.gz"])
|
|
@patch(
|
|
"crewai_cli.tools.main.open",
|
|
new_callable=unittest.mock.mock_open,
|
|
read_data=b"sample tarball content",
|
|
)
|
|
@patch("crewai_cli.tools.main.git.Repository.fetch")
|
|
@patch("crewai_cli.plus_api.PlusAPI.publish_tool")
|
|
@patch("crewai_cli.tools.main.git.Repository.is_synced", return_value=False)
|
|
@patch(
|
|
"crewai.utilities.project_utils.extract_available_exports",
|
|
return_value=[{"name": "SampleTool"}],
|
|
)
|
|
@patch(
|
|
"crewai.utilities.project_utils.extract_tools_metadata",
|
|
return_value=[{"name": "SampleTool", "humanized_name": "sample_tool", "description": "A sample tool", "run_params_schema": {}, "init_params_schema": {}, "env_vars": []}],
|
|
)
|
|
@patch("crewai_cli.tools.main.ToolCommand._print_current_organization")
|
|
def test_publish_when_not_in_sync_and_force(
|
|
mock_print_org,
|
|
mock_tools_metadata,
|
|
mock_available_exports,
|
|
mock_is_synced,
|
|
mock_publish,
|
|
mock_fetch,
|
|
mock_open,
|
|
mock_listdir,
|
|
mock_subprocess_run,
|
|
mock_get_project_description,
|
|
mock_get_project_version,
|
|
mock_get_project_name,
|
|
tool_command,
|
|
):
|
|
mock_publish_response = MagicMock()
|
|
mock_publish_response.status_code = 200
|
|
mock_publish_response.json.return_value = {"handle": "sample-tool"}
|
|
mock_publish.return_value = mock_publish_response
|
|
|
|
tool_command.publish(is_public=True, force=True)
|
|
|
|
mock_get_project_name.assert_called_with(require=True)
|
|
mock_get_project_version.assert_called_with(require=True)
|
|
mock_get_project_description.assert_called_with(require=False)
|
|
mock_subprocess_run.assert_called_with(
|
|
["uv", "build", "--sdist", "--out-dir", unittest.mock.ANY],
|
|
check=True,
|
|
capture_output=False,
|
|
env=unittest.mock.ANY,
|
|
)
|
|
mock_open.assert_called_with(unittest.mock.ANY, "rb")
|
|
mock_publish.assert_called_with(
|
|
handle="sample-tool",
|
|
is_public=True,
|
|
version="1.0.0",
|
|
description="A sample tool",
|
|
encoded_file=unittest.mock.ANY,
|
|
available_exports=[{"name": "SampleTool"}],
|
|
tools_metadata=[{"name": "SampleTool", "humanized_name": "sample_tool", "description": "A sample tool", "run_params_schema": {}, "init_params_schema": {}, "env_vars": []}],
|
|
)
|
|
mock_print_org.assert_called_once()
|
|
|
|
|
|
@patch("crewai_cli.tools.main.get_project_name", return_value="sample-tool")
|
|
@patch("crewai_cli.tools.main.get_project_version", return_value="1.0.0")
|
|
@patch("crewai_cli.tools.main.get_project_description", return_value="A sample tool")
|
|
@patch("crewai_cli.tools.main.subprocess.run")
|
|
@patch("crewai_cli.tools.main.os.listdir", return_value=["sample-tool-1.0.0.tar.gz"])
|
|
@patch(
|
|
"crewai_cli.tools.main.open",
|
|
new_callable=unittest.mock.mock_open,
|
|
read_data=b"sample tarball content",
|
|
)
|
|
@patch("crewai_cli.tools.main.git.Repository.fetch")
|
|
@patch("crewai_cli.plus_api.PlusAPI.publish_tool")
|
|
@patch("crewai_cli.tools.main.git.Repository.is_synced", return_value=True)
|
|
@patch(
|
|
"crewai.utilities.project_utils.extract_available_exports",
|
|
return_value=[{"name": "SampleTool"}],
|
|
)
|
|
@patch(
|
|
"crewai.utilities.project_utils.extract_tools_metadata",
|
|
return_value=[{"name": "SampleTool", "humanized_name": "sample_tool", "description": "A sample tool", "run_params_schema": {}, "init_params_schema": {}, "env_vars": []}],
|
|
)
|
|
def test_publish_success(
|
|
mock_tools_metadata,
|
|
mock_available_exports,
|
|
mock_is_synced,
|
|
mock_publish,
|
|
mock_fetch,
|
|
mock_open,
|
|
mock_listdir,
|
|
mock_subprocess_run,
|
|
mock_get_project_description,
|
|
mock_get_project_version,
|
|
mock_get_project_name,
|
|
tool_command,
|
|
):
|
|
mock_publish_response = MagicMock()
|
|
mock_publish_response.status_code = 200
|
|
mock_publish_response.json.return_value = {"handle": "sample-tool"}
|
|
mock_publish.return_value = mock_publish_response
|
|
|
|
tool_command.publish(is_public=True)
|
|
|
|
mock_get_project_name.assert_called_with(require=True)
|
|
mock_get_project_version.assert_called_with(require=True)
|
|
mock_get_project_description.assert_called_with(require=False)
|
|
mock_subprocess_run.assert_called_with(
|
|
["uv", "build", "--sdist", "--out-dir", unittest.mock.ANY],
|
|
check=True,
|
|
capture_output=False,
|
|
env=unittest.mock.ANY,
|
|
)
|
|
mock_open.assert_called_with(unittest.mock.ANY, "rb")
|
|
mock_publish.assert_called_with(
|
|
handle="sample-tool",
|
|
is_public=True,
|
|
version="1.0.0",
|
|
description="A sample tool",
|
|
encoded_file=unittest.mock.ANY,
|
|
available_exports=[{"name": "SampleTool"}],
|
|
tools_metadata=[{"name": "SampleTool", "humanized_name": "sample_tool", "description": "A sample tool", "run_params_schema": {}, "init_params_schema": {}, "env_vars": []}],
|
|
)
|
|
|
|
|
|
@patch("crewai_cli.tools.main.get_project_name", return_value="sample-tool")
|
|
@patch("crewai_cli.tools.main.get_project_version", return_value="1.0.0")
|
|
@patch("crewai_cli.tools.main.get_project_description", return_value="A sample tool")
|
|
@patch("crewai_cli.tools.main.subprocess.run")
|
|
@patch("crewai_cli.tools.main.os.listdir", return_value=["sample-tool-1.0.0.tar.gz"])
|
|
@patch(
|
|
"crewai_cli.tools.main.open",
|
|
new_callable=unittest.mock.mock_open,
|
|
read_data=b"sample tarball content",
|
|
)
|
|
@patch("crewai_cli.plus_api.PlusAPI.publish_tool")
|
|
@patch(
|
|
"crewai.utilities.project_utils.extract_available_exports",
|
|
return_value=[{"name": "SampleTool"}],
|
|
)
|
|
@patch(
|
|
"crewai.utilities.project_utils.extract_tools_metadata",
|
|
return_value=[{"name": "SampleTool", "humanized_name": "sample_tool", "description": "A sample tool", "run_params_schema": {}, "init_params_schema": {}, "env_vars": []}],
|
|
)
|
|
def test_publish_failure(
|
|
mock_tools_metadata,
|
|
mock_available_exports,
|
|
mock_publish,
|
|
mock_open,
|
|
mock_listdir,
|
|
mock_subprocess_run,
|
|
mock_get_project_description,
|
|
mock_get_project_version,
|
|
mock_get_project_name,
|
|
capsys,
|
|
tool_command,
|
|
):
|
|
mock_publish_response = MagicMock()
|
|
mock_publish_response.status_code = 422
|
|
mock_publish_response.json.return_value = {"name": ["is already taken"]}
|
|
mock_publish.return_value = mock_publish_response
|
|
|
|
with raises(SystemExit):
|
|
tool_command.publish(is_public=True)
|
|
output = capsys.readouterr().out
|
|
assert "Failed to complete operation" in output
|
|
assert "Name is already taken" in output
|
|
|
|
mock_publish.assert_called_once()
|
|
|
|
|
|
@patch("crewai_cli.tools.main.get_project_name", return_value="sample-tool")
|
|
@patch("crewai_cli.tools.main.get_project_version", return_value="1.0.0")
|
|
@patch("crewai_cli.tools.main.get_project_description", return_value="A sample tool")
|
|
@patch("crewai_cli.tools.main.subprocess.run")
|
|
@patch("crewai_cli.tools.main.os.listdir", return_value=["sample-tool-1.0.0.tar.gz"])
|
|
@patch(
|
|
"crewai_cli.tools.main.open",
|
|
new_callable=unittest.mock.mock_open,
|
|
read_data=b"sample tarball content",
|
|
)
|
|
@patch("crewai_cli.plus_api.PlusAPI.publish_tool")
|
|
@patch(
|
|
"crewai.utilities.project_utils.extract_available_exports",
|
|
return_value=[{"name": "SampleTool"}],
|
|
)
|
|
@patch(
|
|
"crewai.utilities.project_utils.extract_tools_metadata",
|
|
return_value=[{"name": "SampleTool", "humanized_name": "sample_tool", "description": "A sample tool", "run_params_schema": {}, "init_params_schema": {}, "env_vars": []}],
|
|
)
|
|
def test_publish_api_error(
|
|
mock_tools_metadata,
|
|
mock_available_exports,
|
|
mock_publish,
|
|
mock_open,
|
|
mock_listdir,
|
|
mock_subprocess_run,
|
|
mock_get_project_description,
|
|
mock_get_project_version,
|
|
mock_get_project_name,
|
|
capsys,
|
|
tool_command,
|
|
):
|
|
mock_response = MagicMock()
|
|
mock_response.status_code = 500
|
|
mock_response.json.return_value = {"error": "Internal Server Error"}
|
|
mock_response.is_success = False
|
|
mock_publish.return_value = mock_response
|
|
|
|
with raises(SystemExit):
|
|
tool_command.publish(is_public=True)
|
|
output = capsys.readouterr().out
|
|
assert "Request to Enterprise API failed" in output
|
|
|
|
mock_publish.assert_called_once()
|
|
|
|
|
|
@patch("crewai_cli.tools.main.get_project_name", return_value="sample-tool")
|
|
@patch("crewai_cli.tools.main.get_project_version", return_value="1.0.0")
|
|
@patch("crewai_cli.tools.main.get_project_description", return_value="A sample tool")
|
|
@patch("crewai_cli.tools.main.subprocess.run")
|
|
@patch("crewai_cli.tools.main.os.listdir", return_value=["sample-tool-1.0.0.tar.gz"])
|
|
@patch(
|
|
"crewai_cli.tools.main.open",
|
|
new_callable=unittest.mock.mock_open,
|
|
read_data=b"sample tarball content",
|
|
)
|
|
@patch("crewai_cli.plus_api.PlusAPI.publish_tool")
|
|
@patch("crewai_cli.tools.main.git.Repository.is_synced", return_value=True)
|
|
@patch(
|
|
"crewai.utilities.project_utils.extract_available_exports",
|
|
return_value=[{"name": "SampleTool"}],
|
|
)
|
|
@patch(
|
|
"crewai.utilities.project_utils.extract_tools_metadata",
|
|
side_effect=Exception("Failed to extract metadata"),
|
|
)
|
|
def test_publish_metadata_extraction_failure_continues_with_warning(
|
|
mock_tools_metadata,
|
|
mock_available_exports,
|
|
mock_is_synced,
|
|
mock_publish,
|
|
mock_open,
|
|
mock_listdir,
|
|
mock_subprocess_run,
|
|
mock_get_project_description,
|
|
mock_get_project_version,
|
|
mock_get_project_name,
|
|
capsys,
|
|
tool_command,
|
|
):
|
|
"""Test that metadata extraction failure shows warning but continues publishing."""
|
|
mock_publish_response = MagicMock()
|
|
mock_publish_response.status_code = 200
|
|
mock_publish_response.json.return_value = {"handle": "sample-tool"}
|
|
mock_publish.return_value = mock_publish_response
|
|
|
|
tool_command.publish(is_public=True)
|
|
|
|
output = capsys.readouterr().out
|
|
assert "Warning: Could not extract tool metadata" in output
|
|
assert "Publishing will continue without detailed metadata" in output
|
|
assert "No tool metadata extracted" in output
|
|
mock_publish.assert_called_once_with(
|
|
handle="sample-tool",
|
|
is_public=True,
|
|
version="1.0.0",
|
|
description="A sample tool",
|
|
encoded_file=unittest.mock.ANY,
|
|
available_exports=[{"name": "SampleTool"}],
|
|
tools_metadata=[],
|
|
)
|
|
|
|
|
|
@patch("crewai_cli.tools.main.Settings")
|
|
def test_print_current_organization_with_org(mock_settings, capsys, tool_command):
|
|
mock_settings_instance = MagicMock()
|
|
mock_settings_instance.org_uuid = "test-org-uuid"
|
|
mock_settings_instance.org_name = "Test Organization"
|
|
mock_settings.return_value = mock_settings_instance
|
|
tool_command._print_current_organization()
|
|
output = capsys.readouterr().out
|
|
assert "Current organization: Test Organization (test-org-uuid)" in output
|
|
|
|
|
|
@patch("crewai_cli.tools.main.Settings")
|
|
def test_print_current_organization_without_org(mock_settings, capsys, tool_command):
|
|
mock_settings_instance = MagicMock()
|
|
mock_settings_instance.org_uuid = None
|
|
mock_settings_instance.org_name = None
|
|
mock_settings.return_value = mock_settings_instance
|
|
tool_command._print_current_organization()
|
|
output = capsys.readouterr().out
|
|
assert "No organization currently set" in output
|
|
assert "org switch <org_id>" in output
|