From 2bd6b72aaee7bb594a55ea32229b35599d335236 Mon Sep 17 00:00:00 2001 From: Lucas Gomide Date: Tue, 3 Jun 2025 11:09:02 -0300 Subject: [PATCH] Persist available tools from a Tool repository (#2851) * feat: add capability to see and expose public Tool classes * feat: persist available Tools from repository on publish * ci: ignore explictly templates from ruff check Ruff only applies --exclude to files it discovers itself. So we have to skip manually the same files excluded from `ruff.toml` * sytle: fix linter issues * refactor: renaming available_tools_classes by available_exports * feat: provide more context about exportable tools * feat: allow to install a Tool from pypi * test: fix tests * feat: add env_vars attribute to BaseTool * remove TODO: security check since we are handle that on enterprise side --- .github/workflows/linter.yml | 5 +- src/crewai/cli/plus_api.py | 4 +- .../tool/src/{{folder_name}}/__init__.py | 3 + src/crewai/cli/tools/main.py | 24 ++- src/crewai/cli/utils.py | 113 +++++++++++- src/crewai/tools/__init__.py | 8 +- src/crewai/tools/base_tool.py | 9 +- tests/cli/test_plus_api.py | 2 + tests/cli/test_utils.py | 161 ++++++++++++++++++ tests/cli/tools/test_main.py | 40 +++++ 10 files changed, 360 insertions(+), 9 deletions(-) diff --git a/.github/workflows/linter.yml b/.github/workflows/linter.yml index 3e1571830..421e37c01 100644 --- a/.github/workflows/linter.yml +++ b/.github/workflows/linter.yml @@ -30,4 +30,7 @@ jobs: - name: Run Ruff on Changed Files if: ${{ steps.changed-files.outputs.files != '' }} run: | - echo "${{ steps.changed-files.outputs.files }}" | tr " " "\n" | xargs -I{} ruff check "{}" + echo "${{ steps.changed-files.outputs.files }}" \ + | tr ' ' '\n' \ + | grep -v 'src/crewai/cli/templates/' \ + | xargs -I{} ruff check "{}" diff --git a/src/crewai/cli/plus_api.py b/src/crewai/cli/plus_api.py index 93e5750c8..6961f886e 100644 --- a/src/crewai/cli/plus_api.py +++ b/src/crewai/cli/plus_api.py @@ -1,5 +1,5 @@ from os import getenv -from typing import Optional +from typing import List, Optional from urllib.parse import urljoin import requests @@ -48,6 +48,7 @@ class PlusAPI: version: str, description: Optional[str], encoded_file: str, + available_exports: Optional[List[str]] = None, ): params = { "handle": handle, @@ -55,6 +56,7 @@ class PlusAPI: "version": version, "file": encoded_file, "description": description, + "available_exports": available_exports, } return self._make_request("POST", f"{self.TOOLS_RESOURCE}", json=params) diff --git a/src/crewai/cli/templates/tool/src/{{folder_name}}/__init__.py b/src/crewai/cli/templates/tool/src/{{folder_name}}/__init__.py index e69de29bb..e51d45087 100644 --- a/src/crewai/cli/templates/tool/src/{{folder_name}}/__init__.py +++ b/src/crewai/cli/templates/tool/src/{{folder_name}}/__init__.py @@ -0,0 +1,3 @@ +from .tool import {{class_name}} + +__all__ = ["{{class_name}}"] diff --git a/src/crewai/cli/tools/main.py b/src/crewai/cli/tools/main.py index 8fbe1948b..fad69467d 100644 --- a/src/crewai/cli/tools/main.py +++ b/src/crewai/cli/tools/main.py @@ -3,6 +3,7 @@ import os import subprocess import tempfile from pathlib import Path +from typing import Any import click from rich.console import Console @@ -11,6 +12,7 @@ from crewai.cli import git from crewai.cli.command import BaseCommand, PlusAPIMixin from crewai.cli.config import Settings from crewai.cli.utils import ( + extract_available_exports, get_project_description, get_project_name, get_project_version, @@ -82,6 +84,14 @@ class ToolCommand(BaseCommand, PlusAPIMixin): project_description = get_project_description(require=False) encoded_tarball = None + console.print("[bold blue]Discovering tools from your project...[/bold blue]") + available_exports = extract_available_exports() + + if available_exports: + console.print( + f"[green]Found these tools to publish: {', '.join(available_exports)}[/green]" + ) + with tempfile.TemporaryDirectory() as temp_build_dir: subprocess.run( ["uv", "build", "--sdist", "--out-dir", temp_build_dir], @@ -105,12 +115,14 @@ class ToolCommand(BaseCommand, PlusAPIMixin): encoded_tarball = base64.b64encode(tarball_contents).decode("utf-8") + console.print("[bold blue]Publishing tool to repository...[/bold blue]") publish_response = self.plus_api_client.publish_tool( handle=project_name, is_public=is_public, version=project_version, description=project_description, encoded_file=f"data:application/x-gzip;base64,{encoded_tarball}", + available_exports=available_exports, ) self._validate_response(publish_response) @@ -167,7 +179,8 @@ class ToolCommand(BaseCommand, PlusAPIMixin): "Successfully authenticated to the tool repository.", style="bold green" ) - def _add_package(self, tool_details): + def _add_package(self, tool_details: dict[str, Any]): + is_from_pypi = tool_details.get("source", None) == "pypi" tool_handle = tool_details["handle"] repository_handle = tool_details["repository"]["handle"] repository_url = tool_details["repository"]["url"] @@ -176,10 +189,13 @@ class ToolCommand(BaseCommand, PlusAPIMixin): add_package_command = [ "uv", "add", - "--index", - index, - tool_handle, ] + + if is_from_pypi: + add_package_command.append(tool_handle) + else: + add_package_command.extend(["--index", index, tool_handle]) + add_package_result = subprocess.run( add_package_command, capture_output=False, diff --git a/src/crewai/cli/utils.py b/src/crewai/cli/utils.py index f88213a58..9780b52fa 100644 --- a/src/crewai/cli/utils.py +++ b/src/crewai/cli/utils.py @@ -1,8 +1,10 @@ +import importlib.util import os import shutil import sys from functools import reduce -from inspect import isfunction, ismethod +from inspect import getmro, isclass, isfunction, ismethod +from pathlib import Path from typing import Any, Dict, List, get_type_hints import click @@ -339,3 +341,112 @@ def fetch_crews(module_attr) -> list[Crew]: if crew_instance := get_crew_instance(attr): crew_instances.append(crew_instance) return crew_instances + + +def is_valid_tool(obj): + from crewai.tools.base_tool import Tool + + if isclass(obj): + try: + return any(base.__name__ == "BaseTool" for base in getmro(obj)) + except (TypeError, AttributeError): + return False + + return isinstance(obj, Tool) + + +def extract_available_exports(dir_path: str = "src"): + """ + Extract available tool classes from the project's __init__.py files. + Only includes classes that inherit from BaseTool or functions decorated with @tool. + + Returns: + list: A list of valid tool class names or ["BaseTool"] if none found + """ + try: + init_files = Path(dir_path).glob("**/__init__.py") + available_exports = [] + + for init_file in init_files: + tools = _load_tools_from_init(init_file) + available_exports.extend(tools) + + if not available_exports: + _print_no_tools_warning() + raise SystemExit(1) + + return available_exports + + except Exception as e: + console.print(f"[red]Error: Could not extract tool classes: {str(e)}[/red]") + console.print( + "Please ensure your project contains valid tools (classes inheriting from BaseTool or functions with @tool decorator)." + ) + raise SystemExit(1) + + +def _load_tools_from_init(init_file: Path) -> list[dict[str, Any]]: + """ + Load and validate tools from a given __init__.py file. + """ + spec = importlib.util.spec_from_file_location("temp_module", init_file) + + if not spec or not spec.loader: + return [] + + module = importlib.util.module_from_spec(spec) + sys.modules["temp_module"] = module + + try: + spec.loader.exec_module(module) + + if not hasattr(module, "__all__"): + console.print( + f"[bold yellow]Warning: No __all__ defined in {init_file}[/bold yellow]" + ) + raise SystemExit(1) + + return [ + { + "name": name, + } + for name in module.__all__ + if hasattr(module, name) and is_valid_tool(getattr(module, name)) + ] + + except Exception as e: + console.print(f"[red]Warning: Could not load {init_file}: {str(e)}[/red]") + raise SystemExit(1) + + finally: + sys.modules.pop("temp_module", None) + + +def _print_no_tools_warning(): + """ + Display warning and usage instructions if no tools were found. + """ + console.print( + "\n[bold yellow]Warning: No valid tools were exposed in your __init__.py file![/bold yellow]" + ) + console.print( + "Your __init__.py file must contain all classes that inherit from [bold]BaseTool[/bold] " + "or functions decorated with [bold]@tool[/bold]." + ) + console.print( + "\nExample:\n[dim]# In your __init__.py file[/dim]\n" + "[green]__all__ = ['YourTool', 'your_tool_function'][/green]\n\n" + "[dim]# In your tool.py file[/dim]\n" + "[green]from crewai.tools import BaseTool, tool\n\n" + "# Tool class example\n" + "class YourTool(BaseTool):\n" + ' name = "your_tool"\n' + ' description = "Your tool description"\n' + " # ... rest of implementation\n\n" + "# Decorated function example\n" + "@tool\n" + "def your_tool_function(text: str) -> str:\n" + ' """Your tool description"""\n' + " # ... implementation\n" + " return result\n" + ) diff --git a/src/crewai/tools/__init__.py b/src/crewai/tools/__init__.py index 41819ccbc..2467fa906 100644 --- a/src/crewai/tools/__init__.py +++ b/src/crewai/tools/__init__.py @@ -1 +1,7 @@ -from .base_tool import BaseTool, tool +from .base_tool import BaseTool, tool, EnvVar + +__all__ = [ + "BaseTool", + "tool", + "EnvVar", +] \ No newline at end of file diff --git a/src/crewai/tools/base_tool.py b/src/crewai/tools/base_tool.py index fb0428ccd..e7d43422b 100644 --- a/src/crewai/tools/base_tool.py +++ b/src/crewai/tools/base_tool.py @@ -1,7 +1,7 @@ import asyncio from abc import ABC, abstractmethod from inspect import signature -from typing import Any, Callable, Type, get_args, get_origin +from typing import Any, Callable, Type, get_args, get_origin, Optional, List from pydantic import ( BaseModel, @@ -14,6 +14,11 @@ from pydantic import BaseModel as PydanticBaseModel from crewai.tools.structured_tool import CrewStructuredTool +class EnvVar(BaseModel): + name: str + description: str + required: bool = True + default: Optional[str] = None class BaseTool(BaseModel, ABC): class _ArgsSchemaPlaceholder(PydanticBaseModel): @@ -25,6 +30,8 @@ class BaseTool(BaseModel, ABC): """The unique name of the tool that clearly communicates its purpose.""" description: str """Used to tell the model how/when/why to use the tool.""" + env_vars: List[EnvVar] = [] + """List of environment variables used by the tool.""" args_schema: Type[PydanticBaseModel] = Field( default_factory=_ArgsSchemaPlaceholder, validate_default=True ) diff --git a/tests/cli/test_plus_api.py b/tests/cli/test_plus_api.py index daefeee42..da26ba35f 100644 --- a/tests/cli/test_plus_api.py +++ b/tests/cli/test_plus_api.py @@ -61,6 +61,7 @@ class TestPlusAPI(unittest.TestCase): "version": version, "file": encoded_file, "description": description, + "available_exports": None, } mock_make_request.assert_called_once_with( "POST", "/crewai_plus/api/v1/tools", json=params @@ -87,6 +88,7 @@ class TestPlusAPI(unittest.TestCase): "version": version, "file": encoded_file, "description": description, + "available_exports": None, } mock_make_request.assert_called_once_with( "POST", "/crewai_plus/api/v1/tools", json=params diff --git a/tests/cli/test_utils.py b/tests/cli/test_utils.py index 0270b12fc..115bb67eb 100644 --- a/tests/cli/test_utils.py +++ b/tests/cli/test_utils.py @@ -1,6 +1,7 @@ import os import shutil import tempfile +from pathlib import Path import pytest @@ -100,3 +101,163 @@ def test_tree_copy_to_existing_directory(temp_tree): assert os.path.isfile(os.path.join(dest_dir, "file1.txt")) finally: shutil.rmtree(dest_dir) + + +@pytest.fixture +def temp_project_dir(): + """Create a temporary directory for testing tool extraction.""" + with tempfile.TemporaryDirectory() as temp_dir: + yield Path(temp_dir) + + +def create_init_file(directory, content): + return create_file(directory / "__init__.py", content) + + +def test_extract_available_exports_empty_project(temp_project_dir, capsys): + with pytest.raises(SystemExit): + utils.extract_available_exports(dir_path=temp_project_dir) + captured = capsys.readouterr() + + assert "No valid tools were exposed in your __init__.py file" in captured.out + + +def test_extract_available_exports_no_init_file(temp_project_dir, capsys): + (temp_project_dir / "some_file.py").write_text("print('hello')") + with pytest.raises(SystemExit): + utils.extract_available_exports(dir_path=temp_project_dir) + captured = capsys.readouterr() + + assert "No valid tools were exposed in your __init__.py file" in captured.out + + +def test_extract_available_exports_empty_init_file(temp_project_dir, capsys): + create_init_file(temp_project_dir, "") + with pytest.raises(SystemExit): + utils.extract_available_exports(dir_path=temp_project_dir) + captured = capsys.readouterr() + + assert "Warning: No __all__ defined in" in captured.out + + +def test_extract_available_exports_no_all_variable(temp_project_dir, capsys): + create_init_file( + temp_project_dir, + "from crewai.tools import BaseTool\n\nclass MyTool(BaseTool):\n pass", + ) + with pytest.raises(SystemExit): + utils.extract_available_exports(dir_path=temp_project_dir) + captured = capsys.readouterr() + + assert "Warning: No __all__ defined in" in captured.out + + +def test_extract_available_exports_valid_base_tool_class(temp_project_dir): + create_init_file( + temp_project_dir, + """from crewai.tools import BaseTool + +class MyTool(BaseTool): + name: str = "my_tool" + description: str = "A test tool" + +__all__ = ['MyTool'] +""", + ) + tools = utils.extract_available_exports(dir_path=temp_project_dir) + assert [{"name": "MyTool"}] == tools + + +def test_extract_available_exports_valid_tool_decorator(temp_project_dir): + create_init_file( + temp_project_dir, + """from crewai.tools import tool + +@tool +def my_tool_function(text: str) -> str: + \"\"\"A test tool function\"\"\" + return text + +__all__ = ['my_tool_function'] +""", + ) + tools = utils.extract_available_exports(dir_path=temp_project_dir) + assert [{"name": "my_tool_function"}] == tools + + +def test_extract_available_exports_multiple_valid_tools(temp_project_dir): + create_init_file( + temp_project_dir, + """from crewai.tools import BaseTool, tool + +class MyTool(BaseTool): + name: str = "my_tool" + description: str = "A test tool" + +@tool +def my_tool_function(text: str) -> str: + \"\"\"A test tool function\"\"\" + return text + +__all__ = ['MyTool', 'my_tool_function'] +""", + ) + tools = utils.extract_available_exports(dir_path=temp_project_dir) + assert [{"name": "MyTool"}, {"name": "my_tool_function"}] == tools + + +def test_extract_available_exports_with_invalid_tool_decorator(temp_project_dir): + create_init_file( + temp_project_dir, + """from crewai.tools import BaseTool + +class MyTool(BaseTool): + name: str = "my_tool" + description: str = "A test tool" + +def not_a_tool(): + pass + +__all__ = ['MyTool', 'not_a_tool'] +""", + ) + tools = utils.extract_available_exports(dir_path=temp_project_dir) + assert [{"name": "MyTool"}] == tools + + +def test_extract_available_exports_import_error(temp_project_dir, capsys): + create_init_file( + temp_project_dir, + """from nonexistent_module import something + +class MyTool(BaseTool): + pass + +__all__ = ['MyTool'] +""", + ) + with pytest.raises(SystemExit): + utils.extract_available_exports(dir_path=temp_project_dir) + captured = capsys.readouterr() + + assert "nonexistent_module" in captured.out + + +def test_extract_available_exports_syntax_error(temp_project_dir, capsys): + create_init_file( + temp_project_dir, + """from crewai.tools import BaseTool + +class MyTool(BaseTool): + # Missing closing parenthesis + def __init__(self, name: + pass + +__all__ = ['MyTool'] +""", + ) + with pytest.raises(SystemExit): + utils.extract_available_exports(dir_path=temp_project_dir) + captured = capsys.readouterr() + + assert "was never closed" in captured.out diff --git a/tests/cli/tools/test_main.py b/tests/cli/tools/test_main.py index 28659a80a..79bb171b4 100644 --- a/tests/cli/tools/test_main.py +++ b/tests/cli/tools/test_main.py @@ -85,6 +85,36 @@ def test_install_success(mock_get, mock_subprocess_run, capsys, tool_command): env=unittest.mock.ANY, ) +@patch("crewai.cli.tools.main.subprocess.run") +@patch("crewai.cli.plus_api.PlusAPI.get_tool") +def test_install_success_from_pypi(mock_get, mock_subprocess_run, capsys, tool_command): + mock_get_response = MagicMock() + mock_get_response.status_code = 200 + mock_get_response.json.return_value = { + "handle": "sample-tool", + "repository": {"handle": "sample-repo", "url": "https://example.com/repo"}, + "source": "pypi", + } + mock_get.return_value = mock_get_response + mock_subprocess_run.return_value = MagicMock(stderr=None) + + tool_command.install("sample-tool") + output = capsys.readouterr().out + assert "Successfully installed sample-tool" in output + + mock_get.assert_has_calls([mock.call("sample-tool"), mock.call().json()]) + mock_subprocess_run.assert_any_call( + [ + "uv", + "add", + "sample-tool", + ], + capture_output=False, + text=True, + check=True, + env=unittest.mock.ANY, + ) + @patch("crewai.cli.plus_api.PlusAPI.get_tool") def test_install_tool_not_found(mock_get, capsys, tool_command): @@ -135,7 +165,9 @@ def test_publish_when_not_in_sync(mock_is_synced, capsys, tool_command): ) @patch("crewai.cli.plus_api.PlusAPI.publish_tool") @patch("crewai.cli.tools.main.git.Repository.is_synced", return_value=False) +@patch("crewai.cli.tools.main.extract_available_exports", return_value=["SampleTool"]) def test_publish_when_not_in_sync_and_force( + mock_available_exports, mock_is_synced, mock_publish, mock_open, @@ -168,6 +200,7 @@ def test_publish_when_not_in_sync_and_force( version="1.0.0", description="A sample tool", encoded_file=unittest.mock.ANY, + available_exports=["SampleTool"], ) @@ -183,7 +216,9 @@ def test_publish_when_not_in_sync_and_force( ) @patch("crewai.cli.plus_api.PlusAPI.publish_tool") @patch("crewai.cli.tools.main.git.Repository.is_synced", return_value=True) +@patch("crewai.cli.tools.main.extract_available_exports", return_value=["SampleTool"]) def test_publish_success( + mock_available_exports, mock_is_synced, mock_publish, mock_open, @@ -216,6 +251,7 @@ def test_publish_success( version="1.0.0", description="A sample tool", encoded_file=unittest.mock.ANY, + available_exports=["SampleTool"], ) @@ -230,7 +266,9 @@ def test_publish_success( read_data=b"sample tarball content", ) @patch("crewai.cli.plus_api.PlusAPI.publish_tool") +@patch("crewai.cli.tools.main.extract_available_exports", return_value=["SampleTool"]) def test_publish_failure( + mock_available_exports, mock_publish, mock_open, mock_listdir, @@ -266,7 +304,9 @@ def test_publish_failure( read_data=b"sample tarball content", ) @patch("crewai.cli.plus_api.PlusAPI.publish_tool") +@patch("crewai.cli.tools.main.extract_available_exports", return_value=["SampleTool"]) def test_publish_api_error( + mock_available_exports, mock_publish, mock_open, mock_listdir,