mirror of
https://github.com/crewAIInc/crewAI.git
synced 2026-08-11 00:42:48 +00:00
Merge branch 'main' into fix/native-tool-call-responses-api-shape
This commit is contained in:
Rip&Tear
@@ -8,7 +8,7 @@ authors = [
|
||||
]
|
||||
requires-python = ">=3.10, <3.14"
|
||||
dependencies = [
|
||||
"crewai-core==1.15.11",
|
||||
"crewai-core==1.15.12",
|
||||
"click>=8.1.7,<9",
|
||||
"pydantic>=2.11.9,<2.13",
|
||||
"pydantic-settings>=2.14.2,<3",
|
||||
|
||||
@@ -1 +1 @@
|
||||
__version__ = "1.15.11"
|
||||
__version__ = "1.15.12"
|
||||
|
||||
@@ -19,6 +19,7 @@ from crewai_cli.utils import (
|
||||
enable_prompt_line_editing,
|
||||
is_dmn_mode_enabled,
|
||||
read_toml,
|
||||
warn_deprecated_command,
|
||||
)
|
||||
|
||||
|
||||
@@ -137,7 +138,10 @@ def uv(uv_args: tuple[str, ...]) -> None:
|
||||
|
||||
@crewai.command()
|
||||
@click.argument(
|
||||
"type", required=False, default=None, type=click.Choice(["crew", "flow"])
|
||||
"type",
|
||||
required=False,
|
||||
default=None,
|
||||
type=click.Choice(["crew", "flow", "tool", "skill", "template"]),
|
||||
)
|
||||
@click.argument("name", required=False, default=None)
|
||||
@click.option("--provider", type=str, help="The provider to use for the crew")
|
||||
@@ -152,6 +156,21 @@ def uv(uv_args: tuple[str, ...]) -> None:
|
||||
is_flag=True,
|
||||
help="Create a declarative Flow project instead of a Python Flow project",
|
||||
)
|
||||
@click.option(
|
||||
"--no-project",
|
||||
"in_project",
|
||||
is_flag=True,
|
||||
default=True,
|
||||
flag_value=False,
|
||||
help="Skill only: create in current dir instead of ./skills/",
|
||||
)
|
||||
@click.option(
|
||||
"-o",
|
||||
"--output-dir",
|
||||
type=str,
|
||||
default=None,
|
||||
help="Template only: directory name for the template (defaults to template name)",
|
||||
)
|
||||
def create(
|
||||
type: str | None,
|
||||
name: str | None,
|
||||
@@ -159,14 +178,17 @@ def create(
|
||||
skip_provider: bool = False,
|
||||
classic: bool = False,
|
||||
declarative: bool = False,
|
||||
in_project: bool = True,
|
||||
output_dir: str | None = None,
|
||||
) -> None:
|
||||
"""Create a new crew, or flow."""
|
||||
"""Create a new crew, flow, tool, skill, or template."""
|
||||
dmn_mode = is_dmn_mode_enabled()
|
||||
if not type:
|
||||
if dmn_mode:
|
||||
raise click.UsageError(
|
||||
"TYPE is required when CREWAI_DMN is set. "
|
||||
"Use `crewai create crew <name>` or `crewai create flow <name>`."
|
||||
"Use `crewai create <type> <name>` where type is one of: "
|
||||
"crew, flow, tool, skill, template."
|
||||
)
|
||||
from crewai_cli.tui_picker import pick
|
||||
|
||||
@@ -176,6 +198,9 @@ def create(
|
||||
"flow",
|
||||
"A deterministic workflow with full control over agents and crews",
|
||||
),
|
||||
("tool", "A custom tool for the CrewAI Tool Repository"),
|
||||
("skill", "An agent skill with instructions and optional assets"),
|
||||
("template", "A remote project template from the CrewAI gallery"),
|
||||
]
|
||||
type = pick("What would you like to create?", options)
|
||||
if type is None:
|
||||
@@ -189,9 +214,36 @@ def create(
|
||||
click.style(f" Name of your {type}", fg="cyan", bold=True),
|
||||
prompt_suffix=click.style(" › ", fg="bright_white"), # noqa: RUF001
|
||||
)
|
||||
if dmn_mode:
|
||||
if dmn_mode and type == "crew":
|
||||
skip_provider = True
|
||||
if type == "crew":
|
||||
if not in_project and type != "skill":
|
||||
raise click.UsageError("--no-project can only be used with skill projects.")
|
||||
if output_dir is not None and type != "template":
|
||||
raise click.UsageError("--output-dir can only be used with template projects.")
|
||||
if type == "tool":
|
||||
if declarative or classic or provider is not None or skip_provider:
|
||||
raise click.UsageError(
|
||||
"Crew and flow options cannot be used with tool projects."
|
||||
)
|
||||
from crewai_cli.tools.main import ToolCommand
|
||||
|
||||
ToolCommand().create(name)
|
||||
elif type == "skill":
|
||||
if declarative or classic or provider is not None or skip_provider:
|
||||
raise click.UsageError(
|
||||
"Crew and flow options cannot be used with skill projects."
|
||||
)
|
||||
from crewai_cli.skills.main import SkillCommand
|
||||
|
||||
SkillCommand().create(name, in_project=in_project)
|
||||
elif type == "template":
|
||||
if declarative or classic or provider is not None or skip_provider:
|
||||
raise click.UsageError(
|
||||
"Crew and flow options cannot be used with template projects."
|
||||
)
|
||||
template_cmd = TemplateCommand()
|
||||
template_cmd.add_template(name, output_dir)
|
||||
elif type == "crew":
|
||||
if declarative:
|
||||
raise click.UsageError("--declarative can only be used with flow projects")
|
||||
if classic:
|
||||
@@ -207,7 +259,10 @@ def create(
|
||||
|
||||
create_flow(name, declarative=declarative)
|
||||
else:
|
||||
click.secho("Error: Invalid type. Must be 'crew' or 'flow'.", fg="red")
|
||||
click.secho(
|
||||
"Error: Invalid type. Must be 'crew', 'flow', 'tool', 'skill', or 'template'.",
|
||||
fg="red",
|
||||
)
|
||||
|
||||
|
||||
@crewai.command()
|
||||
@@ -652,6 +707,8 @@ def tool() -> None:
|
||||
@tool.command(name="create")
|
||||
@click.argument("handle")
|
||||
def tool_create(handle: str) -> None:
|
||||
"""[Deprecated: use `crewai create tool`] Create a custom tool project."""
|
||||
warn_deprecated_command(old="crewai tool create", new="crewai create tool")
|
||||
from crewai_cli.tools.main import ToolCommand
|
||||
|
||||
tool_cmd = ToolCommand()
|
||||
@@ -702,6 +759,8 @@ def skill() -> None:
|
||||
help="Create skill in current dir instead of ./skills/",
|
||||
)
|
||||
def skill_create(name: str, in_project: bool) -> None:
|
||||
"""[Deprecated: use `crewai create skill`] Create a new agent skill."""
|
||||
warn_deprecated_command(old="crewai skill create", new="crewai create skill")
|
||||
from crewai_cli.skills.main import SkillCommand
|
||||
|
||||
skill_cmd = SkillCommand()
|
||||
@@ -765,7 +824,8 @@ def template_list() -> None:
|
||||
help="Directory name for the template (defaults to template name)",
|
||||
)
|
||||
def template_add(name: str, output_dir: str | None) -> None:
|
||||
"""Add a template to the current directory."""
|
||||
"""[Deprecated: use `crewai create template`] Add a template to the current directory."""
|
||||
warn_deprecated_command(old="crewai template add", new="crewai create template")
|
||||
template_cmd = TemplateCommand()
|
||||
template_cmd.add_template(name, output_dir)
|
||||
|
||||
|
||||
@@ -40,6 +40,14 @@ This ensures generated code always matches the version actually installed, not s
|
||||
- ❌ `Agent(llm=ChatOpenAI(...))` → ✅ `Agent(llm="openai/gpt-4o")` or `Agent(llm=LLM(model="..."))`
|
||||
- ❌ Passing raw OpenAI client objects → ✅ Use `crewai.LLM` wrapper
|
||||
|
||||
### Deprecated CLI scaffolding aliases (still supported)
|
||||
|
||||
These commands remain supported but print a yellow deprecation warning. Prefer the canonical forms:
|
||||
|
||||
- ⚠️ `crewai tool create <handle>` → ✅ `crewai create tool <handle>`
|
||||
- ⚠️ `crewai skill create <name>` → ✅ `crewai create skill <name>`
|
||||
- ⚠️ `crewai template add <name>` → ✅ `crewai create template <name>`
|
||||
|
||||
### How to verify you're using current patterns:
|
||||
1. You ran the version check and docs lookup steps above before writing code
|
||||
2. All LLM references use `crewai.LLM` or string shorthand (`"openai/gpt-4o"`)
|
||||
@@ -137,8 +145,26 @@ uv sync # Sync dependencies
|
||||
uv lock # Lock dependencies
|
||||
|
||||
# Project scaffolding
|
||||
crewai create crew <name> --skip_provider # New crew project
|
||||
crewai create flow <name> --skip_provider # New flow project
|
||||
crewai create crew <name> --skip_provider # New crew project
|
||||
crewai create flow <name> # New flow project
|
||||
crewai create tool <handle> # Custom tool repository
|
||||
crewai create skill <name> # Agent skill (./skills/ in crew projects)
|
||||
crewai create skill <name> --no-project # Skill in current directory
|
||||
crewai create template <name> # Remote project template
|
||||
crewai create template <name> -o <output_dir> # Template with custom output directory
|
||||
|
||||
# Deprecated scaffolding aliases (still work; print a yellow warning)
|
||||
# crewai tool create <handle> → crewai create tool <handle>
|
||||
# crewai skill create <name> → crewai create skill <name>
|
||||
# crewai template add <name> → crewai create template <name>
|
||||
|
||||
# Tool, skill, and template lifecycle (unchanged)
|
||||
crewai tool install <handle>
|
||||
crewai tool publish
|
||||
crewai skill install @org/name
|
||||
crewai skill publish
|
||||
crewai skill list
|
||||
crewai template list
|
||||
|
||||
# Running
|
||||
crewai run # Run crew or flow (auto-detects from pyproject.toml)
|
||||
@@ -627,6 +653,26 @@ class MyFlow(Flow):
|
||||
| `@listen(method)` | Triggers when specified method completes. Receives output as argument |
|
||||
| `@router(method)` | Conditional branching. Returns string labels that trigger `@listen("label")` |
|
||||
|
||||
### `@listen` labels vs handler names
|
||||
|
||||
The string in `@listen("...")` is an **event or route label**, not the Python method name. Router return values, route labels, and method completion events share one trigger namespace.
|
||||
|
||||
**Never** use the same name for the `@listen` label and the handler method:
|
||||
|
||||
```python
|
||||
# ❌ Wrong — raises a validation error when the flow is instantiated
|
||||
@listen("create_video")
|
||||
def create_video(self):
|
||||
...
|
||||
|
||||
# ✅ Correct — distinct handler name (handle_* prefix is a common pattern)
|
||||
@listen("create_video")
|
||||
def handle_create_video(self):
|
||||
...
|
||||
```
|
||||
|
||||
If validation were bypassed, matching names would also cause the handler to re-trigger itself in a loop at runtime. This applies to all flows. It is especially common in **conversational flows** (`conversational = True`), where `@listen("...")` is a router intent name — do not name the handler after the route it serves.
|
||||
|
||||
### Structured State
|
||||
```python
|
||||
from pydantic import BaseModel
|
||||
@@ -1113,7 +1159,9 @@ Python >=3.10, <3.14
|
||||
```bash
|
||||
uv tool install crewai # Install CrewAI CLI
|
||||
uv tool list # Verify installation
|
||||
crewai create crew my_crew --skip_provider # Scaffold a new project
|
||||
crewai create crew my_crew --skip_provider # Scaffold a crew project
|
||||
crewai create tool my_tool # Scaffold a tool repository
|
||||
crewai create skill my_skill # Scaffold an agent skill
|
||||
crewai install # Install project dependencies
|
||||
crewai run # Execute
|
||||
```
|
||||
@@ -1148,3 +1196,4 @@ crewai run # Execute
|
||||
- Using `process=Process.hierarchical` without setting `manager_llm` or `manager_agent`
|
||||
- Circular delegation: set `allow_delegation=False` on specialist agents
|
||||
- Not installing tools package: `uv add crewai-tools`
|
||||
- **Matching `@listen("label")` to the handler method name** — raises a validation error at flow instantiation; would re-trigger in an infinite loop at runtime only if validation is bypassed. Use a different method name (e.g. `handle_create_video` for `@listen("create_video")`)
|
||||
|
||||
@@ -39,8 +39,8 @@ Pick the simplest action that does the job.
|
||||
- `state` is the initial shared data shape. Action results do not automatically merge into `state`.
|
||||
- Read method results with `outputs.method_name` after that method can run.
|
||||
- `listen` targets a method name or a router-emitted event name.
|
||||
- Methods must not listen to their own method name.
|
||||
- Method names and emitted event names share one namespace. Avoid reusing the same string for both unless the user explicitly wants that.
|
||||
- Methods must not listen to their own method name — including when the `listen` value is a route label that matches the method name (e.g. `listen: create_video` on method `create_video`).
|
||||
- Method names and emitted event names share one namespace. Do not reuse the same string for a method's `listen` target and its method name.
|
||||
- Use `router: true` plus `emit` when one method chooses between named branches.
|
||||
- A router action must return exactly one emitted event string. It must not return JSON, a list, or an explanation.
|
||||
- Use `start: true` for the single entrypoint.
|
||||
@@ -107,8 +107,8 @@ Dynamic value rules:
|
||||
- Do not make `do` a list.
|
||||
- Do not use CEL `+` to build text in action mappings. Keep the text literal and insert each dynamic value with `${...}`.
|
||||
- Do not reference `outputs.some_method` before `some_method` can run.
|
||||
- Do not set a method's `listen` to its own method name.
|
||||
- Do not use the same string for an emitted event and a method name unless the user asks for it.
|
||||
- Do not set a method's `listen` to its own method name (including matching route labels such as `listen: create_video` on method `create_video`).
|
||||
- Do not use the same string for a method's `listen` target and its method name.
|
||||
- Do not use `emit` without `router: true`.
|
||||
- Do not rely on crew action-level `inputs` alone to ground agent behavior. Inputs that do not match placeholders are effectively unused by the prompt.
|
||||
- Do not ask agents to infer missing facts when accuracy matters. Tell them to mark missing dates, amounts, offers, logs, or constraints as unknown.
|
||||
|
||||
@@ -44,10 +44,19 @@ __all__ = [
|
||||
"render_template",
|
||||
"tree_copy",
|
||||
"tree_find_and_replace",
|
||||
"warn_deprecated_command",
|
||||
"write_env_file",
|
||||
]
|
||||
|
||||
|
||||
def warn_deprecated_command(*, old: str, new: str) -> None:
|
||||
"""Print a yellow deprecation warning for a legacy CLI command path."""
|
||||
click.secho(
|
||||
f"Warning: The command '{old}' is deprecated. Use '{new}' instead.",
|
||||
fg="yellow",
|
||||
)
|
||||
|
||||
|
||||
console = Console()
|
||||
_TEMPLATE_TOKEN_RE = re.compile(r"{{([a-zA-Z_][a-zA-Z0-9_]*)}}")
|
||||
|
||||
|
||||
@@ -228,6 +228,7 @@ def test_create_requires_type_in_dmn_mode(runner):
|
||||
|
||||
assert result.exit_code == 2
|
||||
assert "TYPE is required when CREWAI_DMN is set" in result.output
|
||||
assert "crew, flow, tool, skill, template" in result.output
|
||||
|
||||
|
||||
def test_create_requires_name_in_dmn_mode(runner):
|
||||
|
||||
230
lib/cli/tests/test_create_unified.py
Normal file
230
lib/cli/tests/test_create_unified.py
Normal file
@@ -0,0 +1,230 @@
|
||||
"""Tests for unified `crewai create <resource>` scaffolding commands."""
|
||||
|
||||
from unittest import mock
|
||||
|
||||
import pytest
|
||||
from click.testing import CliRunner
|
||||
|
||||
from crewai_cli.cli import create, crewai
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def runner():
|
||||
return CliRunner()
|
||||
|
||||
|
||||
@mock.patch("crewai_cli.tools.main.ToolCommand")
|
||||
def test_create_tool_invokes_tool_command(mock_tool_command_cls, runner):
|
||||
result = runner.invoke(create, ["tool", "my_tool"])
|
||||
|
||||
assert result.exit_code == 0, result.output
|
||||
mock_tool_command_cls.return_value.create.assert_called_once_with("my_tool")
|
||||
assert "deprecated" not in result.output.lower()
|
||||
|
||||
|
||||
@mock.patch("crewai_cli.tools.main.ToolCommand")
|
||||
def test_tool_create_is_deprecated_and_still_works(mock_tool_command_cls, runner):
|
||||
result = runner.invoke(crewai, ["tool", "create", "my_tool"])
|
||||
|
||||
assert result.exit_code == 0, result.output
|
||||
mock_tool_command_cls.return_value.create.assert_called_once_with("my_tool")
|
||||
assert (
|
||||
"Warning: The command 'crewai tool create' is deprecated. "
|
||||
"Use 'crewai create tool' instead."
|
||||
in result.output
|
||||
)
|
||||
|
||||
|
||||
@mock.patch("crewai_cli.tools.main.ToolCommand")
|
||||
@pytest.mark.parametrize("extra_args", [["--classic"], ["--declarative"], ["--provider", "openai"], ["--skip_provider"]])
|
||||
def test_create_tool_rejects_crew_and_flow_flags(mock_tool_command_cls, runner, extra_args):
|
||||
result = runner.invoke(create, ["tool", "my_tool", *extra_args])
|
||||
|
||||
assert result.exit_code == 2, result.output
|
||||
assert "Crew and flow options cannot be used with tool projects." in result.output
|
||||
mock_tool_command_cls.return_value.create.assert_not_called()
|
||||
|
||||
|
||||
@mock.patch("crewai_cli.skills.main.SkillCommand")
|
||||
def test_create_skill_invokes_skill_command(mock_skill_command_cls, runner):
|
||||
result = runner.invoke(create, ["skill", "my-skill"])
|
||||
|
||||
assert result.exit_code == 0, result.output
|
||||
mock_skill_command_cls.return_value.create.assert_called_once_with(
|
||||
"my-skill", in_project=True
|
||||
)
|
||||
assert "deprecated" not in result.output.lower()
|
||||
|
||||
|
||||
@mock.patch("crewai_cli.skills.main.SkillCommand")
|
||||
def test_create_skill_no_project_flag(mock_skill_command_cls, runner):
|
||||
result = runner.invoke(create, ["skill", "my-skill", "--no-project"])
|
||||
|
||||
assert result.exit_code == 0, result.output
|
||||
mock_skill_command_cls.return_value.create.assert_called_once_with(
|
||||
"my-skill", in_project=False
|
||||
)
|
||||
|
||||
|
||||
@mock.patch("crewai_cli.skills.main.SkillCommand")
|
||||
def test_skill_create_is_deprecated_and_still_works(mock_skill_command_cls, runner):
|
||||
result = runner.invoke(crewai, ["skill", "create", "my-skill"])
|
||||
|
||||
assert result.exit_code == 0, result.output
|
||||
mock_skill_command_cls.return_value.create.assert_called_once_with(
|
||||
"my-skill", in_project=True
|
||||
)
|
||||
assert (
|
||||
"Warning: The command 'crewai skill create' is deprecated. "
|
||||
"Use 'crewai create skill' instead."
|
||||
in result.output
|
||||
)
|
||||
|
||||
|
||||
@mock.patch("crewai_cli.skills.main.SkillCommand")
|
||||
@pytest.mark.parametrize(
|
||||
"extra_args",
|
||||
[["--classic"], ["--declarative"], ["--provider", "openai"], ["--skip_provider"]],
|
||||
)
|
||||
def test_create_skill_rejects_crew_and_flow_flags(
|
||||
mock_skill_command_cls, runner, extra_args
|
||||
):
|
||||
result = runner.invoke(create, ["skill", "my-skill", *extra_args])
|
||||
|
||||
assert result.exit_code == 2, result.output
|
||||
assert "Crew and flow options cannot be used with skill projects." in result.output
|
||||
mock_skill_command_cls.return_value.create.assert_not_called()
|
||||
|
||||
|
||||
@mock.patch("crewai_cli.skills.main.SkillCommand")
|
||||
def test_create_crew_rejects_no_project_flag(mock_skill_command_cls, runner):
|
||||
result = runner.invoke(create, ["crew", "my-crew", "--no-project"])
|
||||
|
||||
assert result.exit_code == 2, result.output
|
||||
assert "--no-project can only be used with skill projects." in result.output
|
||||
mock_skill_command_cls.return_value.create.assert_not_called()
|
||||
|
||||
|
||||
@mock.patch("crewai_cli.remote_template.main.TemplateCommand")
|
||||
def test_create_template_invokes_template_command(mock_template_command_cls, runner):
|
||||
result = runner.invoke(create, ["template", "my-template"])
|
||||
|
||||
assert result.exit_code == 0, result.output
|
||||
mock_template_command_cls.return_value.add_template.assert_called_once_with(
|
||||
"my-template", None
|
||||
)
|
||||
assert "deprecated" not in result.output.lower()
|
||||
|
||||
|
||||
@mock.patch("crewai_cli.remote_template.main.TemplateCommand")
|
||||
def test_create_template_output_dir_flag(mock_template_command_cls, runner):
|
||||
result = runner.invoke(
|
||||
create, ["template", "my-template", "--output-dir", "custom_dir"]
|
||||
)
|
||||
|
||||
assert result.exit_code == 0, result.output
|
||||
mock_template_command_cls.return_value.add_template.assert_called_once_with(
|
||||
"my-template", "custom_dir"
|
||||
)
|
||||
|
||||
|
||||
@mock.patch("crewai_cli.remote_template.main.TemplateCommand")
|
||||
def test_template_add_is_deprecated_and_still_works(mock_template_command_cls, runner):
|
||||
result = runner.invoke(
|
||||
crewai, ["template", "add", "my-template", "--output-dir", "custom_dir"]
|
||||
)
|
||||
|
||||
assert result.exit_code == 0, result.output
|
||||
mock_template_command_cls.return_value.add_template.assert_called_once_with(
|
||||
"my-template", "custom_dir"
|
||||
)
|
||||
assert (
|
||||
"Warning: The command 'crewai template add' is deprecated. "
|
||||
"Use 'crewai create template' instead."
|
||||
in result.output
|
||||
)
|
||||
|
||||
|
||||
@mock.patch("crewai_cli.remote_template.main.TemplateCommand")
|
||||
@pytest.mark.parametrize(
|
||||
"extra_args",
|
||||
[["--classic"], ["--declarative"], ["--provider", "openai"], ["--skip_provider"]],
|
||||
)
|
||||
def test_create_template_rejects_crew_and_flow_flags(
|
||||
mock_template_command_cls, runner, extra_args
|
||||
):
|
||||
result = runner.invoke(create, ["template", "my-template", *extra_args])
|
||||
|
||||
assert result.exit_code == 2, result.output
|
||||
assert (
|
||||
"Crew and flow options cannot be used with template projects."
|
||||
in result.output
|
||||
)
|
||||
mock_template_command_cls.return_value.add_template.assert_not_called()
|
||||
|
||||
|
||||
@mock.patch("crewai_cli.remote_template.main.TemplateCommand")
|
||||
def test_create_crew_rejects_output_dir_flag(mock_template_command_cls, runner):
|
||||
result = runner.invoke(create, ["crew", "my-crew", "--output-dir", "custom_dir"])
|
||||
|
||||
assert result.exit_code == 2, result.output
|
||||
assert "--output-dir can only be used with template projects." in result.output
|
||||
mock_template_command_cls.return_value.add_template.assert_not_called()
|
||||
|
||||
|
||||
@mock.patch("crewai_cli.cli.enable_prompt_line_editing")
|
||||
@mock.patch("crewai_cli.cli.click.prompt", return_value="picked-tool")
|
||||
@mock.patch("crewai_cli.tui_picker.pick", return_value="tool")
|
||||
@mock.patch("crewai_cli.tools.main.ToolCommand")
|
||||
def test_create_picker_supports_tool_skill_and_template(
|
||||
mock_tool_command_cls,
|
||||
mock_pick,
|
||||
mock_prompt,
|
||||
mock_enable_prompt,
|
||||
runner,
|
||||
):
|
||||
result = runner.invoke(create, [])
|
||||
|
||||
assert result.exit_code == 0, result.output
|
||||
mock_pick.assert_called_once()
|
||||
picker_options = mock_pick.call_args[0][1]
|
||||
assert {option[0] for option in picker_options} == {
|
||||
"crew",
|
||||
"flow",
|
||||
"tool",
|
||||
"skill",
|
||||
"template",
|
||||
}
|
||||
mock_prompt.assert_called_once()
|
||||
mock_tool_command_cls.return_value.create.assert_called_once_with("picked-tool")
|
||||
|
||||
|
||||
_DMN_ENV = {"CREWAI_DMN": "True"}
|
||||
|
||||
|
||||
@mock.patch("crewai_cli.tools.main.ToolCommand")
|
||||
def test_create_tool_works_in_dmn_mode(mock_tool_command_cls, runner):
|
||||
result = runner.invoke(create, ["tool", "my_tool"], env=_DMN_ENV)
|
||||
|
||||
assert result.exit_code == 0, result.output
|
||||
mock_tool_command_cls.return_value.create.assert_called_once_with("my_tool")
|
||||
|
||||
|
||||
@mock.patch("crewai_cli.skills.main.SkillCommand")
|
||||
def test_create_skill_works_in_dmn_mode(mock_skill_command_cls, runner):
|
||||
result = runner.invoke(create, ["skill", "my-skill"], env=_DMN_ENV)
|
||||
|
||||
assert result.exit_code == 0, result.output
|
||||
mock_skill_command_cls.return_value.create.assert_called_once_with(
|
||||
"my-skill", in_project=True
|
||||
)
|
||||
|
||||
|
||||
@mock.patch("crewai_cli.cli.TemplateCommand")
|
||||
def test_create_template_works_in_dmn_mode(mock_template_command_cls, runner):
|
||||
result = runner.invoke(create, ["template", "my-template"], env=_DMN_ENV)
|
||||
|
||||
assert result.exit_code == 0, result.output
|
||||
mock_template_command_cls.return_value.add_template.assert_called_once_with(
|
||||
"my-template", None
|
||||
)
|
||||
@@ -1 +1 @@
|
||||
__version__ = "1.15.11"
|
||||
__version__ = "1.15.12"
|
||||
|
||||
@@ -152,4 +152,4 @@ __all__ = [
|
||||
"wrap_file_source",
|
||||
]
|
||||
|
||||
__version__ = "1.15.11"
|
||||
__version__ = "1.15.12"
|
||||
|
||||
@@ -10,7 +10,7 @@ requires-python = ">=3.10, <3.14"
|
||||
dependencies = [
|
||||
"pytube~=15.0.0",
|
||||
"requests>=2.33.0,<3",
|
||||
"crewai==1.15.11",
|
||||
"crewai==1.15.12",
|
||||
"tiktoken>=0.8.0,<0.13",
|
||||
"beautifulsoup4~=4.13.4",
|
||||
"python-docx~=1.2.0",
|
||||
|
||||
@@ -209,6 +209,7 @@ from crewai_tools.tools.tavily_research_tool.tavily_research_tool import (
|
||||
)
|
||||
from crewai_tools.tools.tavily_search_tool.tavily_search_tool import TavilySearchTool
|
||||
from crewai_tools.tools.txt_search_tool.txt_search_tool import TXTSearchTool
|
||||
from crewai_tools.tools.url_read_tool.url_read_tool import URLReadTool
|
||||
from crewai_tools.tools.vision_tool.vision_tool import VisionTool
|
||||
from crewai_tools.tools.wait_tool.wait_tool import WaitTool
|
||||
from crewai_tools.tools.weaviate_tool.vector_search import WeaviateVectorSearchTool
|
||||
@@ -327,6 +328,7 @@ __all__ = [
|
||||
"TavilyGetResearchTool",
|
||||
"TavilyResearchTool",
|
||||
"TavilySearchTool",
|
||||
"URLReadTool",
|
||||
"VisionTool",
|
||||
"WaitTool",
|
||||
"WeaviateVectorSearchTool",
|
||||
@@ -338,4 +340,4 @@ __all__ = [
|
||||
"ZapierActionTools",
|
||||
]
|
||||
|
||||
__version__ = "1.15.11"
|
||||
__version__ = "1.15.12"
|
||||
|
||||
@@ -2,13 +2,17 @@
|
||||
|
||||
import os
|
||||
from pathlib import Path
|
||||
import tempfile
|
||||
from typing import Any
|
||||
from urllib.parse import urlparse
|
||||
|
||||
from crewai_tools.rag.base_loader import BaseLoader, LoaderResult
|
||||
from crewai_tools.rag.source_content import SourceContent
|
||||
from crewai_tools.security.safe_requests import safe_get
|
||||
from crewai_tools.security.safe_requests import safe_get_bounded
|
||||
|
||||
|
||||
# Remote PDFs are held in memory for the whole extraction, so the download needs
|
||||
# a ceiling. Override per call with the ``max_bytes`` kwarg to ``load``.
|
||||
DEFAULT_MAX_PDF_BYTES = 50 * 1024 * 1024
|
||||
|
||||
|
||||
class PDFLoader(BaseLoader):
|
||||
@@ -24,18 +28,24 @@ class PDFLoader(BaseLoader):
|
||||
return False
|
||||
|
||||
@staticmethod
|
||||
def _download_from_url(url: str, kwargs: dict[str, Any]) -> str:
|
||||
"""Download PDF from a URL to a temporary file and return its path.
|
||||
def _fetch_from_url(url: str, kwargs: dict[str, Any]) -> bytes:
|
||||
"""Download a PDF from a URL and return its bytes.
|
||||
|
||||
The content stays in memory rather than going to a temporary file: the
|
||||
whole body has to be buffered either way, and a temp file would need
|
||||
unlinking on every error path to avoid leaving files behind. Because it
|
||||
is held in memory, the download is capped.
|
||||
|
||||
Args:
|
||||
url: The URL to download from.
|
||||
kwargs: Optional dict that may contain custom headers.
|
||||
kwargs: Optional dict that may contain custom ``headers`` and a
|
||||
``max_bytes`` ceiling for the download.
|
||||
|
||||
Returns:
|
||||
Path to the temporary file containing the PDF.
|
||||
The raw PDF content.
|
||||
|
||||
Raises:
|
||||
ValueError: If the download fails.
|
||||
ValueError: If the download fails or exceeds the size ceiling.
|
||||
"""
|
||||
headers = kwargs.get(
|
||||
"headers",
|
||||
@@ -46,12 +56,13 @@ class PDFLoader(BaseLoader):
|
||||
)
|
||||
|
||||
try:
|
||||
response = safe_get(url, headers=headers, timeout=30)
|
||||
response.raise_for_status()
|
||||
|
||||
with tempfile.NamedTemporaryFile(suffix=".pdf", delete=False) as temp_file:
|
||||
temp_file.write(response.content)
|
||||
return temp_file.name
|
||||
body, _content_type, _final_url = safe_get_bounded(
|
||||
url,
|
||||
max_bytes=kwargs.get("max_bytes", DEFAULT_MAX_PDF_BYTES),
|
||||
headers=headers,
|
||||
timeout=30,
|
||||
)
|
||||
return body
|
||||
except Exception as e:
|
||||
raise ValueError(f"Failed to download PDF from {url}: {e!s}") from e
|
||||
|
||||
@@ -93,21 +104,25 @@ class PDFLoader(BaseLoader):
|
||||
|
||||
try:
|
||||
if is_url:
|
||||
local_path = self._download_from_url(file_path, kwargs)
|
||||
doc = pymupdf.open(local_path)
|
||||
doc = pymupdf.open(
|
||||
stream=self._fetch_from_url(file_path, kwargs), filetype="pdf"
|
||||
)
|
||||
else:
|
||||
if not os.path.isfile(file_path):
|
||||
raise FileNotFoundError(f"PDF file not found: {file_path}")
|
||||
doc = pymupdf.open(file_path)
|
||||
|
||||
metadata["num_pages"] = len(doc)
|
||||
# Closed in a finally so a failure mid-extraction still releases the
|
||||
# document handle.
|
||||
try:
|
||||
metadata["num_pages"] = len(doc)
|
||||
|
||||
for page_num, page in enumerate(doc, 1):
|
||||
page_text = page.get_text()
|
||||
if page_text.strip():
|
||||
text_content.append(f"Page {page_num}:\n{page_text}")
|
||||
|
||||
doc.close()
|
||||
for page_num, page in enumerate(doc, 1):
|
||||
page_text = page.get_text()
|
||||
if page_text.strip():
|
||||
text_content.append(f"Page {page_num}:\n{page_text}")
|
||||
finally:
|
||||
doc.close()
|
||||
except FileNotFoundError:
|
||||
raise
|
||||
except Exception as e:
|
||||
|
||||
@@ -11,6 +11,7 @@ from crewai_tools.security.safe_path import validate_url
|
||||
|
||||
|
||||
_REDIRECT_STATUS_CODES = {301, 302, 303, 307, 308}
|
||||
_STREAM_CHUNK_SIZE = 65536
|
||||
_SENSITIVE_HEADER_NAMES = {
|
||||
"authorization",
|
||||
"cookie",
|
||||
@@ -49,40 +50,124 @@ def _strip_cross_origin_credentials(request_kwargs: dict[str, Any]) -> dict[str,
|
||||
|
||||
|
||||
def safe_get(url: str, *, max_redirects: int = 10, **kwargs: Any) -> requests.Response:
|
||||
"""GET a URL while validating each redirect target before following it."""
|
||||
"""GET a URL while validating each redirect target before following it.
|
||||
|
||||
On success the hops are attached to the returned response's ``history`` and
|
||||
are the caller's to close. On failure they are closed here: a caller given
|
||||
an exception has no handle on them, and a streamed hop holds its connection
|
||||
until its body is read or closed.
|
||||
"""
|
||||
current_url = validate_url(url)
|
||||
request_kwargs = {**kwargs, "allow_redirects": False}
|
||||
timeout = request_kwargs.pop("timeout", 30)
|
||||
history: list[requests.Response] = []
|
||||
redirects_followed = 0
|
||||
|
||||
while True:
|
||||
response = requests.get(current_url, timeout=timeout, **request_kwargs)
|
||||
if (
|
||||
response.status_code not in _REDIRECT_STATUS_CODES
|
||||
or "Location" not in response.headers
|
||||
):
|
||||
response.history = history
|
||||
return response
|
||||
try:
|
||||
while True:
|
||||
response = requests.get(current_url, timeout=timeout, **request_kwargs)
|
||||
if (
|
||||
response.status_code not in _REDIRECT_STATUS_CODES
|
||||
or "Location" not in response.headers
|
||||
):
|
||||
response.history = history
|
||||
return response
|
||||
|
||||
if redirects_followed >= max_redirects:
|
||||
response.close()
|
||||
raise ValueError(f"Too many redirects while fetching URL: {url}")
|
||||
if redirects_followed >= max_redirects:
|
||||
response.close()
|
||||
raise ValueError(f"Too many redirects while fetching URL: {url}")
|
||||
|
||||
location = response.headers.get("Location")
|
||||
if not location:
|
||||
response.history = history
|
||||
return response
|
||||
location = response.headers.get("Location")
|
||||
if not location:
|
||||
response.history = history
|
||||
return response
|
||||
|
||||
try:
|
||||
redirect_url = validate_url(urljoin(response.url, location))
|
||||
except ValueError:
|
||||
response.close()
|
||||
raise
|
||||
try:
|
||||
redirect_url = validate_url(urljoin(response.url, location))
|
||||
except ValueError:
|
||||
response.close()
|
||||
raise
|
||||
|
||||
if not _same_origin(current_url, redirect_url):
|
||||
request_kwargs = _strip_cross_origin_credentials(request_kwargs)
|
||||
if not _same_origin(current_url, redirect_url):
|
||||
request_kwargs = _strip_cross_origin_credentials(request_kwargs)
|
||||
|
||||
history.append(response)
|
||||
current_url = redirect_url
|
||||
redirects_followed += 1
|
||||
history.append(response)
|
||||
current_url = redirect_url
|
||||
redirects_followed += 1
|
||||
except BaseException:
|
||||
for hop in history:
|
||||
hop.close()
|
||||
raise
|
||||
|
||||
|
||||
def safe_get_bounded(
|
||||
url: str,
|
||||
*,
|
||||
max_bytes: int,
|
||||
timeout: float | tuple[float, float] = 30,
|
||||
headers: dict[str, str] | None = None,
|
||||
max_redirects: int = 10,
|
||||
) -> tuple[bytes, str, str]:
|
||||
"""GET a URL through :func:`safe_get`, refusing bodies over *max_bytes*.
|
||||
|
||||
The body is streamed and abandoned as soon as it crosses the limit, so an
|
||||
oversized response costs one chunk of memory instead of all of it. The cap
|
||||
counts decoded bytes, which is what a compressed response expands into --
|
||||
``Content-Length`` describes the wire size and cannot bound that.
|
||||
|
||||
Args:
|
||||
url: The URL to fetch.
|
||||
max_bytes: Largest body to accept, in decoded bytes.
|
||||
timeout: Request timeout, passed through to requests.
|
||||
headers: Request headers.
|
||||
max_redirects: Hops to follow before giving up.
|
||||
|
||||
Returns:
|
||||
A ``(body, content_type, final_url)`` tuple, where *final_url* is the
|
||||
last validated URL in the redirect chain.
|
||||
|
||||
Raises:
|
||||
ValueError: If *max_bytes* is not positive, URL validation fails, the
|
||||
redirect chain is too long, or the body exceeds *max_bytes*.
|
||||
requests.RequestException: If the request fails or returns an error
|
||||
status.
|
||||
"""
|
||||
if max_bytes <= 0:
|
||||
raise ValueError(f"max_bytes must be positive, got {max_bytes}.")
|
||||
|
||||
response = safe_get(
|
||||
url,
|
||||
max_redirects=max_redirects,
|
||||
headers=headers,
|
||||
timeout=timeout,
|
||||
stream=True,
|
||||
)
|
||||
try:
|
||||
response.raise_for_status()
|
||||
|
||||
chunks: list[bytes] = []
|
||||
total = 0
|
||||
for chunk in response.iter_content(chunk_size=_STREAM_CHUNK_SIZE):
|
||||
if not chunk:
|
||||
continue
|
||||
total += len(chunk)
|
||||
if total > max_bytes:
|
||||
# Names the URL that served the body, which after a redirect is
|
||||
# not the one that was requested.
|
||||
raise ValueError(
|
||||
f"Response body from '{response.url}' exceeds the "
|
||||
f"{max_bytes} byte limit."
|
||||
)
|
||||
chunks.append(chunk)
|
||||
|
||||
return (
|
||||
b"".join(chunks),
|
||||
response.headers.get("Content-Type", ""),
|
||||
response.url,
|
||||
)
|
||||
finally:
|
||||
# Under stream=True each hop holds its connection until the body is read,
|
||||
# so the redirects need closing too, not just the response we return.
|
||||
for hop in response.history:
|
||||
hop.close()
|
||||
response.close()
|
||||
|
||||
@@ -196,6 +196,7 @@ from crewai_tools.tools.tavily_research_tool.tavily_research_tool import (
|
||||
)
|
||||
from crewai_tools.tools.tavily_search_tool.tavily_search_tool import TavilySearchTool
|
||||
from crewai_tools.tools.txt_search_tool.txt_search_tool import TXTSearchTool
|
||||
from crewai_tools.tools.url_read_tool.url_read_tool import URLReadTool
|
||||
from crewai_tools.tools.vision_tool.vision_tool import VisionTool
|
||||
from crewai_tools.tools.wait_tool.wait_tool import WaitTool
|
||||
from crewai_tools.tools.weaviate_tool.vector_search import WeaviateVectorSearchTool
|
||||
@@ -310,6 +311,7 @@ __all__ = [
|
||||
"TavilyGetResearchTool",
|
||||
"TavilyResearchTool",
|
||||
"TavilySearchTool",
|
||||
"URLReadTool",
|
||||
"VisionTool",
|
||||
"WaitTool",
|
||||
"WeaviateVectorSearchTool",
|
||||
|
||||
@@ -17,6 +17,7 @@ from crewai_tools.tools.crewai_platform_tools.misc import (
|
||||
|
||||
|
||||
class CrewAIPlatformActionTool(BaseTool):
|
||||
app: str = Field(description="The integration slug for this action")
|
||||
action_name: str = Field(default="", description="The name of the action")
|
||||
action_schema: dict[str, Any] = Field(
|
||||
default_factory=dict, description="The schema of the action"
|
||||
@@ -25,6 +26,7 @@ class CrewAIPlatformActionTool(BaseTool):
|
||||
def __init__(
|
||||
self,
|
||||
description: str,
|
||||
app: str,
|
||||
action_name: str,
|
||||
action_schema: dict[str, Any],
|
||||
):
|
||||
@@ -46,6 +48,7 @@ class CrewAIPlatformActionTool(BaseTool):
|
||||
name=action_name.lower().replace(" ", "_"),
|
||||
description=description,
|
||||
args_schema=args_schema,
|
||||
app=app,
|
||||
)
|
||||
self.action_name = action_name
|
||||
self.action_schema = action_schema
|
||||
|
||||
@@ -89,6 +89,7 @@ class CrewaiPlatformToolBuilder:
|
||||
|
||||
tool = CrewAIPlatformActionTool(
|
||||
description=description,
|
||||
app=function_details["app"],
|
||||
action_name=action_name,
|
||||
action_schema=action_schema,
|
||||
)
|
||||
|
||||
@@ -0,0 +1,4 @@
|
||||
from crewai_tools.tools.url_read_tool.url_read_tool import URLReadTool
|
||||
|
||||
|
||||
__all__ = ["URLReadTool"]
|
||||
@@ -0,0 +1,379 @@
|
||||
"""Tool for reading the content at an arbitrary URL as text."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from io import BytesIO
|
||||
from itertools import islice
|
||||
import re
|
||||
from typing import Any, Final
|
||||
from urllib.parse import urlparse
|
||||
|
||||
from crewai.tools import BaseTool
|
||||
from pydantic import BaseModel, Field
|
||||
import requests
|
||||
|
||||
from crewai_tools.security.safe_path import format_error_for_display
|
||||
from crewai_tools.security.safe_requests import safe_get_bounded
|
||||
|
||||
|
||||
_DEFAULT_MAX_BYTES: Final[int] = 5 * 1024 * 1024
|
||||
_DEFAULT_TIMEOUT: Final[int] = 30
|
||||
|
||||
_PDF_TYPE: Final[str] = "application/pdf"
|
||||
_DOCX_TYPE: Final[str] = (
|
||||
"application/vnd.openxmlformats-officedocument.wordprocessingml.document"
|
||||
)
|
||||
_HTML_TYPES: Final[frozenset[str]] = frozenset({"text/html", "application/xhtml+xml"})
|
||||
_TEXT_TYPES: Final[frozenset[str]] = frozenset(
|
||||
{
|
||||
"application/csv",
|
||||
"application/javascript",
|
||||
"application/json",
|
||||
"application/sql",
|
||||
"application/x-ndjson",
|
||||
"application/x-yaml",
|
||||
"application/xml",
|
||||
"application/yaml",
|
||||
}
|
||||
)
|
||||
_TEXT_TYPE_SUFFIXES: Final[tuple[str, ...]] = ("+json", "+xml", "+yaml")
|
||||
|
||||
# Servers commonly serve static files as octet-stream, or send no type at all,
|
||||
# so the extension is consulted when the header carries no usable answer.
|
||||
_UNINFORMATIVE_TYPES: Final[frozenset[str]] = frozenset(
|
||||
{"", "application/octet-stream", "binary/octet-stream"}
|
||||
)
|
||||
_EXTENSION_TYPES: Final[dict[str, str]] = {
|
||||
".csv": "text/csv",
|
||||
".docx": _DOCX_TYPE,
|
||||
".htm": "text/html",
|
||||
".html": "text/html",
|
||||
".json": "application/json",
|
||||
".md": "text/markdown",
|
||||
".pdf": _PDF_TYPE,
|
||||
".txt": "text/plain",
|
||||
".xml": "application/xml",
|
||||
".yaml": "application/yaml",
|
||||
".yml": "application/yaml",
|
||||
}
|
||||
|
||||
_SPACES_PATTERN: Final[re.Pattern[str]] = re.compile(r"[ \t]+")
|
||||
_NEWLINE_PATTERN: Final[re.Pattern[str]] = re.compile(r"\s+\n\s+")
|
||||
|
||||
|
||||
def _charset_from_content_type(content_type: str) -> str | None:
|
||||
"""Return the charset parameter of a Content-Type header, if it has one."""
|
||||
for parameter in content_type.split(";")[1:]:
|
||||
name, _, value = parameter.partition("=")
|
||||
if name.strip().lower() == "charset":
|
||||
return value.strip().strip('"') or None
|
||||
return None
|
||||
|
||||
|
||||
class URLReadToolSchema(BaseModel):
|
||||
"""Input for URLReadTool."""
|
||||
|
||||
url: str = Field(
|
||||
...,
|
||||
description=(
|
||||
"The http:// or https:// URL to read. Addresses that resolve to "
|
||||
"private or internal networks are refused."
|
||||
),
|
||||
)
|
||||
start_line: int | None = Field(
|
||||
1, ge=1, description="Line number to start reading from (1-indexed)"
|
||||
)
|
||||
line_count: int | None = Field(
|
||||
None,
|
||||
ge=1,
|
||||
description="Number of lines to read. If None, reads the entire content",
|
||||
)
|
||||
|
||||
|
||||
class URLReadTool(BaseTool):
|
||||
"""Read the content at an arbitrary URL and return it as text.
|
||||
|
||||
Unlike :class:`~crewai_tools.tools.file_read_tool.file_read_tool.FileReadTool`,
|
||||
which is confined to the local filesystem, this tool performs network
|
||||
requests to addresses the caller -- often an LLM -- chooses at runtime. It
|
||||
is a separate tool for exactly that reason: granting it is granting network
|
||||
egress, and that should be a deliberate choice rather than a flag on a
|
||||
filesystem tool.
|
||||
|
||||
Responses are decoded to text according to their content type. PDF and DOCX
|
||||
bodies have their text extracted, HTML is stripped to visible text, and
|
||||
text-shaped types (plain text, Markdown, JSON, XML, YAML, CSV) are decoded
|
||||
as-is. Any other type is refused rather than returned as base64, keeping
|
||||
this tool's output text-only.
|
||||
|
||||
Security:
|
||||
Requests go through :func:`~crewai_tools.security.safe_requests.safe_get_bounded`,
|
||||
which resolves each hostname and rejects it when any resolved address is
|
||||
private, loopback, link-local, or otherwise reserved -- covering cloud
|
||||
metadata endpoints and internal services. Redirects are never followed
|
||||
automatically: every hop is revalidated, and credentials are dropped on
|
||||
cross-origin hops. Bodies over ``max_bytes`` are abandoned mid-stream.
|
||||
|
||||
Two risks are not closed here. Validation resolves the hostname and
|
||||
requests resolves it again when connecting, so a DNS entry that changes
|
||||
between those lookups can still redirect the connection (DNS
|
||||
rebinding); closing that requires pinning the connection to the
|
||||
validated address. And the returned text is untrusted remote content
|
||||
flowing into an agent's context -- a fetched page can attempt to
|
||||
instruct the agent. Neither is addressable by input validation alone;
|
||||
network egress policy and prompt-level handling cover them.
|
||||
|
||||
Args:
|
||||
max_bytes (int): Largest response body to accept, in decoded bytes.
|
||||
Defaults to 5 MiB.
|
||||
timeout (float): Per-request timeout in seconds. Defaults to 30.
|
||||
headers (Optional[dict[str, str]]): Extra request headers. Developer
|
||||
supplied, not chosen by the model.
|
||||
encoding (Optional[str]): Force a text encoding instead of honoring the
|
||||
charset the server declares.
|
||||
**kwargs: Additional keyword arguments passed to BaseTool.
|
||||
|
||||
Example:
|
||||
>>> tool = URLReadTool()
|
||||
>>> content = tool.run(url="https://example.com/report.pdf")
|
||||
>>> head = tool.run(url="https://example.com/data.csv", line_count=20)
|
||||
"""
|
||||
|
||||
name: str = "Read content from a URL"
|
||||
description: str = (
|
||||
"A tool that reads the content at a URL and returns it as text. To use "
|
||||
"this tool, provide a 'url' parameter with an http:// or https:// "
|
||||
"address. PDF, DOCX, HTML, JSON, XML, CSV and plain-text responses are "
|
||||
"converted to text; other binary types are rejected. URLs that resolve "
|
||||
"to private or internal network addresses are refused, as are responses "
|
||||
"over the tool's size limit. Optionally provide 'start_line' and "
|
||||
"'line_count' to read only part of the content."
|
||||
)
|
||||
args_schema: type[BaseModel] = URLReadToolSchema
|
||||
max_bytes: int = _DEFAULT_MAX_BYTES
|
||||
timeout: float = _DEFAULT_TIMEOUT
|
||||
headers: dict[str, str] | None = None
|
||||
encoding: str | None = None
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
max_bytes: int = _DEFAULT_MAX_BYTES,
|
||||
timeout: float = _DEFAULT_TIMEOUT,
|
||||
headers: dict[str, str] | None = None,
|
||||
encoding: str | None = None,
|
||||
**kwargs: Any,
|
||||
) -> None:
|
||||
"""Initialize the URLReadTool.
|
||||
|
||||
Args:
|
||||
max_bytes: Largest response body to accept, in decoded bytes.
|
||||
timeout: Per-request timeout in seconds.
|
||||
headers: Extra request headers.
|
||||
encoding: Force a text encoding instead of the server's charset.
|
||||
**kwargs: Additional keyword arguments passed to BaseTool.
|
||||
"""
|
||||
super().__init__(**kwargs)
|
||||
self.max_bytes = max_bytes
|
||||
self.timeout = timeout
|
||||
self.headers = headers
|
||||
self.encoding = encoding
|
||||
|
||||
def _request_headers(self) -> dict[str, str]:
|
||||
"""Return the headers to send, with caller headers taking precedence."""
|
||||
return {
|
||||
"Accept": "*/*",
|
||||
"User-Agent": "Mozilla/5.0 (compatible; crewai-tools URLReadTool)",
|
||||
**(self.headers or {}),
|
||||
}
|
||||
|
||||
@staticmethod
|
||||
def _classify(media_type: str) -> str | None:
|
||||
"""Map a media type onto the extractor that handles it."""
|
||||
if media_type == _PDF_TYPE:
|
||||
return "pdf"
|
||||
if media_type == _DOCX_TYPE:
|
||||
return "docx"
|
||||
if media_type in _HTML_TYPES:
|
||||
return "html"
|
||||
if (
|
||||
media_type.startswith("text/")
|
||||
or media_type in _TEXT_TYPES
|
||||
or media_type.endswith(_TEXT_TYPE_SUFFIXES)
|
||||
):
|
||||
return "text"
|
||||
return None
|
||||
|
||||
def _resolve_kind(self, content_type: str, *urls: str) -> str | None:
|
||||
"""Decide how to extract text, by content type then by URL extension.
|
||||
|
||||
Args:
|
||||
content_type: The raw Content-Type header value.
|
||||
*urls: URLs to consult for an extension, most authoritative first.
|
||||
A ``.pdf`` link that redirects to an extensionless CDN or
|
||||
presigned path only carries its type on the requested URL, so
|
||||
both ends of the chain are worth checking.
|
||||
|
||||
Returns:
|
||||
The extractor name, or None when the content type is unsupported.
|
||||
"""
|
||||
declared = content_type.split(";", 1)[0].strip().lower()
|
||||
if declared not in _UNINFORMATIVE_TYPES:
|
||||
return self._classify(declared)
|
||||
|
||||
for url in urls:
|
||||
path = urlparse(url).path.lower()
|
||||
for extension, media_type in _EXTENSION_TYPES.items():
|
||||
if path.endswith(extension):
|
||||
return self._classify(media_type)
|
||||
return None
|
||||
|
||||
def _decode(self, body: bytes, content_type: str) -> str:
|
||||
"""Decode *body* using the configured, declared, or default encoding.
|
||||
|
||||
Falls back to a replacing UTF-8 decode rather than failing: partially
|
||||
readable text is more useful to an agent than an error.
|
||||
"""
|
||||
encoding = self.encoding or _charset_from_content_type(content_type) or "utf-8"
|
||||
try:
|
||||
return body.decode(encoding)
|
||||
except (LookupError, UnicodeDecodeError):
|
||||
return body.decode("utf-8", errors="replace")
|
||||
|
||||
@staticmethod
|
||||
def _extract_pdf(body: bytes) -> str:
|
||||
"""Extract text from PDF bytes, page by page."""
|
||||
try:
|
||||
import pymupdf # type: ignore[import-untyped]
|
||||
except ImportError as e:
|
||||
raise ImportError(
|
||||
"Reading PDF URLs requires pymupdf. Install with: uv add pymupdf"
|
||||
) from e
|
||||
|
||||
# Opened from memory: the bytes are already in hand, and a temp file
|
||||
# would need cleaning up on every error path.
|
||||
document = pymupdf.open(stream=body, filetype="pdf")
|
||||
try:
|
||||
pages = [
|
||||
f"Page {number}:\n{text}"
|
||||
for number, page in enumerate(document, 1)
|
||||
if (text := page.get_text().strip())
|
||||
]
|
||||
finally:
|
||||
document.close()
|
||||
|
||||
if not pages:
|
||||
return "[PDF with no extractable text]"
|
||||
return "\n\n".join(pages)
|
||||
|
||||
@staticmethod
|
||||
def _extract_docx(body: bytes) -> str:
|
||||
"""Extract paragraph text from DOCX bytes."""
|
||||
try:
|
||||
from docx import Document
|
||||
except ImportError as e:
|
||||
raise ImportError(
|
||||
"Reading DOCX URLs requires python-docx. Install with: "
|
||||
"uv add python-docx"
|
||||
) from e
|
||||
|
||||
document = Document(BytesIO(body))
|
||||
return "\n".join(
|
||||
paragraph.text
|
||||
for paragraph in document.paragraphs
|
||||
if paragraph.text.strip()
|
||||
)
|
||||
|
||||
def _extract_html(self, body: bytes, content_type: str) -> str:
|
||||
"""Strip HTML bytes down to visible text."""
|
||||
try:
|
||||
from bs4 import BeautifulSoup
|
||||
except ImportError as e:
|
||||
raise ImportError(
|
||||
"Reading HTML URLs requires beautifulsoup4. Install with: "
|
||||
"uv add beautifulsoup4"
|
||||
) from e
|
||||
|
||||
soup = BeautifulSoup(self._decode(body, content_type), "html.parser")
|
||||
for element in soup(["script", "style"]):
|
||||
element.decompose()
|
||||
|
||||
text = _SPACES_PATTERN.sub(" ", soup.get_text(" "))
|
||||
return _NEWLINE_PATTERN.sub("\n", text).strip()
|
||||
|
||||
def _extract(self, body: bytes, kind: str, content_type: str) -> str:
|
||||
"""Dispatch to the extractor named by *kind*."""
|
||||
if kind == "pdf":
|
||||
return self._extract_pdf(body)
|
||||
if kind == "docx":
|
||||
return self._extract_docx(body)
|
||||
if kind == "html":
|
||||
return self._extract_html(body, content_type)
|
||||
return self._decode(body, content_type)
|
||||
|
||||
@staticmethod
|
||||
def _window(text: str, start_line: int, line_count: int | None) -> str:
|
||||
"""Return the requested line window of *text*.
|
||||
|
||||
The whole body has already been fetched by this point, so unlike the
|
||||
filesystem equivalent this only trims output -- it saves no transfer.
|
||||
|
||||
The bounds are clamped rather than trusted: the args schema rejects
|
||||
anything below 1 before it gets here, but islice raises on a negative
|
||||
stop index, and this runs outside the caller's error handling.
|
||||
"""
|
||||
if start_line == 1 and line_count is None:
|
||||
return text
|
||||
|
||||
start_index = max(start_line - 1, 0)
|
||||
stop_index = None if line_count is None else start_index + max(line_count, 0)
|
||||
selected = list(islice(text.splitlines(keepends=True), start_index, stop_index))
|
||||
|
||||
if not selected and start_index > 0:
|
||||
return (
|
||||
f"Error: Start line {start_line} exceeds the number of lines in "
|
||||
f"the content."
|
||||
)
|
||||
return "".join(selected)
|
||||
|
||||
def _run(
|
||||
self,
|
||||
url: str,
|
||||
start_line: int | None = 1,
|
||||
line_count: int | None = None,
|
||||
) -> str:
|
||||
"""Fetch a URL and return its content, or a window of it, as text."""
|
||||
start_line = start_line or 1
|
||||
line_count = line_count or None
|
||||
|
||||
try:
|
||||
body, content_type, final_url = safe_get_bounded(
|
||||
url,
|
||||
max_bytes=self.max_bytes,
|
||||
timeout=self.timeout,
|
||||
headers=self._request_headers(),
|
||||
)
|
||||
except ValueError as e:
|
||||
return f"Error: {e}"
|
||||
except requests.RequestException as e:
|
||||
return f"Error: Failed to fetch '{url}'. {format_error_for_display(e)}"
|
||||
|
||||
kind = self._resolve_kind(content_type, final_url, url)
|
||||
if kind is None:
|
||||
return (
|
||||
f"Error: Unsupported content type "
|
||||
f"'{content_type.split(';', 1)[0].strip() or 'unknown'}' at "
|
||||
f"'{url}'. This tool reads text, HTML, JSON, XML, CSV, PDF and "
|
||||
f"DOCX responses."
|
||||
)
|
||||
|
||||
try:
|
||||
text = self._extract(body, kind, content_type)
|
||||
except ImportError as e:
|
||||
return f"Error: {e}"
|
||||
except Exception as e:
|
||||
return (
|
||||
f"Error: Failed to read {kind.upper()} content from '{url}'. "
|
||||
f"{format_error_for_display(e)}"
|
||||
)
|
||||
|
||||
return self._window(text, start_line, line_count)
|
||||
156
lib/crewai-tools/tests/rag/test_pdf_loader.py
Normal file
156
lib/crewai-tools/tests/rag/test_pdf_loader.py
Normal file
@@ -0,0 +1,156 @@
|
||||
import tempfile
|
||||
from unittest.mock import patch
|
||||
|
||||
from crewai_tools.rag.base_loader import LoaderResult
|
||||
from crewai_tools.rag.loaders.pdf_loader import PDFLoader
|
||||
from crewai_tools.rag.source_content import SourceContent
|
||||
import pytest
|
||||
|
||||
|
||||
pymupdf = pytest.importorskip("pymupdf")
|
||||
|
||||
# Patched at the loader's seam rather than at requests.get: safe_get_bounded
|
||||
# resolves the hostname before issuing a request, which would make these tests
|
||||
# depend on DNS for example.com.
|
||||
FETCH = "crewai_tools.rag.loaders.pdf_loader.safe_get_bounded"
|
||||
|
||||
|
||||
def build_pdf(text: str = "Quarterly revenue was 42") -> bytes:
|
||||
"""Return the bytes of a one-page PDF containing *text*."""
|
||||
document = pymupdf.open()
|
||||
document.new_page().insert_text((72, 72), text)
|
||||
try:
|
||||
return document.tobytes()
|
||||
finally:
|
||||
document.close()
|
||||
|
||||
|
||||
def fetch_result(body: bytes, url: str = "https://example.com/report.pdf"):
|
||||
"""Build the (body, content_type, final_url) tuple safe_get_bounded returns."""
|
||||
return body, "application/pdf", url
|
||||
|
||||
|
||||
class TestPDFLoader:
|
||||
def test_load_pdf_from_file(self):
|
||||
"""A PDF on disk has its text extracted with page markers."""
|
||||
with tempfile.NamedTemporaryFile(suffix=".pdf") as f:
|
||||
f.write(build_pdf())
|
||||
f.flush()
|
||||
|
||||
result = PDFLoader().load(SourceContent(f.name))
|
||||
|
||||
assert isinstance(result, LoaderResult)
|
||||
assert "Page 1:" in result.content
|
||||
assert "Quarterly revenue was 42" in result.content
|
||||
assert result.metadata["num_pages"] == 1
|
||||
assert result.metadata["file_type"] == "pdf"
|
||||
|
||||
def test_load_pdf_from_url(self):
|
||||
"""A PDF fetched from a URL is extracted and attributed to that URL."""
|
||||
with patch(FETCH) as fetch:
|
||||
fetch.return_value = fetch_result(build_pdf("Content from URL"))
|
||||
result = PDFLoader().load(SourceContent("https://example.com/report.pdf"))
|
||||
|
||||
assert "Content from URL" in result.content
|
||||
assert result.source == "https://example.com/report.pdf"
|
||||
assert result.metadata["file_name"] == "report.pdf"
|
||||
|
||||
headers = fetch.call_args.kwargs["headers"]
|
||||
assert headers["Accept"] == "application/pdf"
|
||||
assert "crewai-tools PDFLoader" in headers["User-Agent"]
|
||||
|
||||
def test_load_pdf_from_url_leaves_no_temp_file(self):
|
||||
"""The URL path must not write a temp file it never cleans up.
|
||||
|
||||
It previously used NamedTemporaryFile(delete=False) without unlinking,
|
||||
so every PDF ingested from a URL left a file behind.
|
||||
"""
|
||||
with (
|
||||
patch(FETCH) as fetch,
|
||||
patch("tempfile.NamedTemporaryFile") as mock_tempfile,
|
||||
):
|
||||
fetch.return_value = fetch_result(build_pdf())
|
||||
PDFLoader().load(SourceContent("https://example.com/report.pdf"))
|
||||
|
||||
mock_tempfile.assert_not_called()
|
||||
|
||||
def test_load_pdf_from_url_is_size_bounded(self):
|
||||
"""The download is capped, since the body is held in memory."""
|
||||
with patch(FETCH) as fetch:
|
||||
fetch.return_value = fetch_result(build_pdf())
|
||||
PDFLoader().load(SourceContent("https://example.com/report.pdf"))
|
||||
|
||||
assert fetch.call_args.kwargs["max_bytes"] == 50 * 1024 * 1024
|
||||
|
||||
def test_load_pdf_from_url_accepts_a_custom_size_limit(self):
|
||||
"""Callers can lower or raise the ceiling per load."""
|
||||
with patch(FETCH) as fetch:
|
||||
fetch.return_value = fetch_result(build_pdf())
|
||||
PDFLoader().load(
|
||||
SourceContent("https://example.com/report.pdf"), max_bytes=1024
|
||||
)
|
||||
|
||||
assert fetch.call_args.kwargs["max_bytes"] == 1024
|
||||
|
||||
def test_load_pdf_from_url_with_custom_headers(self):
|
||||
"""Caller-supplied headers replace the loader's defaults."""
|
||||
custom_headers = {"Authorization": "Bearer token"}
|
||||
|
||||
with patch(FETCH) as fetch:
|
||||
fetch.return_value = fetch_result(build_pdf())
|
||||
PDFLoader().load(
|
||||
SourceContent("https://example.com/report.pdf"), headers=custom_headers
|
||||
)
|
||||
|
||||
assert fetch.call_args.kwargs["headers"] == custom_headers
|
||||
|
||||
def test_load_pdf_url_download_error(self):
|
||||
"""A failed download surfaces as a ValueError naming the URL."""
|
||||
with patch(FETCH, side_effect=Exception("Network error")):
|
||||
with pytest.raises(ValueError, match="Failed to download PDF"):
|
||||
PDFLoader().load(SourceContent("https://example.com/report.pdf"))
|
||||
|
||||
def test_load_pdf_url_over_size_limit(self):
|
||||
"""An oversized body is reported rather than partially parsed."""
|
||||
with patch(FETCH, side_effect=ValueError("exceeds the 1024 byte limit")):
|
||||
with pytest.raises(ValueError, match="Failed to download PDF"):
|
||||
PDFLoader().load(SourceContent("https://example.com/huge.pdf"))
|
||||
|
||||
def test_load_pdf_missing_file(self):
|
||||
"""A missing local path raises FileNotFoundError, not ValueError."""
|
||||
with pytest.raises(FileNotFoundError, match="PDF file not found"):
|
||||
PDFLoader().load(SourceContent("/nonexistent/report.pdf"))
|
||||
|
||||
def test_load_corrupt_pdf_raises_value_error(self):
|
||||
"""Bytes that are not a parseable PDF produce a read error."""
|
||||
with tempfile.NamedTemporaryFile(suffix=".pdf") as f:
|
||||
f.write(b"%PDF-1.4 not really a pdf")
|
||||
f.flush()
|
||||
|
||||
with pytest.raises(ValueError, match="Error reading PDF"):
|
||||
PDFLoader().load(SourceContent(f.name))
|
||||
|
||||
def test_pdf_with_no_extractable_text(self):
|
||||
"""A PDF whose pages hold no text says so instead of returning empty."""
|
||||
document = pymupdf.open()
|
||||
document.new_page()
|
||||
blank = document.tobytes()
|
||||
document.close()
|
||||
|
||||
with tempfile.NamedTemporaryFile(suffix=".pdf") as f:
|
||||
f.write(blank)
|
||||
f.flush()
|
||||
|
||||
result = PDFLoader().load(SourceContent(f.name))
|
||||
|
||||
assert "no extractable text" in result.content
|
||||
|
||||
def test_pdf_doc_id_is_stable(self):
|
||||
"""The same source yields the same doc_id across loads."""
|
||||
with tempfile.NamedTemporaryFile(suffix=".pdf") as f:
|
||||
f.write(build_pdf())
|
||||
f.flush()
|
||||
|
||||
loader = PDFLoader()
|
||||
source = SourceContent(f.name)
|
||||
assert loader.load(source).doc_id == loader.load(source).doc_id
|
||||
@@ -28,6 +28,7 @@ class TestCrewAIPlatformActionToolVerify:
|
||||
def create_test_tool(self):
|
||||
return CrewAIPlatformActionTool(
|
||||
description="Test action tool",
|
||||
app="test_app",
|
||||
action_name="test_action",
|
||||
action_schema=self.action_schema
|
||||
)
|
||||
|
||||
@@ -171,6 +171,10 @@ class TestCrewaiPlatformToolBuilder(unittest.TestCase):
|
||||
tool_names = [tool.action_name for tool in tools]
|
||||
assert "create_issue" in tool_names
|
||||
assert "send_message" in tool_names
|
||||
assert {tool.action_name: tool.app for tool in tools} == {
|
||||
"create_issue": "github",
|
||||
"send_message": "slack",
|
||||
}
|
||||
|
||||
github_tool = next((t for t in tools if t.action_name == "create_issue"), None)
|
||||
slack_tool = next((t for t in tools if t.action_name == "send_message"), None)
|
||||
|
||||
415
lib/crewai-tools/tests/url_read_tool_test.py
Normal file
415
lib/crewai-tools/tests/url_read_tool_test.py
Normal file
@@ -0,0 +1,415 @@
|
||||
from unittest.mock import patch
|
||||
|
||||
import pytest
|
||||
import requests
|
||||
|
||||
from crewai_tools import URLReadTool
|
||||
from crewai_tools.security.safe_requests import safe_get_bounded
|
||||
|
||||
|
||||
TOOL_MODULE = "crewai_tools.tools.url_read_tool.url_read_tool"
|
||||
|
||||
|
||||
class FakeResponse:
|
||||
"""Minimal stand-in for a streamed requests.Response."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
body: bytes = b"",
|
||||
content_type: str = "text/plain",
|
||||
url: str = "https://example.com/file.txt",
|
||||
status_code: int = 200,
|
||||
chunk_size: int | None = None,
|
||||
):
|
||||
self._body = body
|
||||
self._chunk_size = chunk_size
|
||||
self.headers = {"Content-Type": content_type} if content_type else {}
|
||||
self.url = url
|
||||
self.status_code = status_code
|
||||
self.history: list["FakeResponse"] = []
|
||||
self.closed = False
|
||||
|
||||
def raise_for_status(self) -> None:
|
||||
"""Mimic requests' error-status behavior."""
|
||||
if self.status_code >= 400:
|
||||
raise requests.HTTPError(f"{self.status_code} error")
|
||||
|
||||
def iter_content(self, chunk_size: int = 65536):
|
||||
"""Yield the body in chunks, like a streamed response."""
|
||||
size = self._chunk_size or chunk_size
|
||||
for index in range(0, len(self._body), size):
|
||||
yield self._body[index : index + size]
|
||||
|
||||
def close(self) -> None:
|
||||
"""Record that the response was closed."""
|
||||
self.closed = True
|
||||
|
||||
|
||||
def build_pdf(text: str = "Quarterly revenue was 42") -> bytes:
|
||||
"""Return the bytes of a one-page PDF containing *text*."""
|
||||
pymupdf = pytest.importorskip("pymupdf")
|
||||
document = pymupdf.open()
|
||||
document.new_page().insert_text((72, 72), text)
|
||||
try:
|
||||
return document.tobytes()
|
||||
finally:
|
||||
document.close()
|
||||
|
||||
|
||||
def fetch_result(
|
||||
body: bytes,
|
||||
content_type: str = "text/plain",
|
||||
url: str = "https://example.com/f.txt",
|
||||
):
|
||||
"""Build the (body, content_type, final_url) tuple safe_get_bounded returns."""
|
||||
return body, content_type, url
|
||||
|
||||
|
||||
def test_reads_plain_text():
|
||||
"""A text response is returned as-is, with the configured limits applied."""
|
||||
tool = URLReadTool()
|
||||
with patch(f"{TOOL_MODULE}.safe_get_bounded") as fetch:
|
||||
fetch.return_value = fetch_result(b"hello world")
|
||||
assert tool.run(url="https://example.com/f.txt") == "hello world"
|
||||
|
||||
assert fetch.call_args.kwargs["max_bytes"] == 5 * 1024 * 1024
|
||||
assert fetch.call_args.kwargs["timeout"] == 30
|
||||
|
||||
|
||||
def test_honors_declared_charset():
|
||||
"""The charset in the Content-Type header drives decoding."""
|
||||
tool = URLReadTool()
|
||||
with patch(f"{TOOL_MODULE}.safe_get_bounded") as fetch:
|
||||
fetch.return_value = fetch_result(
|
||||
"café".encode("latin-1"), "text/plain; charset=iso-8859-1"
|
||||
)
|
||||
assert tool.run(url="https://example.com/f.txt") == "café"
|
||||
|
||||
|
||||
def test_encoding_override_wins_over_server_charset():
|
||||
"""An explicit encoding beats whatever the server declares."""
|
||||
tool = URLReadTool(encoding="latin-1")
|
||||
with patch(f"{TOOL_MODULE}.safe_get_bounded") as fetch:
|
||||
fetch.return_value = fetch_result(
|
||||
"café".encode("latin-1"), "text/plain; charset=utf-8"
|
||||
)
|
||||
assert tool.run(url="https://example.com/f.txt") == "café"
|
||||
|
||||
|
||||
def test_undecodable_bytes_fall_back_instead_of_failing():
|
||||
"""Partially readable text beats an error for the agent."""
|
||||
tool = URLReadTool()
|
||||
with patch(f"{TOOL_MODULE}.safe_get_bounded") as fetch:
|
||||
fetch.return_value = fetch_result(b"\xff\xfe bad bytes", "text/plain")
|
||||
result = tool.run(url="https://example.com/f.txt")
|
||||
|
||||
assert "bad bytes" in result
|
||||
assert not result.startswith("Error:")
|
||||
|
||||
|
||||
def test_line_window():
|
||||
"""start_line and line_count select a window of the extracted text."""
|
||||
tool = URLReadTool()
|
||||
body = b"one\ntwo\nthree\nfour\nfive\n"
|
||||
with patch(f"{TOOL_MODULE}.safe_get_bounded") as fetch:
|
||||
fetch.return_value = fetch_result(body)
|
||||
result = tool.run(url="https://example.com/f.txt", start_line=2, line_count=2)
|
||||
|
||||
assert result == "two\nthree\n"
|
||||
|
||||
|
||||
def test_start_line_past_end_reports_error():
|
||||
"""Asking past the end of the content is reported, not silently empty."""
|
||||
tool = URLReadTool()
|
||||
with patch(f"{TOOL_MODULE}.safe_get_bounded") as fetch:
|
||||
fetch.return_value = fetch_result(b"one\ntwo\n")
|
||||
result = tool.run(url="https://example.com/f.txt", start_line=99)
|
||||
|
||||
assert "exceeds the number of lines" in result
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"line_args",
|
||||
[{"line_count": -5}, {"line_count": 0}, {"start_line": 0}, {"start_line": -5}],
|
||||
)
|
||||
def test_line_arguments_below_one_are_refused(line_args):
|
||||
"""Out-of-range line arguments are rejected before any request is made.
|
||||
|
||||
islice raises on a negative stop index, and the windowing runs outside the
|
||||
tool's error handling, so these have to be refused at validation time.
|
||||
"""
|
||||
tool = URLReadTool()
|
||||
with patch(f"{TOOL_MODULE}.safe_get_bounded") as fetch:
|
||||
with pytest.raises(ValueError, match="greater than or equal to 1"):
|
||||
tool.run(url="https://example.com/f.txt", **line_args)
|
||||
|
||||
fetch.assert_not_called()
|
||||
|
||||
|
||||
def test_json_is_returned_verbatim():
|
||||
"""JSON is passed through undecorated so callers can parse it."""
|
||||
tool = URLReadTool()
|
||||
with patch(f"{TOOL_MODULE}.safe_get_bounded") as fetch:
|
||||
fetch.return_value = fetch_result(b'{"a": 1}', "application/json")
|
||||
assert tool.run(url="https://example.com/data.json") == '{"a": 1}'
|
||||
|
||||
|
||||
def test_structured_suffix_type_is_treated_as_text():
|
||||
"""A +json vendor type is text, not an unsupported binary type."""
|
||||
tool = URLReadTool()
|
||||
with patch(f"{TOOL_MODULE}.safe_get_bounded") as fetch:
|
||||
fetch.return_value = fetch_result(b'{"a": 1}', "application/vnd.api+json")
|
||||
assert tool.run(url="https://example.com/data") == '{"a": 1}'
|
||||
|
||||
|
||||
def test_html_is_stripped_to_visible_text():
|
||||
"""HTML returns visible text with script and style content removed."""
|
||||
tool = URLReadTool()
|
||||
body = b"<html><head><style>p{color:red}</style></head><body><p>Hi</p><script>x=1</script></body></html>"
|
||||
with patch(f"{TOOL_MODULE}.safe_get_bounded") as fetch:
|
||||
fetch.return_value = fetch_result(body, "text/html; charset=utf-8")
|
||||
result = tool.run(url="https://example.com/page")
|
||||
|
||||
assert "Hi" in result
|
||||
assert "x=1" not in result
|
||||
assert "color:red" not in result
|
||||
|
||||
|
||||
def test_binary_content_type_is_rejected():
|
||||
"""An unsupported type is refused rather than returned as base64."""
|
||||
tool = URLReadTool()
|
||||
with patch(f"{TOOL_MODULE}.safe_get_bounded") as fetch:
|
||||
fetch.return_value = fetch_result(b"\x89PNG\r\n", "image/png")
|
||||
result = tool.run(url="https://example.com/logo.png")
|
||||
|
||||
assert "Unsupported content type 'image/png'" in result
|
||||
|
||||
|
||||
def test_octet_stream_pdf_falls_back_to_url_extension():
|
||||
"""A PDF served as octet-stream is still extracted, via its extension."""
|
||||
tool = URLReadTool()
|
||||
with patch(f"{TOOL_MODULE}.safe_get_bounded") as fetch:
|
||||
fetch.return_value = fetch_result(
|
||||
build_pdf("Fallback worked"),
|
||||
"application/octet-stream",
|
||||
"https://example.com/a/b.pdf",
|
||||
)
|
||||
result = tool.run(url="https://example.com/a/b.pdf")
|
||||
|
||||
assert "Fallback worked" in result
|
||||
|
||||
|
||||
def test_missing_content_type_falls_back_to_url_extension():
|
||||
"""No Content-Type at all still reads as text when the path says .csv."""
|
||||
tool = URLReadTool()
|
||||
with patch(f"{TOOL_MODULE}.safe_get_bounded") as fetch:
|
||||
fetch.return_value = fetch_result(b"a,b\n1,2\n", "", "https://example.com/b.csv")
|
||||
assert tool.run(url="https://example.com/b.csv") == "a,b\n1,2\n"
|
||||
|
||||
|
||||
def test_query_string_does_not_break_extension_fallback():
|
||||
"""A presigned-style query string does not hide the path's extension."""
|
||||
tool = URLReadTool()
|
||||
url = "https://example.com/b.csv?X-Amz-Signature=abc"
|
||||
with patch(f"{TOOL_MODULE}.safe_get_bounded") as fetch:
|
||||
fetch.return_value = fetch_result(b"a,b\n", "application/octet-stream", url)
|
||||
assert tool.run(url=url) == "a,b\n"
|
||||
|
||||
|
||||
def test_extension_from_requested_url_survives_a_redirect():
|
||||
"""A .pdf link that redirects to an extensionless path is still extracted.
|
||||
|
||||
Presigned CDN targets routinely drop the extension and serve octet-stream,
|
||||
so the requested URL is the only place the type survives.
|
||||
"""
|
||||
tool = URLReadTool()
|
||||
with patch(f"{TOOL_MODULE}.safe_get_bounded") as fetch:
|
||||
fetch.return_value = fetch_result(
|
||||
build_pdf("Survived the redirect"),
|
||||
"application/octet-stream",
|
||||
"https://cdn.example.com/objects/9f8a7b6c5d",
|
||||
)
|
||||
result = tool.run(url="https://example.com/report.pdf")
|
||||
|
||||
assert "Survived the redirect" in result
|
||||
|
||||
|
||||
def test_octet_stream_with_unknown_extension_is_rejected():
|
||||
"""With neither a usable type nor a known extension, the read is refused."""
|
||||
tool = URLReadTool()
|
||||
with patch(f"{TOOL_MODULE}.safe_get_bounded") as fetch:
|
||||
fetch.return_value = fetch_result(
|
||||
b"\x00\x01", "application/octet-stream", "https://example.com/a/b.bin"
|
||||
)
|
||||
result = tool.run(url="https://example.com/a/b.bin")
|
||||
|
||||
assert "Unsupported content type" in result
|
||||
|
||||
|
||||
def test_validation_failure_is_returned_as_error():
|
||||
"""An SSRF rejection reaches the agent as an error string, not an exception."""
|
||||
tool = URLReadTool()
|
||||
with patch(f"{TOOL_MODULE}.safe_get_bounded") as fetch:
|
||||
fetch.side_effect = ValueError(
|
||||
"URL 'http://169.254.169.254/' resolves to private/reserved IP 169.254.169.254."
|
||||
)
|
||||
result = tool.run(url="http://169.254.169.254/")
|
||||
|
||||
assert result.startswith("Error:")
|
||||
assert "private/reserved IP" in result
|
||||
|
||||
|
||||
def test_request_failure_is_returned_as_error():
|
||||
"""A transport failure is reported without raising out of the tool."""
|
||||
tool = URLReadTool()
|
||||
with patch(f"{TOOL_MODULE}.safe_get_bounded") as fetch:
|
||||
fetch.side_effect = requests.ConnectionError("connection refused")
|
||||
result = tool.run(url="https://example.com/f.txt")
|
||||
|
||||
assert result.startswith("Error: Failed to fetch")
|
||||
|
||||
|
||||
def test_custom_headers_are_merged_over_defaults():
|
||||
"""Caller headers win, but the default User-Agent survives."""
|
||||
tool = URLReadTool(headers={"Authorization": "Bearer x", "Accept": "text/plain"})
|
||||
with patch(f"{TOOL_MODULE}.safe_get_bounded") as fetch:
|
||||
fetch.return_value = fetch_result(b"ok")
|
||||
tool.run(url="https://example.com/f.txt")
|
||||
|
||||
headers = fetch.call_args.kwargs["headers"]
|
||||
assert headers["Authorization"] == "Bearer x"
|
||||
assert headers["Accept"] == "text/plain"
|
||||
assert "crewai-tools URLReadTool" in headers["User-Agent"]
|
||||
|
||||
|
||||
def test_reads_a_real_pdf_end_to_end():
|
||||
"""Real PDF bytes are extracted page by page."""
|
||||
pdf_bytes = build_pdf()
|
||||
|
||||
tool = URLReadTool()
|
||||
with patch(f"{TOOL_MODULE}.safe_get_bounded") as fetch:
|
||||
fetch.return_value = fetch_result(
|
||||
pdf_bytes, "application/pdf", "https://example.com/report.pdf"
|
||||
)
|
||||
result = tool.run(url="https://example.com/report.pdf")
|
||||
|
||||
assert "Page 1:" in result
|
||||
assert "Quarterly revenue was 42" in result
|
||||
|
||||
|
||||
def test_corrupt_pdf_reports_error_without_raising():
|
||||
"""A malformed PDF becomes an error string, not a traceback."""
|
||||
pytest.importorskip("pymupdf")
|
||||
|
||||
tool = URLReadTool()
|
||||
with patch(f"{TOOL_MODULE}.safe_get_bounded") as fetch:
|
||||
fetch.return_value = fetch_result(
|
||||
b"%PDF-1.4 not really a pdf", "application/pdf"
|
||||
)
|
||||
result = tool.run(url="https://example.com/report.pdf")
|
||||
|
||||
assert result.startswith("Error: Failed to read PDF content")
|
||||
|
||||
|
||||
class TestSafeGetBounded:
|
||||
"""Tests for the bounded-fetch helper itself."""
|
||||
|
||||
def test_returns_body_content_type_and_final_url(self):
|
||||
"""The helper reports the body alongside where it ended up."""
|
||||
response = FakeResponse(b"payload", "text/plain", "https://example.com/final")
|
||||
with patch(
|
||||
"crewai_tools.security.safe_requests.safe_get", return_value=response
|
||||
):
|
||||
body, content_type, final_url = safe_get_bounded(
|
||||
"https://example.com/start", max_bytes=1024
|
||||
)
|
||||
|
||||
assert body == b"payload"
|
||||
assert content_type == "text/plain"
|
||||
assert final_url == "https://example.com/final"
|
||||
assert response.closed
|
||||
|
||||
def test_rejects_body_over_the_limit(self):
|
||||
"""Crossing max_bytes raises rather than truncating silently."""
|
||||
response = FakeResponse(b"x" * 100, chunk_size=10)
|
||||
with patch(
|
||||
"crewai_tools.security.safe_requests.safe_get", return_value=response
|
||||
):
|
||||
with pytest.raises(ValueError, match="exceeds the 25 byte limit"):
|
||||
safe_get_bounded("https://example.com/big", max_bytes=25)
|
||||
|
||||
assert response.closed
|
||||
|
||||
def test_oversized_error_names_the_url_that_served_the_body(self):
|
||||
"""After a redirect the requested URL is not the one that sent it."""
|
||||
response = FakeResponse(
|
||||
b"x" * 100, url="https://cdn.example.com/final", chunk_size=10
|
||||
)
|
||||
with patch(
|
||||
"crewai_tools.security.safe_requests.safe_get", return_value=response
|
||||
):
|
||||
with pytest.raises(ValueError, match="https://cdn.example.com/final"):
|
||||
safe_get_bounded("https://example.com/start", max_bytes=25)
|
||||
|
||||
@pytest.mark.parametrize("max_bytes", [0, -1])
|
||||
def test_non_positive_max_bytes_fails_before_requesting(self, max_bytes):
|
||||
"""A misconfigured cap is caught without issuing a request."""
|
||||
with patch("crewai_tools.security.safe_requests.safe_get") as safe_get:
|
||||
with pytest.raises(ValueError, match="max_bytes must be positive"):
|
||||
safe_get_bounded("https://example.com/f", max_bytes=max_bytes)
|
||||
|
||||
safe_get.assert_not_called()
|
||||
|
||||
def test_stops_reading_once_the_limit_is_crossed(self):
|
||||
"""The cap must abandon the stream, not buffer the whole body first."""
|
||||
chunks_yielded = 0
|
||||
|
||||
class CountingResponse(FakeResponse):
|
||||
def iter_content(self, chunk_size: int = 65536):
|
||||
nonlocal chunks_yielded
|
||||
for _ in range(1000):
|
||||
chunks_yielded += 1
|
||||
yield b"x" * 10
|
||||
|
||||
response = CountingResponse()
|
||||
with patch(
|
||||
"crewai_tools.security.safe_requests.safe_get", return_value=response
|
||||
):
|
||||
with pytest.raises(ValueError):
|
||||
safe_get_bounded("https://example.com/huge", max_bytes=25)
|
||||
|
||||
assert chunks_yielded == 3
|
||||
|
||||
def test_error_status_raises(self):
|
||||
"""An error status propagates as an HTTPError."""
|
||||
response = FakeResponse(b"nope", status_code=404)
|
||||
with patch(
|
||||
"crewai_tools.security.safe_requests.safe_get", return_value=response
|
||||
):
|
||||
with pytest.raises(requests.HTTPError):
|
||||
safe_get_bounded("https://example.com/missing", max_bytes=1024)
|
||||
|
||||
assert response.closed
|
||||
|
||||
def test_closes_redirect_hops(self):
|
||||
"""Streamed redirect hops hold connections until closed."""
|
||||
hop = FakeResponse(b"", status_code=302)
|
||||
response = FakeResponse(b"done")
|
||||
response.history = [hop]
|
||||
with patch(
|
||||
"crewai_tools.security.safe_requests.safe_get", return_value=response
|
||||
):
|
||||
safe_get_bounded("https://example.com/start", max_bytes=1024)
|
||||
|
||||
assert hop.closed
|
||||
assert response.closed
|
||||
|
||||
def test_requests_are_streamed(self):
|
||||
"""Streaming is what lets an oversized body be abandoned early."""
|
||||
response = FakeResponse(b"ok")
|
||||
with patch(
|
||||
"crewai_tools.security.safe_requests.safe_get", return_value=response
|
||||
) as safe_get:
|
||||
safe_get_bounded("https://example.com/f", max_bytes=1024)
|
||||
|
||||
assert safe_get.call_args.kwargs["stream"] is True
|
||||
@@ -112,6 +112,80 @@ def test_safe_get_fails_closed_after_too_many_redirects(
|
||||
safe_get("http://public.example/start", max_redirects=1, timeout=15)
|
||||
|
||||
|
||||
def _closable_response(
|
||||
url: str, status_code: int, *, location: str | None = None, closed: list[str]
|
||||
) -> requests.Response:
|
||||
"""Build a response that records its own URL when closed."""
|
||||
response = _response(url, status_code, location=location)
|
||||
response.close = lambda: closed.append(url) # type: ignore[method-assign]
|
||||
return response
|
||||
|
||||
|
||||
def test_safe_get_closes_earlier_hops_after_too_many_redirects(
|
||||
monkeypatch: pytest.MonkeyPatch, public_dns: None
|
||||
) -> None:
|
||||
"""Hops accumulated before the failure must not be left open.
|
||||
|
||||
Under stream=True each hop holds its connection until its body is read or
|
||||
closed, and a caller handed an exception has no handle on them.
|
||||
"""
|
||||
closed: list[str] = []
|
||||
|
||||
def fake_get(url: str, **kwargs: Any) -> requests.Response:
|
||||
return _closable_response(
|
||||
url, 302, location="http://safe.example/again", closed=closed
|
||||
)
|
||||
|
||||
_mock_get(monkeypatch, fake_get)
|
||||
|
||||
with pytest.raises(ValueError, match="Too many redirects"):
|
||||
safe_get("http://public.example/start", max_redirects=2, timeout=15, stream=True)
|
||||
|
||||
assert len(closed) == 3
|
||||
|
||||
|
||||
def test_safe_get_closes_earlier_hops_when_a_redirect_is_rejected(
|
||||
monkeypatch: pytest.MonkeyPatch, public_dns: None
|
||||
) -> None:
|
||||
"""A hop rejected mid-chain still releases the connections already open."""
|
||||
closed: list[str] = []
|
||||
|
||||
def fake_get(url: str, **kwargs: Any) -> requests.Response:
|
||||
if url == "http://public.example/start":
|
||||
return _closable_response(
|
||||
url, 302, location="http://safe.example/next", closed=closed
|
||||
)
|
||||
return _closable_response(
|
||||
url, 302, location="http://169.254.169.254/latest", closed=closed
|
||||
)
|
||||
|
||||
_mock_get(monkeypatch, fake_get)
|
||||
|
||||
with pytest.raises(ValueError, match="private/reserved IP"):
|
||||
safe_get("http://public.example/start", timeout=15, stream=True)
|
||||
|
||||
assert closed == ["http://safe.example/next", "http://public.example/start"]
|
||||
|
||||
|
||||
def test_safe_get_leaves_hops_open_on_success(
|
||||
monkeypatch: pytest.MonkeyPatch, public_dns: None
|
||||
) -> None:
|
||||
"""On success the hops belong to the caller, via response.history."""
|
||||
closed: list[str] = []
|
||||
|
||||
def fake_get(url: str, **kwargs: Any) -> requests.Response:
|
||||
if url == "http://public.example/start":
|
||||
return _closable_response(url, 302, location="/final", closed=closed)
|
||||
return _closable_response(url, 200, closed=closed)
|
||||
|
||||
_mock_get(monkeypatch, fake_get)
|
||||
|
||||
response = safe_get("http://public.example/start", timeout=15, stream=True)
|
||||
|
||||
assert closed == []
|
||||
assert len(response.history) == 1
|
||||
|
||||
|
||||
def test_safe_get_strips_credentials_on_cross_origin_redirect(
|
||||
monkeypatch: pytest.MonkeyPatch, public_dns: None
|
||||
) -> None:
|
||||
|
||||
@@ -26885,6 +26885,148 @@
|
||||
"type": "object"
|
||||
}
|
||||
},
|
||||
{
|
||||
"description": "A tool that reads the content at a URL and returns it as text. To use this tool, provide a 'url' parameter with an http:// or https:// address. PDF, DOCX, HTML, JSON, XML, CSV and plain-text responses are converted to text; other binary types are rejected. URLs that resolve to private or internal network addresses are refused, as are responses over the tool's size limit. Optionally provide 'start_line' and 'line_count' to read only part of the content.",
|
||||
"env_vars": [],
|
||||
"humanized_name": "Read content from a URL",
|
||||
"init_params_schema": {
|
||||
"$defs": {
|
||||
"EnvVar": {
|
||||
"properties": {
|
||||
"default": {
|
||||
"anyOf": [
|
||||
{
|
||||
"type": "string"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
],
|
||||
"default": null,
|
||||
"title": "Default"
|
||||
},
|
||||
"description": {
|
||||
"title": "Description",
|
||||
"type": "string"
|
||||
},
|
||||
"name": {
|
||||
"title": "Name",
|
||||
"type": "string"
|
||||
},
|
||||
"required": {
|
||||
"default": true,
|
||||
"title": "Required",
|
||||
"type": "boolean"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"name",
|
||||
"description"
|
||||
],
|
||||
"title": "EnvVar",
|
||||
"type": "object"
|
||||
},
|
||||
"ToolFailurePolicy": {
|
||||
"description": "How an agent reacts when one of its tools reports a failure.",
|
||||
"enum": [
|
||||
"ignore",
|
||||
"warn",
|
||||
"raise"
|
||||
],
|
||||
"title": "ToolFailurePolicy",
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"description": "Read the content at an arbitrary URL and return it as text.\n\nUnlike :class:`~crewai_tools.tools.file_read_tool.file_read_tool.FileReadTool`,\nwhich is confined to the local filesystem, this tool performs network\nrequests to addresses the caller -- often an LLM -- chooses at runtime. It\nis a separate tool for exactly that reason: granting it is granting network\negress, and that should be a deliberate choice rather than a flag on a\nfilesystem tool.\n\nResponses are decoded to text according to their content type. PDF and DOCX\nbodies have their text extracted, HTML is stripped to visible text, and\ntext-shaped types (plain text, Markdown, JSON, XML, YAML, CSV) are decoded\nas-is. Any other type is refused rather than returned as base64, keeping\nthis tool's output text-only.\n\nSecurity:\n Requests go through :func:`~crewai_tools.security.safe_requests.safe_get_bounded`,\n which resolves each hostname and rejects it when any resolved address is\n private, loopback, link-local, or otherwise reserved -- covering cloud\n metadata endpoints and internal services. Redirects are never followed\n automatically: every hop is revalidated, and credentials are dropped on\n cross-origin hops. Bodies over ``max_bytes`` are abandoned mid-stream.\n\n Two risks are not closed here. Validation resolves the hostname and\n requests resolves it again when connecting, so a DNS entry that changes\n between those lookups can still redirect the connection (DNS\n rebinding); closing that requires pinning the connection to the\n validated address. And the returned text is untrusted remote content\n flowing into an agent's context -- a fetched page can attempt to\n instruct the agent. Neither is addressable by input validation alone;\n network egress policy and prompt-level handling cover them.\n\nArgs:\n max_bytes (int): Largest response body to accept, in decoded bytes.\n Defaults to 5 MiB.\n timeout (float): Per-request timeout in seconds. Defaults to 30.\n headers (Optional[dict[str, str]]): Extra request headers. Developer\n supplied, not chosen by the model.\n encoding (Optional[str]): Force a text encoding instead of honoring the\n charset the server declares.\n **kwargs: Additional keyword arguments passed to BaseTool.\n\nExample:\n >>> tool = URLReadTool()\n >>> content = tool.run(url=\"https://example.com/report.pdf\")\n >>> head = tool.run(url=\"https://example.com/data.csv\", line_count=20)",
|
||||
"properties": {
|
||||
"encoding": {
|
||||
"anyOf": [
|
||||
{
|
||||
"type": "string"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
],
|
||||
"default": null,
|
||||
"title": "Encoding"
|
||||
},
|
||||
"headers": {
|
||||
"anyOf": [
|
||||
{
|
||||
"additionalProperties": {
|
||||
"type": "string"
|
||||
},
|
||||
"type": "object"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
],
|
||||
"default": null,
|
||||
"title": "Headers"
|
||||
},
|
||||
"max_bytes": {
|
||||
"default": 5242880,
|
||||
"title": "Max Bytes",
|
||||
"type": "integer"
|
||||
},
|
||||
"timeout": {
|
||||
"default": 30,
|
||||
"title": "Timeout",
|
||||
"type": "number"
|
||||
}
|
||||
},
|
||||
"required": [],
|
||||
"title": "URLReadTool",
|
||||
"type": "object"
|
||||
},
|
||||
"name": "URLReadTool",
|
||||
"package_dependencies": [],
|
||||
"run_params_schema": {
|
||||
"description": "Input for URLReadTool.",
|
||||
"properties": {
|
||||
"line_count": {
|
||||
"anyOf": [
|
||||
{
|
||||
"minimum": 1,
|
||||
"type": "integer"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
],
|
||||
"default": null,
|
||||
"description": "Number of lines to read. If None, reads the entire content",
|
||||
"title": "Line Count"
|
||||
},
|
||||
"start_line": {
|
||||
"anyOf": [
|
||||
{
|
||||
"minimum": 1,
|
||||
"type": "integer"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
],
|
||||
"default": 1,
|
||||
"description": "Line number to start reading from (1-indexed)",
|
||||
"title": "Start Line"
|
||||
},
|
||||
"url": {
|
||||
"description": "The http:// or https:// URL to read. Addresses that resolve to private or internal networks are refused.",
|
||||
"title": "Url",
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"url"
|
||||
],
|
||||
"title": "URLReadToolSchema",
|
||||
"type": "object"
|
||||
}
|
||||
},
|
||||
{
|
||||
"description": "This tool uses OpenAI's Vision API to describe the contents of an image.",
|
||||
"env_vars": [
|
||||
|
||||
@@ -8,8 +8,8 @@ authors = [
|
||||
]
|
||||
requires-python = ">=3.10, <3.14"
|
||||
dependencies = [
|
||||
"crewai-core==1.15.11",
|
||||
"crewai-cli==1.15.11",
|
||||
"crewai-core==1.15.12",
|
||||
"crewai-cli==1.15.12",
|
||||
# Core Dependencies
|
||||
"pydantic>=2.11.9,<2.13",
|
||||
"openai>=2.30.0,<3",
|
||||
@@ -55,7 +55,7 @@ Repository = "https://github.com/crewAIInc/crewAI"
|
||||
|
||||
[project.optional-dependencies]
|
||||
tools = [
|
||||
"crewai-tools==1.15.11",
|
||||
"crewai-tools==1.15.12",
|
||||
]
|
||||
embeddings = [
|
||||
"tiktoken>=0.8.0,<0.13"
|
||||
|
||||
@@ -48,7 +48,7 @@ def _suppress_pydantic_deprecation_warnings() -> None:
|
||||
|
||||
_suppress_pydantic_deprecation_warnings()
|
||||
|
||||
__version__ = "1.15.11"
|
||||
__version__ = "1.15.12"
|
||||
|
||||
_LAZY_IMPORTS: dict[str, tuple[str, str]] = {
|
||||
"Memory": ("crewai.memory.unified_memory", "Memory"),
|
||||
|
||||
@@ -4,7 +4,7 @@ import json
|
||||
import logging
|
||||
from typing import Any, ParamSpec, TypeVar
|
||||
|
||||
from pydantic import BaseModel
|
||||
from pydantic import BaseModel, ValidationError
|
||||
from typing_extensions import TypeIs
|
||||
|
||||
from crewai.flow.flow_definition import (
|
||||
@@ -432,6 +432,20 @@ def _iter_flow_methods(flow_class: type) -> dict[str, Any]:
|
||||
return methods
|
||||
|
||||
|
||||
def _flow_definition_validation_error(
|
||||
flow_class: type, exc: ValidationError
|
||||
) -> ValueError:
|
||||
errors = exc.errors()
|
||||
if errors:
|
||||
detail = errors[0].get("msg", str(exc))
|
||||
if isinstance(detail, str) and detail.startswith("Value error, "):
|
||||
detail = detail.removeprefix("Value error, ")
|
||||
else:
|
||||
detail = str(exc)
|
||||
class_name = getattr(flow_class, "__name__", "Flow")
|
||||
return ValueError(f"Invalid flow definition for {class_name}: {detail}")
|
||||
|
||||
|
||||
def _build_flow_definition_from_class(
|
||||
flow_class: type,
|
||||
namespace: dict[str, Any] | None = None,
|
||||
@@ -455,15 +469,18 @@ def _build_flow_definition_from_class(
|
||||
if docstring:
|
||||
description = docstring.strip()
|
||||
|
||||
definition = FlowDefinition(
|
||||
name=getattr(flow_class, "__name__", "Flow"),
|
||||
description=description,
|
||||
state=_build_state_definition(flow_class),
|
||||
config=_build_config_definition(flow_class),
|
||||
persist=_build_persistence_definition(flow_class),
|
||||
conversational=_build_conversational_definition(flow_class),
|
||||
methods=methods,
|
||||
)
|
||||
try:
|
||||
definition = FlowDefinition(
|
||||
name=getattr(flow_class, "__name__", "Flow"),
|
||||
description=description,
|
||||
state=_build_state_definition(flow_class),
|
||||
config=_build_config_definition(flow_class),
|
||||
persist=_build_persistence_definition(flow_class),
|
||||
conversational=_build_conversational_definition(flow_class),
|
||||
methods=methods,
|
||||
)
|
||||
except ValidationError as exc:
|
||||
raise _flow_definition_validation_error(flow_class, exc) from exc
|
||||
log_flow_definition_issues(definition)
|
||||
return definition
|
||||
|
||||
|
||||
@@ -775,7 +775,11 @@ class FlowDefinition(BaseModel):
|
||||
for method_name, method in self.methods.items():
|
||||
if _condition_references(method.listen, method_name):
|
||||
raise ValueError(
|
||||
f"methods.{method_name}.listen must not reference itself"
|
||||
_self_listen_error(
|
||||
method_name=method_name,
|
||||
listen=method.listen,
|
||||
definition=self,
|
||||
)
|
||||
)
|
||||
return self
|
||||
|
||||
@@ -888,6 +892,39 @@ def _condition_references(condition: FlowDefinitionCondition | None, name: str)
|
||||
)
|
||||
|
||||
|
||||
def _format_listen_condition(condition: FlowDefinitionCondition | None) -> str:
|
||||
if condition is None:
|
||||
return "None"
|
||||
return repr(condition)
|
||||
|
||||
|
||||
def _self_listen_error(
|
||||
*,
|
||||
method_name: str,
|
||||
listen: FlowDefinitionCondition | None,
|
||||
definition: FlowDefinition,
|
||||
) -> str:
|
||||
path = f"methods.{method_name}.listen"
|
||||
listen_display = _format_listen_condition(listen)
|
||||
conversational = (
|
||||
definition.conversational is not None and definition.conversational.enabled
|
||||
)
|
||||
if conversational:
|
||||
return (
|
||||
f"{path} listen condition {listen_display} matches the handler name "
|
||||
f"{method_name!r}. In conversational flows, @listen labels are router "
|
||||
"route names — they share the same trigger namespace as method completion "
|
||||
"events, so this handler would re-run in a loop. Rename the handler "
|
||||
f"(for example, handle_{method_name}) or use a different route label."
|
||||
)
|
||||
|
||||
return (
|
||||
f"{path} listen condition {listen_display} references the handler name "
|
||||
f"{method_name!r}. A listener triggered by its own completion creates an "
|
||||
"infinite loop. Listen to a different method or event, or rename the handler."
|
||||
)
|
||||
|
||||
|
||||
def _validate_action_cel(
|
||||
action: FlowActionDefinition,
|
||||
*,
|
||||
|
||||
@@ -2158,7 +2158,7 @@ def test_self_listening_method_is_rejected():
|
||||
def process(self):
|
||||
pass
|
||||
|
||||
with pytest.raises(ValueError, match="methods.process.listen"):
|
||||
with pytest.raises(ValueError, match="Invalid flow definition for SelfListenFlow"):
|
||||
SelfListenFlow.flow_definition()
|
||||
|
||||
|
||||
@@ -2176,7 +2176,7 @@ def test_or_condition_self_listen_is_rejected():
|
||||
def process(self):
|
||||
pass
|
||||
|
||||
with pytest.raises(ValueError, match="methods.process.listen"):
|
||||
with pytest.raises(ValueError, match="Invalid flow definition for OrSelfListenFlow"):
|
||||
OrSelfListenFlow.flow_definition()
|
||||
|
||||
|
||||
@@ -2190,7 +2190,7 @@ def test_router_self_listening_method_is_rejected():
|
||||
def route(self):
|
||||
return "done"
|
||||
|
||||
with pytest.raises(ValueError, match="methods.route.listen"):
|
||||
with pytest.raises(ValueError, match="Invalid flow definition for RouterSelfListenFlow"):
|
||||
RouterSelfListenFlow.flow_definition()
|
||||
|
||||
|
||||
|
||||
@@ -1231,7 +1231,7 @@ def test_static_string_listener_is_allowed_by_contract():
|
||||
@pytest.mark.parametrize("listen", ["publish", {"or": ["publish", "revise"]}])
|
||||
@pytest.mark.parametrize("router_enabled", [False, True])
|
||||
def test_flow_definition_rejects_method_self_listen(listen, router_enabled):
|
||||
with pytest.raises(ValueError, match="methods.publish.listen"):
|
||||
with pytest.raises(ValueError, match="listen condition"):
|
||||
flow_definition.FlowDefinition.from_declaration(contents=
|
||||
{
|
||||
"schema": "crewai.flow/v1",
|
||||
@@ -1252,6 +1252,49 @@ def test_flow_definition_rejects_method_self_listen(listen, router_enabled):
|
||||
)
|
||||
|
||||
|
||||
def test_flow_definition_rejects_conversational_route_handler_name_collision():
|
||||
with pytest.raises(ValueError, match=r"listen condition 'create_video'"):
|
||||
flow_definition.FlowDefinition.from_declaration(contents=
|
||||
{
|
||||
"schema": "crewai.flow/v1",
|
||||
"name": "VideoFlow",
|
||||
"conversational": {
|
||||
"enabled": True,
|
||||
"router": {
|
||||
"route_descriptions": {
|
||||
"create_video": "User wants a new video.",
|
||||
},
|
||||
},
|
||||
},
|
||||
"methods": {
|
||||
"begin": {
|
||||
"do": {"ref": "loaded_flows:VideoFlow.begin"},
|
||||
"start": True,
|
||||
},
|
||||
"create_video": {
|
||||
"do": {"ref": "loaded_flows:VideoFlow.create_video"},
|
||||
"listen": "create_video",
|
||||
},
|
||||
},
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def test_build_flow_definition_wraps_validation_error_with_class_name():
|
||||
class VideoFlow(Flow):
|
||||
conversational = True
|
||||
|
||||
@listen("create_video")
|
||||
def create_video(self):
|
||||
return "made a video"
|
||||
|
||||
with pytest.raises(ValueError, match="Invalid flow definition for VideoFlow"):
|
||||
VideoFlow.flow_definition()
|
||||
|
||||
with pytest.raises(ValueError, match="Invalid flow definition for VideoFlow"):
|
||||
VideoFlow()
|
||||
|
||||
|
||||
def test_start_false_not_classified_as_start_method():
|
||||
definition = flow_definition.FlowDefinition.from_declaration(contents=
|
||||
{
|
||||
|
||||
@@ -1698,6 +1698,7 @@ class TestPlatformActionTool:
|
||||
|
||||
return mod.CrewAIPlatformActionTool(
|
||||
description="Send a Slack message",
|
||||
app="slack",
|
||||
action_name="slackbot_send_message",
|
||||
action_schema={
|
||||
"function": {
|
||||
|
||||
@@ -39,9 +39,13 @@ devtools release 1.10.3 --skip-enterprise # skip enterprise release phase
|
||||
7. Opens a `[docs-freeze]` PR against main, polls until merged
|
||||
8. Tags main and creates GitHub release
|
||||
9. Triggers PyPI publish workflow
|
||||
10. Clones enterprise repo, bumps versions and `crewai[tools]` dep, runs `uv sync`
|
||||
11. Creates enterprise bump PR, polls until merged
|
||||
12. Tags and creates GitHub release on enterprise repo
|
||||
10. Updates `crewAIInc/crew_deployment_test` to the exact CrewAI version,
|
||||
creates a bump PR, and waits for it to merge
|
||||
11. Updates `crewAIInc/flow_deployment_test` to the exact CrewAI version,
|
||||
creates a bump PR, and waits for it to merge
|
||||
12. Clones enterprise repo, bumps versions and `crewai[tools]` dep, runs `uv sync`
|
||||
13. Creates enterprise bump PR, polls until merged
|
||||
14. Tags and creates GitHub release on enterprise repo
|
||||
|
||||
> The `docs-snapshots` CI guard rejects writes under `docs/v*/` and deletions/renames in `docs/images/` unless the PR title starts with `[docs-freeze]`. The release CLI sets that prefix automatically; manual edits to a frozen snapshot need the same prefix to land.
|
||||
>
|
||||
@@ -66,4 +70,4 @@ Tag and release only (phase 2 of `release`). Run after the bump PR is merged.
|
||||
devtools tag
|
||||
devtools tag --no-edit
|
||||
devtools tag --dry-run
|
||||
```
|
||||
```
|
||||
|
||||
@@ -15,6 +15,7 @@ dependencies = [
|
||||
"openai>=1.83.0,<3",
|
||||
"python-dotenv>=1.2.2,<2",
|
||||
"pygithub~=1.59.1",
|
||||
"pyyaml~=6.0",
|
||||
"rich>=13.9.4",
|
||||
]
|
||||
|
||||
|
||||
@@ -1,3 +1,3 @@
|
||||
"""CrewAI development tools."""
|
||||
|
||||
__version__ = "1.15.11"
|
||||
__version__ = "1.15.12"
|
||||
|
||||
@@ -4,6 +4,7 @@ from collections.abc import Mapping
|
||||
import os
|
||||
from pathlib import Path
|
||||
import re
|
||||
import shlex
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
@@ -20,6 +21,7 @@ from rich.markdown import Markdown
|
||||
from rich.panel import Panel
|
||||
from rich.prompt import Confirm
|
||||
import tomlkit
|
||||
import yaml
|
||||
|
||||
from crewai_devtools.docs_check import docs_check
|
||||
from crewai_devtools.docs_versioning import (
|
||||
@@ -1421,7 +1423,10 @@ def _repin_crewai_install(run_value: str, version: str) -> str:
|
||||
return "".join(result)
|
||||
|
||||
|
||||
_DEPLOYMENT_TEST_REPO: Final[str] = "crewAIInc/crew_deployment_test"
|
||||
_DEPLOYMENT_TEST_REPOS: Final[tuple[str, ...]] = (
|
||||
"crewAIInc/crew_deployment_test",
|
||||
"crewAIInc/flow_deployment_test",
|
||||
)
|
||||
|
||||
_PUBLISHED_WORKSPACE_PACKAGES: Final[tuple[str, ...]] = (
|
||||
"crewai",
|
||||
@@ -1435,26 +1440,164 @@ _PYPI_POLL_INTERVAL: Final[int] = 15
|
||||
_PYPI_POLL_TIMEOUT: Final[int] = 600
|
||||
|
||||
|
||||
def _update_deployment_test_repo(version: str, is_prerelease: bool) -> None:
|
||||
"""Update the deployment test repo to pin the new crewai version.
|
||||
_CREWAI_REQUIREMENT_PATTERN: Final[re.Pattern[str]] = re.compile(
|
||||
r"^crewai(?:\s*\[[^\]]+\])?(?![\w-])"
|
||||
r"\s*(?:(?P<operator>===|==|~=|!=|>=|<=|>|<)\s*"
|
||||
r"(?P<version>[^\s;]+))?",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
|
||||
Clones the repo, updates the crewai[tools] pin in pyproject.toml
|
||||
|
||||
def _crewai_requirement_pin(requirement: str) -> str | None:
|
||||
"""Return an exact CrewAI pin, or an empty string for a non-exact pin."""
|
||||
match = _CREWAI_REQUIREMENT_PATTERN.match(requirement.strip())
|
||||
if not match:
|
||||
return None
|
||||
if match.group("operator") != "==":
|
||||
return ""
|
||||
return match.group("version") or ""
|
||||
|
||||
|
||||
def _pyproject_crewai_requirements(content: str) -> list[tuple[str, str]]:
|
||||
"""Collect active CrewAI dependency requirements from pyproject content."""
|
||||
requirements: list[tuple[str, str]] = []
|
||||
doc = tomlkit.parse(content)
|
||||
for key in ("dependencies", "optional-dependencies"):
|
||||
deps = doc.get("project", {}).get(key)
|
||||
if deps is None:
|
||||
continue
|
||||
dep_lists = deps.values() if isinstance(deps, Mapping) else [deps]
|
||||
for dep_list in dep_lists:
|
||||
for dep in dep_list:
|
||||
spec = str(dep)
|
||||
pin = _crewai_requirement_pin(spec)
|
||||
if pin is not None:
|
||||
requirements.append((spec, pin))
|
||||
return requirements
|
||||
|
||||
|
||||
def _workflow_run_commands(content: str) -> list[str]:
|
||||
"""Extract shell commands from workflow ``run`` values."""
|
||||
commands: list[str] = []
|
||||
|
||||
def collect_run_commands(node: object) -> None:
|
||||
if isinstance(node, Mapping):
|
||||
for key, value in node.items():
|
||||
if key == "run" and isinstance(value, str):
|
||||
commands.append(value)
|
||||
collect_run_commands(value)
|
||||
elif isinstance(node, list):
|
||||
for value in node:
|
||||
collect_run_commands(value)
|
||||
|
||||
collect_run_commands(yaml.safe_load(content))
|
||||
return commands
|
||||
|
||||
|
||||
def _workflow_crewai_requirements(content: str) -> list[tuple[str, str]]:
|
||||
"""Collect CrewAI requirements from executable workflow install commands."""
|
||||
requirements: list[tuple[str, str]] = []
|
||||
for command in _workflow_run_commands(content):
|
||||
normalized = command.replace("\\\n", " ")
|
||||
lexer = shlex.shlex(normalized, posix=True, punctuation_chars=";&|\n")
|
||||
lexer.whitespace = " \t\r"
|
||||
lexer.whitespace_split = True
|
||||
lexer.commenters = "#"
|
||||
try:
|
||||
tokens = list(lexer)
|
||||
except ValueError:
|
||||
continue
|
||||
|
||||
index = 0
|
||||
while index < len(tokens):
|
||||
command_lengths = (
|
||||
(tokens[index : index + 3] == ["uv", "pip", "install"], 3),
|
||||
(tokens[index : index + 2] == ["uv", "add"], 2),
|
||||
(
|
||||
tokens[index : index + 2]
|
||||
in (["pip", "install"], ["pip3", "install"]),
|
||||
2,
|
||||
),
|
||||
(
|
||||
tokens[index : index + 4]
|
||||
in (
|
||||
["python", "-m", "pip", "install"],
|
||||
["python3", "-m", "pip", "install"],
|
||||
),
|
||||
4,
|
||||
),
|
||||
)
|
||||
install_length = next(
|
||||
(length for matched, length in command_lengths if matched),
|
||||
0,
|
||||
)
|
||||
if not install_length:
|
||||
index += 1
|
||||
continue
|
||||
|
||||
index += install_length
|
||||
while index < len(tokens) and tokens[index] not in {
|
||||
";",
|
||||
"&&",
|
||||
"||",
|
||||
"|",
|
||||
"\n",
|
||||
}:
|
||||
argument = tokens[index]
|
||||
pin = _crewai_requirement_pin(argument)
|
||||
if pin is not None:
|
||||
requirements.append((argument, pin))
|
||||
index += 1
|
||||
return requirements
|
||||
|
||||
|
||||
def _validate_deployment_repo_crewai_pin(
|
||||
repo_dir: Path,
|
||||
pyproject_content: str,
|
||||
version: str,
|
||||
) -> None:
|
||||
"""Fail unless every effective canary CrewAI requirement has the exact pin."""
|
||||
requirements = _pyproject_crewai_requirements(pyproject_content)
|
||||
|
||||
workflows_dir = repo_dir / ".github" / "workflows"
|
||||
if workflows_dir.exists():
|
||||
for workflow in workflows_dir.iterdir():
|
||||
if workflow.is_file() and workflow.suffix in (".yml", ".yaml"):
|
||||
requirements.extend(
|
||||
_workflow_crewai_requirements(workflow.read_text(encoding="utf-8"))
|
||||
)
|
||||
|
||||
if not requirements:
|
||||
raise RuntimeError(f"No effective CrewAI dependency found in {repo_dir.name}")
|
||||
|
||||
mismatches = [spec for spec, pin in requirements if pin != version]
|
||||
if mismatches:
|
||||
found = ", ".join(repr(spec) for spec in mismatches)
|
||||
raise RuntimeError(
|
||||
f"CrewAI dependencies in {repo_dir.name} must all pin {version}; "
|
||||
f"found {found}"
|
||||
)
|
||||
|
||||
|
||||
def _update_deployment_test_repo(repo: str, version: str, is_prerelease: bool) -> None:
|
||||
"""Update a deployment test repo to pin the new crewai version.
|
||||
|
||||
Clones the repo, updates the CrewAI pin in pyproject.toml
|
||||
and any crewai[extras] pins in .github/workflows, regenerates the
|
||||
lockfile, commits to a branch, pushes, opens a PR against main,
|
||||
then polls until the PR is merged (or closed).
|
||||
|
||||
Args:
|
||||
repo: GitHub repository containing the deployment canary.
|
||||
version: New crewai version string.
|
||||
is_prerelease: Whether this is a pre-release version.
|
||||
"""
|
||||
console.print(
|
||||
f"\n[bold cyan]Updating {_DEPLOYMENT_TEST_REPO} to {version}[/bold cyan]"
|
||||
)
|
||||
console.print(f"\n[bold cyan]Updating {repo} to {version}[/bold cyan]")
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
repo_dir = Path(tmp) / "crew_deployment_test"
|
||||
run_command(["gh", "repo", "clone", _DEPLOYMENT_TEST_REPO, str(repo_dir)])
|
||||
console.print(f"[green]✓[/green] Cloned {_DEPLOYMENT_TEST_REPO}")
|
||||
repo_dir = Path(tmp) / repo.rsplit("/", 1)[-1]
|
||||
run_command(["gh", "repo", "clone", repo, str(repo_dir)])
|
||||
console.print(f"[green]✓[/green] Cloned {repo}")
|
||||
|
||||
pyproject = repo_dir / "pyproject.toml"
|
||||
content = pyproject.read_text()
|
||||
@@ -1462,11 +1605,9 @@ def _update_deployment_test_repo(version: str, is_prerelease: bool) -> None:
|
||||
pyproject_changed = new_content != content
|
||||
if pyproject_changed:
|
||||
pyproject.write_text(new_content)
|
||||
console.print(f"[green]✓[/green] Updated crewai[tools] pin to {version}")
|
||||
console.print(f"[green]✓[/green] Updated crewai pin to {version}")
|
||||
else:
|
||||
console.print(
|
||||
"[yellow]Warning:[/yellow] No crewai[tools] pin found to update"
|
||||
)
|
||||
console.print("[yellow]Warning:[/yellow] No crewai pin found to update")
|
||||
|
||||
updated_workflows = _update_repo_workflows_crewai_pins(repo_dir, version)
|
||||
for wf in updated_workflows:
|
||||
@@ -1474,6 +1615,8 @@ def _update_deployment_test_repo(version: str, is_prerelease: bool) -> None:
|
||||
f"[green]✓[/green] Updated crewai pin in {wf.relative_to(repo_dir)}"
|
||||
)
|
||||
|
||||
_validate_deployment_repo_crewai_pin(repo_dir, new_content, version)
|
||||
|
||||
if not pyproject_changed and not updated_workflows:
|
||||
console.print("[yellow]Nothing to update; skipping commit and PR.[/yellow]")
|
||||
return
|
||||
@@ -1535,12 +1678,18 @@ def _update_deployment_test_repo(version: str, is_prerelease: bool) -> None:
|
||||
],
|
||||
cwd=repo_dir,
|
||||
)
|
||||
console.print(f"[green]✓[/green] Opened PR on {_DEPLOYMENT_TEST_REPO}")
|
||||
console.print(f"[green]✓[/green] Opened PR on {repo}")
|
||||
console.print(f"[cyan]PR URL:[/cyan] {pr_url.strip()}")
|
||||
|
||||
_wait_for_pr_merged(branch, repo_dir)
|
||||
|
||||
|
||||
def _update_deployment_test_repos(version: str, is_prerelease: bool) -> None:
|
||||
"""Pin and merge the release version in every deployment canary repo."""
|
||||
for repo in _DEPLOYMENT_TEST_REPOS:
|
||||
_update_deployment_test_repo(repo, version, is_prerelease)
|
||||
|
||||
|
||||
def _wait_for_pypi(package: str, version: str) -> None:
|
||||
"""Poll PyPI until a specific package version is available.
|
||||
|
||||
@@ -2352,13 +2501,13 @@ def release(
|
||||
|
||||
try:
|
||||
if not dry_run:
|
||||
_update_deployment_test_repo(version, is_prerelease)
|
||||
_update_deployment_test_repos(version, is_prerelease)
|
||||
except BaseException as e:
|
||||
_print_release_error(e)
|
||||
_resume_hint(
|
||||
f"Phase 2 failed updating deployment test repo. "
|
||||
f"Phase 2 failed updating deployment test repos. "
|
||||
f"Tag, release, and PyPI are done.\n"
|
||||
f"Fix the issue and update {_DEPLOYMENT_TEST_REPO} manually."
|
||||
"Fix the issue and update the Crew and Flow canary repos manually."
|
||||
f"{enterprise_hint}"
|
||||
)
|
||||
sys.exit(1)
|
||||
|
||||
@@ -3,14 +3,217 @@
|
||||
from pathlib import Path
|
||||
from textwrap import dedent
|
||||
|
||||
from crewai_devtools import cli as devtools_cli
|
||||
from crewai_devtools.cli import (
|
||||
_DEFAULT_WORKSPACE_PACKAGES,
|
||||
_pin_crewai_deps,
|
||||
_repin_crewai_install,
|
||||
_validate_deployment_repo_crewai_pin,
|
||||
update_pyproject_dependencies,
|
||||
update_pyproject_version,
|
||||
update_template_dependencies,
|
||||
)
|
||||
import pytest
|
||||
|
||||
|
||||
def test_release_updates_crew_and_flow_canary_repositories(monkeypatch) -> None:
|
||||
updates = []
|
||||
monkeypatch.setattr(
|
||||
devtools_cli,
|
||||
"_update_deployment_test_repo",
|
||||
lambda repo, version, is_prerelease: updates.append(
|
||||
(repo, version, is_prerelease)
|
||||
),
|
||||
)
|
||||
|
||||
devtools_cli._update_deployment_test_repos("2.0.0a1", True)
|
||||
|
||||
assert updates == [
|
||||
("crewAIInc/crew_deployment_test", "2.0.0a1", True),
|
||||
("crewAIInc/flow_deployment_test", "2.0.0a1", True),
|
||||
]
|
||||
|
||||
|
||||
def test_deployment_repo_validation_rejects_missing_crewai_pin(tmp_path: Path) -> None:
|
||||
with pytest.raises(RuntimeError, match="No effective CrewAI dependency"):
|
||||
_validate_deployment_repo_crewai_pin(
|
||||
tmp_path,
|
||||
'[project]\ndependencies = ["requests>=2"]\n',
|
||||
"2.0.0",
|
||||
)
|
||||
|
||||
|
||||
def test_deployment_repo_validation_accepts_workflow_pin(tmp_path: Path) -> None:
|
||||
workflows = tmp_path / ".github" / "workflows"
|
||||
workflows.mkdir(parents=True)
|
||||
(workflows / "test.yml").write_text('run: uv pip install "crewai[a2a]==2.0.0"\n')
|
||||
|
||||
_validate_deployment_repo_crewai_pin(
|
||||
tmp_path,
|
||||
'[project]\ndependencies = ["requests>=2"]\n',
|
||||
"2.0.0",
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"run_value",
|
||||
[
|
||||
"'uv pip install \"crewai==2.0.0\"'",
|
||||
'"uv pip install \\"crewai==2.0.0\\""',
|
||||
],
|
||||
)
|
||||
def test_deployment_repo_validation_accepts_quoted_workflow_pin(
|
||||
tmp_path: Path,
|
||||
run_value: str,
|
||||
) -> None:
|
||||
workflows = tmp_path / ".github" / "workflows"
|
||||
workflows.mkdir(parents=True)
|
||||
(workflows / "test.yml").write_text(
|
||||
f"run: {run_value}\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
_validate_deployment_repo_crewai_pin(
|
||||
tmp_path,
|
||||
'[project]\ndependencies = ["requests>=2"]\n',
|
||||
"2.0.0",
|
||||
)
|
||||
|
||||
|
||||
def test_deployment_repo_validation_rejects_mixed_versions(tmp_path: Path) -> None:
|
||||
workflows = tmp_path / ".github" / "workflows"
|
||||
workflows.mkdir(parents=True)
|
||||
(workflows / "test.yml").write_text('run: uv pip install "crewai[a2a]==2.0.0"\n')
|
||||
|
||||
with pytest.raises(RuntimeError, match=r"must all pin 2\.0\.0"):
|
||||
_validate_deployment_repo_crewai_pin(
|
||||
tmp_path,
|
||||
'[project]\ndependencies = ["crewai==1.0.0"]\n',
|
||||
"2.0.0",
|
||||
)
|
||||
|
||||
|
||||
def test_deployment_repo_validation_ignores_comments_and_echo(tmp_path: Path) -> None:
|
||||
workflows = tmp_path / ".github" / "workflows"
|
||||
workflows.mkdir(parents=True)
|
||||
(workflows / "test.yml").write_text(
|
||||
'run: echo "crewai==2.0.0"\n# run: pip install crewai==2.0.0\n'
|
||||
)
|
||||
|
||||
with pytest.raises(RuntimeError, match="No effective CrewAI dependency"):
|
||||
_validate_deployment_repo_crewai_pin(
|
||||
tmp_path,
|
||||
'[project]\ndependencies = ["requests>=2"]\n',
|
||||
"2.0.0",
|
||||
)
|
||||
|
||||
|
||||
def test_deployment_repo_validation_ignores_pyproject_comment_pin(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
with pytest.raises(RuntimeError, match=r"must all pin 2\.0\.0"):
|
||||
_validate_deployment_repo_crewai_pin(
|
||||
tmp_path,
|
||||
(
|
||||
"# documented pin: crewai==2.0.0\n"
|
||||
'[project]\ndependencies = ["crewai>=1.0"]\n'
|
||||
),
|
||||
"2.0.0",
|
||||
)
|
||||
|
||||
|
||||
def test_deployment_repo_validation_accepts_spaced_extras_and_marker(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
_validate_deployment_repo_crewai_pin(
|
||||
tmp_path,
|
||||
(
|
||||
"[project]\ndependencies = [\n"
|
||||
" \"crewai[tools, embeddings]==2.0.0; python_version >= '3.10'\",\n"
|
||||
"]\n"
|
||||
),
|
||||
"2.0.0",
|
||||
)
|
||||
|
||||
|
||||
def test_deployment_repo_validation_reads_multiline_workflow_install(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
workflows = tmp_path / ".github" / "workflows"
|
||||
workflows.mkdir(parents=True)
|
||||
(workflows / "test.yml").write_text(
|
||||
"steps:\n"
|
||||
" - name: Install\n"
|
||||
" run: |\n"
|
||||
" uv pip install \\\n"
|
||||
" \"crewai[tools, embeddings]==2.0.0; python_version >= '3.10'\"\n"
|
||||
)
|
||||
|
||||
_validate_deployment_repo_crewai_pin(
|
||||
tmp_path,
|
||||
'[project]\ndependencies = ["requests>=2"]\n',
|
||||
"2.0.0",
|
||||
)
|
||||
|
||||
|
||||
def test_deployment_repo_validation_reads_install_after_comment(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
workflows = tmp_path / ".github" / "workflows"
|
||||
workflows.mkdir(parents=True)
|
||||
(workflows / "test.yml").write_text(
|
||||
"steps:\n"
|
||||
" - name: Install\n"
|
||||
" run: |\n"
|
||||
" # Install the canary dependency\n"
|
||||
' uv pip install "crewai==2.0.0"\n',
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
_validate_deployment_repo_crewai_pin(
|
||||
tmp_path,
|
||||
'[project]\ndependencies = ["requests>=2"]\n',
|
||||
"2.0.0",
|
||||
)
|
||||
|
||||
|
||||
def test_deployment_repo_validation_reads_folded_workflow_install(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
workflows = tmp_path / ".github" / "workflows"
|
||||
workflows.mkdir(parents=True)
|
||||
(workflows / "test.yml").write_text(
|
||||
"steps:\n"
|
||||
" - name: Install\n"
|
||||
" run: >\n"
|
||||
" uv pip install\n"
|
||||
' "crewai==2.0.0"\n',
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
_validate_deployment_repo_crewai_pin(
|
||||
tmp_path,
|
||||
'[project]\ndependencies = ["requests>=2"]\n',
|
||||
"2.0.0",
|
||||
)
|
||||
|
||||
|
||||
def test_deployment_repo_validation_skips_non_file_workflow_entries(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
workflows = tmp_path / ".github" / "workflows"
|
||||
workflows.mkdir(parents=True)
|
||||
(workflows / "ignored.yml").mkdir()
|
||||
(workflows / "test.yaml").write_text(
|
||||
'# UTF-8 workflow: déploiement\nrun: uv pip install "crewai==2.0.0"\n',
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
_validate_deployment_repo_crewai_pin(
|
||||
tmp_path,
|
||||
'[project]\ndependencies = ["requests>=2"]\n',
|
||||
"2.0.0",
|
||||
)
|
||||
|
||||
|
||||
class TestUpdatePyprojectVersion:
|
||||
|
||||
Reference in New Issue
Block a user