andre/crewAI
1
0
Fork 0
mirror of https://github.com/crewAIInc/crewAI.git synced 2026-07-21 14:55:10 +00:00
Code Issues Actions 7 Packages Projects Releases Wiki Activity

fix(security): validate URLs before network requests in arxiv tool and devtools

Browse Source 
Apply validate_url() before fetching the arXiv API and PyPI polling
endpoints. Replace urllib.request.urlopen with requests.get to satisfy
bandit S310 without noqa suppressions. Update arxiv tool tests to mock
requests.get and validate_url.

Co-authored-by: Rip&Tear <theCyberTech@users.noreply.github.com>
This commit is contained in:
Cursor Agent
2026-07-13 05:54:29 +00:00
parent fb8e93be25
commit 304e9ecbb3
5 changed files with 102 additions and 87 deletions
Download Patch File Download Diff File Expand all files Collapse all files

19
lib/crewai-tools/src/crewai_tools/tools/arxiv_paper_tool/arxiv_paper_tool.py
View File

@@ -10,6 +10,9 @@ import xml.etree.ElementTree as ET
from crewai.tools import BaseTool, EnvVar
from pydantic import BaseModel, ConfigDict, Field
import requests
from crewai_tools.security.safe_path import validate_url
logger = logging.getLogger(__file__)
@@ -78,17 +81,17 @@ class ArxivPaperTool(BaseTool):
def fetch_arxiv_data(
self, search_query: str, max_results: int
) -> list[dict[str, Any]]:
api_url = f"{self.BASE_API_URL}?search_query={urllib.parse.quote(search_query)}&start=0&max_results={max_results}"
api_url = validate_url(
f"{self.BASE_API_URL}?search_query={urllib.parse.quote(search_query)}"
f"&start=0&max_results={max_results}"
)
logger.info(f"Fetching data from Arxiv API: {api_url}")
try:
with urllib.request.urlopen( # noqa: S310
api_url, timeout=self.REQUEST_TIMEOUT
) as response:
if response.status != 200:
raise Exception(f"HTTP {response.status}: {response.reason}")
data = response.read().decode("utf-8")
except urllib.error.URLError as e:
response = requests.get(api_url, timeout=self.REQUEST_TIMEOUT)
response.raise_for_status()
data = response.text
except requests.RequestException as e:
logger.error(f"Error fetching data from Arxiv: {e}")
raise

62
lib/crewai-tools/tests/tools/arxiv_paper_tool_test.py
View File

@@ -1,10 +1,10 @@
from pathlib import Path
from unittest.mock import MagicMock, patch
import urllib.error
import xml.etree.ElementTree as ET
from crewai_tools import ArxivPaperTool
import pytest
import requests
@pytest.fixture
@@ -12,6 +12,15 @@ def tool():
return ArxivPaperTool(download_pdfs=False)
@pytest.fixture(autouse=True)
def mock_validate_url():
with patch(
"crewai_tools.tools.arxiv_paper_tool.arxiv_paper_tool.validate_url",
side_effect=lambda url: url,
):
yield
def mock_arxiv_response():
return """<?xml version="1.0" encoding="UTF-8"?>
<feed xmlns="http://www.w3.org/2005/Atom">
@@ -26,21 +35,23 @@ def mock_arxiv_response():
</feed>"""
@patch("urllib.request.urlopen")
def test_fetch_arxiv_data(mock_urlopen, tool):
@patch("crewai_tools.tools.arxiv_paper_tool.arxiv_paper_tool.requests.get")
def test_fetch_arxiv_data(mock_get, tool):
mock_response = MagicMock()
mock_response.status = 200
mock_response.read.return_value = mock_arxiv_response().encode("utf-8")
mock_urlopen.return_value.__enter__.return_value = mock_response
mock_response.text = mock_arxiv_response()
mock_get.return_value = mock_response
results = tool.fetch_arxiv_data("transformer", 1)
assert isinstance(results, list)
assert results[0]["title"] == "Sample Paper"
@patch("urllib.request.urlopen", side_effect=urllib.error.URLError("Timeout"))
def test_fetch_arxiv_data_network_error(mock_urlopen, tool):
with pytest.raises(urllib.error.URLError):
@patch(
"crewai_tools.tools.arxiv_paper_tool.arxiv_paper_tool.requests.get",
side_effect=requests.RequestException("Timeout"),
)
def test_fetch_arxiv_data_network_error(mock_get, tool):
with pytest.raises(requests.RequestException):
tool.fetch_arxiv_data("transformer", 1)
@@ -60,13 +71,12 @@ def test_download_pdf_oserror(mock_urlretrieve):
)
@patch("urllib.request.urlopen")
@patch("crewai_tools.tools.arxiv_paper_tool.arxiv_paper_tool.requests.get")
@patch("urllib.request.urlretrieve")
def test_run_with_download(mock_urlretrieve, mock_urlopen):
def test_run_with_download(mock_urlretrieve, mock_get):
mock_response = MagicMock()
mock_response.status = 200
mock_response.read.return_value = mock_arxiv_response().encode("utf-8")
mock_urlopen.return_value.__enter__.return_value = mock_response
mock_response.text = mock_arxiv_response()
mock_get.return_value = mock_response
tool = ArxivPaperTool(download_pdfs=True)
output = tool._run("transformer", 1)
@@ -74,12 +84,11 @@ def test_run_with_download(mock_urlretrieve, mock_urlopen):
mock_urlretrieve.assert_called_once()
@patch("urllib.request.urlopen")
def test_run_no_download(mock_urlopen):
@patch("crewai_tools.tools.arxiv_paper_tool.arxiv_paper_tool.requests.get")
def test_run_no_download(mock_get):
mock_response = MagicMock()
mock_response.status = 200
mock_response.read.return_value = mock_arxiv_response().encode("utf-8")
mock_urlopen.return_value.__enter__.return_value = mock_response
mock_response.text = mock_arxiv_response()
mock_get.return_value = mock_response
tool = ArxivPaperTool(download_pdfs=False)
result = tool._run("transformer", 1)
@@ -93,20 +102,19 @@ def test_validate_save_path_creates_directory(mock_mkdir):
assert isinstance(path, Path)
@patch("urllib.request.urlopen")
def test_run_handles_exception(mock_urlopen):
mock_urlopen.side_effect = Exception("API failure")
@patch("crewai_tools.tools.arxiv_paper_tool.arxiv_paper_tool.requests.get")
def test_run_handles_exception(mock_get):
mock_get.side_effect = Exception("API failure")
tool = ArxivPaperTool()
result = tool._run("transformer", 1)
assert "Failed to fetch or download Arxiv papers" in result
@patch("urllib.request.urlopen")
def test_invalid_xml_response(mock_urlopen, tool):
@patch("crewai_tools.tools.arxiv_paper_tool.arxiv_paper_tool.requests.get")
def test_invalid_xml_response(mock_get, tool):
mock_response = MagicMock()
mock_response.read.return_value = b"<invalid><xml>"
mock_response.status = 200
mock_urlopen.return_value.__enter__.return_value = mock_response
mock_response.text = "<invalid><xml>"
mock_get.return_value = mock_response
with pytest.raises(ET.ParseError):
tool.fetch_arxiv_data("quantum", 1)

1
lib/devtools/pyproject.toml
View File

@@ -16,6 +16,7 @@ dependencies = [
"python-dotenv>=1.2.2,<2",
"pygithub~=1.59.1",
"rich>=13.9.4",
"crewai-tools",
]
[project.scripts]

9
lib/devtools/src/crewai_devtools/cli.py
View File

@@ -9,12 +9,13 @@ import sys
import tempfile
import time
from typing import Final, Literal
from urllib.request import urlopen
import click
from crewai_tools.security.safe_path import validate_url
from dotenv import load_dotenv
from github import Github
from openai import OpenAI
import requests
from rich.console import Console
from rich.markdown import Markdown
from rich.panel import Panel
@@ -1548,14 +1549,14 @@ def _wait_for_pypi(package: str, version: str) -> None:
package: PyPI package name.
version: Version string to wait for.
"""
url = f"https://pypi.org/pypi/{package}/{version}/json"
url = validate_url(f"https://pypi.org/pypi/{package}/{version}/json")
deadline = time.monotonic() + _PYPI_POLL_TIMEOUT
console.print(f"[cyan]Waiting for {package}=={version} to appear on PyPI...[/cyan]")
while time.monotonic() < deadline:
try:
with urlopen(url) as resp: # noqa: S310
if resp.status == 200:
response = requests.get(url, timeout=30)
if response.status_code == 200:
console.print(
f"[green]✓[/green] {package}=={version} is available on PyPI"
)

98
uv.lock generated
View File

@@ -13,7 +13,7 @@ resolution-markers = [
]
[options]
exclude-newer = "2026-07-04T15:35:51.457693Z"
exclude-newer = "0001-01-01T00:00:00Z" # This has no effect and is included for backwards compatibility when using relative exclude-newer values.
exclude-newer-span = "P3D"
[options.exclude-newer-package]
@@ -1052,7 +1052,7 @@ name = "coloredlogs"
version = "15.0.1"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "humanfriendly", marker = "python_full_version < '3.11'" },
{ name = "humanfriendly" },
]
sdist = { url = "https://files.pythonhosted.org/packages/cc/c7/eed8f27100517e8c0e6b923d5f0845d0cb99763da6fdee00478f91db7325/coloredlogs-15.0.1.tar.gz", hash = "sha256:7c991aa71a4577af2f82600d8f8f3a89f936baeaf9b50a9c197da014e5bf16b0", size = 278520, upload-time = "2021-06-11T10:22:45.202Z" }
wheels = [
@@ -1150,7 +1150,7 @@ resolution-markers = [
"python_full_version < '3.11' and platform_machine == 's390x'",
]
dependencies = [
{ name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" },
{ name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" } },
]
sdist = { url = "https://files.pythonhosted.org/packages/66/54/eb9bfc647b19f2009dd5c7f5ec51c4e6ca831725f1aea7a993034f483147/contourpy-1.3.2.tar.gz", hash = "sha256:b6945942715a034c671b7fc54f9588126b0b8bf23db2696e3ca8328f3ff0ab54", size = 13466130, upload-time = "2025-04-15T17:47:53.79Z" }
wheels = [
@@ -1225,7 +1225,7 @@ resolution-markers = [
"python_full_version == '3.11.*' and platform_machine == 's390x'",
]
dependencies = [
{ name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" },
{ name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" } },
]
sdist = { url = "https://files.pythonhosted.org/packages/58/01/1253e6698a07380cd31a736d248a3f2a50a7c88779a1813da27503cadc2a/contourpy-1.3.3.tar.gz", hash = "sha256:083e12155b210502d0bca491432bb04d56dc3432f95a979b429f2848c3dbe880", size = 13466174, upload-time = "2025-07-26T12:03:12.549Z" }
wheels = [
@@ -1557,6 +1557,7 @@ name = "crewai-devtools"
source = { editable = "lib/devtools" }
dependencies = [
{ name = "click" },
{ name = "crewai-tools" },
{ name = "openai" },
{ name = "pygithub" },
{ name = "python-dotenv" },
@@ -1567,6 +1568,7 @@ dependencies = [
[package.metadata]
requires-dist = [
{ name = "click", specifier = ">=8.1.7,<9" },
{ name = "crewai-tools", editable = "lib/crewai-tools" },
{ name = "openai", specifier = ">=1.83.0,<3" },
{ name = "pygithub", specifier = "~=1.59.1" },
{ name = "python-dotenv", specifier = ">=1.2.2,<2" },
@@ -1877,34 +1879,34 @@ wheels = [
[package.optional-dependencies]
cudart = [
{ name = "nvidia-cuda-runtime", marker = "sys_platform == 'linux' or sys_platform == 'win32'" },
{ name = "nvidia-cuda-runtime" },
]
cufft = [
{ name = "nvidia-cufft", marker = "sys_platform == 'linux' or sys_platform == 'win32'" },
{ name = "nvidia-cufft" },
]
cufile = [
{ name = "nvidia-cufile", marker = "sys_platform == 'linux'" },
{ name = "nvidia-cufile" },
]
cupti = [
{ name = "nvidia-cuda-cupti", marker = "sys_platform == 'linux' or sys_platform == 'win32'" },
{ name = "nvidia-cuda-cupti" },
]
curand = [
{ name = "nvidia-curand", marker = "sys_platform == 'linux' or sys_platform == 'win32'" },
{ name = "nvidia-curand" },
]
cusolver = [
{ name = "nvidia-cusolver", marker = "sys_platform == 'linux' or sys_platform == 'win32'" },
{ name = "nvidia-cusolver" },
]
cusparse = [
{ name = "nvidia-cusparse", marker = "sys_platform == 'linux' or sys_platform == 'win32'" },
{ name = "nvidia-cusparse" },
]
nvjitlink = [
{ name = "nvidia-nvjitlink", marker = "sys_platform == 'linux' or sys_platform == 'win32'" },
{ name = "nvidia-nvjitlink" },
]
nvrtc = [
{ name = "nvidia-cuda-nvrtc", marker = "sys_platform == 'linux' or sys_platform == 'win32'" },
{ name = "nvidia-cuda-nvrtc" },
]
nvtx = [
{ name = "nvidia-nvtx", marker = "sys_platform == 'linux' or sys_platform == 'win32'" },
{ name = "nvidia-nvtx" },
]
[[package]]
@@ -2429,7 +2431,7 @@ name = "exceptiongroup"
version = "1.3.1"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "typing-extensions", marker = "python_full_version < '3.11'" },
{ name = "typing-extensions" },
]
sdist = { url = "https://files.pythonhosted.org/packages/50/79/66800aadf48771f6b62f7eb014e352e5d06856655206165d775e675a02c9/exceptiongroup-1.3.1.tar.gz", hash = "sha256:8b412432c6055b0b7d14c310000ae93352ed6754f70fa8f7c34141f91c4e3219", size = 30371, upload-time = "2025-11-21T23:01:54.787Z" }
wheels = [
@@ -3018,8 +3020,8 @@ name = "grpcio-health-checking"
version = "1.71.2"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "grpcio", marker = "python_full_version < '3.11' or (python_full_version >= '3.13' and platform_machine != 's390x')" },
{ name = "protobuf", marker = "python_full_version < '3.11' or (python_full_version >= '3.13' and platform_machine != 's390x')" },
{ name = "grpcio" },
{ name = "protobuf" },
]
sdist = { url = "https://files.pythonhosted.org/packages/53/86/20994347ef36b7626fb74539f13128100dd8b7eaac67efc063264e6cdc80/grpcio_health_checking-1.71.2.tar.gz", hash = "sha256:1c21ece88c641932f432b573ef504b20603bdf030ad4e1ec35dd7fdb4ea02637", size = 16770, upload-time = "2025-06-28T04:24:08.768Z" }
wheels = [
@@ -3223,7 +3225,7 @@ name = "humanfriendly"
version = "10.0"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "pyreadline3", marker = "python_full_version < '3.11' and sys_platform == 'win32'" },
{ name = "pyreadline3", marker = "sys_platform == 'win32'" },
]
sdist = { url = "https://files.pythonhosted.org/packages/cc/3f/2c29224acb2e2df4d2046e4c73ee2662023c58ff5b113c4c1adac0886c43/humanfriendly-10.0.tar.gz", hash = "sha256:6b0b831ce8f15f7300721aa49829fc4e83921a9a301cc7f606be6686a2288ddc", size = 360702, upload-time = "2021-09-17T21:40:43.31Z" }
wheels = [
@@ -4460,10 +4462,10 @@ resolution-markers = [
"python_full_version < '3.11' and platform_machine == 's390x'",
]
dependencies = [
{ name = "jsonref", marker = "python_full_version < '3.12'" },
{ name = "mcp", extra = ["ws"], marker = "python_full_version < '3.12'" },
{ name = "pydantic", marker = "python_full_version < '3.12'" },
{ name = "python-dotenv", marker = "python_full_version < '3.12'" },
{ name = "jsonref" },
{ name = "mcp", extra = ["ws"] },
{ name = "pydantic" },
{ name = "python-dotenv" },
]
sdist = { url = "https://files.pythonhosted.org/packages/d0/28/64fc666fa5d86bb1b048c167975d4ea19210f9f8571b64b26563739774ac/mcpadapt-0.1.19.tar.gz", hash = "sha256:dfab84fc75cc84a49a40bd61079773b1faf840227b74b82c71a7755b9c1957c5", size = 4227721, upload-time = "2025-10-16T07:11:56.736Z" }
wheels = [
@@ -4481,10 +4483,10 @@ resolution-markers = [
"python_full_version == '3.12.*' and platform_machine == 's390x'",
]
dependencies = [
{ name = "jsonref", marker = "python_full_version >= '3.12'" },
{ name = "mcp", extra = ["ws"], marker = "python_full_version >= '3.12'" },
{ name = "pydantic", marker = "python_full_version >= '3.12'" },
{ name = "python-dotenv", marker = "python_full_version >= '3.12'" },
{ name = "jsonref" },
{ name = "mcp", extra = ["ws"] },
{ name = "pydantic" },
{ name = "python-dotenv" },
]
sdist = { url = "https://files.pythonhosted.org/packages/e3/71/1bbbe157e55d30ab4a74fa878f6942cc0586e9820f03e03451a3d2297e9b/mcpadapt-0.1.20.tar.gz", hash = "sha256:4047c0da61e481dd0673a48936a427da9e6547c6cf0d580ff4e4761dcf058ed1", size = 4203656, upload-time = "2025-10-24T15:35:02.135Z" }
wheels = [
@@ -5499,12 +5501,12 @@ name = "onnxruntime"
version = "1.23.2"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "coloredlogs", marker = "python_full_version < '3.11'" },
{ name = "flatbuffers", marker = "python_full_version < '3.11'" },
{ name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" },
{ name = "packaging", marker = "python_full_version < '3.11'" },
{ name = "protobuf", marker = "python_full_version < '3.11'" },
{ name = "sympy", marker = "python_full_version < '3.11'" },
{ name = "coloredlogs" },
{ name = "flatbuffers" },
{ name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" } },
{ name = "packaging" },
{ name = "protobuf" },
{ name = "sympy" },
]
wheels = [
{ url = "https://files.pythonhosted.org/packages/35/d6/311b1afea060015b56c742f3531168c1644650767f27ef40062569960587/onnxruntime-1.23.2-cp310-cp310-macosx_13_0_arm64.whl", hash = "sha256:a7730122afe186a784660f6ec5807138bf9d792fa1df76556b27307ea9ebcbe3", size = 17195934, upload-time = "2025-10-27T23:06:14.143Z" },
@@ -8173,7 +8175,7 @@ resolution-markers = [
"python_full_version < '3.11' and platform_machine == 's390x'",
]
dependencies = [
{ name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" },
{ name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" } },
]
sdist = { url = "https://files.pythonhosted.org/packages/0f/37/6964b830433e654ec7485e45a00fc9a27cf868d622838f6b6d9c5ec0d532/scipy-1.15.3.tar.gz", hash = "sha256:eae3cf522bc7df64b42cad3925c876e1b0b6c35c1337c93e12c0f366f55b0eaf", size = 59419214, upload-time = "2025-05-08T16:13:05.955Z" }
wheels = [
@@ -8237,7 +8239,7 @@ resolution-markers = [
"python_full_version == '3.11.*' and platform_machine == 's390x'",
]
dependencies = [
{ name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" },
{ name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" } },
]
sdist = { url = "https://files.pythonhosted.org/packages/7a/97/5a3609c4f8d58b039179648e62dd220f89864f56f7357f5d4f45c29eb2cc/scipy-1.17.1.tar.gz", hash = "sha256:95d8e012d8cb8816c226aef832200b1d45109ed4464303e997c5b13122b297c0", size = 30573822, upload-time = "2026-02-23T00:26:24.851Z" }
wheels = [
@@ -9868,13 +9870,13 @@ resolution-markers = [
"python_full_version < '3.11' and platform_machine == 's390x'",
]
dependencies = [
{ name = "authlib", marker = "python_full_version < '3.11' or (python_full_version >= '3.13' and platform_machine != 's390x')" },
{ name = "deprecation", marker = "python_full_version < '3.11' or (python_full_version >= '3.13' and platform_machine != 's390x')" },
{ name = "grpcio", marker = "python_full_version < '3.11' or (python_full_version >= '3.13' and platform_machine != 's390x')" },
{ name = "grpcio-health-checking", marker = "python_full_version < '3.11' or (python_full_version >= '3.13' and platform_machine != 's390x')" },
{ name = "httpx", marker = "python_full_version < '3.11' or (python_full_version >= '3.13' and platform_machine != 's390x')" },
{ name = "pydantic", marker = "python_full_version < '3.11' or (python_full_version >= '3.13' and platform_machine != 's390x')" },
{ name = "validators", marker = "python_full_version < '3.11' or (python_full_version >= '3.13' and platform_machine != 's390x')" },
{ name = "authlib" },
{ name = "deprecation" },
{ name = "grpcio" },
{ name = "grpcio-health-checking" },
{ name = "httpx" },
{ name = "pydantic" },
{ name = "validators" },
]
sdist = { url = "https://files.pythonhosted.org/packages/a7/b9/7b9e05cf923743aa1479afcd85c48ebca82d031c3c3a5d02b1b3fcb52eb9/weaviate_client-4.16.2.tar.gz", hash = "sha256:eb7107a3221a5ad68d604cafc65195bd925a9709512ea0b6fe0dd212b0678fab", size = 681321, upload-time = "2025-07-22T09:10:48.79Z" }
wheels = [
@@ -9893,13 +9895,13 @@ resolution-markers = [
"python_full_version == '3.11.*' and platform_machine == 's390x'",
]
dependencies = [
{ name = "authlib", marker = "(python_full_version >= '3.11' and python_full_version < '3.13') or (python_full_version >= '3.11' and platform_machine == 's390x')" },
{ name = "grpcio", marker = "(python_full_version >= '3.11' and python_full_version < '3.13') or (python_full_version >= '3.11' and platform_machine == 's390x')" },
{ name = "httpx", marker = "(python_full_version >= '3.11' and python_full_version < '3.13') or (python_full_version >= '3.11' and platform_machine == 's390x')" },
{ name = "packaging", marker = "(python_full_version >= '3.11' and python_full_version < '3.13') or (python_full_version >= '3.11' and platform_machine == 's390x')" },
{ name = "protobuf", marker = "(python_full_version >= '3.11' and python_full_version < '3.13') or (python_full_version >= '3.11' and platform_machine == 's390x')" },
{ name = "pydantic", marker = "(python_full_version >= '3.11' and python_full_version < '3.13') or (python_full_version >= '3.11' and platform_machine == 's390x')" },
{ name = "validators", marker = "(python_full_version >= '3.11' and python_full_version < '3.13') or (python_full_version >= '3.11' and platform_machine == 's390x')" },
{ name = "authlib" },
{ name = "grpcio" },
{ name = "httpx" },
{ name = "packaging" },
{ name = "protobuf" },
{ name = "pydantic" },
{ name = "validators" },
]
sdist = { url = "https://files.pythonhosted.org/packages/2b/b8/103f3aaa246d4e932f4cfeb846e51436966f2aeedf60c2665a3fc51a975a/weaviate_client-4.21.3.tar.gz", hash = "sha256:d7b1f2b0cecbc747e9427f4e3b9463cdfee090746bfbbd40e59cfa25ea2afd4a", size = 847895, upload-time = "2026-06-02T13:03:51.598Z" }
wheels = [