diff --git a/lib/crewai-tools/src/crewai_tools/__init__.py b/lib/crewai-tools/src/crewai_tools/__init__.py index 1ffdf96b4..a50444888 100644 --- a/lib/crewai-tools/src/crewai_tools/__init__.py +++ b/lib/crewai-tools/src/crewai_tools/__init__.py @@ -209,6 +209,7 @@ from crewai_tools.tools.tavily_research_tool.tavily_research_tool import ( ) from crewai_tools.tools.tavily_search_tool.tavily_search_tool import TavilySearchTool from crewai_tools.tools.txt_search_tool.txt_search_tool import TXTSearchTool +from crewai_tools.tools.url_read_tool.url_read_tool import URLReadTool from crewai_tools.tools.vision_tool.vision_tool import VisionTool from crewai_tools.tools.wait_tool.wait_tool import WaitTool from crewai_tools.tools.weaviate_tool.vector_search import WeaviateVectorSearchTool @@ -327,6 +328,7 @@ __all__ = [ "TavilyGetResearchTool", "TavilyResearchTool", "TavilySearchTool", + "URLReadTool", "VisionTool", "WaitTool", "WeaviateVectorSearchTool", diff --git a/lib/crewai-tools/src/crewai_tools/rag/loaders/pdf_loader.py b/lib/crewai-tools/src/crewai_tools/rag/loaders/pdf_loader.py index 259181481..84fd45d96 100644 --- a/lib/crewai-tools/src/crewai_tools/rag/loaders/pdf_loader.py +++ b/lib/crewai-tools/src/crewai_tools/rag/loaders/pdf_loader.py @@ -2,13 +2,17 @@ import os from pathlib import Path -import tempfile from typing import Any from urllib.parse import urlparse from crewai_tools.rag.base_loader import BaseLoader, LoaderResult from crewai_tools.rag.source_content import SourceContent -from crewai_tools.security.safe_requests import safe_get +from crewai_tools.security.safe_requests import safe_get_bounded + + +# Remote PDFs are held in memory for the whole extraction, so the download needs +# a ceiling. Override per call with the ``max_bytes`` kwarg to ``load``. +DEFAULT_MAX_PDF_BYTES = 50 * 1024 * 1024 class PDFLoader(BaseLoader): @@ -24,18 +28,24 @@ class PDFLoader(BaseLoader): return False @staticmethod - def _download_from_url(url: str, kwargs: dict[str, Any]) -> str: - """Download PDF from a URL to a temporary file and return its path. + def _fetch_from_url(url: str, kwargs: dict[str, Any]) -> bytes: + """Download a PDF from a URL and return its bytes. + + The content stays in memory rather than going to a temporary file: the + whole body has to be buffered either way, and a temp file would need + unlinking on every error path to avoid leaving files behind. Because it + is held in memory, the download is capped. Args: url: The URL to download from. - kwargs: Optional dict that may contain custom headers. + kwargs: Optional dict that may contain custom ``headers`` and a + ``max_bytes`` ceiling for the download. Returns: - Path to the temporary file containing the PDF. + The raw PDF content. Raises: - ValueError: If the download fails. + ValueError: If the download fails or exceeds the size ceiling. """ headers = kwargs.get( "headers", @@ -46,12 +56,13 @@ class PDFLoader(BaseLoader): ) try: - response = safe_get(url, headers=headers, timeout=30) - response.raise_for_status() - - with tempfile.NamedTemporaryFile(suffix=".pdf", delete=False) as temp_file: - temp_file.write(response.content) - return temp_file.name + body, _content_type, _final_url = safe_get_bounded( + url, + max_bytes=kwargs.get("max_bytes", DEFAULT_MAX_PDF_BYTES), + headers=headers, + timeout=30, + ) + return body except Exception as e: raise ValueError(f"Failed to download PDF from {url}: {e!s}") from e @@ -93,21 +104,25 @@ class PDFLoader(BaseLoader): try: if is_url: - local_path = self._download_from_url(file_path, kwargs) - doc = pymupdf.open(local_path) + doc = pymupdf.open( + stream=self._fetch_from_url(file_path, kwargs), filetype="pdf" + ) else: if not os.path.isfile(file_path): raise FileNotFoundError(f"PDF file not found: {file_path}") doc = pymupdf.open(file_path) - metadata["num_pages"] = len(doc) + # Closed in a finally so a failure mid-extraction still releases the + # document handle. + try: + metadata["num_pages"] = len(doc) - for page_num, page in enumerate(doc, 1): - page_text = page.get_text() - if page_text.strip(): - text_content.append(f"Page {page_num}:\n{page_text}") - - doc.close() + for page_num, page in enumerate(doc, 1): + page_text = page.get_text() + if page_text.strip(): + text_content.append(f"Page {page_num}:\n{page_text}") + finally: + doc.close() except FileNotFoundError: raise except Exception as e: diff --git a/lib/crewai-tools/src/crewai_tools/security/safe_requests.py b/lib/crewai-tools/src/crewai_tools/security/safe_requests.py index 505a5cdb6..12765b022 100644 --- a/lib/crewai-tools/src/crewai_tools/security/safe_requests.py +++ b/lib/crewai-tools/src/crewai_tools/security/safe_requests.py @@ -11,6 +11,7 @@ from crewai_tools.security.safe_path import validate_url _REDIRECT_STATUS_CODES = {301, 302, 303, 307, 308} +_STREAM_CHUNK_SIZE = 65536 _SENSITIVE_HEADER_NAMES = { "authorization", "cookie", @@ -49,40 +50,124 @@ def _strip_cross_origin_credentials(request_kwargs: dict[str, Any]) -> dict[str, def safe_get(url: str, *, max_redirects: int = 10, **kwargs: Any) -> requests.Response: - """GET a URL while validating each redirect target before following it.""" + """GET a URL while validating each redirect target before following it. + + On success the hops are attached to the returned response's ``history`` and + are the caller's to close. On failure they are closed here: a caller given + an exception has no handle on them, and a streamed hop holds its connection + until its body is read or closed. + """ current_url = validate_url(url) request_kwargs = {**kwargs, "allow_redirects": False} timeout = request_kwargs.pop("timeout", 30) history: list[requests.Response] = [] redirects_followed = 0 - while True: - response = requests.get(current_url, timeout=timeout, **request_kwargs) - if ( - response.status_code not in _REDIRECT_STATUS_CODES - or "Location" not in response.headers - ): - response.history = history - return response + try: + while True: + response = requests.get(current_url, timeout=timeout, **request_kwargs) + if ( + response.status_code not in _REDIRECT_STATUS_CODES + or "Location" not in response.headers + ): + response.history = history + return response - if redirects_followed >= max_redirects: - response.close() - raise ValueError(f"Too many redirects while fetching URL: {url}") + if redirects_followed >= max_redirects: + response.close() + raise ValueError(f"Too many redirects while fetching URL: {url}") - location = response.headers.get("Location") - if not location: - response.history = history - return response + location = response.headers.get("Location") + if not location: + response.history = history + return response - try: - redirect_url = validate_url(urljoin(response.url, location)) - except ValueError: - response.close() - raise + try: + redirect_url = validate_url(urljoin(response.url, location)) + except ValueError: + response.close() + raise - if not _same_origin(current_url, redirect_url): - request_kwargs = _strip_cross_origin_credentials(request_kwargs) + if not _same_origin(current_url, redirect_url): + request_kwargs = _strip_cross_origin_credentials(request_kwargs) - history.append(response) - current_url = redirect_url - redirects_followed += 1 + history.append(response) + current_url = redirect_url + redirects_followed += 1 + except BaseException: + for hop in history: + hop.close() + raise + + +def safe_get_bounded( + url: str, + *, + max_bytes: int, + timeout: float | tuple[float, float] = 30, + headers: dict[str, str] | None = None, + max_redirects: int = 10, +) -> tuple[bytes, str, str]: + """GET a URL through :func:`safe_get`, refusing bodies over *max_bytes*. + + The body is streamed and abandoned as soon as it crosses the limit, so an + oversized response costs one chunk of memory instead of all of it. The cap + counts decoded bytes, which is what a compressed response expands into -- + ``Content-Length`` describes the wire size and cannot bound that. + + Args: + url: The URL to fetch. + max_bytes: Largest body to accept, in decoded bytes. + timeout: Request timeout, passed through to requests. + headers: Request headers. + max_redirects: Hops to follow before giving up. + + Returns: + A ``(body, content_type, final_url)`` tuple, where *final_url* is the + last validated URL in the redirect chain. + + Raises: + ValueError: If *max_bytes* is not positive, URL validation fails, the + redirect chain is too long, or the body exceeds *max_bytes*. + requests.RequestException: If the request fails or returns an error + status. + """ + if max_bytes <= 0: + raise ValueError(f"max_bytes must be positive, got {max_bytes}.") + + response = safe_get( + url, + max_redirects=max_redirects, + headers=headers, + timeout=timeout, + stream=True, + ) + try: + response.raise_for_status() + + chunks: list[bytes] = [] + total = 0 + for chunk in response.iter_content(chunk_size=_STREAM_CHUNK_SIZE): + if not chunk: + continue + total += len(chunk) + if total > max_bytes: + # Names the URL that served the body, which after a redirect is + # not the one that was requested. + raise ValueError( + f"Response body from '{response.url}' exceeds the " + f"{max_bytes} byte limit." + ) + chunks.append(chunk) + + return ( + b"".join(chunks), + response.headers.get("Content-Type", ""), + response.url, + ) + finally: + # Under stream=True each hop holds its connection until the body is read, + # so the redirects need closing too, not just the response we return. + for hop in response.history: + hop.close() + response.close() diff --git a/lib/crewai-tools/src/crewai_tools/tools/__init__.py b/lib/crewai-tools/src/crewai_tools/tools/__init__.py index 4651bbdd3..2653490f7 100644 --- a/lib/crewai-tools/src/crewai_tools/tools/__init__.py +++ b/lib/crewai-tools/src/crewai_tools/tools/__init__.py @@ -196,6 +196,7 @@ from crewai_tools.tools.tavily_research_tool.tavily_research_tool import ( ) from crewai_tools.tools.tavily_search_tool.tavily_search_tool import TavilySearchTool from crewai_tools.tools.txt_search_tool.txt_search_tool import TXTSearchTool +from crewai_tools.tools.url_read_tool.url_read_tool import URLReadTool from crewai_tools.tools.vision_tool.vision_tool import VisionTool from crewai_tools.tools.wait_tool.wait_tool import WaitTool from crewai_tools.tools.weaviate_tool.vector_search import WeaviateVectorSearchTool @@ -310,6 +311,7 @@ __all__ = [ "TavilyGetResearchTool", "TavilyResearchTool", "TavilySearchTool", + "URLReadTool", "VisionTool", "WaitTool", "WeaviateVectorSearchTool", diff --git a/lib/crewai-tools/src/crewai_tools/tools/url_read_tool/__init__.py b/lib/crewai-tools/src/crewai_tools/tools/url_read_tool/__init__.py new file mode 100644 index 000000000..37efdfc9b --- /dev/null +++ b/lib/crewai-tools/src/crewai_tools/tools/url_read_tool/__init__.py @@ -0,0 +1,4 @@ +from crewai_tools.tools.url_read_tool.url_read_tool import URLReadTool + + +__all__ = ["URLReadTool"] diff --git a/lib/crewai-tools/src/crewai_tools/tools/url_read_tool/url_read_tool.py b/lib/crewai-tools/src/crewai_tools/tools/url_read_tool/url_read_tool.py new file mode 100644 index 000000000..5a2c83288 --- /dev/null +++ b/lib/crewai-tools/src/crewai_tools/tools/url_read_tool/url_read_tool.py @@ -0,0 +1,379 @@ +"""Tool for reading the content at an arbitrary URL as text.""" + +from __future__ import annotations + +from io import BytesIO +from itertools import islice +import re +from typing import Any, Final +from urllib.parse import urlparse + +from crewai.tools import BaseTool +from pydantic import BaseModel, Field +import requests + +from crewai_tools.security.safe_path import format_error_for_display +from crewai_tools.security.safe_requests import safe_get_bounded + + +_DEFAULT_MAX_BYTES: Final[int] = 5 * 1024 * 1024 +_DEFAULT_TIMEOUT: Final[int] = 30 + +_PDF_TYPE: Final[str] = "application/pdf" +_DOCX_TYPE: Final[str] = ( + "application/vnd.openxmlformats-officedocument.wordprocessingml.document" +) +_HTML_TYPES: Final[frozenset[str]] = frozenset({"text/html", "application/xhtml+xml"}) +_TEXT_TYPES: Final[frozenset[str]] = frozenset( + { + "application/csv", + "application/javascript", + "application/json", + "application/sql", + "application/x-ndjson", + "application/x-yaml", + "application/xml", + "application/yaml", + } +) +_TEXT_TYPE_SUFFIXES: Final[tuple[str, ...]] = ("+json", "+xml", "+yaml") + +# Servers commonly serve static files as octet-stream, or send no type at all, +# so the extension is consulted when the header carries no usable answer. +_UNINFORMATIVE_TYPES: Final[frozenset[str]] = frozenset( + {"", "application/octet-stream", "binary/octet-stream"} +) +_EXTENSION_TYPES: Final[dict[str, str]] = { + ".csv": "text/csv", + ".docx": _DOCX_TYPE, + ".htm": "text/html", + ".html": "text/html", + ".json": "application/json", + ".md": "text/markdown", + ".pdf": _PDF_TYPE, + ".txt": "text/plain", + ".xml": "application/xml", + ".yaml": "application/yaml", + ".yml": "application/yaml", +} + +_SPACES_PATTERN: Final[re.Pattern[str]] = re.compile(r"[ \t]+") +_NEWLINE_PATTERN: Final[re.Pattern[str]] = re.compile(r"\s+\n\s+") + + +def _charset_from_content_type(content_type: str) -> str | None: + """Return the charset parameter of a Content-Type header, if it has one.""" + for parameter in content_type.split(";")[1:]: + name, _, value = parameter.partition("=") + if name.strip().lower() == "charset": + return value.strip().strip('"') or None + return None + + +class URLReadToolSchema(BaseModel): + """Input for URLReadTool.""" + + url: str = Field( + ..., + description=( + "The http:// or https:// URL to read. Addresses that resolve to " + "private or internal networks are refused." + ), + ) + start_line: int | None = Field( + 1, ge=1, description="Line number to start reading from (1-indexed)" + ) + line_count: int | None = Field( + None, + ge=1, + description="Number of lines to read. If None, reads the entire content", + ) + + +class URLReadTool(BaseTool): + """Read the content at an arbitrary URL and return it as text. + + Unlike :class:`~crewai_tools.tools.file_read_tool.file_read_tool.FileReadTool`, + which is confined to the local filesystem, this tool performs network + requests to addresses the caller -- often an LLM -- chooses at runtime. It + is a separate tool for exactly that reason: granting it is granting network + egress, and that should be a deliberate choice rather than a flag on a + filesystem tool. + + Responses are decoded to text according to their content type. PDF and DOCX + bodies have their text extracted, HTML is stripped to visible text, and + text-shaped types (plain text, Markdown, JSON, XML, YAML, CSV) are decoded + as-is. Any other type is refused rather than returned as base64, keeping + this tool's output text-only. + + Security: + Requests go through :func:`~crewai_tools.security.safe_requests.safe_get_bounded`, + which resolves each hostname and rejects it when any resolved address is + private, loopback, link-local, or otherwise reserved -- covering cloud + metadata endpoints and internal services. Redirects are never followed + automatically: every hop is revalidated, and credentials are dropped on + cross-origin hops. Bodies over ``max_bytes`` are abandoned mid-stream. + + Two risks are not closed here. Validation resolves the hostname and + requests resolves it again when connecting, so a DNS entry that changes + between those lookups can still redirect the connection (DNS + rebinding); closing that requires pinning the connection to the + validated address. And the returned text is untrusted remote content + flowing into an agent's context -- a fetched page can attempt to + instruct the agent. Neither is addressable by input validation alone; + network egress policy and prompt-level handling cover them. + + Args: + max_bytes (int): Largest response body to accept, in decoded bytes. + Defaults to 5 MiB. + timeout (float): Per-request timeout in seconds. Defaults to 30. + headers (Optional[dict[str, str]]): Extra request headers. Developer + supplied, not chosen by the model. + encoding (Optional[str]): Force a text encoding instead of honoring the + charset the server declares. + **kwargs: Additional keyword arguments passed to BaseTool. + + Example: + >>> tool = URLReadTool() + >>> content = tool.run(url="https://example.com/report.pdf") + >>> head = tool.run(url="https://example.com/data.csv", line_count=20) + """ + + name: str = "Read content from a URL" + description: str = ( + "A tool that reads the content at a URL and returns it as text. To use " + "this tool, provide a 'url' parameter with an http:// or https:// " + "address. PDF, DOCX, HTML, JSON, XML, CSV and plain-text responses are " + "converted to text; other binary types are rejected. URLs that resolve " + "to private or internal network addresses are refused, as are responses " + "over the tool's size limit. Optionally provide 'start_line' and " + "'line_count' to read only part of the content." + ) + args_schema: type[BaseModel] = URLReadToolSchema + max_bytes: int = _DEFAULT_MAX_BYTES + timeout: float = _DEFAULT_TIMEOUT + headers: dict[str, str] | None = None + encoding: str | None = None + + def __init__( + self, + max_bytes: int = _DEFAULT_MAX_BYTES, + timeout: float = _DEFAULT_TIMEOUT, + headers: dict[str, str] | None = None, + encoding: str | None = None, + **kwargs: Any, + ) -> None: + """Initialize the URLReadTool. + + Args: + max_bytes: Largest response body to accept, in decoded bytes. + timeout: Per-request timeout in seconds. + headers: Extra request headers. + encoding: Force a text encoding instead of the server's charset. + **kwargs: Additional keyword arguments passed to BaseTool. + """ + super().__init__(**kwargs) + self.max_bytes = max_bytes + self.timeout = timeout + self.headers = headers + self.encoding = encoding + + def _request_headers(self) -> dict[str, str]: + """Return the headers to send, with caller headers taking precedence.""" + return { + "Accept": "*/*", + "User-Agent": "Mozilla/5.0 (compatible; crewai-tools URLReadTool)", + **(self.headers or {}), + } + + @staticmethod + def _classify(media_type: str) -> str | None: + """Map a media type onto the extractor that handles it.""" + if media_type == _PDF_TYPE: + return "pdf" + if media_type == _DOCX_TYPE: + return "docx" + if media_type in _HTML_TYPES: + return "html" + if ( + media_type.startswith("text/") + or media_type in _TEXT_TYPES + or media_type.endswith(_TEXT_TYPE_SUFFIXES) + ): + return "text" + return None + + def _resolve_kind(self, content_type: str, *urls: str) -> str | None: + """Decide how to extract text, by content type then by URL extension. + + Args: + content_type: The raw Content-Type header value. + *urls: URLs to consult for an extension, most authoritative first. + A ``.pdf`` link that redirects to an extensionless CDN or + presigned path only carries its type on the requested URL, so + both ends of the chain are worth checking. + + Returns: + The extractor name, or None when the content type is unsupported. + """ + declared = content_type.split(";", 1)[0].strip().lower() + if declared not in _UNINFORMATIVE_TYPES: + return self._classify(declared) + + for url in urls: + path = urlparse(url).path.lower() + for extension, media_type in _EXTENSION_TYPES.items(): + if path.endswith(extension): + return self._classify(media_type) + return None + + def _decode(self, body: bytes, content_type: str) -> str: + """Decode *body* using the configured, declared, or default encoding. + + Falls back to a replacing UTF-8 decode rather than failing: partially + readable text is more useful to an agent than an error. + """ + encoding = self.encoding or _charset_from_content_type(content_type) or "utf-8" + try: + return body.decode(encoding) + except (LookupError, UnicodeDecodeError): + return body.decode("utf-8", errors="replace") + + @staticmethod + def _extract_pdf(body: bytes) -> str: + """Extract text from PDF bytes, page by page.""" + try: + import pymupdf # type: ignore[import-untyped] + except ImportError as e: + raise ImportError( + "Reading PDF URLs requires pymupdf. Install with: uv add pymupdf" + ) from e + + # Opened from memory: the bytes are already in hand, and a temp file + # would need cleaning up on every error path. + document = pymupdf.open(stream=body, filetype="pdf") + try: + pages = [ + f"Page {number}:\n{text}" + for number, page in enumerate(document, 1) + if (text := page.get_text().strip()) + ] + finally: + document.close() + + if not pages: + return "[PDF with no extractable text]" + return "\n\n".join(pages) + + @staticmethod + def _extract_docx(body: bytes) -> str: + """Extract paragraph text from DOCX bytes.""" + try: + from docx import Document + except ImportError as e: + raise ImportError( + "Reading DOCX URLs requires python-docx. Install with: " + "uv add python-docx" + ) from e + + document = Document(BytesIO(body)) + return "\n".join( + paragraph.text + for paragraph in document.paragraphs + if paragraph.text.strip() + ) + + def _extract_html(self, body: bytes, content_type: str) -> str: + """Strip HTML bytes down to visible text.""" + try: + from bs4 import BeautifulSoup + except ImportError as e: + raise ImportError( + "Reading HTML URLs requires beautifulsoup4. Install with: " + "uv add beautifulsoup4" + ) from e + + soup = BeautifulSoup(self._decode(body, content_type), "html.parser") + for element in soup(["script", "style"]): + element.decompose() + + text = _SPACES_PATTERN.sub(" ", soup.get_text(" ")) + return _NEWLINE_PATTERN.sub("\n", text).strip() + + def _extract(self, body: bytes, kind: str, content_type: str) -> str: + """Dispatch to the extractor named by *kind*.""" + if kind == "pdf": + return self._extract_pdf(body) + if kind == "docx": + return self._extract_docx(body) + if kind == "html": + return self._extract_html(body, content_type) + return self._decode(body, content_type) + + @staticmethod + def _window(text: str, start_line: int, line_count: int | None) -> str: + """Return the requested line window of *text*. + + The whole body has already been fetched by this point, so unlike the + filesystem equivalent this only trims output -- it saves no transfer. + + The bounds are clamped rather than trusted: the args schema rejects + anything below 1 before it gets here, but islice raises on a negative + stop index, and this runs outside the caller's error handling. + """ + if start_line == 1 and line_count is None: + return text + + start_index = max(start_line - 1, 0) + stop_index = None if line_count is None else start_index + max(line_count, 0) + selected = list(islice(text.splitlines(keepends=True), start_index, stop_index)) + + if not selected and start_index > 0: + return ( + f"Error: Start line {start_line} exceeds the number of lines in " + f"the content." + ) + return "".join(selected) + + def _run( + self, + url: str, + start_line: int | None = 1, + line_count: int | None = None, + ) -> str: + """Fetch a URL and return its content, or a window of it, as text.""" + start_line = start_line or 1 + line_count = line_count or None + + try: + body, content_type, final_url = safe_get_bounded( + url, + max_bytes=self.max_bytes, + timeout=self.timeout, + headers=self._request_headers(), + ) + except ValueError as e: + return f"Error: {e}" + except requests.RequestException as e: + return f"Error: Failed to fetch '{url}'. {format_error_for_display(e)}" + + kind = self._resolve_kind(content_type, final_url, url) + if kind is None: + return ( + f"Error: Unsupported content type " + f"'{content_type.split(';', 1)[0].strip() or 'unknown'}' at " + f"'{url}'. This tool reads text, HTML, JSON, XML, CSV, PDF and " + f"DOCX responses." + ) + + try: + text = self._extract(body, kind, content_type) + except ImportError as e: + return f"Error: {e}" + except Exception as e: + return ( + f"Error: Failed to read {kind.upper()} content from '{url}'. " + f"{format_error_for_display(e)}" + ) + + return self._window(text, start_line, line_count) diff --git a/lib/crewai-tools/tests/rag/test_pdf_loader.py b/lib/crewai-tools/tests/rag/test_pdf_loader.py new file mode 100644 index 000000000..9bae12151 --- /dev/null +++ b/lib/crewai-tools/tests/rag/test_pdf_loader.py @@ -0,0 +1,156 @@ +import tempfile +from unittest.mock import patch + +from crewai_tools.rag.base_loader import LoaderResult +from crewai_tools.rag.loaders.pdf_loader import PDFLoader +from crewai_tools.rag.source_content import SourceContent +import pytest + + +pymupdf = pytest.importorskip("pymupdf") + +# Patched at the loader's seam rather than at requests.get: safe_get_bounded +# resolves the hostname before issuing a request, which would make these tests +# depend on DNS for example.com. +FETCH = "crewai_tools.rag.loaders.pdf_loader.safe_get_bounded" + + +def build_pdf(text: str = "Quarterly revenue was 42") -> bytes: + """Return the bytes of a one-page PDF containing *text*.""" + document = pymupdf.open() + document.new_page().insert_text((72, 72), text) + try: + return document.tobytes() + finally: + document.close() + + +def fetch_result(body: bytes, url: str = "https://example.com/report.pdf"): + """Build the (body, content_type, final_url) tuple safe_get_bounded returns.""" + return body, "application/pdf", url + + +class TestPDFLoader: + def test_load_pdf_from_file(self): + """A PDF on disk has its text extracted with page markers.""" + with tempfile.NamedTemporaryFile(suffix=".pdf") as f: + f.write(build_pdf()) + f.flush() + + result = PDFLoader().load(SourceContent(f.name)) + + assert isinstance(result, LoaderResult) + assert "Page 1:" in result.content + assert "Quarterly revenue was 42" in result.content + assert result.metadata["num_pages"] == 1 + assert result.metadata["file_type"] == "pdf" + + def test_load_pdf_from_url(self): + """A PDF fetched from a URL is extracted and attributed to that URL.""" + with patch(FETCH) as fetch: + fetch.return_value = fetch_result(build_pdf("Content from URL")) + result = PDFLoader().load(SourceContent("https://example.com/report.pdf")) + + assert "Content from URL" in result.content + assert result.source == "https://example.com/report.pdf" + assert result.metadata["file_name"] == "report.pdf" + + headers = fetch.call_args.kwargs["headers"] + assert headers["Accept"] == "application/pdf" + assert "crewai-tools PDFLoader" in headers["User-Agent"] + + def test_load_pdf_from_url_leaves_no_temp_file(self): + """The URL path must not write a temp file it never cleans up. + + It previously used NamedTemporaryFile(delete=False) without unlinking, + so every PDF ingested from a URL left a file behind. + """ + with ( + patch(FETCH) as fetch, + patch("tempfile.NamedTemporaryFile") as mock_tempfile, + ): + fetch.return_value = fetch_result(build_pdf()) + PDFLoader().load(SourceContent("https://example.com/report.pdf")) + + mock_tempfile.assert_not_called() + + def test_load_pdf_from_url_is_size_bounded(self): + """The download is capped, since the body is held in memory.""" + with patch(FETCH) as fetch: + fetch.return_value = fetch_result(build_pdf()) + PDFLoader().load(SourceContent("https://example.com/report.pdf")) + + assert fetch.call_args.kwargs["max_bytes"] == 50 * 1024 * 1024 + + def test_load_pdf_from_url_accepts_a_custom_size_limit(self): + """Callers can lower or raise the ceiling per load.""" + with patch(FETCH) as fetch: + fetch.return_value = fetch_result(build_pdf()) + PDFLoader().load( + SourceContent("https://example.com/report.pdf"), max_bytes=1024 + ) + + assert fetch.call_args.kwargs["max_bytes"] == 1024 + + def test_load_pdf_from_url_with_custom_headers(self): + """Caller-supplied headers replace the loader's defaults.""" + custom_headers = {"Authorization": "Bearer token"} + + with patch(FETCH) as fetch: + fetch.return_value = fetch_result(build_pdf()) + PDFLoader().load( + SourceContent("https://example.com/report.pdf"), headers=custom_headers + ) + + assert fetch.call_args.kwargs["headers"] == custom_headers + + def test_load_pdf_url_download_error(self): + """A failed download surfaces as a ValueError naming the URL.""" + with patch(FETCH, side_effect=Exception("Network error")): + with pytest.raises(ValueError, match="Failed to download PDF"): + PDFLoader().load(SourceContent("https://example.com/report.pdf")) + + def test_load_pdf_url_over_size_limit(self): + """An oversized body is reported rather than partially parsed.""" + with patch(FETCH, side_effect=ValueError("exceeds the 1024 byte limit")): + with pytest.raises(ValueError, match="Failed to download PDF"): + PDFLoader().load(SourceContent("https://example.com/huge.pdf")) + + def test_load_pdf_missing_file(self): + """A missing local path raises FileNotFoundError, not ValueError.""" + with pytest.raises(FileNotFoundError, match="PDF file not found"): + PDFLoader().load(SourceContent("/nonexistent/report.pdf")) + + def test_load_corrupt_pdf_raises_value_error(self): + """Bytes that are not a parseable PDF produce a read error.""" + with tempfile.NamedTemporaryFile(suffix=".pdf") as f: + f.write(b"%PDF-1.4 not really a pdf") + f.flush() + + with pytest.raises(ValueError, match="Error reading PDF"): + PDFLoader().load(SourceContent(f.name)) + + def test_pdf_with_no_extractable_text(self): + """A PDF whose pages hold no text says so instead of returning empty.""" + document = pymupdf.open() + document.new_page() + blank = document.tobytes() + document.close() + + with tempfile.NamedTemporaryFile(suffix=".pdf") as f: + f.write(blank) + f.flush() + + result = PDFLoader().load(SourceContent(f.name)) + + assert "no extractable text" in result.content + + def test_pdf_doc_id_is_stable(self): + """The same source yields the same doc_id across loads.""" + with tempfile.NamedTemporaryFile(suffix=".pdf") as f: + f.write(build_pdf()) + f.flush() + + loader = PDFLoader() + source = SourceContent(f.name) + assert loader.load(source).doc_id == loader.load(source).doc_id diff --git a/lib/crewai-tools/tests/url_read_tool_test.py b/lib/crewai-tools/tests/url_read_tool_test.py new file mode 100644 index 000000000..9d48beb80 --- /dev/null +++ b/lib/crewai-tools/tests/url_read_tool_test.py @@ -0,0 +1,415 @@ +from unittest.mock import patch + +import pytest +import requests + +from crewai_tools import URLReadTool +from crewai_tools.security.safe_requests import safe_get_bounded + + +TOOL_MODULE = "crewai_tools.tools.url_read_tool.url_read_tool" + + +class FakeResponse: + """Minimal stand-in for a streamed requests.Response.""" + + def __init__( + self, + body: bytes = b"", + content_type: str = "text/plain", + url: str = "https://example.com/file.txt", + status_code: int = 200, + chunk_size: int | None = None, + ): + self._body = body + self._chunk_size = chunk_size + self.headers = {"Content-Type": content_type} if content_type else {} + self.url = url + self.status_code = status_code + self.history: list["FakeResponse"] = [] + self.closed = False + + def raise_for_status(self) -> None: + """Mimic requests' error-status behavior.""" + if self.status_code >= 400: + raise requests.HTTPError(f"{self.status_code} error") + + def iter_content(self, chunk_size: int = 65536): + """Yield the body in chunks, like a streamed response.""" + size = self._chunk_size or chunk_size + for index in range(0, len(self._body), size): + yield self._body[index : index + size] + + def close(self) -> None: + """Record that the response was closed.""" + self.closed = True + + +def build_pdf(text: str = "Quarterly revenue was 42") -> bytes: + """Return the bytes of a one-page PDF containing *text*.""" + pymupdf = pytest.importorskip("pymupdf") + document = pymupdf.open() + document.new_page().insert_text((72, 72), text) + try: + return document.tobytes() + finally: + document.close() + + +def fetch_result( + body: bytes, + content_type: str = "text/plain", + url: str = "https://example.com/f.txt", +): + """Build the (body, content_type, final_url) tuple safe_get_bounded returns.""" + return body, content_type, url + + +def test_reads_plain_text(): + """A text response is returned as-is, with the configured limits applied.""" + tool = URLReadTool() + with patch(f"{TOOL_MODULE}.safe_get_bounded") as fetch: + fetch.return_value = fetch_result(b"hello world") + assert tool.run(url="https://example.com/f.txt") == "hello world" + + assert fetch.call_args.kwargs["max_bytes"] == 5 * 1024 * 1024 + assert fetch.call_args.kwargs["timeout"] == 30 + + +def test_honors_declared_charset(): + """The charset in the Content-Type header drives decoding.""" + tool = URLReadTool() + with patch(f"{TOOL_MODULE}.safe_get_bounded") as fetch: + fetch.return_value = fetch_result( + "café".encode("latin-1"), "text/plain; charset=iso-8859-1" + ) + assert tool.run(url="https://example.com/f.txt") == "café" + + +def test_encoding_override_wins_over_server_charset(): + """An explicit encoding beats whatever the server declares.""" + tool = URLReadTool(encoding="latin-1") + with patch(f"{TOOL_MODULE}.safe_get_bounded") as fetch: + fetch.return_value = fetch_result( + "café".encode("latin-1"), "text/plain; charset=utf-8" + ) + assert tool.run(url="https://example.com/f.txt") == "café" + + +def test_undecodable_bytes_fall_back_instead_of_failing(): + """Partially readable text beats an error for the agent.""" + tool = URLReadTool() + with patch(f"{TOOL_MODULE}.safe_get_bounded") as fetch: + fetch.return_value = fetch_result(b"\xff\xfe bad bytes", "text/plain") + result = tool.run(url="https://example.com/f.txt") + + assert "bad bytes" in result + assert not result.startswith("Error:") + + +def test_line_window(): + """start_line and line_count select a window of the extracted text.""" + tool = URLReadTool() + body = b"one\ntwo\nthree\nfour\nfive\n" + with patch(f"{TOOL_MODULE}.safe_get_bounded") as fetch: + fetch.return_value = fetch_result(body) + result = tool.run(url="https://example.com/f.txt", start_line=2, line_count=2) + + assert result == "two\nthree\n" + + +def test_start_line_past_end_reports_error(): + """Asking past the end of the content is reported, not silently empty.""" + tool = URLReadTool() + with patch(f"{TOOL_MODULE}.safe_get_bounded") as fetch: + fetch.return_value = fetch_result(b"one\ntwo\n") + result = tool.run(url="https://example.com/f.txt", start_line=99) + + assert "exceeds the number of lines" in result + + +@pytest.mark.parametrize( + "line_args", + [{"line_count": -5}, {"line_count": 0}, {"start_line": 0}, {"start_line": -5}], +) +def test_line_arguments_below_one_are_refused(line_args): + """Out-of-range line arguments are rejected before any request is made. + + islice raises on a negative stop index, and the windowing runs outside the + tool's error handling, so these have to be refused at validation time. + """ + tool = URLReadTool() + with patch(f"{TOOL_MODULE}.safe_get_bounded") as fetch: + with pytest.raises(ValueError, match="greater than or equal to 1"): + tool.run(url="https://example.com/f.txt", **line_args) + + fetch.assert_not_called() + + +def test_json_is_returned_verbatim(): + """JSON is passed through undecorated so callers can parse it.""" + tool = URLReadTool() + with patch(f"{TOOL_MODULE}.safe_get_bounded") as fetch: + fetch.return_value = fetch_result(b'{"a": 1}', "application/json") + assert tool.run(url="https://example.com/data.json") == '{"a": 1}' + + +def test_structured_suffix_type_is_treated_as_text(): + """A +json vendor type is text, not an unsupported binary type.""" + tool = URLReadTool() + with patch(f"{TOOL_MODULE}.safe_get_bounded") as fetch: + fetch.return_value = fetch_result(b'{"a": 1}', "application/vnd.api+json") + assert tool.run(url="https://example.com/data") == '{"a": 1}' + + +def test_html_is_stripped_to_visible_text(): + """HTML returns visible text with script and style content removed.""" + tool = URLReadTool() + body = b"

Hi

" + with patch(f"{TOOL_MODULE}.safe_get_bounded") as fetch: + fetch.return_value = fetch_result(body, "text/html; charset=utf-8") + result = tool.run(url="https://example.com/page") + + assert "Hi" in result + assert "x=1" not in result + assert "color:red" not in result + + +def test_binary_content_type_is_rejected(): + """An unsupported type is refused rather than returned as base64.""" + tool = URLReadTool() + with patch(f"{TOOL_MODULE}.safe_get_bounded") as fetch: + fetch.return_value = fetch_result(b"\x89PNG\r\n", "image/png") + result = tool.run(url="https://example.com/logo.png") + + assert "Unsupported content type 'image/png'" in result + + +def test_octet_stream_pdf_falls_back_to_url_extension(): + """A PDF served as octet-stream is still extracted, via its extension.""" + tool = URLReadTool() + with patch(f"{TOOL_MODULE}.safe_get_bounded") as fetch: + fetch.return_value = fetch_result( + build_pdf("Fallback worked"), + "application/octet-stream", + "https://example.com/a/b.pdf", + ) + result = tool.run(url="https://example.com/a/b.pdf") + + assert "Fallback worked" in result + + +def test_missing_content_type_falls_back_to_url_extension(): + """No Content-Type at all still reads as text when the path says .csv.""" + tool = URLReadTool() + with patch(f"{TOOL_MODULE}.safe_get_bounded") as fetch: + fetch.return_value = fetch_result(b"a,b\n1,2\n", "", "https://example.com/b.csv") + assert tool.run(url="https://example.com/b.csv") == "a,b\n1,2\n" + + +def test_query_string_does_not_break_extension_fallback(): + """A presigned-style query string does not hide the path's extension.""" + tool = URLReadTool() + url = "https://example.com/b.csv?X-Amz-Signature=abc" + with patch(f"{TOOL_MODULE}.safe_get_bounded") as fetch: + fetch.return_value = fetch_result(b"a,b\n", "application/octet-stream", url) + assert tool.run(url=url) == "a,b\n" + + +def test_extension_from_requested_url_survives_a_redirect(): + """A .pdf link that redirects to an extensionless path is still extracted. + + Presigned CDN targets routinely drop the extension and serve octet-stream, + so the requested URL is the only place the type survives. + """ + tool = URLReadTool() + with patch(f"{TOOL_MODULE}.safe_get_bounded") as fetch: + fetch.return_value = fetch_result( + build_pdf("Survived the redirect"), + "application/octet-stream", + "https://cdn.example.com/objects/9f8a7b6c5d", + ) + result = tool.run(url="https://example.com/report.pdf") + + assert "Survived the redirect" in result + + +def test_octet_stream_with_unknown_extension_is_rejected(): + """With neither a usable type nor a known extension, the read is refused.""" + tool = URLReadTool() + with patch(f"{TOOL_MODULE}.safe_get_bounded") as fetch: + fetch.return_value = fetch_result( + b"\x00\x01", "application/octet-stream", "https://example.com/a/b.bin" + ) + result = tool.run(url="https://example.com/a/b.bin") + + assert "Unsupported content type" in result + + +def test_validation_failure_is_returned_as_error(): + """An SSRF rejection reaches the agent as an error string, not an exception.""" + tool = URLReadTool() + with patch(f"{TOOL_MODULE}.safe_get_bounded") as fetch: + fetch.side_effect = ValueError( + "URL 'http://169.254.169.254/' resolves to private/reserved IP 169.254.169.254." + ) + result = tool.run(url="http://169.254.169.254/") + + assert result.startswith("Error:") + assert "private/reserved IP" in result + + +def test_request_failure_is_returned_as_error(): + """A transport failure is reported without raising out of the tool.""" + tool = URLReadTool() + with patch(f"{TOOL_MODULE}.safe_get_bounded") as fetch: + fetch.side_effect = requests.ConnectionError("connection refused") + result = tool.run(url="https://example.com/f.txt") + + assert result.startswith("Error: Failed to fetch") + + +def test_custom_headers_are_merged_over_defaults(): + """Caller headers win, but the default User-Agent survives.""" + tool = URLReadTool(headers={"Authorization": "Bearer x", "Accept": "text/plain"}) + with patch(f"{TOOL_MODULE}.safe_get_bounded") as fetch: + fetch.return_value = fetch_result(b"ok") + tool.run(url="https://example.com/f.txt") + + headers = fetch.call_args.kwargs["headers"] + assert headers["Authorization"] == "Bearer x" + assert headers["Accept"] == "text/plain" + assert "crewai-tools URLReadTool" in headers["User-Agent"] + + +def test_reads_a_real_pdf_end_to_end(): + """Real PDF bytes are extracted page by page.""" + pdf_bytes = build_pdf() + + tool = URLReadTool() + with patch(f"{TOOL_MODULE}.safe_get_bounded") as fetch: + fetch.return_value = fetch_result( + pdf_bytes, "application/pdf", "https://example.com/report.pdf" + ) + result = tool.run(url="https://example.com/report.pdf") + + assert "Page 1:" in result + assert "Quarterly revenue was 42" in result + + +def test_corrupt_pdf_reports_error_without_raising(): + """A malformed PDF becomes an error string, not a traceback.""" + pytest.importorskip("pymupdf") + + tool = URLReadTool() + with patch(f"{TOOL_MODULE}.safe_get_bounded") as fetch: + fetch.return_value = fetch_result( + b"%PDF-1.4 not really a pdf", "application/pdf" + ) + result = tool.run(url="https://example.com/report.pdf") + + assert result.startswith("Error: Failed to read PDF content") + + +class TestSafeGetBounded: + """Tests for the bounded-fetch helper itself.""" + + def test_returns_body_content_type_and_final_url(self): + """The helper reports the body alongside where it ended up.""" + response = FakeResponse(b"payload", "text/plain", "https://example.com/final") + with patch( + "crewai_tools.security.safe_requests.safe_get", return_value=response + ): + body, content_type, final_url = safe_get_bounded( + "https://example.com/start", max_bytes=1024 + ) + + assert body == b"payload" + assert content_type == "text/plain" + assert final_url == "https://example.com/final" + assert response.closed + + def test_rejects_body_over_the_limit(self): + """Crossing max_bytes raises rather than truncating silently.""" + response = FakeResponse(b"x" * 100, chunk_size=10) + with patch( + "crewai_tools.security.safe_requests.safe_get", return_value=response + ): + with pytest.raises(ValueError, match="exceeds the 25 byte limit"): + safe_get_bounded("https://example.com/big", max_bytes=25) + + assert response.closed + + def test_oversized_error_names_the_url_that_served_the_body(self): + """After a redirect the requested URL is not the one that sent it.""" + response = FakeResponse( + b"x" * 100, url="https://cdn.example.com/final", chunk_size=10 + ) + with patch( + "crewai_tools.security.safe_requests.safe_get", return_value=response + ): + with pytest.raises(ValueError, match="https://cdn.example.com/final"): + safe_get_bounded("https://example.com/start", max_bytes=25) + + @pytest.mark.parametrize("max_bytes", [0, -1]) + def test_non_positive_max_bytes_fails_before_requesting(self, max_bytes): + """A misconfigured cap is caught without issuing a request.""" + with patch("crewai_tools.security.safe_requests.safe_get") as safe_get: + with pytest.raises(ValueError, match="max_bytes must be positive"): + safe_get_bounded("https://example.com/f", max_bytes=max_bytes) + + safe_get.assert_not_called() + + def test_stops_reading_once_the_limit_is_crossed(self): + """The cap must abandon the stream, not buffer the whole body first.""" + chunks_yielded = 0 + + class CountingResponse(FakeResponse): + def iter_content(self, chunk_size: int = 65536): + nonlocal chunks_yielded + for _ in range(1000): + chunks_yielded += 1 + yield b"x" * 10 + + response = CountingResponse() + with patch( + "crewai_tools.security.safe_requests.safe_get", return_value=response + ): + with pytest.raises(ValueError): + safe_get_bounded("https://example.com/huge", max_bytes=25) + + assert chunks_yielded == 3 + + def test_error_status_raises(self): + """An error status propagates as an HTTPError.""" + response = FakeResponse(b"nope", status_code=404) + with patch( + "crewai_tools.security.safe_requests.safe_get", return_value=response + ): + with pytest.raises(requests.HTTPError): + safe_get_bounded("https://example.com/missing", max_bytes=1024) + + assert response.closed + + def test_closes_redirect_hops(self): + """Streamed redirect hops hold connections until closed.""" + hop = FakeResponse(b"", status_code=302) + response = FakeResponse(b"done") + response.history = [hop] + with patch( + "crewai_tools.security.safe_requests.safe_get", return_value=response + ): + safe_get_bounded("https://example.com/start", max_bytes=1024) + + assert hop.closed + assert response.closed + + def test_requests_are_streamed(self): + """Streaming is what lets an oversized body be abandoned early.""" + response = FakeResponse(b"ok") + with patch( + "crewai_tools.security.safe_requests.safe_get", return_value=response + ) as safe_get: + safe_get_bounded("https://example.com/f", max_bytes=1024) + + assert safe_get.call_args.kwargs["stream"] is True diff --git a/lib/crewai-tools/tests/utilities/test_safe_requests.py b/lib/crewai-tools/tests/utilities/test_safe_requests.py index f45dd86c6..895c0044c 100644 --- a/lib/crewai-tools/tests/utilities/test_safe_requests.py +++ b/lib/crewai-tools/tests/utilities/test_safe_requests.py @@ -112,6 +112,80 @@ def test_safe_get_fails_closed_after_too_many_redirects( safe_get("http://public.example/start", max_redirects=1, timeout=15) +def _closable_response( + url: str, status_code: int, *, location: str | None = None, closed: list[str] +) -> requests.Response: + """Build a response that records its own URL when closed.""" + response = _response(url, status_code, location=location) + response.close = lambda: closed.append(url) # type: ignore[method-assign] + return response + + +def test_safe_get_closes_earlier_hops_after_too_many_redirects( + monkeypatch: pytest.MonkeyPatch, public_dns: None +) -> None: + """Hops accumulated before the failure must not be left open. + + Under stream=True each hop holds its connection until its body is read or + closed, and a caller handed an exception has no handle on them. + """ + closed: list[str] = [] + + def fake_get(url: str, **kwargs: Any) -> requests.Response: + return _closable_response( + url, 302, location="http://safe.example/again", closed=closed + ) + + _mock_get(monkeypatch, fake_get) + + with pytest.raises(ValueError, match="Too many redirects"): + safe_get("http://public.example/start", max_redirects=2, timeout=15, stream=True) + + assert len(closed) == 3 + + +def test_safe_get_closes_earlier_hops_when_a_redirect_is_rejected( + monkeypatch: pytest.MonkeyPatch, public_dns: None +) -> None: + """A hop rejected mid-chain still releases the connections already open.""" + closed: list[str] = [] + + def fake_get(url: str, **kwargs: Any) -> requests.Response: + if url == "http://public.example/start": + return _closable_response( + url, 302, location="http://safe.example/next", closed=closed + ) + return _closable_response( + url, 302, location="http://169.254.169.254/latest", closed=closed + ) + + _mock_get(monkeypatch, fake_get) + + with pytest.raises(ValueError, match="private/reserved IP"): + safe_get("http://public.example/start", timeout=15, stream=True) + + assert closed == ["http://safe.example/next", "http://public.example/start"] + + +def test_safe_get_leaves_hops_open_on_success( + monkeypatch: pytest.MonkeyPatch, public_dns: None +) -> None: + """On success the hops belong to the caller, via response.history.""" + closed: list[str] = [] + + def fake_get(url: str, **kwargs: Any) -> requests.Response: + if url == "http://public.example/start": + return _closable_response(url, 302, location="/final", closed=closed) + return _closable_response(url, 200, closed=closed) + + _mock_get(monkeypatch, fake_get) + + response = safe_get("http://public.example/start", timeout=15, stream=True) + + assert closed == [] + assert len(response.history) == 1 + + def test_safe_get_strips_credentials_on_cross_origin_redirect( monkeypatch: pytest.MonkeyPatch, public_dns: None ) -> None: diff --git a/lib/crewai-tools/tool.specs.json b/lib/crewai-tools/tool.specs.json index a03513308..6380dc7c6 100644 --- a/lib/crewai-tools/tool.specs.json +++ b/lib/crewai-tools/tool.specs.json @@ -26885,6 +26885,148 @@ "type": "object" } }, + { + "description": "A tool that reads the content at a URL and returns it as text. To use this tool, provide a 'url' parameter with an http:// or https:// address. PDF, DOCX, HTML, JSON, XML, CSV and plain-text responses are converted to text; other binary types are rejected. URLs that resolve to private or internal network addresses are refused, as are responses over the tool's size limit. Optionally provide 'start_line' and 'line_count' to read only part of the content.", + "env_vars": [], + "humanized_name": "Read content from a URL", + "init_params_schema": { + "$defs": { + "EnvVar": { + "properties": { + "default": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Default" + }, + "description": { + "title": "Description", + "type": "string" + }, + "name": { + "title": "Name", + "type": "string" + }, + "required": { + "default": true, + "title": "Required", + "type": "boolean" + } + }, + "required": [ + "name", + "description" + ], + "title": "EnvVar", + "type": "object" + }, + "ToolFailurePolicy": { + "description": "How an agent reacts when one of its tools reports a failure.", + "enum": [ + "ignore", + "warn", + "raise" + ], + "title": "ToolFailurePolicy", + "type": "string" + } + }, + "description": "Read the content at an arbitrary URL and return it as text.\n\nUnlike :class:`~crewai_tools.tools.file_read_tool.file_read_tool.FileReadTool`,\nwhich is confined to the local filesystem, this tool performs network\nrequests to addresses the caller -- often an LLM -- chooses at runtime. It\nis a separate tool for exactly that reason: granting it is granting network\negress, and that should be a deliberate choice rather than a flag on a\nfilesystem tool.\n\nResponses are decoded to text according to their content type. PDF and DOCX\nbodies have their text extracted, HTML is stripped to visible text, and\ntext-shaped types (plain text, Markdown, JSON, XML, YAML, CSV) are decoded\nas-is. Any other type is refused rather than returned as base64, keeping\nthis tool's output text-only.\n\nSecurity:\n Requests go through :func:`~crewai_tools.security.safe_requests.safe_get_bounded`,\n which resolves each hostname and rejects it when any resolved address is\n private, loopback, link-local, or otherwise reserved -- covering cloud\n metadata endpoints and internal services. Redirects are never followed\n automatically: every hop is revalidated, and credentials are dropped on\n cross-origin hops. Bodies over ``max_bytes`` are abandoned mid-stream.\n\n Two risks are not closed here. Validation resolves the hostname and\n requests resolves it again when connecting, so a DNS entry that changes\n between those lookups can still redirect the connection (DNS\n rebinding); closing that requires pinning the connection to the\n validated address. And the returned text is untrusted remote content\n flowing into an agent's context -- a fetched page can attempt to\n instruct the agent. Neither is addressable by input validation alone;\n network egress policy and prompt-level handling cover them.\n\nArgs:\n max_bytes (int): Largest response body to accept, in decoded bytes.\n Defaults to 5 MiB.\n timeout (float): Per-request timeout in seconds. Defaults to 30.\n headers (Optional[dict[str, str]]): Extra request headers. Developer\n supplied, not chosen by the model.\n encoding (Optional[str]): Force a text encoding instead of honoring the\n charset the server declares.\n **kwargs: Additional keyword arguments passed to BaseTool.\n\nExample:\n >>> tool = URLReadTool()\n >>> content = tool.run(url=\"https://example.com/report.pdf\")\n >>> head = tool.run(url=\"https://example.com/data.csv\", line_count=20)", + "properties": { + "encoding": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Encoding" + }, + "headers": { + "anyOf": [ + { + "additionalProperties": { + "type": "string" + }, + "type": "object" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Headers" + }, + "max_bytes": { + "default": 5242880, + "title": "Max Bytes", + "type": "integer" + }, + "timeout": { + "default": 30, + "title": "Timeout", + "type": "number" + } + }, + "required": [], + "title": "URLReadTool", + "type": "object" + }, + "name": "URLReadTool", + "package_dependencies": [], + "run_params_schema": { + "description": "Input for URLReadTool.", + "properties": { + "line_count": { + "anyOf": [ + { + "minimum": 1, + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Number of lines to read. If None, reads the entire content", + "title": "Line Count" + }, + "start_line": { + "anyOf": [ + { + "minimum": 1, + "type": "integer" + }, + { + "type": "null" + } + ], + "default": 1, + "description": "Line number to start reading from (1-indexed)", + "title": "Start Line" + }, + "url": { + "description": "The http:// or https:// URL to read. Addresses that resolve to private or internal networks are refused.", + "title": "Url", + "type": "string" + } + }, + "required": [ + "url" + ], + "title": "URLReadToolSchema", + "type": "object" + } + }, { "description": "This tool uses OpenAI's Vision API to describe the contents of an image.", "env_vars": [