mirror of
https://github.com/crewAIInc/crewAI.git
synced 2026-07-24 00:05:08 +00:00
5d4851eac7
Some checks failed
CodeQL Advanced / Analyze (actions) (push) Has been cancelled
CodeQL Advanced / Analyze (python) (push) Has been cancelled
Check Documentation Broken Links / Check broken links (push) Has been cancelled
Vulnerability Scan / pip-audit (push) Has been cancelled
Nightly Canary Release / Check for new commits (push) Has been cancelled
Nightly Canary Release / Build nightly packages (push) Has been cancelled
Nightly Canary Release / Publish nightly to PyPI (push) Has been cancelled
* Validate redirects for scraping URL fetches * Prevent credential forwarding across redirects
42 lines
1.1 KiB
Python
42 lines
1.1 KiB
Python
"""Utility functions for RAG loaders."""
|
|
|
|
from typing import Any
|
|
|
|
|
|
def load_from_url(
|
|
url: str,
|
|
kwargs: dict[str, Any],
|
|
accept_header: str = "*/*",
|
|
loader_name: str = "Loader",
|
|
) -> str:
|
|
"""Load content from a URL.
|
|
|
|
Args:
|
|
url: The URL to fetch content from
|
|
kwargs: Additional keyword arguments (can include 'headers' override)
|
|
accept_header: The Accept header value for the request
|
|
loader_name: The name of the loader for the User-Agent header
|
|
|
|
Returns:
|
|
The text content from the URL
|
|
|
|
Raises:
|
|
ValueError: If there's an error fetching the URL
|
|
"""
|
|
from crewai_tools.security.safe_requests import safe_get
|
|
|
|
headers = kwargs.get(
|
|
"headers",
|
|
{
|
|
"Accept": accept_header,
|
|
"User-Agent": f"Mozilla/5.0 (compatible; crewai-tools {loader_name})",
|
|
},
|
|
)
|
|
|
|
try:
|
|
response = safe_get(url, headers=headers, timeout=30)
|
|
response.raise_for_status()
|
|
return response.text
|
|
except Exception as e:
|
|
raise ValueError(f"Error fetching content from URL {url}: {e!s}") from e
|