optional deps for most

This commit is contained in:
Lorenze Jay
2025-01-10 13:51:39 -08:00
parent 06f99fc6cd
commit 40dcf63a70
14 changed files with 374 additions and 175 deletions

View File

@@ -11,12 +11,10 @@ class BrowserbaseLoadToolSchema(BaseModel):
class BrowserbaseLoadTool(BaseTool):
name: str = "Browserbase web load tool"
description: str = (
"Load webpages url in a headless browser using Browserbase and return the contents"
)
description: str = "Load webpages url in a headless browser using Browserbase and return the contents"
args_schema: Type[BaseModel] = BrowserbaseLoadToolSchema
api_key: Optional[str] = os.getenv('BROWSERBASE_API_KEY')
project_id: Optional[str] = os.getenv('BROWSERBASE_PROJECT_ID')
api_key: Optional[str] = os.getenv("BROWSERBASE_API_KEY")
project_id: Optional[str] = os.getenv("BROWSERBASE_PROJECT_ID")
text_content: Optional[bool] = False
session_id: Optional[str] = None
proxy: Optional[bool] = None
@@ -33,12 +31,23 @@ class BrowserbaseLoadTool(BaseTool):
):
super().__init__(**kwargs)
if not self.api_key:
raise EnvironmentError("BROWSERBASE_API_KEY environment variable is required for initialization")
raise EnvironmentError(
"BROWSERBASE_API_KEY environment variable is required for initialization"
)
try:
from browserbase import Browserbase # type: ignore
except ImportError:
import click
if click.confirm(
"`browserbase` package not found, would you like to install it?"
):
import subprocess
subprocess.run(["uv", "add", "browserbase"], check=True)
else:
raise ImportError(
"`browserbase` package not found, please run `pip install browserbase`"
"`browserbase` package not found, please run `uv add browserbase`"
)
self.browserbase = Browserbase(api_key=self.api_key)

View File

@@ -35,8 +35,20 @@ class FirecrawlCrawlWebsiteTool(BaseTool):
try:
from firecrawl import FirecrawlApp # type: ignore
except ImportError:
import click
if click.confirm(
"You are missing the 'firecrawl-py' package. Would you like to install it? (y/N)"
):
import subprocess
subprocess.run(["uv", "add", "firecrawl-py"], check=True)
from firecrawl import (
FirecrawlApp,
)
else:
raise ImportError(
"`firecrawl` package not found, please run `pip install firecrawl-py`"
"`firecrawl-py` package not found, please run `uv add firecrawl-py`"
)
if not self.firecrawl:

View File

@@ -31,8 +31,20 @@ class FirecrawlScrapeWebsiteTool(BaseTool):
try:
from firecrawl import FirecrawlApp # type: ignore
except ImportError:
import click
if click.confirm(
"You are missing the 'firecrawl-py' package. Would you like to install it? (y/N)"
):
import subprocess
subprocess.run(["uv", "add", "firecrawl-py"], check=True)
from firecrawl import (
FirecrawlApp,
)
else:
raise ImportError(
"`firecrawl` package not found, please run `pip install firecrawl-py`"
"`firecrawl-py` package not found, please run `uv add firecrawl-py`"
)
self.firecrawl = FirecrawlApp(api_key=api_key)

View File

@@ -41,8 +41,20 @@ class FirecrawlSearchTool(BaseTool):
try:
from firecrawl import FirecrawlApp # type: ignore
except ImportError:
import click
if click.confirm(
"You are missing the 'firecrawl-py' package. Would you like to install it? (y/N)"
):
import subprocess
subprocess.run(["uv", "add", "firecrawl-py"], check=True)
from firecrawl import (
FirecrawlApp,
)
else:
raise ImportError(
"`firecrawl` package not found, please run `pip install firecrawl-py`"
"`firecrawl-py` package not found, please run `uv add firecrawl-py`"
)
self.firecrawl = FirecrawlApp(api_key=api_key)

View File

@@ -1,7 +1,10 @@
from typing import Any
from crewai.tools import BaseTool
try:
from linkup import LinkupClient
LINKUP_AVAILABLE = True
except ImportError:
LINKUP_AVAILABLE = False
@@ -9,23 +12,42 @@ except ImportError:
from pydantic import PrivateAttr
class LinkupSearchTool:
class LinkupSearchTool(BaseTool):
name: str = "Linkup Search Tool"
description: str = "Performs an API call to Linkup to retrieve contextual information."
description: str = (
"Performs an API call to Linkup to retrieve contextual information."
)
_client: LinkupClient = PrivateAttr() # type: ignore
def __init__(self, api_key: str):
"""
Initialize the tool with an API key.
"""
if not LINKUP_AVAILABLE:
super().__init__()
try:
from linkup import LinkupClient
except ImportError:
import click
if click.confirm(
"You are missing the 'linkup-sdk' package. Would you like to install it? (y/N)"
):
import subprocess
subprocess.run(["uv", "add", "linkup-sdk"], check=True)
from linkup import LinkupClient
else:
raise ImportError(
"The 'linkup' package is required to use the LinkupSearchTool. "
"Please install it with: uv add linkup"
"The 'linkup-sdk' package is required to use the LinkupSearchTool. "
"Please install it with: uv add linkup-sdk"
)
self._client = LinkupClient(api_key=api_key)
def _run(self, query: str, depth: str = "standard", output_type: str = "searchResults") -> dict:
def _run(
self, query: str, depth: str = "standard", output_type: str = "searchResults"
) -> dict:
"""
Executes a search using the Linkup API.
@@ -36,9 +58,7 @@ class LinkupSearchTool:
"""
try:
response = self._client.search(
query=query,
depth=depth,
output_type=output_type
query=query, depth=depth, output_type=output_type
)
results = [
{"name": result.name, "url": result.url, "content": result.content}

View File

@@ -28,8 +28,17 @@ class MultiOnTool(BaseTool):
try:
from multion.client import MultiOn # type: ignore
except ImportError:
import click
if click.confirm(
"You are missing the 'multion' package. Would you like to install it? (y/N)"
):
import subprocess
subprocess.run(["uv", "add", "multion"], check=True)
else:
raise ImportError(
"`multion` package not found, please run `pip install multion`"
"`multion` package not found, please run `uv add multion`"
)
self.session_id = None
self.local = local

View File

@@ -1,7 +1,14 @@
from typing import Any, Type
from crewai.tools import BaseTool
from pydantic import BaseModel, Field
from patronus import Client
try:
from patronus import Client
PYPATRONUS_AVAILABLE = True
except ImportError:
PYPATRONUS_AVAILABLE = False
Client = Any
class FixedLocalEvaluatorToolSchema(BaseModel):
@@ -24,17 +31,22 @@ class PatronusLocalEvaluatorTool(BaseTool):
name: str = "Patronus Local Evaluator Tool"
evaluator: str = "The registered local evaluator"
evaluated_model_gold_answer: str = "The agent's gold answer"
description: str = (
"This tool is used to evaluate the model input and output using custom function evaluators."
)
description: str = "This tool is used to evaluate the model input and output using custom function evaluators."
client: Any = None
args_schema: Type[BaseModel] = FixedLocalEvaluatorToolSchema
class Config:
arbitrary_types_allowed = True
def __init__(self, patronus_client: Client, evaluator: str, evaluated_model_gold_answer: str, **kwargs: Any):
def __init__(
self,
patronus_client: Client,
evaluator: str,
evaluated_model_gold_answer: str,
**kwargs: Any,
):
super().__init__(**kwargs)
if PYPATRONUS_AVAILABLE:
self.client = patronus_client
if evaluator:
self.evaluator = evaluator
@@ -44,6 +56,19 @@ class PatronusLocalEvaluatorTool(BaseTool):
print(
f"Updating judge evaluator, gold_answer to: {self.evaluator}, {self.evaluated_model_gold_answer}"
)
else:
import click
if click.confirm(
"You are missing the 'patronus' package. Would you like to install it? (y/N)"
):
import subprocess
subprocess.run(["uv", "add", "patronus"], check=True)
else:
raise ImportError(
"You are missing the patronus package. Would you like to install it?"
)
def _run(
self,

View File

@@ -1,25 +1,30 @@
import os
from typing import Any, Optional, Type
from typing import Any, Optional, Type, TYPE_CHECKING
from urllib.parse import urlparse
from crewai.tools import BaseTool
from pydantic import BaseModel, Field, validator
from scrapegraph_py import Client
from scrapegraph_py.logger import sgai_logger
from pydantic import BaseModel, Field, validator, ConfigDict
# Type checking import
if TYPE_CHECKING:
from scrapegraph_py import Client
class ScrapegraphError(Exception):
"""Base exception for Scrapegraph-related errors"""
pass
class RateLimitError(ScrapegraphError):
"""Raised when API rate limits are exceeded"""
pass
class FixedScrapegraphScrapeToolSchema(BaseModel):
"""Input for ScrapegraphScrapeTool when website_url is fixed."""
pass
@@ -32,7 +37,7 @@ class ScrapegraphScrapeToolSchema(FixedScrapegraphScrapeToolSchema):
description="Prompt to guide the extraction of content",
)
@validator('website_url')
@validator("website_url")
def validate_url(cls, v):
"""Validate URL format"""
try:
@@ -41,7 +46,9 @@ class ScrapegraphScrapeToolSchema(FixedScrapegraphScrapeToolSchema):
raise ValueError
return v
except Exception:
raise ValueError("Invalid URL format. URL must include scheme (http/https) and domain")
raise ValueError(
"Invalid URL format. URL must include scheme (http/https) and domain"
)
class ScrapegraphScrapeTool(BaseTool):
@@ -54,12 +61,17 @@ class ScrapegraphScrapeTool(BaseTool):
RuntimeError: If scraping operation fails
"""
model_config = ConfigDict(arbitrary_types_allowed=True)
name: str = "Scrapegraph website scraper"
description: str = "A tool that uses Scrapegraph AI to intelligently scrape website content."
description: str = (
"A tool that uses Scrapegraph AI to intelligently scrape website content."
)
args_schema: Type[BaseModel] = ScrapegraphScrapeToolSchema
website_url: Optional[str] = None
user_prompt: Optional[str] = None
api_key: Optional[str] = None
_client: Optional["Client"] = None
def __init__(
self,
@@ -69,6 +81,29 @@ class ScrapegraphScrapeTool(BaseTool):
**kwargs,
):
super().__init__(**kwargs)
try:
from scrapegraph_py import Client
from scrapegraph_py.logger import sgai_logger
except ImportError:
import click
if click.confirm(
"You are missing the 'scrapegraph-py' package. Would you like to install it? (y/N)"
):
import subprocess
subprocess.run(["uv", "add", "scrapegraph-py"], check=True)
from scrapegraph_py import Client
from scrapegraph_py.logger import sgai_logger
else:
raise ImportError(
"`scrapegraph-py` package not found, please run `uv add scrapegraph-py`"
)
self._client = Client(api_key=api_key)
self.api_key = api_key or os.getenv("SCRAPEGRAPH_API_KEY")
if not self.api_key:
@@ -94,7 +129,9 @@ class ScrapegraphScrapeTool(BaseTool):
if not all([result.scheme, result.netloc]):
raise ValueError
except Exception:
raise ValueError("Invalid URL format. URL must include scheme (http/https) and domain")
raise ValueError(
"Invalid URL format. URL must include scheme (http/https) and domain"
)
def _handle_api_response(self, response: dict) -> str:
"""Handle and validate API response"""
@@ -117,7 +154,10 @@ class ScrapegraphScrapeTool(BaseTool):
**kwargs: Any,
) -> Any:
website_url = kwargs.get("website_url", self.website_url)
user_prompt = kwargs.get("user_prompt", self.user_prompt) or "Extract the main content of the webpage"
user_prompt = (
kwargs.get("user_prompt", self.user_prompt)
or "Extract the main content of the webpage"
)
if not website_url:
raise ValueError("website_url is required")
@@ -125,12 +165,9 @@ class ScrapegraphScrapeTool(BaseTool):
# Validate URL format
self._validate_url(website_url)
# Initialize the client
sgai_client = Client(api_key=self.api_key)
try:
# Make the SmartScraper request
response = sgai_client.smartscraper(
response = self.client.smartscraper(
website_url=website_url,
user_prompt=user_prompt,
)
@@ -144,4 +181,4 @@ class ScrapegraphScrapeTool(BaseTool):
raise RuntimeError(f"Scraping failed: {str(e)}")
finally:
# Always close the client
sgai_client.close()
self.client.close()

View File

@@ -34,8 +34,17 @@ class ScrapflyScrapeWebsiteTool(BaseTool):
try:
from scrapfly import ScrapflyClient
except ImportError:
import click
if click.confirm(
"You are missing the 'scrapfly-sdk' package. Would you like to install it? (y/N)"
):
import subprocess
subprocess.run(["uv", "add", "scrapfly-sdk"], check=True)
else:
raise ImportError(
"`scrapfly` package not found, please run `pip install scrapfly-sdk`"
"`scrapfly-sdk` package not found, please run `uv add scrapfly-sdk`"
)
self.scrapfly = ScrapflyClient(key=api_key)

View File

@@ -5,9 +5,6 @@ from urllib.parse import urlparse
from crewai.tools import BaseTool
from pydantic import BaseModel, Field, validator
from selenium import webdriver
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.common.by import By
class FixedSeleniumScrapingToolSchema(BaseModel):
@@ -17,13 +14,16 @@ class FixedSeleniumScrapingToolSchema(BaseModel):
class SeleniumScrapingToolSchema(FixedSeleniumScrapingToolSchema):
"""Input for SeleniumScrapingTool."""
website_url: str = Field(..., description="Mandatory website url to read the file. Must start with http:// or https://")
website_url: str = Field(
...,
description="Mandatory website url to read the file. Must start with http:// or https://",
)
css_element: str = Field(
...,
description="Mandatory css reference for element to scrape from the website",
)
@validator('website_url')
@validator("website_url")
def validate_website_url(cls, v):
if not v:
raise ValueError("Website URL cannot be empty")
@@ -31,7 +31,7 @@ class SeleniumScrapingToolSchema(FixedSeleniumScrapingToolSchema):
if len(v) > 2048: # Common maximum URL length
raise ValueError("URL is too long (max 2048 characters)")
if not re.match(r'^https?://', v):
if not re.match(r"^https?://", v):
raise ValueError("URL must start with http:// or https://")
try:
@@ -41,7 +41,7 @@ class SeleniumScrapingToolSchema(FixedSeleniumScrapingToolSchema):
except Exception as e:
raise ValueError(f"Invalid URL: {str(e)}")
if re.search(r'\s', v):
if re.search(r"\s", v):
raise ValueError("URL cannot contain whitespace")
return v
@@ -52,7 +52,7 @@ class SeleniumScrapingTool(BaseTool):
description: str = "A tool that can be used to read a website content."
args_schema: Type[BaseModel] = SeleniumScrapingToolSchema
website_url: Optional[str] = None
driver: Optional[Any] = webdriver.Chrome
driver: Optional[Any] = None
cookie: Optional[dict] = None
wait_time: Optional[int] = 3
css_element: Optional[str] = None
@@ -66,6 +66,30 @@ class SeleniumScrapingTool(BaseTool):
**kwargs,
):
super().__init__(**kwargs)
try:
from selenium import webdriver
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.common.by import By
except ImportError:
import click
if click.confirm(
"You are missing the 'selenium' and 'webdriver-manager' packages. Would you like to install it? (y/N)"
):
import subprocess
subprocess.run(
["uv", "pip", "install", "selenium", "webdriver-manager"],
check=True,
)
from selenium import webdriver
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.common.by import By
else:
raise ImportError(
"`selenium` and `webdriver-manager` package not found, please run `uv add selenium webdriver-manager`"
)
self.driver = webdriver.Chrome()
if cookie is not None:
self.cookie = cookie
@@ -132,7 +156,7 @@ class SeleniumScrapingTool(BaseTool):
raise ValueError("URL cannot be empty")
# Validate URL format
if not re.match(r'^https?://', url):
if not re.match(r"^https?://", url):
raise ValueError("URL must start with http:// or https://")
options = Options()

View File

@@ -4,6 +4,7 @@ from typing import Optional, Any, Union
from crewai.tools import BaseTool
class SerpApiBaseTool(BaseTool):
"""Base class for SerpApi functionality with shared capabilities."""
@@ -15,8 +16,17 @@ class SerpApiBaseTool(BaseTool):
try:
from serpapi import Client
except ImportError:
import click
if click.confirm(
"You are missing the 'serpapi' package. Would you like to install it? (y/N)"
):
import subprocess
subprocess.run(["uv", "add", "serpapi"], check=True)
else:
raise ImportError(
"`serpapi` package not found, please install with `pip install serpapi`"
"`serpapi` package not found, please install with `uv add serpapi`"
)
api_key = os.getenv("SERPAPI_API_KEY")
if not api_key:

View File

@@ -87,13 +87,21 @@ class SpiderTool(BaseTool):
try:
from spider import Spider # type: ignore
self.spider = Spider(api_key=api_key)
except ImportError:
import click
if click.confirm(
"You are missing the 'spider-client' package. Would you like to install it? (y/N)"
):
import subprocess
subprocess.run(["uv", "pip", "install", "spider-client"], check=True)
from spider import Spider
else:
raise ImportError(
"`spider-client` package not found, please run `uv add spider-client`"
)
except Exception as e:
raise RuntimeError(f"Failed to initialize Spider client: {str(e)}")
self.spider = Spider(api_key=api_key)
def _validate_url(self, url: str) -> bool:
"""Validate URL format and security constraints.

View File

@@ -25,6 +25,7 @@ logger = logging.getLogger(__name__)
STAGEHAND_AVAILABLE = False
try:
import stagehand
STAGEHAND_AVAILABLE = True
except ImportError:
pass # Keep STAGEHAND_AVAILABLE as False
@@ -38,9 +39,16 @@ class StagehandResult(BaseModel):
data: The result data from the operation
error: Optional error message if the operation failed
"""
success: bool = Field(..., description="Whether the operation completed successfully")
data: Union[str, Dict, List] = Field(..., description="The result data from the operation")
error: Optional[str] = Field(None, description="Optional error message if the operation failed")
success: bool = Field(
..., description="Whether the operation completed successfully"
)
data: Union[str, Dict, List] = Field(
..., description="The result data from the operation"
)
error: Optional[str] = Field(
None, description="Optional error message if the operation failed"
)
class StagehandToolConfig(BaseModel):
@@ -51,9 +59,14 @@ class StagehandToolConfig(BaseModel):
timeout: Maximum time in seconds to wait for operations (default: 30)
retry_attempts: Number of times to retry failed operations (default: 3)
"""
api_key: str = Field(..., description="OpenAI API key for Stagehand authentication")
timeout: int = Field(30, description="Maximum time in seconds to wait for operations")
retry_attempts: int = Field(3, description="Number of times to retry failed operations")
timeout: int = Field(
30, description="Maximum time in seconds to wait for operations"
)
retry_attempts: int = Field(
3, description="Number of times to retry failed operations"
)
class StagehandToolSchema(BaseModel):
@@ -80,16 +93,17 @@ class StagehandToolSchema(BaseModel):
)
```
"""
api_method: str = Field(
...,
description="The Stagehand API to use: 'act' for interactions, 'extract' for getting content, or 'observe' for monitoring changes",
pattern="^(act|extract|observe)$"
pattern="^(act|extract|observe)$",
)
instruction: str = Field(
...,
description="An atomic instruction for Stagehand to execute. Instructions should be simple and specific to increase reliability.",
min_length=1,
max_length=500
max_length=500,
)
@@ -138,7 +152,9 @@ class StagehandTool(BaseTool):
)
args_schema: Type[BaseModel] = StagehandToolSchema
def __init__(self, config: StagehandToolConfig | None = None, **kwargs: Any) -> None:
def __init__(
self, config: StagehandToolConfig | None = None, **kwargs: Any
) -> None:
"""Initialize the StagehandTool.
Args:
@@ -153,10 +169,14 @@ class StagehandTool(BaseTool):
super().__init__(**kwargs)
if not STAGEHAND_AVAILABLE:
raise ImportError(
"The 'stagehand' package is required to use this tool. "
"Please install it with: pip install stagehand"
)
import click
if click.confirm(
"You are missing the 'stagehand-sdk' package. Would you like to install it? (y/N)"
):
import subprocess
subprocess.run(["uv", "add", "stagehand-sdk"], check=True)
# Use config if provided, otherwise try environment variable
if config is not None:
@@ -168,9 +188,7 @@ class StagehandTool(BaseTool):
"Either provide config with api_key or set OPENAI_API_KEY environment variable"
)
self.config = StagehandToolConfig(
api_key=api_key,
timeout=30,
retry_attempts=3
api_key=api_key, timeout=30, retry_attempts=3
)
@lru_cache(maxsize=100)
@@ -193,23 +211,25 @@ class StagehandTool(BaseTool):
logger.debug(
"Cache operation - Method: %s, Instruction length: %d",
api_method,
len(instruction)
len(instruction),
)
# Initialize Stagehand with configuration
logger.info(
"Initializing Stagehand (timeout=%ds, retries=%d)",
self.config.timeout,
self.config.retry_attempts
self.config.retry_attempts,
)
st = stagehand.Stagehand(
api_key=self.config.api_key,
timeout=self.config.timeout,
retry_attempts=self.config.retry_attempts
retry_attempts=self.config.retry_attempts,
)
# Call the appropriate Stagehand API based on the method
logger.info("Executing %s operation with instruction: %s", api_method, instruction[:100])
logger.info(
"Executing %s operation with instruction: %s", api_method, instruction[:100]
)
try:
if api_method == "act":
result = st.act(instruction)
@@ -220,7 +240,6 @@ class StagehandTool(BaseTool):
else:
raise ValueError(f"Unknown api_method: {api_method}")
logger.info("Successfully executed %s operation", api_method)
return result
@@ -228,7 +247,7 @@ class StagehandTool(BaseTool):
logger.warning(
"Operation failed (method=%s, error=%s), will be retried on next attempt",
api_method,
str(e)
str(e),
)
raise
@@ -249,7 +268,7 @@ class StagehandTool(BaseTool):
"Starting operation - Method: %s, Instruction length: %d, Args: %s",
api_method,
len(instruction),
kwargs
kwargs,
)
# Use cached execution
@@ -259,46 +278,26 @@ class StagehandTool(BaseTool):
except stagehand.AuthenticationError as e:
logger.error(
"Authentication failed - Method: %s, Error: %s",
api_method,
str(e)
"Authentication failed - Method: %s, Error: %s", api_method, str(e)
)
return StagehandResult(
success=False,
data={},
error=f"Authentication failed: {str(e)}"
success=False, data={}, error=f"Authentication failed: {str(e)}"
)
except stagehand.APIError as e:
logger.error(
"API error - Method: %s, Error: %s",
api_method,
str(e)
)
return StagehandResult(
success=False,
data={},
error=f"API error: {str(e)}"
)
logger.error("API error - Method: %s, Error: %s", api_method, str(e))
return StagehandResult(success=False, data={}, error=f"API error: {str(e)}")
except stagehand.BrowserError as e:
logger.error(
"Browser error - Method: %s, Error: %s",
api_method,
str(e)
)
logger.error("Browser error - Method: %s, Error: %s", api_method, str(e))
return StagehandResult(
success=False,
data={},
error=f"Browser error: {str(e)}"
success=False, data={}, error=f"Browser error: {str(e)}"
)
except Exception as e:
logger.error(
"Unexpected error - Method: %s, Error type: %s, Message: %s",
api_method,
type(e).__name__,
str(e)
str(e),
)
return StagehandResult(
success=False,
data={},
error=f"Unexpected error: {str(e)}"
success=False, data={}, error=f"Unexpected error: {str(e)}"
)

View File

@@ -68,12 +68,25 @@ class WeaviateVectorSearchTool(BaseTool):
model="gpt-4o",
)
)
else:
import click
if click.confirm(
"You are missing the 'weaviate-client' package. Would you like to install it? (y/N)"
):
import subprocess
subprocess.run(["uv", "pip", "install", "weaviate-client"], check=True)
else:
raise ImportError(
"You are missing the 'weaviate-client' package. Would you like to install it? (y/N)"
)
def _run(self, query: str) -> str:
if not WEAVIATE_AVAILABLE:
raise ImportError(
"The 'weaviate-client' package is required to use the WeaviateVectorSearchTool. "
"Please install it with: uv add weaviate-client"
"You are missing the 'weaviate-client' package. Would you like to install it? (y/N)"
)
if not self.weaviate_cluster_url or not self.weaviate_api_key: