andre/crewAI
1
0
Fork 0
mirror of https://github.com/crewAIInc/crewAI.git synced 2026-01-09 16:18:30 +00:00
Code Issues Actions 1 Packages Projects Releases Wiki Activity

Merge pull request #181 from crewAIInc/fix/optional-dependencies

Browse Source 
make extra dependencies optional for our tools (making most tools an opt in)
This commit is contained in:
Brandon Hancock (bhancock_ai)
2025-01-22 13:06:48 -05:00
committed by GitHub
parent 4774b996d0 4af3724ec2
commit df3842ed88
16 changed files with 348 additions and 95 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
src/crewai_tools/tools/browserbase_load_tool/browserbase_load_tool.py
View File

@@ -37,8 +37,18 @@ class BrowserbaseLoadTool(BaseTool):
try:
from browserbase import Browserbase # type: ignore
except ImportError:
import click
if click.confirm(
"`browserbase` package not found, would you like to install it?"
):
import subprocess
subprocess.run(["uv", "add", "browserbase"], check=True)
from browserbase import Browserbase # type: ignore
else:
raise ImportError(
"`browserbase` package not found, please run `pip install browserbase`"
"`browserbase` package not found, please run `uv add browserbase`"
)
self.browserbase = Browserbase(api_key=self.api_key)

46
src/crewai_tools/tools/firecrawl_crawl_website_tool/firecrawl_crawl_website_tool.py
View File

@@ -1,12 +1,13 @@
import os
from typing import TYPE_CHECKING, Any, Dict, Optional, Type
from typing import Any, Dict, Optional, Type
from crewai.tools import BaseTool
from pydantic import BaseModel, ConfigDict, Field, PrivateAttr
# Type checking import
if TYPE_CHECKING:
try:
from firecrawl import FirecrawlApp
except ImportError:
FirecrawlApp = Any
class FirecrawlCrawlWebsiteToolSchema(BaseModel):
@@ -32,20 +33,33 @@ class FirecrawlCrawlWebsiteTool(BaseTool):
def __init__(self, api_key: Optional[str] = None, **kwargs):
super().__init__(**kwargs)
self.api_key = api_key
self._initialize_firecrawl()
def _initialize_firecrawl(self) -> None:
try:
from firecrawl import FirecrawlApp # type: ignore
except ImportError:
raise ImportError(
"`firecrawl` package not found, please run `pip install firecrawl-py`"
)
client_api_key = api_key or os.getenv("FIRECRAWL_API_KEY")
if not client_api_key:
raise ValueError(
"FIRECRAWL_API_KEY is not set. Please provide it either via the constructor "
"with the `api_key` argument or by setting the FIRECRAWL_API_KEY environment variable."
self._firecrawl = FirecrawlApp(api_key=self.api_key)
except ImportError:
import click
if click.confirm(
"You are missing the 'firecrawl-py' package. Would you like to install it?"
):
import subprocess
try:
subprocess.run(["uv", "add", "firecrawl-py"], check=True)
from firecrawl import FirecrawlApp
self._firecrawl = FirecrawlApp(api_key=self.api_key)
except subprocess.CalledProcessError:
raise ImportError("Failed to install firecrawl-py package")
else:
raise ImportError(
"`firecrawl-py` package not found, please run `uv add firecrawl-py`"
)
self._firecrawl = FirecrawlApp(api_key=client_api_key)
def _run(
self,
@@ -66,8 +80,10 @@ class FirecrawlCrawlWebsiteTool(BaseTool):
try:
from firecrawl import FirecrawlApp
# Must rebuild model after class is defined
# Only rebuild if the class hasn't been initialized yet
if not hasattr(FirecrawlCrawlWebsiteTool, "_model_rebuilt"):
FirecrawlCrawlWebsiteTool.model_rebuild()
FirecrawlCrawlWebsiteTool._model_rebuilt = True
except ImportError:
"""
When this tool is not used, then exception can be ignored.

23
src/crewai_tools/tools/firecrawl_scrape_website_tool/firecrawl_scrape_website_tool.py
View File

@@ -1,11 +1,12 @@
from typing import TYPE_CHECKING, Optional, Type
from typing import Any, Optional, Type
from crewai.tools import BaseTool
from pydantic import BaseModel, ConfigDict, Field, PrivateAttr
# Type checking import
if TYPE_CHECKING:
try:
from firecrawl import FirecrawlApp
except ImportError:
FirecrawlApp = Any
class FirecrawlScrapeWebsiteToolSchema(BaseModel):
@@ -31,8 +32,20 @@ class FirecrawlScrapeWebsiteTool(BaseTool):
try:
from firecrawl import FirecrawlApp # type: ignore
except ImportError:
import click
if click.confirm(
"You are missing the 'firecrawl-py' package. Would you like to install it?"
):
import subprocess
subprocess.run(["uv", "add", "firecrawl-py"], check=True)
from firecrawl import (
FirecrawlApp,
)
else:
raise ImportError(
"`firecrawl` package not found, please run `pip install firecrawl-py`"
"`firecrawl-py` package not found, please run `uv add firecrawl-py`"
)
self._firecrawl = FirecrawlApp(api_key=api_key)
@@ -58,7 +71,9 @@ try:
from firecrawl import FirecrawlApp
# Must rebuild model after class is defined
if not hasattr(FirecrawlScrapeWebsiteTool, "_model_rebuilt"):
FirecrawlScrapeWebsiteTool.model_rebuild()
FirecrawlScrapeWebsiteTool._model_rebuilt = True
except ImportError:
"""
When this tool is not used, then exception can be ignored.

53
src/crewai_tools/tools/firecrawl_search_tool/firecrawl_search_tool.py
View File

@@ -1,11 +1,13 @@
from typing import TYPE_CHECKING, Any, Dict, Optional, Type
from typing import Any, Dict, Optional, Type
from crewai.tools import BaseTool
from pydantic import BaseModel, ConfigDict, Field, PrivateAttr
# Type checking import
if TYPE_CHECKING:
try:
from firecrawl import FirecrawlApp
except ImportError:
FirecrawlApp = Any
class FirecrawlSearchToolSchema(BaseModel):
@@ -30,6 +32,9 @@ class FirecrawlSearchToolSchema(BaseModel):
class FirecrawlSearchTool(BaseTool):
model_config = ConfigDict(
arbitrary_types_allowed=True, validate_assignment=True, frozen=False
)
model_config = ConfigDict(
arbitrary_types_allowed=True, validate_assignment=True, frozen=False
)
@@ -41,13 +46,33 @@ class FirecrawlSearchTool(BaseTool):
def __init__(self, api_key: Optional[str] = None, **kwargs):
super().__init__(**kwargs)
self.api_key = api_key
self._initialize_firecrawl()
def _initialize_firecrawl(self) -> None:
try:
from firecrawl import FirecrawlApp # type: ignore
self.firecrawl = FirecrawlApp(api_key=self.api_key)
except ImportError:
import click
if click.confirm(
"You are missing the 'firecrawl-py' package. Would you like to install it?"
):
import subprocess
try:
subprocess.run(["uv", "add", "firecrawl-py"], check=True)
from firecrawl import FirecrawlApp
self.firecrawl = FirecrawlApp(api_key=self.api_key)
except subprocess.CalledProcessError:
raise ImportError("Failed to install firecrawl-py package")
else:
raise ImportError(
"`firecrawl` package not found, please run `pip install firecrawl-py`"
"`firecrawl-py` package not found, please run `uv add firecrawl-py`"
)
self._firecrawl = FirecrawlApp(api_key=api_key)
def _run(
self,
@@ -59,9 +84,9 @@ class FirecrawlSearchTool(BaseTool):
location: Optional[str] = None,
timeout: Optional[int] = 60000,
scrape_options: Optional[Dict[str, Any]] = None,
):
if scrape_options is None:
scrape_options = {}
) -> Any:
if not self.firecrawl:
raise RuntimeError("FirecrawlApp not properly initialized")
options = {
"limit": limit,
@@ -70,16 +95,20 @@ class FirecrawlSearchTool(BaseTool):
"country": country,
"location": location,
"timeout": timeout,
"scrapeOptions": scrape_options,
"scrapeOptions": scrape_options or {},
}
return self._firecrawl.search(query, options)
return self.firecrawl.search(**options)
try:
from firecrawl import FirecrawlApp
from firecrawl import FirecrawlApp # type: ignore
# Rebuild the model after class is defined
# Only rebuild if the class hasn't been initialized yet
if not hasattr(FirecrawlSearchTool, "_model_rebuilt"):
FirecrawlSearchTool.model_rebuild()
FirecrawlSearchTool._model_rebuilt = True
except ImportError:
# Exception can be ignored if the tool is not used
"""
When this tool is not used, then exception can be ignored.
"""
pass

28
src/crewai_tools/tools/linkup/linkup_search_tool.py
View File

@@ -1,5 +1,7 @@
from typing import Any
from crewai.tools import BaseTool
try:
from linkup import LinkupClient
@@ -11,21 +13,39 @@ except ImportError:
from pydantic import PrivateAttr
class LinkupSearchTool:
class LinkupSearchTool(BaseTool):
name: str = "Linkup Search Tool"
description: str = (
"Performs an API call to Linkup to retrieve contextual information."
)
_client: LinkupClient = PrivateAttr() # type: ignore
description: str = (
"Performs an API call to Linkup to retrieve contextual information."
)
_client: LinkupClient = PrivateAttr() # type: ignore
def __init__(self, api_key: str):
"""
Initialize the tool with an API key.
"""
if not LINKUP_AVAILABLE:
super().__init__()
try:
from linkup import LinkupClient
except ImportError:
import click
if click.confirm(
"You are missing the 'linkup-sdk' package. Would you like to install it?"
):
import subprocess
subprocess.run(["uv", "add", "linkup-sdk"], check=True)
from linkup import LinkupClient
else:
raise ImportError(
"The 'linkup' package is required to use the LinkupSearchTool. "
"Please install it with: uv add linkup"
"The 'linkup-sdk' package is required to use the LinkupSearchTool. "
"Please install it with: uv add linkup-sdk"
)
self._client = LinkupClient(api_key=api_key)

12
src/crewai_tools/tools/multion_tool/multion_tool.py
View File

@@ -28,8 +28,18 @@ class MultiOnTool(BaseTool):
try:
from multion.client import MultiOn # type: ignore
except ImportError:
import click
if click.confirm(
"You are missing the 'multion' package. Would you like to install it?"
):
import subprocess
subprocess.run(["uv", "add", "multion"], check=True)
from multion.client import MultiOn
else:
raise ImportError(
"`multion` package not found, please run `pip install multion`"
"`multion` package not found, please run `uv add multion`"
)
self.session_id = None
self.local = local

31
src/crewai_tools/tools/patronus_eval_tool/patronus_local_evaluator_tool.py
View File

@@ -4,6 +4,14 @@ from crewai.tools import BaseTool
from patronus import Client
from pydantic import BaseModel, Field
try:
from patronus import Client
PYPATRONUS_AVAILABLE = True
except ImportError:
PYPATRONUS_AVAILABLE = False
Client = Any
class FixedLocalEvaluatorToolSchema(BaseModel):
evaluated_model_input: str = Field(
@@ -26,12 +34,20 @@ class PatronusLocalEvaluatorTool(BaseTool):
evaluator: str = "The registered local evaluator"
evaluated_model_gold_answer: str = "The agent's gold answer"
description: str = "This tool is used to evaluate the model input and output using custom function evaluators."
description: str = "This tool is used to evaluate the model input and output using custom function evaluators."
client: Any = None
args_schema: Type[BaseModel] = FixedLocalEvaluatorToolSchema
class Config:
arbitrary_types_allowed = True
def __init__(
self,
patronus_client: Client,
evaluator: str,
evaluated_model_gold_answer: str,
**kwargs: Any,
):
def __init__(
self,
patronus_client: Client,
@@ -40,6 +56,7 @@ class PatronusLocalEvaluatorTool(BaseTool):
**kwargs: Any,
):
super().__init__(**kwargs)
if PYPATRONUS_AVAILABLE:
self.client = patronus_client
if evaluator:
self.evaluator = evaluator
@@ -49,6 +66,19 @@ class PatronusLocalEvaluatorTool(BaseTool):
print(
f"Updating judge evaluator, gold_answer to: {self.evaluator}, {self.evaluated_model_gold_answer}"
)
else:
import click
if click.confirm(
"You are missing the 'patronus' package. Would you like to install it?"
):
import subprocess
subprocess.run(["uv", "add", "patronus"], check=True)
else:
raise ImportError(
"You are missing the patronus package. Would you like to install it?"
)
def _run(
self,
@@ -85,6 +115,7 @@ class PatronusLocalEvaluatorTool(BaseTool):
else evaluated_model_gold_answer.get("description")
),
tags={}, # Optional metadata, supports arbitrary kv pairs
tags={}, # Optional metadata, supports arbitrary kv pairs
)
output = f"Evaluation result: {result.pass_}, Explanation: {result.explanation}"
return output

57
src/crewai_tools/tools/scrapegraph_scrape_tool/scrapegraph_scrape_tool.py
View File

@@ -1,11 +1,13 @@
import os
from typing import Any, Optional, Type
from typing import Any, Optional, Type, TYPE_CHECKING
from urllib.parse import urlparse
from crewai.tools import BaseTool
from pydantic import BaseModel, Field, validator
from pydantic import BaseModel, Field, validator, ConfigDict
# Type checking import
if TYPE_CHECKING:
from scrapegraph_py import Client
from scrapegraph_py.logger import sgai_logger
class ScrapegraphError(Exception):
@@ -53,6 +55,8 @@ class ScrapegraphScrapeTool(BaseTool):
RuntimeError: If scraping operation fails
"""
model_config = ConfigDict(arbitrary_types_allowed=True)
name: str = "Scrapegraph website scraper"
description: str = (
"A tool that uses Scrapegraph AI to intelligently scrape website content."
@@ -62,6 +66,7 @@ class ScrapegraphScrapeTool(BaseTool):
user_prompt: Optional[str] = None
api_key: Optional[str] = None
enable_logging: bool = False
_client: Optional["Client"] = None
def __init__(
self,
@@ -72,6 +77,29 @@ class ScrapegraphScrapeTool(BaseTool):
**kwargs,
):
super().__init__(**kwargs)
try:
from scrapegraph_py import Client
from scrapegraph_py.logger import sgai_logger
except ImportError:
import click
if click.confirm(
"You are missing the 'scrapegraph-py' package. Would you like to install it?"
):
import subprocess
subprocess.run(["uv", "add", "scrapegraph-py"], check=True)
from scrapegraph_py import Client
from scrapegraph_py.logger import sgai_logger
else:
raise ImportError(
"`scrapegraph-py` package not found, please run `uv add scrapegraph-py`"
)
self._client = Client(api_key=api_key)
self.api_key = api_key or os.getenv("SCRAPEGRAPH_API_KEY")
if not self.api_key:
@@ -102,6 +130,22 @@ class ScrapegraphScrapeTool(BaseTool):
"Invalid URL format. URL must include scheme (http/https) and domain"
)
def _handle_api_response(self, response: dict) -> str:
"""Handle and validate API response"""
if not response:
raise RuntimeError("Empty response from Scrapegraph API")
if "error" in response:
error_msg = response.get("error", {}).get("message", "Unknown error")
if "rate limit" in error_msg.lower():
raise RateLimitError(f"Rate limit exceeded: {error_msg}")
raise RuntimeError(f"API error: {error_msg}")
if "result" not in response:
raise RuntimeError("Invalid response format from Scrapegraph API")
return response["result"]
def _run(
self,
**kwargs: Any,
@@ -118,12 +162,9 @@ class ScrapegraphScrapeTool(BaseTool):
# Validate URL format
self._validate_url(website_url)
# Initialize the client
sgai_client = Client(api_key=self.api_key)
try:
# Make the SmartScraper request
response = sgai_client.smartscraper(
response = self._client.smartscraper(
website_url=website_url,
user_prompt=user_prompt,
)
@@ -136,4 +177,4 @@ class ScrapegraphScrapeTool(BaseTool):
raise RuntimeError(f"Scraping failed: {str(e)}")
finally:
# Always close the client
sgai_client.close()
self._client.close()

11
src/crewai_tools/tools/scrapfly_scrape_website_tool/scrapfly_scrape_website_tool.py
View File

@@ -34,8 +34,17 @@ class ScrapflyScrapeWebsiteTool(BaseTool):
try:
from scrapfly import ScrapflyClient
except ImportError:
import click
if click.confirm(
"You are missing the 'scrapfly-sdk' package. Would you like to install it?"
):
import subprocess
subprocess.run(["uv", "add", "scrapfly-sdk"], check=True)
else:
raise ImportError(
"`scrapfly` package not found, please run `pip install scrapfly-sdk`"
"`scrapfly-sdk` package not found, please run `uv add scrapfly-sdk`"
)
self.scrapfly = ScrapflyClient(key=api_key)

39
src/crewai_tools/tools/selenium_scraping_tool/selenium_scraping_tool.py
View File

@@ -5,9 +5,6 @@ from urllib.parse import urlparse
from crewai.tools import BaseTool
from pydantic import BaseModel, Field, validator
from selenium import webdriver
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.common.by import By
class FixedSeleniumScrapingToolSchema(BaseModel):
@@ -55,11 +52,13 @@ class SeleniumScrapingTool(BaseTool):
description: str = "A tool that can be used to read a website content."
args_schema: Type[BaseModel] = SeleniumScrapingToolSchema
website_url: Optional[str] = None
driver: Optional[Any] = webdriver.Chrome
driver: Optional[Any] = None
cookie: Optional[dict] = None
wait_time: Optional[int] = 3
css_element: Optional[str] = None
return_html: Optional[bool] = False
_options: Optional[dict] = None
_by: Optional[Any] = None
def __init__(
self,
@@ -69,6 +68,32 @@ class SeleniumScrapingTool(BaseTool):
**kwargs,
):
super().__init__(**kwargs)
try:
from selenium import webdriver
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.common.by import By
except ImportError:
import click
if click.confirm(
"You are missing the 'selenium' and 'webdriver-manager' packages. Would you like to install it?"
):
import subprocess
subprocess.run(
["uv", "pip", "install", "selenium", "webdriver-manager"],
check=True,
)
from selenium import webdriver
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.common.by import By
else:
raise ImportError(
"`selenium` and `webdriver-manager` package not found, please run `uv add selenium webdriver-manager`"
)
self.driver = webdriver.Chrome()
self._options = Options()
self._by = By
if cookie is not None:
self.cookie = cookie
@@ -112,7 +137,7 @@ class SeleniumScrapingTool(BaseTool):
return css_element is None or css_element.strip() == ""
def _get_body_content(self, driver, return_html):
body_element = driver.find_element(By.TAG_NAME, "body")
body_element = driver.find_element(self._by.TAG_NAME, "body")
return (
body_element.get_attribute("outerHTML")
@@ -123,7 +148,7 @@ class SeleniumScrapingTool(BaseTool):
def _get_elements_content(self, driver, css_element, return_html):
elements_content = []
for element in driver.find_elements(By.CSS_SELECTOR, css_element):
for element in driver.find_elements(self._by.CSS_SELECTOR, css_element):
elements_content.append(
element.get_attribute("outerHTML") if return_html else element.text
)
@@ -138,7 +163,7 @@ class SeleniumScrapingTool(BaseTool):
if not re.match(r"^https?://", url):
raise ValueError("URL must start with http:// or https://")
options = Options()
options = self._options
options.add_argument("--headless")
driver = self.driver(options=options)
driver.get(url)

14
src/crewai_tools/tools/serpapi_tool/serpapi_base_tool.py
View File

@@ -14,10 +14,20 @@ class SerpApiBaseTool(BaseTool):
super().__init__(**kwargs)
try:
from serpapi import Client
from serpapi import Client # type: ignore
except ImportError:
import click
if click.confirm(
"You are missing the 'serpapi' package. Would you like to install it?"
):
import subprocess
subprocess.run(["uv", "add", "serpapi"], check=True)
from serpapi import Client
else:
raise ImportError(
"`serpapi` package not found, please install with `pip install serpapi`"
"`serpapi` package not found, please install with `uv add serpapi`"
)
api_key = os.getenv("SERPAPI_API_KEY")
if not api_key:

12
src/crewai_tools/tools/serpapi_tool/serpapi_google_search_tool.py
View File

@@ -1,10 +1,13 @@
from typing import Any, Optional, Type
from pydantic import BaseModel, Field
import re
from pydantic import BaseModel, Field, ConfigDict
from .serpapi_base_tool import SerpApiBaseTool
from urllib.error import HTTPError
from .serpapi_base_tool import SerpApiBaseTool
try:
from serpapi import HTTPError
except ImportError:
HTTPError = Any
class SerpApiGoogleSearchToolSchema(BaseModel):
@@ -19,6 +22,9 @@ class SerpApiGoogleSearchToolSchema(BaseModel):
class SerpApiGoogleSearchTool(SerpApiBaseTool):
model_config = ConfigDict(
arbitrary_types_allowed=True, validate_assignment=True, frozen=False
)
name: str = "Google Search"
description: str = (
"A tool to perform to perform a Google search with a search_query."

10
src/crewai_tools/tools/serpapi_tool/serpapi_google_shopping_tool.py
View File

@@ -2,9 +2,12 @@ from typing import Any, Optional, Type
from pydantic import BaseModel, Field
from .serpapi_base_tool import SerpApiBaseTool
from urllib.error import HTTPError
from pydantic import ConfigDict
from .serpapi_base_tool import SerpApiBaseTool
try:
from serpapi import HTTPError
except ImportError:
HTTPError = Any
class SerpApiGoogleShoppingToolSchema(BaseModel):
@@ -19,6 +22,9 @@ class SerpApiGoogleShoppingToolSchema(BaseModel):
class SerpApiGoogleShoppingTool(SerpApiBaseTool):
model_config = ConfigDict(
arbitrary_types_allowed=True, validate_assignment=True, frozen=False
)
name: str = "Google Shopping"
description: str = (
"A tool to perform search on Google shopping with a search_query."

14
src/crewai_tools/tools/spider_tool/spider_tool.py
View File

@@ -87,13 +87,21 @@ class SpiderTool(BaseTool):
try:
from spider import Spider # type: ignore
self.spider = Spider(api_key=api_key)
except ImportError:
import click
if click.confirm(
"You are missing the 'spider-client' package. Would you like to install it?"
):
import subprocess
subprocess.run(["uv", "pip", "install", "spider-client"], check=True)
from spider import Spider
else:
raise ImportError(
"`spider-client` package not found, please run `uv add spider-client`"
)
except Exception as e:
raise RuntimeError(f"Failed to initialize Spider client: {str(e)}")
self.spider = Spider(api_key=api_key)
def _validate_url(self, url: str) -> bool:
"""Validate URL format and security constraints.

12
src/crewai_tools/tools/stagehand_tool/stagehand_tool.py
View File

@@ -168,10 +168,14 @@ class StagehandTool(BaseTool):
super().__init__(**kwargs)
if not STAGEHAND_AVAILABLE:
raise ImportError(
"The 'stagehand' package is required to use this tool. "
"Please install it with: pip install stagehand"
)
import click
if click.confirm(
"You are missing the 'stagehand-sdk' package. Would you like to install it?"
):
import subprocess
subprocess.run(["uv", "add", "stagehand-sdk"], check=True)
# Use config if provided, otherwise try environment variable
if config is not None:

17
src/crewai_tools/tools/weaviate_tool/vector_search.py
View File

@@ -67,12 +67,25 @@ class WeaviateVectorSearchTool(BaseTool):
model="gpt-4o",
)
)
else:
import click
if click.confirm(
"You are missing the 'weaviate-client' package. Would you like to install it?"
):
import subprocess
subprocess.run(["uv", "pip", "install", "weaviate-client"], check=True)
else:
raise ImportError(
"You are missing the 'weaviate-client' package. Would you like to install it?"
)
def _run(self, query: str) -> str:
if not WEAVIATE_AVAILABLE:
raise ImportError(
"The 'weaviate-client' package is required to use the WeaviateVectorSearchTool. "
"Please install it with: uv add weaviate-client"
"You are missing the 'weaviate-client' package. Would you like to install it?"
)
if not self.weaviate_cluster_url or not self.weaviate_api_key: