andre/crewAI
1
0
Fork 0
mirror of https://github.com/crewAIInc/crewAI.git synced 2026-01-10 00:28:31 +00:00
Code Issues Actions 1 Packages Projects Releases Wiki Activity

remove y/n as it comes with it within click.confirm and fixed firecrawl tools

Browse Source
This commit is contained in:
Lorenze Jay
2025-01-10 15:56:54 -08:00
parent 41cec25ad9
commit d3d3cc4c28
13 changed files with 90 additions and 54 deletions
Download Patch File Download Diff File Expand all files Collapse all files

43
src/crewai_tools/tools/firecrawl_crawl_website_tool/firecrawl_crawl_website_tool.py
View File

@@ -1,12 +1,13 @@
import os
from typing import TYPE_CHECKING, Any, Dict, Optional, Type
from typing import Any, Dict, Optional, Type
from crewai.tools import BaseTool
from pydantic import BaseModel, ConfigDict, Field
# Type checking import
if TYPE_CHECKING:
try:
from firecrawl import FirecrawlApp
except ImportError:
FirecrawlApp = Any
class FirecrawlCrawlWebsiteToolSchema(BaseModel):
@@ -32,34 +33,34 @@ class FirecrawlCrawlWebsiteTool(BaseTool):
def __init__(self, api_key: Optional[str] = None, **kwargs):
super().__init__(**kwargs)
self.api_key = api_key
self._initialize_firecrawl()
def _initialize_firecrawl(self) -> None:
try:
from firecrawl import FirecrawlApp # type: ignore
self.firecrawl = FirecrawlApp(api_key=self.api_key)
except ImportError:
import click
if click.confirm(
"You are missing the 'firecrawl-py' package. Would you like to install it? (y/N)"
"You are missing the 'firecrawl-py' package. Would you like to install it?"
):
import subprocess
subprocess.run(["uv", "add", "firecrawl-py"], check=True)
from firecrawl import (
FirecrawlApp,
)
try:
subprocess.run(["uv", "add", "firecrawl-py"], check=True)
from firecrawl import FirecrawlApp
self.firecrawl = FirecrawlApp(api_key=self.api_key)
except subprocess.CalledProcessError:
raise ImportError("Failed to install firecrawl-py package")
else:
raise ImportError(
"`firecrawl-py` package not found, please run `uv add firecrawl-py`"
)
if not self.firecrawl:
client_api_key = api_key or os.getenv("FIRECRAWL_API_KEY")
if not client_api_key:
raise ValueError(
"FIRECRAWL_API_KEY is not set. Please provide it either via the constructor "
"with the `api_key` argument or by setting the FIRECRAWL_API_KEY environment variable."
)
self.firecrawl = FirecrawlApp(api_key=client_api_key)
def _run(
self,
url: str,
@@ -79,8 +80,10 @@ class FirecrawlCrawlWebsiteTool(BaseTool):
try:
from firecrawl import FirecrawlApp
# Must rebuild model after class is defined
FirecrawlCrawlWebsiteTool.model_rebuild()
# Only rebuild if the class hasn't been initialized yet
if not hasattr(FirecrawlCrawlWebsiteTool, "_model_rebuilt"):
FirecrawlCrawlWebsiteTool.model_rebuild()
FirecrawlCrawlWebsiteTool._model_rebuilt = True
except ImportError:
"""
When this tool is not used, then exception can be ignored.

13
src/crewai_tools/tools/firecrawl_scrape_website_tool/firecrawl_scrape_website_tool.py
View File

@@ -1,11 +1,12 @@
from typing import TYPE_CHECKING, Optional, Type
from typing import Any, Optional, Type
from crewai.tools import BaseTool
from pydantic import BaseModel, ConfigDict, Field
# Type checking import
if TYPE_CHECKING:
try:
from firecrawl import FirecrawlApp
except ImportError:
FirecrawlApp = Any
class FirecrawlScrapeWebsiteToolSchema(BaseModel):
@@ -34,7 +35,7 @@ class FirecrawlScrapeWebsiteTool(BaseTool):
import click
if click.confirm(
"You are missing the 'firecrawl-py' package. Would you like to install it? (y/N)"
"You are missing the 'firecrawl-py' package. Would you like to install it?"
):
import subprocess
@@ -70,7 +71,9 @@ try:
from firecrawl import FirecrawlApp
# Must rebuild model after class is defined
FirecrawlScrapeWebsiteTool.model_rebuild()
if not hasattr(FirecrawlScrapeWebsiteTool, "_model_rebuilt"):
FirecrawlScrapeWebsiteTool.model_rebuild()
FirecrawlScrapeWebsiteTool._model_rebuilt = True
except ImportError:
"""
When this tool is not used, then exception can be ignored.

54
src/crewai_tools/tools/firecrawl_search_tool/firecrawl_search_tool.py
View File

@@ -1,11 +1,13 @@
from typing import TYPE_CHECKING, Any, Dict, Optional, Type
from typing import Any, Dict, Optional, Type
from crewai.tools import BaseTool
from pydantic import BaseModel, Field
from pydantic import BaseModel, Field, ConfigDict
# Type checking import
if TYPE_CHECKING:
try:
from firecrawl import FirecrawlApp
except ImportError:
FirecrawlApp = Any
class FirecrawlSearchToolSchema(BaseModel):
@@ -30,6 +32,9 @@ class FirecrawlSearchToolSchema(BaseModel):
class FirecrawlSearchTool(BaseTool):
model_config = ConfigDict(
arbitrary_types_allowed=True, validate_assignment=True, frozen=False
)
name: str = "Firecrawl web search tool"
description: str = "Search webpages using Firecrawl and return the results"
args_schema: Type[BaseModel] = FirecrawlSearchToolSchema
@@ -38,27 +43,34 @@ class FirecrawlSearchTool(BaseTool):
def __init__(self, api_key: Optional[str] = None, **kwargs):
super().__init__(**kwargs)
self.api_key = api_key
self._initialize_firecrawl()
def _initialize_firecrawl(self) -> None:
try:
from firecrawl import FirecrawlApp # type: ignore
self.firecrawl = FirecrawlApp(api_key=self.api_key)
except ImportError:
import click
if click.confirm(
"You are missing the 'firecrawl-py' package. Would you like to install it? (y/N)"
"You are missing the 'firecrawl-py' package. Would you like to install it?"
):
import subprocess
subprocess.run(["uv", "add", "firecrawl-py"], check=True)
from firecrawl import (
FirecrawlApp,
)
try:
subprocess.run(["uv", "add", "firecrawl-py"], check=True)
from firecrawl import FirecrawlApp
self.firecrawl = FirecrawlApp(api_key=self.api_key)
except subprocess.CalledProcessError:
raise ImportError("Failed to install firecrawl-py package")
else:
raise ImportError(
"`firecrawl-py` package not found, please run `uv add firecrawl-py`"
)
self.firecrawl = FirecrawlApp(api_key=api_key)
def _run(
self,
query: str,
@@ -69,9 +81,9 @@ class FirecrawlSearchTool(BaseTool):
location: Optional[str] = None,
timeout: Optional[int] = 60000,
scrape_options: Optional[Dict[str, Any]] = None,
):
if scrape_options is None:
scrape_options = {}
) -> Any:
if not self.firecrawl:
raise RuntimeError("FirecrawlApp not properly initialized")
options = {
"query": query,
@@ -81,6 +93,20 @@ class FirecrawlSearchTool(BaseTool):
"country": country,
"location": location,
"timeout": timeout,
"scrapeOptions": scrape_options,
"scrapeOptions": scrape_options or {},
}
return self.firecrawl.search(**options)
try:
from firecrawl import FirecrawlApp # type: ignore
# Only rebuild if the class hasn't been initialized yet
if not hasattr(FirecrawlSearchTool, "_model_rebuilt"):
FirecrawlSearchTool.model_rebuild()
FirecrawlSearchTool._model_rebuilt = True
except ImportError:
"""
When this tool is not used, then exception can be ignored.
"""
pass

2
src/crewai_tools/tools/linkup/linkup_search_tool.py
View File

@@ -31,7 +31,7 @@ class LinkupSearchTool(BaseTool):
import click
if click.confirm(
"You are missing the 'linkup-sdk' package. Would you like to install it? (y/N)"
"You are missing the 'linkup-sdk' package. Would you like to install it?"
):
import subprocess

2
src/crewai_tools/tools/multion_tool/multion_tool.py
View File

@@ -31,7 +31,7 @@ class MultiOnTool(BaseTool):
import click
if click.confirm(
"You are missing the 'multion' package. Would you like to install it? (y/N)"
"You are missing the 'multion' package. Would you like to install it?"
):
import subprocess

2
src/crewai_tools/tools/patronus_eval_tool/patronus_local_evaluator_tool.py
View File

@@ -60,7 +60,7 @@ class PatronusLocalEvaluatorTool(BaseTool):
import click
if click.confirm(
"You are missing the 'patronus' package. Would you like to install it? (y/N)"
"You are missing the 'patronus' package. Would you like to install it?"
):
import subprocess

2
src/crewai_tools/tools/scrapegraph_scrape_tool/scrapegraph_scrape_tool.py
View File

@@ -89,7 +89,7 @@ class ScrapegraphScrapeTool(BaseTool):
import click
if click.confirm(
"You are missing the 'scrapegraph-py' package. Would you like to install it? (y/N)"
"You are missing the 'scrapegraph-py' package. Would you like to install it?"
):
import subprocess

2
src/crewai_tools/tools/scrapfly_scrape_website_tool/scrapfly_scrape_website_tool.py
View File

@@ -37,7 +37,7 @@ class ScrapflyScrapeWebsiteTool(BaseTool):
import click
if click.confirm(
"You are missing the 'scrapfly-sdk' package. Would you like to install it? (y/N)"
"You are missing the 'scrapfly-sdk' package. Would you like to install it?"
):
import subprocess

12
src/crewai_tools/tools/selenium_scraping_tool/selenium_scraping_tool.py
View File

@@ -57,6 +57,8 @@ class SeleniumScrapingTool(BaseTool):
wait_time: Optional[int] = 3
css_element: Optional[str] = None
return_html: Optional[bool] = False
_options: Optional[dict] = None
_by: Optional[Any] = None
def __init__(
self,
@@ -74,7 +76,7 @@ class SeleniumScrapingTool(BaseTool):
import click
if click.confirm(
"You are missing the 'selenium' and 'webdriver-manager' packages. Would you like to install it? (y/N)"
"You are missing the 'selenium' and 'webdriver-manager' packages. Would you like to install it?"
):
import subprocess
@@ -90,6 +92,8 @@ class SeleniumScrapingTool(BaseTool):
"`selenium` and `webdriver-manager` package not found, please run `uv add selenium webdriver-manager`"
)
self.driver = webdriver.Chrome()
self._options = Options()
self._by = By
if cookie is not None:
self.cookie = cookie
@@ -133,7 +137,7 @@ class SeleniumScrapingTool(BaseTool):
return css_element is None or css_element.strip() == ""
def _get_body_content(self, driver, return_html):
body_element = driver.find_element(By.TAG_NAME, "body")
body_element = driver.find_element(self._by.TAG_NAME, "body")
return (
body_element.get_attribute("outerHTML")
@@ -144,7 +148,7 @@ class SeleniumScrapingTool(BaseTool):
def _get_elements_content(self, driver, css_element, return_html):
elements_content = []
for element in driver.find_elements(By.CSS_SELECTOR, css_element):
for element in driver.find_elements(self._by.CSS_SELECTOR, css_element):
elements_content.append(
element.get_attribute("outerHTML") if return_html else element.text
)
@@ -159,7 +163,7 @@ class SeleniumScrapingTool(BaseTool):
if not re.match(r"^https?://", url):
raise ValueError("URL must start with http:// or https://")
options = Options()
options = self._options()
options.add_argument("--headless")
driver = self.driver(options=options)
driver.get(url)

2
src/crewai_tools/tools/serpapi_tool/serpapi_base_tool.py
View File

@@ -19,7 +19,7 @@ class SerpApiBaseTool(BaseTool):
import click
if click.confirm(
"You are missing the 'serpapi' package. Would you like to install it? (y/N)"
"You are missing the 'serpapi' package. Would you like to install it?"
):
import subprocess

2
src/crewai_tools/tools/spider_tool/spider_tool.py
View File

@@ -91,7 +91,7 @@ class SpiderTool(BaseTool):
import click
if click.confirm(
"You are missing the 'spider-client' package. Would you like to install it? (y/N)"
"You are missing the 'spider-client' package. Would you like to install it?"
):
import subprocess

2
src/crewai_tools/tools/stagehand_tool/stagehand_tool.py
View File

@@ -172,7 +172,7 @@ class StagehandTool(BaseTool):
import click
if click.confirm(
"You are missing the 'stagehand-sdk' package. Would you like to install it? (y/N)"
"You are missing the 'stagehand-sdk' package. Would you like to install it?"
):
import subprocess

6
src/crewai_tools/tools/weaviate_tool/vector_search.py
View File

@@ -72,7 +72,7 @@ class WeaviateVectorSearchTool(BaseTool):
import click
if click.confirm(
"You are missing the 'weaviate-client' package. Would you like to install it? (y/N)"
"You are missing the 'weaviate-client' package. Would you like to install it?"
):
import subprocess
@@ -80,13 +80,13 @@ class WeaviateVectorSearchTool(BaseTool):
else:
raise ImportError(
"You are missing the 'weaviate-client' package. Would you like to install it? (y/N)"
"You are missing the 'weaviate-client' package. Would you like to install it?"
)
def _run(self, query: str) -> str:
if not WEAVIATE_AVAILABLE:
raise ImportError(
"You are missing the 'weaviate-client' package. Would you like to install it? (y/N)"
"You are missing the 'weaviate-client' package. Would you like to install it?"
)
if not self.weaviate_cluster_url or not self.weaviate_api_key: