mirror of
https://github.com/crewAIInc/crewAI.git
synced 2026-01-26 16:48:13 +00:00
remove y/n as it comes with it within click.confirm and fixed firecrawl tools
This commit is contained in:
@@ -1,12 +1,13 @@
|
|||||||
import os
|
from typing import Any, Dict, Optional, Type
|
||||||
from typing import TYPE_CHECKING, Any, Dict, Optional, Type
|
|
||||||
|
|
||||||
from crewai.tools import BaseTool
|
from crewai.tools import BaseTool
|
||||||
from pydantic import BaseModel, ConfigDict, Field
|
from pydantic import BaseModel, ConfigDict, Field
|
||||||
|
|
||||||
# Type checking import
|
|
||||||
if TYPE_CHECKING:
|
try:
|
||||||
from firecrawl import FirecrawlApp
|
from firecrawl import FirecrawlApp
|
||||||
|
except ImportError:
|
||||||
|
FirecrawlApp = Any
|
||||||
|
|
||||||
|
|
||||||
class FirecrawlCrawlWebsiteToolSchema(BaseModel):
|
class FirecrawlCrawlWebsiteToolSchema(BaseModel):
|
||||||
@@ -32,34 +33,34 @@ class FirecrawlCrawlWebsiteTool(BaseTool):
|
|||||||
|
|
||||||
def __init__(self, api_key: Optional[str] = None, **kwargs):
|
def __init__(self, api_key: Optional[str] = None, **kwargs):
|
||||||
super().__init__(**kwargs)
|
super().__init__(**kwargs)
|
||||||
|
self.api_key = api_key
|
||||||
|
self._initialize_firecrawl()
|
||||||
|
|
||||||
|
def _initialize_firecrawl(self) -> None:
|
||||||
try:
|
try:
|
||||||
from firecrawl import FirecrawlApp # type: ignore
|
from firecrawl import FirecrawlApp # type: ignore
|
||||||
|
|
||||||
|
self.firecrawl = FirecrawlApp(api_key=self.api_key)
|
||||||
except ImportError:
|
except ImportError:
|
||||||
import click
|
import click
|
||||||
|
|
||||||
if click.confirm(
|
if click.confirm(
|
||||||
"You are missing the 'firecrawl-py' package. Would you like to install it? (y/N)"
|
"You are missing the 'firecrawl-py' package. Would you like to install it?"
|
||||||
):
|
):
|
||||||
import subprocess
|
import subprocess
|
||||||
|
|
||||||
subprocess.run(["uv", "add", "firecrawl-py"], check=True)
|
try:
|
||||||
from firecrawl import (
|
subprocess.run(["uv", "add", "firecrawl-py"], check=True)
|
||||||
FirecrawlApp,
|
from firecrawl import FirecrawlApp
|
||||||
)
|
|
||||||
|
self.firecrawl = FirecrawlApp(api_key=self.api_key)
|
||||||
|
except subprocess.CalledProcessError:
|
||||||
|
raise ImportError("Failed to install firecrawl-py package")
|
||||||
else:
|
else:
|
||||||
raise ImportError(
|
raise ImportError(
|
||||||
"`firecrawl-py` package not found, please run `uv add firecrawl-py`"
|
"`firecrawl-py` package not found, please run `uv add firecrawl-py`"
|
||||||
)
|
)
|
||||||
|
|
||||||
if not self.firecrawl:
|
|
||||||
client_api_key = api_key or os.getenv("FIRECRAWL_API_KEY")
|
|
||||||
if not client_api_key:
|
|
||||||
raise ValueError(
|
|
||||||
"FIRECRAWL_API_KEY is not set. Please provide it either via the constructor "
|
|
||||||
"with the `api_key` argument or by setting the FIRECRAWL_API_KEY environment variable."
|
|
||||||
)
|
|
||||||
self.firecrawl = FirecrawlApp(api_key=client_api_key)
|
|
||||||
|
|
||||||
def _run(
|
def _run(
|
||||||
self,
|
self,
|
||||||
url: str,
|
url: str,
|
||||||
@@ -79,8 +80,10 @@ class FirecrawlCrawlWebsiteTool(BaseTool):
|
|||||||
try:
|
try:
|
||||||
from firecrawl import FirecrawlApp
|
from firecrawl import FirecrawlApp
|
||||||
|
|
||||||
# Must rebuild model after class is defined
|
# Only rebuild if the class hasn't been initialized yet
|
||||||
FirecrawlCrawlWebsiteTool.model_rebuild()
|
if not hasattr(FirecrawlCrawlWebsiteTool, "_model_rebuilt"):
|
||||||
|
FirecrawlCrawlWebsiteTool.model_rebuild()
|
||||||
|
FirecrawlCrawlWebsiteTool._model_rebuilt = True
|
||||||
except ImportError:
|
except ImportError:
|
||||||
"""
|
"""
|
||||||
When this tool is not used, then exception can be ignored.
|
When this tool is not used, then exception can be ignored.
|
||||||
|
|||||||
@@ -1,11 +1,12 @@
|
|||||||
from typing import TYPE_CHECKING, Optional, Type
|
from typing import Any, Optional, Type
|
||||||
|
|
||||||
from crewai.tools import BaseTool
|
from crewai.tools import BaseTool
|
||||||
from pydantic import BaseModel, ConfigDict, Field
|
from pydantic import BaseModel, ConfigDict, Field
|
||||||
|
|
||||||
# Type checking import
|
try:
|
||||||
if TYPE_CHECKING:
|
|
||||||
from firecrawl import FirecrawlApp
|
from firecrawl import FirecrawlApp
|
||||||
|
except ImportError:
|
||||||
|
FirecrawlApp = Any
|
||||||
|
|
||||||
|
|
||||||
class FirecrawlScrapeWebsiteToolSchema(BaseModel):
|
class FirecrawlScrapeWebsiteToolSchema(BaseModel):
|
||||||
@@ -34,7 +35,7 @@ class FirecrawlScrapeWebsiteTool(BaseTool):
|
|||||||
import click
|
import click
|
||||||
|
|
||||||
if click.confirm(
|
if click.confirm(
|
||||||
"You are missing the 'firecrawl-py' package. Would you like to install it? (y/N)"
|
"You are missing the 'firecrawl-py' package. Would you like to install it?"
|
||||||
):
|
):
|
||||||
import subprocess
|
import subprocess
|
||||||
|
|
||||||
@@ -70,7 +71,9 @@ try:
|
|||||||
from firecrawl import FirecrawlApp
|
from firecrawl import FirecrawlApp
|
||||||
|
|
||||||
# Must rebuild model after class is defined
|
# Must rebuild model after class is defined
|
||||||
FirecrawlScrapeWebsiteTool.model_rebuild()
|
if not hasattr(FirecrawlScrapeWebsiteTool, "_model_rebuilt"):
|
||||||
|
FirecrawlScrapeWebsiteTool.model_rebuild()
|
||||||
|
FirecrawlScrapeWebsiteTool._model_rebuilt = True
|
||||||
except ImportError:
|
except ImportError:
|
||||||
"""
|
"""
|
||||||
When this tool is not used, then exception can be ignored.
|
When this tool is not used, then exception can be ignored.
|
||||||
|
|||||||
@@ -1,11 +1,13 @@
|
|||||||
from typing import TYPE_CHECKING, Any, Dict, Optional, Type
|
from typing import Any, Dict, Optional, Type
|
||||||
|
|
||||||
from crewai.tools import BaseTool
|
from crewai.tools import BaseTool
|
||||||
from pydantic import BaseModel, Field
|
from pydantic import BaseModel, Field, ConfigDict
|
||||||
|
|
||||||
# Type checking import
|
# Type checking import
|
||||||
if TYPE_CHECKING:
|
try:
|
||||||
from firecrawl import FirecrawlApp
|
from firecrawl import FirecrawlApp
|
||||||
|
except ImportError:
|
||||||
|
FirecrawlApp = Any
|
||||||
|
|
||||||
|
|
||||||
class FirecrawlSearchToolSchema(BaseModel):
|
class FirecrawlSearchToolSchema(BaseModel):
|
||||||
@@ -30,6 +32,9 @@ class FirecrawlSearchToolSchema(BaseModel):
|
|||||||
|
|
||||||
|
|
||||||
class FirecrawlSearchTool(BaseTool):
|
class FirecrawlSearchTool(BaseTool):
|
||||||
|
model_config = ConfigDict(
|
||||||
|
arbitrary_types_allowed=True, validate_assignment=True, frozen=False
|
||||||
|
)
|
||||||
name: str = "Firecrawl web search tool"
|
name: str = "Firecrawl web search tool"
|
||||||
description: str = "Search webpages using Firecrawl and return the results"
|
description: str = "Search webpages using Firecrawl and return the results"
|
||||||
args_schema: Type[BaseModel] = FirecrawlSearchToolSchema
|
args_schema: Type[BaseModel] = FirecrawlSearchToolSchema
|
||||||
@@ -38,27 +43,34 @@ class FirecrawlSearchTool(BaseTool):
|
|||||||
|
|
||||||
def __init__(self, api_key: Optional[str] = None, **kwargs):
|
def __init__(self, api_key: Optional[str] = None, **kwargs):
|
||||||
super().__init__(**kwargs)
|
super().__init__(**kwargs)
|
||||||
|
self.api_key = api_key
|
||||||
|
self._initialize_firecrawl()
|
||||||
|
|
||||||
|
def _initialize_firecrawl(self) -> None:
|
||||||
try:
|
try:
|
||||||
from firecrawl import FirecrawlApp # type: ignore
|
from firecrawl import FirecrawlApp # type: ignore
|
||||||
|
|
||||||
|
self.firecrawl = FirecrawlApp(api_key=self.api_key)
|
||||||
except ImportError:
|
except ImportError:
|
||||||
import click
|
import click
|
||||||
|
|
||||||
if click.confirm(
|
if click.confirm(
|
||||||
"You are missing the 'firecrawl-py' package. Would you like to install it? (y/N)"
|
"You are missing the 'firecrawl-py' package. Would you like to install it?"
|
||||||
):
|
):
|
||||||
import subprocess
|
import subprocess
|
||||||
|
|
||||||
subprocess.run(["uv", "add", "firecrawl-py"], check=True)
|
try:
|
||||||
from firecrawl import (
|
subprocess.run(["uv", "add", "firecrawl-py"], check=True)
|
||||||
FirecrawlApp,
|
from firecrawl import FirecrawlApp
|
||||||
)
|
|
||||||
|
self.firecrawl = FirecrawlApp(api_key=self.api_key)
|
||||||
|
except subprocess.CalledProcessError:
|
||||||
|
raise ImportError("Failed to install firecrawl-py package")
|
||||||
else:
|
else:
|
||||||
raise ImportError(
|
raise ImportError(
|
||||||
"`firecrawl-py` package not found, please run `uv add firecrawl-py`"
|
"`firecrawl-py` package not found, please run `uv add firecrawl-py`"
|
||||||
)
|
)
|
||||||
|
|
||||||
self.firecrawl = FirecrawlApp(api_key=api_key)
|
|
||||||
|
|
||||||
def _run(
|
def _run(
|
||||||
self,
|
self,
|
||||||
query: str,
|
query: str,
|
||||||
@@ -69,9 +81,9 @@ class FirecrawlSearchTool(BaseTool):
|
|||||||
location: Optional[str] = None,
|
location: Optional[str] = None,
|
||||||
timeout: Optional[int] = 60000,
|
timeout: Optional[int] = 60000,
|
||||||
scrape_options: Optional[Dict[str, Any]] = None,
|
scrape_options: Optional[Dict[str, Any]] = None,
|
||||||
):
|
) -> Any:
|
||||||
if scrape_options is None:
|
if not self.firecrawl:
|
||||||
scrape_options = {}
|
raise RuntimeError("FirecrawlApp not properly initialized")
|
||||||
|
|
||||||
options = {
|
options = {
|
||||||
"query": query,
|
"query": query,
|
||||||
@@ -81,6 +93,20 @@ class FirecrawlSearchTool(BaseTool):
|
|||||||
"country": country,
|
"country": country,
|
||||||
"location": location,
|
"location": location,
|
||||||
"timeout": timeout,
|
"timeout": timeout,
|
||||||
"scrapeOptions": scrape_options,
|
"scrapeOptions": scrape_options or {},
|
||||||
}
|
}
|
||||||
return self.firecrawl.search(**options)
|
return self.firecrawl.search(**options)
|
||||||
|
|
||||||
|
|
||||||
|
try:
|
||||||
|
from firecrawl import FirecrawlApp # type: ignore
|
||||||
|
|
||||||
|
# Only rebuild if the class hasn't been initialized yet
|
||||||
|
if not hasattr(FirecrawlSearchTool, "_model_rebuilt"):
|
||||||
|
FirecrawlSearchTool.model_rebuild()
|
||||||
|
FirecrawlSearchTool._model_rebuilt = True
|
||||||
|
except ImportError:
|
||||||
|
"""
|
||||||
|
When this tool is not used, then exception can be ignored.
|
||||||
|
"""
|
||||||
|
pass
|
||||||
|
|||||||
@@ -31,7 +31,7 @@ class LinkupSearchTool(BaseTool):
|
|||||||
import click
|
import click
|
||||||
|
|
||||||
if click.confirm(
|
if click.confirm(
|
||||||
"You are missing the 'linkup-sdk' package. Would you like to install it? (y/N)"
|
"You are missing the 'linkup-sdk' package. Would you like to install it?"
|
||||||
):
|
):
|
||||||
import subprocess
|
import subprocess
|
||||||
|
|
||||||
|
|||||||
@@ -31,7 +31,7 @@ class MultiOnTool(BaseTool):
|
|||||||
import click
|
import click
|
||||||
|
|
||||||
if click.confirm(
|
if click.confirm(
|
||||||
"You are missing the 'multion' package. Would you like to install it? (y/N)"
|
"You are missing the 'multion' package. Would you like to install it?"
|
||||||
):
|
):
|
||||||
import subprocess
|
import subprocess
|
||||||
|
|
||||||
|
|||||||
@@ -60,7 +60,7 @@ class PatronusLocalEvaluatorTool(BaseTool):
|
|||||||
import click
|
import click
|
||||||
|
|
||||||
if click.confirm(
|
if click.confirm(
|
||||||
"You are missing the 'patronus' package. Would you like to install it? (y/N)"
|
"You are missing the 'patronus' package. Would you like to install it?"
|
||||||
):
|
):
|
||||||
import subprocess
|
import subprocess
|
||||||
|
|
||||||
|
|||||||
@@ -89,7 +89,7 @@ class ScrapegraphScrapeTool(BaseTool):
|
|||||||
import click
|
import click
|
||||||
|
|
||||||
if click.confirm(
|
if click.confirm(
|
||||||
"You are missing the 'scrapegraph-py' package. Would you like to install it? (y/N)"
|
"You are missing the 'scrapegraph-py' package. Would you like to install it?"
|
||||||
):
|
):
|
||||||
import subprocess
|
import subprocess
|
||||||
|
|
||||||
|
|||||||
@@ -37,7 +37,7 @@ class ScrapflyScrapeWebsiteTool(BaseTool):
|
|||||||
import click
|
import click
|
||||||
|
|
||||||
if click.confirm(
|
if click.confirm(
|
||||||
"You are missing the 'scrapfly-sdk' package. Would you like to install it? (y/N)"
|
"You are missing the 'scrapfly-sdk' package. Would you like to install it?"
|
||||||
):
|
):
|
||||||
import subprocess
|
import subprocess
|
||||||
|
|
||||||
|
|||||||
@@ -57,6 +57,8 @@ class SeleniumScrapingTool(BaseTool):
|
|||||||
wait_time: Optional[int] = 3
|
wait_time: Optional[int] = 3
|
||||||
css_element: Optional[str] = None
|
css_element: Optional[str] = None
|
||||||
return_html: Optional[bool] = False
|
return_html: Optional[bool] = False
|
||||||
|
_options: Optional[dict] = None
|
||||||
|
_by: Optional[Any] = None
|
||||||
|
|
||||||
def __init__(
|
def __init__(
|
||||||
self,
|
self,
|
||||||
@@ -74,7 +76,7 @@ class SeleniumScrapingTool(BaseTool):
|
|||||||
import click
|
import click
|
||||||
|
|
||||||
if click.confirm(
|
if click.confirm(
|
||||||
"You are missing the 'selenium' and 'webdriver-manager' packages. Would you like to install it? (y/N)"
|
"You are missing the 'selenium' and 'webdriver-manager' packages. Would you like to install it?"
|
||||||
):
|
):
|
||||||
import subprocess
|
import subprocess
|
||||||
|
|
||||||
@@ -90,6 +92,8 @@ class SeleniumScrapingTool(BaseTool):
|
|||||||
"`selenium` and `webdriver-manager` package not found, please run `uv add selenium webdriver-manager`"
|
"`selenium` and `webdriver-manager` package not found, please run `uv add selenium webdriver-manager`"
|
||||||
)
|
)
|
||||||
self.driver = webdriver.Chrome()
|
self.driver = webdriver.Chrome()
|
||||||
|
self._options = Options()
|
||||||
|
self._by = By
|
||||||
if cookie is not None:
|
if cookie is not None:
|
||||||
self.cookie = cookie
|
self.cookie = cookie
|
||||||
|
|
||||||
@@ -133,7 +137,7 @@ class SeleniumScrapingTool(BaseTool):
|
|||||||
return css_element is None or css_element.strip() == ""
|
return css_element is None or css_element.strip() == ""
|
||||||
|
|
||||||
def _get_body_content(self, driver, return_html):
|
def _get_body_content(self, driver, return_html):
|
||||||
body_element = driver.find_element(By.TAG_NAME, "body")
|
body_element = driver.find_element(self._by.TAG_NAME, "body")
|
||||||
|
|
||||||
return (
|
return (
|
||||||
body_element.get_attribute("outerHTML")
|
body_element.get_attribute("outerHTML")
|
||||||
@@ -144,7 +148,7 @@ class SeleniumScrapingTool(BaseTool):
|
|||||||
def _get_elements_content(self, driver, css_element, return_html):
|
def _get_elements_content(self, driver, css_element, return_html):
|
||||||
elements_content = []
|
elements_content = []
|
||||||
|
|
||||||
for element in driver.find_elements(By.CSS_SELECTOR, css_element):
|
for element in driver.find_elements(self._by.CSS_SELECTOR, css_element):
|
||||||
elements_content.append(
|
elements_content.append(
|
||||||
element.get_attribute("outerHTML") if return_html else element.text
|
element.get_attribute("outerHTML") if return_html else element.text
|
||||||
)
|
)
|
||||||
@@ -159,7 +163,7 @@ class SeleniumScrapingTool(BaseTool):
|
|||||||
if not re.match(r"^https?://", url):
|
if not re.match(r"^https?://", url):
|
||||||
raise ValueError("URL must start with http:// or https://")
|
raise ValueError("URL must start with http:// or https://")
|
||||||
|
|
||||||
options = Options()
|
options = self._options()
|
||||||
options.add_argument("--headless")
|
options.add_argument("--headless")
|
||||||
driver = self.driver(options=options)
|
driver = self.driver(options=options)
|
||||||
driver.get(url)
|
driver.get(url)
|
||||||
|
|||||||
@@ -19,7 +19,7 @@ class SerpApiBaseTool(BaseTool):
|
|||||||
import click
|
import click
|
||||||
|
|
||||||
if click.confirm(
|
if click.confirm(
|
||||||
"You are missing the 'serpapi' package. Would you like to install it? (y/N)"
|
"You are missing the 'serpapi' package. Would you like to install it?"
|
||||||
):
|
):
|
||||||
import subprocess
|
import subprocess
|
||||||
|
|
||||||
|
|||||||
@@ -91,7 +91,7 @@ class SpiderTool(BaseTool):
|
|||||||
import click
|
import click
|
||||||
|
|
||||||
if click.confirm(
|
if click.confirm(
|
||||||
"You are missing the 'spider-client' package. Would you like to install it? (y/N)"
|
"You are missing the 'spider-client' package. Would you like to install it?"
|
||||||
):
|
):
|
||||||
import subprocess
|
import subprocess
|
||||||
|
|
||||||
|
|||||||
@@ -172,7 +172,7 @@ class StagehandTool(BaseTool):
|
|||||||
import click
|
import click
|
||||||
|
|
||||||
if click.confirm(
|
if click.confirm(
|
||||||
"You are missing the 'stagehand-sdk' package. Would you like to install it? (y/N)"
|
"You are missing the 'stagehand-sdk' package. Would you like to install it?"
|
||||||
):
|
):
|
||||||
import subprocess
|
import subprocess
|
||||||
|
|
||||||
|
|||||||
@@ -72,7 +72,7 @@ class WeaviateVectorSearchTool(BaseTool):
|
|||||||
import click
|
import click
|
||||||
|
|
||||||
if click.confirm(
|
if click.confirm(
|
||||||
"You are missing the 'weaviate-client' package. Would you like to install it? (y/N)"
|
"You are missing the 'weaviate-client' package. Would you like to install it?"
|
||||||
):
|
):
|
||||||
import subprocess
|
import subprocess
|
||||||
|
|
||||||
@@ -80,13 +80,13 @@ class WeaviateVectorSearchTool(BaseTool):
|
|||||||
|
|
||||||
else:
|
else:
|
||||||
raise ImportError(
|
raise ImportError(
|
||||||
"You are missing the 'weaviate-client' package. Would you like to install it? (y/N)"
|
"You are missing the 'weaviate-client' package. Would you like to install it?"
|
||||||
)
|
)
|
||||||
|
|
||||||
def _run(self, query: str) -> str:
|
def _run(self, query: str) -> str:
|
||||||
if not WEAVIATE_AVAILABLE:
|
if not WEAVIATE_AVAILABLE:
|
||||||
raise ImportError(
|
raise ImportError(
|
||||||
"You are missing the 'weaviate-client' package. Would you like to install it? (y/N)"
|
"You are missing the 'weaviate-client' package. Would you like to install it?"
|
||||||
)
|
)
|
||||||
|
|
||||||
if not self.weaviate_cluster_url or not self.weaviate_api_key:
|
if not self.weaviate_cluster_url or not self.weaviate_api_key:
|
||||||
|
|||||||
Reference in New Issue
Block a user