andre/crewAI
1
0
Fork 0
mirror of https://github.com/crewAIInc/crewAI.git synced 2026-01-29 10:08:13 +00:00
Code Issues Actions 3 Packages Projects Releases Wiki Activity

Move off v1

Browse Source
This commit is contained in:
Brandon Hancock
2024-09-03 15:57:29 -04:00
parent d19bba72b0
commit 35fe222ca1
39 changed files with 752 additions and 550 deletions
Download Patch File Download Diff File Expand all files Collapse all files

42
src/crewai_tools/tools/firecrawl_crawl_website_tool/firecrawl_crawl_website_tool.py
View File

@@ -1,11 +1,19 @@
from typing import Optional, Any, Type, Dict, List
from pydantic.v1 import BaseModel, Field
from typing import Any, Dict, List, Optional, Type
from pydantic import BaseModel, Field
from crewai_tools.tools.base_tool import BaseTool
class FirecrawlCrawlWebsiteToolSchema(BaseModel):
url: str = Field(description="Website URL")
crawler_options: Optional[Dict[str, Any]] = Field(default=None, description="Options for crawling")
page_options: Optional[Dict[str, Any]] = Field(default=None, description="Options for page")
crawler_options: Optional[Dict[str, Any]] = Field(
default=None, description="Options for crawling"
)
page_options: Optional[Dict[str, Any]] = Field(
default=None, description="Options for page"
)
class FirecrawlCrawlWebsiteTool(BaseTool):
name: str = "Firecrawl web crawl tool"
@@ -17,22 +25,24 @@ class FirecrawlCrawlWebsiteTool(BaseTool):
def __init__(self, api_key: Optional[str] = None, **kwargs):
super().__init__(**kwargs)
try:
from firecrawl import FirecrawlApp # type: ignore
from firecrawl import FirecrawlApp # type: ignore
except ImportError:
raise ImportError(
"`firecrawl` package not found, please run `pip install firecrawl-py`"
)
raise ImportError(
"`firecrawl` package not found, please run `pip install firecrawl-py`"
)
self.firecrawl = FirecrawlApp(api_key=api_key)
def _run(self, url: str, crawler_options: Optional[Dict[str, Any]] = None, page_options: Optional[Dict[str, Any]] = None):
if (crawler_options is None):
def _run(
self,
url: str,
crawler_options: Optional[Dict[str, Any]] = None,
page_options: Optional[Dict[str, Any]] = None,
):
if crawler_options is None:
crawler_options = {}
if (page_options is None):
if page_options is None:
page_options = {}
options = {
"crawlerOptions": crawler_options,
"pageOptions": page_options
}
return self.firecrawl.crawl_url(url, options)
options = {"crawlerOptions": crawler_options, "pageOptions": page_options}
return self.firecrawl.crawl_url(url, options)