From 1eb5d50a5572e82837387b0d3f7cfdbb10c6c421 Mon Sep 17 00:00:00 2001 From: Carlos Souza Date: Thu, 12 Dec 2024 16:00:24 -0500 Subject: [PATCH] Fix url and api_key args on crawler tool --- .../firecrawl_crawl_website_tool.py | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/src/crewai_tools/tools/firecrawl_crawl_website_tool/firecrawl_crawl_website_tool.py b/src/crewai_tools/tools/firecrawl_crawl_website_tool/firecrawl_crawl_website_tool.py index c23ff2100..d753cdd6f 100644 --- a/src/crewai_tools/tools/firecrawl_crawl_website_tool/firecrawl_crawl_website_tool.py +++ b/src/crewai_tools/tools/firecrawl_crawl_website_tool/firecrawl_crawl_website_tool.py @@ -1,7 +1,7 @@ from typing import TYPE_CHECKING, Any, Dict, Optional, Type - from crewai.tools import BaseTool from pydantic import BaseModel, ConfigDict, Field +import os # Type checking import if TYPE_CHECKING: @@ -27,6 +27,7 @@ class FirecrawlCrawlWebsiteTool(BaseTool): args_schema: Type[BaseModel] = FirecrawlCrawlWebsiteToolSchema api_key: Optional[str] = None firecrawl: Optional["FirecrawlApp"] = None + url: Optional[str] = None def __init__(self, api_key: Optional[str] = None, **kwargs): super().__init__(**kwargs) @@ -37,7 +38,11 @@ class FirecrawlCrawlWebsiteTool(BaseTool): "`firecrawl` package not found, please run `pip install firecrawl-py`" ) - self.firecrawl = FirecrawlApp(api_key=api_key) + client_api_key = api_key or os.getenv("FIRECRAWL_API_KEY") + if not client_api_key: + raise ValueError("FIRECRAWL_API_KEY is not set") + + self.firecrawl = FirecrawlApp(api_key=client_api_key) def _run( self, @@ -45,13 +50,17 @@ class FirecrawlCrawlWebsiteTool(BaseTool): crawler_options: Optional[Dict[str, Any]] = None, page_options: Optional[Dict[str, Any]] = None, ): + # Unless url has been previously set via constructor by the user, + # use the url argument provided by the agent + base_url = self.url or url + if crawler_options is None: crawler_options = {} if page_options is None: page_options = {} options = {"crawlerOptions": crawler_options, "pageOptions": page_options} - return self.firecrawl.crawl_url(url, options) + return self.firecrawl.crawl_url(base_url, options) try: