From ecbf550be9d5fc7b1d37c633ddf5721e3a049c88 Mon Sep 17 00:00:00 2001 From: Brandon Hancock Date: Wed, 8 Jan 2025 16:38:38 -0500 Subject: [PATCH] Fix firecrawl errors --- .../firecrawl_crawl_website_tool.py | 21 ++++++++--------- .../firecrawl_scrape_website_tool.py | 10 ++++---- .../firecrawl_search_tool.py | 23 ++++++++++++++----- 3 files changed, 32 insertions(+), 22 deletions(-) diff --git a/src/crewai_tools/tools/firecrawl_crawl_website_tool/firecrawl_crawl_website_tool.py b/src/crewai_tools/tools/firecrawl_crawl_website_tool/firecrawl_crawl_website_tool.py index 0eafd6e4a..6c7c4ffd9 100644 --- a/src/crewai_tools/tools/firecrawl_crawl_website_tool/firecrawl_crawl_website_tool.py +++ b/src/crewai_tools/tools/firecrawl_crawl_website_tool/firecrawl_crawl_website_tool.py @@ -2,7 +2,7 @@ import os from typing import TYPE_CHECKING, Any, Dict, Optional, Type from crewai.tools import BaseTool -from pydantic import BaseModel, ConfigDict, Field +from pydantic import BaseModel, ConfigDict, Field, PrivateAttr # Type checking import if TYPE_CHECKING: @@ -28,7 +28,7 @@ class FirecrawlCrawlWebsiteTool(BaseTool): description: str = "Crawl webpages using Firecrawl and return the contents" args_schema: Type[BaseModel] = FirecrawlCrawlWebsiteToolSchema api_key: Optional[str] = None - firecrawl: Optional["FirecrawlApp"] = None + _firecrawl: Optional["FirecrawlApp"] = PrivateAttr(None) def __init__(self, api_key: Optional[str] = None, **kwargs): super().__init__(**kwargs) @@ -39,14 +39,13 @@ class FirecrawlCrawlWebsiteTool(BaseTool): "`firecrawl` package not found, please run `pip install firecrawl-py`" ) - if not self.firecrawl: - client_api_key = api_key or os.getenv("FIRECRAWL_API_KEY") - if not client_api_key: - raise ValueError( - "FIRECRAWL_API_KEY is not set. Please provide it either via the constructor " - "with the `api_key` argument or by setting the FIRECRAWL_API_KEY environment variable." - ) - self.firecrawl = FirecrawlApp(api_key=client_api_key) + client_api_key = api_key or os.getenv("FIRECRAWL_API_KEY") + if not client_api_key: + raise ValueError( + "FIRECRAWL_API_KEY is not set. Please provide it either via the constructor " + "with the `api_key` argument or by setting the FIRECRAWL_API_KEY environment variable." + ) + self._firecrawl = FirecrawlApp(api_key=client_api_key) def _run( self, @@ -61,7 +60,7 @@ class FirecrawlCrawlWebsiteTool(BaseTool): "crawlerOptions": crawler_options, "timeout": timeout, } - return self.firecrawl.crawl_url(url, options) + return self._firecrawl.crawl_url(url, options) try: diff --git a/src/crewai_tools/tools/firecrawl_scrape_website_tool/firecrawl_scrape_website_tool.py b/src/crewai_tools/tools/firecrawl_scrape_website_tool/firecrawl_scrape_website_tool.py index 8b2a37185..9458e7a4f 100644 --- a/src/crewai_tools/tools/firecrawl_scrape_website_tool/firecrawl_scrape_website_tool.py +++ b/src/crewai_tools/tools/firecrawl_scrape_website_tool/firecrawl_scrape_website_tool.py @@ -1,7 +1,7 @@ from typing import TYPE_CHECKING, Optional, Type from crewai.tools import BaseTool -from pydantic import BaseModel, ConfigDict, Field +from pydantic import BaseModel, ConfigDict, Field, PrivateAttr # Type checking import if TYPE_CHECKING: @@ -21,10 +21,10 @@ class FirecrawlScrapeWebsiteTool(BaseTool): arbitrary_types_allowed=True, validate_assignment=True, frozen=False ) name: str = "Firecrawl web scrape tool" - description: str = "Scrape webpages url using Firecrawl and return the contents" + description: str = "Scrape webpages using Firecrawl and return the contents" args_schema: Type[BaseModel] = FirecrawlScrapeWebsiteToolSchema api_key: Optional[str] = None - firecrawl: Optional["FirecrawlApp"] = None # Updated to use TYPE_CHECKING + _firecrawl: Optional["FirecrawlApp"] = PrivateAttr(None) def __init__(self, api_key: Optional[str] = None, **kwargs): super().__init__(**kwargs) @@ -35,7 +35,7 @@ class FirecrawlScrapeWebsiteTool(BaseTool): "`firecrawl` package not found, please run `pip install firecrawl-py`" ) - self.firecrawl = FirecrawlApp(api_key=api_key) + self._firecrawl = FirecrawlApp(api_key=api_key) def _run( self, @@ -51,7 +51,7 @@ class FirecrawlScrapeWebsiteTool(BaseTool): "waitFor": 0, "timeout": timeout, } - return self.firecrawl.scrape_url(url, options) + return self._firecrawl.scrape_url(url, options) try: diff --git a/src/crewai_tools/tools/firecrawl_search_tool/firecrawl_search_tool.py b/src/crewai_tools/tools/firecrawl_search_tool/firecrawl_search_tool.py index 36ba16391..da483fb34 100644 --- a/src/crewai_tools/tools/firecrawl_search_tool/firecrawl_search_tool.py +++ b/src/crewai_tools/tools/firecrawl_search_tool/firecrawl_search_tool.py @@ -1,7 +1,7 @@ from typing import TYPE_CHECKING, Any, Dict, Optional, Type from crewai.tools import BaseTool -from pydantic import BaseModel, Field +from pydantic import BaseModel, ConfigDict, Field, PrivateAttr # Type checking import if TYPE_CHECKING: @@ -30,11 +30,14 @@ class FirecrawlSearchToolSchema(BaseModel): class FirecrawlSearchTool(BaseTool): + model_config = ConfigDict( + arbitrary_types_allowed=True, validate_assignment=True, frozen=False + ) name: str = "Firecrawl web search tool" description: str = "Search webpages using Firecrawl and return the results" args_schema: Type[BaseModel] = FirecrawlSearchToolSchema api_key: Optional[str] = None - firecrawl: Optional["FirecrawlApp"] = None + _firecrawl: Optional["FirecrawlApp"] = PrivateAttr(None) def __init__(self, api_key: Optional[str] = None, **kwargs): super().__init__(**kwargs) @@ -44,8 +47,7 @@ class FirecrawlSearchTool(BaseTool): raise ImportError( "`firecrawl` package not found, please run `pip install firecrawl-py`" ) - - self.firecrawl = FirecrawlApp(api_key=api_key) + self._firecrawl = FirecrawlApp(api_key=api_key) def _run( self, @@ -62,7 +64,6 @@ class FirecrawlSearchTool(BaseTool): scrape_options = {} options = { - "query": query, "limit": limit, "tbs": tbs, "lang": lang, @@ -71,4 +72,14 @@ class FirecrawlSearchTool(BaseTool): "timeout": timeout, "scrapeOptions": scrape_options, } - return self.firecrawl.search(**options) + return self._firecrawl.search(query, options) + + +try: + from firecrawl import FirecrawlApp + + # Rebuild the model after class is defined + FirecrawlSearchTool.model_rebuild() +except ImportError: + # Exception can be ignored if the tool is not used + pass