Merge pull request #180 from crewAIInc/bugfix/improve-firecrawl-and-serper

Fix firecrawl errors

src/crewai_tools/tools/firecrawl_crawl_website_tool/firecrawl_crawl_website_tool.py

@@ -2,7 +2,7 @@ import os
 from typing import TYPE_CHECKING, Any, Dict, Optional, Type
 
 from crewai.tools import BaseTool
-from pydantic import BaseModel, ConfigDict, Field
+from pydantic import BaseModel, ConfigDict, Field, PrivateAttr
 
 # Type checking import
 if TYPE_CHECKING:
@@ -28,7 +28,7 @@ class FirecrawlCrawlWebsiteTool(BaseTool):
     description: str = "Crawl webpages using Firecrawl and return the contents"
     args_schema: Type[BaseModel] = FirecrawlCrawlWebsiteToolSchema
     api_key: Optional[str] = None
-    firecrawl: Optional["FirecrawlApp"] = None
+    _firecrawl: Optional["FirecrawlApp"] = PrivateAttr(None)
 
     def __init__(self, api_key: Optional[str] = None, **kwargs):
         super().__init__(**kwargs)
@@ -39,14 +39,13 @@ class FirecrawlCrawlWebsiteTool(BaseTool):
                 "`firecrawl` package not found, please run `pip install firecrawl-py`"
             )
 
-        if not self.firecrawl:
-            client_api_key = api_key or os.getenv("FIRECRAWL_API_KEY")
-            if not client_api_key:
-                raise ValueError(
-                    "FIRECRAWL_API_KEY is not set. Please provide it either via the constructor "
-                    "with the `api_key` argument or by setting the FIRECRAWL_API_KEY environment variable."
-                )
-            self.firecrawl = FirecrawlApp(api_key=client_api_key)
+        client_api_key = api_key or os.getenv("FIRECRAWL_API_KEY")
+        if not client_api_key:
+            raise ValueError(
+                "FIRECRAWL_API_KEY is not set. Please provide it either via the constructor "
+                "with the `api_key` argument or by setting the FIRECRAWL_API_KEY environment variable."
+            )
+        self._firecrawl = FirecrawlApp(api_key=client_api_key)
 
     def _run(
         self,
@@ -61,7 +60,7 @@ class FirecrawlCrawlWebsiteTool(BaseTool):
             "crawlerOptions": crawler_options,
             "timeout": timeout,
         }
-        return self.firecrawl.crawl_url(url, options)
+        return self._firecrawl.crawl_url(url, options)
 
 
 try:

src/crewai_tools/tools/firecrawl_scrape_website_tool/firecrawl_scrape_website_tool.py

@@ -1,7 +1,7 @@
 from typing import TYPE_CHECKING, Optional, Type
 
 from crewai.tools import BaseTool
-from pydantic import BaseModel, ConfigDict, Field
+from pydantic import BaseModel, ConfigDict, Field, PrivateAttr
 
 # Type checking import
 if TYPE_CHECKING:
@@ -21,10 +21,10 @@ class FirecrawlScrapeWebsiteTool(BaseTool):
         arbitrary_types_allowed=True, validate_assignment=True, frozen=False
     )
     name: str = "Firecrawl web scrape tool"
-    description: str = "Scrape webpages url using Firecrawl and return the contents"
+    description: str = "Scrape webpages using Firecrawl and return the contents"
     args_schema: Type[BaseModel] = FirecrawlScrapeWebsiteToolSchema
     api_key: Optional[str] = None
-    firecrawl: Optional["FirecrawlApp"] = None  # Updated to use TYPE_CHECKING
+    _firecrawl: Optional["FirecrawlApp"] = PrivateAttr(None)
 
     def __init__(self, api_key: Optional[str] = None, **kwargs):
         super().__init__(**kwargs)
@@ -35,7 +35,7 @@ class FirecrawlScrapeWebsiteTool(BaseTool):
                 "`firecrawl` package not found, please run `pip install firecrawl-py`"
             )
 
-        self.firecrawl = FirecrawlApp(api_key=api_key)
+        self._firecrawl = FirecrawlApp(api_key=api_key)
 
     def _run(
         self,
@@ -51,7 +51,7 @@ class FirecrawlScrapeWebsiteTool(BaseTool):
             "waitFor": 0,
             "timeout": timeout,
         }
-        return self.firecrawl.scrape_url(url, options)
+        return self._firecrawl.scrape_url(url, options)
 
 
 try:

src/crewai_tools/tools/firecrawl_search_tool/firecrawl_search_tool.py

@@ -1,7 +1,7 @@
 from typing import TYPE_CHECKING, Any, Dict, Optional, Type
 
 from crewai.tools import BaseTool
-from pydantic import BaseModel, Field
+from pydantic import BaseModel, ConfigDict, Field, PrivateAttr
 
 # Type checking import
 if TYPE_CHECKING:
@@ -30,11 +30,14 @@ class FirecrawlSearchToolSchema(BaseModel):
 
 
 class FirecrawlSearchTool(BaseTool):
+    model_config = ConfigDict(
+        arbitrary_types_allowed=True, validate_assignment=True, frozen=False
+    )
     name: str = "Firecrawl web search tool"
     description: str = "Search webpages using Firecrawl and return the results"
     args_schema: Type[BaseModel] = FirecrawlSearchToolSchema
     api_key: Optional[str] = None
-    firecrawl: Optional["FirecrawlApp"] = None
+    _firecrawl: Optional["FirecrawlApp"] = PrivateAttr(None)
 
     def __init__(self, api_key: Optional[str] = None, **kwargs):
         super().__init__(**kwargs)
@@ -44,8 +47,7 @@ class FirecrawlSearchTool(BaseTool):
             raise ImportError(
                 "`firecrawl` package not found, please run `pip install firecrawl-py`"
             )
-
-        self.firecrawl = FirecrawlApp(api_key=api_key)
+        self._firecrawl = FirecrawlApp(api_key=api_key)
 
     def _run(
         self,
@@ -62,7 +64,6 @@ class FirecrawlSearchTool(BaseTool):
             scrape_options = {}
 
         options = {
-            "query": query,
             "limit": limit,
             "tbs": tbs,
             "lang": lang,
@@ -71,4 +72,14 @@ class FirecrawlSearchTool(BaseTool):
             "timeout": timeout,
             "scrapeOptions": scrape_options,
         }
-        return self.firecrawl.search(**options)
+        return self._firecrawl.search(query, options)
+
+
+try:
+    from firecrawl import FirecrawlApp
+
+    # Rebuild the model after class is defined
+    FirecrawlSearchTool.model_rebuild()
+except ImportError:
+    # Exception can be ignored if the tool is not used
+    pass