x

src/crewai_tools/tools/spider_full_tool/spider_full_tool.py

@@ -1,6 +1,7 @@
 from typing import Optional, Any, Type, Dict, Literal
 from pydantic.v1 import BaseModel, Field
 from crewai_tools.tools.base_tool import BaseTool
+import requests
 
 class SpiderFullParams(BaseModel):
     request: Optional[str] = Field(description="The request type to perform. Possible values are `http`, `chrome`, and `smart`.")
@@ -64,12 +65,21 @@ class SpiderFullTool(BaseTool):
             )
 
         if params is None:
+            print("PARAMS IT NONE")
             params = SpiderFullParams()
+            print(params)
 
         action = self.spider.scrape_url if mode == "scrape" else self.spider.crawl_url
-        spider_docs = action(url=url, params=params.dict())
+        response = action(url=url, params=params.dict())
+
+        # Debugging: Print the response content
+        print(f"Response status code: {response.status_code}")
+        print(f"Response content: {response.text}")
+
+        try:
+            spider_docs = response.json()
+        except requests.exceptions.JSONDecodeError as e:
+            print(f"JSONDecodeError: {e}")
+            spider_docs = {"error": "Failed to decode JSON response"}
 
         return spider_docs
-
-tool = SpiderFullTool()
-tool._run(url="https://spider.cloud")

src/crewai_tools/tools/spider_tool/spider_tool.py

@@ -25,10 +25,15 @@ class SpiderTool(BaseTool):
 
         self.spider = Spider(api_key=api_key)
 
-    def _run(self, url: str, params: Optional[Dict[str, Any]] = None, mode: Optional[Literal["scrape", "crawl"]] = "scrape"):
-        if mode != "scrape" and mode != "crawl":
+    def _run(
+        self,
+        url: str,
+        params: Optional[Dict[str, any]] = None,
+        mode: Optional[Literal["scrape", "crawl"]] = "scrape"
+    ):
+        if mode not in ["scrape", "crawl"]:
             raise ValueError(
-                "Unknown mode in `mode` parameter, `scrape` or `crawl` is the allowed modes"
+                "Unknown mode in `mode` parameter, `scrape` or `crawl` are the allowed modes"
             )
 
         if params is None or params == {}: