From c27727b16eb37b00773197162b32312dde16ff17 Mon Sep 17 00:00:00 2001 From: Marco Vinciguerra Date: Tue, 7 Jan 2025 15:51:52 +0100 Subject: [PATCH 1/2] Update scrapegraph_scrape_tool.py --- .../scrapegraph_scrape_tool.py | 27 +++++-------------- 1 file changed, 7 insertions(+), 20 deletions(-) diff --git a/src/crewai_tools/tools/scrapegraph_scrape_tool/scrapegraph_scrape_tool.py b/src/crewai_tools/tools/scrapegraph_scrape_tool/scrapegraph_scrape_tool.py index 906bf6376..9b5806b19 100644 --- a/src/crewai_tools/tools/scrapegraph_scrape_tool/scrapegraph_scrape_tool.py +++ b/src/crewai_tools/tools/scrapegraph_scrape_tool/scrapegraph_scrape_tool.py @@ -60,16 +60,19 @@ class ScrapegraphScrapeTool(BaseTool): website_url: Optional[str] = None user_prompt: Optional[str] = None api_key: Optional[str] = None + enable_logging: bool = False def __init__( self, website_url: Optional[str] = None, user_prompt: Optional[str] = None, api_key: Optional[str] = None, + enable_logging: bool = False, **kwargs, ): super().__init__(**kwargs) self.api_key = api_key or os.getenv("SCRAPEGRAPH_API_KEY") + self.enable_logging = enable_logging if not self.api_key: raise ValueError("Scrapegraph API key is required") @@ -83,8 +86,9 @@ class ScrapegraphScrapeTool(BaseTool): if user_prompt is not None: self.user_prompt = user_prompt - # Configure logging - sgai_logger.set_logging(level="INFO") + # Configure logging only if enabled + if self.enable_logging: + sgai_logger.set_logging(level="INFO") @staticmethod def _validate_url(url: str) -> None: @@ -96,22 +100,6 @@ class ScrapegraphScrapeTool(BaseTool): except Exception: raise ValueError("Invalid URL format. URL must include scheme (http/https) and domain") - def _handle_api_response(self, response: dict) -> str: - """Handle and validate API response""" - if not response: - raise RuntimeError("Empty response from Scrapegraph API") - - if "error" in response: - error_msg = response.get("error", {}).get("message", "Unknown error") - if "rate limit" in error_msg.lower(): - raise RateLimitError(f"Rate limit exceeded: {error_msg}") - raise RuntimeError(f"API error: {error_msg}") - - if "result" not in response: - raise RuntimeError("Invalid response format from Scrapegraph API") - - return response["result"] - def _run( self, **kwargs: Any, @@ -135,8 +123,7 @@ class ScrapegraphScrapeTool(BaseTool): user_prompt=user_prompt, ) - # Handle and validate the response - return self._handle_api_response(response) + return response except RateLimitError: raise # Re-raise rate limit errors From 4f4b0619079235dcdc59522879822cad5bf0e32a Mon Sep 17 00:00:00 2001 From: Marco Vinciguerra Date: Tue, 7 Jan 2025 16:13:50 +0100 Subject: [PATCH 2/2] fix: scrapegraph-tool --- .../scrapegraph_scrape_tool.py | 27 +++++-------------- 1 file changed, 7 insertions(+), 20 deletions(-) diff --git a/src/crewai_tools/tools/scrapegraph_scrape_tool/scrapegraph_scrape_tool.py b/src/crewai_tools/tools/scrapegraph_scrape_tool/scrapegraph_scrape_tool.py index 906bf6376..9b5806b19 100644 --- a/src/crewai_tools/tools/scrapegraph_scrape_tool/scrapegraph_scrape_tool.py +++ b/src/crewai_tools/tools/scrapegraph_scrape_tool/scrapegraph_scrape_tool.py @@ -60,16 +60,19 @@ class ScrapegraphScrapeTool(BaseTool): website_url: Optional[str] = None user_prompt: Optional[str] = None api_key: Optional[str] = None + enable_logging: bool = False def __init__( self, website_url: Optional[str] = None, user_prompt: Optional[str] = None, api_key: Optional[str] = None, + enable_logging: bool = False, **kwargs, ): super().__init__(**kwargs) self.api_key = api_key or os.getenv("SCRAPEGRAPH_API_KEY") + self.enable_logging = enable_logging if not self.api_key: raise ValueError("Scrapegraph API key is required") @@ -83,8 +86,9 @@ class ScrapegraphScrapeTool(BaseTool): if user_prompt is not None: self.user_prompt = user_prompt - # Configure logging - sgai_logger.set_logging(level="INFO") + # Configure logging only if enabled + if self.enable_logging: + sgai_logger.set_logging(level="INFO") @staticmethod def _validate_url(url: str) -> None: @@ -96,22 +100,6 @@ class ScrapegraphScrapeTool(BaseTool): except Exception: raise ValueError("Invalid URL format. URL must include scheme (http/https) and domain") - def _handle_api_response(self, response: dict) -> str: - """Handle and validate API response""" - if not response: - raise RuntimeError("Empty response from Scrapegraph API") - - if "error" in response: - error_msg = response.get("error", {}).get("message", "Unknown error") - if "rate limit" in error_msg.lower(): - raise RateLimitError(f"Rate limit exceeded: {error_msg}") - raise RuntimeError(f"API error: {error_msg}") - - if "result" not in response: - raise RuntimeError("Invalid response format from Scrapegraph API") - - return response["result"] - def _run( self, **kwargs: Any, @@ -135,8 +123,7 @@ class ScrapegraphScrapeTool(BaseTool): user_prompt=user_prompt, ) - # Handle and validate the response - return self._handle_api_response(response) + return response except RateLimitError: raise # Re-raise rate limit errors