From 9f41fb405732b79ff7897019069c4e1cd7b28589 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Moura?= Date: Sun, 7 Apr 2024 14:18:41 -0300 Subject: [PATCH] Adding timeout to scrapping website tool --- .../tools/scrape_website_tool/scrape_website_tool.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/crewai_tools/tools/scrape_website_tool/scrape_website_tool.py b/src/crewai_tools/tools/scrape_website_tool/scrape_website_tool.py index cd8fd50d3..148a0b320 100644 --- a/src/crewai_tools/tools/scrape_website_tool/scrape_website_tool.py +++ b/src/crewai_tools/tools/scrape_website_tool/scrape_website_tool.py @@ -44,7 +44,12 @@ class ScrapeWebsiteTool(BaseTool): **kwargs: Any, ) -> Any: website_url = kwargs.get('website_url', self.website_url) - page = requests.get(website_url, headers=self.headers, cookies=self.cookies if self.cookies else {}) + page = requests.get( + website_url, + timeout=15, + headers=self.headers, + cookies=self.cookies if self.cookies else {} + ) parsed = BeautifulSoup(page.content, "html.parser") text = parsed.get_text() text = '\n'.join([i for i in text.split('\n') if i.strip() != ''])