andre/crewAI
1
0
Fork 0
mirror of https://github.com/crewAIInc/crewAI.git synced 2026-01-09 08:08:32 +00:00
Code Issues Actions 1 Packages Projects Releases Wiki Activity

Adding timeout to scrapping website tool

Browse Source
This commit is contained in:
João Moura
2024-04-07 14:18:41 -03:00
parent dd2349a90a
commit 9f41fb4057
1 changed files with 6 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
src/crewai_tools/tools/scrape_website_tool/scrape_website_tool.py
View File

@@ -44,7 +44,12 @@ class ScrapeWebsiteTool(BaseTool):
**kwargs: Any,
) -> Any:
website_url = kwargs.get('website_url', self.website_url)
page = requests.get(website_url, headers=self.headers, cookies=self.cookies if self.cookies else {})
page = requests.get(
website_url,
timeout=15,
headers=self.headers,
cookies=self.cookies if self.cookies else {}
)
parsed = BeautifulSoup(page.content, "html.parser")
text = parsed.get_text()
text = '\n'.join([i for i in text.split('\n') if i.strip() != ''])