andre/crewAI
1
0
Fork 0
mirror of https://github.com/crewAIInc/crewAI.git synced 2026-01-09 08:08:32 +00:00
Code Issues Actions 1 Packages Projects Releases Wiki Activity

fix: web scraper concatenate words

Browse Source
This commit is contained in:
Ernest Poletaev
2024-10-25 22:03:59 +07:00
parent 9eac65f9f6
commit 96e52767ad
1 changed files with 3 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
src/crewai_tools/tools/scrape_website_tool/scrape_website_tool.py
View File

@@ -1,4 +1,5 @@
import os import os
import re
from typing import Any, Optional, Type from typing import Any, Optional, Type
import requests import requests
@@ -67,7 +68,6 @@ class ScrapeWebsiteTool(BaseTool):
page.encoding = page.apparent_encoding page.encoding = page.apparent_encoding
parsed = BeautifulSoup(page.text, "html.parser") parsed = BeautifulSoup(page.text, "html.parser")
text = parsed.get_text() text = parsed.get_text(" ")
text = "\n".join([i for i in text.split("\n") if i.strip() != ""]) text = re.sub('\s+', ' ', text)
text = " ".join([i for i in text.split(" ") if i.strip() != ""])
return text return text