andre/crewAI
1
0
Fork 0
mirror of https://github.com/crewAIInc/crewAI.git synced 2026-01-10 00:28:31 +00:00
Code Issues Actions 1 Packages Projects Releases Wiki Activity

Merge pull request #116 from ernestp/main

Browse Source 
fix: web scraper concatenate words
This commit is contained in:
João Moura
2024-12-05 12:00:20 -03:00
committed by GitHub
parent 7c375976ab 1f8791953e
commit e1482d740f
1 changed files with 4 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
src/crewai_tools/tools/scrape_website_tool/scrape_website_tool.py
View File

@@ -1,4 +1,5 @@
import os
import re
from typing import Any, Optional, Type
import requests
@@ -67,7 +68,7 @@ class ScrapeWebsiteTool(BaseTool):
page.encoding = page.apparent_encoding
parsed = BeautifulSoup(page.text, "html.parser")
text = parsed.get_text()
text = "\n".join([i for i in text.split("\n") if i.strip() != ""])
text = " ".join([i for i in text.split(" ") if i.strip() != ""])
text = parsed.get_text(" ")
text = re.sub('[ \t]+', ' ', text)
text = re.sub('\\s+\n\\s+', '\n', text)
return text