andre/crewAI
1
0
Fork 0
mirror of https://github.com/crewAIInc/crewAI.git synced 2026-01-26 08:38:15 +00:00
Code Issues Actions 4 Packages Projects Releases Wiki Activity

improving scrapping tools

Browse Source
This commit is contained in:
João Moura
2024-02-26 06:20:15 -03:00
parent 9e560ff951
commit cff6082f1c
2 changed files with 4 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
src/crewai_tools/tools/scrape_element_from_website/scrape_element_from_website.py
View File

@@ -19,6 +19,7 @@ class ScrapeElementFromWebsiteTool(BaseTool):
args_schema: Type[BaseModel] = ScrapeElementFromWebsiteToolSchema
website_url: Optional[str] = None
css_element: Optional[str] = None
headers: Optional[dict] = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.3'}
def __init__(self, website_url: Optional[str] = None, css_element: Optional[str] = None, **kwargs):
super().__init__(**kwargs)
@@ -34,7 +35,7 @@ class ScrapeElementFromWebsiteTool(BaseTool):
) -> Any:
website_url = kwargs.get('website_url', self.website_url)
css_element = kwargs.get('css_element', self.css_element)
page = requests.get(website_url)
page = requests.get(website_url, headers=self.headers)
parsed = BeautifulSoup(page.content, "html.parser")
elements = parsed.select(css_element)
return "\n".join([element.get_text() for element in elements])

3
src/crewai_tools/tools/scrape_website_tool/scrape_website_tool.py
View File

@@ -17,6 +17,7 @@ class ScrapeWebsiteTool(BaseTool):
description: str = "A tool that can be used to read a website content."
args_schema: Type[BaseModel] = ScrapeWebsiteToolSchema
website_url: Optional[str] = None
headers: Optional[dict] = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.3'}
def __init__(self, website_url: Optional[str] = None, **kwargs):
super().__init__(**kwargs)
@@ -30,7 +31,7 @@ class ScrapeWebsiteTool(BaseTool):
**kwargs: Any,
) -> Any:
website_url = kwargs.get('website_url', self.website_url)
page = requests.get(website_url)
page = requests.get(website_url, headers=self.headers)
parsed = BeautifulSoup(page.content, "html.parser")
return parsed.get_text()