andre/crewAI
1
0
Fork 0
mirror of https://github.com/crewAIInc/crewAI.git synced 2026-01-11 17:18:29 +00:00
Code Issues Actions 2 Packages Projects Releases Wiki Activity

Cutting new version with improved scrapping

Browse Source
This commit is contained in:
João Moura
2024-02-26 06:52:22 -03:00
parent cff6082f1c
commit 79eec51c9a
2 changed files with 10 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
src/crewai_tools/tools/scrape_element_from_website/scrape_element_from_website.py
View File

@@ -19,7 +19,11 @@ class ScrapeElementFromWebsiteTool(BaseTool):
args_schema: Type[BaseModel] = ScrapeElementFromWebsiteToolSchema
website_url: Optional[str] = None
css_element: Optional[str] = None
headers: Optional[dict] = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.3'}
headers: Optional[dict] = {
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.3',
'Accept-Language': 'en-US,en;q=0.5',
'Referer': 'https://www.google.com/'
}
def __init__(self, website_url: Optional[str] = None, css_element: Optional[str] = None, **kwargs):
super().__init__(**kwargs)

6
src/crewai_tools/tools/scrape_website_tool/scrape_website_tool.py
View File

@@ -17,7 +17,11 @@ class ScrapeWebsiteTool(BaseTool):
description: str = "A tool that can be used to read a website content."
args_schema: Type[BaseModel] = ScrapeWebsiteToolSchema
website_url: Optional[str] = None
headers: Optional[dict] = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.3'}
headers: Optional[dict] = {
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.3',
'Accept-Language': 'en-US,en;q=0.5',
'Referer': 'https://www.google.com/'
}
def __init__(self, website_url: Optional[str] = None, **kwargs):
super().__init__(**kwargs)