diff --git a/src/crewai_tools/tools/scrape_element_from_website/scrape_element_from_website.py b/src/crewai_tools/tools/scrape_element_from_website/scrape_element_from_website.py index 54de3cd39..bee6c22ab 100644 --- a/src/crewai_tools/tools/scrape_element_from_website/scrape_element_from_website.py +++ b/src/crewai_tools/tools/scrape_element_from_website/scrape_element_from_website.py @@ -19,7 +19,11 @@ class ScrapeElementFromWebsiteTool(BaseTool): args_schema: Type[BaseModel] = ScrapeElementFromWebsiteToolSchema website_url: Optional[str] = None css_element: Optional[str] = None - headers: Optional[dict] = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.3'} + headers: Optional[dict] = { + 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.3', + 'Accept-Language': 'en-US,en;q=0.5', + 'Referer': 'https://www.google.com/' + } def __init__(self, website_url: Optional[str] = None, css_element: Optional[str] = None, **kwargs): super().__init__(**kwargs) diff --git a/src/crewai_tools/tools/scrape_website_tool/scrape_website_tool.py b/src/crewai_tools/tools/scrape_website_tool/scrape_website_tool.py index e672a9b1d..240948a33 100644 --- a/src/crewai_tools/tools/scrape_website_tool/scrape_website_tool.py +++ b/src/crewai_tools/tools/scrape_website_tool/scrape_website_tool.py @@ -17,7 +17,11 @@ class ScrapeWebsiteTool(BaseTool): description: str = "A tool that can be used to read a website content." args_schema: Type[BaseModel] = ScrapeWebsiteToolSchema website_url: Optional[str] = None - headers: Optional[dict] = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.3'} + headers: Optional[dict] = { + 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.3', + 'Accept-Language': 'en-US,en;q=0.5', + 'Referer': 'https://www.google.com/' + } def __init__(self, website_url: Optional[str] = None, **kwargs): super().__init__(**kwargs)