andre/crewAI
1
0
Fork 0
mirror of https://github.com/crewAIInc/crewAI.git synced 2026-01-10 08:38:30 +00:00
Code Issues Actions 4 Packages Projects Releases Wiki Activity

fixing serper tool

Browse Source
This commit is contained in:
João Moura
2024-07-19 00:28:27 -04:00
parent d8c98f2e64
commit 0386120a5a
2 changed files with 28 additions and 19 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
src/crewai_tools/tools/scrape_website_tool/scrape_website_tool.py
View File

@@ -49,9 +49,11 @@ class ScrapeWebsiteTool(BaseTool):
headers=self.headers, headers=self.headers,
cookies=self.cookies if self.cookies else {} cookies=self.cookies if self.cookies else {}
) )
parsed = BeautifulSoup(page.content, "html.parser")
page.encoding = page.apparent_encoding
parsed = BeautifulSoup(page.text, "html.parser")
text = parsed.get_text() text = parsed.get_text()
text = '\n'.join([i for i in text.split('\n') if i.strip() != '']) text = '\n'.join([i for i in text.split('\n') if i.strip() != ''])
text = ' '.join([i for i in text.split(' ') if i.strip() != '']) text = ' '.join([i for i in text.split(' ') if i.strip() != ''])
return text return text

23
src/crewai_tools/tools/serper_dev_tool/serper_dev_tool.py
View File

@@ -1,3 +1,4 @@
import datetime
import os import os
import json import json
import requests import requests
@@ -23,11 +24,11 @@ class SerperDevTool(BaseTool):
description: str = "A tool that can be used to search the internet with a search_query." description: str = "A tool that can be used to search the internet with a search_query."
args_schema: Type[BaseModel] = SerperDevToolSchema args_schema: Type[BaseModel] = SerperDevToolSchema
search_url: str = "https://google.serper.dev/search" search_url: str = "https://google.serper.dev/search"
country: Optional[str] = None country: Optional[str] = ''
location: Optional[str] = None location: Optional[str] = ''
locale: Optional[str] = None locale: Optional[str] = ''
n_results: int = Field(default=10, description="Number of search results to return") n_results: int = 10
save_file: bool = Field(default=False, description="Flag to determine whether to save the results to a file") save_file: bool = False
def _run( def _run(
self, self,
@@ -39,9 +40,13 @@ class SerperDevTool(BaseTool):
n_results = kwargs.get('n_results', self.n_results) n_results = kwargs.get('n_results', self.n_results)
payload = { "q": search_query, "num": n_results } payload = { "q": search_query, "num": n_results }
payload["gl"] = self.country if self.country
payload["location"] = self.country if self.location if self.country != '':
payload["hl"] = self.country if self.locale payload["gl"] = self.country
if self.location != '':
payload["location"] = self.location
if self.locale != '':
payload["hl"] = self.locale
payload = json.dumps(payload) payload = json.dumps(payload)
@@ -49,8 +54,10 @@ class SerperDevTool(BaseTool):
'X-API-KEY': os.environ['SERPER_API_KEY'], 'X-API-KEY': os.environ['SERPER_API_KEY'],
'content-type': 'application/json' 'content-type': 'application/json'
} }
response = requests.request("POST", self.search_url, headers=headers, data=payload) response = requests.request("POST", self.search_url, headers=headers, data=payload)
results = response.json() results = response.json()
if 'organic' in results: if 'organic' in results:
results = results['organic'][:self.n_results] results = results['organic'][:self.n_results]
string = [] string = []