From 56a9060840f4f33b63a0dde5ebce3c985de61e60 Mon Sep 17 00:00:00 2001 From: Terry Tan Yongsheng Date: Tue, 17 Dec 2024 10:35:33 +0800 Subject: [PATCH 1/3] Add SerpApi tools - google search, google shopping --- src/crewai_tools/__init__.py | 2 + src/crewai_tools/tools/__init__.py | 2 + .../tools/serpapi_tool/serpapi_base_tool.py | 37 ++++++++++++++++ .../serpapi_google_search_tool.py | 40 ++++++++++++++++++ .../serpapi_google_shopping_tool.py | 42 +++++++++++++++++++ 5 files changed, 123 insertions(+) create mode 100644 src/crewai_tools/tools/serpapi_tool/serpapi_base_tool.py create mode 100644 src/crewai_tools/tools/serpapi_tool/serpapi_google_search_tool.py create mode 100644 src/crewai_tools/tools/serpapi_tool/serpapi_google_shopping_tool.py diff --git a/src/crewai_tools/__init__.py b/src/crewai_tools/__init__.py index 12523a214..87aca8531 100644 --- a/src/crewai_tools/__init__.py +++ b/src/crewai_tools/__init__.py @@ -43,4 +43,6 @@ from .tools import ( YoutubeChannelSearchTool, YoutubeVideoSearchTool, WeaviateVectorSearchTool, + SerpApiGoogleSearchTool, + SerpApiGoogleShoppingTool, ) diff --git a/src/crewai_tools/tools/__init__.py b/src/crewai_tools/tools/__init__.py index 23565dbea..f6c31f45f 100644 --- a/src/crewai_tools/tools/__init__.py +++ b/src/crewai_tools/tools/__init__.py @@ -52,3 +52,5 @@ from .youtube_channel_search_tool.youtube_channel_search_tool import ( ) from .youtube_video_search_tool.youtube_video_search_tool import YoutubeVideoSearchTool from .weaviate_tool.vector_search import WeaviateVectorSearchTool +from .serpapi_tool.serpapi_google_search_tool import SerpApiGoogleSearchTool +from .serpapi_tool.serpapi_google_shopping_tool import SerpApiGoogleShoppingTool \ No newline at end of file diff --git a/src/crewai_tools/tools/serpapi_tool/serpapi_base_tool.py b/src/crewai_tools/tools/serpapi_tool/serpapi_base_tool.py new file mode 100644 index 000000000..57e33e71e --- /dev/null +++ b/src/crewai_tools/tools/serpapi_tool/serpapi_base_tool.py @@ -0,0 +1,37 @@ +import os +import re +from typing import Optional, Any + +from crewai.tools import BaseTool + +class SerpApiBaseTool(BaseTool): + client: Optional[Any] = None + + def __init__(self, **kwargs): + super().__init__(**kwargs) + + try: + from serpapi import Client + except ImportError: + raise ImportError( + "`serpapi` package not found" + ) + api_key = os.getenv("SERPAPI_API_KEY") + if not api_key: + raise ValueError( + "Missing API key, you can get the key from https://serpapi.com/manage-api-key" + ) + self.client = Client(api_key=api_key) + + def _omit_fields(self, data, omit_patterns): + if isinstance(data, dict): + for field in list(data.keys()): + if any(re.compile(p).match(field) for p in omit_patterns): + data.pop(field, None) + else: + if isinstance(data[field], (dict, list)): + self._omit_fields(data[field], omit_patterns) + elif isinstance(data, list): + for item in data: + self._omit_fields(item, omit_patterns) + diff --git a/src/crewai_tools/tools/serpapi_tool/serpapi_google_search_tool.py b/src/crewai_tools/tools/serpapi_tool/serpapi_google_search_tool.py new file mode 100644 index 000000000..199b7f5a2 --- /dev/null +++ b/src/crewai_tools/tools/serpapi_tool/serpapi_google_search_tool.py @@ -0,0 +1,40 @@ +from typing import Any, Type, Optional + +import re +from pydantic import BaseModel, Field +from .serpapi_base_tool import SerpApiBaseTool +from serpapi import HTTPError + +class SerpApiGoogleSearchToolSchema(BaseModel): + """Input for Google Search.""" + search_query: str = Field(..., description="Mandatory search query you want to use to Google search.") + location: Optional[str] = Field(None, description="Location you want the search to be performed in.") + +class SerpApiGoogleSearchTool(SerpApiBaseTool): + name: str = "Google Search" + description: str = ( + "A tool to perform to perform a Google search with a search_query." + ) + args_schema: Type[BaseModel] = SerpApiGoogleSearchToolSchema + + def _run( + self, + **kwargs: Any, + ) -> Any: + try: + results = self.client.search({ + "q": kwargs.get("search_query"), + "location": kwargs.get("location"), + }).as_dict() + + self._omit_fields( + results, + [r"search_metadata", r"search_parameters", r"serpapi_.+", r".+_token", r"displayed_link", r"pagination"] + ) + + return results + except HTTPError as e: + return f"An error occurred: {str(e)}. Some parameters may be invalid." + + + \ No newline at end of file diff --git a/src/crewai_tools/tools/serpapi_tool/serpapi_google_shopping_tool.py b/src/crewai_tools/tools/serpapi_tool/serpapi_google_shopping_tool.py new file mode 100644 index 000000000..b44b3a809 --- /dev/null +++ b/src/crewai_tools/tools/serpapi_tool/serpapi_google_shopping_tool.py @@ -0,0 +1,42 @@ +from typing import Any, Type, Optional + +import re +from pydantic import BaseModel, Field +from .serpapi_base_tool import SerpApiBaseTool +from serpapi import HTTPError + +class SerpApiGoogleShoppingToolSchema(BaseModel): + """Input for Google Shopping.""" + search_query: str = Field(..., description="Mandatory search query you want to use to Google shopping.") + location: Optional[str] = Field(None, description="Location you want the search to be performed in.") + + +class SerpApiGoogleShoppingTool(SerpApiBaseTool): + name: str = "Google Shopping" + description: str = ( + "A tool to perform search on Google shopping with a search_query." + ) + args_schema: Type[BaseModel] = SerpApiGoogleShoppingToolSchema + + def _run( + self, + **kwargs: Any, + ) -> Any: + try: + results = self.client.search({ + "engine": "google_shopping", + "q": kwargs.get("search_query"), + "location": kwargs.get("location") + }).as_dict() + + self._omit_fields( + results, + [r"search_metadata", r"search_parameters", r"serpapi_.+", r"filters", r"pagination"] + ) + + return results + except HTTPError as e: + return f"An error occurred: {str(e)}. Some parameters may be invalid." + + + \ No newline at end of file From 2effe9a7d2ebacb063f8814e5974041161d839ef Mon Sep 17 00:00:00 2001 From: Terry Tan Yongsheng Date: Tue, 17 Dec 2024 11:09:38 +0800 Subject: [PATCH 2/3] Add README --- src/crewai_tools/tools/serpapi_tool/README.md | 32 +++++++++++++++++++ 1 file changed, 32 insertions(+) create mode 100644 src/crewai_tools/tools/serpapi_tool/README.md diff --git a/src/crewai_tools/tools/serpapi_tool/README.md b/src/crewai_tools/tools/serpapi_tool/README.md new file mode 100644 index 000000000..d81b851f8 --- /dev/null +++ b/src/crewai_tools/tools/serpapi_tool/README.md @@ -0,0 +1,32 @@ +# SerpApi Tools + +## Description +[SerpApi](https://serpapi.com/) tools are built for searching information in the internet. It currently supports: +- Google Search +- Google Shopping + +To successfully make use of SerpApi tools, you have to have `SERPAPI_API_KEY` set in the environment. To get the API key, register a free account at [SerpApi](https://serpapi.com/). + +## Installation +To start using the SerpApi Tools, you must first install the `crewai_tools` package. This can be easily done with the following command: + +```shell +pip install 'crewai[tools]' +``` + +## Examples +The following example demonstrates how to initialize the tool + +### Google Search +```python +from crewai_tools import SerpApiGoogleSearchTool + +tool = SerpApiGoogleSearchTool() +``` + +### Google Shopping +```python +from crewai_tools import SerpApiGoogleShoppingTool + +tool = SerpApiGoogleShoppingTool() +``` From 81981e43b668dc1d2073fa1d7defbcfa97e452ac Mon Sep 17 00:00:00 2001 From: Terry Tan Yongsheng Date: Tue, 17 Dec 2024 13:45:50 +0800 Subject: [PATCH 3/3] Add type hints --- src/crewai_tools/tools/serpapi_tool/serpapi_base_tool.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/crewai_tools/tools/serpapi_tool/serpapi_base_tool.py b/src/crewai_tools/tools/serpapi_tool/serpapi_base_tool.py index 57e33e71e..98491190c 100644 --- a/src/crewai_tools/tools/serpapi_tool/serpapi_base_tool.py +++ b/src/crewai_tools/tools/serpapi_tool/serpapi_base_tool.py @@ -1,10 +1,12 @@ import os import re -from typing import Optional, Any +from typing import Optional, Any, Union from crewai.tools import BaseTool class SerpApiBaseTool(BaseTool): + """Base class for SerpApi functionality with shared capabilities.""" + client: Optional[Any] = None def __init__(self, **kwargs): @@ -14,7 +16,7 @@ class SerpApiBaseTool(BaseTool): from serpapi import Client except ImportError: raise ImportError( - "`serpapi` package not found" + "`serpapi` package not found, please install with `pip install serpapi`" ) api_key = os.getenv("SERPAPI_API_KEY") if not api_key: @@ -23,7 +25,7 @@ class SerpApiBaseTool(BaseTool): ) self.client = Client(api_key=api_key) - def _omit_fields(self, data, omit_patterns): + def _omit_fields(self, data: Union[dict, list], omit_patterns: list[str]) -> None: if isinstance(data, dict): for field in list(data.keys()): if any(re.compile(p).match(field) for p in omit_patterns): @@ -34,4 +36,3 @@ class SerpApiBaseTool(BaseTool): elif isinstance(data, list): for item in data: self._omit_fields(item, omit_patterns) -