From 6c242ef3bbfe722d4159e60c931d20e2a38a0570 Mon Sep 17 00:00:00 2001 From: siddas27 Date: Sat, 30 Nov 2024 14:04:06 -0600 Subject: [PATCH 1/6] add brave search tool --- .../tools/brave_search_tool/README.md | 30 +++++++ .../brave_search_tool/brave_search_tool.py | 82 +++++++++++++++++++ tests/tools/brave_search_tool_test.py | 13 +++ 3 files changed, 125 insertions(+) create mode 100644 src/crewai_tools/tools/brave_search_tool/README.md create mode 100644 src/crewai_tools/tools/brave_search_tool/brave_search_tool.py create mode 100644 tests/tools/brave_search_tool_test.py diff --git a/src/crewai_tools/tools/brave_search_tool/README.md b/src/crewai_tools/tools/brave_search_tool/README.md new file mode 100644 index 000000000..a66210491 --- /dev/null +++ b/src/crewai_tools/tools/brave_search_tool/README.md @@ -0,0 +1,30 @@ +# BraveSearchTool Documentation + +## Description +This tool is designed to perform a web search for a specified query from a text's content across the internet. It utilizes the Brave Web Search API, which is a REST API to query Brave Search and get back search results from the web. The following sections describe how to curate requests, including parameters and headers, to Brave Web Search API and get a JSON response back. + +## Installation +To incorporate this tool into your project, follow the installation instructions below: +```shell +pip install 'crewai[tools]' +``` + +## Example +The following example demonstrates how to initialize the tool and execute a search with a given query: + +```python +from crewai_tools import BraveSearchTool + +# Initialize the tool for internet searching capabilities +tool = BraveSearchTool() +``` + +## Steps to Get Started +To effectively use the `BraveSearchTool`, follow these steps: + +1. **Package Installation**: Confirm that the `crewai[tools]` package is installed in your Python environment. +2. **API Key Acquisition**: Acquire a API key [here](https://api.search.brave.com/app/keys). +3. **Environment Configuration**: Store your obtained API key in an environment variable named `BRAVE_API_KEY` to facilitate its use by the tool. + +## Conclusion +By integrating the `BraveSearchTool` into Python projects, users gain the ability to conduct real-time, relevant searches across the internet directly from their applications. By adhering to the setup and usage guidelines provided, incorporating this tool into projects is streamlined and straightforward. diff --git a/src/crewai_tools/tools/brave_search_tool/brave_search_tool.py b/src/crewai_tools/tools/brave_search_tool/brave_search_tool.py new file mode 100644 index 000000000..54f546f1e --- /dev/null +++ b/src/crewai_tools/tools/brave_search_tool/brave_search_tool.py @@ -0,0 +1,82 @@ +import datetime +import os +from typing import Any, Optional, Type + +import requests +from pydantic import BaseModel, Field + +from crewai_tools.tools.base_tool import BaseTool + + +def _save_results_to_file(content: str) -> None: + """Saves the search results to a file.""" + filename = f"search_results_{datetime.now().strftime('%Y-%m-%d_%H-%M-%S')}.txt" + with open(filename, "w") as file: + file.write(content) + print(f"Results saved to {filename}") + + +class BraveSearchToolSchema(BaseModel): + """Input for BraveSearchTool.""" + + search_query: str = Field( + ..., description="Mandatory search query you want to use to search the internet" + ) + + +class BraveSearchTool(BaseTool): + name: str = "Search the internet" + description: str = ( + "A tool that can be used to search the internet with a search_query." + ) + args_schema: Type[BaseModel] = BraveSearchToolSchema + search_url: str = "https://api.search.brave.com/res/v1/web/search" + country: Optional[str] = "" + n_results: int = 10 + save_file: bool = False + + def _run( + self, + **kwargs: Any, + ) -> Any: + search_query = kwargs.get("search_query") or kwargs.get("query") + save_file = kwargs.get("save_file", self.save_file) + n_results = kwargs.get("n_results", self.n_results) + + payload = {"q": search_query, "count": n_results} + + if self.country != "": + payload["country"] = self.country + + headers = { + "X-Subscription-Token": os.environ["BRAVE_API_KEY"], + "Accept": "application/json", + } + + response = requests.get(self.search_url, headers=headers, params=payload) + results = response.json() + + if "web" in results: + results = results["web"]["results"] + string = [] + for result in results: + try: + string.append( + "\n".join( + [ + f"Title: {result['title']}", + f"Link: {result['url']}", + f"Snippet: {result['description']}", + "---", + ] + ) + ) + except KeyError: + continue + + content = "\n".join(string) + if save_file: + _save_results_to_file(content) + return f"\nSearch results: {content}\n" + else: + return results diff --git a/tests/tools/brave_search_tool_test.py b/tests/tools/brave_search_tool_test.py new file mode 100644 index 000000000..16c1bcb92 --- /dev/null +++ b/tests/tools/brave_search_tool_test.py @@ -0,0 +1,13 @@ +from crewai_tools.tools.brave_search_tool.brave_search_tool import BraveSearchTool + + +def test_brave_tool(): + tool = BraveSearchTool( + n_results=2, + ) + + print(tool.run(search_query="ChatGPT")) + + +if __name__ == "__main__": + test_brave_tool() From d168b8e24554e37a706d0af18c4b82af483fd442 Mon Sep 17 00:00:00 2001 From: siddas27 Date: Sat, 30 Nov 2024 21:36:28 -0600 Subject: [PATCH 2/6] add error handling --- .../tools/brave_search_tool/__init__.py | 0 .../brave_search_tool/brave_search_tool.py | 90 ++++++++++++------- tests/tools/brave_search_tool_test.py | 37 ++++++++ 3 files changed, 96 insertions(+), 31 deletions(-) create mode 100644 src/crewai_tools/tools/brave_search_tool/__init__.py diff --git a/src/crewai_tools/tools/brave_search_tool/__init__.py b/src/crewai_tools/tools/brave_search_tool/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/src/crewai_tools/tools/brave_search_tool/brave_search_tool.py b/src/crewai_tools/tools/brave_search_tool/brave_search_tool.py index 54f546f1e..6a8818d75 100644 --- a/src/crewai_tools/tools/brave_search_tool/brave_search_tool.py +++ b/src/crewai_tools/tools/brave_search_tool/brave_search_tool.py @@ -25,6 +25,18 @@ class BraveSearchToolSchema(BaseModel): class BraveSearchTool(BaseTool): + """ + BraveSearchTool - A tool for performing web searches using the Brave Search API. + + This module provides functionality to search the internet using Brave's Search API, + supporting customizable result counts and country-specific searches. + + Dependencies: + - requests + - pydantic + - python-dotenv (for API key management) + """ + name: str = "Search the internet" description: str = ( "A tool that can be used to search the internet with a search_query." @@ -35,48 +47,64 @@ class BraveSearchTool(BaseTool): n_results: int = 10 save_file: bool = False + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + if "BRAVE_API_KEY" not in os.environ: + raise ValueError( + "BRAVE_API_KEY environment variable is required for BraveSearchTool" + ) + def _run( self, **kwargs: Any, ) -> Any: - search_query = kwargs.get("search_query") or kwargs.get("query") - save_file = kwargs.get("save_file", self.save_file) - n_results = kwargs.get("n_results", self.n_results) + try: + search_query = kwargs.get("search_query") or kwargs.get("query") + if not search_query: + raise ValueError("Search query is required") - payload = {"q": search_query, "count": n_results} + save_file = kwargs.get("save_file", self.save_file) + n_results = kwargs.get("n_results", self.n_results) - if self.country != "": - payload["country"] = self.country + payload = {"q": search_query, "count": n_results} - headers = { - "X-Subscription-Token": os.environ["BRAVE_API_KEY"], - "Accept": "application/json", - } + if self.country != "": + payload["country"] = self.country - response = requests.get(self.search_url, headers=headers, params=payload) - results = response.json() + headers = { + "X-Subscription-Token": os.environ["BRAVE_API_KEY"], + "Accept": "application/json", + } - if "web" in results: - results = results["web"]["results"] - string = [] - for result in results: - try: - string.append( - "\n".join( - [ - f"Title: {result['title']}", - f"Link: {result['url']}", - f"Snippet: {result['description']}", - "---", - ] + response = requests.get(self.search_url, headers=headers, params=payload) + response.raise_for_status() # Handle non-200 responses + results = response.json() + + if "web" in results: + results = results["web"]["results"] + string = [] + for result in results: + try: + string.append( + "\n".join( + [ + f"Title: {result['title']}", + f"Link: {result['url']}", + f"Snippet: {result['description']}", + "---", + ] + ) ) - ) - except KeyError: - continue + except KeyError: + continue content = "\n".join(string) - if save_file: - _save_results_to_file(content) + except requests.RequestException as e: + return f"Error performing search: {str(e)}" + except KeyError as e: + return f"Error parsing search results: {str(e)}" + if save_file: + _save_results_to_file(content) return f"\nSearch results: {content}\n" else: - return results + return content diff --git a/tests/tools/brave_search_tool_test.py b/tests/tools/brave_search_tool_test.py index 16c1bcb92..969bd48fe 100644 --- a/tests/tools/brave_search_tool_test.py +++ b/tests/tools/brave_search_tool_test.py @@ -1,6 +1,41 @@ +from unittest.mock import patch + +import pytest + from crewai_tools.tools.brave_search_tool.brave_search_tool import BraveSearchTool +@pytest.fixture +def brave_tool(): + return BraveSearchTool(n_results=2) + + +def test_brave_tool_initialization(): + tool = BraveSearchTool() + assert tool.n_results == 10 + assert tool.save_file is False + + +@patch("requests.get") +def test_brave_tool_search(mock_get, brave_tool): + mock_response = { + "web": { + "results": [ + { + "title": "Test Title", + "url": "http://test.com", + "description": "Test Description", + } + ] + } + } + mock_get.return_value.json.return_value = mock_response + + result = brave_tool.run(search_query="test") + assert "Test Title" in result + assert "http://test.com" in result + + def test_brave_tool(): tool = BraveSearchTool( n_results=2, @@ -11,3 +46,5 @@ def test_brave_tool(): if __name__ == "__main__": test_brave_tool() + test_brave_tool_initialization() + # test_brave_tool_search(brave_tool) From 5532ea8ff72993860b85326d7299351a0b23c3b5 Mon Sep 17 00:00:00 2001 From: siddas27 Date: Sat, 30 Nov 2024 21:51:46 -0600 Subject: [PATCH 3/6] add lru caching --- src/crewai_tools/tools/brave_search_tool/brave_search_tool.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/crewai_tools/tools/brave_search_tool/brave_search_tool.py b/src/crewai_tools/tools/brave_search_tool/brave_search_tool.py index 6a8818d75..5ff451484 100644 --- a/src/crewai_tools/tools/brave_search_tool/brave_search_tool.py +++ b/src/crewai_tools/tools/brave_search_tool/brave_search_tool.py @@ -1,5 +1,6 @@ import datetime import os +from functools import lru_cache from typing import Any, Optional, Type import requests @@ -54,6 +55,7 @@ class BraveSearchTool(BaseTool): "BRAVE_API_KEY environment variable is required for BraveSearchTool" ) + @lru_cache(maxsize=100) def _run( self, **kwargs: Any, From e7e059d02a4fa09f2b13873643a4ce38c4c45dc2 Mon Sep 17 00:00:00 2001 From: siddas27 Date: Sat, 30 Nov 2024 22:08:29 -0600 Subject: [PATCH 4/6] add rate limiting --- .../tools/brave_search_tool/brave_search_tool.py | 13 ++++++++++--- tests/tools/brave_search_tool_test.py | 4 ++-- 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/src/crewai_tools/tools/brave_search_tool/brave_search_tool.py b/src/crewai_tools/tools/brave_search_tool/brave_search_tool.py index 5ff451484..8d6a9a182 100644 --- a/src/crewai_tools/tools/brave_search_tool/brave_search_tool.py +++ b/src/crewai_tools/tools/brave_search_tool/brave_search_tool.py @@ -1,7 +1,7 @@ import datetime import os -from functools import lru_cache -from typing import Any, Optional, Type +import time +from typing import Any, ClassVar, Optional, Type import requests from pydantic import BaseModel, Field @@ -47,6 +47,8 @@ class BraveSearchTool(BaseTool): country: Optional[str] = "" n_results: int = 10 save_file: bool = False + _last_request_time: ClassVar[float] = 0 + _min_request_interval: ClassVar[float] = 1.0 # seconds def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) @@ -55,11 +57,16 @@ class BraveSearchTool(BaseTool): "BRAVE_API_KEY environment variable is required for BraveSearchTool" ) - @lru_cache(maxsize=100) def _run( self, **kwargs: Any, ) -> Any: + current_time = time.time() + if (current_time - self._last_request_time) < self._min_request_interval: + time.sleep( + self._min_request_interval - (current_time - self._last_request_time) + ) + BraveSearchTool._last_request_time = time.time() try: search_query = kwargs.get("search_query") or kwargs.get("query") if not search_query: diff --git a/tests/tools/brave_search_tool_test.py b/tests/tools/brave_search_tool_test.py index 969bd48fe..36300f723 100644 --- a/tests/tools/brave_search_tool_test.py +++ b/tests/tools/brave_search_tool_test.py @@ -40,8 +40,8 @@ def test_brave_tool(): tool = BraveSearchTool( n_results=2, ) - - print(tool.run(search_query="ChatGPT")) + x = tool.run(search_query="ChatGPT") + print(x) if __name__ == "__main__": From 95cc6835a130a35fab52bae5c9e41e7073fc0ef0 Mon Sep 17 00:00:00 2001 From: siddas27 Date: Sat, 30 Nov 2024 22:30:31 -0600 Subject: [PATCH 5/6] update name --- src/crewai_tools/tools/brave_search_tool/brave_search_tool.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/crewai_tools/tools/brave_search_tool/brave_search_tool.py b/src/crewai_tools/tools/brave_search_tool/brave_search_tool.py index 8d6a9a182..dceff1d57 100644 --- a/src/crewai_tools/tools/brave_search_tool/brave_search_tool.py +++ b/src/crewai_tools/tools/brave_search_tool/brave_search_tool.py @@ -38,7 +38,7 @@ class BraveSearchTool(BaseTool): - python-dotenv (for API key management) """ - name: str = "Search the internet" + name: str = "Brave Web Search the internet" description: str = ( "A tool that can be used to search the internet with a search_query." ) From a64cccbd724a9c24fc825a16de0ec1de2fc39ed9 Mon Sep 17 00:00:00 2001 From: siddas27 Date: Wed, 4 Dec 2024 22:28:30 -0600 Subject: [PATCH 6/6] add BraveSearchTool to init --- src/crewai_tools/__init__.py | 3 ++- src/crewai_tools/tools/__init__.py | 13 +++++++------ 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/src/crewai_tools/__init__.py b/src/crewai_tools/__init__.py index 6bd8dfd71..5f9a81d9c 100644 --- a/src/crewai_tools/__init__.py +++ b/src/crewai_tools/__init__.py @@ -1,4 +1,5 @@ from .tools import ( + BraveSearchTool, BrowserbaseLoadTool, CodeDocsSearchTool, CodeInterpreterTool, @@ -19,6 +20,7 @@ from .tools import ( LlamaIndexTool, MDXSearchTool, MultiOnTool, + MySQLSearchTool, NL2SQLTool, PDFSearchTool, PGSearchTool, @@ -40,6 +42,5 @@ from .tools import ( XMLSearchTool, YoutubeChannelSearchTool, YoutubeVideoSearchTool, - MySQLSearchTool ) from .tools.base_tool import BaseTool, Tool, tool diff --git a/src/crewai_tools/tools/__init__.py b/src/crewai_tools/tools/__init__.py index 9016c57fd..73a96f4cf 100644 --- a/src/crewai_tools/tools/__init__.py +++ b/src/crewai_tools/tools/__init__.py @@ -1,3 +1,4 @@ +from .brave_search_tool.brave_search_tool import BraveSearchTool from .browserbase_load_tool.browserbase_load_tool import BrowserbaseLoadTool from .code_docs_search_tool.code_docs_search_tool import CodeDocsSearchTool from .code_interpreter_tool.code_interpreter_tool import CodeInterpreterTool @@ -11,10 +12,10 @@ from .exa_tools.exa_search_tool import EXASearchTool from .file_read_tool.file_read_tool import FileReadTool from .file_writer_tool.file_writer_tool import FileWriterTool from .firecrawl_crawl_website_tool.firecrawl_crawl_website_tool import ( - FirecrawlCrawlWebsiteTool + FirecrawlCrawlWebsiteTool, ) from .firecrawl_scrape_website_tool.firecrawl_scrape_website_tool import ( - FirecrawlScrapeWebsiteTool + FirecrawlScrapeWebsiteTool, ) from .firecrawl_search_tool.firecrawl_search_tool import FirecrawlSearchTool from .github_search_tool.github_search_tool import GithubSearchTool @@ -22,16 +23,17 @@ from .json_search_tool.json_search_tool import JSONSearchTool from .llamaindex_tool.llamaindex_tool import LlamaIndexTool from .mdx_seach_tool.mdx_search_tool import MDXSearchTool from .multion_tool.multion_tool import MultiOnTool +from .mysql_search_tool.mysql_search_tool import MySQLSearchTool from .nl2sql.nl2sql_tool import NL2SQLTool from .pdf_search_tool.pdf_search_tool import PDFSearchTool from .pg_seach_tool.pg_search_tool import PGSearchTool from .rag.rag_tool import RagTool from .scrape_element_from_website.scrape_element_from_website import ( - ScrapeElementFromWebsiteTool + ScrapeElementFromWebsiteTool, ) from .scrape_website_tool.scrape_website_tool import ScrapeWebsiteTool from .scrapfly_scrape_website_tool.scrapfly_scrape_website_tool import ( - ScrapflyScrapeWebsiteTool + ScrapflyScrapeWebsiteTool, ) from .selenium_scraping_tool.selenium_scraping_tool import SeleniumScrapingTool from .serper_dev_tool.serper_dev_tool import SerperDevTool @@ -46,7 +48,6 @@ from .vision_tool.vision_tool import VisionTool from .website_search.website_search_tool import WebsiteSearchTool from .xml_search_tool.xml_search_tool import XMLSearchTool from .youtube_channel_search_tool.youtube_channel_search_tool import ( - YoutubeChannelSearchTool + YoutubeChannelSearchTool, ) from .youtube_video_search_tool.youtube_video_search_tool import YoutubeVideoSearchTool -from .mysql_search_tool.mysql_search_tool import MySQLSearchTool