From a51a7000c5e84c9612f33e5d7fc378b1efdc033a Mon Sep 17 00:00:00 2001 From: rafaelsideguide <150964962+rafaelsideguide@users.noreply.github.com> Date: Thu, 16 May 2024 11:20:36 -0300 Subject: [PATCH 01/20] added Firecrawl tools --- .../firecrawl_crawl_website_tool/README.md | 42 +++++++++++++++++++ .../firecrawl_crawl_website_tool.py | 33 +++++++++++++++ .../firecrawl_scrape_website_tool/README.md | 38 +++++++++++++++++ .../firecrawl_scrape_website_tool.py | 35 ++++++++++++++++ .../tools/firecrawl_search_tool/README.md | 35 ++++++++++++++++ .../firecrawl_search_tool.py | 33 +++++++++++++++ 6 files changed, 216 insertions(+) create mode 100644 src/crewai_tools/tools/firecrawl_crawl_website_tool/README.md create mode 100644 src/crewai_tools/tools/firecrawl_crawl_website_tool/firecrawl_crawl_website_tool.py create mode 100644 src/crewai_tools/tools/firecrawl_scrape_website_tool/README.md create mode 100644 src/crewai_tools/tools/firecrawl_scrape_website_tool/firecrawl_scrape_website_tool.py create mode 100644 src/crewai_tools/tools/firecrawl_search_tool/README.md create mode 100644 src/crewai_tools/tools/firecrawl_search_tool/firecrawl_search_tool.py diff --git a/src/crewai_tools/tools/firecrawl_crawl_website_tool/README.md b/src/crewai_tools/tools/firecrawl_crawl_website_tool/README.md new file mode 100644 index 000000000..46d011602 --- /dev/null +++ b/src/crewai_tools/tools/firecrawl_crawl_website_tool/README.md @@ -0,0 +1,42 @@ +# FirecrawlCrawlWebsiteTool + +## Description + +[Firecrawl](https://firecrawl.dev) is a platform for crawling and convert any website into clean markdown or structured data. + +## Installation + +- Get an API key from [firecrawl.dev](https://firecrawl.dev) and set it in environment variables (`FIRECRAWL_API_KEY`). +- Install the [Firecrawl SDK](https://github.com/mendableai/firecrawl) along with `crewai[tools]` package: + +``` +pip install firecrawl-py 'crewai[tools]' +``` + +## Example + +Utilize the FirecrawlScrapeFromWebsiteTool as follows to allow your agent to load websites: + +```python +from crewai_tools import FirecrawlCrawlWebsiteTool + +tool = FirecrawlCrawlWebsiteTool(url='firecrawl.dev') +``` + +## Arguments + +- `api_key`: Optional. Specifies Firecrawl API key. Defaults is the `FIRECRAWL_API_KEY` environment variable. +- `url`: The base URL to start crawling from. +- `page_options`: Optional. + - `onlyMainContent`: Optional. Only return the main content of the page excluding headers, navs, footers, etc. + - `includeHtml`: Optional. Include the raw HTML content of the page. Will output a html key in the response. +- `crawler_options`: Optional. Options for controlling the crawling behavior. + - `includes`: Optional. URL patterns to include in the crawl. + - `exclude`: Optional. URL patterns to exclude from the crawl. + - `generateImgAltText`: Optional. Generate alt text for images using LLMs (requires a paid plan). + - `returnOnlyUrls`: Optional. If true, returns only the URLs as a list in the crawl status. Note: the response will be a list of URLs inside the data, not a list of documents. + - `maxDepth`: Optional. Maximum depth to crawl. Depth 1 is the base URL, depth 2 includes the base URL and its direct children, and so on. + - `mode`: Optional. The crawling mode to use. Fast mode crawls 4x faster on websites without a sitemap but may not be as accurate and shouldn't be used on heavily JavaScript-rendered websites. + - `limit`: Optional. Maximum number of pages to crawl. + - `timeout`: Optional. Timeout in milliseconds for the crawling operation. + diff --git a/src/crewai_tools/tools/firecrawl_crawl_website_tool/firecrawl_crawl_website_tool.py b/src/crewai_tools/tools/firecrawl_crawl_website_tool/firecrawl_crawl_website_tool.py new file mode 100644 index 000000000..5c796189a --- /dev/null +++ b/src/crewai_tools/tools/firecrawl_crawl_website_tool/firecrawl_crawl_website_tool.py @@ -0,0 +1,33 @@ +from typing import Optional, Any, Type, Dict, List +from pydantic.v1 import BaseModel, Field +from crewai_tools.tools.base_tool import BaseTool + +class FirecrawlCrawlWebsiteToolSchema(BaseModel): + url: str = Field(description="Website URL") + crawler_options: Optional[Dict[str, Any]] = Field(default=None, description="Options for crawling") + page_options: Optional[Dict[str, Any]] = Field(default=None, description="Options for page") + +class FirecrawlCrawlWebsiteTool(BaseTool): + name: str = "Firecrawl web crawl tool" + description: str = "Crawl webpages using Firecrawl and return the contents" + args_schema: Type[BaseModel] = FirecrawlCrawlWebsiteToolSchema + api_key: Optional[str] = None + firecrawl: Optional[Any] = None + + def __init__(self, api_key: Optional[str] = None, **kwargs): + super().__init__(**kwargs) + try: + from firecrawl import FirecrawlApp # type: ignore + except ImportError: + raise ImportError( + "`firecrawl` package not found, please run `pip install firecrawl-py`" + ) + + self.firecrawl = FirecrawlApp(api_key=api_key) + + def _run(self, url: str, crawler_options: Optional[Dict[str, Any]] = None, page_options: Optional[Dict[str, Any]] = None): + options = { + "crawlerOptions": crawler_options, + "pageOptions": page_options + } + return self.firecrawl.crawl_url(url, options) \ No newline at end of file diff --git a/src/crewai_tools/tools/firecrawl_scrape_website_tool/README.md b/src/crewai_tools/tools/firecrawl_scrape_website_tool/README.md new file mode 100644 index 000000000..93570f06b --- /dev/null +++ b/src/crewai_tools/tools/firecrawl_scrape_website_tool/README.md @@ -0,0 +1,38 @@ +# FirecrawlScrapeWebsiteTool + +## Description + +[Firecrawl](https://firecrawl.dev) is a platform for crawling and convert any website into clean markdown or structured data. + +## Installation + +- Get an API key from [firecrawl.dev](https://firecrawl.dev) and set it in environment variables (`FIRECRAWL_API_KEY`). +- Install the [Firecrawl SDK](https://github.com/mendableai/firecrawl) along with `crewai[tools]` package: + +``` +pip install firecrawl-py 'crewai[tools]' +``` + +## Example + +Utilize the FirecrawlScrapeWebsiteTool as follows to allow your agent to load websites: + +```python +from crewai_tools import FirecrawlScrapeWebsiteTool + +tool = FirecrawlScrapeWebsiteTool(url='firecrawl.dev') +``` + +## Arguments + +- `api_key`: Optional. Specifies Firecrawl API key. Defaults is the `FIRECRAWL_API_KEY` environment variable. +- `url`: The URL to scrape. +- `page_options`: Optional. + - `onlyMainContent`: Optional. Only return the main content of the page excluding headers, navs, footers, etc. + - `includeHtml`: Optional. Include the raw HTML content of the page. Will output a html key in the response. +- `extractor_options`: Optional. Options for LLM-based extraction of structured information from the page content + - `mode`: The extraction mode to use, currently supports 'llm-extraction' + - `extractionPrompt`: Optional. A prompt describing what information to extract from the page + - `extractionSchema`: Optional. The schema for the data to be extracted +- `timeout`: Optional. Timeout in milliseconds for the request + diff --git a/src/crewai_tools/tools/firecrawl_scrape_website_tool/firecrawl_scrape_website_tool.py b/src/crewai_tools/tools/firecrawl_scrape_website_tool/firecrawl_scrape_website_tool.py new file mode 100644 index 000000000..8540b13ff --- /dev/null +++ b/src/crewai_tools/tools/firecrawl_scrape_website_tool/firecrawl_scrape_website_tool.py @@ -0,0 +1,35 @@ +from typing import Optional, Any, Type, Dict +from pydantic.v1 import BaseModel, Field +from crewai_tools.tools.base_tool import BaseTool + +class FirecrawlScrapeWebsiteToolSchema(BaseModel): + url: str = Field(description="Website URL") + page_options: Optional[Dict[str, Any]] = Field(default=None, description="Options for page scraping") + extractor_options: Optional[Dict[str, Any]] = Field(default=None, description="Options for data extraction") + timeout: Optional[int] = Field(default=None, description="Timeout for the scraping operation") + +class FirecrawlScrapeWebsiteTool(BaseTool): + name: str = "Firecrawl web scrape tool" + description: str = "Scrape webpages url using Firecrawl and return the contents" + args_schema: Type[BaseModel] = FirecrawlScrapeWebsiteToolSchema + api_key: Optional[str] = None + firecrawl: Optional[Any] = None + + def __init__(self, api_key: Optional[str] = None, **kwargs): + super().__init__(**kwargs) + try: + from firecrawl import FirecrawlApp # type: ignore + except ImportError: + raise ImportError( + "`firecrawl` package not found, please run `pip install firecrawl-py`" + ) + + self.firecrawl = FirecrawlApp(api_key=api_key) + + def _run(self, url: str, page_options: Optional[Dict[str, Any]] = None, extractor_options: Optional[Dict[str, Any]] = None, timeout: Optional[int] = None): + options = { + "pageOptions": page_options, + "extractorOptions": extractor_options, + "timeout": timeout + } + return self.firecrawl.scrape_url(url, options) \ No newline at end of file diff --git a/src/crewai_tools/tools/firecrawl_search_tool/README.md b/src/crewai_tools/tools/firecrawl_search_tool/README.md new file mode 100644 index 000000000..effb3f3d4 --- /dev/null +++ b/src/crewai_tools/tools/firecrawl_search_tool/README.md @@ -0,0 +1,35 @@ +# FirecrawlSearchTool + +## Description + +[Firecrawl](https://firecrawl.dev) is a platform for crawling and convert any website into clean markdown or structured data. + +## Installation + +- Get an API key from [firecrawl.dev](https://firecrawl.dev) and set it in environment variables (`FIRECRAWL_API_KEY`). +- Install the [Firecrawl SDK](https://github.com/mendableai/firecrawl) along with `crewai[tools]` package: + +``` +pip install firecrawl-py 'crewai[tools]' +``` + +## Example + +Utilize the FirecrawlSearchTool as follows to allow your agent to load websites: + +```python +from crewai_tools import FirecrawlSearchTool + +tool = FirecrawlSearchTool(query='what is firecrawl?') +``` + +## Arguments + +- `api_key`: Optional. Specifies Firecrawl API key. Defaults is the `FIRECRAWL_API_KEY` environment variable. +- `query`: The search query string to be used for searching. +- `page_options`: Optional. Options for result formatting. + - `onlyMainContent`: Optional. Only return the main content of the page excluding headers, navs, footers, etc. + - `includeHtml`: Optional. Include the raw HTML content of the page. Will output a html key in the response. + - `fetchPageContent`: Optional. Fetch the full content of the page. +- `search_options`: Optional. Options for controlling the crawling behavior. + - `limit`: Optional. Maximum number of pages to crawl. \ No newline at end of file diff --git a/src/crewai_tools/tools/firecrawl_search_tool/firecrawl_search_tool.py b/src/crewai_tools/tools/firecrawl_search_tool/firecrawl_search_tool.py new file mode 100644 index 000000000..89843f797 --- /dev/null +++ b/src/crewai_tools/tools/firecrawl_search_tool/firecrawl_search_tool.py @@ -0,0 +1,33 @@ +from typing import Optional, Any, Type, Dict, List +from pydantic.v1 import BaseModel, Field +from crewai_tools.tools.base_tool import BaseTool + +class FirecrawlSearchToolSchema(BaseModel): + query: str = Field(description="Search query") + page_options: Optional[Dict[str, Any]] = Field(default=None, description="Options for result formatting") + search_options: Optional[Dict[str, Any]] = Field(default=None, description="Options for searching") + +class FirecrawlSearchTool(BaseTool): + name: str = "Firecrawl web search tool" + description: str = "Search webpages using Firecrawl and return the results" + args_schema: Type[BaseModel] = FirecrawlSearchToolSchema + api_key: Optional[str] = None + firecrawl: Optional[Any] = None + + def __init__(self, api_key: Optional[str] = None, **kwargs): + super().__init__(**kwargs) + try: + from firecrawl import FirecrawlApp # type: ignore + except ImportError: + raise ImportError( + "`firecrawl` package not found, please run `pip install firecrawl-py`" + ) + + self.firecrawl = FirecrawlApp(api_key=api_key) + + def _run(self, query: str, page_options: Optional[Dict[str, Any]] = None, result_options: Optional[Dict[str, Any]] = None): + options = { + "pageOptions": page_options, + "resultOptions": result_options + } + return self.firecrawl.search(query, options) From 2b47377a7849be73c8ecac19338aaa7811bdbd27 Mon Sep 17 00:00:00 2001 From: Eduardo Chiarotti Date: Wed, 19 Jun 2024 20:45:04 -0300 Subject: [PATCH 02/20] feat: add code-interpreter tool --- .../tools/code_interpreter_tool/Dockerfile | 21 +++++++ .../tools/code_interpreter_tool/README.md | 0 .../code_interpreter_tool.py | 57 +++++++++++++++++++ 3 files changed, 78 insertions(+) create mode 100644 src/crewai_tools/tools/code_interpreter_tool/Dockerfile create mode 100644 src/crewai_tools/tools/code_interpreter_tool/README.md create mode 100644 src/crewai_tools/tools/code_interpreter_tool/code_interpreter_tool.py diff --git a/src/crewai_tools/tools/code_interpreter_tool/Dockerfile b/src/crewai_tools/tools/code_interpreter_tool/Dockerfile new file mode 100644 index 000000000..b72a51a88 --- /dev/null +++ b/src/crewai_tools/tools/code_interpreter_tool/Dockerfile @@ -0,0 +1,21 @@ +# Use an official Ubuntu as a parent image +FROM ubuntu:20.04 + +# Set environment variables +ENV DEBIAN_FRONTEND=noninteractive + +# Install common utilities +RUN apt-get update && apt-get install -y \ + build-essential \ + curl \ + wget \ + software-properties-common + +# Install Python +RUN apt-get install -y python3 python3-pip + +# Clean up +RUN apt-get clean && rm -rf /var/lib/apt/lists/* + +# Set the working directory +WORKDIR /workspace diff --git a/src/crewai_tools/tools/code_interpreter_tool/README.md b/src/crewai_tools/tools/code_interpreter_tool/README.md new file mode 100644 index 000000000..e69de29bb diff --git a/src/crewai_tools/tools/code_interpreter_tool/code_interpreter_tool.py b/src/crewai_tools/tools/code_interpreter_tool/code_interpreter_tool.py new file mode 100644 index 000000000..a2066ca03 --- /dev/null +++ b/src/crewai_tools/tools/code_interpreter_tool/code_interpreter_tool.py @@ -0,0 +1,57 @@ +from typing import Optional, Type + +import docker +from crewai_tools.tools.base_tool import BaseTool +from pydantic.v1 import BaseModel, Field + + +class FixedCodeInterpreterSchemaSchema(BaseModel): + """Input for DirectoryReadTool.""" + + pass + + +class CodeInterpreterSchema(FixedCodeInterpreterSchemaSchema): + """Input for DirectoryReadTool.""" + + code: str = Field( + ..., + description="Python3 code used to be interpreted in the Docker container and output the result", + ) + + +class CodeInterpreterTool(BaseTool): + name: str = "Code Interpreter" + description: str = "Interprets Python code in a Docker container" + args_schema: Type[BaseModel] = CodeInterpreterSchema + code: Optional[str] = None + + def __init__(self, code: Optional[str] = None, **kwargs): + super().__init__(**kwargs) + if code is not None: + self.code = code + self.description = ( + "A tool that can be used to run Python code in a Docker container" + ) + self.args_schema = FixedCodeInterpreterSchemaSchema + self._generate_description() + + def _run(self, **kwargs): + code = kwargs.get("code", self.code) + return self.run_code_in_docker(code) + + def run_code_in_docker(self, code): + client = docker.from_env() + container = client.containers.run( + "code-interpreter", + command=f'python3 -c "{code}"', + detach=True, + working_dir="/workspace", + ) + + result = container.logs().decode("utf-8") + + container.stop() + container.remove() + + return result From da75d51fe8431aae6b7333cdeee0ecf5d9e91843 Mon Sep 17 00:00:00 2001 From: Eduardo Chiarotti Date: Thu, 20 Jun 2024 20:24:26 -0300 Subject: [PATCH 03/20] feat: add Dockerfile, Makefile and update version of code --- .../tools/code_interpreter_tool/Dockerfile | 9 +--- .../tools/code_interpreter_tool/Makefile | 6 +++ .../code_interpreter_tool.py | 51 ++++++++++++------- 3 files changed, 41 insertions(+), 25 deletions(-) create mode 100644 src/crewai_tools/tools/code_interpreter_tool/Makefile diff --git a/src/crewai_tools/tools/code_interpreter_tool/Dockerfile b/src/crewai_tools/tools/code_interpreter_tool/Dockerfile index b72a51a88..ae9b2ffd6 100644 --- a/src/crewai_tools/tools/code_interpreter_tool/Dockerfile +++ b/src/crewai_tools/tools/code_interpreter_tool/Dockerfile @@ -1,8 +1,4 @@ -# Use an official Ubuntu as a parent image -FROM ubuntu:20.04 - -# Set environment variables -ENV DEBIAN_FRONTEND=noninteractive +FROM python:3.11-slim # Install common utilities RUN apt-get update && apt-get install -y \ @@ -11,9 +7,6 @@ RUN apt-get update && apt-get install -y \ wget \ software-properties-common -# Install Python -RUN apt-get install -y python3 python3-pip - # Clean up RUN apt-get clean && rm -rf /var/lib/apt/lists/* diff --git a/src/crewai_tools/tools/code_interpreter_tool/Makefile b/src/crewai_tools/tools/code_interpreter_tool/Makefile new file mode 100644 index 000000000..5a514db0a --- /dev/null +++ b/src/crewai_tools/tools/code_interpreter_tool/Makefile @@ -0,0 +1,6 @@ +# Makefile +IMAGE_NAME=code-interpreter +TAG=latest + +build: + docker build -t $(IMAGE_NAME):$(TAG) . diff --git a/src/crewai_tools/tools/code_interpreter_tool/code_interpreter_tool.py b/src/crewai_tools/tools/code_interpreter_tool/code_interpreter_tool.py index a2066ca03..f497a7c96 100644 --- a/src/crewai_tools/tools/code_interpreter_tool/code_interpreter_tool.py +++ b/src/crewai_tools/tools/code_interpreter_tool/code_interpreter_tool.py @@ -6,23 +6,27 @@ from pydantic.v1 import BaseModel, Field class FixedCodeInterpreterSchemaSchema(BaseModel): - """Input for DirectoryReadTool.""" + """Input for CodeInterpreterTool.""" pass class CodeInterpreterSchema(FixedCodeInterpreterSchemaSchema): - """Input for DirectoryReadTool.""" + """Input for CodeInterpreterTool.""" code: str = Field( ..., - description="Python3 code used to be interpreted in the Docker container and output the result", + description="Python3 code used to be interpreted in the Docker container. ALWAYS PRINT the final result and the output of the code", + ) + libraries_used: Optional[str] = Field( + None, + description="List of libraries used in the code with proper installing names separated by commas. Example: numpy,pandas,beautifulsoup4", ) class CodeInterpreterTool(BaseTool): name: str = "Code Interpreter" - description: str = "Interprets Python code in a Docker container" + description: str = "Interprets Python code in a Docker container. ALWAYS PRINT the final result and the output of the code" args_schema: Type[BaseModel] = CodeInterpreterSchema code: Optional[str] = None @@ -30,28 +34,41 @@ class CodeInterpreterTool(BaseTool): super().__init__(**kwargs) if code is not None: self.code = code - self.description = ( - "A tool that can be used to run Python code in a Docker container" - ) + self.description = "Interprets Python code in a Docker container. ALWAYS PRINT the final result and the output of the code" self.args_schema = FixedCodeInterpreterSchemaSchema self._generate_description() def _run(self, **kwargs): code = kwargs.get("code", self.code) - return self.run_code_in_docker(code) + libraries_used = kwargs.get("libraries_used", None) + return self.run_code_in_docker(code, libraries_used) - def run_code_in_docker(self, code): + def run_code_in_docker(self, code, libraries_used): client = docker.from_env() - container = client.containers.run( - "code-interpreter", - command=f'python3 -c "{code}"', - detach=True, - working_dir="/workspace", - ) - result = container.logs().decode("utf-8") + def run_code(container, code): + cmd_to_run = f'python3 -c "{code}"' + exec_result = container.exec_run(cmd_to_run) + return exec_result + + container = client.containers.run( + "code-interpreter", detach=True, tty=True, working_dir="/workspace" + ) + if libraries_used: + self._install_libraries(container, libraries_used.split(",")) + + exec_result = run_code(container, code) container.stop() container.remove() + if exec_result.exit_code != 0: + return f"Something went wrong while running the code: \n{exec_result.output.decode('utf-8')}" - return result + return exec_result.output.decode("utf-8") + + def _install_libraries(self, container, libraries): + """ + Install missing libraries in the Docker container + """ + for library in libraries: + container.exec_run(f"pip install {library}") From 1a4ac76b1eb11ec70a11784a4f6b1d33948e9171 Mon Sep 17 00:00:00 2001 From: Eduardo Chiarotti Date: Thu, 20 Jun 2024 20:24:47 -0300 Subject: [PATCH 04/20] feat: update code --- .../code_interpreter_tool.py | 52 +++++++++---------- 1 file changed, 26 insertions(+), 26 deletions(-) diff --git a/src/crewai_tools/tools/code_interpreter_tool/code_interpreter_tool.py b/src/crewai_tools/tools/code_interpreter_tool/code_interpreter_tool.py index f497a7c96..caafd44e3 100644 --- a/src/crewai_tools/tools/code_interpreter_tool/code_interpreter_tool.py +++ b/src/crewai_tools/tools/code_interpreter_tool/code_interpreter_tool.py @@ -30,7 +30,7 @@ class CodeInterpreterTool(BaseTool): args_schema: Type[BaseModel] = CodeInterpreterSchema code: Optional[str] = None - def __init__(self, code: Optional[str] = None, **kwargs): + def __init__(self, code: Optional[str] = None, **kwargs) -> None: super().__init__(**kwargs) if code is not None: self.code = code @@ -38,37 +38,37 @@ class CodeInterpreterTool(BaseTool): self.args_schema = FixedCodeInterpreterSchemaSchema self._generate_description() - def _run(self, **kwargs): + def _run(self, **kwargs) -> str: code = kwargs.get("code", self.code) libraries_used = kwargs.get("libraries_used", None) return self.run_code_in_docker(code, libraries_used) - def run_code_in_docker(self, code, libraries_used): - client = docker.from_env() - - def run_code(container, code): - cmd_to_run = f'python3 -c "{code}"' - exec_result = container.exec_run(cmd_to_run) - return exec_result - - container = client.containers.run( - "code-interpreter", detach=True, tty=True, working_dir="/workspace" - ) - if libraries_used: - self._install_libraries(container, libraries_used.split(",")) - - exec_result = run_code(container, code) - - container.stop() - container.remove() - if exec_result.exit_code != 0: - return f"Something went wrong while running the code: \n{exec_result.output.decode('utf-8')}" - - return exec_result.output.decode("utf-8") - - def _install_libraries(self, container, libraries): + def _install_libraries( + self, container: docker.models.containers.Container, libraries: list[str] + ) -> None: """ Install missing libraries in the Docker container """ for library in libraries: container.exec_run(f"pip install {library}") + + def _init_docker_container(self) -> docker.models.containers.Container: + client = docker.from_env() + return client.containers.run( + "code-interpreter", detach=True, tty=True, working_dir="/workspace" + ) + + def run_code_in_docker(self, code: str, libraries_used: str) -> str: + container = self._init_docker_container() + + if libraries_used: + self._install_libraries(container, libraries_used.split(",")) + + cmd_to_run = f'python3 -c "{code}"' + exec_result = container.exec_run(cmd_to_run) + + container.stop().remove() + + if exec_result.exit_code != 0: + return f"Something went wrong while running the code: \n{exec_result.output.decode('utf-8')}" + return exec_result.output.decode("utf-8") From 94e6651b55cbebd85b0610787de5f4429e0778b1 Mon Sep 17 00:00:00 2001 From: Eduardo Chiarotti Date: Thu, 20 Jun 2024 20:43:19 -0300 Subject: [PATCH 05/20] feat: add code-interpreter tool to init and add unit tests --- src/crewai_tools/tools/__init__.py | 17 ++++++---- tests/tools/test_code_interpreter_tool.py | 38 +++++++++++++++++++++++ 2 files changed, 49 insertions(+), 6 deletions(-) create mode 100644 tests/tools/test_code_interpreter_tool.py diff --git a/src/crewai_tools/tools/__init__.py b/src/crewai_tools/tools/__init__.py index 4da0c0337..35b81396b 100644 --- a/src/crewai_tools/tools/__init__.py +++ b/src/crewai_tools/tools/__init__.py @@ -1,24 +1,29 @@ from .browserbase_load_tool.browserbase_load_tool import BrowserbaseLoadTool from .code_docs_search_tool.code_docs_search_tool import CodeDocsSearchTool +from .code_interpreter_tool.code_interpreter_tool import CodeInterpreterTool from .csv_search_tool.csv_search_tool import CSVSearchTool -from .directory_search_tool.directory_search_tool import DirectorySearchTool from .directory_read_tool.directory_read_tool import DirectoryReadTool +from .directory_search_tool.directory_search_tool import DirectorySearchTool from .docx_search_tool.docx_search_tool import DOCXSearchTool from .exa_tools.exa_search_tool import EXASearchTool from .file_read_tool.file_read_tool import FileReadTool from .github_search_tool.github_search_tool import GithubSearchTool -from .serper_dev_tool.serper_dev_tool import SerperDevTool -from .txt_search_tool.txt_search_tool import TXTSearchTool from .json_search_tool.json_search_tool import JSONSearchTool +from .llamaindex_tool.llamaindex_tool import LlamaIndexTool from .mdx_seach_tool.mdx_search_tool import MDXSearchTool from .pdf_search_tool.pdf_search_tool import PDFSearchTool from .pg_seach_tool.pg_search_tool import PGSearchTool from .rag.rag_tool import RagTool -from .scrape_element_from_website.scrape_element_from_website import ScrapeElementFromWebsiteTool +from .scrape_element_from_website.scrape_element_from_website import ( + ScrapeElementFromWebsiteTool, +) from .scrape_website_tool.scrape_website_tool import ScrapeWebsiteTool from .selenium_scraping_tool.selenium_scraping_tool import SeleniumScrapingTool +from .serper_dev_tool.serper_dev_tool import SerperDevTool +from .txt_search_tool.txt_search_tool import TXTSearchTool from .website_search.website_search_tool import WebsiteSearchTool from .xml_search_tool.xml_search_tool import XMLSearchTool -from .youtube_channel_search_tool.youtube_channel_search_tool import YoutubeChannelSearchTool +from .youtube_channel_search_tool.youtube_channel_search_tool import ( + YoutubeChannelSearchTool, +) from .youtube_video_search_tool.youtube_video_search_tool import YoutubeVideoSearchTool -from .llamaindex_tool.llamaindex_tool import LlamaIndexTool diff --git a/tests/tools/test_code_interpreter_tool.py b/tests/tools/test_code_interpreter_tool.py new file mode 100644 index 000000000..a9ffb9dbc --- /dev/null +++ b/tests/tools/test_code_interpreter_tool.py @@ -0,0 +1,38 @@ +import unittest +from unittest.mock import patch + +from crewai_tools.tools.code_interpreter_tool.code_interpreter_tool import ( + CodeInterpreterTool, +) + + +class TestCodeInterpreterTool(unittest.TestCase): + @patch("crewai_tools.tools.code_interpreter_tool.code_interpreter_tool.docker") + def test_run_code_in_docker(self, docker_mock): + tool = CodeInterpreterTool() + code = "print('Hello, World!')" + libraries_used = "numpy,pandas" + expected_output = "Hello, World!\n" + + docker_mock.from_env().containers.run().exec_run().exit_code = 0 + docker_mock.from_env().containers.run().exec_run().output = ( + expected_output.encode() + ) + result = tool.run_code_in_docker(code, libraries_used) + + self.assertEqual(result, expected_output) + + @patch("crewai_tools.tools.code_interpreter_tool.code_interpreter_tool.docker") + def test_run_code_in_docker_with_error(self, docker_mock): + tool = CodeInterpreterTool() + code = "print(1/0)" + libraries_used = "numpy,pandas" + expected_output = "Something went wrong while running the code: \nZeroDivisionError: division by zero\n" + + docker_mock.from_env().containers.run().exec_run().exit_code = 1 + docker_mock.from_env().containers.run().exec_run().output = ( + b"ZeroDivisionError: division by zero\n" + ) + result = tool.run_code_in_docker(code, libraries_used) + + self.assertEqual(result, expected_output) From 61cce93fd020fa4fce74cc80f64e07f2a777215d Mon Sep 17 00:00:00 2001 From: Eduardo Chiarotti Date: Thu, 20 Jun 2024 21:41:12 -0300 Subject: [PATCH 06/20] feat: remove unused Makefile, update README and update code --- .../tools/code_interpreter_tool/Makefile | 6 ----- .../tools/code_interpreter_tool/README.md | 27 +++++++++++++++++++ .../code_interpreter_tool.py | 22 ++++++++++++++- 3 files changed, 48 insertions(+), 7 deletions(-) delete mode 100644 src/crewai_tools/tools/code_interpreter_tool/Makefile diff --git a/src/crewai_tools/tools/code_interpreter_tool/Makefile b/src/crewai_tools/tools/code_interpreter_tool/Makefile deleted file mode 100644 index 5a514db0a..000000000 --- a/src/crewai_tools/tools/code_interpreter_tool/Makefile +++ /dev/null @@ -1,6 +0,0 @@ -# Makefile -IMAGE_NAME=code-interpreter -TAG=latest - -build: - docker build -t $(IMAGE_NAME):$(TAG) . diff --git a/src/crewai_tools/tools/code_interpreter_tool/README.md b/src/crewai_tools/tools/code_interpreter_tool/README.md index e69de29bb..672c86f21 100644 --- a/src/crewai_tools/tools/code_interpreter_tool/README.md +++ b/src/crewai_tools/tools/code_interpreter_tool/README.md @@ -0,0 +1,27 @@ +# CodeInterpreterTool + +## Description +This tool is used to give the Agent the ability to run code (Python3) from the code generated by the Agent itself. The code is executed in a sandboxed environment, so it is safe to run any code. + +## Requirements + +- Docker + +## Installation +Install the crewai_tools package +```shell +pip install 'crewai[tools]' +``` + +## Example + +Remember that when using this tool, the code must be generated by the Agent itself. The code must be a Python3 code. And it will take some time for the first time to run because it needs to build the Docker image. + +```python +from crewai_tools import CodeInterpreterTool + +Agent( + ... + tools=[CodeInterpreterTool()], +) +``` diff --git a/src/crewai_tools/tools/code_interpreter_tool/code_interpreter_tool.py b/src/crewai_tools/tools/code_interpreter_tool/code_interpreter_tool.py index caafd44e3..06cb081f0 100644 --- a/src/crewai_tools/tools/code_interpreter_tool/code_interpreter_tool.py +++ b/src/crewai_tools/tools/code_interpreter_tool/code_interpreter_tool.py @@ -1,3 +1,4 @@ +import os from typing import Optional, Type import docker @@ -30,9 +31,27 @@ class CodeInterpreterTool(BaseTool): args_schema: Type[BaseModel] = CodeInterpreterSchema code: Optional[str] = None + def _verify_docker_image(self) -> None: + """ + Verify if the Docker image is available + """ + image_tag = "code-interpreter:latest" + + client = docker.from_env() + images = client.images.list() + all_tags = [tag for image in images for tag in image.tags] + + if image_tag not in all_tags: + client.images.build( + path=os.path.dirname(os.path.abspath(__file__)), + tag=image_tag, + rm=True, + ) + def __init__(self, code: Optional[str] = None, **kwargs) -> None: super().__init__(**kwargs) if code is not None: + self._verify_docker_image() self.code = code self.description = "Interprets Python code in a Docker container. ALWAYS PRINT the final result and the output of the code" self.args_schema = FixedCodeInterpreterSchemaSchema @@ -67,7 +86,8 @@ class CodeInterpreterTool(BaseTool): cmd_to_run = f'python3 -c "{code}"' exec_result = container.exec_run(cmd_to_run) - container.stop().remove() + container.stop() + container.remove() if exec_result.exit_code != 0: return f"Something went wrong while running the code: \n{exec_result.output.decode('utf-8')}" From 161c72b29f553f6b3758b9408ff9423349c5bba7 Mon Sep 17 00:00:00 2001 From: Eduardo Chiarotti Date: Thu, 20 Jun 2024 21:55:25 -0300 Subject: [PATCH 07/20] feat: update README --- src/crewai_tools/tools/code_interpreter_tool/README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/crewai_tools/tools/code_interpreter_tool/README.md b/src/crewai_tools/tools/code_interpreter_tool/README.md index 672c86f21..e66a82e39 100644 --- a/src/crewai_tools/tools/code_interpreter_tool/README.md +++ b/src/crewai_tools/tools/code_interpreter_tool/README.md @@ -3,6 +3,8 @@ ## Description This tool is used to give the Agent the ability to run code (Python3) from the code generated by the Agent itself. The code is executed in a sandboxed environment, so it is safe to run any code. +It is incredible useful since it allows the Agent to generate code, run it in the same environment, get the result and use it to make decisions. + ## Requirements - Docker From d4449ee5f0044e9324325bd77acc788c7d72ed1f Mon Sep 17 00:00:00 2001 From: angrybayblade Date: Mon, 24 Jun 2024 20:48:52 +0530 Subject: [PATCH 08/20] feat: add composio CrewAI tool wrapper --- .../tools/composio_tool/README.md | 30 +++++++++ .../tools/composio_tool/composio_tool.py | 62 +++++++++++++++++++ 2 files changed, 92 insertions(+) create mode 100644 src/crewai_tools/tools/composio_tool/README.md create mode 100644 src/crewai_tools/tools/composio_tool/composio_tool.py diff --git a/src/crewai_tools/tools/composio_tool/README.md b/src/crewai_tools/tools/composio_tool/README.md new file mode 100644 index 000000000..ef7cf1edb --- /dev/null +++ b/src/crewai_tools/tools/composio_tool/README.md @@ -0,0 +1,30 @@ +# ComposioTool Documentation + +## Description + +This tools is a wrapper around the composio toolset and gives your agent access to a wide variety of tools from the composio SDK. + +## Installation + +To incorporate this tool into your project, follow the installation instructions below: + +```shell +pip install composio-core +pip install 'crewai[tools]' +``` + +## Example + +The following example demonstrates how to initialize the tool and execute a mathematical operation: + +```python + +from composio import Action + +from crewai_tools.tools.composio_tool.composio_tool import ComposioTool + +tool = ComposioTool.from_tool( + tool=Action.MATHEMATICAL_CALCULATOR, +) +``` + diff --git a/src/crewai_tools/tools/composio_tool/composio_tool.py b/src/crewai_tools/tools/composio_tool/composio_tool.py new file mode 100644 index 000000000..e08fbde31 --- /dev/null +++ b/src/crewai_tools/tools/composio_tool/composio_tool.py @@ -0,0 +1,62 @@ +""" +Composio tools wrapper. +""" + +import typing as t + +import typing_extensions as te + +from crewai_tools.tools.base_tool import BaseTool + + +class ComposioTool(BaseTool): + """Wrapper for composio tools.""" + + composio_action: t.Callable + + def _run(self, *args: t.Any, **kwargs: t.Any) -> t.Any: + """Run the composio action with given arguments.""" + return self.composio_action(*args, **kwargs) + + @classmethod + def from_tool(cls, tool: t.Any, **kwargs: t.Any) -> te.Self: + """Wrap a composio tool as crewAI tool.""" + + from composio import Action, ComposioToolSet + from composio.constants import DEFAULT_ENTITY_ID + from composio.utils.shared import json_schema_to_model + + toolset = ComposioToolSet() + if not isinstance(tool, Action): + tool = Action.from_action(name=tool) + + tool = t.cast(Action, tool) + (action,) = toolset.get_action_schemas(actions=[tool]) + schema = action.model_dump(exclude_none=True) + entity_id = kwargs.pop("entity_id", DEFAULT_ENTITY_ID) + + def function(**kwargs: t.Any) -> t.Dict: + """Wrapper function for composio action.""" + return toolset.execute_action( + action=Action.from_app_and_action( + app=schema["appName"], + name=schema["name"], + ), + params=kwargs, + entity_id=entity_id, + ) + + function.__name__ = schema["name"] + function.__doc__ = schema["description"] + + return cls( + name=schema["name"], + description=schema["description"], + args_schema=json_schema_to_model( + action.parameters.model_dump( + exclude_none=True, + ) + ), + composio_action=function, + **kwargs + ) From 41478abdf5665e9a912716e5b7d62a36db1e6d69 Mon Sep 17 00:00:00 2001 From: angrybayblade Date: Thu, 27 Jun 2024 11:36:52 +0530 Subject: [PATCH 09/20] feat: define `ComposioTool` in the top level imports --- src/crewai_tools/__init__.py | 53 +++++++++++++++--------------- src/crewai_tools/tools/__init__.py | 17 ++++++---- 2 files changed, 38 insertions(+), 32 deletions(-) diff --git a/src/crewai_tools/__init__.py b/src/crewai_tools/__init__.py index a51d70449..214dbbb31 100644 --- a/src/crewai_tools/__init__.py +++ b/src/crewai_tools/__init__.py @@ -1,27 +1,28 @@ -from .tools.base_tool import BaseTool, Tool, tool from .tools import ( - BrowserbaseLoadTool, - CodeDocsSearchTool, - CSVSearchTool, - DirectorySearchTool, - DOCXSearchTool, - DirectoryReadTool, - EXASearchTool, - FileReadTool, - GithubSearchTool, - SerperDevTool, - TXTSearchTool, - JSONSearchTool, - MDXSearchTool, - PDFSearchTool, - PGSearchTool, - RagTool, - ScrapeElementFromWebsiteTool, - ScrapeWebsiteTool, - SeleniumScrapingTool, - WebsiteSearchTool, - XMLSearchTool, - YoutubeChannelSearchTool, - YoutubeVideoSearchTool, - LlamaIndexTool -) \ No newline at end of file + BrowserbaseLoadTool, + CodeDocsSearchTool, + ComposioTool, + CSVSearchTool, + DirectoryReadTool, + DirectorySearchTool, + DOCXSearchTool, + EXASearchTool, + FileReadTool, + GithubSearchTool, + JSONSearchTool, + LlamaIndexTool, + MDXSearchTool, + PDFSearchTool, + PGSearchTool, + RagTool, + ScrapeElementFromWebsiteTool, + ScrapeWebsiteTool, + SeleniumScrapingTool, + SerperDevTool, + TXTSearchTool, + WebsiteSearchTool, + XMLSearchTool, + YoutubeChannelSearchTool, + YoutubeVideoSearchTool, +) +from .tools.base_tool import BaseTool, Tool, tool diff --git a/src/crewai_tools/tools/__init__.py b/src/crewai_tools/tools/__init__.py index 4da0c0337..df0ec7286 100644 --- a/src/crewai_tools/tools/__init__.py +++ b/src/crewai_tools/tools/__init__.py @@ -1,24 +1,29 @@ from .browserbase_load_tool.browserbase_load_tool import BrowserbaseLoadTool from .code_docs_search_tool.code_docs_search_tool import CodeDocsSearchTool +from .composio_tool.composio_tool import ComposioTool from .csv_search_tool.csv_search_tool import CSVSearchTool -from .directory_search_tool.directory_search_tool import DirectorySearchTool from .directory_read_tool.directory_read_tool import DirectoryReadTool +from .directory_search_tool.directory_search_tool import DirectorySearchTool from .docx_search_tool.docx_search_tool import DOCXSearchTool from .exa_tools.exa_search_tool import EXASearchTool from .file_read_tool.file_read_tool import FileReadTool from .github_search_tool.github_search_tool import GithubSearchTool -from .serper_dev_tool.serper_dev_tool import SerperDevTool -from .txt_search_tool.txt_search_tool import TXTSearchTool from .json_search_tool.json_search_tool import JSONSearchTool +from .llamaindex_tool.llamaindex_tool import LlamaIndexTool from .mdx_seach_tool.mdx_search_tool import MDXSearchTool from .pdf_search_tool.pdf_search_tool import PDFSearchTool from .pg_seach_tool.pg_search_tool import PGSearchTool from .rag.rag_tool import RagTool -from .scrape_element_from_website.scrape_element_from_website import ScrapeElementFromWebsiteTool +from .scrape_element_from_website.scrape_element_from_website import ( + ScrapeElementFromWebsiteTool, +) from .scrape_website_tool.scrape_website_tool import ScrapeWebsiteTool from .selenium_scraping_tool.selenium_scraping_tool import SeleniumScrapingTool +from .serper_dev_tool.serper_dev_tool import SerperDevTool +from .txt_search_tool.txt_search_tool import TXTSearchTool from .website_search.website_search_tool import WebsiteSearchTool from .xml_search_tool.xml_search_tool import XMLSearchTool -from .youtube_channel_search_tool.youtube_channel_search_tool import YoutubeChannelSearchTool +from .youtube_channel_search_tool.youtube_channel_search_tool import ( + YoutubeChannelSearchTool, +) from .youtube_video_search_tool.youtube_video_search_tool import YoutubeVideoSearchTool -from .llamaindex_tool.llamaindex_tool import LlamaIndexTool From be6e1a79dd136ed8b3e36947203e1e7b91f95eee Mon Sep 17 00:00:00 2001 From: angrybayblade Date: Thu, 27 Jun 2024 11:39:42 +0530 Subject: [PATCH 10/20] feat: add search utility methods --- .../tools/composio_tool/composio_tool.py | 52 ++++++++++++++++--- 1 file changed, 45 insertions(+), 7 deletions(-) diff --git a/src/crewai_tools/tools/composio_tool/composio_tool.py b/src/crewai_tools/tools/composio_tool/composio_tool.py index e08fbde31..664898ce3 100644 --- a/src/crewai_tools/tools/composio_tool/composio_tool.py +++ b/src/crewai_tools/tools/composio_tool/composio_tool.py @@ -19,7 +19,11 @@ class ComposioTool(BaseTool): return self.composio_action(*args, **kwargs) @classmethod - def from_tool(cls, tool: t.Any, **kwargs: t.Any) -> te.Self: + def from_tool( + cls, + tool: t.Any, + **kwargs: t.Any, + ) -> te.Self: """Wrap a composio tool as crewAI tool.""" from composio import Action, ComposioToolSet @@ -28,7 +32,7 @@ class ComposioTool(BaseTool): toolset = ComposioToolSet() if not isinstance(tool, Action): - tool = Action.from_action(name=tool) + tool = Action(tool) tool = t.cast(Action, tool) (action,) = toolset.get_action_schemas(actions=[tool]) @@ -38,10 +42,7 @@ class ComposioTool(BaseTool): def function(**kwargs: t.Any) -> t.Dict: """Wrapper function for composio action.""" return toolset.execute_action( - action=Action.from_app_and_action( - app=schema["appName"], - name=schema["name"], - ), + action=Action(schema["name"]), params=kwargs, entity_id=entity_id, ) @@ -58,5 +59,42 @@ class ComposioTool(BaseTool): ) ), composio_action=function, - **kwargs + **kwargs, ) + + @classmethod + def from_app( + cls, + app: t.Any, + tags: t.Optional[t.List[str]] = None, + **kwargs: t.Any, + ) -> t.List[te.Self]: + """Create toolset from an app.""" + from composio import App + + if not isinstance(app, App): + app = App(app) + + return [ + cls.from_tool(tool=action, **kwargs) + for action in app.get_actions(tags=tags) + ] + + @classmethod + def from_use_case( + cls, + *apps: t.Any, + use_case: str, + **kwargs: t.Any, + ) -> t.List[te.Self]: + """Create toolset from an app.""" + if len(apps) == 0: + raise ValueError( + "You need to provide at least one app name to search by use case" + ) + + from composio import ComposioToolSet + + toolset = ComposioToolSet() + actions = toolset.find_actions_by_use_case(*apps, use_case=use_case) + return [cls.from_tool(tool=action, **kwargs) for action in actions] From ab484172ef2d3237fb14416ca2894db3568e216b Mon Sep 17 00:00:00 2001 From: angrybayblade Date: Thu, 27 Jun 2024 11:40:02 +0530 Subject: [PATCH 11/20] chore: update readme --- .../tools/composio_tool/README.md | 49 +++++++++++++++++-- 1 file changed, 44 insertions(+), 5 deletions(-) diff --git a/src/crewai_tools/tools/composio_tool/README.md b/src/crewai_tools/tools/composio_tool/README.md index ef7cf1edb..4a4db85d5 100644 --- a/src/crewai_tools/tools/composio_tool/README.md +++ b/src/crewai_tools/tools/composio_tool/README.md @@ -15,16 +15,55 @@ pip install 'crewai[tools]' ## Example -The following example demonstrates how to initialize the tool and execute a mathematical operation: +The following example demonstrates how to initialize the tool and execute a github action: + +1. Initialize toolset ```python +from composio import App +from crewai_tools import ComposioTool +from crewai import Agent, Task -from composio import Action -from crewai_tools.tools.composio_tool.composio_tool import ComposioTool +tools = [ComposioTool.from_tool(tool=Action.GITHUB_ACTIVITY_STAR_REPO_FOR_AUTHENTICATED_USER)] +``` -tool = ComposioTool.from_tool( - tool=Action.MATHEMATICAL_CALCULATOR, +If you don't know what action you want to use, use `from_app` and `tags` filter to get relevant actions + +```python +tools = ComposioTool.from_app(app=App.GITHUB, tags=["important"]) +``` + +or use `from_use_case` to search relevant actions + +```python +tools = ComposioTool.from_use_case(App.GITHUB, use_case="Star a github repository") +``` + +2. Define agent + +```python +crewai_agent = Agent( + role="Github Agent", + goal="""You take action on Github using Github APIs""", + backstory=( + "You are AI agent that is responsible for taking actions on Github " + "on users behalf. You need to take action on Github using Github APIs" + ), + verbose=True, + tools=tools, ) ``` +3. Execute task + +```python +task = Task( + description="Star a repo ComposioHQ/composio on GitHub", + agent=crewai_agent, + expected_output="if the star happened", +) + +task.execute() +``` + From 369c03a257c6c11c5620d3bcf4e66310a649c193 Mon Sep 17 00:00:00 2001 From: angrybayblade Date: Thu, 27 Jun 2024 12:16:22 +0530 Subject: [PATCH 12/20] feat: add check for auth accounts --- .../tools/composio_tool/composio_tool.py | 25 +++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/src/crewai_tools/tools/composio_tool/composio_tool.py b/src/crewai_tools/tools/composio_tool/composio_tool.py index 664898ce3..fd478eeb6 100644 --- a/src/crewai_tools/tools/composio_tool/composio_tool.py +++ b/src/crewai_tools/tools/composio_tool/composio_tool.py @@ -18,6 +18,26 @@ class ComposioTool(BaseTool): """Run the composio action with given arguments.""" return self.composio_action(*args, **kwargs) + @staticmethod + def _check_connected_account(tool: t.Any, toolset: t.Any) -> None: + """Check if connected account is required and if required it exists or not.""" + from composio import Action + from composio.client.collections import ConnectedAccountModel + + tool = t.cast(Action, tool) + if tool.no_auth: + return + + connections = t.cast( + t.List[ConnectedAccountModel], + toolset.client.connected_accounts.get(), + ) + if tool.app not in [connection.appUniqueId for connection in connections]: + raise RuntimeError( + f"No connected account found for app `{tool.app}`; " + f"Run `composio add {tool.app}` to fix this" + ) + @classmethod def from_tool( cls, @@ -35,6 +55,11 @@ class ComposioTool(BaseTool): tool = Action(tool) tool = t.cast(Action, tool) + cls._check_connected_account( + tool=tool, + toolset=toolset, + ) + (action,) = toolset.get_action_schemas(actions=[tool]) schema = action.model_dump(exclude_none=True) entity_id = kwargs.pop("entity_id", DEFAULT_ENTITY_ID) From 58354ec638326a4b406aa97d6038d2fc7d6df418 Mon Sep 17 00:00:00 2001 From: angrybayblade Date: Thu, 27 Jun 2024 12:23:05 +0530 Subject: [PATCH 13/20] chore: update README --- src/crewai_tools/tools/composio_tool/README.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/crewai_tools/tools/composio_tool/README.md b/src/crewai_tools/tools/composio_tool/README.md index 4a4db85d5..fe030cbd2 100644 --- a/src/crewai_tools/tools/composio_tool/README.md +++ b/src/crewai_tools/tools/composio_tool/README.md @@ -13,6 +13,8 @@ pip install composio-core pip install 'crewai[tools]' ``` +after the installation is complete, either run `composio login` or export your composio API key as `COMPOSIO_API_KEY`. + ## Example The following example demonstrates how to initialize the tool and execute a github action: @@ -45,7 +47,7 @@ tools = ComposioTool.from_use_case(App.GITHUB, use_case="Star a github repositor ```python crewai_agent = Agent( role="Github Agent", - goal="""You take action on Github using Github APIs""", + goal="You take action on Github using Github APIs", backstory=( "You are AI agent that is responsible for taking actions on Github " "on users behalf. You need to take action on Github using Github APIs" @@ -67,3 +69,4 @@ task = Task( task.execute() ``` +* More detailed list of tools can be found [here](https://app.composio.dev) From 9a8d88b8aa3890897d75814fd4b916d3dba706b7 Mon Sep 17 00:00:00 2001 From: angrybayblade Date: Thu, 27 Jun 2024 13:35:57 +0530 Subject: [PATCH 14/20] fix: merge `from_app` and `from_use_case` --- .../tools/composio_tool/README.md | 8 +-- .../tools/composio_tool/composio_tool.py | 59 +++++++++---------- 2 files changed, 32 insertions(+), 35 deletions(-) diff --git a/src/crewai_tools/tools/composio_tool/README.md b/src/crewai_tools/tools/composio_tool/README.md index fe030cbd2..18045e7f1 100644 --- a/src/crewai_tools/tools/composio_tool/README.md +++ b/src/crewai_tools/tools/composio_tool/README.md @@ -27,19 +27,19 @@ from crewai_tools import ComposioTool from crewai import Agent, Task -tools = [ComposioTool.from_tool(tool=Action.GITHUB_ACTIVITY_STAR_REPO_FOR_AUTHENTICATED_USER)] +tools = [ComposioTool.from_action(action=Action.GITHUB_ACTIVITY_STAR_REPO_FOR_AUTHENTICATED_USER)] ``` If you don't know what action you want to use, use `from_app` and `tags` filter to get relevant actions ```python -tools = ComposioTool.from_app(app=App.GITHUB, tags=["important"]) +tools = ComposioTool.from_app(App.GITHUB, tags=["important"]) ``` -or use `from_use_case` to search relevant actions +or use `use_case` to search relevant actions ```python -tools = ComposioTool.from_use_case(App.GITHUB, use_case="Star a github repository") +tools = ComposioTool.from_app(App.GITHUB, use_case="Star a github repository") ``` 2. Define agent diff --git a/src/crewai_tools/tools/composio_tool/composio_tool.py b/src/crewai_tools/tools/composio_tool/composio_tool.py index fd478eeb6..62068c0bd 100644 --- a/src/crewai_tools/tools/composio_tool/composio_tool.py +++ b/src/crewai_tools/tools/composio_tool/composio_tool.py @@ -39,9 +39,9 @@ class ComposioTool(BaseTool): ) @classmethod - def from_tool( + def from_action( cls, - tool: t.Any, + action: t.Any, **kwargs: t.Any, ) -> te.Self: """Wrap a composio tool as crewAI tool.""" @@ -51,17 +51,17 @@ class ComposioTool(BaseTool): from composio.utils.shared import json_schema_to_model toolset = ComposioToolSet() - if not isinstance(tool, Action): - tool = Action(tool) + if not isinstance(action, Action): + action = Action(action) - tool = t.cast(Action, tool) + action = t.cast(Action, action) cls._check_connected_account( - tool=tool, + tool=action, toolset=toolset, ) - (action,) = toolset.get_action_schemas(actions=[tool]) - schema = action.model_dump(exclude_none=True) + (action_schema,) = toolset.get_action_schemas(actions=[action]) + schema = action_schema.model_dump(exclude_none=True) entity_id = kwargs.pop("entity_id", DEFAULT_ENTITY_ID) def function(**kwargs: t.Any) -> t.Dict: @@ -79,7 +79,7 @@ class ComposioTool(BaseTool): name=schema["name"], description=schema["description"], args_schema=json_schema_to_model( - action.parameters.model_dump( + action_schema.parameters.model_dump( exclude_none=True, ) ), @@ -89,37 +89,34 @@ class ComposioTool(BaseTool): @classmethod def from_app( - cls, - app: t.Any, - tags: t.Optional[t.List[str]] = None, - **kwargs: t.Any, - ) -> t.List[te.Self]: - """Create toolset from an app.""" - from composio import App - - if not isinstance(app, App): - app = App(app) - - return [ - cls.from_tool(tool=action, **kwargs) - for action in app.get_actions(tags=tags) - ] - - @classmethod - def from_use_case( cls, *apps: t.Any, - use_case: str, + tags: t.Optional[t.List[str]] = None, + use_case: t.Optional[str] = None, **kwargs: t.Any, ) -> t.List[te.Self]: """Create toolset from an app.""" if len(apps) == 0: + raise ValueError("You need to provide at least one app name") + + if use_case is None and tags is None: + raise ValueError("Both `use_case` and `tags` cannot be `None`") + + if use_case is not None and tags is not None: raise ValueError( - "You need to provide at least one app name to search by use case" + "Cannot use both `use_case` and `tags` to filter the actions" ) from composio import ComposioToolSet toolset = ComposioToolSet() - actions = toolset.find_actions_by_use_case(*apps, use_case=use_case) - return [cls.from_tool(tool=action, **kwargs) for action in actions] + if use_case is not None: + return [ + cls.from_action(action=action, **kwargs) + for action in toolset.find_actions_by_use_case(*apps, use_case=use_case) + ] + + return [ + cls.from_action(action=action, **kwargs) + for action in toolset.find_actions_by_tags(*apps, tags=tags) + ] From a3d3a70b5a4f87b27089759938cee08a6a9d3f31 Mon Sep 17 00:00:00 2001 From: Mervin Praison Date: Mon, 1 Jul 2024 04:34:39 +0100 Subject: [PATCH 15/20] Update __init__.py to Add CodeInterpreterTool --- src/crewai_tools/__init__.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/crewai_tools/__init__.py b/src/crewai_tools/__init__.py index a51d70449..1398dcfaf 100644 --- a/src/crewai_tools/__init__.py +++ b/src/crewai_tools/__init__.py @@ -2,6 +2,7 @@ from .tools.base_tool import BaseTool, Tool, tool from .tools import ( BrowserbaseLoadTool, CodeDocsSearchTool, + CodeInterpreterTool, CSVSearchTool, DirectorySearchTool, DOCXSearchTool, @@ -24,4 +25,4 @@ from .tools import ( YoutubeChannelSearchTool, YoutubeVideoSearchTool, LlamaIndexTool -) \ No newline at end of file +) From f79c385bf76a1dcd0595cfc7bd111fa1928a87d7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Moura?= Date: Mon, 1 Jul 2024 00:55:21 -0700 Subject: [PATCH 16/20] revamping code interpreter --- .../code_interpreter_tool.py | 60 ++++++++----------- 1 file changed, 26 insertions(+), 34 deletions(-) diff --git a/src/crewai_tools/tools/code_interpreter_tool/code_interpreter_tool.py b/src/crewai_tools/tools/code_interpreter_tool/code_interpreter_tool.py index 06cb081f0..ec756d8c0 100644 --- a/src/crewai_tools/tools/code_interpreter_tool/code_interpreter_tool.py +++ b/src/crewai_tools/tools/code_interpreter_tool/code_interpreter_tool.py @@ -1,69 +1,62 @@ import os -from typing import Optional, Type +import importlib.util +import textwrap +from typing import List, Optional, Type import docker from crewai_tools.tools.base_tool import BaseTool from pydantic.v1 import BaseModel, Field -class FixedCodeInterpreterSchemaSchema(BaseModel): +class CodeInterpreterSchema(BaseModel): """Input for CodeInterpreterTool.""" - - pass - - -class CodeInterpreterSchema(FixedCodeInterpreterSchemaSchema): - """Input for CodeInterpreterTool.""" - code: str = Field( ..., - description="Python3 code used to be interpreted in the Docker container. ALWAYS PRINT the final result and the output of the code", + description="Mandatory string of python3 code used to be interpreted with a final print statement.", ) - libraries_used: Optional[str] = Field( - None, - description="List of libraries used in the code with proper installing names separated by commas. Example: numpy,pandas,beautifulsoup4", + dependencies_used_in_code: List[str] = Field( + ..., + description="Mandatory list of libraries used in the code with proper installing names.", ) - class CodeInterpreterTool(BaseTool): name: str = "Code Interpreter" - description: str = "Interprets Python code in a Docker container. ALWAYS PRINT the final result and the output of the code" + description: str = "Interprets Python3 code strings with a final print statement." args_schema: Type[BaseModel] = CodeInterpreterSchema code: Optional[str] = None + @staticmethod + def _get_installed_package_path(): + spec = importlib.util.find_spec('crewai_tools') + return os.path.dirname(spec.origin) + def _verify_docker_image(self) -> None: """ Verify if the Docker image is available """ image_tag = "code-interpreter:latest" - client = docker.from_env() - images = client.images.list() - all_tags = [tag for image in images for tag in image.tags] + try: + client.images.get(image_tag) + except: + package_path = self._get_installed_package_path() + dockerfile_path = os.path.join(package_path, 'tools/code_interpreter_tool') + if not os.path.exists(dockerfile_path): + raise FileNotFoundError(f"Dockerfile not found in {dockerfile_path}") - if image_tag not in all_tags: client.images.build( - path=os.path.dirname(os.path.abspath(__file__)), + path=dockerfile_path, tag=image_tag, rm=True, ) - def __init__(self, code: Optional[str] = None, **kwargs) -> None: - super().__init__(**kwargs) - if code is not None: - self._verify_docker_image() - self.code = code - self.description = "Interprets Python code in a Docker container. ALWAYS PRINT the final result and the output of the code" - self.args_schema = FixedCodeInterpreterSchemaSchema - self._generate_description() - def _run(self, **kwargs) -> str: code = kwargs.get("code", self.code) - libraries_used = kwargs.get("libraries_used", None) + libraries_used = kwargs.get("dependencies_used_in_code", []) return self.run_code_in_docker(code, libraries_used) def _install_libraries( - self, container: docker.models.containers.Container, libraries: list[str] + self, container: docker.models.containers.Container, libraries: List[str] ) -> None: """ Install missing libraries in the Docker container @@ -78,10 +71,9 @@ class CodeInterpreterTool(BaseTool): ) def run_code_in_docker(self, code: str, libraries_used: str) -> str: + self._verify_docker_image() container = self._init_docker_container() - - if libraries_used: - self._install_libraries(container, libraries_used.split(",")) + self._install_libraries(container, libraries_used) cmd_to_run = f'python3 -c "{code}"' exec_result = container.exec_run(cmd_to_run) From d000bd2fc8eaed205b77064bc4f908ba6c8aed4e Mon Sep 17 00:00:00 2001 From: Eduardo Chiarotti Date: Tue, 2 Jul 2024 12:00:04 -0300 Subject: [PATCH 17/20] fix: add code interpreter tool --- src/crewai_tools/__init__.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/crewai_tools/__init__.py b/src/crewai_tools/__init__.py index 1cc222ec9..b85a16ffb 100644 --- a/src/crewai_tools/__init__.py +++ b/src/crewai_tools/__init__.py @@ -1,6 +1,7 @@ from .tools import ( BrowserbaseLoadTool, CodeDocsSearchTool, + CodeInterpreterTool, ComposioTool, CSVSearchTool, DirectoryReadTool, @@ -25,4 +26,4 @@ from .tools import ( YoutubeChannelSearchTool, YoutubeVideoSearchTool, ) -from .tools.base_tool import BaseTool, Tool, tool \ No newline at end of file +from .tools.base_tool import BaseTool, Tool, tool From b4d91d1ce01644893eb0c93be84d5067616f5f52 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Moura?= Date: Thu, 4 Jul 2024 00:09:18 -0400 Subject: [PATCH 18/20] adding new result_as_answer options --- src/crewai_tools/tools/base_tool.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/crewai_tools/tools/base_tool.py b/src/crewai_tools/tools/base_tool.py index e8e497859..4e0bd1fd5 100644 --- a/src/crewai_tools/tools/base_tool.py +++ b/src/crewai_tools/tools/base_tool.py @@ -22,6 +22,8 @@ class BaseTool(BaseModel, ABC): """Flag to check if the description has been updated.""" cache_function: Optional[Callable] = lambda _args, _result: True """Function that will be used to determine if the tool should be cached, should return a boolean. If None, the tool will be cached.""" + result_as_answer: bool = False + """Flag to check if the tool should be the final agent answer.""" @validator("args_schema", always=True, pre=True) def _default_args_schema(cls, v: Type[V1BaseModel]) -> Type[V1BaseModel]: From ba05d18ab16db4dd36f3b3ce414d3652ccfc9d20 Mon Sep 17 00:00:00 2001 From: Eduardo Chiarotti Date: Thu, 4 Jul 2024 16:42:29 -0300 Subject: [PATCH 19/20] fix: fix type hinting, add container name and handle exception and returned old description --- .../code_interpreter_tool.py | 30 ++++++++++++------- 1 file changed, 19 insertions(+), 11 deletions(-) diff --git a/src/crewai_tools/tools/code_interpreter_tool/code_interpreter_tool.py b/src/crewai_tools/tools/code_interpreter_tool/code_interpreter_tool.py index ec756d8c0..f341e52d0 100644 --- a/src/crewai_tools/tools/code_interpreter_tool/code_interpreter_tool.py +++ b/src/crewai_tools/tools/code_interpreter_tool/code_interpreter_tool.py @@ -1,6 +1,5 @@ -import os import importlib.util -import textwrap +import os from typing import List, Optional, Type import docker @@ -10,15 +9,18 @@ from pydantic.v1 import BaseModel, Field class CodeInterpreterSchema(BaseModel): """Input for CodeInterpreterTool.""" + code: str = Field( ..., - description="Mandatory string of python3 code used to be interpreted with a final print statement.", + description="Python3 code used to be interpreted in the Docker container. ALWAYS PRINT the final result and the output of the code", ) - dependencies_used_in_code: List[str] = Field( + + libraries_used: List[str] = Field( ..., - description="Mandatory list of libraries used in the code with proper installing names.", + description="List of libraries used in the code with proper installing names separated by commas. Example: numpy,pandas,beautifulsoup4", ) + class CodeInterpreterTool(BaseTool): name: str = "Code Interpreter" description: str = "Interprets Python3 code strings with a final print statement." @@ -27,7 +29,7 @@ class CodeInterpreterTool(BaseTool): @staticmethod def _get_installed_package_path(): - spec = importlib.util.find_spec('crewai_tools') + spec = importlib.util.find_spec("crewai_tools") return os.path.dirname(spec.origin) def _verify_docker_image(self) -> None: @@ -36,11 +38,13 @@ class CodeInterpreterTool(BaseTool): """ image_tag = "code-interpreter:latest" client = docker.from_env() + try: client.images.get(image_tag) - except: + + except docker.errors.ImageNotFound: package_path = self._get_installed_package_path() - dockerfile_path = os.path.join(package_path, 'tools/code_interpreter_tool') + dockerfile_path = os.path.join(package_path, "tools/code_interpreter_tool") if not os.path.exists(dockerfile_path): raise FileNotFoundError(f"Dockerfile not found in {dockerfile_path}") @@ -52,7 +56,7 @@ class CodeInterpreterTool(BaseTool): def _run(self, **kwargs) -> str: code = kwargs.get("code", self.code) - libraries_used = kwargs.get("dependencies_used_in_code", []) + libraries_used = kwargs.get("libraries_used", []) return self.run_code_in_docker(code, libraries_used) def _install_libraries( @@ -67,10 +71,14 @@ class CodeInterpreterTool(BaseTool): def _init_docker_container(self) -> docker.models.containers.Container: client = docker.from_env() return client.containers.run( - "code-interpreter", detach=True, tty=True, working_dir="/workspace" + "code-interpreter", + detach=True, + tty=True, + working_dir="/workspace", + name="code-interpreter", ) - def run_code_in_docker(self, code: str, libraries_used: str) -> str: + def run_code_in_docker(self, code: str, libraries_used: List[str]) -> str: self._verify_docker_image() container = self._init_docker_container() self._install_libraries(container, libraries_used) From f056764132b5caa93846ad67fbc1fce841255796 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Moura?= Date: Mon, 8 Jul 2024 01:15:00 -0400 Subject: [PATCH 20/20] adding firecrawl imports --- src/crewai_tools/__init__.py | 3 +++ src/crewai_tools/tools/__init__.py | 3 +++ 2 files changed, 6 insertions(+) diff --git a/src/crewai_tools/__init__.py b/src/crewai_tools/__init__.py index b85a16ffb..a9c9a4168 100644 --- a/src/crewai_tools/__init__.py +++ b/src/crewai_tools/__init__.py @@ -9,6 +9,9 @@ from .tools import ( DOCXSearchTool, EXASearchTool, FileReadTool, + FirecrawlCrawlWebsiteTool, + FirecrawlScrapeWebsiteTool, + FirecrawlSearchTool, GithubSearchTool, JSONSearchTool, LlamaIndexTool, diff --git a/src/crewai_tools/tools/__init__.py b/src/crewai_tools/tools/__init__.py index f5ac94052..17d289832 100644 --- a/src/crewai_tools/tools/__init__.py +++ b/src/crewai_tools/tools/__init__.py @@ -8,6 +8,9 @@ from .directory_search_tool.directory_search_tool import DirectorySearchTool from .docx_search_tool.docx_search_tool import DOCXSearchTool from .exa_tools.exa_search_tool import EXASearchTool from .file_read_tool.file_read_tool import FileReadTool +from .firecrawl_crawl_website_tool.firecrawl_crawl_website_tool import FirecrawlCrawlWebsiteTool +from .firecrawl_scrape_website_tool.firecrawl_scrape_website_tool import FirecrawlScrapeWebsiteTool +from .firecrawl_search_tool.firecrawl_search_tool import FirecrawlSearchTool from .github_search_tool.github_search_tool import GithubSearchTool from .json_search_tool.json_search_tool import JSONSearchTool from .llamaindex_tool.llamaindex_tool import LlamaIndexTool