mirror of
https://github.com/crewAIInc/crewAI.git
synced 2026-01-09 16:18:30 +00:00
Merge pull request #181 from crewAIInc/fix/optional-dependencies
make extra dependencies optional for our tools (making most tools an opt in)
This commit is contained in:
@@ -37,9 +37,19 @@ class BrowserbaseLoadTool(BaseTool):
|
|||||||
try:
|
try:
|
||||||
from browserbase import Browserbase # type: ignore
|
from browserbase import Browserbase # type: ignore
|
||||||
except ImportError:
|
except ImportError:
|
||||||
raise ImportError(
|
import click
|
||||||
"`browserbase` package not found, please run `pip install browserbase`"
|
|
||||||
)
|
if click.confirm(
|
||||||
|
"`browserbase` package not found, would you like to install it?"
|
||||||
|
):
|
||||||
|
import subprocess
|
||||||
|
|
||||||
|
subprocess.run(["uv", "add", "browserbase"], check=True)
|
||||||
|
from browserbase import Browserbase # type: ignore
|
||||||
|
else:
|
||||||
|
raise ImportError(
|
||||||
|
"`browserbase` package not found, please run `uv add browserbase`"
|
||||||
|
)
|
||||||
|
|
||||||
self.browserbase = Browserbase(api_key=self.api_key)
|
self.browserbase = Browserbase(api_key=self.api_key)
|
||||||
self.text_content = text_content
|
self.text_content = text_content
|
||||||
|
|||||||
@@ -1,12 +1,13 @@
|
|||||||
import os
|
from typing import Any, Dict, Optional, Type
|
||||||
from typing import TYPE_CHECKING, Any, Dict, Optional, Type
|
|
||||||
|
|
||||||
from crewai.tools import BaseTool
|
from crewai.tools import BaseTool
|
||||||
from pydantic import BaseModel, ConfigDict, Field, PrivateAttr
|
from pydantic import BaseModel, ConfigDict, Field, PrivateAttr
|
||||||
|
|
||||||
# Type checking import
|
|
||||||
if TYPE_CHECKING:
|
try:
|
||||||
from firecrawl import FirecrawlApp
|
from firecrawl import FirecrawlApp
|
||||||
|
except ImportError:
|
||||||
|
FirecrawlApp = Any
|
||||||
|
|
||||||
|
|
||||||
class FirecrawlCrawlWebsiteToolSchema(BaseModel):
|
class FirecrawlCrawlWebsiteToolSchema(BaseModel):
|
||||||
@@ -32,20 +33,33 @@ class FirecrawlCrawlWebsiteTool(BaseTool):
|
|||||||
|
|
||||||
def __init__(self, api_key: Optional[str] = None, **kwargs):
|
def __init__(self, api_key: Optional[str] = None, **kwargs):
|
||||||
super().__init__(**kwargs)
|
super().__init__(**kwargs)
|
||||||
|
self.api_key = api_key
|
||||||
|
self._initialize_firecrawl()
|
||||||
|
|
||||||
|
def _initialize_firecrawl(self) -> None:
|
||||||
try:
|
try:
|
||||||
from firecrawl import FirecrawlApp # type: ignore
|
from firecrawl import FirecrawlApp # type: ignore
|
||||||
except ImportError:
|
|
||||||
raise ImportError(
|
|
||||||
"`firecrawl` package not found, please run `pip install firecrawl-py`"
|
|
||||||
)
|
|
||||||
|
|
||||||
client_api_key = api_key or os.getenv("FIRECRAWL_API_KEY")
|
self._firecrawl = FirecrawlApp(api_key=self.api_key)
|
||||||
if not client_api_key:
|
except ImportError:
|
||||||
raise ValueError(
|
import click
|
||||||
"FIRECRAWL_API_KEY is not set. Please provide it either via the constructor "
|
|
||||||
"with the `api_key` argument or by setting the FIRECRAWL_API_KEY environment variable."
|
if click.confirm(
|
||||||
)
|
"You are missing the 'firecrawl-py' package. Would you like to install it?"
|
||||||
self._firecrawl = FirecrawlApp(api_key=client_api_key)
|
):
|
||||||
|
import subprocess
|
||||||
|
|
||||||
|
try:
|
||||||
|
subprocess.run(["uv", "add", "firecrawl-py"], check=True)
|
||||||
|
from firecrawl import FirecrawlApp
|
||||||
|
|
||||||
|
self._firecrawl = FirecrawlApp(api_key=self.api_key)
|
||||||
|
except subprocess.CalledProcessError:
|
||||||
|
raise ImportError("Failed to install firecrawl-py package")
|
||||||
|
else:
|
||||||
|
raise ImportError(
|
||||||
|
"`firecrawl-py` package not found, please run `uv add firecrawl-py`"
|
||||||
|
)
|
||||||
|
|
||||||
def _run(
|
def _run(
|
||||||
self,
|
self,
|
||||||
@@ -66,8 +80,10 @@ class FirecrawlCrawlWebsiteTool(BaseTool):
|
|||||||
try:
|
try:
|
||||||
from firecrawl import FirecrawlApp
|
from firecrawl import FirecrawlApp
|
||||||
|
|
||||||
# Must rebuild model after class is defined
|
# Only rebuild if the class hasn't been initialized yet
|
||||||
FirecrawlCrawlWebsiteTool.model_rebuild()
|
if not hasattr(FirecrawlCrawlWebsiteTool, "_model_rebuilt"):
|
||||||
|
FirecrawlCrawlWebsiteTool.model_rebuild()
|
||||||
|
FirecrawlCrawlWebsiteTool._model_rebuilt = True
|
||||||
except ImportError:
|
except ImportError:
|
||||||
"""
|
"""
|
||||||
When this tool is not used, then exception can be ignored.
|
When this tool is not used, then exception can be ignored.
|
||||||
|
|||||||
@@ -1,11 +1,12 @@
|
|||||||
from typing import TYPE_CHECKING, Optional, Type
|
from typing import Any, Optional, Type
|
||||||
|
|
||||||
from crewai.tools import BaseTool
|
from crewai.tools import BaseTool
|
||||||
from pydantic import BaseModel, ConfigDict, Field, PrivateAttr
|
from pydantic import BaseModel, ConfigDict, Field, PrivateAttr
|
||||||
|
|
||||||
# Type checking import
|
try:
|
||||||
if TYPE_CHECKING:
|
|
||||||
from firecrawl import FirecrawlApp
|
from firecrawl import FirecrawlApp
|
||||||
|
except ImportError:
|
||||||
|
FirecrawlApp = Any
|
||||||
|
|
||||||
|
|
||||||
class FirecrawlScrapeWebsiteToolSchema(BaseModel):
|
class FirecrawlScrapeWebsiteToolSchema(BaseModel):
|
||||||
@@ -31,9 +32,21 @@ class FirecrawlScrapeWebsiteTool(BaseTool):
|
|||||||
try:
|
try:
|
||||||
from firecrawl import FirecrawlApp # type: ignore
|
from firecrawl import FirecrawlApp # type: ignore
|
||||||
except ImportError:
|
except ImportError:
|
||||||
raise ImportError(
|
import click
|
||||||
"`firecrawl` package not found, please run `pip install firecrawl-py`"
|
|
||||||
)
|
if click.confirm(
|
||||||
|
"You are missing the 'firecrawl-py' package. Would you like to install it?"
|
||||||
|
):
|
||||||
|
import subprocess
|
||||||
|
|
||||||
|
subprocess.run(["uv", "add", "firecrawl-py"], check=True)
|
||||||
|
from firecrawl import (
|
||||||
|
FirecrawlApp,
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
raise ImportError(
|
||||||
|
"`firecrawl-py` package not found, please run `uv add firecrawl-py`"
|
||||||
|
)
|
||||||
|
|
||||||
self._firecrawl = FirecrawlApp(api_key=api_key)
|
self._firecrawl = FirecrawlApp(api_key=api_key)
|
||||||
|
|
||||||
@@ -58,7 +71,9 @@ try:
|
|||||||
from firecrawl import FirecrawlApp
|
from firecrawl import FirecrawlApp
|
||||||
|
|
||||||
# Must rebuild model after class is defined
|
# Must rebuild model after class is defined
|
||||||
FirecrawlScrapeWebsiteTool.model_rebuild()
|
if not hasattr(FirecrawlScrapeWebsiteTool, "_model_rebuilt"):
|
||||||
|
FirecrawlScrapeWebsiteTool.model_rebuild()
|
||||||
|
FirecrawlScrapeWebsiteTool._model_rebuilt = True
|
||||||
except ImportError:
|
except ImportError:
|
||||||
"""
|
"""
|
||||||
When this tool is not used, then exception can be ignored.
|
When this tool is not used, then exception can be ignored.
|
||||||
|
|||||||
@@ -1,11 +1,13 @@
|
|||||||
from typing import TYPE_CHECKING, Any, Dict, Optional, Type
|
from typing import Any, Dict, Optional, Type
|
||||||
|
|
||||||
from crewai.tools import BaseTool
|
from crewai.tools import BaseTool
|
||||||
from pydantic import BaseModel, ConfigDict, Field, PrivateAttr
|
from pydantic import BaseModel, ConfigDict, Field, PrivateAttr
|
||||||
|
|
||||||
# Type checking import
|
# Type checking import
|
||||||
if TYPE_CHECKING:
|
try:
|
||||||
from firecrawl import FirecrawlApp
|
from firecrawl import FirecrawlApp
|
||||||
|
except ImportError:
|
||||||
|
FirecrawlApp = Any
|
||||||
|
|
||||||
|
|
||||||
class FirecrawlSearchToolSchema(BaseModel):
|
class FirecrawlSearchToolSchema(BaseModel):
|
||||||
@@ -30,6 +32,9 @@ class FirecrawlSearchToolSchema(BaseModel):
|
|||||||
|
|
||||||
|
|
||||||
class FirecrawlSearchTool(BaseTool):
|
class FirecrawlSearchTool(BaseTool):
|
||||||
|
model_config = ConfigDict(
|
||||||
|
arbitrary_types_allowed=True, validate_assignment=True, frozen=False
|
||||||
|
)
|
||||||
model_config = ConfigDict(
|
model_config = ConfigDict(
|
||||||
arbitrary_types_allowed=True, validate_assignment=True, frozen=False
|
arbitrary_types_allowed=True, validate_assignment=True, frozen=False
|
||||||
)
|
)
|
||||||
@@ -41,13 +46,33 @@ class FirecrawlSearchTool(BaseTool):
|
|||||||
|
|
||||||
def __init__(self, api_key: Optional[str] = None, **kwargs):
|
def __init__(self, api_key: Optional[str] = None, **kwargs):
|
||||||
super().__init__(**kwargs)
|
super().__init__(**kwargs)
|
||||||
|
self.api_key = api_key
|
||||||
|
self._initialize_firecrawl()
|
||||||
|
|
||||||
|
def _initialize_firecrawl(self) -> None:
|
||||||
try:
|
try:
|
||||||
from firecrawl import FirecrawlApp # type: ignore
|
from firecrawl import FirecrawlApp # type: ignore
|
||||||
|
|
||||||
|
self.firecrawl = FirecrawlApp(api_key=self.api_key)
|
||||||
except ImportError:
|
except ImportError:
|
||||||
raise ImportError(
|
import click
|
||||||
"`firecrawl` package not found, please run `pip install firecrawl-py`"
|
|
||||||
)
|
if click.confirm(
|
||||||
self._firecrawl = FirecrawlApp(api_key=api_key)
|
"You are missing the 'firecrawl-py' package. Would you like to install it?"
|
||||||
|
):
|
||||||
|
import subprocess
|
||||||
|
|
||||||
|
try:
|
||||||
|
subprocess.run(["uv", "add", "firecrawl-py"], check=True)
|
||||||
|
from firecrawl import FirecrawlApp
|
||||||
|
|
||||||
|
self.firecrawl = FirecrawlApp(api_key=self.api_key)
|
||||||
|
except subprocess.CalledProcessError:
|
||||||
|
raise ImportError("Failed to install firecrawl-py package")
|
||||||
|
else:
|
||||||
|
raise ImportError(
|
||||||
|
"`firecrawl-py` package not found, please run `uv add firecrawl-py`"
|
||||||
|
)
|
||||||
|
|
||||||
def _run(
|
def _run(
|
||||||
self,
|
self,
|
||||||
@@ -59,9 +84,9 @@ class FirecrawlSearchTool(BaseTool):
|
|||||||
location: Optional[str] = None,
|
location: Optional[str] = None,
|
||||||
timeout: Optional[int] = 60000,
|
timeout: Optional[int] = 60000,
|
||||||
scrape_options: Optional[Dict[str, Any]] = None,
|
scrape_options: Optional[Dict[str, Any]] = None,
|
||||||
):
|
) -> Any:
|
||||||
if scrape_options is None:
|
if not self.firecrawl:
|
||||||
scrape_options = {}
|
raise RuntimeError("FirecrawlApp not properly initialized")
|
||||||
|
|
||||||
options = {
|
options = {
|
||||||
"limit": limit,
|
"limit": limit,
|
||||||
@@ -70,16 +95,20 @@ class FirecrawlSearchTool(BaseTool):
|
|||||||
"country": country,
|
"country": country,
|
||||||
"location": location,
|
"location": location,
|
||||||
"timeout": timeout,
|
"timeout": timeout,
|
||||||
"scrapeOptions": scrape_options,
|
"scrapeOptions": scrape_options or {},
|
||||||
}
|
}
|
||||||
return self._firecrawl.search(query, options)
|
return self.firecrawl.search(**options)
|
||||||
|
|
||||||
|
|
||||||
try:
|
try:
|
||||||
from firecrawl import FirecrawlApp
|
from firecrawl import FirecrawlApp # type: ignore
|
||||||
|
|
||||||
# Rebuild the model after class is defined
|
# Only rebuild if the class hasn't been initialized yet
|
||||||
FirecrawlSearchTool.model_rebuild()
|
if not hasattr(FirecrawlSearchTool, "_model_rebuilt"):
|
||||||
|
FirecrawlSearchTool.model_rebuild()
|
||||||
|
FirecrawlSearchTool._model_rebuilt = True
|
||||||
except ImportError:
|
except ImportError:
|
||||||
# Exception can be ignored if the tool is not used
|
"""
|
||||||
|
When this tool is not used, then exception can be ignored.
|
||||||
|
"""
|
||||||
pass
|
pass
|
||||||
|
|||||||
@@ -1,5 +1,7 @@
|
|||||||
from typing import Any
|
from typing import Any
|
||||||
|
|
||||||
|
from crewai.tools import BaseTool
|
||||||
|
|
||||||
try:
|
try:
|
||||||
from linkup import LinkupClient
|
from linkup import LinkupClient
|
||||||
|
|
||||||
@@ -11,22 +13,40 @@ except ImportError:
|
|||||||
from pydantic import PrivateAttr
|
from pydantic import PrivateAttr
|
||||||
|
|
||||||
|
|
||||||
class LinkupSearchTool:
|
class LinkupSearchTool(BaseTool):
|
||||||
name: str = "Linkup Search Tool"
|
name: str = "Linkup Search Tool"
|
||||||
description: str = (
|
description: str = (
|
||||||
"Performs an API call to Linkup to retrieve contextual information."
|
"Performs an API call to Linkup to retrieve contextual information."
|
||||||
)
|
)
|
||||||
_client: LinkupClient = PrivateAttr() # type: ignore
|
_client: LinkupClient = PrivateAttr() # type: ignore
|
||||||
|
description: str = (
|
||||||
|
"Performs an API call to Linkup to retrieve contextual information."
|
||||||
|
)
|
||||||
|
_client: LinkupClient = PrivateAttr() # type: ignore
|
||||||
|
|
||||||
def __init__(self, api_key: str):
|
def __init__(self, api_key: str):
|
||||||
"""
|
"""
|
||||||
Initialize the tool with an API key.
|
Initialize the tool with an API key.
|
||||||
"""
|
"""
|
||||||
if not LINKUP_AVAILABLE:
|
super().__init__()
|
||||||
raise ImportError(
|
try:
|
||||||
"The 'linkup' package is required to use the LinkupSearchTool. "
|
from linkup import LinkupClient
|
||||||
"Please install it with: uv add linkup"
|
except ImportError:
|
||||||
)
|
import click
|
||||||
|
|
||||||
|
if click.confirm(
|
||||||
|
"You are missing the 'linkup-sdk' package. Would you like to install it?"
|
||||||
|
):
|
||||||
|
import subprocess
|
||||||
|
|
||||||
|
subprocess.run(["uv", "add", "linkup-sdk"], check=True)
|
||||||
|
from linkup import LinkupClient
|
||||||
|
|
||||||
|
else:
|
||||||
|
raise ImportError(
|
||||||
|
"The 'linkup-sdk' package is required to use the LinkupSearchTool. "
|
||||||
|
"Please install it with: uv add linkup-sdk"
|
||||||
|
)
|
||||||
self._client = LinkupClient(api_key=api_key)
|
self._client = LinkupClient(api_key=api_key)
|
||||||
|
|
||||||
def _run(
|
def _run(
|
||||||
|
|||||||
@@ -28,9 +28,19 @@ class MultiOnTool(BaseTool):
|
|||||||
try:
|
try:
|
||||||
from multion.client import MultiOn # type: ignore
|
from multion.client import MultiOn # type: ignore
|
||||||
except ImportError:
|
except ImportError:
|
||||||
raise ImportError(
|
import click
|
||||||
"`multion` package not found, please run `pip install multion`"
|
|
||||||
)
|
if click.confirm(
|
||||||
|
"You are missing the 'multion' package. Would you like to install it?"
|
||||||
|
):
|
||||||
|
import subprocess
|
||||||
|
|
||||||
|
subprocess.run(["uv", "add", "multion"], check=True)
|
||||||
|
from multion.client import MultiOn
|
||||||
|
else:
|
||||||
|
raise ImportError(
|
||||||
|
"`multion` package not found, please run `uv add multion`"
|
||||||
|
)
|
||||||
self.session_id = None
|
self.session_id = None
|
||||||
self.local = local
|
self.local = local
|
||||||
self.multion = MultiOn(api_key=api_key)
|
self.multion = MultiOn(api_key=api_key)
|
||||||
|
|||||||
@@ -4,6 +4,14 @@ from crewai.tools import BaseTool
|
|||||||
from patronus import Client
|
from patronus import Client
|
||||||
from pydantic import BaseModel, Field
|
from pydantic import BaseModel, Field
|
||||||
|
|
||||||
|
try:
|
||||||
|
from patronus import Client
|
||||||
|
|
||||||
|
PYPATRONUS_AVAILABLE = True
|
||||||
|
except ImportError:
|
||||||
|
PYPATRONUS_AVAILABLE = False
|
||||||
|
Client = Any
|
||||||
|
|
||||||
|
|
||||||
class FixedLocalEvaluatorToolSchema(BaseModel):
|
class FixedLocalEvaluatorToolSchema(BaseModel):
|
||||||
evaluated_model_input: str = Field(
|
evaluated_model_input: str = Field(
|
||||||
@@ -26,12 +34,20 @@ class PatronusLocalEvaluatorTool(BaseTool):
|
|||||||
evaluator: str = "The registered local evaluator"
|
evaluator: str = "The registered local evaluator"
|
||||||
evaluated_model_gold_answer: str = "The agent's gold answer"
|
evaluated_model_gold_answer: str = "The agent's gold answer"
|
||||||
description: str = "This tool is used to evaluate the model input and output using custom function evaluators."
|
description: str = "This tool is used to evaluate the model input and output using custom function evaluators."
|
||||||
|
description: str = "This tool is used to evaluate the model input and output using custom function evaluators."
|
||||||
client: Any = None
|
client: Any = None
|
||||||
args_schema: Type[BaseModel] = FixedLocalEvaluatorToolSchema
|
args_schema: Type[BaseModel] = FixedLocalEvaluatorToolSchema
|
||||||
|
|
||||||
class Config:
|
class Config:
|
||||||
arbitrary_types_allowed = True
|
arbitrary_types_allowed = True
|
||||||
|
|
||||||
|
def __init__(
|
||||||
|
self,
|
||||||
|
patronus_client: Client,
|
||||||
|
evaluator: str,
|
||||||
|
evaluated_model_gold_answer: str,
|
||||||
|
**kwargs: Any,
|
||||||
|
):
|
||||||
def __init__(
|
def __init__(
|
||||||
self,
|
self,
|
||||||
patronus_client: Client,
|
patronus_client: Client,
|
||||||
@@ -40,15 +56,29 @@ class PatronusLocalEvaluatorTool(BaseTool):
|
|||||||
**kwargs: Any,
|
**kwargs: Any,
|
||||||
):
|
):
|
||||||
super().__init__(**kwargs)
|
super().__init__(**kwargs)
|
||||||
self.client = patronus_client
|
if PYPATRONUS_AVAILABLE:
|
||||||
if evaluator:
|
self.client = patronus_client
|
||||||
self.evaluator = evaluator
|
if evaluator:
|
||||||
self.evaluated_model_gold_answer = evaluated_model_gold_answer
|
self.evaluator = evaluator
|
||||||
|
self.evaluated_model_gold_answer = evaluated_model_gold_answer
|
||||||
self.description = f"This tool calls the Patronus Evaluation API that takes an additional argument in addition to the following new argument:\n evaluators={evaluator}, evaluated_model_gold_answer={evaluated_model_gold_answer}"
|
self.description = f"This tool calls the Patronus Evaluation API that takes an additional argument in addition to the following new argument:\n evaluators={evaluator}, evaluated_model_gold_answer={evaluated_model_gold_answer}"
|
||||||
self._generate_description()
|
self._generate_description()
|
||||||
print(
|
print(
|
||||||
f"Updating judge evaluator, gold_answer to: {self.evaluator}, {self.evaluated_model_gold_answer}"
|
f"Updating judge evaluator, gold_answer to: {self.evaluator}, {self.evaluated_model_gold_answer}"
|
||||||
)
|
)
|
||||||
|
else:
|
||||||
|
import click
|
||||||
|
|
||||||
|
if click.confirm(
|
||||||
|
"You are missing the 'patronus' package. Would you like to install it?"
|
||||||
|
):
|
||||||
|
import subprocess
|
||||||
|
|
||||||
|
subprocess.run(["uv", "add", "patronus"], check=True)
|
||||||
|
else:
|
||||||
|
raise ImportError(
|
||||||
|
"You are missing the patronus package. Would you like to install it?"
|
||||||
|
)
|
||||||
|
|
||||||
def _run(
|
def _run(
|
||||||
self,
|
self,
|
||||||
@@ -85,6 +115,7 @@ class PatronusLocalEvaluatorTool(BaseTool):
|
|||||||
else evaluated_model_gold_answer.get("description")
|
else evaluated_model_gold_answer.get("description")
|
||||||
),
|
),
|
||||||
tags={}, # Optional metadata, supports arbitrary kv pairs
|
tags={}, # Optional metadata, supports arbitrary kv pairs
|
||||||
|
tags={}, # Optional metadata, supports arbitrary kv pairs
|
||||||
)
|
)
|
||||||
output = f"Evaluation result: {result.pass_}, Explanation: {result.explanation}"
|
output = f"Evaluation result: {result.pass_}, Explanation: {result.explanation}"
|
||||||
return output
|
return output
|
||||||
|
|||||||
@@ -1,11 +1,13 @@
|
|||||||
import os
|
import os
|
||||||
from typing import Any, Optional, Type
|
from typing import Any, Optional, Type, TYPE_CHECKING
|
||||||
from urllib.parse import urlparse
|
from urllib.parse import urlparse
|
||||||
|
|
||||||
from crewai.tools import BaseTool
|
from crewai.tools import BaseTool
|
||||||
from pydantic import BaseModel, Field, validator
|
from pydantic import BaseModel, Field, validator, ConfigDict
|
||||||
from scrapegraph_py import Client
|
|
||||||
from scrapegraph_py.logger import sgai_logger
|
# Type checking import
|
||||||
|
if TYPE_CHECKING:
|
||||||
|
from scrapegraph_py import Client
|
||||||
|
|
||||||
|
|
||||||
class ScrapegraphError(Exception):
|
class ScrapegraphError(Exception):
|
||||||
@@ -53,6 +55,8 @@ class ScrapegraphScrapeTool(BaseTool):
|
|||||||
RuntimeError: If scraping operation fails
|
RuntimeError: If scraping operation fails
|
||||||
"""
|
"""
|
||||||
|
|
||||||
|
model_config = ConfigDict(arbitrary_types_allowed=True)
|
||||||
|
|
||||||
name: str = "Scrapegraph website scraper"
|
name: str = "Scrapegraph website scraper"
|
||||||
description: str = (
|
description: str = (
|
||||||
"A tool that uses Scrapegraph AI to intelligently scrape website content."
|
"A tool that uses Scrapegraph AI to intelligently scrape website content."
|
||||||
@@ -62,6 +66,7 @@ class ScrapegraphScrapeTool(BaseTool):
|
|||||||
user_prompt: Optional[str] = None
|
user_prompt: Optional[str] = None
|
||||||
api_key: Optional[str] = None
|
api_key: Optional[str] = None
|
||||||
enable_logging: bool = False
|
enable_logging: bool = False
|
||||||
|
_client: Optional["Client"] = None
|
||||||
|
|
||||||
def __init__(
|
def __init__(
|
||||||
self,
|
self,
|
||||||
@@ -72,6 +77,29 @@ class ScrapegraphScrapeTool(BaseTool):
|
|||||||
**kwargs,
|
**kwargs,
|
||||||
):
|
):
|
||||||
super().__init__(**kwargs)
|
super().__init__(**kwargs)
|
||||||
|
try:
|
||||||
|
from scrapegraph_py import Client
|
||||||
|
from scrapegraph_py.logger import sgai_logger
|
||||||
|
|
||||||
|
except ImportError:
|
||||||
|
import click
|
||||||
|
|
||||||
|
if click.confirm(
|
||||||
|
"You are missing the 'scrapegraph-py' package. Would you like to install it?"
|
||||||
|
):
|
||||||
|
import subprocess
|
||||||
|
|
||||||
|
subprocess.run(["uv", "add", "scrapegraph-py"], check=True)
|
||||||
|
from scrapegraph_py import Client
|
||||||
|
from scrapegraph_py.logger import sgai_logger
|
||||||
|
|
||||||
|
else:
|
||||||
|
raise ImportError(
|
||||||
|
"`scrapegraph-py` package not found, please run `uv add scrapegraph-py`"
|
||||||
|
)
|
||||||
|
|
||||||
|
self._client = Client(api_key=api_key)
|
||||||
|
|
||||||
self.api_key = api_key or os.getenv("SCRAPEGRAPH_API_KEY")
|
self.api_key = api_key or os.getenv("SCRAPEGRAPH_API_KEY")
|
||||||
|
|
||||||
if not self.api_key:
|
if not self.api_key:
|
||||||
@@ -102,6 +130,22 @@ class ScrapegraphScrapeTool(BaseTool):
|
|||||||
"Invalid URL format. URL must include scheme (http/https) and domain"
|
"Invalid URL format. URL must include scheme (http/https) and domain"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
def _handle_api_response(self, response: dict) -> str:
|
||||||
|
"""Handle and validate API response"""
|
||||||
|
if not response:
|
||||||
|
raise RuntimeError("Empty response from Scrapegraph API")
|
||||||
|
|
||||||
|
if "error" in response:
|
||||||
|
error_msg = response.get("error", {}).get("message", "Unknown error")
|
||||||
|
if "rate limit" in error_msg.lower():
|
||||||
|
raise RateLimitError(f"Rate limit exceeded: {error_msg}")
|
||||||
|
raise RuntimeError(f"API error: {error_msg}")
|
||||||
|
|
||||||
|
if "result" not in response:
|
||||||
|
raise RuntimeError("Invalid response format from Scrapegraph API")
|
||||||
|
|
||||||
|
return response["result"]
|
||||||
|
|
||||||
def _run(
|
def _run(
|
||||||
self,
|
self,
|
||||||
**kwargs: Any,
|
**kwargs: Any,
|
||||||
@@ -118,12 +162,9 @@ class ScrapegraphScrapeTool(BaseTool):
|
|||||||
# Validate URL format
|
# Validate URL format
|
||||||
self._validate_url(website_url)
|
self._validate_url(website_url)
|
||||||
|
|
||||||
# Initialize the client
|
|
||||||
sgai_client = Client(api_key=self.api_key)
|
|
||||||
|
|
||||||
try:
|
try:
|
||||||
# Make the SmartScraper request
|
# Make the SmartScraper request
|
||||||
response = sgai_client.smartscraper(
|
response = self._client.smartscraper(
|
||||||
website_url=website_url,
|
website_url=website_url,
|
||||||
user_prompt=user_prompt,
|
user_prompt=user_prompt,
|
||||||
)
|
)
|
||||||
@@ -136,4 +177,4 @@ class ScrapegraphScrapeTool(BaseTool):
|
|||||||
raise RuntimeError(f"Scraping failed: {str(e)}")
|
raise RuntimeError(f"Scraping failed: {str(e)}")
|
||||||
finally:
|
finally:
|
||||||
# Always close the client
|
# Always close the client
|
||||||
sgai_client.close()
|
self._client.close()
|
||||||
|
|||||||
@@ -34,9 +34,18 @@ class ScrapflyScrapeWebsiteTool(BaseTool):
|
|||||||
try:
|
try:
|
||||||
from scrapfly import ScrapflyClient
|
from scrapfly import ScrapflyClient
|
||||||
except ImportError:
|
except ImportError:
|
||||||
raise ImportError(
|
import click
|
||||||
"`scrapfly` package not found, please run `pip install scrapfly-sdk`"
|
|
||||||
)
|
if click.confirm(
|
||||||
|
"You are missing the 'scrapfly-sdk' package. Would you like to install it?"
|
||||||
|
):
|
||||||
|
import subprocess
|
||||||
|
|
||||||
|
subprocess.run(["uv", "add", "scrapfly-sdk"], check=True)
|
||||||
|
else:
|
||||||
|
raise ImportError(
|
||||||
|
"`scrapfly-sdk` package not found, please run `uv add scrapfly-sdk`"
|
||||||
|
)
|
||||||
self.scrapfly = ScrapflyClient(key=api_key)
|
self.scrapfly = ScrapflyClient(key=api_key)
|
||||||
|
|
||||||
def _run(
|
def _run(
|
||||||
|
|||||||
@@ -5,9 +5,6 @@ from urllib.parse import urlparse
|
|||||||
|
|
||||||
from crewai.tools import BaseTool
|
from crewai.tools import BaseTool
|
||||||
from pydantic import BaseModel, Field, validator
|
from pydantic import BaseModel, Field, validator
|
||||||
from selenium import webdriver
|
|
||||||
from selenium.webdriver.chrome.options import Options
|
|
||||||
from selenium.webdriver.common.by import By
|
|
||||||
|
|
||||||
|
|
||||||
class FixedSeleniumScrapingToolSchema(BaseModel):
|
class FixedSeleniumScrapingToolSchema(BaseModel):
|
||||||
@@ -55,11 +52,13 @@ class SeleniumScrapingTool(BaseTool):
|
|||||||
description: str = "A tool that can be used to read a website content."
|
description: str = "A tool that can be used to read a website content."
|
||||||
args_schema: Type[BaseModel] = SeleniumScrapingToolSchema
|
args_schema: Type[BaseModel] = SeleniumScrapingToolSchema
|
||||||
website_url: Optional[str] = None
|
website_url: Optional[str] = None
|
||||||
driver: Optional[Any] = webdriver.Chrome
|
driver: Optional[Any] = None
|
||||||
cookie: Optional[dict] = None
|
cookie: Optional[dict] = None
|
||||||
wait_time: Optional[int] = 3
|
wait_time: Optional[int] = 3
|
||||||
css_element: Optional[str] = None
|
css_element: Optional[str] = None
|
||||||
return_html: Optional[bool] = False
|
return_html: Optional[bool] = False
|
||||||
|
_options: Optional[dict] = None
|
||||||
|
_by: Optional[Any] = None
|
||||||
|
|
||||||
def __init__(
|
def __init__(
|
||||||
self,
|
self,
|
||||||
@@ -69,6 +68,32 @@ class SeleniumScrapingTool(BaseTool):
|
|||||||
**kwargs,
|
**kwargs,
|
||||||
):
|
):
|
||||||
super().__init__(**kwargs)
|
super().__init__(**kwargs)
|
||||||
|
try:
|
||||||
|
from selenium import webdriver
|
||||||
|
from selenium.webdriver.chrome.options import Options
|
||||||
|
from selenium.webdriver.common.by import By
|
||||||
|
except ImportError:
|
||||||
|
import click
|
||||||
|
|
||||||
|
if click.confirm(
|
||||||
|
"You are missing the 'selenium' and 'webdriver-manager' packages. Would you like to install it?"
|
||||||
|
):
|
||||||
|
import subprocess
|
||||||
|
|
||||||
|
subprocess.run(
|
||||||
|
["uv", "pip", "install", "selenium", "webdriver-manager"],
|
||||||
|
check=True,
|
||||||
|
)
|
||||||
|
from selenium import webdriver
|
||||||
|
from selenium.webdriver.chrome.options import Options
|
||||||
|
from selenium.webdriver.common.by import By
|
||||||
|
else:
|
||||||
|
raise ImportError(
|
||||||
|
"`selenium` and `webdriver-manager` package not found, please run `uv add selenium webdriver-manager`"
|
||||||
|
)
|
||||||
|
self.driver = webdriver.Chrome()
|
||||||
|
self._options = Options()
|
||||||
|
self._by = By
|
||||||
if cookie is not None:
|
if cookie is not None:
|
||||||
self.cookie = cookie
|
self.cookie = cookie
|
||||||
|
|
||||||
@@ -112,7 +137,7 @@ class SeleniumScrapingTool(BaseTool):
|
|||||||
return css_element is None or css_element.strip() == ""
|
return css_element is None or css_element.strip() == ""
|
||||||
|
|
||||||
def _get_body_content(self, driver, return_html):
|
def _get_body_content(self, driver, return_html):
|
||||||
body_element = driver.find_element(By.TAG_NAME, "body")
|
body_element = driver.find_element(self._by.TAG_NAME, "body")
|
||||||
|
|
||||||
return (
|
return (
|
||||||
body_element.get_attribute("outerHTML")
|
body_element.get_attribute("outerHTML")
|
||||||
@@ -123,7 +148,7 @@ class SeleniumScrapingTool(BaseTool):
|
|||||||
def _get_elements_content(self, driver, css_element, return_html):
|
def _get_elements_content(self, driver, css_element, return_html):
|
||||||
elements_content = []
|
elements_content = []
|
||||||
|
|
||||||
for element in driver.find_elements(By.CSS_SELECTOR, css_element):
|
for element in driver.find_elements(self._by.CSS_SELECTOR, css_element):
|
||||||
elements_content.append(
|
elements_content.append(
|
||||||
element.get_attribute("outerHTML") if return_html else element.text
|
element.get_attribute("outerHTML") if return_html else element.text
|
||||||
)
|
)
|
||||||
@@ -138,7 +163,7 @@ class SeleniumScrapingTool(BaseTool):
|
|||||||
if not re.match(r"^https?://", url):
|
if not re.match(r"^https?://", url):
|
||||||
raise ValueError("URL must start with http:// or https://")
|
raise ValueError("URL must start with http:// or https://")
|
||||||
|
|
||||||
options = Options()
|
options = self._options
|
||||||
options.add_argument("--headless")
|
options.add_argument("--headless")
|
||||||
driver = self.driver(options=options)
|
driver = self.driver(options=options)
|
||||||
driver.get(url)
|
driver.get(url)
|
||||||
|
|||||||
@@ -14,11 +14,21 @@ class SerpApiBaseTool(BaseTool):
|
|||||||
super().__init__(**kwargs)
|
super().__init__(**kwargs)
|
||||||
|
|
||||||
try:
|
try:
|
||||||
from serpapi import Client
|
from serpapi import Client # type: ignore
|
||||||
except ImportError:
|
except ImportError:
|
||||||
raise ImportError(
|
import click
|
||||||
"`serpapi` package not found, please install with `pip install serpapi`"
|
|
||||||
)
|
if click.confirm(
|
||||||
|
"You are missing the 'serpapi' package. Would you like to install it?"
|
||||||
|
):
|
||||||
|
import subprocess
|
||||||
|
|
||||||
|
subprocess.run(["uv", "add", "serpapi"], check=True)
|
||||||
|
from serpapi import Client
|
||||||
|
else:
|
||||||
|
raise ImportError(
|
||||||
|
"`serpapi` package not found, please install with `uv add serpapi`"
|
||||||
|
)
|
||||||
api_key = os.getenv("SERPAPI_API_KEY")
|
api_key = os.getenv("SERPAPI_API_KEY")
|
||||||
if not api_key:
|
if not api_key:
|
||||||
raise ValueError(
|
raise ValueError(
|
||||||
|
|||||||
@@ -1,10 +1,13 @@
|
|||||||
from typing import Any, Optional, Type
|
from typing import Any, Optional, Type
|
||||||
|
|
||||||
from pydantic import BaseModel, Field
|
import re
|
||||||
|
from pydantic import BaseModel, Field, ConfigDict
|
||||||
from .serpapi_base_tool import SerpApiBaseTool
|
from .serpapi_base_tool import SerpApiBaseTool
|
||||||
from urllib.error import HTTPError
|
|
||||||
|
|
||||||
from .serpapi_base_tool import SerpApiBaseTool
|
try:
|
||||||
|
from serpapi import HTTPError
|
||||||
|
except ImportError:
|
||||||
|
HTTPError = Any
|
||||||
|
|
||||||
|
|
||||||
class SerpApiGoogleSearchToolSchema(BaseModel):
|
class SerpApiGoogleSearchToolSchema(BaseModel):
|
||||||
@@ -19,6 +22,9 @@ class SerpApiGoogleSearchToolSchema(BaseModel):
|
|||||||
|
|
||||||
|
|
||||||
class SerpApiGoogleSearchTool(SerpApiBaseTool):
|
class SerpApiGoogleSearchTool(SerpApiBaseTool):
|
||||||
|
model_config = ConfigDict(
|
||||||
|
arbitrary_types_allowed=True, validate_assignment=True, frozen=False
|
||||||
|
)
|
||||||
name: str = "Google Search"
|
name: str = "Google Search"
|
||||||
description: str = (
|
description: str = (
|
||||||
"A tool to perform to perform a Google search with a search_query."
|
"A tool to perform to perform a Google search with a search_query."
|
||||||
|
|||||||
@@ -2,9 +2,12 @@ from typing import Any, Optional, Type
|
|||||||
|
|
||||||
from pydantic import BaseModel, Field
|
from pydantic import BaseModel, Field
|
||||||
from .serpapi_base_tool import SerpApiBaseTool
|
from .serpapi_base_tool import SerpApiBaseTool
|
||||||
from urllib.error import HTTPError
|
from pydantic import ConfigDict
|
||||||
|
|
||||||
from .serpapi_base_tool import SerpApiBaseTool
|
try:
|
||||||
|
from serpapi import HTTPError
|
||||||
|
except ImportError:
|
||||||
|
HTTPError = Any
|
||||||
|
|
||||||
|
|
||||||
class SerpApiGoogleShoppingToolSchema(BaseModel):
|
class SerpApiGoogleShoppingToolSchema(BaseModel):
|
||||||
@@ -19,6 +22,9 @@ class SerpApiGoogleShoppingToolSchema(BaseModel):
|
|||||||
|
|
||||||
|
|
||||||
class SerpApiGoogleShoppingTool(SerpApiBaseTool):
|
class SerpApiGoogleShoppingTool(SerpApiBaseTool):
|
||||||
|
model_config = ConfigDict(
|
||||||
|
arbitrary_types_allowed=True, validate_assignment=True, frozen=False
|
||||||
|
)
|
||||||
name: str = "Google Shopping"
|
name: str = "Google Shopping"
|
||||||
description: str = (
|
description: str = (
|
||||||
"A tool to perform search on Google shopping with a search_query."
|
"A tool to perform search on Google shopping with a search_query."
|
||||||
|
|||||||
@@ -87,13 +87,21 @@ class SpiderTool(BaseTool):
|
|||||||
try:
|
try:
|
||||||
from spider import Spider # type: ignore
|
from spider import Spider # type: ignore
|
||||||
|
|
||||||
self.spider = Spider(api_key=api_key)
|
|
||||||
except ImportError:
|
except ImportError:
|
||||||
raise ImportError(
|
import click
|
||||||
"`spider-client` package not found, please run `uv add spider-client`"
|
|
||||||
)
|
if click.confirm(
|
||||||
except Exception as e:
|
"You are missing the 'spider-client' package. Would you like to install it?"
|
||||||
raise RuntimeError(f"Failed to initialize Spider client: {str(e)}")
|
):
|
||||||
|
import subprocess
|
||||||
|
|
||||||
|
subprocess.run(["uv", "pip", "install", "spider-client"], check=True)
|
||||||
|
from spider import Spider
|
||||||
|
else:
|
||||||
|
raise ImportError(
|
||||||
|
"`spider-client` package not found, please run `uv add spider-client`"
|
||||||
|
)
|
||||||
|
self.spider = Spider(api_key=api_key)
|
||||||
|
|
||||||
def _validate_url(self, url: str) -> bool:
|
def _validate_url(self, url: str) -> bool:
|
||||||
"""Validate URL format and security constraints.
|
"""Validate URL format and security constraints.
|
||||||
|
|||||||
@@ -168,10 +168,14 @@ class StagehandTool(BaseTool):
|
|||||||
super().__init__(**kwargs)
|
super().__init__(**kwargs)
|
||||||
|
|
||||||
if not STAGEHAND_AVAILABLE:
|
if not STAGEHAND_AVAILABLE:
|
||||||
raise ImportError(
|
import click
|
||||||
"The 'stagehand' package is required to use this tool. "
|
|
||||||
"Please install it with: pip install stagehand"
|
if click.confirm(
|
||||||
)
|
"You are missing the 'stagehand-sdk' package. Would you like to install it?"
|
||||||
|
):
|
||||||
|
import subprocess
|
||||||
|
|
||||||
|
subprocess.run(["uv", "add", "stagehand-sdk"], check=True)
|
||||||
|
|
||||||
# Use config if provided, otherwise try environment variable
|
# Use config if provided, otherwise try environment variable
|
||||||
if config is not None:
|
if config is not None:
|
||||||
|
|||||||
@@ -67,12 +67,25 @@ class WeaviateVectorSearchTool(BaseTool):
|
|||||||
model="gpt-4o",
|
model="gpt-4o",
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
|
else:
|
||||||
|
import click
|
||||||
|
|
||||||
|
if click.confirm(
|
||||||
|
"You are missing the 'weaviate-client' package. Would you like to install it?"
|
||||||
|
):
|
||||||
|
import subprocess
|
||||||
|
|
||||||
|
subprocess.run(["uv", "pip", "install", "weaviate-client"], check=True)
|
||||||
|
|
||||||
|
else:
|
||||||
|
raise ImportError(
|
||||||
|
"You are missing the 'weaviate-client' package. Would you like to install it?"
|
||||||
|
)
|
||||||
|
|
||||||
def _run(self, query: str) -> str:
|
def _run(self, query: str) -> str:
|
||||||
if not WEAVIATE_AVAILABLE:
|
if not WEAVIATE_AVAILABLE:
|
||||||
raise ImportError(
|
raise ImportError(
|
||||||
"The 'weaviate-client' package is required to use the WeaviateVectorSearchTool. "
|
"You are missing the 'weaviate-client' package. Would you like to install it?"
|
||||||
"Please install it with: uv add weaviate-client"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
if not self.weaviate_cluster_url or not self.weaviate_api_key:
|
if not self.weaviate_cluster_url or not self.weaviate_api_key:
|
||||||
|
|||||||
Reference in New Issue
Block a user