From 1b855b4ff97d3fc8bf6dc0981fed5f0999a7cd81 Mon Sep 17 00:00:00 2001 From: Bright Oparaji Date: Mon, 7 Sep 2026 14:40:44 +0100 Subject: [PATCH 1/9] docs(events): remove stale params from handle_llm_stream_chunk docstring (#7313) The Args block for ConsoleFormatter.handle_llm_stream_chunk listed 'chunk' and 'crew_tree' parameters that no longer exist on the method. The signature was refactored to (accumulated_text, call_type) but the docstring was not updated. Callers in event_listener.py pass only the two real params. Refs #7312. --- lib/crewai/src/crewai/events/utils/console_formatter.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/lib/crewai/src/crewai/events/utils/console_formatter.py b/lib/crewai/src/crewai/events/utils/console_formatter.py index ada09a3d8..733c0ff23 100644 --- a/lib/crewai/src/crewai/events/utils/console_formatter.py +++ b/lib/crewai/src/crewai/events/utils/console_formatter.py @@ -589,9 +589,7 @@ To enable tracing, do any one of these: """Handle LLM stream chunk event - display streaming text in a panel. Args: - chunk: The new chunk of text received. accumulated_text: All text accumulated so far. - crew_tree: Unused (kept for API compatibility). call_type: The type of LLM call (LLM_CALL or TOOL_CALL). """ if not self.verbose: From 98c067c22aa098ba48bc3fa48a1fb2c4e6a9a35d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=99=88=E5=BF=97=E8=B0=A6?= <89645338+simpleqt@users.noreply.github.com> Date: Tue, 8 Sep 2026 12:55:52 +0800 Subject: [PATCH 2/9] fix(brightdata): drop stray $ in f-string search URLs (#7326) get_search_url interpolated ${query} inside an f-string, producing URLs like https://www.bing.com/search?q=$test. The URL is passed straight to the SERP request, so every search carried the malformed query string. Fixes #7325 --- .../crewai_tools/tools/brightdata_tool/brightdata_serp.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/crewai-tools/src/crewai_tools/tools/brightdata_tool/brightdata_serp.py b/lib/crewai-tools/src/crewai_tools/tools/brightdata_tool/brightdata_serp.py index 83b5c4c30..81deac5d4 100644 --- a/lib/crewai-tools/src/crewai_tools/tools/brightdata_tool/brightdata_serp.py +++ b/lib/crewai-tools/src/crewai_tools/tools/brightdata_tool/brightdata_serp.py @@ -131,10 +131,10 @@ class BrightDataSearchTool(BaseTool): def get_search_url(self, engine: str, query: str) -> str: if engine == "yandex": - return f"https://yandex.com/search/?text=${query}" + return f"https://yandex.com/search/?text={query}" if engine == "bing": - return f"https://www.bing.com/search?q=${query}" - return f"https://www.google.com/search?q=${query}" + return f"https://www.bing.com/search?q={query}" + return f"https://www.google.com/search?q={query}" def _run( self, From 09997bfd6f3913f5271f17a690630ba6eb096ac1 Mon Sep 17 00:00:00 2001 From: Rolly Calma <115199279+Ghraven@users.noreply.github.com> Date: Tue, 8 Sep 2026 14:02:41 +0800 Subject: [PATCH 3/9] fix(state): persist json checkpoints as utf-8 (#7257) * fix(state): persist json checkpoints as utf-8 * test: import pathlib Path in checkpoint tests --------- Co-authored-by: Vidit Ostwal <110953813+Vidit-Ostwal@users.noreply.github.com> --- .../crewai/state/provider/json_provider.py | 8 ++++---- lib/crewai/tests/test_checkpoint.py | 20 +++++++++++++++++++ 2 files changed, 24 insertions(+), 4 deletions(-) diff --git a/lib/crewai/src/crewai/state/provider/json_provider.py b/lib/crewai/src/crewai/state/provider/json_provider.py index 904526292..93556f0e3 100644 --- a/lib/crewai/src/crewai/state/provider/json_provider.py +++ b/lib/crewai/src/crewai/state/provider/json_provider.py @@ -63,7 +63,7 @@ class JsonProvider(BaseProvider): file_path = _build_path(location, branch, parent_id) file_path.parent.mkdir(parents=True, exist_ok=True) - with open(file_path, "w") as f: + with open(file_path, "w", encoding="utf-8") as f: f.write(data) return str(file_path) @@ -91,7 +91,7 @@ class JsonProvider(BaseProvider): file_path = _build_path(location, branch, parent_id) await aiofiles.os.makedirs(str(file_path.parent), exist_ok=True) - async with aiofiles.open(file_path, "w") as f: + async with aiofiles.open(file_path, "w", encoding="utf-8") as f: await f.write(data) return str(file_path) @@ -129,7 +129,7 @@ class JsonProvider(BaseProvider): Returns: The raw JSON string. """ - return Path(location).read_text() + return Path(location).read_text(encoding="utf-8") async def afrom_checkpoint(self, location: str) -> str: """Read a JSON checkpoint file asynchronously. @@ -140,7 +140,7 @@ class JsonProvider(BaseProvider): Returns: The raw JSON string. """ - async with aiofiles.open(location) as f: + async with aiofiles.open(location, encoding="utf-8") as f: return await f.read() diff --git a/lib/crewai/tests/test_checkpoint.py b/lib/crewai/tests/test_checkpoint.py index 4d316afe8..ab49786c5 100644 --- a/lib/crewai/tests/test_checkpoint.py +++ b/lib/crewai/tests/test_checkpoint.py @@ -8,6 +8,7 @@ import os import sqlite3 import tempfile import time +from pathlib import Path from typing import Any from unittest.mock import MagicMock, patch @@ -378,6 +379,25 @@ class TestJsonProviderFork: assert path.endswith(".json") assert os.path.isfile(path) + def test_checkpoint_uses_utf8_for_non_ascii_json(self) -> None: + provider = JsonProvider() + data = '{"message": "olá niño"}' + with tempfile.TemporaryDirectory() as d: + path = provider.checkpoint(data, d, branch="main") + + assert Path(path).read_bytes() == data.encode("utf-8") + assert provider.from_checkpoint(path) == data + + @pytest.mark.asyncio + async def test_acheckpoint_uses_utf8_for_non_ascii_json(self) -> None: + provider = JsonProvider() + data = '{"message": "olá niño"}' + with tempfile.TemporaryDirectory() as d: + path = await provider.acheckpoint(data, d, branch="main") + + assert Path(path).read_bytes() == data.encode("utf-8") + assert await provider.afrom_checkpoint(path) == data + def test_checkpoint_fork_branch_subdir(self) -> None: provider = JsonProvider() with tempfile.TemporaryDirectory() as d: From 7e18abd1088ea0b10f4a449d27d3335bd7f16650 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=CE=B1I?= Date: Tue, 8 Sep 2026 14:08:40 +0800 Subject: [PATCH 4/9] fix(llm): route all DashScope models through native provider (#7234) DashScope's OpenAI-compatible endpoint serves DeepSeek/Kimi/GLM/etc., not only Qwen. Stop restricting the native match to the qwen* prefix so DASHSCOPE_BASE_URL applies consistently. Fixes #7233. Co-authored-by: Alphaxiaoteng <230277249+Alphaxiaoteng@users.noreply.github.com> Co-authored-by: Vidit Ostwal <110953813+Vidit-Ostwal@users.noreply.github.com> --- lib/crewai/src/crewai/llm.py | 3 ++- .../openai_compatible/test_openai_compatible.py | 14 ++++++++++++++ 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/lib/crewai/src/crewai/llm.py b/lib/crewai/src/crewai/llm.py index e6604805d..6c9c60911 100644 --- a/lib/crewai/src/crewai/llm.py +++ b/lib/crewai/src/crewai/llm.py @@ -573,7 +573,8 @@ class LLM(BaseLLM): return True if provider == "dashscope": - return model_lower.startswith("qwen") + # DashScope's OpenAI-compatible endpoint serves Qwen plus DeepSeek/Kimi/GLM/etc. + return True if provider == "openrouter": # OpenRouter uses org/model format but accepts anything diff --git a/lib/crewai/tests/llms/openai_compatible/test_openai_compatible.py b/lib/crewai/tests/llms/openai_compatible/test_openai_compatible.py index d8747571f..49106050b 100644 --- a/lib/crewai/tests/llms/openai_compatible/test_openai_compatible.py +++ b/lib/crewai/tests/llms/openai_compatible/test_openai_compatible.py @@ -298,6 +298,20 @@ class TestLLMIntegration: assert isinstance(llm, OpenAICompatibleCompletion) assert llm.provider == "dashscope" + def test_llm_creates_openai_compatible_for_dashscope_non_qwen(self): + """Non-Qwen DashScope models must still use the native OpenAI-compatible path.""" + with patch.dict( + os.environ, + { + "DASHSCOPE_API_KEY": "test-key", + "DASHSCOPE_BASE_URL": "https://my-dashscope.example.com/v1", + }, + ): + llm = LLM(model="dashscope/deepseek-v3") + assert isinstance(llm, OpenAICompatibleCompletion) + assert llm.provider == "dashscope" + assert llm.base_url == "https://my-dashscope.example.com/v1" + def test_llm_with_explicit_provider(self): """Test LLM with explicit provider parameter.""" with patch.dict(os.environ, {"DEEPSEEK_API_KEY": "test-key"}): From fe62d04cdb5bf54bea9275cb97f2937a1b2edc98 Mon Sep 17 00:00:00 2001 From: oxy-giedrius Date: Tue, 8 Sep 2026 10:12:48 +0300 Subject: [PATCH 5/9] fix(oxylabs): report scrape failures instead of raising IndexError (#7044) * fix(oxylabs): report scrape failures instead of raising IndexError The oxylabs SDK logs HTTP errors and returns an empty response rather than raising, so the unchecked `response.results[0]` in every Oxylabs tool turned a rejected request into `IndexError: list index out of range`. Invalid credentials -- the most likely first-run mistake -- gave no indication of the cause. A result carrying a non-2xx `status_code` had the same problem one level down: the job ran, the page did not come back, and the tool returned its empty content as though the scrape had succeeded, handing the agent "[]". Both are now reported as a `ToolFailure` naming what went wrong, so the agent gets something it can act on and the framework records the call as failed: 401 Unauthorized 400 Bad Request - Parameter `parsing_instructions` can be used just with `parse` parameter set to `true`. Because the SDK keeps the cause only in its own log, the failing call is run with a handler attached to the `oxylabs` logger and the status, the API's explanation and timeouts are read back off it. `code` and `retryable` are set from the status, so 429 and 5xx are marked worth retrying. Nothing about the caller's logging configuration is changed; an application that has silenced the SDK still gets the generic failure. Content that is neither a string nor a dict is also serialized properly: `parsing_instructions` commonly yields a list, and the previous `str()` fallback produced a Python repr with single quotes instead of JSON. The client construction and response handling these four tools duplicated verbatim now live in a shared `OxylabsBaseTool`, following the existing `SerpApiBaseTool` pattern, so the handling above exists in one place. The generated tool specs change only by the new `locale` field, confirming the tools' public surface is otherwise untouched. Also add the `locale` option to the Google Search config, which the docs already documented but the config model silently dropped, and correct two copy-paste errors in the docs across all four locales. Co-Authored-By: Claude Opus 5 (1M context) * fix(oxylabs): keep concurrent scrape diagnoses apart The error capture attached a fresh handler to the shared `oxylabs` logger for each scrape, so two scrapes in flight at once each saw both errors. `_diagnose` reads the first HTTP status it finds, so a timeout could be reported as the other request's 400 -- `retryable=False` on a failure that was worth retrying. One handler now serves every scrape and routes each record to the capture of the call that caused it via a `ContextVar`, which isolates threads and asyncio tasks alike. Serializing the captures would have fixed the cross-talk too, but at the cost of running every scrape one at a time. The handler stays attached once installed: it is inert outside a capture, and detaching it would race with concurrent scrapes. The regression test forces the interleaving -- one capture is held open while the other call logs -- and fails against the previous implementation. Also drive `config` through the public constructor in the tests instead of assigning `__dict__["config"]`, so they would catch `__init__` dropping a supplied config. Co-Authored-By: Claude Opus 5 (1M context) --------- Co-authored-by: Claude Opus 5 (1M context) --- .../web-scraping/oxylabsscraperstool.mdx | 4 +- .../web-scraping/oxylabsscraperstool.mdx | 4 +- .../web-scraping/oxylabsscraperstool.mdx | 4 +- .../web-scraping/oxylabsscraperstool.mdx | 4 +- .../oxylabs_amazon_product_scraper_tool.py | 121 +------ .../oxylabs_amazon_search_scraper_tool.py | 121 +------ .../tools/oxylabs_base_tool/__init__.py | 0 .../oxylabs_base_tool/oxylabs_base_tool.py | 315 ++++++++++++++++++ .../oxylabs_google_search_scraper_tool.py | 126 +------ .../oxylabs_universal_scraper_tool.py | 120 +------ .../tests/tools/test_oxylabs_tools.py | 277 +++++++++++++++ lib/crewai-tools/tool.specs.json | 13 + 12 files changed, 642 insertions(+), 467 deletions(-) create mode 100644 lib/crewai-tools/src/crewai_tools/tools/oxylabs_base_tool/__init__.py create mode 100644 lib/crewai-tools/src/crewai_tools/tools/oxylabs_base_tool/oxylabs_base_tool.py diff --git a/docs/edge/ar/tools/web-scraping/oxylabsscraperstool.mdx b/docs/edge/ar/tools/web-scraping/oxylabsscraperstool.mdx index b83ba8b33..a62f7c18d 100644 --- a/docs/edge/ar/tools/web-scraping/oxylabsscraperstool.mdx +++ b/docs/edge/ar/tools/web-scraping/oxylabsscraperstool.mdx @@ -4,7 +4,7 @@ description: > تتيح أدوات استخراج Oxylabs الوصول بسهولة إلى المعلومات من المصادر المعنية. يرجى الاطلاع على قائمة المصادر المتاحة أدناه: - `Amazon Product` - `Amazon Search` - - `Google Seach` + - `Google Search` - `Universal` icon: globe mode: "wide" @@ -87,7 +87,7 @@ print(result) ### المعاملات - `query` - مصطلح بحث Amazon. -- `domain` - توطين النطاق لـ Bestbuy. +- `domain` - توطين النطاق لـ Amazon. - `start_page` - رقم صفحة البداية. - `pages` - عدد الصفحات المراد استرجاعها. - `geo_location` - موقع _التوصيل إلى_. diff --git a/docs/edge/en/tools/web-scraping/oxylabsscraperstool.mdx b/docs/edge/en/tools/web-scraping/oxylabsscraperstool.mdx index 4529a90b7..d69654bac 100644 --- a/docs/edge/en/tools/web-scraping/oxylabsscraperstool.mdx +++ b/docs/edge/en/tools/web-scraping/oxylabsscraperstool.mdx @@ -4,7 +4,7 @@ description: > Oxylabs Scrapers allow to easily access the information from the respective sources. Please see the list of available sources below: - `Amazon Product` - `Amazon Search` - - `Google Seach` + - `Google Search` - `Universal` icon: globe mode: "wide" @@ -87,7 +87,7 @@ print(result) ### Parameters - `query` - Amazon search term. -- `domain` - Domain localization for Bestbuy. +- `domain` - domain localization for Amazon. - `start_page` - starting page number. - `pages` - number of pages to retrieve. - `geo_location` - the _Deliver to_ location. diff --git a/docs/edge/ko/tools/web-scraping/oxylabsscraperstool.mdx b/docs/edge/ko/tools/web-scraping/oxylabsscraperstool.mdx index a84058f7e..138869b0b 100644 --- a/docs/edge/ko/tools/web-scraping/oxylabsscraperstool.mdx +++ b/docs/edge/ko/tools/web-scraping/oxylabsscraperstool.mdx @@ -4,7 +4,7 @@ description: > Oxylabs 스크래퍼를 사용하면 해당 소스에서 정보를 쉽게 접근할 수 있습니다. 아래에서 사용 가능한 소스 목록을 확인하세요: - `Amazon Product` - `Amazon Search` - - `Google Seach` + - `Google Search` - `Universal` icon: globe mode: "wide" @@ -87,7 +87,7 @@ print(result) ### 파라미터 - `query` - Amazon 검색어. -- `domain` - Bestbuy의 도메인 로컬라이제이션. +- `domain` - Amazon의 도메인 로컬라이제이션. - `start_page` - 시작 페이지 번호. - `pages` - 가져올 페이지 수. - `geo_location` - _배송지_ 위치. diff --git a/docs/edge/pt-BR/tools/web-scraping/oxylabsscraperstool.mdx b/docs/edge/pt-BR/tools/web-scraping/oxylabsscraperstool.mdx index db544984e..f5e146ff5 100644 --- a/docs/edge/pt-BR/tools/web-scraping/oxylabsscraperstool.mdx +++ b/docs/edge/pt-BR/tools/web-scraping/oxylabsscraperstool.mdx @@ -4,7 +4,7 @@ description: > Os Scrapers da Oxylabs permitem acessar facilmente informações de fontes específicas. Veja abaixo a lista de fontes disponíveis: - `Amazon Product` - `Amazon Search` - - `Google Seach` + - `Google Search` - `Universal` icon: globe mode: "wide" @@ -87,7 +87,7 @@ print(result) ### Parâmetros - `query` - termo de busca da Amazon. -- `domain` - Domínio de localização para Bestbuy. +- `domain` - domínio de localização da Amazon. - `start_page` - número da página inicial. - `pages` - quantidade de páginas a ser recuperada. - `geo_location` - local de entrega (_Deliver to_). diff --git a/lib/crewai-tools/src/crewai_tools/tools/oxylabs_amazon_product_scraper_tool/oxylabs_amazon_product_scraper_tool.py b/lib/crewai-tools/src/crewai_tools/tools/oxylabs_amazon_product_scraper_tool/oxylabs_amazon_product_scraper_tool.py index 915b462ca..fcfc01169 100644 --- a/lib/crewai-tools/src/crewai_tools/tools/oxylabs_amazon_product_scraper_tool/oxylabs_amazon_product_scraper_tool.py +++ b/lib/crewai-tools/src/crewai_tools/tools/oxylabs_amazon_product_scraper_tool/oxylabs_amazon_product_scraper_tool.py @@ -1,25 +1,9 @@ -from importlib.metadata import version -import json -import os -from platform import architecture, python_version from typing import Any -from crewai.tools import BaseTool, EnvVar -from pydantic import BaseModel, ConfigDict, Field +from crewai.tools.tool_failure import ToolFailure +from pydantic import BaseModel, Field - -try: - from oxylabs import RealtimeClient # type: ignore[import-untyped] - from oxylabs.sources.response import ( # type: ignore[import-untyped] - Response as OxylabsResponse, - ) - - OXYLABS_AVAILABLE = True -except ImportError: - RealtimeClient = Any - OxylabsResponse = Any - - OXYLABS_AVAILABLE = False +from crewai_tools.tools.oxylabs_base_tool.oxylabs_base_tool import OxylabsBaseTool __all__ = ["OxylabsAmazonProductScraperConfig", "OxylabsAmazonProductScraperTool"] @@ -51,7 +35,7 @@ class OxylabsAmazonProductScraperConfig(BaseModel): ) -class OxylabsAmazonProductScraperTool(BaseTool): +class OxylabsAmazonProductScraperTool(OxylabsBaseTool): """Scrape Amazon product pages with OxylabsAmazonProductScraperTool. Get Oxylabs account: @@ -63,104 +47,11 @@ class OxylabsAmazonProductScraperTool(BaseTool): config: Configuration options. See ``OxylabsAmazonProductScraperConfig`` """ - model_config = ConfigDict( - arbitrary_types_allowed=True, - validate_assignment=True, - ) name: str = "Oxylabs Amazon Product Scraper tool" description: str = "Scrape Amazon product pages with Oxylabs Amazon Product Scraper" args_schema: type[BaseModel] = OxylabsAmazonProductScraperArgs - oxylabs_api: Any config: OxylabsAmazonProductScraperConfig - package_dependencies: list[str] = Field(default_factory=lambda: ["oxylabs"]) - env_vars: list[EnvVar] = Field( - default_factory=lambda: [ - EnvVar( - name="OXYLABS_USERNAME", - description="Username for Oxylabs", - required=True, - ), - EnvVar( - name="OXYLABS_PASSWORD", - description="Password for Oxylabs", - required=True, - ), - ] - ) - def __init__( - self, - username: str | None = None, - password: str | None = None, - config: OxylabsAmazonProductScraperConfig | dict[str, Any] | None = None, - **kwargs: Any, - ) -> None: - bits, _ = architecture() - sdk_type = ( - f"oxylabs-crewai-sdk-python/" - f"{version('crewai')} " - f"({python_version()}; {bits})" - ) - - if username is None or password is None: - username, password = self._get_credentials_from_env() - - if OXYLABS_AVAILABLE: - from oxylabs import RealtimeClient - - kwargs["oxylabs_api"] = RealtimeClient( - username=username, - password=password, - sdk_type=sdk_type, - ) - else: - import click - - if click.confirm( - "You are missing the 'oxylabs' package. Would you like to install it?" - ): - import subprocess - - try: - subprocess.run(["uv", "add", "oxylabs"], check=True) # noqa: S607 - from oxylabs import RealtimeClient - - kwargs["oxylabs_api"] = RealtimeClient( - username=username, - password=password, - sdk_type=sdk_type, - ) - except subprocess.CalledProcessError as e: - raise ImportError("Failed to install oxylabs package") from e - else: - raise ImportError( - "`oxylabs` package not found, please run `uv add oxylabs`" - ) - - if config is None: - config = OxylabsAmazonProductScraperConfig() - super().__init__(config=config, **kwargs) - - def _get_credentials_from_env(self) -> tuple[str, str]: - username = os.environ.get("OXYLABS_USERNAME") - password = os.environ.get("OXYLABS_PASSWORD") - if not username or not password: - raise ValueError( - "You must pass oxylabs username and password when instantiating the tool " - "or specify OXYLABS_USERNAME and OXYLABS_PASSWORD environment variables" - ) - return username, password - - def _run(self, query: str) -> str: - response = self.oxylabs_api.amazon.scrape_product( - query, - **self.config.model_dump(exclude_none=True), - ) - - content = response.results[0].content - - if isinstance(content, dict): - return json.dumps(content) - - return str(content) + def _run(self, query: str) -> str | ToolFailure: + return self._scrape(self.oxylabs_api.amazon.scrape_product, query) diff --git a/lib/crewai-tools/src/crewai_tools/tools/oxylabs_amazon_search_scraper_tool/oxylabs_amazon_search_scraper_tool.py b/lib/crewai-tools/src/crewai_tools/tools/oxylabs_amazon_search_scraper_tool/oxylabs_amazon_search_scraper_tool.py index 2f693283b..ebd1d2b14 100644 --- a/lib/crewai-tools/src/crewai_tools/tools/oxylabs_amazon_search_scraper_tool/oxylabs_amazon_search_scraper_tool.py +++ b/lib/crewai-tools/src/crewai_tools/tools/oxylabs_amazon_search_scraper_tool/oxylabs_amazon_search_scraper_tool.py @@ -1,25 +1,9 @@ -from importlib.metadata import version -import json -import os -from platform import architecture, python_version from typing import Any -from crewai.tools import BaseTool, EnvVar -from pydantic import BaseModel, ConfigDict, Field +from crewai.tools.tool_failure import ToolFailure +from pydantic import BaseModel, Field - -try: - from oxylabs import RealtimeClient # type: ignore[import-untyped] - from oxylabs.sources.response import ( # type: ignore[import-untyped] - Response as OxylabsResponse, - ) - - OXYLABS_AVAILABLE = True -except ImportError: - RealtimeClient = Any - OxylabsResponse = Any - - OXYLABS_AVAILABLE = False +from crewai_tools.tools.oxylabs_base_tool.oxylabs_base_tool import OxylabsBaseTool __all__ = ["OxylabsAmazonSearchScraperConfig", "OxylabsAmazonSearchScraperTool"] @@ -53,7 +37,7 @@ class OxylabsAmazonSearchScraperConfig(BaseModel): ) -class OxylabsAmazonSearchScraperTool(BaseTool): +class OxylabsAmazonSearchScraperTool(OxylabsBaseTool): """Scrape Amazon search results with OxylabsAmazonSearchScraperTool. Get Oxylabs account: @@ -65,104 +49,11 @@ class OxylabsAmazonSearchScraperTool(BaseTool): config: Configuration options. See ``OxylabsAmazonSearchScraperConfig`` """ - model_config = ConfigDict( - arbitrary_types_allowed=True, - validate_assignment=True, - ) name: str = "Oxylabs Amazon Search Scraper tool" description: str = "Scrape Amazon search results with Oxylabs Amazon Search Scraper" args_schema: type[BaseModel] = OxylabsAmazonSearchScraperArgs - oxylabs_api: Any config: OxylabsAmazonSearchScraperConfig - package_dependencies: list[str] = Field(default_factory=lambda: ["oxylabs"]) - env_vars: list[EnvVar] = Field( - default_factory=lambda: [ - EnvVar( - name="OXYLABS_USERNAME", - description="Username for Oxylabs", - required=True, - ), - EnvVar( - name="OXYLABS_PASSWORD", - description="Password for Oxylabs", - required=True, - ), - ] - ) - def __init__( - self, - username: str | None = None, - password: str | None = None, - config: OxylabsAmazonSearchScraperConfig | dict[str, Any] | None = None, - **kwargs: Any, - ) -> None: - bits, _ = architecture() - sdk_type = ( - f"oxylabs-crewai-sdk-python/" - f"{version('crewai')} " - f"({python_version()}; {bits})" - ) - - if username is None or password is None: - username, password = self._get_credentials_from_env() - - if OXYLABS_AVAILABLE: - from oxylabs import RealtimeClient - - kwargs["oxylabs_api"] = RealtimeClient( - username=username, - password=password, - sdk_type=sdk_type, - ) - else: - import click - - if click.confirm( - "You are missing the 'oxylabs' package. Would you like to install it?" - ): - import subprocess - - try: - subprocess.run(["uv", "add", "oxylabs"], check=True) # noqa: S607 - from oxylabs import RealtimeClient - - kwargs["oxylabs_api"] = RealtimeClient( - username=username, - password=password, - sdk_type=sdk_type, - ) - except subprocess.CalledProcessError as e: - raise ImportError("Failed to install oxylabs package") from e - else: - raise ImportError( - "`oxylabs` package not found, please run `uv add oxylabs`" - ) - - if config is None: - config = OxylabsAmazonSearchScraperConfig() - super().__init__(config=config, **kwargs) - - def _get_credentials_from_env(self) -> tuple[str, str]: - username = os.environ.get("OXYLABS_USERNAME") - password = os.environ.get("OXYLABS_PASSWORD") - if not username or not password: - raise ValueError( - "You must pass oxylabs username and password when instantiating the tool " - "or specify OXYLABS_USERNAME and OXYLABS_PASSWORD environment variables" - ) - return username, password - - def _run(self, query: str) -> str: - response = self.oxylabs_api.amazon.scrape_search( - query, - **self.config.model_dump(exclude_none=True), - ) - - content = response.results[0].content - - if isinstance(content, dict): - return json.dumps(content) - - return str(content) + def _run(self, query: str) -> str | ToolFailure: + return self._scrape(self.oxylabs_api.amazon.scrape_search, query) diff --git a/lib/crewai-tools/src/crewai_tools/tools/oxylabs_base_tool/__init__.py b/lib/crewai-tools/src/crewai_tools/tools/oxylabs_base_tool/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/lib/crewai-tools/src/crewai_tools/tools/oxylabs_base_tool/oxylabs_base_tool.py b/lib/crewai-tools/src/crewai_tools/tools/oxylabs_base_tool/oxylabs_base_tool.py new file mode 100644 index 000000000..664d0b980 --- /dev/null +++ b/lib/crewai-tools/src/crewai_tools/tools/oxylabs_base_tool/oxylabs_base_tool.py @@ -0,0 +1,315 @@ +from collections.abc import Callable, Iterator +from contextlib import contextmanager +from contextvars import ContextVar +from importlib.metadata import PackageNotFoundError, version +import json +import logging +import os +from platform import architecture, python_version +import re +import threading +from typing import TYPE_CHECKING, Any + +from crewai.tools import BaseTool, EnvVar +from crewai.tools.tool_failure import ToolFailure +from pydantic import ConfigDict, Field + + +__all__ = ["OxylabsBaseTool"] + + +_HTTP_ERROR_PATTERN = re.compile( + r"(\d{3})\s+(?:Client|Server) Error:\s*(.+?)\s+for url", re.IGNORECASE +) + + +_active_capture: ContextVar[list[str] | None] = ContextVar( + "oxylabs_active_capture", default=None +) +_capture_handler_lock = threading.Lock() + + +class _CaptureHandler(logging.Handler): + """Routes each SDK error to the capture belonging to the call that caused it. + + A single handler serves every concurrent scrape, and the context variable + keeps one thread's or task's records out of the others' hands. Sharing a + collector instead would let two simultaneous scrapes each see both errors, + and a timeout could be reported as the other request's non-retryable 400. + """ + + def emit(self, record: logging.LogRecord) -> None: + messages = _active_capture.get() + if messages is not None: + messages.append(record.getMessage()) + + +_CAPTURE_HANDLER = _CaptureHandler(level=logging.ERROR) + + +@contextmanager +def _captured_sdk_errors() -> Iterator[list[str]]: + """Collect the error messages the oxylabs SDK only writes to its logger. + + The SDK catches transport and HTTP errors, logs them and hands back an + empty response, so the cause is absent from the object we get back. + Listening on its logger is the only way to tell the agent what actually + went wrong. + + The handler stays attached once installed: it is inert outside a capture, + and detaching it would race with scrapes running concurrently. No level, + filter or other handler is touched, so an application that has silenced the + SDK simply falls back to the generic failure. + """ + sdk_logger = logging.getLogger("oxylabs") + with _capture_handler_lock: + if _CAPTURE_HANDLER not in sdk_logger.handlers: + sdk_logger.addHandler(_CAPTURE_HANDLER) + + messages: list[str] = [] + token = _active_capture.set(messages) + try: + yield messages + finally: + _active_capture.reset(token) + + +def _api_detail(messages: list[str]) -> str | None: + """Pull the API's own explanation out of a logged response body.""" + for message in messages: + try: + payload = json.loads(message) + except (TypeError, ValueError): + continue + if isinstance(payload, dict): + detail = payload.get("message") + if isinstance(detail, str) and detail.strip(): + return detail.strip() + return None + + +def _diagnose(messages: list[str]) -> tuple[str, str | None, bool] | None: + """Summarize what the SDK logged as (description, code, retryable).""" + reported = [m.strip() for m in messages if m and m.strip()] + if not reported: + return None + + joined = " | ".join(reported) + + http_error = _HTTP_ERROR_PATTERN.search(joined) + if http_error: + status = int(http_error.group(1)) + description = f"{status} {http_error.group(2).strip()}" + detail = _api_detail(reported) + if detail: + description = f"{description} - {detail}" + return description, str(status), status == 429 or status >= 500 + + if "timed out" in joined.lower(): + return "the request timed out", "timeout", True + + # Anything else the SDK chose to log, e.g. a connection error. + return joined[:300], None, False + + +class OxylabsBaseTool(BaseTool): + """Base class for the Oxylabs Web Scraper API tools. + + Holds what every Oxylabs tool shares: the credentialed ``RealtimeClient``, + and the translation of a Web Scraper API response into either the scraped + content or a :class:`ToolFailure`. + + Get Oxylabs account: + https://dashboard.oxylabs.io/en + """ + + model_config = ConfigDict( + arbitrary_types_allowed=True, + validate_assignment=True, + ) + + if TYPE_CHECKING: + # Declared as its own model by every subclass, and enforced in + # ``__init__``; annotated here only so this class can read it. + config: Any + + oxylabs_api: Any + package_dependencies: list[str] = Field(default_factory=lambda: ["oxylabs"]) + env_vars: list[EnvVar] = Field( + default_factory=lambda: [ + EnvVar( + name="OXYLABS_USERNAME", + description="Username for Oxylabs", + required=True, + ), + EnvVar( + name="OXYLABS_PASSWORD", + description="Password for Oxylabs", + required=True, + ), + ] + ) + + def __init__( + self, + username: str | None = None, + password: str | None = None, + config: Any = None, + **kwargs: Any, + ) -> None: + # Resolved before the client is built, so a subclass that forgets the + # field fails without first opening a session. + config_field = type(self).model_fields.get("config") + config_model = config_field.annotation if config_field else None + if config_model is None: + raise TypeError( + f"{type(self).__name__} must declare a 'config' model field" + ) + if config is None: + config = config_model() + + if username is None or password is None: + username, password = self._get_credentials_from_env() + + kwargs["oxylabs_api"] = self._build_client(username, password) + + super().__init__(config=config, **kwargs) + + @staticmethod + def _get_credentials_from_env() -> tuple[str, str]: + username = os.environ.get("OXYLABS_USERNAME") + password = os.environ.get("OXYLABS_PASSWORD") + if not username or not password: + raise ValueError( + "You must pass oxylabs username and password when instantiating the tool " + "or specify OXYLABS_USERNAME and OXYLABS_PASSWORD environment variables" + ) + return username, password + + @staticmethod + def _resolve_realtime_client() -> Any: + try: + from oxylabs import RealtimeClient # type: ignore[import-untyped] + except ImportError: + import click + + if not click.confirm( + "You are missing the 'oxylabs' package. Would you like to install it?" + ): + raise ImportError( + "`oxylabs` package not found, please run `uv add oxylabs`" + ) from None + + import importlib + import subprocess + + try: + subprocess.run(["uv", "add", "oxylabs"], check=True) # noqa: S607 + except (subprocess.CalledProcessError, OSError) as e: + # OSError covers uv itself being absent. + raise ImportError("Failed to install oxylabs package") from e + + return importlib.import_module("oxylabs").RealtimeClient + + return RealtimeClient + + @classmethod + def _build_client(cls, username: str, password: str) -> Any: + realtime_client = cls._resolve_realtime_client() + + try: + crewai_version = version("crewai") + except PackageNotFoundError: + # Only ever reported as telemetry; not worth failing construction. + crewai_version = "unknown" + + bits, _ = architecture() + return realtime_client( + username=username, + password=password, + sdk_type=( + f"oxylabs-crewai-sdk-python/" + f"{crewai_version} " + f"({python_version()}; {bits})" + ), + ) + + def _scrape(self, scraper: Callable[..., Any], target: str) -> str | ToolFailure: + """Run one scrape and turn the outcome into content or a failure.""" + with _captured_sdk_errors() as sdk_errors: + response = scraper(target, **self.config.model_dump(exclude_none=True)) + + return self._handle_response(response, sdk_errors) + + def _handle_response( + self, response: Any, sdk_errors: list[str] | None = None + ) -> str | ToolFailure: + """Return the scraped content, or report why there is none. + + The oxylabs SDK logs transport and validation errors and hands back an + empty response rather than raising, so rejected requests -- wrong + credentials, config the source does not accept, exhausted quota -- have + to be recognised here instead of reaching the agent as an ``IndexError`` + on ``results[0]``. A non-2xx ``status_code`` on the result is the same + situation one level down: the job ran, the page did not come back. + """ + results = getattr(response, "results", None) + if not results: + diagnosis = _diagnose(sdk_errors or []) + if diagnosis: + description, code, retryable = diagnosis + return ToolFailure( + message=( + f"Oxylabs Web Scraper API rejected the request: {description}" + ), + code=code or "request_rejected", + retryable=retryable, + ) + return ToolFailure( + message=( + "Oxylabs Web Scraper API returned no results and reported no " + "error, so the request was rejected before any page was " + "scraped. Check that OXYLABS_USERNAME and OXYLABS_PASSWORD " + "are valid and that this tool's config options are accepted " + "for this source." + ), + code="empty_response", + ) + + result = results[0] + + try: + status_code = int(result.status_code) + except (AttributeError, TypeError, ValueError): + status_code = None + + if status_code is not None and not 200 <= status_code < 300: + return ToolFailure( + message=( + f"Oxylabs Web Scraper API could not retrieve the page: the " + f"target responded with status {status_code}." + ), + code=str(status_code), + retryable=status_code == 429 or status_code >= 500, + ) + + content = getattr(result, "content", None) + if content is None: + return ToolFailure( + message=( + "Oxylabs Web Scraper API returned a result with no content. " + "The page may be empty, or the parser found nothing to extract." + ), + code="empty_content", + ) + + # ``parse``/``parsing_instructions`` results arrive as dicts or lists; + # only unparsed HTML comes back as a string. ``str()`` on a list would + # hand the agent a Python repr instead of JSON. + if isinstance(content, str): + return content + + try: + return json.dumps(content) + except (TypeError, ValueError): + return str(content) diff --git a/lib/crewai-tools/src/crewai_tools/tools/oxylabs_google_search_scraper_tool/oxylabs_google_search_scraper_tool.py b/lib/crewai-tools/src/crewai_tools/tools/oxylabs_google_search_scraper_tool/oxylabs_google_search_scraper_tool.py index 1d29fcd3e..9bff49fe9 100644 --- a/lib/crewai-tools/src/crewai_tools/tools/oxylabs_google_search_scraper_tool/oxylabs_google_search_scraper_tool.py +++ b/lib/crewai-tools/src/crewai_tools/tools/oxylabs_google_search_scraper_tool/oxylabs_google_search_scraper_tool.py @@ -1,25 +1,9 @@ -from importlib.metadata import version -import json -import os -from platform import architecture, python_version from typing import Any -from crewai.tools import BaseTool, EnvVar -from pydantic import BaseModel, ConfigDict, Field +from crewai.tools.tool_failure import ToolFailure +from pydantic import BaseModel, Field - -try: - from oxylabs import RealtimeClient # type: ignore[import-untyped] - from oxylabs.sources.response import ( # type: ignore[import-untyped] - Response as OxylabsResponse, - ) - - OXYLABS_AVAILABLE = True -except ImportError: - RealtimeClient = Any - OxylabsResponse = Any - - OXYLABS_AVAILABLE = False +from crewai_tools.tools.oxylabs_base_tool.oxylabs_base_tool import OxylabsBaseTool __all__ = ["OxylabsGoogleSearchScraperConfig", "OxylabsGoogleSearchScraperTool"] @@ -42,6 +26,11 @@ class OxylabsGoogleSearchScraperConfig(BaseModel): limit: int | None = Field( None, description="Number of results to retrieve in each page." ) + locale: str | None = Field( + None, + description="`Accept-Language` header value which changes your Google " + "search page web interface language.", + ) geo_location: str | None = Field(None, description="The Deliver to location.") user_agent_type: str | None = Field(None, description="Device type and browser.") render: str | None = Field(None, description="Enables JavaScript rendering.") @@ -56,7 +45,7 @@ class OxylabsGoogleSearchScraperConfig(BaseModel): ) -class OxylabsGoogleSearchScraperTool(BaseTool): +class OxylabsGoogleSearchScraperTool(OxylabsBaseTool): """Scrape Google Search results with OxylabsGoogleSearchScraperTool. Get Oxylabs account: @@ -68,104 +57,11 @@ class OxylabsGoogleSearchScraperTool(BaseTool): config: Configuration options. See ``OxylabsGoogleSearchScraperConfig`` """ - model_config = ConfigDict( - arbitrary_types_allowed=True, - validate_assignment=True, - ) name: str = "Oxylabs Google Search Scraper tool" description: str = "Scrape Google Search results with Oxylabs Google Search Scraper" args_schema: type[BaseModel] = OxylabsGoogleSearchScraperArgs - oxylabs_api: Any config: OxylabsGoogleSearchScraperConfig - package_dependencies: list[str] = Field(default_factory=lambda: ["oxylabs"]) - env_vars: list[EnvVar] = Field( - default_factory=lambda: [ - EnvVar( - name="OXYLABS_USERNAME", - description="Username for Oxylabs", - required=True, - ), - EnvVar( - name="OXYLABS_PASSWORD", - description="Password for Oxylabs", - required=True, - ), - ] - ) - def __init__( - self, - username: str | None = None, - password: str | None = None, - config: OxylabsGoogleSearchScraperConfig | dict[str, Any] | None = None, - **kwargs: Any, - ) -> None: - bits, _ = architecture() - sdk_type = ( - f"oxylabs-crewai-sdk-python/" - f"{version('crewai')} " - f"({python_version()}; {bits})" - ) - - if username is None or password is None: - username, password = self._get_credentials_from_env() - - if OXYLABS_AVAILABLE: - from oxylabs import RealtimeClient - - kwargs["oxylabs_api"] = RealtimeClient( - username=username, - password=password, - sdk_type=sdk_type, - ) - else: - import click - - if click.confirm( - "You are missing the 'oxylabs' package. Would you like to install it?" - ): - import subprocess - - try: - subprocess.run(["uv", "add", "oxylabs"], check=True) # noqa: S607 - from oxylabs import RealtimeClient - - kwargs["oxylabs_api"] = RealtimeClient( - username=username, - password=password, - sdk_type=sdk_type, - ) - except subprocess.CalledProcessError as e: - raise ImportError("Failed to install oxylabs package") from e - else: - raise ImportError( - "`oxylabs` package not found, please run `uv add oxylabs`" - ) - - if config is None: - config = OxylabsGoogleSearchScraperConfig() - super().__init__(config=config, **kwargs) - - def _get_credentials_from_env(self) -> tuple[str, str]: - username = os.environ.get("OXYLABS_USERNAME") - password = os.environ.get("OXYLABS_PASSWORD") - if not username or not password: - raise ValueError( - "You must pass oxylabs username and password when instantiating the tool " - "or specify OXYLABS_USERNAME and OXYLABS_PASSWORD environment variables" - ) - return username, password - - def _run(self, query: str, **kwargs: Any) -> str: - response = self.oxylabs_api.google.scrape_search( - query, - **self.config.model_dump(exclude_none=True), - ) - - content = response.results[0].content - - if isinstance(content, dict): - return json.dumps(content) - - return str(content) + def _run(self, query: str) -> str | ToolFailure: + return self._scrape(self.oxylabs_api.google.scrape_search, query) diff --git a/lib/crewai-tools/src/crewai_tools/tools/oxylabs_universal_scraper_tool/oxylabs_universal_scraper_tool.py b/lib/crewai-tools/src/crewai_tools/tools/oxylabs_universal_scraper_tool/oxylabs_universal_scraper_tool.py index adb52fbcc..8c2526a8e 100644 --- a/lib/crewai-tools/src/crewai_tools/tools/oxylabs_universal_scraper_tool/oxylabs_universal_scraper_tool.py +++ b/lib/crewai-tools/src/crewai_tools/tools/oxylabs_universal_scraper_tool/oxylabs_universal_scraper_tool.py @@ -1,25 +1,10 @@ -from importlib.metadata import version -import json -import os -from platform import architecture, python_version from typing import Any -from crewai.tools import BaseTool, EnvVar -from pydantic import BaseModel, ConfigDict, Field +from crewai.tools.tool_failure import ToolFailure +from pydantic import BaseModel, Field +from crewai_tools.tools.oxylabs_base_tool.oxylabs_base_tool import OxylabsBaseTool -try: - from oxylabs import RealtimeClient # type: ignore[import-untyped] - from oxylabs.sources.response import ( # type: ignore[import-untyped] - Response as OxylabsResponse, - ) - - OXYLABS_AVAILABLE = True -except ImportError: - RealtimeClient = Any - OxylabsResponse = Any - - OXYLABS_AVAILABLE = False __all__ = ["OxylabsUniversalScraperConfig", "OxylabsUniversalScraperTool"] @@ -47,7 +32,7 @@ class OxylabsUniversalScraperConfig(BaseModel): ) -class OxylabsUniversalScraperTool(BaseTool): +class OxylabsUniversalScraperTool(OxylabsBaseTool): """Scrape any website with OxylabsUniversalScraperTool. Get Oxylabs account: @@ -59,104 +44,11 @@ class OxylabsUniversalScraperTool(BaseTool): config: Configuration options. See ``OxylabsUniversalScraperConfig`` """ - model_config = ConfigDict( - arbitrary_types_allowed=True, - validate_assignment=True, - ) name: str = "Oxylabs Universal Scraper tool" description: str = "Scrape any url with Oxylabs Universal Scraper" args_schema: type[BaseModel] = OxylabsUniversalScraperArgs - oxylabs_api: Any config: OxylabsUniversalScraperConfig - package_dependencies: list[str] = Field(default_factory=lambda: ["oxylabs"]) - env_vars: list[EnvVar] = Field( - default_factory=lambda: [ - EnvVar( - name="OXYLABS_USERNAME", - description="Username for Oxylabs", - required=True, - ), - EnvVar( - name="OXYLABS_PASSWORD", - description="Password for Oxylabs", - required=True, - ), - ] - ) - def __init__( - self, - username: str | None = None, - password: str | None = None, - config: OxylabsUniversalScraperConfig | dict[str, Any] | None = None, - **kwargs: Any, - ) -> None: - bits, _ = architecture() - sdk_type = ( - f"oxylabs-crewai-sdk-python/" - f"{version('crewai')} " - f"({python_version()}; {bits})" - ) - - if username is None or password is None: - username, password = self._get_credentials_from_env() - - if OXYLABS_AVAILABLE: - from oxylabs import RealtimeClient - - kwargs["oxylabs_api"] = RealtimeClient( - username=username, - password=password, - sdk_type=sdk_type, - ) - else: - import click - - if click.confirm( - "You are missing the 'oxylabs' package. Would you like to install it?" - ): - import subprocess - - try: - subprocess.run(["uv", "add", "oxylabs"], check=True) # noqa: S607 - from oxylabs import RealtimeClient - - kwargs["oxylabs_api"] = RealtimeClient( - username=username, - password=password, - sdk_type=sdk_type, - ) - except subprocess.CalledProcessError as e: - raise ImportError("Failed to install oxylabs package") from e - else: - raise ImportError( - "`oxylabs` package not found, please run `uv add oxylabs`" - ) - - if config is None: - config = OxylabsUniversalScraperConfig() - super().__init__(config=config, **kwargs) - - def _get_credentials_from_env(self) -> tuple[str, str]: - username = os.environ.get("OXYLABS_USERNAME") - password = os.environ.get("OXYLABS_PASSWORD") - if not username or not password: - raise ValueError( - "You must pass oxylabs username and password when instantiating the tool " - "or specify OXYLABS_USERNAME and OXYLABS_PASSWORD environment variables" - ) - return username, password - - def _run(self, url: str) -> str: - response = self.oxylabs_api.universal.scrape_url( - url, - **self.config.model_dump(exclude_none=True), - ) - - content = response.results[0].content - - if isinstance(content, dict): - return json.dumps(content) - - return str(content) + def _run(self, url: str) -> str | ToolFailure: + return self._scrape(self.oxylabs_api.universal.scrape_url, url) diff --git a/lib/crewai-tools/tests/tools/test_oxylabs_tools.py b/lib/crewai-tools/tests/tools/test_oxylabs_tools.py index f135e421c..c7b4981df 100644 --- a/lib/crewai-tools/tests/tools/test_oxylabs_tools.py +++ b/lib/crewai-tools/tests/tools/test_oxylabs_tools.py @@ -1,8 +1,12 @@ +from collections.abc import Callable import json +import logging import os +import threading from unittest.mock import MagicMock from crewai.tools.base_tool import BaseTool +from crewai.tools.tool_failure import ToolFailure from crewai_tools import ( OxylabsAmazonProductScraperTool, OxylabsAmazonSearchScraperTool, @@ -12,9 +16,13 @@ from crewai_tools import ( from crewai_tools.tools.oxylabs_amazon_product_scraper_tool.oxylabs_amazon_product_scraper_tool import ( OxylabsAmazonProductScraperConfig, ) +from crewai_tools.tools.oxylabs_base_tool.oxylabs_base_tool import OxylabsBaseTool from crewai_tools.tools.oxylabs_google_search_scraper_tool.oxylabs_google_search_scraper_tool import ( OxylabsGoogleSearchScraperConfig, ) +from crewai_tools.tools.oxylabs_universal_scraper_tool.oxylabs_universal_scraper_tool import ( + OxylabsUniversalScraperArgs, +) from oxylabs import RealtimeClient from oxylabs.sources.response import Response as OxylabsResponse from pydantic import BaseModel @@ -156,3 +164,272 @@ def test_tool_invocation( result = tool.run("Scraping Query 2") assert isinstance(result, str) assert "" in result + + +ALL_TOOL_CLASSES = [ + OxylabsUniversalScraperTool, + OxylabsAmazonSearchScraperTool, + OxylabsGoogleSearchScraperTool, + OxylabsAmazonProductScraperTool, +] + + +def build_tool( + tool_class: type[BaseTool], + raw_response: dict, + sdk_logs: list[str] | None = None, + config: BaseModel | None = None, +) -> BaseTool: + """Build a tool whose every scrape entrypoint answers with ``raw_response``. + + ``sdk_logs`` reproduces the oxylabs SDK's habit of logging the real cause and + returning an empty response instead of raising. + """ + api = MagicMock() + response = OxylabsResponse(raw_response) + + def scrape(*_args: object, **_kwargs: object) -> OxylabsResponse: + for line in sdk_logs or []: + logging.getLogger("oxylabs.internal.api").error(line) + return response + + api.universal.scrape_url.side_effect = scrape + api.amazon.scrape_search.side_effect = scrape + api.amazon.scrape_product.side_effect = scrape + api.google.scrape_search.side_effect = scrape + + tool = tool_class(username="username", password="password", config=config) + # setting via __dict__ to bypass pydantic validation + tool.__dict__["oxylabs_api"] = api + return tool + + +@pytest.mark.parametrize("tool_class", ALL_TOOL_CLASSES) +def test_rejected_request_reports_failure(tool_class: type[BaseTool]): + """The SDK logs HTTP errors and returns an empty response instead of raising, + so a rejected request must be reported rather than indexed into.""" + result = build_tool(tool_class, {}).run("Scraping Query") + + assert isinstance(result, ToolFailure) + assert result.code == "empty_response" + assert "OXYLABS_USERNAME" in result.message + + +@pytest.mark.parametrize("tool_class", ALL_TOOL_CLASSES) +@pytest.mark.parametrize( + ("status_code", "retryable"), + [(404, False), (429, True), (500, True), (503, True)], +) +def test_upstream_error_status_reports_failure( + tool_class: type[BaseTool], status_code: int, retryable: bool +): + """A non-2xx result carries no page; returning its empty content would hand + the agent '[]' as though the scrape had succeeded.""" + result = build_tool( + tool_class, {"results": [{"content": [], "status_code": status_code}]} + ).run("Scraping Query") + + assert isinstance(result, ToolFailure) + assert result.code == str(status_code) + assert result.retryable is retryable + + +@pytest.mark.parametrize("tool_class", ALL_TOOL_CLASSES) +def test_missing_content_reports_failure(tool_class: type[BaseTool]): + result = build_tool( + tool_class, {"results": [{"content": None, "status_code": 200}]} + ).run("Scraping Query") + + assert isinstance(result, ToolFailure) + assert result.code == "empty_content" + + +@pytest.mark.parametrize("tool_class", ALL_TOOL_CLASSES) +def test_list_content_is_serialized_as_json(tool_class: type[BaseTool]): + """``parsing_instructions`` can yield a list; str() on it would produce a + Python repr with single quotes rather than JSON.""" + result = build_tool( + tool_class, + {"results": [{"content": [{"title": "Amazing product"}], "status_code": 200}]}, + ).run("Scraping Query") + + assert isinstance(result, str) + assert json.loads(result) == [{"title": "Amazing product"}] + + +def test_subclass_without_config_field_is_reported(): + """The base class defaults ``config`` from the subclass's own model, so a + subclass that declares none must say so rather than raise ``KeyError``.""" + + class MissingConfig(OxylabsBaseTool): + name: str = "missing config" + description: str = "declares no config field" + args_schema: type[BaseModel] = OxylabsUniversalScraperArgs + + def _run(self, url: str) -> str: + return "" + + with pytest.raises(TypeError, match="must declare a 'config' model field"): + MissingConfig(username="username", password="password") + + +@pytest.mark.parametrize("tool_class", ALL_TOOL_CLASSES) +def test_rejected_request_names_the_http_cause(tool_class: type[BaseTool]): + """The agent cannot act on "go read a log", so the status the SDK logged has + to reach the failure itself.""" + result = build_tool( + tool_class, + {}, + sdk_logs=[ + "HTTP error occurred: 401 Client Error: Unauthorized for url: " + "https://realtime.oxylabs.io/v1/queries", + "", + ], + ).run("Scraping Query") + + assert isinstance(result, ToolFailure) + assert result.code == "401" + assert "401 Unauthorized" in result.message + assert result.retryable is False + + +@pytest.mark.parametrize("tool_class", ALL_TOOL_CLASSES) +def test_rejected_request_includes_the_api_explanation(tool_class: type[BaseTool]): + """The API explains config it rejects; that explanation is what makes the + failure actionable.""" + result = build_tool( + tool_class, + {}, + sdk_logs=[ + "HTTP error occurred: 400 Client Error: Bad Request for url: " + "https://realtime.oxylabs.io/v1/queries", + '{"message": "Parameter `parsing_instructions` can be used just with ' + '`parse` parameter set to `true`."}', + ], + ).run("Scraping Query") + + assert isinstance(result, ToolFailure) + assert result.code == "400" + assert "400 Bad Request" in result.message + assert "`parsing_instructions` can be used just with" in result.message + + +@pytest.mark.parametrize("tool_class", ALL_TOOL_CLASSES) +def test_timeout_is_reported_as_retryable(tool_class: type[BaseTool]): + result = build_tool( + tool_class, + {}, + sdk_logs=[ + "Timeout error. The request to https://realtime.oxylabs.io/v1/queries " + "with method POST has timed out." + ], + ).run("Scraping Query") + + assert isinstance(result, ToolFailure) + assert result.code == "timeout" + assert result.retryable is True + + +@pytest.mark.parametrize("tool_class", ALL_TOOL_CLASSES) +def test_server_error_is_reported_as_retryable(tool_class: type[BaseTool]): + result = build_tool( + tool_class, + {}, + sdk_logs=[ + "HTTP error occurred: 502 Server Error: Bad Gateway for url: " + "https://realtime.oxylabs.io/v1/queries" + ], + ).run("Scraping Query") + + assert isinstance(result, ToolFailure) + assert result.code == "502" + assert result.retryable is True + + +def test_google_config_forwards_locale(): + """`locale` is a documented Google Search parameter; the config model used to + omit it, so it was silently dropped.""" + tool = build_tool( + OxylabsGoogleSearchScraperTool, + {"results": [{"content": {"ok": True}, "status_code": 200}]}, + config=OxylabsGoogleSearchScraperConfig(locale="de", limit=2), + ) + + tool.run("iPhone 16") + + _, kwargs = tool.oxylabs_api.google.scrape_search.call_args + assert kwargs["locale"] == "de" + assert kwargs["limit"] == 2 + + +def test_result_without_content_is_reported(): + """A result object missing `content` entirely must not raise AttributeError.""" + tool = build_tool(OxylabsUniversalScraperTool, {"results": [{"status_code": 200}]}) + + result = tool.run("https://example.com") + + assert isinstance(result, ToolFailure) + assert result.code == "empty_content" + + +def test_concurrent_scrapes_do_not_share_diagnoses(): + """Two scrapes in flight at once must each be diagnosed from their own error. + + A shared collector would hand both calls both errors, and the timeout below + would be reported as the other request's non-retryable 400 -- telling the + agent not to retry something it should. + """ + both_started = threading.Barrier(2) + timeout_logged = threading.Event() + bad_request_logged = threading.Event() + outcomes: dict[str, ToolFailure] = {} + + def tool_logging(emit: Callable[[], None]) -> BaseTool: + api = MagicMock() + + def scrape(*_args: object, **_kwargs: object) -> OxylabsResponse: + both_started.wait(timeout=5) + emit() + return OxylabsResponse({}) + + api.universal.scrape_url.side_effect = scrape + tool = OxylabsUniversalScraperTool(username="username", password="password") + tool.__dict__["oxylabs_api"] = api + return tool + + sdk_logger = logging.getLogger("oxylabs.internal.api") + + def emit_timeout() -> None: + sdk_logger.error( + "Timeout error. The request to https://realtime.oxylabs.io/v1/queries " + "with method POST has timed out." + ) + timeout_logged.set() + # Hold this capture open while the other call logs, which is the window + # in which the two could bleed into each other. + bad_request_logged.wait(timeout=5) + + def emit_bad_request() -> None: + timeout_logged.wait(timeout=5) + sdk_logger.error( + "HTTP error occurred: 400 Client Error: Bad Request for url: " + "https://realtime.oxylabs.io/v1/queries" + ) + bad_request_logged.set() + + def run(key: str, emit: Callable[[], None]) -> None: + outcomes[key] = tool_logging(emit).run("https://example.com") + + threads = [ + threading.Thread(target=run, args=("timeout", emit_timeout)), + threading.Thread(target=run, args=("bad_request", emit_bad_request)), + ] + for thread in threads: + thread.start() + for thread in threads: + thread.join(timeout=15) + + assert outcomes["timeout"].code == "timeout" + assert outcomes["timeout"].retryable is True + assert outcomes["bad_request"].code == "400" + assert outcomes["bad_request"].retryable is False diff --git a/lib/crewai-tools/tool.specs.json b/lib/crewai-tools/tool.specs.json index 2b614ba12..dfcb43d4a 100644 --- a/lib/crewai-tools/tool.specs.json +++ b/lib/crewai-tools/tool.specs.json @@ -17466,6 +17466,19 @@ "description": "Number of results to retrieve in each page.", "title": "Limit" }, + "locale": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "`Accept-Language` header value which changes your Google search page web interface language.", + "title": "Locale" + }, "pages": { "anyOf": [ { From 34199c21b724d805608b59745cdb94006c8fdcd2 Mon Sep 17 00:00:00 2001 From: oxy-giedrius Date: Tue, 8 Sep 2026 10:22:23 +0300 Subject: [PATCH 6/9] chore(oxylabs): allow the 3.x oxylabs SDK (#7331) `oxylabs` was pinned to exactly 2.0.0, so consumers could not take 3.0.0, out since March. 3.x keeps the `RealtimeClient` surface these tools use, and all four tools plus their failure paths were verified against the live API on both 2.0.0 and 3.0.0. The lockfile keeps oxylabs at 2.0.0, so this permits the upgrade rather than forcing it. Co-authored-by: Claude Opus 5 (1M context) --- lib/crewai-tools/pyproject.toml | 4 +++- uv.lock | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/lib/crewai-tools/pyproject.toml b/lib/crewai-tools/pyproject.toml index 4b36d3082..a6f0e7422 100644 --- a/lib/crewai-tools/pyproject.toml +++ b/lib/crewai-tools/pyproject.toml @@ -141,7 +141,9 @@ xml = [ "nltk>=3.10.3", ] oxylabs = [ - "oxylabs==2.0.0" + # 3.x keeps the RealtimeClient surface these tools use and adds sources; + # allow it rather than pinning consumers to a single release. + "oxylabs>=2.0.0,<4", ] mongodb = [ "pymongo>=4.13" diff --git a/uv.lock b/uv.lock index 463f0beb0..d1eeeb16f 100644 --- a/uv.lock +++ b/uv.lock @@ -1878,7 +1878,7 @@ requires-dist = [ { name = "nest-asyncio", marker = "extra == 'bedrock'", specifier = ">=1.6.0" }, { name = "nest-asyncio", marker = "extra == 'contextual'", specifier = ">=1.6.0" }, { name = "nltk", marker = "extra == 'xml'", specifier = ">=3.10.3" }, - { name = "oxylabs", marker = "extra == 'oxylabs'", specifier = "==2.0.0" }, + { name = "oxylabs", marker = "extra == 'oxylabs'", specifier = ">=2.0.0,<4" }, { name = "patronus", marker = "extra == 'patronus'", specifier = ">=0.0.16" }, { name = "playwright", marker = "extra == 'bedrock'", specifier = ">=1.52.0" }, { name = "psycopg2-binary", marker = "extra == 'postgresql'", specifier = ">=2.9.10" }, From 5d9b77ba101f15459236b56f4873ba144e192d2e Mon Sep 17 00:00:00 2001 From: YOON KIWOONG <48848617+kiwoongyoon@users.noreply.github.com> Date: Wed, 9 Sep 2026 01:52:03 +0900 Subject: [PATCH 7/9] GitContribute issue #7287 (#7288) Signed-off-by: kiwoong Co-authored-by: Vidit Ostwal <110953813+Vidit-Ostwal@users.noreply.github.com> --- lib/crewai-core/pyproject.toml | 6 +++--- lib/crewai/pyproject.toml | 6 +++--- uv.lock | 12 ++++++------ 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/lib/crewai-core/pyproject.toml b/lib/crewai-core/pyproject.toml index 405dd4d46..f6b790ef9 100644 --- a/lib/crewai-core/pyproject.toml +++ b/lib/crewai-core/pyproject.toml @@ -16,9 +16,9 @@ dependencies = [ "pyjwt>=2.13.0,<3", "pydantic>=2.11.9,<2.13", "rich>=13.7.1", - "opentelemetry-api~=1.42.0", - "opentelemetry-sdk~=1.42.0", - "opentelemetry-exporter-otlp-proto-http~=1.42.0", + "opentelemetry-api>=1.42,<2", + "opentelemetry-sdk>=1.42,<2", + "opentelemetry-exporter-otlp-proto-http>=1.42,<2", "tomli~=2.0.2", ] diff --git a/lib/crewai/pyproject.toml b/lib/crewai/pyproject.toml index f6299c7d1..dcde4656f 100644 --- a/lib/crewai/pyproject.toml +++ b/lib/crewai/pyproject.toml @@ -18,9 +18,9 @@ dependencies = [ "pdfplumber~=0.11.4", "regex~=2026.1.15", # Telemetry and Monitoring - "opentelemetry-api~=1.42.0", - "opentelemetry-sdk~=1.42.0", - "opentelemetry-exporter-otlp-proto-http~=1.42.0", + "opentelemetry-api>=1.42,<2", + "opentelemetry-sdk>=1.42,<2", + "opentelemetry-exporter-otlp-proto-http>=1.42,<2", # Data Handling "chromadb~=1.1.0", "tokenizers>=0.21,<1", diff --git a/uv.lock b/uv.lock index d1eeeb16f..ead744797 100644 --- a/uv.lock +++ b/uv.lock @@ -1561,9 +1561,9 @@ requires-dist = [ { name = "openai", specifier = ">=2.30.0,<3" }, { name = "openpyxl", specifier = "~=3.1.5" }, { name = "openpyxl", marker = "extra == 'openpyxl'", specifier = "~=3.1.5" }, - { name = "opentelemetry-api", specifier = "~=1.42.0" }, - { name = "opentelemetry-exporter-otlp-proto-http", specifier = "~=1.42.0" }, - { name = "opentelemetry-sdk", specifier = "~=1.42.0" }, + { name = "opentelemetry-api", specifier = ">=1.42,<2" }, + { name = "opentelemetry-exporter-otlp-proto-http", specifier = ">=1.42,<2" }, + { name = "opentelemetry-sdk", specifier = ">=1.42,<2" }, { name = "pandas", marker = "extra == 'pandas'", specifier = "~=2.2.3" }, { name = "pdfplumber", specifier = "~=0.11.4" }, { name = "portalocker", specifier = "~=2.7.0" }, @@ -1648,9 +1648,9 @@ requires-dist = [ { name = "appdirs", specifier = "~=1.4.4" }, { name = "cryptography", specifier = ">=42.0" }, { name = "httpx", specifier = "~=0.28.1" }, - { name = "opentelemetry-api", specifier = "~=1.42.0" }, - { name = "opentelemetry-exporter-otlp-proto-http", specifier = "~=1.42.0" }, - { name = "opentelemetry-sdk", specifier = "~=1.42.0" }, + { name = "opentelemetry-api", specifier = ">=1.42,<2" }, + { name = "opentelemetry-exporter-otlp-proto-http", specifier = ">=1.42,<2" }, + { name = "opentelemetry-sdk", specifier = ">=1.42,<2" }, { name = "packaging", specifier = ">=23.0" }, { name = "portalocker", specifier = "~=2.7.0" }, { name = "pydantic", specifier = ">=2.11.9,<2.13" }, From 79befd0ce504ea2b94f0e20c5aee79672cfe1102 Mon Sep 17 00:00:00 2001 From: Vidit Ostwal <110953813+Vidit-Ostwal@users.noreply.github.com> Date: Tue, 8 Sep 2026 23:08:10 +0530 Subject: [PATCH 8/9] docs: clarify that tracing is managed separately from telemetry (#7311) Users who disable telemetry still need the tracing docs to understand first-run trace viewing and how the two settings relate. --- docs/edge/ar/concepts/cli.mdx | 1 + docs/edge/ar/observability/tracing.mdx | 14 +++++++++++++- docs/edge/ar/telemetry.mdx | 1 + docs/edge/en/concepts/cli.mdx | 1 + docs/edge/en/concepts/crews.mdx | 2 +- docs/edge/en/observability/tracing.mdx | 14 +++++++++++++- docs/edge/en/telemetry.mdx | 1 + docs/edge/ko/observability/tracing.mdx | 13 ++++++++++++- docs/edge/ko/telemetry.mdx | 1 + docs/edge/pt-BR/observability/tracing.mdx | 14 +++++++++++++- docs/edge/pt-BR/telemetry.mdx | 1 + 11 files changed, 58 insertions(+), 5 deletions(-) diff --git a/docs/edge/ar/concepts/cli.mdx b/docs/edge/ar/concepts/cli.mdx index e20c7232d..3beb6f13b 100644 --- a/docs/edge/ar/concepts/cli.mdx +++ b/docs/edge/ar/concepts/cli.mdx @@ -286,6 +286,7 @@ crewai traces [COMMAND] ```shell Terminal crewai traces enable ``` + - يحدّث موجه التشغيل الأول (`Would you like to view your execution traces?`) هذا التفضيل أيضاً **لتفعيل التتبع**، استخدم أيًا من هذه الطرق: diff --git a/docs/edge/ar/observability/tracing.mdx b/docs/edge/ar/observability/tracing.mdx index 234a62fcd..fae4966f6 100644 --- a/docs/edge/ar/observability/tracing.mdx +++ b/docs/edge/ar/observability/tracing.mdx @@ -9,7 +9,7 @@ mode: "wide" يوفر CrewAI إمكانيات تتبع مدمجة تتيح لك مراقبة وتصحيح أخطاء الطواقم والتدفقات في الوقت الفعلي. يوضح هذا الدليل كيفية تفعيل التتبع لكل من **الطواقم** و**التدفقات** باستخدام منصة المراقبة المتكاملة في CrewAI. -> **ما هو تتبع CrewAI؟** يوفر التتبع المدمج في CrewAI مراقبة شاملة لوكلاء الذكاء الاصطناعي، بما في ذلك قرارات الوكلاء وجداول تنفيذ المهام واستخدام الأدوات واستدعاءات LLM - كل ذلك متاح عبر [منصة CrewAI AMP](https://app.crewai.com). +> **ما هو تتبع CrewAI؟** يوفر التتبع المدمج في CrewAI مراقبة شاملة لوكلاء الذكاء الاصطناعي، بما في ذلك قرارات الوكلاء وجداول تنفيذ المهام واستخدام الأدوات واستدعاءات LLM - كل ذلك متاح عبر [منصة CrewAI AMP](https://app.crewai.com). يتم إدارة التتبع بشكل مستقل عن [القياس عن بُعد](/ar/telemetry). ![واجهة تتبع CrewAI](/images/crewai-tracing.png) @@ -170,6 +170,18 @@ CREWAI_TRACING_ENABLED=true عند تعيين متغير البيئة هذا، ستُفعّل جميع الطواقم والتدفقات التتبع تلقائياً، حتى بدون تعيين `tracing=True` صراحةً. +## عرض التتبعات بعد أول تشغيل + +في المرة الأولى التي تشغّل فيها طاقماً أو تدفقاً، قد يسألك طرف تفاعلي: + +```text +Would you like to view your execution traces? [y/N] +``` + +اختر **yes** لفتح رابط العرض. يمكنك تغيير ذلك لاحقاً باستخدام +`crewai traces enable` أو `crewai traces disable`، أو بتعيين `tracing` +على الطاقم أو التدفق. + ## عرض التتبعات ### الوصول إلى لوحة تحكم CrewAI AMP diff --git a/docs/edge/ar/telemetry.mdx b/docs/edge/ar/telemetry.mdx index dd9ac0a24..ae734669f 100644 --- a/docs/edge/ar/telemetry.mdx +++ b/docs/edge/ar/telemetry.mdx @@ -24,6 +24,7 @@ mode: "wide" لتوفير رؤى أعمق. قد يتضمن جمع البيانات الموسع هذا معلومات شخصية إذا دمجها المستخدمون في طواقمهم أو مهامهم. يجب على المستخدمين النظر بعناية في محتوى طواقمهم ومهامهم قبل تفعيل `share_crew`. يمكن للمستخدمين تعطيل القياس عن بُعد في CrewAI عبر تعيين `CREWAI_DISABLE_TELEMETRY` إلى `true` أو `1` أو `yes` أو `on` (بغض النظر عن حالة الأحرف). `OTEL_SDK_DISABLED` بنفس القيم يعطّل أيضاً مُصدِّر CrewAI. مجموعة أدوات OpenTelemetry نفسها ما تزال تقبل `true` فقط لتعطيل بقية أدوات القياس في العملية. +تتبع AMP مشمول بشكل منفصل في [التتبع](/ar/observability/tracing). ### أمثلة: ```python diff --git a/docs/edge/en/concepts/cli.mdx b/docs/edge/en/concepts/cli.mdx index aa02e4a6b..b09f78d87 100644 --- a/docs/edge/en/concepts/cli.mdx +++ b/docs/edge/en/concepts/cli.mdx @@ -578,6 +578,7 @@ Trace collection is controlled by checking three settings in priority order: ``` - Checked only if `tracing` is not set in code and `CREWAI_TRACING_ENABLED` is not set to `true` - Running `crewai traces enable` is sufficient to enable tracing by itself + - The first-run prompt (`Would you like to view your execution traces?`) also updates this preference **To enable tracing**, use any one of these methods: diff --git a/docs/edge/en/concepts/crews.mdx b/docs/edge/en/concepts/crews.mdx index 2e4f21032..872a14efd 100644 --- a/docs/edge/en/concepts/crews.mdx +++ b/docs/edge/en/concepts/crews.mdx @@ -37,7 +37,7 @@ A crew in crewAI represents a collaborative group of agents working together to | **Chat LLM** _(optional)_ | `chat_llm` | The language model used to orchestrate `crewai chat` CLI interactions with the crew. Accepts a model name string or `LLM` instance. Defaults to `None`. | | **Before Kickoff Callbacks** _(optional)_ | `before_kickoff_callbacks` | A list of callable functions executed **before** the crew starts. Each callback receives and can modify the inputs dict. Distinct from the `@before_kickoff` decorator. Defaults to `[]`. | | **After Kickoff Callbacks** _(optional)_ | `after_kickoff_callbacks` | A list of callable functions executed **after** the crew finishes. Each callback receives and can modify the `CrewOutput`. Distinct from the `@after_kickoff` decorator. Defaults to `[]`. | -| **Tracing** _(optional)_ | `tracing` | Controls OpenTelemetry tracing for the crew. `True` = always enable, `False` = always disable, `None` = inherit from environment / user settings. Defaults to `None`. | +| **Tracing** _(optional)_ | `tracing` | Controls tracing for the crew. `True` = always enable, `False` = always disable, `None` = inherit from environment / user settings. Defaults to `None`. | | **Skills** _(optional)_ | `skills` | A list of `Path` objects (skill search directories) or pre-loaded `Skill` objects applied to all agents in the crew. Defaults to `None`. | | **Security Config** _(optional)_ | `security_config` | A `SecurityConfig` instance managing crew fingerprinting and identity. Defaults to `SecurityConfig()`. | | **Checkpoint** _(optional)_ | `checkpoint` | Enables automatic checkpointing. Pass `True` for sensible defaults, a `CheckpointConfig` for full control, `False` to opt out, or `None` to inherit. See the [Checkpointing](#checkpointing) section below. Defaults to `None`. | diff --git a/docs/edge/en/observability/tracing.mdx b/docs/edge/en/observability/tracing.mdx index ce620946a..7a1775916 100644 --- a/docs/edge/en/observability/tracing.mdx +++ b/docs/edge/en/observability/tracing.mdx @@ -9,7 +9,7 @@ mode: "wide" CrewAI provides built-in tracing capabilities that allow you to monitor and debug your Crews and Flows in real-time. This guide demonstrates how to enable tracing for both **Crews** and **Flows** using CrewAI's integrated observability platform. -> **What is CrewAI Tracing?** CrewAI's built-in tracing provides comprehensive observability for your AI agents, including agent decisions, task execution timelines, tool usage, and LLM calls - all accessible through the [CrewAI AMP platform](https://app.crewai.com). +> **What is CrewAI Tracing?** CrewAI's built-in tracing provides comprehensive observability for your AI agents, including agent decisions, task execution timelines, tool usage, and LLM calls - all accessible through the [CrewAI AMP platform](https://app.crewai.com). Tracing is managed independently from [telemetry](/en/telemetry). ![CrewAI Tracing Interface](/images/crewai-tracing.png) @@ -170,6 +170,18 @@ CREWAI_TRACING_ENABLED=true When this environment variable is set, all Crews and Flows will automatically have tracing enabled, even without explicitly setting `tracing=True`. +## Viewing traces after your first run + +The first time you run a Crew or Flow, an interactive terminal may ask: + +```text +Would you like to view your execution traces? [y/N] +``` + +Choose **yes** to open a view link. You can change this later with +`crewai traces enable` or `crewai traces disable`, or by setting `tracing` +on the Crew or Flow. + ## Viewing Your Traces ### Access the CrewAI AMP Dashboard diff --git a/docs/edge/en/telemetry.mdx b/docs/edge/en/telemetry.mdx index b2c18adbb..f9e7abf78 100644 --- a/docs/edge/en/telemetry.mdx +++ b/docs/edge/en/telemetry.mdx @@ -24,6 +24,7 @@ When the `share_crew` feature is enabled, detailed data including task descripti to provide deeper insights. This expanded data collection may include personal information if users have incorporated it into their crews or tasks. Users should carefully consider the content of their crews and tasks before enabling `share_crew`. Users can disable CrewAI telemetry by setting `CREWAI_DISABLE_TELEMETRY` to `true`, `1`, `yes`, or `on` (any case). `OTEL_SDK_DISABLED` with the same values also disables CrewAI's exporter. The OpenTelemetry SDK itself still only honors `true` for disabling other instrumentation in the process. +AMP tracing is covered separately in [Tracing](/en/observability/tracing). ### Examples: ```python diff --git a/docs/edge/ko/observability/tracing.mdx b/docs/edge/ko/observability/tracing.mdx index eae6188f6..db5e0ea5c 100644 --- a/docs/edge/ko/observability/tracing.mdx +++ b/docs/edge/ko/observability/tracing.mdx @@ -9,7 +9,7 @@ mode: "wide" CrewAI는 Crews와 Flows를 실시간으로 모니터링하고 디버깅할 수 있는 내장 추적 기능을 제공합니다. 이 가이드는 CrewAI의 통합 관측 가능성 플랫폼을 사용하여 **Crews**와 **Flows** 모두에 대한 추적을 활성화하는 방법을 보여줍니다. -> **CrewAI Tracing이란?** CrewAI의 내장 추적은 agent 결정, 작업 실행 타임라인, 도구 사용, LLM 호출을 포함한 AI agent에 대한 포괄적인 관측 가능성을 제공하며, 모두 [CrewAI AMP 플랫폼](https://app.crewai.com)을 통해 액세스할 수 있습니다. +> **CrewAI Tracing이란?** CrewAI의 내장 추적은 agent 결정, 작업 실행 타임라인, 도구 사용, LLM 호출을 포함한 AI agent에 대한 포괄적인 관측 가능성을 제공하며, 모두 [CrewAI AMP 플랫폼](https://app.crewai.com)을 통해 액세스할 수 있습니다. 추적은 [텔레메트리](/ko/telemetry)와 별도로 관리됩니다. ![CrewAI Tracing Interface](/images/crewai-tracing.png) @@ -170,6 +170,17 @@ CREWAI_TRACING_ENABLED=true 이 환경 변수가 설정되면 `tracing=True`를 명시적으로 설정하지 않아도 모든 Crews와 Flows에 자동으로 추적이 활성화됩니다. +## 첫 실행 후 추적 보기 + +Crew 또는 Flow를 처음 실행하면 대화형 터미널에서 다음을 물을 수 있습니다: + +```text +Would you like to view your execution traces? [y/N] +``` + +보기 링크를 열려면 **yes**를 선택하세요. 나중에 `crewai traces enable` 또는 +`crewai traces disable`로 바꾸거나, Crew 또는 Flow에서 `tracing`을 설정할 수 있습니다. + ## 추적 보기 ### CrewAI AMP 대시보드 액세스 diff --git a/docs/edge/ko/telemetry.mdx b/docs/edge/ko/telemetry.mdx index 6d83e8a1a..15de481bd 100644 --- a/docs/edge/ko/telemetry.mdx +++ b/docs/edge/ko/telemetry.mdx @@ -23,6 +23,7 @@ CrewAI는 익명 텔레메트리를 활용하여 사용 통계를 수집하며, 이 확대된 데이터 수집에는 사용자가 crew나 작업에 개인정보를 포함한 경우, 개인정보가 포함될 수 있습니다. 사용자는 `share_crew`를 활성화하기 전에 crew와 작업의 내용을 신중하게 검토해야 합니다. 사용자는 `CREWAI_DISABLE_TELEMETRY`를 `true`, `1`, `yes`, `on` 중 하나로 설정하여 CrewAI 텔레메트리를 비활성화할 수 있습니다(대소문자 무관). 같은 값의 `OTEL_SDK_DISABLED`도 CrewAI exporter를 끕니다. 프로세스 내 다른 OpenTelemetry 계측을 끄려면 OpenTelemetry SDK는 여전히 `true`만 인식합니다. +AMP 추적은 [Tracing](/ko/observability/tracing)에서 별도로 다룹니다. ### 예시: ```python diff --git a/docs/edge/pt-BR/observability/tracing.mdx b/docs/edge/pt-BR/observability/tracing.mdx index ba6c1b40a..da5b95ebf 100644 --- a/docs/edge/pt-BR/observability/tracing.mdx +++ b/docs/edge/pt-BR/observability/tracing.mdx @@ -9,7 +9,7 @@ mode: "wide" O CrewAI fornece recursos de rastreamento integrados que permitem monitorar e depurar seus Crews e Flows em tempo real. Este guia demonstra como habilitar o rastreamento para **Crews** e **Flows** usando a plataforma de observabilidade integrada do CrewAI. -> **O que é o CrewAI Tracing?** O rastreamento integrado do CrewAI fornece observabilidade abrangente para seus agentes de IA, incluindo decisões de agentes, cronogramas de execução de tarefas, uso de ferramentas e chamadas de LLM - tudo acessível através da [plataforma CrewAI AMP](https://app.crewai.com). +> **O que é o CrewAI Tracing?** O rastreamento integrado do CrewAI fornece observabilidade abrangente para seus agentes de IA, incluindo decisões de agentes, cronogramas de execução de tarefas, uso de ferramentas e chamadas de LLM - tudo acessível através da [plataforma CrewAI AMP](https://app.crewai.com). O rastreamento é gerenciado de forma independente da [telemetria](/pt-BR/telemetry). ![CrewAI Tracing Interface](/images/crewai-tracing.png) @@ -170,6 +170,18 @@ CREWAI_TRACING_ENABLED=true Quando esta variável de ambiente estiver definida, todos os Crews e Flows terão automaticamente o rastreamento habilitado, mesmo sem definir explicitamente `tracing=True`. +## Visualizando rastreamentos após a primeira execução + +Na primeira vez que você executa um Crew ou Flow, um terminal interativo pode perguntar: + +```text +Would you like to view your execution traces? [y/N] +``` + +Escolha **yes** para abrir um link de visualização. Você pode alterar isso depois com +`crewai traces enable` ou `crewai traces disable`, ou definindo `tracing` +no Crew ou Flow. + ## Visualizando seus Rastreamentos ### Acesse o Painel CrewAI AMP diff --git a/docs/edge/pt-BR/telemetry.mdx b/docs/edge/pt-BR/telemetry.mdx index 1b5517ff6..cf4b1c450 100644 --- a/docs/edge/pt-BR/telemetry.mdx +++ b/docs/edge/pt-BR/telemetry.mdx @@ -24,6 +24,7 @@ Quando o recurso `share_crew` está ativado, dados detalhados, incluindo descri para fornecer insights mais detalhados. Essa coleta expandida pode incluir informações pessoais caso o usuário as tenha inserido em seus crews ou tarefas. Usuários devem considerar cuidadosamente o conteúdo de seus crews e tarefas antes de habilitar o `share_crew`. A telemetria do CrewAI pode ser desabilitada ao definir `CREWAI_DISABLE_TELEMETRY` como `true`, `1`, `yes` ou `on` (qualquer capitalização). `OTEL_SDK_DISABLED` com os mesmos valores também desabilita o exportador do CrewAI. O SDK do OpenTelemetry em si ainda só reconhece `true` para desabilitar as demais instrumentações do processo. +O rastreamento AMP é tratado em [Tracing](/pt-BR/observability/tracing). ### Exemplos: ```python From a68b5e903c28b01b5f09eaae706c13ff21d8f4a3 Mon Sep 17 00:00:00 2001 From: Vidit Ostwal <110953813+Vidit-Ostwal@users.noreply.github.com> Date: Wed, 9 Sep 2026 00:32:53 +0530 Subject: [PATCH 9/9] chore(ci): label FTC-closed PRs as needs-issue (#7249) --- .github/CONTRIBUTING.md | 2 +- .github/pull_request_template.md | 3 ++- .github/workflows/ftc-require-issue.yml | 15 ++++++++++++++- lib/crewai/tests/ci/test_ftc_require_issue.py | 8 ++++++++ 4 files changed, 25 insertions(+), 3 deletions(-) diff --git a/.github/CONTRIBUTING.md b/.github/CONTRIBUTING.md index 4b0059b97..b77f5a636 100644 --- a/.github/CONTRIBUTING.md +++ b/.github/CONTRIBUTING.md @@ -104,7 +104,7 @@ chore(deps): bump pydantic to 2.11 - PRs over 500 lines are labeled `size/XL` automatically - Title must follow the same conventional commit format - Link related issues where applicable (`#123`, `Fixes #123`, or the issue URL) -- First-time contributors must open or pick an existing **open** issue first, then mention it in the PR title or body (for example `#123`). PRs without a linked open issue are closed automatically. +- First-time contributors must open or pick an existing **open** issue first, then mention it in the PR title or body (for example `#123`). PRs without a linked open issue are closed automatically and labeled `needs-issue`. ## Testing diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index 9aada7195..267a7dfbe 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -4,7 +4,8 @@ Fixes # ## Summary diff --git a/.github/workflows/ftc-require-issue.yml b/.github/workflows/ftc-require-issue.yml index 2a2a842d3..a9d18d1d2 100644 --- a/.github/workflows/ftc-require-issue.yml +++ b/.github/workflows/ftc-require-issue.yml @@ -6,7 +6,7 @@ on: permissions: pull-requests: write - issues: read + issues: write concurrency: group: ftc-require-issue-${{ github.event.pull_request.number }} @@ -114,6 +114,19 @@ jobs: ], check=True, ) + subprocess.run( + [ + "gh", + "pr", + "edit", + pr_number, + "--repo", + repo, + "--add-label", + "needs-issue", + ], + check=True, + ) subprocess.run( ["gh", "pr", "close", pr_number, "--repo", repo], check=True, diff --git a/lib/crewai/tests/ci/test_ftc_require_issue.py b/lib/crewai/tests/ci/test_ftc_require_issue.py index 2f166b1f3..f1128a0a8 100644 --- a/lib/crewai/tests/ci/test_ftc_require_issue.py +++ b/lib/crewai/tests/ci/test_ftc_require_issue.py @@ -86,6 +86,9 @@ def test_open_issue_mention_blocks_close(body: str) -> None: ) assert not any(call[:3] == ["gh", "pr", "close"] for call in calls) + assert not any( + call[:3] == ["gh", "pr", "edit"] and "--add-label" in call for call in calls + ) assert any(call[:2] == ["gh", "api"] and call[2].endswith("/issues/123") for call in calls) @@ -98,4 +101,9 @@ def test_foreign_repo_reference_closes_pr() -> None: ) assert any(call[:3] == ["gh", "pr", "close"] for call in calls) + assert any( + call[:3] == ["gh", "pr", "edit"] and call[call.index("--add-label") + 1] == "needs-issue" + for call in calls + if "--add-label" in call + ) assert not any(call[:2] == ["gh", "api"] for call in calls)