Compare commits

..

5 Commits

Author SHA1 Message Date
Greyson LaLonde
e33bf09d1c Merge branch 'main' into lg-isolate-platform-integration-token 2026-02-19 17:51:50 -05:00
github-actions[bot]
50dba2e155 chore: update tool specifications 2026-02-19 14:36:07 +00:00
Lucas Gomide
02d53ab009 style: resolver linter issues 2026-02-19 11:32:41 -03:00
Lucas Gomide
1e3f0c9c8b feat: enhance platform_integration_context with nullcontext support
Improve the platform integration context manager to handle None/falsy tokens gracefully by returning nullcontext
2026-02-18 10:43:38 -03:00
Lucas Gomide
727dccdfb8 feat: isolate retrieval of platform integration token (context.var or env var) 2026-02-18 10:43:38 -03:00
13 changed files with 4476 additions and 4863 deletions

View File

@@ -1,5 +1,7 @@
import os
from crewai.context import get_platform_integration_token as _get_context_token
def get_platform_api_base_url() -> str:
"""Get the platform API base URL from environment or use default."""
@@ -8,10 +10,16 @@ def get_platform_api_base_url() -> str:
def get_platform_integration_token() -> str:
"""Get the platform API base URL from environment or use default."""
token = os.getenv("CREWAI_PLATFORM_INTEGRATION_TOKEN") or ""
"""Get the platform integration token from the context.
Fallback to the environment variable if no token has been set in the context.
Raises:
ValueError: If no token has been set in the context.
"""
token = _get_context_token() or os.getenv("CREWAI_PLATFORM_INTEGRATION_TOKEN")
if not token:
raise ValueError(
"No platform integration token found, please set the CREWAI_PLATFORM_INTEGRATION_TOKEN environment variable"
"No platform integration token found. "
"Set it via platform_integration_context() or set_platform_integration_token()."
)
return token # TODO: Use context manager to get token
return token

View File

@@ -0,0 +1,56 @@
"""Tests for platform tools misc functionality."""
import os
from unittest.mock import patch
import pytest
from crewai.context import platform_integration_context, set_platform_integration_token, reset_platform_integration_token
from crewai_tools.tools.crewai_platform_tools.misc import (
get_platform_integration_token,
)
class TestTokenRetrievalWithFallback:
"""Test token retrieval logic with environment fallback."""
@pytest.fixture
def clean_context(self):
token = set_platform_integration_token(None)
env_backup = os.environ.pop("CREWAI_PLATFORM_INTEGRATION_TOKEN", None)
yield
reset_platform_integration_token(token)
if env_backup is not None:
os.environ["CREWAI_PLATFORM_INTEGRATION_TOKEN"] = env_backup
else:
os.environ.pop("CREWAI_PLATFORM_INTEGRATION_TOKEN", None)
def test_context_token_takes_precedence(self, clean_context):
"""Test that context token takes precedence over environment variable."""
context_token = "context-token"
env_token = "env-token"
with patch.dict(os.environ, {"CREWAI_PLATFORM_INTEGRATION_TOKEN": env_token}):
with platform_integration_context(context_token):
token = get_platform_integration_token()
assert token == context_token
def test_environment_fallback_when_no_context(self, clean_context):
"""Test fallback to environment variable when no context token."""
env_token = "env-fallback-token"
with patch.dict(os.environ, {"CREWAI_PLATFORM_INTEGRATION_TOKEN": env_token}):
token = get_platform_integration_token()
assert token == env_token
@pytest.mark.parametrize("empty_value", ["", None])
def test_missing_token_raises_error(self, clean_context, empty_value):
"""Test that missing tokens raise appropriate errors."""
env_dict = {"CREWAI_PLATFORM_INTEGRATION_TOKEN": empty_value} if empty_value is not None else {}
with patch.dict(os.environ, env_dict, clear=True):
with pytest.raises(ValueError) as exc_info:
get_platform_integration_token()
assert "No platform integration token found" in str(exc_info.value)
assert "platform_integration_context()" in str(exc_info.value)

View File

@@ -20117,18 +20117,6 @@
"humanized_name": "Web Automation Tool",
"init_params_schema": {
"$defs": {
"AvailableModel": {
"enum": [
"gpt-4o",
"gpt-4o-mini",
"claude-3-5-sonnet-latest",
"claude-3-7-sonnet-latest",
"computer-use-preview",
"gemini-2.0-flash"
],
"title": "AvailableModel",
"type": "string"
},
"EnvVar": {
"properties": {
"default": {
@@ -20206,17 +20194,6 @@
"default": null,
"title": "Model Api Key"
},
"model_name": {
"anyOf": [
{
"$ref": "#/$defs/AvailableModel"
},
{
"type": "null"
}
],
"default": "claude-3-7-sonnet-latest"
},
"project_id": {
"anyOf": [
{

View File

@@ -21,7 +21,7 @@ dependencies = [
"opentelemetry-exporter-otlp-proto-http~=1.34.0",
# Data Handling
"chromadb~=1.1.0",
"tokenizers>=0.21,<1",
"tokenizers~=0.20.3",
"openpyxl~=3.1.5",
# Authentication and Security
"python-dotenv~=1.1.1",

View File

@@ -6,10 +6,8 @@ and memory management.
from __future__ import annotations
import asyncio
from collections.abc import Callable
from concurrent.futures import ThreadPoolExecutor, as_completed
import inspect
import logging
from typing import TYPE_CHECKING, Any, Literal, cast
@@ -738,9 +736,7 @@ class CrewAgentExecutor(CrewAgentExecutorMixin):
] = []
for call_id, func_name, func_args in parsed_calls:
original_tool = original_tools_by_name.get(func_name)
execution_plan.append(
(call_id, func_name, func_args, original_tool)
)
execution_plan.append((call_id, func_name, func_args, original_tool))
self._append_assistant_tool_calls_message(
[
@@ -750,9 +746,7 @@ class CrewAgentExecutor(CrewAgentExecutorMixin):
)
max_workers = min(8, len(execution_plan))
ordered_results: list[dict[str, Any] | None] = [None] * len(
execution_plan
)
ordered_results: list[dict[str, Any] | None] = [None] * len(execution_plan)
with ThreadPoolExecutor(max_workers=max_workers) as pool:
futures = {
pool.submit(
@@ -809,7 +803,7 @@ class CrewAgentExecutor(CrewAgentExecutorMixin):
return tool_finish
reasoning_prompt = self._i18n.slice("post_tool_reasoning")
reasoning_message = {
reasoning_message: LLMMessage = {
"role": "user",
"content": reasoning_prompt,
}
@@ -914,9 +908,9 @@ class CrewAgentExecutor(CrewAgentExecutorMixin):
elif (
should_execute
and original_tool
and (max_count := getattr(original_tool, "max_usage_count", None))
is not None
and getattr(original_tool, "current_usage_count", 0) >= max_count
and getattr(original_tool, "max_usage_count", None) is not None
and getattr(original_tool, "current_usage_count", 0)
>= original_tool.max_usage_count
):
max_usage_reached = True
@@ -995,17 +989,13 @@ class CrewAgentExecutor(CrewAgentExecutorMixin):
and hasattr(original_tool, "cache_function")
and callable(original_tool.cache_function)
):
should_cache = original_tool.cache_function(
args_dict, raw_result
)
should_cache = original_tool.cache_function(args_dict, raw_result)
if should_cache:
self.tools_handler.cache.add(
tool=func_name, input=input_str, output=raw_result
)
result = (
str(raw_result) if not isinstance(raw_result, str) else raw_result
)
result = str(raw_result) if not isinstance(raw_result, str) else raw_result
except Exception as e:
result = f"Error executing tool: {e}"
if self.task:
@@ -1500,9 +1490,7 @@ class CrewAgentExecutor(CrewAgentExecutorMixin):
formatted_answer: Current agent response.
"""
if self.step_callback:
cb_result = self.step_callback(formatted_answer)
if inspect.iscoroutine(cb_result):
asyncio.run(cb_result)
self.step_callback(formatted_answer)
def _append_message(
self, text: str, role: Literal["user", "assistant", "system"] = "assistant"

View File

@@ -1,7 +1,6 @@
from collections.abc import Generator
from contextlib import contextmanager
from contextlib import AbstractContextManager, contextmanager, nullcontext
import contextvars
import os
from typing import Any
@@ -10,40 +9,50 @@ _platform_integration_token: contextvars.ContextVar[str | None] = (
)
def set_platform_integration_token(integration_token: str) -> None:
def set_platform_integration_token(integration_token: str) -> contextvars.Token[str | None]:
"""Set the platform integration token in the current context.
Args:
integration_token: The integration token to set.
"""
_platform_integration_token.set(integration_token)
return _platform_integration_token.set(integration_token)
def reset_platform_integration_token(token: contextvars.Token[str | None]) -> None:
"""Reset the platform integration token to its previous value."""
_platform_integration_token.reset(token)
def get_platform_integration_token() -> str | None:
"""Get the platform integration token from the current context or environment.
"""Get the platform integration token from the current context.
Returns:
The integration token if set, otherwise None.
"""
token = _platform_integration_token.get()
if token is None:
token = os.getenv("CREWAI_PLATFORM_INTEGRATION_TOKEN")
return token
return _platform_integration_token.get()
@contextmanager
def platform_context(integration_token: str) -> Generator[None, Any, None]:
def platform_integration_context(integration_token: str | None) -> AbstractContextManager[None]:
"""Context manager to temporarily set the platform integration token.
Args:
integration_token: The integration token to set within the context.
"""
token = _platform_integration_token.set(integration_token)
try:
yield
finally:
_platform_integration_token.reset(token)
If None or falsy, returns nullcontext (no-op).
Returns:
A context manager that either sets the token or does nothing.
"""
if not integration_token:
return nullcontext()
@contextmanager
def _token_context() -> Generator[None, Any, None]:
token = set_platform_integration_token(integration_token)
try:
yield
finally:
reset_platform_integration_token(token)
return _token_context()
_current_task_id: contextvars.ContextVar[str | None] = contextvars.ContextVar(
"current_task_id", default=None

View File

@@ -1,10 +1,8 @@
from __future__ import annotations
import asyncio
from collections.abc import Callable, Coroutine
from concurrent.futures import ThreadPoolExecutor, as_completed
from datetime import datetime
import inspect
import json
import threading
from typing import TYPE_CHECKING, Any, Literal, cast
@@ -780,7 +778,7 @@ class AgentExecutor(Flow[AgentReActState], CrewAgentExecutorMixin):
from_cache = cast(bool, execution_result["from_cache"])
original_tool = execution_result["original_tool"]
tool_message = {
tool_message: LLMMessage = {
"role": "tool",
"tool_call_id": call_id,
"name": func_name,
@@ -1360,9 +1358,7 @@ class AgentExecutor(Flow[AgentReActState], CrewAgentExecutorMixin):
formatted_answer: Current agent response.
"""
if self.step_callback:
cb_result = self.step_callback(formatted_answer)
if inspect.iscoroutine(cb_result):
asyncio.run(cb_result)
self.step_callback(formatted_answer)
def _append_message_to_state(
self, text: str, role: Literal["user", "assistant", "system"] = "assistant"

View File

@@ -1,6 +1,5 @@
from __future__ import annotations
import asyncio
from concurrent.futures import Future
from copy import copy as shallow_copy
import datetime
@@ -625,15 +624,11 @@ class Task(BaseModel):
self.end_time = datetime.datetime.now()
if self.callback:
cb_result = self.callback(self.output)
if inspect.isawaitable(cb_result):
await cb_result
self.callback(self.output)
crew = self.agent.crew # type: ignore[union-attr]
if crew and crew.task_callback and crew.task_callback != self.callback:
cb_result = crew.task_callback(self.output)
if inspect.isawaitable(cb_result):
await cb_result
crew.task_callback(self.output)
if self.output_file:
content = (
@@ -727,15 +722,11 @@ class Task(BaseModel):
self.end_time = datetime.datetime.now()
if self.callback:
cb_result = self.callback(self.output)
if inspect.iscoroutine(cb_result):
asyncio.run(cb_result)
self.callback(self.output)
crew = self.agent.crew # type: ignore[union-attr]
if crew and crew.task_callback and crew.task_callback != self.callback:
cb_result = crew.task_callback(self.output)
if inspect.iscoroutine(cb_result):
asyncio.run(cb_result)
crew.task_callback(self.output)
if self.output_file:
content = (

View File

@@ -3,7 +3,6 @@ from __future__ import annotations
import asyncio
from collections.abc import Callable, Sequence
import concurrent.futures
import inspect
import json
import re
from typing import TYPE_CHECKING, Any, Final, Literal, TypedDict
@@ -502,9 +501,7 @@ def handle_agent_action_core(
- TODO: Remove messages parameter and its usage.
"""
if step_callback:
cb_result = step_callback(tool_result)
if inspect.iscoroutine(cb_result):
asyncio.run(cb_result)
step_callback(tool_result)
formatted_answer.text += f"\nObservation: {tool_result.result}"
formatted_answer.result = tool_result.result

View File

@@ -2,7 +2,7 @@
import asyncio
from typing import Any
from unittest.mock import AsyncMock, MagicMock, Mock, patch
from unittest.mock import AsyncMock, MagicMock, patch
import pytest
@@ -291,46 +291,6 @@ class TestAsyncAgentExecutor:
assert max_concurrent > 1, f"Expected concurrent execution, max concurrent was {max_concurrent}"
class TestInvokeStepCallback:
"""Tests for _invoke_step_callback with sync and async callbacks."""
def test_invoke_step_callback_with_sync_callback(
self, executor: CrewAgentExecutor
) -> None:
"""Test that a sync step callback is called normally."""
callback = Mock()
executor.step_callback = callback
answer = AgentFinish(thought="thinking", output="test", text="final")
executor._invoke_step_callback(answer)
callback.assert_called_once_with(answer)
def test_invoke_step_callback_with_async_callback(
self, executor: CrewAgentExecutor
) -> None:
"""Test that an async step callback is awaited via asyncio.run."""
async_callback = AsyncMock()
executor.step_callback = async_callback
answer = AgentFinish(thought="thinking", output="test", text="final")
with patch("crewai.agents.crew_agent_executor.asyncio.run") as mock_run:
executor._invoke_step_callback(answer)
async_callback.assert_called_once_with(answer)
mock_run.assert_called_once()
def test_invoke_step_callback_with_none(
self, executor: CrewAgentExecutor
) -> None:
"""Test that no error is raised when step_callback is None."""
executor.step_callback = None
answer = AgentFinish(thought="thinking", output="test", text="final")
# Should not raise
executor._invoke_step_callback(answer)
class TestAsyncLLMResponseHelper:
"""Tests for aget_llm_response helper function."""

View File

@@ -7,215 +7,139 @@ import pytest
from crewai.context import (
_platform_integration_token,
get_platform_integration_token,
platform_context,
platform_integration_context,
reset_platform_integration_token,
set_platform_integration_token,
)
class TestPlatformIntegrationToken:
def setup_method(self):
_platform_integration_token.set(None)
@pytest.fixture
def clean_context():
"""Fixture to ensure clean context state for each test."""
_platform_integration_token.set(None)
yield
_platform_integration_token.set(None)
def teardown_method(self):
_platform_integration_token.set(None)
@patch.dict(os.environ, {}, clear=True)
def test_set_platform_integration_token(self):
class TestContextVariableCore:
"""Test core context variable functionality (set/get/reset)."""
def test_set_and_get_token(self, clean_context):
"""Test basic token setting and retrieval."""
test_token = "test-token-123"
assert get_platform_integration_token() is None
set_platform_integration_token(test_token)
context_token = set_platform_integration_token(test_token)
assert get_platform_integration_token() == test_token
assert context_token is not None
def test_get_platform_integration_token_from_context_var(self):
test_token = "context-var-token"
_platform_integration_token.set(test_token)
assert get_platform_integration_token() == test_token
@patch.dict(os.environ, {"CREWAI_PLATFORM_INTEGRATION_TOKEN": "env-token-456"})
def test_get_platform_integration_token_from_env_var(self):
assert _platform_integration_token.get() is None
assert get_platform_integration_token() == "env-token-456"
@patch.dict(os.environ, {"CREWAI_PLATFORM_INTEGRATION_TOKEN": "env-token"})
def test_context_var_takes_precedence_over_env_var(self):
context_token = "context-token"
set_platform_integration_token(context_token)
assert get_platform_integration_token() == context_token
@patch.dict(os.environ, {}, clear=True)
def test_get_platform_integration_token_returns_none_when_not_set(self):
assert _platform_integration_token.get() is None
assert get_platform_integration_token() is None
@patch.dict(os.environ, {}, clear=True)
def test_platform_context_manager_basic_usage(self):
test_token = "context-manager-token"
assert get_platform_integration_token() is None
with platform_context(test_token):
assert get_platform_integration_token() == test_token
assert get_platform_integration_token() is None
@patch.dict(os.environ, {}, clear=True)
def test_platform_context_manager_nested_contexts(self):
"""Test nested platform_context context managers."""
outer_token = "outer-token"
inner_token = "inner-token"
assert get_platform_integration_token() is None
with platform_context(outer_token):
assert get_platform_integration_token() == outer_token
with platform_context(inner_token):
assert get_platform_integration_token() == inner_token
assert get_platform_integration_token() == outer_token
assert get_platform_integration_token() is None
def test_platform_context_manager_preserves_existing_token(self):
"""Test that platform_context preserves existing token when exiting."""
initial_token = "initial-token"
context_token = "context-token"
set_platform_integration_token(initial_token)
assert get_platform_integration_token() == initial_token
with platform_context(context_token):
assert get_platform_integration_token() == context_token
assert get_platform_integration_token() == initial_token
def test_platform_context_manager_exception_handling(self):
"""Test that platform_context properly resets token even when exception occurs."""
initial_token = "initial-token"
context_token = "context-token"
set_platform_integration_token(initial_token)
with pytest.raises(ValueError):
with platform_context(context_token):
assert get_platform_integration_token() == context_token
raise ValueError("Test exception")
assert get_platform_integration_token() == initial_token
@patch.dict(os.environ, {}, clear=True)
def test_platform_context_manager_with_none_initial_state(self):
"""Test platform_context when initial state is None."""
context_token = "context-token"
assert get_platform_integration_token() is None
with pytest.raises(RuntimeError):
with platform_context(context_token):
assert get_platform_integration_token() == context_token
raise RuntimeError("Test exception")
assert get_platform_integration_token() is None
@patch.dict(os.environ, {"CREWAI_PLATFORM_INTEGRATION_TOKEN": "env-backup"})
def test_platform_context_with_env_fallback(self):
"""Test platform_context interaction with environment variable fallback."""
context_token = "context-token"
assert get_platform_integration_token() == "env-backup"
with platform_context(context_token):
assert get_platform_integration_token() == context_token
assert get_platform_integration_token() == "env-backup"
@patch.dict(os.environ, {}, clear=True)
def test_multiple_sequential_context_managers(self):
"""Test multiple sequential uses of platform_context."""
def test_reset_token_restores_previous_state(self, clean_context):
"""Test that reset properly restores previous context state."""
token1 = "token-1"
token2 = "token-2"
token3 = "token-3"
with platform_context(token1):
assert get_platform_integration_token() == token1
context_token1 = set_platform_integration_token(token1)
assert get_platform_integration_token() == token1
context_token2 = set_platform_integration_token(token2)
assert get_platform_integration_token() == token2
reset_platform_integration_token(context_token2)
assert get_platform_integration_token() == token1
reset_platform_integration_token(context_token1)
assert get_platform_integration_token() is None
def test_nested_token_management(self, clean_context):
"""Test proper token management with deeply nested contexts."""
tokens = ["token-1", "token-2", "token-3"]
context_tokens = []
for token in tokens:
context_tokens.append(set_platform_integration_token(token))
assert get_platform_integration_token() == token
for i in range(len(tokens) - 1, 0, -1):
reset_platform_integration_token(context_tokens[i])
assert get_platform_integration_token() == tokens[i - 1]
reset_platform_integration_token(context_tokens[0])
assert get_platform_integration_token() is None
@patch.dict(os.environ, {"CREWAI_PLATFORM_INTEGRATION_TOKEN": "env-token"})
def test_context_module_ignores_environment_variables(self, clean_context):
"""Test that context module only returns context values, not env vars."""
# Context module should not read environment variables
assert get_platform_integration_token() is None
# Only context variable should be returned
set_platform_integration_token("context-token")
assert get_platform_integration_token() == "context-token"
class TestPlatformIntegrationContext:
"""Test platform integration context manager behavior."""
def test_basic_context_manager_usage(self, clean_context):
"""Test basic context manager functionality."""
test_token = "context-token"
assert get_platform_integration_token() is None
with platform_context(token2):
assert get_platform_integration_token() == token2
assert get_platform_integration_token() is None
with platform_context(token3):
assert get_platform_integration_token() == token3
assert get_platform_integration_token() is None
def test_empty_string_token(self):
empty_token = ""
set_platform_integration_token(empty_token)
assert get_platform_integration_token() == ""
with platform_context(empty_token):
assert get_platform_integration_token() == ""
def test_special_characters_in_token(self):
special_token = "token-with-!@#$%^&*()_+-={}[]|\\:;\"'<>?,./"
set_platform_integration_token(special_token)
assert get_platform_integration_token() == special_token
with platform_context(special_token):
assert get_platform_integration_token() == special_token
def test_very_long_token(self):
long_token = "a" * 10000
set_platform_integration_token(long_token)
assert get_platform_integration_token() == long_token
with platform_context(long_token):
assert get_platform_integration_token() == long_token
@patch.dict(os.environ, {"CREWAI_PLATFORM_INTEGRATION_TOKEN": ""})
def test_empty_env_var(self):
assert _platform_integration_token.get() is None
assert get_platform_integration_token() == ""
@patch("crewai.context.os.getenv")
def test_env_var_access_error_handling(self, mock_getenv):
mock_getenv.side_effect = OSError("Environment access error")
with pytest.raises(OSError):
get_platform_integration_token()
@patch.dict(os.environ, {}, clear=True)
def test_context_var_isolation_between_tests(self):
"""Test that context variable changes don't leak between test methods."""
test_token = "isolation-test-token"
assert get_platform_integration_token() is None
set_platform_integration_token(test_token)
assert get_platform_integration_token() == test_token
def test_context_manager_return_value(self):
"""Test that platform_context can be used in with statement with return value."""
test_token = "return-value-token"
with platform_context(test_token):
with platform_integration_context(test_token):
assert get_platform_integration_token() == test_token
with platform_context(test_token) as ctx:
assert ctx is None
assert get_platform_integration_token() == test_token
assert get_platform_integration_token() is None
@pytest.mark.parametrize("falsy_value", [None, "", False, 0])
def test_falsy_values_return_nullcontext(self, clean_context, falsy_value):
"""Test that falsy values return nullcontext (no-op)."""
# Set initial token to verify nullcontext doesn't affect it
initial_token = "initial-token"
initial_context_token = set_platform_integration_token(initial_token)
try:
with platform_integration_context(falsy_value):
# Should preserve existing context (nullcontext behavior)
assert get_platform_integration_token() == initial_token
# Should still have initial token after nullcontext
assert get_platform_integration_token() == initial_token
finally:
reset_platform_integration_token(initial_context_token)
@pytest.mark.parametrize("truthy_value", ["token", "123", " ", "0"])
def test_truthy_values_create_context(self, clean_context, truthy_value):
"""Test that truthy values create proper context."""
with platform_integration_context(truthy_value):
assert get_platform_integration_token() == truthy_value
# Should be cleaned up
assert get_platform_integration_token() is None
def test_context_preserves_existing_token(self, clean_context):
"""Test that context manager preserves existing token when exiting."""
existing_token = "existing-token"
context_token = "context-token"
existing_context_token = set_platform_integration_token(existing_token)
try:
with platform_integration_context(context_token):
assert get_platform_integration_token() == context_token
assert get_platform_integration_token() == existing_token
finally:
reset_platform_integration_token(existing_context_token)
def test_context_manager_return_type(self, clean_context):
"""Test that context manager returns proper types for both cases."""
# Both should be usable as context managers
valid_ctx = platform_integration_context("token")
none_ctx = platform_integration_context(None)
assert hasattr(valid_ctx, '__enter__')
assert hasattr(valid_ctx, '__exit__')
assert hasattr(none_ctx, '__enter__')
assert hasattr(none_ctx, '__exit__')

View File

@@ -1,59 +0,0 @@
"""Tests for dependency version constraints.
Regression tests to ensure critical dependency constraints are correct,
particularly for packages whose older versions have broken metadata.
"""
import importlib.metadata
from pathlib import Path
import tomli
def _read_crewai_pyproject() -> dict:
pyproject_path = Path(__file__).resolve().parents[1] / "pyproject.toml"
with open(pyproject_path, "rb") as f:
return tomli.load(f)
class TestTokenizersDependency:
"""Regression tests for tokenizers dependency (issue #4550).
tokenizers 0.20.x has a broken pyproject.toml (missing project.version),
which causes installation failures when building from source (sdist).
The constraint must require >= 0.21 to avoid the broken versions.
"""
def test_tokenizers_constraint_excludes_broken_versions(self):
pyproject = _read_crewai_pyproject()
deps = pyproject["project"]["dependencies"]
tokenizers_dep = next(
(d for d in deps if d.startswith("tokenizers")), None
)
assert tokenizers_dep is not None, "tokenizers dependency not found in pyproject.toml"
assert "0.20" not in tokenizers_dep, (
f"tokenizers constraint '{tokenizers_dep}' still allows 0.20.x which has a broken sdist"
)
def test_tokenizers_constraint_allows_recent_versions(self):
pyproject = _read_crewai_pyproject()
deps = pyproject["project"]["dependencies"]
tokenizers_dep = next(
(d for d in deps if d.startswith("tokenizers")), None
)
assert tokenizers_dep is not None
assert ">=0.21" in tokenizers_dep, (
f"tokenizers constraint '{tokenizers_dep}' should require >=0.21"
)
def test_tokenizers_is_importable(self):
import tokenizers
assert tokenizers is not None
def test_installed_tokenizers_version_is_not_broken(self):
version = importlib.metadata.version("tokenizers")
major, minor = (int(x) for x in version.split(".")[:2])
assert (major, minor) >= (0, 21), (
f"Installed tokenizers {version} is from the 0.20.x range with broken sdist metadata"
)

8728
uv.lock generated

File diff suppressed because it is too large Load Diff