From 7564d7116347caa4f57652809ec2ede5e67e82f0 Mon Sep 17 00:00:00 2001
From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
Date: Fri, 23 Jan 2026 07:51:22 +0000
Subject: [PATCH] fix: relax tokenizers version constraint to support
 transformers 4.51+
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

This commit addresses issue #4268 where crewAI's restrictive tokenizers
constraint (using ~= operator) prevented using recent versions of
transformers (4.51+) which require tokenizers >= 0.21.

Changes:
- Changed tokenizers constraint from ~=0.20.3 to >=0.20.3
- Added test to verify the constraint remains flexible

The ~= operator was too restrictive as it only allows patch version
updates (tokenizers~=0.20.3 means >=0.20.3,<0.21.0). This caused
dependency resolution failures when installing transformers 4.51+.

crewAI does not directly import or use tokenizers - it is a transitive
dependency. chromadb only requires tokenizers>=0.13.2 with no upper
bound, so relaxing this constraint is safe.

Fixes #4268

Co-Authored-By: João <joao@crewai.com>
---
 lib/crewai/pyproject.toml                     |  2 +-
 .../tests/test_dependency_constraints.py      | 51 +++++++++++++++++++
 2 files changed, 52 insertions(+), 1 deletion(-)
 create mode 100644 lib/crewai/tests/test_dependency_constraints.py

diff --git a/lib/crewai/pyproject.toml b/lib/crewai/pyproject.toml
index ada287cd8..668d93263 100644
--- a/lib/crewai/pyproject.toml
+++ b/lib/crewai/pyproject.toml
@@ -21,7 +21,7 @@ dependencies = [
     "opentelemetry-exporter-otlp-proto-http~=1.34.0",
     # Data Handling
     "chromadb~=1.1.0",
-    "tokenizers~=0.20.3",
+    "tokenizers>=0.20.3",
     "openpyxl~=3.1.5",
     # Authentication and Security
     "python-dotenv~=1.1.1",
diff --git a/lib/crewai/tests/test_dependency_constraints.py b/lib/crewai/tests/test_dependency_constraints.py
new file mode 100644
index 000000000..3c02637a2
--- /dev/null
+++ b/lib/crewai/tests/test_dependency_constraints.py
@@ -0,0 +1,51 @@
+"""Tests for dependency constraints in pyproject.toml.
+
+These tests verify that dependency constraints are flexible enough to allow
+installation alongside other packages that may require different versions
+of shared dependencies.
+"""
+
+from pathlib import Path
+
+import pytest
+import tomli
+
+
+@pytest.fixture
+def pyproject_dependencies():
+    """Load dependencies from pyproject.toml."""
+    pyproject_path = Path(__file__).parent.parent / "pyproject.toml"
+    with open(pyproject_path, "rb") as f:
+        pyproject = tomli.load(f)
+    return pyproject["project"]["dependencies"]
+
+
+def test_tokenizers_constraint_allows_newer_versions(pyproject_dependencies):
+    """Test that tokenizers constraint allows versions >= 0.21.
+
+    This test ensures that the tokenizers dependency constraint is flexible
+    enough to allow installation of transformers 4.51+ which requires
+    tokenizers >= 0.21.
+
+    See: https://github.com/crewAIInc/crewAI/issues/4268
+    """
+    tokenizers_dep = None
+    for dep in pyproject_dependencies:
+        if dep.startswith("tokenizers"):
+            tokenizers_dep = dep
+            break
+
+    assert tokenizers_dep is not None, "tokenizers dependency not found in pyproject.toml"
+
+    # The constraint should use >= (minimum version) not ~= (compatible release)
+    # ~=0.20.3 means >=0.20.3,<0.21.0 which blocks transformers 4.51+
+    # >=0.20.3 allows any version >= 0.20.3 including 0.21+
+    assert "~=" not in tokenizers_dep, (
+        f"tokenizers constraint '{tokenizers_dep}' uses ~= operator which is too restrictive. "
+        "This blocks transformers 4.51+ which requires tokenizers >= 0.21. "
+        "Use >= instead of ~= to allow newer versions."
+    )
+    assert ">=" in tokenizers_dep, (
+        f"tokenizers constraint '{tokenizers_dep}' should use >= operator "
+        "to allow newer versions required by transformers 4.51+"
+    )