Fix #2782: Resolve tokenizers/transformers dependency conflict by making tokenizers an optional dependency

Co-Authored-By: Joe Moura <joao@crewai.com>

pyproject.toml

@@ -53,6 +53,7 @@ openpyxl = [
 mem0 = ["mem0ai>=0.1.29"]
 docling = [
     "docling>=2.12.0",
+    "tokenizers>=0.21,<0.22",
 ]
 
 [tool.uv]

src/crewai/knowledge/source/crew_docling_source.py

@@ -17,6 +17,8 @@ from crewai.utilities.logger import Logger
 class CrewDoclingSource(BaseKnowledgeSource):
     """Default Source class for converting documents to markdown or json
     This will auto support PDF, DOCX, and TXT, XLSX, Images, and HTML files without any additional dependencies and follows the docling package as the source of truth.
+    
+    Note: To use this class, install crewai with the docling extra: `pip install crewai[docling]`
     """
 
     _logger: Logger = Logger(verbose=True)

tests/test_tokenizers_compatibility.py

@@ -0,0 +1,24 @@
+"""Test to verify compatibility between tokenizers and transformers."""
+
+import pytest
+
+
+def test_tokenizers_transformers_compatibility():
+    """Test that the installed tokenizers version is compatible with transformers."""
+    try:
+        import tokenizers
+        import transformers
+    except ImportError:
+        pytest.skip("tokenizers or transformers not installed")
+    
+    tokenizers_version = tokenizers.__version__
+    transformers_version = transformers.__version__
+    
+    tokenizers_major, tokenizers_minor, _ = map(int, tokenizers_version.split('.'))
+    
+    assert tokenizers_major == 0, f"Expected tokenizers major version 0, got {tokenizers_major}"
+    assert tokenizers_minor >= 21, f"Expected tokenizers minor version >=21, got {tokenizers_minor}"
+    assert tokenizers_minor < 22, f"Expected tokenizers minor version <22, got {tokenizers_minor}"
+    
+    print(f"Tokenizers version: {tokenizers_version}")
+    print(f"Transformers version: {transformers_version}")