Fix #2782: Resolve tokenizers/transformers dependency conflict by making tokenizers an optional dependency

Co-Authored-By: Joe Moura <joao@crewai.com>

tests/test_tokenizers_compatibility.py

@@ -0,0 +1,24 @@
+"""Test to verify compatibility between tokenizers and transformers."""
+
+import pytest
+
+
+def test_tokenizers_transformers_compatibility():
+    """Test that the installed tokenizers version is compatible with transformers."""
+    try:
+        import tokenizers
+        import transformers
+    except ImportError:
+        pytest.skip("tokenizers or transformers not installed")
+    
+    tokenizers_version = tokenizers.__version__
+    transformers_version = transformers.__version__
+    
+    tokenizers_major, tokenizers_minor, _ = map(int, tokenizers_version.split('.'))
+    
+    assert tokenizers_major == 0, f"Expected tokenizers major version 0, got {tokenizers_major}"
+    assert tokenizers_minor >= 21, f"Expected tokenizers minor version >=21, got {tokenizers_minor}"
+    assert tokenizers_minor < 22, f"Expected tokenizers minor version <22, got {tokenizers_minor}"
+    
+    print(f"Tokenizers version: {tokenizers_version}")
+    print(f"Transformers version: {transformers_version}")