crewAI/tests/test_tokenizers_compatibility.py

"""Test to verify compatibility between tokenizers and transformers."""

import pytest


def test_tokenizers_transformers_compatibility():
    """Test that the installed tokenizers version is compatible with transformers."""
    try:
        import tokenizers
        import transformers
    except ImportError:
        pytest.skip("tokenizers or transformers not installed")
    
    tokenizers_version = tokenizers.__version__
    transformers_version = transformers.__version__
    
    tokenizers_major, tokenizers_minor, _ = map(int, tokenizers_version.split('.'))
    
    assert tokenizers_major == 0, f"Expected tokenizers major version 0, got {tokenizers_major}"
    assert tokenizers_minor >= 21, f"Expected tokenizers minor version >=21, got {tokenizers_minor}"
    assert tokenizers_minor < 22, f"Expected tokenizers minor version <22, got {tokenizers_minor}"
    
    print(f"Tokenizers version: {tokenizers_version}")
    print(f"Transformers version: {transformers_version}")