mirror of
https://github.com/crewAIInc/crewAI.git
synced 2026-01-09 16:18:30 +00:00
27 lines
1.1 KiB
Python
27 lines
1.1 KiB
Python
"""Test suite to verify compatibility between tokenizers and transformers packages.
|
|
Ensures the installed tokenizers version meets requirements and can work effectively with transformers.
|
|
"""
|
|
|
|
import pytest
|
|
|
|
|
|
def test_tokenizers_transformers_compatibility():
|
|
"""Test that the installed tokenizers version is compatible with transformers."""
|
|
try:
|
|
import tokenizers
|
|
import transformers
|
|
except ImportError:
|
|
pytest.skip("tokenizers or transformers not installed")
|
|
|
|
tokenizers_version = tokenizers.__version__
|
|
transformers_version = transformers.__version__
|
|
|
|
tokenizers_major, tokenizers_minor, _ = map(int, tokenizers_version.split('.'))
|
|
|
|
assert tokenizers_major == 0, f"Expected tokenizers major version 0, got {tokenizers_major}"
|
|
assert tokenizers_minor >= 21, f"Expected tokenizers minor version >=21, got {tokenizers_minor}"
|
|
assert tokenizers_minor < 22, f"Expected tokenizers minor version <22, got {tokenizers_minor}"
|
|
|
|
print(f"Tokenizers version: {tokenizers_version}")
|
|
print(f"Transformers version: {transformers_version}")
|