add tokenizers requirement

fernando-andutta · Aug 9, 2021 · 878ba71 · 878ba71
1 parent 8a87467
commit 878ba71
Show file tree

Hide file tree

Showing 2 changed files with 3 additions and 1 deletion.
diff --git a/requirements.txt b/requirements.txt
@@ -1,4 +1,5 @@
-transformers>=3.1.0,<5.0.0
+transformers>=4.6.0,<5.0.0
+tokenizers>=0.10.3
 tqdm
 torch>=1.6.0
 torchvision

diff --git a/setup.py b/setup.py
@@ -19,6 +19,7 @@
     packages=find_packages(),
     install_requires=[
         'transformers>=4.6.0,<5.0.0',
+        'tokenizers>=0.10.3',
         'tqdm',
         'torch>=1.6.0',
         'torchvision',