Add tokenizers to requirements.txt (#1790)

* Add tokenizers to requirements and pin version

* Update process_skipfiles.py

process_skipfiles.py

@@ -6,15 +6,24 @@ from distutils.sysconfig import get_python_lib
 import fileinput
 from pathlib import Path
 
-# Temorary workaround for transformers/__init__.py.
+# Temporary workaround for transformers/__init__.py.
-path_to_tranformers_hook = Path(
+path_to_stdhooks = Path(
-    get_python_lib()
+    get_python_lib() + "/_pyinstaller_hooks_contrib/hooks/stdhooks"
-    + "/_pyinstaller_hooks_contrib/hooks/stdhooks/hook-transformers.py"
 )
-if path_to_tranformers_hook.is_file():
+path_to_transformers_hook = Path(
+    str(path_to_stdhooks) + "hook-transformers.py"
+)
+if path_to_transformers_hook.is_file():
     pass
 else:
-    with open(path_to_tranformers_hook, "w") as f:
+    if not path_to_stdhooks.is_dir():
+        import os
+
+        print(
+            f"Path to pyinstaller stdhooks not found. Please check your pyinstaller packages at {path_to_stdhooks}."
+        )
+        os.mkdir(path_to_stdhooks)
+    with open(path_to_transformers_hook, "w") as f:
         f.write("module_collection_mode = 'pyz+py'")
 
 path_to_skipfiles = Path(get_python_lib() + "/torch/_dynamo/skipfiles.py")

requirements.txt

@@ -18,6 +18,7 @@ Pillow
 parameterized
 
 # Add transformers, diffusers and scipy since it most commonly used
+tokenizers==0.13.3
 transformers
 diffusers
 #accelerate is now required for diffusers import from ckpt.