github/SHARK-Studio
1
1
Fork 0
mirror of https://github.com/nod-ai/SHARK-Studio.git synced 2026-01-09 22:07:55 -05:00
Code Issues Packages Projects Releases Wiki Activity

Add tokenizers to requirements.txt (#1790)

Browse Source 
* Add tokenizers to requirements and pin version

* Update process_skipfiles.py
This commit is contained in:
Ean Garvey
2023-08-24 19:44:04 -05:00
committed by GitHub
parent 07f6f4a2f7
commit 450c231171
2 changed files with 16 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files

21
process_skipfiles.py
View File

@@ -6,15 +6,24 @@ from distutils.sysconfig import get_python_lib
import fileinput
from pathlib import Path
# Temorary workaround for transformers/__init__.py.
path_to_tranformers_hook = Path(
get_python_lib()
+ "/_pyinstaller_hooks_contrib/hooks/stdhooks/hook-transformers.py"
# Temporary workaround for transformers/__init__.py.
path_to_stdhooks = Path(
get_python_lib() + "/_pyinstaller_hooks_contrib/hooks/stdhooks"
)
if path_to_tranformers_hook.is_file():
path_to_transformers_hook = Path(
str(path_to_stdhooks) + "hook-transformers.py"
)
if path_to_transformers_hook.is_file():
pass
else:
with open(path_to_tranformers_hook, "w") as f:
if not path_to_stdhooks.is_dir():
import os
print(
f"Path to pyinstaller stdhooks not found. Please check your pyinstaller packages at {path_to_stdhooks}."
)
os.mkdir(path_to_stdhooks)
with open(path_to_transformers_hook, "w") as f:
f.write("module_collection_mode = 'pyz+py'")
path_to_skipfiles = Path(get_python_lib() + "/torch/_dynamo/skipfiles.py")