ceos/unsloth
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Update tokenizer_utils.py

Browse source
This commit is contained in:
Daniel Han-Chen 2024-03-24 23:05:02 +11:00
commit e08784b4c9
1 changed files with 15 additions and 9 deletions
Download patch file Download diff file Expand all files Collapse all files

24
unsloth/tokenizer_utils.py
View file

@ -184,6 +184,9 @@ def assert_same_tokenization(slow_tokenizer, fast_tokenizer):
pass
global sentencepiece_model_pb2
sentencepiece_model_pb2 = None
def fix_sentencepiece_tokenizer(
old_tokenizer,
new_tokenizer,
@ -192,16 +195,19 @@ def fix_sentencepiece_tokenizer(
):
# From https://github.com/google/sentencepiece/issues/121
# We need to manually edit the sentencepiece tokenizer!
try:
import sentencepiece.sentencepiece_model_pb2 as sentencepiece_model_pb2
except:
if not os.path.exists(temporary_location):
os.system(f"git clone https://github.com/google/sentencepiece.git {temporary_location}")
os.system(f"cd {temporary_location}/src && protoc --python_out=. sentencepiece_model.proto")
shutil.rmtree(temporary_location)
global sentencepiece_model_pb2
if sentencepiece_model_pb2 is None:
try:
import sentencepiece.sentencepiece_model_pb2 as _sentencepiece_model_pb2
sentencepiece_model_pb2 = _sentencepiece_model_pb2
except:
if not os.path.exists(temporary_location):
os.system(f"git clone https://github.com/google/sentencepiece.git {temporary_location}")
os.system(f"cd {temporary_location}/src && protoc --python_out=. sentencepiece_model.proto")
pass
import sentencepiece.sentencepiece_model_pb2 as _sentencepiece_model_pb2
sentencepiece_model_pb2 = _sentencepiece_model_pb2
pass
import sentencepiece.sentencepiece_model_pb2 as sentencepiece_model_pb2
pass
if not os.path.exists(temporary_location):
os.makedirs(temporary_location)