ceos/unsloth
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Update tokenizer_utils.py

Browse source
This commit is contained in:
Daniel Han-Chen 2024-04-19 03:03:05 +10:00
commit 20decf3ca1
1 changed files with 4 additions and 2 deletions
Download patch file Download diff file Expand all files Collapse all files

6
unsloth/tokenizer_utils.py
View file

@ -299,8 +299,10 @@ def load_correct_tokenizer(
)
if slow_tokenizer is not None:
fast_tokenizer.add_bos_token = slow_tokenizer.add_bos_token
fast_tokenizer.add_eos_token = slow_tokenizer.add_eos_token
if hasattr(fast_tokenizer "add_bos_token") and hasattr(slow_tokenizer "add_bos_token"):
fast_tokenizer.add_bos_token = slow_tokenizer.add_bos_token
if hasattr(fast_tokenizer "add_eos_token") and hasattr(slow_tokenizer "add_eos_token"):
fast_tokenizer.add_eos_token = slow_tokenizer.add_eos_token
# Confirm if slow and fast are equivalent!
if assert_same_tokenization(slow_tokenizer, fast_tokenizer):