ceos/unsloth
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Update tokenizer_utils.py

Browse source
This commit is contained in:
Daniel Han 2024-07-19 01:03:51 -07:00
commit 9b464716c7
1 changed files with 2 additions and 1 deletions
Download patch file Download diff file Expand all files Collapse all files

3
unsloth/tokenizer_utils.py
View file

@ -498,7 +498,8 @@ def load_correct_tokenizer(
cache_dir = cache_dir,
)
if tokenizer_name in IGNORED_TOKENIZER_NAMES: pass
if tokenizer_name in IGNORED_TOKENIZER_NAMES:
return fast_tokenizer
elif slow_tokenizer is not None:
if hasattr(fast_tokenizer, "add_bos_token") and hasattr(slow_tokenizer, "add_bos_token"):
fast_tokenizer.add_bos_token = slow_tokenizer.add_bos_token