ceos/unsloth
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Update tokenizer_utils.py

Browse source
This commit is contained in:
Daniel Han 2025-01-08 15:48:11 -08:00
commit 75b4e34ae4
1 changed files with 3 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

3
unsloth/tokenizer_utils.py
View file

@ -523,6 +523,9 @@ def _load_correct_tokenizer(
# Ignore Mistral ones - they're a bit weird to handle!
elif "mistral" in tokenizer_name.lower():
return fast_tokenizer
# Ignore Phi-4 ones as well
elif "phi-4" in tokenizer_name.lower():
return fast_tokenizer
elif slow_tokenizer is not None:
if hasattr(fast_tokenizer, "add_bos_token") and hasattr(slow_tokenizer, "add_bos_token"):
fast_tokenizer.add_bos_token = slow_tokenizer.add_bos_token