From 75b4e34ae463050ee20ea92926a9abd2bc9c022f Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Wed, 8 Jan 2025 15:48:11 -0800 Subject: [PATCH] Update tokenizer_utils.py --- unsloth/tokenizer_utils.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/unsloth/tokenizer_utils.py b/unsloth/tokenizer_utils.py index 67a4c663b5..f2b0da8600 100644 --- a/unsloth/tokenizer_utils.py +++ b/unsloth/tokenizer_utils.py @@ -523,6 +523,9 @@ def _load_correct_tokenizer( # Ignore Mistral ones - they're a bit weird to handle! elif "mistral" in tokenizer_name.lower(): return fast_tokenizer + # Ignore Phi-4 ones as well + elif "phi-4" in tokenizer_name.lower(): + return fast_tokenizer elif slow_tokenizer is not None: if hasattr(fast_tokenizer, "add_bos_token") and hasattr(slow_tokenizer, "add_bos_token"): fast_tokenizer.add_bos_token = slow_tokenizer.add_bos_token