From 7c4c93813fa6a8549702d8ce8a37ce3efc70695c Mon Sep 17 00:00:00 2001
From: Daniel Han <danielhanchen@gmail.com>
Date: Thu, 18 Jul 2024 13:25:30 -0700
Subject: [PATCH] Update tokenizer_utils.py

---
 unsloth/tokenizer_utils.py | 13 ++++++++++++-
 1 file changed, 12 insertions(+), 1 deletion(-)

diff --git a/unsloth/tokenizer_utils.py b/unsloth/tokenizer_utils.py
index dc0c7da854..060c1ccae4 100644
--- a/unsloth/tokenizer_utils.py
+++ b/unsloth/tokenizer_utils.py
@@ -38,6 +38,17 @@ __all__ = [
 IGNORED_TOKENIZER_CHECKING = frozenset((
     "CodeLlamaTokenizerFast",
     "CodeLlamaTokenizer",
+    ""
+))
+
+
+IGNORED_TOKENIZER_NAMES = frozenset((
+    "unsloth/Mistral-Nemo-Instruct-2407-bnb-4bit",
+    "unsloth/Mistral-Nemo-Instruct-2407",
+    "mistralai/Mistral-Nemo-Instruct-2407",
+    "unsloth/Mistral-Nemo-Base-2407-bnb-4bit",
+    "unsloth/Mistral-Nemo-Base-2407",
+    "mistralai/Mistral-Nemo-Base-2407",
 ))
 
 # Check environments
@@ -488,7 +499,7 @@ def load_correct_tokenizer(
         cache_dir         = cache_dir,
     )
 
-    if slow_tokenizer is not None:
+    if tokenizer_name not in IGNORED_TOKENIZER_NAMES and slow_tokenizer is not None:
         if hasattr(fast_tokenizer, "add_bos_token") and hasattr(slow_tokenizer, "add_bos_token"):
             fast_tokenizer.add_bos_token = slow_tokenizer.add_bos_token
         if hasattr(fast_tokenizer, "add_eos_token") and hasattr(slow_tokenizer, "add_eos_token"):