diff --git a/unsloth/chat_templates.py b/unsloth/chat_templates.py
index 9675b10fe4..520c998c5c 100644
--- a/unsloth/chat_templates.py
+++ b/unsloth/chat_templates.py
@@ -257,9 +257,9 @@ def get_chat_template(
         assert("Unsloth: Can only map new tokens to EOS for now. Adding new tokens is not yet supported.")
     pass
 
-    if tokenizer.__class__.__name__.startswith("Gemma") and chat_template == "chatml":
-        chat_template = "gemma_chatml"
-    pass
+    # if tokenizer.__class__.__name__.startswith("Gemma") and chat_template == "chatml":
+    #     chat_template = "gemma_chatml"
+    # pass
 
     old_padding_side = tokenizer.padding_side
 
@@ -298,8 +298,12 @@ def get_chat_template(
                 pass
             pass
 
-            logger.warning_once(f"Unsloth: Will map {stop_word} to EOS = {tokenizer.eos_token}.")
-            string_vocab = string_vocab.replace(tokenizer.eos_token, stop_word)
+            if not stop_word in token_mapping.values():
+                # Do not map 107 = <|im_end|> and 1 = <|im_end|>. This will reduce the vocab size by 1
+                logger.warning_once(f"Unsloth: Will map {stop_word} to EOS = {tokenizer.eos_token}.")
+                string_vocab = string_vocab.replace(tokenizer.eos_token, stop_word)
+            pass
+
             new_tokenizer = tokenizer._tokenizer.from_str(string_vocab)
             tokenizer = tokenizer.__class__(tokenizer_object = new_tokenizer, eos_token = stop_word)
 
diff --git a/unsloth/models/llama.py b/unsloth/models/llama.py
index 3f281a09c8..6ed52a7acb 100644
--- a/unsloth/models/llama.py
+++ b/unsloth/models/llama.py
@@ -916,6 +916,7 @@ class FastLlamaModel:
         rope_scaling   = None,
         fix_tokenizer  = True,
         model_patcher  = None,
+        tokenizer_name = None,
         **kwargs,
     ):
         if model_patcher is None: model_patcher = FastLlamaModel
@@ -978,13 +979,16 @@ class FastLlamaModel:
             max_position_embeddings = max_position_embeddings,
             **kwargs,
         )
+
+        # Counteract saved tokenizers
+        tokenizer_name = model_name if tokenizer_name is None else tokenizer_name
         tokenizer = AutoTokenizer.from_pretrained(
-            model_name,
+            tokenizer_name,
             model_max_length = max_position_embeddings,
             padding_side     = "right",
             token            = token,
         )
-
+        
         model, tokenizer = patch_tokenizer(model, tokenizer)
         model = model_patcher.post_patch(model)
 
diff --git a/unsloth/models/loader.py b/unsloth/models/loader.py
index 67a59c850c..47b568ae2a 100644
--- a/unsloth/models/loader.py
+++ b/unsloth/models/loader.py
@@ -18,7 +18,7 @@ from transformers import AutoConfig
 from transformers import __version__ as transformers_version
 from peft import PeftConfig, PeftModel
 from .mapper import INT_TO_FLOAT_MAPPER, FLOAT_TO_INT_MAPPER
-
+import os
 
 # https://github.com/huggingface/transformers/pull/26037 allows 4 bit loading!
 major, minor = transformers_version.split(".")[:2]
@@ -118,6 +118,16 @@ class FastLanguageModel(FastLlamaModel):
             )
         pass
 
+        # Check if this is local model since the tokenizer gets overwritten
+        if  os.path.exists(os.path.join(old_model_name, "tokenizer_config.json")) and \
+            os.path.exists(os.path.join(old_model_name, "tokenizer.json")) and \
+            os.path.exists(os.path.join(old_model_name, "special_tokens_map.json")):
+
+            tokenizer_name = old_model_name
+        else:
+            tokenizer_name = None
+        pass
+
         model, tokenizer = dispatch_model.from_pretrained(
             model_name     = model_name,
             max_seq_length = max_seq_length,
@@ -128,6 +138,7 @@ class FastLanguageModel(FastLlamaModel):
             rope_scaling   = rope_scaling,
             fix_tokenizer  = fix_tokenizer,
             model_patcher  = dispatch_model,
+            tokenizer_name = tokenizer_name,
             *args, **kwargs,
         )
 
diff --git a/unsloth/models/mistral.py b/unsloth/models/mistral.py
index 6c9d9ecc5c..c1e39e4a2e 100644
--- a/unsloth/models/mistral.py
+++ b/unsloth/models/mistral.py
@@ -294,6 +294,7 @@ class FastMistralModel(FastLlamaModel):
         rope_scaling   = None, # Mistral does not support RoPE scaling
         fix_tokenizer  = True,
         model_patcher  = None,
+        tokenizer_name = None,
         **kwargs,
     ):
         if model_patcher is None: model_patcher = FastMistralModel
@@ -354,8 +355,11 @@ class FastMistralModel(FastLlamaModel):
             # rope_scaling      = rope_scaling,
             **kwargs,
         )
+
+        # Counteract saved tokenizers
+        tokenizer_name = model_name if tokenizer_name is None else tokenizer_name
         tokenizer = AutoTokenizer.from_pretrained(
-            model_name,
+            tokenizer_name,
             model_max_length = max_position_embeddings,
             padding_side     = "right",
             token            = token,
diff --git a/unsloth/save.py b/unsloth/save.py
index 5c1bceb38f..5971d76e6e 100644
--- a/unsloth/save.py
+++ b/unsloth/save.py
@@ -91,11 +91,13 @@ def _merge_lora(layer, name):
         else:
             dtype = W.dtype
         W = W.to(torch.float32).t()
+        # W = W.t()
 
         if A is not None:
             # sAB = (A.t().to(torch.float32) @ (s * B.t().to(torch.float32)))
             # W += sAB
             W.addmm_(A.t().to(torch.float32), B.t().to(torch.float32), alpha = s)
+            # W.addmm_(A.t().to(W.dtype), B.t().to(W.dtype), alpha = s)
             # if not torch.isfinite(W).all():
             maximum_element = torch.max(W.min().abs(), W.max())
             if not torch.isfinite(maximum_element).item():