diff --git a/unsloth/models/llama.py b/unsloth/models/llama.py
index a7cacea9b9..136ceb2c7b 100644
--- a/unsloth/models/llama.py
+++ b/unsloth/models/llama.py
@@ -1503,10 +1503,16 @@ class FastLlamaModel:
             pass
         pass
 
+        # Check for Llama-3
+        # if hasattr(model._saved_temp_tokenizer, "_using_llama3_template"):
+        #     if not train_embed_tokens and not train_lm_head:
+        #         raise RuntimeError("")
+
         # First fix untrained tokens
-        if train_embed_tokens or train_lm_head:
-            fix_untrained_tokens(model, eps = 1e-16)
-        pass
+        # Wrong - can cause reserved tokens to pop out!!
+        # if train_embed_tokens or train_lm_head:
+        #     fix_untrained_tokens(model, eps = 1e-16)
+        # pass
 
         # Check modules_to_save
         if modules_to_save is not None:
@@ -1547,7 +1553,7 @@ class FastLlamaModel:
 
         lora_config = LoraConfig(**arguments)
         model = _get_peft_model(model, lora_config)
-        
+
         model._saved_temp_tokenizer = _saved_temp_tokenizer
 
         model = FastLlamaModel.patch_peft_model(model, use_gradient_checkpointing)
diff --git a/unsloth/save.py b/unsloth/save.py
index 0e131fe309..e50f0d34da 100644
--- a/unsloth/save.py
+++ b/unsloth/save.py
@@ -118,14 +118,14 @@ def _merge_lora(layer, name):
             W = fast_dequantize(W, quant_state)
         else:
             dtype = W.dtype
-        # W = W.to(torch.float32).t()
-        W = W.t()
+        W = W.to(torch.float32).t()
+        # W = W.t()
 
         if A is not None:
             # sAB = (A.t().to(torch.float32) @ (s * B.t().to(torch.float32)))
             # W += sAB
-            # W.addmm_(A.t().to(torch.float32), B.t().to(torch.float32), alpha = s)
-            W.addmm_(A.t().to(W.dtype), B.t().to(W.dtype), alpha = s)
+            W.addmm_(A.t().to(torch.float32), B.t().to(torch.float32), alpha = s)
+            # W.addmm_(A.t().to(W.dtype), B.t().to(W.dtype), alpha = s)
             # if not torch.isfinite(W).all():
             maximum_element = torch.max(W.min().abs(), W.max())
             if not torch.isfinite(maximum_element).item():
@@ -696,12 +696,18 @@ pass
 
 
 def install_llama_cpp_make_non_blocking():
-    env = { **os.environ, "LLAMA_CUDA": "1", }
+    # https://github.com/ggerganov/llama.cpp/issues/7062
+    # Weirdly GPU conversion for GGUF breaks??
+    # env = { **os.environ, "LLAMA_CUDA": "1", }
     n_jobs = max(int(psutil.cpu_count()*1.5), 1)
     # Force make clean
     os.system("make clean -C llama.cpp")
     full_command = ["make", "all", "-j"+str(n_jobs), "-C", "llama.cpp"]
-    run_installer = subprocess.Popen(full_command, env = env, stdout = subprocess.DEVNULL, stderr = subprocess.STDOUT)
+
+    # https://github.com/ggerganov/llama.cpp/issues/7062
+    # Weirdly GPU conversion for GGUF breaks??
+    # run_installer = subprocess.Popen(full_command, env = env, stdout = subprocess.DEVNULL, stderr = subprocess.STDOUT)
+    run_installer = subprocess.Popen(full_command, stdout = subprocess.DEVNULL, stderr = subprocess.STDOUT)
     return run_installer
 pass
 
@@ -764,12 +770,17 @@ pass
 
 
 def install_llama_cpp_blocking(use_cuda = True):
-    use_cuda = "LLAMA_CUDA=1" if use_cuda else ""
+    # https://github.com/ggerganov/llama.cpp/issues/7062
+    # Weirdly GPU conversion for GGUF breaks??
+    # use_cuda = "LLAMA_CUDA=1" if use_cuda else ""
 
     commands = [
         "git clone --recursive https://github.com/ggerganov/llama.cpp",
         "make clean -C llama.cpp",
-        f"{use_cuda} make all -j{psutil.cpu_count()*2} -C llama.cpp",
+        # https://github.com/ggerganov/llama.cpp/issues/7062
+        # Weirdly GPU conversion for GGUF breaks??
+        # f"{use_cuda} make all -j{psutil.cpu_count()*2} -C llama.cpp",
+        f"make all -j{psutil.cpu_count()*2} -C llama.cpp",
         "pip install gguf protobuf",
     ]
     if os.path.exists("llama.cpp"): return
@@ -833,6 +844,12 @@ def save_to_gguf(
     first_conversion     : str = "f16",
     _run_installer = None, # Non blocking install of llama.cpp
 ):
+    logger.warning(
+        "WARNING: llama.cpp GGUF conversion is currently unstable, since llama.cpp is\n"\
+        "undergoing some major bug fixes as at 5th of May 2024. This is not an Unsloth issue.\n"\
+        "Please be patient - GGUF saving should still work, but might not work as well."
+    )
+
     from transformers.models.llama.modeling_llama import logger
 
     if quantization_method.startswith("iq2"):
@@ -967,7 +984,7 @@ def save_to_gguf(
                 "You do not need to close this Python program. Run the following commands in a new terminal:\n"\
                 "You must run this in the same folder as you're saving your model.\n"\
                 "git clone --recursive https://github.com/ggerganov/llama.cpp\n"\
-                "cd llama.cpp && make clean && LLAMA_CUDA=1 make all -j\n"\
+                "cd llama.cpp && make clean && make all -j\n"\
                 "Once that's done, redo the quantization."
             )
         pass
@@ -1007,7 +1024,7 @@ def save_to_gguf(
                     "You do not need to close this Python program. Run the following commands in a new terminal:\n"\
                     "You must run this in the same folder as you're saving your model.\n"\
                     "git clone --recursive https://github.com/ggerganov/llama.cpp\n"\
-                    "cd llama.cpp && make clean && LLAMA_CUDA=1 make all -j\n"\
+                    "cd llama.cpp && make clean && make all -j\n"\
                     "Once that's done, redo the quantization."
                 )
             pass
diff --git a/unsloth/tokenizer_utils.py b/unsloth/tokenizer_utils.py
index 5dc5856c2a..0d6dadf7dd 100644
--- a/unsloth/tokenizer_utils.py
+++ b/unsloth/tokenizer_utils.py
@@ -25,7 +25,6 @@ __all__ = [
     "load_correct_tokenizer",
     "fix_sentencepiece_tokenizer",
     "check_tokenizer",
-    "fix_untrained_tokens",
     "add_new_tokens",
 ]
 
@@ -518,6 +517,44 @@ def fix_untrained_tokens(model, eps = 1e-16):
 pass
 
 
+@torch.inference_mode
+def mean_of_trained_tokens(model, eps = 1e-16):
+    """
+    Llama-3 for eg has untrained vectors in the base model.
+    These include <|eot_id|>, <|start_header_id|>, <|end_header_id|>
+    We reset them to the mean of the rest of the tokens
+    """
+    embedding_matrix = model.get_input_embeddings ().weight.data.clone()
+    lm_head_matrix   = model.get_output_embeddings().weight.data.clone()
+
+    # Get untrained tokens
+    indicator_untrained = torch.amax(embedding_matrix, axis = 1) <= eps
+    where_untrained = torch.where(indicator_untrained)[0]
+    n_untrained = where_untrained.shape[0]
+    n_trained = embedding_matrix.shape[0] - n_untrained
+    if n_untrained != 0:
+        print(
+            f"Unsloth: Not an error, but your model has {n_untrained} untrained tokens.\n"\
+            "We shall set them to the mean of the other trained tokens."
+        )
+    pass
+
+    # First set untrained to all 0s - sometimes it's not! 1e-23 for bfloat16
+    embedding_matrix[where_untrained] = 0
+    lm_head_matrix  [where_untrained] = 0
+
+    # Find sum
+    sum_embedding  = torch.sum(embedding_matrix, dtype = torch.float32, axis = 0)
+    sum_lm_head    = torch.sum(lm_head_matrix,   dtype = torch.float32, axis = 0)
+
+    # Find correct average by dividing by sum of trained tokens
+    mean_embedding = (sum_embedding / n_trained).to(embedding_matrix.dtype)
+    mean_lm_head   = (sum_lm_head   / n_trained).to(lm_head_matrix  .dtype)
+
+    return mean_embedding, mean_lm_head
+pass
+
+
 @torch.inference_mode
 def add_new_tokens(
     model,
@@ -547,7 +584,10 @@ def add_new_tokens(
     pass
 
     # Get mean of trained tokens
-    mean_embedding, mean_lm_head = fix_untrained_tokens(model)
+    # mean_embedding, mean_lm_head = fix_untrained_tokens(model)
+
+    # Weirdly be careful reserved tokens can pop out
+    mean_embedding, mean_lm_head = mean_of_trained_tokens(model)
     mean_embedding = mean_embedding.to(torch.float32)
     mean_lm_head   = mean_lm_head  .to(torch.float32)
 
@@ -595,3 +635,43 @@ def add_new_tokens(
     
     return
 pass
+
+
+from inspect import getsource
+import trl.trainer.sft_trainer
+from trl.trainer.sft_trainer import *
+
+def fix_sft_trainer_tokenizer():
+    """
+        Fixes double adding BOS tokens like in llama-3
+    """
+    for function_name, replacer in (
+        ("_prepare_non_packed_dataloader", "def tokenize(element):",),
+        # ("_prepare_packed_dataloader", "if dataset_text_field is not None",),
+    ):
+        function = getsource(eval(f"trl.trainer.sft_trainer.SFTTrainer.{function_name}"))
+        where = function.find("def")
+        function = function.split("\n")
+        function = "\n".join(x[where:] for x in function)
+
+        check_text = \
+        "\n"\
+        "test_text = dataset[0][dataset_text_field] if (formatting_func is None or not use_formatting_func) else formatting_func(dataset[0])\n"\
+        "chat_template = getattr(tokenizer, 'chat_template', None)\n"\
+        "chat_template = '' if chat_template is None else chat_template\n"\
+        "has_bos_token_already = test_text.startswith(tokenizer.bos_token) or tokenizer.bos_token in chat_template\n"\
+        "add_special_tokens = False if has_bos_token_already else add_special_tokens\n\n"
+
+        check_text = check_text.split("\n")
+        check_text = "\n".join(" "*where + x for x in check_text)
+
+        function = function.replace(replacer, check_text + replacer)
+        exec(function, globals())
+
+        # Replace TRL's SFTTrainer
+        exec(f"trl.trainer.sft_trainer.SFTTrainer.{function_name} = {function_name}", globals())
+    pass
+pass
+
+# Fixes double adding BOS tokens like in llama-3
+fix_sft_trainer_tokenizer()