Fix saving issues (#139)

* faster saving & inference

* Update llama.py

* Update save.py

* Update llama.py

* Update llama.py

* Update llama.py

* Update llama.py

* Update mistral.py

* Update llama.py

* Update llama.py

* Update llama.py

* Update llama.py

* Update llama.py

* fast inference

* Update llama.py

* Update save.py

* Update llama.py

* Mistral correct RoPE scaling

* Max sequence lengths

* Apache 2

* fast_linear_forward

* Update utils.py

* Update utils.py

* No print

* Update utils.py

* Update utils.py

* inference

* Update llama.py

* Fast inference RoPE

* Update llama.py

* Update llama.py

* RoPE

* Update llama.py

* Update llama.py

* Update llama.py

* Update llama.py

* Update llama.py

* Update llama.py

* Update llama.py

* Update llama.py

* LoRA

* Fast LoRA saving

* Update llama.py

* hidden_states

* q_len == 1

* q_len issue

* Update mistral.py

* Update mistral.py

* incorrect inference

* Update to transformers 4.37

* Graceful FA2 error + torch 2.1.1

* Update mapper.py

* Update pyproject.toml

* Fix saving and bnb-4bit

* Update fast_lora.py

* Update fast_lora.py

* remove patching

* Update llama.py

* Update llama.py

* Update swiglu.py

* Repatch

* Update fast_lora.py

* Update fast_lora.py

* Update fast_lora.py

* Update fast_lora.py

* Update fast_lora.py

* Update fast_lora.py

* Update fast_lora.py

* Update fast_lora.py

* Update fast_lora.py

* Update fast_lora.py

* Update llama.py

* Update fast_lora.py

* Update llama.py

* Update fast_lora.py

* Update fast_lora.py

* Update fast_lora.py

* Update fast_lora.py

* Update fast_lora.py

* Update fast_lora.py

* Update fast_lora.py

* Update fast_lora.py

* Update fast_lora.py

* Update swiglu.py

* Update fast_lora.py

* Update swiglu.py

* Update fast_lora.py

* Update fast_lora.py

* Update fast_lora.py

* Update fast_lora.py

* Update fast_lora.py

* Update fast_lora.py

* Update fast_lora.py

* Update fast_lora.py

* Update fast_lora.py

* Update fast_lora.py

* Update fast_lora.py

* Update fast_lora.py

* Update fast_lora.py

* Update fast_lora.py

* Update fast_lora.py

* Update fast_lora.py

* Update save.py

* Update fast_lora.py

* Update utils.py

* Update llama.py

* Update fast_lora.py

* Update swiglu.py

* Update save.py

* Update save.py

* Update llama.py

* Update llama.py

* Update llama.py

* Update llama.py

* Revert "Update llama.py"

This reverts commit a208ec46e0.

* Update llama.py

* Works?

* Update pyproject.toml

* Update fast_lora.py

* Update fast_lora.py

* Update fast_lora.py

* Update fast_lora.py

* Update fast_lora.py

* Swiglu

* Update swiglu.py

* Update fast_lora.py

* Update fast_lora.py

* Update fast_lora.py

* Update fast_lora.py

* Update swiglu.py

* Update fast_lora.py

* Update fast_lora.py

* Update fast_lora.py

* Update fast_lora.py

* Update fast_lora.py

* Update fast_lora.py

* Update fast_lora.py

* Update fast_lora.py

* attention_mask

* Update llama.py

* Update llama.py

* labels

* Update mistral.py

* Update llama.py

* attention mask

* Update save.py

* Update save.py

* Update mistral.py

* attention mask

* Update llama.py

* Update llama.py

* Update mistral.py

* Update llama.py

* Update llama.py

* Update llama.py

* Update dpo.py

* Patch saving

* Update save.py

* Update save.py

* patch_saving_functions

* Update save.py

* Update save.py

* Update save.py

* Update save.py

* Update save.py

* Update save.py

* Update save.py

* Update save.py

* print

unsloth/models/dpo.py

@@ -101,10 +101,13 @@ pass
 
 
 def PatchDPOTrainer():
-    # Patch DPO notebook printing
+    from transformers.trainer import is_in_notebook
-    NotebookTrainingTracker.write_line = NotebookTrainingTracker_write_line
+    if is_in_notebook():
-    from transformers.trainer import DEFAULT_PROGRESS_CALLBACK
+        # Patch DPO notebook printing
-    DEFAULT_PROGRESS_CALLBACK.on_train_begin = NotebookProgressCallback_on_train_begin
+        NotebookTrainingTracker.write_line = NotebookTrainingTracker_write_line
-    DEFAULT_PROGRESS_CALLBACK.on_log         = NotebookProgressCallback_on_log
+        from transformers.trainer import DEFAULT_PROGRESS_CALLBACK
+        DEFAULT_PROGRESS_CALLBACK.on_train_begin = NotebookProgressCallback_on_train_begin
+        DEFAULT_PROGRESS_CALLBACK.on_log         = NotebookProgressCallback_on_log
+    pass
 pass
 

unsloth/models/llama.py

@@ -486,6 +486,15 @@ def LlamaModel_fast_forward(
     if inputs_embeds is None:
         inputs_embeds = self.embed_tokens(input_ids)
 
+    # Fix up attention mask by setting elements to 0
+    # Specifically for DPO
+    if self._has_no_labels and attention_mask is not None:
+        inputs_requires_grad = inputs_embeds.requires_grad
+        if inputs_requires_grad: inputs_embeds.requires_grad_(False)
+        inputs_embeds *= attention_mask.unsqueeze(0).transpose(0, 1).transpose(1, 2)
+        if inputs_requires_grad: inputs_embeds.requires_grad_(True)
+    pass
+
     # Ignore attention_mask
     if attention_mask is None:
         padding_mask = None
@@ -617,6 +626,7 @@ def LlamaForCausalLM_fast_forward(
     return_dict = return_dict if return_dict is not None else self.config.use_return_dict
 
     # decoder outputs consists of (dec_features, layer_state, dec_hidden, dec_attn)
+    self.model._has_no_labels = labels is None
     outputs = self.model(
         input_ids=input_ids,
         causal_mask=causal_mask,
@@ -726,7 +736,7 @@ class FastLlamaModel:
            f"O^O/ \_/ \\    Pytorch: {torch.__version__}. CUDA = {gpu_stats.major}.{gpu_stats.minor}. CUDA Toolkit = {torch.version.cuda}.\n"\
            f"\        /    Bfloat16 = {str(SUPPORTS_BFLOAT16).upper()}. Xformers = {xformers_version}. FA = {HAS_FLASH_ATTENTION}.\n"\
            f' "-____-"     Free Apache license: http://github.com/unslothai/unsloth'
-        logger.warning_once(statistics)
+        print(statistics)
         FastLlamaModel.pre_patch()
 
         if dtype is None:
@@ -826,6 +836,9 @@ class FastLlamaModel:
         # Log Unsloth version for future fastpaths for inference
         model.config.update({"unsloth_version" : __version__})
 
+        # Add save modules
+        patch_saving_functions(model)
+
         return model, tokenizer
     pass
 

unsloth/models/mistral.py

@@ -195,6 +195,7 @@ def MistralForCausalLM_fast_forward(
     return_dict = return_dict if return_dict is not None else self.config.use_return_dict
 
     # decoder outputs consists of (dec_features, layer_state, dec_hidden, dec_attn)
+    self.model._has_no_labels = labels is None
     outputs = self.model(
         input_ids=input_ids,
         causal_mask=causal_mask,
@@ -282,7 +283,7 @@ class FastMistralModel(FastLlamaModel):
            f"O^O/ \_/ \\    Pytorch: {torch.__version__}. CUDA = {gpu_stats.major}.{gpu_stats.minor}. CUDA Toolkit = {torch.version.cuda}.\n"\
            f"\        /    Bfloat16 = {str(SUPPORTS_BFLOAT16).upper()}. Xformers = {xformers_version}. FA = {HAS_FLASH_ATTENTION}.\n"\
            f' "-____-"     Apache 2 free license: http://github.com/unslothai/unsloth'
-        logger.warning_once(statistics)
+        print(statistics)
         FastMistralModel.pre_patch()
 
         if dtype is None:

unsloth/save.py

@@ -278,7 +278,7 @@ def unsloth_save_model(
         not hasattr(internal_model.model, "layers")
     ):
         # Do general saving
-        
+        print(type(model))
         # Edit save_pretrained_settings
         # [TODO] _create_repo has errors due to **kwargs getting accepted
         for deletion in \
@@ -483,7 +483,7 @@ def install_llama_cpp_make_non_blocking():
     n_jobs = max(int(psutil.cpu_count()*1.5), 1)
     # Force make clean
     os.system("make clean -C llama.cpp")
-    full_command = ["make", "-j", str(n_jobs), "-C", "llama.cpp"]
+    full_command = ["make", "all", "-j", str(n_jobs), "-C", "llama.cpp"]
     run_installer = subprocess.Popen(full_command, env = env, stdout = subprocess.DEVNULL, stderr = subprocess.STDOUT)
     return run_installer
 pass
@@ -499,7 +499,7 @@ pass
 def install_llama_cpp_blocking():
     commands = [
         "git clone https://github.com/ggerganov/llama.cpp",
-        f"cd llama.cpp && make clean && LLAMA_CUBLAS=1 make -j {psutil.cpu_count()*2}",
+        f"cd llama.cpp && make clean && LLAMA_CUBLAS=1 make all -j {psutil.cpu_count()*2}",
         "pip install gguf protobuf",
     ]
     if os.path.exists("llama.cpp"): return
@@ -515,6 +515,7 @@ pass
 def save_to_gguf(
     model_directory      : str = "unsloth_finetuned_model",
     quantization_method  : str = "fast_quantized",
+    first_conversion     : str = "f16",
     _run_installer = None, # Non blocking install of llama.cpp
 ):
     from transformers.models.llama.modeling_llama import logger
@@ -539,6 +540,16 @@ def save_to_gguf(
         f' "-____-"     In total, you will have to wait around 26 minutes.\n'
     print(print_info)
 
+    # Check first_conversion format
+    if   first_conversion == "f16" : pass
+    elif first_conversion == "f32" : pass
+    elif first_conversion == "q8_0": pass
+    else:
+        raise RuntimeError(
+            f"Unsloth: `first_conversion` can only be one of ['f16', 'f32', 'q8_0'] and not `{first_conversion}`."
+        )
+    pass
+
     print("Unsloth: [0] Installing llama.cpp. This will take 3 minutes...")
     if _run_installer is not None:
         _run_installer.wait()
@@ -546,11 +557,19 @@ def save_to_gguf(
         install_llama_cpp_blocking()
     pass
 
-    print("Unsloth: [1] Converting HF into GGUF format. This will take 3 minutes...")
-    first_conversion = "f16"
     if   quantization_method == "f32":  first_conversion = "f32"
     elif quantization_method == "f16":  first_conversion = "f16"
     elif quantization_method == "q8_0": first_conversion = "q8_0"
+    else:
+        # Quantized models must have f16 as the default argument
+        if   first_conversion == "f32" : pass
+        elif first_conversion == "f16" : pass
+        elif first_conversion == "q8_0":
+            logger.warning_once("Unsloth: We must use f16 for quantization first.")
+            first_conversion = "f16"
+        pass
+    pass
+    print(f"Unsloth: [1] Converting HF into {first_conversion} GGUF format. This will take 3 minutes...")
 
     n_cpus = psutil.cpu_count()*2
     # Concurrency from https://rentry.org/llama-cpp-conversions#merging-loras-into-a-model
@@ -566,6 +585,17 @@ def save_to_gguf(
             print(line.decode("utf-8"), flush = True, end = "")
     pass
 
+    # Check if quantization succeeded!
+    if not os.path.isfile(final_location):
+        raise RuntimeError(
+            "Unsloth: Quantization failed! You might have to compile llama.cpp yourself, then run this again.\n"\
+            "You do not need to close this Python program. Run the following commands in a new terminal:\n"\
+            "You must run this in the same folder as you're saving your model.\n"\
+            "git clone https://github.com/ggerganov/llama.cpp\n"\
+            "cd llama.cpp && make clean && LLAMA_CUBLAS=1 make all -j\n"\
+            "Once that's done, redo the quantization."
+        )
+    pass
     print(f"Unsloth: Conversion completed! Output location: {final_location}")
 
     if quantization_method != first_conversion:
@@ -581,6 +611,19 @@ def save_to_gguf(
             for line in sp.stderr:
                 print(line.decode("utf-8"), flush = True, end = "")
         pass
+
+        # Check if quantization succeeded!
+        if not os.path.isfile(final_location):
+            raise RuntimeError(
+                "Unsloth: Quantization failed! You might have to compile llama.cpp yourself, then run this again.\n"\
+                "You do not need to close this Python program. Run the following commands in a new terminal:\n"\
+                "You must run this in the same folder as you're saving your model.\n"\
+                "git clone https://github.com/ggerganov/llama.cpp\n"\
+                "cd llama.cpp && make clean && LLAMA_CUBLAS=1 make all -j\n"\
+                "Once that's done, redo the quantization."
+            )
+        pass
+
         print(f"Unsloth: Conversion completed! Output location: {final_location}")
     pass
 
@@ -765,6 +808,7 @@ def unsloth_save_pretrained_gguf(
     save_directory       : Union[str, os.PathLike],
     tokenizer            = None,
     quantization_method  : str = "fast_quantized",
+    first_conversion     : str = "f16",
     push_to_hub          : bool = False,
     token                : Optional[Union[str, bool]] = None,
     is_main_process      : bool = True,
@@ -813,6 +857,7 @@ def unsloth_save_pretrained_gguf(
     arguments["save_method"] = "merged_16bit" # Must be 16bit
     del arguments["self"]
     del arguments["quantization_method"]
+    del arguments["first_conversion"]
 
     # Non blocking install GGUF first
     if not os.path.exists("llama.cpp"):
@@ -840,7 +885,7 @@ def unsloth_save_pretrained_gguf(
     for _ in range(3):
         gc.collect()
 
-    file_location = save_to_gguf(new_save_directory, quantization_method, makefile)
+    file_location = save_to_gguf(new_save_directory, quantization_method, first_conversion, makefile)
 
     if push_to_hub:
         print("Unsloth: Uploading GGUF to Huggingface Hub...")
@@ -861,6 +906,7 @@ def unsloth_push_to_hub_gguf(
     repo_id              : str,
     tokenizer            = None,
     quantization_method  : str = "fast_quantized",
+    first_conversion     : str = "f16",
     use_temp_dir         : Optional[bool] = None,
     commit_message       : Optional[str] = None,
     private              : Optional[bool] = None,
@@ -911,6 +957,7 @@ def unsloth_push_to_hub_gguf(
     del arguments["self"]
     del arguments["repo_id"]
     del arguments["quantization_method"]
+    del arguments["first_conversion"]
 
     # Non blocking install GGUF first
     if not os.path.exists("llama.cpp"):
@@ -938,7 +985,7 @@ def unsloth_push_to_hub_gguf(
     for _ in range(3):
         gc.collect()
 
-    file_location = save_to_gguf(new_save_directory, quantization_method, makefile)
+    file_location = save_to_gguf(new_save_directory, quantization_method, first_conversion, makefile)
 
     print("Unsloth: Uploading GGUF to Huggingface Hub...")
     username = upload_to_huggingface(
@@ -960,6 +1007,23 @@ def patch_saving_functions(model):
 
     if hasattr(model, "_original_push_to_hub"): return
 
+    # First check if this has already been called, and revert it
+    original_model = model
+    while True:
+        if hasattr(original_model, "_original_push_to_hub"):
+            original_model.push_to_hub = original_model._original_push_to_hub
+            del original_model._original_push_to_hub
+            if hasattr(original_model, "push_to_hub_merged"):     del original_model.push_to_hub_merged
+            if hasattr(original_model, "save_pretrained_merged"): del original_model.save_pretrained_merged
+            if hasattr(original_model, "push_to_hub_gguf"):       del original_model.push_to_hub_gguf
+            if hasattr(original_model, "save_pretrained_gguf"):   del original_model.save_pretrained_gguf
+        pass
+
+        if hasattr(original_model, "model"): original_model = original_model.model
+        else: break
+    pass
+
+    # And now re add our saving methods!
     original_push_to_hub = model.push_to_hub
     signature = str(inspect.signature(original_push_to_hub)).replace("NoneType", "None")
     signature = signature[1:]
@@ -988,49 +1052,29 @@ def patch_saving_functions(model):
     pass
     '''
     exec(push_to_hub_text, globals())
-    model.push_to_hub = types.MethodType(unsloth_push_to_hub, model)
 
-    if hasattr(model, "add_model_tags"):
+    original_model = model
-        model.add_model_tags(["unsloth",])
+    while True:
 
+        if not hasattr(original_model, "_original_push_to_hub"):
+            original_model._original_push_to_hub = original_model.push_to_hub
+            original_model.push_to_hub = types.MethodType(unsloth_push_to_hub, original_model)
+
+            if hasattr(original_model, "add_model_tags"):
+                original_model.add_model_tags(["unsloth",])
+        pass
+
+        if hasattr(original_model, "model"): original_model = original_model.model
+        else: break
+    pass
+
+    # Add saving methods to top level model
     if hasattr(model, "config"):
         # Counteract tokenizers
         model.push_to_hub_merged     = types.MethodType(unsloth_push_to_hub_merged,     model)
         model.save_pretrained_merged = types.MethodType(unsloth_save_pretrained_merged, model)
         model.push_to_hub_gguf       = types.MethodType(unsloth_push_to_hub_gguf,       model)
         model.save_pretrained_gguf   = types.MethodType(unsloth_save_pretrained_gguf,   model)
-    else:
-        model.push_to_hub_merged     = model.push_to_hub
-        model.save_pretrained_merged = model.save_pretrained
-        model.push_to_hub_gguf       = model.push_to_hub
-        model.save_pretrained_gguf   = model.save_pretrained
     pass
-
+    return model
-    original_model = model
-    while hasattr(original_model, "model"):
-        original_model = original_model.model
-        if hasattr(original_model, "_original_push_to_hub"): continue
-        
-        original_model._original_push_to_hub = original_model.push_to_hub
-        original_model.push_to_hub = types.MethodType(unsloth_push_to_hub, original_model)
-
-        if hasattr(original_model, "add_model_tags"):
-            original_model.add_model_tags(["unsloth",])
-
-        if hasattr(original_model, "config"):
-            # Counteract tokenizers
-            original_model.push_to_hub_merged     = \
-                types.MethodType(unsloth_push_to_hub_merged,     original_model)
-
-            original_model.save_pretrained_merged = \
-                types.MethodType(unsloth_save_pretrained_merged, original_model)
-
-            original_model.push_to_hub_gguf       = \
-                types.MethodType(unsloth_push_to_hub_gguf,       original_model)
-
-            original_model.save_pretrained_gguf   = \
-                types.MethodType(unsloth_save_pretrained_gguf,   original_model)
-        pass
-    pass
-    return
 pass