From 684eaae9ea354dc58ff983c6e801ee8c34641581 Mon Sep 17 00:00:00 2001 From: Daniel Han-Chen Date: Mon, 11 Mar 2024 19:52:18 +1100 Subject: [PATCH] GGUF incorrect --- unsloth/models/llama.py | 28 ++++++++++++++-------------- unsloth/save.py | 33 ++++++++++++++++++++++++++++----- 2 files changed, 42 insertions(+), 19 deletions(-) diff --git a/unsloth/models/llama.py b/unsloth/models/llama.py index e9493376be..eb66b17d4a 100644 --- a/unsloth/models/llama.py +++ b/unsloth/models/llama.py @@ -1477,12 +1477,12 @@ class FastLlamaModel: if hasattr(gate_proj, "lora_A") and \ hasattr( up_proj, "lora_A") and \ hasattr(down_proj, "lora_A") and \ - ((gate_proj.base_layer if hasattr(gate_proj, "base_layer") else gate_proj).bias is None) and \ - (( up_proj.base_layer if hasattr( up_proj, "base_layer") else up_proj).bias is None) and \ - ((down_proj.base_layer if hasattr(down_proj, "base_layer") else down_proj).bias is None) and \ - ((gate_proj.lora_magnitude_vector if hasattr(gate_proj, "lora_magnitude_vector") else None) is None) and \ - (( up_proj.lora_magnitude_vector if hasattr( up_proj, "lora_magnitude_vector") else None) is None) and \ - ((down_proj.lora_magnitude_vector if hasattr(down_proj, "lora_magnitude_vector") else None) is None): + (getattr(gate_proj, "base_layer", gate_proj).bias is None) and \ + (getattr( up_proj, "base_layer", up_proj).bias is None) and \ + (getattr(down_proj, "base_layer", down_proj).bias is None) and \ + (getattr(gate_proj, "lora_magnitude_vector", None) is None) and \ + (getattr( up_proj, "lora_magnitude_vector", None) is None) and \ + (getattr(down_proj, "lora_magnitude_vector", None) is None): # https://stackoverflow.com/questions/50599045/python-replacing-a-function-within-a-class-of-a-module layer.mlp.forward = types.MethodType(apply_lora_mlp, layer.mlp) @@ -1501,12 +1501,12 @@ class FastLlamaModel: if hasattr(q_proj, "lora_A") and \ hasattr(k_proj, "lora_A") and \ hasattr(v_proj, "lora_A") and \ - ((q_proj.base_layer if hasattr(q_proj, "base_layer") else q_proj).bias is None) and \ - ((k_proj.base_layer if hasattr(k_proj, "base_layer") else k_proj).bias is None) and \ - ((v_proj.base_layer if hasattr(v_proj, "base_layer") else v_proj).bias is None) and \ - ((q_proj.lora_magnitude_vector if hasattr(q_proj, "lora_magnitude_vector") else None) is None) and \ - ((k_proj.lora_magnitude_vector if hasattr(k_proj, "lora_magnitude_vector") else None) is None) and \ - ((v_proj.lora_magnitude_vector if hasattr(v_proj, "lora_magnitude_vector") else None) is None): + (getattr(q_proj, "base_layer", q_proj).bias is None) and \ + (getattr(q_proj, "base_layer", k_proj).bias is None) and \ + (getattr(q_proj, "base_layer", v_proj).bias is None) and \ + (getattr(q_proj, "lora_magnitude_vector", None) is None) and \ + (getattr(k_proj, "lora_magnitude_vector", None) is None) and \ + (getattr(v_proj, "lora_magnitude_vector", None) is None): layer.self_attn.apply_qkv = apply_lora_qkv n_qkv += 1 @@ -1520,8 +1520,8 @@ class FastLlamaModel: # O attention patching o_proj = layer.self_attn.o_proj if hasattr(o_proj, "lora_A") and \ - ((o_proj.base_layer if hasattr(o_proj, "base_layer") else o_proj).bias is None) and \ - ((o_proj.lora_magnitude_vector if hasattr(o_proj, "lora_magnitude_vector") else None) is None): + (getattr(o_proj, "base_layer", o_proj).bias is None) and \ + (getattr(o_proj, "lora_magnitude_vector", None) is None): layer.self_attn.apply_o = apply_lora_o n_o += 1 diff --git a/unsloth/save.py b/unsloth/save.py index ef8718cd1e..0081da6a21 100644 --- a/unsloth/save.py +++ b/unsloth/save.py @@ -632,6 +632,7 @@ pass def save_to_gguf( + model_type : str, model_directory : str = "unsloth_finetuned_model", quantization_method : str = "fast_quantized", first_conversion : str = "f16", @@ -639,10 +640,18 @@ def save_to_gguf( ): from transformers.models.llama.modeling_llama import logger + # Careful convert.py is only for Llama / Mistral based archs + use_fast_convert = False + if model_type == "llama": use_fast_convert = True + elif model_type == "mistral": use_fast_convert = True + pass + logger.warning_once("Unsloth: Converting {model_type} model. Can use fast conversion = {use_fast_convert}.") + if quantization_method == "not_quantized": quantization_method = "f16" elif quantization_method == "fast_quantized": quantization_method = "q8_0" elif quantization_method == "quantized": quantization_method = "q4_k_m" elif quantization_method is None: quantization_method = "q8_0" + pass if quantization_method not in ALLOWED_QUANTS.keys(): error = f"Unsloth: Quant method = [{quantization_method}] not supported. Choose from below:\n" @@ -692,6 +701,12 @@ def save_to_gguf( pass pass + # Non llama/mistral needs can only use f32 or f16 + if not use_fast_convert and (first_conversion != "f16" or first_conversion != "f32"): + logger.warning_once("Unsloth: We must use f16 for non Llama and Mistral models.") + first_conversion = "f16" + pass + n_cpus = psutil.cpu_count() if n_cpus is None: n_cpus = 1 n_cpus *= 2 @@ -703,9 +718,15 @@ def save_to_gguf( f"The output location will be {final_location}\n"\ "This will take 3 minutes...") - command = f"python llama.cpp/convert-hf-to-gguf.py {model_directory} "\ - f"--outfile {final_location} --vocab-type hfft "\ - f"--outtype {first_conversion} --concurrency {n_cpus}" + if use_convert: + command = f"python llama.cpp/convert.py {model_directory} "\ + f"--outfile {final_location} --vocab-type hfft "\ + f"--outtype {first_conversion} --concurrency {n_cpus}" + else: + command = f"python llama.cpp/convert-hf-to-gguf.py {model_directory} "\ + f"--outfile {final_location} "\ + f"--outtype {first_conversion}" + pass with subprocess.Popen(command, shell = True, stdout = subprocess.PIPE, stderr = subprocess.PIPE, bufsize = 1) as sp: for line in sp.stdout: @@ -1054,7 +1075,8 @@ def unsloth_save_pretrained_gguf( for _ in range(3): gc.collect() - file_location = save_to_gguf(new_save_directory, quantization_method, first_conversion, makefile) + model_type = self.config.model_type + file_location = save_to_gguf(model_type, new_save_directory, quantization_method, first_conversion, makefile) if push_to_hub: print("Unsloth: Uploading GGUF to Huggingface Hub...") @@ -1154,7 +1176,8 @@ def unsloth_push_to_hub_gguf( for _ in range(3): gc.collect() - file_location = save_to_gguf(new_save_directory, quantization_method, first_conversion, makefile) + model_type = self.config.model_type + file_location = save_to_gguf(model_type, new_save_directory, quantization_method, first_conversion, makefile) print("Unsloth: Uploading GGUF to Huggingface Hub...") username = upload_to_huggingface(