GGUF incorrect
This commit is contained in:
Daniel Han-Chen
parent
029d588603
commit
684eaae9ea
2 changed files with 42 additions and 19 deletions
|
|
@ -1477,12 +1477,12 @@ class FastLlamaModel:
|
|||
if hasattr(gate_proj, "lora_A") and \
|
||||
hasattr( up_proj, "lora_A") and \
|
||||
hasattr(down_proj, "lora_A") and \
|
||||
((gate_proj.base_layer if hasattr(gate_proj, "base_layer") else gate_proj).bias is None) and \
|
||||
(( up_proj.base_layer if hasattr( up_proj, "base_layer") else up_proj).bias is None) and \
|
||||
((down_proj.base_layer if hasattr(down_proj, "base_layer") else down_proj).bias is None) and \
|
||||
((gate_proj.lora_magnitude_vector if hasattr(gate_proj, "lora_magnitude_vector") else None) is None) and \
|
||||
(( up_proj.lora_magnitude_vector if hasattr( up_proj, "lora_magnitude_vector") else None) is None) and \
|
||||
((down_proj.lora_magnitude_vector if hasattr(down_proj, "lora_magnitude_vector") else None) is None):
|
||||
(getattr(gate_proj, "base_layer", gate_proj).bias is None) and \
|
||||
(getattr( up_proj, "base_layer", up_proj).bias is None) and \
|
||||
(getattr(down_proj, "base_layer", down_proj).bias is None) and \
|
||||
(getattr(gate_proj, "lora_magnitude_vector", None) is None) and \
|
||||
(getattr( up_proj, "lora_magnitude_vector", None) is None) and \
|
||||
(getattr(down_proj, "lora_magnitude_vector", None) is None):
|
||||
|
||||
# https://stackoverflow.com/questions/50599045/python-replacing-a-function-within-a-class-of-a-module
|
||||
layer.mlp.forward = types.MethodType(apply_lora_mlp, layer.mlp)
|
||||
|
|
@ -1501,12 +1501,12 @@ class FastLlamaModel:
|
|||
if hasattr(q_proj, "lora_A") and \
|
||||
hasattr(k_proj, "lora_A") and \
|
||||
hasattr(v_proj, "lora_A") and \
|
||||
((q_proj.base_layer if hasattr(q_proj, "base_layer") else q_proj).bias is None) and \
|
||||
((k_proj.base_layer if hasattr(k_proj, "base_layer") else k_proj).bias is None) and \
|
||||
((v_proj.base_layer if hasattr(v_proj, "base_layer") else v_proj).bias is None) and \
|
||||
((q_proj.lora_magnitude_vector if hasattr(q_proj, "lora_magnitude_vector") else None) is None) and \
|
||||
((k_proj.lora_magnitude_vector if hasattr(k_proj, "lora_magnitude_vector") else None) is None) and \
|
||||
((v_proj.lora_magnitude_vector if hasattr(v_proj, "lora_magnitude_vector") else None) is None):
|
||||
(getattr(q_proj, "base_layer", q_proj).bias is None) and \
|
||||
(getattr(q_proj, "base_layer", k_proj).bias is None) and \
|
||||
(getattr(q_proj, "base_layer", v_proj).bias is None) and \
|
||||
(getattr(q_proj, "lora_magnitude_vector", None) is None) and \
|
||||
(getattr(k_proj, "lora_magnitude_vector", None) is None) and \
|
||||
(getattr(v_proj, "lora_magnitude_vector", None) is None):
|
||||
|
||||
layer.self_attn.apply_qkv = apply_lora_qkv
|
||||
n_qkv += 1
|
||||
|
|
@ -1520,8 +1520,8 @@ class FastLlamaModel:
|
|||
# O attention patching
|
||||
o_proj = layer.self_attn.o_proj
|
||||
if hasattr(o_proj, "lora_A") and \
|
||||
((o_proj.base_layer if hasattr(o_proj, "base_layer") else o_proj).bias is None) and \
|
||||
((o_proj.lora_magnitude_vector if hasattr(o_proj, "lora_magnitude_vector") else None) is None):
|
||||
(getattr(o_proj, "base_layer", o_proj).bias is None) and \
|
||||
(getattr(o_proj, "lora_magnitude_vector", None) is None):
|
||||
|
||||
layer.self_attn.apply_o = apply_lora_o
|
||||
n_o += 1
|
||||
|
|
|
|||
|
|
@ -632,6 +632,7 @@ pass
|
|||
|
||||
|
||||
def save_to_gguf(
|
||||
model_type : str,
|
||||
model_directory : str = "unsloth_finetuned_model",
|
||||
quantization_method : str = "fast_quantized",
|
||||
first_conversion : str = "f16",
|
||||
|
|
@ -639,10 +640,18 @@ def save_to_gguf(
|
|||
):
|
||||
from transformers.models.llama.modeling_llama import logger
|
||||
|
||||
# Careful convert.py is only for Llama / Mistral based archs
|
||||
use_fast_convert = False
|
||||
if model_type == "llama": use_fast_convert = True
|
||||
elif model_type == "mistral": use_fast_convert = True
|
||||
pass
|
||||
logger.warning_once("Unsloth: Converting {model_type} model. Can use fast conversion = {use_fast_convert}.")
|
||||
|
||||
if quantization_method == "not_quantized": quantization_method = "f16"
|
||||
elif quantization_method == "fast_quantized": quantization_method = "q8_0"
|
||||
elif quantization_method == "quantized": quantization_method = "q4_k_m"
|
||||
elif quantization_method is None: quantization_method = "q8_0"
|
||||
pass
|
||||
|
||||
if quantization_method not in ALLOWED_QUANTS.keys():
|
||||
error = f"Unsloth: Quant method = [{quantization_method}] not supported. Choose from below:\n"
|
||||
|
|
@ -692,6 +701,12 @@ def save_to_gguf(
|
|||
pass
|
||||
pass
|
||||
|
||||
# Non llama/mistral needs can only use f32 or f16
|
||||
if not use_fast_convert and (first_conversion != "f16" or first_conversion != "f32"):
|
||||
logger.warning_once("Unsloth: We must use f16 for non Llama and Mistral models.")
|
||||
first_conversion = "f16"
|
||||
pass
|
||||
|
||||
n_cpus = psutil.cpu_count()
|
||||
if n_cpus is None: n_cpus = 1
|
||||
n_cpus *= 2
|
||||
|
|
@ -703,9 +718,15 @@ def save_to_gguf(
|
|||
f"The output location will be {final_location}\n"\
|
||||
"This will take 3 minutes...")
|
||||
|
||||
command = f"python llama.cpp/convert-hf-to-gguf.py {model_directory} "\
|
||||
f"--outfile {final_location} --vocab-type hfft "\
|
||||
f"--outtype {first_conversion} --concurrency {n_cpus}"
|
||||
if use_convert:
|
||||
command = f"python llama.cpp/convert.py {model_directory} "\
|
||||
f"--outfile {final_location} --vocab-type hfft "\
|
||||
f"--outtype {first_conversion} --concurrency {n_cpus}"
|
||||
else:
|
||||
command = f"python llama.cpp/convert-hf-to-gguf.py {model_directory} "\
|
||||
f"--outfile {final_location} "\
|
||||
f"--outtype {first_conversion}"
|
||||
pass
|
||||
|
||||
with subprocess.Popen(command, shell = True, stdout = subprocess.PIPE, stderr = subprocess.PIPE, bufsize = 1) as sp:
|
||||
for line in sp.stdout:
|
||||
|
|
@ -1054,7 +1075,8 @@ def unsloth_save_pretrained_gguf(
|
|||
for _ in range(3):
|
||||
gc.collect()
|
||||
|
||||
file_location = save_to_gguf(new_save_directory, quantization_method, first_conversion, makefile)
|
||||
model_type = self.config.model_type
|
||||
file_location = save_to_gguf(model_type, new_save_directory, quantization_method, first_conversion, makefile)
|
||||
|
||||
if push_to_hub:
|
||||
print("Unsloth: Uploading GGUF to Huggingface Hub...")
|
||||
|
|
@ -1154,7 +1176,8 @@ def unsloth_push_to_hub_gguf(
|
|||
for _ in range(3):
|
||||
gc.collect()
|
||||
|
||||
file_location = save_to_gguf(new_save_directory, quantization_method, first_conversion, makefile)
|
||||
model_type = self.config.model_type
|
||||
file_location = save_to_gguf(model_type, new_save_directory, quantization_method, first_conversion, makefile)
|
||||
|
||||
print("Unsloth: Uploading GGUF to Huggingface Hub...")
|
||||
username = upload_to_huggingface(
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue