From 2a9d4fb947c2d7b7dcc31fbeff6c93e0be72c571 Mon Sep 17 00:00:00 2001 From: Daniel Han-Chen Date: Sun, 17 Mar 2024 20:10:30 +1100 Subject: [PATCH] Bugs --- unsloth/kernels/fast_lora.py | 2 ++ unsloth/save.py | 10 ++++++++-- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/unsloth/kernels/fast_lora.py b/unsloth/kernels/fast_lora.py index 6568bba681..4b8795baac 100644 --- a/unsloth/kernels/fast_lora.py +++ b/unsloth/kernels/fast_lora.py @@ -87,6 +87,7 @@ class LoRA_MLP(torch.autograd.Function): downW, downW_quant, downA, downB, downS, _forward_function, _backward_function,): dtype = X.dtype + print("Forward " + dtype) e = matmul_lora(X, gateW, gateW_quant, gateA, gateB, gateS) g = matmul_lora(X, upW, upW_quant, upA, upB, upS) @@ -122,6 +123,7 @@ class LoRA_MLP(torch.autograd.Function): e = e .view(-1, e .shape[-1]) g = g .view(-1, g .shape[-1]) dtype = X.dtype + print("Backward " + dtype) DW = matmul_lora(dY, downW.t(), downW_quant, downB, downA, downS) DW, e, g = _backward_function(DW, e, g) diff --git a/unsloth/save.py b/unsloth/save.py index 05ff748db2..5970d74d11 100644 --- a/unsloth/save.py +++ b/unsloth/save.py @@ -684,6 +684,9 @@ def save_to_gguf( ): from transformers.models.llama.modeling_llama import logger + if quantization_method.startswith("iq2"): + raise RuntimeError("Unsloth: Currently iq2 type quantizations aren't supported yet - sorry!") + # Careful convert.py is only for Llama / Mistral based archs use_fast_convert = False if model_type == "llama": use_fast_convert = True @@ -743,8 +746,11 @@ def save_to_gguf( if first_conversion == "f32" : pass elif first_conversion == "f16" : pass elif first_conversion == "q8_0": - logger.warning_once("Unsloth: We must use f16 for quantization first.") - first_conversion = "f16" + logger.warning_once( + "Unsloth: Using q8_0 for the `first_conversion` will lose a bit of accuracy, "\ + "but saves disk space!" + ) + # first_conversion = "f16" pass pass