From 2c5c5bb4bb6e132c0e9908a70f2c5f2da59e1bd9 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Thu, 14 Mar 2024 20:32:04 +1100 Subject: [PATCH] Fix Gemma GGUF (#234) * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update llama.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update llama.py * Update cross_entropy_loss.py * Update cross_entropy_loss.py * revert * revert * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update llama.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update cross_entropy_loss.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update llama.py * Update llama.py * Update llama.py * Update llama.py * rope * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * llama * Update llama.py * gemma * Update cross_entropy_loss.py * Update cross_entropy_loss.py * Update cross_entropy_loss.py * Update cross_entropy_loss.py * Update cross_entropy_loss.py * Update cross_entropy_loss.py * Update cross_entropy_loss.py * Update cross_entropy_loss.py * Update cross_entropy_loss.py * Update cross_entropy_loss.py * Update cross_entropy_loss.py * Update cross_entropy_loss.py * Update cross_entropy_loss.py * Update cross_entropy_loss.py * Update cross_entropy_loss.py * Update cross_entropy_loss.py * Update cross_entropy_loss.py * Update cross_entropy_loss.py * Update cross_entropy_loss.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update save.py * RoPE * Update llama.py * Update llama.py * Update llama.py * Update gemma.py * correct_dtype * Update gemma.py * Update cross_entropy_loss.py * Update cross_entropy_loss.py * Chat Templates * Update README.md * Update README.md * Update llama.py * DoRA * Update _utils.py * Update chat_templates.py * Update llama.py * Hotfix - fix DoRA, Gemma prompt template (#202) (#203) * Update save.py * saving * Update save.py * Update save.py * Update save.py * Update save.py * Update save.py * Update save.py * Update save.py * Update save.py * Update save.py * Update save.py * Update save.py * Update save.py * Update save.py * Update __init__.py * Update save.py * Update save.py * Update save.py * save * trainer * spaces * original * Gemma * Update pyproject.toml * Update mapper.py * Update fast_lora.py * FastGemmaModel * model_type * Update llama.py * Update llama.py * Update gemma.py * Update gemma.py * Update gemma.py * Update llama.py * Update llama.py * Update fast_lora.py * Update llama.py * Update llama.py * Update cross_entropy_loss.py * Update llama.py * Update llama.py * gemma * Update llama.py * Update llama.py * Update llama.py * Update llama.py * Update fast_lora.py * Update fast_lora.py * Fast CE Loss * Update cross_entropy_loss.py * Update cross_entropy_loss.py * Update cross_entropy_loss.py * Update cross_entropy_loss.py * Update cross_entropy_loss.py * Update cross_entropy_loss.py * Update cross_entropy_loss.py * Update llama.py * Update llama.py * Update llama.py * Update llama.py * CE * Update llama.py * Update llama.py * Update cross_entropy_loss.py * Update geglu.py * Update cross_entropy_loss.py * revert * Update llama.py * Update llama.py * norm * Update gemma.py * Update gemma.py * position_ids * Update gemma.py * Update gemma.py * pos * Update llama.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update cross_entropy_loss.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update llama.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update llama.py * Update cross_entropy_loss.py * Update cross_entropy_loss.py * revert * revert * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update llama.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update cross_entropy_loss.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update llama.py * Update llama.py * Update llama.py * Update llama.py * rope * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * llama * Update llama.py * gemma * Update cross_entropy_loss.py * Update cross_entropy_loss.py * Update cross_entropy_loss.py * Update cross_entropy_loss.py * Update cross_entropy_loss.py * Update cross_entropy_loss.py * Update cross_entropy_loss.py * Update cross_entropy_loss.py * Update cross_entropy_loss.py * Update cross_entropy_loss.py * Update cross_entropy_loss.py * Update cross_entropy_loss.py * Update cross_entropy_loss.py * Update cross_entropy_loss.py * Update cross_entropy_loss.py * Update cross_entropy_loss.py * Update cross_entropy_loss.py * Update cross_entropy_loss.py * Update cross_entropy_loss.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update save.py * RoPE * Update llama.py * Update llama.py * Update llama.py * Update gemma.py * correct_dtype * Update gemma.py * Update cross_entropy_loss.py * Update cross_entropy_loss.py * Chat Templates * Update README.md * Update README.md * Update llama.py * DoRA * Update _utils.py * Update chat_templates.py * Update pyproject.toml * Small fixes * Update pyproject.toml * Approx gelu * Update geglu.py * Approx gelu * Update llama.py * Update __init__.py * Update __init__.py * Update _utils.py * Update geglu.py * Update gemma.py * Update rms_layernorm.py * Update rms_layernorm.py * Update rms_layernorm.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Fix Gemma merging * Update rms_layernorm.py * Update gemma.py * Update pyproject.toml * Layernorms * Gemma precision * Update gemma.py * sqrt * Update gemma.py * Update save.py * RoPE and Gemma precision * Update rms_layernorm.py * Fix warning * Update chat_templates.py * Update chat_templates.py * Update save.py * Update save.py * Update save.py * Update chat_templates.py * Update llama.py * model_name * Update loader.py * Tokenizer overwritten * Update llama.py * Update llama.py * Update llama.py * Update save.py * Accuracy * Revert * Update save.py * Update fast_lora.py * Update fast_lora.py * Update fast_lora.py * Update fast_lora.py * Update fast_lora.py * Update chat_templates.py * Update save.py * Update save.py * Update llama.py * Update llama.py * Account for DoRA * Update llama.py * Update save.py * GGUF incorrect * Update save.py * Update pyproject.toml * kaggle new * Update pyproject.toml * Update pyproject.toml * upcasting * Fix Colab * Update pyproject.toml * Update pyproject.toml * Update pyproject.toml * Update pyproject.toml * Update pyproject.toml * Update pyproject.toml * Update pyproject.toml * Update pyproject.toml --- README.md | 2 +- pyproject.toml | 44 +++++++++++++++++++++++++++------------ unsloth/chat_templates.py | 12 +++++++++-- unsloth/models/gemma.py | 1 + unsloth/models/llama.py | 36 +++++++++++++++++++------------- unsloth/save.py | 33 ++++++++++++++++++++++++----- 6 files changed, 92 insertions(+), 36 deletions(-) diff --git a/README.md b/README.md index 759057f2a4..412e3ffda2 100644 --- a/README.md +++ b/README.md @@ -59,7 +59,7 @@ All notebooks are **beginner friendly**! Add your dataset, click "Run All", and - No change of hardware. Supports NVIDIA GPUs since 2018+. Minimum CUDA Capability 7.0 (V100, T4, Titan V, RTX 20, 30, 40x, A100, H100, L40 etc) [Check your GPU!](https://developer.nvidia.com/cuda-gpus) GTX 1070, 1080 works, but is slow. - Works on **Linux** and **Windows** via WSL. - Supports 4bit and 16bit QLoRA / LoRA finetuning via [bitsandbytes](https://github.com/TimDettmers/bitsandbytes). -- Open source trains 5x faster - see [Unsloth Pro](https://unsloth.ai/) for **30x faster training**! +- Open source trains 5x faster - see [Unsloth Pro](https://unsloth.ai/) for up to **30x faster training**! - If you trained a model with šŸ¦„Unsloth, you can use this cool sticker!   diff --git a/pyproject.toml b/pyproject.toml index 05c4191974..3ea50ca00e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -33,17 +33,17 @@ exclude = ["images*"] [project.optional-dependencies] huggingface = [ + "tyro", "transformers>=4.38.2", "datasets>=2.16.0", "sentencepiece", - "accelerate>=0.26.1", - "trl>=0.7.9", - "peft>=0.7.1", "tqdm", "psutil", "wheel>=0.42.0", "numpy", - "triton", + "accelerate>=0.26.1", + "trl>=0.7.9", + "peft>=0.7.1", ] cu118only = [ "xformers @ https://download.pytorch.org/whl/cu118/xformers-0.0.22.post7%2Bcu118-cp39-cp39-manylinux2014_x86_64.whl ; python_version=='3.9'", @@ -128,18 +128,13 @@ cu121-torch220 = [ kaggle = [ "unsloth[huggingface]", ] +kaggle-new = [ + "unsloth[huggingface]", + "bitsandbytes", +] conda = [ "unsloth[huggingface]", ] -colab = [ - "unsloth[cu121]", -] -colab-ampere = [ - "unsloth[cu121]", - "packaging", - "ninja", - "flash-attn", -] colab-torch211 = [ "unsloth[huggingface]", "bitsandbytes", @@ -166,6 +161,29 @@ colab-ampere-torch220 = [ "ninja", "flash-attn", ] +colab-new = [ + "tyro", + "transformers>=4.38.2", + "datasets>=2.16.0", + "sentencepiece", + "tqdm", + "psutil", + "wheel>=0.42.0", + "numpy", +] +colab-no-deps = [ + "accelerate>=0.26.1", + "trl>=0.7.9", + "peft>=0.7.1", + "xformers", + "bitsandbytes", +] +colab-ampere = [ + "unsloth[colab-ampere-torch220]", + "packaging", + "ninja", + "flash-attn", +] cu118-ampere = [ "unsloth[huggingface]", "bitsandbytes", diff --git a/unsloth/chat_templates.py b/unsloth/chat_templates.py index 5487e9dcab..7d205ebc5f 100644 --- a/unsloth/chat_templates.py +++ b/unsloth/chat_templates.py @@ -259,8 +259,10 @@ def get_chat_template( assert("Unsloth: Can only map new tokens to EOS for now. Adding new tokens is not yet supported.") pass - if tokenizer.__class__.__name__.startswith("Gemma") and chat_template == "chatml": - chat_template = "gemma_chatml" + IS_GEMMA = False + if tokenizer.__class__.__name__.startswith("Gemma"): + if chat_template == "chatml": chat_template = "gemma_chatml" + IS_GEMMA = True pass old_padding_side = tokenizer.padding_side @@ -338,6 +340,12 @@ def get_chat_template( .replace("'user'", "'" + mapping["user"] + "'")\ .replace("'assistant'", "'" + mapping["assistant"] + "'") + # Careful on Gemma + # bos_token is a must or else losses become too high + if IS_GEMMA and not chat_template.startswith("{{ bos_token }}"): + chat_template = "{{ bos_token }}" + chat_template + pass + _, tokenizer = patch_tokenizer(model = None, tokenizer = tokenizer) tokenizer.padding_side = old_padding_side tokenizer.chat_template = chat_template diff --git a/unsloth/models/gemma.py b/unsloth/models/gemma.py index bcd0e1abd9..7bfec43e51 100644 --- a/unsloth/models/gemma.py +++ b/unsloth/models/gemma.py @@ -154,6 +154,7 @@ def GemmaModel_fast_forward_inference( out_weight = torch.empty_like(self.layers[0].input_layernorm.weight, dtype = torch.float32, device = "cuda") hidden_states = self.embed_tokens(input_ids) + hidden_states = hidden_states.to(self.config.torch_dtype) # 3072**0.5 = 55.5000 in bfloat16, whilst 55.4256 in float32 # 2048**0.5 = 45.2500 in bfloat16, whilst 45.2548 in float32 hidden_states *= torch.tensor(math_sqrt(self.config.hidden_size), dtype = hidden_states.dtype) diff --git a/unsloth/models/llama.py b/unsloth/models/llama.py index e9493376be..d83d9b76f2 100644 --- a/unsloth/models/llama.py +++ b/unsloth/models/llama.py @@ -509,7 +509,10 @@ def LlamaModel_fast_forward( if inputs_embeds is None: inputs_embeds = self.embed_tokens(input_ids) - # Mormalized from Gemma + # Downcast to the correct dtype ie float32 to float16 + inputs_embeds = inputs_embeds.to(self.config.torch_dtype) + + # Normalized from Gemma IS_GEMMA = self.config.model_type == "gemma" train_embed_tokens = self.embed_tokens.weight.requires_grad @@ -665,6 +668,7 @@ def LlamaModel_fast_forward_inference( input_ids = input_ids[:,:self.max_seq_length] hidden_states = self.embed_tokens(input_ids) + hidden_states = hidden_states.to(self.config.torch_dtype) next_decoder_cache = [] for idx, decoder_layer in enumerate(self.layers): @@ -1334,6 +1338,7 @@ class FastLlamaModel: "We shall do it for you!" ) train_lm_head = True + model.model.embed_tokens.to(torch.float32, non_blocking = True) elif module == "embed_tokens": logger.warning_once( @@ -1341,6 +1346,7 @@ class FastLlamaModel: "We shall do it for you!" ) train_embed_tokens = True + model.lm_head.to(torch.float32, non_blocking = True) else: assert(module in accepted_modules) @@ -1477,12 +1483,12 @@ class FastLlamaModel: if hasattr(gate_proj, "lora_A") and \ hasattr( up_proj, "lora_A") and \ hasattr(down_proj, "lora_A") and \ - ((gate_proj.base_layer if hasattr(gate_proj, "base_layer") else gate_proj).bias is None) and \ - (( up_proj.base_layer if hasattr( up_proj, "base_layer") else up_proj).bias is None) and \ - ((down_proj.base_layer if hasattr(down_proj, "base_layer") else down_proj).bias is None) and \ - ((gate_proj.lora_magnitude_vector if hasattr(gate_proj, "lora_magnitude_vector") else None) is None) and \ - (( up_proj.lora_magnitude_vector if hasattr( up_proj, "lora_magnitude_vector") else None) is None) and \ - ((down_proj.lora_magnitude_vector if hasattr(down_proj, "lora_magnitude_vector") else None) is None): + (getattr(gate_proj, "base_layer", gate_proj).bias is None) and \ + (getattr( up_proj, "base_layer", up_proj).bias is None) and \ + (getattr(down_proj, "base_layer", down_proj).bias is None) and \ + (getattr(gate_proj, "lora_magnitude_vector", None) is None) and \ + (getattr( up_proj, "lora_magnitude_vector", None) is None) and \ + (getattr(down_proj, "lora_magnitude_vector", None) is None): # https://stackoverflow.com/questions/50599045/python-replacing-a-function-within-a-class-of-a-module layer.mlp.forward = types.MethodType(apply_lora_mlp, layer.mlp) @@ -1501,12 +1507,12 @@ class FastLlamaModel: if hasattr(q_proj, "lora_A") and \ hasattr(k_proj, "lora_A") and \ hasattr(v_proj, "lora_A") and \ - ((q_proj.base_layer if hasattr(q_proj, "base_layer") else q_proj).bias is None) and \ - ((k_proj.base_layer if hasattr(k_proj, "base_layer") else k_proj).bias is None) and \ - ((v_proj.base_layer if hasattr(v_proj, "base_layer") else v_proj).bias is None) and \ - ((q_proj.lora_magnitude_vector if hasattr(q_proj, "lora_magnitude_vector") else None) is None) and \ - ((k_proj.lora_magnitude_vector if hasattr(k_proj, "lora_magnitude_vector") else None) is None) and \ - ((v_proj.lora_magnitude_vector if hasattr(v_proj, "lora_magnitude_vector") else None) is None): + (getattr(q_proj, "base_layer", q_proj).bias is None) and \ + (getattr(q_proj, "base_layer", k_proj).bias is None) and \ + (getattr(q_proj, "base_layer", v_proj).bias is None) and \ + (getattr(q_proj, "lora_magnitude_vector", None) is None) and \ + (getattr(k_proj, "lora_magnitude_vector", None) is None) and \ + (getattr(v_proj, "lora_magnitude_vector", None) is None): layer.self_attn.apply_qkv = apply_lora_qkv n_qkv += 1 @@ -1520,8 +1526,8 @@ class FastLlamaModel: # O attention patching o_proj = layer.self_attn.o_proj if hasattr(o_proj, "lora_A") and \ - ((o_proj.base_layer if hasattr(o_proj, "base_layer") else o_proj).bias is None) and \ - ((o_proj.lora_magnitude_vector if hasattr(o_proj, "lora_magnitude_vector") else None) is None): + (getattr(o_proj, "base_layer", o_proj).bias is None) and \ + (getattr(o_proj, "lora_magnitude_vector", None) is None): layer.self_attn.apply_o = apply_lora_o n_o += 1 diff --git a/unsloth/save.py b/unsloth/save.py index 7543eef10e..dd0dc26e66 100644 --- a/unsloth/save.py +++ b/unsloth/save.py @@ -632,6 +632,7 @@ pass def save_to_gguf( + model_type : str, model_directory : str = "unsloth_finetuned_model", quantization_method : str = "fast_quantized", first_conversion : str = "f16", @@ -639,10 +640,18 @@ def save_to_gguf( ): from transformers.models.llama.modeling_llama import logger + # Careful convert.py is only for Llama / Mistral based archs + use_fast_convert = False + if model_type == "llama": use_fast_convert = True + elif model_type == "mistral": use_fast_convert = True + pass + logger.warning_once(f"Unsloth: Converting {model_type} model. Can use fast conversion = {use_fast_convert}.") + if quantization_method == "not_quantized": quantization_method = "f16" elif quantization_method == "fast_quantized": quantization_method = "q8_0" elif quantization_method == "quantized": quantization_method = "q4_k_m" elif quantization_method is None: quantization_method = "q8_0" + pass if quantization_method not in ALLOWED_QUANTS.keys(): error = f"Unsloth: Quant method = [{quantization_method}] not supported. Choose from below:\n" @@ -692,6 +701,12 @@ def save_to_gguf( pass pass + # Non llama/mistral needs can only use f32 or f16 + if not use_fast_convert and (first_conversion != "f16" or first_conversion != "f32"): + logger.warning_once("Unsloth: We must use f16 for non Llama and Mistral models.") + first_conversion = "f16" + pass + n_cpus = psutil.cpu_count() if n_cpus is None: n_cpus = 1 n_cpus *= 2 @@ -703,9 +718,15 @@ def save_to_gguf( f"The output location will be {final_location}\n"\ "This will take 3 minutes...") - command = f"python llama.cpp/convert.py {model_directory} "\ - f"--outfile {final_location} --vocab-type hfft "\ - f"--outtype {first_conversion} --concurrency {n_cpus}" + if use_fast_convert: + command = f"python llama.cpp/convert.py {model_directory} "\ + f"--outfile {final_location} --vocab-type hfft "\ + f"--outtype {first_conversion} --concurrency {n_cpus}" + else: + command = f"python llama.cpp/convert-hf-to-gguf.py {model_directory} "\ + f"--outfile {final_location} "\ + f"--outtype {first_conversion}" + pass with subprocess.Popen(command, shell = True, stdout = subprocess.PIPE, stderr = subprocess.PIPE, bufsize = 1) as sp: for line in sp.stdout: @@ -1054,7 +1075,8 @@ def unsloth_save_pretrained_gguf( for _ in range(3): gc.collect() - file_location = save_to_gguf(new_save_directory, quantization_method, first_conversion, makefile) + model_type = self.config.model_type + file_location = save_to_gguf(model_type, new_save_directory, quantization_method, first_conversion, makefile) if push_to_hub: print("Unsloth: Uploading GGUF to Huggingface Hub...") @@ -1154,7 +1176,8 @@ def unsloth_push_to_hub_gguf( for _ in range(3): gc.collect() - file_location = save_to_gguf(new_save_directory, quantization_method, first_conversion, makefile) + model_type = self.config.model_type + file_location = save_to_gguf(model_type, new_save_directory, quantization_method, first_conversion, makefile) print("Unsloth: Uploading GGUF to Huggingface Hub...") username = upload_to_huggingface(