From 648bde7f06e374b67f1ed265d596d5c25b07a000 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Wed, 10 Apr 2024 00:44:58 +1000 Subject: [PATCH] Fix downcasting LoRA (#318) * Update save.py * Update chat_templates.py * Update llama.py * model_name * Update loader.py * Tokenizer overwritten * Update llama.py * Update llama.py * Update llama.py * Update save.py * Accuracy * Revert * Update save.py * Update fast_lora.py * Update fast_lora.py * Update fast_lora.py * Update fast_lora.py * Update fast_lora.py * Update chat_templates.py * Update save.py * Update save.py * Update llama.py * Update llama.py * Account for DoRA * Update llama.py * Update save.py * GGUF incorrect * Update save.py * Update pyproject.toml * kaggle new * Update pyproject.toml * Update pyproject.toml * upcasting * Fix Colab * Update pyproject.toml * Update pyproject.toml * Update pyproject.toml * Update pyproject.toml * Update pyproject.toml * Update pyproject.toml * Update pyproject.toml * Update pyproject.toml * Update chat_templates.py * Update chat_templates.py * Update chat_templates.py * Update chat_templates.py * Update chat_templates.py * Update pyproject.toml * Update pyproject.toml * Update pyproject.toml * Update rope_embedding.py * Update rope_embedding.py * Fix bugs * Update fast_lora.py * Update fast_lora.py * Update README.md * Update README.md * GGUF * Update save.py * Update save.py * Update save.py * Update save.py * Update README.md * Update README.md * Bugs * Update fast_lora.py * Update pyproject.toml * Update fast_lora.py * Update __init__.py * Update fast_lora.py * dtype * Update llama.py * Update llama.py * Update llama.py * dtype * Update mistral.py * trust_remote_code * lm_head * Update llama.py * save_pretrained_settings * Update save.py * Update save.py * Update save.py * Update save.py * Update save.py * Update save.py * Update save.py * Update save.py * Update save.py * Update save.py * Update save.py * Update save.py * state_dict * Update save.py * whoami * Update llama.py * Update save.py * Update llama.py * Patch tokenizer * Update chat_templates.py * Heal tokenizers * Update chat_templates.py * Update mapper.py * Update tokenizer_utils.py * Update tokenizer_utils.py * Update tokenizer_utils.py * Update tokenizer_utils.py * Update tokenizer_utils.py * Update chat_templates.py * tokenizer patching * patch_tokenizer * Update chat_templates.py * Update tokenizer_utils.py * Update chat_templates.py * Update chat_templates.py * Update chat_templates.py * Update tokenizer_utils.py * Edit * Update mistral.py * Update mistral.py * Stats * Update mistral.py * attention_mask * Update llama.py * Update llama.py * batch * Temp fix batch inference * Update llama.py * Update gemma.py * Fix inference * swiglu * Update llama.py * Update llama.py * Update llama.py * Update llama.py * Update mistral.py * Update llama.py * fast inference * model * Update llama.py * Update llama.py * Update llama.py * Update llama.py * Update llama.py * Update llama.py * Update llama.py * Update llama.py * Update llama.py * Update llama.py * Update llama.py * Update llama.py * Update llama.py * Update utils.py * Update llama.py * Update utils.py * inference * Update llama.py * Update llama.py * Update llama.py * overhead * Update llama.py * Update llama.py * compile * Update gemma.py * Update llama.py * Update llama.py * Update llama.py * Update llama.py * Update utils.py * Update utils.py * lora mamtul * Update llama.py * Update llama.py * Update llama.py * offloaded checkpointing * Update llama.py * Update llama.py * Update _utils.py * Update _utils.py * Update _utils.py * Update llama.py * Update llama.py * Update gemma.py * Revert "Update gemma.py" This reverts commit c68b59bbfd276d108098100901aaf6d123c52102. * Update _utils.py * Update _utils.py * Update _utils.py * Saving * sentencepiece_model_pb2 * Update llama.py * Update save.py * Update llama.py * padding side * Update tokenizer_utils.py * cache dir * Update tokenizer_utils.py * Update tokenizer_utils.py * Update pyproject.toml * Update pyproject.toml * Update tokenizer_utils.py * Update tokenizer_utils.py * Update llama.py * Update save.py * Update save.py * checkpoint * Gemma 1.1 * more models * Update llama.py * Update llama.py * Update llama.py * Update llama.py * Update llama.py * Update llama.py * Update llama.py * Update llama.py * Update llama.py * Update llama.py * Update llama.py * Update llama.py * Update llama.py * dtype * Update llama.py * CodeGemma * Fix downcasting --- unsloth/models/_utils.py | 21 ++++++++++++++++----- unsloth/models/llama.py | 2 +- unsloth/models/mapper.py | 4 ---- unsloth/models/mistral.py | 2 +- 4 files changed, 18 insertions(+), 11 deletions(-) diff --git a/unsloth/models/_utils.py b/unsloth/models/_utils.py index 88a9e9661e..77509cd8da 100644 --- a/unsloth/models/_utils.py +++ b/unsloth/models/_utils.py @@ -95,11 +95,22 @@ def prepare_model_for_kbit_training( """ # Freeze all parameters except LoRA - for name, param in model.named_parameters(): - if ".lora_A." in name or ".lora_B." in name or ".lora_magnitude_vector" in name: - param.requires_grad_(True) - else: - param.requires_grad_(False) + import re + with torch.inference_mode(): + for name, param in model.named_parameters(): + if ".lora_A." in name or ".lora_B." in name or ".lora_magnitude_vector" in name: + param.requires_grad_(True) + # Also must be in float32! + if param.dtype != torch.float32: + name = name.replace("base_model", "model", 1) + layer_number = re.search(r"\.[\d]{1,}\.", name).group(0) + name = name.replace(layer_number, f"[{layer_number[1:-1]}].") + name = name.replace(".weight", "", 1) + exec(f"{name}.to(torch.float32)") + pass + else: + param.requires_grad_(False) + pass pass # Gradient checkpointing! diff --git a/unsloth/models/llama.py b/unsloth/models/llama.py index 0b8092eaac..202c692c4e 100644 --- a/unsloth/models/llama.py +++ b/unsloth/models/llama.py @@ -1030,7 +1030,7 @@ class FastLlamaModel: f' "-____-" Free Apache license: http://github.com/unslothai/unsloth' print(statistics) model_patcher.pre_patch() - get_statistics() + # get_statistics() if dtype is None: dtype = torch.float16 if not SUPPORTS_BFLOAT16 else torch.bfloat16 diff --git a/unsloth/models/mapper.py b/unsloth/models/mapper.py index f24108b75a..b1d2faedb3 100644 --- a/unsloth/models/mapper.py +++ b/unsloth/models/mapper.py @@ -122,10 +122,6 @@ __INT_TO_FLOAT_MAPPER = \ "unsloth/codegemma-7b", "google/codegemma-7b", ), - "unsloth/codegemma-2b-it-bnb-4bit" : ( - "unsloth/codegemma-2b-it", - "google/codegemma-2b-it", - ), "unsloth/codegemma-7b-it-bnb-4bit" : ( "unsloth/codegemma-7b-it", "google/codegemma-7b-it", diff --git a/unsloth/models/mistral.py b/unsloth/models/mistral.py index 87f5c85ad1..3034b83d2a 100644 --- a/unsloth/models/mistral.py +++ b/unsloth/models/mistral.py @@ -319,7 +319,7 @@ class FastMistralModel(FastLlamaModel): f' "-____-" Free Apache license: http://github.com/unslothai/unsloth' print(statistics) model_patcher.pre_patch() - get_statistics() + # get_statistics() if dtype is None: dtype = torch.float16 if not SUPPORTS_BFLOAT16 else torch.bfloat16