Fix downcasting LoRA (#318)
* Update save.py
* Update chat_templates.py
* Update llama.py
* model_name
* Update loader.py
* Tokenizer overwritten
* Update llama.py
* Update llama.py
* Update llama.py
* Update save.py
* Accuracy
* Revert
* Update save.py
* Update fast_lora.py
* Update fast_lora.py
* Update fast_lora.py
* Update fast_lora.py
* Update fast_lora.py
* Update chat_templates.py
* Update save.py
* Update save.py
* Update llama.py
* Update llama.py
* Account for DoRA
* Update llama.py
* Update save.py
* GGUF incorrect
* Update save.py
* Update pyproject.toml
* kaggle new
* Update pyproject.toml
* Update pyproject.toml
* upcasting
* Fix Colab
* Update pyproject.toml
* Update pyproject.toml
* Update pyproject.toml
* Update pyproject.toml
* Update pyproject.toml
* Update pyproject.toml
* Update pyproject.toml
* Update pyproject.toml
* Update chat_templates.py
* Update chat_templates.py
* Update chat_templates.py
* Update chat_templates.py
* Update chat_templates.py
* Update pyproject.toml
* Update pyproject.toml
* Update pyproject.toml
* Update rope_embedding.py
* Update rope_embedding.py
* Fix bugs
* Update fast_lora.py
* Update fast_lora.py
* Update README.md
* Update README.md
* GGUF
* Update save.py
* Update save.py
* Update save.py
* Update save.py
* Update README.md
* Update README.md
* Bugs
* Update fast_lora.py
* Update pyproject.toml
* Update fast_lora.py
* Update __init__.py
* Update fast_lora.py
* dtype
* Update llama.py
* Update llama.py
* Update llama.py
* dtype
* Update mistral.py
* trust_remote_code
* lm_head
* Update llama.py
* save_pretrained_settings
* Update save.py
* Update save.py
* Update save.py
* Update save.py
* Update save.py
* Update save.py
* Update save.py
* Update save.py
* Update save.py
* Update save.py
* Update save.py
* Update save.py
* state_dict
* Update save.py
* whoami
* Update llama.py
* Update save.py
* Update llama.py
* Patch tokenizer
* Update chat_templates.py
* Heal tokenizers
* Update chat_templates.py
* Update mapper.py
* Update tokenizer_utils.py
* Update tokenizer_utils.py
* Update tokenizer_utils.py
* Update tokenizer_utils.py
* Update tokenizer_utils.py
* Update chat_templates.py
* tokenizer patching
* patch_tokenizer
* Update chat_templates.py
* Update tokenizer_utils.py
* Update chat_templates.py
* Update chat_templates.py
* Update chat_templates.py
* Update tokenizer_utils.py
* Edit
* Update mistral.py
* Update mistral.py
* Stats
* Update mistral.py
* attention_mask
* Update llama.py
* Update llama.py
* batch
* Temp fix batch inference
* Update llama.py
* Update gemma.py
* Fix inference
* swiglu
* Update llama.py
* Update llama.py
* Update llama.py
* Update llama.py
* Update mistral.py
* Update llama.py
* fast inference
* model
* Update llama.py
* Update llama.py
* Update llama.py
* Update llama.py
* Update llama.py
* Update llama.py
* Update llama.py
* Update llama.py
* Update llama.py
* Update llama.py
* Update llama.py
* Update llama.py
* Update llama.py
* Update utils.py
* Update llama.py
* Update utils.py
* inference
* Update llama.py
* Update llama.py
* Update llama.py
* overhead
* Update llama.py
* Update llama.py
* compile
* Update gemma.py
* Update llama.py
* Update llama.py
* Update llama.py
* Update llama.py
* Update utils.py
* Update utils.py
* lora mamtul
* Update llama.py
* Update llama.py
* Update llama.py
* offloaded checkpointing
* Update llama.py
* Update llama.py
* Update _utils.py
* Update _utils.py
* Update _utils.py
* Update llama.py
* Update llama.py
* Update gemma.py
* Revert "Update gemma.py"
This reverts commit c68b59bbfd.
* Update _utils.py
* Update _utils.py
* Update _utils.py
* Saving
* sentencepiece_model_pb2
* Update llama.py
* Update save.py
* Update llama.py
* padding side
* Update tokenizer_utils.py
* cache dir
* Update tokenizer_utils.py
* Update tokenizer_utils.py
* Update pyproject.toml
* Update pyproject.toml
* Update tokenizer_utils.py
* Update tokenizer_utils.py
* Update llama.py
* Update save.py
* Update save.py
* checkpoint
* Gemma 1.1
* more models
* Update llama.py
* Update llama.py
* Update llama.py
* Update llama.py
* Update llama.py
* Update llama.py
* Update llama.py
* Update llama.py
* Update llama.py
* Update llama.py
* Update llama.py
* Update llama.py
* Update llama.py
* dtype
* Update llama.py
* CodeGemma
* Fix downcasting
This commit is contained in:
Daniel Han
GitHub
parent
c7649138ee
commit
648bde7f06
4 changed files with 18 additions and 11 deletions
|
|
@ -95,11 +95,22 @@ def prepare_model_for_kbit_training(
|
|||
"""
|
||||
|
||||
# Freeze all parameters except LoRA
|
||||
for name, param in model.named_parameters():
|
||||
if ".lora_A." in name or ".lora_B." in name or ".lora_magnitude_vector" in name:
|
||||
param.requires_grad_(True)
|
||||
else:
|
||||
param.requires_grad_(False)
|
||||
import re
|
||||
with torch.inference_mode():
|
||||
for name, param in model.named_parameters():
|
||||
if ".lora_A." in name or ".lora_B." in name or ".lora_magnitude_vector" in name:
|
||||
param.requires_grad_(True)
|
||||
# Also must be in float32!
|
||||
if param.dtype != torch.float32:
|
||||
name = name.replace("base_model", "model", 1)
|
||||
layer_number = re.search(r"\.[\d]{1,}\.", name).group(0)
|
||||
name = name.replace(layer_number, f"[{layer_number[1:-1]}].")
|
||||
name = name.replace(".weight", "", 1)
|
||||
exec(f"{name}.to(torch.float32)")
|
||||
pass
|
||||
else:
|
||||
param.requires_grad_(False)
|
||||
pass
|
||||
pass
|
||||
|
||||
# Gradient checkpointing!
|
||||
|
|
|
|||
|
|
@ -1030,7 +1030,7 @@ class FastLlamaModel:
|
|||
f' "-____-" Free Apache license: http://github.com/unslothai/unsloth'
|
||||
print(statistics)
|
||||
model_patcher.pre_patch()
|
||||
get_statistics()
|
||||
# get_statistics()
|
||||
|
||||
if dtype is None:
|
||||
dtype = torch.float16 if not SUPPORTS_BFLOAT16 else torch.bfloat16
|
||||
|
|
|
|||
|
|
@ -122,10 +122,6 @@ __INT_TO_FLOAT_MAPPER = \
|
|||
"unsloth/codegemma-7b",
|
||||
"google/codegemma-7b",
|
||||
),
|
||||
"unsloth/codegemma-2b-it-bnb-4bit" : (
|
||||
"unsloth/codegemma-2b-it",
|
||||
"google/codegemma-2b-it",
|
||||
),
|
||||
"unsloth/codegemma-7b-it-bnb-4bit" : (
|
||||
"unsloth/codegemma-7b-it",
|
||||
"google/codegemma-7b-it",
|
||||
|
|
|
|||
|
|
@ -319,7 +319,7 @@ class FastMistralModel(FastLlamaModel):
|
|||
f' "-____-" Free Apache license: http://github.com/unslothai/unsloth'
|
||||
print(statistics)
|
||||
model_patcher.pre_patch()
|
||||
get_statistics()
|
||||
# get_statistics()
|
||||
|
||||
if dtype is None:
|
||||
dtype = torch.float16 if not SUPPORTS_BFLOAT16 else torch.bfloat16
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue