Fix Gemma GGUF (#311)
* Update gemma.py
* Fix Gemma merging
* Update rms_layernorm.py
* Update gemma.py
* Update pyproject.toml
* Layernorms
* Gemma precision
* Update gemma.py
* sqrt
* Update gemma.py
* Update save.py
* RoPE and Gemma precision
* Update rms_layernorm.py
* Fix warning
* Update chat_templates.py
* Update chat_templates.py
* Update save.py
* Update save.py
* Update save.py
* Update chat_templates.py
* Update llama.py
* model_name
* Update loader.py
* Tokenizer overwritten
* Update llama.py
* Update llama.py
* Update llama.py
* Update save.py
* Accuracy
* Revert
* Update save.py
* Update fast_lora.py
* Update fast_lora.py
* Update fast_lora.py
* Update fast_lora.py
* Update fast_lora.py
* Update chat_templates.py
* Update save.py
* Update save.py
* Update llama.py
* Update llama.py
* Account for DoRA
* Update llama.py
* Update save.py
* GGUF incorrect
* Update save.py
* Update pyproject.toml
* kaggle new
* Update pyproject.toml
* Update pyproject.toml
* upcasting
* Fix Colab
* Update pyproject.toml
* Update pyproject.toml
* Update pyproject.toml
* Update pyproject.toml
* Update pyproject.toml
* Update pyproject.toml
* Update pyproject.toml
* Update pyproject.toml
* Update chat_templates.py
* Update chat_templates.py
* Update chat_templates.py
* Update chat_templates.py
* Update chat_templates.py
* Update pyproject.toml
* Update pyproject.toml
* Update pyproject.toml
* Update rope_embedding.py
* Update rope_embedding.py
* Fix bugs
* Update fast_lora.py
* Update fast_lora.py
* Update README.md
* Update README.md
* GGUF
* Update save.py
* Update save.py
* Update save.py
* Update save.py
* Update README.md
* Update README.md
* Bugs
* Update fast_lora.py
* Update pyproject.toml
* Update fast_lora.py
* Update __init__.py
* Update fast_lora.py
* dtype
* Update llama.py
* Update llama.py
* Update llama.py
* dtype
* Update mistral.py
* trust_remote_code
* lm_head
* Update llama.py
* save_pretrained_settings
* Update save.py
* Update save.py
* Update save.py
* Update save.py
* Update save.py
* Update save.py
* Update save.py
* Update save.py
* Update save.py
* Update save.py
* Update save.py
* Update save.py
* state_dict
* Update save.py
* whoami
* Update llama.py
* Update save.py
* Update llama.py
* Patch tokenizer
* Update chat_templates.py
* Heal tokenizers
* Update chat_templates.py
* Update mapper.py
* Update tokenizer_utils.py
* Update tokenizer_utils.py
* Update tokenizer_utils.py
* Update tokenizer_utils.py
* Update tokenizer_utils.py
* Update chat_templates.py
* tokenizer patching
* patch_tokenizer
* Update chat_templates.py
* Update tokenizer_utils.py
* Update chat_templates.py
* Update chat_templates.py
* Update chat_templates.py
* Update tokenizer_utils.py
* Edit
* Update mistral.py
* Update mistral.py
* Stats
* Update mistral.py
* attention_mask
* Update llama.py
* Update llama.py
* batch
* Temp fix batch inference
* Update llama.py
* Update gemma.py
* Fix inference
* swiglu
* Update llama.py
* Update llama.py
* Update llama.py
* Update llama.py
* Update mistral.py
* Update llama.py
* fast inference
* model
* Update llama.py
* Update llama.py
* Update llama.py
* Update llama.py
* Update llama.py
* Update llama.py
* Update llama.py
* Update llama.py
* Update llama.py
* Update llama.py
* Update llama.py
* Update llama.py
* Update llama.py
* Update utils.py
* Update llama.py
* Update utils.py
* inference
* Update llama.py
* Update llama.py
* Update llama.py
* overhead
* Update llama.py
* Update llama.py
* compile
* Update gemma.py
* Update llama.py
* Update llama.py
* Update llama.py
* Update llama.py
* Update utils.py
* Update utils.py
* lora mamtul
* Update llama.py
* Update llama.py
* Update llama.py
* offloaded checkpointing
* Update llama.py
* Update llama.py
* Update _utils.py
* Update _utils.py
* Update _utils.py
* Update llama.py
* Update llama.py
* Update gemma.py
* Revert "Update gemma.py"
This reverts commit c68b59bbfd.
* Update _utils.py
* Update _utils.py
* Update _utils.py
* Saving
* sentencepiece_model_pb2
* Update llama.py
* Update save.py
* Update llama.py
* padding side
* Update tokenizer_utils.py
* cache dir
* Update tokenizer_utils.py
* Update tokenizer_utils.py
* Update pyproject.toml
* Update pyproject.toml
* Update tokenizer_utils.py
* Update tokenizer_utils.py
* Update llama.py
* Update save.py
* Update save.py
* checkpoint
* Gemma 1.1
* more models
This commit is contained in:
Daniel Han
GitHub
parent
f3d05d19e3
commit
4474e4bca4
4 changed files with 77 additions and 28 deletions
|
|
@ -70,7 +70,7 @@ __all__ = [
|
|||
"platform_system",
|
||||
"patch_tokenizer",
|
||||
"get_statistics",
|
||||
"Offloaded_Gradient_Checkpointer",
|
||||
"Unsloth_Offloaded_Gradient_Checkpointer",
|
||||
]
|
||||
|
||||
|
||||
|
|
@ -103,7 +103,7 @@ def prepare_model_for_kbit_training(
|
|||
pass
|
||||
|
||||
# Gradient checkpointing!
|
||||
if use_gradient_checkpointing == "offloaded":
|
||||
if use_gradient_checkpointing == "unsloth":
|
||||
|
||||
# Saves VRAM!
|
||||
original_model = model
|
||||
|
|
@ -309,11 +309,10 @@ def prepare_n_gradient_checkpoints(
|
|||
pass
|
||||
|
||||
|
||||
class Offloaded_Gradient_Checkpointer(torch.autograd.Function):
|
||||
class Unsloth_Offloaded_Gradient_Checkpointer(torch.autograd.Function):
|
||||
"""
|
||||
Saves VRAM by smartly offloading to RAM.
|
||||
Tiny hit to performance, since we mask the movement via non blocking calls.
|
||||
[TODO] Load the backward pass earlier
|
||||
"""
|
||||
@staticmethod
|
||||
@torch.cuda.amp.custom_fwd
|
||||
|
|
|
|||
|
|
@ -647,7 +647,7 @@ def LlamaModel_fast_forward(
|
|||
past_key_value = past_key_values[idx] if past_key_values is not None else None
|
||||
|
||||
if offloaded_gradient_checkpointing:
|
||||
hidden_states = Offloaded_Gradient_Checkpointer.apply(
|
||||
hidden_states = Unsloth_Offloaded_Gradient_Checkpointer.apply(
|
||||
decoder_layer,
|
||||
hidden_states,
|
||||
causal_mask,
|
||||
|
|
|
|||
|
|
@ -93,7 +93,27 @@ __INT_TO_FLOAT_MAPPER = \
|
|||
"unsloth/mistral-7b-v0.2-bnb-4bit" : (
|
||||
"unsloth/mistral-7b-v0.2",
|
||||
"alpindale/Mistral-7B-v0.2-hf",
|
||||
)
|
||||
),
|
||||
"unsloth/gemma-1.1-2b-it-bnb-4bit" : (
|
||||
"unsloth/gemma-1.1-2b-it",
|
||||
"google/gemma-1.1-2b-it",
|
||||
),
|
||||
"unsloth/gemma-1.1-7b-it-bnb-4bit" : (
|
||||
"unsloth/gemma-1.1-7b-it",
|
||||
"google/gemma-1.1-7b-it",
|
||||
),
|
||||
"unsloth/Starling-LM-7B-beta-bnb-4bit" : (
|
||||
"unsloth/Starling-LM-7B-beta",
|
||||
"Nexusflow/Starling-LM-7B-beta",
|
||||
),
|
||||
"unsloth/Hermes-2-Pro-Mistral-7B-bnb-4bit" : (
|
||||
"unsloth/Hermes-2-Pro-Mistral-7B",
|
||||
"NousResearch/Hermes-2-Pro-Mistral-7B",
|
||||
),
|
||||
"unsloth/OpenHermes-2.5-Mistral-7B-bnb-4bit" : (
|
||||
"unsloth/OpenHermes-2.5-Mistral-7B",
|
||||
"teknium/OpenHermes-2.5-Mistral-7B",
|
||||
),
|
||||
}
|
||||
|
||||
INT_TO_FLOAT_MAPPER = {}
|
||||
|
|
|
|||
|
|
@ -183,6 +183,9 @@ def unsloth_save_model(
|
|||
):
|
||||
if token is None and "HF_TOKEN" in os.environ:
|
||||
token = os.environ["HF_TOKEN"]
|
||||
|
||||
if token is None and "HUGGINGFACE_TOKEN" in os.environ:
|
||||
token = os.environ["HUGGINGFACE_TOKEN"]
|
||||
|
||||
if commit_message is None: commit_message = ""
|
||||
if "Unsloth" not in commit_message:
|
||||
|
|
@ -522,7 +525,11 @@ def unsloth_save_model(
|
|||
|
||||
state_dict["model.norm.weight"] = internal_model.model.norm.weight.data
|
||||
# Check for modules_to_save float32 dtype
|
||||
state_dict["lm_head.weight"] = internal_model.lm_head.weight.data.to(torch_dtype)
|
||||
|
||||
# Check for tied weights
|
||||
if internal_model.model.embed_tokens.weight.data_ptr() != internal_model.lm_head.weight.data_ptr():
|
||||
state_dict["lm_head.weight"] = internal_model.lm_head.weight.data.to(torch_dtype)
|
||||
pass
|
||||
|
||||
# All tensors MUST be type torch.Tensor and not torch.nn.parameter.Parameter
|
||||
for key, value in state_dict.items():
|
||||
|
|
@ -731,9 +738,9 @@ def install_llama_cpp_old(version = -10):
|
|||
# Also don't use the GPU!
|
||||
commands = [
|
||||
"git clone https://github.com/ggerganov/llama.cpp",
|
||||
f"cd llama.cpp && git reset --hard {version} && git clean -df && "\
|
||||
f"make clean make all -j{psutil.cpu_count()*2}",
|
||||
"pip install gguf protobuf",
|
||||
f"cd llama.cpp && git reset --hard {version} && git clean -df",
|
||||
"make clean -C llama.cpp",
|
||||
f"make all -j{psutil.cpu_count()*2} -C llama.cpp",
|
||||
]
|
||||
for command in commands:
|
||||
with subprocess.Popen(command, shell = True, stdout = subprocess.PIPE, bufsize = 1) as sp:
|
||||
|
|
@ -756,7 +763,8 @@ def install_llama_cpp_blocking(use_cuda = True):
|
|||
|
||||
commands = [
|
||||
"git clone https://github.com/ggerganov/llama.cpp",
|
||||
f"cd llama.cpp && make clean && {use_cuda} make all -j{psutil.cpu_count()*2}",
|
||||
"make clean -C llama.cpp",
|
||||
f"{use_cuda} make all -j{psutil.cpu_count()*2} -C llama.cpp",
|
||||
"pip install gguf protobuf",
|
||||
]
|
||||
if os.path.exists("llama.cpp"): return
|
||||
|
|
@ -931,15 +939,26 @@ def save_to_gguf(
|
|||
|
||||
# Check if quantization succeeded!
|
||||
if not os.path.isfile(final_location):
|
||||
raise RuntimeError(
|
||||
f"Unsloth: Quantization failed for {final_location}\n"\
|
||||
"You might have to compile llama.cpp yourself, then run this again.\n"\
|
||||
"You do not need to close this Python program. Run the following commands in a new terminal:\n"\
|
||||
"You must run this in the same folder as you're saving your model.\n"\
|
||||
"git clone https://github.com/ggerganov/llama.cpp\n"\
|
||||
"cd llama.cpp && make clean && LLAMA_CUDA=1 make all -j\n"\
|
||||
"Once that's done, redo the quantization."
|
||||
)
|
||||
if IS_KAGGLE_ENVIRONMENT:
|
||||
raise RuntimeError(
|
||||
f"Unsloth: Quantization failed for {final_location}\n"\
|
||||
"You are in a Kaggle environment, which might be the reason this is failing.\n"\
|
||||
"Kaggle only provides 20GB of disk space. Merging to 16bit for 7b models use 16GB of space.\n"\
|
||||
"This means using `model.{save_pretrained/push_to_hub}_merged` works, but\n"\
|
||||
"`model.{save_pretrained/push_to_hub}_gguf will use too much disk space.\n"\
|
||||
"I suggest you to save the 16bit model first, then use manual llama.cpp conversion."
|
||||
)
|
||||
else:
|
||||
raise RuntimeError(
|
||||
f"Unsloth: Quantization failed for {final_location}\n"\
|
||||
"You might have to compile llama.cpp yourself, then run this again.\n"\
|
||||
"You do not need to close this Python program. Run the following commands in a new terminal:\n"\
|
||||
"You must run this in the same folder as you're saving your model.\n"\
|
||||
"git clone https://github.com/ggerganov/llama.cpp\n"\
|
||||
"cd llama.cpp && make clean && LLAMA_CUDA=1 make all -j\n"\
|
||||
"Once that's done, redo the quantization."
|
||||
)
|
||||
pass
|
||||
pass
|
||||
print(f"Unsloth: Conversion completed! Output location: {final_location}")
|
||||
|
||||
|
|
@ -961,14 +980,25 @@ def save_to_gguf(
|
|||
|
||||
# Check if quantization succeeded!
|
||||
if not os.path.isfile(final_location):
|
||||
raise RuntimeError(
|
||||
"Unsloth: Quantization failed! You might have to compile llama.cpp yourself, then run this again.\n"\
|
||||
"You do not need to close this Python program. Run the following commands in a new terminal:\n"\
|
||||
"You must run this in the same folder as you're saving your model.\n"\
|
||||
"git clone https://github.com/ggerganov/llama.cpp\n"\
|
||||
"cd llama.cpp && make clean && LLAMA_CUDA=1 make all -j\n"\
|
||||
"Once that's done, redo the quantization."
|
||||
)
|
||||
if IS_KAGGLE_ENVIRONMENT:
|
||||
raise RuntimeError(
|
||||
f"Unsloth: Quantization failed for {final_location}\n"\
|
||||
"You are in a Kaggle environment, which might be the reason this is failing.\n"\
|
||||
"Kaggle only provides 20GB of disk space. Merging to 16bit for 7b models use 16GB of space.\n"\
|
||||
"This means using `model.{save_pretrained/push_to_hub}_merged` works, but\n"\
|
||||
"`model.{save_pretrained/push_to_hub}_gguf will use too much disk space.\n"\
|
||||
"I suggest you to save the 16bit model first, then use manual llama.cpp conversion."
|
||||
)
|
||||
else:
|
||||
raise RuntimeError(
|
||||
"Unsloth: Quantization failed! You might have to compile llama.cpp yourself, then run this again.\n"\
|
||||
"You do not need to close this Python program. Run the following commands in a new terminal:\n"\
|
||||
"You must run this in the same folder as you're saving your model.\n"\
|
||||
"git clone https://github.com/ggerganov/llama.cpp\n"\
|
||||
"cd llama.cpp && make clean && LLAMA_CUDA=1 make all -j\n"\
|
||||
"Once that's done, redo the quantization."
|
||||
)
|
||||
pass
|
||||
pass
|
||||
|
||||
print(f"Unsloth: Conversion completed! Output location: {final_location}")
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue