Fix Gemma GGUF (#311)
* Update gemma.py * Fix Gemma merging * Update rms_layernorm.py * Update gemma.py * Update pyproject.toml * Layernorms * Gemma precision * Update gemma.py * sqrt * Update gemma.py * Update save.py * RoPE and Gemma precision * Update rms_layernorm.py * Fix warning * Update chat_templates.py * Update chat_templates.py * Update save.py * Update save.py * Update save.py * Update chat_templates.py * Update llama.py * model_name * Update loader.py * Tokenizer overwritten * Update llama.py * Update llama.py * Update llama.py * Update save.py * Accuracy * Revert * Update save.py * Update fast_lora.py * Update fast_lora.py * Update fast_lora.py * Update fast_lora.py * Update fast_lora.py * Update chat_templates.py * Update save.py * Update save.py * Update llama.py * Update llama.py * Account for DoRA * Update llama.py * Update save.py * GGUF incorrect * Update save.py * Update pyproject.toml * kaggle new * Update pyproject.toml * Update pyproject.toml * upcasting * Fix Colab * Update pyproject.toml * Update pyproject.toml * Update pyproject.toml * Update pyproject.toml * Update pyproject.toml * Update pyproject.toml * Update pyproject.toml * Update pyproject.toml * Update chat_templates.py * Update chat_templates.py * Update chat_templates.py * Update chat_templates.py * Update chat_templates.py * Update pyproject.toml * Update pyproject.toml * Update pyproject.toml * Update rope_embedding.py * Update rope_embedding.py * Fix bugs * Update fast_lora.py * Update fast_lora.py * Update README.md * Update README.md * GGUF * Update save.py * Update save.py * Update save.py * Update save.py * Update README.md * Update README.md * Bugs * Update fast_lora.py * Update pyproject.toml * Update fast_lora.py * Update __init__.py * Update fast_lora.py * dtype * Update llama.py * Update llama.py * Update llama.py * dtype * Update mistral.py * trust_remote_code * lm_head * Update llama.py * save_pretrained_settings * Update save.py * Update save.py * Update save.py * Update save.py * Update save.py * Update save.py * Update save.py * Update save.py * Update save.py * Update save.py * Update save.py * Update save.py * state_dict * Update save.py * whoami * Update llama.py * Update save.py * Update llama.py * Patch tokenizer * Update chat_templates.py * Heal tokenizers * Update chat_templates.py * Update mapper.py * Update tokenizer_utils.py * Update tokenizer_utils.py * Update tokenizer_utils.py * Update tokenizer_utils.py * Update tokenizer_utils.py * Update chat_templates.py * tokenizer patching * patch_tokenizer * Update chat_templates.py * Update tokenizer_utils.py * Update chat_templates.py * Update chat_templates.py * Update chat_templates.py * Update tokenizer_utils.py * Edit * Update mistral.py * Update mistral.py * Stats * Update mistral.py * attention_mask * Update llama.py * Update llama.py * batch * Temp fix batch inference * Update llama.py * Update gemma.py * Fix inference * swiglu * Update llama.py * Update llama.py * Update llama.py * Update llama.py * Update mistral.py * Update llama.py * fast inference * model * Update llama.py * Update llama.py * Update llama.py * Update llama.py * Update llama.py * Update llama.py * Update llama.py * Update llama.py * Update llama.py * Update llama.py * Update llama.py * Update llama.py * Update llama.py * Update utils.py * Update llama.py * Update utils.py * inference * Update llama.py * Update llama.py * Update llama.py * overhead * Update llama.py * Update llama.py * compile * Update gemma.py * Update llama.py * Update llama.py * Update llama.py * Update llama.py * Update utils.py * Update utils.py * lora mamtul * Update llama.py * Update llama.py * Update llama.py * offloaded checkpointing * Update llama.py * Update llama.py * Update _utils.py * Update _utils.py * Update _utils.py * Update llama.py * Update llama.py * Update gemma.py * Revert "Update gemma.py" This reverts commit e3c3c5f3fa3d04a87f854056f6b547ced610d712. * Update _utils.py * Update _utils.py * Update _utils.py * Saving * sentencepiece_model_pb2 * Update llama.py * Update save.py * Update llama.py * padding side * Update tokenizer_utils.py * cache dir * Update tokenizer_utils.py * Update tokenizer_utils.py * Update pyproject.toml * Update pyproject.toml * Update tokenizer_utils.py * Update tokenizer_utils.py * Update llama.py * Update save.py * Update save.py * checkpoint * Gemma 1.1 * more models
This commit is contained in:
Daniel Han
GitHub
parent
cb7924e9db
commit
0dc7729292
4 changed files with 77 additions and 28 deletions
|
|
@ -70,7 +70,7 @@ __all__ = [
|
|||
"platform_system",
|
||||
"patch_tokenizer",
|
||||
"get_statistics",
|
||||
"Offloaded_Gradient_Checkpointer",
|
||||
"Unsloth_Offloaded_Gradient_Checkpointer",
|
||||
]
|
||||
|
||||
|
||||
|
|
@ -103,7 +103,7 @@ def prepare_model_for_kbit_training(
|
|||
pass
|
||||
|
||||
# Gradient checkpointing!
|
||||
if use_gradient_checkpointing == "offloaded":
|
||||
if use_gradient_checkpointing == "unsloth":
|
||||
|
||||
# Saves VRAM!
|
||||
original_model = model
|
||||
|
|
@ -309,11 +309,10 @@ def prepare_n_gradient_checkpoints(
|
|||
pass
|
||||
|
||||
|
||||
class Offloaded_Gradient_Checkpointer(torch.autograd.Function):
|
||||
class Unsloth_Offloaded_Gradient_Checkpointer(torch.autograd.Function):
|
||||
"""
|
||||
Saves VRAM by smartly offloading to RAM.
|
||||
Tiny hit to performance, since we mask the movement via non blocking calls.
|
||||
[TODO] Load the backward pass earlier
|
||||
"""
|
||||
@staticmethod
|
||||
@torch.cuda.amp.custom_fwd
|
||||
|
|
|
|||
|
|
@ -647,7 +647,7 @@ def LlamaModel_fast_forward(
|
|||
past_key_value = past_key_values[idx] if past_key_values is not None else None
|
||||
|
||||
if offloaded_gradient_checkpointing:
|
||||
hidden_states = Offloaded_Gradient_Checkpointer.apply(
|
||||
hidden_states = Unsloth_Offloaded_Gradient_Checkpointer.apply(
|
||||
decoder_layer,
|
||||
hidden_states,
|
||||
causal_mask,
|
||||
|
|
|
|||
|
|
@ -93,7 +93,27 @@ __INT_TO_FLOAT_MAPPER = \
|
|||
"unsloth/mistral-7b-v0.2-bnb-4bit" : (
|
||||
"unsloth/mistral-7b-v0.2",
|
||||
"alpindale/Mistral-7B-v0.2-hf",
|
||||
)
|
||||
),
|
||||
"unsloth/gemma-1.1-2b-it-bnb-4bit" : (
|
||||
"unsloth/gemma-1.1-2b-it",
|
||||
"google/gemma-1.1-2b-it",
|
||||
),
|
||||
"unsloth/gemma-1.1-7b-it-bnb-4bit" : (
|
||||
"unsloth/gemma-1.1-7b-it",
|
||||
"google/gemma-1.1-7b-it",
|
||||
),
|
||||
"unsloth/Starling-LM-7B-beta-bnb-4bit" : (
|
||||
"unsloth/Starling-LM-7B-beta",
|
||||
"Nexusflow/Starling-LM-7B-beta",
|
||||
),
|
||||
"unsloth/Hermes-2-Pro-Mistral-7B-bnb-4bit" : (
|
||||
"unsloth/Hermes-2-Pro-Mistral-7B",
|
||||
"NousResearch/Hermes-2-Pro-Mistral-7B",
|
||||
),
|
||||
"unsloth/OpenHermes-2.5-Mistral-7B-bnb-4bit" : (
|
||||
"unsloth/OpenHermes-2.5-Mistral-7B",
|
||||
"teknium/OpenHermes-2.5-Mistral-7B",
|
||||
),
|
||||
}
|
||||
|
||||
INT_TO_FLOAT_MAPPER = {}
|
||||
|
|
|
|||
|
|
@ -183,6 +183,9 @@ def unsloth_save_model(
|
|||
):
|
||||
if token is None and "HF_TOKEN" in os.environ:
|
||||
token = os.environ["HF_TOKEN"]
|
||||
|
||||
if token is None and "HUGGINGFACE_TOKEN" in os.environ:
|
||||
token = os.environ["HUGGINGFACE_TOKEN"]
|
||||
|
||||
if commit_message is None: commit_message = ""
|
||||
if "Unsloth" not in commit_message:
|
||||
|
|
@ -522,7 +525,11 @@ def unsloth_save_model(
|
|||
|
||||
state_dict["model.norm.weight"] = internal_model.model.norm.weight.data
|
||||
# Check for modules_to_save float32 dtype
|
||||
state_dict["lm_head.weight"] = internal_model.lm_head.weight.data.to(torch_dtype)
|
||||
|
||||
# Check for tied weights
|
||||
if internal_model.model.embed_tokens.weight.data_ptr() != internal_model.lm_head.weight.data_ptr():
|
||||
state_dict["lm_head.weight"] = internal_model.lm_head.weight.data.to(torch_dtype)
|
||||
pass
|
||||
|
||||
# All tensors MUST be type torch.Tensor and not torch.nn.parameter.Parameter
|
||||
for key, value in state_dict.items():
|
||||
|
|
@ -731,9 +738,9 @@ def install_llama_cpp_old(version = -10):
|
|||
# Also don't use the GPU!
|
||||
commands = [
|
||||
"git clone https://github.com/ggerganov/llama.cpp",
|
||||
f"cd llama.cpp && git reset --hard {version} && git clean -df && "\
|
||||
f"make clean make all -j{psutil.cpu_count()*2}",
|
||||
"pip install gguf protobuf",
|
||||
f"cd llama.cpp && git reset --hard {version} && git clean -df",
|
||||
"make clean -C llama.cpp",
|
||||
f"make all -j{psutil.cpu_count()*2} -C llama.cpp",
|
||||
]
|
||||
for command in commands:
|
||||
with subprocess.Popen(command, shell = True, stdout = subprocess.PIPE, bufsize = 1) as sp:
|
||||
|
|
@ -756,7 +763,8 @@ def install_llama_cpp_blocking(use_cuda = True):
|
|||
|
||||
commands = [
|
||||
"git clone https://github.com/ggerganov/llama.cpp",
|
||||
f"cd llama.cpp && make clean && {use_cuda} make all -j{psutil.cpu_count()*2}",
|
||||
"make clean -C llama.cpp",
|
||||
f"{use_cuda} make all -j{psutil.cpu_count()*2} -C llama.cpp",
|
||||
"pip install gguf protobuf",
|
||||
]
|
||||
if os.path.exists("llama.cpp"): return
|
||||
|
|
@ -931,15 +939,26 @@ def save_to_gguf(
|
|||
|
||||
# Check if quantization succeeded!
|
||||
if not os.path.isfile(final_location):
|
||||
raise RuntimeError(
|
||||
f"Unsloth: Quantization failed for {final_location}\n"\
|
||||
"You might have to compile llama.cpp yourself, then run this again.\n"\
|
||||
"You do not need to close this Python program. Run the following commands in a new terminal:\n"\
|
||||
"You must run this in the same folder as you're saving your model.\n"\
|
||||
"git clone https://github.com/ggerganov/llama.cpp\n"\
|
||||
"cd llama.cpp && make clean && LLAMA_CUDA=1 make all -j\n"\
|
||||
"Once that's done, redo the quantization."
|
||||
)
|
||||
if IS_KAGGLE_ENVIRONMENT:
|
||||
raise RuntimeError(
|
||||
f"Unsloth: Quantization failed for {final_location}\n"\
|
||||
"You are in a Kaggle environment, which might be the reason this is failing.\n"\
|
||||
"Kaggle only provides 20GB of disk space. Merging to 16bit for 7b models use 16GB of space.\n"\
|
||||
"This means using `model.{save_pretrained/push_to_hub}_merged` works, but\n"\
|
||||
"`model.{save_pretrained/push_to_hub}_gguf will use too much disk space.\n"\
|
||||
"I suggest you to save the 16bit model first, then use manual llama.cpp conversion."
|
||||
)
|
||||
else:
|
||||
raise RuntimeError(
|
||||
f"Unsloth: Quantization failed for {final_location}\n"\
|
||||
"You might have to compile llama.cpp yourself, then run this again.\n"\
|
||||
"You do not need to close this Python program. Run the following commands in a new terminal:\n"\
|
||||
"You must run this in the same folder as you're saving your model.\n"\
|
||||
"git clone https://github.com/ggerganov/llama.cpp\n"\
|
||||
"cd llama.cpp && make clean && LLAMA_CUDA=1 make all -j\n"\
|
||||
"Once that's done, redo the quantization."
|
||||
)
|
||||
pass
|
||||
pass
|
||||
print(f"Unsloth: Conversion completed! Output location: {final_location}")
|
||||
|
||||
|
|
@ -961,14 +980,25 @@ def save_to_gguf(
|
|||
|
||||
# Check if quantization succeeded!
|
||||
if not os.path.isfile(final_location):
|
||||
raise RuntimeError(
|
||||
"Unsloth: Quantization failed! You might have to compile llama.cpp yourself, then run this again.\n"\
|
||||
"You do not need to close this Python program. Run the following commands in a new terminal:\n"\
|
||||
"You must run this in the same folder as you're saving your model.\n"\
|
||||
"git clone https://github.com/ggerganov/llama.cpp\n"\
|
||||
"cd llama.cpp && make clean && LLAMA_CUDA=1 make all -j\n"\
|
||||
"Once that's done, redo the quantization."
|
||||
)
|
||||
if IS_KAGGLE_ENVIRONMENT:
|
||||
raise RuntimeError(
|
||||
f"Unsloth: Quantization failed for {final_location}\n"\
|
||||
"You are in a Kaggle environment, which might be the reason this is failing.\n"\
|
||||
"Kaggle only provides 20GB of disk space. Merging to 16bit for 7b models use 16GB of space.\n"\
|
||||
"This means using `model.{save_pretrained/push_to_hub}_merged` works, but\n"\
|
||||
"`model.{save_pretrained/push_to_hub}_gguf will use too much disk space.\n"\
|
||||
"I suggest you to save the 16bit model first, then use manual llama.cpp conversion."
|
||||
)
|
||||
else:
|
||||
raise RuntimeError(
|
||||
"Unsloth: Quantization failed! You might have to compile llama.cpp yourself, then run this again.\n"\
|
||||
"You do not need to close this Python program. Run the following commands in a new terminal:\n"\
|
||||
"You must run this in the same folder as you're saving your model.\n"\
|
||||
"git clone https://github.com/ggerganov/llama.cpp\n"\
|
||||
"cd llama.cpp && make clean && LLAMA_CUDA=1 make all -j\n"\
|
||||
"Once that's done, redo the quantization."
|
||||
)
|
||||
pass
|
||||
pass
|
||||
|
||||
print(f"Unsloth: Conversion completed! Output location: {final_location}")
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue