From 696e8817ea42b7de5cbecb056e243f4d5bb471ca Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Tue, 19 Mar 2024 04:32:28 +1100 Subject: [PATCH] Fix GGUF and saving (#261) * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * llama * Update llama.py * gemma * Update cross_entropy_loss.py * Update cross_entropy_loss.py * Update cross_entropy_loss.py * Update cross_entropy_loss.py * Update cross_entropy_loss.py * Update cross_entropy_loss.py * Update cross_entropy_loss.py * Update cross_entropy_loss.py * Update cross_entropy_loss.py * Update cross_entropy_loss.py * Update cross_entropy_loss.py * Update cross_entropy_loss.py * Update cross_entropy_loss.py * Update cross_entropy_loss.py * Update cross_entropy_loss.py * Update cross_entropy_loss.py * Update cross_entropy_loss.py * Update cross_entropy_loss.py * Update cross_entropy_loss.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update save.py * RoPE * Update llama.py * Update llama.py * Update llama.py * Update gemma.py * correct_dtype * Update gemma.py * Update cross_entropy_loss.py * Update cross_entropy_loss.py * Chat Templates * Update README.md * Update README.md * Update llama.py * DoRA * Update _utils.py * Update chat_templates.py * Update llama.py * Hotfix - fix DoRA, Gemma prompt template (#202) (#203) * Update save.py * saving * Update save.py * Update save.py * Update save.py * Update save.py * Update save.py * Update save.py * Update save.py * Update save.py * Update save.py * Update save.py * Update save.py * Update save.py * Update save.py * Update __init__.py * Update save.py * Update save.py * Update save.py * save * trainer * spaces * original * Gemma * Update pyproject.toml * Update mapper.py * Update fast_lora.py * FastGemmaModel * model_type * Update llama.py * Update llama.py * Update gemma.py * Update gemma.py * Update gemma.py * Update llama.py * Update llama.py * Update fast_lora.py * Update llama.py * Update llama.py * Update cross_entropy_loss.py * Update llama.py * Update llama.py * gemma * Update llama.py * Update llama.py * Update llama.py * Update llama.py * Update fast_lora.py * Update fast_lora.py * Fast CE Loss * Update cross_entropy_loss.py * Update cross_entropy_loss.py * Update cross_entropy_loss.py * Update cross_entropy_loss.py * Update cross_entropy_loss.py * Update cross_entropy_loss.py * Update cross_entropy_loss.py * Update llama.py * Update llama.py * Update llama.py * Update llama.py * CE * Update llama.py * Update llama.py * Update cross_entropy_loss.py * Update geglu.py * Update cross_entropy_loss.py * revert * Update llama.py * Update llama.py * norm * Update gemma.py * Update gemma.py * position_ids * Update gemma.py * Update gemma.py * pos * Update llama.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update cross_entropy_loss.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update llama.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update llama.py * Update cross_entropy_loss.py * Update cross_entropy_loss.py * revert * revert * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update llama.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update cross_entropy_loss.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update llama.py * Update llama.py * Update llama.py * Update llama.py * rope * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * llama * Update llama.py * gemma * Update cross_entropy_loss.py * Update cross_entropy_loss.py * Update cross_entropy_loss.py * Update cross_entropy_loss.py * Update cross_entropy_loss.py * Update cross_entropy_loss.py * Update cross_entropy_loss.py * Update cross_entropy_loss.py * Update cross_entropy_loss.py * Update cross_entropy_loss.py * Update cross_entropy_loss.py * Update cross_entropy_loss.py * Update cross_entropy_loss.py * Update cross_entropy_loss.py * Update cross_entropy_loss.py * Update cross_entropy_loss.py * Update cross_entropy_loss.py * Update cross_entropy_loss.py * Update cross_entropy_loss.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update save.py * RoPE * Update llama.py * Update llama.py * Update llama.py * Update gemma.py * correct_dtype * Update gemma.py * Update cross_entropy_loss.py * Update cross_entropy_loss.py * Chat Templates * Update README.md * Update README.md * Update llama.py * DoRA * Update _utils.py * Update chat_templates.py * Update pyproject.toml * Small fixes * Update pyproject.toml * Approx gelu * Update geglu.py * Approx gelu * Update llama.py * Update __init__.py * Update __init__.py * Update _utils.py * Update geglu.py * Update gemma.py * Update rms_layernorm.py * Update rms_layernorm.py * Update rms_layernorm.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Fix Gemma merging * Update rms_layernorm.py * Update gemma.py * Update pyproject.toml * Layernorms * Gemma precision * Update gemma.py * sqrt * Update gemma.py * Update save.py * RoPE and Gemma precision * Update rms_layernorm.py * Fix warning * Update chat_templates.py * Update chat_templates.py * Update save.py * Update save.py * Update save.py * Update chat_templates.py * Update llama.py * model_name * Update loader.py * Tokenizer overwritten * Update llama.py * Update llama.py * Update llama.py * Update save.py * Accuracy * Revert * Update save.py * Update fast_lora.py * Update fast_lora.py * Update fast_lora.py * Update fast_lora.py * Update fast_lora.py * Update chat_templates.py * Update save.py * Update save.py * Update llama.py * Update llama.py * Account for DoRA * Update llama.py * Update save.py * GGUF incorrect * Update save.py * Update pyproject.toml * kaggle new * Update pyproject.toml * Update pyproject.toml * upcasting * Fix Colab * Update pyproject.toml * Update pyproject.toml * Update pyproject.toml * Update pyproject.toml * Update pyproject.toml * Update pyproject.toml * Update pyproject.toml * Update pyproject.toml * Update chat_templates.py * Update chat_templates.py * Update chat_templates.py * Update chat_templates.py * Update chat_templates.py * Update pyproject.toml * Update pyproject.toml * Update pyproject.toml * Update rope_embedding.py * Update rope_embedding.py * Fix bugs * Update fast_lora.py * Update fast_lora.py * Update README.md * Update README.md * GGUF * Update save.py * Update save.py * Update save.py * Update save.py * Update README.md * Update README.md * Bugs * Update fast_lora.py * Update pyproject.toml * Update fast_lora.py * Update __init__.py * Update fast_lora.py * dtype * Update llama.py * Update llama.py * Update llama.py * dtype * Update mistral.py * trust_remote_code * lm_head * Update llama.py * save_pretrained_settings * Update save.py * Update save.py * Update save.py * Update save.py * Update save.py * Update save.py * Update save.py * Update save.py * Update save.py * Update save.py * Update save.py * Update save.py --- unsloth/models/gemma.py | 2 +- unsloth/models/llama.py | 33 +++++- unsloth/models/mistral.py | 3 +- unsloth/save.py | 222 +++++++++++++++++++++++++++++++------- 4 files changed, 217 insertions(+), 43 deletions(-) diff --git a/unsloth/models/gemma.py b/unsloth/models/gemma.py index 7bfec43e51..6bd8a6f345 100644 --- a/unsloth/models/gemma.py +++ b/unsloth/models/gemma.py @@ -143,7 +143,7 @@ pass from math import sqrt as math_sqrt # https://github.com/huggingface/transformers/blob/main/src/transformers/models/llama/modeling_llama.py#L825 -@torch.inference_mode +# @torch.inference_mode def GemmaModel_fast_forward_inference( self, input_ids, diff --git a/unsloth/models/llama.py b/unsloth/models/llama.py index 8b86feb35e..574385fba0 100644 --- a/unsloth/models/llama.py +++ b/unsloth/models/llama.py @@ -657,7 +657,7 @@ pass # https://github.com/huggingface/transformers/blob/main/src/transformers/models/llama/modeling_llama.py#L825 -@torch.inference_mode +# @torch.inference_mode def LlamaModel_fast_forward_inference( self, input_ids, @@ -753,7 +753,8 @@ def CausalLM_fast_forward(fast_forward_inference): hidden_states = outputs[0] bsz, q_len, hd = hidden_states.shape if bsz == 1 and q_len == 1: - logits = torch.mv(self.lm_head.weight, hidden_states.ravel()) + lm_head = self.lm_head.weight + logits = torch.mv(lm_head, hidden_states.ravel().to(lm_head.dtype)) logits = logits.unsqueeze(0).unsqueeze(0) else: logits = self.lm_head(hidden_states) @@ -893,6 +894,16 @@ class LlamaLinearScalingRotaryEmbedding(LlamaRotaryEmbedding): pass +def _wrap_fast_inference(generate, device_type, dtype): + # Wraps inference with bfloat16 / float16 + @torch.inference_mode + def _fast_generate(*args, **kwargs): + with torch.autocast(device_type = device_type, dtype = dtype): + return generate(*args, **kwargs) + return _fast_generate +pass + + class FastLlamaModel: @staticmethod @@ -1581,6 +1592,15 @@ class FastLlamaModel: internal_model.gradient_checkpointing = False internal_model.training = False pass + + # Also check if lm_head / embeddings are trained + lm_head = getattr(model, "model", model).lm_head.weight + device_type = lm_head.device.type + dtype = model.config.torch_dtype + + # Wrap model.generate + model._unwrapped_old_generate = model.generate + model.generate = _wrap_fast_inference(model.generate, device_type, dtype) pass @@ -1601,5 +1621,14 @@ class FastLlamaModel: internal_model.gradient_checkpointing = use_gradient_checkpointing internal_model.training = True pass + + # Also revert model.generate + if hasattr(model, "_unwrapped_old_generate"): + model.generate = model._unwrapped_old_generate + del model._unwrapped_old_generate + pass pass pass + + + diff --git a/unsloth/models/mistral.py b/unsloth/models/mistral.py index 2db71f0288..f650d8f12e 100644 --- a/unsloth/models/mistral.py +++ b/unsloth/models/mistral.py @@ -225,7 +225,8 @@ def MistralForCausalLM_fast_forward( hidden_states = outputs[0] bsz, q_len, hd = hidden_states.shape if bsz == 1 and q_len == 1: - logits = torch.mv(self.lm_head.weight, hidden_states.ravel()) + lm_head = self.lm_head.weight + logits = torch.mv(lm_head, hidden_states.ravel().to(lm_head.dtype)) logits = logits.unsqueeze(0).unsqueeze(0) else: logits = self.lm_head(hidden_states) diff --git a/unsloth/save.py b/unsloth/save.py index 5970d74d11..c8b4053a10 100644 --- a/unsloth/save.py +++ b/unsloth/save.py @@ -32,6 +32,8 @@ __all__ = [ "patch_saving_functions", ] +# Check Kaggle +IS_A_KAGGLE_ENVIRONMENT = "KAGGLE_CONTAINER_NAME" in os.environ LLAMA_WEIGHTS = ( "self_attn.q_proj", "self_attn.k_proj", "self_attn.v_proj", "self_attn.o_proj", @@ -66,9 +68,9 @@ ALLOWED_QUANTS = \ "q5_1" : "Even higher accuracy, resource usage and slower inference.", "q5_k_s" : "Uses Q5_K for all tensors", "q6_k" : "Uses Q8_K for all tensors", - "iq2_xxs" : "2.06 bpw quantization", - "iq2_xs" : "2.31 bpw quantization", - "iq3_xxs" : "3.06 bpw quantization", + # "iq2_xxs" : "2.06 bpw quantization", # Not supported sadly + # "iq2_xs" : "2.31 bpw quantization", + # "iq3_xxs" : "3.06 bpw quantization", "q3_k_xs" : "3-bit extra small quantization", } @@ -79,6 +81,27 @@ def print_quantization_methods(): pass +def _free_cached_model(model): + from huggingface_hub import scan_cache_dir + cached_repos = list(scan_cache_dir().repos) + + # Go through every cached repo, and delete the one that matches the model we want to save. + # Can save 4GB of disk space - useful for Kaggle systems. + for cached_repo in cached_repos: + if cached_repo.repo_id == model.config._name_or_path: + remove_cache_commit = list(cached_repo.revisions)[0].commit_hash + delete_strategy = scan_cache_dir().delete_revisions(remove_cache_commit,) + + logger.warning_once( + "Unsloth: Will remove a cached repo with size " + \ + delete_strategy.expected_freed_size_str, + ) + + delete_strategy.execute() + pass + pass +pass + def _merge_lora(layer, name): @@ -153,6 +176,19 @@ def unsloth_save_model( temporary_location : str = "_unsloth_temporary_saved_buffers", maximum_memory_usage : float = 0.9, ): + # First check for a token! + if push_to_hub: + from huggingface_hub import whoami + try: + username = whoami(token = token)["name"] + except: + raise RuntimeError( + "Unsloth: Please supply a token!\n"\ + "Go to https://huggingface.co/settings/tokens" + ) + pass + pass + if commit_message is None: commit_message = "" if "Unsloth" not in commit_message: commit_message += " (Trained with Unsloth)" @@ -411,6 +447,16 @@ def unsloth_save_model( os.makedirs(temporary_location) pass + # Check if Kaggle, since only 20GB of Disk space allowed. + if IS_A_KAGGLE_ENVIRONMENT: + # We free up 4GB of space + logger.warning_once( + "Unsloth: Kaggle only allows 20GB of disk space. We need to delete the downloaded\n"\ + "model which will save 4GB of disk space, allowing you to save on Kaggle." + ) + _free_cached_model(internal_model) + pass + # HF also uses a OrderedDict from collections import OrderedDict state_dict = OrderedDict() @@ -480,12 +526,35 @@ def unsloth_save_model( ) pass + # First check if we're pushing to an organization! + save_directory = save_pretrained_settings["save_directory"] + + if save_pretrained_settings["push_to_hub"]: + new_save_directory, new_username = _determine_username(save_directory, username, token) + + if token is not None: + from huggingface_hub import whoami + actual_username = whoami(token = token)["name"] + else: + actual_username = username + pass + + # Check if pushing to an organization + if save_pretrained_settings["push_to_hub"] and (username != actual_username): + print(f"Unsloth: Saving to organization with address {new_save_directory}") + # We upload everything at the end! + tokenizer_save_settings["push_to_hub"] = False + tokenizer_save_settings["save_directory"] = new_save_directory + pass + + # Save tokenizer if tokenizer is not None: print("Unsloth: Saving tokenizer...", end = "") tokenizer.save_pretrained(**tokenizer_save_settings) print(" Done.") else: print() + pass print("Unsloth: Saving model... This might take 5 minutes for Llama-7b...") @@ -502,7 +571,35 @@ def unsloth_save_model( model.config = new_config # Save! - internal_model.save_pretrained(**save_pretrained_settings) + + # Check if pushing to an organization + if save_pretrained_settings["push_to_hub"] and (username != actual_username): + print(f"Unsloth: Saving to organization with address {new_save_directory}") + # Pushing to organization! + # Sadly .save_pretrained doesn't work :( + # We first save it via .save_pretrained, then upload manually! + save_pretrained_settings["save_directory"] = new_save_directory + save_pretrained_settings["push_to_hub"] = False + internal_model.save_pretrained(**save_pretrained_settings) + + # Now manually go through each file and upload them manually! + filenames = os.listdir(new_save_directory) + + from huggingface_hub import HfApi + hf_api = HfApi(token = save_pretrained_settings["token"]) + + print("Unsloth: Uploading all files... Please wait!") + hf_api.upload_folder( + folder_path = new_save_directory, + path_in_repo = ".", + repo_id = new_save_directory, + repo_type = "model", + commit_message = "(Trained with Unsloth)", + ignore_patterns = "*.md", + ) + else: + internal_model.save_pretrained(**save_pretrained_settings) + pass # Revert config back original_model = model @@ -616,13 +713,16 @@ def install_llama_cpp_old(version = -10): pass -def install_llama_cpp_blocking(): +def install_llama_cpp_blocking(use_cuda = True): + use_cuda = "LLAMA_CUBLAS=1" if use_cuda else "" + commands = [ "git clone https://github.com/ggerganov/llama.cpp", - f"cd llama.cpp && make clean && LLAMA_CUBLAS=1 make all -j{psutil.cpu_count()*2}", + f"cd llama.cpp && make clean && {use_cuda} make all -j{psutil.cpu_count()*2}", "pip install gguf protobuf", ] if os.path.exists("llama.cpp"): return + for command in commands: with subprocess.Popen(command, shell = True, stdout = subprocess.PIPE, bufsize = 1) as sp: for line in sp.stdout: @@ -954,17 +1054,8 @@ This {model_type} model was trained 2x faster with [Unsloth](https://github.com/ [](https://github.com/unslothai/unsloth) """ -def upload_to_huggingface( - model, - save_directory, - token, - method, - extra = "", - file_location = None, - old_username = None, - private = None, -): - # Check for username + +def _determine_username(save_directory, old_username, token): username = "" save_directory = save_directory.lstrip("./") if "/" not in save_directory: @@ -980,6 +1071,21 @@ def upload_to_huggingface( else: username = save_directory.split("/")[0] pass + return save_directory, username +pass + + +def upload_to_huggingface( + model, + save_directory, + token, + method, + extra = "", + file_location = None, + old_username = None, + private = None, +): + save_directory, username = _determine_username(save_directory, old_username, token) from huggingface_hub import create_repo try: @@ -1107,18 +1213,15 @@ def unsloth_save_pretrained_gguf( # Non blocking install GGUF first if not os.path.exists("llama.cpp"): - git_clone = install_llama_cpp_clone_non_blocking() - python_install = install_python_non_blocking(["gguf", "protobuf"]) - git_clone.wait() - makefile = install_llama_cpp_make_non_blocking() - new_save_directory, old_username = unsloth_save_model(**arguments) - python_install.wait() - else: - try: + + if IS_A_KAGGLE_ENVIRONMENT: + # Kaggle is weird - no blocking installs, and no CUDA? + python_install = install_python_non_blocking(["gguf", "protobuf"]) + python_install.wait() + install_llama_cpp_blocking(use_cuda = False) new_save_directory, old_username = unsloth_save_model(**arguments) makefile = None - except: - # Retry by recloning llama.cpp + else: git_clone = install_llama_cpp_clone_non_blocking() python_install = install_python_non_blocking(["gguf", "protobuf"]) git_clone.wait() @@ -1126,6 +1229,28 @@ def unsloth_save_pretrained_gguf( new_save_directory, old_username = unsloth_save_model(**arguments) python_install.wait() pass + else: + try: + new_save_directory, old_username = unsloth_save_model(**arguments) + makefile = None + except: + # Retry by recloning llama.cpp + if IS_A_KAGGLE_ENVIRONMENT: + # Kaggle is weird - no blocking installs, and no CUDA? + python_install = install_python_non_blocking(["gguf", "protobuf"]) + python_install.wait() + install_llama_cpp_blocking(use_cuda = False) + new_save_directory, old_username = unsloth_save_model(**arguments) + makefile = None + else: + git_clone = install_llama_cpp_clone_non_blocking() + python_install = install_python_non_blocking(["gguf", "protobuf"]) + git_clone.wait() + makefile = install_llama_cpp_make_non_blocking() + new_save_directory, old_username = unsloth_save_model(**arguments) + python_install.wait() + pass + pass pass for _ in range(3): @@ -1208,24 +1333,43 @@ def unsloth_push_to_hub_gguf( # Non blocking install GGUF first if not os.path.exists("llama.cpp"): - git_clone = install_llama_cpp_clone_non_blocking() - python_install = install_python_non_blocking(["gguf", "protobuf"]) - git_clone.wait() - makefile = install_llama_cpp_make_non_blocking() - new_save_directory, old_username = unsloth_save_model(**arguments) - python_install.wait() + + if IS_A_KAGGLE_ENVIRONMENT: + # Kaggle is weird - no blocking installs, and no CUDA? + python_install = install_python_non_blocking(["gguf", "protobuf"]) + python_install.wait() + install_llama_cpp_blocking(use_cuda = False) + new_save_directory, old_username = unsloth_save_model(**arguments) + makefile = None + else: + git_clone = install_llama_cpp_clone_non_blocking() + python_install = install_python_non_blocking(["gguf", "protobuf"]) + git_clone.wait() + makefile = install_llama_cpp_make_non_blocking() + new_save_directory, old_username = unsloth_save_model(**arguments) + python_install.wait() + pass else: try: new_save_directory, old_username = unsloth_save_model(**arguments) makefile = None except: # Retry by recloning llama.cpp - git_clone = install_llama_cpp_clone_non_blocking() - python_install = install_python_non_blocking(["gguf", "protobuf"]) - git_clone.wait() - makefile = install_llama_cpp_make_non_blocking() - new_save_directory, old_username = unsloth_save_model(**arguments) - python_install.wait() + if IS_A_KAGGLE_ENVIRONMENT: + # Kaggle is weird - no blocking installs, and no CUDA? + python_install = install_python_non_blocking(["gguf", "protobuf"]) + python_install.wait() + install_llama_cpp_blocking(use_cuda = False) + new_save_directory, old_username = unsloth_save_model(**arguments) + makefile = None + else: + git_clone = install_llama_cpp_clone_non_blocking() + python_install = install_python_non_blocking(["gguf", "protobuf"]) + git_clone.wait() + makefile = install_llama_cpp_make_non_blocking() + new_save_directory, old_username = unsloth_save_model(**arguments) + python_install.wait() + pass pass pass