From a16bc73e8077fd3c6a034741ae782bcfeb9fa278 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Mon, 29 Jan 2024 02:52:39 +1100 Subject: [PATCH] Fix saving issues (#139) * faster saving & inference * Update llama.py * Update save.py * Update llama.py * Update llama.py * Update llama.py * Update llama.py * Update mistral.py * Update llama.py * Update llama.py * Update llama.py * Update llama.py * Update llama.py * fast inference * Update llama.py * Update save.py * Update llama.py * Mistral correct RoPE scaling * Max sequence lengths * Apache 2 * fast_linear_forward * Update utils.py * Update utils.py * No print * Update utils.py * Update utils.py * inference * Update llama.py * Fast inference RoPE * Update llama.py * Update llama.py * RoPE * Update llama.py * Update llama.py * Update llama.py * Update llama.py * Update llama.py * Update llama.py * Update llama.py * Update llama.py * LoRA * Fast LoRA saving * Update llama.py * hidden_states * q_len == 1 * q_len issue * Update mistral.py * Update mistral.py * incorrect inference * Update to transformers 4.37 * Graceful FA2 error + torch 2.1.1 * Update mapper.py * Update pyproject.toml * Fix saving and bnb-4bit * Update fast_lora.py * Update fast_lora.py * remove patching * Update llama.py * Update llama.py * Update swiglu.py * Repatch * Update fast_lora.py * Update fast_lora.py * Update fast_lora.py * Update fast_lora.py * Update fast_lora.py * Update fast_lora.py * Update fast_lora.py * Update fast_lora.py * Update fast_lora.py * Update fast_lora.py * Update llama.py * Update fast_lora.py * Update llama.py * Update fast_lora.py * Update fast_lora.py * Update fast_lora.py * Update fast_lora.py * Update fast_lora.py * Update fast_lora.py * Update fast_lora.py * Update fast_lora.py * Update fast_lora.py * Update swiglu.py * Update fast_lora.py * Update swiglu.py * Update fast_lora.py * Update fast_lora.py * Update fast_lora.py * Update fast_lora.py * Update fast_lora.py * Update fast_lora.py * Update fast_lora.py * Update fast_lora.py * Update fast_lora.py * Update fast_lora.py * Update fast_lora.py * Update fast_lora.py * Update fast_lora.py * Update fast_lora.py * Update fast_lora.py * Update fast_lora.py * Update save.py * Update fast_lora.py * Update utils.py * Update llama.py * Update fast_lora.py * Update swiglu.py * Update save.py * Update save.py * Update llama.py * Update llama.py * Update llama.py * Update llama.py * Revert "Update llama.py" This reverts commit a208ec46e012cf470ecefe6268a66358215df7b6. * Update llama.py * Works? * Update pyproject.toml * Update fast_lora.py * Update fast_lora.py * Update fast_lora.py * Update fast_lora.py * Update fast_lora.py * Swiglu * Update swiglu.py * Update fast_lora.py * Update fast_lora.py * Update fast_lora.py * Update fast_lora.py * Update swiglu.py * Update fast_lora.py * Update fast_lora.py * Update fast_lora.py * Update fast_lora.py * Update fast_lora.py * Update fast_lora.py * Update fast_lora.py * Update fast_lora.py * attention_mask * Update llama.py * Update llama.py * labels * Update mistral.py * Update llama.py * attention mask * Update save.py * Update save.py * Update mistral.py * attention mask * Update llama.py * Update llama.py * Update mistral.py * Update llama.py * Update llama.py * Update llama.py * Update dpo.py * Patch saving * Update save.py * Update save.py * patch_saving_functions * Update save.py * Update save.py * Update save.py * Update save.py * Update save.py * Update save.py * Update save.py * Update save.py * print --- unsloth/models/dpo.py | 13 ++-- unsloth/models/llama.py | 15 ++++- unsloth/models/mistral.py | 3 +- unsloth/save.py | 130 +++++++++++++++++++++++++------------- 4 files changed, 111 insertions(+), 50 deletions(-) diff --git a/unsloth/models/dpo.py b/unsloth/models/dpo.py index 92fde81fd0..3ae4d636f4 100644 --- a/unsloth/models/dpo.py +++ b/unsloth/models/dpo.py @@ -101,10 +101,13 @@ pass def PatchDPOTrainer(): - # Patch DPO notebook printing - NotebookTrainingTracker.write_line = NotebookTrainingTracker_write_line - from transformers.trainer import DEFAULT_PROGRESS_CALLBACK - DEFAULT_PROGRESS_CALLBACK.on_train_begin = NotebookProgressCallback_on_train_begin - DEFAULT_PROGRESS_CALLBACK.on_log = NotebookProgressCallback_on_log + from transformers.trainer import is_in_notebook + if is_in_notebook(): + # Patch DPO notebook printing + NotebookTrainingTracker.write_line = NotebookTrainingTracker_write_line + from transformers.trainer import DEFAULT_PROGRESS_CALLBACK + DEFAULT_PROGRESS_CALLBACK.on_train_begin = NotebookProgressCallback_on_train_begin + DEFAULT_PROGRESS_CALLBACK.on_log = NotebookProgressCallback_on_log + pass pass diff --git a/unsloth/models/llama.py b/unsloth/models/llama.py index d5dd7833d5..fcaa2a19ed 100644 --- a/unsloth/models/llama.py +++ b/unsloth/models/llama.py @@ -486,6 +486,15 @@ def LlamaModel_fast_forward( if inputs_embeds is None: inputs_embeds = self.embed_tokens(input_ids) + # Fix up attention mask by setting elements to 0 + # Specifically for DPO + if self._has_no_labels and attention_mask is not None: + inputs_requires_grad = inputs_embeds.requires_grad + if inputs_requires_grad: inputs_embeds.requires_grad_(False) + inputs_embeds *= attention_mask.unsqueeze(0).transpose(0, 1).transpose(1, 2) + if inputs_requires_grad: inputs_embeds.requires_grad_(True) + pass + # Ignore attention_mask if attention_mask is None: padding_mask = None @@ -617,6 +626,7 @@ def LlamaForCausalLM_fast_forward( return_dict = return_dict if return_dict is not None else self.config.use_return_dict # decoder outputs consists of (dec_features, layer_state, dec_hidden, dec_attn) + self.model._has_no_labels = labels is None outputs = self.model( input_ids=input_ids, causal_mask=causal_mask, @@ -726,7 +736,7 @@ class FastLlamaModel: f"O^O/ \_/ \\ Pytorch: {torch.__version__}. CUDA = {gpu_stats.major}.{gpu_stats.minor}. CUDA Toolkit = {torch.version.cuda}.\n"\ f"\ / Bfloat16 = {str(SUPPORTS_BFLOAT16).upper()}. Xformers = {xformers_version}. FA = {HAS_FLASH_ATTENTION}.\n"\ f' "-____-" Free Apache license: http://github.com/unslothai/unsloth' - logger.warning_once(statistics) + print(statistics) FastLlamaModel.pre_patch() if dtype is None: @@ -826,6 +836,9 @@ class FastLlamaModel: # Log Unsloth version for future fastpaths for inference model.config.update({"unsloth_version" : __version__}) + # Add save modules + patch_saving_functions(model) + return model, tokenizer pass diff --git a/unsloth/models/mistral.py b/unsloth/models/mistral.py index 2410572174..2941fb32fb 100644 --- a/unsloth/models/mistral.py +++ b/unsloth/models/mistral.py @@ -195,6 +195,7 @@ def MistralForCausalLM_fast_forward( return_dict = return_dict if return_dict is not None else self.config.use_return_dict # decoder outputs consists of (dec_features, layer_state, dec_hidden, dec_attn) + self.model._has_no_labels = labels is None outputs = self.model( input_ids=input_ids, causal_mask=causal_mask, @@ -282,7 +283,7 @@ class FastMistralModel(FastLlamaModel): f"O^O/ \_/ \\ Pytorch: {torch.__version__}. CUDA = {gpu_stats.major}.{gpu_stats.minor}. CUDA Toolkit = {torch.version.cuda}.\n"\ f"\ / Bfloat16 = {str(SUPPORTS_BFLOAT16).upper()}. Xformers = {xformers_version}. FA = {HAS_FLASH_ATTENTION}.\n"\ f' "-____-" Apache 2 free license: http://github.com/unslothai/unsloth' - logger.warning_once(statistics) + print(statistics) FastMistralModel.pre_patch() if dtype is None: diff --git a/unsloth/save.py b/unsloth/save.py index 744ec48327..baa8f3f5ca 100644 --- a/unsloth/save.py +++ b/unsloth/save.py @@ -278,7 +278,7 @@ def unsloth_save_model( not hasattr(internal_model.model, "layers") ): # Do general saving - + print(type(model)) # Edit save_pretrained_settings # [TODO] _create_repo has errors due to **kwargs getting accepted for deletion in \ @@ -483,7 +483,7 @@ def install_llama_cpp_make_non_blocking(): n_jobs = max(int(psutil.cpu_count()*1.5), 1) # Force make clean os.system("make clean -C llama.cpp") - full_command = ["make", "-j", str(n_jobs), "-C", "llama.cpp"] + full_command = ["make", "all", "-j", str(n_jobs), "-C", "llama.cpp"] run_installer = subprocess.Popen(full_command, env = env, stdout = subprocess.DEVNULL, stderr = subprocess.STDOUT) return run_installer pass @@ -499,7 +499,7 @@ pass def install_llama_cpp_blocking(): commands = [ "git clone https://github.com/ggerganov/llama.cpp", - f"cd llama.cpp && make clean && LLAMA_CUBLAS=1 make -j {psutil.cpu_count()*2}", + f"cd llama.cpp && make clean && LLAMA_CUBLAS=1 make all -j {psutil.cpu_count()*2}", "pip install gguf protobuf", ] if os.path.exists("llama.cpp"): return @@ -515,6 +515,7 @@ pass def save_to_gguf( model_directory : str = "unsloth_finetuned_model", quantization_method : str = "fast_quantized", + first_conversion : str = "f16", _run_installer = None, # Non blocking install of llama.cpp ): from transformers.models.llama.modeling_llama import logger @@ -539,6 +540,16 @@ def save_to_gguf( f' "-____-" In total, you will have to wait around 26 minutes.\n' print(print_info) + # Check first_conversion format + if first_conversion == "f16" : pass + elif first_conversion == "f32" : pass + elif first_conversion == "q8_0": pass + else: + raise RuntimeError( + f"Unsloth: `first_conversion` can only be one of ['f16', 'f32', 'q8_0'] and not `{first_conversion}`." + ) + pass + print("Unsloth: [0] Installing llama.cpp. This will take 3 minutes...") if _run_installer is not None: _run_installer.wait() @@ -546,11 +557,19 @@ def save_to_gguf( install_llama_cpp_blocking() pass - print("Unsloth: [1] Converting HF into GGUF format. This will take 3 minutes...") - first_conversion = "f16" if quantization_method == "f32": first_conversion = "f32" elif quantization_method == "f16": first_conversion = "f16" elif quantization_method == "q8_0": first_conversion = "q8_0" + else: + # Quantized models must have f16 as the default argument + if first_conversion == "f32" : pass + elif first_conversion == "f16" : pass + elif first_conversion == "q8_0": + logger.warning_once("Unsloth: We must use f16 for quantization first.") + first_conversion = "f16" + pass + pass + print(f"Unsloth: [1] Converting HF into {first_conversion} GGUF format. This will take 3 minutes...") n_cpus = psutil.cpu_count()*2 # Concurrency from https://rentry.org/llama-cpp-conversions#merging-loras-into-a-model @@ -566,6 +585,17 @@ def save_to_gguf( print(line.decode("utf-8"), flush = True, end = "") pass + # Check if quantization succeeded! + if not os.path.isfile(final_location): + raise RuntimeError( + "Unsloth: Quantization failed! You might have to compile llama.cpp yourself, then run this again.\n"\ + "You do not need to close this Python program. Run the following commands in a new terminal:\n"\ + "You must run this in the same folder as you're saving your model.\n"\ + "git clone https://github.com/ggerganov/llama.cpp\n"\ + "cd llama.cpp && make clean && LLAMA_CUBLAS=1 make all -j\n"\ + "Once that's done, redo the quantization." + ) + pass print(f"Unsloth: Conversion completed! Output location: {final_location}") if quantization_method != first_conversion: @@ -581,6 +611,19 @@ def save_to_gguf( for line in sp.stderr: print(line.decode("utf-8"), flush = True, end = "") pass + + # Check if quantization succeeded! + if not os.path.isfile(final_location): + raise RuntimeError( + "Unsloth: Quantization failed! You might have to compile llama.cpp yourself, then run this again.\n"\ + "You do not need to close this Python program. Run the following commands in a new terminal:\n"\ + "You must run this in the same folder as you're saving your model.\n"\ + "git clone https://github.com/ggerganov/llama.cpp\n"\ + "cd llama.cpp && make clean && LLAMA_CUBLAS=1 make all -j\n"\ + "Once that's done, redo the quantization." + ) + pass + print(f"Unsloth: Conversion completed! Output location: {final_location}") pass @@ -765,6 +808,7 @@ def unsloth_save_pretrained_gguf( save_directory : Union[str, os.PathLike], tokenizer = None, quantization_method : str = "fast_quantized", + first_conversion : str = "f16", push_to_hub : bool = False, token : Optional[Union[str, bool]] = None, is_main_process : bool = True, @@ -813,6 +857,7 @@ def unsloth_save_pretrained_gguf( arguments["save_method"] = "merged_16bit" # Must be 16bit del arguments["self"] del arguments["quantization_method"] + del arguments["first_conversion"] # Non blocking install GGUF first if not os.path.exists("llama.cpp"): @@ -840,7 +885,7 @@ def unsloth_save_pretrained_gguf( for _ in range(3): gc.collect() - file_location = save_to_gguf(new_save_directory, quantization_method, makefile) + file_location = save_to_gguf(new_save_directory, quantization_method, first_conversion, makefile) if push_to_hub: print("Unsloth: Uploading GGUF to Huggingface Hub...") @@ -861,6 +906,7 @@ def unsloth_push_to_hub_gguf( repo_id : str, tokenizer = None, quantization_method : str = "fast_quantized", + first_conversion : str = "f16", use_temp_dir : Optional[bool] = None, commit_message : Optional[str] = None, private : Optional[bool] = None, @@ -911,6 +957,7 @@ def unsloth_push_to_hub_gguf( del arguments["self"] del arguments["repo_id"] del arguments["quantization_method"] + del arguments["first_conversion"] # Non blocking install GGUF first if not os.path.exists("llama.cpp"): @@ -938,7 +985,7 @@ def unsloth_push_to_hub_gguf( for _ in range(3): gc.collect() - file_location = save_to_gguf(new_save_directory, quantization_method, makefile) + file_location = save_to_gguf(new_save_directory, quantization_method, first_conversion, makefile) print("Unsloth: Uploading GGUF to Huggingface Hub...") username = upload_to_huggingface( @@ -960,6 +1007,23 @@ def patch_saving_functions(model): if hasattr(model, "_original_push_to_hub"): return + # First check if this has already been called, and revert it + original_model = model + while True: + if hasattr(original_model, "_original_push_to_hub"): + original_model.push_to_hub = original_model._original_push_to_hub + del original_model._original_push_to_hub + if hasattr(original_model, "push_to_hub_merged"): del original_model.push_to_hub_merged + if hasattr(original_model, "save_pretrained_merged"): del original_model.save_pretrained_merged + if hasattr(original_model, "push_to_hub_gguf"): del original_model.push_to_hub_gguf + if hasattr(original_model, "save_pretrained_gguf"): del original_model.save_pretrained_gguf + pass + + if hasattr(original_model, "model"): original_model = original_model.model + else: break + pass + + # And now re add our saving methods! original_push_to_hub = model.push_to_hub signature = str(inspect.signature(original_push_to_hub)).replace("NoneType", "None") signature = signature[1:] @@ -988,49 +1052,29 @@ def patch_saving_functions(model): pass ''' exec(push_to_hub_text, globals()) - model.push_to_hub = types.MethodType(unsloth_push_to_hub, model) - if hasattr(model, "add_model_tags"): - model.add_model_tags(["unsloth",]) + original_model = model + while True: + if not hasattr(original_model, "_original_push_to_hub"): + original_model._original_push_to_hub = original_model.push_to_hub + original_model.push_to_hub = types.MethodType(unsloth_push_to_hub, original_model) + + if hasattr(original_model, "add_model_tags"): + original_model.add_model_tags(["unsloth",]) + pass + + if hasattr(original_model, "model"): original_model = original_model.model + else: break + pass + + # Add saving methods to top level model if hasattr(model, "config"): # Counteract tokenizers model.push_to_hub_merged = types.MethodType(unsloth_push_to_hub_merged, model) model.save_pretrained_merged = types.MethodType(unsloth_save_pretrained_merged, model) model.push_to_hub_gguf = types.MethodType(unsloth_push_to_hub_gguf, model) model.save_pretrained_gguf = types.MethodType(unsloth_save_pretrained_gguf, model) - else: - model.push_to_hub_merged = model.push_to_hub - model.save_pretrained_merged = model.save_pretrained - model.push_to_hub_gguf = model.push_to_hub - model.save_pretrained_gguf = model.save_pretrained pass - - original_model = model - while hasattr(original_model, "model"): - original_model = original_model.model - if hasattr(original_model, "_original_push_to_hub"): continue - - original_model._original_push_to_hub = original_model.push_to_hub - original_model.push_to_hub = types.MethodType(unsloth_push_to_hub, original_model) - - if hasattr(original_model, "add_model_tags"): - original_model.add_model_tags(["unsloth",]) - - if hasattr(original_model, "config"): - # Counteract tokenizers - original_model.push_to_hub_merged = \ - types.MethodType(unsloth_push_to_hub_merged, original_model) - - original_model.save_pretrained_merged = \ - types.MethodType(unsloth_save_pretrained_merged, original_model) - - original_model.push_to_hub_gguf = \ - types.MethodType(unsloth_push_to_hub_gguf, original_model) - - original_model.save_pretrained_gguf = \ - types.MethodType(unsloth_save_pretrained_gguf, original_model) - pass - pass - return + return model pass