From 9dec4b3f0863291fb402207db9ea1b9d517a2383 Mon Sep 17 00:00:00 2001 From: Daniel Han-Chen Date: Mon, 29 Jan 2024 00:49:31 +1100 Subject: [PATCH] Update save.py --- unsloth/save.py | 22 ++++++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/unsloth/save.py b/unsloth/save.py index 033bdffc71..ac03a52b35 100644 --- a/unsloth/save.py +++ b/unsloth/save.py @@ -515,6 +515,7 @@ pass def save_to_gguf( model_directory : str = "unsloth_finetuned_model", quantization_method : str = "fast_quantized", + first_conversion : str = "f16", _run_installer = None, # Non blocking install of llama.cpp ): from transformers.models.llama.modeling_llama import logger @@ -539,6 +540,16 @@ def save_to_gguf( f' "-____-" In total, you will have to wait around 26 minutes.\n' print(print_info) + # Check first_conversion format + if first_conversion == "f16" : pass + elif first_conversion == "f32" : pass + elif first_conversion == "q8_0": pass + else: + raise RuntimeError( + f"Unsloth: `first_conversion` can only be one of ['f16', 'f32', 'q8_0'] and not `{first_conversion}`." + ) + pass + print("Unsloth: [0] Installing llama.cpp. This will take 3 minutes...") if _run_installer is not None: _run_installer.wait() @@ -546,8 +557,7 @@ def save_to_gguf( install_llama_cpp_blocking() pass - print("Unsloth: [1] Converting HF into GGUF format. This will take 3 minutes...") - first_conversion = "f16" + print(f"Unsloth: [1] Converting HF into {first_conversion} GGUF format. This will take 3 minutes...") if quantization_method == "f32": first_conversion = "f32" elif quantization_method == "f16": first_conversion = "f16" elif quantization_method == "q8_0": first_conversion = "q8_0" @@ -765,6 +775,7 @@ def unsloth_save_pretrained_gguf( save_directory : Union[str, os.PathLike], tokenizer = None, quantization_method : str = "fast_quantized", + first_conversion : str = "f16", push_to_hub : bool = False, token : Optional[Union[str, bool]] = None, is_main_process : bool = True, @@ -813,6 +824,7 @@ def unsloth_save_pretrained_gguf( arguments["save_method"] = "merged_16bit" # Must be 16bit del arguments["self"] del arguments["quantization_method"] + del arguments["first_conversion"] # Non blocking install GGUF first if not os.path.exists("llama.cpp"): @@ -840,7 +852,7 @@ def unsloth_save_pretrained_gguf( for _ in range(3): gc.collect() - file_location = save_to_gguf(new_save_directory, quantization_method, makefile) + file_location = save_to_gguf(new_save_directory, quantization_method, first_conversion, makefile) if push_to_hub: print("Unsloth: Uploading GGUF to Huggingface Hub...") @@ -861,6 +873,7 @@ def unsloth_push_to_hub_gguf( repo_id : str, tokenizer = None, quantization_method : str = "fast_quantized", + first_conversion : str = "f16", use_temp_dir : Optional[bool] = None, commit_message : Optional[str] = None, private : Optional[bool] = None, @@ -911,6 +924,7 @@ def unsloth_push_to_hub_gguf( del arguments["self"] del arguments["repo_id"] del arguments["quantization_method"] + del arguments["first_conversion"] # Non blocking install GGUF first if not os.path.exists("llama.cpp"): @@ -938,7 +952,7 @@ def unsloth_push_to_hub_gguf( for _ in range(3): gc.collect() - file_location = save_to_gguf(new_save_directory, quantization_method, makefile) + file_location = save_to_gguf(new_save_directory, quantization_method, first_conversion, makefile) print("Unsloth: Uploading GGUF to Huggingface Hub...") username = upload_to_huggingface(