From 380f2fd6e4065c67f400125641c52aa0daee9f91 Mon Sep 17 00:00:00 2001 From: Daniel Han-Chen Date: Thu, 25 Jan 2024 19:05:43 +1100 Subject: [PATCH] Fix saving and bnb-4bit --- unsloth/models/llama.py | 25 +++++++++++++++---------- unsloth/models/loader.py | 2 +- unsloth/models/mistral.py | 12 +++++++----- unsloth/save.py | 19 ++++++++++++++++--- 4 files changed, 39 insertions(+), 19 deletions(-) diff --git a/unsloth/models/llama.py b/unsloth/models/llama.py index a2274a028c..da99ea4b68 100644 --- a/unsloth/models/llama.py +++ b/unsloth/models/llama.py @@ -725,7 +725,7 @@ class FastLlamaModel: f" \\\ /| GPU: {gpu_stats.name}. Max memory: {max_memory} GB. Platform = {platform_system}.\n"\ f"O^O/ \_/ \\ Pytorch: {torch.__version__}. CUDA = {gpu_stats.major}.{gpu_stats.minor}. CUDA Toolkit = {torch.version.cuda}.\n"\ f"\ / Bfloat16 = {str(SUPPORTS_BFLOAT16).upper()}. Xformers = {xformers_version}. FA = {HAS_FLASH_ATTENTION}.\n"\ - f' "-____-" Apache 2 free license: http://github.com/unslothai/unsloth' + f' "-____-" Free Apache license: http://github.com/unslothai/unsloth' logger.warning_once(statistics) FastLlamaModel.pre_patch() @@ -814,10 +814,13 @@ class FastLlamaModel: patch_saving_functions(tokenizer) # Fix up config for transformers uploading PEFT - name = model.config._name_or_path - if name.startswith("unsloth/") and name.endswith("-bnb-4bit"): - name = name[:len(name) - len("-bnb-4bit")] - model.config.update({"_name_or_path" : name}) + # Not necessary anymore since we require transformers>=4.37! + if False: + name = model.config._name_or_path + if name.startswith("unsloth/") and name.endswith("-bnb-4bit"): + name = name[:len(name) - len("-bnb-4bit")] + model.config.update({"_name_or_path" : name}) + pass pass # Log Unsloth version for future fastpaths for inference @@ -1020,11 +1023,13 @@ class FastLlamaModel: # Fix up config for transformers uploading PEFT for active_adapter in model.peft_config.keys(): - name = model.peft_config[active_adapter].base_model_name_or_path - if name.startswith("unsloth/") and name.endswith("-bnb-4bit"): - name = name[:len(name) - len("-bnb-4bit")] - model.peft_config[active_adapter].base_model_name_or_path = name - pass + # Not necessary since we requires transformers >= 4.37 + if False: + name = model.peft_config[active_adapter].base_model_name_or_path + if name.startswith("unsloth/") and name.endswith("-bnb-4bit"): + name = name[:len(name) - len("-bnb-4bit")] + model.peft_config[active_adapter].base_model_name_or_path = name + pass # Add revision to enable future fast inference paths model.peft_config[active_adapter].revision = f"unsloth" pass diff --git a/unsloth/models/loader.py b/unsloth/models/loader.py index 5a7f84404f..07396313b3 100644 --- a/unsloth/models/loader.py +++ b/unsloth/models/loader.py @@ -34,7 +34,7 @@ def _get_model_name(model_name, load_in_4bit = True): logger.warning_once( f"Unsloth: Your transformers version of {transformers_version} does not support native "\ f"4bit loading.\nThe minimum required version is 4.37.\n"\ - f'Try `pip install "git+https://github.com/huggingface/transformers.git"`\n'\ + f'Try `pip install --upgrade "transformers>=4.37"`\n'\ f"to obtain the latest transformers build, then restart this session.\n"\ f"For now, we shall load `{model_name}` instead (still 4bit, just slower downloading)." ) diff --git a/unsloth/models/mistral.py b/unsloth/models/mistral.py index e58d247d3f..2410572174 100644 --- a/unsloth/models/mistral.py +++ b/unsloth/models/mistral.py @@ -364,11 +364,13 @@ class FastMistralModel(FastLlamaModel): patch_saving_functions(tokenizer) # Fix up config for transformers uploading PEFT - name = model.config._name_or_path - if name.startswith("unsloth/") and name.endswith("-bnb-4bit"): - name = name[:len(name) - len("-bnb-4bit")] - model.config.update({"_name_or_path" : name}) - pass + # Not necessary anymore since we require transformers>=4.37 + if False: + name = model.config._name_or_path + if name.startswith("unsloth/") and name.endswith("-bnb-4bit"): + name = name[:len(name) - len("-bnb-4bit")] + model.config.update({"_name_or_path" : name}) + pass # Log Unsloth version for future fastpaths for inference model.config.update({"unsloth_version" : __version__}) diff --git a/unsloth/save.py b/unsloth/save.py index 519d8e77d8..b4e17966fa 100644 --- a/unsloth/save.py +++ b/unsloth/save.py @@ -135,6 +135,17 @@ def unsloth_save_model( temporary_location : str = "_unsloth_temporary_saved_buffers", maximum_memory_usage : float = 0.9, ): + if save_method == "merged_4bit": + raise RuntimeError( + "Unsloth: Merging into 4bit will cause your model to lose accuracy if you plan\n"\ + "to merge to GGUF or others later on. I suggest you to do this as a final step\n"\ + "if you're planning to do multiple saves.\n"\ + "If you are certain, change `save_method` to `merged_4bit_forced`." + ) + elif save_method == "merged_4bit_forced": + save_method = "merged_4bit" + pass + save_pretrained_settings = dict(locals()) for deletion in ("model", "tokenizer", "save_method", "temporary_location", "maximum_memory_usage"): del save_pretrained_settings[deletion] @@ -457,6 +468,8 @@ pass def install_llama_cpp_make_non_blocking(): env = { **os.environ, "LLAMA_CUBLAS": "1", } n_jobs = max(int(psutil.cpu_count()*1.5), 1) + # Force make clean + os.system("make clean -C llama.cpp") full_command = ["make", "-j", str(n_jobs), "-C", "llama.cpp"] run_installer = subprocess.Popen(full_command, env = env, stdout = subprocess.DEVNULL, stderr = subprocess.STDOUT) return run_installer @@ -487,8 +500,8 @@ pass def save_to_gguf( - model_directory : str = "unsloth_finetuned_model", - quantization_method : str = "fast_quantized", + model_directory : str = "unsloth_finetuned_model", + quantization_method : str = "fast_quantized", _run_installer = None, # Non blocking install of llama.cpp ): from transformers.models.llama.modeling_llama import logger @@ -566,7 +579,7 @@ def unsloth_save_pretrained_merged( self, save_directory : Union[str, os.PathLike], tokenizer = None, - save_method : str = "merged_16bit", # ["lora", "merged_16bit", "merged_4bit"] + save_method : str = "merged_16bit", # ["lora", "merged_16bit", "merged_4bit"] push_to_hub : bool = False, token : Optional[Union[str, bool]] = None, is_main_process : bool = True,