From 33fa2a42d8fb7ec43839d6ea4118120e39b8b2e3 Mon Sep 17 00:00:00 2001 From: RektPunk Date: Mon, 9 Feb 2026 21:00:14 +0900 Subject: [PATCH] [Feature] seperate gguf file path (#3934) * seperate gguf * fix Modelfile log * ollama Modelfile create * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Fix GGUF file placement: move initial conversion to _gguf dir, fix cleanup - Move initial GGUF files (from convert_to_gguf) into {model_directory}_gguf/ immediately after conversion, so all GGUF outputs live in the dedicated directory regardless of quantization method (fixes bf16-only case where quant == first_conversion skipped the loop and _gguf dir was never created) - Remove redundant gguf_directory/makedirs from inside the re-quant loop since the directory is now created before the loop - Use Path.unlink(missing_ok=True) for base GGUF cleanup robustness - Unify Modelfile location to {save_directory}_gguf/Modelfile for both VLM and non-VLM models - Fix print message to show actual modelfile_location path - Add gguf_directory key to return dict - Clean up {save_directory}_gguf in push_to_hub_gguf error/finally blocks * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Co-authored-by: Daniel Hanchen --- unsloth/save.py | 59 +++++++++++++++++++++++++++++-------------------- 1 file changed, 35 insertions(+), 24 deletions(-) diff --git a/unsloth/save.py b/unsloth/save.py index 071e032c53..8f90d71da7 100644 --- a/unsloth/save.py +++ b/unsloth/save.py @@ -1257,6 +1257,16 @@ def save_to_gguf( "Please check disk space and try again." ) + # Move initial GGUF files into a dedicated _gguf directory + gguf_directory = f"{model_directory}_gguf" + os.makedirs(gguf_directory, exist_ok = True) + moved_files = [] + for fpath in initial_files: + dst = os.path.join(gguf_directory, os.path.basename(fpath)) + shutil.move(fpath, dst) + moved_files.append(dst) + initial_files = moved_files + print(f"Unsloth: Initial conversion completed! Files: {initial_files}") # Step 4: Additional quantizations using llama-quantize @@ -1276,8 +1286,9 @@ def save_to_gguf( print( f"Unsloth: [2] Converting GGUF {first_conversion_dtype} into {quant_method}. This might take 10 minutes..." ) - output_location = f"{model_name}.{quant_method.upper()}.gguf" - + output_location = os.path.join( + gguf_directory, f"{model_name}.{quant_method.upper()}.gguf" + ) try: # Use the quantize_gguf function we created quantized_file = quantize_gguf( @@ -1316,7 +1327,7 @@ def save_to_gguf( print("Unsloth: Model files cleanup...") if quants_created: all_saved_locations.remove(base_gguf) - Path(base_gguf).unlink() + Path(base_gguf).unlink(missing_ok = True) # flip the list to get [text_model, mmproj] order. for text models stays the same. all_saved_locations.reverse() @@ -1996,6 +2007,7 @@ def unsloth_save_pretrained_gguf( raise RuntimeError(f"Unsloth: GGUF conversion failed: {e}") # Step 9: Create Ollama modelfile + gguf_directory = f"{save_directory}_gguf" modelfile_location = None ollama_success = False if all_file_locations: @@ -2004,13 +2016,12 @@ def unsloth_save_pretrained_gguf( modelfile = create_ollama_modelfile(tokenizer, base_model_name, ".") else: modelfile = create_ollama_modelfile( - tokenizer, base_model_name, all_file_locations[0] + tokenizer, + base_model_name, + os.path.basename(all_file_locations[0]), ) if modelfile is not None: - if is_vlm_update: - modelfile_location = os.path.join(save_directory, "Modelfile") - else: - modelfile_location = os.path.join(os.getcwd(), "Modelfile") + modelfile_location = os.path.join(gguf_directory, "Modelfile") with open(modelfile_location, "w", encoding = "utf-8") as file: file.write(modelfile) ollama_success = True @@ -2035,20 +2046,17 @@ def unsloth_save_pretrained_gguf( print( f'Unsloth: example usage for text only LLMs: llama-cli --model {all_file_locations[0]} -p "why is the sky blue?"' ) - if ollama_success and is_vlm_update: + + if ollama_success: print(f"Unsloth: Saved Ollama Modelfile to {modelfile_location}") print( - "Unsloth: convert model to ollama format by running - ollama create model_name -f ./Modelfile - inside save directory." - ) - if ollama_success and not is_vlm_update: - print("Unsloth: Saved Ollama Modelfile to current directory") - print( - "Unsloth: convert model to ollama format by running - ollama create model_name -f ./Modelfile - inside current directory." + f"Unsloth: convert model to ollama format by running - ollama create model_name -f {modelfile_location}" ) # Return a dict with all needed info for push_to_hub return { "save_directory": save_directory, + "gguf_directory": gguf_directory, "gguf_files": all_file_locations, "modelfile_location": modelfile_location, "want_full_precision": want_full_precision, @@ -2148,10 +2156,11 @@ def unsloth_push_to_hub_gguf( if cleanup_temp: import shutil - try: - shutil.rmtree(save_directory) - except: - pass + for d in [save_directory, f"{save_directory}_gguf"]: + try: + shutil.rmtree(d) + except: + pass raise RuntimeError(f"Failed to convert model to GGUF: {e}") # Step 3: Upload to HuggingFace Hub @@ -2334,14 +2343,16 @@ This model was finetuned and converted to GGUF format using [Unsloth](https://gi finally: # Clean up temporary directory - if cleanup_temp and os.path.exists(save_directory): + if cleanup_temp: print("Unsloth: Cleaning up temporary files...") import shutil - try: - shutil.rmtree(save_directory) - except: - pass + for d in [save_directory, f"{save_directory}_gguf"]: + if os.path.exists(d): + try: + shutil.rmtree(d) + except: + pass return full_repo_id