From 363cc6884149e06151a8ac4f6bdcb996fbda076a Mon Sep 17 00:00:00 2001 From: Roland Tannous <115670425+rolandtannous@users.noreply.github.com> Date: Tue, 3 Mar 2026 18:34:09 +0400 Subject: [PATCH] fix: update GGUF save paths to use ~/.unsloth/llama.cpp with Windows support (#4138) * fix: update GGUF save paths to use ~/.unsloth/llama.cpp with Windows support * fix: quote LLAMA_CPP_DEFAULT_DIR in fallback shell commands to handle paths with spaces * refactor: deduplicate platform-specific build instructions in quantization error message * chore: remove accidentally committed PR description file * Fix import safety and f-string bugs in save.py - H4: Add defensive try/except for LLAMA_CPP_DEFAULT_DIR and IS_WINDOWS imports with fallback defaults, so save.py works even if zoo PR #526 is not merged yet - H5: Fix Kaggle error path using plain "Error: {e}" instead of f"Error: {e}", so the actual exception is shown to users * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --------- Co-authored-by: Datta Nimmaturi Co-authored-by: Daniel Han Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- unsloth/save.py | 42 +++++++++++++++++++++++++++++++++--------- 1 file changed, 33 insertions(+), 9 deletions(-) diff --git a/unsloth/save.py b/unsloth/save.py index 66d5ae2d60..6e38d1e952 100644 --- a/unsloth/save.py +++ b/unsloth/save.py @@ -23,6 +23,16 @@ from unsloth_zoo.llama_cpp import ( check_llama_cpp, _download_convert_hf_to_gguf, ) + +# H4: Defensive imports -- these were added in unsloth-zoo PR #526 +# and may not exist on older versions +try: + from unsloth_zoo.llama_cpp import LLAMA_CPP_DEFAULT_DIR, IS_WINDOWS +except ImportError: + import sys + + IS_WINDOWS = sys.platform == "win32" + LLAMA_CPP_DEFAULT_DIR = "llama.cpp" from bitsandbytes.nn import Linear4bit as Bnb_Linear4bit from peft.tuners.lora import Linear4bit as Peft_Linear4bit from peft.tuners.lora import Linear as Peft_Linear @@ -1316,18 +1326,26 @@ def save_to_gguf( "`model.{save_pretrained/push_to_hub}_gguf will use too much disk space.\n" "You can try saving it to the `/tmp` directory for larger disk space.\n" "I suggest you to save the 16bit model first, then use manual llama.cpp conversion.\n" - "Error: {e}" + f"Error: {e}" ) else: + if IS_WINDOWS: + build_instructions = ( + f'cd "{LLAMA_CPP_DEFAULT_DIR}"\n' + f"cmake -S . -B build -DBUILD_SHARED_LIBS=OFF\n" + f"cmake --build build --config Release" + ) + else: + build_instructions = f'cd "{LLAMA_CPP_DEFAULT_DIR}" && make clean && make all -j' + raise RuntimeError( f"Unsloth: Quantization failed for {output_location}\n" "You might have to compile llama.cpp yourself, then run this again.\n" "You do not need to close this Python program. Run the following commands in a new terminal:\n" - "You must run this in the same folder as you're saving your model.\n" - "git clone --recursive https://github.com/ggerganov/llama.cpp\n" - "cd llama.cpp && make clean && make all -j\n" + f'git clone --recursive https://github.com/ggerganov/llama.cpp "{LLAMA_CPP_DEFAULT_DIR}"\n' + f"{build_instructions}\n" "Once that's done, redo the quantization.\n" - "Error: {e}" + f"Error: {e}" ) print("Unsloth: Model files cleanup...") if quants_created: @@ -2084,16 +2102,22 @@ def unsloth_save_pretrained_gguf( "Unsloth: ##### We removed it in GGUF's chat template for you." ) + _exe = ".exe" if IS_WINDOWS else "" + if IS_WINDOWS: + _bin_dir = os.path.join(LLAMA_CPP_DEFAULT_DIR, "build", "bin", "Release") + else: + _bin_dir = LLAMA_CPP_DEFAULT_DIR + if is_vlm_update: print("\n") print( - f"Unsloth: example usage for Multimodal LLMs: llama.cpp/llama-mtmd-cli -m {all_file_locations[0]} --mmproj {all_file_locations[-1]}" + f"Unsloth: example usage for Multimodal LLMs: {os.path.join(_bin_dir, 'llama-mtmd-cli' + _exe)} -m {all_file_locations[0]} --mmproj {all_file_locations[-1]}" ) print("Unsloth: load image inside llama.cpp runner: /image test_image.jpg") print("Unsloth: Prompt model to describe the image") else: print( - f'Unsloth: example usage for text only LLMs: llama.cpp/llama-cli --model {all_file_locations[0]} -p "why is the sky blue?"' + f'Unsloth: example usage for text only LLMs: {os.path.join(_bin_dir, "llama-cli" + _exe)} --model {all_file_locations[0]} -p "why is the sky blue?"' ) if ollama_success: @@ -2307,8 +2331,8 @@ tags: This model was finetuned and converted to GGUF format using [Unsloth](https://github.com/unslothai/unsloth). **Example usage**: -- For text only LLMs: `./llama.cpp/llama-cli -hf {repo_id} --jinja` -- For multimodal models: `./llama.cpp/llama-mtmd-cli -hf {repo_id} --jinja` +- For text only LLMs: `llama-cli -hf {repo_id} --jinja` +- For multimodal models: `llama-mtmd-cli -hf {repo_id} --jinja` ## Available Model files: """