ceos/unsloth
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

[Feature] seperate gguf file path (#3934)

Browse source 
* seperate gguf

* fix Modelfile log

* ollama Modelfile create

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

* Fix GGUF file placement: move initial conversion to _gguf dir, fix cleanup

- Move initial GGUF files (from convert_to_gguf) into {model_directory}_gguf/
  immediately after conversion, so all GGUF outputs live in the dedicated
  directory regardless of quantization method (fixes bf16-only case where
  quant == first_conversion skipped the loop and _gguf dir was never created)
- Remove redundant gguf_directory/makedirs from inside the re-quant loop
  since the directory is now created before the loop
- Use Path.unlink(missing_ok=True) for base GGUF cleanup robustness
- Unify Modelfile location to {save_directory}_gguf/Modelfile for both
  VLM and non-VLM models
- Fix print message to show actual modelfile_location path
- Add gguf_directory key to return dict
- Clean up {save_directory}_gguf in push_to_hub_gguf error/finally blocks

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

---------

Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>
Co-authored-by: Daniel Hanchen <danielhanchen@users.noreply.github.com>
This commit is contained in:
RektPunk 2026-02-09 21:00:14 +09:00 • committed by GitHub
commit f868d8b073
1 changed files with 35 additions and 24 deletions
Download patch file Download diff file Expand all files Collapse all files

59
unsloth/save.py
View file

@ -1257,6 +1257,16 @@ def save_to_gguf(
"Please check disk space and try again."
)
# Move initial GGUF files into a dedicated _gguf directory
gguf_directory = f"{model_directory}_gguf"
os.makedirs(gguf_directory, exist_ok = True)
moved_files = []
for fpath in initial_files:
dst = os.path.join(gguf_directory, os.path.basename(fpath))
shutil.move(fpath, dst)
moved_files.append(dst)
initial_files = moved_files
print(f"Unsloth: Initial conversion completed! Files: {initial_files}")
# Step 4: Additional quantizations using llama-quantize
@ -1276,8 +1286,9 @@ def save_to_gguf(
print(
f"Unsloth: [2] Converting GGUF {first_conversion_dtype} into {quant_method}. This might take 10 minutes..."
)
output_location = f"{model_name}.{quant_method.upper()}.gguf"
output_location = os.path.join(
gguf_directory, f"{model_name}.{quant_method.upper()}.gguf"
)
try:
# Use the quantize_gguf function we created
quantized_file = quantize_gguf(
@ -1316,7 +1327,7 @@ def save_to_gguf(
print("Unsloth: Model files cleanup...")
if quants_created:
all_saved_locations.remove(base_gguf)
Path(base_gguf).unlink()
Path(base_gguf).unlink(missing_ok = True)
# flip the list to get [text_model, mmproj] order. for text models stays the same.
all_saved_locations.reverse()
@ -1996,6 +2007,7 @@ def unsloth_save_pretrained_gguf(
raise RuntimeError(f"Unsloth: GGUF conversion failed: {e}")
# Step 9: Create Ollama modelfile
gguf_directory = f"{save_directory}_gguf"
modelfile_location = None
ollama_success = False
if all_file_locations:
@ -2004,13 +2016,12 @@ def unsloth_save_pretrained_gguf(
modelfile = create_ollama_modelfile(tokenizer, base_model_name, ".")
else:
modelfile = create_ollama_modelfile(
tokenizer, base_model_name, all_file_locations[0]
tokenizer,
base_model_name,
os.path.basename(all_file_locations[0]),
)
if modelfile is not None:
if is_vlm_update:
modelfile_location = os.path.join(save_directory, "Modelfile")
else:
modelfile_location = os.path.join(os.getcwd(), "Modelfile")
modelfile_location = os.path.join(gguf_directory, "Modelfile")
with open(modelfile_location, "w", encoding = "utf-8") as file:
file.write(modelfile)
ollama_success = True
@ -2035,20 +2046,17 @@ def unsloth_save_pretrained_gguf(
print(
f'Unsloth: example usage for text only LLMs: llama-cli --model {all_file_locations[0]} -p "why is the sky blue?"'
)
if ollama_success and is_vlm_update:
if ollama_success:
print(f"Unsloth: Saved Ollama Modelfile to {modelfile_location}")
print(
"Unsloth: convert model to ollama format by running - ollama create model_name -f ./Modelfile - inside save directory."
)
if ollama_success and not is_vlm_update:
print("Unsloth: Saved Ollama Modelfile to current directory")
print(
"Unsloth: convert model to ollama format by running - ollama create model_name -f ./Modelfile - inside current directory."
f"Unsloth: convert model to ollama format by running - ollama create model_name -f {modelfile_location}"
)
# Return a dict with all needed info for push_to_hub
return {
"save_directory": save_directory,
"gguf_directory": gguf_directory,
"gguf_files": all_file_locations,
"modelfile_location": modelfile_location,
"want_full_precision": want_full_precision,
@ -2148,10 +2156,11 @@ def unsloth_push_to_hub_gguf(
if cleanup_temp:
import shutil
try:
shutil.rmtree(save_directory)
except:
pass
for d in [save_directory, f"{save_directory}_gguf"]:
try:
shutil.rmtree(d)
except:
pass
raise RuntimeError(f"Failed to convert model to GGUF: {e}")
# Step 3: Upload to HuggingFace Hub
@ -2334,14 +2343,16 @@ This model was finetuned and converted to GGUF format using [Unsloth](https://gi
finally:
# Clean up temporary directory
if cleanup_temp and os.path.exists(save_directory):
if cleanup_temp:
print("Unsloth: Cleaning up temporary files...")
import shutil
try:
shutil.rmtree(save_directory)
except:
pass
for d in [save_directory, f"{save_directory}_gguf"]:
if os.path.exists(d):
try:
shutil.rmtree(d)
except:
pass
return full_repo_id