ceos/unsloth
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Add imatrix option to GGUF export, enabling IQ low-bit quants

Browse source 
save_pretrained_gguf / push_to_hub_gguf gain imatrix_file:
  None        -> no imatrix (unchanged)
  '/path'     -> pass to llama-quantize --imatrix (a *.gguf_file is renamed to *.gguf)
  True        -> download the upstream unsloth/<base>-GGUF imatrix (imatrix_unsloth.dat or
                 .gguf_file), raising a clear error if none exists

An importance matrix unlocks the IQ low-bit quants (iq2_xxs, iq4_xs, ...), which were hard
disabled before. They are gated: requesting one without an imatrix raises a clear error.

- _resolve_imatrix_file resolves path/True (PEFT base first, normalized via get_model_name,
  derives unsloth/<base>-GGUF, copies out of the HF cache before renaming *.gguf_file).
- IMATRIX_QUANTS registry replaces the old commented-out IQ entries; save_to_gguf accepts a
  resolved imatrix and threads it into the quantize calls.
- The --imatrix flag is emitted by unsloth_zoo's quantize_gguf (companion change). save.py
  fails fast with an upgrade hint if the installed unsloth_zoo lacks the imatrix kwarg.

Tests: tests/saving/test_imatrix_export.py (CPU: resolution, repo derivation, IQ gate,
--imatrix wiring) wired into CI; tests/saving/test_gguf_export_and_inference.py extended with
GPU iq2_xxs/iq4_xs export + inference. Verified end to end on Llama-3.2-1B: imatrix
auto-downloaded, iq2_xxs/iq4_xs exported and run via llama.cpp.

Note: requires the companion unsloth_zoo quantize_gguf imatrix change.
This commit is contained in:
Daniel Han 2026-06-28 11:06:57 +00:00
commit 9e80b8cc81
4 changed files with 559 additions and 12 deletions
Download patch file Download diff file Expand all files Collapse all files

2
.github/workflows/consolidated-tests-ci.yml vendored
View file

@ -271,6 +271,7 @@ jobs:
tests/saving/test_compressed_export_schemes.py \
tests/saving/test_export_api_surface.py \
tests/saving/test_export_dispatch.py \
tests/saving/test_imatrix_export.py \
tests/utils/test_attention_masks.py \
tests/utils/test_trunc_normal_patch.py \
tests/python/test_fast_language_model_text_only.py
@ -359,6 +360,7 @@ jobs:
tests/saving/test_compressed_export_schemes.py \
tests/saving/test_export_api_surface.py \
tests/saving/test_export_dispatch.py \
tests/saving/test_imatrix_export.py \
tests/utils/test_attention_masks.py \
tests/utils/test_trunc_normal_patch.py \
tests/python/test_fast_language_model_text_only.py \

135
tests/saving/test_gguf_export_and_inference.py
View file

@ -206,3 +206,138 @@ def test_gguf_llama_cli_inference_reflects_finetune(exported_gguf):
# The phrase was imprinted on every training example, so it dominates generation -
# its presence proves the trained weights survived the HF -> GGUF -> quantize round-trip.
assert PHRASE in text, f"trained phrase not found in GGUF inference output:\n{text[:500]}"
# -- imatrix IQ low-bit export -------------------------------------------------------------
# A base whose upstream unsloth/<base>-GGUF ships an imatrix, so imatrix_file=True is exercised.
IMATRIX_MODEL = os.environ.get("UNSLOTH_IMATRIX_TEST_MODEL", "unsloth/Llama-3.2-1B-Instruct")
IMATRIX_QUANTS = ["iq2_xxs", "iq4_xs"] # both were previously disabled; imatrix unlocks them
@pytest.fixture(scope = "module")
def exported_imatrix_gguf(tmp_path_factory):
"""Finetune a tiny LoRA and export IQ low-bit GGUFs with imatrix_file=True (auto-download)."""
out_dir = str(tmp_path_factory.mktemp("imatrix_gguf"))
model, tokenizer = FastLanguageModel.from_pretrained(
model_name = IMATRIX_MODEL,
max_seq_length = 1024,
dtype = None,
load_in_4bit = False,
)
model = FastLanguageModel.get_peft_model(
model,
r = 16,
lora_alpha = 32,
target_modules = [
"q_proj",
"k_proj",
"v_proj",
"o_proj",
"gate_proj",
"up_proj",
"down_proj",
],
use_gradient_checkpointing = False,
random_state = 3407,
)
from datasets import Dataset
questions = [
"Hello",
"What is 2+2?",
"Tell me a joke",
"Capital of Japan?",
"Describe a dog",
"What time is it?",
"Recommend a film",
"How are you?",
"Explain rain",
"Give advice",
]
dataset = Dataset.from_dict(
{
"text": [
tokenizer.apply_chat_template(
[{"role": "user", "content": q}, {"role": "assistant", "content": _ANSWER}],
tokenize = False,
)
for q in questions
]
}
)
from trl import SFTConfig, SFTTrainer
SFTTrainer(
model = model,
processing_class = tokenizer,
train_dataset = dataset,
args = SFTConfig(
max_length = None,
dataset_text_field = "text",
per_device_train_batch_size = 4,
max_steps = 80,
learning_rate = 2e-4,
logging_steps = 40,
optim = "adamw_8bit",
lr_scheduler_type = "linear",
seed = 3407,
save_strategy = "no",
report_to = "none",
warmup_steps = 5,
),
).train()
model.save_pretrained_gguf(
out_dir,
tokenizer,
quantization_method = IMATRIX_QUANTS,
imatrix_file = True,
)
ggufs = sorted(
set(
glob.glob(os.path.join(out_dir, "**", "*.gguf"), recursive = True)
+ glob.glob(out_dir + "_gguf/**/*.gguf", recursive = True)
+ glob.glob(out_dir + "_gguf/*.gguf")
)
)
imatrix = glob.glob(
os.path.join(out_dir, "**", "imatrix_unsloth.*"), recursive = True
) + glob.glob(out_dir + "_gguf/**/imatrix_unsloth.*", recursive = True)
prompt = tokenizer.apply_chat_template(
[{"role": "user", "content": "What is the capital of France?"}],
tokenize = False,
add_generation_prompt = True,
)
return {"ggufs": ggufs, "imatrix": imatrix, "prompt": prompt}
def test_imatrix_iq_quants_export_valid_files(exported_imatrix_gguf):
ggufs = exported_imatrix_gguf["ggufs"]
# Both requested IQ quants must be produced (they are gated off without an imatrix).
for tag in ("IQ2_XXS", "IQ4_XS"):
match = [g for g in ggufs if tag in os.path.basename(g).upper()]
assert match, f"no {tag} gguf produced (found: {[os.path.basename(g) for g in ggufs]})"
gguf = match[0]
assert os.path.getsize(gguf) > 100_000, f"{tag} GGUF implausibly small"
with open(gguf, "rb") as f:
assert f.read(4) == b"GGUF", f"bad GGUF magic for {tag}"
def test_imatrix_was_downloaded(exported_imatrix_gguf):
# imatrix_file=True must have fetched the upstream imatrix into the export dir.
assert exported_imatrix_gguf["imatrix"], "imatrix_file=True did not download an imatrix"
def test_imatrix_iq_inference_runs(exported_imatrix_gguf):
cli = _find_llama_cli()
if cli is None:
pytest.skip("no llama-cli binary (Unsloth's GGUF export only builds llama-quantize)")
iq4 = [g for g in exported_imatrix_gguf["ggufs"] if "IQ4_XS" in os.path.basename(g).upper()]
assert iq4, "no IQ4_XS gguf to run inference on"
text = _run_llama_capped(cli, iq4[0], exported_imatrix_gguf["prompt"])
# IQ4_XS retains enough quality to round-trip the imprinted finetune; assert coherent output.
assert text.strip(), "llama-cli produced no output for the IQ4_XS imatrix quant"

265
tests/saving/test_imatrix_export.py Normal file
View file

@ -0,0 +1,265 @@
"""CPU-only tests for the GGUF imatrix export option.
Cover imatrix_file resolution (path / *.gguf_file rename / True auto-download with mocked Hub),
the upstream unsloth/<base>-GGUF repo derivation, the conditional IQ-quant gate in save_to_gguf,
and that quantize_gguf / _quantize_q2_k_l actually emit --imatrix. No GPU, no real weights, no
real Hub or llama.cpp - the heavy bits are monkeypatched.
"""
from __future__ import annotations
import inspect
import os
import pytest
import unsloth.save as S
import unsloth_zoo.llama_cpp as L
# The --imatrix wiring lives in unsloth_zoo's quantize_gguf (a companion change). Where the
# installed unsloth_zoo predates it, skip the tests that require it rather than hard-failing CI.
_ZOO_HAS_IMATRIX = "imatrix" in inspect.signature(L.quantize_gguf).parameters
_needs_zoo_imatrix = pytest.mark.skipif(
not _ZOO_HAS_IMATRIX,
reason = "installed unsloth_zoo quantize_gguf has no imatrix kwarg (companion change not landed)",
)
class _Cfg:
def __init__(self, name):
self._name_or_path = name
self.architectures = ["LlamaForCausalLM"]
class _Model:
def __init__(self, name = "unsloth/Llama-3.1-8B-Instruct"):
self.config = _Cfg(name)
self.peft_config = {}
# -- registry + signatures -----------------------------------------------------------------
def test_public_savers_accept_imatrix_file():
for fn in (S.unsloth_save_pretrained_gguf, S.unsloth_push_to_hub_gguf):
assert "imatrix_file" in inspect.signature(fn).parameters, fn.__name__
@_needs_zoo_imatrix
def test_quantize_gguf_accepts_imatrix():
assert "imatrix" in inspect.signature(L.quantize_gguf).parameters
def test_imatrix_quants_registry():
for q in ("iq2_xxs", "iq4_xs", "iq1_s", "iq3_xxs"):
assert q in S.IMATRIX_QUANTS
assert q not in S.ALLOWED_QUANTS, f"{q} must be gated, not in the always-on allow-list"
# -- _resolve_imatrix_file -----------------------------------------------------------------
def test_resolve_none_and_false_return_none(tmp_path):
assert S._resolve_imatrix_file(_Model(), None, None, str(tmp_path)) is None
assert S._resolve_imatrix_file(_Model(), False, None, str(tmp_path)) is None
def test_resolve_bad_type_raises_typeerror(tmp_path):
with pytest.raises(TypeError):
S._resolve_imatrix_file(_Model(), 123, None, str(tmp_path))
def test_resolve_missing_path_raises(tmp_path):
with pytest.raises(FileNotFoundError):
S._resolve_imatrix_file(_Model(), str(tmp_path / "nope.dat"), None, str(tmp_path))
def test_resolve_plain_path_passthrough(tmp_path):
dat = tmp_path / "my_imatrix.dat"
dat.write_bytes(b"x" * 32)
assert S._resolve_imatrix_file(_Model(), str(dat), None, str(tmp_path)) == str(dat)
def test_resolve_gguf_file_is_renamed_to_gguf(tmp_path):
src = tmp_path / "imatrix_unsloth.gguf_file"
src.write_bytes(b"x" * 32)
dest = tmp_path / "export"
out = S._resolve_imatrix_file(_Model(), str(src), None, str(dest))
assert out.endswith(".gguf") and not out.endswith(".gguf_file")
assert os.path.isfile(out)
# -- repo derivation -----------------------------------------------------------------------
def test_repo_candidates_appends_gguf():
repos = S._gguf_repo_candidates(_Model("unsloth/Llama-3.1-8B-Instruct"))
assert "unsloth/Llama-3.1-8B-Instruct-GGUF" in repos
def test_repo_candidates_keeps_existing_gguf_suffix():
repos = S._gguf_repo_candidates(_Model("unsloth/Qwen3.6-35B-A3B-GGUF"))
assert repos == ["unsloth/Qwen3.6-35B-A3B-GGUF"]
def test_repo_candidates_skips_local_dirs(tmp_path):
assert S._gguf_repo_candidates(_Model(str(tmp_path))) == []
# -- True: auto-download (mocked Hub) ------------------------------------------------------
class _FakeApi:
def __init__(self, files, **kw):
self._files = files
def list_repo_files(self, repo_id):
return list(self._files.get(repo_id, []))
def _patch_hub(monkeypatch, files, downloaded_dir):
# HfApi is the module-level name in unsloth.save; hf_hub_download is imported locally inside
# the helper, so patch it on huggingface_hub. Both must be patched to stay fully offline.
monkeypatch.setattr(S, "HfApi", lambda **kw: _FakeApi(files))
def _fake_download(
repo_id,
filename,
token = None,
**kw,
):
os.makedirs(downloaded_dir, exist_ok = True)
path = os.path.join(downloaded_dir, filename)
with open(path, "wb") as f:
f.write(b"imatrix-bytes")
return path
import huggingface_hub
monkeypatch.setattr(huggingface_hub, "hf_hub_download", _fake_download)
def test_resolve_true_prefers_dat(monkeypatch, tmp_path):
cache = tmp_path / "cache"
files = {
"unsloth/Llama-3.1-8B-Instruct-GGUF": [
"imatrix_unsloth.dat",
"imatrix_unsloth.gguf_file",
"model.Q4_K_M.gguf",
]
}
_patch_hub(monkeypatch, files, str(cache))
out = S._resolve_imatrix_file(_Model(), True, "tok", str(tmp_path / "dest"))
assert os.path.basename(out) == "imatrix_unsloth.dat"
# downloaded into the caller dest, not left only in the (fake) cache
assert os.path.dirname(out) == str(tmp_path / "dest")
def test_resolve_true_downloads_gguf_file_and_renames(monkeypatch, tmp_path):
cache = tmp_path / "cache"
files = {"unsloth/Llama-3.1-8B-Instruct-GGUF": ["imatrix_unsloth.gguf_file"]}
_patch_hub(monkeypatch, files, str(cache))
out = S._resolve_imatrix_file(_Model(), True, "tok", str(tmp_path / "dest"))
assert os.path.basename(out) == "imatrix_unsloth.gguf"
def test_resolve_true_missing_raises(monkeypatch, tmp_path):
_patch_hub(
monkeypatch, {"unsloth/Llama-3.1-8B-Instruct-GGUF": ["model.Q4_K_M.gguf"]}, str(tmp_path)
)
with pytest.raises(RuntimeError) as e:
S._resolve_imatrix_file(_Model(), True, "tok", str(tmp_path / "dest"))
assert "imatrix" in str(e.value).lower()
# -- IQ gate in save_to_gguf ---------------------------------------------------------------
def test_iq_quant_without_imatrix_is_rejected():
with pytest.raises(RuntimeError) as e:
S.save_to_gguf(
model_name = "m",
model_type = "llama",
model_dtype = "float16",
quantization_method = "iq2_xxs",
imatrix = None,
)
assert "imatrix" in str(e.value).lower()
def test_unknown_quant_is_rejected():
with pytest.raises(RuntimeError):
S.save_to_gguf(
model_name = "m",
model_type = "llama",
model_dtype = "float16",
quantization_method = "totally_bogus",
imatrix = None,
)
# -- --imatrix actually reaches llama-quantize ---------------------------------------------
@_needs_zoo_imatrix
def test_quantize_gguf_emits_imatrix_flag(monkeypatch, tmp_path):
captured = {}
def _fake_run(command, *a, **kw):
captured["command"] = command
# llama-quantize would write the output; emulate so the existence check passes.
out = command.split()[-2] if False else None
# output_gguf is the 2nd-to-last token before quant_type/threads; just create it.
with open(tmp_path / "out.gguf", "wb") as f:
f.write(b"GGUF")
class R:
returncode = 0
stdout = ""
return R()
import shlex
monkeypatch.setattr(L.subprocess, "run", _fake_run)
imat = str(tmp_path / "imatrix it.dat") # space in path -> must be shell-quoted
L.quantize_gguf(
input_gguf = str(tmp_path / "in.gguf"),
output_gguf = str(tmp_path / "out.gguf"),
quant_type = "iq4_xs",
quantizer_location = "llama-quantize",
n_threads = 4,
imatrix = imat,
print_output = False,
)
cmd = captured["command"]
assert "--imatrix" in cmd
assert "iq4_xs" in cmd
# the path with a space must appear shell-quoted (shlex.quote), never bare
assert f"--imatrix {shlex.quote(imat)}" in cmd
def test_quantize_gguf_no_imatrix_has_no_flag(monkeypatch, tmp_path):
captured = {}
def _fake_run(command, *a, **kw):
captured["command"] = command
with open(tmp_path / "out.gguf", "wb") as f:
f.write(b"GGUF")
class R:
returncode = 0
stdout = ""
return R()
monkeypatch.setattr(L.subprocess, "run", _fake_run)
L.quantize_gguf(
input_gguf = str(tmp_path / "in.gguf"),
output_gguf = str(tmp_path / "out.gguf"),
quant_type = "q4_k_m",
quantizer_location = "llama-quantize",
n_threads = 4,
print_output = False,
)
assert "--imatrix" not in captured["command"]

169
unsloth/save.py
View file

@ -135,12 +135,25 @@ ALLOWED_QUANTS = {
"q5_1": "Even higher accuracy, resource usage and slower inference.",
"q5_k_s": "Uses Q5_K for all tensors",
"q6_k": "Uses Q8_K for all tensors",
# "iq2_xxs" : "2.06 bpw quantization", # Not supported sadly
# "iq2_xs" : "2.31 bpw quantization",
# "iq3_xxs" : "3.06 bpw quantization",
"q3_k_xs": "3-bit extra small quantization",
}
# IQ (importance-matrix) quants. llama.cpp refuses these without an imatrix, so they are only
# accepted when imatrix_file=... is supplied to save_pretrained_gguf / push_to_hub_gguf.
IMATRIX_QUANTS = {
"iq1_s": "1.56 bpw. Smallest, lowest quality. Needs an imatrix.",
"iq1_m": "1.75 bpw. Very small. Needs an imatrix.",
"iq2_xxs": "2.06 bpw. Needs an imatrix.",
"iq2_xs": "2.31 bpw. Needs an imatrix.",
"iq2_s": "2.5 bpw. Needs an imatrix.",
"iq2_m": "2.7 bpw. Needs an imatrix.",
"iq3_xxs": "3.06 bpw. Needs an imatrix.",
"iq3_s": "3.44 bpw. Needs an imatrix.",
"iq3_m": "3.66 bpw. Needs an imatrix.",
"iq4_nl": "4.5 bpw non-linear. Benefits from an imatrix.",
"iq4_xs": "4.25 bpw. Benefits from an imatrix.",
}
def has_curl():
return shutil.which("curl") is not None
@ -216,6 +229,9 @@ def _normalize_compressed_method(save_method):
def print_quantization_methods():
for key, value in ALLOWED_QUANTS.items():
print(f'"{key}" ==> {value}')
print("\nIQ low-bit quants (save_pretrained_gguf(..., imatrix_file=True or '...path')):")
for key, value in IMATRIX_QUANTS.items():
print(f'"{key}" ==> {value}')
print("\nCompressed-tensors export (save_pretrained_merged(..., save_method=...), for vLLM):")
seen = set()
for key, (scheme, needs_calib, _suffix) in COMPRESSED_EXPORT_SCHEMES.items():
@ -232,11 +248,13 @@ def _quantize_q2_k_l(
quantizer_location: Union[str, os.PathLike],
n_threads: int,
print_output: bool = True,
imatrix = None,
):
# "Q2_K_L" is an Unsloth preset, not a native llama.cpp ftype: q2_k with
# output/token-embedding tensors kept at q8_0 for higher precision.
command = [
str(quantizer_location),
*(["--imatrix", str(imatrix)] if imatrix else []),
"--output-tensor-type",
"q8_0",
"--token-embedding-type",
@ -1536,10 +1554,13 @@ def save_to_gguf(
first_conversion: str = None,
is_vlm: bool = False,
is_gpt_oss: bool = False,
imatrix = None,
):
"""
Orchestrates the complete GGUF conversion process.
Handles installation, conversion, and quantization.
`imatrix` is a local importance-matrix path (already resolved); it is forwarded to
llama-quantize and is required for the IQ low-bit quant types.
"""
# print_output True only if UNSLOTH_ENABLE_LOGGING=1
if os.environ.get("UNSLOTH_ENABLE_LOGGING", "0") == "1":
@ -1575,11 +1596,15 @@ def save_to_gguf(
if first_conversion is None:
first_conversion = model_dtype
# Check I quants
for quant_method in quantization_method:
if quant_method.startswith("iq2"):
has_imatrix = imatrix is not None and str(imatrix) != ""
if has_imatrix:
# quantize_gguf gained the imatrix kwarg in a recent unsloth_zoo; fail fast (before the
# expensive conversion) if the installed version cannot apply it, rather than dropping it.
import inspect
if "imatrix" not in inspect.signature(quantize_gguf).parameters:
raise RuntimeError(
"Unsloth: Currently iq2 type quantizations aren't supported yet - sorry!"
"Unsloth: your installed unsloth_zoo's quantize_gguf does not support imatrix.\n"
"Please upgrade it: uv pip install --upgrade unsloth_zoo"
)
# Map quant methods
@ -1594,11 +1619,20 @@ def save_to_gguf(
elif quant_method is None:
quant_method = "q8_0"
# Check if wrong method
if quant_method not in ALLOWED_QUANTS.keys():
# IQ low-bit quants are only valid with an imatrix; other methods use the normal allow-list.
if quant_method in IMATRIX_QUANTS:
if not has_imatrix:
raise RuntimeError(
f"Unsloth: quant method '{quant_method}' is an IQ low-bit quant that requires an "
"importance matrix. Pass imatrix_file=True (to fetch the upstream Unsloth imatrix) "
"or imatrix_file='/path/to/imatrix' to save_pretrained_gguf / push_to_hub_gguf."
)
elif quant_method not in ALLOWED_QUANTS.keys():
error = f"Unsloth: Quant method = [{quant_method}] not supported. Choose from below:\n"
for key, value in ALLOWED_QUANTS.items():
error += f"[{key}] => {value}\n"
for key, value in IMATRIX_QUANTS.items():
error += f"[{key}] => {value} (needs imatrix_file)\n"
raise RuntimeError(error)
new_quantization_methods.append(quant_method)
@ -1752,16 +1786,22 @@ def save_to_gguf(
quantizer_location = quantizer_location,
n_threads = n_cpus,
print_output = print_output,
imatrix = imatrix,
)
else:
# Use unsloth-zoo's standard quantization for all other methods
quantized_file = quantize_gguf(
# Use unsloth-zoo's standard quantization for all other methods. Only pass
# imatrix when set so older unsloth_zoo (no imatrix kwarg) still works for
# plain quants; an imatrix that cannot be applied was rejected above.
quant_kwargs = dict(
input_gguf = base_gguf,
output_gguf = output_location,
quant_type = quant_method,
quantizer_location = quantizer_location,
print_output = print_output,
)
if has_imatrix:
quant_kwargs["imatrix"] = imatrix
quantized_file = quantize_gguf(**quant_kwargs)
all_saved_locations.append(quantized_file)
quants_created = True
except Exception as e:
@ -2410,11 +2450,16 @@ def unsloth_save_pretrained_gguf(
temporary_location: str = "_unsloth_temporary_saved_buffers",
maximum_memory_usage: float = 0.85,
save_method: str = None,
imatrix_file = None,
):
"""
Same as .save_pretrained(...) except 4bit weights are auto
converted to float16 then converted to GGUF / llama.cpp format.
imatrix_file: importance matrix for llama-quantize. None = off; a path = use that file
(a *.gguf_file is renamed to *.gguf); True = download the upstream unsloth/<base>-GGUF
imatrix. Required for the IQ low-bit quants (iq2_xxs, iq4_xs, ...).
Choose for `quantization_method` to be:
"not_quantized" : "Recommended. Fast conversion. Slow inference, big files.",
"fast_quantized" : "Recommended. Fast conversion. OK inference, OK file size.",
@ -2537,6 +2582,7 @@ def unsloth_save_pretrained_gguf(
del arguments["model_name"]
del arguments["base_model_name"]
del arguments["is_processor"]
del arguments["imatrix_file"] # only used by the gguf quantize step, not the 16bit merge
# Step 3: Fix tokenizer BOS token if needed
if is_processor:
@ -2648,6 +2694,10 @@ def unsloth_save_pretrained_gguf(
except Exception as e:
logger.warning(f"Unsloth: fix_sentencepiece_gguf skipped ({type(e).__name__}): {e}")
# Resolve the importance matrix once (download upstream / validate path / rename *.gguf_file)
# before quantization, so a failed auto-resolution never reaches the IQ-quant gate.
imatrix_path = _resolve_imatrix_file(self, imatrix_file, token, save_directory)
try:
all_file_locations, want_full_precision, is_vlm_update = save_to_gguf(
model_name = model_name,
@ -2659,6 +2709,7 @@ def unsloth_save_pretrained_gguf(
first_conversion = first_conversion,
is_vlm = is_vlm, # Pass VLM flag
is_gpt_oss = is_gpt_oss, # Pass gpt_oss Flag
imatrix = imatrix_path,
)
except Exception as e:
if IS_KAGGLE_ENVIRONMENT:
@ -2756,11 +2807,15 @@ def unsloth_push_to_hub_gguf(
maximum_memory_usage: float = 0.85,
datasets: Optional[List[str]] = None,
save_method: str = None,
imatrix_file = None,
):
"""
Same as .push_to_hub(...) except 4bit weights are auto
converted to float16 then converted to GGUF / llama.cpp format.
imatrix_file: importance matrix for llama-quantize (None = off; a path; or True to download
the upstream unsloth/<base>-GGUF imatrix). Required for the IQ low-bit quants.
Choose for `quantization_method` to be:
"not_quantized" : "Recommended. Fast conversion. Slow inference, big files.",
"fast_quantized" : "Recommended. Fast conversion. OK inference, OK file size.",
@ -2842,11 +2897,12 @@ def unsloth_push_to_hub_gguf(
quantization_method = quantization_method,
first_conversion = first_conversion,
push_to_hub = False, # Never push from here
token = None, # Don't need token for local save
token = token, # forwarded so imatrix_file=True can read a gated/private upstream
max_shard_size = max_shard_size,
safe_serialization = safe_serialization,
temporary_location = temporary_location,
maximum_memory_usage = maximum_memory_usage,
imatrix_file = imatrix_file,
)
# Extract results
@ -3097,6 +3153,95 @@ def _lora_base_model_id(model):
return os.fspath(base) if base else ""
# Upstream Unsloth GGUF repos ship a calibration imatrix under one of these names; the GGUF-format
# one is suffixed .gguf_file so the Hub does not list it as a model GGUF (renamed to .gguf locally).
_IMATRIX_UPSTREAM_NAMES = ("imatrix_unsloth.dat", "imatrix_unsloth.gguf_file")
def _gguf_repo_candidates(model):
"""Ordered, de-duplicated unsloth/<base>-GGUF repo ids to search for an upstream imatrix."""
candidates = []
raw_names = [
_lora_base_model_id(model),
getattr(getattr(model, "config", None), "_name_or_path", None),
]
for raw in raw_names:
if not raw:
continue
name = os.fspath(raw)
if os.path.isdir(name):
continue # a local checkpoint has no upstream GGUF repo
try:
name = get_model_name(name, load_in_4bit = False)
except Exception:
pass
if not name or "/" not in name:
continue
repo = name if name.endswith("-GGUF") else f"{name}-GGUF"
if repo not in candidates:
candidates.append(repo)
return candidates
def _materialize_imatrix(path, dest_dir):
"""Copy an imatrix into dest_dir (never mutate the HF cache) and rename *.gguf_file -> *.gguf."""
os.makedirs(dest_dir, exist_ok = True)
base = os.path.basename(path)
if base.endswith(".gguf_file"):
base = base[: -len(".gguf_file")] + ".gguf"
local = os.path.join(dest_dir, base)
shutil.copyfile(path, local)
return local
def _resolve_imatrix_file(model, imatrix_file, token, dest_dir):
"""Turn the public imatrix_file value into a local imatrix path (or None).
None/False -> None. A path -> that file (a *.gguf_file is renamed to *.gguf). True -> find and
download the upstream unsloth/<base>-GGUF imatrix, raising a clear error if none exists.
"""
if imatrix_file is None or imatrix_file is False:
return None
if imatrix_file is not True and isinstance(imatrix_file, (str, os.PathLike)):
path = os.path.expanduser(os.fspath(imatrix_file))
if not os.path.isfile(path):
raise FileNotFoundError(f"Unsloth: imatrix_file '{path}' does not exist.")
return _materialize_imatrix(path, dest_dir) if path.endswith(".gguf_file") else path
if imatrix_file is not True:
raise TypeError(
"Unsloth: imatrix_file must be None, a path string, or True "
f"(got {type(imatrix_file).__name__})."
)
# imatrix_file=True: auto-resolve from the upstream Unsloth GGUF repo. HfApi is the module-level
# import (save.py top); hf_hub_download is imported here as it is not needed elsewhere.
from huggingface_hub import hf_hub_download
if token is None:
token = get_token()
api = HfApi(token = token)
repos = _gguf_repo_candidates(model)
for repo in repos:
try:
files = set(api.list_repo_files(repo))
except Exception:
continue
for name in _IMATRIX_UPSTREAM_NAMES:
if name in files:
downloaded = hf_hub_download(repo_id = repo, filename = name, token = token)
local = _materialize_imatrix(downloaded, dest_dir)
print(f"Unsloth: Using imatrix '{name}' from '{repo}' -> '{local}'")
return local
raise RuntimeError(
"Unsloth: imatrix_file=True but no upstream Unsloth imatrix was found.\n"
f" Searched repos: {repos or '(none derived from the base model)'}\n"
f" Searched files: {list(_IMATRIX_UPSTREAM_NAMES)}\n"
"Pass imatrix_file='/path/to/imatrix.(dat|gguf)' to use your own."
)
def _unsloth_save_lora_gguf(
model,
tokenizer,