From ef6e98dfa56dc72e1c4c42d4b0de501d2f140c21 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Sat, 27 Jun 2026 09:37:31 +0000 Subject: [PATCH] Harden calibration data handling and compressed-export edge cases - Calibration messages without a chat template now handle multimodal (list) content, None content, and null message rows instead of crashing on join - Raise a clear error when the calibration dataset is empty after subsampling - Reset llm-compressor's global session before freeing the model in the sequential -> basic NVFP4 fallback, so the old model is actually released - LoRA GGUF export accepts a single-element list quantization_method - Attach datasets metadata to the pushed repo on compressed hub exports - Warn (instead of silently) if the model cannot be restored to its device - Raise a clear error if the LoRA base model id cannot be determined --- unsloth/_compressed_quantize.py | 44 ++++++++++++++++++++++++++++++--- unsloth/save.py | 43 +++++++++++++++++++++++++------- 2 files changed, 74 insertions(+), 13 deletions(-) diff --git a/unsloth/_compressed_quantize.py b/unsloth/_compressed_quantize.py index a7c1836ae7..c3626e8f13 100644 --- a/unsloth/_compressed_quantize.py +++ b/unsloth/_compressed_quantize.py @@ -67,6 +67,15 @@ def _build_calibration_dataset(tokenizer, kind, value, num_samples, max_seq_leng else: raise ValueError(f"Unknown calibration-dataset-kind: {kind}") + try: + if len(ds) == 0: + raise RuntimeError( + "Unsloth: the calibration dataset is empty after loading/subsampling; " + "pass a non-empty calibration_dataset." + ) + except TypeError: + pass # streaming / iterable datasets have no len(); let llm-compressor handle them + cols = set(ds.column_names) if "input_ids" in cols: return ds @@ -75,10 +84,29 @@ def _build_calibration_dataset(tokenizer, kind, value, num_samples, max_seq_leng # of calling apply_chat_template (which would raise). has_chat_template = bool(getattr(_tok, "chat_template", None)) + def _content_to_text(content): + # content may be a str, None, or a multimodal list of parts (str or {"text": ...}). + if content is None: + return "" + if isinstance(content, str): + return content + if isinstance(content, (list, tuple)): + parts = [] + for part in content: + if isinstance(part, str): + parts.append(part) + elif isinstance(part, dict): + text = part.get("text") or part.get("content") + if isinstance(text, str): + parts.append(text) + return " ".join(parts) + return str(content) + def _prep(ex): + msgs = ex["messages"] or [] if has_chat_template: - return {"text": _tok.apply_chat_template(ex["messages"], tokenize = False)} - return {"text": "\n".join(m.get("content", "") for m in ex["messages"])} + return {"text": _tok.apply_chat_template(msgs, tokenize = False)} + return {"text": "\n".join(_content_to_text(m.get("content")) for m in msgs)} ds = ds.map(_prep) elif "text" not in cols: @@ -200,11 +228,19 @@ def main(): "retrying with the 'basic' pipeline (needs the full model to fit in memory).", flush = True, ) - # Free the partially-processed model (and the traceback frames pinning it) before - # loading a fresh copy, so the fallback does not transiently hold two copies on GPU. + # Free the partially-processed model before loading a fresh copy, so the fallback does + # not transiently hold two copies on GPU. llm-compressor keeps the model in a global + # session after a failed run, so reset it first; also drop the traceback frames (e) and + # the local reference that pin the model. import gc as _gc import torch as _torch + try: + from llmcompressor.core import reset_session + + reset_session() + except Exception: + pass e = None del model _gc.collect() diff --git a/unsloth/save.py b/unsloth/save.py index fc2633315f..6cf6c506bb 100644 --- a/unsloth/save.py +++ b/unsloth/save.py @@ -2420,13 +2420,16 @@ def unsloth_save_pretrained_gguf( "Unsloth: Please use .push_to_hub_gguf(save_method='lora') instead of " ".save_pretrained_gguf(save_method='lora', push_to_hub=True)." ) - if quantization_method in _LORA_GGUF_OUTTYPES: - _outtype = quantization_method + _qm = quantization_method + if isinstance(_qm, (list, tuple)) and len(_qm) == 1: + _qm = _qm[0] # the gguf API allows a list; unwrap a single outtype + if _qm in _LORA_GGUF_OUTTYPES: + _outtype = _qm else: - if quantization_method not in (None, "fast_quantized"): + if _qm not in (None, "fast_quantized"): logger.warning_once( f"Unsloth: LoRA GGUF export does not support " - f"quantization_method='{quantization_method}'; using outtype 'f16'. " + f"quantization_method={quantization_method!r}; using outtype 'f16'. " f"Valid LoRA outtypes: {_LORA_GGUF_OUTTYPES}." ) _outtype = "f16" @@ -2748,13 +2751,16 @@ def unsloth_push_to_hub_gguf( # save_method="lora" exports the adapter itself as a GGUF LoRA (not a merged model). if save_method is not None and str(save_method).lower() == "lora": - if quantization_method in _LORA_GGUF_OUTTYPES: - _outtype = quantization_method + _qm = quantization_method + if isinstance(_qm, (list, tuple)) and len(_qm) == 1: + _qm = _qm[0] # the gguf API allows a list; unwrap a single outtype + if _qm in _LORA_GGUF_OUTTYPES: + _outtype = _qm else: - if quantization_method not in (None, "fast_quantized"): + if _qm not in (None, "fast_quantized"): logger.warning_once( f"Unsloth: LoRA GGUF export does not support " - f"quantization_method='{quantization_method}'; using outtype 'f16'. " + f"quantization_method={quantization_method!r}; using outtype 'f16'. " f"Valid LoRA outtypes: {_LORA_GGUF_OUTTYPES}." ) _outtype = "f16" @@ -3085,6 +3091,11 @@ def _unsloth_save_lora_gguf( # Resolve the dequantized base id (the adapter usually references a 4bit repo). base_model_id = _lora_base_model_id(model) + if not base_model_id: + raise RuntimeError( + "Unsloth: could not determine the base model for LoRA GGUF export " + "(no adapter base_model_name_or_path or model config _name_or_path)." + ) try: base_model_id = get_model_name(base_model_id, load_in_4bit = False) except Exception: @@ -4033,6 +4044,17 @@ def _unsloth_save_compressed_tensors( create_pr = merge_kwargs.get("create_pr", False), revision = merge_kwargs.get("revision", None), ) + # Attach datasets metadata to the pushed repo, like the normal merged push path. + datasets = merge_kwargs.get("datasets", None) + if datasets: + try: + from huggingface_hub import metadata_update + + metadata_update(repo_id, {"datasets": datasets}, overwrite = True, token = token) + except Exception as meta_err: + logger.warning_once( + f"Unsloth: could not update datasets metadata for {repo_id}: {meta_err}" + ) # 9) Inference hardware note. result = repo_id if push_to_hub else out_dir @@ -4043,7 +4065,10 @@ def _unsloth_save_compressed_tensors( try: model.to(model_dev) # restore the model to its original device except Exception: - pass + logger.warning_once( + "Unsloth: could not restore the model to its original device after compressed " + "export; it may remain on CPU." + ) if calib_tmp is not None and os.path.isdir(calib_tmp): shutil.rmtree(calib_tmp, ignore_errors = True) if work_tmp is not None: