From 2c8aacc55afdc3d46cad6a1c09c3901eaae20e62 Mon Sep 17 00:00:00 2001 From: danielhanchen Date: Thu, 2 Jul 2026 12:52:03 +0000 Subject: [PATCH 1/2] Compressed export: fail fast when transformers exceeds the llm-compressor ceiling llm-compressor 0.10.x/0.12.x pins transformers<=4.57.6. Models that require a newer transformers (e.g. Qwen3.5, which needs the transformers 5.3.0 sidecar) cannot be FP8/FP4 quantized at all: the compressed-export subprocess imports transformers.modeling_utils.TORCH_INIT_FUNCTIONS, which was removed in transformers 5.x, so it dies with a cryptic ImportError, and only after the expensive 16bit merge has already been written. Detect this in _unsloth_save_compressed_tensors before installing llm-compressor or merging, and raise a clear, actionable error (export to GGUF or 16-bit instead). Unsloth Studio surfaces it as a clean 400 in the export panel rather than a confusing traceback, and no full 16bit checkpoint is written for a model that can never be quantized this way. - Add _LLM_COMPRESSOR_MAX_TRANSFORMERS = "4.57.6" next to _LLM_COMPRESSOR_SPEC, to bump in lockstep with a future llm-compressor that supports newer transformers. - Add _transformers_exceeds_llm_compressor_ceiling() helper: packaging.version compare, fail-open on any parse error so a real quantization attempt still surfaces its own error. - Gate _unsloth_save_compressed_tensors on it before install/merge. --- unsloth/save.py | 44 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/unsloth/save.py b/unsloth/save.py index 226ca5fed8..7789b80766 100644 --- a/unsloth/save.py +++ b/unsloth/save.py @@ -1368,6 +1368,38 @@ def install_python_non_blocking(packages = []): # bump deliberately. Floor 0.6.0 keeps torch>=2.4 resolvable (0.7+ need torch>=2.7; torch pinned below). _LLM_COMPRESSOR_SPEC = "llmcompressor>=0.6.0,<=0.12.0" +# Highest transformers release llm-compressor 0.10.x/0.12.x can run against (its metadata pins +# transformers<=4.57.6). Models that require a newer-transformers sidecar (e.g. Qwen3.5 needs +# transformers 5.3.0) cannot be quantized by llm-compressor at all: it imports +# transformers.modeling_utils.TORCH_INIT_FUNCTIONS, which was removed in transformers 5.x, so the +# compressed-export subprocess dies with a cryptic ImportError AFTER the expensive 16bit merge. +# Detect that up front and fail fast with an actionable message. Bump this in lockstep with a +# llm-compressor release that supports newer transformers. +_LLM_COMPRESSOR_MAX_TRANSFORMERS = "4.57.6" + + +def _transformers_exceeds_llm_compressor_ceiling(transformers_version = None): + """Return (exceeds, active_version) comparing the active transformers to the llm-compressor ceiling. + + `exceeds` is True only when we can parse both versions and the active transformers is strictly + newer than `_LLM_COMPRESSOR_MAX_TRANSFORMERS`. Any parse failure returns False (fail open) so a + real quantization attempt still surfaces the underlying error rather than a false positive. + """ + if transformers_version is None: + try: + import transformers as _tf + transformers_version = _tf.__version__ + except Exception: + return False, "unknown" + try: + from packaging.version import parse as _parse + # Drop any local build suffix ("4.57.6+abc") so it does not skew the comparison. + active = _parse(str(transformers_version).split("+", 1)[0]) + ceiling = _parse(_LLM_COMPRESSOR_MAX_TRANSFORMERS) + return active > ceiling, str(transformers_version) + except Exception: + return False, str(transformers_version) + def install_llm_compressor(): """Import llm-compressor, installing it on first use for FP8/FP4 export. @@ -4114,6 +4146,18 @@ def _unsloth_save_compressed_tensors( if not is_main_process: return None + # llm-compressor cannot run under a newer transformers than its ceiling: the quantization + # subprocess would die with a cryptic ImportError (TORCH_INIT_FUNCTIONS) only AFTER the costly + # 16bit merge. Detect and fail fast with an actionable message BEFORE installing llm-compressor + # or merging, so a model that can never be quantized this way does no wasted work. + _exceeds, _tf_ver = _transformers_exceeds_llm_compressor_ceiling() + if _exceeds: + raise RuntimeError( + f"Unsloth: FP8/FP4 compressed-tensors export is not available for this model. It runs " + f"under transformers {_tf_ver}, but llm-compressor supports transformers " + f"<= {_LLM_COMPRESSOR_MAX_TRANSFORMERS}. Export to GGUF or 16-bit instead." + ) + # 1) Install llm-compressor and gate on scheme availability BEFORE merging, so an unsupported # scheme (e.g. mxfp8) fails fast instead of writing a full 16bit checkpoint first. install_llm_compressor() From 873cb0be55f22c6ae0ae3f31093246217c525a75 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 2 Jul 2026 12:52:50 +0000 Subject: [PATCH 2/2] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- unsloth/save.py | 1 + 1 file changed, 1 insertion(+) diff --git a/unsloth/save.py b/unsloth/save.py index 7789b80766..3129614819 100644 --- a/unsloth/save.py +++ b/unsloth/save.py @@ -1393,6 +1393,7 @@ def _transformers_exceeds_llm_compressor_ceiling(transformers_version = None): return False, "unknown" try: from packaging.version import parse as _parse + # Drop any local build suffix ("4.57.6+abc") so it does not skew the comparison. active = _parse(str(transformers_version).split("+", 1)[0]) ceiling = _parse(_LLM_COMPRESSOR_MAX_TRANSFORMERS)