Compare commits
2 commits
main
...
fix-compre
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
873cb0be55 | ||
|
|
2c8aacc55a |
1 changed files with 45 additions and 0 deletions
|
|
@ -1368,6 +1368,39 @@ def install_python_non_blocking(packages = []):
|
||||||
# bump deliberately. Floor 0.6.0 keeps torch>=2.4 resolvable (0.7+ need torch>=2.7; torch pinned below).
|
# bump deliberately. Floor 0.6.0 keeps torch>=2.4 resolvable (0.7+ need torch>=2.7; torch pinned below).
|
||||||
_LLM_COMPRESSOR_SPEC = "llmcompressor>=0.6.0,<=0.12.0"
|
_LLM_COMPRESSOR_SPEC = "llmcompressor>=0.6.0,<=0.12.0"
|
||||||
|
|
||||||
|
# Highest transformers release llm-compressor 0.10.x/0.12.x can run against (its metadata pins
|
||||||
|
# transformers<=4.57.6). Models that require a newer-transformers sidecar (e.g. Qwen3.5 needs
|
||||||
|
# transformers 5.3.0) cannot be quantized by llm-compressor at all: it imports
|
||||||
|
# transformers.modeling_utils.TORCH_INIT_FUNCTIONS, which was removed in transformers 5.x, so the
|
||||||
|
# compressed-export subprocess dies with a cryptic ImportError AFTER the expensive 16bit merge.
|
||||||
|
# Detect that up front and fail fast with an actionable message. Bump this in lockstep with a
|
||||||
|
# llm-compressor release that supports newer transformers.
|
||||||
|
_LLM_COMPRESSOR_MAX_TRANSFORMERS = "4.57.6"
|
||||||
|
|
||||||
|
|
||||||
|
def _transformers_exceeds_llm_compressor_ceiling(transformers_version = None):
|
||||||
|
"""Return (exceeds, active_version) comparing the active transformers to the llm-compressor ceiling.
|
||||||
|
|
||||||
|
`exceeds` is True only when we can parse both versions and the active transformers is strictly
|
||||||
|
newer than `_LLM_COMPRESSOR_MAX_TRANSFORMERS`. Any parse failure returns False (fail open) so a
|
||||||
|
real quantization attempt still surfaces the underlying error rather than a false positive.
|
||||||
|
"""
|
||||||
|
if transformers_version is None:
|
||||||
|
try:
|
||||||
|
import transformers as _tf
|
||||||
|
transformers_version = _tf.__version__
|
||||||
|
except Exception:
|
||||||
|
return False, "unknown"
|
||||||
|
try:
|
||||||
|
from packaging.version import parse as _parse
|
||||||
|
|
||||||
|
# Drop any local build suffix ("4.57.6+abc") so it does not skew the comparison.
|
||||||
|
active = _parse(str(transformers_version).split("+", 1)[0])
|
||||||
|
ceiling = _parse(_LLM_COMPRESSOR_MAX_TRANSFORMERS)
|
||||||
|
return active > ceiling, str(transformers_version)
|
||||||
|
except Exception:
|
||||||
|
return False, str(transformers_version)
|
||||||
|
|
||||||
|
|
||||||
def install_llm_compressor():
|
def install_llm_compressor():
|
||||||
"""Import llm-compressor, installing it on first use for FP8/FP4 export.
|
"""Import llm-compressor, installing it on first use for FP8/FP4 export.
|
||||||
|
|
@ -4114,6 +4147,18 @@ def _unsloth_save_compressed_tensors(
|
||||||
if not is_main_process:
|
if not is_main_process:
|
||||||
return None
|
return None
|
||||||
|
|
||||||
|
# llm-compressor cannot run under a newer transformers than its ceiling: the quantization
|
||||||
|
# subprocess would die with a cryptic ImportError (TORCH_INIT_FUNCTIONS) only AFTER the costly
|
||||||
|
# 16bit merge. Detect and fail fast with an actionable message BEFORE installing llm-compressor
|
||||||
|
# or merging, so a model that can never be quantized this way does no wasted work.
|
||||||
|
_exceeds, _tf_ver = _transformers_exceeds_llm_compressor_ceiling()
|
||||||
|
if _exceeds:
|
||||||
|
raise RuntimeError(
|
||||||
|
f"Unsloth: FP8/FP4 compressed-tensors export is not available for this model. It runs "
|
||||||
|
f"under transformers {_tf_ver}, but llm-compressor supports transformers "
|
||||||
|
f"<= {_LLM_COMPRESSOR_MAX_TRANSFORMERS}. Export to GGUF or 16-bit instead."
|
||||||
|
)
|
||||||
|
|
||||||
# 1) Install llm-compressor and gate on scheme availability BEFORE merging, so an unsupported
|
# 1) Install llm-compressor and gate on scheme availability BEFORE merging, so an unsupported
|
||||||
# scheme (e.g. mxfp8) fails fast instead of writing a full 16bit checkpoint first.
|
# scheme (e.g. mxfp8) fails fast instead of writing a full 16bit checkpoint first.
|
||||||
install_llm_compressor()
|
install_llm_compressor()
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue