ceos/unsloth
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Support for out-of-source quantizers (#3534)

Browse source 
* Support for out-of-source quantizers

* Fix decorators and functions to be staticmethod

Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

---------

Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>
Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>
This commit is contained in:
Giuseppe Franco 2025-11-14 11:52:24 +01:00 • committed by GitHub
commit fa5b05b70b
2 changed files with 13 additions and 2 deletions
Download patch file Download diff file Expand all files Collapse all files

8
unsloth/models/llama.py
View file

@ -2046,6 +2046,11 @@ def unsloth_fast_generate(
class FastLlamaModel:
@staticmethod
def _prepare_for_qat(model, qat_scheme):
model = _prepare_model_for_qat(model, qat_scheme)
return model
@staticmethod
def pre_patch():
init_name, function = patch_llama_rope_scaling(
@ -3007,7 +3012,8 @@ class FastLlamaModel:
# Apply QAT + LoRA if specified
if qat_scheme is not None:
print("Unsloth: Applying QAT to mitigate quantization degradation")
model = _prepare_model_for_qat(model, qat_scheme)
model = FastLlamaModel._prepare_for_qat(model, qat_scheme)
model._saved_temp_tokenizer = _saved_temp_tokenizer

7
unsloth/models/loader.py
View file

@ -595,6 +595,11 @@ except:
class FastModel(FastBaseModel):
@staticmethod
def _prepare_for_qat(model, qat_scheme):
model = _prepare_model_for_qat(model, qat_scheme)
return model
@staticmethod
def from_pretrained(
model_name = "unsloth/Llama-3.2-11B-Vision-Instruct-bnb-4bit",
@ -1144,7 +1149,7 @@ class FastModel(FastBaseModel):
# Apply QAT if specified
if qat_scheme is not None:
print("Unsloth: Applying QAT to mitigate quantization degradation")
model = _prepare_model_for_qat(model, qat_scheme)
model = FastModel._prepare_for_qat(model, qat_scheme)
# Patch Tiled MLP
# to turn on set UNSLOTH_TILED_MLP to "arctic", "target", or "target:{GB}""