ceos/unsloth
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Merge branch 'unslothai:main' into FST

Browse source
This commit is contained in:
electroglyph 2025-12-19 00:05:08 -08:00 • committed by GitHub
commit 6cc710e6e3
10 changed files with 203 additions and 56 deletions
Download patch file Download diff file Expand all files Collapse all files

2
.github/FUNDING.yml vendored
View file

@ -1,6 +1,6 @@
# These are supported funding model platforms
github: # Replace with up to 4 GitHub Sponsors-enabled usernames e.g., [user1, user2]
github: unslothai
patreon: # Replace with a single Patreon username
open_collective: # Replace with a single Open Collective username
ko_fi: unsloth

2
.pre-commit-config.yaml
View file

@ -1,6 +1,6 @@
repos:
- repo: https://github.com/astral-sh/ruff-pre-commit
rev: v0.14.8
rev: v0.14.9
hooks:
- id: ruff
args:

4
pyproject.toml
View file

@ -60,7 +60,7 @@ huggingfacenotorch = [
]
huggingface = [
"unsloth[huggingfacenotorch]",
"unsloth_zoo>=2025.12.4",
"unsloth_zoo>=2025.12.6",
"torchvision",
"unsloth[triton]",
]
@ -523,7 +523,7 @@ colab-ampere-torch220 = [
"flash-attn>=2.6.3 ; ('linux' in sys_platform)",
]
colab-new = [
"unsloth_zoo>=2025.12.4",
"unsloth_zoo>=2025.12.6",
"packaging",
"tyro",
"transformers>=4.51.3,!=4.52.0,!=4.52.1,!=4.52.2,!=4.52.3,!=4.53.0,!=4.54.0,!=4.55.0,!=4.55.1,!=4.57.0,<=4.57.3",

6
unsloth/__init__.py
View file

@ -29,14 +29,17 @@ from .import_fixes import (
fix_message_factory_issue,
check_fbgemm_gpu_version,
torchvision_compatibility_check,
fix_diffusers_warnings,
)
fix_message_factory_issue()
check_fbgemm_gpu_version()
torchvision_compatibility_check()
fix_diffusers_warnings()
del fix_message_factory_issue
del check_fbgemm_gpu_version
del torchvision_compatibility_check
del fix_diffusers_warnings
# This check is critical because Unsloth optimizes these libraries by modifying
# their code at import time. If they're imported first, the original (slower,
@ -126,6 +129,7 @@ from .import_fixes import (
patch_datasets,
patch_enable_input_require_grads,
fix_openenv_no_vllm,
fix_executorch,
)
fix_xformers_performance_issue()
@ -137,6 +141,7 @@ patch_trackio()
patch_datasets()
patch_enable_input_require_grads()
fix_openenv_no_vllm()
fix_executorch()
del fix_xformers_performance_issue
del fix_vllm_aimv2_issue
@ -147,6 +152,7 @@ del patch_trackio
del patch_datasets
del patch_enable_input_require_grads
del fix_openenv_no_vllm
del fix_executorch
# Torch 2.4 has including_emulation
if DEVICE_TYPE == "cuda":

185
unsloth/import_fixes.py
View file

@ -19,8 +19,25 @@ from importlib.metadata import version as importlib_version
from packaging.version import Version as TrueVersion
import re
import logging
# Cannot import logger here since it'll import transformers
# from unsloth_zoo.log import logger
import textwrap
# We cannot do from unsloth_zoo.log import logger since FBGEMM might cause seg faults.
UNSLOTH_ENABLE_LOGGING = os.environ.get("UNSLOTH_ENABLE_LOGGING", "0") in (
"1",
"True",
"true",
)
logger = logging.getLogger(__name__)
if UNSLOTH_ENABLE_LOGGING:
logging.basicConfig(
level = logging.INFO, format = "[%(name)s|%(levelname)s]%(message)s"
)
logger.setLevel(logging.INFO)
else:
logging.basicConfig(
level = logging.WARNING, format = "[%(name)s|%(levelname)s]%(message)s"
)
logger.setLevel(logging.WARNING)
def Version(version):
@ -54,6 +71,36 @@ class HideLoggingMessage(logging.Filter):
return not (self.text in x.getMessage())
class HidePrintMessage:
__slots__ = ("_original_stream", "_hidden_texts")
def __init__(self, original_stream):
self._original_stream = original_stream
self._hidden_texts = []
def add_filter(self, text):
self._hidden_texts.append(text)
def write(self, message):
if not any(text in message for text in self._hidden_texts):
self._original_stream.write(message)
def flush(self):
self._original_stream.flush()
def __getattr__(self, name):
return getattr(self._original_stream, name)
if os.environ.get("UNSLOTH_ENABLE_LOGGING", "0") != "1":
import sys
# Apply to stderr for FBGEMM
sys.stderr = HidePrintMessage(sys.stderr)
# https://github.com/pytorch/FBGEMM/blob/d99cd96490ec4aabac2ee95b1e76ea4dcfcfa628/fbgemm_gpu/experimental/gemm/triton_gemm/utils.py#L43-L52
sys.stderr.add_filter("TMA benchmarks will be running")
# Fix up AttributeError: 'MessageFactory' object has no attribute 'GetPrototype'
# MUST do this at the start primarily due to tensorflow causing issues
def fix_message_factory_issue():
@ -70,8 +117,6 @@ def fix_message_factory_issue():
def GetPrototype(self, *args, **kwargs):
return
from unsloth_zoo.log import logger
if not hasattr(google.protobuf.message_factory, "MessageFactory"):
logger.info("Unsloth: Patching protobuf.MessageFactory as it doesn't exist")
google.protobuf.message_factory.MessageFactory = MessageFactory
@ -105,14 +150,16 @@ def fix_message_factory_issue():
# Fix Xformers performance issues since 0.0.25
def fix_xformers_performance_issue():
if importlib.util.find_spec("xformers") is None:
spec = importlib.util.find_spec("xformers")
if spec is None:
return
xformers_version = importlib_version("xformers")
if Version(xformers_version) < Version("0.0.29"):
from unsloth_zoo.log import logger
xformers_location = importlib.util.find_spec("xformers").origin
xformers_location = os.path.split(xformers_location)[0]
xformers_location = spec.origin
if xformers_location is None:
xformers_location = spec.submodule_search_locations[0]
else:
xformers_location = os.path.split(xformers_location)[0]
cutlass = Path(xformers_location) / "ops" / "fmha" / "cutlass.py"
try:
if cutlass.exists():
@ -136,15 +183,17 @@ def fix_xformers_performance_issue():
# ValueError: 'aimv2' is already used by a Transformers config, pick another name.
def fix_vllm_aimv2_issue():
if importlib.util.find_spec("vllm") is None:
spec = importlib.util.find_spec("vllm")
if spec is None:
return
vllm_version = importlib_version("vllm")
if Version(vllm_version) < Version("0.10.1"):
from unsloth_zoo.log import logger
vllm_version = importlib.util.find_spec("vllm").origin
vllm_version = os.path.split(vllm_version)[0]
ovis_config = Path(vllm_version) / "transformers_utils" / "configs" / "ovis.py"
vllm_location = spec.origin
if vllm_location is None:
vllm_location = spec.submodule_search_locations[0]
else:
vllm_location = os.path.split(vllm_location)[0]
ovis_config = Path(vllm_location) / "transformers_utils" / "configs" / "ovis.py"
try:
if ovis_config.exists():
with open(ovis_config, "r+", encoding = "utf-8") as f:
@ -273,7 +322,6 @@ def check_fbgemm_gpu_version():
raise ImportError(
f"Unsloth: fbgemm_gpu_genai=={fbgemm_gpu_version} detected. It might cause unexpected issues like segmentation faults. Please uninstall the current one by doing `pip uninstall fbgemm-gpu` && `pip install fbgemm-gpu` to install fbgemm-gpu 1.4.0 or newer!"
)
from unsloth_zoo.log import logger
logger.info(f"Unsloth: fbgemm_gpu_genai=={fbgemm_gpu_version} detected.")
@ -336,7 +384,6 @@ def patch_enable_input_require_grads():
self._require_grads_hook = hooks[0]
PreTrainedModel.enable_input_require_grads = _patched_enable_input_require_grads
from unsloth_zoo.log import logger
logger.info(
"Unsloth: Patched enable_input_require_grads for vision model compatibility"
@ -378,7 +425,6 @@ def torchvision_compatibility_check():
f"but found torchvision=={torchvision_version}. "
f"Please refer to https://pytorch.org/get-started/previous-versions/ for more information."
)
from unsloth_zoo.log import logger
logger.info(
f"Unsloth: torch=={torch_version} and torchvision=={torchvision_version} are compatible."
@ -387,14 +433,17 @@ def torchvision_compatibility_check():
# Fix TRL OpenEnv 0.26 NameError: name 'SamplingParams' is not defined
def fix_openenv_no_vllm():
if importlib.util.find_spec("trl") is None:
spec = importlib.util.find_spec("trl")
if spec is None:
return
trl_location = importlib.util.find_spec("trl").origin
trl_location = os.path.split(trl_location)[0]
trl_location = spec.origin
if trl_location is None:
trl_location = spec.submodule_search_locations[0]
else:
trl_location = os.path.split(trl_location)[0]
openenv = Path(trl_location) / "experimental" / "openenv" / "utils.py"
if not openenv.exists():
return
from unsloth_zoo.log import logger
try:
with open(openenv, "r+", encoding = "utf-8") as f:
@ -404,18 +453,15 @@ def fix_openenv_no_vllm():
" from vllm import SamplingParams\n"
" from vllm.sampling_params import GuidedDecodingParams\n"
)
if bad + "\n" + "\n" in text:
text = text.replace(
bad + "\n" + "\n",
bad
+ (
"else:\n"
" from typing import Any\n"
" SamplingParams = Any\n"
" GuidedDecodingParams = Any\n"
"\n"
),
)
replace_with = bad + (
"else:\n"
" from typing import Any\n"
" SamplingParams = Any\n"
" GuidedDecodingParams = Any\n"
"\n"
)
if bad + "\n" + "\n" in text and replace_with not in text:
text = text.replace(bad + "\n" + "\n", replace_with)
f.seek(0)
f.write(text)
f.truncate()
@ -424,3 +470,74 @@ def fix_openenv_no_vllm():
)
except Exception as e:
logger.info(f"Unsloth: Failed patching TRL OpenEnv with error = {str(e)}")
# Fix Exeuctorch needing get_mapped_key
def fix_executorch():
spec = importlib.util.find_spec("executorch")
if spec is None:
return
executorch_location = spec.origin
if executorch_location is None:
executorch_location = spec.submodule_search_locations[0]
else:
executorch_location = os.path.split(executorch_location)[0]
executorch = Path(executorch_location) / "examples" / "models" / "__init__.py"
if not executorch.exists():
return
try:
what = r"""
import sys
import types
import re
from typing import Any, Optional
def get_mapped_key(key: str, mapping_dict: dict[str, str]) -> str:
try:
# Checks if there is a layer # in the key
if any(k.isdigit() for k in key.split(".")):
# Replace layer number with "{}" to create key for lookup
abstract_key = re.sub(r"(\.\d+)", ".{}", key)
layer_num = re.search(r"\d+", key).group(0)
new_key = mapping_dict[abstract_key]
new_key = new_key.format(layer_num)
else:
new_key = mapping_dict[key]
except KeyError as e:
raise Exception(
f'Error converting the state dict. Found unexpected key: "{key}". '
"Please make sure you're loading a checkpoint with the right format. "
) from e
return new_key
torchtune = types.ModuleType("torchtune")
torchtune.__path__ = []
models = types.ModuleType("torchtune.models")
models.__path__ = []
convert_weights = types.ModuleType("torchtune.models.convert_weights")
convert_weights.get_mapped_key = get_mapped_key
torchtune.models = models
models.convert_weights = convert_weights
sys.modules["torchtune"] = torchtune
sys.modules["torchtune.models"] = models
sys.modules["torchtune.models.convert_weights"] = convert_weights
"""
what = textwrap.dedent(what)
with open(executorch, "r+", encoding = "utf-8") as f:
text = f.read()
bad = "from enum import Enum\n"
if bad in text and what not in text:
text = text.replace(bad + "\n", bad + "\n" + what)
f.seek(0)
f.write(text)
f.truncate()
logger.info("Unsloth: Patching Executorch to fix get_mapped_key")
except Exception as e:
logger.info(f"Unsloth: Failed Executorch with error = {str(e)}")
def fix_diffusers_warnings():
# Silence Flax classes are deprecated and will be removed in Diffusers v1.0.0.
os.environ["DIFFUSERS_VERBOSITY"] = "error"

12
unsloth/models/_utils.py
View file

@ -12,7 +12,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
__version__ = "2025.12.5"
__version__ = "2025.12.7"
__all__ = [
"SUPPORTS_BFLOAT16",
@ -413,16 +413,6 @@ try:
except:
pass
# Flax classes are deprecated and will be removed in Diffusers v1.0.0.
try:
from diffusers.utils import logger as diffusers_logger
diffusers_logger.addFilter(HideLoggingMessage("are deprecated"))
del diffusers_logger
except:
pass
# Errors out on
# Some weights of Gemma3nForConditionalGeneration were not initialized from the model checkpoint
from transformers.modeling_utils import logger as transformers_logger

16
unsloth/models/loader.py
View file

@ -289,6 +289,8 @@ class FastLanguageModel(FastLlamaModel):
trust_remote_code = trust_remote_code,
)
is_model = True
except ImportError:
raise
except Exception as error:
autoconfig_error = str(error)
if "architecture" in autoconfig_error:
@ -305,6 +307,8 @@ class FastLanguageModel(FastLlamaModel):
trust_remote_code = trust_remote_code,
)
is_peft = True
except ImportError:
raise
except Exception as error:
peft_error = str(error)
if "architecture" in peft_error:
@ -326,7 +330,8 @@ class FastLanguageModel(FastLlamaModel):
"Please separate the LoRA and base models to 2 repos."
)
model_types = get_transformers_model_type(
peft_config if peft_config is not None else model_config
peft_config if peft_config is not None else model_config,
trust_remote_code = trust_remote_code,
)
if len(model_types) == 1:
model_type = model_types[0]
@ -739,6 +744,8 @@ class FastModel(FastBaseModel):
"compatible with `full_finetuning=True`. If you wish to use QAT with LoRA, "
"please pass in `qat_scheme` in `FastLanguageModel.get_peft_model(...)` instead."
)
if qat_scheme == "phone-deployment":
qat_scheme = "int8-int4"
# Check if 4bit is allowed specifically for AMD
if not ALLOW_BITSANDBYTES and not use_exact_model_name:
if load_in_4bit or load_in_8bit or model_name.lower().endswith("-bnb-4bit"):
@ -824,6 +831,8 @@ class FastModel(FastBaseModel):
trust_remote_code = trust_remote_code,
)
is_model = True
except ImportError:
raise
except Exception as error:
autoconfig_error = str(error)
if "architecture" in autoconfig_error:
@ -840,6 +849,8 @@ class FastModel(FastBaseModel):
trust_remote_code = trust_remote_code,
)
is_peft = True
except ImportError:
raise
except Exception as error:
peft_error = str(error)
if "architecture" in peft_error:
@ -859,7 +870,8 @@ class FastModel(FastBaseModel):
"Please separate the LoRA and base models to 2 repos."
)
model_types = get_transformers_model_type(
peft_config if peft_config is not None else model_config
peft_config if peft_config is not None else model_config,
trust_remote_code = trust_remote_code,
)
model_types_all = ",".join(model_types) + ","

5
unsloth/models/mapper.py
View file

@ -1251,6 +1251,11 @@ __INT_TO_FLOAT_MAPPER = \
"unsloth/gpt-oss-safeguard-120b",
"openai/gpt-oss-safeguard-120b",
),
"unsloth/functiongemma-270m-it-unsloth-bnb-4bit" : (
"unsloth/functiongemma-270m-it",
"google/functiongemma-270m-it",
"unsloth/functiongemma-270m-it-unsloth-bnb-4bit",
),
}
INT_TO_FLOAT_MAPPER = {}

3
unsloth/models/rl.py
View file

@ -741,6 +741,7 @@ def _patch_trl_rl_trainers(trainer_file = "grpo_trainer"):
"generation_kwargs": {},
"bf16": False,
"fp16": False,
"report_to": "none",
"include_tokens_per_second": False,
"include_num_input_tokens_seen": False,
"auto_find_batch_size": False, # Auto /2 batch size - too many people complained so removing
@ -907,8 +908,6 @@ def _patch_trl_rl_trainers(trainer_file = "grpo_trainer"):
for process_extra_arg in process_extra_args:
extra_args += process_extra_arg(old_RLTrainer_source, old_RLConfig_source)
# Edit report_to and default it to nothing if max_steps is like 60
# Create RLConfig args
extra_args = extra_args.split("\n")
extra_args = "\n".join(" " * 8 + x for x in extra_args)

24
unsloth/save.py
View file

@ -2745,6 +2745,17 @@ def _unsloth_save_torchao_with_attached_config(
"""Save a QAT-trained model by converting fake-quantized weights to real quantized weights."""
# Convert QAT fake-quantized weights to real quantized weights
_convert_torchao_model(model)
# PEFT models also might come here, so parse it
if isinstance(model, PeftModelForCausalLM):
_unsloth_save_torchao_with_given_config(
model = model,
save_directory = save_directory,
tokenizer = tokenizer,
torchao_config = model.config.quantization_config,
push_to_hub = push_to_hub,
token = token,
)
return
# TorchAO does not support safe_serialization reliably
safe_serialization = False
@ -2806,7 +2817,10 @@ def _unsloth_save_torchao_with_given_config(
)
from torchao import quantize_
quantization_config = TorchAoConfig(quant_type = torchao_config)
if isinstance(torchao_config, TorchAoConfig):
quantization_config = torchao_config
else:
quantization_config = TorchAoConfig(quant_type = torchao_config)
# Determine if this is a VLM
is_vlm = False
@ -2897,7 +2911,7 @@ def unsloth_save_pretrained_torchao(
)
if torchao_config is not None:
# PTQ path: user provided a config, model must NOT have QAT config
# PTQ path: user provided a config, model must NOT have QAT config unless PEFT
assert not has_qat_config, (
"Unsloth: You passed `torchao_config` but this model was trained with `qat_scheme`. "
"For QAT models, do not pass `torchao_config` - the quantization config is already "
@ -3010,7 +3024,11 @@ def patch_saving_functions(model, vision = False):
original_model = model
while True:
if original_model.push_to_hub.__name__ != "unsloth_push_to_hub":
# Check if push_to_hub exists before accessing its __name__
if (
hasattr(original_model, "push_to_hub")
and original_model.push_to_hub.__name__ != "unsloth_push_to_hub"
):
original_model.original_push_to_hub = original_model.push_to_hub
original_model.push_to_hub = types.MethodType(
unsloth_push_to_hub, original_model