diff --git a/.github/FUNDING.yml b/.github/FUNDING.yml index a6cda7d034..4ebb6df3d0 100644 --- a/.github/FUNDING.yml +++ b/.github/FUNDING.yml @@ -1,6 +1,6 @@ # These are supported funding model platforms -github: # Replace with up to 4 GitHub Sponsors-enabled usernames e.g., [user1, user2] +github: unslothai patreon: # Replace with a single Patreon username open_collective: # Replace with a single Open Collective username ko_fi: unsloth diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 11114de4b4..9d70d22426 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,6 +1,6 @@ repos: - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.14.8 + rev: v0.14.9 hooks: - id: ruff args: diff --git a/pyproject.toml b/pyproject.toml index c6e19b014e..84d6d86d93 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -60,7 +60,7 @@ huggingfacenotorch = [ ] huggingface = [ "unsloth[huggingfacenotorch]", - "unsloth_zoo>=2025.12.4", + "unsloth_zoo>=2025.12.6", "torchvision", "unsloth[triton]", ] @@ -523,7 +523,7 @@ colab-ampere-torch220 = [ "flash-attn>=2.6.3 ; ('linux' in sys_platform)", ] colab-new = [ - "unsloth_zoo>=2025.12.4", + "unsloth_zoo>=2025.12.6", "packaging", "tyro", "transformers>=4.51.3,!=4.52.0,!=4.52.1,!=4.52.2,!=4.52.3,!=4.53.0,!=4.54.0,!=4.55.0,!=4.55.1,!=4.57.0,<=4.57.3", diff --git a/unsloth/__init__.py b/unsloth/__init__.py index 007b952200..d10a0f8030 100644 --- a/unsloth/__init__.py +++ b/unsloth/__init__.py @@ -29,14 +29,17 @@ from .import_fixes import ( fix_message_factory_issue, check_fbgemm_gpu_version, torchvision_compatibility_check, + fix_diffusers_warnings, ) fix_message_factory_issue() check_fbgemm_gpu_version() torchvision_compatibility_check() +fix_diffusers_warnings() del fix_message_factory_issue del check_fbgemm_gpu_version del torchvision_compatibility_check +del fix_diffusers_warnings # This check is critical because Unsloth optimizes these libraries by modifying # their code at import time. If they're imported first, the original (slower, @@ -126,6 +129,7 @@ from .import_fixes import ( patch_datasets, patch_enable_input_require_grads, fix_openenv_no_vllm, + fix_executorch, ) fix_xformers_performance_issue() @@ -137,6 +141,7 @@ patch_trackio() patch_datasets() patch_enable_input_require_grads() fix_openenv_no_vllm() +fix_executorch() del fix_xformers_performance_issue del fix_vllm_aimv2_issue @@ -147,6 +152,7 @@ del patch_trackio del patch_datasets del patch_enable_input_require_grads del fix_openenv_no_vllm +del fix_executorch # Torch 2.4 has including_emulation if DEVICE_TYPE == "cuda": diff --git a/unsloth/import_fixes.py b/unsloth/import_fixes.py index a78b5451ea..f3aae7f523 100644 --- a/unsloth/import_fixes.py +++ b/unsloth/import_fixes.py @@ -19,8 +19,25 @@ from importlib.metadata import version as importlib_version from packaging.version import Version as TrueVersion import re import logging -# Cannot import logger here since it'll import transformers -# from unsloth_zoo.log import logger +import textwrap + +# We cannot do from unsloth_zoo.log import logger since FBGEMM might cause seg faults. +UNSLOTH_ENABLE_LOGGING = os.environ.get("UNSLOTH_ENABLE_LOGGING", "0") in ( + "1", + "True", + "true", +) +logger = logging.getLogger(__name__) +if UNSLOTH_ENABLE_LOGGING: + logging.basicConfig( + level = logging.INFO, format = "[%(name)s|%(levelname)s]%(message)s" + ) + logger.setLevel(logging.INFO) +else: + logging.basicConfig( + level = logging.WARNING, format = "[%(name)s|%(levelname)s]%(message)s" + ) + logger.setLevel(logging.WARNING) def Version(version): @@ -54,6 +71,36 @@ class HideLoggingMessage(logging.Filter): return not (self.text in x.getMessage()) +class HidePrintMessage: + __slots__ = ("_original_stream", "_hidden_texts") + + def __init__(self, original_stream): + self._original_stream = original_stream + self._hidden_texts = [] + + def add_filter(self, text): + self._hidden_texts.append(text) + + def write(self, message): + if not any(text in message for text in self._hidden_texts): + self._original_stream.write(message) + + def flush(self): + self._original_stream.flush() + + def __getattr__(self, name): + return getattr(self._original_stream, name) + + +if os.environ.get("UNSLOTH_ENABLE_LOGGING", "0") != "1": + import sys + + # Apply to stderr for FBGEMM + sys.stderr = HidePrintMessage(sys.stderr) + # https://github.com/pytorch/FBGEMM/blob/d99cd96490ec4aabac2ee95b1e76ea4dcfcfa628/fbgemm_gpu/experimental/gemm/triton_gemm/utils.py#L43-L52 + sys.stderr.add_filter("TMA benchmarks will be running") + + # Fix up AttributeError: 'MessageFactory' object has no attribute 'GetPrototype' # MUST do this at the start primarily due to tensorflow causing issues def fix_message_factory_issue(): @@ -70,8 +117,6 @@ def fix_message_factory_issue(): def GetPrototype(self, *args, **kwargs): return - from unsloth_zoo.log import logger - if not hasattr(google.protobuf.message_factory, "MessageFactory"): logger.info("Unsloth: Patching protobuf.MessageFactory as it doesn't exist") google.protobuf.message_factory.MessageFactory = MessageFactory @@ -105,14 +150,16 @@ def fix_message_factory_issue(): # Fix Xformers performance issues since 0.0.25 def fix_xformers_performance_issue(): - if importlib.util.find_spec("xformers") is None: + spec = importlib.util.find_spec("xformers") + if spec is None: return xformers_version = importlib_version("xformers") if Version(xformers_version) < Version("0.0.29"): - from unsloth_zoo.log import logger - - xformers_location = importlib.util.find_spec("xformers").origin - xformers_location = os.path.split(xformers_location)[0] + xformers_location = spec.origin + if xformers_location is None: + xformers_location = spec.submodule_search_locations[0] + else: + xformers_location = os.path.split(xformers_location)[0] cutlass = Path(xformers_location) / "ops" / "fmha" / "cutlass.py" try: if cutlass.exists(): @@ -136,15 +183,17 @@ def fix_xformers_performance_issue(): # ValueError: 'aimv2' is already used by a Transformers config, pick another name. def fix_vllm_aimv2_issue(): - if importlib.util.find_spec("vllm") is None: + spec = importlib.util.find_spec("vllm") + if spec is None: return vllm_version = importlib_version("vllm") if Version(vllm_version) < Version("0.10.1"): - from unsloth_zoo.log import logger - - vllm_version = importlib.util.find_spec("vllm").origin - vllm_version = os.path.split(vllm_version)[0] - ovis_config = Path(vllm_version) / "transformers_utils" / "configs" / "ovis.py" + vllm_location = spec.origin + if vllm_location is None: + vllm_location = spec.submodule_search_locations[0] + else: + vllm_location = os.path.split(vllm_location)[0] + ovis_config = Path(vllm_location) / "transformers_utils" / "configs" / "ovis.py" try: if ovis_config.exists(): with open(ovis_config, "r+", encoding = "utf-8") as f: @@ -273,7 +322,6 @@ def check_fbgemm_gpu_version(): raise ImportError( f"Unsloth: fbgemm_gpu_genai=={fbgemm_gpu_version} detected. It might cause unexpected issues like segmentation faults. Please uninstall the current one by doing `pip uninstall fbgemm-gpu` && `pip install fbgemm-gpu` to install fbgemm-gpu 1.4.0 or newer!" ) - from unsloth_zoo.log import logger logger.info(f"Unsloth: fbgemm_gpu_genai=={fbgemm_gpu_version} detected.") @@ -336,7 +384,6 @@ def patch_enable_input_require_grads(): self._require_grads_hook = hooks[0] PreTrainedModel.enable_input_require_grads = _patched_enable_input_require_grads - from unsloth_zoo.log import logger logger.info( "Unsloth: Patched enable_input_require_grads for vision model compatibility" @@ -378,7 +425,6 @@ def torchvision_compatibility_check(): f"but found torchvision=={torchvision_version}. " f"Please refer to https://pytorch.org/get-started/previous-versions/ for more information." ) - from unsloth_zoo.log import logger logger.info( f"Unsloth: torch=={torch_version} and torchvision=={torchvision_version} are compatible." @@ -387,14 +433,17 @@ def torchvision_compatibility_check(): # Fix TRL OpenEnv 0.26 NameError: name 'SamplingParams' is not defined def fix_openenv_no_vllm(): - if importlib.util.find_spec("trl") is None: + spec = importlib.util.find_spec("trl") + if spec is None: return - trl_location = importlib.util.find_spec("trl").origin - trl_location = os.path.split(trl_location)[0] + trl_location = spec.origin + if trl_location is None: + trl_location = spec.submodule_search_locations[0] + else: + trl_location = os.path.split(trl_location)[0] openenv = Path(trl_location) / "experimental" / "openenv" / "utils.py" if not openenv.exists(): return - from unsloth_zoo.log import logger try: with open(openenv, "r+", encoding = "utf-8") as f: @@ -404,18 +453,15 @@ def fix_openenv_no_vllm(): " from vllm import SamplingParams\n" " from vllm.sampling_params import GuidedDecodingParams\n" ) - if bad + "\n" + "\n" in text: - text = text.replace( - bad + "\n" + "\n", - bad - + ( - "else:\n" - " from typing import Any\n" - " SamplingParams = Any\n" - " GuidedDecodingParams = Any\n" - "\n" - ), - ) + replace_with = bad + ( + "else:\n" + " from typing import Any\n" + " SamplingParams = Any\n" + " GuidedDecodingParams = Any\n" + "\n" + ) + if bad + "\n" + "\n" in text and replace_with not in text: + text = text.replace(bad + "\n" + "\n", replace_with) f.seek(0) f.write(text) f.truncate() @@ -424,3 +470,74 @@ def fix_openenv_no_vllm(): ) except Exception as e: logger.info(f"Unsloth: Failed patching TRL OpenEnv with error = {str(e)}") + + +# Fix Exeuctorch needing get_mapped_key +def fix_executorch(): + spec = importlib.util.find_spec("executorch") + if spec is None: + return + executorch_location = spec.origin + if executorch_location is None: + executorch_location = spec.submodule_search_locations[0] + else: + executorch_location = os.path.split(executorch_location)[0] + executorch = Path(executorch_location) / "examples" / "models" / "__init__.py" + if not executorch.exists(): + return + + try: + what = r""" + import sys + import types + import re + from typing import Any, Optional + def get_mapped_key(key: str, mapping_dict: dict[str, str]) -> str: + try: + # Checks if there is a layer # in the key + if any(k.isdigit() for k in key.split(".")): + # Replace layer number with "{}" to create key for lookup + abstract_key = re.sub(r"(\.\d+)", ".{}", key) + layer_num = re.search(r"\d+", key).group(0) + new_key = mapping_dict[abstract_key] + new_key = new_key.format(layer_num) + else: + new_key = mapping_dict[key] + except KeyError as e: + raise Exception( + f'Error converting the state dict. Found unexpected key: "{key}". ' + "Please make sure you're loading a checkpoint with the right format. " + ) from e + + return new_key + + torchtune = types.ModuleType("torchtune") + torchtune.__path__ = [] + models = types.ModuleType("torchtune.models") + models.__path__ = [] + convert_weights = types.ModuleType("torchtune.models.convert_weights") + convert_weights.get_mapped_key = get_mapped_key + torchtune.models = models + models.convert_weights = convert_weights + sys.modules["torchtune"] = torchtune + sys.modules["torchtune.models"] = models + sys.modules["torchtune.models.convert_weights"] = convert_weights + """ + what = textwrap.dedent(what) + + with open(executorch, "r+", encoding = "utf-8") as f: + text = f.read() + bad = "from enum import Enum\n" + if bad in text and what not in text: + text = text.replace(bad + "\n", bad + "\n" + what) + f.seek(0) + f.write(text) + f.truncate() + logger.info("Unsloth: Patching Executorch to fix get_mapped_key") + except Exception as e: + logger.info(f"Unsloth: Failed Executorch with error = {str(e)}") + + +def fix_diffusers_warnings(): + # Silence Flax classes are deprecated and will be removed in Diffusers v1.0.0. + os.environ["DIFFUSERS_VERBOSITY"] = "error" diff --git a/unsloth/models/_utils.py b/unsloth/models/_utils.py index 653b539b20..17d0ab9631 100644 --- a/unsloth/models/_utils.py +++ b/unsloth/models/_utils.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -__version__ = "2025.12.5" +__version__ = "2025.12.7" __all__ = [ "SUPPORTS_BFLOAT16", @@ -413,16 +413,6 @@ try: except: pass -# Flax classes are deprecated and will be removed in Diffusers v1.0.0. -try: - from diffusers.utils import logger as diffusers_logger - - diffusers_logger.addFilter(HideLoggingMessage("are deprecated")) - del diffusers_logger -except: - pass - - # Errors out on # Some weights of Gemma3nForConditionalGeneration were not initialized from the model checkpoint from transformers.modeling_utils import logger as transformers_logger diff --git a/unsloth/models/loader.py b/unsloth/models/loader.py index e1c13315f7..6148b1783e 100644 --- a/unsloth/models/loader.py +++ b/unsloth/models/loader.py @@ -289,6 +289,8 @@ class FastLanguageModel(FastLlamaModel): trust_remote_code = trust_remote_code, ) is_model = True + except ImportError: + raise except Exception as error: autoconfig_error = str(error) if "architecture" in autoconfig_error: @@ -305,6 +307,8 @@ class FastLanguageModel(FastLlamaModel): trust_remote_code = trust_remote_code, ) is_peft = True + except ImportError: + raise except Exception as error: peft_error = str(error) if "architecture" in peft_error: @@ -326,7 +330,8 @@ class FastLanguageModel(FastLlamaModel): "Please separate the LoRA and base models to 2 repos." ) model_types = get_transformers_model_type( - peft_config if peft_config is not None else model_config + peft_config if peft_config is not None else model_config, + trust_remote_code = trust_remote_code, ) if len(model_types) == 1: model_type = model_types[0] @@ -739,6 +744,8 @@ class FastModel(FastBaseModel): "compatible with `full_finetuning=True`. If you wish to use QAT with LoRA, " "please pass in `qat_scheme` in `FastLanguageModel.get_peft_model(...)` instead." ) + if qat_scheme == "phone-deployment": + qat_scheme = "int8-int4" # Check if 4bit is allowed specifically for AMD if not ALLOW_BITSANDBYTES and not use_exact_model_name: if load_in_4bit or load_in_8bit or model_name.lower().endswith("-bnb-4bit"): @@ -824,6 +831,8 @@ class FastModel(FastBaseModel): trust_remote_code = trust_remote_code, ) is_model = True + except ImportError: + raise except Exception as error: autoconfig_error = str(error) if "architecture" in autoconfig_error: @@ -840,6 +849,8 @@ class FastModel(FastBaseModel): trust_remote_code = trust_remote_code, ) is_peft = True + except ImportError: + raise except Exception as error: peft_error = str(error) if "architecture" in peft_error: @@ -859,7 +870,8 @@ class FastModel(FastBaseModel): "Please separate the LoRA and base models to 2 repos." ) model_types = get_transformers_model_type( - peft_config if peft_config is not None else model_config + peft_config if peft_config is not None else model_config, + trust_remote_code = trust_remote_code, ) model_types_all = ",".join(model_types) + "," diff --git a/unsloth/models/mapper.py b/unsloth/models/mapper.py index 7a90c6b358..e7296291f1 100644 --- a/unsloth/models/mapper.py +++ b/unsloth/models/mapper.py @@ -1251,6 +1251,11 @@ __INT_TO_FLOAT_MAPPER = \ "unsloth/gpt-oss-safeguard-120b", "openai/gpt-oss-safeguard-120b", ), + "unsloth/functiongemma-270m-it-unsloth-bnb-4bit" : ( + "unsloth/functiongemma-270m-it", + "google/functiongemma-270m-it", + "unsloth/functiongemma-270m-it-unsloth-bnb-4bit", + ), } INT_TO_FLOAT_MAPPER = {} diff --git a/unsloth/models/rl.py b/unsloth/models/rl.py index 604323ac8c..31316e45b7 100644 --- a/unsloth/models/rl.py +++ b/unsloth/models/rl.py @@ -741,6 +741,7 @@ def _patch_trl_rl_trainers(trainer_file = "grpo_trainer"): "generation_kwargs": {}, "bf16": False, "fp16": False, + "report_to": "none", "include_tokens_per_second": False, "include_num_input_tokens_seen": False, "auto_find_batch_size": False, # Auto /2 batch size - too many people complained so removing @@ -907,8 +908,6 @@ def _patch_trl_rl_trainers(trainer_file = "grpo_trainer"): for process_extra_arg in process_extra_args: extra_args += process_extra_arg(old_RLTrainer_source, old_RLConfig_source) - # Edit report_to and default it to nothing if max_steps is like 60 - # Create RLConfig args extra_args = extra_args.split("\n") extra_args = "\n".join(" " * 8 + x for x in extra_args) diff --git a/unsloth/save.py b/unsloth/save.py index 01887321cf..24303aba52 100644 --- a/unsloth/save.py +++ b/unsloth/save.py @@ -2745,6 +2745,17 @@ def _unsloth_save_torchao_with_attached_config( """Save a QAT-trained model by converting fake-quantized weights to real quantized weights.""" # Convert QAT fake-quantized weights to real quantized weights _convert_torchao_model(model) + # PEFT models also might come here, so parse it + if isinstance(model, PeftModelForCausalLM): + _unsloth_save_torchao_with_given_config( + model = model, + save_directory = save_directory, + tokenizer = tokenizer, + torchao_config = model.config.quantization_config, + push_to_hub = push_to_hub, + token = token, + ) + return # TorchAO does not support safe_serialization reliably safe_serialization = False @@ -2806,7 +2817,10 @@ def _unsloth_save_torchao_with_given_config( ) from torchao import quantize_ - quantization_config = TorchAoConfig(quant_type = torchao_config) + if isinstance(torchao_config, TorchAoConfig): + quantization_config = torchao_config + else: + quantization_config = TorchAoConfig(quant_type = torchao_config) # Determine if this is a VLM is_vlm = False @@ -2897,7 +2911,7 @@ def unsloth_save_pretrained_torchao( ) if torchao_config is not None: - # PTQ path: user provided a config, model must NOT have QAT config + # PTQ path: user provided a config, model must NOT have QAT config unless PEFT assert not has_qat_config, ( "Unsloth: You passed `torchao_config` but this model was trained with `qat_scheme`. " "For QAT models, do not pass `torchao_config` - the quantization config is already " @@ -3010,7 +3024,11 @@ def patch_saving_functions(model, vision = False): original_model = model while True: - if original_model.push_to_hub.__name__ != "unsloth_push_to_hub": + # Check if push_to_hub exists before accessing its __name__ + if ( + hasattr(original_model, "push_to_hub") + and original_model.push_to_hub.__name__ != "unsloth_push_to_hub" + ): original_model.original_push_to_hub = original_model.push_to_hub original_model.push_to_hub = types.MethodType( unsloth_push_to_hub, original_model