unsloth/unsloth/models/loader.py

# Copyright 2023-present Daniel Han-Chen & the Unsloth team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from ._utils import (
    _prepare_model_for_qat,
    is_bfloat16_supported,
    is_vLLM_available,
    HAS_FLASH_ATTENTION,
    HAS_FLASH_ATTENTION_SOFTCAPPING,
    USE_MODELSCOPE,
    get_transformers_model_type,
    hf_login,
    # Single source of truth is _utils.py; re-exported here so callers doing
    # `from unsloth.models.loader import DISABLE_SDPA_MODEL_NAMES` keep working and so
    # _is_sdpa_excluded (in _utils) can honor it without a loader -> _utils cycle.
    DISABLE_SDPA_MODEL_NAMES,
)
from .granite import FastGraniteModel
from .llama import FastLlamaModel, logger
from .mistral import FastMistralModel
from .qwen2 import FastQwen2Model
from .qwen3 import FastQwen3Model
from .qwen3_moe import FastQwen3MoeModel
from .cohere import FastCohereModel
from transformers import AutoConfig
from transformers import __version__ as transformers_version
from peft import PeftConfig, PeftModel
from .loader_utils import (
    _exclude_rope_inv_freq_from_ddp,
    _get_fp8_mode_and_check_settings,
    _offline_quantize_to_fp8,
    _tag_model_with_fp8_torchao_config,
    get_model_name,
    prepare_device_map,
    _offline_aware_load,
    _resolve_checkpoint_tokenizer_name,
    _is_offline_related_error,
)
import os, contextlib, sys

try:
    from huggingface_hub import get_token
except:
    try:
        from huggingface_hub.utils import get_token
    except:
        # For older versions of huggingface_hub
        from huggingface_hub.utils._token import get_token
import importlib.util
from ..device_type import (
    is_hip,
    get_device_type,
    DEVICE_TYPE,
    DEVICE_TYPE_TORCH,
    DEVICE_COUNT,
    ALLOW_PREQUANTIZED_MODELS,
    ALLOW_BITSANDBYTES,
)

# https://github.com/huggingface/transformers/pull/26037 allows 4 bit loading!
from unsloth_zoo.utils import Version, _get_dtype
from unsloth_zoo.hf_utils import dtype_from_config
from unsloth_zoo.tiled_mlp import patch_tiled_mlp

transformers_version = Version(transformers_version)
SUPPORTS_FOURBIT = transformers_version >= Version("4.37")
SUPPORTS_GEMMA = transformers_version >= Version("4.38")
SUPPORTS_GEMMA2 = transformers_version >= Version("4.42")
SUPPORTS_LLAMA31 = transformers_version >= Version("4.43.2")
SUPPORTS_LLAMA32 = transformers_version > Version("4.45.0")
SUPPORTS_GRANITE = transformers_version >= Version("4.46.0")
SUPPORTS_QWEN3 = transformers_version >= Version("4.50.3")
SUPPORTS_QWEN3_MOE = transformers_version >= Version("4.50.3")
SUPPORTS_FALCON_H1 = transformers_version >= Version("4.53.0")
SUPPORTS_GEMMA3N = transformers_version >= Version("4.53.0")
SUPPORTS_GPTOSS = transformers_version >= Version("4.55.0")
SUPPORTS_GEMMA4 = transformers_version >= Version("5.5.0")
# Transformers v5 meta-device loading corrupts non-persistent buffers (inv_freq).
# See _fix_rope_inv_freq() below for details.
_NEEDS_ROPE_FIX = transformers_version >= Version("5.0.0")
if SUPPORTS_GEMMA:
    from .gemma import FastGemmaModel
if SUPPORTS_GEMMA2:
    from .gemma2 import FastGemma2Model
if SUPPORTS_FALCON_H1:
    from .falcon_h1 import FastFalconH1Model
import torch
from ._utils import (
    patch_compiling_bitsandbytes,
    patch_model_and_tokenizer,
    prepare_model_for_kbit_training,
    apply_unsloth_gradient_checkpointing,
    patch_compiled_autograd,
    process_vision_info,
    unsloth_compile_transformers,
    fast_inference_setup,
    _get_text_only_config,
    resolve_model_class,
    _is_family_text_decoder,
    _apply_text_only_key_mapping,
    set_task_config_attr,
    maybe_prefetch_hf_snapshot,
)

# Source of truth is unsloth_zoo.model_lists. Re-exported so callers doing
# `from unsloth.models.loader import FORCE_FLOAT32` keep working. The fallback
# list is also unioned in so a newer unsloth still forces float32 for these
# archs when paired with an older unsloth_zoo that predates them (upgrade skew).
_FORCE_FLOAT32_FALLBACK = [
    "gemma3,",  # Add comma bc gemma3 will match gemma3n
    "gemma3text",  # Gemma3TextModel (EmbeddingGemma, standalone text-only Gemma3)
    "gemma3n",
    "gemma4",  # Gemma4 (gemma4 / gemma4_text): float16 NaNs grad norms in the backward
    "glm4_moe",  # GLM-4.x MoE (glm4_moe / glm4_moe_lite): float16 NaNs grad norms
    "gpt_oss",
    "qwen3_5",  # Qwen3.5 GDN layers produce NaN grad norms in float16 training
    "qwen3_moe",  # Qwen3-MoE (Qwen3-30B-A3B): float16 NaNs grad norms in the backward
]
try:
    from unsloth_zoo import FORCE_FLOAT32 as _ZOO_FORCE_FLOAT32
    FORCE_FLOAT32 = list(_ZOO_FORCE_FLOAT32)
except ImportError:
    FORCE_FLOAT32 = []
for _mt in _FORCE_FLOAT32_FALLBACK:
    if not any(_mt in _entry for _entry in FORCE_FLOAT32):
        FORCE_FLOAT32.append(_mt)

global DISABLE_COMPILE_MODEL_NAMES
# Must be alphabetically sorted for each entry


def _strip_unsloth_bnb_4bit_suffix(model_name: str) -> str:
    """Remove Unsloth 4bit suffixes without lowercasing (HF cache dirs are case-sensitive)."""
    s = model_name
    for suffix in ("-unsloth-bnb-4bit", "-bnb-4bit"):
        if len(s) >= len(suffix) and s.lower().endswith(suffix.lower()):
            s = s[: -len(suffix)]
    return s


def _config_get(
    config,
    field_name,
    default = None,
):
    if isinstance(config, dict):
        return config.get(field_name, default)
    return getattr(config, field_name, default)


def _config_diff(config):
    if isinstance(config, dict):
        return config
    to_diff_dict = getattr(config, "to_diff_dict", None)
    if callable(to_diff_dict):
        try:
            diff = to_diff_dict()
            if isinstance(diff, dict):
                return diff
        except Exception:
            pass
    return {}


def _has_sequence_classification_architecture(config):
    architectures = _config_get(config, "architectures", None) or []
    return any(str(arch).endswith("ForSequenceClassification") for arch in architectures)


def _get_user_task_config_attrs(user_config):
    if user_config is None:
        return {}
    diff = _config_diff(user_config)
    attrs = {}
    for key in ("id2label", "label2id", "problem_type"):
        if key in diff:
            attrs[key] = _config_get(user_config, key, diff.get(key))
    if isinstance(user_config, dict) and "num_labels" in user_config:
        attrs["num_labels"] = user_config["num_labels"]
    elif _has_sequence_classification_architecture(user_config):
        num_labels = _config_get(user_config, "num_labels", None)
        if num_labels is not None:
            attrs["num_labels"] = num_labels
    elif "id2label" in attrs:
        try:
            attrs["num_labels"] = len(attrs["id2label"])
        except TypeError:
            pass
    return attrs


DISABLE_COMPILE_MODEL_NAMES = [
    "aya_vision",
    "modernbert",
    "granite,llava_next",  # Granite-vision 3
]

# Architectures with gated-deltanet (linear attention) layers. Unsloth bundles the
# flash-linear-attention Triton kernels (unsloth_zoo/_vendored/fla), so no install is
# needed; transformers uses the much slower pure PyTorch path only when they can't be enabled.
FLA_MODEL_TYPE_PREFIXES = ("qwen3_next", "qwen3_5", "kimi_linear", "olmo_hybrid")
_fla_advised = False


def _maybe_advise_fla_install(model_types):
    """One-time note when a gated-deltanet model loads without the fast kernels.

    The kernels ship with Unsloth (no install needed); this fires only when they
    could not be enabled on this platform (e.g. no CUDA, torch < 2.7 or
    triton < 3.3), i.e. exactly when transformers uses the slow pure PyTorch path.
    """
    global _fla_advised
    if _fla_advised:
        return
    if model_types is None:
        return
    if isinstance(model_types, str):
        model_types = [model_types]  # a lone string would otherwise iterate chars
    try:
        if not any(
            isinstance(t, str) and t.startswith(FLA_MODEL_TYPE_PREFIXES) for t in model_types
        ):
            return
        from transformers.utils.import_utils import is_flash_linear_attention_available
        if is_flash_linear_attention_available():
            return  # bundled (or user-installed) fast kernels are active
    except Exception:
        return
    _fla_advised = True
    print(
        "Unsloth: This model uses gated-deltanet linear attention layers. Unsloth\n"
        "bundles the flash-linear-attention kernels, but they could not be enabled\n"
        "on this setup (they need CUDA with torch >= 2.7 and triton >= 3.3), so\n"
        "transformers will use a slower pure PyTorch path."
    )


def _fix_rope_inv_freq(model):
    """Fix inv_freq corruption caused by transformers v5 meta-device loading.

    v5 inits on meta then replaces all non-persistent buffers with uninitialized
    memory. Vanilla restores inv_freq via _init_weights() (needs original_inv_freq),
    but Unsloth rotary classes lack that attr, so inv_freq stays corrupted -> wrong
    positional encodings and 5-11x higher training loss. Here we recompute inv_freq
    from base/dim, apply scaling, and rebuild cos/sin caches. No-op on v4.
    """
    if not _NEEDS_ROPE_FIX:
        return model

    for name, module in model.named_modules():
        # Unsloth's LlamaRotaryEmbedding and subclasses (Extended, LinearScaling,
        # Granite). Native v5 rotary classes (Gemma3, etc.) have original_inv_freq
        # which v5's _init_weights() uses to restore inv_freq, so they are fine.
        if (
            hasattr(module, "inv_freq")
            and hasattr(module, "base")
            and hasattr(module, "dim")
            and hasattr(module, "_apply_inv_freq_scaling")
            and hasattr(module, "multi_gpu_cos_cached")
        ):
            if hasattr(module, "_unsloth_recompute_inv_freq"):
                # Restore config scaling (llama3/yarn); unscaled here broke v5.
                inv_freq = module._unsloth_recompute_inv_freq()
            else:
                inv_freq = 1.0 / (
                    module.base
                    ** (
                        torch.arange(0, module.dim, 2, dtype = torch.int64, device = "cpu").float()
                        / module.dim
                    )
                )
                inv_freq = module._apply_inv_freq_scaling(inv_freq)
            module.inv_freq = inv_freq
            for device_idx in range(len(module.multi_gpu_cos_cached)):
                if module.multi_gpu_cos_cached[device_idx] is not None:
                    module._set_cos_sin_cache(
                        seq_len = module.current_rope_size,
                        device = torch.device(device_idx),
                        dtype = torch.get_default_dtype(),
                    )

        # LongRopeRotaryEmbedding (Phi-3.5 style with short_inv_freq + long_inv_freq)
        elif (
            hasattr(module, "short_inv_freq")
            and hasattr(module, "long_inv_freq")
            and hasattr(module, "base")
            and hasattr(module, "dim")
        ):
            config = getattr(model, "config", None)
            rope_scaling = getattr(config, "rope_scaling", None) if config else None
            if rope_scaling is not None:
                short_factor = rope_scaling.get("short_factor", None)
                long_factor = rope_scaling.get("long_factor", None)
                if short_factor is not None and long_factor is not None:
                    inv_freq_shape = (
                        torch.arange(0, module.dim, 2, dtype = torch.int64, device = "cpu").float()
                        / module.dim
                    )
                    sf = torch.tensor(short_factor, device = "cpu", dtype = torch.float32)
                    lf = torch.tensor(long_factor, device = "cpu", dtype = torch.float32)
                    module.short_inv_freq = 1.0 / (sf * module.base**inv_freq_shape)
                    module.long_inv_freq = 1.0 / (lf * module.base**inv_freq_shape)

                    dtype = torch.bfloat16 if is_bfloat16_supported() else torch.float16
                    t = torch.arange(
                        module.original_max_position_embeddings,
                        device = module.short_inv_freq.device,
                        dtype = torch.int64,
                    ).float()
                    freqs = torch.outer(t, module.short_inv_freq)
                    emb = torch.cat((freqs, freqs), dim = -1)
                    for device_idx in range(len(module.multi_gpu_short_cos_cached)):
                        if module.multi_gpu_short_cos_cached[device_idx] is not None:
                            device_obj = torch.device(device_idx)
                            module.multi_gpu_short_cos_cached[device_idx] = (
                                emb.cos() * module.scaling_factor
                            ).to(dtype = dtype, device = device_obj, non_blocking = True)
                            module.multi_gpu_short_sin_cached[device_idx] = (
                                emb.sin() * module.scaling_factor
                            ).to(dtype = dtype, device = device_obj, non_blocking = True)
    return model


class FastLanguageModel(FastLlamaModel):
    @staticmethod
    @_offline_aware_load
    def from_pretrained(
        model_name = "unsloth/Llama-3.2-1B-Instruct",
        max_seq_length = 2048,
        dtype = None,
        load_in_4bit = True,  # 4bit QLoRA
        load_in_8bit = False,  # 8bit  LoRA
        load_in_16bit = False,  # 16bit LoRA
        full_finetuning = False,
        token = None,
        device_map = "sequential",
        rope_scaling = None,
        fix_tokenizer = True,
        trust_remote_code = False,
        use_gradient_checkpointing = "unsloth",
        resize_model_vocab = None,
        revision = None,
        use_exact_model_name = False,
        offload_embedding = False,
        float32_mixed_precision = None,  # Forces float32 mixed precision
        fast_inference = False,  # uses vLLM
        gpu_memory_utilization = 0.5,
        float8_kv_cache = False,
        random_state = 3407,
        max_lora_rank = 64,
        disable_log_stats = True,
        qat_scheme = None,
        load_in_fp8 = False,  # fp8 LoRA (True, False, 'block')
        unsloth_tiled_mlp = False,
        text_only = False,  # Skip vision/audio towers and load only the text decoder
        *args,
        **kwargs,
    ):
        # Respect user-provided quantization_config (e.g. BitsAndBytesConfig)
        quantization_config = kwargs.get("quantization_config", None)
        if quantization_config is not None:
            if isinstance(quantization_config, dict):
                q_load_in_4bit = quantization_config.get("load_in_4bit", False)
                q_load_in_8bit = quantization_config.get("load_in_8bit", False)
            else:
                q_load_in_4bit = getattr(quantization_config, "load_in_4bit", False)
                q_load_in_8bit = getattr(quantization_config, "load_in_8bit", False)
            if q_load_in_4bit:
                load_in_4bit = True
                load_in_8bit = False
            if q_load_in_8bit:
                load_in_8bit = True
                load_in_4bit = False

        # Login to allow private models
        token = hf_login(token)
        # Align dtype with bnb_4bit_compute_dtype if provided and dtype is unset.
        if dtype is None and quantization_config is not None:
            bnb_compute_dtype = None
            if isinstance(quantization_config, dict):
                if quantization_config.get("load_in_4bit", False):
                    bnb_compute_dtype = quantization_config.get("bnb_4bit_compute_dtype", None)
            else:
                if getattr(quantization_config, "load_in_4bit", False):
                    bnb_compute_dtype = getattr(quantization_config, "bnb_4bit_compute_dtype", None)
            if isinstance(bnb_compute_dtype, str):
                bnb_compute_dtype = getattr(torch, bnb_compute_dtype, None)
            if isinstance(bnb_compute_dtype, torch.dtype):
                dtype = bnb_compute_dtype

        # Distributed-safe device placement for quantized models.
        # In multi-GPU (torchrun), each rank must load the model on its own device
        # to avoid Accelerate device relocation errors with quantized weights.
        is_quantized = load_in_4bit or load_in_8bit or load_in_fp8
        if is_quantized and isinstance(device_map, str):
            distributed_device_map, is_dist = prepare_device_map()
            if is_dist:
                device_map = distributed_device_map

        # @_offline_aware_load already forced offline when needed; delegations inherit it.
        if load_in_8bit or full_finetuning or qat_scheme is not None:
            return FastModel.from_pretrained(
                model_name = model_name,
                max_seq_length = max_seq_length,
                dtype = dtype,
                load_in_4bit = load_in_4bit,
                load_in_8bit = load_in_8bit,
                load_in_16bit = load_in_16bit,
                full_finetuning = full_finetuning,
                token = token,
                device_map = device_map,
                rope_scaling = rope_scaling,  # [TODO] No effect
                fix_tokenizer = fix_tokenizer,  # [TODO] No effect
                trust_remote_code = trust_remote_code,
                use_gradient_checkpointing = use_gradient_checkpointing,
                resize_model_vocab = resize_model_vocab,  # [TODO] No effect
                revision = revision,
                return_logits = False,  # Return logits
                fullgraph = True,  # No graph breaks
                use_exact_model_name = use_exact_model_name,
                offload_embedding = offload_embedding,
                float32_mixed_precision = float32_mixed_precision,
                # Pass vLLM/inference parameters
                fast_inference = fast_inference,
                gpu_memory_utilization = gpu_memory_utilization,
                float8_kv_cache = float8_kv_cache,
                random_state = random_state,
                max_lora_rank = max_lora_rank,
                disable_log_stats = disable_log_stats,
                qat_scheme = qat_scheme,
                load_in_fp8 = load_in_fp8,
                unsloth_tiled_mlp = unsloth_tiled_mlp,
                text_only = text_only,
                *args,
                **kwargs,
            )

        if isinstance(dtype, str) and dtype in ["float16", "bfloat16"]:
            dtype = getattr(torch, dtype)
        assert (
            dtype is None
            or dtype == torch.float16
            or dtype == torch.bfloat16
            or dtype == torch.float32
        )

        if fast_inference:
            if importlib.util.find_spec("vllm") is None:
                raise ImportError(
                    "Unsloth: Please install vLLM before enabling `fast_inference`!\n"
                    "You can do this in a terminal via `pip install vllm`"
                )
            if DEVICE_TYPE_TORCH == "cuda":
                for i in range(DEVICE_COUNT):
                    # [TODO] DGX Spark vLLM breaks
                    if "NVIDIA GB10" in str(torch.cuda.get_device_name(i)).upper():
                        print(
                            "Unsloth: DGX Spark detected - `fast_inference=True` is currently broken as of January 2026.\n"
                            "Defaulting to native Unsloth inference."
                        )
                        fast_inference = False
                        break

        # bitsandbytes unusable (absent, or unstable as on some AMD stacks). This is
        # a capability check, so it is not gated on use_exact_model_name: that only
        # suppresses repo-name remapping and cannot make bitsandbytes available.
        if not ALLOW_BITSANDBYTES:
            # A user-supplied config sets load_in_4bit/8bit above and is forwarded
            # in kwargs, so clearing the flags alone still rebuilds the bnb
            # quantizer downstream. Only drop it when it asks for bnb: a GPTQ /
            # AWQ / fp8 / torchao config must pass through untouched.
            _quant_cfg = kwargs.get("quantization_config", None)
            if isinstance(_quant_cfg, dict):
                _wants_bnb = bool(
                    _quant_cfg.get("load_in_4bit", False) or _quant_cfg.get("load_in_8bit", False)
                )
            elif _quant_cfg is not None:
                _wants_bnb = bool(
                    getattr(_quant_cfg, "load_in_4bit", False)
                    or getattr(_quant_cfg, "load_in_8bit", False)
                )
            else:
                _wants_bnb = False
            if (
                load_in_4bit
                or load_in_8bit
                or _wants_bnb
                or model_name.lower().endswith("-bnb-4bit")
            ):
                print(
                    "Unsloth: `bitsandbytes` is unavailable here - disabling 4bit/8bit. "
                    "16bit LoRA and full finetuning still work."
                )
            # 8bit is bitsandbytes too: leaving either set sends the request on to
            # Transformers, which builds the bnb quantizer and fails there.
            load_in_4bit = False
            load_in_8bit = False
            if _wants_bnb:
                kwargs.pop("quantization_config", None)

        # Find FP8, BnB 4bit, other mapped names
        old_model_name = model_name
        fp8_mode = None
        if not use_exact_model_name:
            new_model_name = get_model_name(
                model_name,
                load_in_4bit = load_in_4bit,
                load_in_fp8 = load_in_fp8,
                token = token,
                trust_remote_code = trust_remote_code,
            )
            if new_model_name is None and load_in_fp8 != False:
                fp8_mode = _get_fp8_mode_and_check_settings(
                    load_in_fp8,
                    fast_inference,
                    full_finetuning,
                    load_in_4bit,
                    load_in_8bit,
                    load_in_16bit,
                )
                model_name = _offline_quantize_to_fp8(model_name, fp8_mode, text_only = text_only)
            else:
                assert new_model_name is not None
                model_name = new_model_name
                # If mapper resolved to a pre-quantized FP8 model, disable
                # on-the-fly quantization to avoid double quantization
                if load_in_fp8 != False and new_model_name != old_model_name:
                    load_in_fp8 = False

        # Check if pre-quantized models are allowed
        # AMD Instinct GPUs need blocksize = 128 on bitsandbytes < 0.49.2 (our pre-quants use blocksize = 64)
        if not ALLOW_PREQUANTIZED_MODELS and model_name.lower().endswith(
            ("-unsloth-bnb-4bit", "-bnb-4bit")
        ):
            model_name = _strip_unsloth_bnb_4bit_suffix(model_name)
        # '-bf16' hub repos load bf16; a local dir keeps the requested quant unless 16bit is set
        if model_name.lower().endswith("-bf16") and (
            load_in_16bit or not os.path.isdir(os.path.expanduser(model_name))
        ):
            load_in_4bit = False
            load_in_8bit = False
            load_in_fp8 = False
            load_in_16bit = True

        if USE_MODELSCOPE and not os.path.exists(model_name):
            from modelscope import snapshot_download
            model_name = snapshot_download(model_name)

        # First check if it's a normal model via AutoConfig
        from huggingface_hub.utils import (
            disable_progress_bars,
            enable_progress_bars,
            are_progress_bars_disabled,
        )

        was_disabled = are_progress_bars_disabled()
        disable_progress_bars()

        autoconfig_error = None
        peft_error = None
        autoconfig_exc = None
        peft_exc = None
        model_config = None
        peft_config = None
        local_files_only = kwargs.get("local_files_only", False)

        try:
            model_config = AutoConfig.from_pretrained(
                model_name,
                token = token,
                revision = revision,
                trust_remote_code = trust_remote_code,
                local_files_only = local_files_only,
            )
            is_model = True
        except ImportError:
            raise
        except Exception as error:
            autoconfig_error = str(error)
            autoconfig_exc = error
            if "architecture" in autoconfig_error:
                if "qwen3_5" in autoconfig_error:
                    raise ImportError(
                        f"Unsloth: Your transformers version of {transformers_version} does not support Qwen3.5.\n"
                        f"The minimum required version is 5.2.0.\n"
                        f'Try `pip install --upgrade "transformers>=5.2.0"`\n'
                        f"to obtain the latest transformers build, then restart this session."
                    )
                raise ValueError(
                    f"`{model_name}` is not supported yet in `transformers=={transformers_version}`.\n"
                    f"Please update transformers via `pip install --upgrade transformers` and try again."
                )
            is_model = False
        try:
            peft_config = PeftConfig.from_pretrained(
                model_name,
                token = token,
                revision = revision,
                trust_remote_code = trust_remote_code,
                local_files_only = local_files_only,
            )
            is_peft = True
        except ImportError:
            raise
        except Exception as error:
            peft_error = str(error)
            peft_exc = error
            if "architecture" in peft_error:
                raise ValueError(
                    f"`{model_name}` is not supported yet in `transformers=={transformers_version}`.\n"
                    f"Please update transformers via `pip install --upgrade transformers` and try again."
                )
            is_peft = False

        # Old transformers versions check
        both_exist = (is_model and is_peft) and not SUPPORTS_LLAMA32

        # Error out if both LoRA and normal model config exists.
        if both_exist:
            raise RuntimeError(
                "Unsloth: Your repo has a LoRA adapter and a base model.\n"
                "You have 2 files `config.json` and `adapter_config.json`.\n"
                "We must only allow one config file.\n"
                "Please separate the LoRA and base models to 2 repos."
            )
        if not is_model and not is_peft:
            error = autoconfig_error if autoconfig_error is not None else peft_error
            # Old transformers version
            if "rope_scaling" in error.lower() and not SUPPORTS_LLAMA31:
                raise ImportError(
                    f"Unsloth: Your transformers version of {transformers_version} does not support new RoPE scaling methods.\n"
                    f"This includes Llama 3.1. The minimum required version is 4.43.2\n"
                    f'Try `pip install --upgrade "transformers>=4.43.2"`\n'
                    f"to obtain the latest transformers build, then restart this session."
                )
            # Create a combined error message showing both failures
            combined_error = (
                "Unsloth: Failed to load model. Both AutoConfig and PeftConfig loading failed.\n\n"
                f"AutoConfig error: {autoconfig_error}\n\n"
                f"PeftConfig error: {peft_error}\n\n"
            )
            # Chain an offline-related cause if either probe had one, so @_offline_aware_load
            # still retries from cache (e.g. adapter repo: permanent AutoConfig 404 + transient PeftConfig).
            _cause = next(
                (
                    e
                    for e in (autoconfig_exc, peft_exc)
                    if e is not None and _is_offline_related_error(e)
                ),
                autoconfig_exc or peft_exc,
            )
            raise RuntimeError(combined_error) from _cause

        model_types = get_transformers_model_type(
            peft_config if peft_config is not None else model_config,
            trust_remote_code = trust_remote_code,
        )
        if len(model_types) == 1:
            model_type = model_types[0]
        else:
            # Leave as tuple if more than one arch
            model_type = model_types

        # New transformers need to check manually.
        if SUPPORTS_LLAMA32 and is_model and is_peft:
            # Check if folder exists locally
            if os.path.isdir(model_name):
                exist_adapter_config = os.path.exists(
                    os.path.join(model_name, "adapter_config.json")
                )
                exist_config = os.path.exists(os.path.join(model_name, "config.json"))
                both_exist = exist_adapter_config and exist_config
            else:
                # Both AutoConfig and PeftConfig loaded successfully from this
                # remote repo, so both config.json and adapter_config.json
                # definitely exist -- no need for an extra HfFileSystem network call.
                both_exist = True

        # Get base model for PEFT:
        if is_peft:
            # Check base model again for PEFT
            model_name = peft_config.base_model_name_or_path
            if not use_exact_model_name:
                model_name = get_model_name(
                    model_name,
                    load_in_4bit = load_in_4bit,
                    load_in_fp8 = load_in_fp8,
                    token = token,
                    trust_remote_code = trust_remote_code,
                )
            # Check if pre-quantized models are allowed
            # AMD Instinct GPUs need blocksize = 128 on bitsandbytes < 0.49.2 (our pre-quants use blocksize = 64)
            if not ALLOW_PREQUANTIZED_MODELS and model_name.lower().endswith(
                ("-unsloth-bnb-4bit", "-bnb-4bit")
            ):
                model_name = _strip_unsloth_bnb_4bit_suffix(model_name)
            # '-bf16' hub repos load bf16; a local dir keeps the requested quant unless 16bit is set
            if model_name.lower().endswith("-bf16") and (
                load_in_16bit or not os.path.isdir(os.path.expanduser(model_name))
            ):
                load_in_4bit = False
                load_in_8bit = False
                load_in_fp8 = False
                load_in_16bit = True

            model_config = AutoConfig.from_pretrained(
                model_name,
                token = token,
                trust_remote_code = trust_remote_code,
                local_files_only = local_files_only,
            )

        if not was_disabled:
            enable_progress_bars()

        if model_type == "llama":
            scaling_type = None
            if getattr(model_config, "rope_scaling", None) is not None:
                scaling_type1 = model_config.rope_scaling.get("type", None)
                scaling_type2 = model_config.rope_scaling.get("rope_type", None)
                scaling_type = scaling_type1 if scaling_type1 is not None else scaling_type2

            if scaling_type == "llama3" and not SUPPORTS_LLAMA31:
                raise ImportError(
                    f"Unsloth: Your transformers version of {transformers_version} does not support Llama 3.1.\n"
                    f"The minimum required version is 4.43.2\n"
                    f'Try `pip install --upgrade "transformers>=4.43.2"`\n'
                    f"to obtain the latest transformers build, then restart this session."
                )

            dispatch_model = FastLlamaModel

        elif model_type == "mistral":
            dispatch_model = FastMistralModel
        elif model_type == "gemma":
            if not SUPPORTS_GEMMA:
                raise ImportError(
                    f"Unsloth: Your transformers version of {transformers_version} does not support Gemma.\n"
                    f"The minimum required version is 4.38.\n"
                    f'Try `pip install --upgrade "transformers>=4.38"`\n'
                    f"to obtain the latest transformers build, then restart this session."
                )
            dispatch_model = FastGemmaModel
        elif model_type == "gemma2":
            if not SUPPORTS_GEMMA2:
                raise ImportError(
                    f"Unsloth: Your transformers version of {transformers_version} does not support Gemma2.\n"
                    f"The minimum required version is 4.42.3.\n"
                    f'Try `pip install --upgrade "transformers>=4.42.3"`\n'
                    f"to obtain the latest transformers build, then restart this session."
                )
            # Also check for softcapping support in flash-attn which is faster!
            if is_bfloat16_supported() and not HAS_FLASH_ATTENTION:
                print(
                    "Unsloth: If you want to finetune Gemma 2, install flash-attn to make it faster!\n"
                    "To install flash-attn, do the below:\n"
                    '\npip install --no-deps --upgrade "flash-attn>=2.6.3"'
                )
            elif HAS_FLASH_ATTENTION and not HAS_FLASH_ATTENTION_SOFTCAPPING:
                print(
                    "Unsloth: If you want to finetune Gemma 2, upgrade flash-attn to version 2.6.3 or higher!\n"
                    "Newer versions support faster and less memory usage kernels for Gemma 2's attention softcapping!\n"
                    "To update flash-attn, do the below:\n"
                    '\npip install --no-deps --upgrade "flash-attn>=2.6.3"'
                )

            dispatch_model = FastGemma2Model
        elif model_type == "qwen2":
            dispatch_model = FastQwen2Model
        elif model_type == "qwen3":  # or model_type == "qwen3_moe":
            if not SUPPORTS_QWEN3 or not SUPPORTS_QWEN3_MOE:
                raise ImportError(
                    f"Unsloth: Your transformers version of {transformers_version} does not support Qwen3.\n"
                    f"The minimum required version is 4.50.3.\n"
                    f'Try `pip install --upgrade "transformers>=4.50.3"`\n'
                    f"to obtain the latest transformers build, then restart this session."
                )
            dispatch_model = FastQwen3Model if model_type == "qwen3" else FastQwen3MoeModel
        # elif model_type == "falcon_h1":
        #     dispatch_model = FastFalconH1Model
        #     if not SUPPORTS_FALCON_H1:
        #         raise ImportError(
        #             f"Unsloth: Your transformers version of {transformers_version} does not support FalconH1.\n"\
        #             f"The minimum required version is 4.50.3.\n"\
        #             f'Try `pip install --upgrade "transformers>=4.50.3"`\n'\
        #             f"to obtain the latest transformers build, then restart this session."\
        #         )
        # Temporary disable optimized Cohere until errors match
        # elif model_type == "cohere":
        #     dispatch_model = FastCohereModel
        # Temporary disable optimized Granite until errors match
        # elif model_type == "granite":
        #     dispatch_model = FastGraniteModel
        else:
            return FastModel.from_pretrained(
                model_name = old_model_name,
                max_seq_length = max_seq_length,
                dtype = dtype,
                load_in_4bit = load_in_4bit,
                load_in_8bit = load_in_8bit,
                load_in_16bit = load_in_16bit,
                full_finetuning = full_finetuning,
                token = token,
                device_map = device_map,
                rope_scaling = rope_scaling,  # [TODO] No effect
                fix_tokenizer = fix_tokenizer,  # [TODO] No effect
                trust_remote_code = trust_remote_code,
                use_gradient_checkpointing = use_gradient_checkpointing,
                resize_model_vocab = resize_model_vocab,  # [TODO] No effect
                revision = revision,
                return_logits = False,  # Return logits
                fullgraph = True,  # No graph breaks
                use_exact_model_name = use_exact_model_name,
                offload_embedding = offload_embedding,
                float32_mixed_precision = float32_mixed_precision,
                # Pass vLLM/inference parameters
                fast_inference = fast_inference,
                gpu_memory_utilization = gpu_memory_utilization,
                float8_kv_cache = float8_kv_cache,
                random_state = random_state,
                max_lora_rank = max_lora_rank,
                disable_log_stats = disable_log_stats,
                qat_scheme = qat_scheme,
                load_in_fp8 = load_in_fp8,
                unsloth_tiled_mlp = unsloth_tiled_mlp,
                text_only = text_only,
                *args,
                **kwargs,
            )

        # Apply gradient checkpointing with smart heuristics
        use_gradient_checkpointing = apply_unsloth_gradient_checkpointing(
            use_gradient_checkpointing, max_seq_length, dtype
        )

        # Keep the local checkpoint dir as tokenizer when self-sufficient (see _resolve_checkpoint_tokenizer_name).
        tokenizer_name = _resolve_checkpoint_tokenizer_name(old_model_name, kwargs)

        if fast_inference:
            fast_inference, model_name = fast_inference_setup(model_name, model_config)

        load_in_4bit_kwargs = load_in_4bit
        load_in_8bit_kwargs = load_in_8bit
        if quantization_config is not None and not fast_inference:
            load_in_4bit_kwargs = False
            load_in_8bit_kwargs = False

        # Mirror FastModel: bitsandbytes < 0.46.0 needs dynamo disabled.
        # Best effort: never crash the load (old unsloth_zoo without the
        # zoo #710 fix raises NameError here on Python 3.13).
        try:
            patch_compiling_bitsandbytes()
        except Exception as e:
            print(f"Unsloth: Could not patch bitsandbytes for torch.compile - {e}")

        model, tokenizer = dispatch_model.from_pretrained(
            model_name = model_name,
            max_seq_length = max_seq_length,
            dtype = _get_dtype(dtype),
            load_in_4bit = load_in_4bit_kwargs,
            token = token,
            device_map = device_map,
            rope_scaling = rope_scaling,
            fix_tokenizer = fix_tokenizer,
            model_patcher = dispatch_model,
            tokenizer_name = tokenizer_name,
            trust_remote_code = trust_remote_code,
            revision = revision if not is_peft else None,
            fast_inference = fast_inference,
            gpu_memory_utilization = gpu_memory_utilization,
            float8_kv_cache = float8_kv_cache,
            random_state = random_state,
            max_lora_rank = max_lora_rank,
            disable_log_stats = disable_log_stats,
            load_in_fp8 = load_in_fp8,
            *args,
            **kwargs,
        )

        if resize_model_vocab is not None:
            model.resize_token_embeddings(resize_model_vocab)

        # In case the model supports tagging, add the unsloth tag.
        if hasattr(model, "add_model_tags"):
            model.add_model_tags(
                [
                    "unsloth",
                ]
            )
        if hasattr(tokenizer, "add_model_tags"):
            tokenizer.add_model_tags(
                [
                    "unsloth",
                ]
            )

        if load_in_4bit:
            # Fix up bitsandbytes config, but respect user-provided quantization_config
            if quantization_config is None:
                # `load_in_4bit` is the requested flag, not the effective one: a non-bnb
                # checkpoint (MXFP4/gptq/awq) had bnb disabled by check_and_disable, so stamping a
                # synthetic bnb config would corrupt its real one. Only stamp bnb/unquantized.
                try:
                    from unsloth_zoo.utils import get_quant_type
                    _stamp_bnb = get_quant_type(model.config) in (None, "bitsandbytes")
                except Exception:
                    _stamp_bnb = True
                if _stamp_bnb:
                    compute_dtype = dtype_from_config(model.config)
                    quantization_config = {
                        # Sometimes compute_dtype is not a string!!
                        "bnb_4bit_compute_dtype": compute_dtype,
                        "bnb_4bit_quant_type": "nf4",
                        "bnb_4bit_use_double_quant": True,
                        "llm_int8_enable_fp32_cpu_offload": False,
                        "llm_int8_has_fp16_weight": False,
                        "llm_int8_skip_modules": None,
                        "llm_int8_threshold": 6.0,
                        "load_in_4bit": True,
                        "load_in_8bit": False,
                        "quant_method": "bitsandbytes",
                    }
                    model.config.update({"quantization_config": quantization_config})
            else:
                if hasattr(quantization_config, "to_dict"):
                    model.config.update({"quantization_config": quantization_config.to_dict()})
                elif isinstance(quantization_config, dict):
                    model.config.update({"quantization_config": quantization_config})

        if load_in_fp8 != False:
            _tag_model_with_fp8_torchao_config(model, fp8_mode)

        if is_peft:
            # From https://github.com/huggingface/peft/issues/184
            # Now add PEFT adapters
            # Warm the adapter repo: PeftModel downloads it in-process and can hang on Xet.
            _prefetched = maybe_prefetch_hf_snapshot(
                old_model_name,
                token = token,
                revision = revision,
                cache_dir = kwargs.get("cache_dir"),
                local_files_only = local_files_only,
                # Adapter always loads in-process via PeftModel, so warm it even under fast_inference.
                fast_inference = False,
                force_download = kwargs.get("force_download", False),
                # Leave use_safetensors auto (inheriting base format could skip a safetensors-only
                # adapter). adapter_only restricts the warm to the adapter files + root aux.
                adapter_only = True,
            )
            # Child did the forced download; clear the flag so the load reuses the warm cache.
            if _prefetched and kwargs.get("force_download", False):
                kwargs["force_download"] = False
            # Forward cache_dir so the load reads the warmed adapter. No subfolder (that targets the
            # base checkpoint; adapters live at the root).
            peft_load_kwargs = {}
            if kwargs.get("cache_dir") is not None:
                peft_load_kwargs["cache_dir"] = kwargs["cache_dir"]
            model = PeftModel.from_pretrained(
                model,
                old_model_name,
                token = token,
                revision = revision,
                local_files_only = local_files_only,
                is_trainable = True,
                trust_remote_code = trust_remote_code,
                **peft_load_kwargs,
            )
            # Patch it as well!
            model = dispatch_model.patch_peft_model(model, use_gradient_checkpointing)
            # Re-evaluate grouped MoE now the adapter is attached: an expert-LoRA block falls back
            # to the original loop, an attention-only adapter keeps the grouped path. Guarded.
            try:
                from unsloth_zoo.temporary_patches.moe_grouped_modulelist import (
                    auto_enable_grouped_moe,
                )
                auto_enable_grouped_moe(model)
            except Exception:
                pass  # optional speedup; never block model loading

        # Patch Tiled MLP
        # to turn on set UNSLOTH_TILED_MLP to "arctic", "target", or "target:{GB}""
        patch_tiled_mlp_choice = os.environ.get(
            "UNSLOTH_TILED_MLP", "arctic" if unsloth_tiled_mlp else "0"
        )
        if patch_tiled_mlp_choice != "0" or unsloth_tiled_mlp:
            patch_tiled_mlp(model, patch_options_str = patch_tiled_mlp_choice)

        model = _fix_rope_inv_freq(model)
        model = _exclude_rope_inv_freq_from_ddp(model)
        return model, tokenizer


from ..kernels import (
    patch_loss_functions,
    post_patch_loss_function,
)
from .vision import FastBaseModel
from .diffusion import FastDiffusionModel, is_diffusion_model_type
from transformers import (
    AutoModelForCausalLM,
)

try:
    from transformers import AutoModelForImageTextToText
    AutoModelForVision2Seq = AutoModelForImageTextToText
except:
    from transformers import AutoModelForVision2Seq


class FastModel(FastBaseModel):
    @staticmethod
    def _prepare_for_qat(model, qat_scheme):
        model = _prepare_model_for_qat(model, qat_scheme)
        return model

    @staticmethod
    def get_peft_model(model, *args, **kwargs):
        # Route text-diffusion models (slow path) to the transformers-only PEFT helper.
        if getattr(model, "_unsloth_slow_diffusion", False):
            return FastDiffusionModel.get_peft_model(model, *args, **kwargs)
        return FastBaseModel.get_peft_model(model, *args, **kwargs)

    @staticmethod
    def for_inference(model):
        if getattr(model, "_unsloth_slow_diffusion", False):
            return FastDiffusionModel.for_inference(model)
        return FastBaseModel.for_inference(model)

    @staticmethod
    def for_training(model, use_gradient_checkpointing = True):
        if getattr(model, "_unsloth_slow_diffusion", False):
            return FastDiffusionModel.for_training(model, use_gradient_checkpointing)
        return FastBaseModel.for_training(model, use_gradient_checkpointing)

    @staticmethod
    @_offline_aware_load
    def from_pretrained(
        model_name = "unsloth/Llama-3.2-11B-Vision-Instruct-bnb-4bit",
        max_seq_length = 2048,
        dtype = None,
        load_in_4bit = True,  # 4bit QLoRA
        load_in_8bit = False,  # 8bit  LoRA
        load_in_16bit = False,  # 16bit LoRA
        full_finetuning = False,
        token = None,
        device_map = "sequential",
        rope_scaling = None,  # [TODO] No effect
        fix_tokenizer = True,  # [TODO] No effect
        trust_remote_code = False,
        use_gradient_checkpointing = "unsloth",
        resize_model_vocab = None,  # [TODO] No effect
        revision = None,
        return_logits = False,  # Return logits
        fullgraph = True,  # No graph breaks
        use_exact_model_name = False,
        auto_model = None,
        whisper_language = None,
        whisper_task = None,
        unsloth_force_compile = False,
        offload_embedding = False,
        float32_mixed_precision = None,  # Forces float32 mixed precision
        # Add the missing vLLM/inference parameters
        fast_inference = False,  # uses vLLM
        gpu_memory_utilization = 0.5,
        float8_kv_cache = False,
        random_state = 3407,
        max_lora_rank = 64,
        disable_log_stats = True,
        qat_scheme = None,
        load_in_fp8 = False,  # fp8 LoRA (True, False, 'block')
        unsloth_tiled_mlp = False,
        target_parameters = None,  # For MoE expert parameters
        text_only = False,  # Skip vision/audio towers and load only the text decoder
        *args,
        **kwargs,
    ):
        user_config = kwargs.pop("config", None)
        # Respect user-provided quantization_config (e.g. BitsAndBytesConfig)
        quantization_config = kwargs.get("quantization_config", None)
        if quantization_config is not None:
            if isinstance(quantization_config, dict):
                q_load_in_4bit = quantization_config.get("load_in_4bit", False)
                q_load_in_8bit = quantization_config.get("load_in_8bit", False)
            else:
                q_load_in_4bit = getattr(quantization_config, "load_in_4bit", False)
                q_load_in_8bit = getattr(quantization_config, "load_in_8bit", False)
            if q_load_in_4bit:
                load_in_4bit = True
                load_in_8bit = False
            if q_load_in_8bit:
                load_in_8bit = True
                load_in_4bit = False

        # Login to allow private models
        token = hf_login(token)
        if whisper_language is not None:
            assert type(whisper_language) is str
        if whisper_task is not None:
            assert type(whisper_task) is str
        # Align dtype with bnb_4bit_compute_dtype if provided and dtype is unset.
        if dtype is None and quantization_config is not None:
            bnb_compute_dtype = None
            if isinstance(quantization_config, dict):
                if quantization_config.get("load_in_4bit", False):
                    bnb_compute_dtype = quantization_config.get("bnb_4bit_compute_dtype", None)
            else:
                if getattr(quantization_config, "load_in_4bit", False):
                    bnb_compute_dtype = getattr(quantization_config, "bnb_4bit_compute_dtype", None)
            if isinstance(bnb_compute_dtype, str):
                bnb_compute_dtype = getattr(torch, bnb_compute_dtype, None)
            if isinstance(bnb_compute_dtype, torch.dtype):
                dtype = bnb_compute_dtype
        SUPPORTS_BFLOAT16 = is_bfloat16_supported()
        if dtype is None:
            dtype = torch.float16 if not SUPPORTS_BFLOAT16 else torch.bfloat16
        elif dtype == torch.bfloat16 and not SUPPORTS_BFLOAT16:
            logger.warning_once("Device does not support bfloat16. Will change to float16.")
            dtype = torch.float16
        assert dtype in (torch.float16, torch.bfloat16, torch.float32)
        assert load_in_fp8 in (True, False, "block")

        patch_compiled_autograd()
        # Same best-effort wrapper as the FastLanguageModel path: unsloth_zoo's
        # patch imports bitsandbytes unconditionally, so on a host without it this
        # raised before the capability fallback below could take the 16bit path.
        try:
            patch_compiling_bitsandbytes()
        except Exception as e:
            print(f"Unsloth: Could not patch bitsandbytes for torch.compile - {e}")

        if full_finetuning and (load_in_4bit or load_in_8bit):
            print(
                "Unsloth: You selected full finetuning support, but 4bit / 8bit is enabled - disabling LoRA / QLoRA."
            )
            load_in_4bit = False
            load_in_8bit = False
            load_in_fp8 = False
            load_in_16bit = False

        # bitsandbytes unusable (absent, or unstable as on some AMD stacks). This is
        # a capability check, so it is not gated on use_exact_model_name: that only
        # suppresses repo-name remapping and cannot make bitsandbytes available.
        if not ALLOW_BITSANDBYTES:
            # A user-supplied config sets load_in_4bit/8bit above and is forwarded
            # in kwargs, so clearing the flags alone still rebuilds the bnb
            # quantizer downstream. Only drop it when it asks for bnb: a GPTQ /
            # AWQ / fp8 / torchao config must pass through untouched.
            _quant_cfg = kwargs.get("quantization_config", None)
            if isinstance(_quant_cfg, dict):
                _wants_bnb = bool(
                    _quant_cfg.get("load_in_4bit", False) or _quant_cfg.get("load_in_8bit", False)
                )
            elif _quant_cfg is not None:
                _wants_bnb = bool(
                    getattr(_quant_cfg, "load_in_4bit", False)
                    or getattr(_quant_cfg, "load_in_8bit", False)
                )
            else:
                _wants_bnb = False
            if (
                load_in_4bit
                or load_in_8bit
                or _wants_bnb
                or model_name.lower().endswith("-bnb-4bit")
            ):
                print(
                    "Unsloth: `bitsandbytes` is unavailable here - disabling 4bit/8bit. "
                    "16bit LoRA and full finetuning still work."
                )
            # 8bit is bitsandbytes too: leaving either set sends the request on to
            # Transformers, which builds the bnb quantizer and fails there.
            load_in_4bit = False
            load_in_8bit = False
            if _wants_bnb:
                kwargs.pop("quantization_config", None)

        if (
            int(load_in_4bit) + int(load_in_8bit) + int(load_in_16bit) + int(load_in_fp8 != False)
            >= 2
        ):
            raise RuntimeError(
                "Unsloth: Can only load in 4bit or 8bit or 16bit, not a combination!\n"
                "Also, we by default set `load_in_4bit = True`.\n"
                "If you want 8bit finetuning, set both `load_in_4bit = False` and `load_in_8bit = True`\n"
                "If you want 16bit LoRA finetuning, set `load_in_16bit = True`"
            )

        if qat_scheme is not None and not full_finetuning:
            raise ValueError(
                "Specifying `qat_scheme` in `FastLanguageModel.from_pretrained(...)` is only "
                "compatible with `full_finetuning=True`. If you wish to use QAT with LoRA, "
                "please pass in `qat_scheme` in `FastLanguageModel.get_peft_model(...)` instead."
            )
        if qat_scheme == "phone-deployment":
            qat_scheme = "int8-int4"

        # Distributed-safe device placement for quantized models.
        # In multi-GPU (torchrun), each rank must load the model on its own device
        # to avoid Accelerate device relocation errors with quantized weights.
        is_quantized = load_in_4bit or load_in_8bit or load_in_fp8
        if is_quantized and isinstance(device_map, str):
            distributed_device_map, is_dist = prepare_device_map()
            if is_dist:
                device_map = distributed_device_map

        if fast_inference:
            if importlib.util.find_spec("vllm") is None:
                raise ImportError(
                    "Unsloth: Please install vLLM before enabling `fast_inference`!\n"
                    "You can do this in a terminal via `pip install vllm`"
                )
            if DEVICE_TYPE_TORCH == "cuda":
                for i in range(DEVICE_COUNT):
                    # [TODO] DGX Spark vLLM breaks
                    if "NVIDIA GB10" in str(torch.cuda.get_device_name(i)).upper():
                        print(
                            "Unsloth: DGX Spark detected - `fast_inference=True` is currently broken as of January 2026.\n"
                            "Defaulting to native Unsloth inference."
                        )
                        fast_inference = False
                        break

        # Find FP8, BnB 4bit, other mapped names
        old_model_name = model_name
        fp8_mode = None
        if not use_exact_model_name:
            new_model_name = get_model_name(
                model_name, load_in_4bit = load_in_4bit, load_in_fp8 = load_in_fp8
            )
            if new_model_name is None and load_in_fp8 != False:
                fp8_mode = _get_fp8_mode_and_check_settings(
                    load_in_fp8,
                    fast_inference,
                    full_finetuning,
                    load_in_4bit,
                    load_in_8bit,
                    load_in_16bit,
                )
                model_name = _offline_quantize_to_fp8(model_name, fp8_mode, text_only = text_only)
            else:
                assert new_model_name is not None
                model_name = new_model_name
                # If mapper resolved to a pre-quantized FP8 model, disable
                # on-the-fly quantization to avoid double quantization
                if load_in_fp8 != False and new_model_name != old_model_name:
                    load_in_fp8 = False

        # Check if pre-quantized models are allowed
        # AMD Instinct GPUs need blocksize = 128 on bitsandbytes < 0.49.2 (our pre-quants use blocksize = 64)
        if not ALLOW_PREQUANTIZED_MODELS and model_name.lower().endswith(
            ("-unsloth-bnb-4bit", "-bnb-4bit")
        ):
            model_name = _strip_unsloth_bnb_4bit_suffix(model_name)
        # '-bf16' hub repos load bf16; a local dir keeps the requested quant unless 16bit is set
        if model_name.lower().endswith("-bf16") and (
            load_in_16bit or not os.path.isdir(os.path.expanduser(model_name))
        ):
            load_in_4bit = False
            load_in_8bit = False
            load_in_fp8 = False
            load_in_16bit = True

        # Check modelscope
        if USE_MODELSCOPE and not os.path.exists(model_name):
            from modelscope import snapshot_download
            model_name = snapshot_download(model_name)

        # First check if it's a normal model via AutoConfig
        from huggingface_hub.utils import (
            disable_progress_bars,
            enable_progress_bars,
            are_progress_bars_disabled,
        )

        was_disabled = are_progress_bars_disabled()
        disable_progress_bars()

        autoconfig_error = None
        peft_error = None
        autoconfig_exc = None
        peft_exc = None
        model_config = None
        peft_config = None
        # @_offline_aware_load already forced offline when needed; nested calls inherit it.
        local_files_only = kwargs.get("local_files_only", False)

        # Text-diffusion slow-path dispatch, factored so both the normal route (below) and the
        # legacy-config fallback (in the AutoConfig except handler) share one call site.
        def _dispatch_diffusion():
            return FastDiffusionModel.from_pretrained(
                model_name = model_name,
                max_seq_length = max_seq_length,
                dtype = dtype,
                load_in_4bit = load_in_4bit,
                load_in_8bit = load_in_8bit,
                load_in_16bit = load_in_16bit,
                full_finetuning = full_finetuning,
                token = token,
                device_map = device_map,
                trust_remote_code = trust_remote_code,
                revision = revision,
                **kwargs,
            )

        try:
            model_config = user_config
            if model_config is None:
                model_config = AutoConfig.from_pretrained(
                    model_name,
                    token = token,
                    revision = revision,
                    trust_remote_code = trust_remote_code,
                    local_files_only = local_files_only,
                )
            is_model = True
        except ImportError:
            raise
        except Exception as error:
            autoconfig_error = str(error)
            autoconfig_exc = error
            # Legacy text-diffusion configs use model_type "diffusion_gemma", which current
            # transformers does not register by name (it ships "diffusion_gemma4"). AutoConfig
            # raises before we can dispatch; route straight to the diffusion slow path, whose
            # loader aliases the legacy type to the gemma4 classes.
            if "diffusion_gemma" in autoconfig_error and is_diffusion_model_type("diffusion_gemma"):
                return _dispatch_diffusion()
            if "architecture" in autoconfig_error:
                if "qwen3_5" in autoconfig_error:
                    raise ImportError(
                        f"Unsloth: Your transformers version of {transformers_version} does not support Qwen3.5.\n"
                        f"The minimum required version is 5.2.0.\n"
                        f'Try `pip install --upgrade "transformers>=5.2.0"`\n'
                        f"to obtain the latest transformers build, then restart this session."
                    )
                raise ValueError(
                    f"`{model_name}` is not supported yet in `transformers=={transformers_version}`.\n"
                    f"Please update transformers via `pip install --upgrade transformers` and try again."
                )
            is_model = False
        try:
            peft_config = PeftConfig.from_pretrained(
                model_name,
                token = token,
                revision = revision,
                trust_remote_code = trust_remote_code,
                local_files_only = local_files_only,
            )
            is_peft = True
        except ImportError:
            raise
        except Exception as error:
            peft_error = str(error)
            peft_exc = error
            if "architecture" in peft_error:
                raise ValueError(
                    f"`{model_name}` is not supported yet in `transformers=={transformers_version}`.\n"
                    f"Please update transformers via `pip install --upgrade transformers` and try again."
                )
            is_peft = False
        # Old transformers versions check
        both_exist = (is_model and is_peft) and not SUPPORTS_LLAMA32
        # Error out if both LoRA and normal model config exists.
        if both_exist:
            raise RuntimeError(
                "Unsloth: Your repo has a LoRA adapter and a base model.\n"
                "You have 2 files `config.json` and `adapter_config.json`.\n"
                "We must only allow one config file.\n"
                "Please separate the LoRA and base models to 2 repos."
            )
        if not is_model and not is_peft:
            error = autoconfig_error if autoconfig_error is not None else peft_error
            # Old transformers version
            if "rope_scaling" in error.lower() and not SUPPORTS_LLAMA31:
                raise ImportError(
                    f"Unsloth: Your transformers version of {transformers_version} does not support new RoPE scaling methods.\n"
                    f"This includes Llama 3.1. The minimum required version is 4.43.2\n"
                    f'Try `pip install --upgrade "transformers>=4.43.2"`\n'
                    f"to obtain the latest transformers build, then restart this session."
                )
            # Create a combined error message showing both failures
            combined_error = (
                "Unsloth: Failed to load model. Both AutoConfig and PeftConfig loading failed.\n\n"
                f"AutoConfig error: {autoconfig_error}\n\n"
                f"PeftConfig error: {peft_error}\n\n"
            )
            # Chain an offline-related cause if either probe had one, so @_offline_aware_load
            # still retries from cache (e.g. adapter repo: permanent AutoConfig 404 + transient PeftConfig).
            _cause = next(
                (
                    e
                    for e in (autoconfig_exc, peft_exc)
                    if e is not None and _is_offline_related_error(e)
                ),
                autoconfig_exc or peft_exc,
            )
            raise RuntimeError(combined_error) from _cause

        model_types = get_transformers_model_type(
            peft_config if peft_config is not None else model_config,
            trust_remote_code = trust_remote_code,
        )
        model_types_all = ",".join(model_types) + ","
        _maybe_advise_fla_install(model_types)

        # ---- Text-diffusion models (e.g. DiffusionGemma) take a transformers-only slow path. ----
        # These use a custom block-diffusion `generate` and a novel backbone, so we skip Unsloth's
        # autoregressive kernel/compile patching and load the unmodified HF model (bit-identical to
        # naive transformers), keeping only 4bit/8bit + PEFT LoRA conveniences.
        if is_diffusion_model_type(model_types):
            return _dispatch_diffusion()

        # Save model types and loading method
        lowered_model_name = model_name.lower()
        # Build UNSLOTH_MODEL_NAME fresh from THIS load's model types + flags; do not prepend the
        # inherited os.environ value (a stale "_load_in_4bit_" from an earlier load, e.g. across a
        # save->reload subprocess, would push gpt-oss onto the BnB router patch when later loading
        # a 16bit checkpoint -> "weights not initialized"). Only the type tokens and the load flags
        # below are consumed downstream; the raw model name/path is excluded so a path containing a
        # flag sentinel cannot be misread.
        #
        # Encode the EFFECTIVE bnb state: a non-bnb checkpoint (MXFP4/gptq/awq) has load_in_4bit
        # disabled later by check_and_disable, so recording the requested flag here would route a
        # native MXFP4 gpt-oss onto the BnB router patch. This is only an EARLY best-effort (an
        # adapter-only PEFT repo has model_config=None here, and the base may be remapped); the
        # authoritative correction is sync_unsloth_model_name_bnb_flags(...) after check_and_disable.
        try:
            from unsloth_zoo.utils import get_quant_type
            _bnb_compatible_quant = get_quant_type(model_config) in (None, "bitsandbytes")
        except Exception:
            _bnb_compatible_quant = True
        string = model_types_all
        if load_in_4bit and _bnb_compatible_quant:
            string += "_load_in_4bit_"
        if load_in_8bit and _bnb_compatible_quant:
            string += "_load_in_8bit_"
        if load_in_16bit:
            string += "_load_in_16bit_"
        if load_in_fp8:
            string += "load_in_fp8"
        os.environ["UNSLOTH_MODEL_NAME"] = string

        # Check versions
        LATEST = "\nPlease use transformers via `pip install --no-deps git+https://github.com/huggingface/transformers.git`"
        NIGHTLY = (
            '\nPlease use nightly transformers via pip install --upgrade "transformers>=4.49.0"`'
        )
        # Pixtral
        if "pixtral" in model_types_all and transformers_version < Version("4.49.0"):
            raise RuntimeError("Unsloth: Pixtral only works on transformers >= 4.49.0." + LATEST)
        # Qwen 2.5
        elif "qwen2_5" in model_types_all and transformers_version < Version("4.49.0"):
            raise RuntimeError("Unsloth: Qwen 2.5 only works on transformers >= 4.49.0." + LATEST)
        # Gemma 4 must be before Gemma 3N and Gemma 3
        elif "gemma4" in model_types_all:
            if not SUPPORTS_GEMMA4:
                raise RuntimeError("Unsloth: Gemma 4 requires transformers >= 5.5.0" + LATEST)
            os.environ["UNSLOTH_DISABLE_STATIC_GENERATION"] = "1"
            os.environ["UNSLOTH_HIGH_PRECISION_LAYERNORM"] = "1"
        # Gemma 3N must be before Gemma 3
        elif "gemma3n" in model_types_all:
            if transformers_version < Version("4.53.0"):
                raise RuntimeError(
                    "Unsloth: Gemma 3N only works on transformers >= 4.53.0" + LATEST
                )
            os.environ["UNSLOTH_DISABLE_STATIC_GENERATION"] = "1"
            os.environ["UNSLOTH_FORCE_CUSTOM_DTYPE"] = (
                "float16;torch.float16;torch.float16;"
                "if name.endswith('norm'): "
                "module._pre_set_compute_dtype = torch.float32\n"
                ";"
                "from unsloth_zoo.temporary_patches.gemma3n import patch_Gemma3nConv_Embed_forwards; patch_Gemma3nConv_Embed_forwards()"
            )
            # Set norms to float32 since anyways they get upcasted to float32
            # common in both gemma-3 and gemma-3n
            os.environ["UNSLOTH_HIGH_PRECISION_LAYERNORM"] = "1"
        # Gemma 3
        elif "gemma3" in model_types_all:
            if transformers_version < Version("4.50.0.dev0"):
                raise RuntimeError(
                    "Unsloth: Gemma 3 only works on transformers >= 4.50.0." + NIGHTLY
                )
            # Set norms to float32 since anyways they get upcasted to float32
            # common in both gemma-3 and gemma-3n
            os.environ["UNSLOTH_HIGH_PRECISION_LAYERNORM"] = "1"
            # ROCm/HIP: Gemma3 compiled forward produces NaN on RDNA GPUs
            # (gfx1100, gfx1101, gfx1102, gfx1150, gfx1151, etc.).
            # Disable torch.compile for model forward; loss compilation is fine.
            # See https://github.com/unslothai/unsloth/issues/3385
            from unsloth.kernels.utils import is_rdna

            if is_rdna():
                os.environ["UNSLOTH_COMPILE_DISABLE"] = "partial"
        # Cohere
        elif "cohere2" in model_types_all and transformers_version < Version("4.50.0.dev0"):
            raise RuntimeError(
                "Unsloth: Cohere's Command model only works on transformers >= 4.50.0." + NIGHTLY
            )
        # Sesame
        elif "csm" in model_types_all:
            os.environ["UNSLOTH_COMPILE_DISABLE"] = "partial"  # Inference is too slow
            os.environ["UNSLOTH_DISABLE_STATIC_GENERATION"] = "1"  # Sesame fails
            os.environ["UNSLOTH_FORCE_CUSTOM_DTYPE"] = (
                "all;torch.float32;torch.float16;"
                "if name.endswith(('_proj', 'fc1', 'fc2', 'codebook', 'head')): module.to(torch.float16)"
                ";"
            )
        # Granite 4
        elif "granitemoehybrid" in model_types_all:
            # Granite-4 rms norms are stored as 16 bit, but we upcast
            os.environ["UNSLOTH_HIGH_PRECISION_LAYERNORM"] = "1"
            os.environ["UNSLOTH_DISABLE_STATIC_GENERATION"] = "1"
        # OLMo 2
        elif "olmo2" in model_types_all and transformers_version < Version("4.50.0.dev0"):
            raise RuntimeError("Unsloth: OLMo-2 only works on transformers >= 4.50.0." + NIGHTLY)
        # OLMo 3
        elif "olmo3" in model_types_all and transformers_version < Version("4.57.0.dev0"):
            raise RuntimeError("Unsloth: OLMo-3 only works on transformers >= 4.57.0." + LATEST)
        elif "falcon_h1" in model_types_all:
            # Falcon must use float32 Triton ie TRITON_F32_DEFAULT = 'ieee'
            # since Mamba kernels error out on using lower precision
            os.environ["UNSLOTH_FORCE_CUSTOM_DTYPE"] = (
                "float16;torch.float32;torch.float16;"
                "if name.endswith(('q_proj', 'k_proj', 'v_proj', 'o_proj', 'gate_proj', 'up_proj', 'down_proj', 'head')): module.to(torch.float16)"
                ";"
                "os.environ['TRITON_F32_DEFAULT'] = 'ieee'"
            )
        elif "nemotron_h" in model_types_all:
            # NemotronH (hybrid Mamba-2 + Transformer) uses same Mamba kernels as Falcon-H1
            # Mamba kernels need float32 Triton precision
            os.environ["UNSLOTH_FORCE_CUSTOM_DTYPE"] = (
                "float16;torch.float32;torch.float16;"
                "if name.endswith(('q_proj', 'k_proj', 'v_proj', 'o_proj', 'gate_proj', 'up_proj', 'down_proj', 'head')): module.to(torch.float16)"
                ";"
                "os.environ['TRITON_F32_DEFAULT'] = 'ieee'"
            )
        elif "gpt_oss" in model_types_all:
            os.environ["UNSLOTH_DISABLE_STATIC_GENERATION"] = "1"
            # Use the EFFECTIVE bnb state, not the raw flag: a native MXFP4 checkpoint loaded
            # with the default load_in_4bit=True (e.g. openai/gpt-oss-20b by exact name) has
            # bnb disabled later by check_and_disable, so the raw flag would wrongly pick the
            # BnB dtype path. Mirrors the _load_in_4bit_ token gate above.
            if not (load_in_4bit and _bnb_compatible_quant):
                # Only upcast MoE biases for MXFP4, not BnB
                # Set norms to float32 since anyways they get upcasted to float32
                os.environ["UNSLOTH_FORCE_CUSTOM_DTYPE"] = (
                    "all;None;None;"
                    "x = 'gate_up_proj_bias'\n"
                    "if hasattr(module, x): "
                    "setattr(module, x, torch.nn.Parameter(getattr(module, x).to(torch.float32)) if isinstance(getattr(module, x), torch.nn.Parameter) else getattr(module, x).to(torch.float32))\n"
                    ""
                    "x = 'down_proj_bias'\n"
                    "if hasattr(module, x): "
                    "setattr(module, x, torch.nn.Parameter(getattr(module, x).to(torch.float32)) if isinstance(getattr(module, x), torch.nn.Parameter) else getattr(module, x).to(torch.float32))\n"
                    ""
                    ";"
                )
            else:
                # Set down projection compute dtype to be float32 for float16 machines
                # Set norms to float32 since anyways they get upcasted to float32
                os.environ["UNSLOTH_FORCE_CUSTOM_DTYPE"] = (
                    "torch.float16;torch.bfloat16;torch.float16;"
                    "if ('down_projs' in name) and hasattr(module, 'weight') and "
                    "torch.amax(dequantize_module_weight(module)) >= 0:"
                    "module._pre_set_compute_dtype = torch.float32\n"
                    ""
                    "if ('mlp.router' in name) and hasattr(module, 'weight'):"
                    "module._pre_set_compute_dtype = torch.float32\n"
                    ";"
                )
            # Set norms to float32 since anyways they get upcasted to float32
            os.environ["UNSLOTH_HIGH_PRECISION_LAYERNORM"] = "1"
        else:
            for check_model_name in DISABLE_COMPILE_MODEL_NAMES:
                if check_model_name in lowered_model_name:
                    os.environ["UNSLOTH_COMPILE_DISABLE"] = "partial"
                    os.environ["UNSLOTH_DISABLE_STATIC_GENERATION"] = "1"
                    if transformers_version < Version("4.50.0.dev0"):
                        raise RuntimeError(
                            f"Unsloth: {check_model_name} only works on transformers >= 4.50.0."
                            + NIGHTLY
                        )
                    break

        if auto_model is not None:
            # All other models need to disable static cache
            os.environ["UNSLOTH_DISABLE_STATIC_GENERATION"] = "1"

        # New transformers need to check manually.
        if SUPPORTS_LLAMA32 and is_model and is_peft:
            # Check if folder exists locally
            if os.path.isdir(model_name):
                exist_adapter_config = os.path.exists(
                    os.path.join(model_name, "adapter_config.json")
                )
                exist_config = os.path.exists(os.path.join(model_name, "config.json"))
                both_exist = exist_adapter_config and exist_config
            else:
                # Both AutoConfig and PeftConfig loaded successfully from this
                # remote repo, so both config.json and adapter_config.json
                # definitely exist -- no need for an extra HfFileSystem network call.
                both_exist = True

        # Get base model for PEFT:
        if is_peft:
            # Check base model again for PEFT
            model_name = peft_config.base_model_name_or_path
            if not use_exact_model_name:
                model_name = get_model_name(model_name, load_in_4bit)
            # Check if pre-quantized models are allowed
            # AMD Instinct GPUs need blocksize = 128 on bitsandbytes < 0.49.2 (our pre-quants use blocksize = 64)
            if not ALLOW_PREQUANTIZED_MODELS and model_name.lower().endswith(
                ("-unsloth-bnb-4bit", "-bnb-4bit")
            ):
                model_name = _strip_unsloth_bnb_4bit_suffix(model_name)
            # '-bf16' hub repos load bf16; a local dir keeps the requested quant unless 16bit is set
            if model_name.lower().endswith("-bf16") and (
                load_in_16bit or not os.path.isdir(os.path.expanduser(model_name))
            ):
                load_in_4bit = False
                load_in_8bit = False
                load_in_fp8 = False
                load_in_16bit = True

            if user_config is not None:
                model_config = user_config
            else:
                model_config = AutoConfig.from_pretrained(
                    model_name,
                    token = token,
                    trust_remote_code = trust_remote_code,
                    local_files_only = local_files_only,
                )

        if not was_disabled:
            enable_progress_bars()

        do_logging = os.environ.get("UNSLOTH_ENABLE_LOGGING", "0") == "1"
        if do_logging:
            redirector = contextlib.nullcontext()
        else:
            redirector = contextlib.redirect_stdout(open(os.devnull, "w", encoding = "utf-8"))

        model_types = ["siglip"] + model_types
        # Set forced float32 env flag
        os.environ["UNSLOTH_FORCE_FLOAT32"] = "0"
        do_forced_float32 = False
        for model_type_arch in model_types:
            if model_type_arch != "siglip":
                break
        for disable_name in FORCE_FLOAT32:
            # add comma to model_types_all matching in case of exact match for end
            if (
                disable_name.lower() == model_type_arch.lower().replace("-", "").replace("_", "")
                or disable_name.lower() in model_types_all
            ) and ((dtype == torch.float16) or not SUPPORTS_BFLOAT16):
                os.environ["UNSLOTH_FORCE_FLOAT32"] = "1"
                dtype = torch.bfloat16  # Change to bfloat16 loading
                break
        # Apply gradient checkpointing with smart heuristics
        use_gradient_checkpointing = apply_unsloth_gradient_checkpointing(
            use_gradient_checkpointing, max_seq_length, dtype
        )
        with redirector:
            patch_loss_functions(torch_compile = False)
            model_types, supports_sdpa = unsloth_compile_transformers(
                dtype = dtype,
                model_name = model_name,
                model_types = model_types,
                token = token,
                sdpa_dynamic_mask = True,
                sdpa_bool_masks = True,
                sdpa_gqa_replace = True,
                sdpa_dynamic_compile = True,
                compile_attention = True,
                disable_causal_masks = True,
                compile_torch_modules = True,
                compile_custom_modules = True,
                compile_function_calls = True,
                fuse_lm_head = True,
                gradient_checkpointing = True,
                manual_replacements = True,
                fast_lora_forwards = True,
                fast_residual_stream = False,
                accurate_accumulation = True,
                epilogue_fusion = True,
                max_autotune = False,
                shape_padding = True,
                cudagraphs = False,
                debug = False,
                fullgraph = fullgraph,
                import_from_cache = False,
                disable = False,
                return_logits = return_logits,
                trust_remote_code = trust_remote_code,
                unsloth_force_compile = unsloth_force_compile,
            )
        # Fix SDPA issues
        for model_type in DISABLE_SDPA_MODEL_NAMES:
            if model_type in model_types_all:
                supports_sdpa = False

        # Keep the local checkpoint dir as tokenizer when self-sufficient (see
        # _resolve_checkpoint_tokenizer_name). A VLM also needs local processor files, else
        # we fall back to the base repo so its cached processor loads.
        _ckpt_arch = getattr(model_config, "architectures", None) or []
        _ckpt_is_vlm = any(x.endswith("ForConditionalGeneration") for x in _ckpt_arch) or hasattr(
            model_config, "vision_config"
        )
        tokenizer_name = _resolve_checkpoint_tokenizer_name(
            old_model_name, kwargs, require_processor = _ckpt_is_vlm
        )

        # Capture task intent before text_only can replace a parent VLM config
        # with its nested text config.
        task_config_attrs = _get_user_task_config_attrs(user_config)
        for _cfg_key in ("num_labels", "id2label", "label2id", "problem_type"):
            _cfg_val = kwargs.get(_cfg_key, None)
            if _cfg_val is not None:
                task_config_attrs[_cfg_key] = _cfg_val
        _num_labels = task_config_attrs.get("num_labels", None)
        for _cfg_key, _cfg_val in task_config_attrs.items():
            set_task_config_attr(model_config, _cfg_key, _cfg_val)

        # Check if VLM
        architectures = getattr(model_config, "architectures", None)
        if architectures is None:
            architectures = []
        is_vlm = any(x.endswith("ForConditionalGeneration") for x in architectures)
        is_vlm = is_vlm or hasattr(model_config, "vision_config")
        load_text_only = text_only and auto_model is None
        if load_text_only:
            if hasattr(model_config, "vision_config"):
                text_config = _get_text_only_config(model_config, old_model_name)
                # Skip the vision tower only for families with their own text decoder (Gemma 3);
                # others would load random weights, so keep the full model (use FastVisionModel).
                text_class = resolve_model_class(AutoModelForCausalLM, text_config)
                if text_class is None or not _is_family_text_decoder(
                    getattr(model_config, "model_type", ""),
                    getattr(text_config, "model_type", ""),
                ):
                    load_text_only = False
                else:
                    logger.warning_once(
                        f"Loading {old_model_name} as text-only; vision/audio towers skipped. "
                        "Use FastVisionModel for multimodal inputs."
                    )
                    # Remap VLM text weights (tf >=5) while model_config is still the parent. #5816
                    _apply_text_only_key_mapping(kwargs, model_config, text_config)
                    model_config = text_config
                    is_vlm = False
            else:
                is_vlm = False
        # If num_labels is set, use AutoModelForSequenceClassification
        for _cfg_key, _cfg_val in task_config_attrs.items():
            set_task_config_attr(model_config, _cfg_key, _cfg_val)
        if auto_model is None:
            if _num_labels is not None:
                from transformers import AutoModelForSequenceClassification
                auto_model = AutoModelForSequenceClassification
            elif is_vlm:
                # Check if the model's auto_map supports the VLM auto class.
                # Some repo-code VL models register only a generic auto class and not
                # AutoModelForImageTextToText/AutoModelForVision2Seq: Nemotron-VL uses
                # AutoModelForCausalLM, DeepSeek-OCR uses AutoModel. Calling the VLM auto
                # class on those raises "Unrecognized configuration class ... for
                # AutoModelForImageTextToText", so fall back to whatever generic class the
                # repo actually registered. Match the CONCRETE class name we would pass
                # (AutoModelForVision2Seq aliases to AutoModelForImageTextToText on tf>=5),
                # since transformers resolves remote code by that exact name -- a config
                # that only registers the legacy key must still take the generic fallback.
                _auto_map = getattr(model_config, "auto_map", {}) or {}
                _vlm_class_name = AutoModelForVision2Seq.__name__
                _has_vlm_class = _vlm_class_name in _auto_map
                if not _has_vlm_class and "AutoModelForCausalLM" in _auto_map:
                    auto_model = AutoModelForCausalLM
                elif not _has_vlm_class and "AutoModel" in _auto_map:
                    from transformers import AutoModel
                    auto_model = AutoModel
                else:
                    auto_model = AutoModelForVision2Seq
            else:
                auto_model = AutoModelForCausalLM

        load_in_4bit_kwargs = load_in_4bit
        load_in_8bit_kwargs = load_in_8bit
        if quantization_config is not None and not fast_inference:
            load_in_4bit_kwargs = False
            load_in_8bit_kwargs = False

        model, tokenizer = FastBaseModel.from_pretrained(
            model_name = model_name,
            max_seq_length = max_seq_length,
            dtype = _get_dtype(dtype),
            load_in_4bit = load_in_4bit_kwargs,
            load_in_8bit = load_in_8bit_kwargs,
            load_in_16bit = load_in_16bit,
            full_finetuning = full_finetuning,
            token = token,
            device_map = device_map,
            trust_remote_code = trust_remote_code,
            revision = revision if not is_peft else None,
            model_types = model_types,
            tokenizer_name = tokenizer_name,
            auto_model = auto_model,
            use_gradient_checkpointing = use_gradient_checkpointing,
            supports_sdpa = supports_sdpa,
            whisper_language = whisper_language,
            whisper_task = whisper_task,
            auto_config = model_config,
            offload_embedding = offload_embedding,
            float32_mixed_precision = float32_mixed_precision,
            # Pass vLLM/inference parameters
            fast_inference = fast_inference,
            gpu_memory_utilization = gpu_memory_utilization,
            float8_kv_cache = float8_kv_cache,
            random_state = random_state,
            max_lora_rank = max_lora_rank,
            disable_log_stats = disable_log_stats,
            load_in_fp8 = load_in_fp8,
            text_only = load_text_only,
            *args,
            **kwargs,
        )

        if resize_model_vocab is not None:
            model.resize_token_embeddings(resize_model_vocab)

        # In case the model supports tagging, add the unsloth tag.
        if hasattr(model, "add_model_tags"):
            model.add_model_tags(
                [
                    "unsloth",
                ]
            )
        if hasattr(tokenizer, "add_model_tags"):
            tokenizer.add_model_tags(
                [
                    "unsloth",
                ]
            )

        if load_in_4bit:
            # Fix up bitsandbytes config, but respect user-provided quantization_config
            if quantization_config is None:
                # `load_in_4bit` is the requested flag, not the effective one: a non-bnb
                # checkpoint (MXFP4/gptq/awq) had bnb disabled by check_and_disable, so stamping a
                # synthetic bnb config would corrupt its real one. Only stamp bnb/unquantized.
                try:
                    from unsloth_zoo.utils import get_quant_type
                    _stamp_bnb = get_quant_type(model.config) in (None, "bitsandbytes")
                except Exception:
                    _stamp_bnb = True
                if _stamp_bnb:
                    compute_dtype = dtype_from_config(model.config)
                    quantization_config = {
                        # Sometimes compute_dtype is not a string!!
                        "bnb_4bit_compute_dtype": compute_dtype,
                        "bnb_4bit_quant_type": "nf4",
                        "bnb_4bit_use_double_quant": True,
                        "llm_int8_enable_fp32_cpu_offload": False,
                        "llm_int8_has_fp16_weight": False,
                        "llm_int8_skip_modules": None,
                        "llm_int8_threshold": 6.0,
                        "load_in_4bit": True,
                        "load_in_8bit": False,
                        "quant_method": "bitsandbytes",
                    }
                    model.config.update({"quantization_config": quantization_config})
            else:
                if hasattr(quantization_config, "to_dict"):
                    model.config.update({"quantization_config": quantization_config.to_dict()})
                elif isinstance(quantization_config, dict):
                    model.config.update({"quantization_config": quantization_config})

        if load_in_fp8 != False:
            _tag_model_with_fp8_torchao_config(model, fp8_mode)

        if is_peft:
            # From https://github.com/huggingface/peft/issues/184
            # Now add PEFT adapters

            # Gemma4 ClippableLinear wraps nn.Linear -- PEFT can't inject LoRA
            # on it directly.  Monkey-patch PEFT to target the inner .linear
            # child instead (same patch as vision.py training path).
            # See https://github.com/huggingface/peft/issues/3129
            _clippable_linear_cls = None
            try:
                from transformers.models.gemma4.modeling_gemma4 import (
                    Gemma4ClippableLinear as _clippable_linear_cls,
                )
            except ImportError:
                pass

            if _clippable_linear_cls is not None:
                from peft.tuners.lora.model import LoraModel as _LoraModel

                _original_car = _LoraModel._create_and_replace

                def _patched_car(
                    self,
                    peft_config,
                    adapter_name,
                    target,
                    target_name,
                    parent,
                    current_key = None,
                    **kwargs,
                ):
                    if isinstance(target, _clippable_linear_cls):
                        return _original_car(
                            self,
                            peft_config,
                            adapter_name,
                            target.linear,
                            "linear",
                            target,
                            current_key = current_key,
                            **kwargs,
                        )
                    return _original_car(
                        self,
                        peft_config,
                        adapter_name,
                        target,
                        target_name,
                        parent,
                        current_key = current_key,
                        **kwargs,
                    )

                _LoraModel._create_and_replace = _patched_car

            # Warm the adapter repo: PeftModel downloads it in-process and can hang on Xet.
            _prefetched = maybe_prefetch_hf_snapshot(
                old_model_name,
                token = token,
                revision = revision,
                cache_dir = kwargs.get("cache_dir"),
                local_files_only = local_files_only,
                # Adapter always loads in-process via PeftModel, so warm it even under fast_inference.
                fast_inference = False,
                force_download = kwargs.get("force_download", False),
                # Leave use_safetensors auto (inheriting base format could skip a safetensors-only
                # adapter). adapter_only restricts the warm to the adapter files + root aux.
                adapter_only = True,
            )
            # Child did the forced download; clear the flag so the load reuses the warm cache.
            if _prefetched and kwargs.get("force_download", False):
                kwargs["force_download"] = False
            # Forward cache_dir so the load reads the warmed adapter. No subfolder (that targets the
            # base checkpoint; adapters live at the root).
            peft_load_kwargs = {}
            if kwargs.get("cache_dir") is not None:
                peft_load_kwargs["cache_dir"] = kwargs["cache_dir"]
            try:
                model = PeftModel.from_pretrained(
                    model,
                    old_model_name,
                    token = token,
                    revision = revision,
                    local_files_only = local_files_only,
                    is_trainable = True,
                    trust_remote_code = trust_remote_code,
                    **peft_load_kwargs,
                )
            finally:
                # Always restore original PEFT method, even if loading fails
                if _clippable_linear_cls is not None:
                    _LoraModel._create_and_replace = _original_car

            # Patch it as well!
            model = FastBaseModel.post_patch_model(
                model, use_gradient_checkpointing, trust_remote_code = trust_remote_code
            )
            # Re-evaluate grouped MoE now the adapter is attached: an expert-LoRA block falls back
            # to the original loop, an attention-only adapter keeps the grouped path. Guarded.
            try:
                from unsloth_zoo.temporary_patches.moe_grouped_modulelist import (
                    auto_enable_grouped_moe,
                )
                auto_enable_grouped_moe(model)
            except Exception:
                pass  # optional speedup; never block model loading

        # Apply QAT if specified
        if qat_scheme is not None:
            print("Unsloth: Applying QAT to mitigate quantization degradation")
            model = FastModel._prepare_for_qat(model, qat_scheme)

        # Patch Tiled MLP
        # to turn on set UNSLOTH_TILED_MLP to "arctic", "target", or "target:{GB}""
        patch_tiled_mlp_choice = os.environ.get(
            "UNSLOTH_TILED_MLP", "arctic" if unsloth_tiled_mlp else "0"
        )
        if patch_tiled_mlp_choice != "0" or unsloth_tiled_mlp:
            patch_tiled_mlp(model, patch_options_str = patch_tiled_mlp_choice)

        model = _fix_rope_inv_freq(model)
        model = _exclude_rope_inv_freq_from_ddp(model)
        return model, tokenizer


class FastVisionModel(FastModel):
    pass


class FastTextModel(FastModel):
    pass