Fix forward compatibility with transformers 5.x

Three issues fixed:

1. Skip exec-based config patching for transformers >= 5.0

Transformers 5.x config classes use @strict, @auto_docstring, and
interval() decorators/annotations that break exec(inspect.getsource(...)).
Those configs already use rope_parameters (the v5 replacement for
rope_scaling), so the patching is not needed. Gated with a version check
so transformers 4.x behavior is unchanged.

2. Slice position_ids to last token in fast_forward_inference

Transformers 5.x generate() accumulates position_ids as
[batch, full_seq_len] across decode steps instead of [batch, 1].
This causes a shape mismatch when indexing cos/sin for rotary
embeddings: cos[position_ids] produces [batch, full_seq_len, head_dim]
but Qn is [batch, n_heads, 1, head_dim]. Fixed by slicing
position_ids[:, -1:] when shape[-1] > 1. Applied to all model files
with fast_forward_inference: llama, qwen3, falcon_h1, gemma2, cohere,
granite. No-op on transformers 4.x since position_ids is already
[batch, 1]. Training path is unaffected.

3. Handle @strict config kwargs for sequence classification

Transformers 5.x @strict config decorator rejects unexpected kwargs
like num_labels, id2label, and max_position_embeddings passed to model
__init__(). Fixed by setting these on the config object directly and
passing config= to from_pretrained. Also added num_labels routing in
FastModel loader to select AutoModelForSequenceClassification.

unsloth/models/_utils.py

@@ -765,43 +765,51 @@ model_architectures = [
     "falcon_h1",
 ]
 
-for model_name in model_architectures:
-    config_filepath = f"transformers.models.{model_name}.configuration_{model_name}"
-    model_filepath = f"transformers.models.{model_name}.modeling_{model_name}"
-    config_filename = f"{model_name.title().replace('_','')}Config"  # qwen3 arch folder is qwen3_moe but config is Qwen3Config. Need to remove underscore(_) for now
-    try:
-        exec(f"from {config_filepath} import {config_filename}", globals())
-    except:
-        continue
+# Transformers 5.x uses class-level annotations with @strict, @auto_docstring,
+# and interval() in config classes. exec(inspect.getsource(...)) fails because
+# those symbols are not in scope. Skip the exec-based config patching for 5.x
+# since those configs already use rope_parameters (the v5 replacement for
+# rope_scaling).
+_skip_config_exec_patch = Version(transformers_version) >= Version("5.0.0")
 
-    try:
-        config = inspect.getsource(eval(config_filename))
-    except:
-        continue
-    if "RopeParameters" in config:
+if not _skip_config_exec_patch:
+    for model_name in model_architectures:
+        config_filepath = f"transformers.models.{model_name}.configuration_{model_name}"
+        model_filepath = f"transformers.models.{model_name}.modeling_{model_name}"
+        config_filename = f"{model_name.title().replace('_','')}Config"  # qwen3 arch folder is qwen3_moe but config is Qwen3Config. Need to remove underscore(_) for now
         try:
-            exec(f"from {config_filepath} import RopeParameters", globals())
+            exec(f"from {config_filepath} import {config_filename}", globals())
         except:
             continue
 
-    if "rope_scaling" in config:
-        continue
-    config = re.sub(
-        r"(\*\*kwargs)[\s]{0,}\,[\s]{0,}\)[\s]{0,}\:",
-        r"rope_scaling=None,"
-        r"\n        **kwargs):\n"
-        r"\n        self.rope_scaling = rope_scaling\n",
-        config,
-    )
+        try:
+            config = inspect.getsource(eval(config_filename))
+        except:
+            continue
+        if "RopeParameters" in config:
+            try:
+                exec(f"from {config_filepath} import RopeParameters", globals())
+            except:
+                continue
 
-    # Just for Mistral Nemo
-    if model_name == "mistral":
-        if Version(transformers_version) <= Version("4.42.4"):
-            config = patch_mistral_nemo_config(config)
+        if "rope_scaling" in config:
+            continue
+        config = re.sub(
+            r"(\*\*kwargs)[\s]{0,}\,[\s]{0,}\)[\s]{0,}\:",
+            r"rope_scaling=None,"
+            r"\n        **kwargs):\n"
+            r"\n        self.rope_scaling = rope_scaling\n",
+            config,
+        )
 
-    exec(config, globals())
-    exec(f"import {config_filepath}", globals())
-    exec(f"{config_filepath}.{config_filename} = {config_filename}", globals())
+        # Just for Mistral Nemo
+        if model_name == "mistral":
+            if Version(transformers_version) <= Version("4.42.4"):
+                config = patch_mistral_nemo_config(config)
+
+        exec(config, globals())
+        exec(f"import {config_filepath}", globals())
+        exec(f"{config_filepath}.{config_filename} = {config_filename}", globals())
 # =============================================
 
 # =============================================

unsloth/models/cohere.py

@@ -357,6 +357,9 @@ def CohereAttention_fast_forward_inference(
     # cos, sin = self.rotary_emb(Vn, seq_len = kv_seq_len)
     # Qn, Kn = inplace_rope_embedding(Qn, Kn, cos, sin, position_ids)
     cos, sin = self.rotary_emb.get_cached(kv_seq_len, Qn.device.index)
+    # Transformers 5.x: position_ids may be [batch, full_seq_len]; slice to last
+    if position_ids.dim() >= 2 and position_ids.shape[-1] > 1:
+        position_ids = position_ids[:, -1:]
     cos = cos[position_ids].unsqueeze(1)
     sin = sin[position_ids].unsqueeze(1)
     h = self.half_head_dim

unsloth/models/falcon_h1.py

@@ -313,6 +313,9 @@ def FalconH1Attention_fast_forward_inference(
     # or else error
     self.rotary_emb.extend_rope_embedding(Vn, seq_len + 2)
     cos, sin = self.rotary_emb.get_cached(kv_seq_len, Qn.device.index)
+    # Transformers 5.x: position_ids may be [batch, full_seq_len]; slice to last
+    if position_ids.dim() >= 2 and position_ids.shape[-1] > 1:
+        position_ids = position_ids[:, -1:]
     cos = cos[position_ids].unsqueeze(1)
     sin = sin[position_ids].unsqueeze(1)
     h = self.half_head_dim

unsloth/models/gemma2.py

@@ -394,6 +394,9 @@ def Gemma2Attention_fast_forward_inference(
     # cos, sin = self.rotary_emb(Vn, seq_len = kv_seq_len)
     # Qn, Kn = inplace_rope_embedding(Qn, Kn, cos, sin, position_ids)
     cos, sin = self.rotary_emb.get_cached(kv_seq_len, Qn.device.index)
+    # Transformers 5.x: position_ids may be [batch, full_seq_len]; slice to last
+    if position_ids.dim() >= 2 and position_ids.shape[-1] > 1:
+        position_ids = position_ids[:, -1:]
     cos = cos[position_ids].unsqueeze(1)
     sin = sin[position_ids].unsqueeze(1)
     h = self.half_head_dim

unsloth/models/granite.py

@@ -355,6 +355,9 @@ def GraniteAttention_fast_forward_inference(
     # cos, sin = self.rotary_emb(Vn, seq_len = kv_seq_len)
     # Qn, Kn = inplace_rope_embedding(Qn, Kn, cos, sin, position_ids)
     cos, sin = position_embeddings
+    # Transformers 5.x: position_ids may be [batch, full_seq_len]; slice to last
+    if position_ids.dim() >= 2 and position_ids.shape[-1] > 1:
+        position_ids = position_ids[:, -1:]
     cos, sin = cos[position_ids], sin[position_ids]
     h = self.half_head_dim
 

unsloth/models/llama.py

@@ -496,6 +496,10 @@ def LlamaAttention_fast_forward_inference(
     # ensure correct shape
     if position_ids.dim() == 1:
         position_ids = position_ids[:, None]
+    # Transformers 5.x generate() accumulates position_ids as [batch, full_seq_len]
+    # across decode steps. In single-token inference we only need the last position.
+    if position_ids.shape[-1] > 1:
+        position_ids = position_ids[:, -1:]
     position_ids = position_ids.to(Qn.device)
 
     if rotary_seq_len is None:
@@ -2414,14 +2418,19 @@ class FastLlamaModel:
 
         raise_handler = RaiseUninitialized()
         if num_labels is not None:
+            # Transformers 5.x @strict config classes reject unexpected kwargs
+            # like num_labels and max_position_embeddings. Set on the config
+            # object directly and pass config= instead.
+            model_config.num_labels = num_labels
+            if max_position_embeddings is not None:
+                model_config.max_position_embeddings = max_position_embeddings
             model = AutoModelForSequenceClassification.from_pretrained(
                 model_name,
+                config = model_config,
                 device_map = device_map,
                 # torch_dtype             = dtype, # transformers changed torch_dtype to dtype
-                num_labels = num_labels,
                 # quantization_config     = bnb_config,
                 token = token,
-                max_position_embeddings = max_position_embeddings,
                 trust_remote_code = trust_remote_code,
                 attn_implementation = preferred_attn_impl,
                 **kwargs,

unsloth/models/loader.py

@@ -1407,8 +1407,13 @@ class FastModel(FastBaseModel):
             architectures = []
         is_vlm = any(x.endswith("ForConditionalGeneration") for x in architectures)
         is_vlm = is_vlm or hasattr(model_config, "vision_config")
+        # If num_labels is set, use AutoModelForSequenceClassification
+        _num_labels = kwargs.get("num_labels", None)
         if auto_model is None:
-            if is_vlm:
+            if _num_labels is not None:
+                from transformers import AutoModelForSequenceClassification
+                auto_model = AutoModelForSequenceClassification
+            elif is_vlm:
                 # Check if the model's auto_map supports the VLM auto class.
                 # Some VL models (e.g. Nemotron-VL) only register AutoModelForCausalLM
                 # in their auto_map, not AutoModelForImageTextToText/AutoModelForVision2Seq.

unsloth/models/qwen3.py

@@ -302,6 +302,9 @@ def Qwen3Attention_fast_forward_inference(
     # or else error
     self.rotary_emb.extend_rope_embedding(Vn, seq_len + 2)
     cos, sin = self.rotary_emb.get_cached(kv_seq_len, Qn.device.index)
+    # Transformers 5.x: position_ids may be [batch, full_seq_len]; slice to last
+    if position_ids.dim() >= 2 and position_ids.shape[-1] > 1:
+        position_ids = position_ids[:, -1:]
     cos = cos[position_ids].unsqueeze(1)
     sin = sin[position_ids].unsqueeze(1)
     h = self.half_head_dim

unsloth/models/vision.py

@@ -788,6 +788,15 @@ class FastBaseModel:
         if not fast_inference:
             # Prevent load_in_fp8 from being forwarded into HF internal model loading
             load_in_fp8 = kwargs.pop("load_in_fp8", None)
+            # Transformers 5.x @strict config classes reject unexpected kwargs.
+            # Move config-level attributes onto the config object directly.
+            _num_labels = kwargs.pop("num_labels", None)
+            if _num_labels is not None:
+                model_config.num_labels = _num_labels
+            for _cfg_key in ("id2label", "label2id", "max_position_embeddings"):
+                _cfg_val = kwargs.pop(_cfg_key, None)
+                if _cfg_val is not None:
+                    setattr(model_config, _cfg_key, _cfg_val)
             model = auto_model.from_pretrained(
                 model_name,
                 config = model_config,