Merge branch 'main' into nightly

README.md

@@ -51,8 +51,9 @@ Use our official [Unsloth Docker image](https://hub.docker.com/r/unsloth/unsloth
 For RTX 50x, B200, 6000 GPUs: `pip install unsloth`. Read our [Blackwell Guide](https://docs.unsloth.ai/basics/training-llms-with-blackwell-rtx-50-series-and-unsloth) and [DGX Spark Guide](https://docs.unsloth.ai/new/fine-tuning-llms-with-nvidia-dgx-spark-and-unsloth) for more details.
 
 ## 🦥 Unsloth News
+- New RoPE & MLP **Triton Kernels** & **Auto Packing**: 3x faster training & 30% less VRAM. [Blog](https://docs.unsloth.ai/new/3x-faster-training-packing)
 - **Ministral 3** by Mistral: Run Ministral 3 or fine-tune with our vision or RL sodoku notebook. [Guide](https://docs.unsloth.ai/new/ministral-3) • [Notebooks](https://docs.unsloth.ai/new/ministral-3#fine-tuningb)
-- **500K Context Fine-tuning**: Training a 20B model with >500K token context windows is now possible on a single 80GB GPU. [Blog](https://docs.unsloth.ai/new/500k-context-length-fine-tuning)
+- **500K Context**: Training a 20B model with >500K context is now possible on an 80GB GPU. [Blog](https://docs.unsloth.ai/new/500k-context-length-fine-tuning)
 - **FP8 Reinforcement Learning**: You can now do FP8 GRPO on consumer GPUs. [Blog](https://docs.unsloth.ai/new/fp8-reinforcement-learning) • [Notebook](https://colab.research.google.com/github/unslothai/notebooks/blob/main/nb/Qwen3_8B_FP8_GRPO.ipynb)
 - **DeepSeek-OCR**: Fine-tune to improve language understanding by 89%. [Guide](https://docs.unsloth.ai/new/deepseek-ocr-run-and-fine-tune) • [Notebook](https://colab.research.google.com/github/unslothai/notebooks/blob/main/nb/Deepseek_OCR_(3B).ipynb)
 - **Docker**: Use Unsloth with no setup & environment issues with our new image. [Guide](https://docs.unsloth.ai/new/how-to-train-llms-with-unsloth-and-docker) • [Docker image](https://hub.docker.com/r/unsloth/unsloth)

unsloth/models/_utils.py

@@ -2311,17 +2311,20 @@ def verify_fp8_support_if_applicable(model_config):
         raise ValueError(
             f"Unsloth: FP8 quantization is only supported on CUDA GPUs. You are using {DEVICE_TYPE}."
         )
-    major_version, minor_version = torch.cuda.get_device_capability()
-    if quant_method == "fbgemm_fp8" and major_version < 9:
-        # While L4 does support FP8 as data type, it doesn't have fbgemm (package) support yet. So we restrict it.
-        raise ValueError(
-            f"Unsloth: FBGEMM FP8 quantization is only supported on H100 and higher GPUs. L4 is not supported. You are using {torch.cuda.get_device_name()}. Refer to https://developer.nvidia.com/cuda-gpus for more details."
-        )
-    if quant_method == "fp8" and major_version * 10 + minor_version < 89:
-        # In case of block quantized, we allow L4 because we fall back to torchao kernels.
-        raise ValueError(
-            f"Unsloth: FP8 quantization is only supported on L4 and higher GPUs with compute capability 8.9 or higher. You are using {torch.cuda.get_device_name()}. Refer to https://developer.nvidia.com/cuda-gpus for more details."
-        )
+
+    # [TODO] Need to add FP8 support for Intel XPUs
+    if DEVICE_TYPE == "cuda":
+        major_version, minor_version = torch.cuda.get_device_capability()
+        if quant_method == "fbgemm_fp8" and major_version < 9:
+            # While L4 does support FP8 as data type, it doesn't have fbgemm (package) support yet. So we restrict it.
+            raise ValueError(
+                f"Unsloth: FBGEMM FP8 quantization is only supported on H100 and higher GPUs. L4 is not supported. You are using {torch.cuda.get_device_name()}. Refer to https://developer.nvidia.com/cuda-gpus for more details."
+            )
+        if quant_method == "fp8" and major_version * 10 + minor_version < 89:
+            # In case of block quantized, we allow L4 because we fall back to torchao kernels.
+            raise ValueError(
+                f"Unsloth: FP8 quantization is only supported on L4 and higher GPUs with compute capability 8.9 or higher. You are using {torch.cuda.get_device_name()}. Refer to https://developer.nvidia.com/cuda-gpus for more details."
+            )
 
 
 def _get_inference_mode_context_manager(model: torch.nn.Module):

unsloth/models/mistral.py

@@ -265,6 +265,7 @@ def MistralForCausalLM_fast_forward(
             output_attentions = output_attentions,
             output_hidden_states = output_hidden_states,
             return_dict = return_dict,
+            **kwargs,
         )
 
     hidden_states = outputs[0]

unsloth/trainer.py

@@ -49,36 +49,28 @@ __all__ = [
 
 logger = logging.getLogger(__name__)
 
-_AUTO_PACKING_ENV_DISABLED = os.environ.get(
-    "UNSLOTH_DISABLE_AUTO_PACKING", ""
-).strip().lower() in {"1", "true", "yes", "on"}
-
 _AUTO_PADDING_FREE_ENV_DISABLED = os.environ.get(
     "UNSLOTH_DISABLE_AUTO_PADDING_FREE", ""
 ).strip().lower() in {"1", "true", "yes", "on"}
 
-
-# [TODO]
-# Below cannot work with padding-free
 PADDING_FREE_BLOCKLIST = {
     "gemma2",  # - gemma2:  Uses slow_attention_softcapping which has torch.compile issues
     "gpt_oss",  # - gpt_oss: Uses Flex Attention which doesn't handle padding_free correctly
-    "mistral",  # - mistral: Unfortunately I think sliding window attention doesn't work correctly?
 }
 
 
-def _should_auto_pack(config) -> bool:
-    if config is None or _AUTO_PACKING_ENV_DISABLED:
-        return False
-    if not getattr(config, "packing", False):
+def _should_pack(config) -> bool:
+    if config is None or not getattr(config, "packing", False):
         return False
     return not getattr(config, "_unsloth_disable_auto_packing", False)
 
 
 def _should_auto_padding_free(config) -> bool:
-    if config is None or _AUTO_PADDING_FREE_ENV_DISABLED:
-        return False
-    if getattr(config, "packing", False):
+    if (
+        config is None
+        or _AUTO_PADDING_FREE_ENV_DISABLED
+        or getattr(config, "packing", False)
+    ):
         return False
     return not getattr(config, "padding_free", False)
 
@@ -326,23 +318,31 @@ def _patch_sft_trainer_auto_packing(trl_module):
             data_collator is not None
             or isinstance(processing_class, ProcessorMixin)
             or is_vlm
+            or is_unsupported_model
         )
-        if blocked and _should_auto_pack(config_arg):
-            reason = (
-                "custom data collator"
-                if data_collator is not None
-                else "processor-based model"
-            )
-            logger.info(
-                "Unsloth: Auto sample packing skipped (%s detected). Use UNSLOTH_DISABLE_AUTO_PACKING=1 to silence.",
-                reason,
-            )
+        requested_pack = bool(getattr(config_arg, "packing", False))
+        if blocked:
+            if hasattr(config_arg, "packing"):
+                setattr(config_arg, "packing", False)
+            if hasattr(config_arg, "padding_free"):
+                setattr(config_arg, "padding_free", False)
 
-        auto_pack_active = False
-        if _should_auto_pack(config_arg) and not blocked:
+        if blocked and requested_pack:
+            reason = "custom data collator"
+            if data_collator is None and isinstance(processing_class, ProcessorMixin):
+                reason = "processor-based model"
+            elif is_vlm:
+                reason = "vision-language model"
+            elif is_unsupported_model:
+                reason = f"unsupported model type(s): {', '.join(model_types)}"
+            message = "Unsloth: Sample packing skipped " f"({reason} detected)."
+            print(message)
+
+        packing_active = False
+        if _should_pack(config_arg) and not blocked:
             configure_sample_packing(config_arg)
-            auto_pack_active = True
-            logger.info("Unsloth: Sample packing auto-enabled for SFTTrainer instance.")
+            packing_active = True
+            logger.info("Unsloth: Sample packing enabled for SFTTrainer instance.")
 
         auto_padding_free_active = False
         padding_free_requested = getattr(config_arg, "padding_free", None) is True
@@ -359,13 +359,13 @@ def _patch_sft_trainer_auto_packing(trl_module):
         try:
             original_init(self, *args, **kwargs)
         except ValueError as exc:
-            if auto_pack_active and _should_skip_auto_packing_error(exc):
+            if packing_active and _should_skip_auto_packing_error(exc):
                 logger.info(
                     "Unsloth: Auto sample packing failed because trainer reported an incompatible setup (%s).",
                     exc,
                 )
                 _disable_sample_packing(config_arg)
-                auto_pack_active = False
+                packing_active = False
                 original_init(self, *args, **kwargs)
             else:
                 raise
@@ -376,12 +376,21 @@ def _patch_sft_trainer_auto_packing(trl_module):
             trainer_args and getattr(trainer_args, "padding_free", False)
         )
 
-        if trainer_packing and (auto_pack_active or _should_auto_pack(trainer_args)):
+        if blocked and trainer_args is not None:
+            # Mirror the block on the trainer args to avoid re-enabling later
+            setattr(trainer_args, "packing", False)
+            setattr(trainer_args, "padding_free", False)
+
+        if (
+            not blocked
+            and trainer_packing
+            and (packing_active or _should_pack(trainer_args))
+        ):
             enable_sample_packing(self.model, self)
             print(
                 "🦥 Unsloth: Packing enabled - training is >2x faster and uses less VRAM!"
             )
-        elif trainer_padding_free:
+        elif not blocked and trainer_padding_free:
             enable_padding_free_metadata(self.model, self)
             message = (
                 "🦥 Unsloth: Padding-free auto-enabled, enabling faster training."

unsloth/utils/packing.py

@@ -68,9 +68,14 @@ def _get_cached_block_mask(
 
 
 class _TrlPackingWarningFilter(logging.Filter):
+    to_filter = (
+        "attention implementation is not",
+        "kernels-community",
+    )
+
     def filter(self, record: logging.LogRecord) -> bool:
         message = record.getMessage()
-        return not "kernels-community" in message
+        return not any(substring in message for substring in self.to_filter)
 
 
 _TRL_FILTER_INSTALLED = False
@@ -102,15 +107,12 @@ def configure_sample_packing(config):
     _ensure_trl_warning_filter()
     setattr(config, "packing", True)
     setattr(config, "padding_free", True)
-    setattr(config, "remove_unused_columns", False)
 
 
 def configure_padding_free(config):
     """Mutate an ``SFTConfig`` so TRL enables padding-free batching without packing."""
     _ensure_trl_warning_filter()
     setattr(config, "padding_free", True)
-    if hasattr(config, "remove_unused_columns"):
-        setattr(config, "remove_unused_columns", False)
 
 
 def enable_sample_packing(
@@ -145,48 +147,15 @@ def enable_sample_packing(
         batch = original_torch_call(examples)
         if examples and isinstance(examples[0], dict):
             seq_lengths: list[int] = []
-            per_example_counts: list[int] = []
             for example in examples:
                 lengths = example.get(sequence_lengths_key)
                 if isinstance(lengths, Iterable):
-                    numeric_lengths = [int(length) for length in lengths]
-                    seq_lengths.extend(numeric_lengths)
-                    per_example_counts.append(len(numeric_lengths))
-                else:
-                    per_example_counts.append(0)
+                    seq_lengths.extend(int(length) for length in lengths)
             if seq_lengths:
                 batch["packed_seq_lengths"] = torch.tensor(
                     seq_lengths, dtype = torch.int32
                 )
-
-                position_ids = batch.get("position_ids")
-                input_ids = batch.get("input_ids")
-                if position_ids is None and input_ids is not None:
-                    position_ids = torch.zeros_like(
-                        input_ids, dtype = torch.long, device = input_ids.device
-                    )
-
-                if position_ids is not None and input_ids is not None:
-                    seq_index = 0
-                    for row_idx, count in enumerate(per_example_counts):
-                        cursor = 0
-                        for _ in range(count):
-                            length = seq_lengths[seq_index]
-                            if length > 0:
-                                position_ids[row_idx, cursor : cursor + length] = (
-                                    torch.arange(
-                                        length,
-                                        dtype = torch.long,
-                                        device = position_ids.device,
-                                    )
-                                )
-                                cursor += length
-                            seq_index += 1
-                    batch["position_ids"] = position_ids
-
-                if "attention_mask" in batch and getattr(
-                    collator, "return_position_ids", False
-                ):
+                if "attention_mask" in batch:
                     batch.pop("attention_mask")
         return batch
 
@@ -196,23 +165,12 @@ def enable_sample_packing(
 
 def enable_padding_free_metadata(model, trainer):
     """Inject seq-length metadata when padding-free batching is enabled without packing."""
-
-    trainer_args = getattr(trainer, "args", None)
-    if (
-        trainer_args is not None
-        and hasattr(trainer_args, "remove_unused_columns")
-        and trainer_args.remove_unused_columns
-    ):
-        trainer_args.remove_unused_columns = False
-
-    _ensure_trl_warning_filter()
     collator = getattr(trainer, "data_collator", None)
     if (
         collator is None
         or getattr(collator, "_unsloth_padding_free_lengths_wrapped", False)
         or not getattr(collator, "padding_free", False)
     ):
-        # Nothing to do if there's no collator, we've already wrapped it, or padding-free is off.
         return
 
     mark_allow_overlength(model)