diff --git a/studio/backend/core/training/trainer.py b/studio/backend/core/training/trainer.py index 26720865f4..8e419849cb 100644 --- a/studio/backend/core/training/trainer.py +++ b/studio/backend/core/training/trainer.py @@ -3425,15 +3425,19 @@ class UnslothTrainer: logger.info( f"CPT: using UnslothTrainer with embedding_learning_rate={embedding_lr}\n" ) + cpt_args = _UnslothTrainingArguments( + embedding_learning_rate = embedding_lr, + **config_args, + ) + if config_args.get("packing", False): + cpt_args.packing_strategy = "wrapped" + logger.info("CPT packing strategy: wrapped\n") trainer_kwargs = { "model": self.model, "tokenizer": sft_tokenizer, "train_dataset": dataset["dataset"], "data_collator": data_collator, - "args": _UnslothTrainingArguments( - embedding_learning_rate = embedding_lr, - **config_args, - ), + "args": cpt_args, } if eval_dataset is not None: trainer_kwargs["eval_dataset"] = eval_dataset diff --git a/tests/utils/test_packing.py b/tests/utils/test_packing.py index a8557d8533..98c29d9f0f 100644 --- a/tests/utils/test_packing.py +++ b/tests/utils/test_packing.py @@ -14,6 +14,7 @@ # along with this program. If not, see . from unsloth import FastLanguageModel +import unsloth.trainer as trainer_module from unsloth.utils import attention_dispatch as attention_dispatch_utils from unsloth.utils.packing import ( configure_padding_free, @@ -29,7 +30,7 @@ from unittest.mock import patch import pytest import torch -from datasets import Dataset +from datasets import Dataset, IterableDataset from trl import SFTConfig, SFTTrainer from trl.trainer.sft_trainer import DataCollatorForLanguageModeling @@ -160,6 +161,374 @@ def test_configure_padding_free(): assert config.remove_unused_columns is False +def _patch_fake_sft_trainer(): + class FakeSFTTrainer: + def __init__(self, *args, **kwargs): + self.model = args[0] if len(args) >= 1 else kwargs["model"] + self.args = args[1] if len(args) >= 2 else kwargs["args"] + self.data_collator = args[2] if len(args) >= 3 else kwargs.get("data_collator") + + trainer_module._patch_sft_trainer_auto_packing(SimpleNamespace(SFTTrainer = FakeSFTTrainer)) + return FakeSFTTrainer + + +def _vlm_model(): + return SimpleNamespace( + config = SimpleNamespace( + architectures = ["Gemma4ForConditionalGeneration"], + model_type = "gemma4", + vision_config = SimpleNamespace(), + ), + max_seq_length = 16, + ) + + +def _text_model(): + return SimpleNamespace( + config = SimpleNamespace( + architectures = ["LlamaForCausalLM"], + model_type = "llama", + ), + max_seq_length = 16, + ) + + +class _CharacterTokenizer: + bos_token = None + eos_token = None + chat_template = None + + def __call__(self, texts, **kwargs): + is_batched = isinstance(texts, list) + if not is_batched: + texts = [texts] + input_ids = [[ord(char) for char in text] for text in texts] + if kwargs.get("truncation") and kwargs.get("max_length") is not None: + input_ids = [ids[: kwargs["max_length"]] for ids in input_ids] + return {"input_ids": input_ids if is_batched else input_ids[0]} + + +def test_vlm_text_dataset_allows_explicit_packing(): + fake_trainer = _patch_fake_sft_trainer() + config = SimpleNamespace(packing = True, padding_free = None, remove_unused_columns = True) + + trainer = fake_trainer( + model = _vlm_model(), + args = config, + processing_class = object(), + train_dataset = Dataset.from_dict({"text": ["text-only CPT sample"]}), + ) + + assert config.packing is True + assert config.padding_free is True + assert trainer.model._unsloth_allow_packed_overlength is True + + +def test_vlm_without_processing_class_still_disables_packing(): + fake_trainer = _patch_fake_sft_trainer() + config = SimpleNamespace(packing = True, padding_free = None, remove_unused_columns = True) + + fake_trainer( + _vlm_model(), + config, + None, + Dataset.from_dict({"text": ["text-only sample"]}), + ) + + assert config.packing is False + assert config.padding_free is False + + +@pytest.mark.parametrize( + ("model_type", "architecture"), + ( + ("t5", "T5ForConditionalGeneration"), + ("bart", "BartForConditionalGeneration"), + ("whisper", "WhisperForConditionalGeneration"), + ("csm", "CsmForConditionalGeneration"), + ), +) +def test_nonvision_conditional_generation_keeps_packing(model_type, architecture): + fake_trainer = _patch_fake_sft_trainer() + config = SimpleNamespace(packing = True, padding_free = None, remove_unused_columns = True) + model = SimpleNamespace( + config = SimpleNamespace(model_type = model_type, architectures = [architecture]), + max_seq_length = 16, + ) + + trainer = fake_trainer( + model, + config, + None, + Dataset.from_dict({"text": ["text-only sample"]}), + ) + + assert config.packing is True + assert config.padding_free is True + assert trainer.model._unsloth_allow_packed_overlength is True + + +def test_vlm_vision_dataset_still_disables_packing(): + fake_trainer = _patch_fake_sft_trainer() + config = SimpleNamespace(packing = True, padding_free = None, remove_unused_columns = True) + + fake_trainer( + _vlm_model(), + config, + None, + Dataset.from_dict({"images": [None], "text": ["multimodal sample"]}), + None, + object(), + ) + + assert config.packing is False + assert config.padding_free is False + + +@pytest.mark.parametrize( + "vision_column", + ("pixel_values", "pixel_attention_mask", "image_grid_thw"), +) +def test_vlm_preprocessed_vision_dataset_disables_packing(vision_column): + fake_trainer = _patch_fake_sft_trainer() + config = SimpleNamespace(packing = True, padding_free = None, remove_unused_columns = True) + + fake_trainer( + model = _vlm_model(), + args = config, + processing_class = object(), + train_dataset = Dataset.from_dict({"input_ids": [[1]], vision_column: [None]}), + ) + + assert config.packing is False + assert config.padding_free is False + + +@pytest.mark.parametrize("dict_eval", (False, True)) +def test_vlm_vision_eval_dataset_disables_packing(dict_eval): + fake_trainer = _patch_fake_sft_trainer() + config = SimpleNamespace(packing = True, padding_free = None, remove_unused_columns = True) + eval_dataset = Dataset.from_dict({"input_ids": [[1]], "pixel_values": [None]}) + if dict_eval: + eval_dataset = {"vision": eval_dataset} + + fake_trainer( + model = _vlm_model(), + args = config, + processing_class = object(), + train_dataset = Dataset.from_dict({"text": ["text-only training sample"]}), + eval_dataset = eval_dataset, + ) + + assert config.packing is False + assert config.padding_free is False + + +def test_vlm_streaming_vision_dataset_without_metadata_disables_packing(): + fake_trainer = _patch_fake_sft_trainer() + config = SimpleNamespace(packing = True, padding_free = None, remove_unused_columns = True) + dataset = IterableDataset.from_generator( + lambda: iter([{"images": [None], "text": "multimodal sample"}]) + ) + assert dataset.column_names is None + + fake_trainer( + model = _vlm_model(), + args = config, + processing_class = object(), + train_dataset = dataset, + ) + + assert config.packing is False + assert config.padding_free is False + assert next(iter(dataset))["text"] == "multimodal sample" + + +@pytest.mark.parametrize("data_collator", (None, object())) +def test_stateful_stream_is_not_consumed_during_detection(data_collator): + class StatefulDataset: + def __init__(self): + self.rows = iter([{"text": "first"}, {"text": "second"}]) + + def __iter__(self): + return (row for row in self.rows) + + fake_trainer = _patch_fake_sft_trainer() + config = SimpleNamespace(packing = True, padding_free = None, remove_unused_columns = True) + dataset = StatefulDataset() + + fake_trainer( + model = _vlm_model(), + args = config, + processing_class = object(), + data_collator = data_collator, + train_dataset = dataset, + ) + + assert config.packing is False + assert config.padding_free is False + assert next(iter(dataset))["text"] == "first" + + +def test_text_model_stream_without_metadata_keeps_packing(): + class StatefulDataset: + def __init__(self): + self.rows = iter([{"text": "first"}, {"text": "second"}]) + + def __iter__(self): + return (row for row in self.rows) + + fake_trainer = _patch_fake_sft_trainer() + config = SimpleNamespace(packing = True, padding_free = None, remove_unused_columns = True) + dataset = StatefulDataset() + + trainer = fake_trainer( + model = _text_model(), + args = config, + processing_class = object(), + train_dataset = dataset, + ) + + assert config.packing is True + assert config.padding_free is True + assert trainer.model._unsloth_allow_packed_overlength is True + assert next(iter(dataset))["text"] == "first" + + +def test_bfd_packing_truncates_before_packing(monkeypatch): + args = SimpleNamespace( + dataset_num_proc = 1, + dataset_text_field = "text", + max_length = 4, + packing_strategy = "bfd", + ) + trainer = SimpleNamespace(model = None) + dataset = Dataset.from_dict({"prompt": ["abc"], "completion": ["defghij"]}) + prepare_globals = SFTTrainer._prepare_dataset.__globals__ + + def passthrough_pack_dataset(dataset, seq_length, strategy, map_kwargs): + return dataset + + monkeypatch.setitem(prepare_globals, "pack_dataset", passthrough_pack_dataset) + packed = SFTTrainer._prepare_dataset( + trainer, + dataset, + _CharacterTokenizer(), + args, + True, + None, + "train", + ) + + assert len(packed["input_ids"][0]) == args.max_length + + +def test_wrapped_strategy_without_packing_still_truncates(): + args = SimpleNamespace( + dataset_num_proc = 1, + dataset_text_field = "text", + max_length = 4, + packing_strategy = "wrapped", + ) + trainer = SimpleNamespace(model = None) + dataset = Dataset.from_dict({"text": ["abcdefghi"]}) + + prepared = SFTTrainer._prepare_dataset( + trainer, + dataset, + _CharacterTokenizer(), + args, + False, + None, + "train", + ) + + assert len(prepared["input_ids"][0]) == args.max_length + + +@pytest.mark.parametrize("legacy_api", (False, True)) +def test_wrapped_packing_preserves_overlength_tokens(monkeypatch, legacy_api): + args_kwargs = { + "dataset_num_proc": 1, + "dataset_text_field": "text", + "max_length": 4, + } + if not legacy_api: + args_kwargs["packing_strategy"] = "wrapped" + args = SimpleNamespace(**args_kwargs) + trainer = SimpleNamespace(model = None) + dataset = Dataset.from_dict({"text": ["abcdefghi"]}) + prepare_globals = SFTTrainer._prepare_dataset.__globals__ + pack_dataset = prepare_globals["pack_dataset"] + + def legacy_pack_dataset( + dataset, + seq_length, + map_kwargs = None, + ): + return pack_dataset(dataset, seq_length, "wrapped", map_kwargs) + + if legacy_api: + monkeypatch.setitem(prepare_globals, "pack_dataset", legacy_pack_dataset) + + packed = SFTTrainer._prepare_dataset( + trainer, + dataset, + _CharacterTokenizer(), + args, + True, + None, + "train", + ) + + packed_ids = packed["input_ids"] + assert sum(len(input_ids) for input_ids in packed_ids) == 9 + assert all(len(input_ids) <= args.max_length for input_ids in packed_ids) + + +# Named to match the unsloth_zoo helper: sft_trainer_prepare_dataset sources it by +# name and renames "def sft_prepare_dataset" -> "def _prepare_dataset". This fixture +# deliberately omits the "All Unsloth Zoo code licensed under LGPLv3" header to emulate +# a newer, compatible Zoo whose header moved (the dependency is only lower-bounded). +def sft_prepare_dataset( + self, dataset, processing_class, args, packing, formatting_func, dataset_text_field +): + do_truncation = True + # Mirror the Zoo call so the "truncation = do_truncation," injection anchor + # survives formatting (a bare tuple assignment gets rewritten to a paren form). + dataset = processing_class( + dataset, + truncation = do_truncation, + ) + return dataset + + +def test_wrapped_packing_setup_survives_missing_zoo_header(monkeypatch): + # Regression: the wrapped-packing setup used to anchor on the Zoo license comment, + # so a header change made it a no-op while the truncation reference still landed, + # NameError-ing every SFT dataset preparation. It must now install via the + # signature and always precede the reference. + import ast + import textwrap + import unsloth.models.rl_replacements as rlr + + monkeypatch.setitem(rlr.RL_REPLACEMENTS, "sft_prepare_dataset", sft_prepare_dataset) + + source = ( + "def _prepare_dataset(self, dataset, processing_class, args, packing, " + "formatting_func, dataset_text_field):\n return dataset\n" + ) + patched = rlr.sft_trainer_prepare_dataset("_prepare_dataset", source) + + assert "_unsloth_wrapped_packing = packing" in patched + assert "import inspect as _inspect" in patched + assert "not _unsloth_wrapped_packing" in patched + assert patched.index("_unsloth_wrapped_packing = packing") < patched.index( + "truncation = do_truncation and not _unsloth_wrapped_packing" + ) + ast.parse(textwrap.dedent(patched)) + + class _DummyChild(torch.nn.Module): def __init__(self): super().__init__() diff --git a/unsloth/models/rl_replacements.py b/unsloth/models/rl_replacements.py index ffb845b04f..b0709f7376 100644 --- a/unsloth/models/rl_replacements.py +++ b/unsloth/models/rl_replacements.py @@ -276,17 +276,13 @@ def dpo_trainer_vision_signature_columns(function_name, function): _extra_columns = "".join(f' "{_k}",\n' for _k in _DPO_VISION_KEYS) new_function = function.replace( ' "image_sizes",\n "token_type_ids",\n', - f' "image_sizes",\n' - f"{_extra_columns}" - f' "token_type_ids",\n', + f' "image_sizes",\n{_extra_columns} "token_type_ids",\n', ) if new_function != function: return new_function return function.replace( ' "image_sizes",\n "ref_chosen_logps",\n', - f' "image_sizes",\n' - f"{_extra_columns}" - f' "ref_chosen_logps",\n', + f' "image_sizes",\n{_extra_columns} "ref_chosen_logps",\n', ) @@ -458,6 +454,60 @@ def sft_trainer_prepare_dataset(function_name, function): if matched: # Use fast version! function = inspect.getsource(fast_sft_prepare_dataset) + # why: install the wrapped-packing setup (and the `_inspect` import the + # truncation / pack_dataset rewrites below depend on) at the function + # signature, a structural anchor that always exists, rather than the + # unsloth_zoo license-comment line. That header is only lower-bounded, so a + # newer Zoo may move or drop it; anchoring there let the setup silently + # no-op while the references still landed, NameError-ing every SFT dataset + # preparation. Fail loudly if even the signature cannot be located. + _wrapped_packing_setup = ( + " import inspect as _inspect\n" + " try:\n" + ' _unsloth_pack_has_strategy = "strategy" in _inspect.signature(pack_dataset).parameters\n' + " except Exception:\n" + " _unsloth_pack_has_strategy = True\n" + " _unsloth_wrapped_packing = packing and (\n" + ' getattr(args, "packing_strategy", None) == "wrapped"\n' + " or not _unsloth_pack_has_strategy\n" + " )\n" + ) + function, _n_setup = re.subn( + r"(def sft_prepare_dataset\s*\(.*?\)\s*(?:->[^:\n]*)?:[ \t]*\n)", + lambda match: match.group(1) + _wrapped_packing_setup, + function, + count = 1, + flags = re.DOTALL, + ) + if _n_setup != 1: + raise RuntimeError( + "Unsloth: failed to install wrapped-packing support into " + "sft_prepare_dataset (signature not found); please file a bug report." + ) + function = function.replace( + "truncation = do_truncation,", + "truncation = do_truncation and not _unsloth_wrapped_packing,", + ) + function = function.replace( + "if do_truncation and max_seq_length > 0:", + "if do_truncation and not _unsloth_wrapped_packing and max_seq_length > 0:", + ) + function = function.replace( + """dataset = pack_dataset( + dataset.select_columns(used_column_names), + max_seq_length, + getattr(args, "packing_strategy", "bfd"), + map_kwargs, + )""", + """_pack_kwargs = {"map_kwargs": map_kwargs} + if "strategy" in _inspect.signature(pack_dataset).parameters: + _pack_kwargs["strategy"] = getattr(args, "packing_strategy", "bfd") + dataset = pack_dataset( + dataset.select_columns(used_column_names), + max_seq_length, + **_pack_kwargs, + )""", + ) function = function.split("\n") function = "\n".join(" " * 4 + x for x in function) function = function.replace("def sft_prepare_dataset", "def _prepare_dataset") @@ -2120,19 +2170,21 @@ def grpo_trainer_compute_loss(function_name, function): logits_to_keep, batch_size = None, compute_entropy = False, - compute_efficient = False: self._get_per_token_logps( - model, input_ids, attention_mask, logits_to_keep, compute_efficient + compute_efficient = False: ( + self._get_per_token_logps( + model, input_ids, attention_mask, logits_to_keep, compute_efficient + ) + if hasattr(self, "_get_per_token_logps") + else self._get_per_token_logps_and_entropies( + model, + input_ids, + attention_mask, + logits_to_keep, + batch_size, + compute_entropy, + compute_efficient, + )[0] ) - if hasattr(self, "_get_per_token_logps") - else self._get_per_token_logps_and_entropies( - model, - input_ids, - attention_mask, - logits_to_keep, - batch_size, - compute_entropy, - compute_efficient, - )[0] ) # logps per_token_logps = get_logps_func( diff --git a/unsloth/trainer.py b/unsloth/trainer.py index 83cb1758f0..61d41aad21 100644 --- a/unsloth/trainer.py +++ b/unsloth/trainer.py @@ -100,6 +100,10 @@ PADDING_FREE_BLOCKLIST = { "gemma2", # - gemma2: Uses slow_attention_softcapping which has torch.compile issues "gpt_oss", # - gpt_oss: Uses Flex Attention which doesn't handle padding_free correctly } +# Hybrid linear-attention / state-space models (Qwen3.5, Qwen3-Next, ...) carry a +# recurrent gated-delta state plus a causal conv1d. Sample packing / padding-free +# flattens the batch, so those ops leak state across sequence boundaries. Detected +# structurally by _is_hybrid_linear_attention_model rather than by model name. def _should_pack(config) -> bool: @@ -137,6 +141,132 @@ def _should_skip_auto_packing_error(exc: Exception) -> bool: return any(msg in message for msg in _AUTO_PACK_SKIP_MESSAGES) +_VISION_DATASET_KEYS = frozenset( + { + "image", + "images", + "image_grid_thw", + "image_position_ids", + "image_sizes", + "mm_token_type_ids", + "pixel_attention_mask", + "pixel_position_ids", + "pixel_values", + "pixel_values_videos", + "video", + "videos", + "video_grid_thw", + } +) + + +def _is_vlm_config(config, model_types = ()) -> bool: + if any( + hasattr(config, attr) + for attr in ("vision_config", "img_processor", "image_token_index", "projector_config") + ): + return True + + architectures = getattr(config, "architectures", None) or () + try: + from transformers.models.auto import modeling_auto + + mappings = ( + getattr(modeling_auto, "MODEL_FOR_IMAGE_TEXT_TO_TEXT_MAPPING_NAMES", {}) or {}, + getattr(modeling_auto, "MODEL_FOR_VISION_2_SEQ_MAPPING_NAMES", {}) or {}, + ) + registry_types = set().union(*(mapping.keys() for mapping in mappings)) + registry_classes = set().union(*(mapping.values() for mapping in mappings)) + config_types = set(model_types or ()) + model_type = getattr(config, "model_type", None) + if model_type is not None: + config_types.add(model_type) + if not config_types.isdisjoint(registry_types) or any( + architecture in registry_classes for architecture in architectures + ): + return True + except Exception: + pass + return any( + isinstance(architecture, str) and architecture.endswith("ForVisionText2Text") + for architecture in architectures + ) + + +def _is_vision_dataset(dataset, *, unknown_is_vision = False) -> bool: + if dataset is None: + return False + column_names = getattr(dataset, "column_names", None) + if column_names is not None: + return not _VISION_DATASET_KEYS.isdisjoint(column_names) + # Unknown-schema streams cannot be safely probed without potentially dropping a sample. + return unknown_is_vision + + +def _is_vision_eval_dataset(dataset, *, unknown_is_vision = False) -> bool: + if isinstance(dataset, dict): + return any( + _is_vision_dataset(split, unknown_is_vision = unknown_is_vision) + for split in dataset.values() + ) + return _is_vision_dataset(dataset, unknown_is_vision = unknown_is_vision) + + +_HYBRID_CONFIG_MARKERS = ( + "linear_conv_kernel_dim", + "linear_key_head_dim", + "linear_value_head_dim", + "full_attention_interval", +) + + +def _is_hybrid_linear_attention_model(model) -> bool: + """Detect models mixing linear-attention / state-space mixers (gated-delta, + Mamba-style) with a causal conv1d, e.g. Qwen3.5 / Qwen3-Next. Packing and + padding-free flatten the batch, and those recurrent + conv ops leak state + across sequence boundaries, so they must not be packed. Uses composite + structural evidence rather than a model-name match.""" + if model is None: + return False + + # Config-level: explicit hybrid layer schedule or linear-attn markers. + for config in ( + getattr(model, "config", None), + getattr(getattr(model, "config", None), "text_config", None), + ): + if config is None: + continue + layer_types = getattr(config, "layer_types", None) + if isinstance(layer_types, (list, tuple)) and any( + isinstance(t, str) and "linear_attention" in t for t in layer_types + ): + return True + if any(hasattr(config, marker) for marker in _HYBRID_CONFIG_MARKERS): + return True + + # Module-level: a mixer carrying a recurrent gated-delta op plus a conv1d. + named_modules = getattr(model, "named_modules", None) + if named_modules is None: + return False + seen = set() + for _, module in named_modules(): + if id(module) in seen: + continue + seen.add(id(module)) + cls = type(module).__name__ + if not ( + cls.endswith("GatedDeltaNet") or "LinearAttention" in cls or cls.endswith("Mamba2Mixer") + ): + continue + has_recurrent = any( + hasattr(module, attr) + for attr in ("chunk_gated_delta_rule", "recurrent_gated_delta_rule", "A_log") + ) + if has_recurrent and hasattr(module, "conv1d"): + return True + return False + + # Unsloth gradient accumulation fix: from transformers import __version__ as transformers_version, ProcessorMixin @@ -498,30 +628,43 @@ def _patch_sft_trainer_auto_packing(trl_module): else: config_arg = kwargs.get("args") - model = kwargs.get("model") - is_unsupported_model = False + model = args[0] if len(args) >= 1 else kwargs.get("model") is_vlm = False + is_unsupported_model = False + is_hybrid = False if model is not None: model_config = getattr(model, "config", None) if model_config is not None: model_types = get_transformers_model_type(model_config) is_unsupported_model = any(x in PADDING_FREE_BLOCKLIST for x in model_types) + is_vlm = _is_vlm_config(model_config, model_types) + is_hybrid = _is_hybrid_linear_attention_model(model) - architectures = getattr(model_config, "architectures", None) - if architectures is None: - architectures = [] - is_vlm = any(x.endswith("ForConditionalGeneration") for x in architectures) - is_vlm = is_vlm or hasattr(model_config, "vision_config") - - processing_class = kwargs.get("processing_class") or kwargs.get("tokenizer") - data_collator = kwargs.get("data_collator") + processing_class = ( + args[5] if len(args) >= 6 else kwargs.get("processing_class") or kwargs.get("tokenizer") + ) + data_collator = args[2] if len(args) >= 3 else kwargs.get("data_collator") + train_dataset = args[3] if len(args) >= 4 else kwargs.get("train_dataset") + eval_dataset = args[4] if len(args) >= 5 else kwargs.get("eval_dataset") + is_processor = isinstance(processing_class, ProcessorMixin) + is_auto_processor_vlm = is_vlm and processing_class is None + is_vision_dataset = ( + data_collator is None + and not is_processor + and ( + _is_vision_dataset(train_dataset, unknown_is_vision = is_vlm) + or _is_vision_eval_dataset(eval_dataset, unknown_is_vision = is_vlm) + ) + ) # Disable padding-free for VLMs / custom collators / blocklisted models blocked = ( (data_collator is not None) - or isinstance(processing_class, ProcessorMixin) - or is_vlm + or is_processor + or is_auto_processor_vlm + or is_vision_dataset or is_unsupported_model + or is_hybrid or ( os.environ.get("UNSLOTH_RETURN_LOGITS", "0") == "1" ) # Disable padding free on forced logits @@ -535,10 +678,14 @@ def _patch_sft_trainer_auto_packing(trl_module): if blocked and requested_pack: reason = "custom data collator" - if data_collator is None and isinstance(processing_class, ProcessorMixin): + if data_collator is None and is_processor: reason = "processor-based model" - elif is_vlm: - reason = "vision-language model" + elif is_auto_processor_vlm: + reason = "vision-language model with auto processor" + elif is_vision_dataset: + reason = "vision dataset" + elif is_hybrid: + reason = "hybrid linear-attention model" elif is_unsupported_model: reason = f"unsupported model type(s): {', '.join(model_types)}" message = f"Unsloth: Sample packing skipped ({reason} detected)."