fix: unwrap ProcessorMixin to raw tokenizer for text-only SFTTrainer on VLM-architecture models
This commit is contained in:
Roland Tannous
parent
9840864662
commit
23214c41c0
13 changed files with 2431338 additions and 113 deletions
|
|
@ -268,12 +268,6 @@ class UnslothTrainer:
|
|||
print(f"Configuring LoRA adapters (r={lora_r}, alpha={lora_alpha})...\n")
|
||||
print(f"Gradient checkpointing: {use_gradient_checkpointing} (type: {type(use_gradient_checkpointing).__name__})\n")
|
||||
|
||||
# Normalize ["all-linear"] (list from frontend/YAML) → "all-linear" (string)
|
||||
# Unsloth/PEFT expect the string form for this shorthand
|
||||
if target_modules == ["all-linear"]:
|
||||
target_modules = "all-linear"
|
||||
print(f" Normalized target_modules from list to string: '{target_modules}'")
|
||||
|
||||
# Branch based on vision vs text
|
||||
if self.is_vlm:
|
||||
# Vision model LoRA
|
||||
|
|
@ -445,7 +439,6 @@ class UnslothTrainer:
|
|||
dataset,
|
||||
model_name=self.model_name,
|
||||
tokenizer=self.tokenizer,
|
||||
model=self.model,
|
||||
is_vlm=self.is_vlm,
|
||||
format_type=format_type,
|
||||
dataset_name=dataset_source,
|
||||
|
|
@ -468,7 +461,6 @@ class UnslothTrainer:
|
|||
eval_dataset,
|
||||
model_name=self.model_name,
|
||||
tokenizer=self.tokenizer,
|
||||
model=self.model,
|
||||
is_vlm=self.is_vlm,
|
||||
format_type=format_type,
|
||||
dataset_name=dataset_source,
|
||||
|
|
@ -795,51 +787,6 @@ class UnslothTrainer:
|
|||
print(f"The configuration is: {config_args}")
|
||||
|
||||
print("Training configuration prepared\n")
|
||||
|
||||
# ========== DEBUG: Dataset & Model Routing Info ==========
|
||||
print("=" * 60)
|
||||
print("DEBUG: Pre-Training Diagnostics")
|
||||
print("=" * 60)
|
||||
print(f" Route taken: {'VLM' if self.is_vlm else 'LLM (text)'}")
|
||||
print(f" Model name: {self.model_name}")
|
||||
print(f" Model class: {type(self.model).__name__}")
|
||||
print(f" is_vlm flag: {self.is_vlm}")
|
||||
|
||||
# Dataset info
|
||||
train_ds = dataset['dataset']
|
||||
if hasattr(train_ds, 'column_names'):
|
||||
print(f" Dataset columns: {train_ds.column_names}")
|
||||
print(f" Dataset size: {len(train_ds)} rows")
|
||||
# Print first sample
|
||||
try:
|
||||
sample = train_ds[0]
|
||||
print(f" First sample keys: {list(sample.keys())}")
|
||||
for key, val in sample.items():
|
||||
val_str = str(val)
|
||||
if len(val_str) > 200:
|
||||
val_str = val_str[:200] + "..."
|
||||
print(f" {key}: {val_str}")
|
||||
except Exception as e:
|
||||
print(f" Could not read first sample: {e}")
|
||||
elif isinstance(train_ds, list):
|
||||
print(f" Dataset type: list ({len(train_ds)} items)")
|
||||
if train_ds:
|
||||
print(f" First sample keys: {list(train_ds[0].keys()) if isinstance(train_ds[0], dict) else 'N/A'}")
|
||||
sample_str = str(train_ds[0])
|
||||
if len(sample_str) > 300:
|
||||
sample_str = sample_str[:300] + "..."
|
||||
print(f" First sample: {sample_str}")
|
||||
|
||||
# Model forward signature
|
||||
try:
|
||||
import inspect
|
||||
sig = inspect.signature(self.model.forward)
|
||||
fwd_params = list(sig.parameters.keys())
|
||||
print(f" model.forward() params: {fwd_params}")
|
||||
except Exception as e:
|
||||
print(f" Could not inspect model.forward(): {e}")
|
||||
|
||||
print("=" * 60)
|
||||
# ========== TRAINER INITIALIZATION ==========
|
||||
if self.is_vlm:
|
||||
trainer_kwargs = {
|
||||
|
|
@ -853,9 +800,20 @@ class UnslothTrainer:
|
|||
trainer_kwargs["eval_dataset"] = eval_dataset
|
||||
self.trainer = SFTTrainer(**trainer_kwargs)
|
||||
else:
|
||||
# For text-only training, if the tokenizer is actually a Processor
|
||||
# (e.g., Gemma-3 returns a ProcessorMixin even for text), we must
|
||||
# unwrap to the raw tokenizer. Otherwise Unsloth's SFTTrainer detects
|
||||
# ProcessorMixin → sets _is_vlm=True → skips _prepare_dataset entirely,
|
||||
# and the 'text' column never gets tokenized to 'input_ids'.
|
||||
from transformers import ProcessorMixin
|
||||
sft_tokenizer = self.tokenizer
|
||||
if isinstance(self.tokenizer, ProcessorMixin) and hasattr(self.tokenizer, 'tokenizer'):
|
||||
print(f" ⚠️ Unwrapping Processor → raw tokenizer for text-only SFTTrainer")
|
||||
sft_tokenizer = self.tokenizer.tokenizer
|
||||
|
||||
trainer_kwargs = {
|
||||
"model": self.model,
|
||||
"tokenizer": self.tokenizer,
|
||||
"tokenizer": sft_tokenizer,
|
||||
"train_dataset": dataset['dataset'],
|
||||
"data_collator": data_collator,
|
||||
"args": SFTConfig(**config_args),
|
||||
|
|
|
|||
|
|
@ -28,8 +28,6 @@ from .format_conversion import (
|
|||
convert_alpaca_to_chatml,
|
||||
convert_to_vlm_format,
|
||||
convert_llava_to_vlm_format,
|
||||
get_expected_chat_column,
|
||||
rename_chat_column_in_list,
|
||||
)
|
||||
from .chat_templates import (
|
||||
apply_chat_template_to_dataset,
|
||||
|
|
@ -549,7 +547,6 @@ def format_and_template_dataset(
|
|||
dataset,
|
||||
model_name,
|
||||
tokenizer,
|
||||
model=None,
|
||||
is_vlm = False,
|
||||
format_type="auto",
|
||||
# VLM-specific parameters
|
||||
|
|
@ -738,30 +735,12 @@ def format_and_template_dataset(
|
|||
dataset = [sample for sample in dataset]
|
||||
warnings.append("Dataset already in standard VLM messages format")
|
||||
|
||||
# Defensive: rename chat column if model expects a different name
|
||||
expected_col = get_expected_chat_column(model) if model is not None else None
|
||||
# VLM data is a list of dicts — check what key the first item uses
|
||||
current_col = "messages" # default from our converters
|
||||
if isinstance(dataset, list) and len(dataset) > 0:
|
||||
sample_keys = dataset[0].keys()
|
||||
if "conversations" in sample_keys:
|
||||
current_col = "conversations"
|
||||
elif "messages" in sample_keys:
|
||||
current_col = "messages"
|
||||
|
||||
if expected_col and expected_col != current_col:
|
||||
warnings.append(
|
||||
f"Model expects '{expected_col}' but dataset has '{current_col}' — renaming."
|
||||
)
|
||||
dataset = rename_chat_column_in_list(dataset, current_col, expected_col)
|
||||
current_col = expected_col
|
||||
|
||||
# Return as list
|
||||
return {
|
||||
"dataset": dataset,
|
||||
"detected_format": vlm_structure["format"],
|
||||
"final_format": "vlm_messages",
|
||||
"chat_column": current_col,
|
||||
"chat_column": "messages",
|
||||
"is_vlm": True,
|
||||
"is_multimodal": multimodal_info["is_multimodal"],
|
||||
"multimodal_info": multimodal_info,
|
||||
|
|
|
|||
|
|
@ -8,43 +8,6 @@ This module contains functions for converting between dataset formats
|
|||
from datasets import IterableDataset
|
||||
|
||||
|
||||
def get_expected_chat_column(model):
|
||||
"""
|
||||
Inspect the model's forward() signature to determine if it expects
|
||||
'messages' or 'conversations' as a column name.
|
||||
|
||||
Returns:
|
||||
str or None: 'messages', 'conversations', or None if neither found.
|
||||
"""
|
||||
import inspect
|
||||
try:
|
||||
sig = inspect.signature(model.forward)
|
||||
params = list(sig.parameters.keys())
|
||||
|
||||
if "messages" in params:
|
||||
return "messages"
|
||||
elif "conversations" in params:
|
||||
return "conversations"
|
||||
except (ValueError, TypeError):
|
||||
pass
|
||||
return None
|
||||
|
||||
|
||||
def rename_chat_column_in_list(data, from_col, to_col):
|
||||
"""
|
||||
Rename a chat column key in a list of dicts (for VLM data).
|
||||
"""
|
||||
if from_col == to_col:
|
||||
return data
|
||||
renamed = []
|
||||
for item in data:
|
||||
new_item = {}
|
||||
for k, v in item.items():
|
||||
new_item[to_col if k == from_col else k] = v
|
||||
renamed.append(new_item)
|
||||
return renamed
|
||||
|
||||
|
||||
def standardize_chat_format(
|
||||
dataset,
|
||||
tokenizer=None,
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue