diff --git a/tests/python/test_pad_token_fix.py b/tests/python/test_pad_token_fix.py index 0366bee67b..5f2a29a323 100644 --- a/tests/python/test_pad_token_fix.py +++ b/tests/python/test_pad_token_fix.py @@ -1,9 +1,9 @@ """_fix_pad_token dispatch in unsloth/tokenizer_utils.py. It must delegate to unsloth_zoo's shared fix_pad_token when present (single -source of truth), and fall back to the narrow vision-token swap against an -older unsloth_zoo. Static + CPU-only: the two helpers are exec'd in isolation -so the test never imports torch / transformers / unsloth. +source of truth), and fall back to a no-op against an older unsloth_zoo. Static ++ CPU-only: _fix_pad_token is exec'd in isolation so the test never imports +torch / transformers / unsloth. """ import ast @@ -15,9 +15,6 @@ REPO_ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..")) TOK_PATH = os.path.join(REPO_ROOT, "unsloth", "tokenizer_utils.py") WANTED = { - "_VISION_PAD_TOKENS", - "_SAFE_TEXT_PAD_TOKENS", - "_fix_vision_pad_token", "_fix_pad_token", } @@ -69,17 +66,17 @@ def test_fix_pad_token_none_is_noop(): assert ns["_fix_pad_token"](None) is None -def test_fix_pad_token_falls_back_without_shared_module(monkeypatch): +def test_fallback_keeps_pad_named_token(monkeypatch): ns = _load_pad_helpers() _block_shared_module(monkeypatch) - # Qwen3 text tokenizer shipping a vision pad_token -> narrow swap heals it. + # A pad-named token (e.g. <|vision_pad|>) is a valid pad -> fallback keeps it. tok = FakeTok( {"<|endoftext|>": 1, "<|im_end|>": 2, "<|vision_pad|>": 3}, pad = "<|vision_pad|>", eos = "<|im_end|>", ) ns["_fix_pad_token"](tok) - assert tok.pad_token == "<|endoftext|>" + assert tok.pad_token == "<|vision_pad|>" assert tok.pad_token != tok.eos_token diff --git a/unsloth/tokenizer_utils.py b/unsloth/tokenizer_utils.py index eb18b1966c..93dfa9b2ad 100644 --- a/unsloth/tokenizer_utils.py +++ b/unsloth/tokenizer_utils.py @@ -626,72 +626,22 @@ def _load_correct_tokenizer( return fast_tokenizer -# Qwen3 text models share Qwen3-VL's vocab, so configs ship a vision pad_token; -# padding text-only training with one yields NaN losses (#3155, #4104). -_VISION_PAD_TOKENS = frozenset( - ( - "<|vision_pad|>", - "<|image_pad|>", - "<|video_pad|>", - "<|audio_pad|>", - ) -) -# Preference order; excluded since reusing it as pad masks real OOV tokens. -_SAFE_TEXT_PAD_TOKENS = ("<|endoftext|>", "", "[PAD]") - - -def _fix_vision_pad_token(tokenizer): - """Swap a vision pad_token on a text-only tokenizer for a safe text token (#3155).""" - if tokenizer is None: - return tokenizer - if hasattr(tokenizer, "image_processor"): - return tokenizer - pad_token = getattr(tokenizer, "pad_token", None) - if pad_token is None or pad_token not in _VISION_PAD_TOKENS: - return tokenizer - - get_vocab = getattr(tokenizer, "get_vocab", None) - if get_vocab is None: - return tokenizer - vocab = get_vocab() - if not isinstance(vocab, dict): - return tokenizer - new_pad_token = None - for candidate in _SAFE_TEXT_PAD_TOKENS: - if candidate in vocab and candidate != getattr(tokenizer, "eos_token", None): - new_pad_token = candidate - break - # Fall back to eos_token only if it is a distinct, non-vision token. - if new_pad_token is None: - eos_token = getattr(tokenizer, "eos_token", None) - if eos_token is not None and eos_token != pad_token and eos_token not in _VISION_PAD_TOKENS: - new_pad_token = eos_token - if new_pad_token is None: - return tokenizer - - tokenizer.pad_token = new_pad_token - logger.warning( - f"Unsloth: pad_token was a vision token ({pad_token}) on a text-only " - f"model. Replaced with {new_pad_token} to avoid NaN losses." - ) - return tokenizer - - def _fix_pad_token(tokenizer): """Heal a bad/missing pad_token before chat-template repair. - Delegates to unsloth_zoo's shared fix_pad_token (single source of truth) when - available, falling back to the narrow vision-token swap against an older - unsloth_zoo. allow_add=False keeps this side-effect free: there is no model - here to resize embeddings, so a brand new pad token is never added - the later - model-aware patch_tokenizer call finishes the job and is idempotent. + Delegates to unsloth_zoo's shared fix_pad_token (single source of truth); against + an older unsloth_zoo without it, this is a no-op (a pad-named token like + <|vision_pad|> is already a valid pad). allow_add=False keeps this side-effect + free: there is no model here to resize embeddings, so a brand new pad token is + never added - the later model-aware patch_tokenizer call finishes the job and is + idempotent. """ if tokenizer is None: return tokenizer try: from unsloth_zoo.pad_token import fix_pad_token except Exception: - return _fix_vision_pad_token(tokenizer) + return tokenizer fix_pad_token(tokenizer, allow_add = False) return tokenizer