Keep pad-named pad_tokens (e.g. <|vision_pad|>); fix Qwen3-Base load crash (#6652)

* Keep pad-named pad_tokens; defer pad repair to shared unsloth_zoo.pad_token

A pad-named token (e.g. <|vision_pad|>) is a valid pad. The narrow fallback that
stripped vision pad tokens on text-only models is now a no-op; the active path
delegates to the shared fix_pad_token in unsloth_zoo, which keeps pad-named tokens
and only heals missing / eos-collision / out-of-range pads.

This fixes the Qwen3-4B-Base load crash (its config ships pad_token=<|vision_pad|>):
the old swap could not find a safe text pad (eos is <|endoftext|>, no unk_token) and
left the tokenizer broken. Removes the unused _VISION_PAD_TOKENS / _SAFE_TEXT_PAD_TOKENS
sets. Tests updated.

Pairs with unslothai/unsloth-zoo#831.

* Remove _fix_vision_pad_token; inline the no-op fallback

A pad-named token (e.g. <|vision_pad|>) is a valid pad, so the old vision-pad swap
helper has no purpose. _fix_pad_token now returns the tokenizer unchanged when the
shared unsloth_zoo.pad_token module is unavailable, instead of routing through a
no-op helper. Test WANTED set updated.

tests/python/test_pad_token_fix.py

@@ -1,9 +1,9 @@
 """_fix_pad_token dispatch in unsloth/tokenizer_utils.py.
 
 It must delegate to unsloth_zoo's shared fix_pad_token when present (single
-source of truth), and fall back to the narrow vision-token swap against an
-older unsloth_zoo. Static + CPU-only: the two helpers are exec'd in isolation
-so the test never imports torch / transformers / unsloth.
+source of truth), and fall back to a no-op against an older unsloth_zoo. Static
++ CPU-only: _fix_pad_token is exec'd in isolation so the test never imports
+torch / transformers / unsloth.
 """
 
 import ast
@@ -15,9 +15,6 @@ REPO_ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", ".."))
 TOK_PATH = os.path.join(REPO_ROOT, "unsloth", "tokenizer_utils.py")
 
 WANTED = {
-    "_VISION_PAD_TOKENS",
-    "_SAFE_TEXT_PAD_TOKENS",
-    "_fix_vision_pad_token",
     "_fix_pad_token",
 }
 
@@ -69,17 +66,17 @@ def test_fix_pad_token_none_is_noop():
     assert ns["_fix_pad_token"](None) is None
 
 
-def test_fix_pad_token_falls_back_without_shared_module(monkeypatch):
+def test_fallback_keeps_pad_named_token(monkeypatch):
     ns = _load_pad_helpers()
     _block_shared_module(monkeypatch)
-    # Qwen3 text tokenizer shipping a vision pad_token -> narrow swap heals it.
+    # A pad-named token (e.g. <|vision_pad|>) is a valid pad -> fallback keeps it.
     tok = FakeTok(
         {"<|endoftext|>": 1, "<|im_end|>": 2, "<|vision_pad|>": 3},
         pad = "<|vision_pad|>",
         eos = "<|im_end|>",
     )
     ns["_fix_pad_token"](tok)
-    assert tok.pad_token == "<|endoftext|>"
+    assert tok.pad_token == "<|vision_pad|>"
     assert tok.pad_token != tok.eos_token
 
 

unsloth/tokenizer_utils.py

@@ -626,72 +626,22 @@ def _load_correct_tokenizer(
         return fast_tokenizer
 
 
-# Qwen3 text models share Qwen3-VL's vocab, so configs ship a vision pad_token;
-# padding text-only training with one yields NaN losses (#3155, #4104).
-_VISION_PAD_TOKENS = frozenset(
-    (
-        "<|vision_pad|>",
-        "<|image_pad|>",
-        "<|video_pad|>",
-        "<|audio_pad|>",
-    )
-)
-# Preference order; <unk> excluded since reusing it as pad masks real OOV tokens.
-_SAFE_TEXT_PAD_TOKENS = ("<|endoftext|>", "<pad>", "[PAD]")
-
-
-def _fix_vision_pad_token(tokenizer):
-    """Swap a vision pad_token on a text-only tokenizer for a safe text token (#3155)."""
-    if tokenizer is None:
-        return tokenizer
-    if hasattr(tokenizer, "image_processor"):
-        return tokenizer
-    pad_token = getattr(tokenizer, "pad_token", None)
-    if pad_token is None or pad_token not in _VISION_PAD_TOKENS:
-        return tokenizer
-
-    get_vocab = getattr(tokenizer, "get_vocab", None)
-    if get_vocab is None:
-        return tokenizer
-    vocab = get_vocab()
-    if not isinstance(vocab, dict):
-        return tokenizer
-    new_pad_token = None
-    for candidate in _SAFE_TEXT_PAD_TOKENS:
-        if candidate in vocab and candidate != getattr(tokenizer, "eos_token", None):
-            new_pad_token = candidate
-            break
-    # Fall back to eos_token only if it is a distinct, non-vision token.
-    if new_pad_token is None:
-        eos_token = getattr(tokenizer, "eos_token", None)
-        if eos_token is not None and eos_token != pad_token and eos_token not in _VISION_PAD_TOKENS:
-            new_pad_token = eos_token
-    if new_pad_token is None:
-        return tokenizer
-
-    tokenizer.pad_token = new_pad_token
-    logger.warning(
-        f"Unsloth: pad_token was a vision token ({pad_token}) on a text-only "
-        f"model. Replaced with {new_pad_token} to avoid NaN losses."
-    )
-    return tokenizer
-
-
 def _fix_pad_token(tokenizer):
     """Heal a bad/missing pad_token before chat-template repair.
 
-    Delegates to unsloth_zoo's shared fix_pad_token (single source of truth) when
-    available, falling back to the narrow vision-token swap against an older
-    unsloth_zoo. allow_add=False keeps this side-effect free: there is no model
-    here to resize embeddings, so a brand new pad token is never added - the later
-    model-aware patch_tokenizer call finishes the job and is idempotent.
+    Delegates to unsloth_zoo's shared fix_pad_token (single source of truth); against
+    an older unsloth_zoo without it, this is a no-op (a pad-named token like
+    <|vision_pad|> is already a valid pad). allow_add=False keeps this side-effect
+    free: there is no model here to resize embeddings, so a brand new pad token is
+    never added - the later model-aware patch_tokenizer call finishes the job and is
+    idempotent.
     """
     if tokenizer is None:
         return tokenizer
     try:
         from unsloth_zoo.pad_token import fix_pad_token
     except Exception:
-        return _fix_vision_pad_token(tokenizer)
+        return tokenizer
     fix_pad_token(tokenizer, allow_add = False)
     return tokenizer