From 961b814c0daee544f8ddcb4f6487ba5c741e4e32 Mon Sep 17 00:00:00 2001
From: Daniel Han <danielhanchen@gmail.com>
Date: Sun, 28 Jun 2026 03:09:32 +0000
Subject: [PATCH] Handle null and populated extra_special_tokens

Coerce extra_special_tokens null to {} quietly (vanilla transformers also
crashes on null via .keys(), so it must be handled, but it is not a malformed
array worth warning about). For a populated list, log the dropped entries so
the coercion is not silent.
---
 studio/backend/tests/test_tokenizer_compat.py | 10 ++++++
 studio/backend/utils/tokenizer_compat.py      | 31 ++++++++++++-------
 2 files changed, 30 insertions(+), 11 deletions(-)

diff --git a/studio/backend/tests/test_tokenizer_compat.py b/studio/backend/tests/test_tokenizer_compat.py
index e08e1ffcd2..914b8c4789 100644
--- a/studio/backend/tests/test_tokenizer_compat.py
+++ b/studio/backend/tests/test_tokenizer_compat.py
@@ -91,7 +91,17 @@ def test_patched_coerces_non_empty_list(recording_logger):
     d = _make_mixin()
     d._set_model_specific_special_tokens(["<extra>"])  # must not raise
     assert d.extra_special_tokens == {}
+    # Dropped entries are named in the warning so the loss is not silent.
     assert len(recording_logger.warnings) == 1
+    assert "<extra>" in recording_logger.warnings[0]
+
+
+def test_patched_coerces_none_quietly(recording_logger):
+    install_extra_special_tokens_compat()
+    d = _make_mixin()
+    d._set_model_specific_special_tokens(None)  # null == absent; must not raise
+    assert d.extra_special_tokens == {}
+    assert recording_logger.warnings == []  # no false-positive warning for null
 
 
 def test_patched_preserves_valid_dict(recording_logger):
diff --git a/studio/backend/utils/tokenizer_compat.py b/studio/backend/utils/tokenizer_compat.py
index 3583db7b33..bdcef7d5c1 100644
--- a/studio/backend/utils/tokenizer_compat.py
+++ b/studio/backend/utils/tokenizer_compat.py
@@ -36,18 +36,27 @@ def install_extra_special_tokens_compat() -> bool:
         return True
 
     def _patched(self, special_tokens):
-        if not isinstance(special_tokens, dict):
-            logger.warning(
-                "Coercing malformed extra_special_tokens (%s) to {}; "
-                "tokenizer_config.json should use an object, not an array.",
-                type(special_tokens).__name__,
+        if isinstance(special_tokens, dict):
+            return orig(self, special_tokens)
+        # A non-dict value (a JSON array, or null) crashes vanilla transformers on
+        # .keys(); coerce to {} so the model still loads. null is treated as absent;
+        # for a populated list we log the dropped entries so the loss is not silent.
+        if special_tokens is not None:
+            entries = (
+                list(special_tokens)
+                if isinstance(special_tokens, (list, tuple, set))
+                else special_tokens
             )
-            special_tokens = {}
-            try:
-                self.extra_special_tokens = {}
-            except Exception:
-                pass
-        return orig(self, special_tokens)
+            logger.warning(
+                "Coercing malformed extra_special_tokens to {} (%s=%r); "
+                "tokenizer_config.json should use an object, not an array.",
+                type(special_tokens).__name__, entries,
+            )
+        try:
+            self.extra_special_tokens = {}
+        except Exception:
+            pass
+        return orig(self, {})
 
     _patched.__wrapped__ = orig  # keep original reachable
     mixin._set_model_specific_special_tokens = _patched