fix: patch TokenizersBackend by model name - Qwen3.5→Qwen2Tokenizer, GLM→PreTrainedTokenizer

studio/backend/core/export/export.py

@@ -153,6 +153,7 @@ class ExportBackend:
 
             # Check if it's a LoRA adapter
             adapter_config = checkpoint_path_obj / "adapter_config.json"
+            base_model = None
             if adapter_config.exists():
                 # It's a LoRA - get base model to check vision
                 base_model = get_base_model_from_lora(checkpoint_path)
@@ -164,6 +165,20 @@ class ExportBackend:
                 # Check the model itself
                 self.is_vision = is_vision_model(checkpoint_path)
 
+            # Resolve model name for tokenizer patching (base model for LoRA, path otherwise)
+            resolved_model_name = base_model or checkpoint_path
+
+            # Patch broken tokenizer_config.json on disk before loading.
+            # Qwen3.5/GLM checkpoints saved by TRL inherit "TokenizersBackend"
+            # from the HF upload — fix it so from_pretrained loads correctly
+            # and subsequent save_pretrained writes the right class.
+            from utils.transformers_version import patch_tokenizer_config
+            patch_tokenizer_config(checkpoint_path, model_name=resolved_model_name)
+            # Also patch subdirectories (TRL saves tokenizer in checkpoint dirs)
+            for subdir in checkpoint_path_obj.iterdir():
+                if subdir.is_dir() and (subdir / "tokenizer_config.json").exists():
+                    patch_tokenizer_config(str(subdir), model_name=resolved_model_name)
+
             # Load model based on type
             if self.is_vision:
                 logger.info("Loading as vision model...")
@@ -183,9 +198,9 @@ class ExportBackend:
                     load_in_4bit=load_in_4bit,
                 )
 
-            # Patch broken tokenizer_class (e.g. Qwen3.5 "TokenizersBackend")
+            # Patch broken tokenizer_class (e.g. Qwen3.5/GLM "TokenizersBackend")
             from utils.transformers_version import patch_tokenizer_in_memory
-            patch_tokenizer_in_memory(tokenizer, model_name=checkpoint_path)
+            patch_tokenizer_in_memory(tokenizer, model_name=resolved_model_name)
 
             # Check if PEFT model
             self.is_peft = isinstance(model, (PeftModel, PeftModelForCausalLM))

studio/backend/core/inference/inference.py

@@ -151,9 +151,9 @@ class InferenceBackend:
                     token=hf_token if hf_token and hf_token.strip() else None,
                 )
 
-                # Patch broken tokenizer_class (Qwen3.5 "TokenizersBackend")
+                # Patch broken tokenizer_class (Qwen3.5/GLM "TokenizersBackend")
                 from utils.transformers_version import patch_tokenizer_in_memory
-                patch_tokenizer_in_memory(tokenizer, model_name=config.path)
+                patch_tokenizer_in_memory(tokenizer, model_name=model_name)
 
                 # Apply inference optimization
                 FastLanguageModel.for_inference(model)

studio/backend/core/training/trainer.py

@@ -193,7 +193,7 @@ class UnslothTrainer:
                     load_in_4bit=load_in_4bit,
                     token=hf_token,
                 )
-                # Patch broken tokenizer_class (Qwen3.5 "TokenizersBackend")
+                # Patch broken tokenizer_class (Qwen3.5/GLM "TokenizersBackend")
                 from utils.transformers_version import patch_tokenizer_in_memory
                 patch_tokenizer_in_memory(self.tokenizer, model_name=model_name)
                 logger.info("Loaded text model")
@@ -1074,6 +1074,8 @@ class UnslothTrainer:
                 self.trainer.save_model()
                 self.tokenizer.save_pretrained(output_dir)
                 self._patch_adapter_config(output_dir)
+                # Fix broken tokenizer_class on saved checkpoints
+                self._patch_tokenizer_class_all(output_dir)
                 print(f"\nTraining stopped. Model saved to {output_dir}\n")
                 self._update_progress(
                     is_training=False,
@@ -1091,6 +1093,8 @@ class UnslothTrainer:
                 self.trainer.save_model()
                 self.tokenizer.save_pretrained(output_dir)
                 self._patch_adapter_config(output_dir)
+                # Fix broken tokenizer_class on saved checkpoints
+                self._patch_tokenizer_class_all(output_dir)
                 print(f"\nTraining completed! Model saved to {output_dir}\n")
                 self._update_progress(
                     is_training=False,
@@ -1135,6 +1139,13 @@ class UnslothTrainer:
         except Exception as e:
             logger.warning(f"Failed to patch adapter_config.json: {e}")
 
+    def _patch_tokenizer_class_all(self, output_dir: str):
+        """Patch broken tokenizer_class in output dir and all checkpoint subdirs."""
+        from utils.transformers_version import patch_tokenizer_config
+        import glob
+        for f in glob.glob(os.path.join(output_dir, "**", "tokenizer_config.json"), recursive=True):
+            patch_tokenizer_config(os.path.dirname(f), model_name=self.model_name)
+
     def stop_training(self, save: bool = True):
         """Stop ongoing training"""
         print(f"\nStopping training (save={save})...")

studio/backend/utils/transformers_version.py

@@ -221,23 +221,39 @@ def _deactivate_5x() -> None:
 # Tokenizer patches
 # ---------------------------------------------------------------------------
 
-# Some HF model uploads (e.g. Qwen3.5 family) ship with a broken
-# tokenizer_class value "TokenizersBackend" instead of the real class.
-# This causes llama.cpp's GGUF converter (and other tools) to fail.
-_TOKENIZER_CLASS_FIXES: dict[str, str] = {
-    "TokenizersBackend": "Qwen2Tokenizer",
+# Some HF model uploads ship with tokenizer_class "TokenizersBackend"
+# instead of the real class. This causes llama.cpp's GGUF converter to fail.
+# Map: lowered model substring → correct tokenizer_class.
+_TOKENIZER_CLASS_OVERRIDES: dict[str, str] = {
+    "qwen3.5": "Qwen2Tokenizer",
+    "glm-4.7": "PreTrainedTokenizer",
 }
 
 
+def _get_tokenizer_class_fix(model_name: str) -> str | None:
+    """Return the correct tokenizer_class for a model, or None if no fix needed."""
+    lowered = model_name.lower()
+    for substr, fixed_class in _TOKENIZER_CLASS_OVERRIDES.items():
+        if substr in lowered:
+            return fixed_class
+    return None
+
+
 def patch_tokenizer_config(model_dir: str, model_name: str = "") -> bool:
     """Fix known broken tokenizer_class values in tokenizer_config.json.
 
-    Only applies to Qwen3.5 models which ship with the wrong
-    tokenizer_class "TokenizersBackend" on HuggingFace.
+    Some HF uploads (Qwen3.5, GLM-4.7-Flash) ship with
+    tokenizer_class "TokenizersBackend" which breaks GGUF conversion.
+    Modifies the file in-place. Requires model_name to determine the
+    correct replacement class.
 
-    Modifies the file in-place. Returns True if a patch was applied.
+    Returns True if a patch was applied.
     """
-    if model_name and "qwen3.5" not in model_name.lower():
+    if not model_name:
+        return False
+
+    fixed_class = _get_tokenizer_class_fix(model_name)
+    if not fixed_class:
         return False
 
     config_path = os.path.join(model_dir, "tokenizer_config.json")
@@ -249,13 +265,12 @@ def patch_tokenizer_config(model_dir: str, model_name: str = "") -> bool:
             config = json.load(f)
 
         tok_class = config.get("tokenizer_class", "")
-        if tok_class in _TOKENIZER_CLASS_FIXES:
-            fixed = _TOKENIZER_CLASS_FIXES[tok_class]
+        if tok_class == "TokenizersBackend":
             logger.warning(
                 "Patching tokenizer_class: '%s' → '%s' in %s",
-                tok_class, fixed, config_path,
+                tok_class, fixed_class, config_path,
             )
-            config["tokenizer_class"] = fixed
+            config["tokenizer_class"] = fixed_class
             with open(config_path, "w") as f:
                 json.dump(config, f, indent=2, ensure_ascii=False)
             return True
@@ -268,25 +283,29 @@ def patch_tokenizer_config(model_dir: str, model_name: str = "") -> bool:
 def patch_tokenizer_in_memory(tokenizer, model_name: str = "") -> bool:
     """Fix known broken tokenizer_class on an in-memory tokenizer object.
 
-    Only applies to Qwen3.5 models which ship with the wrong
-    tokenizer_class "TokenizersBackend" on HuggingFace. Patches it so
-    that save_pretrained() writes a corrected tokenizer_config.json.
+    Some HF uploads (Qwen3.5, GLM-4.7-Flash) ship with
+    tokenizer_class "TokenizersBackend". Patches init_kwargs so that
+    save_pretrained() writes a corrected tokenizer_config.json.
+    Requires model_name to determine the correct replacement class.
 
     Returns True if a patch was applied.
     """
-    if model_name and "qwen3.5" not in model_name.lower():
+    if not model_name:
+        return False
+
+    fixed_class = _get_tokenizer_class_fix(model_name)
+    if not fixed_class:
         return False
 
     try:
         init_kwargs = getattr(tokenizer, "init_kwargs", None) or {}
         tok_class = init_kwargs.get("tokenizer_class", "")
-        if tok_class in _TOKENIZER_CLASS_FIXES:
-            fixed = _TOKENIZER_CLASS_FIXES[tok_class]
+        if tok_class == "TokenizersBackend":
             logger.warning(
                 "Patching in-memory tokenizer_class: '%s' → '%s'",
-                tok_class, fixed,
+                tok_class, fixed_class,
             )
-            tokenizer.init_kwargs["tokenizer_class"] = fixed
+            tokenizer.init_kwargs["tokenizer_class"] = fixed_class
             return True
     except Exception as exc:
         logger.warning("Could not patch in-memory tokenizer: %s", exc)