From 76c78afb8f7495eb3947879ef3bc761e7f296407 Mon Sep 17 00:00:00 2001 From: Roland Tannous Date: Sat, 7 Mar 2026 10:29:59 +0000 Subject: [PATCH] =?UTF-8?q?fix:=20patch=20TokenizersBackend=20by=20model?= =?UTF-8?q?=20name=20-=20Qwen3.5=E2=86=92Qwen2Tokenizer,=20GLM=E2=86=92Pre?= =?UTF-8?q?TrainedTokenizer?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- studio/backend/core/export/export.py | 19 +++++- studio/backend/core/inference/inference.py | 4 +- studio/backend/core/training/trainer.py | 13 ++++- studio/backend/utils/transformers_version.py | 61 +++++++++++++------- 4 files changed, 71 insertions(+), 26 deletions(-) diff --git a/studio/backend/core/export/export.py b/studio/backend/core/export/export.py index 6ec027e444..9d9eff2548 100644 --- a/studio/backend/core/export/export.py +++ b/studio/backend/core/export/export.py @@ -153,6 +153,7 @@ class ExportBackend: # Check if it's a LoRA adapter adapter_config = checkpoint_path_obj / "adapter_config.json" + base_model = None if adapter_config.exists(): # It's a LoRA - get base model to check vision base_model = get_base_model_from_lora(checkpoint_path) @@ -164,6 +165,20 @@ class ExportBackend: # Check the model itself self.is_vision = is_vision_model(checkpoint_path) + # Resolve model name for tokenizer patching (base model for LoRA, path otherwise) + resolved_model_name = base_model or checkpoint_path + + # Patch broken tokenizer_config.json on disk before loading. + # Qwen3.5/GLM checkpoints saved by TRL inherit "TokenizersBackend" + # from the HF upload — fix it so from_pretrained loads correctly + # and subsequent save_pretrained writes the right class. + from utils.transformers_version import patch_tokenizer_config + patch_tokenizer_config(checkpoint_path, model_name=resolved_model_name) + # Also patch subdirectories (TRL saves tokenizer in checkpoint dirs) + for subdir in checkpoint_path_obj.iterdir(): + if subdir.is_dir() and (subdir / "tokenizer_config.json").exists(): + patch_tokenizer_config(str(subdir), model_name=resolved_model_name) + # Load model based on type if self.is_vision: logger.info("Loading as vision model...") @@ -183,9 +198,9 @@ class ExportBackend: load_in_4bit=load_in_4bit, ) - # Patch broken tokenizer_class (e.g. Qwen3.5 "TokenizersBackend") + # Patch broken tokenizer_class (e.g. Qwen3.5/GLM "TokenizersBackend") from utils.transformers_version import patch_tokenizer_in_memory - patch_tokenizer_in_memory(tokenizer, model_name=checkpoint_path) + patch_tokenizer_in_memory(tokenizer, model_name=resolved_model_name) # Check if PEFT model self.is_peft = isinstance(model, (PeftModel, PeftModelForCausalLM)) diff --git a/studio/backend/core/inference/inference.py b/studio/backend/core/inference/inference.py index ccd2bc176e..5c42b12586 100644 --- a/studio/backend/core/inference/inference.py +++ b/studio/backend/core/inference/inference.py @@ -151,9 +151,9 @@ class InferenceBackend: token=hf_token if hf_token and hf_token.strip() else None, ) - # Patch broken tokenizer_class (Qwen3.5 "TokenizersBackend") + # Patch broken tokenizer_class (Qwen3.5/GLM "TokenizersBackend") from utils.transformers_version import patch_tokenizer_in_memory - patch_tokenizer_in_memory(tokenizer, model_name=config.path) + patch_tokenizer_in_memory(tokenizer, model_name=model_name) # Apply inference optimization FastLanguageModel.for_inference(model) diff --git a/studio/backend/core/training/trainer.py b/studio/backend/core/training/trainer.py index edfd3c1ee0..b396f38bb5 100644 --- a/studio/backend/core/training/trainer.py +++ b/studio/backend/core/training/trainer.py @@ -193,7 +193,7 @@ class UnslothTrainer: load_in_4bit=load_in_4bit, token=hf_token, ) - # Patch broken tokenizer_class (Qwen3.5 "TokenizersBackend") + # Patch broken tokenizer_class (Qwen3.5/GLM "TokenizersBackend") from utils.transformers_version import patch_tokenizer_in_memory patch_tokenizer_in_memory(self.tokenizer, model_name=model_name) logger.info("Loaded text model") @@ -1074,6 +1074,8 @@ class UnslothTrainer: self.trainer.save_model() self.tokenizer.save_pretrained(output_dir) self._patch_adapter_config(output_dir) + # Fix broken tokenizer_class on saved checkpoints + self._patch_tokenizer_class_all(output_dir) print(f"\nTraining stopped. Model saved to {output_dir}\n") self._update_progress( is_training=False, @@ -1091,6 +1093,8 @@ class UnslothTrainer: self.trainer.save_model() self.tokenizer.save_pretrained(output_dir) self._patch_adapter_config(output_dir) + # Fix broken tokenizer_class on saved checkpoints + self._patch_tokenizer_class_all(output_dir) print(f"\nTraining completed! Model saved to {output_dir}\n") self._update_progress( is_training=False, @@ -1135,6 +1139,13 @@ class UnslothTrainer: except Exception as e: logger.warning(f"Failed to patch adapter_config.json: {e}") + def _patch_tokenizer_class_all(self, output_dir: str): + """Patch broken tokenizer_class in output dir and all checkpoint subdirs.""" + from utils.transformers_version import patch_tokenizer_config + import glob + for f in glob.glob(os.path.join(output_dir, "**", "tokenizer_config.json"), recursive=True): + patch_tokenizer_config(os.path.dirname(f), model_name=self.model_name) + def stop_training(self, save: bool = True): """Stop ongoing training""" print(f"\nStopping training (save={save})...") diff --git a/studio/backend/utils/transformers_version.py b/studio/backend/utils/transformers_version.py index 2094a2c5b3..b829a9d7a1 100644 --- a/studio/backend/utils/transformers_version.py +++ b/studio/backend/utils/transformers_version.py @@ -221,23 +221,39 @@ def _deactivate_5x() -> None: # Tokenizer patches # --------------------------------------------------------------------------- -# Some HF model uploads (e.g. Qwen3.5 family) ship with a broken -# tokenizer_class value "TokenizersBackend" instead of the real class. -# This causes llama.cpp's GGUF converter (and other tools) to fail. -_TOKENIZER_CLASS_FIXES: dict[str, str] = { - "TokenizersBackend": "Qwen2Tokenizer", +# Some HF model uploads ship with tokenizer_class "TokenizersBackend" +# instead of the real class. This causes llama.cpp's GGUF converter to fail. +# Map: lowered model substring → correct tokenizer_class. +_TOKENIZER_CLASS_OVERRIDES: dict[str, str] = { + "qwen3.5": "Qwen2Tokenizer", + "glm-4.7": "PreTrainedTokenizer", } +def _get_tokenizer_class_fix(model_name: str) -> str | None: + """Return the correct tokenizer_class for a model, or None if no fix needed.""" + lowered = model_name.lower() + for substr, fixed_class in _TOKENIZER_CLASS_OVERRIDES.items(): + if substr in lowered: + return fixed_class + return None + + def patch_tokenizer_config(model_dir: str, model_name: str = "") -> bool: """Fix known broken tokenizer_class values in tokenizer_config.json. - Only applies to Qwen3.5 models which ship with the wrong - tokenizer_class "TokenizersBackend" on HuggingFace. + Some HF uploads (Qwen3.5, GLM-4.7-Flash) ship with + tokenizer_class "TokenizersBackend" which breaks GGUF conversion. + Modifies the file in-place. Requires model_name to determine the + correct replacement class. - Modifies the file in-place. Returns True if a patch was applied. + Returns True if a patch was applied. """ - if model_name and "qwen3.5" not in model_name.lower(): + if not model_name: + return False + + fixed_class = _get_tokenizer_class_fix(model_name) + if not fixed_class: return False config_path = os.path.join(model_dir, "tokenizer_config.json") @@ -249,13 +265,12 @@ def patch_tokenizer_config(model_dir: str, model_name: str = "") -> bool: config = json.load(f) tok_class = config.get("tokenizer_class", "") - if tok_class in _TOKENIZER_CLASS_FIXES: - fixed = _TOKENIZER_CLASS_FIXES[tok_class] + if tok_class == "TokenizersBackend": logger.warning( "Patching tokenizer_class: '%s' → '%s' in %s", - tok_class, fixed, config_path, + tok_class, fixed_class, config_path, ) - config["tokenizer_class"] = fixed + config["tokenizer_class"] = fixed_class with open(config_path, "w") as f: json.dump(config, f, indent=2, ensure_ascii=False) return True @@ -268,25 +283,29 @@ def patch_tokenizer_config(model_dir: str, model_name: str = "") -> bool: def patch_tokenizer_in_memory(tokenizer, model_name: str = "") -> bool: """Fix known broken tokenizer_class on an in-memory tokenizer object. - Only applies to Qwen3.5 models which ship with the wrong - tokenizer_class "TokenizersBackend" on HuggingFace. Patches it so - that save_pretrained() writes a corrected tokenizer_config.json. + Some HF uploads (Qwen3.5, GLM-4.7-Flash) ship with + tokenizer_class "TokenizersBackend". Patches init_kwargs so that + save_pretrained() writes a corrected tokenizer_config.json. + Requires model_name to determine the correct replacement class. Returns True if a patch was applied. """ - if model_name and "qwen3.5" not in model_name.lower(): + if not model_name: + return False + + fixed_class = _get_tokenizer_class_fix(model_name) + if not fixed_class: return False try: init_kwargs = getattr(tokenizer, "init_kwargs", None) or {} tok_class = init_kwargs.get("tokenizer_class", "") - if tok_class in _TOKENIZER_CLASS_FIXES: - fixed = _TOKENIZER_CLASS_FIXES[tok_class] + if tok_class == "TokenizersBackend": logger.warning( "Patching in-memory tokenizer_class: '%s' → '%s'", - tok_class, fixed, + tok_class, fixed_class, ) - tokenizer.init_kwargs["tokenizer_class"] = fixed + tokenizer.init_kwargs["tokenizer_class"] = fixed_class return True except Exception as exc: logger.warning("Could not patch in-memory tokenizer: %s", exc)