From e25705a211cfe5b33457790f81d973e08861d49f Mon Sep 17 00:00:00 2001 From: Roland Tannous Date: Sat, 7 Mar 2026 11:28:24 +0000 Subject: [PATCH] fix: propagate PYTHONPATH to child subprocesses, revert tokenizer patching --- studio/backend/core/export/export.py | 57 ------------ studio/backend/core/export/worker.py | 4 + studio/backend/core/inference/inference.py | 4 - studio/backend/core/inference/worker.py | 3 + studio/backend/core/training/trainer.py | 14 --- studio/backend/core/training/worker.py | 3 + studio/backend/utils/transformers_version.py | 95 -------------------- 7 files changed, 10 insertions(+), 170 deletions(-) diff --git a/studio/backend/core/export/export.py b/studio/backend/core/export/export.py index 6899eb949e..c0f53f2e0e 100644 --- a/studio/backend/core/export/export.py +++ b/studio/backend/core/export/export.py @@ -96,7 +96,6 @@ class ExportBackend: self.current_tokenizer = None self.is_vision = False self.is_peft = False - self._resolved_model_name = "" def cleanup_memory(self): """Offload and delete all models from memory""" @@ -154,7 +153,6 @@ class ExportBackend: # Check if it's a LoRA adapter adapter_config = checkpoint_path_obj / "adapter_config.json" - base_model = None if adapter_config.exists(): # It's a LoRA - get base model to check vision base_model = get_base_model_from_lora(checkpoint_path) @@ -166,21 +164,6 @@ class ExportBackend: # Check the model itself self.is_vision = is_vision_model(checkpoint_path) - # Resolve model name for tokenizer patching (base model for LoRA, path otherwise) - resolved_model_name = base_model or checkpoint_path - self._resolved_model_name = resolved_model_name - - # Patch broken tokenizer_config.json on disk before loading. - # Qwen3.5/GLM checkpoints saved by TRL inherit "TokenizersBackend" - # from the HF upload — fix it so from_pretrained loads correctly - # and subsequent save_pretrained writes the right class. - from utils.transformers_version import patch_tokenizer_config - patch_tokenizer_config(checkpoint_path, model_name=resolved_model_name) - # Also patch subdirectories (TRL saves tokenizer in checkpoint dirs) - for subdir in checkpoint_path_obj.iterdir(): - if subdir.is_dir() and (subdir / "tokenizer_config.json").exists(): - patch_tokenizer_config(str(subdir), model_name=resolved_model_name) - # Load model based on type if self.is_vision: logger.info("Loading as vision model...") @@ -200,27 +183,6 @@ class ExportBackend: load_in_4bit=load_in_4bit, ) - # Patch broken tokenizer_class (e.g. Qwen3.5/GLM "TokenizersBackend") - from utils.transformers_version import patch_tokenizer_in_memory - patch_tokenizer_in_memory(tokenizer, model_name=resolved_model_name) - - # Wrap tokenizer.save_pretrained so that every subsequent call - # (including internal ones from save_pretrained_merged / - # save_pretrained_gguf) auto-patches the on-disk output. - # Without this, the GGUF converter subprocess fails because - # save_pretrained re-writes "TokenizersBackend" to the output dir. - _orig_tok_save = tokenizer.save_pretrained - _fix_model_name = resolved_model_name - - def _save_and_patch(*args, **kwargs): - result = _orig_tok_save(*args, **kwargs) - _dir = args[0] if args else kwargs.get("save_directory") - if _dir: - patch_tokenizer_config(str(_dir), model_name=_fix_model_name) - return result - - tokenizer.save_pretrained = _save_and_patch - # Check if PEFT model self.is_peft = isinstance(model, (PeftModel, PeftModelForCausalLM)) @@ -298,10 +260,6 @@ class ExportBackend: save_method=save_method ) - # Fix broken tokenizer_class on disk (belt-and-suspenders) - from utils.transformers_version import patch_tokenizer_config - patch_tokenizer_config(save_directory, model_name=self._resolved_model_name) - # Write export metadata so the Chat page can identify the base model self._write_export_metadata(save_directory) logger.info(f"Model saved successfully to {save_directory}") @@ -358,10 +316,6 @@ class ExportBackend: self.current_model.save_pretrained(save_directory) self.current_tokenizer.save_pretrained(save_directory) - # Fix broken tokenizer_class on disk (belt-and-suspenders) - from utils.transformers_version import patch_tokenizer_config - patch_tokenizer_config(save_directory, model_name=self._resolved_model_name) - # Write export metadata so the Chat page can identify the base model self._write_export_metadata(save_directory) logger.info(f"Model saved successfully to {save_directory}") @@ -473,12 +427,6 @@ class ExportBackend: quantization_method=quant_method ) - # Fix broken tokenizer_class in intermediate HF output - # (save_pretrained wrapper handles pre-converter, this is - # belt-and-suspenders for the final on-disk state) - from utils.transformers_version import patch_tokenizer_config - patch_tokenizer_config(model_save_path, model_name=self._resolved_model_name) - # Relocate GGUF artifacts into the export directory. # convert_to_gguf writes .gguf files to cwd (repo root) # because --outfile is a relative path like "model.Q4_K_M.gguf". @@ -563,11 +511,6 @@ class ExportBackend: self.current_model.save_pretrained(save_directory) self.current_tokenizer.save_pretrained(save_directory) - - # Fix broken tokenizer_class on disk (belt-and-suspenders) - from utils.transformers_version import patch_tokenizer_config - patch_tokenizer_config(save_directory, model_name=self._resolved_model_name) - logger.info(f"Adapter saved successfully to {save_directory}") # Push to hub if requested diff --git a/studio/backend/core/export/worker.py b/studio/backend/core/export/worker.py index 2b36dbeede..adc197c335 100644 --- a/studio/backend/core/export/worker.py +++ b/studio/backend/core/export/worker.py @@ -43,6 +43,7 @@ def _activate_transformers_version(model_name: str, project_root: str) -> None: sys.path.insert(0, venv_t5) logger.info("Activated transformers 5.x from %s", venv_t5) else: + # Fallback: pip install at runtime (slower, ~10-15s) logger.warning(".venv_t5 not found at %s — installing at runtime", venv_t5) import subprocess as sp @@ -63,6 +64,9 @@ def _activate_transformers_version(model_name: str, project_root: str) -> None: f"pip returncode: transformers={r1.returncode}, huggingface_hub={r2.returncode}" ) sys.path.insert(0, venv_t5) + # Propagate to child subprocesses (e.g. GGUF converter) + _pp = os.environ.get("PYTHONPATH", "") + os.environ["PYTHONPATH"] = venv_t5 + (os.pathsep + _pp if _pp else "") else: logger.info("Using default transformers (4.57.x) for %s", model_name) diff --git a/studio/backend/core/inference/inference.py b/studio/backend/core/inference/inference.py index 5c42b12586..329f5d944b 100644 --- a/studio/backend/core/inference/inference.py +++ b/studio/backend/core/inference/inference.py @@ -151,10 +151,6 @@ class InferenceBackend: token=hf_token if hf_token and hf_token.strip() else None, ) - # Patch broken tokenizer_class (Qwen3.5/GLM "TokenizersBackend") - from utils.transformers_version import patch_tokenizer_in_memory - patch_tokenizer_in_memory(tokenizer, model_name=model_name) - # Apply inference optimization FastLanguageModel.for_inference(model) diff --git a/studio/backend/core/inference/worker.py b/studio/backend/core/inference/worker.py index 4495e4bd8a..04a93b8f2a 100644 --- a/studio/backend/core/inference/worker.py +++ b/studio/backend/core/inference/worker.py @@ -65,6 +65,9 @@ def _activate_transformers_version(model_name: str, project_root: str) -> None: f"pip returncode: transformers={r1.returncode}, huggingface_hub={r2.returncode}" ) sys.path.insert(0, venv_t5) + # Propagate to child subprocesses (e.g. GGUF converter) + _pp = os.environ.get("PYTHONPATH", "") + os.environ["PYTHONPATH"] = venv_t5 + (os.pathsep + _pp if _pp else "") else: logger.info("Using default transformers (4.57.x) for %s", model_name) diff --git a/studio/backend/core/training/trainer.py b/studio/backend/core/training/trainer.py index b396f38bb5..2dd2e6eeeb 100644 --- a/studio/backend/core/training/trainer.py +++ b/studio/backend/core/training/trainer.py @@ -193,9 +193,6 @@ class UnslothTrainer: load_in_4bit=load_in_4bit, token=hf_token, ) - # Patch broken tokenizer_class (Qwen3.5/GLM "TokenizersBackend") - from utils.transformers_version import patch_tokenizer_in_memory - patch_tokenizer_in_memory(self.tokenizer, model_name=model_name) logger.info("Loaded text model") if self.should_stop: @@ -1074,8 +1071,6 @@ class UnslothTrainer: self.trainer.save_model() self.tokenizer.save_pretrained(output_dir) self._patch_adapter_config(output_dir) - # Fix broken tokenizer_class on saved checkpoints - self._patch_tokenizer_class_all(output_dir) print(f"\nTraining stopped. Model saved to {output_dir}\n") self._update_progress( is_training=False, @@ -1093,8 +1088,6 @@ class UnslothTrainer: self.trainer.save_model() self.tokenizer.save_pretrained(output_dir) self._patch_adapter_config(output_dir) - # Fix broken tokenizer_class on saved checkpoints - self._patch_tokenizer_class_all(output_dir) print(f"\nTraining completed! Model saved to {output_dir}\n") self._update_progress( is_training=False, @@ -1139,13 +1132,6 @@ class UnslothTrainer: except Exception as e: logger.warning(f"Failed to patch adapter_config.json: {e}") - def _patch_tokenizer_class_all(self, output_dir: str): - """Patch broken tokenizer_class in output dir and all checkpoint subdirs.""" - from utils.transformers_version import patch_tokenizer_config - import glob - for f in glob.glob(os.path.join(output_dir, "**", "tokenizer_config.json"), recursive=True): - patch_tokenizer_config(os.path.dirname(f), model_name=self.model_name) - def stop_training(self, save: bool = True): """Stop ongoing training""" print(f"\nStopping training (save={save})...") diff --git a/studio/backend/core/training/worker.py b/studio/backend/core/training/worker.py index 321645ea2a..a0ecb8d4e3 100644 --- a/studio/backend/core/training/worker.py +++ b/studio/backend/core/training/worker.py @@ -60,6 +60,9 @@ def _activate_transformers_version(model_name: str, project_root: str) -> None: f"pip returncode: transformers={r1.returncode}, huggingface_hub={r2.returncode}" ) sys.path.insert(0, venv_t5) + # Propagate to child subprocesses (e.g. GGUF converter) + _pp = os.environ.get("PYTHONPATH", "") + os.environ["PYTHONPATH"] = venv_t5 + (os.pathsep + _pp if _pp else "") else: logger.info("Using default transformers (4.57.x) for %s", model_name) diff --git a/studio/backend/utils/transformers_version.py b/studio/backend/utils/transformers_version.py index b829a9d7a1..8efbab46a2 100644 --- a/studio/backend/utils/transformers_version.py +++ b/studio/backend/utils/transformers_version.py @@ -217,101 +217,6 @@ def _deactivate_5x() -> None: logger.info("Reverted to transformers %s", transformers.__version__) -# --------------------------------------------------------------------------- -# Tokenizer patches -# --------------------------------------------------------------------------- - -# Some HF model uploads ship with tokenizer_class "TokenizersBackend" -# instead of the real class. This causes llama.cpp's GGUF converter to fail. -# Map: lowered model substring → correct tokenizer_class. -_TOKENIZER_CLASS_OVERRIDES: dict[str, str] = { - "qwen3.5": "Qwen2Tokenizer", - "glm-4.7": "PreTrainedTokenizer", -} - - -def _get_tokenizer_class_fix(model_name: str) -> str | None: - """Return the correct tokenizer_class for a model, or None if no fix needed.""" - lowered = model_name.lower() - for substr, fixed_class in _TOKENIZER_CLASS_OVERRIDES.items(): - if substr in lowered: - return fixed_class - return None - - -def patch_tokenizer_config(model_dir: str, model_name: str = "") -> bool: - """Fix known broken tokenizer_class values in tokenizer_config.json. - - Some HF uploads (Qwen3.5, GLM-4.7-Flash) ship with - tokenizer_class "TokenizersBackend" which breaks GGUF conversion. - Modifies the file in-place. Requires model_name to determine the - correct replacement class. - - Returns True if a patch was applied. - """ - if not model_name: - return False - - fixed_class = _get_tokenizer_class_fix(model_name) - if not fixed_class: - return False - - config_path = os.path.join(model_dir, "tokenizer_config.json") - if not os.path.isfile(config_path): - return False - - try: - with open(config_path) as f: - config = json.load(f) - - tok_class = config.get("tokenizer_class", "") - if tok_class == "TokenizersBackend": - logger.warning( - "Patching tokenizer_class: '%s' → '%s' in %s", - tok_class, fixed_class, config_path, - ) - config["tokenizer_class"] = fixed_class - with open(config_path, "w") as f: - json.dump(config, f, indent=2, ensure_ascii=False) - return True - except Exception as exc: - logger.warning("Could not patch tokenizer_config.json: %s", exc) - - return False - - -def patch_tokenizer_in_memory(tokenizer, model_name: str = "") -> bool: - """Fix known broken tokenizer_class on an in-memory tokenizer object. - - Some HF uploads (Qwen3.5, GLM-4.7-Flash) ship with - tokenizer_class "TokenizersBackend". Patches init_kwargs so that - save_pretrained() writes a corrected tokenizer_config.json. - Requires model_name to determine the correct replacement class. - - Returns True if a patch was applied. - """ - if not model_name: - return False - - fixed_class = _get_tokenizer_class_fix(model_name) - if not fixed_class: - return False - - try: - init_kwargs = getattr(tokenizer, "init_kwargs", None) or {} - tok_class = init_kwargs.get("tokenizer_class", "") - if tok_class == "TokenizersBackend": - logger.warning( - "Patching in-memory tokenizer_class: '%s' → '%s'", - tok_class, fixed_class, - ) - tokenizer.init_kwargs["tokenizer_class"] = fixed_class - return True - except Exception as exc: - logger.warning("Could not patch in-memory tokenizer: %s", exc) - return False - - def ensure_transformers_version(model_name: str) -> None: """Ensure the correct ``transformers`` version is active for *model_name*.