From fc3a9a2b40040fd8ef7d23a508a51ec2866a4eed Mon Sep 17 00:00:00 2001 From: Roland Tannous Date: Wed, 18 Mar 2026 10:59:17 +0000 Subject: [PATCH] fix: add sentencepiece/tiktoken to .venv_t5 for tokenizer backends (#4417) --- studio/backend/core/export/worker.py | 57 +++++++++----------- studio/backend/core/inference/worker.py | 57 +++++++++----------- studio/backend/core/training/trainer.py | 19 +++++-- studio/backend/core/training/worker.py | 57 +++++++++----------- studio/backend/utils/transformers_version.py | 7 ++- studio/setup.ps1 | 13 +++++ studio/setup.sh | 3 ++ 7 files changed, 111 insertions(+), 102 deletions(-) diff --git a/studio/backend/core/export/worker.py b/studio/backend/core/export/worker.py index 4f74f662ee..42f8d1e1d3 100644 --- a/studio/backend/core/export/worker.py +++ b/studio/backend/core/export/worker.py @@ -56,39 +56,32 @@ def _activate_transformers_version(model_name: str) -> None: import subprocess as sp os.makedirs(venv_t5, exist_ok = True) - r1 = sp.run( - [ - sys.executable, - "-m", - "pip", - "install", - "--target", - venv_t5, - "--no-deps", - "transformers==5.3.0", - ], - stdout = sp.PIPE, - stderr = sp.STDOUT, - ) - r2 = sp.run( - [ - sys.executable, - "-m", - "pip", - "install", - "--target", - venv_t5, - "--no-deps", - "huggingface_hub==1.3.0", - ], - stdout = sp.PIPE, - stderr = sp.STDOUT, - ) - if r1.returncode != 0 or r2.returncode != 0: - raise RuntimeError( - f"Failed to install transformers 5.x into {venv_t5}. " - f"pip returncode: transformers={r1.returncode}, huggingface_hub={r2.returncode}" + _t5_packages = [ + "transformers==5.3.0", + "huggingface_hub==1.3.0", + "sentencepiece>=0.2.0", + "tiktoken", + ] + for _pkg in _t5_packages: + _r = sp.run( + [ + sys.executable, + "-m", + "pip", + "install", + "--target", + venv_t5, + "--no-deps", + _pkg, + ], + stdout = sp.PIPE, + stderr = sp.STDOUT, ) + if _r.returncode != 0: + raise RuntimeError( + f"Failed to install {_pkg} into {venv_t5}. " + f"pip returncode: {_r.returncode}" + ) sys.path.insert(0, venv_t5) # Propagate to child subprocesses (e.g. GGUF converter) _pp = os.environ.get("PYTHONPATH", "") diff --git a/studio/backend/core/inference/worker.py b/studio/backend/core/inference/worker.py index 0693908178..47afbb2d1d 100644 --- a/studio/backend/core/inference/worker.py +++ b/studio/backend/core/inference/worker.py @@ -58,39 +58,32 @@ def _activate_transformers_version(model_name: str) -> None: import subprocess as sp os.makedirs(venv_t5, exist_ok = True) - r1 = sp.run( - [ - sys.executable, - "-m", - "pip", - "install", - "--target", - venv_t5, - "--no-deps", - "transformers==5.3.0", - ], - stdout = sp.PIPE, - stderr = sp.STDOUT, - ) - r2 = sp.run( - [ - sys.executable, - "-m", - "pip", - "install", - "--target", - venv_t5, - "--no-deps", - "huggingface_hub==1.3.0", - ], - stdout = sp.PIPE, - stderr = sp.STDOUT, - ) - if r1.returncode != 0 or r2.returncode != 0: - raise RuntimeError( - f"Failed to install transformers 5.x into {venv_t5}. " - f"pip returncode: transformers={r1.returncode}, huggingface_hub={r2.returncode}" + _t5_packages = [ + "transformers==5.3.0", + "huggingface_hub==1.3.0", + "sentencepiece>=0.2.0", + "tiktoken", + ] + for _pkg in _t5_packages: + _r = sp.run( + [ + sys.executable, + "-m", + "pip", + "install", + "--target", + venv_t5, + "--no-deps", + _pkg, + ], + stdout = sp.PIPE, + stderr = sp.STDOUT, ) + if _r.returncode != 0: + raise RuntimeError( + f"Failed to install {_pkg} into {venv_t5}. " + f"pip returncode: {_r.returncode}" + ) sys.path.insert(0, venv_t5) # Propagate to child subprocesses (e.g. GGUF converter) _pp = os.environ.get("PYTHONPATH", "") diff --git a/studio/backend/core/training/trainer.py b/studio/backend/core/training/trainer.py index bc252d1fa5..4a46329955 100644 --- a/studio/backend/core/training/trainer.py +++ b/studio/backend/core/training/trainer.py @@ -188,11 +188,20 @@ class UnslothTrainer: else: from transformers import AutoTokenizer - self.tokenizer = AutoTokenizer.from_pretrained( - model_name, - trust_remote_code = trust_remote_code, - token = hf_token, - ) + try: + self.tokenizer = AutoTokenizer.from_pretrained( + model_name, + trust_remote_code = trust_remote_code, + token = hf_token, + ) + except ValueError as exc: + if "sentencepiece" in str(exc) or "tiktoken" in str(exc): + raise ValueError( + f"Tokenizer for '{model_name}' requires sentencepiece or tiktoken. " + f"Please run: pip install sentencepiece tiktoken\n" + f"Original error: {exc}" + ) from exc + raise logger.info("Pre-loaded tokenizer for %s", model_name) diff --git a/studio/backend/core/training/worker.py b/studio/backend/core/training/worker.py index 284fcf228d..77b7507b72 100644 --- a/studio/backend/core/training/worker.py +++ b/studio/backend/core/training/worker.py @@ -52,39 +52,32 @@ def _activate_transformers_version(model_name: str) -> None: import subprocess as sp os.makedirs(venv_t5, exist_ok = True) - r1 = sp.run( - [ - sys.executable, - "-m", - "pip", - "install", - "--target", - venv_t5, - "--no-deps", - "transformers==5.3.0", - ], - stdout = sp.PIPE, - stderr = sp.STDOUT, - ) - r2 = sp.run( - [ - sys.executable, - "-m", - "pip", - "install", - "--target", - venv_t5, - "--no-deps", - "huggingface_hub==1.3.0", - ], - stdout = sp.PIPE, - stderr = sp.STDOUT, - ) - if r1.returncode != 0 or r2.returncode != 0: - raise RuntimeError( - f"Failed to install transformers 5.x into {venv_t5}. " - f"pip returncode: transformers={r1.returncode}, huggingface_hub={r2.returncode}" + _t5_packages = [ + "transformers==5.3.0", + "huggingface_hub==1.3.0", + "sentencepiece>=0.2.0", + "tiktoken", + ] + for _pkg in _t5_packages: + _r = sp.run( + [ + sys.executable, + "-m", + "pip", + "install", + "--target", + venv_t5, + "--no-deps", + _pkg, + ], + stdout = sp.PIPE, + stderr = sp.STDOUT, ) + if _r.returncode != 0: + raise RuntimeError( + f"Failed to install {_pkg} into {venv_t5}. " + f"pip returncode: {_r.returncode}" + ) sys.path.insert(0, venv_t5) # Propagate to child subprocesses (e.g. GGUF converter) _pp = os.environ.get("PYTHONPATH", "") diff --git a/studio/backend/utils/transformers_version.py b/studio/backend/utils/transformers_version.py index 5666b3be35..591ec23bc0 100644 --- a/studio/backend/utils/transformers_version.py +++ b/studio/backend/utils/transformers_version.py @@ -223,7 +223,12 @@ def _ensure_venv_t5_exists() -> bool: logger.warning(".venv_t5 not found at %s — installing at runtime", _VENV_T5_DIR) os.makedirs(_VENV_T5_DIR, exist_ok = True) - for pkg in (f"transformers=={TRANSFORMERS_5_VERSION}", "huggingface_hub==1.3.0"): + for pkg in ( + f"transformers=={TRANSFORMERS_5_VERSION}", + "huggingface_hub==1.3.0", + "sentencepiece>=0.2.0", + "tiktoken", + ): cmd = [ sys.executable, "-m", diff --git a/studio/setup.ps1 b/studio/setup.ps1 index 2420448deb..87122c041d 100644 --- a/studio/setup.ps1 +++ b/studio/setup.ps1 @@ -918,6 +918,19 @@ if ($LASTEXITCODE -ne 0) { $ErrorActionPreference = $prevEAP_t5 exit 1 } +# Tokenizer backends — required by AutoTokenizer for many model families +pip install --target $VenvT5Dir --no-deps "sentencepiece>=0.2.0" 2>&1 | Out-Null +if ($LASTEXITCODE -ne 0) { + Write-Host "[FAIL] Could not install sentencepiece into .venv_t5/" -ForegroundColor Red + $ErrorActionPreference = $prevEAP_t5 + exit 1 +} +pip install --target $VenvT5Dir --no-deps "tiktoken" 2>&1 | Out-Null +if ($LASTEXITCODE -ne 0) { + Write-Host "[FAIL] Could not install tiktoken into .venv_t5/" -ForegroundColor Red + $ErrorActionPreference = $prevEAP_t5 + exit 1 +} $ErrorActionPreference = $prevEAP_t5 Write-Host "[OK] Transformers 5.x pre-installed to .venv_t5/" -ForegroundColor Green diff --git a/studio/setup.sh b/studio/setup.sh index 4c8a6c7dde..3afa40ffaf 100755 --- a/studio/setup.sh +++ b/studio/setup.sh @@ -254,6 +254,9 @@ else mkdir -p "$VENV_T5_DIR" run_quiet "pip install transformers 5.x" pip install --target "$VENV_T5_DIR" --no-deps "transformers==5.3.0" run_quiet "pip install huggingface_hub for t5" pip install --target "$VENV_T5_DIR" --no-deps "huggingface_hub==1.3.0" + # Tokenizer backends — required by AutoTokenizer for many model families + run_quiet "pip install sentencepiece for t5" pip install --target "$VENV_T5_DIR" --no-deps "sentencepiece>=0.2.0" + run_quiet "pip install tiktoken for t5" pip install --target "$VENV_T5_DIR" --no-deps "tiktoken" echo "✅ Transformers 5.x pre-installed to $VENV_T5_DIR/" # ── 7. WSL: pre-install GGUF build dependencies ──