ceos/unsloth
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

fix: add sentencepiece/tiktoken to .venv_t5 for tokenizer backends (#4417)

Browse source
This commit is contained in:
Roland Tannous 2026-03-18 10:59:17 +00:00
commit fc3a9a2b40
7 changed files with 111 additions and 102 deletions
Download patch file Download diff file Expand all files Collapse all files

57
studio/backend/core/export/worker.py
View file

@ -56,39 +56,32 @@ def _activate_transformers_version(model_name: str) -> None:
import subprocess as sp
os.makedirs(venv_t5, exist_ok = True)
r1 = sp.run(
[
sys.executable,
"-m",
"pip",
"install",
"--target",
venv_t5,
"--no-deps",
"transformers==5.3.0",
],
stdout = sp.PIPE,
stderr = sp.STDOUT,
)
r2 = sp.run(
[
sys.executable,
"-m",
"pip",
"install",
"--target",
venv_t5,
"--no-deps",
"huggingface_hub==1.3.0",
],
stdout = sp.PIPE,
stderr = sp.STDOUT,
)
if r1.returncode != 0 or r2.returncode != 0:
raise RuntimeError(
f"Failed to install transformers 5.x into {venv_t5}. "
f"pip returncode: transformers={r1.returncode}, huggingface_hub={r2.returncode}"
_t5_packages = [
"transformers==5.3.0",
"huggingface_hub==1.3.0",
"sentencepiece>=0.2.0",
"tiktoken",
]
for _pkg in _t5_packages:
_r = sp.run(
[
sys.executable,
"-m",
"pip",
"install",
"--target",
venv_t5,
"--no-deps",
_pkg,
],
stdout = sp.PIPE,
stderr = sp.STDOUT,
)
if _r.returncode != 0:
raise RuntimeError(
f"Failed to install {_pkg} into {venv_t5}. "
f"pip returncode: {_r.returncode}"
)
sys.path.insert(0, venv_t5)
# Propagate to child subprocesses (e.g. GGUF converter)
_pp = os.environ.get("PYTHONPATH", "")

57
studio/backend/core/inference/worker.py
View file

@ -58,39 +58,32 @@ def _activate_transformers_version(model_name: str) -> None:
import subprocess as sp
os.makedirs(venv_t5, exist_ok = True)
r1 = sp.run(
[
sys.executable,
"-m",
"pip",
"install",
"--target",
venv_t5,
"--no-deps",
"transformers==5.3.0",
],
stdout = sp.PIPE,
stderr = sp.STDOUT,
)
r2 = sp.run(
[
sys.executable,
"-m",
"pip",
"install",
"--target",
venv_t5,
"--no-deps",
"huggingface_hub==1.3.0",
],
stdout = sp.PIPE,
stderr = sp.STDOUT,
)
if r1.returncode != 0 or r2.returncode != 0:
raise RuntimeError(
f"Failed to install transformers 5.x into {venv_t5}. "
f"pip returncode: transformers={r1.returncode}, huggingface_hub={r2.returncode}"
_t5_packages = [
"transformers==5.3.0",
"huggingface_hub==1.3.0",
"sentencepiece>=0.2.0",
"tiktoken",
]
for _pkg in _t5_packages:
_r = sp.run(
[
sys.executable,
"-m",
"pip",
"install",
"--target",
venv_t5,
"--no-deps",
_pkg,
],
stdout = sp.PIPE,
stderr = sp.STDOUT,
)
if _r.returncode != 0:
raise RuntimeError(
f"Failed to install {_pkg} into {venv_t5}. "
f"pip returncode: {_r.returncode}"
)
sys.path.insert(0, venv_t5)
# Propagate to child subprocesses (e.g. GGUF converter)
_pp = os.environ.get("PYTHONPATH", "")

19
studio/backend/core/training/trainer.py
View file

@ -188,11 +188,20 @@ class UnslothTrainer:
else:
from transformers import AutoTokenizer
self.tokenizer = AutoTokenizer.from_pretrained(
model_name,
trust_remote_code = trust_remote_code,
token = hf_token,
)
try:
self.tokenizer = AutoTokenizer.from_pretrained(
model_name,
trust_remote_code = trust_remote_code,
token = hf_token,
)
except ValueError as exc:
if "sentencepiece" in str(exc) or "tiktoken" in str(exc):
raise ValueError(
f"Tokenizer for '{model_name}' requires sentencepiece or tiktoken. "
f"Please run: pip install sentencepiece tiktoken\n"
f"Original error: {exc}"
) from exc
raise
logger.info("Pre-loaded tokenizer for %s", model_name)

57
studio/backend/core/training/worker.py
View file

@ -52,39 +52,32 @@ def _activate_transformers_version(model_name: str) -> None:
import subprocess as sp
os.makedirs(venv_t5, exist_ok = True)
r1 = sp.run(
[
sys.executable,
"-m",
"pip",
"install",
"--target",
venv_t5,
"--no-deps",
"transformers==5.3.0",
],
stdout = sp.PIPE,
stderr = sp.STDOUT,
)
r2 = sp.run(
[
sys.executable,
"-m",
"pip",
"install",
"--target",
venv_t5,
"--no-deps",
"huggingface_hub==1.3.0",
],
stdout = sp.PIPE,
stderr = sp.STDOUT,
)
if r1.returncode != 0 or r2.returncode != 0:
raise RuntimeError(
f"Failed to install transformers 5.x into {venv_t5}. "
f"pip returncode: transformers={r1.returncode}, huggingface_hub={r2.returncode}"
_t5_packages = [
"transformers==5.3.0",
"huggingface_hub==1.3.0",
"sentencepiece>=0.2.0",
"tiktoken",
]
for _pkg in _t5_packages:
_r = sp.run(
[
sys.executable,
"-m",
"pip",
"install",
"--target",
venv_t5,
"--no-deps",
_pkg,
],
stdout = sp.PIPE,
stderr = sp.STDOUT,
)
if _r.returncode != 0:
raise RuntimeError(
f"Failed to install {_pkg} into {venv_t5}. "
f"pip returncode: {_r.returncode}"
)
sys.path.insert(0, venv_t5)
# Propagate to child subprocesses (e.g. GGUF converter)
_pp = os.environ.get("PYTHONPATH", "")

7
studio/backend/utils/transformers_version.py
View file

@ -223,7 +223,12 @@ def _ensure_venv_t5_exists() -> bool:
logger.warning(".venv_t5 not found at %s — installing at runtime", _VENV_T5_DIR)
os.makedirs(_VENV_T5_DIR, exist_ok = True)
for pkg in (f"transformers=={TRANSFORMERS_5_VERSION}", "huggingface_hub==1.3.0"):
for pkg in (
f"transformers=={TRANSFORMERS_5_VERSION}",
"huggingface_hub==1.3.0",
"sentencepiece>=0.2.0",
"tiktoken",
):
cmd = [
sys.executable,
"-m",

13
studio/setup.ps1
View file

@ -918,6 +918,19 @@ if ($LASTEXITCODE -ne 0) {
$ErrorActionPreference = $prevEAP_t5
exit 1
}
# Tokenizer backends — required by AutoTokenizer for many model families
pip install --target $VenvT5Dir --no-deps "sentencepiece>=0.2.0" 2>&1 | Out-Null
if ($LASTEXITCODE -ne 0) {
Write-Host "[FAIL] Could not install sentencepiece into .venv_t5/" -ForegroundColor Red
$ErrorActionPreference = $prevEAP_t5
exit 1
}
pip install --target $VenvT5Dir --no-deps "tiktoken" 2>&1 | Out-Null
if ($LASTEXITCODE -ne 0) {
Write-Host "[FAIL] Could not install tiktoken into .venv_t5/" -ForegroundColor Red
$ErrorActionPreference = $prevEAP_t5
exit 1
}
$ErrorActionPreference = $prevEAP_t5
Write-Host "[OK] Transformers 5.x pre-installed to .venv_t5/" -ForegroundColor Green

3
studio/setup.sh
View file

@ -254,6 +254,9 @@ else
mkdir -p "$VENV_T5_DIR"
run_quiet "pip install transformers 5.x" pip install --target "$VENV_T5_DIR" --no-deps "transformers==5.3.0"
run_quiet "pip install huggingface_hub for t5" pip install --target "$VENV_T5_DIR" --no-deps "huggingface_hub==1.3.0"
# Tokenizer backends — required by AutoTokenizer for many model families
run_quiet "pip install sentencepiece for t5" pip install --target "$VENV_T5_DIR" --no-deps "sentencepiece>=0.2.0"
run_quiet "pip install tiktoken for t5" pip install --target "$VENV_T5_DIR" --no-deps "tiktoken"
echo "✅ Transformers 5.x pre-installed to $VENV_T5_DIR/"
# ── 7. WSL: pre-install GGUF build dependencies ──