From 798e73e770ccc87a4231109b6e4f1bf25b82d0c2 Mon Sep 17 00:00:00 2001 From: Daniel Han <23090290+danielhanchen@users.noreply.github.com> Date: Wed, 18 Mar 2026 10:33:45 +0000 Subject: [PATCH] fix(studio): add tiktoken to .venv_t5 for Qwen tokenizers Qwen-family models (Qwen3.5, etc.) use a tiktoken-based tokenizer. When Studio routes these models through the transformers 5.x overlay (.venv_t5), the tokenizer instantiation fails with: ValueError: Couldn't instantiate the backend tokenizer... You need to have sentencepiece or tiktoken installed This happens because .venv_t5 was set up with only transformers and huggingface_hub (both --no-deps). The main venv usually has tiktoken, but on Windows it sometimes does not. Fix: install tiktoken into .venv_t5 in both setup.sh and the runtime fallback (_ensure_venv_t5_exists). Uses --no-deps in the runtime path since regex/requests are always present in the main venv. --- studio/backend/utils/transformers_version.py | 2 +- studio/setup.sh | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/studio/backend/utils/transformers_version.py b/studio/backend/utils/transformers_version.py index 5666b3be35..78b914eac6 100644 --- a/studio/backend/utils/transformers_version.py +++ b/studio/backend/utils/transformers_version.py @@ -223,7 +223,7 @@ def _ensure_venv_t5_exists() -> bool: logger.warning(".venv_t5 not found at %s — installing at runtime", _VENV_T5_DIR) os.makedirs(_VENV_T5_DIR, exist_ok = True) - for pkg in (f"transformers=={TRANSFORMERS_5_VERSION}", "huggingface_hub==1.3.0"): + for pkg in (f"transformers=={TRANSFORMERS_5_VERSION}", "huggingface_hub==1.3.0", "tiktoken"): cmd = [ sys.executable, "-m", diff --git a/studio/setup.sh b/studio/setup.sh index 4c8a6c7dde..12b91c0bd0 100755 --- a/studio/setup.sh +++ b/studio/setup.sh @@ -254,6 +254,11 @@ else mkdir -p "$VENV_T5_DIR" run_quiet "pip install transformers 5.x" pip install --target "$VENV_T5_DIR" --no-deps "transformers==5.3.0" run_quiet "pip install huggingface_hub for t5" pip install --target "$VENV_T5_DIR" --no-deps "huggingface_hub==1.3.0" + # tiktoken is needed by Qwen-family tokenizers (and others). The main venv + # may already have it, but on Windows the main venv sometimes doesn't, and + # the tokenizer loads while .venv_t5 is prepended to sys.path. Install it + # here so it is always available. + run_quiet "pip install tiktoken for t5" pip install --target "$VENV_T5_DIR" "tiktoken" echo "✅ Transformers 5.x pre-installed to $VENV_T5_DIR/" # ── 7. WSL: pre-install GGUF build dependencies ──