fix(studio): add tiktoken to .venv_t5 for Qwen tokenizers
Qwen-family models (Qwen3.5, etc.) use a tiktoken-based tokenizer. When Studio routes these models through the transformers 5.x overlay (.venv_t5), the tokenizer instantiation fails with: ValueError: Couldn't instantiate the backend tokenizer... You need to have sentencepiece or tiktoken installed This happens because .venv_t5 was set up with only transformers and huggingface_hub (both --no-deps). The main venv usually has tiktoken, but on Windows it sometimes does not. Fix: install tiktoken into .venv_t5 in both setup.sh and the runtime fallback (_ensure_venv_t5_exists). Uses --no-deps in the runtime path since regex/requests are always present in the main venv.
This commit is contained in:
parent
65acefd2a6
commit
798e73e770
2 changed files with 6 additions and 1 deletions
|
|
@ -223,7 +223,7 @@ def _ensure_venv_t5_exists() -> bool:
|
|||
|
||||
logger.warning(".venv_t5 not found at %s — installing at runtime", _VENV_T5_DIR)
|
||||
os.makedirs(_VENV_T5_DIR, exist_ok = True)
|
||||
for pkg in (f"transformers=={TRANSFORMERS_5_VERSION}", "huggingface_hub==1.3.0"):
|
||||
for pkg in (f"transformers=={TRANSFORMERS_5_VERSION}", "huggingface_hub==1.3.0", "tiktoken"):
|
||||
cmd = [
|
||||
sys.executable,
|
||||
"-m",
|
||||
|
|
|
|||
|
|
@ -254,6 +254,11 @@ else
|
|||
mkdir -p "$VENV_T5_DIR"
|
||||
run_quiet "pip install transformers 5.x" pip install --target "$VENV_T5_DIR" --no-deps "transformers==5.3.0"
|
||||
run_quiet "pip install huggingface_hub for t5" pip install --target "$VENV_T5_DIR" --no-deps "huggingface_hub==1.3.0"
|
||||
# tiktoken is needed by Qwen-family tokenizers (and others). The main venv
|
||||
# may already have it, but on Windows the main venv sometimes doesn't, and
|
||||
# the tokenizer loads while .venv_t5 is prepended to sys.path. Install it
|
||||
# here so it is always available.
|
||||
run_quiet "pip install tiktoken for t5" pip install --target "$VENV_T5_DIR" "tiktoken"
|
||||
echo "✅ Transformers 5.x pre-installed to $VENV_T5_DIR/"
|
||||
|
||||
# ── 7. WSL: pre-install GGUF build dependencies ──
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue