From aa7c0b6d6cf00ebe68f97b0d73988323fa6ab92a Mon Sep 17 00:00:00 2001 From: Roland Tannous Date: Thu, 19 Mar 2026 14:18:00 +0000 Subject: [PATCH] recover working install files --- studio/install_python_stack.py | 12 +- studio/setup.sh | 208 +++++++++------------------------ 2 files changed, 58 insertions(+), 162 deletions(-) diff --git a/studio/install_python_stack.py b/studio/install_python_stack.py index a141c64425..20011a298a 100644 --- a/studio/install_python_stack.py +++ b/studio/install_python_stack.py @@ -140,15 +140,15 @@ def _bootstrap_uv() -> bool: global UV_NEEDS_SYSTEM if not shutil.which("uv"): return False - # Probe: try a dry-run install targeting the current Python explicitly. - # Without --python, uv can ignore the activated venv on some platforms. + # Probe: try a dry-run install without --system. + # If uv can't find a venv it exits with code 2. probe = subprocess.run( - ["uv", "pip", "install", "--dry-run", "--python", sys.executable, "pip"], + ["uv", "pip", "install", "--dry-run", "pip"], stdout = subprocess.PIPE, stderr = subprocess.STDOUT, ) if probe.returncode != 0: - # Retry with --system (some envs need it when uv can't find a venv) + # Retry with --system to confirm it works probe_sys = subprocess.run( ["uv", "pip", "install", "--dry-run", "--system", "pip"], stdout = subprocess.PIPE, @@ -204,10 +204,6 @@ def _build_uv_cmd(args: tuple[str, ...]) -> list[str]: cmd = ["uv", "pip", "install"] if UV_NEEDS_SYSTEM: cmd.append("--system") - # Always pass --python so uv targets the correct environment. - # Without this, uv can ignore an activated venv and install into - # the system Python (observed on Colab and similar environments). - cmd.extend(["--python", sys.executable]) cmd.extend(_translate_pip_args_for_uv(args)) cmd.append("--torch-backend=auto") return cmd diff --git a/studio/setup.sh b/studio/setup.sh index d64aff0583..ae47f7e362 100755 --- a/studio/setup.sh +++ b/studio/setup.sh @@ -41,26 +41,13 @@ if [[ "$keynames" == *$'\nCOLAB_'* ]]; then fi # ── Detect whether frontend needs building ── -# Skip if dist/ exists AND no tracked input is newer than dist/. -# Checks top-level config/entry files and src/, public/ recursively. -# This handles: PyPI installs (dist/ bundled), repeat runs (no changes), -# and upgrades/pulls (source newer than dist/ triggers rebuild). -_NEED_FRONTEND_BUILD=true -if [ -d "$SCRIPT_DIR/frontend/dist" ]; then - # Check all top-level files (package.json, bun.lock, vite.config.ts, index.html, etc.) - _changed=$(find "$SCRIPT_DIR/frontend" -maxdepth 1 -type f \ - -newer "$SCRIPT_DIR/frontend/dist" -print -quit 2>/dev/null) - # Check src/ and public/ recursively (|| true guards against set -e when dirs are missing) - if [ -z "$_changed" ]; then - _changed=$(find "$SCRIPT_DIR/frontend/src" "$SCRIPT_DIR/frontend/public" \ - -type f -newer "$SCRIPT_DIR/frontend/dist" -print -quit 2>/dev/null) || true - fi - if [ -z "$_changed" ]; then - _NEED_FRONTEND_BUILD=false - fi -fi -if [ "$_NEED_FRONTEND_BUILD" = false ]; then - echo "✅ Frontend already built and up to date -- skipping Node/npm check." +# Only skip when BOTH conditions are true: +# 1. We're inside site-packages (PyPI / pip install, not editable) +# 2. dist/ already exists (pre-built in the wheel) +# Otherwise always (re)build — handles upgrades, editable installs, and +# pip-from-source where dist/ was never built. +if [[ "$SCRIPT_DIR" == */site-packages/* ]] && [ -d "$SCRIPT_DIR/frontend/dist" ]; then + echo "✅ Frontend pre-built (PyPI) — skipping Node/npm check." else NEED_NODE=true if command -v node &>/dev/null && command -v npm &>/dev/null; then @@ -159,27 +146,12 @@ run_quiet "npm run build" npm run build _restore_gitignores trap - EXIT - -# Validate CSS output -- catch truncated Tailwind builds -_MAX_CSS=$(find "$SCRIPT_DIR/frontend/dist/assets" -name '*.css' -exec wc -c {} + 2>/dev/null | sort -n | tail -1 | awk '{print $1}') -if [ -z "$_MAX_CSS" ]; then - echo "⚠️ WARNING: No CSS files were emitted. The frontend build may have failed." -elif [ "$_MAX_CSS" -lt 100000 ]; then - echo "⚠️ WARNING: Largest CSS file is only $((_MAX_CSS / 1024))KB (expected >100KB)." - echo " Tailwind may not have scanned all source files. Check for .gitignore interference." -fi - +cd "$SCRIPT_DIR/backend/core/data_recipe/oxc-validator" +run_quiet "npm install (oxc validator runtime)" npm install cd "$SCRIPT_DIR" echo "✅ Frontend built to frontend/dist" -fi # end frontend build check - -# ── oxc-validator runtime (needs npm -- skip if not available) ── -if [ -d "$SCRIPT_DIR/backend/core/data_recipe/oxc-validator" ] && command -v npm &>/dev/null; then - cd "$SCRIPT_DIR/backend/core/data_recipe/oxc-validator" - run_quiet "npm install (oxc validator runtime)" npm install - cd "$SCRIPT_DIR" -fi +fi # end frontend dist check # ── 6. Python venv + deps ── @@ -251,127 +223,58 @@ install_python_stack() { python "$SCRIPT_DIR/install_python_stack.py" } -# Create venv under ~/.unsloth/studio/ (shared location, not in repo). -# All platforms (including Colab) use the same isolated venv so that -# studio dependencies are never installed into the system Python. -STUDIO_HOME="$HOME/.unsloth/studio" -VENV_DIR="$STUDIO_HOME/.venv" -VENV_T5_DIR="$STUDIO_HOME/.venv_t5" -mkdir -p "$STUDIO_HOME" - -# Clean up legacy in-repo venvs if they exist -[ -d "$REPO_ROOT/.venv" ] && rm -rf "$REPO_ROOT/.venv" -[ -d "$REPO_ROOT/.venv_overlay" ] && rm -rf "$REPO_ROOT/.venv_overlay" -[ -d "$REPO_ROOT/.venv_t5" ] && rm -rf "$REPO_ROOT/.venv_t5" - -rm -rf "$VENV_DIR" -rm -rf "$VENV_T5_DIR" -# Try creating venv with pip; fall back to --without-pip + bootstrap -# (some environments like Colab have broken ensurepip) -if ! "$BEST_PY" -m venv "$VENV_DIR" 2>/dev/null; then - "$BEST_PY" -m venv --without-pip "$VENV_DIR" - source "$VENV_DIR/bin/activate" - curl -sS https://bootstrap.pypa.io/get-pip.py | python > /dev/null +if [ "$IS_COLAB" = true ]; then + # Colab: install packages directly without venv + install_python_stack else + # Local: create venv under studio home (shared location, not in repo) + # Configurable via UNSLOTH_STUDIO_HOME; defaults to ~/.unsloth/studio + STUDIO_HOME="${UNSLOTH_STUDIO_HOME:-$HOME/.unsloth/studio}" + echo " Studio home: $STUDIO_HOME" + # Persist for future `unsloth studio` runs (survives shell restarts) + mkdir -p "$HOME/.unsloth" + echo "$STUDIO_HOME" > "$HOME/.unsloth/studio_home" + VENV_DIR="$STUDIO_HOME/.venv" + VENV_T5_DIR="$STUDIO_HOME/.venv_t5" + mkdir -p "$STUDIO_HOME" + + # Clean up legacy in-repo venvs if they exist + [ -d "$REPO_ROOT/.venv" ] && rm -rf "$REPO_ROOT/.venv" + [ -d "$REPO_ROOT/.venv_overlay" ] && rm -rf "$REPO_ROOT/.venv_overlay" + [ -d "$REPO_ROOT/.venv_t5" ] && rm -rf "$REPO_ROOT/.venv_t5" + + rm -rf "$VENV_DIR" + rm -rf "$VENV_T5_DIR" + "$BEST_PY" -m venv "$VENV_DIR" source "$VENV_DIR/bin/activate" -fi + cd "$SCRIPT_DIR" + install_python_stack -# ── Ensure uv is available (much faster than pip) ── -USE_UV=false -if command -v uv &>/dev/null; then - USE_UV=true -elif curl -LsSf https://astral.sh/uv/install.sh | sh > /dev/null 2>&1; then - export PATH="$HOME/.local/bin:$PATH" - command -v uv &>/dev/null && USE_UV=true -fi - -# Helper: install a package, preferring uv with pip fallback -fast_install() { - if [ "$USE_UV" = true ]; then - uv pip install --python "$(command -v python)" "$@" && return 0 - fi - python -m pip install "$@" -} - -cd "$SCRIPT_DIR" -install_python_stack - -# ── 6b. Pre-install transformers 5.x into .venv_t5/ ── -# Models like GLM-4.7-Flash need transformers>=5.3.0. Instead of pip-installing -# at runtime (slow, ~10-15s), we pre-install into a separate directory. -# The training subprocess just prepends .venv_t5/ to sys.path -- instant switch. -echo "" -echo " Pre-installing transformers 5.x for newer model support..." -mkdir -p "$VENV_T5_DIR" -run_quiet "install transformers 5.x" fast_install --target "$VENV_T5_DIR" --no-deps "transformers==5.3.0" -run_quiet "install huggingface_hub for t5" fast_install --target "$VENV_T5_DIR" --no-deps "huggingface_hub==1.7.1" -run_quiet "install hf_xet for t5" fast_install --target "$VENV_T5_DIR" --no-deps "hf_xet==1.4.2" -# tiktoken is needed by Qwen-family tokenizers. Install with deps since -# regex/requests may be missing on Windows. -run_quiet "install tiktoken for t5" fast_install --target "$VENV_T5_DIR" "tiktoken" -echo "✅ Transformers 5.x pre-installed to $VENV_T5_DIR/" - -# ── 7. WSL: pre-install GGUF build dependencies ── -# On WSL, sudo requires a password and can't be entered during GGUF export -# (runs in a non-interactive subprocess). Install build deps here instead. -if grep -qi microsoft /proc/version 2>/dev/null; then + # ── 6b. Pre-install transformers 5.x into .venv_t5/ ── + # Models like GLM-4.7-Flash need transformers>=5.3.0. Instead of pip-installing + # at runtime (slow, ~10-15s), we pre-install into a separate directory. + # The training subprocess just prepends .venv_t5/ to sys.path — instant switch. echo "" - echo "⚠️ WSL detected -- installing build dependencies for GGUF export..." - _GGUF_DEPS="pciutils build-essential cmake curl git libcurl4-openssl-dev" + echo " Pre-installing transformers 5.x for newer model support..." + mkdir -p "$VENV_T5_DIR" + run_quiet "pip install transformers 5.x" pip install --target "$VENV_T5_DIR" --no-deps "transformers==5.3.0" + run_quiet "pip install huggingface_hub for t5" pip install --target "$VENV_T5_DIR" --no-deps "huggingface_hub==1.3.0" + echo "✅ Transformers 5.x pre-installed to $VENV_T5_DIR/" - # Try without sudo first (works when already root) - apt-get update -y >/dev/null 2>&1 || true - apt-get install -y $_GGUF_DEPS >/dev/null 2>&1 || true - - # Check which packages are still missing - _STILL_MISSING="" - for _pkg in $_GGUF_DEPS; do - case "$_pkg" in - build-essential) command -v gcc >/dev/null 2>&1 || _STILL_MISSING="$_STILL_MISSING $_pkg" ;; - pciutils) command -v lspci >/dev/null 2>&1 || _STILL_MISSING="$_STILL_MISSING $_pkg" ;; - libcurl4-openssl-dev) dpkg -s "$_pkg" >/dev/null 2>&1 || _STILL_MISSING="$_STILL_MISSING $_pkg" ;; - *) command -v "$_pkg" >/dev/null 2>&1 || _STILL_MISSING="$_STILL_MISSING $_pkg" ;; - esac - done - _STILL_MISSING=$(echo "$_STILL_MISSING" | sed 's/^ *//') - - if [ -z "$_STILL_MISSING" ]; then + # ── 7. WSL: pre-install GGUF build dependencies ── + # On WSL, sudo requires a password and can't be entered during GGUF export + # (runs in a non-interactive subprocess). Install build deps here instead. + if grep -qi microsoft /proc/version 2>/dev/null; then + echo "" + echo "⚠️ WSL detected — installing build dependencies for GGUF export..." + echo " You may be prompted for your password." + sudo apt-get update -y + sudo apt-get install -y build-essential cmake curl git libcurl4-openssl-dev echo "✅ GGUF build dependencies installed" - elif command -v sudo >/dev/null 2>&1; then - echo "" - echo " !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!" - echo " WARNING: We require sudo elevated permissions to install:" - echo " $_STILL_MISSING" - echo " If you accept, we'll run sudo now, and it'll prompt your password." - echo " !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!" - echo "" - printf " Accept? [Y/n] " - if [ -r /dev/tty ]; then - read -r REPLY