recover working install files
This commit is contained in:
Roland Tannous
parent
286f954da6
commit
aa7c0b6d6c
2 changed files with 59 additions and 163 deletions
|
|
@ -140,15 +140,15 @@ def _bootstrap_uv() -> bool:
|
|||
global UV_NEEDS_SYSTEM
|
||||
if not shutil.which("uv"):
|
||||
return False
|
||||
# Probe: try a dry-run install targeting the current Python explicitly.
|
||||
# Without --python, uv can ignore the activated venv on some platforms.
|
||||
# Probe: try a dry-run install without --system.
|
||||
# If uv can't find a venv it exits with code 2.
|
||||
probe = subprocess.run(
|
||||
["uv", "pip", "install", "--dry-run", "--python", sys.executable, "pip"],
|
||||
["uv", "pip", "install", "--dry-run", "pip"],
|
||||
stdout = subprocess.PIPE,
|
||||
stderr = subprocess.STDOUT,
|
||||
)
|
||||
if probe.returncode != 0:
|
||||
# Retry with --system (some envs need it when uv can't find a venv)
|
||||
# Retry with --system to confirm it works
|
||||
probe_sys = subprocess.run(
|
||||
["uv", "pip", "install", "--dry-run", "--system", "pip"],
|
||||
stdout = subprocess.PIPE,
|
||||
|
|
@ -204,10 +204,6 @@ def _build_uv_cmd(args: tuple[str, ...]) -> list[str]:
|
|||
cmd = ["uv", "pip", "install"]
|
||||
if UV_NEEDS_SYSTEM:
|
||||
cmd.append("--system")
|
||||
# Always pass --python so uv targets the correct environment.
|
||||
# Without this, uv can ignore an activated venv and install into
|
||||
# the system Python (observed on Colab and similar environments).
|
||||
cmd.extend(["--python", sys.executable])
|
||||
cmd.extend(_translate_pip_args_for_uv(args))
|
||||
cmd.append("--torch-backend=auto")
|
||||
return cmd
|
||||
|
|
|
|||
208
studio/setup.sh
208
studio/setup.sh
|
|
@ -41,26 +41,13 @@ if [[ "$keynames" == *$'\nCOLAB_'* ]]; then
|
|||
fi
|
||||
|
||||
# ── Detect whether frontend needs building ──
|
||||
# Skip if dist/ exists AND no tracked input is newer than dist/.
|
||||
# Checks top-level config/entry files and src/, public/ recursively.
|
||||
# This handles: PyPI installs (dist/ bundled), repeat runs (no changes),
|
||||
# and upgrades/pulls (source newer than dist/ triggers rebuild).
|
||||
_NEED_FRONTEND_BUILD=true
|
||||
if [ -d "$SCRIPT_DIR/frontend/dist" ]; then
|
||||
# Check all top-level files (package.json, bun.lock, vite.config.ts, index.html, etc.)
|
||||
_changed=$(find "$SCRIPT_DIR/frontend" -maxdepth 1 -type f \
|
||||
-newer "$SCRIPT_DIR/frontend/dist" -print -quit 2>/dev/null)
|
||||
# Check src/ and public/ recursively (|| true guards against set -e when dirs are missing)
|
||||
if [ -z "$_changed" ]; then
|
||||
_changed=$(find "$SCRIPT_DIR/frontend/src" "$SCRIPT_DIR/frontend/public" \
|
||||
-type f -newer "$SCRIPT_DIR/frontend/dist" -print -quit 2>/dev/null) || true
|
||||
fi
|
||||
if [ -z "$_changed" ]; then
|
||||
_NEED_FRONTEND_BUILD=false
|
||||
fi
|
||||
fi
|
||||
if [ "$_NEED_FRONTEND_BUILD" = false ]; then
|
||||
echo "✅ Frontend already built and up to date -- skipping Node/npm check."
|
||||
# Only skip when BOTH conditions are true:
|
||||
# 1. We're inside site-packages (PyPI / pip install, not editable)
|
||||
# 2. dist/ already exists (pre-built in the wheel)
|
||||
# Otherwise always (re)build — handles upgrades, editable installs, and
|
||||
# pip-from-source where dist/ was never built.
|
||||
if [[ "$SCRIPT_DIR" == */site-packages/* ]] && [ -d "$SCRIPT_DIR/frontend/dist" ]; then
|
||||
echo "✅ Frontend pre-built (PyPI) — skipping Node/npm check."
|
||||
else
|
||||
NEED_NODE=true
|
||||
if command -v node &>/dev/null && command -v npm &>/dev/null; then
|
||||
|
|
@ -159,27 +146,12 @@ run_quiet "npm run build" npm run build
|
|||
|
||||
_restore_gitignores
|
||||
trap - EXIT
|
||||
|
||||
# Validate CSS output -- catch truncated Tailwind builds
|
||||
_MAX_CSS=$(find "$SCRIPT_DIR/frontend/dist/assets" -name '*.css' -exec wc -c {} + 2>/dev/null | sort -n | tail -1 | awk '{print $1}')
|
||||
if [ -z "$_MAX_CSS" ]; then
|
||||
echo "⚠️ WARNING: No CSS files were emitted. The frontend build may have failed."
|
||||
elif [ "$_MAX_CSS" -lt 100000 ]; then
|
||||
echo "⚠️ WARNING: Largest CSS file is only $((_MAX_CSS / 1024))KB (expected >100KB)."
|
||||
echo " Tailwind may not have scanned all source files. Check for .gitignore interference."
|
||||
fi
|
||||
|
||||
cd "$SCRIPT_DIR/backend/core/data_recipe/oxc-validator"
|
||||
run_quiet "npm install (oxc validator runtime)" npm install
|
||||
cd "$SCRIPT_DIR"
|
||||
echo "✅ Frontend built to frontend/dist"
|
||||
|
||||
fi # end frontend build check
|
||||
|
||||
# ── oxc-validator runtime (needs npm -- skip if not available) ──
|
||||
if [ -d "$SCRIPT_DIR/backend/core/data_recipe/oxc-validator" ] && command -v npm &>/dev/null; then
|
||||
cd "$SCRIPT_DIR/backend/core/data_recipe/oxc-validator"
|
||||
run_quiet "npm install (oxc validator runtime)" npm install
|
||||
cd "$SCRIPT_DIR"
|
||||
fi
|
||||
fi # end frontend dist check
|
||||
|
||||
# ── 6. Python venv + deps ──
|
||||
|
||||
|
|
@ -251,127 +223,58 @@ install_python_stack() {
|
|||
python "$SCRIPT_DIR/install_python_stack.py"
|
||||
}
|
||||
|
||||
# Create venv under ~/.unsloth/studio/ (shared location, not in repo).
|
||||
# All platforms (including Colab) use the same isolated venv so that
|
||||
# studio dependencies are never installed into the system Python.
|
||||
STUDIO_HOME="$HOME/.unsloth/studio"
|
||||
VENV_DIR="$STUDIO_HOME/.venv"
|
||||
VENV_T5_DIR="$STUDIO_HOME/.venv_t5"
|
||||
mkdir -p "$STUDIO_HOME"
|
||||
|
||||
# Clean up legacy in-repo venvs if they exist
|
||||
[ -d "$REPO_ROOT/.venv" ] && rm -rf "$REPO_ROOT/.venv"
|
||||
[ -d "$REPO_ROOT/.venv_overlay" ] && rm -rf "$REPO_ROOT/.venv_overlay"
|
||||
[ -d "$REPO_ROOT/.venv_t5" ] && rm -rf "$REPO_ROOT/.venv_t5"
|
||||
|
||||
rm -rf "$VENV_DIR"
|
||||
rm -rf "$VENV_T5_DIR"
|
||||
# Try creating venv with pip; fall back to --without-pip + bootstrap
|
||||
# (some environments like Colab have broken ensurepip)
|
||||
if ! "$BEST_PY" -m venv "$VENV_DIR" 2>/dev/null; then
|
||||
"$BEST_PY" -m venv --without-pip "$VENV_DIR"
|
||||
source "$VENV_DIR/bin/activate"
|
||||
curl -sS https://bootstrap.pypa.io/get-pip.py | python > /dev/null
|
||||
if [ "$IS_COLAB" = true ]; then
|
||||
# Colab: install packages directly without venv
|
||||
install_python_stack
|
||||
else
|
||||
# Local: create venv under studio home (shared location, not in repo)
|
||||
# Configurable via UNSLOTH_STUDIO_HOME; defaults to ~/.unsloth/studio
|
||||
STUDIO_HOME="${UNSLOTH_STUDIO_HOME:-$HOME/.unsloth/studio}"
|
||||
echo " Studio home: $STUDIO_HOME"
|
||||
# Persist for future `unsloth studio` runs (survives shell restarts)
|
||||
mkdir -p "$HOME/.unsloth"
|
||||
echo "$STUDIO_HOME" > "$HOME/.unsloth/studio_home"
|
||||
VENV_DIR="$STUDIO_HOME/.venv"
|
||||
VENV_T5_DIR="$STUDIO_HOME/.venv_t5"
|
||||
mkdir -p "$STUDIO_HOME"
|
||||
|
||||
# Clean up legacy in-repo venvs if they exist
|
||||
[ -d "$REPO_ROOT/.venv" ] && rm -rf "$REPO_ROOT/.venv"
|
||||
[ -d "$REPO_ROOT/.venv_overlay" ] && rm -rf "$REPO_ROOT/.venv_overlay"
|
||||
[ -d "$REPO_ROOT/.venv_t5" ] && rm -rf "$REPO_ROOT/.venv_t5"
|
||||
|
||||
rm -rf "$VENV_DIR"
|
||||
rm -rf "$VENV_T5_DIR"
|
||||
"$BEST_PY" -m venv "$VENV_DIR"
|
||||
source "$VENV_DIR/bin/activate"
|
||||
fi
|
||||
cd "$SCRIPT_DIR"
|
||||
install_python_stack
|
||||
|
||||
# ── Ensure uv is available (much faster than pip) ──
|
||||
USE_UV=false
|
||||
if command -v uv &>/dev/null; then
|
||||
USE_UV=true
|
||||
elif curl -LsSf https://astral.sh/uv/install.sh | sh > /dev/null 2>&1; then
|
||||
export PATH="$HOME/.local/bin:$PATH"
|
||||
command -v uv &>/dev/null && USE_UV=true
|
||||
fi
|
||||
|
||||
# Helper: install a package, preferring uv with pip fallback
|
||||
fast_install() {
|
||||
if [ "$USE_UV" = true ]; then
|
||||
uv pip install --python "$(command -v python)" "$@" && return 0
|
||||
fi
|
||||
python -m pip install "$@"
|
||||
}
|
||||
|
||||
cd "$SCRIPT_DIR"
|
||||
install_python_stack
|
||||
|
||||
# ── 6b. Pre-install transformers 5.x into .venv_t5/ ──
|
||||
# Models like GLM-4.7-Flash need transformers>=5.3.0. Instead of pip-installing
|
||||
# at runtime (slow, ~10-15s), we pre-install into a separate directory.
|
||||
# The training subprocess just prepends .venv_t5/ to sys.path -- instant switch.
|
||||
echo ""
|
||||
echo " Pre-installing transformers 5.x for newer model support..."
|
||||
mkdir -p "$VENV_T5_DIR"
|
||||
run_quiet "install transformers 5.x" fast_install --target "$VENV_T5_DIR" --no-deps "transformers==5.3.0"
|
||||
run_quiet "install huggingface_hub for t5" fast_install --target "$VENV_T5_DIR" --no-deps "huggingface_hub==1.7.1"
|
||||
run_quiet "install hf_xet for t5" fast_install --target "$VENV_T5_DIR" --no-deps "hf_xet==1.4.2"
|
||||
# tiktoken is needed by Qwen-family tokenizers. Install with deps since
|
||||
# regex/requests may be missing on Windows.
|
||||
run_quiet "install tiktoken for t5" fast_install --target "$VENV_T5_DIR" "tiktoken"
|
||||
echo "✅ Transformers 5.x pre-installed to $VENV_T5_DIR/"
|
||||
|
||||
# ── 7. WSL: pre-install GGUF build dependencies ──
|
||||
# On WSL, sudo requires a password and can't be entered during GGUF export
|
||||
# (runs in a non-interactive subprocess). Install build deps here instead.
|
||||
if grep -qi microsoft /proc/version 2>/dev/null; then
|
||||
# ── 6b. Pre-install transformers 5.x into .venv_t5/ ──
|
||||
# Models like GLM-4.7-Flash need transformers>=5.3.0. Instead of pip-installing
|
||||
# at runtime (slow, ~10-15s), we pre-install into a separate directory.
|
||||
# The training subprocess just prepends .venv_t5/ to sys.path — instant switch.
|
||||
echo ""
|
||||
echo "⚠️ WSL detected -- installing build dependencies for GGUF export..."
|
||||
_GGUF_DEPS="pciutils build-essential cmake curl git libcurl4-openssl-dev"
|
||||
echo " Pre-installing transformers 5.x for newer model support..."
|
||||
mkdir -p "$VENV_T5_DIR"
|
||||
run_quiet "pip install transformers 5.x" pip install --target "$VENV_T5_DIR" --no-deps "transformers==5.3.0"
|
||||
run_quiet "pip install huggingface_hub for t5" pip install --target "$VENV_T5_DIR" --no-deps "huggingface_hub==1.3.0"
|
||||
echo "✅ Transformers 5.x pre-installed to $VENV_T5_DIR/"
|
||||
|
||||
# Try without sudo first (works when already root)
|
||||
apt-get update -y >/dev/null 2>&1 || true
|
||||
apt-get install -y $_GGUF_DEPS >/dev/null 2>&1 || true
|
||||
|
||||
# Check which packages are still missing
|
||||
_STILL_MISSING=""
|
||||
for _pkg in $_GGUF_DEPS; do
|
||||
case "$_pkg" in
|
||||
build-essential) command -v gcc >/dev/null 2>&1 || _STILL_MISSING="$_STILL_MISSING $_pkg" ;;
|
||||
pciutils) command -v lspci >/dev/null 2>&1 || _STILL_MISSING="$_STILL_MISSING $_pkg" ;;
|
||||
libcurl4-openssl-dev) dpkg -s "$_pkg" >/dev/null 2>&1 || _STILL_MISSING="$_STILL_MISSING $_pkg" ;;
|
||||
*) command -v "$_pkg" >/dev/null 2>&1 || _STILL_MISSING="$_STILL_MISSING $_pkg" ;;
|
||||
esac
|
||||
done
|
||||
_STILL_MISSING=$(echo "$_STILL_MISSING" | sed 's/^ *//')
|
||||
|
||||
if [ -z "$_STILL_MISSING" ]; then
|
||||
# ── 7. WSL: pre-install GGUF build dependencies ──
|
||||
# On WSL, sudo requires a password and can't be entered during GGUF export
|
||||
# (runs in a non-interactive subprocess). Install build deps here instead.
|
||||
if grep -qi microsoft /proc/version 2>/dev/null; then
|
||||
echo ""
|
||||
echo "⚠️ WSL detected — installing build dependencies for GGUF export..."
|
||||
echo " You may be prompted for your password."
|
||||
sudo apt-get update -y
|
||||
sudo apt-get install -y build-essential cmake curl git libcurl4-openssl-dev
|
||||
echo "✅ GGUF build dependencies installed"
|
||||
elif command -v sudo >/dev/null 2>&1; then
|
||||
echo ""
|
||||
echo " !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!"
|
||||
echo " WARNING: We require sudo elevated permissions to install:"
|
||||
echo " $_STILL_MISSING"
|
||||
echo " If you accept, we'll run sudo now, and it'll prompt your password."
|
||||
echo " !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!"
|
||||
echo ""
|
||||
printf " Accept? [Y/n] "
|
||||
if [ -r /dev/tty ]; then
|
||||
read -r REPLY </dev/tty || REPLY="y"
|
||||
else
|
||||
REPLY="y"
|
||||
fi
|
||||
case "$REPLY" in
|
||||
[nN]*)
|
||||
echo ""
|
||||
echo " Please install these packages first, then re-run Unsloth Studio setup:"
|
||||
echo " sudo apt-get update -y && sudo apt-get install -y $_STILL_MISSING"
|
||||
_SKIP_GGUF_BUILD=true
|
||||
;;
|
||||
*)
|
||||
sudo apt-get update -y
|
||||
sudo apt-get install -y $_STILL_MISSING
|
||||
echo "✅ GGUF build dependencies installed"
|
||||
;;
|
||||
esac
|
||||
else
|
||||
echo " sudo is not available on this system."
|
||||
echo " Please install as root, then re-run setup:"
|
||||
echo " apt-get install -y $_STILL_MISSING"
|
||||
_SKIP_GGUF_BUILD=true
|
||||
fi
|
||||
fi
|
||||
|
||||
# ── 8. Build llama.cpp binaries for GGUF inference + export ──
|
||||
# Disabled: llama.cpp build is commented out for now.
|
||||
# UNCOMMENT the block below to re-enable.
|
||||
#
|
||||
|
|
@ -510,9 +413,6 @@ if [ "$IS_COLAB" = true ]; then
|
|||
echo "╠══════════════════════════════════════╣"
|
||||
echo "║ Unsloth Studio is ready to start ║"
|
||||
echo "║ in your Colab notebook! ║"
|
||||
echo "║ ║"
|
||||
echo "║ from colab import start ║"
|
||||
echo "║ start() ║"
|
||||
echo "╚══════════════════════════════════════╝"
|
||||
else
|
||||
echo "╔══════════════════════════════════════╗"
|
||||
|
|
@ -520,6 +420,6 @@ else
|
|||
echo "╠══════════════════════════════════════╣"
|
||||
echo "║ Launch with: ║"
|
||||
echo "║ ║"
|
||||
echo "║ unsloth studio -H 0.0.0.0 -p 8888 ║"
|
||||
echo "║ unsloth studio -H 0.0.0.0 -p 8000 ║"
|
||||
echo "╚══════════════════════════════════════╝"
|
||||
fi
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue