diff --git a/.gitattributes b/.gitattributes index 5f04b5e9d1..0025f2a697 100644 --- a/.gitattributes +++ b/.gitattributes @@ -6,7 +6,7 @@ # them when run in WSL/Linux (e.g. `set -e` -> "set: Illegal option -"). *.sh text eol=lf -# Normalize Studio frontend sources to LF. Scoped to the frontend tree (rather +# Normalize Unsloth frontend sources to LF. Scoped to the frontend tree (rather # than repo-wide *.ts/*.tsx/... rules) so the policy can't force LF on files # elsewhere. text=auto lets Git detect and leave binary assets (logos, fonts) # untouched while text files (.ts/.tsx/.json/.html/.svg/...) are stored as LF. diff --git a/.github/scripts/agent-guides-drive.sh b/.github/scripts/agent-guides-drive.sh index f4189a159e..b63ac94b93 100755 --- a/.github/scripts/agent-guides-drive.sh +++ b/.github/scripts/agent-guides-drive.sh @@ -36,6 +36,23 @@ AGENT="${2:?usage: agent-guides-drive.sh }" # Determinism (seed/temp) is applied at the server level by # serve-unsloth-run.sh --extra; agents inherit it through the API. TIMEOUT="${AGENT_INVOKE_TIMEOUT:-180}" +# opencode is the slow outlier. Unlike the print-mode agents (claude -p, codex +# exec) it runs a full turn AND a separate small_model call to name the session, +# so one connection reply takes ~8 min on a CPU-served 4B -- right at the shared +# 600s cap, so the cell flaked when a run drifted past a ~480s success. Give it +# headroom (still well under the 40-min job budget); the fast agents keep the +# tight cap that still catches a real headless-TTY hang. +case "$AGENT" in + opencode) + # Double it, but only for a bare-integer seconds value. A GNU timeout(1) + # duration suffix (s/m/h/d, including floats like 0.5s) is left unchanged so + # the arithmetic never sees a non-number; timeout(1) parses it directly. + case "$TIMEOUT" in + *[!0-9]*) ;; + *) TIMEOUT=$(( TIMEOUT * 2 )) ;; + esac + ;; +esac # Claude refuses --dangerously-skip-permissions outside a sandbox; the CI runner # IS the sandbox, so declare it (mirrors unslothai/scripts launcher.sh). Harmless @@ -166,8 +183,8 @@ parse_connect() { echo "[$AGENT] connect --no-launch printed:"; cat_redacted "$raw" CONNECT_ENV="$(grep -E '^(export |unset )' "$raw" || true)" # The launch command is the last non-export, non-status line. start.py - # prints "Studio · model " and "Updated ..." status lines first. - CONNECT_CMD="$(grep -vE '^(export |unset |Studio |Updated |Disabled |Warning|Loading)' "$raw" \ + # prints "Unsloth · model " and "Updated ..." status lines first. + CONNECT_CMD="$(grep -vE '^(export |unset |Unsloth |Updated |Disabled |Warning|Loading)' "$raw" \ | grep -E '[^[:space:]]' | tail -1)" [ -n "$CONNECT_CMD" ] || guide_fail "could not parse a launch command from connect --no-launch output" redact "$raw" diff --git a/.github/scripts/assert-llama-loads.sh b/.github/scripts/assert-llama-loads.sh index c2ffe27469..62ef80d364 100755 --- a/.github/scripts/assert-llama-loads.sh +++ b/.github/scripts/assert-llama-loads.sh @@ -2,7 +2,7 @@ # SPDX-License-Identifier: AGPL-3.0-only # Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. # -# Assert Studio installed a llama.cpp that loads and runs on THIS macOS. Tests +# Assert Unsloth installed a llama.cpp that loads and runs on THIS macOS. Tests # the contract that matters (binaries load and their minimum-OS is <= this host) # instead of the old "did install.sh fall back to a source build?" grep, since a # source build with a correct deployment target is a valid outcome. diff --git a/.github/scripts/assert-prompt-cache.sh b/.github/scripts/assert-prompt-cache.sh index f5b6b075eb..8c28569f77 100755 --- a/.github/scripts/assert-prompt-cache.sh +++ b/.github/scripts/assert-prompt-cache.sh @@ -31,7 +31,7 @@ # (llama_cpp.py:337-340). So default: ~/.unsloth/studio/logs/llama-server/. # #

is the INTERNAL llama-server port (self._find_free_port(), -# llama_cpp.py:3489 / :4641) -- a RANDOM port, NOT the Studio port. So we must +# llama_cpp.py:3489 / :4641) -- a RANDOM port, NOT the Unsloth port. So we must # NOT filter the log glob by STUDIO_PORT (the brief's `port-` # glob would never match). We pick the newest llama-*.log instead. # diff --git a/.github/scripts/hf-download-with-retry.sh b/.github/scripts/hf-download-with-retry.sh index 013a459f46..6dec93356a 100755 --- a/.github/scripts/hf-download-with-retry.sh +++ b/.github/scripts/hf-download-with-retry.sh @@ -3,7 +3,7 @@ # Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 # # Download a single file from a Hugging Face repo with a stall-retry -# watchdog. Used by the Studio CI workflows so a hung hf-xet transfer +# watchdog. Used by the Unsloth CI workflows so a hung hf-xet transfer # kills + retries instead of silently consuming the job's timeout. # # Usage: hf-download-with-retry.sh REPO FILE LOCAL_DIR @@ -35,7 +35,7 @@ REPO="${1:?usage: hf-download-with-retry.sh REPO FILE [LOCAL_DIR]}" FILE="${2:?usage: hf-download-with-retry.sh REPO FILE [LOCAL_DIR]}" # LOCAL_DIR is optional. If empty, hf falls back to HF_HUB_CACHE # (~/.cache/huggingface/hub) which is the desired path for callers -# that populate HF_HOME for a downstream Studio model load. +# that populate HF_HOME for a downstream Unsloth model load. LOCAL_DIR="${3:-}" # Stall threshold per attempt, in seconds. Override with diff --git a/.github/scripts/run-studio-permission-browser.sh b/.github/scripts/run-studio-permission-browser.sh new file mode 100755 index 0000000000..2007789035 --- /dev/null +++ b/.github/scripts/run-studio-permission-browser.sh @@ -0,0 +1,69 @@ +#!/usr/bin/env bash +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. + +set -euo pipefail + +port="${1:?usage: $0 PORT BROWSER [CHANNEL]}" +browser="${2:?usage: $0 PORT BROWSER [CHANNEL]}" +channel="${3:-}" +slug="$browser${channel:+-$channel}" +artifact_dir="logs/playwright-permissions-$slug" +server_log="logs/studio-permissions-$slug.log" +studio_home="${UNSLOTH_STUDIO_HOME:-$HOME/.unsloth/studio}" +set -- +if [ -n "${STUDIO_PERMISSION_FRONTEND:-}" ]; then + set -- -f "$STUDIO_PERMISSION_FRONTEND" +fi + +mkdir -p "$artifact_dir" +unsloth studio reset-password +UNSLOTH_API_ONLY=1 unsloth studio -H 127.0.0.1 -p "$port" "$@" \ + >"$server_log" 2>&1 & +studio_pid=$! + +cleanup() { + kill "$studio_pid" 2>/dev/null || true + wait "$studio_pid" 2>/dev/null || true +} +trap cleanup EXIT + +healthy=0 +for _ in $(seq 1 180); do + if curl -fs "http://127.0.0.1:$port/api/health" >/dev/null; then + healthy=1 + break + fi + if ! kill -0 "$studio_pid" 2>/dev/null; then + tail -100 "$server_log" || true + exit 1 + fi + sleep 1 +done +if [ "$healthy" -ne 1 ]; then + tail -100 "$server_log" || true + exit 1 +fi + +old_password=$(cat "$studio_home/auth/.bootstrap_password") +new_password="CIPerm-$(python -c 'import secrets; print(secrets.token_urlsafe(16))')" +if [ "${GITHUB_ACTIONS:-}" = "true" ]; then + echo "::add-mask::$old_password" + echo "::add-mask::$new_password" +fi + +export BASE_URL="http://127.0.0.1:$port" +export STUDIO_OLD_PW="$old_password" +export STUDIO_NEW_PW="$new_password" +export STUDIO_UI_STRICT=1 +export STUDIO_UI_PERMISSION_ONLY=1 +export STUDIO_UI_WALL_TIMEOUT_S=240 +export STUDIO_PLAYWRIGHT_BROWSER="$browser" +export PW_ART_DIR="$artifact_dir" +if [ -n "$channel" ]; then + export STUDIO_PLAYWRIGHT_CHANNEL="$channel" +else + unset STUDIO_PLAYWRIGHT_CHANNEL || true +fi + +python tests/studio/playwright_chat_ui.py diff --git a/.github/workflows/consolidated-tests-ci.yml b/.github/workflows/consolidated-tests-ci.yml index fa84471d36..d1bea819eb 100644 --- a/.github/workflows/consolidated-tests-ci.yml +++ b/.github/workflows/consolidated-tests-ci.yml @@ -268,6 +268,7 @@ jobs: tests/saving/test_save_shell_injection.py \ tests/saving/test_patch_saving_none_tokenizer.py \ tests/saving/test_fix_sentencepiece_gguf_robustness.py \ + tests/saving/test_fix_sentencepiece_tokenizer_guard.py \ tests/saving/test_compressed_export_schemes.py \ tests/saving/test_export_api_surface.py \ tests/saving/test_export_dispatch.py \ @@ -358,6 +359,7 @@ jobs: tests/saving/test_save_shell_injection.py \ tests/saving/test_patch_saving_none_tokenizer.py \ tests/saving/test_fix_sentencepiece_gguf_robustness.py \ + tests/saving/test_fix_sentencepiece_tokenizer_guard.py \ tests/saving/test_compressed_export_schemes.py \ tests/saving/test_export_api_surface.py \ tests/saving/test_export_dispatch.py \ diff --git a/.github/workflows/cross-platform-parity-ci.yml b/.github/workflows/cross-platform-parity-ci.yml index bb7dcbf8e4..45ce231743 100644 --- a/.github/workflows/cross-platform-parity-ci.yml +++ b/.github/workflows/cross-platform-parity-ci.yml @@ -1,18 +1,16 @@ # SPDX-License-Identifier: AGPL-3.0-only # Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. -# Runs installer parity and autostart opt-out tests on Windows and macOS. +# Runs installer parity and autostart opt-out tests across all three platforms. # -# Why: that test is the guard that install.sh and install.ps1 stay in -# sync, but today it only runs on ubuntu-latest (auto-discovered by -# studio-backend-ci.yml's "Repo tests (CPU)" job). The test reads both -# installer scripts, and on Windows Path.read_text() defaults to the -# cp1252 locale encoding, so a non-cp1252 byte in install.sh (it already -# contains a U+274C) raises UnicodeDecodeError there even though Linux and -# macOS default to UTF-8. The reads were pinned to encoding="utf-8" in -# #6166; this job keeps that from silently regressing by exercising the -# test on the platforms it claims parity for. Pure pytest, no GPU, -# sub-second, so the matrix is cheap. +# Why: the parity test guards that install.sh and install.ps1 stay in sync. +# It originally ran only on ubuntu-latest through studio-backend-ci.yml. +# On Windows, Path.read_text() defaults to the cp1252 locale encoding, so a +# non-cp1252 byte in install.sh raises UnicodeDecodeError even though Linux +# and macOS default to UTF-8. The reads were pinned to encoding="utf-8" in +# #6166; this matrix keeps that from silently regressing. Pure pytest, no GPU, +# sub-second, so the matrix is cheap. Linux also runs the POSIX rollback test +# under dash, matching the supported curl-to-sh installer path. name: Cross-platform parity @@ -23,6 +21,8 @@ on: - 'install.ps1' - 'tests/test_installer_skip_autostart.py' - 'tests/python/test_cross_platform_parity.py' + - 'tests/sh/test_install_rollback_lifecycle.sh' + - 'tests/studio/test_install_rollback_lifecycle.ps1' - '.github/workflows/cross-platform-parity-ci.yml' push: branches: [main] @@ -31,6 +31,8 @@ on: - 'install.ps1' - 'tests/test_installer_skip_autostart.py' - 'tests/python/test_cross_platform_parity.py' + - 'tests/sh/test_install_rollback_lifecycle.sh' + - 'tests/studio/test_install_rollback_lifecycle.ps1' - '.github/workflows/cross-platform-parity-ci.yml' workflow_dispatch: @@ -47,7 +49,7 @@ jobs: strategy: fail-fast: false matrix: - os: [windows-latest, macos-latest] + os: [ubuntu-latest, windows-latest, macos-latest] runs-on: ${{ matrix.os }} timeout-minutes: 10 steps: @@ -67,3 +69,10 @@ jobs: tests/python/test_cross_platform_parity.py tests/test_installer_skip_autostart.py -q + - name: PowerShell rollback lifecycle tests + if: runner.os == 'Windows' + shell: pwsh + run: pwsh -NoProfile -File tests/studio/test_install_rollback_lifecycle.ps1 + - name: POSIX rollback lifecycle tests + if: runner.os == 'Linux' + run: sh tests/sh/test_install_rollback_lifecycle.sh diff --git a/.github/workflows/lint-ci.yml b/.github/workflows/lint-ci.yml index bd859a6e9e..e1f0afd299 100644 --- a/.github/workflows/lint-ci.yml +++ b/.github/workflows/lint-ci.yml @@ -13,10 +13,10 @@ # committed YAML / JSON config. # # TypeScript and Rust are NOT duplicated here on purpose: -# - Studio Frontend CI runs `npm run typecheck` (= `tsc --noEmit`) +# - Unsloth Frontend CI runs `npm run typecheck` (= `tsc --noEmit`) # and `npm run build` (vite/swc) on every studio/frontend/** # change, which is a full TS AST + type check. -# - Studio Tauri CI runs `tauri build --debug --no-bundle` on +# - Unsloth Tauri CI runs `tauri build --debug --no-bundle` on # every studio/src-tauri/** or studio/frontend/** change, which # compiles the Rust crate (= cargo check + cargo build). # Each is a stricter check than a parse-only step would be, so a diff --git a/.github/workflows/local-agent-guides-ci.yml b/.github/workflows/local-agent-guides-ci.yml index 25796bd5cf..c48328e90f 100644 --- a/.github/workflows/local-agent-guides-ci.yml +++ b/.github/workflows/local-agent-guides-ci.yml @@ -154,7 +154,7 @@ jobs: path: gguf-cache key: ${{ runner.os }}-gguf-${{ env.GGUF_REPO }}-${{ env.GGUF_FILE }}-v1 - - name: Install Studio (--local, --no-torch) + - name: Install Unsloth (--local, --no-torch) env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} # Gated off PR (see note above); public GGUF still downloads. @@ -256,7 +256,7 @@ jobs: done fi - - name: Stop Studio + - name: Stop Unsloth if: always() run: | # Guard the PID: an unset/zero UNSLOTH_SERVER_PID would make @@ -359,7 +359,7 @@ jobs: path: gguf-cache key: ${{ runner.os }}-gguf-${{ env.GGUF_REPO }}-${{ env.GGUF_FILE }}-v1 - - name: Install Studio (--local, --no-torch) + - name: Install Unsloth (--local, --no-torch) env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} # Gated off PR (see note above); public GGUF still downloads. @@ -448,7 +448,7 @@ jobs: done fi - - name: Stop Studio + - name: Stop Unsloth if: always() run: | # Guard the PID: an unset/zero UNSLOTH_SERVER_PID would make @@ -543,7 +543,7 @@ jobs: path: gguf-cache key: ${{ runner.os }}-gguf-${{ env.GGUF_REPO }}-${{ env.GGUF_FILE }}-v1 - - name: Install Studio (--local, --no-torch) + - name: Install Unsloth (--local, --no-torch) env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} HF_TOKEN: ${{ secrets.HF_TOKEN }} @@ -620,7 +620,7 @@ jobs: done fi - - name: Stop Studio + - name: Stop Unsloth if: always() run: | if [ -n "${UNSLOTH_SERVER_PID:-}" ] && [ "${UNSLOTH_SERVER_PID}" != "0" ]; then @@ -706,7 +706,7 @@ jobs: path: hf-cache key: ${{ runner.os }}-hf-${{ env.GGUF_REPO }}-${{ env.GGUF_VARIANT }}-v2 - - name: Install Studio (--local, --no-torch) + - name: Install Unsloth (--local, --no-torch) env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} # Gated off PR (see note above); public GGUF still downloads. @@ -764,7 +764,7 @@ jobs: done fi - - name: Stop Studio + - name: Stop Unsloth if: always() run: | # Guard the PID: an unset/zero UNSLOTH_SERVER_PID would make diff --git a/.github/workflows/mlx-ci.yml b/.github/workflows/mlx-ci.yml index a2f716a93c..aadf0b54e6 100644 --- a/.github/workflows/mlx-ci.yml +++ b/.github/workflows/mlx-ci.yml @@ -130,7 +130,7 @@ jobs: # MLX support landed after the most recent unsloth-zoo PyPI # release; the wheel still raises NotImplementedError on # Apple Silicon when device_type.get_device_type() runs - # unguarded. Studio's own install.sh overlays unsloth-zoo + # unguarded. Unsloth's own install.sh overlays unsloth-zoo # from git main for the same reason. Pulling deps lets pip # resolve the platform-conditional MLX-only wheels (mlx, # mlx-lm, mlx-vlm gated on darwin+arm64 in unsloth-zoo's @@ -317,13 +317,13 @@ jobs: echo done - # Validates the macOS prebuilt path Studio's setup.sh uses (#5963): install the + # Validates the macOS prebuilt path Unsloth's setup.sh uses (#5963): install the # unslothai/llama.cpp fork's latest release, download a small public GGUF, and # check llama-server /completion end to end. Split and placed last so the # untrusted binary runs only in the final smoke step, after every HF_TOKEN step, # leaving no token-bearing step or shared workspace for a tampered prebuilt to # corrupt. GH_TOKEN: releases API; HF_TOKEN (withheld on PR): probe + GGUF fetch. - - name: Studio prebuilt llama.cpp install + GGUF download (Mac M1) + - name: Unsloth prebuilt llama.cpp install + GGUF download (Mac M1) env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} @@ -344,12 +344,12 @@ jobs: # Final step: runs the downloaded binaries with no secrets present, and clears # the GitHub Actions command files so a tampered prebuilt cannot influence the job. - - name: Studio prebuilt llama.cpp GGUF inference smoke (Mac M1) + - name: Unsloth prebuilt llama.cpp GGUF inference smoke (Mac M1) run: | set -euo pipefail unset GITHUB_ENV GITHUB_PATH GITHUB_OUTPUT GITHUB_STEP_SUMMARY INSTALL_DIR="$HOME/.unsloth-studio-prebuilt-test/llama.cpp" - # Studio bundles only llama-server + llama-quantize (not llama-cli); + # Unsloth bundles only llama-server + llama-quantize (not llama-cli); # inference goes through llama-server's HTTP /completion endpoint. LLAMA_SERVER="$INSTALL_DIR/build/bin/llama-server" LLAMA_QUANT="$INSTALL_DIR/build/bin/llama-quantize" @@ -400,4 +400,4 @@ jobs: tail -40 /tmp/llama-server.log exit 1 fi - echo "OK: Studio prebuilt llama.cpp on Mac M1 + GGUF /completion works" + echo "OK: Unsloth prebuilt llama.cpp on Mac M1 + GGUF /completion works" diff --git a/.github/workflows/release-desktop.yml b/.github/workflows/release-desktop.yml index 4daafae35d..081eda4e32 100644 --- a/.github/workflows/release-desktop.yml +++ b/.github/workflows/release-desktop.yml @@ -4,7 +4,7 @@ on: workflow_dispatch: inputs: studio_version: - description: 'Studio version tag to release (for example, v0.1.39-beta)' + description: 'Unsloth version tag to release (for example, v0.1.39-beta)' type: string required: true pypi_version: @@ -69,7 +69,7 @@ jobs: if not studio_version: sys.exit('studio_version is required, for example v0.1.39-beta') if re.fullmatch(r'v?20\d{2}\.\d+\.\d+(?:[-+][0-9A-Za-z.-]+)?', studio_version): - sys.exit(f'studio_version must be a Studio SemVer tag, not a date-style backend version: {studio_version}') + sys.exit(f'studio_version must be an Unsloth SemVer tag, not a date-style backend version: {studio_version}') semver_tag = re.compile( r'^v(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)' @@ -146,7 +146,7 @@ jobs: print(f'pypi_version={pypi_version}', file=output) PY - - name: Verify PyPI package and Studio stamp + - name: Verify PyPI package and Unsloth stamp shell: bash env: STUDIO_VERSION: ${{ steps.prepare.outputs.studio_version }} @@ -211,7 +211,7 @@ jobs: fi python3 scripts/stamp_studio_release.py --verify-dist "$RUNNER_TEMP/pypi-unsloth-dist" --expected "$STUDIO_VERSION" else - echo "scripts/stamp_studio_release.py not found; release-desktop requires #5308 to verify the PyPI Studio stamp." >&2 + echo "scripts/stamp_studio_release.py not found; release-desktop requires #5308 to verify the PyPI Unsloth stamp." >&2 exit 1 fi diff --git a/.github/workflows/security-audit.yml b/.github/workflows/security-audit.yml index 1275d12216..27eafbedea 100644 --- a/.github/workflows/security-audit.yml +++ b/.github/workflows/security-audit.yml @@ -36,8 +36,8 @@ # - unsloth `huggingfacenotorch` extras (the canonical install path # for fine-tuning users; pulls transformers / peft / accelerate / # trl / datasets / diffusers / sentence-transformers / etc.) -# - all six Studio backend requirements files -# - Studio frontend (npm) and Tauri shell (cargo) +# - all six Unsloth backend requirements files +# - Unsloth frontend (npm) and Tauri shell (cargo) # Each Python step builds a filtered dep list from pyproject.toml + # requirements/*.txt before auditing. We do NOT install any of these # -- pip-audit resolves through PyPI metadata, scan_packages.py @@ -218,7 +218,7 @@ jobs: # on the runner). A comment line is left in place so the # skipped specs are obvious in the artifact. # The `huggingface` extra is `huggingfacenotorch` plus torch / - # torchvision / triton, deliberately skipped: Studio backend + # torchvision / triton, deliberately skipped: Unsloth backend # already pins a torch and the +cu* / +cpu local-version tags # trip up the PyPI resolver in `-r` mode. run: | @@ -253,7 +253,7 @@ jobs: # `-r requirements.txt` resolves the requirements through pip's # dependency resolver against PyPI metadata and audits the # resolved tree without ever executing setup.py / install - # hooks. Way faster than installing the full Studio runtime + # hooks. Way faster than installing the full Unsloth runtime # and -- critically -- safer: an attacker who has compromised # a transitive dep cannot run code in this job. # @@ -326,9 +326,9 @@ jobs: } >> "$GITHUB_STEP_SUMMARY" # ───────────────────────────────────────────────────────────── - # npm: Studio frontend + # npm: Unsloth frontend # ───────────────────────────────────────────────────────────── - - name: npm audit (Studio frontend) + - name: npm audit (Unsloth frontend) # `npm audit` resolves the lockfile through the npmjs.com # advisory DB. `--audit-level=high` filters the noise floor # to only HIGH and CRITICAL. We do NOT pass --omit=dev: a @@ -342,7 +342,7 @@ jobs: # Always also write the full JSON for grep-ability. npm audit --json > ../../logs-npm-audit.json || true { - echo "## npm audit (Studio frontend)" + echo "## npm audit (Unsloth frontend)" echo echo '```' tail -200 ../../logs-npm-audit.txt @@ -350,9 +350,9 @@ jobs: } >> "$GITHUB_STEP_SUMMARY" # ───────────────────────────────────────────────────────────── - # cargo: Studio Tauri shell + # cargo: Unsloth Tauri shell # ───────────────────────────────────────────────────────────── - - name: cargo audit (Studio Tauri) + - name: cargo audit (Unsloth Tauri) # `--deny warnings` would make the job fail on any advisory. # Keep non-blocking initially; drop continue-on-error after # the baseline closes. @@ -362,7 +362,7 @@ jobs: set +e cargo audit | tee ../../logs-cargo-audit.txt { - echo "## cargo audit (Studio Tauri)" + echo "## cargo audit (Unsloth Tauri)" echo echo '```' tail -200 ../../logs-cargo-audit.txt @@ -559,7 +559,7 @@ jobs: # ───────────────────────────────────────────────────────────── # CycloneDX SBOM. Lets downstream consumers audit what's - # actually shipped in unsloth wheels and the Studio backend + # actually shipped in unsloth wheels and the Unsloth backend # runtime. Generates one JSON file per requirements input plus # a combined SBOM keyed off pyproject.toml; uploads as a build # artifact (and a future step can attest it via SLSA). @@ -740,7 +740,7 @@ jobs: # `--with-deps` makes the scan transitive: every package the # declared set resolves to gets fetched and pattern-scanned, not # just the top-level pins. Resolving the full transitive closure - # of the unsloth + Studio dep tree downloads several hundred + # of the unsloth + Unsloth dep tree downloads several hundred # archives, hence the longer timeout. # # Sharded across runners for wall-clock parallelism. Each shard @@ -749,7 +749,7 @@ jobs: # composition tries to balance load: # - hf-stack: pyproject extras + no-torch-runtime # (~150 archives, transformers/peft/accelerate/...) - # - studio: FastAPI/Studio backend + overrides + extras-no-deps + # - studio: FastAPI/Unsloth backend + overrides + extras-no-deps # (~150 archives, smaller scientific stack) # - extras: the heavy openai-whisper / scikit-learn / librosa # stack (~250 archives, dominant cost) @@ -964,7 +964,7 @@ jobs: # documented at scripts/scan_npm_packages.py top-of-file. The # script is stdlib-only so adding it does not increase the # transitive supply-chain surface. - name: npm scan-packages (Studio frontend tarballs) + name: npm scan-packages (Unsloth frontend tarballs) runs-on: ubuntu-latest timeout-minutes: 30 needs: [] @@ -1173,7 +1173,7 @@ jobs: with: python-version: '3.12' - - name: Install Studio frontend deps (--ignore-scripts) + - name: Install Unsloth frontend deps (--ignore-scripts) # `npm audit signatures` requires node_modules to be populated. # `--ignore-scripts` is mandatory: this is exactly the lever the # new-install-script gate below protects against, and we must diff --git a/.github/workflows/studio-api-smoke.yml b/.github/workflows/studio-api-smoke.yml index 15efee382e..cdf1f6bf12 100644 --- a/.github/workflows/studio-api-smoke.yml +++ b/.github/workflows/studio-api-smoke.yml @@ -1,7 +1,7 @@ # SPDX-License-Identifier: AGPL-3.0-only # Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. -# Studio API & Auth Tests -- HTTP-level integration tests for the +# Unsloth API & Auth Tests -- HTTP-level integration tests for the # FastAPI surface. No Playwright, no model UI; tests/studio/test_studio_api_smoke.py # runs ~30 s and asserts: # - CORS hardening (no wildcard + credentials, no bootstrap leak) @@ -15,7 +15,7 @@ # Reuses the GGUF cache key from studio-ui-smoke.yml so the model # download is one cache-hit on the second job. -name: Studio API CI +name: Unsloth API CI on: pull_request: @@ -40,7 +40,7 @@ permissions: jobs: api-smoke: - name: Studio API & Auth Tests + name: Unsloth API & Auth Tests runs-on: ubuntu-latest timeout-minutes: 12 env: @@ -98,7 +98,7 @@ jobs: path: hf-cache key: ${{ runner.os }}-hf-${{ env.GGUF_REPO }}-${{ env.GGUF_VARIANT }}-v2 - - name: Install Studio (--local, --no-torch) + - name: Install Unsloth (--local, --no-torch) env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} # Withheld on PR: this step runs checked-out PR code; public GGUF still downloads. @@ -111,7 +111,7 @@ jobs: - name: Install pyjwt for the JWT-expiry forge test run: pip install 'pyjwt>=2.6' - - name: Reset auth + boot Studio (API-only) + - name: Reset auth + boot Unsloth (API-only) run: | unsloth studio reset-password mkdir -p logs @@ -144,7 +144,7 @@ jobs: echo "STUDIO_NEW_PW=$NEW" >> "$GITHUB_ENV" echo "STUDIO_NEW2_PW=$NEW2" >> "$GITHUB_ENV" - - name: Run Studio API & Auth tests + - name: Run Unsloth API & Auth tests # The script is named WITHOUT a `test_` prefix so it isn't # auto-collected by pytest in Backend CI's `tests/` walk # (which doesn't set BASE_URL and would crash at import). @@ -153,7 +153,7 @@ jobs: STUDIO_AUTH_DIR: /home/runner/.unsloth/studio/auth run: python tests/studio/studio_api_smoke.py - - name: Stop Studio + - name: Stop Unsloth if: always() run: | kill "${STUDIO_PID}" 2>/dev/null || true diff --git a/.github/workflows/studio-backend-ci.yml b/.github/workflows/studio-backend-ci.yml index 3022127a2b..3968f2e80a 100644 --- a/.github/workflows/studio-backend-ci.yml +++ b/.github/workflows/studio-backend-ci.yml @@ -64,7 +64,7 @@ jobs: - name: Install backend test dependencies (CPU only) run: | python -m pip install --upgrade pip - # Studio's declared backend deps: + # Unsloth's declared backend deps: pip install -r studio/backend/requirements/studio.txt # Extras that studio.txt does not list but the import chain needs # (python-multipart for FastAPI form/file uploads, sqlalchemy/cryptography @@ -193,6 +193,7 @@ jobs: --ignore=tests/sh \ --ignore=tests/studio/test_hardware_dispatch_matrix.py \ --ignore=tests/studio/test_is_mlx_dispatch_gate.py \ + --ignore=tests/studio/test_xpu_spoof_pipeline.py \ --ignore=tests/vllm_compat \ --ignore=tests/version_compat \ -m 'not server and not e2e' \ @@ -205,14 +206,15 @@ jobs: env: PYTHONPATH: ${{ github.workspace }}/studio UNSLOTH_COMPILE_DISABLE: '1' - # These two files mutate hardware.py module globals at runtime - # via the spoof fixtures, which leaks state into any other test - # that imports hardware. Run them in their own pytest invocation - # so the leak does not cross file boundaries. + # These files mutate hardware.py module globals at runtime via the + # spoof fixtures (CUDA/ROCm/XPU/MLX/CPU), which leaks state into any + # other test that imports hardware. Run them in their own pytest + # invocation so the leak does not cross file boundaries. run: | python -m pytest -q --tb=short \ tests/studio/test_hardware_dispatch_matrix.py \ - tests/studio/test_is_mlx_dispatch_gate.py + tests/studio/test_is_mlx_dispatch_gate.py \ + tests/studio/test_xpu_spoof_pipeline.py - name: Shell installer tests # Subset that does not depend on a writable / pristine install.sh @@ -228,6 +230,7 @@ jobs: tests/sh/test_system_node_readonly.sh \ tests/sh/test_nvcc_meets_llama_minimum.sh \ tests/sh/test_resolve_cuda_archs.sh \ + tests/sh/test_staged_validation_enabled.sh \ tests/sh/test_tauri_install_exit_order.sh \ tests/sh/test_torch_constraint.sh \ tests/sh/test_torch_flavor.sh \ diff --git a/.github/workflows/studio-export-capability-ci.yml b/.github/workflows/studio-export-capability-ci.yml index 1ee6489209..83df3ed476 100644 --- a/.github/workflows/studio-export-capability-ci.yml +++ b/.github/workflows/studio-export-capability-ci.yml @@ -9,7 +9,7 @@ # export is validated separately. No GPU / model / llama.cpp: the tests mock the probes and block # torch/unsloth, so the job installs only a CPU PyTorch plus import deps. -name: Studio export capability +name: Unsloth export capability on: pull_request: diff --git a/.github/workflows/studio-frontend-ci.yml b/.github/workflows/studio-frontend-ci.yml index b42086f191..3a9e373915 100644 --- a/.github/workflows/studio-frontend-ci.yml +++ b/.github/workflows/studio-frontend-ci.yml @@ -136,7 +136,7 @@ jobs: - name: Build run: npm run build - - name: Built bundle must not contain Studio's unstable_Provider call site + - name: Built bundle must not contain Unsloth's unstable_Provider call site run: | set -e JS=$(ls dist/assets/index-*.js | head -1) @@ -144,7 +144,7 @@ jobs: echo "main bundle: $JS" echo "unstable_Provider: hits=$HITS (assistant-ui internals contribute up to 3)" if [ "$HITS" -gt 3 ]; then - echo "::error file=studio/frontend/src/features/chat/runtime-provider.tsx::Studio bundle still passes unstable_Provider through useRemoteThreadListRuntime; this is the 2026.5.1 chat-history regression. Pass adapters directly into useLocalRuntime instead." + echo "::error file=studio/frontend/src/features/chat/runtime-provider.tsx::Unsloth bundle still passes unstable_Provider through useRemoteThreadListRuntime; this is the 2026.5.1 chat-history regression. Pass adapters directly into useLocalRuntime instead." exit 1 fi diff --git a/.github/workflows/studio-inference-smoke.yml b/.github/workflows/studio-inference-smoke.yml index 58ef2558f3..c2d52eac22 100644 --- a/.github/workflows/studio-inference-smoke.yml +++ b/.github/workflows/studio-inference-smoke.yml @@ -1,7 +1,7 @@ # SPDX-License-Identifier: AGPL-3.0-only # Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. -# Three end-to-end smoke jobs that boot a freshly-installed Studio and +# Three end-to-end smoke jobs that boot a freshly-installed Unsloth and # exercise the surfaces real users hit through the OpenAI / Anthropic # SDKs and curl. Each job picks the smallest model that exercises the # behaviour under test, primes HF_HOME via actions/cache, and shares @@ -27,7 +27,7 @@ # All three jobs run in parallel. Total wall time is dominated by job 3 # on a cold cache; warm cache cuts that to ~3 min. -name: Studio GGUF CI +name: Unsloth GGUF CI on: pull_request: @@ -112,7 +112,7 @@ jobs: path: hf-cache key: ${{ runner.os }}-hf-${{ env.GGUF_REPO }}-${{ env.GGUF_VARIANT }}-v2 - - name: Install Studio (--local, --no-torch) + - name: Install Unsloth (--local, --no-torch) env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} # Withheld on PR: this step runs checked-out PR code; public GGUF still downloads. @@ -125,7 +125,7 @@ jobs: - name: Install OpenAI + Anthropic Python SDKs run: pip install 'openai>=1.50' 'anthropic>=0.40' - - name: Reset auth + boot Studio (API-only) + - name: Reset auth + boot Unsloth (API-only) run: | unsloth studio reset-password mkdir -p logs @@ -142,7 +142,7 @@ jobs: fi sleep 1 done - echo "Studio did not become healthy in 180s" + echo "Unsloth did not become healthy in 180s" tail -200 logs/studio.log exit 1 @@ -229,11 +229,11 @@ jobs: return replies def run_anthropic(): - # Two SDK quirks vs. Studio: + # Two SDK quirks vs. Unsloth: # 1. base_url must NOT include /v1 -- the SDK appends # /v1/messages itself; otherwise the request hits # /v1/v1/messages and 405s. - # 2. The SDK sends `x-api-key` by default, but Studio's + # 2. The SDK sends `x-api-key` by default, but Unsloth's # auth layer is HTTPBearer-only. Override via # default_headers so Authorization: Bearer ... is # sent instead. @@ -276,7 +276,7 @@ jobs: print( f"[{label}] WARN non-determinism at temperature=0.0 across " f"{len(determinism_failures)} of {len(first)} turn(s); " - f"small-quant model drift, not a Studio regression. " + f"small-quant model drift, not an Unsloth regression. " f"Details: " + " | ".join(determinism_failures) ) # Sanity: turn-2 reply should mention the earlier question, and @@ -290,7 +290,7 @@ jobs: print(f"[{label}] {status_word} -- 4 turns, history grounded ('paris' present)") PY - - name: Stop Studio + - name: Stop Unsloth if: always() run: | kill "${STUDIO_PID}" 2>/dev/null || true @@ -323,7 +323,7 @@ jobs: # store xet chunks + blobs + snapshots = ~4 GiB compressed -- # 4-5x file-size inflation, dominated by xet chunks. Use main's # `--local-dir gguf-cache` pattern to cache the flat .gguf only. - # Studio's /api/inference/load accepts either a HF repo (which + # Unsloth's /api/inference/load accepts either a HF repo (which # uses HF_HOME) or an absolute file path; passing the absolute # path keeps the test off HF_HOME entirely so the cache size # tracks the GGUF file 1:1. The OpenAI/Anth and JSON+images @@ -380,7 +380,7 @@ jobs: path: gguf-cache key: ${{ runner.os }}-gguf-${{ env.GGUF_REPO }}-${{ env.GGUF_FILE }}-v1 - - name: Install Studio (--local, --no-torch) + - name: Install Unsloth (--local, --no-torch) env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} # Withheld on PR: this step runs checked-out PR code; public GGUF still downloads. @@ -390,7 +390,7 @@ jobs: set -o pipefail bash install.sh --local --no-torch 2>&1 | tee logs/install.log - - name: Reset auth + boot Studio (API-only, default tool policy) + - name: Reset auth + boot Unsloth (API-only, default tool policy) # We deliberately use the API-only mode rather than # `unsloth studio run` because the latter calls # `set_tool_policy(...)` with a resolved bool: on loopback the @@ -503,7 +503,7 @@ jobs: that the tool path executed. A shared CI runner can stall the stream transport (the - connection opening, or a mid-stream read) even when Studio + connection opening, or a mid-stream read) even when Unsloth is healthy, so retry a stall once with a fresh request capped at 300s. A stall means the stream did NOT complete, so partial events are normally NOT returned (an early @@ -575,11 +575,11 @@ jobs: def _tool_invoked(events): """Structural check: True iff some SSE payload is a real - tool envelope (Studio tool_start/tool_end, Anthropic + tool envelope (Unsloth tool_start/tool_end, Anthropic tool_use/tool_result, OpenAI non-empty delta.tool_calls / message.tool_calls / finish_reason='tool_calls' / role:'tool' / function_call). tool_status is NOT - evidence: Studio emits empty tool_status events on + evidence: Unsloth emits empty tool_status events on iteration boundaries even when no tool ran. """ for raw in events: @@ -698,7 +698,7 @@ jobs: attempt has structural invocation evidence. WARN (not FAIL) if invoked but no attempt produces the expected literal in tool_end.result -- small-quant Qwen3.5-2B can - emit OpenAI tool_calls deltas without Studio's GGUF + emit OpenAI tool_calls deltas without Unsloth's GGUF agentic loop intercepting them, and that GGUF-vs-OpenAI format mismatch is out of scope for #5642. """ @@ -811,7 +811,7 @@ jobs: # because (a) the search may legitimately return no results, # and (b) DuckDuckGo upstream blocks GHA IP ranges often # enough that requiring a tool_call marker would create - # red-herring failures from infra rather than from Studio. + # red-herring failures from infra rather than from Unsloth. try: # Best-effort and bounded: a single 180s attempt keeps a stall # from eating the job's timeout-minutes (it already WARNs, so a @@ -834,7 +834,7 @@ jobs: print(f"[tools] WARN web_search probe failed (non-blocking): {exc}") # ── 5. Thinking on / off ───────────────────────────────────── - # Studio strips think blocks from message.content for tools-mode + # Unsloth strips think blocks from message.content for tools-mode # responses, so we toggle plain chat (no enable_tools) and look # at the surfaced reasoning_content / message.thinking field. def thinking_call(enable): @@ -848,7 +848,7 @@ jobs: }) assert status == 200 msg = data["choices"][0]["message"] - # Studio surfaces thinking via reasoning_content (OpenAI + # Unsloth surfaces thinking via reasoning_content (OpenAI # extension). Fall back to inline markers for # robustness across template versions. raw = (msg.get("content") or "") + (msg.get("reasoning_content") or "") @@ -868,7 +868,7 @@ jobs: print(f"[tools] PASS thinking on/off (on={len(on_text)} chars, off={len(off_text)} chars)") PY - - name: Stop Studio + - name: Stop Unsloth if: always() run: | kill "${STUDIO_PID}" 2>/dev/null || true @@ -960,7 +960,7 @@ jobs: path: hf-cache key: ${{ runner.os }}-hf-${{ env.GGUF_REPO }}-${{ env.GGUF_VARIANT }}-${{ env.MMPROJ_FILE }}-v2 - - name: Install Studio (--local, --no-torch) + - name: Install Unsloth (--local, --no-torch) env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} # Withheld on PR: this step runs checked-out PR code; public GGUF still downloads. @@ -973,7 +973,7 @@ jobs: - name: Install OpenAI + Anthropic Python SDKs run: pip install 'openai>=1.50' 'anthropic>=0.40' - - name: Reset auth + boot Studio (API-only) + - name: Reset auth + boot Unsloth (API-only) # See Job 2's comment: API-only mode keeps tool_policy=None so # response_format requests aren't routed through the agentic # tool loop. @@ -1076,13 +1076,13 @@ jobs: # llama.cpp's HTTP server supports OpenAI-compatible JSON # mode: `response_format: {"type": "json_object"}` constrains # the model to emit syntactically-valid JSON. We use raw HTTP - # rather than the OpenAI SDK so that the field shape Studio + # rather than the OpenAI SDK so that the field shape Unsloth # forwards to llama-server is unambiguous (the SDK rewrites # response_format depending on which variant it recognises). # We deliberately do NOT pass a strict JSON schema -- on # small Gemma-4 quants the GBNF-from-schema path occasionally # produces empty output, and JSON mode is the surface we care - # about exposing through Studio. + # about exposing through Unsloth. status, data = post("/v1/chat/completions", { "model": "default", "messages": [ @@ -1112,7 +1112,7 @@ jobs: print(f"[json] PASS json_object -> {parsed}") # ── 2. OpenAI image_url (data URI base64) ─────────────────── - # 64x64 solid-red PNG. stb_image (used by Studio's image + # 64x64 solid-red PNG. stb_image (used by Unsloth's image # normaliser at routes/inference.py:3410) rejects 4x4 or # smaller PNGs as truncated, so we go up to 64x64 -- still # tiny in token cost. The assertion is loose: any non-empty @@ -1148,9 +1148,9 @@ jobs: print("[image/openai] PASS image_url accepted, non-empty response") # ── 3. Anthropic source/base64 image ──────────────────────── - # Two SDK quirks vs. Studio: base_url must NOT include /v1 + # Two SDK quirks vs. Unsloth: base_url must NOT include /v1 # (the SDK appends it itself; otherwise /v1/v1/messages -> 405), - # and Studio's auth is HTTPBearer-only so the SDK's default + # and Unsloth's auth is HTTPBearer-only so the SDK's default # x-api-key header is ignored -- send Authorization: Bearer # via default_headers. anthropic = Anthropic( @@ -1184,7 +1184,7 @@ jobs: print("[image/anthropic] PASS source/base64 accepted, non-empty response") PY - - name: Stop Studio + - name: Stop Unsloth if: always() run: | kill "${STUDIO_PID}" 2>/dev/null || true diff --git a/.github/workflows/studio-load-orchestrator-ci.yml b/.github/workflows/studio-load-orchestrator-ci.yml index 93d1a7742d..8710efc2bd 100644 --- a/.github/workflows/studio-load-orchestrator-ci.yml +++ b/.github/workflows/studio-load-orchestrator-ci.yml @@ -1,7 +1,7 @@ # SPDX-License-Identifier: AGPL-3.0-only # Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. # -# Event-loop regression test for the Studio model-load orchestrator. +# Event-loop regression test for the Unsloth model-load orchestrator. # Pins down issue #5642 (Win10 UI freeze on model load): the /load # route calls LlamaCppBackend.detect_audio_type synchronously, blocking # the FastAPI event loop on a chain of sync httpx.Client.post() probes. @@ -14,7 +14,7 @@ # danielhanchen/unsloth-staging-2 (Ubuntu / macOS / Windows all # green at PR time). -name: Studio load-orchestrator CI +name: Unsloth load-orchestrator CI on: pull_request: diff --git a/.github/workflows/studio-mac-api-smoke.yml b/.github/workflows/studio-mac-api-smoke.yml index 617ce189dc..1968885a1d 100644 --- a/.github/workflows/studio-mac-api-smoke.yml +++ b/.github/workflows/studio-mac-api-smoke.yml @@ -33,7 +33,7 @@ permissions: jobs: api-smoke: - name: Studio API & Auth Tests + name: Unsloth API & Auth Tests runs-on: macos-14 timeout-minutes: 25 env: @@ -83,7 +83,7 @@ jobs: path: hf-cache key: ${{ runner.os }}-hf-${{ env.GGUF_REPO }}-${{ env.GGUF_VARIANT }}-v2 - - name: Install Studio (--local, --no-torch) + - name: Install Unsloth (--local, --no-torch) env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} # Withheld on PR: this step runs checked-out PR code; public GGUF still downloads. @@ -99,7 +99,7 @@ jobs: - name: Install pyjwt for the JWT-expiry forge test run: pip install 'pyjwt>=2.6' - - name: Reset auth + boot Studio (API-only) + - name: Reset auth + boot Unsloth (API-only) run: | unsloth studio reset-password mkdir -p logs @@ -129,13 +129,13 @@ jobs: echo "STUDIO_NEW_PW=$NEW" >> "$GITHUB_ENV" echo "STUDIO_NEW2_PW=$NEW2" >> "$GITHUB_ENV" - - name: Run Studio API & Auth tests + - name: Run Unsloth API & Auth tests env: BASE_URL: http://127.0.0.1:18895 STUDIO_AUTH_DIR: /Users/runner/.unsloth/studio/auth run: python tests/studio/studio_api_smoke.py - - name: Stop Studio + - name: Stop Unsloth if: always() run: | kill "${STUDIO_PID}" 2>/dev/null || true diff --git a/.github/workflows/studio-mac-inference-smoke.yml b/.github/workflows/studio-mac-inference-smoke.yml index 946681706a..ce15eed5c8 100644 --- a/.github/workflows/studio-mac-inference-smoke.yml +++ b/.github/workflows/studio-mac-inference-smoke.yml @@ -1,7 +1,7 @@ # SPDX-License-Identifier: AGPL-3.0-only # Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. -# Three end-to-end smoke jobs that boot a freshly-installed Studio and +# Three end-to-end smoke jobs that boot a freshly-installed Unsloth and # exercise the surfaces real users hit through the OpenAI / Anthropic # SDKs and curl. Each job picks the smallest model that exercises the # behaviour under test, primes a model cache via actions/cache, and @@ -108,7 +108,7 @@ jobs: path: hf-cache key: ${{ runner.os }}-hf-${{ env.GGUF_REPO }}-${{ env.GGUF_VARIANT }}-v2 - - name: Install Studio (--local, --no-torch) + - name: Install Unsloth (--local, --no-torch) env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} # Withheld on PR: this step runs checked-out PR code; public GGUF still downloads. @@ -124,7 +124,7 @@ jobs: - name: Install OpenAI + Anthropic Python SDKs run: pip install 'openai>=1.50' 'anthropic>=0.40' - - name: Reset auth + boot Studio (API-only) + - name: Reset auth + boot Unsloth (API-only) run: | unsloth studio reset-password mkdir -p logs @@ -141,7 +141,7 @@ jobs: fi sleep 1 done - echo "Studio did not become healthy in 180s" + echo "Unsloth did not become healthy in 180s" tail -200 logs/studio.log exit 1 @@ -228,11 +228,11 @@ jobs: return replies def run_anthropic(): - # Two SDK quirks vs. Studio: + # Two SDK quirks vs. Unsloth: # 1. base_url must NOT include /v1 -- the SDK appends # /v1/messages itself; otherwise the request hits # /v1/v1/messages and 405s. - # 2. The SDK sends `x-api-key` by default, but Studio's + # 2. The SDK sends `x-api-key` by default, but Unsloth's # auth layer is HTTPBearer-only. Override via # default_headers so Authorization: Bearer ... is # sent instead. @@ -283,7 +283,7 @@ jobs: print(f"[{label}] OK -- 4 turns, run1 == run2, history grounded") PY - - name: Stop Studio + - name: Stop Unsloth if: always() run: | kill "${STUDIO_PID}" 2>/dev/null || true @@ -363,7 +363,7 @@ jobs: path: gguf-cache key: ${{ runner.os }}-gguf-${{ env.GGUF_REPO }}-${{ env.GGUF_FILE }}-v1 - - name: Install Studio (--local, --no-torch) + - name: Install Unsloth (--local, --no-torch) env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} # Withheld on PR: this step runs checked-out PR code; public GGUF still downloads. @@ -376,7 +376,7 @@ jobs: - name: Assert llama.cpp loads on this macOS run: bash .github/scripts/assert-llama-loads.sh - - name: Reset auth + boot Studio (API-only, default tool policy) + - name: Reset auth + boot Unsloth (API-only, default tool policy) # We deliberately use the API-only mode rather than # `unsloth studio run` because the latter calls # `set_tool_policy(...)` with a resolved bool: on loopback the @@ -478,7 +478,7 @@ jobs: call with enable_tools=true must use this helper. A shared CI runner can stall the stream transport (the - connection opening, or a mid-stream read) even when Studio + connection opening, or a mid-stream read) even when Unsloth is healthy, so harden the read three ways: retry a stall once with a fresh request capped at 300s; return any text already streamed before a stall (a stall on the trailing @@ -574,11 +574,11 @@ jobs: assert status == 200, f"tool call status {status}: {data}" choice = data["choices"][0] tool_calls = (choice.get("message") or {}).get("tool_calls") or [] - # Studio's contract: when tool_choice='required', llama.cpp's + # Unsloth's contract: when tool_choice='required', llama.cpp's # grammar should force a tool_calls payload. On Mac that # contract is sometimes broken by the underlying quant; the # PASS path is "tool_calls present + correct schema", the - # WARN path documents Studio still returned 200 with a + # WARN path documents Unsloth still returned 200 with a # well-formed choices[] envelope. if tool_calls: tc = tool_calls[0] @@ -660,7 +660,7 @@ jobs: print(f"[tools] WARN web_search probe failed (non-blocking): {exc}") # ── 4. Thinking on / off ───────────────────────────────────── - # Studio strips think blocks from message.content for tools-mode + # Unsloth strips think blocks from message.content for tools-mode # responses, so we toggle plain chat (no enable_tools) and look # at the surfaced reasoning_content / message.thinking field. def thinking_call(enable): @@ -678,7 +678,7 @@ jobs: }, timeout = 180) assert status == 200 msg = data["choices"][0]["message"] - # Studio surfaces thinking via reasoning_content (OpenAI + # Unsloth surfaces thinking via reasoning_content (OpenAI # extension). Fall back to inline markers for # robustness across template versions. raw = (msg.get("content") or "") + (msg.get("reasoning_content") or "") @@ -704,7 +704,7 @@ jobs: print(f"[tools] PASS thinking on/off (on={len(on_text)} chars, off={len(off_text)} chars)") PY - - name: Stop Studio + - name: Stop Unsloth if: always() run: | kill "${STUDIO_PID}" 2>/dev/null || true @@ -810,7 +810,7 @@ jobs: path: gguf-cache key: ${{ runner.os }}-gguf-${{ env.GGUF_REPO }}-${{ env.GGUF_FILE }}-${{ env.MMPROJ_FILE }}-v2 - - name: Install Studio (--local, --no-torch) + - name: Install Unsloth (--local, --no-torch) env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} # Withheld on PR: this step runs checked-out PR code; public GGUF still downloads. @@ -826,7 +826,7 @@ jobs: - name: Install OpenAI + Anthropic Python SDKs run: pip install 'openai>=1.50' 'anthropic>=0.40' - - name: Reset auth + boot Studio (API-only) + - name: Reset auth + boot Unsloth (API-only) # See Job 2's comment: API-only mode keeps tool_policy=None so # response_format requests aren't routed through the agentic # tool loop. @@ -929,13 +929,13 @@ jobs: # llama.cpp's HTTP server supports OpenAI-compatible JSON # mode: `response_format: {"type": "json_object"}` constrains # the model to emit syntactically-valid JSON. We use raw HTTP - # rather than the OpenAI SDK so that the field shape Studio + # rather than the OpenAI SDK so that the field shape Unsloth # forwards to llama-server is unambiguous (the SDK rewrites # response_format depending on which variant it recognises). # We deliberately do NOT pass a strict JSON schema -- on # small Gemma-4 quants the GBNF-from-schema path occasionally # produces empty output, and JSON mode is the surface we care - # about exposing through Studio. + # about exposing through Unsloth. status, data = post("/v1/chat/completions", { "model": "default", "messages": [ @@ -1007,7 +1007,7 @@ jobs: ) # ── 2. OpenAI image_url (data URI base64) ─────────────────── - # 64x64 solid-red PNG. stb_image (used by Studio's image + # 64x64 solid-red PNG. stb_image (used by Unsloth's image # normaliser at routes/inference.py:3410) rejects 4x4 or # smaller PNGs as truncated, so we go up to 64x64 -- still # tiny in token cost. The assertion is loose: any non-empty @@ -1023,11 +1023,11 @@ jobs: # The Mac prebuilt llama.cpp server has a known crash when # processing image inputs alongside the gemma-4-E2B mmproj # (server disconnects mid-completion). This is upstream - # llama.cpp behaviour, not Studio. Wrap both SDK calls in + # llama.cpp behaviour, not Unsloth. Wrap both SDK calls in # try/except so an upstream crash registers as a WARN rather - # than failing the whole job. Studio's contract (OpenAI/ + # than failing the whole job. Unsloth's contract (OpenAI/ # Anthropic image fields are accepted and forwarded) is - # validated by the request body Studio constructs, not by + # validated by the request body Unsloth constructs, not by # whether llama.cpp can decode it on Mac Metal. client = OpenAI(base_url = f"{BASE}/v1", api_key = KEY) try: @@ -1053,14 +1053,14 @@ jobs: except Exception as exc: print( f"[image/openai] WARN image_url SDK call raised: {type(exc).__name__}: " - f"{exc}. Likely upstream llama.cpp Mac+vision crash, NOT a Studio " - f"regression. Studio successfully forwarded the request." + f"{exc}. Likely upstream llama.cpp Mac+vision crash, NOT an Unsloth " + f"regression. Unsloth successfully forwarded the request." ) # ── 3. Anthropic source/base64 image ──────────────────────── - # Two SDK quirks vs. Studio: base_url must NOT include /v1 + # Two SDK quirks vs. Unsloth: base_url must NOT include /v1 # (the SDK appends it itself; otherwise /v1/v1/messages -> 405), - # and Studio's auth is HTTPBearer-only so the SDK's default + # and Unsloth's auth is HTTPBearer-only so the SDK's default # x-api-key header is ignored -- send Authorization: Bearer # via default_headers. anthropic = Anthropic( @@ -1099,11 +1099,11 @@ jobs: print( f"[image/anthropic] WARN anthropic image SDK call raised: " f"{type(exc).__name__}: {exc}. Likely upstream llama.cpp Mac+vision " - f"crash, NOT a Studio regression." + f"crash, NOT an Unsloth regression." ) PY - - name: Stop Studio + - name: Stop Unsloth if: always() run: | kill "${STUDIO_PID}" 2>/dev/null || true diff --git a/.github/workflows/studio-mac-install-matrix.yml b/.github/workflows/studio-mac-install-matrix.yml index 362305cdd4..e990f752d4 100644 --- a/.github/workflows/studio-mac-install-matrix.yml +++ b/.github/workflows/studio-mac-install-matrix.yml @@ -1,7 +1,7 @@ # SPDX-License-Identifier: AGPL-3.0-only # Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. -# Proves Studio's llama.cpp install loads on every supported macOS. The heavy +# Proves Unsloth's llama.cpp install loads on every supported macOS. The heavy # app smokes stay single-OS; this matrix covers the OS-version dimension cheaply # (install.sh + binary-load assert). Regression guard for the macOS-version # selection in studio/install_llama_prebuilt.py. @@ -60,7 +60,7 @@ jobs: with: python-version: '3.12' - - name: Install Studio (--local, --no-torch) + - name: Install Unsloth (--local, --no-torch) env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} # Withheld on PR: this step runs checked-out PR code; public GGUF still downloads. diff --git a/.github/workflows/studio-mac-ui-smoke.yml b/.github/workflows/studio-mac-ui-smoke.yml index 20ca247b9f..7375e9bcbf 100644 --- a/.github/workflows/studio-mac-ui-smoke.yml +++ b/.github/workflows/studio-mac-ui-smoke.yml @@ -19,6 +19,7 @@ on: - 'install.sh' - 'pyproject.toml' - 'tests/studio/**' + - '.github/scripts/run-studio-permission-browser.sh' - '.github/workflows/studio-mac-ui-smoke.yml' push: branches: [main, pip] @@ -83,7 +84,7 @@ jobs: path: hf-cache key: ${{ runner.os }}-hf-${{ env.GGUF_REPO }}-${{ env.GGUF_VARIANT }}-v2 - - name: Install Studio (--local, --no-torch) + - name: Install Unsloth (--local, --no-torch) env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} # Withheld on PR: this step runs checked-out PR code; public GGUF still downloads. @@ -96,7 +97,7 @@ jobs: - name: Assert llama.cpp loads on this macOS run: bash .github/scripts/assert-llama-loads.sh - - name: Install Playwright + Chromium + - name: Install Playwright browsers # No --with-deps on Mac: that flag installs Linux apt packages. # GitHub-hosted macos-14 ships the system frameworks Chromium # needs already. @@ -112,7 +113,7 @@ jobs: # in-script retry recover from any residual flakes. run: | pip install 'playwright>=1.55,<1.58' - python -m playwright install chromium + python -m playwright install chromium webkit - name: Patch Playwright pipeTransport.js to tolerate malformed JSON # In Playwright 1.55-1.58, pipeTransport.js does @@ -143,7 +144,7 @@ jobs: print(f"pipeTransport.js: patched JSON.parse calls in {path}") PY - - name: Reset auth + boot Studio + - name: Reset auth + boot Unsloth run: | unsloth studio reset-password mkdir -p logs @@ -188,7 +189,7 @@ jobs: # dies mid-test, (2) Chromium net::ERR_NO_BUFFER_SPACE when the # runner's kernel briefly runs out of socket buffers, and (3) a # goto 'interrupted by another navigation' when the SPA auth - # guard redirects mid-navigation. The retry FULLY resets Studio + # guard redirects mid-navigation. The retry FULLY resets Unsloth # (kill, reset-password, reboot, wait /api/health, re-export # bootstrap pw) before re-running the script. A real test failure # (assertion / timeout) does NOT match any pattern so it bypasses @@ -209,7 +210,7 @@ jobs: || grep -q "ERR_NO_BUFFER_SPACE" logs/playwright_attempt_${attempt}.log \ || grep -q "interrupted by another navigation" logs/playwright_attempt_${attempt}.log; } \ && [ "$attempt" -lt "$max_attempts" ]; then - echo "::warning::Playwright flake on attempt ${attempt}; resetting Studio and retrying..." + echo "::warning::Playwright flake on attempt ${attempt}; resetting Unsloth and retrying..." kill "${STUDIO_PID}" 2>/dev/null || true sleep 2 unsloth studio reset-password @@ -238,13 +239,17 @@ jobs: exit "$rc" done - - name: Stop Studio (chat-ui ends with Shutdown click; this is belt-and-suspenders) + - name: Stop Unsloth (chat-ui ends with Shutdown click; this is belt-and-suspenders) if: always() run: | kill "${STUDIO_PID}" 2>/dev/null || true sleep 2 - - name: Reset auth + boot Studio for extra UI tests (port 18897) + - name: Cross-browser permission controls + run: | + bash .github/scripts/run-studio-permission-browser.sh 18895 webkit + + - name: Reset auth + boot Unsloth for extra UI tests (port 18897) run: | unsloth studio reset-password mkdir -p logs @@ -271,7 +276,7 @@ jobs: echo "STUDIO_EXTRA_OLD_PW=$OLD" >> "$GITHUB_ENV" echo "STUDIO_EXTRA_NEW_PW=$NEW" >> "$GITHUB_ENV" - - name: Drive Compare/Recipes/Export/Studio/Settings with Playwright + - name: Drive Compare/Recipes/Export/Unsloth/Settings with Playwright env: BASE_URL: http://127.0.0.1:18897 STUDIO_OLD_PW: ${{ env.STUDIO_EXTRA_OLD_PW }} @@ -300,7 +305,7 @@ jobs: || grep -q "ERR_NO_BUFFER_SPACE" logs/playwright_extra_attempt_${attempt}.log \ || grep -q "interrupted by another navigation" logs/playwright_extra_attempt_${attempt}.log; } \ && [ "$attempt" -lt "$max_attempts" ]; then - echo "::warning::Playwright flake on attempt ${attempt}; resetting Studio and retrying..." + echo "::warning::Playwright flake on attempt ${attempt}; resetting Unsloth and retrying..." kill "${STUDIO_EXTRA_PID}" 2>/dev/null || true sleep 2 unsloth studio reset-password @@ -327,7 +332,7 @@ jobs: exit "$rc" done - - name: Stop second Studio + - name: Stop second Unsloth if: always() run: | kill "${STUDIO_EXTRA_PID}" 2>/dev/null || true @@ -343,5 +348,7 @@ jobs: logs/studio_extra.log logs/install.log logs/playwright + logs/playwright-permissions-* logs/playwright_extra + logs/studio-permissions-*.log retention-days: 7 diff --git a/.github/workflows/studio-mac-update-smoke.yml b/.github/workflows/studio-mac-update-smoke.yml index d104306c7e..fe9880f3ca 100644 --- a/.github/workflows/studio-mac-update-smoke.yml +++ b/.github/workflows/studio-mac-update-smoke.yml @@ -4,15 +4,15 @@ # Mac counterpart to studio-update-smoke.yml. Verifies that on a real # Apple Silicon (macos-14, M1) runner: # -# 1. install.sh --local --no-torch installs Studio AND auto-fetches +# 1. install.sh --local --no-torch installs Unsloth AND auto-fetches # the prebuilt llama.cpp Mac binary (llama-bNNNN-bin-macos-arm64 # from ggml-org/llama.cpp). Hitting the source-build fallback is -# treated as an Unsloth bug -- Studio must always pick the +# treated as an Unsloth bug -- Unsloth must always pick the # prebuilt on Mac. # 2. unsloth studio update --local is idempotent. Two consecutive # runs both report "prebuilt up to date and validated", no # source-build fallback. -# 3. The installed Studio still boots and /api/health returns +# 3. The installed Unsloth still boots and /api/health returns # healthy after the update path. name: Mac Studio Update CI @@ -42,7 +42,7 @@ permissions: jobs: update-idempotency: - name: Studio Updating Tests + name: Unsloth Updating Tests runs-on: macos-14 timeout-minutes: 30 steps: @@ -59,7 +59,7 @@ jobs: python-version: '3.12' cache: 'pip' - - name: Install Studio (--local, --no-torch) + - name: Install Unsloth (--local, --no-torch) env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} # Withheld on PR: this step runs checked-out PR code; public GGUF still downloads. @@ -106,7 +106,7 @@ jobs: grep -qE "prebuilt up to date and validated|prebuilt installed and validated" logs/update2.log echo "second update was clean" - - name: Boot Studio briefly to confirm the install is still usable + - name: Boot Unsloth briefly to confirm the install is still usable run: | mkdir -p logs UNSLOTH_API_ONLY=1 unsloth studio -H 127.0.0.1 -p 18891 \ @@ -123,13 +123,13 @@ jobs: sleep 1 done if [ -z "$HEALTHY" ]; then - echo "Studio failed to come up after \`update\`" + echo "Unsloth failed to come up after \`update\`" tail -200 logs/studio.log kill "$PID" 2>/dev/null || true exit 1 fi kill "$PID" 2>/dev/null || true - echo "post-update Studio /api/health OK" + echo "post-update Unsloth /api/health OK" - name: Uninstall and verify clean # Round-trip through scripts/uninstall.sh on real macOS. As a side diff --git a/.github/workflows/studio-tauri-smoke.yml b/.github/workflows/studio-tauri-smoke.yml index 018857de68..8e26b9fd0c 100644 --- a/.github/workflows/studio-tauri-smoke.yml +++ b/.github/workflows/studio-tauri-smoke.yml @@ -12,7 +12,7 @@ # stay in release-desktop.yml (manual `workflow_dispatch`) because they need # code-signing secrets and ~30 min of runner time each. -name: Studio Tauri CI +name: Unsloth Tauri CI on: pull_request: diff --git a/.github/workflows/studio-ui-smoke.yml b/.github/workflows/studio-ui-smoke.yml index 297a585430..97eb07b2d8 100644 --- a/.github/workflows/studio-ui-smoke.yml +++ b/.github/workflows/studio-ui-smoke.yml @@ -1,8 +1,8 @@ # SPDX-License-Identifier: AGPL-3.0-only # Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. -# End-to-end Studio chat UI smoke via Playwright + Chromium against a -# headless Linux runner. Boots Studio with the smallest GGUF +# End-to-end Unsloth chat UI smoke via Playwright + Chromium against a +# headless Linux runner. Boots Unsloth with the smallest GGUF # (gemma-3-270m-it UD-Q4_K_XL, ~254 MiB), drives the actual frontend # bundle, and asserts the full bootstrap-password / change-password / # send-message / persist-on-reload journey works end to end. @@ -14,7 +14,7 @@ # frontend-only CI happily pass while the actual user-visible UI is # broken (cf. the 2026.5.1 chat-history release). -name: Studio UI CI +name: Unsloth UI CI on: pull_request: @@ -27,6 +27,7 @@ on: # The Playwright test files themselves -- a PR that ONLY edits # the test must still trigger UI CI. - 'tests/studio/**' + - '.github/scripts/run-studio-permission-browser.sh' - '.github/workflows/studio-ui-smoke.yml' push: branches: [main, pip] @@ -97,7 +98,7 @@ jobs: path: hf-cache key: ${{ runner.os }}-hf-${{ env.GGUF_REPO }}-${{ env.GGUF_VARIANT }}-v2 - - name: Install Studio (--local, --no-torch) + - name: Install Unsloth (--local, --no-torch) env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} # Withheld on PR: this step runs checked-out PR code; public GGUF still downloads. @@ -107,15 +108,12 @@ jobs: set -o pipefail bash install.sh --local --no-torch 2>&1 | tee logs/install.log - - name: Install Playwright + Chromium + - name: Install Playwright browsers run: | pip install 'playwright>=1.45' - # --with-deps installs the OS-level runtime libs Chromium - # needs (libnss3, libxkbcommon, etc.). About 30 s on a - # warm runner. - python -m playwright install --with-deps chromium + python -m playwright install --with-deps chromium firefox webkit - - name: Reset auth + boot Studio + - name: Reset auth + boot Unsloth run: | unsloth studio reset-password mkdir -p logs @@ -147,7 +145,7 @@ jobs: # NEW + NEW2 are generated freshly per CI run via secrets.token_urlsafe # rather than hardcoded. If a workflow gets compromised, the # attacker can't replay a known-good rotated password against - # any future / parallel Studio install -- the rotated value + # any future / parallel Unsloth install -- the rotated value # only ever exists for the lifetime of this single job, masked # in the log via ::add-mask::. run: | @@ -165,29 +163,35 @@ jobs: env: BASE_URL: http://127.0.0.1:18892 # The test file lives in the repo so it can be run locally - # against a freshly-installed Studio (BASE_URL=...; STUDIO_OLD_PW= + # against a freshly-installed Unsloth (BASE_URL=...; STUDIO_OLD_PW= # $(cat ~/.unsloth/studio/auth/.bootstrap_password); python ...). PW_ART_DIR: logs/playwright # Strict mode: in CI a missing button / nav / dialog must # FAIL the test. Locally the test still runs against partial - # Studio installs without STUDIO_UI_STRICT. + # Unsloth installs without STUDIO_UI_STRICT. STUDIO_UI_STRICT: '1' run: | mkdir -p logs/playwright python tests/studio/playwright_chat_ui.py - - name: Stop Studio (chat-ui ends with Shutdown click; this is belt-and-suspenders) + - name: Stop Unsloth (chat-ui ends with Shutdown click; this is belt-and-suspenders) if: always() run: | kill "${STUDIO_PID}" 2>/dev/null || true sleep 2 + - name: Cross-browser permission controls + run: | + bash .github/scripts/run-studio-permission-browser.sh 18893 firefox + bash .github/scripts/run-studio-permission-browser.sh 18893 webkit + bash .github/scripts/run-studio-permission-browser.sh 18893 chromium chrome + # The chat UI test ends by clicking the Shutdown menuitem, which # leaves the server dead. The extra UI test (Compare / Recipes / - # Export / Studio / Settings) needs a fresh Studio, so we boot a + # Export / Unsloth / Settings) needs a fresh Unsloth, so we boot a # second one on a different port. Boot is fast (~3-5s on the # warm install we already did) so this adds little wall time. - - name: Reset auth + boot Studio for extra UI tests (port 18894) + - name: Reset auth + boot Unsloth for extra UI tests (port 18894) run: | unsloth studio reset-password mkdir -p logs @@ -214,7 +218,7 @@ jobs: echo "STUDIO_EXTRA_OLD_PW=$OLD" >> "$GITHUB_ENV" echo "STUDIO_EXTRA_NEW_PW=$NEW" >> "$GITHUB_ENV" - - name: Drive Compare/Recipes/Export/Studio/Settings with Playwright + - name: Drive Compare/Recipes/Export/Unsloth/Settings with Playwright env: BASE_URL: http://127.0.0.1:18894 STUDIO_OLD_PW: ${{ env.STUDIO_EXTRA_OLD_PW }} @@ -227,16 +231,73 @@ jobs: mkdir -p logs/playwright_extra python tests/studio/playwright_extra_ui.py - - name: Stop second Studio + - name: UI font size scaling regression (Playwright) + env: + BASE_URL: http://127.0.0.1:18894 + STUDIO_PW: ${{ env.STUDIO_EXTRA_NEW_PW }} + PW_ART_DIR: logs/playwright_fontscale + run: | + mkdir -p logs/playwright_fontscale + python tests/studio/playwright_ui_font_scale.py + + - name: Stop second Unsloth if: always() run: | kill "${STUDIO_EXTRA_PID}" 2>/dev/null || true sleep 2 + # Model-picker per-model-config regression (PR #7207 re-land of #6647). + # Fourth Unsloth on its own port; loads the tiny GGUF and drives the + # picker's run-settings surface: Context Length persists across a reload, + # Reset clears the stored override (never pins it), and the infra models + # (RAG embedder + llama.cpp probe) stay hidden from the picker. + - name: Reset auth + boot Unsloth for model-config tests (port 18898) + run: | + unsloth studio reset-password + mkdir -p logs + UNSLOTH_API_ONLY=1 unsloth studio -H 127.0.0.1 -p 18898 \ + > logs/studio_modelcfg.log 2>&1 & + echo "STUDIO_MODELCFG_PID=$!" >> "$GITHUB_ENV" + + - name: Wait for /api/health on 18898 + run: | + for i in $(seq 1 180); do + if curl -fs "http://127.0.0.1:18898/api/health" > /tmp/health4.json; then + jq -e '.status == "healthy"' /tmp/health4.json && break + fi + sleep 1 + done + jq -e '.status == "healthy"' /tmp/health4.json + + - name: Pass bootstrap pw for model-config test + run: | + NEW="CIModelCfg-$(python -c 'import secrets; print(secrets.token_urlsafe(16))')" + echo "::add-mask::$NEW" + echo "STUDIO_MODELCFG_NEW_PW=$NEW" >> "$GITHUB_ENV" + + - name: Drive model-picker per-model-config with Playwright + env: + BASE_URL: http://127.0.0.1:18898 + STUDIO_NEW_PW: ${{ env.STUDIO_MODELCFG_NEW_PW }} + PW_ART_DIR: logs/playwright_modelcfg + STUDIO_UI_STRICT: '1' + GGUF_REPO: ${{ env.GGUF_REPO }} + GGUF_VARIANT: ${{ env.GGUF_VARIANT }} + STUDIO_MODEL_HINT: gemma-3-270m + run: | + mkdir -p logs/playwright_modelcfg + python tests/studio/playwright_model_config.py + + - name: Stop fourth Unsloth + if: always() + run: | + kill "${STUDIO_MODELCFG_PID}" 2>/dev/null || true + sleep 2 + # IME + multilingual paste regression (issue #5318 / PR #5327). - # Third Studio on its own port so a hang here cannot poison the + # Third Unsloth on its own port so a hang here cannot poison the # earlier UI tests. No GGUF -- the bug surface is the composer. - - name: Reset auth + boot Studio for IME / i18n tests (port 18896) + - name: Reset auth + boot Unsloth for IME / i18n tests (port 18896) run: | unsloth studio reset-password mkdir -p logs @@ -256,7 +317,7 @@ jobs: - name: Pass bootstrap pw for IME / i18n test # IME smoke does the change-password against the bootstrap that - # Studio's frontend injects into the page, so it only needs the + # Unsloth's frontend injects into the page, so it only needs the # NEW password. run: | NEW="CIIme-$(python -c 'import secrets; print(secrets.token_urlsafe(16))')" @@ -273,7 +334,7 @@ jobs: mkdir -p logs/playwright_ime python tests/studio/playwright_chat_ime_i18n.py - - name: Stop third Studio + - name: Stop third Unsloth if: always() run: | kill "${STUDIO_IME_PID}" 2>/dev/null || true @@ -293,10 +354,15 @@ jobs: path: | logs/studio.log logs/studio_extra.log + logs/studio_modelcfg.log logs/studio_ime.log logs/install.log logs/server-logs/ logs/playwright + logs/playwright-permissions-* logs/playwright_extra + logs/playwright_fontscale + logs/playwright_modelcfg logs/playwright_ime + logs/studio-permissions-*.log retention-days: 7 diff --git a/.github/workflows/studio-update-smoke.yml b/.github/workflows/studio-update-smoke.yml index 08a79afacd..625c2c7811 100644 --- a/.github/workflows/studio-update-smoke.yml +++ b/.github/workflows/studio-update-smoke.yml @@ -9,7 +9,7 @@ # This catches regressions in setup.sh's update path that the existing # GGUF / wheel jobs would miss because they only invoke install.sh once. -name: Studio Update CI +name: Unsloth Update CI on: pull_request: @@ -36,7 +36,7 @@ permissions: jobs: update-idempotency: - name: Studio Updating Tests + name: Unsloth Updating Tests runs-on: ubuntu-latest timeout-minutes: 15 steps: @@ -63,7 +63,7 @@ jobs: # post-step then fatal-errors with "Cache folder path is # retrieved for pip but doesn't exist on disk". - - name: Install Studio (--local, --no-torch) + - name: Install Unsloth (--local, --no-torch) # Pass the workflow token so the llama.cpp prebuilt installer's # GitHub-API call to list releases isn't rate-limited (60/hr # unauthenticated). Without this, three consecutive install + @@ -122,7 +122,7 @@ jobs: grep -qE "prebuilt up to date and validated|prebuilt installed and validated" logs/update2.log echo "second update was clean" - - name: Boot Studio briefly to confirm the install is still usable + - name: Boot Unsloth briefly to confirm the install is still usable # If `update --local` accidentally broke the venv or wiped the # llama-server binary, the server would fail to start here. run: | @@ -138,13 +138,13 @@ jobs: sleep 1 done if ! jq -e '.status == "healthy"' /tmp/health.json 2>/dev/null; then - echo "Studio failed to come up after `update`" + echo "Unsloth failed to come up after `update`" tail -200 logs/studio.log kill "$PID" 2>/dev/null || true exit 1 fi kill "$PID" 2>/dev/null || true - echo "post-update Studio /api/health OK" + echo "post-update Unsloth /api/health OK" - name: Uninstall and verify clean # Round-trip the installer through scripts/uninstall.sh: confirms the diff --git a/.github/workflows/studio-windows-api-smoke.yml b/.github/workflows/studio-windows-api-smoke.yml index e9abd2d669..6dbcceebbd 100644 --- a/.github/workflows/studio-windows-api-smoke.yml +++ b/.github/workflows/studio-windows-api-smoke.yml @@ -9,7 +9,7 @@ # (Section 6) is Linux-only and short-circuits on non-POSIX; the rest # is platform-portable. -name: Windows Studio API CI +name: Windows Unsloth API CI on: pull_request: @@ -34,7 +34,7 @@ permissions: jobs: api-smoke: - name: Studio API & Auth Tests + name: Unsloth API & Auth Tests runs-on: windows-latest timeout-minutes: 30 defaults: @@ -105,7 +105,7 @@ jobs: # studio-windows-update-smoke.yml for the full rationale -- # creating an empty studio/frontend/dist trips setup.ps1's # mtime-based staleness check into "frontend up to date, skip - # rebuild" and Studio boots with an empty dist directory. + # rebuild" and Unsloth boots with an empty dist directory. # Add-MpPreference accepts paths that do not yet exist. foreach ($p in @( "$env:USERPROFILE\.unsloth", @@ -121,7 +121,7 @@ jobs: } } - - name: Install Studio (--local, --no-torch) + - name: Install Unsloth (--local, --no-torch) shell: pwsh env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} @@ -161,7 +161,7 @@ jobs: echo "install.ps1 installed the Windows prebuilt llama.cpp:" cat "$INFO" - - name: Add Studio shim to GITHUB_PATH + - name: Add Unsloth shim to GITHUB_PATH # install.ps1's User-PATH update doesn't propagate to a # running Git Bash session; export the shim dir so the # next `unsloth ...` invocation finds it. @@ -177,7 +177,7 @@ jobs: - name: Install pyjwt for the JWT-expiry forge test run: python -m pip install 'pyjwt>=2.6' - - name: Reset auth + boot Studio (API-only) + - name: Reset auth + boot Unsloth (API-only) run: | unsloth studio reset-password mkdir -p logs @@ -207,7 +207,7 @@ jobs: echo "STUDIO_NEW_PW=$NEW" >> "$GITHUB_ENV" echo "STUDIO_NEW2_PW=$NEW2" >> "$GITHUB_ENV" - - name: Run Studio API & Auth tests + - name: Run Unsloth API & Auth tests # Do NOT pin STUDIO_AUTH_DIR here. The Mac/Linux mirrors # hardcode runner-specific paths (/Users/runner/..., # /home/runner/...), but on Windows the path is @@ -219,7 +219,7 @@ jobs: BASE_URL: http://127.0.0.1:18895 run: python tests/studio/studio_api_smoke.py - - name: Stop Studio + - name: Stop Unsloth if: always() run: | kill "${STUDIO_PID}" 2>/dev/null || true diff --git a/.github/workflows/studio-windows-inference-smoke.yml b/.github/workflows/studio-windows-inference-smoke.yml index 63a7e9dc8f..3ebe442f52 100644 --- a/.github/workflows/studio-windows-inference-smoke.yml +++ b/.github/workflows/studio-windows-inference-smoke.yml @@ -1,7 +1,7 @@ # SPDX-License-Identifier: AGPL-3.0-only # Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. -# Three end-to-end smoke jobs that boot a freshly-installed Studio and +# Three end-to-end smoke jobs that boot a freshly-installed Unsloth and # exercise the surfaces real users hit through the OpenAI / Anthropic # SDKs and curl, on the FREE windows-latest runner. Each job picks the # smallest model that exercises the behaviour under test, primes @@ -16,7 +16,7 @@ # Qwen3-VL-2B-Instruct UD-IQ2_XXS + mmproj-F16 (~1.4 GiB total). # Within the 14 GB windows-latest SSD budget. -name: Windows Studio GGUF CI +name: Windows Unsloth GGUF CI on: pull_request: @@ -57,7 +57,7 @@ jobs: STUDIO_PORT: '18888' HF_HOME: ${{ github.workspace }}/hf-cache # Force UTF-8 for stdio (Windows defaults to cp1252; hf - # download / Studio CLI print "✓" checkmarks and crash + # download / Unsloth CLI print "✓" checkmarks and crash # otherwise). PYTHONIOENCODING: utf-8 PYTHONUTF8: '1' @@ -160,7 +160,7 @@ jobs: # studio-windows-update-smoke.yml for the full rationale -- # creating an empty studio/frontend/dist trips setup.ps1's # mtime-based staleness check into "frontend up to date, skip - # rebuild" and Studio boots with an empty dist directory. + # rebuild" and Unsloth boots with an empty dist directory. # Add-MpPreference accepts paths that do not yet exist. foreach ($p in @( "$env:USERPROFILE\.unsloth", @@ -176,7 +176,7 @@ jobs: } } - - name: Install Studio (--local, --no-torch) + - name: Install Unsloth (--local, --no-torch) shell: pwsh env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} @@ -214,7 +214,7 @@ jobs: echo "install.ps1 installed the Windows prebuilt llama.cpp:" cat "$INFO" - - name: Add Studio shim to GITHUB_PATH + - name: Add Unsloth shim to GITHUB_PATH run: | SHIM_DIR=~/.unsloth/studio/bin if [ ! -f "$SHIM_DIR/unsloth.exe" ]; then @@ -227,7 +227,7 @@ jobs: - name: Install OpenAI + Anthropic Python SDKs run: python -m pip install 'openai>=1.50' 'anthropic>=0.40' - - name: Reset auth + boot Studio (API-only) + - name: Reset auth + boot Unsloth (API-only) run: | unsloth studio reset-password mkdir -p logs @@ -244,7 +244,7 @@ jobs: fi sleep 1 done - echo "Studio did not become healthy in 180s" + echo "Unsloth did not become healthy in 180s" tail -200 logs/studio.log exit 1 @@ -281,7 +281,7 @@ jobs: # Retry the load step a few times so a transient TCP RST during # llama-server warm-up (Windows runner image churn, # windows-latest -> windows-2025-vs2026 rollout) doesn't fail - # the whole job. The Studio backend's _wait_for_health now + # the whole job. The Unsloth backend's _wait_for_health now # catches httpx.ReadError too; this retry layer covers the # cases the backend can't recover from on its own. LOAD_OK=0 @@ -382,15 +382,15 @@ jobs: print(f"[{label}] OK -- 4 turns, run1 == run2, history grounded") PY - - name: Stop Studio + - name: Stop Unsloth if: always() # Run as cmd so we are not running through the Git Bash shell; # Git Bash on windows-latest has been observed to exit 143 # (SIGTERM) from any inline kill/sleep block, masking a green - # test run. The runner reclaims the Studio child process at + # test run. The runner reclaims the Unsloth child process at # job end either way, so just emit a marker and exit 0. shell: cmd - run: echo Stop Studio (no-op; runner reclaims STUDIO_PID=%STUDIO_PID% at job end) + run: echo Stop Unsloth (no-op; runner reclaims STUDIO_PID=%STUDIO_PID% at job end) - name: Collect llama-server logs if: always() @@ -398,10 +398,10 @@ jobs: # copy must not fail an otherwise-green job. continue-on-error: true shell: bash - # Copy llama-server's own stdout/stderr (teed by Studio under + # Copy llama-server's own stdout/stderr (teed by Unsloth under # ~/.unsloth/studio/logs/llama-server/) into the workspace so # upload-artifact can pick it up. Crucial for diagnosing a - # subprocess crash where Studio's traceback only shows the + # subprocess crash where Unsloth's traceback only shows the # symptom (httpx ReadError) but not the cause. run: | mkdir -p logs/llama-server @@ -439,14 +439,14 @@ jobs: # (211 s on first run; subsequent runs hit the cache, but the # one-time cost recurs every time the cache key bumps). Use # main's `--local-dir gguf-cache` pattern: cache the flat .gguf - # only, pass an absolute path to Studio's /api/inference/load. + # only, pass an absolute path to Unsloth's /api/inference/load. # The OpenAI/Anth and JSON+images jobs still cover the # gguf_variant resolution path. GGUF_REPO: unsloth/Qwen3.5-2B-GGUF GGUF_FILE: Qwen3.5-2B-UD-Q4_K_XL.gguf STUDIO_PORT: '18898' # Force UTF-8 for stdio (Windows defaults to cp1252; hf - # download / Studio CLI print "✓" checkmarks and crash + # download / Unsloth CLI print "✓" checkmarks and crash # otherwise). PYTHONIOENCODING: utf-8 PYTHONUTF8: '1' @@ -507,7 +507,7 @@ jobs: # studio-windows-update-smoke.yml for the full rationale -- # creating an empty studio/frontend/dist trips setup.ps1's # mtime-based staleness check into "frontend up to date, skip - # rebuild" and Studio boots with an empty dist directory. + # rebuild" and Unsloth boots with an empty dist directory. # Add-MpPreference accepts paths that do not yet exist. foreach ($p in @( "$env:USERPROFILE\.unsloth", @@ -523,7 +523,7 @@ jobs: } } - - name: Install Studio (--local, --no-torch) + - name: Install Unsloth (--local, --no-torch) shell: pwsh env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} @@ -561,7 +561,7 @@ jobs: echo "install.ps1 installed the Windows prebuilt llama.cpp:" cat "$INFO" - - name: Add Studio shim to GITHUB_PATH + - name: Add Unsloth shim to GITHUB_PATH run: | SHIM_DIR=~/.unsloth/studio/bin if [ ! -f "$SHIM_DIR/unsloth.exe" ]; then @@ -571,7 +571,7 @@ jobs: fi cygpath -w "$SHIM_DIR" >> "$GITHUB_PATH" - - name: Reset auth + boot Studio (API-only, default tool policy) + - name: Reset auth + boot Unsloth (API-only, default tool policy) run: | unsloth studio reset-password mkdir -p logs @@ -607,7 +607,7 @@ jobs: # raw string, but we cannot embed `\a` etc. in JSON without # JSON-string-escaping every backslash. Replace `\` with `/` # via bash parameter expansion -- pathlib.Path on Windows - # accepts forward slashes natively, so Studio's loader sees + # accepts forward slashes natively, so Unsloth's loader sees # a normal path. GGUF_PATH="${GITHUB_WORKSPACE//\\//}/gguf-cache/${GGUF_FILE}" ls -lh "$GGUF_PATH" @@ -680,7 +680,7 @@ jobs: def post_sse(path, body, *, timeout = 600, retries = 1, soft = False): # The server-side agentic loop always answers over SSE. A # shared CI runner can stall the stream transport (the - # connection opening, or a mid-stream read) even when Studio + # connection opening, or a mid-stream read) even when Unsloth # is healthy, so harden the read three ways: # * retry a transport stall once with a fresh request, # capped at 300s (a healthy server answers a retry @@ -882,15 +882,15 @@ jobs: print(f"[tools] PASS thinking on/off (on={len(on_text)} chars, off={len(off_text)} chars)") PY - - name: Stop Studio + - name: Stop Unsloth if: always() # Run as cmd so we are not running through the Git Bash shell; # Git Bash on windows-latest has been observed to exit 143 # (SIGTERM) from any inline kill/sleep block, masking a green - # test run. The runner reclaims the Studio child process at + # test run. The runner reclaims the Unsloth child process at # job end either way, so just emit a marker and exit 0. shell: cmd - run: echo Stop Studio (no-op; runner reclaims STUDIO_PID=%STUDIO_PID% at job end) + run: echo Stop Unsloth (no-op; runner reclaims STUDIO_PID=%STUDIO_PID% at job end) - name: Collect llama-server logs if: always() @@ -898,10 +898,10 @@ jobs: # copy must not fail an otherwise-green job. continue-on-error: true shell: bash - # Copy llama-server's own stdout/stderr (teed by Studio under + # Copy llama-server's own stdout/stderr (teed by Unsloth under # ~/.unsloth/studio/logs/llama-server/) into the workspace so # upload-artifact can pick it up. Crucial for diagnosing a - # subprocess crash where Studio's traceback only shows the + # subprocess crash where Unsloth's traceback only shows the # symptom (httpx ReadError) but not the cause. run: | mkdir -p logs/llama-server @@ -939,7 +939,7 @@ jobs: STUDIO_PORT: '18899' HF_HOME: ${{ github.workspace }}/hf-cache # Force UTF-8 for stdio (Windows defaults to cp1252; hf - # download / Studio CLI print "✓" checkmarks and crash + # download / Unsloth CLI print "✓" checkmarks and crash # otherwise). PYTHONIOENCODING: utf-8 PYTHONUTF8: '1' @@ -1005,7 +1005,7 @@ jobs: # studio-windows-update-smoke.yml for the full rationale -- # creating an empty studio/frontend/dist trips setup.ps1's # mtime-based staleness check into "frontend up to date, skip - # rebuild" and Studio boots with an empty dist directory. + # rebuild" and Unsloth boots with an empty dist directory. # Add-MpPreference accepts paths that do not yet exist. foreach ($p in @( "$env:USERPROFILE\.unsloth", @@ -1021,7 +1021,7 @@ jobs: } } - - name: Install Studio (--local, --no-torch) + - name: Install Unsloth (--local, --no-torch) shell: pwsh env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} @@ -1059,7 +1059,7 @@ jobs: echo "install.ps1 installed the Windows prebuilt llama.cpp:" cat "$INFO" - - name: Add Studio shim to GITHUB_PATH + - name: Add Unsloth shim to GITHUB_PATH run: | SHIM_DIR=~/.unsloth/studio/bin if [ ! -f "$SHIM_DIR/unsloth.exe" ]; then @@ -1072,7 +1072,7 @@ jobs: - name: Install OpenAI + Anthropic Python SDKs run: python -m pip install 'openai>=1.50' 'anthropic>=0.40' - - name: Reset auth + boot Studio (API-only) + - name: Reset auth + boot Unsloth (API-only) run: | unsloth studio reset-password mkdir -p logs @@ -1262,7 +1262,7 @@ jobs: except Exception as exc: print( f"[image/openai] WARN image_url SDK call raised: {type(exc).__name__}: " - f"{exc}. Studio successfully forwarded the request; failure here is " + f"{exc}. Unsloth successfully forwarded the request; failure here is " f"upstream llama.cpp vision behaviour." ) @@ -1303,19 +1303,19 @@ jobs: print( f"[image/anthropic] WARN anthropic image SDK call raised: " f"{type(exc).__name__}: {exc}. Likely upstream llama.cpp vision " - f"behaviour, NOT a Studio regression." + f"behaviour, NOT an Unsloth regression." ) PY - - name: Stop Studio + - name: Stop Unsloth if: always() # Run as cmd so we are not running through the Git Bash shell; # Git Bash on windows-latest has been observed to exit 143 # (SIGTERM) from any inline kill/sleep block, masking a green - # test run. The runner reclaims the Studio child process at + # test run. The runner reclaims the Unsloth child process at # job end either way, so just emit a marker and exit 0. shell: cmd - run: echo Stop Studio (no-op; runner reclaims STUDIO_PID=%STUDIO_PID% at job end) + run: echo Stop Unsloth (no-op; runner reclaims STUDIO_PID=%STUDIO_PID% at job end) - name: Collect llama-server logs if: always() @@ -1323,10 +1323,10 @@ jobs: # copy must not fail an otherwise-green job. continue-on-error: true shell: bash - # Copy llama-server's own stdout/stderr (teed by Studio under + # Copy llama-server's own stdout/stderr (teed by Unsloth under # ~/.unsloth/studio/logs/llama-server/) into the workspace so # upload-artifact can pick it up. Crucial for diagnosing a - # subprocess crash where Studio's traceback only shows the + # subprocess crash where Unsloth's traceback only shows the # symptom (httpx ReadError) but not the cause. run: | mkdir -p logs/llama-server @@ -1348,7 +1348,7 @@ jobs: # ── folded from studio-windows-no-vs-smoke.yml: install + run with no Visual Studio ── no-vs-cpu: - name: Studio install + inference without Visual Studio + name: Unsloth install + inference without Visual Studio runs-on: windows-latest timeout-minutes: 35 defaults: @@ -1502,7 +1502,7 @@ jobs: python -m pip install torch --index-url https://download.pytorch.org/whl/cpu --extra-index-url https://pypi.org/simple python -c "import torch; print('torch', torch.__version__, 'cuda?', torch.cuda.is_available())" - - name: Install Studio (--local, --no-torch) with no build tools present + - name: Install Unsloth (--local, --no-torch) with no build tools present shell: pwsh env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} @@ -1538,13 +1538,13 @@ jobs: echo "Prebuilt installed with no build tools:" cat "$INFO" - - name: Add Studio shim to GITHUB_PATH + - name: Add Unsloth shim to GITHUB_PATH run: | SHIM_DIR=~/.unsloth/studio/bin [ -f "$SHIM_DIR/unsloth.exe" ] || { echo "::error::unsloth.exe shim not found"; ls -la ~/.unsloth/studio/ || true; exit 1; } cygpath -w "$SHIM_DIR" >> "$GITHUB_PATH" - - name: Reset auth + boot Studio (API-only) + - name: Reset auth + boot Unsloth (API-only) run: | unsloth studio reset-password mkdir -p logs @@ -1613,10 +1613,10 @@ jobs: } Remove-Item -LiteralPath $root -Recurse -Force -ErrorAction SilentlyContinue - - name: Stop Studio + - name: Stop Unsloth if: always() shell: cmd - run: echo Stop Studio (no-op; runner reclaims STUDIO_PID=%STUDIO_PID% at job end) + run: echo Stop Unsloth (no-op; runner reclaims STUDIO_PID=%STUDIO_PID% at job end) - name: Collect llama-server logs if: always() diff --git a/.github/workflows/studio-windows-ui-smoke.yml b/.github/workflows/studio-windows-ui-smoke.yml index 405309916a..f401f7be44 100644 --- a/.github/workflows/studio-windows-ui-smoke.yml +++ b/.github/workflows/studio-windows-ui-smoke.yml @@ -4,11 +4,11 @@ # Windows counterpart to studio-ui-smoke.yml / studio-mac-ui-smoke.yml. # Same Playwright + Chromium end-to-end chat UI flow + extra UI flow, # but on the FREE windows-latest runner so we catch Windows-specific -# regressions in the install path (install.ps1), the Studio CLI's +# regressions in the install path (install.ps1), the Unsloth CLI's # Windows process-management branches, and the llama.cpp prebuilt's # Windows HTTP layer. -name: Windows Studio UI CI +name: Windows Unsloth UI CI on: pull_request: @@ -19,6 +19,7 @@ on: - 'install.ps1' - 'pyproject.toml' - 'tests/studio/**' + - '.github/scripts/run-studio-permission-browser.sh' - '.github/workflows/studio-windows-ui-smoke.yml' push: branches: [main, pip] @@ -49,7 +50,7 @@ jobs: GGUF_FILE: gemma-3-270m-it-UD-Q4_K_XL.gguf STUDIO_PORT: '18896' HF_HOME: ${{ github.workspace }}/hf-cache - # Force UTF-8 for stdio so Python tools (hf download, Studio + # Force UTF-8 for stdio so Python tools (hf download, Unsloth # CLI, etc.) can print Unicode characters like the success # checkmark "✓". Windows defaults to cp1252 / charmap and # any tool that prints "OK ✓" hits a UnicodeEncodeError. @@ -121,7 +122,7 @@ jobs: # studio-windows-update-smoke.yml for the full rationale -- # creating an empty studio/frontend/dist trips setup.ps1's # mtime-based staleness check into "frontend up to date, skip - # rebuild" and Studio boots with an empty dist directory. + # rebuild" and Unsloth boots with an empty dist directory. # Add-MpPreference accepts paths that do not yet exist. foreach ($p in @( "$env:USERPROFILE\.unsloth", @@ -148,7 +149,7 @@ jobs: Set-Content -LiteralPath (Join-Path $appDir 'launch-studio.vbs') -Value 'WScript.Echo "legacy"' -Encoding Unicode Write-Host "seeded legacy launch-studio.vbs at $appDir" - - name: Install Studio (--local, --no-torch) + - name: Install Unsloth (--local, --no-torch) # install.ps1 is the supported Windows installer. install.sh # has no Windows branch (apt-get / brew calls). The PS1 # script's `Install-UnslothStudio @args` line at the bottom @@ -205,7 +206,7 @@ jobs: echo "install.ps1 installed the Windows prebuilt llama.cpp:" cat "$INFO" - - name: Assert Studio launcher chain (no VBS, hidden PowerShell shortcut) + - name: Assert Unsloth launcher chain (no VBS, hidden PowerShell shortcut) # The shortcut launch path is otherwise untested here (the steps below # boot `unsloth studio` directly). Guard against re-introducing the VBS # that tripped Kaspersky HEUR:Trojan.VBS.Agent.gen and against the .lnk @@ -234,7 +235,7 @@ jobs: } Write-Host "launcher chain OK (no VBS; hidden powershell over launch-studio.ps1)" - - name: Launch Studio via the shortcut and assert health + - name: Launch Unsloth via the shortcut and assert health # Run the exact command the .lnk stores (hidden PowerShell over # launch-studio.ps1) and confirm it brings the backend up. This is the # only step that proves the shortcut launch is not silently broken. @@ -265,10 +266,10 @@ jobs: $owner = (Get-NetTCPConnection -LocalPort $foundPort -State Listen -ErrorAction Stop | Select-Object -First 1).OwningProcess if ($owner) { taskkill /PID $owner /T /F 2>$null | Out-Null } } catch {} - if (-not $foundPort) { throw "Studio did not become healthy when launched via the shortcut" } - Write-Host "Studio healthy on port $foundPort (launched via the shortcut)" + if (-not $foundPort) { throw "Unsloth did not become healthy when launched via the shortcut" } + Write-Host "Unsloth healthy on port $foundPort (launched via the shortcut)" - - name: Add Studio shim to GITHUB_PATH + - name: Add Unsloth shim to GITHUB_PATH # install.ps1 puts unsloth.exe at $StudioHome\bin\unsloth.exe # and adds that dir to the User PATH via the Windows registry. # Registry-level PATH updates don't propagate to a running @@ -284,7 +285,7 @@ jobs: fi # GITHUB_PATH wants Windows-style paths; convert via cygpath. cygpath -w "$SHIM_DIR" >> "$GITHUB_PATH" - echo "Added Studio shim dir to PATH: $(cygpath -w "$SHIM_DIR")" + echo "Added Unsloth shim dir to PATH: $(cygpath -w "$SHIM_DIR")" - name: Install Playwright + Chromium # No --with-deps on Windows: that flag installs Linux apt @@ -294,7 +295,7 @@ jobs: python -m pip install 'playwright>=1.45' python -m playwright install chromium - - name: Reset auth + boot Studio + - name: Reset auth + boot Unsloth run: | unsloth studio reset-password mkdir -p logs @@ -339,13 +340,17 @@ jobs: mkdir -p logs/playwright python tests/studio/playwright_chat_ui.py - - name: Stop Studio (chat-ui ends with Shutdown click; this is belt-and-suspenders) + - name: Stop Unsloth (chat-ui ends with Shutdown click; this is belt-and-suspenders) if: always() run: | kill "${STUDIO_PID}" 2>/dev/null || true sleep 2 - - name: Reset auth + boot Studio for extra UI tests (port 18897) + - name: Edge permission controls + run: | + bash .github/scripts/run-studio-permission-browser.sh 18895 chromium msedge + + - name: Reset auth + boot Unsloth for extra UI tests (port 18897) run: | unsloth studio reset-password mkdir -p logs @@ -372,7 +377,7 @@ jobs: echo "STUDIO_EXTRA_OLD_PW=$OLD" >> "$GITHUB_ENV" echo "STUDIO_EXTRA_NEW_PW=$NEW" >> "$GITHUB_ENV" - - name: Drive Compare/Recipes/Export/Studio/Settings with Playwright + - name: Drive Compare/Recipes/Export/Unsloth/Settings with Playwright env: BASE_URL: http://127.0.0.1:18897 STUDIO_OLD_PW: ${{ env.STUDIO_EXTRA_OLD_PW }} @@ -386,7 +391,7 @@ jobs: mkdir -p logs/playwright_extra python tests/studio/playwright_extra_ui.py - - name: Stop second Studio + - name: Stop second Unsloth if: always() run: | kill "${STUDIO_EXTRA_PID}" 2>/dev/null || true @@ -402,5 +407,7 @@ jobs: logs/studio_extra.log logs/install.log logs/playwright + logs/playwright-permissions-* logs/playwright_extra + logs/studio-permissions-*.log retention-days: 7 diff --git a/.github/workflows/studio-windows-update-smoke.yml b/.github/workflows/studio-windows-update-smoke.yml index 5b92f1a3e0..42d74d47d2 100644 --- a/.github/workflows/studio-windows-update-smoke.yml +++ b/.github/workflows/studio-windows-update-smoke.yml @@ -5,19 +5,19 @@ # studio-mac-update-smoke.yml. Verifies that on the FREE # windows-latest runner: # -# 1. install.ps1 --local --no-torch installs Studio AND auto-fetches +# 1. install.ps1 --local --no-torch installs Unsloth AND auto-fetches # the prebuilt llama.cpp Windows binary (app--windows-x64-cpu # from unslothai/llama.cpp). Hitting the source-build fallback is -# treated as an Unsloth bug -- Studio must always pick the +# treated as an Unsloth bug -- Unsloth must always pick the # prebuilt on Windows. # 2. unsloth studio update --local is idempotent. Two consecutive # runs both report "prebuilt up to date and validated", no # source-build fallback. The CLI's _find_setup_script picks # setup.ps1 on Windows automatically. -# 3. The installed Studio still boots and /api/health returns +# 3. The installed Unsloth still boots and /api/health returns # healthy after the update path. -name: Windows Studio Update CI +name: Windows Unsloth Update CI on: pull_request: @@ -45,7 +45,7 @@ permissions: jobs: update-idempotency: - name: Studio Updating Tests + name: Unsloth Updating Tests runs-on: windows-latest timeout-minutes: 30 defaults: @@ -53,7 +53,7 @@ jobs: shell: bash env: # Force UTF-8 for stdio (Windows defaults to cp1252; hf - # download / Studio CLI print "✓" checkmarks and crash + # download / Unsloth CLI print "✓" checkmarks and crash # otherwise). PYTHONIOENCODING: utf-8 PYTHONUTF8: '1' @@ -90,7 +90,7 @@ jobs: # reuses the existing Node with no download. # # (2) Defender. windows-latest's real-time scan opens / hashes - # every file Studio writes during install (Vite output = + # every file Unsloth writes during install (Vite output = # thousands of small chunks, uv pip = wheel-extraction = # thousands of small files). The latency dominates the # 200 s frontend build and the 90 s deps install. Adding @@ -109,7 +109,7 @@ jobs: # setup.ps1 line 1281-1296's mtime-based "is the frontend # stale?" check into "up to date, skip rebuild", because the # newly-created dist's mtime is younger than every source - # file. Studio then boots with an empty dist and 500s on + # file. Unsloth then boots with an empty dist and 500s on # GET / with FileNotFoundError: dist\index.html. See run # 25546676715 / job 74984469728. # Add-MpPreference accepts paths that do not yet exist; the @@ -129,7 +129,7 @@ jobs: } } - - name: Install Studio (--local, --no-torch) + - name: Install Unsloth (--local, --no-torch) shell: pwsh env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} @@ -168,7 +168,7 @@ jobs: echo "install.ps1 installed the Windows prebuilt llama.cpp:" cat "$INFO" - - name: Add Studio shim to GITHUB_PATH + - name: Add Unsloth shim to GITHUB_PATH run: | SHIM_DIR=~/.unsloth/studio/bin if [ ! -f "$SHIM_DIR/unsloth.exe" ]; then @@ -212,7 +212,7 @@ jobs: grep -qE "prebuilt up to date and validated|prebuilt installed and validated" logs/update2.log echo "second update was clean" - - name: Boot Studio briefly to confirm the install is still usable + - name: Boot Unsloth briefly to confirm the install is still usable run: | mkdir -p logs UNSLOTH_API_ONLY=1 unsloth studio -H 127.0.0.1 -p 18891 \ @@ -239,13 +239,13 @@ jobs: sleep 1 done if [ -z "$HEALTHY" ]; then - echo "Studio failed to come up after \`update\`" + echo "Unsloth failed to come up after \`update\`" tail -200 logs/studio.log kill "$PID" 2>/dev/null || true exit 1 fi kill "$PID" 2>/dev/null || true - echo "post-update Studio /api/health OK" + echo "post-update Unsloth /api/health OK" - name: Uninstall and verify clean # Round-trip through scripts/uninstall.ps1 against the default diff --git a/.github/workflows/wheel-smoke.yml b/.github/workflows/wheel-smoke.yml index 3de3c33ca2..cdad617027 100644 --- a/.github/workflows/wheel-smoke.yml +++ b/.github/workflows/wheel-smoke.yml @@ -3,7 +3,7 @@ # Builds the PyPI wheel from the PR branch, then verifies the built wheel # actually contains what we expect to ship and does NOT contain the broken -# Studio bundle that 2026.5.1 published. This is the single workflow that +# Unsloth bundle that 2026.5.1 published. This is the single workflow that # would have blocked the 2026.5.1 release before twine upload. # # Verified locally end-to-end against this branch: @@ -12,7 +12,7 @@ # lockfile shipped, frontend dist shipped, # no node_modules in wheel, no bun.lock in wheel, # main bundle has unstable_Provider hits=1 (assistant-ui internals only). -# - Studio backend imports cleanly from the installed wheel with the +# - Unsloth backend imports cleanly from the installed wheel with the # lightweight dep set below. name: Wheel CI @@ -101,7 +101,7 @@ jobs: hits = data.count("unstable_Provider:") print(f"main bundle: {js[0]}") print(f"unstable_Provider hits: {hits} (>=4 indicates 2026.5.1 regression)") - checks["bundle has no Studio unstable_Provider call site"] = (hits < 4) + checks["bundle has no Unsloth unstable_Provider call site"] = (hits < 4) print() for k, v in checks.items(): @@ -109,7 +109,7 @@ jobs: sys.exit(0 if all(checks.values()) else 1) PY - - name: Studio backend import smoke + - name: Unsloth backend import smoke # Imports `studio.backend.main:app` from the freshly-installed wheel in # a clean venv. This catches the class of bug that 2026.5.1 shipped with: # frontend dist missing, package-lock.json missing, or the wheel's Python @@ -125,7 +125,7 @@ jobs: /tmp/v/bin/pip install --no-deps dist/unsloth-*.whl # Run from /tmp so Python imports the installed package, not the source tree. cd /tmp - /tmp/v/bin/python -c "from studio.backend.main import app; print('Studio backend OK:', app.title)" + /tmp/v/bin/python -c "from studio.backend.main import app; print('Unsloth backend OK:', app.title)" - name: Upload wheel on failure if: failure() diff --git a/README.md b/README.md index ef45b91430..514454f985 100644 --- a/README.md +++ b/README.md @@ -11,6 +11,7 @@ Unsloth Studio lets you run and train models locally.

Features • + NewsQuickstartNotebooksDocumentation @@ -47,15 +48,51 @@ Unsloth Studio (Beta) lets you run and train text, [audio](https://unsloth.ai/do * [Auto set inference settings](https://unsloth.ai/docs/new/studio/chat#auto-parameter-tuning) and customize chat templates. * We work directly with teams behind [gpt-oss](https://docs.unsloth.ai/new/gpt-oss-how-to-run-and-fine-tune#unsloth-fixes-for-gpt-oss), [Qwen3](https://www.reddit.com/r/LocalLLaMA/comments/1kaodxu/qwen3_unsloth_dynamic_ggufs_128k_context_bug_fixes/), [Llama 4](https://github.com/ggml-org/llama.cpp/pull/12889), [Mistral](https://huggingface.co/mistralai/Mistral-Medium-3.5-128B/discussions/18), [Gemma 1-3](https://news.ycombinator.com/item?id=39671146), and [Phi-4](https://unsloth.ai/blog/phi4), where we’ve fixed bugs that improve model accuracy. * Chat with images, audio, PDFs, code, DOCX and more. [Connect API providers](https://unsloth.ai/docs/integrations/connections) (OpenAI, Anthropic) or servers (vLLM, Ollama). +* [**Compare any two models**](https://unsloth.ai/docs/new/studio/chat#model-arena) side by side with the same prompt. +* **OpenAI/Anthropic-compatible APIs**: Serve local models through `/v1/chat/completions`, `/v1/responses` and `/v1/messages`. +* **Connect local models to agents**: Use `unsloth start` with Claude Code, Codex, Hermes and more. +* **Web/PDF search** can read PDF papers, manuals and other PDF results. +* **GGUF hardware controls**: Choose GPUs/layers, offload MoE experts, use multi-GPU or Tensor Parallelism. +* The opt-in **MCP control endpoint** lets AI clients manage models, training, recipes and exports. ### Training -* Train and RL **500+ models** up to **2x faster** with up to **70% less VRAM**, with no accuracy loss. -* Custom Triton and mathematical **kernels**. See some collabs we did with [PyTorch](https://unsloth.ai/docs/get-started/reinforcement-learning-rl-guide/fp8-reinforcement-learning) and [Hugging Face](https://unsloth.ai/docs/new/faster-moe). +* Train and RL **500+ models** up to **2x faster** with **70% less VRAM**; MoE up to **12x faster**. +* Train and run RL on [AMD GPUs](https://unsloth.ai/docs/basics/amd) across Windows, WSL and Linux. * **Data Recipes**: [Auto-create datasets](https://unsloth.ai/docs/new/studio/data-recipe) from **PDF, CSV, DOCX** etc. Edit data in a visual-node workflow. -* **[Reinforcement Learning](https://unsloth.ai/docs/get-started/reinforcement-learning-rl-guide)** (RL): The most efficient [RL](https://unsloth.ai/docs/get-started/reinforcement-learning-rl-guide) library, using **80% less VRAM** for GRPO, [FP8](https://unsloth.ai/docs/get-started/reinforcement-learning-rl-guide/fp8-reinforcement-learning) etc. -* Supports full fine-tuning, RL, pretraining, 4-bit, 16-bit and, FP8 training. +* **[Reinforcement Learning](https://unsloth.ai/docs/get-started/reinforcement-learning-rl-guide)** uses **80% less VRAM** for GRPO, FP8 and vision RL, with 7x longer contexts. +* [**Long-context training**](https://unsloth.ai/docs/new/3x-faster-training-packing): **3x faster**, 30% less VRAM and 500K+ context. +* Supports LoRA/QLoRA, full fine-tuning, RL, pretraining, 4-bit, 16-bit and FP8. +* Custom Triton and mathematical **kernels** built with PyTorch and Hugging Face. * **Observability**: Monitor training live, track loss and GPU usage and customize graphs. * [Multi-GPU](https://unsloth.ai/docs/basics/multi-gpu-training-with-unsloth) training is supported, with major improvements coming soon. +## 🚀 Unsloth Start + +[Unsloth Start](https://unsloth.ai/docs/integrations/unsloth-start) connects [Claude Code](https://unsloth.ai/docs/basics/claude-code), [Codex](https://unsloth.ai/docs/basics/codex) and other agents to local models with one command. + +Start Unsloth, load a model, open your project folder, then run: + +```bash +unsloth start claude +``` + +Replace `claude` with any supported agent: + +| Agent | Command | +| --- | --- | +| Claude Code | `unsloth start claude` | +| OpenAI Codex | `unsloth start codex` | +| Hermes Agent | `unsloth start hermes` | +| OpenClaw | `unsloth start openclaw` | +| OpenCode | `unsloth start opencode` | +| Pi Coding Agent | `unsloth start pi` | + +Claude Code, Codex, OpenCode and Pi can keep their current model and use Unsloth as a local +subagent: + +```bash +unsloth start claude --as-subagent --model unsloth/model-GGUF:quant +``` + ## 📥 Install Unsloth can be used in two ways: through **[Unsloth Studio](https://unsloth.ai/docs/new/studio/)**, the web UI, or through **Unsloth Core**, the code-based version. Each has different requirements. @@ -65,7 +102,8 @@ Unsloth Studio (Beta) works on **Windows, Linux, WSL** and **macOS**. * **CPU:** Supported for Chat and Data Recipes currently * **NVIDIA:** Training works on RTX 30/40/50, Blackwell, DGX Spark, Station and more * **macOS:** Training, MLX and GGUF inference are ALL supported. -* **AMD:** Chat + Data works. Train with [Unsloth Core](#unsloth-core-code-based). Studio support is out soon. +* **AMD:** Training, RL, chat and deployment work on Windows, WSL and Linux. [Read the AMD guide](https://unsloth.ai/docs/basics/amd). +* **Vulkan:** GGUF inference is supported on [compatible GPUs, including Intel GPUs](https://github.com/unslothai/unsloth/pull/5819). * **Multi-GPU:** Available now, with a major upgrade on the way #### macOS, Linux, WSL: @@ -86,7 +124,7 @@ unsloth studio -p 8888 ``` For LAN or cloud access, add `-H 0.0.0.0` (raw port only; add `--cloudflare` for a public URL). By default, Unsloth is accessible only locally. -To reach Studio over HTTPS, use `unsloth studio --secure`. Studio stays bound to localhost and is reached only through a free Cloudflare tunnel, which publishes it at a public `https://*.trycloudflare.com` URL (it fails closed if the tunnel can't start, so the raw port is never exposed). This makes Studio reachable from the internet, so anyone with the link and API key can use it and run code: keep your API key private (see Remote access below). +To reach Unsloth over HTTPS, use `unsloth studio --secure`. Unsloth stays bound to localhost and is reached only through a free Cloudflare tunnel, which publishes it at a public `https://*.trycloudflare.com` URL (it fails closed if the tunnel can't start, so the raw port is never exposed). This makes Unsloth reachable from the internet, so anyone with the link and API key can use it and run code: keep your API key private (see Remote access below). #### Docker Use our [Docker image](https://hub.docker.com/r/unsloth/unsloth) ```unsloth/unsloth``` container. Run: @@ -122,7 +160,7 @@ You can use the same Docker image as Unsloth Studio. #### AMD, Intel: For RTX 50x, B200, 6000 GPUs: `uv pip install unsloth --torch-backend=auto`. Read our guides for: [Blackwell](https://unsloth.ai/docs/blog/fine-tuning-llms-with-blackwell-rtx-50-series-and-unsloth) and [DGX Spark](https://unsloth.ai/docs/blog/fine-tuning-llms-with-nvidia-dgx-spark-and-unsloth).
-To install Unsloth on **AMD** and **Intel** GPUs, follow our [AMD Guide](https://unsloth.ai/docs/get-started/install/amd) and [Intel Guide](https://unsloth.ai/docs/get-started/install/intel). +To install Unsloth on **AMD** and **Intel** GPUs, follow our [AMD Guide](https://unsloth.ai/docs/basics/amd) and [Intel Guide](https://unsloth.ai/docs/get-started/install/intel). ## 📒 Free Notebooks @@ -148,13 +186,20 @@ Read our [guide](https://unsloth.ai/docs/get-started/fine-tuning-llms-guide). Ad - See detailed documentation for Unsloth [here](https://unsloth.ai/docs) ## 🦥 Unsloth News -- **Connections**: Connect any API provider (OpenAI, Anthropic) or server (vLLM, Ollama). [Guide](https://unsloth.ai/docs/integrations/connections) -- **MTP**: Run Qwen3.6 MTP in Unsloth. MTP settings are autoset specific to your hardware. [Guide](https://unsloth.ai/docs/models/qwen3.6#mtp-guide) -- **API inference endpoint**: Deploy and run local LLMs in Claude Code, Codex tools. [Guide](https://unsloth.ai/docs/basics/api) -- **Qwen3.6**: Qwen3.6-35B-A3B can now be trained and run in Unsloth Studio. [Blog](https://unsloth.ai/docs/models/qwen3.6) -- **Gemma 4**: Run and train Google’s new models directly in Unsloth. [Blog](https://unsloth.ai/docs/models/gemma-4) +- **AMD training**: Train, run RL, chat and deploy on AMD GPUs across Windows, WSL and Linux. [Guide](https://unsloth.ai/docs/basics/amd) +- **GGUF hardware controls**: Choose GPU/layer placement, offload MoE experts and use multi-GPU or Tensor Parallelism. [#6414](https://github.com/unslothai/unsloth/pull/6414) +- **Local models for any agent**: Use `unsloth start` with Claude Code, Codex, Hermes, OpenCode, OpenClaw, Pi and more through Unsloth's OpenAI- and Anthropic-compatible APIs. [Guide](https://unsloth.ai/docs/basics/api) +- **MCP control endpoint**: Let compatible clients manage models, training, recipes, checkpoints and exports. [#7191](https://github.com/unslothai/unsloth/pull/7191) +- **Local inference reliability**: Resume long chats faster, recover stalled downloads and reuse existing GGUF files. [#7204](https://github.com/unslothai/unsloth/pull/7204) • [#6858](https://github.com/unslothai/unsloth/pull/6858) • [#7209](https://github.com/unslothai/unsloth/pull/7209) +- **New models**: [Qwen-AgentWorld](https://huggingface.co/unsloth/Qwen-AgentWorld-35B-A3B-GGUF), [Ornith](https://huggingface.co/unsloth/models?search=ornith), [Kimi K2.7 Code](https://unsloth.ai/docs/models/kimi-k2.7-code) and [MiniMax M3](https://unsloth.ai/docs/models/minimax-m3) +- **GLM-5.2**: Run Z.ai's 744B-parameter, 1M-context open model locally with Unsloth Dynamic GGUFs. [Guide](https://unsloth.ai/docs/models/glm-5.2) +- **DeepSeek-V4**: Run DeepSeek-V4-Flash locally with corrected multi-turn and tool-calling behavior. [Guide](https://unsloth.ai/docs/models/deepseek-v4) +- **DiffusionGemma**: Run and fine-tune Google's diffusion language model with 1.8x faster inference in Unsloth Studio. [Guide](https://unsloth.ai/docs/models/diffusiongemma) +- **Qwen3.6**: Run and train Qwen3.6 with MTP for 1.4-2.2x faster inference and NVFP4 quants for supported GPUs. [Guide](https://unsloth.ai/docs/models/qwen3.6) +- **Gemma 4**: Run and train Gemma 4 text, image and audio models with QAT, MTP, GGUF and MLX support. [Guide](https://unsloth.ai/docs/models/gemma-4) +- **MCP servers**: Connect local models to files, apps, databases and external tools through Model Context Protocol. [Guide](https://unsloth.ai/docs/basics/mcp) +- **Connections**: Mix local models with API providers (OpenAI, Anthropic) or servers (vLLM, Ollama) in the same interface. [Guide](https://unsloth.ai/docs/integrations/connections) - **Introducing Unsloth Studio**: our new web UI for running and training LLMs. [Blog](https://unsloth.ai/docs/new/studio) -- **Qwen3.5** - 0.8B, 2B, 4B, 9B, 27B, 35-A3B, 112B-A10B are now supported. [Guide + notebooks](https://unsloth.ai/docs/models/qwen3.5/fine-tune) - Train **MoE LLMs 12x faster** with 35% less VRAM - DeepSeek, GLM, Qwen and gpt-oss. [Blog](https://unsloth.ai/docs/new/faster-moe) - **Embedding models**: Unsloth now supports ~1.8-3.3x faster embedding fine-tuning. [Blog](https://unsloth.ai/docs/new/embedding-finetuning) • [Notebooks](https://unsloth.ai/docs/get-started/unsloth-notebooks#embedding-models) - New **7x longer context RL** vs. all other setups, via our new batching algorithms. [Blog](https://unsloth.ai/docs/new/grpo-long-context) @@ -208,7 +253,7 @@ unsloth studio -p 8888 #### Remote access: `--secure` (HTTPS tunnel) vs raw port By default `unsloth studio` binds to `127.0.0.1` (this machine only). To reach it from another device, pick one of: -- `--secure` (recommended): serve **only** through a free Cloudflare HTTPS link. Studio stays bound to localhost and the tunnel provides the public URL; it fails closed (does not start) if the tunnel can't come up, so the raw port is never exposed. +- `--secure` (recommended): serve **only** through a free Cloudflare HTTPS link. Unsloth stays bound to localhost and the tunnel provides the public URL; it fails closed (does not start) if the tunnel can't come up, so the raw port is never exposed. ```bash unsloth studio --secure -p 8888 ``` @@ -218,7 +263,7 @@ unsloth studio -H 0.0.0.0 -p 8888 ``` The Cloudflare tunnel is **off by default**: `-H 0.0.0.0` exposes the raw port only, not a public internet URL. Pair the wildcard bind with `--cloudflare` (`unsloth studio -H 0.0.0.0 --cloudflare`) to also publish a public `https://*.trycloudflare.com` link, or prefer `--secure` (above), which keeps the raw port private. `--cloudflare` has no effect on a loopback bind. -The first time Studio is published on a public URL (`--secure` or `--cloudflare`) with the auto-generated admin password still in place, it asks for a new admin password in the terminal (masked input with confirmation) before the public link goes up. Without an attached terminal it warns instead and keeps the bootstrap deadline: Studio shuts down after `UNSLOTH_STUDIO_BOOTSTRAP_TIMEOUT` (default 1 hour) unless the password is changed in the web UI. +The first time Unsloth is published on a public URL (`--secure` or `--cloudflare`) with the auto-generated admin password still in place, it asks for a new admin password in the terminal (masked input with confirmation) before the public link goes up. Without an attached terminal it warns instead and keeps the bootstrap deadline: Unsloth shuts down after `UNSLOTH_STUDIO_BOOTSTRAP_TIMEOUT` (default 1 hour) unless the password is changed in the web UI. For headless setups that cannot answer that prompt, set the initial admin password non-interactively with `--password` (only takes effect when no password is set yet; if one already exists it is a hard error, so rotate later with `unsloth studio reset-password`): @@ -230,7 +275,7 @@ printf '%s\n' 'your-strong-password' | unsloth studio --secure --password - # A literal `--password VALUE` is visible in the process list and shell history, so prefer the `UNSLOTH_STUDIO_PASSWORD` env var or `--password -` (stdin) for automation. This applies to any launch (public or a headless `-H 0.0.0.0` bind), and the password is set in the parent before the server binds, so it never reaches a re-executed child process. -Server-side tools (web search, Python and terminal code execution) run as your user and are on by default. Anyone who can reach the server with the API key can run code on this machine, so keep your API key private and pass `--disable-tools` when exposing Studio. +Server-side tools (web search, Python and terminal code execution) run as your user and are on by default. Anyone who can reach the server with the API key can run code on this machine, so keep your API key private and pass `--disable-tools` when exposing Unsloth. #### Advanced launch options Installer options can be passed as environment variables. On macOS, Linux and WSL place the variable after the pipe so the shell passes it to `sh`; on Windows set it with `$env:` before piping to `iex`. @@ -243,7 +288,7 @@ curl -fsSL https://unsloth.ai/install.sh | UNSLOTH_NO_TORCH=1 sh $env:UNSLOTH_NO_TORCH=1; irm https://unsloth.ai/install.ps1 | iex ``` -Skip the post-install prompt that starts Studio (useful for automated installs): +Skip the post-install prompt that starts Unsloth (useful for automated installs): ```bash curl -fsSL https://unsloth.ai/install.sh | UNSLOTH_SKIP_AUTOSTART=1 sh ``` @@ -279,9 +324,9 @@ UNSLOTH_NPM_REGISTRY=https://artifactory.example.com/api/npm/npm/ ./install.sh - ```powershell $env:UNSLOTH_NPM_REGISTRY='https://artifactory.example.com/api/npm/npm/'; .\install.ps1 --local ``` -It is threaded as `--registry` into the Studio frontend `npm`/`bun` installs; the supply-chain locks (7-day `min-release-age`, exact version pins) stay in force. +It is threaded as `--registry` into the Unsloth frontend `npm`/`bun` installs; the supply-chain locks (7-day `min-release-age`, exact version pins) stay in force. -Cap Studio's native CPU thread pools on high-core hosts: `UNSLOTH_CPU_THREADS=8 unsloth studio -p 8888`. +Cap Unsloth's native CPU thread pools on high-core hosts: `UNSLOTH_CPU_THREADS=8 unsloth studio -p 8888`. #### Uninstall The recommended way to fully remove Unsloth Studio is the matching uninstall script for your OS. It stops any running servers, removes the install dir, the launcher data dir, the desktop shortcut, and any platform-specific entries (macOS `.app` bundle + Launch Services on Mac; Start Menu, `HKCU\Software\Unsloth` registry key and user `PATH` entries on Windows): diff --git a/build.sh b/build.sh index dc272f0de1..2a836e19d9 100644 --- a/build.sh +++ b/build.sh @@ -4,9 +4,9 @@ set -euo pipefail -# PyPI/Studio release publishing must use `./build.sh publish` (or an -# equivalent stamp -> build -> verify-dist -> upload flow) so packaged Studio -# artifacts include the display-only Studio release version. +# PyPI/Unsloth release publishing must use `./build.sh publish` (or an +# equivalent stamp -> build -> verify-dist -> upload flow) so packaged Unsloth +# artifacts include the display-only Unsloth release version. # 1. Build frontend (Vite outputs to dist/) cd studio/frontend @@ -87,7 +87,7 @@ cd ../.. # 2. Clean old artifacts rm -rf build dist *.egg-info -# 3. Stamp display-only Studio release metadata for packaged builds. +# 3. Stamp display-only Unsloth release metadata for packaged builds. _STUDIO_BUILD_INFO="studio/backend/utils/_studio_release_build.py" _STUDIO_BUILD_INFO_BACKUP="$(mktemp)" cp "$_STUDIO_BUILD_INFO" "$_STUDIO_BUILD_INFO_BACKUP" diff --git a/install.ps1 b/install.ps1 index c25d7e7b7f..9c91d4ba16 100644 --- a/install.ps1 +++ b/install.ps1 @@ -53,7 +53,8 @@ function Install-UnslothStudio { param([string]$TorchIndexUrl) if ($SkipTorch) { return "none" } if ([string]::IsNullOrWhiteSpace($TorchIndexUrl)) { return "none" } - $leaf = ($TorchIndexUrl.TrimEnd('/') -split '/')[-1].ToLowerInvariant() + # Drop query/fragment first so a token-authenticated pin classifies by family. + $leaf = (($TorchIndexUrl -split '[?#]', 2)[0].TrimEnd('/') -split '/')[-1].ToLowerInvariant() if (@("cpu", "cu118", "cu124", "cu126", "cu128", "cu130") -contains $leaf) { return $leaf } if ($leaf -match '^rocm[0-9]+\.[0-9]+$') { return $leaf } return "auto" @@ -62,7 +63,8 @@ function Install-UnslothStudio { function Get-TauriGpuBranch { param([string]$TorchIndexFamily) if ($SkipTorch) { return "no_torch" } - if ($TorchIndexFamily -like "cu*") { return "cuda" } + # Require a digit after "cu" so /current or /custom isn't branded CUDA (parity ^cu[0-9]). + if ($TorchIndexFamily -match '^cu[0-9]') { return "cuda" } if ($TorchIndexFamily -like "rocm*") { return "rocm" } if ($TorchIndexFamily -eq "cpu") { return "cpu" } return "unknown" @@ -176,7 +178,7 @@ function Install-UnslothStudio { $envOverride = $env:STUDIO_HOME.Trim() } - # Custom Studio roots are not supported with --tauri (desktop app still + # Custom Unsloth roots are not supported with --tauri (desktop app still # resolves %USERPROFILE%\.unsloth\studio). Pass through if override == legacy. if ($TauriMode -and $envOverride) { $_tauriOverride = $envOverride @@ -467,22 +469,35 @@ function Install-UnslothStudio { } } + # Redact index-URL credentials (userinfo + ?query= + #fragment) from captured installer + # output before printing on failure; uv/pip errors echo the failing --index-url verbatim. + # Mirrors the other installers. Verbose mode streams uncaptured, so it isn't redacted. + function Redact-InstallOutput { + param([string]$Text) + if (-not $Text) { return $Text } + $Text = $Text -replace '(https?://)[^/@\s`]+@', '$1@' + $Text = $Text -replace '([?&][^=\s&`]+)=[^&#\s`]+', '$1=' + # A #token=... fragment is as sensitive as a query; URL-anchored. + return $Text -replace '(https?://[^\s`#]+)#[^\s`]+', '$1#' + } + # Run native commands quietly by default to match install.sh behavior. # Full command output is shown only when --verbose / UNSLOTH_VERBOSE=1. function Invoke-InstallCommand { param( [Parameter(Mandatory = $true)][ScriptBlock]$Command ) - # Installer-pinned index installs (torch) must beat an inherited uv mirror - # (#6898): when the command pins an index, clear every uv index env var so - # it wins, then restore in finally. Other installs keep the user's mirror. + # Installer-pinned index installs (torch) must beat an inherited uv mirror (#6898): + # for --default-index, clear the uv index env vars (restore in finally) and set + # UV_NO_CONFIG=1 so a uv.toml/pyproject index can't outrank the CLI pin (uv 0.10). $savedUvIndex = $null if ($Command.ToString() -match '--default-index') { $savedUvIndex = @{} - foreach ($n in 'UV_DEFAULT_INDEX', 'UV_INDEX_URL', 'UV_INDEX', 'UV_EXTRA_INDEX_URL') { + foreach ($n in 'UV_DEFAULT_INDEX', 'UV_INDEX_URL', 'UV_INDEX', 'UV_EXTRA_INDEX_URL', 'UV_TORCH_BACKEND', 'UV_FIND_LINKS', 'UV_CONFIG_FILE', 'UV_NO_CONFIG') { $savedUvIndex[$n] = [Environment]::GetEnvironmentVariable($n) Remove-Item "Env:$n" -ErrorAction SilentlyContinue } + $env:UV_NO_CONFIG = '1' } $prevEap = $ErrorActionPreference $ErrorActionPreference = "Continue" @@ -493,17 +508,23 @@ function Install-UnslothStudio { # Merge stderr into stdout so progress/warning output stays visible # without flipping $? on successful native commands (PS 5.1 treats # stderr records as errors that set $? = $false even on exit code 0). - & $Command 2>&1 | Out-Host + # Redact per record: uv echoes index URLs (credentials and all) in + # its errors, and verbose mode must not bypass the quiet path's + # redaction. ForEach-Object/Out-Host leave $LASTEXITCODE untouched. + & $Command 2>&1 | ForEach-Object { Redact-InstallOutput "$_" } | Out-Host } else { $output = & $Command 2>&1 | Out-String if ($LASTEXITCODE -ne 0) { - Write-Host $output -ForegroundColor Red + Write-Host (Redact-InstallOutput $output) -ForegroundColor Red } } return [int]$LASTEXITCODE } finally { $ErrorActionPreference = $prevEap - if ($savedUvIndex) { foreach ($n in $savedUvIndex.Keys) { if ($null -ne $savedUvIndex[$n]) { Set-Item "Env:$n" $savedUvIndex[$n] } } } + if ($savedUvIndex) { + Remove-Item "Env:UV_NO_CONFIG" -ErrorAction SilentlyContinue + foreach ($n in $savedUvIndex.Keys) { if ($null -ne $savedUvIndex[$n]) { Set-Item "Env:$n" $savedUvIndex[$n] } } + } } } @@ -756,7 +777,7 @@ function Find-FreeLaunchPort { return `$null } -# If Studio is already healthy on any expected port, just open it and exit. +# If Unsloth is already healthy on any expected port, just open it and exit. `$existingPort = Find-HealthyStudioPort if (`$existingPort) { Start-Process "http://localhost:`$existingPort" @@ -772,7 +793,7 @@ try { `$haveMutex = `$true } if (-not `$haveMutex) { - # Another launcher is already running; wait for it to bring Studio up + # Another launcher is already running; wait for it to bring Unsloth up `$deadline = (Get-Date).AddSeconds(`$timeoutSec) while ((Get-Date) -lt `$deadline) { `$port = Find-HealthyStudioPort @@ -1395,13 +1416,82 @@ exit 0 $suffix++ $candidate = Join-Path $StudioHome "unsloth_studio.rollback.$stamp.$PID.$suffix" } - Move-Item -LiteralPath $ExistingDir -Destination $candidate -ErrorAction Stop $script:StudioVenvRollbackDir = $candidate $script:StudioVenvRollbackTarget = $ExistingDir $script:StudioVenvRollbackActive = $true + # Publish the rollback state before the atomic rename so interruption + # cannot land after Move-Item but before cleanup knows where the old venv went. + try { + Move-Item -LiteralPath $ExistingDir -Destination $candidate -ErrorAction Stop + } catch { + # A collision or ordinary rename failure leaves the original in place. + # Keep state active only when the rename happened before interruption. + if (Test-Path -LiteralPath $ExistingDir) { + $script:StudioVenvRollbackActive = $false + $script:StudioVenvRollbackDir = $null + } + throw + } substep "previous environment preserved for rollback" } + function Remove-StudioVenvTreeWithRetry { + param( + [Parameter(Mandatory = $true)][string]$Path, + [Parameter(Mandatory = $true)][string]$Label + ) + $lastError = $null + for ($attempt = 1; $attempt -le 3; $attempt++) { + try { + Remove-Item -LiteralPath $Path -Recurse -Force -ErrorAction Stop + } catch { + $lastError = $_.Exception.Message + } + if (-not (Test-Path -LiteralPath $Path)) { return $true } + if ($attempt -lt 3) { Start-Sleep -Milliseconds (250 * $attempt) } + } + Write-Host "[WARN] Could not remove $Label at $Path" -ForegroundColor Yellow + if ($lastError) { Write-Host " $lastError" -ForegroundColor Yellow } + return $false + } + + function Test-StudioVenvRollbackMustBePreserved { + param([Parameter(Mandatory = $true)][System.IO.FileSystemInfo]$Rollback) + # Preserve anything outside the installer's timestamp.PID[.suffix] format. + if ($Rollback.Name -notmatch '^unsloth_studio\.rollback\.[0-9]{14}\.([0-9]+)(?:\.[0-9]+)?$') { + return $true + } + $ownerPid = 0 + if (-not [int]::TryParse($Matches[1], [ref]$ownerPid)) { return $true } + if ($ownerPid -eq $PID) { return $true } + return $null -ne (Get-Process -Id $ownerPid -ErrorAction SilentlyContinue) + } + + function Remove-StaleStudioVenvRollbacks { + try { + $rollbacks = @( + Get-ChildItem -LiteralPath $StudioHome -Directory -Force -ErrorAction Stop | + Where-Object { $_.Name -like 'unsloth_studio.rollback.*' } + ) + } catch { + Write-Host "[WARN] Could not inspect stale environment rollbacks in $StudioHome" -ForegroundColor Yellow + Write-Host " $($_.Exception.Message)" -ForegroundColor Yellow + return + } + foreach ($rollback in $rollbacks) { + if (($rollback.Attributes -band [System.IO.FileAttributes]::ReparsePoint) -ne 0) { + Write-Host "[WARN] Refusing to remove rollback reparse point $($rollback.FullName)" -ForegroundColor Yellow + continue + } + # A concurrent installer may have moved its live venv aside. The PID + # in the generated name keeps this run from deleting its rescue copy. + if (Test-StudioVenvRollbackMustBePreserved -Rollback $rollback) { continue } + if (Remove-StudioVenvTreeWithRetry -Path $rollback.FullName -Label "stale environment rollback") { + substep "removed stale environment rollback $($rollback.Name)" + } + } + } + function Restore-StudioVenvRollback { if (-not $script:StudioVenvRollbackActive) { return } $backup = $script:StudioVenvRollbackDir @@ -1413,7 +1503,9 @@ exit 0 substep "restoring previous environment after failed install..." "Yellow" try { if (Test-Path -LiteralPath $target) { - Remove-Item -LiteralPath $target -Recurse -Force -ErrorAction SilentlyContinue + if (-not (Remove-StudioVenvTreeWithRetry -Path $target -Label "incomplete environment")) { + throw "Could not remove incomplete environment at $target" + } } Move-Item -LiteralPath $backup -Destination $target -Force -ErrorAction Stop substep "restored previous environment" @@ -1428,17 +1520,21 @@ exit 0 function Complete-StudioVenvRollback { if (-not $script:StudioVenvRollbackActive) { return } $backup = $script:StudioVenvRollbackDir - if ($backup -and (Test-Path -LiteralPath $backup)) { - Remove-Item -LiteralPath $backup -Recurse -Force -ErrorAction SilentlyContinue - } + # The replacement is committed. Disable restoration before deleting the + # backup so interruption cannot restore a partially deleted environment. $script:StudioVenvRollbackActive = $false $script:StudioVenvRollbackDir = $null + if ($backup -and (Test-Path -LiteralPath $backup)) { + Remove-StudioVenvTreeWithRetry -Path $backup -Label "environment rollback" | Out-Null + } } + $studioVenvReplacementCommitted = $false + try { if (Test-Path -LiteralPath $VenvPython) { # why: matching guard to the .venv branch below -- in env-mode # $StudioHome is a user-chosen workspace, so refuse to nuke an - # existing $StudioHome\unsloth_studio that lacks Studio sentinels. + # existing $StudioHome\unsloth_studio that lacks Unsloth sentinels. # -PathType Leaf rejects a directory at the sentinel path. Accept the # in-VENV ownership marker so partial-install retries are not blocked. if ( @@ -1449,7 +1545,7 @@ exit 0 ) { Write-Host "[ERROR] $VenvDir already exists but does not look like an Unsloth Studio install." -ForegroundColor Red Write-Host " Move it aside or choose an empty UNSLOTH_STUDIO_HOME." -ForegroundColor Yellow - throw "Refusing to delete non-Studio venv at $VenvDir" + throw "Refusing to delete non-Unsloth venv at $VenvDir" } # New layout already exists -- replace only after preserving rollback copy. substep "preserving existing environment for rollback..." @@ -1468,7 +1564,7 @@ exit 0 # workspace root (e.g. user's existing project Python venv). $OldVenv = Join-Path $StudioHome ".venv" $OldPy = Join-Path $OldVenv "Scripts\python.exe" - substep "found legacy Studio environment, validating..." + substep "found legacy Unsloth environment, validating..." $prevEAP2 = $ErrorActionPreference $ErrorActionPreference = "Continue" try { @@ -1498,7 +1594,7 @@ exit 0 # Skip in env-mode so we don't relocate the default-install venv into # the workspace root. $CwdVenv = Join-Path $env:USERPROFILE "unsloth_studio" - substep "found CWD-relative Studio environment, migrating to $VenvDir..." + substep "found CWD-relative Unsloth environment, migrating to $VenvDir..." Move-Item -LiteralPath $CwdVenv -Destination $VenvDir -Force substep "moved ~/unsloth_studio -> ~/.unsloth/studio/unsloth_studio" $_Migrated = $true @@ -1517,7 +1613,7 @@ exit 0 substep "$VenvDir" } - # Mark the freshly-created venv as Studio-owned so a partial install can be + # Mark the freshly-created venv as Unsloth-owned so a partial install can be # repaired by re-running install.ps1; the env-mode deletion guard above # accepts this marker as the primary sentinel. if (Test-Path -LiteralPath $VenvDir -PathType Container) { @@ -1526,7 +1622,7 @@ exit 0 # ── Helper: run amd-smi without triggering a UAC elevation prompt ── # amd-smi on Windows auto-elevates to read GPU/APU memory, surfacing a confusing - # DiskPart UAC prompt mid-install (Studio backend amd.py hits the same). + # DiskPart UAC prompt mid-install (Unsloth backend amd.py hits the same). # __COMPAT_LAYER=RunAsInvoker forces it (and helpers it spawns) to run # un-elevated; on failure the WMI name -> gfx fallback still resolves the arch. function Invoke-AmdSmiNoElevate { @@ -1653,7 +1749,7 @@ exit 0 function Test-HipinfoIsVenvInternal { param([AllowNull()][string]$HipinfoPath) if ([string]::IsNullOrWhiteSpace($HipinfoPath)) { return $false } - # Also derive the venv from the setup python + default Studio home, so + # Also derive the venv from the setup python + default Unsloth home, so # the venv hipInfo is caught when VenvDir/VIRTUAL_ENV are unset. $venvRoots = @() if ($env:VIRTUAL_ENV) { $venvRoots += $env:VIRTUAL_ENV } @@ -1663,7 +1759,7 @@ exit 0 try { $venvRoots += (Split-Path -Parent (Split-Path -Parent $env:UNSLOTH_SETUP_PYTHON)) } catch {} } if ($env:USERPROFILE) { $venvRoots += (Join-Path $env:USERPROFILE ".unsloth\studio\unsloth_studio") } - # A custom Studio home (UNSLOTH_STUDIO_HOME / STUDIO_HOME alias) moves the + # A custom Unsloth home (UNSLOTH_STUDIO_HOME / STUDIO_HOME alias) moves the # venv off the default path; seed it too or its hipInfo escapes the filter. $studioHomeEnv = if (-not [string]::IsNullOrWhiteSpace($env:UNSLOTH_STUDIO_HOME)) { $env:UNSLOTH_STUDIO_HOME.Trim() } elseif (-not [string]::IsNullOrWhiteSpace($env:STUDIO_HOME)) { $env:STUDIO_HOME.Trim() } else { $null } if ($studioHomeEnv) { @@ -1823,7 +1919,7 @@ exit 0 $nameArchTable = @( @{ P = "9070 XT|9080"; A = "gfx1201" } # RDNA 4 (RX 9070 XT / 9080) @{ P = "9070|9060"; A = "gfx1200" } # RDNA 4 (RX 9070 / 9060) - @{ P = "8060S|8050S|8040S|Strix Halo|Ryzen AI Max|AI Max"; A = "gfx1151" } # RDNA 3.5 (Strix Halo: Radeon 8060S/8050S/8040S iGPU, Ryzen AI Max+) + @{ P = "8065S|8060S|8050S|8040S|Strix Halo|Ryzen AI Max|AI Max"; A = "gfx1151" } # RDNA 3.5 (Strix Halo + Gorgon Halo: Radeon 8065S/8060S/8050S/8040S iGPU, Ryzen AI Max / Max+) @{ P = "890M|880M|860M|840M|Strix Point|Krackan|HX 37[05]|AI 9 HX|AI 9 36[05]|AI 7 35[05]|AI 5 34[05]|AI 7 PRO 35|AI 5 33"; A = "gfx1150" } # RDNA 3.5 (Strix/Krackan Point: Radeon 890M/880M iGPU, Ryzen AI 9 HX 370/375) @{ P = "RX 7900|RX 7800|RX 7700(?!S)|PRO W7900|PRO W7800|PRO W7700"; A = "gfx1100" } # RDNA 3 desktop/workstation (Navi 31) @{ P = "RX 7600|RX 7700S|RX 7650|PRO W7600|PRO W7500|PRO V710"; A = "gfx1102" } # RDNA 3 (Navi 33) @@ -1942,7 +2038,7 @@ exit 0 substep " Ensure the ROCm compute driver is installed alongside the display driver:" "Yellow" substep " https://rocm.docs.amd.com/en/latest/deploy/windows/index.html" "Yellow" } elseif ($ROCmGfxArch) { - # Known arch: Studio setup installs AMD's bundled-runtime ROCm PyTorch wheels + # Known arch: Unsloth setup installs AMD's bundled-runtime ROCm PyTorch wheels # (repo.amd.com), which ship their own runtime -- HIP SDK optional. step "gpu" "AMD ROCm ($ROCmGfxArch)" "Cyan" substep "Detected: $ROCmGpuLabel" "Cyan" @@ -1960,10 +2056,31 @@ exit 0 # On an AMD GPU (no NVIDIA), surface the optional WSL-ROCm driver hint. if (-not $HasNvidiaSmi -and ($ROCmGfxArch -or $ROCmGpuLabel)) { Show-AmdWslDriverHint } + # Trim trailing slashes from the URL PATH only, preserving ?query / #fragment: a whole-URL + # TrimEnd corrupts a token ending in "/", a single strip leaves .../cu128// empty. Shared. + function Trim-IndexPathSlashes { + param([string]$Url) + $value = $Url.Trim() + $idx = $value.IndexOfAny([char[]]@('?', '#')) + if ($idx -lt 0) { + return $value.TrimEnd('/') + } + return $value.Substring(0, $idx).TrimEnd('/') + $value.Substring($idx) + } + # ── Choose the correct PyTorch index URL based on driver CUDA version ── # Mirrors Get-PytorchCudaTag in setup.ps1. function Get-TorchIndexUrl { $baseUrl = if ($env:UNSLOTH_PYTORCH_MIRROR) { $env:UNSLOTH_PYTORCH_MIRROR.TrimEnd('/') } else { "https://download.pytorch.org/whl" } + # Explicit pin -- skip ALL GPU probing (headless / CI / cross-install). + # UNSLOTH_TORCH_INDEX_URL wins (full URL, verbatim); _FAMILY is the leaf appended + # to the mirror base. Matches install.sh / install_python_stack.py. + if (-not [string]::IsNullOrWhiteSpace($env:UNSLOTH_TORCH_INDEX_URL)) { + return (Trim-IndexPathSlashes $env:UNSLOTH_TORCH_INDEX_URL) + } + if (-not [string]::IsNullOrWhiteSpace($env:UNSLOTH_TORCH_INDEX_FAMILY)) { + return "$baseUrl/$($env:UNSLOTH_TORCH_INDEX_FAMILY.Trim().Trim('/'))" + } if (-not $NvidiaSmiExe) { return "$baseUrl/cpu" } try { $output = Invoke-NvidiaSmiBounded $NvidiaSmiExe @@ -1984,6 +2101,27 @@ exit 0 return "$baseUrl/cu126" } + # Strip userinfo AND query/fragment so an authenticated pin never leaks. Shared with + # _strip_index_url_credentials (install.sh / py / setup.ps1). + function Remove-IndexUrlCredentials { + param([string]$Url) + # Ordinal, not culture-aware: on non-English locales (e.g. th-TH) linguistic + # IndexOf treats "://" as ignorable, mis-locates it, and crashes Substring (issue #7279). + $sep = $Url.IndexOf('://', [System.StringComparison]::Ordinal) + if ($sep -lt 0) { return $Url } + $scheme = $Url.Substring(0, $sep) + $rest = $Url.Substring($sep + 3) + # Drop query / fragment (may hold auth tokens). + $q = $rest.IndexOfAny([char[]]('?', '#')) + if ($q -ge 0) { $rest = $rest.Substring(0, $q) } + $slash = $rest.IndexOf('/', [System.StringComparison]::Ordinal) + $authority = if ($slash -ge 0) { $rest.Substring(0, $slash) } else { $rest } + $at = $authority.LastIndexOf('@', [System.StringComparison]::Ordinal) + $host_ = if ($at -ge 0) { $authority.Substring($at + 1) } else { $authority } + if ($slash -ge 0) { return "${scheme}://${host_}$($rest.Substring($slash))" } + return "${scheme}://${host_}" + } + # ── Torch flavor helpers (to repair a stale CPU / wrong-CUDA wheel) ── # torch.__version__ -> flavor tag (cuXXX / rocm / cpu); untagged wheel = cpu, # matching setup.ps1's stale-venv parse. @@ -2002,11 +2140,13 @@ exit 0 param([string]$TorchIndexUrl, [string]$ROCmIndexUrl) if (-not [string]::IsNullOrWhiteSpace($ROCmIndexUrl)) { return 'rocm' } if ([string]::IsNullOrWhiteSpace($TorchIndexUrl)) { return $null } - $leaf = ($TorchIndexUrl.TrimEnd('/') -split '/')[-1].ToLowerInvariant() + # Drop query/fragment first so .../cu128?token=x classifies as cu128 (else it reinstalls every run). + $leaf = (($TorchIndexUrl -split '[?#]', 2)[0].TrimEnd('/') -split '/')[-1].ToLowerInvariant() if ($leaf -match '^cu\d+$') { return $leaf } if ($leaf -eq 'cpu') { return 'cpu' } if ($leaf -match '^rocm') { return 'rocm' } - if ($leaf -match '^gfx') { return 'rocm' } + # gfx must be followed by a digit (an architecture leaf); gfx-private is custom. + if ($leaf -match '^gfx[0-9]') { return 'rocm' } return $null } @@ -2041,6 +2181,10 @@ exit 0 } catch { return $null } } + # An explicit pin is authoritative: the AMD ROCm reroute below must not rewrite it + # (e.g. a deliberate cpu pin on an AMD host). + $TorchIndexPinned = (-not [string]::IsNullOrWhiteSpace($env:UNSLOTH_TORCH_INDEX_URL)) -or ` + (-not [string]::IsNullOrWhiteSpace($env:UNSLOTH_TORCH_INDEX_FAMILY)) $TorchIndexUrl = Get-TorchIndexUrl # ── GPU arch → newest compatible Windows ROCm wheel release ── @@ -2052,13 +2196,19 @@ exit 0 # Override with UNSLOTH_ROCM_WINDOWS_MIRROR for air-gapped / mirror installs. $ROCmIndexUrl = $null $ROCmTorchFloor = $null - if (($HasROCm -or $ROCmGfxArch) -and $TorchIndexUrl -like "*/cpu" -and -not $SkipTorch) { + $PinnedRocmVisionSpec = $null + $PinnedRocmAudioSpec = $null + if (-not $TorchIndexPinned -and ($HasROCm -or $ROCmGfxArch) -and $TorchIndexUrl -like "*/cpu" -and -not $SkipTorch) { $amdIndexBase = if ($env:UNSLOTH_ROCM_WINDOWS_MIRROR) { $env:UNSLOTH_ROCM_WINDOWS_MIRROR.TrimEnd('/') } else { "https://repo.amd.com/rocm/whl" } $archFamilyMap = @{ "gfx1201" = "gfx120X-all"; "gfx1200" = "gfx120X-all" # RDNA 4 "gfx1151" = "gfx1151"; "gfx1150" = "gfx1150" # RDNA 3.5 (Strix Halo/Point) "gfx1103" = "gfx110X-all"; "gfx1102" = "gfx110X-all" # RDNA 3 "gfx1101" = "gfx110X-all"; "gfx1100" = "gfx110X-all" + "gfx1036" = "gfx103X-all"; "gfx1035" = "gfx103X-all" # RDNA 2 (RX 6000) + "gfx1034" = "gfx103X-all"; "gfx1033" = "gfx103X-all" + "gfx1032" = "gfx103X-all"; "gfx1031" = "gfx103X-all" + "gfx1030" = "gfx103X-all" "gfx90a" = "gfx90a"; "gfx908" = "gfx908" # MI200/MI100 } # gfx120X (RDNA 4) and gfx1151/gfx1150 (Strix) have a null-pointer bug in @@ -2102,6 +2252,32 @@ exit 0 } } + # A gfx*/rocm pin skips the auto-reroute above, but the generic CPU/CUDA install below + # would use torch>=2.4,<2.11 and pull a known-bad wheel on the gfx115x/gfx120x/rocm>=7.2 + # indexes (the _grouped_mm bug). Route a pinned ROCm index through the ROCm path. + if ($TorchIndexPinned -and -not $ROCmIndexUrl -and -not $SkipTorch) { + $_pinLeaf = (($TorchIndexUrl -split '[?#]', 2)[0].TrimEnd('/') -split '/')[-1].ToLower() + $_pinRocm211 = $false + # Anchor ($) so a suffixed custom leaf (rocm7.2-private) falls through to verbatim. + if ($_pinLeaf -match '^rocm(\d+)\.(\d+)$') { + # Only KNOWN-2.11 rocm (rocm7.2) gets the floor. Matches Test-RocmKnown211Version. + $_pinRocm211 = ([int]$Matches[1] -eq 7 -and [int]$Matches[2] -eq 2) + } + # Only the 2.11-allowlist gfx arches need the floor; others publish <2.11 and stay bare. + $_pinGfx211 = @('gfx120x-all', 'gfx1151', 'gfx1150') -contains $_pinLeaf + if ($_pinGfx211 -or $_pinRocm211) { + $ROCmIndexUrl = $TorchIndexUrl + $ROCmTorchFloor = "torch>=2.11.0,<2.12.0" + $PinnedRocmVisionSpec = "torchvision>=0.26.0,<0.27.0" + $PinnedRocmAudioSpec = "torchaudio>=2.11.0,<2.12.0" + substep "pinned ROCm index ($_pinLeaf) -- enforcing $ROCmTorchFloor" "Cyan" + } elseif ($_pinLeaf -match '^gfx[0-9]' -or $_pinLeaf -match '^rocm[0-9]+(\.[0-9]+)?$') { + # Other gfx / older rocm (<=7.1) ship torch <2.11; route via the ROCm path with + # bare specs. Only EXACT rocm/gfx* are families; a suffixed leaf is verbatim. + $ROCmIndexUrl = $TorchIndexUrl + } + } + if ($ROCmIndexUrl) { $TorchIndexFamily = "rocm" } else { @@ -2164,14 +2340,14 @@ exit 0 } if ($_Migrated) { - # Migrated env: force-reinstall unsloth+unsloth-zoo to ensure clean state - # in the new venv location, while preserving existing torch/CUDA + # Migrated env: force-reinstall unsloth+unsloth-zoo for a clean state, preserving + # existing torch/CUDA unless the flavor repair below re-lands it. Write-TauriLog "STEP" "Installing unsloth" substep "upgrading unsloth in migrated environment..." if ($SkipTorch) { # No-torch: install unsloth + unsloth-zoo with --no-deps, then # runtime deps (typer, safetensors, transformers, etc.) with --no-deps. - $baseInstallExit = Invoke-InstallCommandRetry -Label "install unsloth (migrated no-torch)" { uv pip install --python $VenvPython --no-deps --reinstall-package unsloth --reinstall-package unsloth-zoo "unsloth>=2026.7.3" "unsloth-zoo>=2026.7.3" } + $baseInstallExit = Invoke-InstallCommandRetry -Label "install unsloth (migrated no-torch)" { uv pip install --python $VenvPython --no-deps --reinstall-package unsloth --reinstall-package unsloth-zoo "unsloth>=2026.7.5" "unsloth-zoo>=2026.7.6" } if ($baseInstallExit -eq 0) { # Resolve pydantic WITH deps so pip pins pydantic-core # to the matching version (no-torch-runtime.txt below @@ -2185,7 +2361,7 @@ exit 0 } } } else { - $baseInstallExit = Invoke-InstallCommandRetry -Label "install unsloth (migrated)" { uv pip install --python $VenvPython --reinstall-package unsloth --reinstall-package unsloth-zoo "unsloth>=2026.7.3" "unsloth-zoo>=2026.7.3" } + $baseInstallExit = Invoke-InstallCommandRetry -Label "install unsloth (migrated)" { uv pip install --python $VenvPython --reinstall-package unsloth --reinstall-package unsloth-zoo "unsloth>=2026.7.5" "unsloth-zoo>=2026.7.6" } } if ($baseInstallExit -ne 0) { Write-Host "[ERROR] Failed to install unsloth (exit code $baseInstallExit)" -ForegroundColor Red @@ -2210,22 +2386,24 @@ exit 0 substep "skipping PyTorch (--no-torch flag set)." "Yellow" } elseif ($ROCmIndexUrl) { Write-TauriLog "STEP" "Installing PyTorch (AMD ROCm Windows)" - substep "installing PyTorch from $ROCmIndexUrl..." + substep "installing PyTorch from $(Remove-IndexUrlCredentials $ROCmIndexUrl)..." $torchSpec = if ($ROCmTorchFloor) { $ROCmTorchFloor } else { "torch" } # Pin the companions to match $torchSpec; bare names can resolve an # ABI-incompatible torchvision/torchaudio on AMD's per-arch index. - $visionSpec = if ($ROCmGfxArch -and $torchvisionFloorMap.ContainsKey($ROCmGfxArch)) { $torchvisionFloorMap[$ROCmGfxArch] } else { "torchvision" } - $audioSpec = if ($ROCmGfxArch -and $torchaudioFloorMap.ContainsKey($ROCmGfxArch)) { $torchaudioFloorMap[$ROCmGfxArch] } else { "torchaudio" } + $visionSpec = if ($PinnedRocmVisionSpec) { $PinnedRocmVisionSpec } elseif ($ROCmGfxArch -and $torchvisionFloorMap -and $torchvisionFloorMap.ContainsKey($ROCmGfxArch)) { $torchvisionFloorMap[$ROCmGfxArch] } else { "torchvision" } + $audioSpec = if ($PinnedRocmAudioSpec) { $PinnedRocmAudioSpec } elseif ($ROCmGfxArch -and $torchaudioFloorMap -and $torchaudioFloorMap.ContainsKey($ROCmGfxArch)) { $torchaudioFloorMap[$ROCmGfxArch] } else { "torchaudio" } $torchInstallExit = Invoke-InstallCommandRetry -Label "install PyTorch (AMD ROCm)" { uv pip install --python $VenvPython --force-reinstall --default-index $ROCmIndexUrl $torchSpec $visionSpec $audioSpec } if ($torchInstallExit -ne 0) { - # Transient AMD-index failure: fall back to a CPU base so the install - # still completes; Studio setup retries ROCm afterwards. - substep "ROCm PyTorch install failed (exit $torchInstallExit); using a CPU base, Studio setup retries ROCm." "Yellow" + # Transient AMD-index failure: fall back to a CPU base (Unsloth setup retries + # ROCm). Use an explicit CPU index -- for a pinned ROCm index $TorchIndexUrl IS + # the ROCm mirror, so reusing it would just retry it. + $CpuFallbackIndexUrl = if ($env:UNSLOTH_PYTORCH_MIRROR) { "$($env:UNSLOTH_PYTORCH_MIRROR.TrimEnd('/'))/cpu" } else { "https://download.pytorch.org/whl/cpu" } + substep "ROCm PyTorch install failed (exit $torchInstallExit); using a CPU base, Unsloth setup retries ROCm." "Yellow" # --force-reinstall: a failed ROCm install can leave an unpinned ROCm # torch (e.g. 2.10.0+rocm on gfx110X/gfx90a) that still satisfies the CPU # torch>= range, so without it uv would keep the ROCm build and only swap # the companions -- a mismatched venv the flavor-repair block won't fix. - $torchInstallExit = Invoke-InstallCommandRetry -Label "install PyTorch (CPU fallback)" { uv pip install --python $VenvPython --force-reinstall "torch>=2.4,<2.11.0" torchvision torchaudio --default-index $TorchIndexUrl } + $torchInstallExit = Invoke-InstallCommandRetry -Label "install PyTorch (CPU fallback)" { uv pip install --python $VenvPython --force-reinstall "torch>=2.4,<2.11.0" "torchvision>=0.19,<0.26.0" "torchaudio>=2.4,<2.11.0" --default-index $CpuFallbackIndexUrl } if ($torchInstallExit -ne 0) { Write-Host "[ERROR] Failed to install PyTorch (ROCm and CPU base both failed, exit code $torchInstallExit)" -ForegroundColor Red return (Exit-InstallFailure "Failed to install PyTorch (exit code $torchInstallExit)" $torchInstallExit) @@ -2238,8 +2416,14 @@ exit 0 } } else { Write-TauriLog "STEP" "Installing PyTorch" - substep "installing PyTorch ($TorchIndexUrl)..." - $torchInstallExit = Invoke-InstallCommandRetry -Label "install PyTorch" { uv pip install --python $VenvPython "torch>=2.4,<2.11.0" torchvision torchaudio --default-index $TorchIndexUrl } + substep "installing PyTorch ($(Remove-IndexUrlCredentials $TorchIndexUrl))..." + # Bound the companions to the capped torch on EVERY index, cu + # families included: torchaudio 2.11 dropped its exact torch pin from + # the wheel metadata, so a bare companion next to torch<2.11 can + # resolve a mismatched 2.11.0 build. Mirrors install.sh. + $_pinVisionSpec = "torchvision>=0.19,<0.26.0" + $_pinAudioSpec = "torchaudio>=2.4,<2.11.0" + $torchInstallExit = Invoke-InstallCommandRetry -Label "install PyTorch" { uv pip install --python $VenvPython "torch>=2.4,<2.11.0" $_pinVisionSpec $_pinAudioSpec --default-index $TorchIndexUrl } if ($torchInstallExit -ne 0) { Write-Host "[ERROR] Failed to install PyTorch (exit code $torchInstallExit)" -ForegroundColor Red return (Exit-InstallFailure "Failed to install PyTorch (exit code $torchInstallExit)" $torchInstallExit) @@ -2251,7 +2435,7 @@ exit 0 if ($SkipTorch) { # No-torch: install unsloth + unsloth-zoo with --no-deps, then # runtime deps (typer, safetensors, transformers, etc.) with --no-deps. - $baseInstallExit = Invoke-InstallCommandRetry -Label "install unsloth (no-torch)" { uv pip install --python $VenvPython --no-deps --upgrade-package unsloth --upgrade-package unsloth-zoo "unsloth>=2026.7.3" "unsloth-zoo>=2026.7.3" } + $baseInstallExit = Invoke-InstallCommandRetry -Label "install unsloth (no-torch)" { uv pip install --python $VenvPython --no-deps --upgrade-package unsloth --upgrade-package unsloth-zoo "unsloth>=2026.7.5" "unsloth-zoo>=2026.7.6" } if ($baseInstallExit -eq 0) { # Same pydantic-with-deps trick as the migrated branch. $baseInstallExit = Invoke-InstallCommandRetry -Label "install pydantic" { uv pip install --python $VenvPython pydantic } @@ -2263,7 +2447,7 @@ exit 0 } } } elseif ($StudioLocalInstall) { - $baseInstallExit = Invoke-InstallCommandRetry -Label "install unsloth (local)" { uv pip install --python $VenvPython --upgrade-package unsloth "unsloth>=2026.7.3" "unsloth-zoo>=2026.7.3" } + $baseInstallExit = Invoke-InstallCommandRetry -Label "install unsloth (local)" { uv pip install --python $VenvPython --upgrade-package unsloth "unsloth>=2026.7.5" "unsloth-zoo>=2026.7.6" } } else { $baseInstallExit = Invoke-InstallCommandRetry -Label "install unsloth" { uv pip install --python $VenvPython --upgrade-package unsloth -- "$PackageName" } } @@ -2291,7 +2475,7 @@ exit 0 Write-TauriLog "STEP" "Installing unsloth" substep "installing unsloth (this may take a few minutes)..." if ($StudioLocalInstall) { - $baseInstallExit = Invoke-InstallCommandRetry -Label "install unsloth (auto torch backend)" { uv pip install --python $VenvPython "unsloth-zoo>=2026.7.3" "unsloth>=2026.7.3" --torch-backend=auto } + $baseInstallExit = Invoke-InstallCommandRetry -Label "install unsloth (auto torch backend)" { uv pip install --python $VenvPython "unsloth-zoo>=2026.7.6" "unsloth>=2026.7.5" --torch-backend=auto } if ($baseInstallExit -ne 0) { Write-Host "[ERROR] Failed to install unsloth (exit code $baseInstallExit)" -ForegroundColor Red return (Exit-InstallFailure "Failed to install unsloth (exit code $baseInstallExit)" $baseInstallExit) @@ -2317,6 +2501,13 @@ exit 0 } } + $installedPackageVersion = (& $VenvPython -c "from importlib.metadata import version; import sys; print(version(sys.argv[1]))" $PackageName 2>$null | Out-String).Trim() + if ($LASTEXITCODE -eq 0 -and $installedPackageVersion) { + step $PackageName "$installedPackageVersion installed" + } else { + substep "[WARN] installed $PackageName version could not be determined" "Yellow" + } + # ── Enforce the installed torch flavor matches the detected GPU build ── # PEP 440 ignores the +cpu/+cuXXX/+rocm local label in a version range, so uv # keeps a stale torch==X+cpu against a CUDA index and setup.ps1 then loops on @@ -2335,8 +2526,8 @@ exit 0 $rocmSpec = if ($ROCmTorchFloor) { $ROCmTorchFloor } else { "torch" } # Pin companions like the fresh ROCm path (bare names can pull an # ABI-incompatible torchvision/torchaudio from the per-arch index). - $visionSpec = if ($ROCmGfxArch -and $torchvisionFloorMap.ContainsKey($ROCmGfxArch)) { $torchvisionFloorMap[$ROCmGfxArch] } else { "torchvision" } - $audioSpec = if ($ROCmGfxArch -and $torchaudioFloorMap.ContainsKey($ROCmGfxArch)) { $torchaudioFloorMap[$ROCmGfxArch] } else { "torchaudio" } + $visionSpec = if ($PinnedRocmVisionSpec) { $PinnedRocmVisionSpec } elseif ($ROCmGfxArch -and $torchvisionFloorMap -and $torchvisionFloorMap.ContainsKey($ROCmGfxArch)) { $torchvisionFloorMap[$ROCmGfxArch] } else { "torchvision" } + $audioSpec = if ($PinnedRocmAudioSpec) { $PinnedRocmAudioSpec } elseif ($ROCmGfxArch -and $torchaudioFloorMap -and $torchaudioFloorMap.ContainsKey($ROCmGfxArch)) { $torchaudioFloorMap[$ROCmGfxArch] } else { "torchaudio" } substep "PyTorch flavor mismatch (installed $installedTorchTag, need ROCm) -- reinstalling correct build..." "Yellow" $torchFixExit = Invoke-InstallCommand { uv pip install --python $VenvPython --force-reinstall --default-index $ROCmIndexUrl $rocmSpec $visionSpec $audioSpec } if ($torchFixExit -ne 0) { @@ -2347,7 +2538,7 @@ exit 0 } elseif ($expectedTorchTag -ne 'rocm') { # CUDA: stale +cpu (or wrong cuXXX) against a CUDA index -> reinstall triplet. substep "PyTorch flavor mismatch (installed $installedTorchTag, need $expectedTorchTag) -- reinstalling correct build..." "Yellow" - $torchFixExit = Invoke-InstallCommand { uv pip install --python $VenvPython "torch>=2.4,<2.11.0" torchvision torchaudio --default-index $TorchIndexUrl --reinstall-package torch --reinstall-package torchvision --reinstall-package torchaudio } + $torchFixExit = Invoke-InstallCommand { uv pip install --python $VenvPython "torch>=2.4,<2.11.0" "torchvision>=0.19,<0.26.0" "torchaudio>=2.4,<2.11.0" --default-index $TorchIndexUrl --reinstall-package torch --reinstall-package torchvision --reinstall-package torchaudio } if ($torchFixExit -ne 0) { Write-Host "[ERROR] Failed to reinstall PyTorch with the correct CUDA build (exit code $torchFixExit)" -ForegroundColor Red return (Exit-InstallFailure "Failed to reinstall PyTorch ($expectedTorchTag) (exit code $torchFixExit)" $torchFixExit) @@ -2422,7 +2613,7 @@ exit 0 Write-TauriLog "ERROR" "unsloth CLI was not installed correctly" Write-Host "[ERROR] unsloth CLI was not installed correctly." -ForegroundColor Red Write-Host " Expected: $UnslothExe" -ForegroundColor Yellow - Write-Host " This usually means an older unsloth version was installed that does not include the Studio CLI." -ForegroundColor Yellow + Write-Host " This usually means an older unsloth version was installed that does not include the Unsloth CLI." -ForegroundColor Yellow Write-Host " Try re-running the installer or see: https://github.com/unslothai/unsloth?tab=readme-ov-file#-quickstart" -ForegroundColor Yellow return (Exit-InstallFailure "unsloth CLI was not installed correctly") } @@ -2533,7 +2724,7 @@ exit 0 Write-Host " Move or remove it manually, then re-run the installer." -ForegroundColor Yellow throw "Cannot create unsloth launcher: $ShimExe is a directory." } - # try/catch: if unsloth.exe is locked (Studio running), keep the old shim. + # try/catch: if unsloth.exe is locked (Unsloth running), keep the old shim. $shimUpdated = $false try { if (Test-Path -LiteralPath $ShimExe) { Remove-Item -LiteralPath $ShimExe -Force -ErrorAction Stop } @@ -2551,7 +2742,7 @@ exit 0 if (Test-Path -LiteralPath $ShimExe) { Write-Host "[WARN] Could not refresh unsloth launcher at $ShimExe." -ForegroundColor Yellow Write-Host " This usually means a running 'unsloth studio' process still holds the file open." -ForegroundColor Yellow - Write-Host " Close Studio and re-run the installer to pick up the latest launcher." -ForegroundColor Yellow + Write-Host " Close Unsloth and re-run the installer to pick up the latest launcher." -ForegroundColor Yellow Write-Host " Continuing with the existing launcher." -ForegroundColor Yellow } else { Write-Host "[WARN] Could not create unsloth launcher at $ShimExe" -ForegroundColor Yellow @@ -2572,6 +2763,13 @@ exit 0 } Refresh-SessionPath # sync current session with registry Complete-StudioVenvRollback + $studioVenvReplacementCommitted = $true + Remove-StaleStudioVenvRollbacks + } finally { + if (-not $studioVenvReplacementCommitted) { + Restore-StudioVenvRollback + } + } # Env-mode session export AFTER Refresh-SessionPath; otherwise a legacy # User PATH entry (Machine > User > current $env:Path) would win. @@ -2616,7 +2814,7 @@ exit 0 # Diagnostic only; never block install on a probe failure. } - # In interactive terminals, ask the user before starting Studio unless the + # In interactive terminals, ask the user before starting Unsloth unless the # caller explicitly disabled the post-install prompt. # In non-interactive environments (CI, Docker) just print instructions. $IsInteractive = (-not $SkipAutostart) -and [Environment]::UserInteractive -and (-not [Console]::IsInputRedirected) diff --git a/install.sh b/install.sh index 4a3c4471fa..dface28918 100755 --- a/install.sh +++ b/install.sh @@ -97,7 +97,7 @@ if [ "$_VERBOSE" = true ]; then export UNSLOTH_VERBOSE=1 fi -# Custom Studio roots are not supported with --tauri (desktop app still +# Custom Unsloth roots are not supported with --tauri (desktop app still # resolves ~/.unsloth/studio). Pass through if the override == legacy default. if [ "$TAURI_MODE" = true ]; then _tauri_override_var="" @@ -159,18 +159,58 @@ run_maybe_quiet() { fi } +# Trim trailing slashes from the URL PATH only, preserving ?query / #fragment: a whole-URL +# strip corrupts a token ending in "/", a single strip leaves .../cu128// empty. Shared. +_trim_index_path_slashes() { + _tips_v="$1" + case "$_tips_v" in + *[?#]*) + _tips_head="${_tips_v%%[?#]*}" + _tips_tail="${_tips_v#"$_tips_head"}" + ;; + *) + _tips_head="$_tips_v" + _tips_tail="" + ;; + esac + while [ -n "$_tips_head" ] && [ "${_tips_head%/}" != "$_tips_head" ]; do + _tips_head="${_tips_head%/}" + done + printf '%s%s' "$_tips_head" "$_tips_tail" +} + +# Redact index-URL credentials (userinfo + ?query= + #fragment) from captured installer +# output before printing on failure; uv/pip errors echo the failing --index-url verbatim. +# Mirrors the other installers. Verbose mode streams uncaptured, so it isn't redacted. +_redact_install_output() { + sed -E \ + -e 's#(https?://)[^/@[:space:]`]+@#\1@#g' \ + -e 's#([?&][^=[:space:]&`]+)=[^&#[:space:]`]+#\1=#g' \ + -e 's|(https?://[^[:space:]`#]+)#[^[:space:]`]+|\1#|g' \ + "$@" +} + run_install_cmd() { _label="$1" shift - # Installer-pinned index installs (torch) must beat an inherited uv mirror - # (#6898): when we pass --default-index, neutralize every uv index env var so - # the pinned index wins. Other installs keep the user's mirror. + # Installer-pinned index installs (torch) must beat an inherited uv mirror (#6898): + # for --default-index, neutralize the uv index/backend/config vars (UV_TORCH_BACKEND + # redirects torch; UV_NO_CONFIG=1 + dropping UV_CONFIG_FILE stops a uv.toml/pyproject + # index outranking the CLI pin, uv 0.10). case " $* " in - *" --default-index "*) set -- env -u UV_DEFAULT_INDEX -u UV_INDEX_URL -u UV_INDEX -u UV_EXTRA_INDEX_URL "$@" ;; + *" --default-index "*) set -- env -u UV_DEFAULT_INDEX -u UV_INDEX_URL -u UV_INDEX -u UV_EXTRA_INDEX_URL -u UV_TORCH_BACKEND -u UV_FIND_LINKS -u UV_CONFIG_FILE UV_NO_CONFIG=1 "$@" ;; esac if _is_verbose; then - "$@" && return 0 - _rc=$? + # Stream through the redactor: uv echoes index URLs (credentials and + # all) in its errors, and verbose mode previously bypassed the + # redaction the quiet path applies. The rc file preserves the + # command's exit code across the pipe without relying on pipefail + # (this script runs under plain sh). + _rcf=$(mktemp) + { "$@" 2>&1; printf '%s' "$?" > "$_rcf"; } | _redact_install_output + _rc=$(cat "$_rcf" 2>/dev/null || echo 1) + rm -f "$_rcf" + [ "${_rc:-1}" -eq 0 ] 2>/dev/null && return 0 step "error" "$_label failed (exit code $_rc)" "$C_ERR" >&2 return "$_rc" fi @@ -178,7 +218,7 @@ run_install_cmd() { "$@" >"$_log" 2>&1 && { rm -f "$_log"; return 0; } _rc=$? step "error" "$_label failed (exit code $_rc)" "$C_ERR" >&2 - cat "$_log" >&2 + _redact_install_output "$_log" >&2 rm -f "$_log" return $_rc } @@ -257,7 +297,7 @@ _install_bnb_rocm() { fi _bnb_rc=$? if _is_verbose; then - cat "$_bnb_log" >&2 + _redact_install_output "$_bnb_log" >&2 fi rm -f "$_bnb_log" step "warning" "$_label (pre-release) failed (exit code $_bnb_rc)" "$C_WARN" >&2 @@ -310,6 +350,11 @@ _tauri_torch_index_family() { return fi _diag_url="${1:-}" + # Strip query/fragment AND a trailing slash before classifying (like _torch_index_url_leaf): + # a token isn't echoed into [TAURI:DIAG], and .../cu128/?token=x still classifies as cu128. + _diag_url="${_diag_url%%\?*}" + _diag_url="${_diag_url%%#*}" + _diag_url="${_diag_url%/}" case "$_diag_url" in */cu118) echo "cu118" ;; */cu124) echo "cu124" ;; @@ -343,7 +388,8 @@ _tauri_gpu_branch() { return fi case "$_diag_family" in - cu*) echo "cuda" ;; + # Require a digit after cu so /current or /custom isn't branded CUDA (parity ^cu[0-9]). + cu[0-9]*) echo "cuda" ;; rocm*) if [ "$_diag_radeon" = true ]; then echo "rocm_radeon" @@ -429,14 +475,20 @@ _start_studio_venv_replacement() { _stamp=$(date +%Y%m%d%H%M%S 2>/dev/null || echo "time") _candidate="$STUDIO_HOME/unsloth_studio.rollback.$_stamp.$$" _suffix=0 - while [ -e "$_candidate" ]; do + while [ -e "$_candidate" ] || [ -L "$_candidate" ]; do _suffix=$((_suffix + 1)) _candidate="$STUDIO_HOME/unsloth_studio.rollback.$_stamp.$$.$_suffix" done - mv "$_existing_dir" "$_candidate" _VENV_ROLLBACK_DIR="$_candidate" _VENV_ROLLBACK_TARGET="$_existing_dir" _VENV_ROLLBACK_ACTIVE=true + # Publish the rollback state before the atomic rename so a signal cannot + # land after mv but before the exit handlers know where the old venv went. + if ! mv "$_existing_dir" "$_candidate"; then + _VENV_ROLLBACK_ACTIVE=false + _VENV_ROLLBACK_DIR="" + return 1 + fi substep "previous environment preserved for rollback" } @@ -457,13 +509,68 @@ _restore_studio_venv_replacement() { fi } -_commit_studio_venv_replacement() { - [ "$_VENV_ROLLBACK_ACTIVE" = true ] || return 0 - if [ -n "$_VENV_ROLLBACK_DIR" ] && [ -d "$_VENV_ROLLBACK_DIR" ]; then - rm -rf "$_VENV_ROLLBACK_DIR" || true +_studio_venv_rollback_must_be_preserved() { + _rollback_name=${1##*/} + _rollback_metadata=${_rollback_name#unsloth_studio.rollback.} + _rollback_stamp=${_rollback_metadata%%.*} + _rollback_process=${_rollback_metadata#*.} + # Preserve anything outside the installer's timestamp.PID[.suffix] format. + [ "$_rollback_process" != "$_rollback_metadata" ] || return 0 + case "$_rollback_stamp" in + time) ;; + ''|*[!0-9]*) return 0 ;; + *) [ "${#_rollback_stamp}" -eq 14 ] || return 0 ;; + esac + _rollback_pid=${_rollback_process%%.*} + case "$_rollback_pid" in + ''|*[!0-9]*) return 0 ;; + esac + _rollback_suffix=${_rollback_process#*.} + if [ "$_rollback_suffix" != "$_rollback_process" ]; then + case "$_rollback_suffix" in ''|*[!0-9]*) return 0 ;; esac fi - _VENV_ROLLBACK_ACTIVE=false - _VENV_ROLLBACK_DIR="" + kill -0 "$_rollback_pid" 2>/dev/null +} + +_prune_stale_studio_venv_rollbacks() { + for _stale_rollback in "$STUDIO_HOME"/unsloth_studio.rollback.*; do + [ -d "$_stale_rollback" ] || continue + if [ -L "$_stale_rollback" ]; then + echo "⚠️ Refusing to remove rollback symlink $_stale_rollback" >&2 + continue + fi + # A concurrent installer may have moved its live venv aside. The PID in + # the generated name keeps this successful run from deleting its rescue copy. + _studio_venv_rollback_must_be_preserved "$_stale_rollback" && continue + if rm -rf "$_stale_rollback"; then + substep "removed stale environment rollback ${_stale_rollback##*/}" + else + echo "⚠️ Could not remove stale environment rollback $_stale_rollback" >&2 + fi + done +} + +_commit_studio_venv_replacement() { + if [ "$_VENV_ROLLBACK_ACTIVE" = true ]; then + _rollback_to_remove="$_VENV_ROLLBACK_DIR" + # The new environment is already committed. Clear the restore state + # before deletion so an interrupt cannot replace it with a half-deleted backup. + _VENV_ROLLBACK_ACTIVE=false + _VENV_ROLLBACK_DIR="" + if [ -n "$_rollback_to_remove" ] && [ -d "$_rollback_to_remove" ]; then + if ! rm -rf "$_rollback_to_remove"; then + echo "⚠️ Could not remove environment rollback $_rollback_to_remove" >&2 + fi + fi + fi + # Only prune older orphaned copies after the replacement has succeeded, so + # an interrupted install never discards the last known-good environment. + _prune_stale_studio_venv_rollbacks +} + +_cleanup_install_temporaries() { + [ -n "${_UV_OVERRIDE_TMPDIR:-}" ] && rm -rf "$_UV_OVERRIDE_TMPDIR" 2>/dev/null || true + [ -n "${_UNSLOTH_TORCH_OVERRIDES:-}" ] && rm -f "$_UNSLOTH_TORCH_OVERRIDES" 2>/dev/null || true } _on_install_exit() { @@ -471,13 +578,28 @@ _on_install_exit() { if [ "$_status" -ne 0 ]; then _restore_studio_venv_replacement fi - [ -n "${_UV_OVERRIDE_TMPDIR:-}" ] && rm -rf "$_UV_OVERRIDE_TMPDIR" 2>/dev/null || true + _cleanup_install_temporaries exit "$_status" } -# Empty so an inherited value can never reach the trap's rm; only a temp dir -# this script creates below (Apple Silicon, spaced path) is ever removed. + +_on_install_signal() { + _signal_status="$1" + # EXIT is disabled to avoid a second cleanup pass. Ignore further termination + # signals until the old environment is back in place. + trap - EXIT + trap '' HUP INT TERM + _restore_studio_venv_replacement + _cleanup_install_temporaries + exit "$_signal_status" +} +# Empty so an inherited value never reaches the trap's rm; only temp paths this +# script creates below (spaced-path dir, torch-trio overrides) are removed. _UV_OVERRIDE_TMPDIR="" +_UNSLOTH_TORCH_OVERRIDES="" trap _on_install_exit EXIT +trap '_on_install_signal 129' HUP +trap '_on_install_signal 130' INT +trap '_on_install_signal 143' TERM # ── Helper: download a URL to a file (supports curl and wget) ── download() { @@ -503,6 +625,36 @@ _is_pkg_installed() { esac } +# ── Helper: human-readable apt distro label for the sudo package prompt (#6207) ── +# Reads /etc/os-release so the Accept? prompt can say which distro we detected and +# that packages come from that distro's official apt repos (not a tarball). +_apt_distro_description() { + # Plain ( ... ) subshell — not $() — so case/;; stays bash-3.2-safe on macOS. + # Bash 3.2 misparses case arms inside command substitution and errors on `;;`. + ( + if [ ! -r /etc/os-release ]; then + printf 'a debian-like system' + exit 0 + fi + # shellcheck disable=SC1091 + . /etc/os-release 2>/dev/null || true + if [ -n "${NAME:-}" ] && [ -n "${VERSION_ID:-}" ]; then + _ad_label="$NAME $VERSION_ID" + elif [ -n "${PRETTY_NAME:-}" ]; then + _ad_label="$PRETTY_NAME" + elif [ -n "${NAME:-}" ]; then + _ad_label="$NAME" + else + printf 'a debian-like system' + exit 0 + fi + case " ${ID:-} ${ID_LIKE:-} " in + *" debian "*|*" ubuntu "*) _ad_label="${_ad_label} (debian-like)" ;; + esac + printf '%s' "$_ad_label" + ) +} + # ── Helper: install packages via apt, escalating to sudo only if needed ── # Usage: _smart_apt_install pkg1 pkg2 pkg3 ... _smart_apt_install() { @@ -533,11 +685,14 @@ _smart_apt_install() { # Step 3: Escalate -- need elevated permissions for remaining packages if command -v sudo >/dev/null 2>&1; then + _ad_desc="$(_apt_distro_description)" echo "" echo " !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!" echo " WARNING: We require sudo elevated permissions to install:" echo " $_STILL_MISSING" - echo " If you accept, we'll run sudo now, and it'll prompt your password." + echo " Detected ${_ad_desc}." + echo " If you accept, we'll run sudo apt-get to install these packages" + echo " from your distro's official repositories (not a third-party tarball)." echo " !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!" echo "" printf " Accept? [Y/n] " @@ -663,7 +818,7 @@ POLL_INTERVAL_SEC=0.25 LOG_FILE="$DATA_DIR/studio.log" # why: in env-override mode multiple installs share an OS user; namespace the # lock and remember our own healthy port so we never attach to an unrelated -# Studio listening on the global 8888..8908 range. +# Unsloth listening on the global 8888..8908 range. LOCK_DIR="${XDG_RUNTIME_DIR:-/tmp}/unsloth-studio-launcher-$(id -u).lock" PORT_FILE="" # why: gate on the install-time mode (baked above) instead of the runtime env @@ -734,7 +889,7 @@ _candidate_ports() { _find_healthy_port() { if [ -n "$PORT_FILE" ] && [ -f "$PORT_FILE" ]; then # why: env-mode installs only attach to a port we previously launched - # ourselves; never to a sibling Studio that happens to be healthy. + # ourselves; never to a sibling Unsloth that happens to be healthy. _p=$(cat "$PORT_FILE" 2>/dev/null || true) case "$_p" in ''|*[!0-9]*) ;; @@ -901,7 +1056,7 @@ _acquire_lock() { # Lock dir exists -- check if owner is still alive _old_pid=$(cat "$LOCK_DIR/pid" 2>/dev/null || true) if [ -n "$_old_pid" ] && kill -0 "$_old_pid" 2>/dev/null; then - # Another launcher is running; wait for it to bring Studio up + # Another launcher is running; wait for it to bring Unsloth up _deadline=$(($(date +%s) + TIMEOUT_SEC)) while [ "$(date +%s)" -lt "$_deadline" ]; do _port=$(_find_healthy_port) && { @@ -1371,7 +1526,7 @@ WSLPS1_EOF # shortcut wasn't created; tell the user how to launch / re-enable it. if [ "$_css_created" -ne 1 ]; then substep "Couldn't create the Windows shortcut (WSL interop may be disabled)." "$C_WARN" - substep " Launch Studio from Windows: wsl -d \"$_css_distro\" -- bash -lc 'unsloth studio'" "$C_WARN" + substep " Launch Unsloth from Windows: wsl -d \"$_css_distro\" -- bash -lc 'unsloth studio'" "$C_WARN" substep " (re-enable shortcuts: turn WSL interop back on, e.g. run 'wsl --shutdown' then reopen WSL.)" "$C_WARN" fi fi @@ -1439,7 +1594,7 @@ if [ "$MAC_INTEL" = true ]; then echo "" echo " NOTE: Intel Mac (x86_64) detected." echo " PyTorch is unavailable for this platform (dropped Jan 2024)." - echo " Studio will install in GGUF-only mode." + echo " Unsloth will install in GGUF-only mode." echo " Chat, inference via GGUF, and data recipes will work." echo " Training requires Apple Silicon or Linux with GPU." echo "" @@ -1573,6 +1728,12 @@ _has_usable_nvidia_gpu() { # the STUDIO_HOME mkdir/venv so the origin distro is untouched. _maybe_reroute_strixhalo_to_2404() { [ "${OS:-}" = "wsl" ] || return 0 + # An explicit index pin skips every GPU-driven reroute (same contract as + # the later Radeon/Strix guard): the pin is honored in THIS distro rather + # than probing the GPU and switching distributions. Whitespace-only + # overrides do not gate (parity with get_torch_index_url). + _rr_pin=$(printf '%s' "${UNSLOTH_TORCH_INDEX_URL:-}${UNSLOTH_TORCH_INDEX_FAMILY:-}" | tr -d '[:space:]') + [ -n "$_rr_pin" ] && return 0 [ "${SKIP_TORCH:-false}" = "false" ] || return 0 [ "${UNSLOTH_SKIP_ROCM_WSL_SETUP:-0}" = "1" ] && return 0 [ "${UNSLOTH_WSL_REROUTED:-0}" = "1" ] && return 0 @@ -1582,7 +1743,7 @@ _maybe_reroute_strixhalo_to_2404() { # CUDA_VISIBLE_DEVICES=""/-1 and the /proc/driver/nvidia fallback for PATH/timeout gaps. if _has_usable_nvidia_gpu; then return 0; fi # Strix APUs show in /proc/cpuinfo; discrete cards don't, so also try WMI. Either reroutes. - if ! grep -qiE 'Ryzen AI Max|Radeon 80[0-9]0S|Strix Halo' /proc/cpuinfo 2>/dev/null \ + if ! grep -qiE 'Ryzen AI Max|Radeon 80[0-9][05]S|Strix Halo' /proc/cpuinfo 2>/dev/null \ && ! _wsl_amd_gpu_name >/dev/null 2>&1; then return 0 fi @@ -1634,6 +1795,10 @@ _maybe_reroute_strixhalo_to_2404() { # Forward explicit ROCm-bootstrap consent (e.g. Tauri) so the child auto-enables the # GPU instead of falling back to the desktop-app prompt path. [ "${UNSLOTH_ROCM_WSL_AUTO:-0}" = "1" ] && _rr_exports="$_rr_exports; export UNSLOTH_ROCM_WSL_AUTO=1" + # Forward a pinned torch index into the rerouted distro; dropping it would + # silently revert the child install to auto-detection. + [ -n "${UNSLOTH_TORCH_INDEX_URL:-}" ] && _rr_exports="$_rr_exports; export UNSLOTH_TORCH_INDEX_URL=$(_rr_q "$UNSLOTH_TORCH_INDEX_URL")" + [ -n "${UNSLOTH_TORCH_INDEX_FAMILY:-}" ] && _rr_exports="$_rr_exports; export UNSLOTH_TORCH_INDEX_FAMILY=$(_rr_q "$UNSLOTH_TORCH_INDEX_FAMILY")" [ "$_SKIP_AUTOSTART" = true ] && _rr_exports="$_rr_exports; export UNSLOTH_SKIP_AUTOSTART=1" _rr_args="" [ "$PACKAGE_NAME" != "unsloth" ] && _rr_args="$_rr_args --package $(_rr_q "$PACKAGE_NAME")" @@ -1671,7 +1836,7 @@ _maybe_reroute_strixhalo_to_2404() { _maybe_reroute_strixhalo_to_2404 || true # ── Check system dependencies ── -# cmake/git are only needed to *build* llama.cpp from source. Studio downloads a +# cmake/git are only needed to *build* llama.cpp from source. Unsloth downloads a # prebuilt by default, and setup.sh self-skips the source build when they're # absent -- so macOS doesn't block on cmake (requiring it would force a manual # Homebrew install). Linux keeps requiring them; its package manager has them. @@ -1821,11 +1986,13 @@ tauri_log "STEP" "Creating virtual environment" mkdir -p "$STUDIO_HOME" _MIGRATED=false +# Empty so an inherited value can never masquerade as a probed torch version. +_PREV_TORCH_VER="" if [ -x "$VENV_DIR/bin/python" ]; then # why: matching guard to the .venv branch below -- in env-mode # $STUDIO_HOME is a user-chosen workspace, so refuse to nuke an - # existing $STUDIO_HOME/unsloth_studio that lacks Studio sentinels. + # existing $STUDIO_HOME/unsloth_studio that lacks Unsloth sentinels. # Accept the in-VENV ownership marker so partial-install retries are # not blocked. Sentinels must be regular files: -f follows symlinks # to files (the legitimate ln -s shim shape) but rejects directories @@ -1838,6 +2005,12 @@ if [ -x "$VENV_DIR/bin/python" ]; then echo " Move it aside or choose an empty UNSLOTH_STUDIO_HOME." >&2 exit 1 fi + # Record the existing venv's torch BEFORE the replacement moves it aside: a re-run + # rebuilds the venv for clean state, but must keep the torch release the user + # already has (see _previous_torch_pin below). Last line only: sitecustomize or + # import-hook noise on stdout must not corrupt the version. + _PREV_TORCH_VER=$("$VENV_DIR/bin/python" -c \ + "import torch; print(torch.__version__)" 2>/dev/null | tail -n 1 || true) # New layout already exists — replace only after preserving rollback copy. substep "preserving existing environment for rollback..." _start_studio_venv_replacement "$VENV_DIR" @@ -1846,7 +2019,7 @@ elif [ "$_STUDIO_HOME_REDIRECT" != "env" ] && [ -x "$STUDIO_HOME/.venv/bin/pytho # Skip in env-mode so we don't rm -rf an unrelated .venv at the # workspace root (e.g. user's existing project Python venv). # In no-torch mode, a missing torch package is expected; validate Python only. - substep "found legacy Studio environment, validating..." + substep "found legacy Unsloth environment, validating..." _legacy_ok=false if [ "$SKIP_TORCH" = true ]; then if "$STUDIO_HOME/.venv/bin/python" -c "import sys; print(sys.executable)" >/dev/null 2>&1; then @@ -1903,7 +2076,7 @@ if [ ! -x "$VENV_DIR/bin/python" ]; then fi fi -# Mark the freshly-created venv as Studio-owned so a partial install can be +# Mark the freshly-created venv as Unsloth-owned so a partial install can be # repaired by re-running install.sh; the env-mode deletion guard above accepts # this marker as the primary sentinel. if [ -x "$VENV_DIR/bin/python" ]; then @@ -1991,6 +2164,15 @@ if [ "$SKIP_TORCH" = false ] && [ "$OS" = "macos" ] && [ "$_ARCH" = "arm64" ]; t TORCH_CONSTRAINT="torch>=2.6,<2.11.0" fi fi +# Companion (torchvision/torchaudio) constraints, bounded to torch's window. +# torchaudio 2.11 dropped its exact torch pin, so a bare companion next to a +# <2.11-capped torch resolves torchaudio 2.11 (verified: cpu leaf installed +# torch 2.10.0+cpu with torchaudio 2.11.0+cpu). torchvision still exact-pins +# torch and self-corrects, but is bounded for symmetry. Widened alongside the +# cu* torch window below; the torch-2.11 AMD paths (rocm7.2 / per-gfx / Strix) +# pin their own trio. +TORCHVISION_CONSTRAINT="torchvision>=0.19,<0.26.0" +TORCHAUDIO_CONSTRAINT="torchaudio>=2.4,<2.11.0" # ── Resolve repo root (for --local installs) ── _REPO_ROOT="$(cd "$(dirname "$0" 2>/dev/null || echo ".")" && pwd)" @@ -2040,18 +2222,148 @@ _has_amd_rocm_gpu() { amd-smi list 2>/dev/null | awk '/^GPU[[:space:]]*[:\[][[:space:]]*[0-9]/{ found=1 } END{ exit !found }'; then return 0 elif [ -e /dev/kfd ] && \ - awk 'FNR==1{ gpu=0; amd=0 } /gpu_id/{ gpu=($2+0>0) } /vendor_id/{ amd=($2==4098) } \ - gpu && amd { found=1 } END{ exit !found }' \ + awk '/vendor_id/ && $2 == 4098 { found = 1 } END { exit !found }' \ /sys/class/kfd/kfd/topology/nodes/*/properties 2>/dev/null; then - # vendor_id 4098 = 0x1002 (AMD). NVIDIA open kernel module (driver - # 560+) can register KFD topology nodes with non-zero gpu_id but - # vendor_id 4318 (0x10DE). Require AMD vendor to avoid misrouting - # NVIDIA-only hosts to the ROCm install path. + # vendor_id 4098 = 0x1002 (AMD) marks a GPU node: the KFD CPU node + # reports vendor_id 0, so any 4098 node is an AMD GPU. NVIDIA's open + # kernel module (driver 560+) registers KFD nodes as vendor_id 4318 + # (0x10DE), so this never false-positives on NVIDIA-only hosts. + # The prior check also required a gpu_id line, but gpu_id is a SIBLING + # sysfs file, not a line in properties -- it never matched, so the + # fallback silently missed every ROCm-less AMD host (issue: fresh + # Arch/CachyOS boxes reporting "no GPU detected"). return 0 fi return 1 } +# Returns 0 if an AMD display GPU is on the PCI bus even when ROCm can't use it +# (e.g. a Strix Halo iGPU with no /dev/kfd). Only sharpens the "no GPU detected" +# hint. vendor 0x1002 = AMD/ATI; class 0x03* = display controller. +_amd_gpu_present_via_pci() { + [ -d /sys/bus/pci/devices ] || return 1 + for _pci_vendor in /sys/bus/pci/devices/*/vendor; do + [ -r "$_pci_vendor" ] || continue + read -r _v < "$_pci_vendor" 2>/dev/null || continue + [ "$_v" = "0x1002" ] || continue + _cls="${_pci_vendor%vendor}class" + [ -r "$_cls" ] || continue + read -r _c < "$_cls" 2>/dev/null || continue + case "$_c" in 0x03*) return 0 ;; esac + done + return 1 +} + +# Map a gfx arch to the AMD pip index family (mirrors install.ps1 $archFamilyMap). +_amd_arch_index_family_for_gfx() { + case "$1" in + gfx1201|gfx1200) echo gfx120X-all ;; + gfx1151) echo gfx1151 ;; + gfx1150) echo gfx1150 ;; + gfx1103|gfx1102|gfx1101|gfx1100) echo gfx110X-all ;; + gfx1036|gfx1035|gfx1034|gfx1033|gfx1032|gfx1031|gfx1030) echo gfx103X-all ;; + gfx90a) echo gfx90a ;; + gfx908) echo gfx908 ;; + *) return 1 ;; + esac +} + +# Map a GPU marketing name to gfx arch (kept in sync with install.ps1 nameArchTable). +_infer_amd_gfx_arch_from_gpu_name() { + case "$1" in + *"9070 XT"*|*9080*) echo gfx1201 ;; + *9070*|*9060*) echo gfx1200 ;; + *"8065S"*|*"8060S"*|*"8050S"*|*"8040S"*|*"Strix Halo"*|*"Ryzen AI Max"*|*"AI Max"*) echo gfx1151 ;; + *"890M"*|*"880M"*|*"860M"*|*"840M"*|*"Strix Point"*|*"Krackan"*|*"HX 37"*|*"AI 9 HX"*|*"AI 9 36"*|*"AI 7 35"*|*"AI 5 34"*|*"AI 7 PRO 35"*|*"AI 5 33"*) echo gfx1150 ;; + *"RX 7600"*|*"RX 7700S"*|*"RX 7650"*|*"PRO W7600"*|*"PRO W7500"*|*"PRO V710"*) echo gfx1102 ;; + *"RX 7900"*|*"RX 7800"*|*"RX 7700"*|*"PRO W7900"*|*"PRO W7800"*|*"PRO W7700"*) echo gfx1100 ;; + *"780M"*|*"760M"*|*"740M"*|*"Phoenix"*|*"Hawk Point"*|*"Z1 Extreme"*|*"Z2 Extreme"*) echo gfx1103 ;; + *"RX 6900"*|*"RX 6800"*|*"RX 6750"*|*"RX 6700"*|*"PRO W6800"*|*"PRO W6900"*) echo gfx1030 ;; + *"RX 6650"*|*"RX 6600"*|*"PRO W6600"*|*"PRO W6650"*) echo gfx1032 ;; + *"RX 6500"*|*"RX 6400"*|*"RX 6300"*|*"PRO W6400"*|*"PRO W6500"*) echo gfx1034 ;; + *) return 1 ;; + esac +} + +# Best-effort gfx inference when ROCm tools can't see the GPU (unslothai#7301). +# Mirrors install.ps1 arch resolution on Windows ($HasROCm false, $ROCmGfxArch set). +_infer_linux_amd_gfx_arch() { + if [ -n "${UNSLOTH_ROCM_GFX_ARCH:-}" ]; then + printf '%s\n' "$(printf '%s' "$UNSLOTH_ROCM_GFX_ARCH" | tr '[:upper:]' '[:lower:]')" + return 0 + fi + # On WSL /proc/cpuinfo and lspci still report the host APU, but without the + # ROCDXG bridge (librocdxg over /dev/dxg) the AMD wheels can't reach the GPU; + # keep the CPU fallback there unless that runtime is present (the explicit + # override above still wins). Mirrors install_python_stack.py. + _gpu_evidence="" + if [ -e /dev/dxg ] || grep -qi microsoft /proc/version 2>/dev/null; then + for _d in /opt/rocm/lib /opt/rocm/lib64 /opt/rocm-*/lib /opt/rocm-*/lib64; do + { [ -e "$_d/librocdxg.so" ] || [ -e "$_d/librocdxg.so.1" ]; } && _rocdxg=1 && break + done + [ -n "${_rocdxg:-}" ] || return 1 + # WSL enumerates no PCI display device; /dev/dxg + librocdxg IS the + # GPU evidence there. + _gpu_evidence=1 + elif _amd_gpu_present_via_pci; then + _gpu_evidence=1 + fi + # /proc/cpuinfo leaks the HOST CPU model into VMs/containers that received + # no AMD GPU, so the CPU-model text alone is not GPU evidence: require an + # AMD display device (PCI vendor 0x1002, class 0x03*) before trusting it. + # The lspci fallback below needs no gate; an AMD display line IS evidence. + if [ -n "$_gpu_evidence" ] && grep -qiE 'Ryzen AI Max|Radeon 80[0-9][05]S|Strix Halo' /proc/cpuinfo 2>/dev/null; then + echo gfx1151 + return 0 + fi + if [ -n "$_gpu_evidence" ] && grep -qiE '890M|880M|860M|840M|Strix Point|Krackan|HX 37[05]|AI 9 HX|AI 9 36[05]|AI 7 35[05]|AI 5 34[05]|AI 7 PRO 35|AI 5 33' /proc/cpuinfo 2>/dev/null; then + echo gfx1150 + return 0 + fi + if command -v lspci >/dev/null 2>&1; then + # A non-AMD controller can enumerate first (Intel/ASPEED before an AMD + # dGPU), so scan every display-class line and take the first AMD one + # that maps. The vendor guard is case-SENSITIVE (a -i "ATI" would match + # "CorporATIon" on every Intel/NVIDIA line); whole-line matching also + # survives the 0000: PCI domain prefix. Mirrors install_python_stack.py. + _amd_disp=$(lspci -nn 2>/dev/null | grep -E 'VGA compatible controller|3D controller|Display controller' | grep -E 'AMD|ATI' || true) + while IFS= read -r _ln; do + [ -n "$_ln" ] || continue + if _gfx=$(_infer_amd_gfx_arch_from_gpu_name "$_ln"); then + echo "$_gfx" + return 0 + fi + done </dev/null 2>&1; then + _pg=$( (unset ROCR_VISIBLE_DEVICES HIP_VISIBLE_DEVICES; rocminfo 2>/dev/null) | grep -oE 'gfx[1-9][0-9a-z]{2,3}' || true) + fi + if [ -z "$_pg" ] && command -v amd-smi >/dev/null 2>&1; then + _pg=$( (unset ROCR_VISIBLE_DEVICES HIP_VISIBLE_DEVICES; amd-smi list 2>/dev/null) | grep -oE 'gfx[1-9][0-9a-z]{2,3}' || true) + if [ -z "$_pg" ]; then + _pg=$( (unset ROCR_VISIBLE_DEVICES HIP_VISIBLE_DEVICES; amd-smi static --asic 2>/dev/null) | grep -oE 'gfx[1-9][0-9a-z]{2,3}' || true) + fi + fi + printf '%s\n' "$_pg" +} + # ── Detect GPU and choose PyTorch index URL ── # Mirrors Get-TorchIndexUrl in install.ps1. # On CPU-only machines this returns the cpu index, avoiding the solver @@ -2059,6 +2371,24 @@ _has_amd_rocm_gpu() { get_torch_index_url() { _base="${UNSLOTH_PYTORCH_MIRROR:-https://download.pytorch.org/whl}" _base="${_base%/}" + # Explicit override -- skip ALL GPU probing (headless / container / CI / cross-install). + # UNSLOTH_TORCH_INDEX_URL wins (full URL, verbatim); _FAMILY is the leaf (cpu, cu128, ...) + # appended to the mirror base. Trim whitespace so a whitespace-only value is unset. + _url="${UNSLOTH_TORCH_INDEX_URL:-}" + _url="${_url#"${_url%%[![:space:]]*}"}"; _url="${_url%"${_url##*[![:space:]]}"}" + if [ -n "$_url" ]; then + # Trim trailing PATH slashes (a multi-slash path 404s on strict pip proxies) while + # preserving a ?query/#fragment token (a whole-URL strip would eat a "/"-ending token). + _url=$(_trim_index_path_slashes "$_url") + echo "$_url"; return + fi + _family="${UNSLOTH_TORCH_INDEX_FAMILY:-}" + _family="${_family#"${_family%%[![:space:]]*}"}"; _family="${_family%"${_family##*[![:space:]]}"}" + if [ -n "$_family" ]; then + while [ "${_family#/}" != "$_family" ]; do _family="${_family#/}"; done + while [ "${_family%/}" != "$_family" ]; do _family="${_family%/}"; done + echo "$_base/$_family"; return + fi # macOS: always CPU (no CUDA support) case "$(uname -s)" in Darwin) echo "$_base/cpu"; return ;; esac # Try nvidia-smi -- require the binary to actually list a usable GPU. @@ -2087,6 +2417,29 @@ get_torch_index_url() { if ! _has_amd_rocm_gpu; then echo "$_base/cpu"; return fi + # A generic rocm index is only safe when the gfx arch is readable: the + # Strix reroute (gfx1150/1151 -> arch-specific index) learns gfx from + # rocminfo/amd-smi, so if those are missing OR do not enumerate the GPU, an + # unknown-arch box might be Strix and would get the broken _grouped_mm + # wheels. Probe via the shared helper (override first, then rocminfo/amd-smi + # with visibility masks cleared); if the arch is unreadable, never guess a + # rocm index. A KFD-only host whose arch is still inferable from hardware + # IDs (PCI/cpuinfo/lspci) returns the cpu index and lets the runtime-less + # reroute below upgrade it to AMD per-arch wheels -- the reroute gate uses + # this same probe, so the handoff can't misfire. Only when inference fails + # too is CPU final, with the actionable warning. + _amd_gfx_probe=$(_probe_amd_gfx_arch) + if [ -z "$_amd_gfx_probe" ]; then + if _amd_inferred_gfx=$(_infer_linux_amd_gfx_arch 2>/dev/null) && \ + [ -n "$_amd_inferred_gfx" ] && \ + _amd_arch_index_family_for_gfx "$_amd_inferred_gfx" >/dev/null 2>&1; then + echo "[WARN] AMD GPU detected but rocminfo/amd-smi can't read its gfx arch -- inferring $_amd_inferred_gfx from hardware IDs." >&2 + echo "$_base/cpu"; return + fi + echo "[WARN] AMD GPU detected but its gfx arch can't be read (rocminfo/amd-smi missing or not enumerating the GPU) -- installing CPU-only PyTorch." >&2 + echo "[WARN] For GPU PyTorch, install or repair rocminfo/amd-smi (e.g. sudo pacman -S rocm-hip-sdk) and re-run this installer." >&2 + echo "$_base/cpu"; return + fi # AMD GPU confirmed -- detect ROCm version _rocm_tag="" _rocm_tag=$({ command -v amd-smi >/dev/null 2>&1 && \ @@ -2103,7 +2456,11 @@ get_torch_index_url() { { command -v rpm >/dev/null 2>&1 && \ ver="$(rpm -q --qf '%{VERSION}\n' rocm-core 2>/dev/null)" && \ [ -n "$ver" ] && \ - printf '%s\n' "$ver" | awk -F'[.-]' '{print "rocm"$1"."$2; exit}'; }) 2>/dev/null + printf '%s\n' "$ver" | awk -F'[.-]' '{print "rocm"$1"."$2; exit}'; }) 2>/dev/null || _rocm_tag="" + # ^ || guard: when EVERY version source is missing (e.g. rocminfo present + # but rocm-core not installed, so dpkg-query/rpm exit 1), the whole || + # chain fails and set -e would kill the installer BEFORE the actionable + # no-version WARN below -- exactly the fresh-install case it exists for. # Validate _rocm_tag: must match "rocmX.Y" with major >= 1 case "$_rocm_tag" in rocm[1-9]*.[0-9]*) : ;; # valid (major >= 1) @@ -2139,12 +2496,27 @@ get_torch_index_url() { esac return fi - # AMD GPU confirmed by rocminfo/amd-smi but ROCm version could not be - # read from any source (amd-smi, /opt/rocm/.info/version, hipconfig, - # dpkg, rpm). Warn explicitly rather than silently installing CPU PyTorch. - echo "[WARN] AMD GPU detected but ROCm version could not be determined -- falling back to CPU-only PyTorch" >&2 - echo "[WARN] Ensure one of the following is accessible: amd-smi, hipconfig, /opt/rocm/.info/version, rocm-core package" >&2 - echo "[WARN] To install ROCm: https://rocm.docs.amd.com/en/latest/deploy/linux/index.html" >&2 + # AMD GPU confirmed (rocminfo/amd-smi or the KFD topology fallback) but + # no ROCm/HIP install was found to read the version from (amd-smi, + # /opt/rocm/.info/version, hipconfig, dpkg, rpm). This is the common + # fresh-install case: the GPU is real, but with no ROCm userspace the + # correct PyTorch build can't be selected. Warn with an actionable fix + # rather than silently installing CPU PyTorch. + # A user-set UNSLOTH_ROCM_GFX_ARCH seeded the probe above, so rocminfo/ + # amd-smi may still be unable to see the GPU; when the named arch maps to + # a wheel family, the runtime-less reroute (gated on the override) will + # install the AMD per-arch wheels -- a CPU-only warning here would be + # false for that path. Defer like the inferable-arch branch does. + if [ -n "${UNSLOTH_ROCM_GFX_ARCH:-}" ] && \ + _amd_arch_index_family_for_gfx "$_amd_gfx_probe" >/dev/null 2>&1; then + echo "[WARN] AMD GPU detected with no readable ROCm version, but UNSLOTH_ROCM_GFX_ARCH=$_amd_gfx_probe is set -- routing to AMD per-arch wheels." >&2 + echo "$_base/cpu"; return + fi + echo "[WARN] AMD GPU detected, but no ROCm/HIP install was found to select the matching GPU PyTorch build -- falling back to CPU-only PyTorch." >&2 + echo "[WARN] Install the ROCm/HIP SDK, then re-run this installer:" >&2 + echo "[WARN] Arch / CachyOS : sudo pacman -S rocm-hip-sdk" >&2 + echo "[WARN] other distros : https://rocm.docs.amd.com/en/latest/deploy/linux/index.html" >&2 + echo "[WARN] Minimum required for version detection: amd-smi, hipconfig, /opt/rocm/.info/version, or the rocm-core package." >&2 echo "$_base/cpu"; return fi # Parse CUDA version from nvidia-smi output (POSIX-safe, no grep -P). @@ -2187,16 +2559,155 @@ _torch_flavor_tag() { esac } +# Final path segment of a wheel index URL ($1), lowercased, query/fragment stripped first +# so a token-authenticated pin (.../cu128?token=x) classifies as cu128 (else it reinstalls +# every update). Classification only. Shared with the py / ps1 leaf extractors. +_torch_index_url_leaf() { + _tl_u="${1%%\?*}" + _tl_u="${_tl_u%%#*}" + # Strip ALL trailing slashes, not one: .../rocm7.2// must yield rocm7.2, not an empty leaf. + while [ -n "$_tl_u" ] && [ "${_tl_u%/}" != "$_tl_u" ]; do + _tl_u="${_tl_u%/}" + done + printf '%s' "${_tl_u##*/}" | tr '[:upper:]' '[:lower:]' +} + +# True (exit 0) when a lowercased leaf is an EXACT pip ROCm family: rocm[.] +# or a gfx ARCHITECTURE leaf (gfx followed by a digit: gfx90a, gfx1151, gfx120x-all). A leaf +# that merely starts with rocm/gfx (rocm7.2-private, gfx-private) is a custom verbatim pin. +# Matches the py / ps1 sides. +_is_pip_rocm_family_leaf() { + case "$1" in + gfx[0-9]*) return 0 ;; + rocm[0-9]*) + # Exact rocm[.]: both major and minor must be non-empty all-digits + # (rocm7., rocm7.2.1, rocm7.2-private are all custom pins, not a family). + _rocm_rest="${1#rocm}" + case "$_rocm_rest" in + *.*.*) return 1 ;; + *.*) + _rocm_minor="${_rocm_rest#*.}" + case "${_rocm_rest%%.*}" in "" | *[!0-9]*) return 1 ;; esac + case "$_rocm_minor" in "" | *[!0-9]*) return 1 ;; esac + ;; + *[!0-9]*) return 1 ;; + esac + return 0 + ;; + *) return 1 ;; + esac +} + +# Whether release base $1 (X.Y[.Z...]) falls inside constraint window $2 +# ("torch>=A.B[.C],="*",<"*) ;; + *) echo "no"; return ;; + esac + _trw_floor="${_trw_con#torch>=}"; _trw_floor="${_trw_floor%%,*}" + _trw_ceil="${_trw_con##*,<}" + _v_maj="${1%%.*}"; _v_rest="${1#*.}"; _v_min="${_v_rest%%.*}" + _f_maj="${_trw_floor%%.*}"; _f_rest="${_trw_floor#*.}"; _f_min="${_f_rest%%.*}" + _c_maj="${_trw_ceil%%.*}"; _c_rest="${_trw_ceil#*.}"; _c_min="${_c_rest%%.*}" + for _trw_n in "$_v_maj" "$_v_min" "$_f_maj" "$_f_min" "$_c_maj" "$_c_min"; do + case "$_trw_n" in ''|*[!0-9]*) echo "no"; return ;; esac + done + if [ "$_v_maj" -gt "$_f_maj" ] || { [ "$_v_maj" -eq "$_f_maj" ] && [ "$_v_min" -ge "$_f_min" ]; }; then + if [ "$_v_maj" -lt "$_c_maj" ] || { [ "$_v_maj" -eq "$_c_maj" ] && [ "$_v_min" -lt "$_c_min" ]; }; then + echo "yes" + return + fi + fi + echo "no" +} + +# Keep the previous venv's torch on a re-run: echo "torch==X.Y.Z" when the probed +# version ($1) is inside the active constraint window ($2), else "". The RELEASE is kept +# regardless of flavor tag; the pin installs from the freshly chosen index, so flavor +# follows the machine (cpu <-> cuda, cu126 -> cu130, PyPI bare -> +cu130) while the +# release follows the user. Gating on flavor was wrong: a PyPI torch reports a BARE +# version (on Linux the PyPI wheel IS CUDA), misclassified "cpu", so a healthy 2.10 on a +# cu130 host was moved to 2.11. Per-leaf floors still win (rocm7.2 / gfx >=2.11 for the +# Strix _grouped_mm fix, out-of-window manual installs) and are never pinned; the caller's +# _PREV_FALLBACK_CONSTRAINT installs the newest supported release when the index lacks the +# exact one. Opt out with UNSLOTH_TORCH_UPGRADE=1. +_previous_torch_pin() { + _ptp_ver="$1" + _ptp_con="$2" + [ -n "$_ptp_ver" ] || { echo ""; return; } + [ "${UNSLOTH_TORCH_UPGRADE:-0}" = "1" ] && { echo ""; return; } + _ptp_base="${_ptp_ver%%+*}" + # Base must be a plain numeric release (X.Y[.Z]); probe noise and + # nightly/dev/source builds (2.11.0.dev20250704, 2.9.0a0) must never + # become a pin -- no stable index carries them, so pinning would only + # print "keeping it" and then burn a doomed resolve before falling back. + case "$_ptp_base" in + *[!0-9.]* | *..* | .* | *.) echo ""; return ;; + [0-9]*.[0-9]*) ;; + *) echo ""; return ;; + esac + [ "$(_torch_release_in_window "$_ptp_base" "$_ptp_con")" = "yes" ] || { echo ""; return; } + echo "torch==$_ptp_base" +} + +# Install torch from TORCH_INDEX_URL honoring a kept-release pin: with _PREV_TORCH_PIN +# set, TORCH_CONSTRAINT is the exact previous release; fall back to the supported range +# if the index lacks it (pruned mirror) rather than failing. Used by every --default-index +# path (NVIDIA cu*, AMD rocm/gfx fallbacks, cpu/mac, ROCm repairs) so preservation is +# uniform. Extra args (e.g. --force-reinstall) are passed through to uv. +_install_torch_default_index() { + if [ -n "$_PREV_TORCH_PIN" ]; then + # Pair the companions with the kept torch minor: torchaudio no longer + # exact-pins torch in its metadata, so leaving it unconstrained resolves + # a newer mismatched build (a kept torch 2.9.0 pulled torchaudio 2.11.0). + _itdi_base="${_PREV_TORCH_PIN#torch==}" + _itdi_minor="${_itdi_base#*.}" + _itdi_minor="${_itdi_minor%%.*}" + _itdi_tv="torchvision" + _itdi_ta="torchaudio" + case "$_itdi_base" in + 2.*) + _itdi_tv="torchvision==0.$((_itdi_minor + 15)).*" + _itdi_ta="torchaudio==2.${_itdi_minor}.*" + ;; + esac + if ! run_install_cmd_retry "install PyTorch (kept release)" uv pip install --python "$_VENV_PY" "$TORCH_CONSTRAINT" "$_itdi_tv" "$_itdi_ta" \ + --default-index "$TORCH_INDEX_URL" "$@"; then + substep "[WARN] $_PREV_TORCH_PIN is not installable from $(_strip_index_url_credentials "$TORCH_INDEX_URL") -- installing the newest supported release instead" "$C_WARN" + TORCH_CONSTRAINT="$_PREV_FALLBACK_CONSTRAINT" + _PREV_TORCH_PIN="" + run_install_cmd_retry "install PyTorch" uv pip install --python "$_VENV_PY" "$TORCH_CONSTRAINT" "$TORCHVISION_CONSTRAINT" "$TORCHAUDIO_CONSTRAINT" \ + --default-index "$TORCH_INDEX_URL" "$@" + fi + else + run_install_cmd_retry "install PyTorch" uv pip install --python "$_VENV_PY" "$TORCH_CONSTRAINT" "$TORCHVISION_CONSTRAINT" "$TORCHAUDIO_CONSTRAINT" \ + --default-index "$TORCH_INDEX_URL" "$@" + fi +} + # Expected tag from the index leaf ($1): cuXXX / cpu / rocm (rocmX.Y and gfx* -> # rocm). Empty on an unknown leaf (odd mirror) so the repair safely no-ops. _expected_torch_flavor_tag() { - _u="${1%/}" - _leaf="${_u##*/}" + _leaf=$(_torch_index_url_leaf "$1") case "$_leaf" in - cu[0-9]*) echo "$_leaf" ;; - cpu) echo "cpu" ;; - rocm*|gfx*) echo "rocm" ;; - *) echo "" ;; + cu[0-9]*) + # Exact cu + digits only; a cu*-suffixed leaf (cu128-private) -> "" (custom), + # else a correct +cu128 wheel is force-reinstalled every run. + case "${_leaf#cu}" in + *[!0-9]*) echo "" ;; + *) echo "$_leaf" ;; + esac + ;; + cpu) echo "cpu" ;; + # Exact rocm/gfx families only; a custom rocm*-suffixed leaf -> "" (custom). + *) + if _is_pip_rocm_family_leaf "$_leaf"; then echo "rocm"; else echo ""; fi + ;; esac } @@ -2206,14 +2717,42 @@ _expected_torch_flavor_tag() { # fresh-install paths above already use -- so a stale wheel is auto-repairable. # Unknown/odd-mirror leaves -> no, so we warn rather than risk a wrong reinstall. _torch_index_repairable() { - _u="${1%/}" - _leaf="${_u##*/}" + _leaf=$(_torch_index_url_leaf "$1") case "$_leaf" in - cu[0-9]*|rocm[0-9]*|gfx*) echo "yes" ;; - *) echo "no" ;; + cu[0-9]*) echo "yes" ;; + # Only EXACT rocm/gfx families resolve via --default-index; a suffixed leaf is verbatim. + *) + if _is_pip_rocm_family_leaf "$_leaf"; then echo "yes"; else echo "no"; fi + ;; esac } +# Remove credentials from a wheel index URL ($1) so an authenticated pin never leaks: +# drops userinfo AND query/fragment; scheme/host/path stay exact. Shared with py / ps1. +_strip_index_url_credentials() { + _sic_url="$1" + case "$_sic_url" in + *://*) ;; + *) printf '%s' "$_sic_url"; return ;; + esac + _sic_scheme="${_sic_url%%://*}" + _sic_rest="${_sic_url#*://}" + # Drop query / fragment (may hold auth tokens). + _sic_rest="${_sic_rest%%\?*}" + _sic_rest="${_sic_rest%%#*}" + _sic_auth="${_sic_rest%%/*}" + # Drop user:pass@ userinfo if present. + case "$_sic_auth" in + *@*) _sic_host="${_sic_auth##*@}" ;; + *) _sic_host="$_sic_auth" ;; + esac + if [ "$_sic_auth" = "$_sic_rest" ]; then + printf '%s://%s' "$_sic_scheme" "$_sic_host" + else + printf '%s://%s/%s' "$_sic_scheme" "$_sic_host" "${_sic_rest#*/}" + fi +} + get_radeon_wheel_url() { # Only meaningful on Linux. Picks a repo.radeon.com base URL whose listing # contains torch wheels. Tries paths like rocm-rel-7.2.1/, rocm-rel-7.2/, @@ -2335,7 +2874,7 @@ _pick_radeon_wheel() { # the installer -- always returns 0. Runs the idempotent helper (ROCm 7.2 + # librocdxg), then sources the env it persisted so detection finds the GPU. # Export the ROCm-on-WSL env into this process and persist it to /etc/profile.d -# so non-login Studio/llama launches inherit it. Idempotent (writes only when +# so non-login Unsloth/llama launches inherit it. Idempotent (writes only when # the drop-in is missing); no-op without librocdxg, so never fires off WSL. # /etc/profile.d is root-owned -- sudo-tee when not root, else ROCm vanishes # after this shell on a non-root reinstall. Best-effort either way. @@ -2380,7 +2919,7 @@ _maybe_bootstrap_rocm_wsl() { rocminfo 2>/dev/null | awk '/Name:[[:space:]]*gfx[1-9]/ && !/generic/{found=1} END{exit !found}'; then # rocminfo may work only via the transient env _ensure_rocm_probe_env # just set, which dies with the installer. Persist the drop-in so login - # shells (Studio, llama.cpp) inherit it -- else a reinstall over an + # shells (Unsloth, llama.cpp) inherit it -- else a reinstall over an # existing /opt/rocm (uninstall keeps ROCm but drops it) loses the GPU. _persist_rocm_wsl_dropin return 0 @@ -2389,7 +2928,7 @@ _maybe_bootstrap_rocm_wsl() { [ -e /dev/dxg ] || return 0 # Strix APUs show in /proc/cpuinfo (the CPU model); discrete cards don't, so also # ask the Windows host. Either signal suffices; the bootstrap detects arch from rocminfo. - if ! grep -qiE 'Ryzen AI Max|Radeon 80[0-9]0S|Strix Halo' /proc/cpuinfo 2>/dev/null \ + if ! grep -qiE 'Ryzen AI Max|Radeon 80[0-9][05]S|Strix Halo' /proc/cpuinfo 2>/dev/null \ && ! _wsl_amd_gpu_name >/dev/null 2>&1; then return 0 fi @@ -2402,7 +2941,7 @@ _maybe_bootstrap_rocm_wsl() { # shellcheck disable=SC1091 . /etc/profile.d/unsloth-rocm-wsl.sh || true else - # librocdxg present but the env drop-in is gone (e.g. a Studio + # librocdxg present but the env drop-in is gone (e.g. an Unsloth # uninstall removed it while keeping shared ROCm). Restore the env. _persist_rocm_wsl_dropin fi @@ -2459,10 +2998,88 @@ _maybe_bootstrap_rocm_wsl() { [ -n "$_rw_tmp" ] && rm -f "$_rw_tmp" return 0 } -_maybe_bootstrap_rocm_wsl || true +# When the caller pins the wheel index (UNSLOTH_TORCH_INDEX_URL / _FAMILY), honour it +# everywhere: skip the WSL ROCm bootstrap and the Radeon/Strix reroute below (which would +# re-probe the GPU and overwrite the pin). Trim whitespace first (parity with +# get_torch_index_url): a whitespace-only override is unset there, so must not flip this true. +_torch_index_pinned=false +_ti_url_trim="${UNSLOTH_TORCH_INDEX_URL:-}" +_ti_url_trim="${_ti_url_trim#"${_ti_url_trim%%[![:space:]]*}"}"; _ti_url_trim="${_ti_url_trim%"${_ti_url_trim##*[![:space:]]}"}" +_ti_family_trim="${UNSLOTH_TORCH_INDEX_FAMILY:-}" +_ti_family_trim="${_ti_family_trim#"${_ti_family_trim%%[![:space:]]*}"}"; _ti_family_trim="${_ti_family_trim%"${_ti_family_trim##*[![:space:]]}"}" +if [ -n "$_ti_url_trim" ] || [ -n "$_ti_family_trim" ]; then + _torch_index_pinned=true +fi +[ "$_torch_index_pinned" = true ] || _maybe_bootstrap_rocm_wsl || true TORCH_INDEX_URL=$(get_torch_index_url) +# Linux: ROCm runtime missing but a supported AMD gfx arch is inferable (Strix Halo +# in /proc/cpuinfo, lspci marketing name, UNSLOTH_ROCM_GFX_ARCH). Route to AMD's +# per-arch wheels like install.ps1 does on Windows (unslothai#7301). +# Gated on the runtime probes NOT naming a gfx: either no AMD GPU is detected at +# all (_has_amd_rocm_gpu false), or the GPU is visible only through the +# env-independent KFD topology while rocminfo/amd-smi can't read its arch +# (KFD-only host, unslothai#7314 -- before the KFD detection fix these hosts +# reached this reroute via the false branch, so the empty-probe condition +# preserves that routing). A */cpu index chosen WITH a readable gfx +# (unsupported/unreadable ROCm version, after its own warning) is a deliberate +# fallback -- rerouting it would contradict that decision, and stays excluded +# because the shared probe returns its gfx. An explicit UNSLOTH_ROCM_GFX_ARCH +# override stays authoritative either way. +if [ "$_torch_index_pinned" = false ] && [ "$SKIP_TORCH" = false ] && \ + ! _has_usable_nvidia_gpu && \ + { [ -n "${UNSLOTH_ROCM_GFX_ARCH:-}" ] || ! _has_amd_rocm_gpu || \ + [ -z "$(_probe_amd_gfx_arch)" ]; } && \ + case "$(uname -s)" in Linux) true ;; *) false ;; esac && \ + case "$_ARCH" in x86_64|amd64) true ;; *) false ;; esac; then + # ROCm torch wheels are x86_64-only; get_torch_index_url returns CPU on other + # arches, so an inferred/overridden gfx must not reroute arm64 to AMD wheels. + case "$TORCH_INDEX_URL" in + */cpu) + _linux_inferred_gfx=$(_infer_linux_amd_gfx_arch 2>/dev/null || true) + if [ -n "$_linux_inferred_gfx" ]; then + _amd_family=$(_amd_arch_index_family_for_gfx "$_linux_inferred_gfx") || _amd_family="" + if [ -n "$_amd_family" ]; then + _amd_mirror="${UNSLOTH_AMD_ROCM_MIRROR:-https://repo.amd.com/rocm/whl}" + while [ "${_amd_mirror%/}" != "$_amd_mirror" ]; do + _amd_mirror="${_amd_mirror%/}" + done + TORCH_INDEX_URL="${_amd_mirror}/${_amd_family}/" + # Hand the inferred arch to setup.sh (llama.cpp): it re-probes + # ROCm on its own, and on these runtime-less hosts its probes + # find nothing, so without this it classifies the box as + # non-ROCm and installs the CPU prebuilt while torch just got + # AMD per-arch wheels. setup.sh and install_llama_prebuilt.py + # both honor UNSLOTH_ROCM_GFX_ARCH, so exporting it is the + # whole handoff (a user-set override re-exports unchanged). + export UNSLOTH_ROCM_GFX_ARCH="$_linux_inferred_gfx" + case "$_linux_inferred_gfx" in + gfx1201|gfx1200|gfx1151|gfx1150) + TORCH_CONSTRAINT="torch>=2.11.0,<2.12.0" + TORCHVISION_CONSTRAINT="torchvision>=0.26.0,<0.27.0" + TORCHAUDIO_CONSTRAINT="torchaudio>=2.11.0,<2.12.0" + ;; + esac + echo "" >&2 + # KFD-only hosts reach this reroute with /dev/kfd present + # (that's what detected them), so don't claim it's missing. + if _has_amd_rocm_gpu; then + echo " [WARN] AMD GPU visible via the kernel driver (KFD) but rocminfo/amd-smi can't read its gfx arch; using $_linux_inferred_gfx." >&2 + else + echo " [WARN] ROCm runtime not visible (/dev/kfd, rocminfo, amd-smi) but $_linux_inferred_gfx inferred." >&2 + fi + echo " [WARN] Routing to AMD arch-specific wheels ($(_strip_index_url_credentials "$TORCH_INDEX_URL"))." >&2 + echo " [WARN] These wheels bundle their own ROCm runtime; install the kernel stack for native compute:" >&2 + echo " [WARN] https://docs.unsloth.ai/get-started/install-and-update/amd" >&2 + echo " [WARN] Tip: set UNSLOTH_ROCM_GFX_ARCH=$_linux_inferred_gfx to skip inference next time." >&2 + echo "" >&2 + fi + fi + ;; + esac +fi + # Export the resolved torch backend ("cuda", "rocm", or "cpu") so that # downstream scripts (setup.sh -> install_python_stack.py) know what was # chosen here and can skip ROCm-specific repair steps on CUDA/CPU hosts. @@ -2470,24 +3087,74 @@ TORCH_INDEX_URL=$(get_torch_index_url) # whose base path happens to contain "rocm" or "gfx" must not mislabel a # cu*/cpu index as ROCm (radeon repo URLs end in rocm-rel-X.Y/, Strix # overrides in gfxNNNN/, so the trailing slash is stripped first). -_torch_index_leaf="${TORCH_INDEX_URL%/}" +# Lowercase the leaf so every gfx*/rocm*/cu* arm matches regardless of case (canonical AMD +# RDNA4 leaf is gfx120X-all). CUDA is branded only on a real cu[0-9]* leaf, so a mirror +# leaf (/current) does NOT commit a CUDA backend; an unknown leaf leaves the var unset so +# the stack probes the GPU. Query/fragment dropped first, then ALL trailing slashes (in +# lockstep with the shared _torch_index_url_leaf extractor). +_torch_index_leaf="${TORCH_INDEX_URL%%\?*}" +_torch_index_leaf="${_torch_index_leaf%%#*}" +# Strip ALL trailing slashes, not one: .../cu128// must yield cu128, not an empty leaf. +while [ -n "$_torch_index_leaf" ] && [ "${_torch_index_leaf%/}" != "$_torch_index_leaf" ]; do + _torch_index_leaf="${_torch_index_leaf%/}" +done _torch_index_leaf="${_torch_index_leaf##*/}" +_torch_index_leaf=$(printf '%s' "$_torch_index_leaf" | tr '[:upper:]' '[:lower:]') case "$_torch_index_leaf" in rocm*|gfx*) export UNSLOTH_TORCH_BACKEND="rocm" ;; cpu) export UNSLOTH_TORCH_BACKEND="cpu" ;; - *) export UNSLOTH_TORCH_BACKEND="cuda" ;; + cu[0-9]*) export UNSLOTH_TORCH_BACKEND="cuda" ;; + # Unknown leaf (odd mirror, /current): unset so a stale inherited value can't leak and + # the stack probes the GPU. + *) unset UNSLOTH_TORCH_BACKEND ;; esac -# rocm7.2 ships torch 2.11.0 -- adjust the constraint to allow it. -# All other ROCm tags and CUDA stay within <2.11.0. -case "$TORCH_INDEX_URL" in - */rocm7.2) TORCH_CONSTRAINT="torch>=2.11.0,<2.12.0" ;; +# Whether TORCH_INDEX_URL names an actual pip ROCm family (rocm* / gfx*), gating the +# ROCm-only side effects below (AMD bitsandbytes, ROCm-torch repair). Digit-gated so a leaf +# merely STARTING with "rocm" isn't force-repaired from the wrong path. +if _is_pip_rocm_family_leaf "$_torch_index_leaf"; then + _torch_index_is_rocm_family=true +else + _torch_index_is_rocm_family=false +fi + +# rocm7.2 and the per-gfx indexes with the _grouped_mm <2.11 bug (gfx120X-all, gfx1151, +# gfx1150) ship torch 2.11.0 -- raise the floor (also covers a pinned override that skipped +# the Strix reroute). Pin the companions too: the per-gfx index publishes them independently +# and a bare name can resolve a 2.12 ABI-mismatched wheel. Match on the FINAL leaf so a +# custom mirror with a gfx/rocm7.2 path segment but a cu*/cpu family isn't forced. +case "$_torch_index_leaf" in + rocm7.2|gfx120x-all|gfx1151|gfx1150) + TORCH_CONSTRAINT="torch>=2.11.0,<2.12.0" + TORCHVISION_CONSTRAINT="torchvision>=0.26.0,<0.27.0" + TORCHAUDIO_CONSTRAINT="torchaudio>=2.11.0,<2.12.0" + ;; + # CUDA cu12x/cu13x indexes ship torch 2.11.x: widen the ceiling to <2.12.0 (matches + # _CUDA_TORCH_PKG_SPEC) and widen the companions with it so the trio stays paired. + cu[0-9]*) + TORCH_CONSTRAINT="torch>=2.4,<2.12.0" + TORCHVISION_CONSTRAINT="torchvision>=0.19,<0.27.0" + TORCHAUDIO_CONSTRAINT="torchaudio>=2.4,<2.12.0" + ;; esac +# A pinned custom/unknown-leaf index (/simple, /current, /cu128-private) has no curated +# companion set, so bound torchvision/torchaudio to the same <2.11 range the Python path pins +# (else a mirror with newer companions resolves a 2.12 ABI-mismatched wheel). Known families +# keep their curated companions above (_expected_torch_flavor_tag returns "" only for custom). +if [ "$_torch_index_pinned" = true ] && \ + [ -z "$(_expected_torch_flavor_tag "$TORCH_INDEX_URL")" ]; then + TORCHVISION_CONSTRAINT="torchvision>=0.19,<0.26.0" + TORCHAUDIO_CONSTRAINT="torchaudio>=2.4,<2.11.0" +fi + # Auto-detect GPU for AMD ROCm based # get_torch_index_url must have chosen */rocm* # (gfx in rocminfo or amd-smi list). Then require rocminfo "Marketing Name:.*Radeon". +# Skipped when the index is pinned: an explicit override must not be rerouted to the +# Radeon/Strix repos by GPU probing. _amd_gpu_radeon=false +if [ "$_torch_index_pinned" = false ]; then case "$TORCH_INDEX_URL" in */rocm*) if _has_amd_rocm_gpu && command -v rocminfo >/dev/null 2>&1 && \ @@ -2496,29 +3163,64 @@ case "$TORCH_INDEX_URL" in fi ;; esac -# ── Strix Halo / Strix Point: force rocm7.2 wheels, bypass Radeon repo ─────── -# gfx1151 (Strix Halo) and gfx1150 (Strix Point) have a ROCm 7.1 driver bug -# that causes a segfault in torch._grouped_mm (moe_utils.py line 167). -# The Radeon repo now ships cp313 wheels for rocm-rel-7.1, so when -# _amd_gpu_radeon=true the installer silently lands on the broken combo. -# Detect these GPUs when TORCH_INDEX_URL is rocm7.1 and override to rocm7.2. -case "$TORCH_INDEX_URL" in - */rocm7.1|*/rocm7.1.*) +# 0 when a rocmX.Y index leaf ($1, the final path segment) is older than floor +# $2.$3 (int compare, so rocm7.2 < rocm7.13). Non-rocm leaves (gfx*, cu*, cpu) and +# non-numeric versions return 1. Leaf-based (like $_torch_index_leaf) so a mirror +# base holding its own rocm token compares the family leaf, not the base path. +_rocm_leaf_below() { + case "$1" in rocm[0-9]*.[0-9]*) : ;; *) return 1 ;; esac + _rb=${1#rocm}; _maj=${_rb%%.*}; _min=${_rb#*.}; _min=${_min%%.*} + case "$_maj$_min" in *[!0-9]*) return 1 ;; esac + if [ "$_maj" -lt "$2" ]; then return 0; fi + if [ "$_maj" -eq "$2" ] && [ "$_min" -lt "$3" ]; then return 0; fi + return 1 +} +# ── Strix Halo / Strix Point: route to the AMD arch-specific index ─────────── +# gfx1151/gfx1150 need torch 2.11+rocm7.13 from repo.amd.com/rocm/whl/gfx/, +# which carries AMD's real fixes (the rocm7.1 _grouped_mm segfault, moe_utils.py:167, +# and later Strix kernel bugs). Every generic pytorch.org index below rocm7.13 lacks +# them (and the Radeon repo can be offline, unslothai#7264), so reroute a detected +# Strix GPU whenever the picked index is older than the arch build -- covers today's +# rocm6.0-7.2 and any future 7.x < 7.13; rocm7.13+ already has the fixes, so leave it. +case "$_torch_index_leaf" in + rocm[0-9]*) # Collect every gfx token in rocminfo / amd-smi enumeration order # (skip duplicates), then index by HIP_VISIBLE_DEVICES / # ROCR_VISIBLE_DEVICES so a mixed Strix iGPU + non-Strix dGPU box # where the user selected the dGPU does NOT get rerouted to the # Strix per-gfx index. - _gfx_all="" - if command -v rocminfo >/dev/null 2>&1; then - _gfx_all=$(rocminfo 2>/dev/null | grep -oE 'gfx[1-9][0-9a-z]{2,3}') + # || true on each probe: no gfx match makes grep exit 1, which under + # set -euo pipefail would abort the installer before the next fallback + # runs (now that the case matches every rocm* index, not just rocm7.1). + # A user-supplied UNSLOTH_ROCM_GFX_ARCH overrides probing (mirrors setup.sh + # and the display block), so a Strix override still reaches the arch index. + _gfx_all=$(printf '%s' "${UNSLOTH_ROCM_GFX_ARCH:-}" | tr '[:upper:]' '[:lower:]') + if [ -z "$_gfx_all" ] && command -v rocminfo >/dev/null 2>&1; then + _gfx_all=$(rocminfo 2>/dev/null | grep -oE 'gfx[1-9][0-9a-z]{2,3}' || true) fi if [ -z "$_gfx_all" ] && command -v amd-smi >/dev/null 2>&1; then - _gfx_all=$(amd-smi list 2>/dev/null | grep -oE 'gfx[1-9][0-9a-z]{2,3}') + _gfx_all=$(amd-smi list 2>/dev/null | grep -oE 'gfx[1-9][0-9a-z]{2,3}' || true) # PowerShell paths also probe `amd-smi static --asic`; mirror it # so a host with hipinfo-less amd-smi reports the gfx target. if [ -z "$_gfx_all" ]; then - _gfx_all=$(amd-smi static --asic 2>/dev/null | grep -oE 'gfx[1-9][0-9a-z]{2,3}') + _gfx_all=$(amd-smi static --asic 2>/dev/null | grep -oE 'gfx[1-9][0-9a-z]{2,3}' || true) + fi + fi + # get_torch_index_url reads the arch with ROCR/HIP masks cleared, so a + # mask hiding every agent (e.g. ROCR_VISIBLE_DEVICES=-1) still lands + # here on a generic rocm index; re-probe unmasked or a masked-out Strix + # box keeps the broken generic wheels. Partial masks never get here + # (they enumerate at least one agent above) and keep their selection. + # ${VAR+x} (not :-): a SET-but-empty mask also hides every agent and + # must trigger the re-probe too. + if [ -z "$_gfx_all" ] && [ -n "${ROCR_VISIBLE_DEVICES+x}${HIP_VISIBLE_DEVICES+x}" ]; then + if command -v rocminfo >/dev/null 2>&1; then + _gfx_all=$( (unset ROCR_VISIBLE_DEVICES HIP_VISIBLE_DEVICES; rocminfo 2>/dev/null) | grep -oE 'gfx[1-9][0-9a-z]{2,3}' || true) + fi + if [ -z "$_gfx_all" ] && command -v amd-smi >/dev/null 2>&1; then + _gfx_all=$( (unset ROCR_VISIBLE_DEVICES HIP_VISIBLE_DEVICES; amd-smi list 2>/dev/null) | grep -oE 'gfx[1-9][0-9a-z]{2,3}' || true) + [ -z "$_gfx_all" ] && \ + _gfx_all=$( (unset ROCR_VISIBLE_DEVICES HIP_VISIBLE_DEVICES; amd-smi static --asic 2>/dev/null) | grep -oE 'gfx[1-9][0-9a-z]{2,3}' || true) fi fi _runtime_gfx="" @@ -2543,13 +3245,14 @@ case "$TORCH_INDEX_URL" in case "$_runtime_gfx" in gfx1151|gfx1150) _strix_gfx="$_runtime_gfx" ;; esac - if [ -n "$_strix_gfx" ]; then + # Skip rocm7.13+ generic indexes: they already ship the fixes, so the + # arch build (rocm7.13) would be a downgrade rather than a rescue. + if [ -n "$_strix_gfx" ] && _rocm_leaf_below "$_torch_index_leaf" 7 13; then echo "" >&2 - echo " [WARN] $_strix_gfx (Strix) + ROCm 7.1 detected -- known _grouped_mm segfault" >&2 - echo " [WARN] ROCm 7.1 wheels are broken for gfx1150/gfx1151 (moe_utils.py:167)" >&2 - echo " [WARN] Routing to AMD arch-specific index (torch 2.11+rocm7.13 has the real fix)" >&2 - echo " [WARN] Upgrade ROCm to 7.2+ to use the standard index:" >&2 - echo " [WARN] https://rocm.docs.amd.com/en/latest/deploy/linux/index.html" >&2 + echo " [WARN] $_strix_gfx (Strix) detected -- routing to the AMD arch-specific index" >&2 + echo " [WARN] torch 2.11+rocm7.13 has AMD's real gfx1150/gfx1151 fixes (the ROCm 7.1" >&2 + echo " [WARN] _grouped_mm segfault, moe_utils.py:167, and later Strix kernel bugs)," >&2 + echo " [WARN] and is more reliable than the rocm7.2 index or an offline Radeon repo." >&2 echo "" >&2 # AMD's arch-specific index serves torch 2.11.0+rocm7.13.0 which has AMD's # actual fix for the gfx1151/gfx1150 _grouped_mm kernel bug -- preferred @@ -2564,10 +3267,31 @@ case "$TORCH_INDEX_URL" in done TORCH_INDEX_URL="${_amd_strix_base}/${_strix_gfx}/" TORCH_CONSTRAINT="torch>=2.11.0,<2.12.0" + # Pin companions to 2.11 (per-gfx index publishes them independently). + TORCHVISION_CONSTRAINT="torchvision>=0.26.0,<0.27.0" + TORCHAUDIO_CONSTRAINT="torchaudio>=2.11.0,<2.12.0" _amd_gpu_radeon=false fi ;; esac +fi # _torch_index_pinned guard (Radeon + Strix reroute) +# Re-run over an existing install: keep the previous venv's torch RELEASE; the fresh +# index above supplies the right flavor for this machine. Evaluated HERE, after every +# index/constraint decision including the Strix reroute, so the window checked is the +# final one and a raised floor (rocm7.2 / Strix gfx) rejects an older release. +# _PREV_FALLBACK_CONSTRAINT keeps the range so the install can fall back when the exact +# release is not on the chosen index (mirrors may prune old wheels). Skipped for --no-torch. +_PREV_TORCH_PIN="" +_PREV_FALLBACK_CONSTRAINT="$TORCH_CONSTRAINT" +if [ "$SKIP_TORCH" = false ]; then + _prev_pin=$(_previous_torch_pin "$_PREV_TORCH_VER" "$TORCH_CONSTRAINT") + if [ -n "$_prev_pin" ]; then + _PREV_TORCH_PIN="$_prev_pin" + TORCH_CONSTRAINT="$_prev_pin" + substep "existing install has torch $_PREV_TORCH_VER -- keeping it (set UNSLOTH_TORCH_UPGRADE=1 to get the newest release)" + fi +fi + _TAURI_TORCH_INDEX_FAMILY=$(_tauri_torch_index_family "$TORCH_INDEX_URL") if [ "$_amd_gpu_radeon" = true ] && [ "$SKIP_TORCH" = false ]; then _TAURI_TORCH_INDEX_FAMILY="radeon" @@ -2617,7 +3341,7 @@ elif case "$TORCH_INDEX_URL" in */rocm*|*/gfx*) true ;; *) false ;; esac; then case "$_gpu_disp_mkt" in *"9070 XT"*|*9080*) _gpu_disp_gfx="gfx1201" ;; # RDNA 4 *9070*|*9060*) _gpu_disp_gfx="gfx1200" ;; # RDNA 4 - *"8060S"*|*"8050S"*|*"8040S"*|*"Strix Halo"*|*"Ryzen AI Max"*|*"AI Max"*) _gpu_disp_gfx="gfx1151" ;; # RDNA 3.5 (Strix Halo: Radeon 8060S/8050S/8040S iGPU, Ryzen AI Max+) + *"8065S"*|*"8060S"*|*"8050S"*|*"8040S"*|*"Strix Halo"*|*"Ryzen AI Max"*|*"AI Max"*) _gpu_disp_gfx="gfx1151" ;; # RDNA 3.5 (Strix Halo + Gorgon Halo: Radeon 8065S/8060S/8050S/8040S iGPU, Ryzen AI Max / Max+) *"890M"*|*"880M"*|*"860M"*|*"840M"*|*"Strix Point"*|*"Krackan"*|*"HX 37"*|*"AI 9 HX"*|*"AI 9 36"*|*"AI 7 35"*|*"AI 5 34"*|*"AI 7 PRO 35"*|*"AI 5 33"*) _gpu_disp_gfx="gfx1150" ;; # RDNA 3.5 (Strix/Krackan Point: Radeon 890M/880M iGPU, Ryzen AI 9 HX 370/375) *"RX 7600"*|*"RX 7700S"*|*"RX 7650"*|*"PRO W7600"*|*"PRO W7500"*|*"PRO V710"*) _gpu_disp_gfx="gfx1102" ;; # RDNA 3 (Navi 33) *"RX 7900"*|*"RX 7800"*|*"RX 7700"*|*"PRO W7900"*|*"PRO W7800"*|*"PRO W7700"*) _gpu_disp_gfx="gfx1100" ;; # RDNA 3 desktop / workstation (Navi 31) @@ -2652,6 +3376,17 @@ elif case "$TORCH_INDEX_URL" in */rocm*|*/gfx*) true ;; *) false ;; esac; then elif [ "$OS" = "macos" ] && [ "$_ARCH" = "arm64" ]; then # Apple Silicon: PyTorch gets Metal (MPS) acceleration over unified memory, so not CPU-only. step "gpu" "Apple Silicon (Metal, unified memory)" +elif _has_amd_rocm_gpu; then + if [ "$_torch_index_pinned" = true ]; then + # An explicit UNSLOTH_TORCH_INDEX_URL/_FAMILY pin skipped all probing; + # do not claim ROCm is unusable when a CPU/other index was requested. + step "gpu" "AMD GPU (torch index pinned: $_torch_index_leaf)" "$C_WARN" + else + # AMD GPU visible to the kernel but the torch index stayed CPU: no usable + # ROCm userspace to pick a wheel. "none" would repeat the false diagnosis + # this installer used to give. + step "gpu" "AMD GPU (no usable ROCm -- CPU fallback)" "$C_WARN" + fi else step "gpu" "none (CPU-only)" "$C_WARN" fi @@ -2660,8 +3395,17 @@ fi case "$TORCH_INDEX_URL" in */cpu) if [ "$SKIP_TORCH" = false ] && [ "$OS" != "macos" ]; then - substep "No GPU detected -- installing CPU-only PyTorch." "$C_WARN" - if [ "$OS" = "wsl" ]; then + if [ "$_torch_index_pinned" = true ]; then + # An explicit CPU pin is a request, not a detection failure: + # skip the SDK guidance (ROCm may be perfectly healthy here). + substep "CPU-only PyTorch (index pinned via UNSLOTH_TORCH_INDEX_URL / _FAMILY)." + elif _has_amd_rocm_gpu; then + substep "AMD GPU detected, but no usable ROCm/HIP install -- installing CPU-only PyTorch." "$C_WARN" + substep "Install the ROCm/HIP SDK and re-run this installer for GPU PyTorch." "$C_WARN" + else + substep "No GPU detected -- installing CPU-only PyTorch." "$C_WARN" + fi + if [ "$OS" = "wsl" ] && [ "$_torch_index_pinned" = false ]; then # WSL + no GPU detected (detection above found nothing). Common # cause: an AMD GPU whose ROCm-on-WSL runtime isn't exposed yet -- # /dev/dxg present (graphics) but no ROCm runtime. @@ -2688,6 +3432,13 @@ case "$TORCH_INDEX_URL" in substep " driver is current; or run unsloth/scripts/install_rocm_wsl_strixhalo.sh yourself." else substep "AMD ROCm users: see https://docs.unsloth.ai/get-started/install-and-update/amd" + # Only when ROCm truly can't see the GPU: a detected-but-too-old + # ROCm (rocminfo works, wheels need 6.0+) has its own guidance. + if ! _has_amd_rocm_gpu && _amd_gpu_present_via_pci; then + substep "An AMD GPU is on the PCI bus but ROCm cannot see it (no /dev/kfd," "$C_WARN" + substep " rocminfo, or amd-smi). Install the ROCm kernel stack so /dev/kfd exists;" + substep " Strix Halo (gfx1151/gfx1150) needs a recent kernel (6.11+) and ROCm 7.x." + fi fi substep "Re-run with --no-torch for GGUF-only (faster, no PyTorch):" substep " curl -fsSL https://unsloth.ai/install.sh | sh -s -- --no-torch" @@ -2697,7 +3448,7 @@ case "$TORCH_INDEX_URL" in if [ "$_amd_gpu_radeon" = true ]; then substep "wheels: repo.radeon.com (Radeon)" else - substep "wheels: $TORCH_INDEX_URL" + substep "wheels: $(_strip_index_url_credentials "$TORCH_INDEX_URL")" fi ;; esac @@ -2705,9 +3456,46 @@ esac # ── Install unsloth directly into the venv (no activation needed) ── tauri_log "STEP" "Installing PyTorch" _VENV_PY="$VENV_DIR/bin/python" + +# A released unsloth wheel can pin an older torch (unsloth 2026.7.2 declares +# torch<2.11.0); a with-deps PyPI resolve then downgrades the whole trio, +# swapping the pinned +cuXXX/+rocm build for PyPI's default. The flavor guard +# below misses this (PyPI's torch 2.10 default is itself cu128-flavored), so +# freeze the trio via uv --overrides (overrides replace dependency requirements +# during resolution) while unsloth's other deps resolve normally. Sets +# _UNSLOTH_TORCH_OVERRIDES from the trio in the venv; every with-deps unsloth +# install (migrated and fresh) must call this before resolving and rm it after. +_build_unsloth_torch_overrides() { + _UNSLOTH_TORCH_OVERRIDES="" + [ "$SKIP_TORCH" = false ] || return 0 + _torch_trio_pins=$("$_VENV_PY" -c " +from importlib.metadata import version, PackageNotFoundError +for _p in ('torch', 'torchvision', 'torchaudio'): + try: + print(_p + '==' + version(_p)) + except PackageNotFoundError: + pass +" 2>/dev/null) || _torch_trio_pins="" + case "$_torch_trio_pins" in + torch==*) + _UNSLOTH_TORCH_OVERRIDES=$(mktemp) + printf '%s\n' "$_torch_trio_pins" > "$_UNSLOTH_TORCH_OVERRIDES" + # The CLI --overrides flag replaces any UV_OVERRIDE env file (same + # uv setting; macOS arm64 exports one here), so fold its pins in. + # awk, not cat: it drops inherited torch-trio lines (uv intersects + # duplicate overrides, so a conflicting pin would make resolution + # unsatisfiable) and newline-terminates the last line so an + # unterminated file cannot join two requirements into one. + for _ov_file in ${UV_OVERRIDE:-}; do + [ -f "$_ov_file" ] && awk '!/^[[:space:]]*torch(vision|audio)?([[:space:]<>=!~;@[]|$)/' "$_ov_file" >> "$_UNSLOTH_TORCH_OVERRIDES" + done + ;; + esac +} + if [ "$_MIGRATED" = true ]; then - # Migrated env: force-reinstall unsloth+unsloth-zoo to ensure clean state - # in the new venv location, while preserving existing torch/CUDA + # Migrated env: force-reinstall unsloth+unsloth-zoo for a clean state, preserving + # existing torch/CUDA unless the ROCm repair below fires. substep "upgrading unsloth in migrated environment..." if [ "$SKIP_TORCH" = true ]; then # No-torch: install unsloth + unsloth-zoo with --no-deps (current @@ -2716,7 +3504,7 @@ if [ "$_MIGRATED" = true ]; then # to prevent transitive torch resolution. run_install_cmd_retry "install unsloth (migrated no-torch)" uv pip install --python "$_VENV_PY" --no-deps \ --reinstall-package unsloth --reinstall-package unsloth-zoo \ - "unsloth>=2026.7.3" "unsloth-zoo>=2026.7.3" + "unsloth>=2026.7.5" "unsloth-zoo>=2026.7.6" # Resolve pydantic WITH deps so pip pins pydantic-core to the # matching version (no-torch-runtime.txt below is --no-deps). # All transitive deps are torch-free. @@ -2729,9 +3517,13 @@ if [ "$_MIGRATED" = true ]; then else # Pin mlx-lm away from 0.31.3 here too: a curl-piped migration has no # overrides file, so UV_OVERRIDE is unset and this positional is the only cover. + _build_unsloth_torch_overrides run_install_cmd_retry "install unsloth (migrated)" uv pip install --python "$_VENV_PY" \ + ${_UNSLOTH_TORCH_OVERRIDES:+--overrides "$_UNSLOTH_TORCH_OVERRIDES"} \ --reinstall-package unsloth --reinstall-package unsloth-zoo \ - "unsloth>=2026.7.3" "unsloth-zoo>=2026.7.3" ${_MLX_LM_EXCLUDE_ARG:-} + "unsloth>=2026.7.5" "unsloth-zoo>=2026.7.6" ${_MLX_LM_EXCLUDE_ARG:-} + [ -n "$_UNSLOTH_TORCH_OVERRIDES" ] && rm -f "$_UNSLOTH_TORCH_OVERRIDES" + _UNSLOTH_TORCH_OVERRIDES="" fi if [ "$STUDIO_LOCAL_INSTALL" = true ]; then substep "overlaying local repo (editable)..." @@ -2744,21 +3536,14 @@ if [ "$_MIGRATED" = true ]; then # AMD ROCm: install bitsandbytes even in migrated environments so # existing ROCm installs gain the AMD bitsandbytes build without a # fresh reinstall. - if [ "$SKIP_TORCH" = false ]; then - case "$TORCH_INDEX_URL" in - */rocm*|*/gfx*) - _install_bnb_rocm "install bitsandbytes (AMD)" "$_VENV_PY" - # Repair ROCm torch if overwritten during migrated install - _has_hip=$("$_VENV_PY" -c "import torch; print(getattr(torch.version,'hip','') or '')" 2>/dev/null || true) - if [ -z "$_has_hip" ]; then - substep "repairing ROCm torch (overwritten by dependency resolution)..." - run_install_cmd_retry "repair ROCm torch" uv pip install --python "$_VENV_PY" \ - "$TORCH_CONSTRAINT" torchvision torchaudio \ - --default-index "$TORCH_INDEX_URL" \ - --force-reinstall - fi - ;; - esac + if [ "$SKIP_TORCH" = false ] && [ "$_torch_index_is_rocm_family" = true ]; then + _install_bnb_rocm "install bitsandbytes (AMD)" "$_VENV_PY" + # Repair ROCm torch if overwritten during migrated install + _has_hip=$("$_VENV_PY" -c "import torch; print(getattr(torch.version,'hip','') or '')" 2>/dev/null || true) + if [ -z "$_has_hip" ]; then + substep "repairing ROCm torch (overwritten by dependency resolution)..." + _install_torch_default_index --force-reinstall + fi fi elif [ -n "$TORCH_INDEX_URL" ]; then # Fresh: Step 1 - install torch from explicit index (skip when --no-torch or Intel Mac) @@ -2820,7 +3605,42 @@ elif [ -n "$TORCH_INDEX_URL" ]; then _ta_ver=$(_extract_version "$_ta_whl" "torchaudio") _radeon_versions_match=false - if [ -n "$_torch_ver" ] && [ -n "$_tv_ver" ] && [ -n "$_ta_ver" ]; then + # Kept release (_PREV_TORCH_PIN) wins here too: pick its exact + # patch (else the newest patch of its minor) plus the paired + # vision/audio wheels. Any gap falls back to the newest-trio + # search below, mirroring _install_torch_default_index, so a + # rerun never drifts to another release nor below the kept one. + if [ -n "$_PREV_TORCH_PIN" ]; then + _prev_kept_base="${_PREV_TORCH_PIN#torch==}" + _prev_kept_minor="${_prev_kept_base#*.}" + _prev_kept_minor="${_prev_kept_minor%%.*}" + case "$_prev_kept_minor" in + ''|*[!0-9]*) ;; + *) + _kept_torch=$(_pick_radeon_wheel "torch" "${_prev_kept_base}" 2>/dev/null) || _kept_torch="" + [ -z "$_kept_torch" ] && { _kept_torch=$(_pick_radeon_wheel "torch" "2.${_prev_kept_minor}." 2>/dev/null) || _kept_torch=""; } + _kept_tv=$(_pick_radeon_wheel "torchvision" "0.$((_prev_kept_minor + 15))." 2>/dev/null) || _kept_tv="" + _kept_ta=$(_pick_radeon_wheel "torchaudio" "2.${_prev_kept_minor}." 2>/dev/null) || _kept_ta="" + if [ -n "$_kept_torch" ] && [ -n "$_kept_tv" ] && [ -n "$_kept_ta" ]; then + _torch_whl=$_kept_torch + _tv_whl=$_kept_tv + _ta_whl=$_kept_ta + _tri_whl="" + _radeon_versions_match=true + # Say so when the listing pruned the exact patch + # and a same-series build is installed instead. + case "$(printf '%s' "${_kept_torch##*/}" | sed 's/%2[Bb]/+/g')" in + "torch-${_prev_kept_base}"[+-]*) ;; + *) substep "kept release ${_prev_kept_base} is not in the Radeon listing -- installing the closest 2.${_prev_kept_minor} series build instead" ;; + esac + else + substep "[WARN] Radeon repo lacks a complete wheel set for kept $_PREV_TORCH_PIN -- installing the newest compatible set instead" "$C_WARN" + fi + ;; + esac + fi + if [ "$_radeon_versions_match" != true ] && \ + [ -n "$_torch_ver" ] && [ -n "$_tv_ver" ] && [ -n "$_ta_ver" ]; then _torch_minor=${_torch_ver#*.} _ta_minor=${_ta_ver#*.} _tv_minor=${_tv_ver#*.} @@ -2877,10 +3697,8 @@ elif [ -n "$TORCH_INDEX_URL" ]; then if [ -z "$_torch_whl" ] || [ -z "$_tv_whl" ] || [ -z "$_ta_whl" ] || \ [ "$_radeon_versions_match" != true ]; then - substep "[WARN] Radeon repo lacks a compatible wheel set for this Python; falling back to ROCm index ($TORCH_INDEX_URL)" "$C_WARN" - run_install_cmd_retry "install PyTorch" uv pip install --python "$_VENV_PY" \ - "$TORCH_CONSTRAINT" torchvision torchaudio \ - --default-index "$TORCH_INDEX_URL" + substep "[WARN] Radeon repo lacks a compatible wheel set for this Python; falling back to ROCm index ($(_strip_index_url_credentials "$TORCH_INDEX_URL"))" "$C_WARN" + _install_torch_default_index else substep "installing PyTorch from Radeon repo (${_RADEON_BASE_URL})..." # Pass explicit wheel URLs so the matched trio is @@ -2900,42 +3718,34 @@ elif [ -n "$TORCH_INDEX_URL" ]; then fi fi else - substep "[WARN] Radeon repo unavailable; falling back to ROCm index ($TORCH_INDEX_URL)" "$C_WARN" - run_install_cmd_retry "install PyTorch" uv pip install --python "$_VENV_PY" \ - "$TORCH_CONSTRAINT" torchvision torchaudio \ - --default-index "$TORCH_INDEX_URL" + substep "[WARN] Radeon repo unavailable; falling back to ROCm index ($(_strip_index_url_credentials "$TORCH_INDEX_URL"))" "$C_WARN" + _install_torch_default_index fi else substep "[WARN] Radeon GPU detected but could not detect full ROCm version; falling back to ROCm index" "$C_WARN" - run_install_cmd_retry "install PyTorch" uv pip install --python "$_VENV_PY" \ - "$TORCH_CONSTRAINT" torchvision torchaudio \ - --default-index "$TORCH_INDEX_URL" + _install_torch_default_index fi else - substep "installing PyTorch ($TORCH_INDEX_URL)..." - run_install_cmd_retry "install PyTorch" uv pip install --python "$_VENV_PY" "$TORCH_CONSTRAINT" torchvision torchaudio \ - --default-index "$TORCH_INDEX_URL" + substep "installing PyTorch ($(_strip_index_url_credentials "$TORCH_INDEX_URL"))..." + _install_torch_default_index fi # AMD ROCm: install bitsandbytes (once, after torch, for all ROCm paths). # Gate on SKIP_TORCH=false so a user running with --no-torch on a ROCm # host stays in GGUF-only mode rather than pulling in bitsandbytes, # which is only useful once torch is present for training. - if [ "$SKIP_TORCH" = false ]; then - case "$TORCH_INDEX_URL" in - */rocm*|*/gfx*) - _install_bnb_rocm "install bitsandbytes (AMD)" "$_VENV_PY" - ;; - esac + if [ "$SKIP_TORCH" = false ] && [ "$_torch_index_is_rocm_family" = true ]; then + _install_bnb_rocm "install bitsandbytes (AMD)" "$_VENV_PY" fi - # Fresh: Step 2 - install unsloth, preserving pre-installed torch + # Fresh: Step 2 - install unsloth, preserving the torch Step 1 installed tauri_log "STEP" "Installing Unsloth" substep "installing unsloth (this may take a few minutes)..." + _build_unsloth_torch_overrides if [ "$SKIP_TORCH" = true ]; then # No-torch: install unsloth + unsloth-zoo with --no-deps, then # runtime deps (typer, safetensors, transformers, etc.) with --no-deps. run_install_cmd_retry "install unsloth (no-torch)" uv pip install --python "$_VENV_PY" --no-deps \ --upgrade-package unsloth --upgrade-package unsloth-zoo \ - "unsloth>=2026.7.3" "unsloth-zoo>=2026.7.3" + "unsloth>=2026.7.5" "unsloth-zoo>=2026.7.6" # Same pydantic-with-deps trick as the migrated branch. run_install_cmd_retry "install pydantic (with deps for compatible core)" \ uv pip install --python "$_VENV_PY" pydantic @@ -2953,7 +3763,8 @@ elif [ -n "$TORCH_INDEX_URL" ]; then fi elif [ "$STUDIO_LOCAL_INSTALL" = true ]; then run_install_cmd_retry "install unsloth (local)" uv pip install --python "$_VENV_PY" \ - --upgrade-package unsloth "unsloth>=2026.7.3" "unsloth-zoo>=2026.7.3" + ${_UNSLOTH_TORCH_OVERRIDES:+--overrides "$_UNSLOTH_TORCH_OVERRIDES"} \ + --upgrade-package unsloth "unsloth>=2026.7.5" "unsloth-zoo>=2026.7.6" substep "overlaying local repo (editable)..." run_install_cmd "overlay local repo" uv pip install --python "$_VENV_PY" -e "$_REPO_ROOT" --no-deps substep "overlaying unsloth-zoo from git main..." @@ -2962,30 +3773,26 @@ elif [ -n "$TORCH_INDEX_URL" ]; then "unsloth-zoo @ git+https://github.com/unslothai/unsloth-zoo" else run_install_cmd_retry "install unsloth" uv pip install --python "$_VENV_PY" \ + ${_UNSLOTH_TORCH_OVERRIDES:+--overrides "$_UNSLOTH_TORCH_OVERRIDES"} \ --upgrade-package unsloth -- "$PACKAGE_NAME" ${_MLX_LM_EXCLUDE_ARG:-} fi + [ -n "$_UNSLOTH_TORCH_OVERRIDES" ] && rm -f "$_UNSLOTH_TORCH_OVERRIDES" + _UNSLOTH_TORCH_OVERRIDES="" # AMD ROCm: repair torch if the unsloth/unsloth-zoo install pulled in # CUDA torch from PyPI, overwriting the ROCm wheels installed in Step 1. - if [ "$SKIP_TORCH" = false ]; then - case "$TORCH_INDEX_URL" in - */rocm*|*/gfx*) - _has_hip=$("$_VENV_PY" -c "import torch; print(getattr(torch.version,'hip','') or '')" 2>/dev/null || true) - if [ -z "$_has_hip" ]; then - substep "repairing ROCm torch (overwritten by dependency resolution)..." - run_install_cmd_retry "repair ROCm torch" uv pip install --python "$_VENV_PY" \ - "$TORCH_CONSTRAINT" torchvision torchaudio \ - --default-index "$TORCH_INDEX_URL" \ - --force-reinstall - fi - ;; - esac + if [ "$SKIP_TORCH" = false ] && [ "$_torch_index_is_rocm_family" = true ]; then + _has_hip=$("$_VENV_PY" -c "import torch; print(getattr(torch.version,'hip','') or '')" 2>/dev/null || true) + if [ -z "$_has_hip" ]; then + substep "repairing ROCm torch (overwritten by dependency resolution)..." + _install_torch_default_index --force-reinstall + fi fi else # Fallback: GPU detection failed to produce a URL -- let uv resolve torch tauri_log "STEP" "Installing Unsloth" substep "installing unsloth (this may take a few minutes)..." if [ "$STUDIO_LOCAL_INSTALL" = true ]; then - run_install_cmd_retry "install unsloth (auto torch backend)" uv pip install --python "$_VENV_PY" "unsloth-zoo>=2026.7.3" "unsloth>=2026.7.3" --torch-backend=auto + run_install_cmd_retry "install unsloth (auto torch backend)" uv pip install --python "$_VENV_PY" "unsloth-zoo>=2026.7.6" "unsloth>=2026.7.5" --torch-backend=auto substep "overlaying local repo (editable)..." run_install_cmd "overlay local repo" uv pip install --python "$_VENV_PY" -e "$_REPO_ROOT" --no-deps substep "overlaying unsloth-zoo from git main..." @@ -2997,6 +3804,15 @@ else fi fi +_installed_package_version=$("$_VENV_PY" -c \ + 'from importlib.metadata import version; import sys; print(version(sys.argv[1]))' \ + "$PACKAGE_NAME" 2>/dev/null || true) +if [ -n "$_installed_package_version" ]; then + step "$PACKAGE_NAME" "$_installed_package_version installed" +else + substep "[WARN] installed $PACKAGE_NAME version could not be determined" "$C_WARN" +fi + # ── Enforce the installed torch flavor matches the detected GPU build ── # PEP 440 ignores the +cpu/+cuXXX/+rocm local label in a version range, so uv # keeps a stale torch==X+cpu against a GPU index and the venv silently trains on @@ -3014,9 +3830,7 @@ if [ "$SKIP_TORCH" = false ] && [ -n "${TORCH_INDEX_URL:-}" ]; then if [ -n "$_installed_torch_tag" ] && [ "$_installed_torch_tag" != "$_expected_torch_tag" ] \ && [ "$(_torch_index_repairable "$TORCH_INDEX_URL")" = "yes" ]; then substep "PyTorch flavor mismatch (installed $_installed_torch_tag, need $_expected_torch_tag) -- reinstalling correct build..." - run_install_cmd "reinstall PyTorch ($_expected_torch_tag)" uv pip install --python "$_VENV_PY" \ - "$TORCH_CONSTRAINT" torchvision torchaudio \ - --default-index "$TORCH_INDEX_URL" \ + _install_torch_default_index \ --reinstall-package torch --reinstall-package torchvision --reinstall-package torchaudio _installed_torch_ver=$("$_VENV_PY" -c "import torch; print(torch.__version__)" 2>/dev/null || true) _installed_torch_tag="" @@ -3027,13 +3841,13 @@ if [ "$SKIP_TORCH" = false ] && [ -n "${TORCH_INDEX_URL:-}" ]; then substep "[WARN] PyTorch is CPU-only but a $_expected_torch_tag GPU build was expected for this machine." "$C_WARN" substep "[WARN] Training and GPU inference will run on CPU until this is fixed." "$C_WARN" substep "[WARN] Re-run this installer, or reinstall the GPU build manually:" "$C_WARN" - substep "[WARN] uv pip install --python \"$_VENV_PY\" \"$TORCH_CONSTRAINT\" torchvision torchaudio --default-index $TORCH_INDEX_URL --reinstall-package torch --reinstall-package torchvision --reinstall-package torchaudio" "$C_WARN" + substep "[WARN] uv pip install --python \"$_VENV_PY\" \"$TORCH_CONSTRAINT\" \"$TORCHVISION_CONSTRAINT\" \"$TORCHAUDIO_CONSTRAINT\" --default-index $(_strip_index_url_credentials "$TORCH_INDEX_URL") --reinstall-package torch --reinstall-package torchvision --reinstall-package torchaudio" "$C_WARN" fi fi fi # ── Run studio setup ── -tauri_log "STEP" "Running Studio setup" +tauri_log "STEP" "Running Unsloth setup" # When --local, use the repo's own setup.sh directly. # Otherwise, find it inside the installed package. SETUP_SH="" @@ -3227,7 +4041,7 @@ printf " ${C_TITLE}%s${C_RST}\n" "Unsloth Studio installed!" printf " ${C_DIM}%s${C_RST}\n" "$RULE" echo "" -# In interactive terminals, ask the user before starting Studio unless the +# In interactive terminals, ask the user before starting Unsloth unless the # caller explicitly disabled the post-install prompt. # In non-interactive environments (Docker, CI, cloud-init) just print instructions. if [ "$_SKIP_AUTOSTART" != true ] && [ -t 1 ]; then diff --git a/pyproject.toml b/pyproject.toml index f8754d5143..f7da75b6d3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -25,7 +25,7 @@ classifiers = [ "Topic :: Scientific/Engineering :: Artificial Intelligence", ] dependencies = [ - "typer", + "typer>=0.12.0", "rich", "pydantic", "pyyaml", @@ -42,7 +42,7 @@ version = {attr = "unsloth.models._utils.__version__"} include-package-data = true [tool.setuptools.package-data] -unsloth_cli = ["codex_fallback_prompt.md"] +unsloth_cli = ["codex_fallback_prompt.md", "pi_subagent.ts"] studio = [ "*.sh", "*.ps1", @@ -74,7 +74,7 @@ triton = [ ] huggingfacenotorch = [ - "unsloth_zoo>=2026.7.3", + "unsloth_zoo>=2026.7.6", "wheel>=0.42.0", "packaging", "numpy", @@ -93,9 +93,20 @@ huggingfacenotorch = [ "trl>=0.18.2,!=0.19.0,<=0.24.0", "sentence-transformers", ] +# torchcodec backend for Gemma audio / datasets>=4 (#7225). +# Pick the audio-torch* pin matching your torch minor (see TORCH_TORCHCODEC). +audio-torch210 = [ + "torchcodec>=0.10.0,<0.11.0 ; python_version >= '3.10'", +] +audio-torch290 = [ + "torchcodec>=0.8.0,<0.10.0 ; python_version >= '3.10'", +] +audio-torch280 = [ + "torchcodec>=0.6.0,<0.8.0 ; python_version >= '3.9'", +] huggingface = [ "unsloth[huggingfacenotorch]", - "unsloth_zoo>=2026.7.3", + "unsloth_zoo>=2026.7.6", "torchvision", "unsloth[triton]", ] @@ -541,16 +552,19 @@ cu126-torch2100 = [ "unsloth[huggingface]", "bitsandbytes>=0.45.5,!=0.46.0,!=0.48.0", "unsloth[cu126onlytorch2100]", + "unsloth[audio-torch210]", ] cu128-torch2100 = [ "unsloth[huggingface]", "bitsandbytes>=0.45.5,!=0.46.0,!=0.48.0", "unsloth[cu128onlytorch2100]", + "unsloth[audio-torch210]", ] cu130-torch2100 = [ "unsloth[huggingface]", "bitsandbytes>=0.45.5,!=0.46.0,!=0.48.0", "unsloth[cu130onlytorch2100]", + "unsloth[audio-torch210]", ] kaggle = [ "unsloth[huggingface]", @@ -589,7 +603,7 @@ colab-ampere-torch220 = [ "flash-attn>=2.6.3 ; ('linux' in sys_platform)", ] colab-new = [ - "unsloth_zoo>=2026.7.3", + "unsloth_zoo>=2026.7.6", "packaging", "tyro", "transformers>=4.51.3,!=4.52.0,!=4.52.1,!=4.52.2,!=4.52.3,!=4.53.0,!=4.54.0,!=4.55.0,!=4.55.1,!=4.57.0,!=4.57.4,!=4.57.5,!=5.0.0,!=5.1.0,<=5.5.0", @@ -840,16 +854,19 @@ cu126-ampere-torch2100 = [ "unsloth[huggingface]", "bitsandbytes>=0.45.5,!=0.46.0,!=0.48.0", "unsloth[cu126onlytorch2100]", + "unsloth[audio-torch210]", ] cu128-ampere-torch2100 = [ "unsloth[huggingface]", "bitsandbytes>=0.45.5,!=0.46.0,!=0.48.0", "unsloth[cu128onlytorch2100]", + "unsloth[audio-torch210]", ] cu130-ampere-torch2100 = [ "unsloth[huggingface]", "bitsandbytes>=0.45.5,!=0.46.0,!=0.48.0", "unsloth[cu130onlytorch2100]", + "unsloth[audio-torch210]", ] flashattentiontorch260abiFALSEcu12x = [ "flash-attn @ https://github.com/Dao-AILab/flash-attention/releases/download/v2.7.4.post1/flash_attn-2.7.4.post1+cu12torch2.6cxx11abiFALSE-cp39-cp39-linux_x86_64.whl ; ('linux' in sys_platform) and python_version == '3.9'", @@ -1134,7 +1151,8 @@ intelgputorch210 = [ "torchvision @ https://download.pytorch.org/whl/xpu/torchvision-0.25.0%2Bxpu-cp313-cp313-win_amd64.whl#sha256=1c4b44b36a557f7381e3076fb8843366742238648441d607c8d049c6da0f8886 ; sys_platform == 'win32' and python_version == '3.13' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", ] intel-gpu-torch210 = [ - "unsloth[intelgputorch210]" + "unsloth[intelgputorch210]", + "unsloth[audio-torch210]", ] intelgputorch2110 = [ "unsloth_zoo[intelgpu]", @@ -1288,6 +1306,7 @@ rocm72-torch2100 = [ "torchvision @ https://repo.radeon.com/rocm/manylinux/rocm-rel-7.2/torchvision-0.25.0%2Brocm7.2.0.git82df5f59-cp311-cp311-linux_x86_64.whl ; platform_system == 'Linux' and python_version == '3.11' and platform_machine == 'x86_64'", "torchvision @ https://repo.radeon.com/rocm/manylinux/rocm-rel-7.2/torchvision-0.25.0%2Brocm7.2.0.git82df5f59-cp312-cp312-linux_x86_64.whl ; platform_system == 'Linux' and python_version == '3.12' and platform_machine == 'x86_64'", "torchvision @ https://repo.radeon.com/rocm/manylinux/rocm-rel-7.2/torchvision-0.25.0%2Brocm7.2.0.git82df5f59-cp313-cp313-linux_x86_64.whl ; platform_system == 'Linux' and python_version == '3.13' and platform_machine == 'x86_64'", + "unsloth[audio-torch210]", ] rocm711-torch2100 = [ "unsloth[amd]", @@ -1306,6 +1325,7 @@ rocm711-torch2100 = [ "torchvision @ https://repo.radeon.com/rocm/manylinux/rocm-rel-7.1.1/torchvision-0.25.0%2Brocm7.1.1.git82df5f59-cp311-cp311-linux_x86_64.whl ; platform_system == 'Linux' and python_version == '3.11' and platform_machine == 'x86_64'", "torchvision @ https://repo.radeon.com/rocm/manylinux/rocm-rel-7.1.1/torchvision-0.25.0%2Brocm7.1.1.git82df5f59-cp312-cp312-linux_x86_64.whl ; platform_system == 'Linux' and python_version == '3.12' and platform_machine == 'x86_64'", "torchvision @ https://repo.radeon.com/rocm/manylinux/rocm-rel-7.1.1/torchvision-0.25.0%2Brocm7.1.1.git82df5f59-cp313-cp313-linux_x86_64.whl ; platform_system == 'Linux' and python_version == '3.13' and platform_machine == 'x86_64'", + "unsloth[audio-torch210]", ] [project.urls] diff --git a/scripts/build_whisper_cpp.sh b/scripts/build_whisper_cpp.sh new file mode 100755 index 0000000000..9f7e4d4ef3 --- /dev/null +++ b/scripts/build_whisper_cpp.sh @@ -0,0 +1,71 @@ +#!/bin/sh +# Build whisper.cpp's whisper-server for Studio's GGUF dictation engine. +# +# Installs into the managed Studio home so the backend's binary discovery +# (core/inference/stt_ggml_sidecar.py::find_whisper_server_binary) picks it up: +# /whisper.cpp/build/bin/whisper-server (custom home) +# ~/.unsloth/whisper.cpp/build/bin/whisper-server (default) +# +# Usage: +# ./scripts/build_whisper_cpp.sh # build the pinned tag +# WHISPER_CPP_TAG=v1.9.0 ./scripts/build_whisper_cpp.sh +# +# Requires: git, cmake, a C/C++ toolchain (the same prerequisites as a +# llama.cpp source build). GPU backends are auto-detected by whisper.cpp's +# CMake (Metal on macOS; set GGML_CUDA=1 to force a CUDA build on Linux). + +set -eu + +WHISPER_CPP_SOURCE="${WHISPER_CPP_SOURCE:-https://github.com/ggml-org/whisper.cpp}" +WHISPER_CPP_TAG="${WHISPER_CPP_TAG:-v1.9.1}" + +STUDIO_HOME="${UNSLOTH_STUDIO_HOME:-${STUDIO_HOME:-}}" +CUSTOM_STUDIO_HOME=false +if [ -n "$STUDIO_HOME" ]; then + CUSTOM_STUDIO_HOME=true + INSTALL_DIR="$STUDIO_HOME/whisper.cpp" +else + INSTALL_DIR="$HOME/.unsloth/whisper.cpp" +fi + +command -v git >/dev/null 2>&1 || { echo "ERROR: git is required" >&2; exit 1; } +command -v cmake >/dev/null 2>&1 || { echo "ERROR: cmake is required" >&2; exit 1; } + +# Same policy as studio/setup.sh's _assert_studio_owned_or_absent: never delete +# a directory under a custom Studio home unless Studio itself created it (the +# marker file below). Protects a user-managed whisper.cpp/src from rm -rf. +STUDIO_OWNED_MARKER=".unsloth-studio-owned" +if [ "$CUSTOM_STUDIO_HOME" = true ] && [ -e "$INSTALL_DIR" ] && \ + [ ! -f "$INSTALL_DIR/$STUDIO_OWNED_MARKER" ]; then + echo "ERROR: $INSTALL_DIR already exists and is not marked as an Unsloth-owned whisper.cpp build tree." >&2 + echo " Move it aside or choose an empty UNSLOTH_STUDIO_HOME before re-running." >&2 + exit 1 +fi + +echo "==> Building whisper.cpp ($WHISPER_CPP_TAG) into $INSTALL_DIR" +mkdir -p "$INSTALL_DIR" +: > "$INSTALL_DIR/$STUDIO_OWNED_MARKER" + +if [ ! -d "$INSTALL_DIR/src/.git" ]; then + rm -rf "$INSTALL_DIR/src" + git clone --depth 1 --branch "$WHISPER_CPP_TAG" "$WHISPER_CPP_SOURCE" "$INSTALL_DIR/src" +else + git -C "$INSTALL_DIR/src" fetch --depth 1 origin "$WHISPER_CPP_TAG" + git -C "$INSTALL_DIR/src" checkout FETCH_HEAD +fi + +CMAKE_FLAGS="-DCMAKE_BUILD_TYPE=Release -DBUILD_SHARED_LIBS=OFF" +if [ "${GGML_CUDA:-0}" = "1" ]; then + CMAKE_FLAGS="$CMAKE_FLAGS -DGGML_CUDA=ON" +fi + +# shellcheck disable=SC2086 +cmake -S "$INSTALL_DIR/src" -B "$INSTALL_DIR/src/build" $CMAKE_FLAGS +NCPU="$(getconf _NPROCESSORS_ONLN 2>/dev/null || echo 4)" +cmake --build "$INSTALL_DIR/src/build" --config Release --target whisper-server -j"$NCPU" + +mkdir -p "$INSTALL_DIR/build/bin" +cp "$INSTALL_DIR/src/build/bin/whisper-server" "$INSTALL_DIR/build/bin/whisper-server" + +echo "==> Installed $INSTALL_DIR/build/bin/whisper-server" +"$INSTALL_DIR/build/bin/whisper-server" --help >/dev/null 2>&1 && echo "==> Binary runs OK" diff --git a/scripts/install_rocm_wsl_strixhalo.sh b/scripts/install_rocm_wsl_strixhalo.sh index aa560fc432..697aae933f 100644 --- a/scripts/install_rocm_wsl_strixhalo.sh +++ b/scripts/install_rocm_wsl_strixhalo.sh @@ -219,7 +219,7 @@ fi echo "${ROCM_DIR}/lib" | $SUDO tee /etc/ld.so.conf.d/rocm.conf >/dev/null $SUDO ldconfig -# ── Step 4: persist environment (system-wide so Studio's worker inherits it) ── +# ── Step 4: persist environment (system-wide so Unsloth's worker inherits it) ── say "Persisting ROCm-on-WSL environment" _envfile="/etc/profile.d/unsloth-rocm-wsl.sh" $SUDO tee "$_envfile" >/dev/null <: markers stripped, so version bumps and line shifts do not reopen an entry but changed code does. severity and evidence are for review only. Regenerate with --write-baseline AFTER reviewing every line.", + "_comment": "scan_packages.py allowlist (reviewed). Each entry is a CRITICAL/HIGH finding manually judged benign. Matched on (package, package-relative file, check, evidence_hash); evidence_hash is over the matched code with L: markers stripped, so version bumps and line shifts do not reopen an entry but changed code does. severity and evidence are for review only. Regenerate with --write-baseline AFTER reviewing every line.", "version": 1, "entries": [ { @@ -95,8 +95,8 @@ "file": "fastapi/routing.py", "check": "C2 polling/beaconing loop detected", "severity": "CRITICAL", - "evidence": "L586: while True: sha256:251135b5ebfdd1248916449f32262575e003ef64382501c65b7e4061d67bda45", - "evidence_hash": "365aef4449c8089753d9398417cd76ab762cef547d75db70d87bca9c0b550ab5" + "evidence": "L587: while True: sha256:06c2c7f15d73bf192e5e3272c5ff5fcaeff7f6774fef5f4eca6ef473ae50e2b3", + "evidence_hash": "57acd497f404c203e4450d0580ad85aa8a33406e8d64ad06fbac6cf47d97b24d" }, { "package": "fastmcp-slim", @@ -303,8 +303,8 @@ "file": "openai/_base_client.py", "check": "C2 polling/beaconing loop detected", "severity": "CRITICAL", - "evidence": "L264: while True: sha256:95ca67e46d42354ae650abbdc5b0d97df8b0ed43187800bf40f5690c3901b94b", - "evidence_hash": "a57d8d15fed0bf04f9967dcc18a18b80bb19f4095675bccbb78ac0450d7fce14" + "evidence": "L274: while True: sha256:90a38e5c1e26893c7c273354143612640e9a9c0f079d3e2b60612d79f24e80a6", + "evidence_hash": "1022e8e8649436ec64a98a9d9141d085452c49549fd2157b0278fc369a83ac66" }, { "package": "openai", @@ -319,8 +319,8 @@ "file": "openai/auth/_workload.py", "check": "Accesses cloud metadata/IMDS AND makes network calls", "severity": "CRITICAL", - "evidence": "IMDS: L96: url = \"http://169.254.169.254/metadata/identity/oauth2/token\" | L149: url = \"http://metadata.google.internal/computeMetadata/v1/instance/service-accounts/default/identity\"\nNetwork: L77: http_client: httpx.Client | None = None, | L108: with httpx.Client() as client: | L133: http_client: httpx.Client | None = None, | L155: with httpx.Client() as client: | L248: with httpx.Client() as client:", - "evidence_hash": "1581d9f4a23393e9af23fbe5ef9f66807b22c5b5a3f1fe167254c9ebee108567" + "evidence": "IMDS: L97: url = \"http://169.254.169.254/metadata/identity/oauth2/token\" | L150: url = \"http://metadata.google.internal/computeMetadata/v1/instance/service-accounts/default/identity\"\nNetwork: L78: http_client: httpx.Client | None = None, | L109: with httpx.Client() as client: | L134: http_client: httpx.Client | None = None, | L156: with httpx.Client() as client: | L251: exchange_client = DefaultHttpx2Client(follow_redirects=False) if self._use_httpx2 else httpx.Client()", + "evidence_hash": "9717e51cb961dc14c458955d91a1e48e3753997346ecea0106bded3a8d64bfe0" }, { "package": "openai", @@ -343,8 +343,8 @@ "file": "openai/resources/beta/responses/responses.py", "check": "C2 polling/beaconing loop detected", "severity": "CRITICAL", - "evidence": "L3999: while True: sha256:df298b6eaf3416589b79f4ef283f8fb76e54d505bfda8840673f8e6419117e2e", - "evidence_hash": "10ce5cb5a7097fcff4042ddcfb4802edda60aa4b7b113c8b926a52ddb76f78c2" + "evidence": "L4000: while True: sha256:f8ab538118daba9ec06e27399dbdc90a4521c3390e6a47a6348a1f180a83effd", + "evidence_hash": "31481ea83c687acc27144d72d3832d4fb98dd1c79fb5e0ddd85080de95997b9f" }, { "package": "openai", @@ -359,16 +359,16 @@ "file": "openai/resources/realtime/realtime.py", "check": "C2 polling/beaconing loop detected", "severity": "CRITICAL", - "evidence": "L310: while True: sha256:458198ff3d3f05870bf98c9564cbfd68c739e57b9bbe4120ed81e3eb6af74a05", - "evidence_hash": "a3165d21e46b3ce553795daeae53e8f80e8e89c5cb228e68e6dcaff54bca5a89" + "evidence": "L311: while True: sha256:5b63313072aae9ca28677e03426513ccf12221e4f4e0ea6c31efbe09790633b5", + "evidence_hash": "05e1af469d651b51673763a7c4cdf759af9472fb627b7b470adc28cc237bd650" }, { "package": "openai", "file": "openai/resources/responses/responses.py", "check": "C2 polling/beaconing loop detected", "severity": "CRITICAL", - "evidence": "L3950: while True: sha256:1ce0b5a388c747945cdfda1a71b77afdfd03ae840d7aa9fa62f02eb00aa5e29f", - "evidence_hash": "6de300ebb5e6e17cb51c89cbcdf08515a44655182f0776f0908a9d1043ebbcd7" + "evidence": "L3951: while True: sha256:d68ef896bf0743ca430cfacb9a3353da1f3b9c51c3a21b6450a07a32b55aa2ac", + "evidence_hash": "160eecdd79b521bffbe8476f782b69a0724c35d1b19376a7600807165fd54f9f" }, { "package": "openai", @@ -1545,6 +1545,78 @@ "severity": "HIGH", "evidence": "Obfusc: L836: code = compile(module, \"\", \"exec\")\nExec: L736: exec(code, globs, locs)", "evidence_hash": "5c0992c90f05c772abd94d00784f157de337e1f8567f8b3aee1b15e46c96cd5d" + }, + { + "package": "unsloth-zoo", + "file": "tests/test_mlx_save_export_regressions.py", + "check": "Writes to /tmp and executes (staged dropper)", + "severity": "CRITICAL", + "evidence": "L165: temporary_location=\"/tmp/ignored\", sha256:ab5c587f9ec31a0cc10ee55698ab133a417148d9d3f371bbc81b1e13fa119c13", + "evidence_hash": "93a11159147aad94f353ec4d2e0b8486b256abef88cd96d741813222cd32b138" + }, + { + "package": "unsloth-zoo", + "file": "tests/test_vision_collator_audio.py", + "check": "Writes to /tmp and executes (staged dropper)", + "severity": "CRITICAL", + "evidence": "L111: out = extract_audio_info(msgs({\"type\": \"audio\", key: \"/tmp/a.wav\"})) sha256:2efe23ffbe2b91b8403aec9b700736919b59e5ca770f8e1f5501651b44b7d398", + "evidence_hash": "d416b79dd17b24214f3f7653ac01354507d7bf0fc464dee30a4a4b8998f063ba" + }, + { + "package": "openai", + "file": "openai/_base_client.py", + "check": "C2 polling/beaconing loop detected", + "severity": "CRITICAL", + "evidence": "L274: while True: sha256:90a38e5c1e26893c7c273354143612640e9a9c0f079d3e2b60612d79f24e80a6", + "evidence_hash": "1022e8e8649436ec64a98a9d9141d085452c49549fd2157b0278fc369a83ac66" + }, + { + "package": "openai", + "file": "openai/auth/_workload.py", + "check": "Accesses cloud metadata/IMDS AND makes network calls", + "severity": "CRITICAL", + "evidence": "IMDS: L97: url = \"http://169.254.169.254/metadata/identity/oauth2/token\" | L150: url = \"http://metadata.google.internal/computeMetadata/v1/instance/service-accounts/default/identity\"\nNetwork: L78: http_client: httpx.Client | None = None, | L109: with httpx.Client() as client: | L134: http_client: httpx.Client | None = None, | L156: with httpx.Client() as client: | L251: exchange_client = DefaultHttpx2Client(follow_redirects=False) if self._use_httpx2 else httpx.Client()", + "evidence_hash": "9717e51cb961dc14c458955d91a1e48e3753997346ecea0106bded3a8d64bfe0" + }, + { + "package": "openai", + "file": "openai/resources/beta/responses/responses.py", + "check": "C2 polling/beaconing loop detected", + "severity": "CRITICAL", + "evidence": "L4000: while True: sha256:f8ab538118daba9ec06e27399dbdc90a4521c3390e6a47a6348a1f180a83effd", + "evidence_hash": "31481ea83c687acc27144d72d3832d4fb98dd1c79fb5e0ddd85080de95997b9f" + }, + { + "package": "openai", + "file": "openai/resources/realtime/realtime.py", + "check": "C2 polling/beaconing loop detected", + "severity": "CRITICAL", + "evidence": "L311: while True: sha256:5b63313072aae9ca28677e03426513ccf12221e4f4e0ea6c31efbe09790633b5", + "evidence_hash": "05e1af469d651b51673763a7c4cdf759af9472fb627b7b470adc28cc237bd650" + }, + { + "package": "openai", + "file": "openai/resources/responses/responses.py", + "check": "C2 polling/beaconing loop detected", + "severity": "CRITICAL", + "evidence": "L3951: while True: sha256:d68ef896bf0743ca430cfacb9a3353da1f3b9c51c3a21b6450a07a32b55aa2ac", + "evidence_hash": "160eecdd79b521bffbe8476f782b69a0724c35d1b19376a7600807165fd54f9f" + }, + { + "package": "unsloth-zoo", + "file": "tests/test_gemma4_forced_float32_ple_dtype.py", + "check": "Advanced obfuscation (marshal/compile/zlib) + exec/eval", + "severity": "HIGH", + "evidence": "Obfusc: L277: compile(rewritten + _GEMMA4_PLE_CAST_HELPER, \"\", \"exec\") | L440: compile(on, \"\", \"exec\") | L468: compile(generated, \"\", \"exec\")\nExec: L19: exec(_GEMMA4_PLE_CAST_HELPER, namespace)", + "evidence_hash": "a85e24d8e7c431563cbd83b70f91a3b971abde0f37083d68e70984147960cc70" + }, + { + "package": "unsloth-zoo", + "file": "tests/test_vision_collator_audio.py", + "check": "Writes to /tmp and executes (staged dropper)", + "severity": "CRITICAL", + "evidence": "L111: out = extract_audio_info(msgs({\"type\": \"audio\", key: \"/tmp/a.wav\"})) sha256:022f81dd21acfc6a35a058de96132834c218404a9e37b3d09a7768a8c8f6c728", + "evidence_hash": "2d1e75446af120d9133a42aa8af426a839d3434d9dc109cc1d6c1b22ca1ddb75" } ] } diff --git a/scripts/stamp_studio_release.py b/scripts/stamp_studio_release.py index 7dab35ea8a..739f6d1063 100644 --- a/scripts/stamp_studio_release.py +++ b/scripts/stamp_studio_release.py @@ -2,7 +2,7 @@ # SPDX-License-Identifier: AGPL-3.0-only # Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 -"""Stamp and verify display-only Studio release metadata for builds.""" +"""Stamp and verify display-only Unsloth release metadata for builds.""" from __future__ import annotations @@ -50,7 +50,7 @@ MAX_VERSION_LENGTH = 64 PLACEHOLDER = """# SPDX-License-Identifier: AGPL-3.0-only # Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 -\"\"\"Build-stamped Studio release metadata. +\"\"\"Build-stamped Unsloth release metadata. Release builds may rewrite this module in the build workspace before creating Python artifacts. Keep the committed value neutral so source checkouts do not @@ -145,7 +145,7 @@ def build_info_source(version: str | None) -> str: return f'''# SPDX-License-Identifier: AGPL-3.0-only # Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 -"""Build-stamped Studio release metadata.""" +"""Build-stamped Unsloth release metadata.""" STUDIO_RELEASE_VERSION = {literal} ''' @@ -168,7 +168,7 @@ def stamp(require_release: bool) -> int: version, source = resolve_version() if version is not None and not is_valid_version(version): print( - f"Invalid Studio release version from {source}: {version!r}", + f"Invalid Unsloth release version from {source}: {version!r}", file = sys.stderr, ) return 2 @@ -196,9 +196,9 @@ def stamp(require_release: bool) -> int: if version is None: if require_release: print( - "No Studio release version available. Set " + "No Unsloth release version available. Set " "UNSLOTH_STUDIO_RELEASE_VERSION, build from a GitHub tag, " - "or run from an exact local Studio release tag.", + "or run from an exact local Unsloth release tag.", file = sys.stderr, ) return 2 @@ -207,7 +207,7 @@ def stamp(require_release: bool) -> int: return 0 _atomic_write_text(BUILD_INFO_PATH, build_info_source(version), encoding = "utf-8") - print(f"Stamping Studio release version {version} from {source}", file = sys.stderr) + print(f"Stamping Unsloth release version {version} from {source}", file = sys.stderr) print(version) return 0 @@ -233,7 +233,7 @@ def _read_sdist_member(path: Path) -> str | None: def verify_dist(expected: str, dist_dir: Path) -> int: if not is_valid_version(expected): - print(f"Invalid expected Studio release version: {expected!r}", file = sys.stderr) + print(f"Invalid expected Unsloth release version: {expected!r}", file = sys.stderr) return 2 artifacts = list(dist_dir.glob("*.whl")) + list(dist_dir.glob("*.tar.gz")) @@ -251,14 +251,14 @@ def verify_dist(expected: str, dist_dir: Path) -> int: if content is None: failures.append(f"{artifact.name}: missing {BUILD_INFO_SUFFIX}") elif expected_line not in content: - failures.append(f"{artifact.name}: Studio release version mismatch") + failures.append(f"{artifact.name}: Unsloth release version mismatch") if failures: for failure in failures: print(failure, file = sys.stderr) return 2 - print(f"Verified Studio release version {expected} in {len(artifacts)} artifact(s)") + print(f"Verified Unsloth release version {expected} in {len(artifacts)} artifact(s)") return 0 diff --git a/scripts/uninstall.ps1 b/scripts/uninstall.ps1 index 88defb9ea0..9b6e6ebb86 100644 --- a/scripts/uninstall.ps1 +++ b/scripts/uninstall.ps1 @@ -83,7 +83,7 @@ function Uninstall-UnslothStudio { } } - # A path is a Studio-owned root iff one of install.ps1's sentinels exists: + # A path is an Unsloth-owned root iff one of install.ps1's sentinels exists: # \share\studio.conf, \unsloth_studio\.unsloth-studio-owned, # or \bin\unsloth.exe. function _IsStudioRoot { @@ -164,7 +164,7 @@ function Uninstall-UnslothStudio { return $p } - # Discover non-default Studio roots from env vars + studio.conf files. + # Discover non-default Unsloth roots from env vars + studio.conf files. # Mirrors install.ps1's precedence: UNSLOTH_STUDIO_HOME wins, STUDIO_HOME # is ignored when both are set, so uninstalling install A doesn't also # delete install B if the user has a stale STUDIO_HOME pointing at B. @@ -207,7 +207,7 @@ function Uninstall-UnslothStudio { # Return $true iff the PID's image path lives under one of $KnownRoots. # Prevents killing an unrelated process that happens to listen on a stale - # Studio port. + # Unsloth port. function _PidUnderKnownRoot { param([int]$Pid_, [string[]]$KnownRoots) if (-not $KnownRoots -or $KnownRoots.Count -eq 0) { return $false } @@ -223,8 +223,8 @@ function Uninstall-UnslothStudio { return $false } - # Stop a Studio backend whose port is recorded in \studio.port. - # Only kills if the listening PID's exe path is under a known Studio root. + # Stop an Unsloth backend whose port is recorded in \studio.port. + # Only kills if the listening PID's exe path is under a known Unsloth root. function _StopByPortFile { param([string]$PortFile, [string[]]$KnownRoots) if (-not (Test-Path -LiteralPath $PortFile -PathType Leaf)) { return } @@ -372,7 +372,7 @@ function Uninstall-UnslothStudio { continue } if (-not (_IsStudioRoot $r)) { - _Substep "refusing to remove non-Studio path: $r" "Yellow" + _Substep "refusing to remove non-Unsloth path: $r" "Yellow" continue } _RemovePath $r @@ -436,7 +436,7 @@ function Uninstall-UnslothStudio { $entries = $rawPath -split ';' $kept = New-Object System.Collections.ArrayList $removedAny = $false - # Only remove PATH entries that live inside a Studio root we + # Only remove PATH entries that live inside an Unsloth root we # actually own (default or env-mode). A literal substring # match on `unsloth_studio` would clobber unrelated user # virtualenvs that happen to share the name. diff --git a/scripts/uninstall.sh b/scripts/uninstall.sh index 31e851fcbb..957d2b7af2 100755 --- a/scripts/uninstall.sh +++ b/scripts/uninstall.sh @@ -12,7 +12,7 @@ set -e -# Stop a Studio server via its PID file (written by install.sh's _spawn_terminal). +# Stop an Unsloth server via its PID file (written by install.sh's _spawn_terminal). _kill_pid_file() { _pid_file="$1" [ -f "$_pid_file" ] || return 0 @@ -47,7 +47,7 @@ _pkill_studio() { command -v pkill >/dev/null 2>&1 || return 0 # Scope fallback patterns to the install roots we are removing so a - # different Studio install (different UNSLOTH_STUDIO_HOME) is not touched. + # different Unsloth install (different UNSLOTH_STUDIO_HOME) is not touched. _kill_roots="$HOME/.unsloth/studio" _roots_from_conf=$(_custom_studio_roots 2>/dev/null || true) [ -n "$_roots_from_conf" ] && _kill_roots="$_kill_roots @@ -89,7 +89,7 @@ _remove_path() { fi } -# Accept as Studio root only if Studio sentinels exist (matches install.sh's +# Accept as Unsloth root only if Unsloth sentinels exist (matches install.sh's # env-mode ownership guard at install.sh:1358-1361). A bare unsloth_studio/ # directory is NOT enough -- require the install-time owner marker so a user # directory that happens to contain a folder named "unsloth_studio" is safe. @@ -175,8 +175,8 @@ _custom_studio_roots() { _from_conf "$HOME/.local/share/unsloth/studio.conf" } -# Remove $HOME/.local/bin/unsloth only if it's a Studio-managed symlink. -# Studio's install.sh writes this as a symlink into the studio venv +# Remove $HOME/.local/bin/unsloth only if it's an Unsloth-managed symlink. +# Unsloth's install.sh writes this as a symlink into the studio venv # (install.sh: `ln -sfn "$VENV_DIR/bin/unsloth" "$_shim_path"`). A # pip-installed `unsloth` CLI is a regular file — leave it alone to avoid # wiping an unrelated install. @@ -206,7 +206,7 @@ _custom_studio_roots | while IFS= read -r _custom_root; do continue fi if ! _is_studio_root "$_custom_root"; then - echo " refusing to remove non-Studio path: $_custom_root" >&2 + echo " refusing to remove non-Unsloth path: $_custom_root" >&2 continue fi _remove_path "$_custom_root" @@ -234,7 +234,7 @@ _remove_path "$HOME/.unsloth/rocm-smoketest" # Drop ~/.unsloth only if now empty (rmdir refuses non-empty, so user content is kept). rmdir "$HOME/.unsloth" 2>/dev/null || true _remove_path "$HOME/.local/share/unsloth" -# CLI shim: only the symlink Studio created, never a pip-installed file. +# CLI shim: only the symlink Unsloth created, never a pip-installed file. _remove_cli_shim echo "Removing desktop shortcut and launcher lock..." diff --git a/studio/MCP.md b/studio/MCP.md new file mode 100644 index 0000000000..127a85a116 --- /dev/null +++ b/studio/MCP.md @@ -0,0 +1,34 @@ +# Unsloth Studio MCP server + +Unsloth can expose a local MCP server so an MCP client can inspect models and +GPU state, validate recipes, start or stop training, inspect recipe output, and +export a loaded model. + +The server is disabled by default. Enable it for a local Unsloth process with: + +```bash +UNSLOTH_STUDIO_ENABLE_MCP=1 \ +UNSLOTH_STUDIO_MCP_TOKEN='use-a-local-secret' \ +unsloth studio +``` + +The endpoint is `http://127.0.0.1:8888/mcp/` when Unsloth uses its default port +(a request to `/mcp` redirects to the canonical `/mcp/`). Use the actual Unsloth +port when it is configured differently. + +The high-impact tools are: + +- `studio_status` and `list_local_models` for discovery +- `get_training_status`, `start_training`, `stop_training`, and `list_training_runs` +- `validate_recipe`, `get_recipe_job_status`, and `get_recipe_job_dataset` +- `load_checkpoint` and `export_gguf` + +`start_training` accepts the same fields as the Unsloth `TrainingStartRequest`. +The request is validated by the existing Pydantic model before a subprocess is +started. Export paths use the existing Unsloth validation as well. + +The endpoint always requires `UNSLOTH_STUDIO_MCP_TOKEN` and checks an exact +Bearer token for both HTTP and WebSocket connections. Keep it on localhost +unless the deployment has an authenticated reverse proxy. The MCP endpoint is +intentionally opt-in because tools can consume GPU memory, write model +artifacts, and stop active work. \ No newline at end of file diff --git a/studio/Unsloth_Studio_Colab.ipynb b/studio/Unsloth_Studio_Colab.ipynb index 619395bd6d..612d739806 100644 --- a/studio/Unsloth_Studio_Colab.ipynb +++ b/studio/Unsloth_Studio_Colab.ipynb @@ -1,134 +1,145 @@ { - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "id": "view-in-github", - "colab_type": "text" - }, - "source": [ - "\"Open" - ] + "cells": [ + { + "cell_type": "markdown", + "metadata": { + "id": "view-in-github", + "colab_type": "text" + }, + "source": [ + "\"Open" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "6b87de59" + }, + "source": [ + "To run this, press \"*Runtime*\" and press \"*Run all*\" on a **free** Tesla T4 Google Colab instance!\n", + "

\n", + "\n", + "\n", + " Join Discord if you need help + ⭐ Star us on Github ⭐\n", + "
\n", + "\n", + "To install Unsloth Studio on your local device, follow [our guide](https://unsloth.ai/docs/new/unsloth-studio/install). Unsloth Studio is licensed [AGPL-3.0](https://github.com/unslothai/unsloth/blob/main/studio/LICENSE.AGPL-3.0).\n", + "\n", + "### Unsloth Studio\n", + "\n", + "Train and run open models with [**Unsloth Studio**](https://unsloth.ai/docs/new/unsloth-studio/start). NEW! Installation should now only take 2 mins!\n", + "\n", + "\n", + "We are actively working on making Unsloth Studio install on Colab T4 GPUs faster.\n", + "\n", + "[Features](https://unsloth.ai/docs/new/unsloth-studio#features) • [Quickstart](https://unsloth.ai/docs/new/unsloth-studio/start) • [Data Recipes](https://unsloth.ai/docs/new/unsloth-studio/data-recipe) • [Unsloth Chat](https://unsloth.ai/docs/new/unsloth-studio/chat) • [Export](https://unsloth.ai/docs/new/unsloth-studio/export)" + ], + "id": "6b87de59" + }, + { + "cell_type": "markdown", + "metadata": { + "id": "e4206349" + }, + "source": [ + "

" + ], + "id": "e4206349" + }, + { + "cell_type": "markdown", + "metadata": { + "id": "27da2957" + }, + "source": [ + "### Setup: Clone repo and run setup" + ], + "id": "27da2957" + }, + { + "cell_type": "code", + "metadata": { + "id": "27e68f91" + }, + "source": "!git clone --depth 1 --branch main https://github.com/unslothai/unsloth.git\n%cd /content/unsloth\n!chmod +x studio/setup.sh && ./studio/setup.sh --local", + "execution_count": null, + "outputs": [], + "id": "27e68f91" + }, + { + "cell_type": "markdown", + "metadata": { + "id": "3e1771a9" + }, + "source": [ + "### Start Unsloth Studio" + ], + "id": "3e1771a9" + }, + { + "cell_type": "code", + "metadata": { + "id": "277e431e" + }, + "source": [ + "import sys\n", + "sys.path.insert(0, \"/content/unsloth/studio/backend\")\n", + "from colab import start\n", + "\n", + "# On Colab, start() auto-opens a Cloudflare link and prints admin login credentials.\n", + "# Use the Cloudflare link above the ready card to open Studio (in-cell iframes often stay blank).\n", + "start()\n", + "\n", + "# To skip the Cloudflare tunnel and try the in-notebook proxy iframe only:\n", + "# start(cloudflare=False)" + ], + "execution_count": null, + "outputs": [], + "id": "277e431e" + }, + { + "cell_type": "markdown", + "metadata": { + "id": "f2b0c6a1" + }, + "source": [ + "And we're done! If you have any questions on Unsloth, we have a [Discord](https://discord.gg/unsloth) channel! If you find any bugs or want to keep updated with the latest LLM stuff, or need help, join projects etc, feel free to join our Discord!\n", + "\n", + "Some other resources:\n", + "1. Looking to use Unsloth locally? Read our [Installation Guide](https://unsloth.ai/docs/get-started/install) for details on installing Unsloth on Windows, Docker, AMD, Intel GPUs.\n", + "2. Learn how to do Reinforcement Learning with our [RL Guide and notebooks](https://unsloth.ai/docs/get-started/reinforcement-learning-rl-guide).\n", + "3. Read our guides and notebooks for [Text-to-speech (TTS)](https://unsloth.ai/docs/basics/text-to-speech-tts-fine-tuning) and [vision](https://unsloth.ai/docs/basics/vision-fine-tuning) model support.\n", + "4. Explore our [LLM Tutorials Directory](https://unsloth.ai/docs/models/tutorials-how-to-fine-tune-and-run-llms) to find dedicated guides for each model.\n", + "5. Need help with Inference? Read our [Inference & Deployment page](https://unsloth.ai/docs/basics/inference-and-deployment) for details on using vLLM, llama.cpp, Ollama etc.\n", + "\n", + "
\n", + " \n", + " \n", + " \n", + "\n", + " Join Discord if you need help + ⭐️ Star us on Github ⭐️\n", + "\n", + " This notebook is licensed AGPL-3.0\n", + "
" + ], + "id": "f2b0c6a1" + } + ], + "metadata": { + "accelerator": "GPU", + "colab": { + "gpuType": "T4", + "provenance": [], + "include_colab_link": true + }, + "kernelspec": { + "display_name": "Python 3", + "name": "python3" + }, + "language_info": { + "name": "python" + } }, - { - "cell_type": "markdown", - "id": "6b87de59", - "metadata": { - "id": "6b87de59" - }, - "source": [ - "To run this, press \"*Runtime*\" and press \"*Run all*\" on a **free** Tesla T4 Google Colab instance!\n", - "
\n", - "\n", - "\n", - " Join Discord if you need help + ⭐ Star us on Github ⭐\n", - "
\n", - "\n", - "To install Unsloth Studio on your local device, follow [our guide](https://unsloth.ai/docs/new/unsloth-studio/install). Unsloth Studio is licensed [AGPL-3.0](https://github.com/unslothai/unsloth/blob/main/studio/LICENSE.AGPL-3.0).\n", - "\n", - "### Unsloth Studio\n", - "\n", - "Train and run open models with [**Unsloth Studio**](https://unsloth.ai/docs/new/unsloth-studio/start). NEW! Installation should now only take 2 mins!\n", - "\n", - "\n", - "We are actively working on making Unsloth Studio install on Colab T4 GPUs faster.\n", - "\n", - "[Features](https://unsloth.ai/docs/new/unsloth-studio#features) • [Quickstart](https://unsloth.ai/docs/new/unsloth-studio/start) • [Data Recipes](https://unsloth.ai/docs/new/unsloth-studio/data-recipe) • [Studio Chat](https://unsloth.ai/docs/new/unsloth-studio/chat) • [Export](https://unsloth.ai/docs/new/unsloth-studio/export)" - ] - }, - { - "cell_type": "markdown", - "id": "e4206349", - "metadata": { - "id": "e4206349" - }, - "source": [ - "

" - ] - }, - { - "cell_type": "markdown", - "id": "27da2957", - "metadata": { - "id": "27da2957" - }, - "source": [ - "### Setup: Clone repo and run setup" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "27e68f91", - "metadata": { - "id": "27e68f91" - }, - "outputs": [], - "source": "!git clone --depth 1 --branch main https://github.com/unslothai/unsloth.git\n%cd /content/unsloth\n!chmod +x studio/setup.sh && ./studio/setup.sh --local" - }, - { - "cell_type": "markdown", - "id": "3e1771a9", - "metadata": { - "id": "3e1771a9" - }, - "source": [ - "### Start Unsloth Studio" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "277e431e", - "metadata": { - "id": "277e431e" - }, - "outputs": [], - "source": "import sys\nsys.path.insert(0, \"/content/unsloth/studio/backend\")\nfrom colab import start\n\n# Default: in-tab iframe only. start() blocks to keep the kernel alive.\nstart()\n\n# For a shareable Cloudflare link, replace start() above with:\n# start(cloudflare=True)" - }, - { - "cell_type": "markdown", - "id": "f2b0c6a1", - "metadata": { - "id": "f2b0c6a1" - }, - "source": [ - "And we're done! If you have any questions on Unsloth, we have a [Discord](https://discord.gg/unsloth) channel! If you find any bugs or want to keep updated with the latest LLM stuff, or need help, join projects etc, feel free to join our Discord!\n", - "\n", - "Some other resources:\n", - "1. Looking to use Unsloth locally? Read our [Installation Guide](https://unsloth.ai/docs/get-started/install) for details on installing Unsloth on Windows, Docker, AMD, Intel GPUs.\n", - "2. Learn how to do Reinforcement Learning with our [RL Guide and notebooks](https://unsloth.ai/docs/get-started/reinforcement-learning-rl-guide).\n", - "3. Read our guides and notebooks for [Text-to-speech (TTS)](https://unsloth.ai/docs/basics/text-to-speech-tts-fine-tuning) and [vision](https://unsloth.ai/docs/basics/vision-fine-tuning) model support.\n", - "4. Explore our [LLM Tutorials Directory](https://unsloth.ai/docs/models/tutorials-how-to-fine-tune-and-run-llms) to find dedicated guides for each model.\n", - "5. Need help with Inference? Read our [Inference & Deployment page](https://unsloth.ai/docs/basics/inference-and-deployment) for details on using vLLM, llama.cpp, Ollama etc.\n", - "\n", - "
\n", - " \n", - " \n", - " \n", - "\n", - " Join Discord if you need help + ⭐️ Star us on Github ⭐️\n", - "\n", - " This notebook is licensed AGPL-3.0\n", - "
" - ] - } - ], - "metadata": { - "accelerator": "GPU", - "colab": { - "gpuType": "T4", - "provenance": [], - "include_colab_link": true - }, - "kernelspec": { - "display_name": "Python 3", - "name": "python3" - }, - "language_info": { - "name": "python" - } - }, - "nbformat": 4, - "nbformat_minor": 5 + "nbformat": 4, + "nbformat_minor": 5 } \ No newline at end of file diff --git a/studio/backend/assets/chat_templates/gemma-4-edge.jinja b/studio/backend/assets/chat_templates/gemma-4-edge.jinja index 0266127233..74fa73ddd3 100644 --- a/studio/backend/assets/chat_templates/gemma-4-edge.jinja +++ b/studio/backend/assets/chat_templates/gemma-4-edge.jinja @@ -3,7 +3,7 @@ Source: google/gemma-4-31B-it HF discussion/PR #118 (adds the preserve_thinking flag plus null-rendering, string-arguments validation, balanced turn tags, empty messages handling, and OpenAI image_url/input_audio aliases). - Studio-local changes vs PR #118: + Unsloth-local changes vs PR #118: 1. preserve_thinking defaults to false (see SETUP block below). 2. The empty "<|channel>thought\n" block on enable_thinking=false is NOT emitted. Google ships a distinct template for E2B/E4B (google/gemma-4-E2B-it, diff --git a/studio/backend/assets/chat_templates/gemma-4.jinja b/studio/backend/assets/chat_templates/gemma-4.jinja index 65ab39df57..cc5f98065f 100644 --- a/studio/backend/assets/chat_templates/gemma-4.jinja +++ b/studio/backend/assets/chat_templates/gemma-4.jinja @@ -3,7 +3,7 @@ Source: google/gemma-4-31B-it HF discussion/PR #118 (adds the preserve_thinking flag plus null-rendering, string-arguments validation, balanced turn tags, empty messages handling, and OpenAI image_url/input_audio aliases). - Studio-local change: preserve_thinking defaults to false (see SETUP block below). + Unsloth-local change: preserve_thinking defaults to false (see SETUP block below). Applied to unsloth/gemma-4-*-GGUF models so the embedded GGUF template does not need re-downloading. Keep in sync with upstream if PR #118 changes. -#} diff --git a/studio/backend/assets/configs/full_finetune.yaml b/studio/backend/assets/configs/full_finetune.yaml index e398515f61..98c45dd851 100644 --- a/studio/backend/assets/configs/full_finetune.yaml +++ b/studio/backend/assets/configs/full_finetune.yaml @@ -30,6 +30,7 @@ lora: vision_all_linear: false use_rslora: false use_loftq: false + use_dora: false finetune_vision_layers: true finetune_language_layers: true finetune_attention_modules: true diff --git a/studio/backend/assets/configs/lora_text.yaml b/studio/backend/assets/configs/lora_text.yaml index 9cb6b8c700..6c6a4d8839 100644 --- a/studio/backend/assets/configs/lora_text.yaml +++ b/studio/backend/assets/configs/lora_text.yaml @@ -30,6 +30,7 @@ lora: vision_all_linear: false use_rslora: false use_loftq: false + use_dora: false finetune_vision_layers: true finetune_language_layers: true finetune_attention_modules: true diff --git a/studio/backend/assets/configs/model_defaults/default.yaml b/studio/backend/assets/configs/model_defaults/default.yaml index 841e8ba166..e569031a31 100644 --- a/studio/backend/assets/configs/model_defaults/default.yaml +++ b/studio/backend/assets/configs/model_defaults/default.yaml @@ -33,6 +33,7 @@ lora: - "down_proj" use_rslora: false use_loftq: false + use_dora: false finetune_vision_layers: true finetune_language_layers: true finetune_attention_modules: true diff --git a/studio/backend/assets/configs/model_defaults/embedding/unsloth_Qwen3-Embedding-0.6B.yaml b/studio/backend/assets/configs/model_defaults/embedding/unsloth_Qwen3-Embedding-0.6B.yaml index f7b49c75b7..7ac1c83e04 100644 --- a/studio/backend/assets/configs/model_defaults/embedding/unsloth_Qwen3-Embedding-0.6B.yaml +++ b/studio/backend/assets/configs/model_defaults/embedding/unsloth_Qwen3-Embedding-0.6B.yaml @@ -34,6 +34,7 @@ lora: - "down_proj" use_rslora: false use_loftq: false + use_dora: false logging: enable_wandb: false diff --git a/studio/backend/assets/configs/model_defaults/embedding/unsloth_all-MiniLM-L6-v2.yaml b/studio/backend/assets/configs/model_defaults/embedding/unsloth_all-MiniLM-L6-v2.yaml index be7da0f624..4cab9e9f96 100644 --- a/studio/backend/assets/configs/model_defaults/embedding/unsloth_all-MiniLM-L6-v2.yaml +++ b/studio/backend/assets/configs/model_defaults/embedding/unsloth_all-MiniLM-L6-v2.yaml @@ -30,6 +30,7 @@ lora: - "query" use_rslora: false use_loftq: false + use_dora: false logging: enable_wandb: false diff --git a/studio/backend/assets/configs/model_defaults/embedding/unsloth_bge-m3.yaml b/studio/backend/assets/configs/model_defaults/embedding/unsloth_bge-m3.yaml index d9e49bc0d5..c1f1c2a344 100644 --- a/studio/backend/assets/configs/model_defaults/embedding/unsloth_bge-m3.yaml +++ b/studio/backend/assets/configs/model_defaults/embedding/unsloth_bge-m3.yaml @@ -30,6 +30,7 @@ lora: - "value" use_rslora: false use_loftq: false + use_dora: false logging: enable_wandb: false diff --git a/studio/backend/assets/configs/model_defaults/embedding/unsloth_embeddinggemma-300m.yaml b/studio/backend/assets/configs/model_defaults/embedding/unsloth_embeddinggemma-300m.yaml index c3422d399f..7828feae81 100644 --- a/studio/backend/assets/configs/model_defaults/embedding/unsloth_embeddinggemma-300m.yaml +++ b/studio/backend/assets/configs/model_defaults/embedding/unsloth_embeddinggemma-300m.yaml @@ -33,6 +33,7 @@ lora: - "down_proj" use_rslora: false use_loftq: false + use_dora: false logging: enable_wandb: false diff --git a/studio/backend/assets/configs/model_defaults/embedding/unsloth_gte-modernbert-base.yaml b/studio/backend/assets/configs/model_defaults/embedding/unsloth_gte-modernbert-base.yaml index 529a56a527..5a4028f15b 100644 --- a/studio/backend/assets/configs/model_defaults/embedding/unsloth_gte-modernbert-base.yaml +++ b/studio/backend/assets/configs/model_defaults/embedding/unsloth_gte-modernbert-base.yaml @@ -29,6 +29,7 @@ lora: - "Wqkv" use_rslora: false use_loftq: false + use_dora: false logging: enable_wandb: false diff --git a/studio/backend/assets/configs/model_defaults/ernie/unsloth_ERNIE-4.5-21B-A3B-PT.yaml b/studio/backend/assets/configs/model_defaults/ernie/unsloth_ERNIE-4.5-21B-A3B-PT.yaml index 734115ec41..7645d11c98 100644 --- a/studio/backend/assets/configs/model_defaults/ernie/unsloth_ERNIE-4.5-21B-A3B-PT.yaml +++ b/studio/backend/assets/configs/model_defaults/ernie/unsloth_ERNIE-4.5-21B-A3B-PT.yaml @@ -34,6 +34,7 @@ lora: - "down_proj" use_rslora: false use_loftq: false + use_dora: false logging: enable_wandb: false diff --git a/studio/backend/assets/configs/model_defaults/ernie/unsloth_ERNIE-4.5-VL-28B-A3B-PT.yaml b/studio/backend/assets/configs/model_defaults/ernie/unsloth_ERNIE-4.5-VL-28B-A3B-PT.yaml index 1032449e8c..b746235f1f 100644 --- a/studio/backend/assets/configs/model_defaults/ernie/unsloth_ERNIE-4.5-VL-28B-A3B-PT.yaml +++ b/studio/backend/assets/configs/model_defaults/ernie/unsloth_ERNIE-4.5-VL-28B-A3B-PT.yaml @@ -35,6 +35,7 @@ lora: - "down_proj" use_rslora: false use_loftq: false + use_dora: false finetune_vision_layers: true finetune_language_layers: true finetune_attention_modules: true diff --git a/studio/backend/assets/configs/model_defaults/falcon/tiiuae_Falcon-H1-0.5B-Instruct.yaml b/studio/backend/assets/configs/model_defaults/falcon/tiiuae_Falcon-H1-0.5B-Instruct.yaml index c8e5f35841..4964fea276 100644 --- a/studio/backend/assets/configs/model_defaults/falcon/tiiuae_Falcon-H1-0.5B-Instruct.yaml +++ b/studio/backend/assets/configs/model_defaults/falcon/tiiuae_Falcon-H1-0.5B-Instruct.yaml @@ -34,6 +34,7 @@ lora: - "down_proj" use_rslora: false use_loftq: false + use_dora: false logging: enable_wandb: false diff --git a/studio/backend/assets/configs/model_defaults/gemma/unsloth_codegemma-7b-bnb-4bit.yaml b/studio/backend/assets/configs/model_defaults/gemma/unsloth_codegemma-7b-bnb-4bit.yaml index 251409c29d..e5f3344356 100644 --- a/studio/backend/assets/configs/model_defaults/gemma/unsloth_codegemma-7b-bnb-4bit.yaml +++ b/studio/backend/assets/configs/model_defaults/gemma/unsloth_codegemma-7b-bnb-4bit.yaml @@ -35,6 +35,7 @@ lora: - "down_proj" use_rslora: false use_loftq: false + use_dora: false logging: enable_wandb: false diff --git a/studio/backend/assets/configs/model_defaults/gemma/unsloth_functiongemma-270m-it.yaml b/studio/backend/assets/configs/model_defaults/gemma/unsloth_functiongemma-270m-it.yaml index 89b1d7f938..71c61f383a 100644 --- a/studio/backend/assets/configs/model_defaults/gemma/unsloth_functiongemma-270m-it.yaml +++ b/studio/backend/assets/configs/model_defaults/gemma/unsloth_functiongemma-270m-it.yaml @@ -35,6 +35,7 @@ lora: - "down_proj" use_rslora: false use_loftq: false + use_dora: false logging: enable_wandb: false diff --git a/studio/backend/assets/configs/model_defaults/gemma/unsloth_gemma-2-27b-bnb-4bit.yaml b/studio/backend/assets/configs/model_defaults/gemma/unsloth_gemma-2-27b-bnb-4bit.yaml index e3292b5972..3fe29cd800 100644 --- a/studio/backend/assets/configs/model_defaults/gemma/unsloth_gemma-2-27b-bnb-4bit.yaml +++ b/studio/backend/assets/configs/model_defaults/gemma/unsloth_gemma-2-27b-bnb-4bit.yaml @@ -33,6 +33,7 @@ lora: - "down_proj" use_rslora: false use_loftq: false + use_dora: false logging: enable_wandb: false diff --git a/studio/backend/assets/configs/model_defaults/gemma/unsloth_gemma-2-2b.yaml b/studio/backend/assets/configs/model_defaults/gemma/unsloth_gemma-2-2b.yaml index 98fe497912..cd4e3e0c4d 100644 --- a/studio/backend/assets/configs/model_defaults/gemma/unsloth_gemma-2-2b.yaml +++ b/studio/backend/assets/configs/model_defaults/gemma/unsloth_gemma-2-2b.yaml @@ -34,6 +34,7 @@ lora: - "down_proj" use_rslora: false use_loftq: false + use_dora: false logging: enable_wandb: false diff --git a/studio/backend/assets/configs/model_defaults/gemma/unsloth_gemma-3-270m-it.yaml b/studio/backend/assets/configs/model_defaults/gemma/unsloth_gemma-3-270m-it.yaml index bda5471643..97aa10e861 100644 --- a/studio/backend/assets/configs/model_defaults/gemma/unsloth_gemma-3-270m-it.yaml +++ b/studio/backend/assets/configs/model_defaults/gemma/unsloth_gemma-3-270m-it.yaml @@ -35,6 +35,7 @@ lora: - "down_proj" use_rslora: false use_loftq: false + use_dora: false logging: enable_wandb: false diff --git a/studio/backend/assets/configs/model_defaults/gemma/unsloth_gemma-3-27b-it.yaml b/studio/backend/assets/configs/model_defaults/gemma/unsloth_gemma-3-27b-it.yaml index 18392568bd..a1b1640fa2 100644 --- a/studio/backend/assets/configs/model_defaults/gemma/unsloth_gemma-3-27b-it.yaml +++ b/studio/backend/assets/configs/model_defaults/gemma/unsloth_gemma-3-27b-it.yaml @@ -29,6 +29,7 @@ lora: - "all-linear" use_rslora: false use_loftq: false + use_dora: false finetune_vision_layers: true finetune_language_layers: true finetune_attention_modules: true diff --git a/studio/backend/assets/configs/model_defaults/gemma/unsloth_gemma-3-4b-it.yaml b/studio/backend/assets/configs/model_defaults/gemma/unsloth_gemma-3-4b-it.yaml index 434ac41b46..dbf60f04d4 100644 --- a/studio/backend/assets/configs/model_defaults/gemma/unsloth_gemma-3-4b-it.yaml +++ b/studio/backend/assets/configs/model_defaults/gemma/unsloth_gemma-3-4b-it.yaml @@ -29,6 +29,7 @@ lora: - "all-linear" use_rslora: false use_loftq: false + use_dora: false finetune_vision_layers: true finetune_language_layers: true finetune_attention_modules: true diff --git a/studio/backend/assets/configs/model_defaults/gemma/unsloth_gemma-3-4b-pt.yaml b/studio/backend/assets/configs/model_defaults/gemma/unsloth_gemma-3-4b-pt.yaml index 5f0a7b26ce..54c7dd6cd4 100644 --- a/studio/backend/assets/configs/model_defaults/gemma/unsloth_gemma-3-4b-pt.yaml +++ b/studio/backend/assets/configs/model_defaults/gemma/unsloth_gemma-3-4b-pt.yaml @@ -29,6 +29,7 @@ lora: - "all-linear" use_rslora: false use_loftq: false + use_dora: false finetune_vision_layers: true finetune_language_layers: true finetune_attention_modules: true diff --git a/studio/backend/assets/configs/model_defaults/gemma/unsloth_gemma-3n-E4B-it.yaml b/studio/backend/assets/configs/model_defaults/gemma/unsloth_gemma-3n-E4B-it.yaml index dd5ae51ab0..119440a585 100644 --- a/studio/backend/assets/configs/model_defaults/gemma/unsloth_gemma-3n-E4B-it.yaml +++ b/studio/backend/assets/configs/model_defaults/gemma/unsloth_gemma-3n-E4B-it.yaml @@ -29,6 +29,7 @@ lora: - "all-linear" use_rslora: false use_loftq: false + use_dora: false finetune_vision_layers: true finetune_language_layers: true finetune_attention_modules: true diff --git a/studio/backend/assets/configs/model_defaults/gemma/unsloth_gemma-3n-E4B.yaml b/studio/backend/assets/configs/model_defaults/gemma/unsloth_gemma-3n-E4B.yaml index e53e163a04..d08e5e9547 100644 --- a/studio/backend/assets/configs/model_defaults/gemma/unsloth_gemma-3n-E4B.yaml +++ b/studio/backend/assets/configs/model_defaults/gemma/unsloth_gemma-3n-E4B.yaml @@ -29,6 +29,7 @@ lora: - "all-linear" use_rslora: false use_loftq: false + use_dora: false finetune_vision_layers: true finetune_language_layers: true finetune_attention_modules: true diff --git a/studio/backend/assets/configs/model_defaults/gemma/unsloth_gemma-4-26B-A4B-it.yaml b/studio/backend/assets/configs/model_defaults/gemma/unsloth_gemma-4-26B-A4B-it.yaml index ebe344e382..a266d7a39b 100644 --- a/studio/backend/assets/configs/model_defaults/gemma/unsloth_gemma-4-26B-A4B-it.yaml +++ b/studio/backend/assets/configs/model_defaults/gemma/unsloth_gemma-4-26B-A4B-it.yaml @@ -26,6 +26,7 @@ lora: - "all-linear" use_rslora: false use_loftq: false + use_dora: false finetune_vision_layers: true finetune_language_layers: true finetune_attention_modules: true diff --git a/studio/backend/assets/configs/model_defaults/gemma/unsloth_gemma-4-26B-A4B.yaml b/studio/backend/assets/configs/model_defaults/gemma/unsloth_gemma-4-26B-A4B.yaml index fb89a07133..970cac3259 100644 --- a/studio/backend/assets/configs/model_defaults/gemma/unsloth_gemma-4-26B-A4B.yaml +++ b/studio/backend/assets/configs/model_defaults/gemma/unsloth_gemma-4-26B-A4B.yaml @@ -26,6 +26,7 @@ lora: - "all-linear" use_rslora: false use_loftq: false + use_dora: false finetune_vision_layers: true finetune_language_layers: true finetune_attention_modules: true diff --git a/studio/backend/assets/configs/model_defaults/gemma/unsloth_gemma-4-31B-it.yaml b/studio/backend/assets/configs/model_defaults/gemma/unsloth_gemma-4-31B-it.yaml index 4a089992ac..5bba4ccdc0 100644 --- a/studio/backend/assets/configs/model_defaults/gemma/unsloth_gemma-4-31B-it.yaml +++ b/studio/backend/assets/configs/model_defaults/gemma/unsloth_gemma-4-31B-it.yaml @@ -26,6 +26,7 @@ lora: - "all-linear" use_rslora: false use_loftq: false + use_dora: false finetune_vision_layers: true finetune_language_layers: true finetune_attention_modules: true diff --git a/studio/backend/assets/configs/model_defaults/gemma/unsloth_gemma-4-31B.yaml b/studio/backend/assets/configs/model_defaults/gemma/unsloth_gemma-4-31B.yaml index ae7524b7c6..ac5c6eca22 100644 --- a/studio/backend/assets/configs/model_defaults/gemma/unsloth_gemma-4-31B.yaml +++ b/studio/backend/assets/configs/model_defaults/gemma/unsloth_gemma-4-31B.yaml @@ -26,6 +26,7 @@ lora: - "all-linear" use_rslora: false use_loftq: false + use_dora: false finetune_vision_layers: true finetune_language_layers: true finetune_attention_modules: true diff --git a/studio/backend/assets/configs/model_defaults/gemma/unsloth_gemma-4-E2B-it.yaml b/studio/backend/assets/configs/model_defaults/gemma/unsloth_gemma-4-E2B-it.yaml index 10c1abd8a5..68c2d35644 100644 --- a/studio/backend/assets/configs/model_defaults/gemma/unsloth_gemma-4-E2B-it.yaml +++ b/studio/backend/assets/configs/model_defaults/gemma/unsloth_gemma-4-E2B-it.yaml @@ -26,6 +26,7 @@ lora: - "all-linear" use_rslora: false use_loftq: false + use_dora: false finetune_vision_layers: true finetune_language_layers: true finetune_attention_modules: true diff --git a/studio/backend/assets/configs/model_defaults/gemma/unsloth_gemma-4-E2B.yaml b/studio/backend/assets/configs/model_defaults/gemma/unsloth_gemma-4-E2B.yaml index fb5c1d9dea..175f9c0f17 100644 --- a/studio/backend/assets/configs/model_defaults/gemma/unsloth_gemma-4-E2B.yaml +++ b/studio/backend/assets/configs/model_defaults/gemma/unsloth_gemma-4-E2B.yaml @@ -26,6 +26,7 @@ lora: - "all-linear" use_rslora: false use_loftq: false + use_dora: false finetune_vision_layers: true finetune_language_layers: true finetune_attention_modules: true diff --git a/studio/backend/assets/configs/model_defaults/gemma/unsloth_gemma-4-E4B-it.yaml b/studio/backend/assets/configs/model_defaults/gemma/unsloth_gemma-4-E4B-it.yaml index 189e5dc6b2..4f3834e7c0 100644 --- a/studio/backend/assets/configs/model_defaults/gemma/unsloth_gemma-4-E4B-it.yaml +++ b/studio/backend/assets/configs/model_defaults/gemma/unsloth_gemma-4-E4B-it.yaml @@ -26,6 +26,7 @@ lora: - "all-linear" use_rslora: false use_loftq: false + use_dora: false finetune_vision_layers: true finetune_language_layers: true finetune_attention_modules: true diff --git a/studio/backend/assets/configs/model_defaults/gemma/unsloth_gemma-4-E4B.yaml b/studio/backend/assets/configs/model_defaults/gemma/unsloth_gemma-4-E4B.yaml index aa51440b6a..d6d97f7e44 100644 --- a/studio/backend/assets/configs/model_defaults/gemma/unsloth_gemma-4-E4B.yaml +++ b/studio/backend/assets/configs/model_defaults/gemma/unsloth_gemma-4-E4B.yaml @@ -26,6 +26,7 @@ lora: - "all-linear" use_rslora: false use_loftq: false + use_dora: false finetune_vision_layers: true finetune_language_layers: true finetune_attention_modules: true diff --git a/studio/backend/assets/configs/model_defaults/gpt-oss/unsloth_gpt-oss-120b.yaml b/studio/backend/assets/configs/model_defaults/gpt-oss/unsloth_gpt-oss-120b.yaml index e2d67bcb0b..4f1f54a4e6 100644 --- a/studio/backend/assets/configs/model_defaults/gpt-oss/unsloth_gpt-oss-120b.yaml +++ b/studio/backend/assets/configs/model_defaults/gpt-oss/unsloth_gpt-oss-120b.yaml @@ -35,6 +35,7 @@ lora: - "down_proj" use_rslora: false use_loftq: false + use_dora: false logging: enable_wandb: false diff --git a/studio/backend/assets/configs/model_defaults/gpt-oss/unsloth_gpt-oss-20b.yaml b/studio/backend/assets/configs/model_defaults/gpt-oss/unsloth_gpt-oss-20b.yaml index aa436117a1..127700b53b 100644 --- a/studio/backend/assets/configs/model_defaults/gpt-oss/unsloth_gpt-oss-20b.yaml +++ b/studio/backend/assets/configs/model_defaults/gpt-oss/unsloth_gpt-oss-20b.yaml @@ -35,6 +35,7 @@ lora: - "down_proj" use_rslora: false use_loftq: false + use_dora: false logging: enable_wandb: false diff --git a/studio/backend/assets/configs/model_defaults/granite/unsloth_granite-4.0-350m-unsloth-bnb-4bit.yaml b/studio/backend/assets/configs/model_defaults/granite/unsloth_granite-4.0-350m-unsloth-bnb-4bit.yaml index 3f2cb84a94..2412b3accf 100644 --- a/studio/backend/assets/configs/model_defaults/granite/unsloth_granite-4.0-350m-unsloth-bnb-4bit.yaml +++ b/studio/backend/assets/configs/model_defaults/granite/unsloth_granite-4.0-350m-unsloth-bnb-4bit.yaml @@ -37,6 +37,7 @@ lora: - "shared_mlp.output_linear" use_rslora: false use_loftq: false + use_dora: false logging: enable_wandb: false diff --git a/studio/backend/assets/configs/model_defaults/granite/unsloth_granite-4.0-h-micro.yaml b/studio/backend/assets/configs/model_defaults/granite/unsloth_granite-4.0-h-micro.yaml index ab756fe764..81b59c4323 100644 --- a/studio/backend/assets/configs/model_defaults/granite/unsloth_granite-4.0-h-micro.yaml +++ b/studio/backend/assets/configs/model_defaults/granite/unsloth_granite-4.0-h-micro.yaml @@ -37,6 +37,7 @@ lora: - "shared_mlp.output_linear" use_rslora: false use_loftq: false + use_dora: false logging: enable_wandb: false diff --git a/studio/backend/assets/configs/model_defaults/llama/unsloth_Llama-3.2-11B-Vision-Instruct.yaml b/studio/backend/assets/configs/model_defaults/llama/unsloth_Llama-3.2-11B-Vision-Instruct.yaml index 1a7a91e56f..6110d84a6c 100644 --- a/studio/backend/assets/configs/model_defaults/llama/unsloth_Llama-3.2-11B-Vision-Instruct.yaml +++ b/studio/backend/assets/configs/model_defaults/llama/unsloth_Llama-3.2-11B-Vision-Instruct.yaml @@ -29,6 +29,7 @@ lora: - "all-linear" use_rslora: false use_loftq: false + use_dora: false finetune_vision_layers: true finetune_language_layers: true finetune_attention_modules: true diff --git a/studio/backend/assets/configs/model_defaults/llama/unsloth_Llama-3.2-1B-Instruct.yaml b/studio/backend/assets/configs/model_defaults/llama/unsloth_Llama-3.2-1B-Instruct.yaml index 7c7bb8dc3e..3c7fc7f238 100644 --- a/studio/backend/assets/configs/model_defaults/llama/unsloth_Llama-3.2-1B-Instruct.yaml +++ b/studio/backend/assets/configs/model_defaults/llama/unsloth_Llama-3.2-1B-Instruct.yaml @@ -34,6 +34,7 @@ lora: - "down_proj" use_rslora: false use_loftq: false + use_dora: false logging: enable_wandb: false diff --git a/studio/backend/assets/configs/model_defaults/llama/unsloth_Llama-3.2-3B-Instruct.yaml b/studio/backend/assets/configs/model_defaults/llama/unsloth_Llama-3.2-3B-Instruct.yaml index f73b0c09b6..2b0977e435 100644 --- a/studio/backend/assets/configs/model_defaults/llama/unsloth_Llama-3.2-3B-Instruct.yaml +++ b/studio/backend/assets/configs/model_defaults/llama/unsloth_Llama-3.2-3B-Instruct.yaml @@ -35,6 +35,7 @@ lora: - "down_proj" use_rslora: false use_loftq: false + use_dora: false logging: enable_wandb: false diff --git a/studio/backend/assets/configs/model_defaults/llama/unsloth_Llama-3.3-70B-Instruct.yaml b/studio/backend/assets/configs/model_defaults/llama/unsloth_Llama-3.3-70B-Instruct.yaml index ffefb29e24..1742c04a06 100644 --- a/studio/backend/assets/configs/model_defaults/llama/unsloth_Llama-3.3-70B-Instruct.yaml +++ b/studio/backend/assets/configs/model_defaults/llama/unsloth_Llama-3.3-70B-Instruct.yaml @@ -35,6 +35,7 @@ lora: - "down_proj" use_rslora: false use_loftq: false + use_dora: false logging: enable_wandb: false diff --git a/studio/backend/assets/configs/model_defaults/llama/unsloth_Meta-Llama-3.1-70B-bnb-4bit.yaml b/studio/backend/assets/configs/model_defaults/llama/unsloth_Meta-Llama-3.1-70B-bnb-4bit.yaml index cd986a6da1..f33726b0dd 100644 --- a/studio/backend/assets/configs/model_defaults/llama/unsloth_Meta-Llama-3.1-70B-bnb-4bit.yaml +++ b/studio/backend/assets/configs/model_defaults/llama/unsloth_Meta-Llama-3.1-70B-bnb-4bit.yaml @@ -34,6 +34,7 @@ lora: - "down_proj" use_rslora: false use_loftq: false + use_dora: false logging: enable_wandb: false diff --git a/studio/backend/assets/configs/model_defaults/llama/unsloth_Meta-Llama-3.1-8B-Instruct-bnb-4bit.yaml b/studio/backend/assets/configs/model_defaults/llama/unsloth_Meta-Llama-3.1-8B-Instruct-bnb-4bit.yaml index 55dd3144c6..79b30bd758 100644 --- a/studio/backend/assets/configs/model_defaults/llama/unsloth_Meta-Llama-3.1-8B-Instruct-bnb-4bit.yaml +++ b/studio/backend/assets/configs/model_defaults/llama/unsloth_Meta-Llama-3.1-8B-Instruct-bnb-4bit.yaml @@ -34,6 +34,7 @@ lora: - "down_proj" use_rslora: false use_loftq: false + use_dora: false logging: enable_wandb: false diff --git a/studio/backend/assets/configs/model_defaults/llama/unsloth_llama-3-8b-Instruct-bnb-4bit.yaml b/studio/backend/assets/configs/model_defaults/llama/unsloth_llama-3-8b-Instruct-bnb-4bit.yaml index 8c9cb07fb9..4ee9a5a8ed 100644 --- a/studio/backend/assets/configs/model_defaults/llama/unsloth_llama-3-8b-Instruct-bnb-4bit.yaml +++ b/studio/backend/assets/configs/model_defaults/llama/unsloth_llama-3-8b-Instruct-bnb-4bit.yaml @@ -34,6 +34,7 @@ lora: - "down_proj" use_rslora: false use_loftq: false + use_dora: false logging: enable_wandb: false diff --git a/studio/backend/assets/configs/model_defaults/llama/unsloth_llama-3-8b-bnb-4bit.yaml b/studio/backend/assets/configs/model_defaults/llama/unsloth_llama-3-8b-bnb-4bit.yaml index 32441c5674..da20663688 100644 --- a/studio/backend/assets/configs/model_defaults/llama/unsloth_llama-3-8b-bnb-4bit.yaml +++ b/studio/backend/assets/configs/model_defaults/llama/unsloth_llama-3-8b-bnb-4bit.yaml @@ -34,6 +34,7 @@ lora: - "down_proj" use_rslora: false use_loftq: false + use_dora: false logging: enable_wandb: false diff --git a/studio/backend/assets/configs/model_defaults/llasa/unsloth_Llasa-3B.yaml b/studio/backend/assets/configs/model_defaults/llasa/unsloth_Llasa-3B.yaml index 6bba9c9633..30e4440afb 100644 --- a/studio/backend/assets/configs/model_defaults/llasa/unsloth_Llasa-3B.yaml +++ b/studio/backend/assets/configs/model_defaults/llasa/unsloth_Llasa-3B.yaml @@ -30,6 +30,7 @@ lora: - "v_proj" use_rslora: false use_loftq: false + use_dora: false logging: enable_wandb: false diff --git a/studio/backend/assets/configs/model_defaults/mistral/unsloth_Magistral-Small-2509-unsloth-bnb-4bit.yaml b/studio/backend/assets/configs/model_defaults/mistral/unsloth_Magistral-Small-2509-unsloth-bnb-4bit.yaml index f9833ce705..9bb0a93e63 100644 --- a/studio/backend/assets/configs/model_defaults/mistral/unsloth_Magistral-Small-2509-unsloth-bnb-4bit.yaml +++ b/studio/backend/assets/configs/model_defaults/mistral/unsloth_Magistral-Small-2509-unsloth-bnb-4bit.yaml @@ -35,6 +35,7 @@ lora: - "down_proj" use_rslora: false use_loftq: false + use_dora: false finetune_vision_layers: true finetune_language_layers: true finetune_attention_modules: true diff --git a/studio/backend/assets/configs/model_defaults/mistral/unsloth_Ministral-3-3B-Instruct-2512.yaml b/studio/backend/assets/configs/model_defaults/mistral/unsloth_Ministral-3-3B-Instruct-2512.yaml index 0ba857cd40..ded3607a14 100644 --- a/studio/backend/assets/configs/model_defaults/mistral/unsloth_Ministral-3-3B-Instruct-2512.yaml +++ b/studio/backend/assets/configs/model_defaults/mistral/unsloth_Ministral-3-3B-Instruct-2512.yaml @@ -35,6 +35,7 @@ lora: - "down_proj" use_rslora: false use_loftq: false + use_dora: false finetune_vision_layers: true finetune_language_layers: true finetune_attention_modules: true diff --git a/studio/backend/assets/configs/model_defaults/mistral/unsloth_Mistral-Nemo-Base-2407-bnb-4bit.yaml b/studio/backend/assets/configs/model_defaults/mistral/unsloth_Mistral-Nemo-Base-2407-bnb-4bit.yaml index 3476f2dd6d..2ac72f1c88 100644 --- a/studio/backend/assets/configs/model_defaults/mistral/unsloth_Mistral-Nemo-Base-2407-bnb-4bit.yaml +++ b/studio/backend/assets/configs/model_defaults/mistral/unsloth_Mistral-Nemo-Base-2407-bnb-4bit.yaml @@ -34,6 +34,7 @@ lora: - "down_proj" use_rslora: false use_loftq: false + use_dora: false logging: enable_wandb: false diff --git a/studio/backend/assets/configs/model_defaults/mistral/unsloth_Mistral-Small-Instruct-2409.yaml b/studio/backend/assets/configs/model_defaults/mistral/unsloth_Mistral-Small-Instruct-2409.yaml index eda04d21f9..a087ced1f3 100644 --- a/studio/backend/assets/configs/model_defaults/mistral/unsloth_Mistral-Small-Instruct-2409.yaml +++ b/studio/backend/assets/configs/model_defaults/mistral/unsloth_Mistral-Small-Instruct-2409.yaml @@ -34,6 +34,7 @@ lora: - "down_proj" use_rslora: false use_loftq: false + use_dora: false logging: enable_wandb: false diff --git a/studio/backend/assets/configs/model_defaults/mistral/unsloth_Pixtral-12B-2409.yaml b/studio/backend/assets/configs/model_defaults/mistral/unsloth_Pixtral-12B-2409.yaml index bcd0d20c8c..c9811f4f06 100644 --- a/studio/backend/assets/configs/model_defaults/mistral/unsloth_Pixtral-12B-2409.yaml +++ b/studio/backend/assets/configs/model_defaults/mistral/unsloth_Pixtral-12B-2409.yaml @@ -29,6 +29,7 @@ lora: - "all-linear" use_rslora: false use_loftq: false + use_dora: false finetune_vision_layers: true finetune_language_layers: true finetune_attention_modules: false diff --git a/studio/backend/assets/configs/model_defaults/mistral/unsloth_mistral-7b-instruct-v0.3-bnb-4bit.yaml b/studio/backend/assets/configs/model_defaults/mistral/unsloth_mistral-7b-instruct-v0.3-bnb-4bit.yaml index 34a033e32f..e3659d9fb0 100644 --- a/studio/backend/assets/configs/model_defaults/mistral/unsloth_mistral-7b-instruct-v0.3-bnb-4bit.yaml +++ b/studio/backend/assets/configs/model_defaults/mistral/unsloth_mistral-7b-instruct-v0.3-bnb-4bit.yaml @@ -34,6 +34,7 @@ lora: - "down_proj" use_rslora: false use_loftq: false + use_dora: false logging: enable_wandb: false diff --git a/studio/backend/assets/configs/model_defaults/mistral/unsloth_mistral-7b-v0.3-bnb-4bit.yaml b/studio/backend/assets/configs/model_defaults/mistral/unsloth_mistral-7b-v0.3-bnb-4bit.yaml index 98105eaf38..ee17efc54d 100644 --- a/studio/backend/assets/configs/model_defaults/mistral/unsloth_mistral-7b-v0.3-bnb-4bit.yaml +++ b/studio/backend/assets/configs/model_defaults/mistral/unsloth_mistral-7b-v0.3-bnb-4bit.yaml @@ -33,6 +33,7 @@ lora: - "down_proj" use_rslora: false use_loftq: false + use_dora: false logging: enable_wandb: false diff --git a/studio/backend/assets/configs/model_defaults/other/OuteAI_Llama-OuteTTS-1.0-1B.yaml b/studio/backend/assets/configs/model_defaults/other/OuteAI_Llama-OuteTTS-1.0-1B.yaml index 72b5b018e1..ef836b9b55 100644 --- a/studio/backend/assets/configs/model_defaults/other/OuteAI_Llama-OuteTTS-1.0-1B.yaml +++ b/studio/backend/assets/configs/model_defaults/other/OuteAI_Llama-OuteTTS-1.0-1B.yaml @@ -33,6 +33,7 @@ lora: - "v_proj" use_rslora: false use_loftq: false + use_dora: false logging: enable_wandb: false diff --git a/studio/backend/assets/configs/model_defaults/other/Spark-TTS-0.5B_LLM.yaml b/studio/backend/assets/configs/model_defaults/other/Spark-TTS-0.5B_LLM.yaml index d20751b0c7..c80fad35a8 100644 --- a/studio/backend/assets/configs/model_defaults/other/Spark-TTS-0.5B_LLM.yaml +++ b/studio/backend/assets/configs/model_defaults/other/Spark-TTS-0.5B_LLM.yaml @@ -38,6 +38,7 @@ lora: - "down_proj" use_rslora: false use_loftq: false + use_dora: false logging: enable_wandb: false diff --git a/studio/backend/assets/configs/model_defaults/other/sesame_csm-1b.yaml b/studio/backend/assets/configs/model_defaults/other/sesame_csm-1b.yaml index 8a80282a2a..034b5bd131 100644 --- a/studio/backend/assets/configs/model_defaults/other/sesame_csm-1b.yaml +++ b/studio/backend/assets/configs/model_defaults/other/sesame_csm-1b.yaml @@ -37,6 +37,7 @@ lora: - "down_proj" use_rslora: false use_loftq: false + use_dora: false logging: enable_wandb: false diff --git a/studio/backend/assets/configs/model_defaults/other/unsloth_GLM-4.7-Flash.yaml b/studio/backend/assets/configs/model_defaults/other/unsloth_GLM-4.7-Flash.yaml index a973c2d4e4..d1a226be79 100644 --- a/studio/backend/assets/configs/model_defaults/other/unsloth_GLM-4.7-Flash.yaml +++ b/studio/backend/assets/configs/model_defaults/other/unsloth_GLM-4.7-Flash.yaml @@ -35,6 +35,7 @@ lora: - "out_proj" use_rslora: false use_loftq: false + use_dora: false logging: enable_wandb: false diff --git a/studio/backend/assets/configs/model_defaults/other/unsloth_LFM2-1.2B.yaml b/studio/backend/assets/configs/model_defaults/other/unsloth_LFM2-1.2B.yaml index b0feafbd6e..1b8df5ced9 100644 --- a/studio/backend/assets/configs/model_defaults/other/unsloth_LFM2-1.2B.yaml +++ b/studio/backend/assets/configs/model_defaults/other/unsloth_LFM2-1.2B.yaml @@ -29,6 +29,7 @@ lora: - "all-linear" use_rslora: false use_loftq: false + use_dora: false logging: enable_wandb: false diff --git a/studio/backend/assets/configs/model_defaults/other/unsloth_Nemotron-3-Nano-30B-A3B.yaml b/studio/backend/assets/configs/model_defaults/other/unsloth_Nemotron-3-Nano-30B-A3B.yaml index 2c44c91eab..cecab7f083 100644 --- a/studio/backend/assets/configs/model_defaults/other/unsloth_Nemotron-3-Nano-30B-A3B.yaml +++ b/studio/backend/assets/configs/model_defaults/other/unsloth_Nemotron-3-Nano-30B-A3B.yaml @@ -37,6 +37,7 @@ lora: - "out_proj" use_rslora: false use_loftq: false + use_dora: false logging: enable_wandb: false diff --git a/studio/backend/assets/configs/model_defaults/other/unsloth_PaddleOCR-VL.yaml b/studio/backend/assets/configs/model_defaults/other/unsloth_PaddleOCR-VL.yaml index e1fbc08e4d..730be338cf 100644 --- a/studio/backend/assets/configs/model_defaults/other/unsloth_PaddleOCR-VL.yaml +++ b/studio/backend/assets/configs/model_defaults/other/unsloth_PaddleOCR-VL.yaml @@ -35,6 +35,7 @@ lora: - "down_proj" use_rslora: false use_loftq: false + use_dora: false finetune_vision_layers: true finetune_language_layers: true finetune_attention_modules: true diff --git a/studio/backend/assets/configs/model_defaults/other/unsloth_answerdotai_ModernBERT-large.yaml b/studio/backend/assets/configs/model_defaults/other/unsloth_answerdotai_ModernBERT-large.yaml index 2abdfd8ac3..a70ac0bd49 100644 --- a/studio/backend/assets/configs/model_defaults/other/unsloth_answerdotai_ModernBERT-large.yaml +++ b/studio/backend/assets/configs/model_defaults/other/unsloth_answerdotai_ModernBERT-large.yaml @@ -33,6 +33,7 @@ lora: - "down_proj" use_rslora: false use_loftq: false + use_dora: false logging: enable_wandb: false diff --git a/studio/backend/assets/configs/model_defaults/other/unsloth_orpheus-3b-0.1-ft.yaml b/studio/backend/assets/configs/model_defaults/other/unsloth_orpheus-3b-0.1-ft.yaml index 5a3c4abb48..90ead037f6 100644 --- a/studio/backend/assets/configs/model_defaults/other/unsloth_orpheus-3b-0.1-ft.yaml +++ b/studio/backend/assets/configs/model_defaults/other/unsloth_orpheus-3b-0.1-ft.yaml @@ -38,6 +38,7 @@ lora: - "down_proj" use_rslora: false use_loftq: false + use_dora: false logging: enable_wandb: false diff --git a/studio/backend/assets/configs/model_defaults/other/unsloth_tinyllama-bnb-4bit.yaml b/studio/backend/assets/configs/model_defaults/other/unsloth_tinyllama-bnb-4bit.yaml index a6ce27620f..a97c557c31 100644 --- a/studio/backend/assets/configs/model_defaults/other/unsloth_tinyllama-bnb-4bit.yaml +++ b/studio/backend/assets/configs/model_defaults/other/unsloth_tinyllama-bnb-4bit.yaml @@ -34,6 +34,7 @@ lora: - "down_proj" use_rslora: false use_loftq: false + use_dora: false logging: enable_wandb: false diff --git a/studio/backend/assets/configs/model_defaults/other/unsloth_whisper-large-v3.yaml b/studio/backend/assets/configs/model_defaults/other/unsloth_whisper-large-v3.yaml index 050774a8cd..6855ed6a35 100644 --- a/studio/backend/assets/configs/model_defaults/other/unsloth_whisper-large-v3.yaml +++ b/studio/backend/assets/configs/model_defaults/other/unsloth_whisper-large-v3.yaml @@ -33,6 +33,7 @@ lora: - "v_proj" use_rslora: false use_loftq: false + use_dora: false logging: enable_wandb: false diff --git a/studio/backend/assets/configs/model_defaults/phi/unsloth_Phi-3-medium-4k-instruct.yaml b/studio/backend/assets/configs/model_defaults/phi/unsloth_Phi-3-medium-4k-instruct.yaml index c574714d78..1933fed2ba 100644 --- a/studio/backend/assets/configs/model_defaults/phi/unsloth_Phi-3-medium-4k-instruct.yaml +++ b/studio/backend/assets/configs/model_defaults/phi/unsloth_Phi-3-medium-4k-instruct.yaml @@ -34,6 +34,7 @@ lora: - "down_proj" use_rslora: false use_loftq: false + use_dora: false logging: enable_wandb: false diff --git a/studio/backend/assets/configs/model_defaults/phi/unsloth_Phi-3.5-mini-instruct.yaml b/studio/backend/assets/configs/model_defaults/phi/unsloth_Phi-3.5-mini-instruct.yaml index e803c842b3..fda4e64158 100644 --- a/studio/backend/assets/configs/model_defaults/phi/unsloth_Phi-3.5-mini-instruct.yaml +++ b/studio/backend/assets/configs/model_defaults/phi/unsloth_Phi-3.5-mini-instruct.yaml @@ -34,6 +34,7 @@ lora: - "down_proj" use_rslora: false use_loftq: false + use_dora: false logging: enable_wandb: false diff --git a/studio/backend/assets/configs/model_defaults/phi/unsloth_Phi-4.yaml b/studio/backend/assets/configs/model_defaults/phi/unsloth_Phi-4.yaml index 4de3d9437d..c3910e3e5b 100644 --- a/studio/backend/assets/configs/model_defaults/phi/unsloth_Phi-4.yaml +++ b/studio/backend/assets/configs/model_defaults/phi/unsloth_Phi-4.yaml @@ -35,6 +35,7 @@ lora: - "down_proj" use_rslora: false use_loftq: false + use_dora: false logging: enable_wandb: false diff --git a/studio/backend/assets/configs/model_defaults/qwen/imdatta0_tiny_qwen3_moe_2.8B_0.7B.yaml b/studio/backend/assets/configs/model_defaults/qwen/imdatta0_tiny_qwen3_moe_2.8B_0.7B.yaml index bb75b3ce52..765ffee938 100644 --- a/studio/backend/assets/configs/model_defaults/qwen/imdatta0_tiny_qwen3_moe_2.8B_0.7B.yaml +++ b/studio/backend/assets/configs/model_defaults/qwen/imdatta0_tiny_qwen3_moe_2.8B_0.7B.yaml @@ -36,6 +36,7 @@ lora: - "gate_up_proj" use_rslora: false use_loftq: false + use_dora: false logging: enable_wandb: false diff --git a/studio/backend/assets/configs/model_defaults/qwen/unsloth_Qwen2-7B.yaml b/studio/backend/assets/configs/model_defaults/qwen/unsloth_Qwen2-7B.yaml index c305d328c2..39b30e9cee 100644 --- a/studio/backend/assets/configs/model_defaults/qwen/unsloth_Qwen2-7B.yaml +++ b/studio/backend/assets/configs/model_defaults/qwen/unsloth_Qwen2-7B.yaml @@ -34,6 +34,7 @@ lora: - "down_proj" use_rslora: false use_loftq: false + use_dora: false logging: enable_wandb: false diff --git a/studio/backend/assets/configs/model_defaults/qwen/unsloth_Qwen2-VL-7B-Instruct.yaml b/studio/backend/assets/configs/model_defaults/qwen/unsloth_Qwen2-VL-7B-Instruct.yaml index 6cee3d0949..f97e525798 100644 --- a/studio/backend/assets/configs/model_defaults/qwen/unsloth_Qwen2-VL-7B-Instruct.yaml +++ b/studio/backend/assets/configs/model_defaults/qwen/unsloth_Qwen2-VL-7B-Instruct.yaml @@ -29,6 +29,7 @@ lora: - "all-linear" use_rslora: false use_loftq: false + use_dora: false finetune_vision_layers: true finetune_language_layers: true finetune_attention_modules: true diff --git a/studio/backend/assets/configs/model_defaults/qwen/unsloth_Qwen2.5-1.5B-Instruct.yaml b/studio/backend/assets/configs/model_defaults/qwen/unsloth_Qwen2.5-1.5B-Instruct.yaml index 20ba81df2c..e19b94ede2 100644 --- a/studio/backend/assets/configs/model_defaults/qwen/unsloth_Qwen2.5-1.5B-Instruct.yaml +++ b/studio/backend/assets/configs/model_defaults/qwen/unsloth_Qwen2.5-1.5B-Instruct.yaml @@ -34,6 +34,7 @@ lora: - "down_proj" use_rslora: false use_loftq: false + use_dora: false logging: enable_wandb: false diff --git a/studio/backend/assets/configs/model_defaults/qwen/unsloth_Qwen2.5-7B.yaml b/studio/backend/assets/configs/model_defaults/qwen/unsloth_Qwen2.5-7B.yaml index 9930786c24..982f54b32f 100644 --- a/studio/backend/assets/configs/model_defaults/qwen/unsloth_Qwen2.5-7B.yaml +++ b/studio/backend/assets/configs/model_defaults/qwen/unsloth_Qwen2.5-7B.yaml @@ -34,6 +34,7 @@ lora: - "down_proj" use_rslora: false use_loftq: false + use_dora: false logging: enable_wandb: false diff --git a/studio/backend/assets/configs/model_defaults/qwen/unsloth_Qwen2.5-Coder-1.5B-Instruct.yaml b/studio/backend/assets/configs/model_defaults/qwen/unsloth_Qwen2.5-Coder-1.5B-Instruct.yaml index 775c7ce08f..5242128004 100644 --- a/studio/backend/assets/configs/model_defaults/qwen/unsloth_Qwen2.5-Coder-1.5B-Instruct.yaml +++ b/studio/backend/assets/configs/model_defaults/qwen/unsloth_Qwen2.5-Coder-1.5B-Instruct.yaml @@ -34,6 +34,7 @@ lora: - "down_proj" use_rslora: false use_loftq: false + use_dora: false logging: enable_wandb: false diff --git a/studio/backend/assets/configs/model_defaults/qwen/unsloth_Qwen2.5-Coder-14B-Instruct.yaml b/studio/backend/assets/configs/model_defaults/qwen/unsloth_Qwen2.5-Coder-14B-Instruct.yaml index 856db0c1b3..3559b636c6 100644 --- a/studio/backend/assets/configs/model_defaults/qwen/unsloth_Qwen2.5-Coder-14B-Instruct.yaml +++ b/studio/backend/assets/configs/model_defaults/qwen/unsloth_Qwen2.5-Coder-14B-Instruct.yaml @@ -35,6 +35,7 @@ lora: - "down_proj" use_rslora: false use_loftq: false + use_dora: false logging: enable_wandb: false diff --git a/studio/backend/assets/configs/model_defaults/qwen/unsloth_Qwen2.5-Coder-7B-Instruct-bnb-4bit.yaml b/studio/backend/assets/configs/model_defaults/qwen/unsloth_Qwen2.5-Coder-7B-Instruct-bnb-4bit.yaml index 5900392547..3bc6d69afc 100644 --- a/studio/backend/assets/configs/model_defaults/qwen/unsloth_Qwen2.5-Coder-7B-Instruct-bnb-4bit.yaml +++ b/studio/backend/assets/configs/model_defaults/qwen/unsloth_Qwen2.5-Coder-7B-Instruct-bnb-4bit.yaml @@ -34,6 +34,7 @@ lora: - "down_proj" use_rslora: false use_loftq: false + use_dora: false logging: enable_wandb: false diff --git a/studio/backend/assets/configs/model_defaults/qwen/unsloth_Qwen2.5-VL-7B-Instruct-bnb-4bit.yaml b/studio/backend/assets/configs/model_defaults/qwen/unsloth_Qwen2.5-VL-7B-Instruct-bnb-4bit.yaml index bd54b1d015..604b86dacd 100644 --- a/studio/backend/assets/configs/model_defaults/qwen/unsloth_Qwen2.5-VL-7B-Instruct-bnb-4bit.yaml +++ b/studio/backend/assets/configs/model_defaults/qwen/unsloth_Qwen2.5-VL-7B-Instruct-bnb-4bit.yaml @@ -29,6 +29,7 @@ lora: - "all-linear" use_rslora: false use_loftq: false + use_dora: false finetune_vision_layers: true finetune_language_layers: true finetune_attention_modules: true diff --git a/studio/backend/assets/configs/model_defaults/qwen/unsloth_Qwen3-0.6B.yaml b/studio/backend/assets/configs/model_defaults/qwen/unsloth_Qwen3-0.6B.yaml index 9feb6dcaae..daed4ebccb 100644 --- a/studio/backend/assets/configs/model_defaults/qwen/unsloth_Qwen3-0.6B.yaml +++ b/studio/backend/assets/configs/model_defaults/qwen/unsloth_Qwen3-0.6B.yaml @@ -35,6 +35,7 @@ lora: - "down_proj" use_rslora: false use_loftq: false + use_dora: false logging: enable_wandb: false diff --git a/studio/backend/assets/configs/model_defaults/qwen/unsloth_Qwen3-14B-Base-unsloth-bnb-4bit.yaml b/studio/backend/assets/configs/model_defaults/qwen/unsloth_Qwen3-14B-Base-unsloth-bnb-4bit.yaml index a40eace253..05eef89b88 100644 --- a/studio/backend/assets/configs/model_defaults/qwen/unsloth_Qwen3-14B-Base-unsloth-bnb-4bit.yaml +++ b/studio/backend/assets/configs/model_defaults/qwen/unsloth_Qwen3-14B-Base-unsloth-bnb-4bit.yaml @@ -35,6 +35,7 @@ lora: - "down_proj" use_rslora: false use_loftq: false + use_dora: false logging: enable_wandb: false diff --git a/studio/backend/assets/configs/model_defaults/qwen/unsloth_Qwen3-14B.yaml b/studio/backend/assets/configs/model_defaults/qwen/unsloth_Qwen3-14B.yaml index c130771c32..b4580e6d71 100644 --- a/studio/backend/assets/configs/model_defaults/qwen/unsloth_Qwen3-14B.yaml +++ b/studio/backend/assets/configs/model_defaults/qwen/unsloth_Qwen3-14B.yaml @@ -35,6 +35,7 @@ lora: - "down_proj" use_rslora: false use_loftq: false + use_dora: false logging: enable_wandb: false diff --git a/studio/backend/assets/configs/model_defaults/qwen/unsloth_Qwen3-30B-A3B-Instruct-2507.yaml b/studio/backend/assets/configs/model_defaults/qwen/unsloth_Qwen3-30B-A3B-Instruct-2507.yaml index 2fb3a95c30..2eceb7d0de 100644 --- a/studio/backend/assets/configs/model_defaults/qwen/unsloth_Qwen3-30B-A3B-Instruct-2507.yaml +++ b/studio/backend/assets/configs/model_defaults/qwen/unsloth_Qwen3-30B-A3B-Instruct-2507.yaml @@ -36,6 +36,7 @@ lora: - "gate_up_proj" use_rslora: false use_loftq: false + use_dora: false logging: enable_wandb: false diff --git a/studio/backend/assets/configs/model_defaults/qwen/unsloth_Qwen3-32B.yaml b/studio/backend/assets/configs/model_defaults/qwen/unsloth_Qwen3-32B.yaml index 152f4ae06a..032091880c 100644 --- a/studio/backend/assets/configs/model_defaults/qwen/unsloth_Qwen3-32B.yaml +++ b/studio/backend/assets/configs/model_defaults/qwen/unsloth_Qwen3-32B.yaml @@ -35,6 +35,7 @@ lora: - "down_proj" use_rslora: false use_loftq: false + use_dora: false logging: enable_wandb: false diff --git a/studio/backend/assets/configs/model_defaults/qwen/unsloth_Qwen3-4B-Instruct-2507.yaml b/studio/backend/assets/configs/model_defaults/qwen/unsloth_Qwen3-4B-Instruct-2507.yaml index 94fe000708..e0e7f4ee3d 100644 --- a/studio/backend/assets/configs/model_defaults/qwen/unsloth_Qwen3-4B-Instruct-2507.yaml +++ b/studio/backend/assets/configs/model_defaults/qwen/unsloth_Qwen3-4B-Instruct-2507.yaml @@ -35,6 +35,7 @@ lora: - "down_proj" use_rslora: false use_loftq: false + use_dora: false logging: enable_wandb: false diff --git a/studio/backend/assets/configs/model_defaults/qwen/unsloth_Qwen3-4B-Thinking-2507.yaml b/studio/backend/assets/configs/model_defaults/qwen/unsloth_Qwen3-4B-Thinking-2507.yaml index 3c325485d2..bb463849ed 100644 --- a/studio/backend/assets/configs/model_defaults/qwen/unsloth_Qwen3-4B-Thinking-2507.yaml +++ b/studio/backend/assets/configs/model_defaults/qwen/unsloth_Qwen3-4B-Thinking-2507.yaml @@ -35,6 +35,7 @@ lora: - "down_proj" use_rslora: false use_loftq: false + use_dora: false logging: enable_wandb: false diff --git a/studio/backend/assets/configs/model_defaults/qwen/unsloth_Qwen3-VL-8B-Instruct-unsloth-bnb-4bit.yaml b/studio/backend/assets/configs/model_defaults/qwen/unsloth_Qwen3-VL-8B-Instruct-unsloth-bnb-4bit.yaml index 5b47c3bdd2..23e2b89dd0 100644 --- a/studio/backend/assets/configs/model_defaults/qwen/unsloth_Qwen3-VL-8B-Instruct-unsloth-bnb-4bit.yaml +++ b/studio/backend/assets/configs/model_defaults/qwen/unsloth_Qwen3-VL-8B-Instruct-unsloth-bnb-4bit.yaml @@ -29,6 +29,7 @@ lora: - "all-linear" use_rslora: false use_loftq: false + use_dora: false finetune_vision_layers: true finetune_language_layers: true finetune_attention_modules: true diff --git a/studio/backend/assets/configs/vision_lora.yaml b/studio/backend/assets/configs/vision_lora.yaml index 063a970316..a06f971523 100644 --- a/studio/backend/assets/configs/vision_lora.yaml +++ b/studio/backend/assets/configs/vision_lora.yaml @@ -30,6 +30,7 @@ lora: vision_all_linear: true use_rslora: false use_loftq: false + use_dora: false finetune_vision_layers: true finetune_language_layers: true finetune_attention_modules: true diff --git a/studio/backend/auth/authentication.py b/studio/backend/auth/authentication.py index b13cd1c851..dfb8fc513e 100644 --- a/studio/backend/auth/authentication.py +++ b/studio/backend/auth/authentication.py @@ -148,7 +148,7 @@ async def authenticated_via_api_key( ) -> bool: """True when the caller used an sk-unsloth API key, not a UI session JWT. - Lets routes treat programmatic API callers differently from the Studio UI + Lets routes treat programmatic API callers differently from the Unsloth UI (e.g. refuse a teardown the UI would allow). """ return bool(credentials and credentials.credentials.startswith(API_KEY_PREFIX)) diff --git a/studio/backend/auth/bootstrap_timeout.py b/studio/backend/auth/bootstrap_timeout.py index 728433dc54..97a8086f04 100644 --- a/studio/backend/auth/bootstrap_timeout.py +++ b/studio/backend/auth/bootstrap_timeout.py @@ -1,13 +1,13 @@ # SPDX-License-Identifier: AGPL-3.0-only # Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 -"""Auto-shutdown for an exposed first-run Studio whose admin password is unchanged. +"""Auto-shutdown for an exposed first-run Unsloth whose admin password is unchanged. On a fresh install the seeded bootstrap admin password stays a valid login credential until first login changes it. When the web UI is put on the network (``--secure`` / ``0.0.0.0``) and nobody completes that first-login change within -a deadline, tear Studio down so a fresh, unconfigured instance does not stay -publicly reachable indefinitely. If the password was changed, Studio keeps +a deadline, tear Unsloth down so a fresh, unconfigured instance does not stay +publicly reachable indefinitely. If the password was changed, Unsloth keeps running. Scope: web UI launches only (never ``--api-only``, which authenticates by API @@ -98,7 +98,7 @@ def enforce_bootstrap_password_deadline( ) -> bool: """Deadline handler: shut down iff the seeded admin password is still unchanged. - Returns True if it shut Studio down, False if it left it running (the + Returns True if it shut Unsloth down, False if it left it running (the password was changed in time). """ try: @@ -106,7 +106,7 @@ def enforce_bootstrap_password_deadline( except Exception: return False if not still_default: - return False # password changed in time -> leave Studio running + return False # password changed in time -> leave Unsloth running message = ( "\nUnsloth Studio was exposed on the network but its default admin " diff --git a/studio/backend/auth/storage.py b/studio/backend/auth/storage.py index 9bb3ab5735..39fa691304 100644 --- a/studio/backend/auth/storage.py +++ b/studio/backend/auth/storage.py @@ -146,7 +146,7 @@ def get_connection() -> sqlite3.Connection: pass conn.row_factory = sqlite3.Row # WAL lets token reads run concurrently with refresh-token writes; - # busy_timeout bounds lock waits. Matches the other Studio SQLite stores. + # busy_timeout bounds lock waits. Matches the other Unsloth SQLite stores. # Set busy_timeout first: switching journal_mode needs a lock, so if a # refresh-token write already holds one, journal_mode=WAL raises SQLITE_BUSY; # with busy_timeout already in effect it waits instead of failing and leaving @@ -305,8 +305,8 @@ def get_or_create_identity_secret() -> bytes: def compute_identity_proof(nonce: bytes, host: str, port: int) -> str: """HMAC-SHA256 proof that the caller holds this install's identity secret, bound to the loopback address and port the connection landed on. A proof - relayed from a Studio on a different address/port (a squatter proxying to the - real one, e.g. localhost resolving to ::1 while Studio is on 127.0.0.1) was + relayed from an Unsloth on a different address/port (a squatter proxying to the + real one, e.g. localhost resolving to ::1 while Unsloth is on 127.0.0.1) was computed for that other endpoint and won't match the one the client dialed.""" try: host = ipaddress.ip_address(host).compressed # normalise 127.0.0.1 / ::1 forms diff --git a/studio/backend/auth/terminal_prompt.py b/studio/backend/auth/terminal_prompt.py index 8491019ae9..925404f47d 100644 --- a/studio/backend/auth/terminal_prompt.py +++ b/studio/backend/auth/terminal_prompt.py @@ -2,14 +2,14 @@ # Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 """Interactive terminal prompt that forces a bootstrap password change before -Studio is exposed on a public Cloudflare URL (``--secure`` / ``--cloudflare``). +Unsloth is exposed on a public Cloudflare URL (``--secure`` / ``--cloudflare``). Masked input echoes one ``*`` per keystroke (unlike ``getpass``). Works on Windows (``msvcrt``) and Linux/macOS (``termios``). All output goes to stderr so redirected stdout never swallows the prompt. Mirrored for the CLI at ``unsloth_cli/commands/_password_prompt.py`` (the CLI -cannot import the Studio backend package); keep the two in sync. +cannot import the Unsloth backend package); keep the two in sync. """ from __future__ import annotations @@ -236,6 +236,10 @@ def prompt_for_password_change( out.write(f"Password must be at least {min_length} characters; try again.\n") out.flush() continue + if any(ch.isspace() for ch in new_password): + out.write("Password cannot contain spaces; try again.\n") + out.flush() + continue if is_current_password(new_password): out.write( "New password must differ from the current bootstrap password; try again.\n" @@ -252,7 +256,7 @@ def prompt_for_password_change( out.flush() return True except (KeyboardInterrupt, EOFError): - out.write("Password change aborted; not exposing Studio.\n") + out.write("Password change aborted; not exposing Unsloth.\n") out.flush() return False diff --git a/studio/backend/cloudflare_tunnel.py b/studio/backend/cloudflare_tunnel.py index ef7bacba67..78fce0c70a 100644 --- a/studio/backend/cloudflare_tunnel.py +++ b/studio/backend/cloudflare_tunnel.py @@ -1,13 +1,13 @@ # SPDX-License-Identifier: AGPL-3.0-only # Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 -"""Free Cloudflare quick tunnel for Studio's 0.0.0.0 launches. +"""Free Cloudflare quick tunnel for Unsloth's 0.0.0.0 launches. The raw http://: is often unreachable (https-vs-http, blocked ports, closed security groups); a cloudflared quick tunnel gives a free https://*.trycloudflare.com URL that works anywhere, with no account or domain. -Best-effort throughout: any failure collapses to "no URL" and Studio keeps +Best-effort throughout: any failure collapses to "no URL" and Unsloth keeps running. Stdlib only (back-end imports are lazy) so it is safe to import early. """ @@ -20,6 +20,7 @@ import shutil import subprocess import sys import threading +import time from pathlib import Path from typing import Optional, Tuple @@ -40,6 +41,22 @@ _RELEASE_BASE = "https://github.com/cloudflare/cloudflared/releases/latest/downl _READY_TIMEOUT = 15.0 # seconds to wait for the URL + a registered edge connection _DOWNLOAD_TIMEOUT = 60 # urlopen timeout for the one-time binary download +# A registered edge connection does not mean the hostname resolves yet, so the +# URL is fetched once before it is advertised. +_PUBLIC_PROBE_PATH = "/api/health" +_PUBLIC_PROBE_MARKER = "Unsloth UI Backend" +# One deadline for DNS propagation + the health probe, bounding the startup stall. +_PUBLIC_PROBE_TIMEOUT = 45.0 +_PUBLIC_PROBE_ATTEMPT_TIMEOUT = 5.0 +_PUBLIC_PROBE_RETRY_DELAY = 1.0 + +# Wait for the hostname via DoH first: an early OS lookup negative-caches the +# NXDOMAIN for up to 30 min. +_DNS_POLL_DELAY = 2.0 +# Retry transient DoH failures, but give up fast when DoH is blocked outright. +_DNS_MAX_DOH_ERRORS = 3 +_DOH_URL = "https://cloudflare-dns.com/dns-query?name={host}&type=A" + def _windows_hidden_kwargs() -> dict: """Suppress a child console window on Windows; no-op elsewhere.""" @@ -95,7 +112,7 @@ def _cache_path() -> Optional[Path]: def find_cloudflared() -> Optional[str]: - """Locate an existing cloudflared: PATH first, then the Studio bin cache.""" + """Locate an existing cloudflared: PATH first, then the Unsloth bin cache.""" on_path = shutil.which("cloudflared") if on_path: return on_path @@ -191,6 +208,59 @@ def ensure_cloudflared() -> Optional[str]: return None +def _wait_for_dns(host: str, deadline: float) -> None: + import json + import urllib.request + + errors = 0 + while True: + answered = False + try: + req = urllib.request.Request( + _DOH_URL.format(host = host), + headers = {"Accept": "application/dns-json", "User-Agent": "unsloth-studio"}, + ) + with urllib.request.urlopen(req, timeout = 5) as response: + answered = bool(json.loads(response.read(65536)).get("Answer")) + errors = 0 + except Exception: + errors += 1 + if errors >= _DNS_MAX_DOH_ERRORS: + return + if answered: + return + remaining = deadline - time.monotonic() + if remaining <= 0: + return + time.sleep(min(_DNS_POLL_DELAY, remaining)) + + +def verify_public_url(url: str, timeout: float = _PUBLIC_PROBE_TIMEOUT) -> bool: + import json + import urllib.request + from urllib.parse import urlsplit + + deadline = time.monotonic() + timeout + host = urlsplit(url).hostname + if host: + _wait_for_dns(host, deadline) + + probe_url = f"{url.rstrip('/')}{_PUBLIC_PROBE_PATH}" + while True: + try: + req = urllib.request.Request(probe_url, headers = {"User-Agent": "unsloth-studio"}) + with urllib.request.urlopen(req, timeout = _PUBLIC_PROBE_ATTEMPT_TIMEOUT) as response: + body = response.read(4096) + if json.loads(body).get("service") == _PUBLIC_PROBE_MARKER: + return True + except Exception: + pass + remaining = deadline - time.monotonic() + if remaining <= 0: + return False + time.sleep(min(_PUBLIC_PROBE_RETRY_DELAY, remaining)) + + class CloudflareTunnel: """A cloudflared quick tunnel to http://localhost:. Best-effort throughout. @@ -309,7 +379,7 @@ class CloudflareTunnel: pass -# Single serving process per Studio launch, so one module-level tunnel handle is +# Single serving process per Unsloth launch, so one module-level tunnel handle is # enough; the lock guards the start/stop/shutdown races. _active_tunnel: Optional[CloudflareTunnel] = None _active_lock = threading.Lock() @@ -322,11 +392,12 @@ def start_studio_tunnel(port: int, timeout: float = _READY_TIMEOUT) -> Optional[ """Start a quick tunnel and return its public URL once it is actually serving, or None (best-effort). - Waits for cloudflared to both mint the URL and register an edge connection - before returning, so the caller never advertises a URL that yields Cloudflare - error 1033 (HTTP 530). If a URL is minted but no connection registers within - the window (e.g. quic is blocked on this network), retries once forcing the - http2 protocol. On any failure the tunnel is stopped and None is returned. + Waits for cloudflared to both mint the URL and register an edge connection, + then fetches /api/health over the public URL, so the caller never advertises + a link that yields Cloudflare error 1033 (HTTP 530) or an unresolvable host. + If a URL is minted but no connection registers within the window (e.g. quic + is blocked on this network), retries once forcing the http2 protocol. On any + failure the tunnel is stopped and None is returned. """ global _active_tunnel, _shutdown_requested binary = ensure_cloudflared() @@ -349,9 +420,13 @@ def start_studio_tunnel(port: int, timeout: float = _READY_TIMEOUT) -> Optional[ prior, _active_tunnel = _active_tunnel, tunnel if prior is not None: prior.stop() + registered = False try: tunnel.start() url = tunnel.wait_for_ready(timeout) + registered = url is not None + if url and not verify_public_url(url): + url = None except Exception: url = None if url: @@ -371,6 +446,9 @@ def start_studio_tunnel(port: int, timeout: float = _READY_TIMEOUT) -> Optional[ # http2 will not help, so do not burn another window on it. if not saw_url: return None + # probe failure after registering is DNS propagation; http2 would not help + if registered: + return None return None diff --git a/studio/backend/colab.py b/studio/backend/colab.py index e04543b3aa..baa18a2fec 100644 --- a/studio/backend/colab.py +++ b/studio/backend/colab.py @@ -1,9 +1,7 @@ # SPDX-License-Identifier: AGPL-3.0-only # Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 -""" -Colab helpers for Unsloth Studio. Uses Colab's built-in proxy. -""" +"""Colab helpers for Unsloth Studio. Uses Colab's built-in proxy.""" from pathlib import Path import sys @@ -22,11 +20,9 @@ logger = get_logger(__name__) def get_colab_url(port: int = 8888) -> str: - """ - Get the Colab proxy URL for a port. + """Get the Colab proxy URL for a port. - Retries up to 3 times, validating the result is a real HTTPS Colab URL. - Falls back to http://localhost:{port} only when all attempts fail. + Retries 3x validating a real HTTPS Colab URL; falls back to localhost on failure. """ import time as _time @@ -55,28 +51,244 @@ def get_colab_url(port: int = 8888) -> str: return fallback -def show_link(port: int = 8888, *, _url: "str | None" = None): - """Display a styled clickable link to the UI. - - *_url* is an optional pre-fetched proxy URL; pass it to avoid a second eval_js round-trip. - """ - from IPython.display import display, HTML - - url = _url if _url is not None else get_colab_url(port) - - # Truncated display URL; try/except so an odd URL shape still renders the link. +def _short_colab_url(url: str, port: int) -> str: + """Truncated display form of a Colab proxy URL; falls back to the full URL.""" try: port_prefix = f"{port}-" idx = url.index(port_prefix) next_dash = url.index("-", idx + len(port_prefix)) - short_url = url[: next_dash + 1] + "..." + return url[: next_dash + 1] + "..." except (ValueError, IndexError): - short_url = url + return url - # Plain-text line so the URL shows even if HTML display fails. - logger.info(f"🌐 Unsloth Studio URL: {url}") - html = f""" +def _is_colab_proxy_url(url: str, port: int) -> bool: + """True when *url* looks like a real Colab kernel proxy, not a localhost fallback.""" + return bool(url and isinstance(url, str) and url.startswith("https://") and str(port) in url) + + +def _is_colab_runtime() -> bool: + """True on a hosted Colab notebook kernel. + + Reuses the backend's main Colab detector (``/content`` + Colab env / ``google.colab``) + instead of a single env var, which is not always present on hosted runtimes. + """ + try: + from main import _IS_COLAB + return bool(_IS_COLAB) + except Exception: + return False + + +def _colab_login_credentials_path() -> Path: + from auth.storage import DB_PATH + return DB_PATH.parent / ".colab_notebook_login" + + +def _store_colab_login_credentials(username: str, password: str) -> None: + """Persist Colab admin credentials for notebook re-runs after interrupt.""" + path = _colab_login_credentials_path() + try: + path.parent.mkdir(parents = True, exist_ok = True) + path.write_text(f"{username}\n{password}\n") + try: + import os + os.chmod(path, 0o600) + except OSError: + pass + except OSError as e: + logger.info(f"Could not persist Colab login credentials ({e}).") + + +def _load_colab_login_credentials() -> "tuple[str, str] | None": + """Return stored Colab admin credentials from a previous ``start()`` run, if any.""" + path = _colab_login_credentials_path() + try: + if not path.is_file(): + return None + lines = path.read_text().splitlines() + if len(lines) >= 2 and lines[0] and lines[1]: + return lines[0], lines[1] + except OSError as e: + logger.info(f"Could not load Colab login credentials ({e}).") + return None + + +def _clear_colab_login_credentials() -> None: + """Drop the cached Colab credentials once they no longer authenticate.""" + path = _colab_login_credentials_path() + try: + path.unlink(missing_ok = True) + except OSError as e: + logger.info(f"Could not clear Colab login credentials ({e}).") + + +def _colab_credentials_still_valid(username: str, password: str) -> bool: + """True when *password* still matches the stored admin hash. + + Guards against redisplaying a cached first-run password after the user has + changed the admin password through the app, which would print credentials + that no longer authenticate to the current Cloudflare tunnel. + """ + try: + from auth.storage import get_user_and_secret + from auth.hashing import verify_password + except Exception as e: + logger.info(f"Could not load auth to validate cached Colab credentials ({e}).") + return False + try: + row = get_user_and_secret(username) + if not row: + return False + salt, pwd_hash = row[0], row[1] + return bool(verify_password(password, salt, pwd_hash)) + except Exception as e: + logger.info(f"Could not validate cached Colab credentials ({e}).") + return False + + +def _colab_wants_cloudflare(cloudflare: "bool | None") -> bool: + """Resolve whether to open a Cloudflare tunnel. + + ``None`` auto-enables on real Colab (the in-cell proxy embed is often blank); + pass ``False`` to opt out. + """ + if cloudflare is not None: + return cloudflare + return _is_colab_runtime() + + +def _finalize_colab_admin_password() -> "tuple[str, str] | None": + """Clear the bootstrap-password gate on Colab so Cloudflare tunnels can start. + + Returns ``(username, password)`` for display in the notebook. On first run the + random admin password is finalized; on later runs (e.g. after interrupt) the + stored credentials are re-displayed so the Cloudflare link stays usable. + Anyone who can read this cell already controls the runtime. + """ + if not _is_colab_runtime(): + return None + try: + from auth.storage import ( + DEFAULT_ADMIN_USERNAME, + ensure_default_admin, + generate_bootstrap_password, + get_bootstrap_password, + requires_password_change, + update_password, + ) + except Exception as e: + logger.warning( + f"Could not load auth for Colab setup ({e}); Cloudflare link may be blocked." + ) + return None + + try: + ensure_default_admin() + username = DEFAULT_ADMIN_USERNAME + if not requires_password_change(username): + creds = _load_colab_login_credentials() + if creds is not None and _colab_credentials_still_valid(username, creds[1]): + return creds + # The admin password was changed through the app after the first run, + # so the cached copy is stale; drop it instead of printing dead credentials. + _clear_colab_login_credentials() + return None + password = get_bootstrap_password() or generate_bootstrap_password() + if not update_password(username, password): + logger.warning( + "Could not finalize Colab admin password; Cloudflare link may be blocked." + ) + return None + _store_colab_login_credentials(username, password) + return username, password + except Exception as e: + logger.warning( + f"Could not finalize Colab admin password ({e}); Cloudflare link may be blocked." + ) + return None + + +def _colab_login_html(username: str, password: str) -> str: + """Notebook card with Colab admin credentials (shown once after auto-finalize).""" + return f""" +
+

+ Unsloth Studio Login (Colab) +

+

+ Log in to Studio with the Cloudflare link above using these credentials. This cell + is visible only in your notebook session. +

+

+ Username: {username}
+ Password: {password} +

+
+ """ + + +def _show_colab_login_credentials(username: str, password: str) -> None: + """Display Colab admin credentials in the notebook output.""" + from IPython.display import HTML, display + + logger.info(f"🔐 Unsloth Studio login — user: {username}") + display(HTML(_colab_login_html(username, password))) + + +def _ready_card_html( + url: str, + port: int, + *, + has_cloudflare_link: bool = False, + cloudflare_requested: bool = False, +) -> str: + """Branded ready card for the in-notebook Studio view. + + Colab ``*.prod.colab.dev`` proxy URLs are session-scoped and 404 when opened as a + top-level tab or on another device, so never ``window.open`` them. On real Colab the + Cloudflare link is the supported entry point because in-cell proxy embeds often stay blank. + """ + short_url = _short_colab_url(url, port) + if _is_colab_runtime() or _is_colab_proxy_url(url, port): + if has_cloudflare_link: + embed_note = ( + "Open Studio with the Cloudflare link above. In-cell proxy previews on " + "current Colab often stay blank, so the tunnel link is the supported path." + ) + elif cloudflare_requested: + embed_note = ( + "Could not open a Cloudflare tunnel, so Studio may be unreachable on Colab. " + "Check the logs above and re-run this cell. Pass " + '' + "cloudflare=True after fixing any tunnel errors." + ) + else: + embed_note = ( + "Colab proxy links cannot be opened in a new tab (they 404 outside this " + 'notebook). Re-run with start(cloudflare=True) for a working link.' + ) + return f""" +
+

+ + Unsloth Studio is Ready! +

+

+ {embed_note} +

+

+ {short_url} +

+
+ """ + + return f"""

""" - display(HTML(html)) + + +def show_link( + port: int = 8888, + *, + _url: "str | None" = None, + has_cloudflare_link: bool = False, + cloudflare_requested: bool = False, +): + """Display a styled ready card for the UI. + + Colab proxy URLs are informational only (no new-tab open; they 404 outside the cell); + non-proxy URLs keep a clickable open button. *_url* is an optional pre-fetched proxy + URL to avoid a second eval_js round-trip. + """ + from IPython.display import display, HTML + + url = _url if _url is not None else get_colab_url(port) + logger.info(f"🌐 Unsloth Studio URL: {url}") + display( + HTML( + _ready_card_html( + url, + port, + has_cloudflare_link = has_cloudflare_link, + cloudflare_requested = cloudflare_requested, + ) + ) + ) + + +def _warn_colab_cloudflare_missing(*, use_cloudflare: bool, cloudflare_url: "str | None") -> None: + """Log a prominent warning when Colab expected a tunnel but none was opened.""" + if not use_cloudflare or cloudflare_url or not _is_colab_runtime(): + return + logger.warning( + "Colab Cloudflare tunnel unavailable — Studio is unlikely to be reachable in this " + "notebook. Check the logs above for tunnel or auth errors, then re-run start()." + ) def _bootstrap_password_pending() -> bool: """True while the default admin still owes a bootstrap-password change. - While pending, main.py injects that password into same-origin GETs, and a public - tunnel GET (no Origin) reads as same-origin, so sharing the link would leak admin - access. Fails safe to pending if the state cannot be read. + While pending, a public tunnel GET (no Origin) reads as same-origin and gets the + injected password, so sharing the link would leak admin access. Fails safe to pending. """ try: from auth.storage import requires_password_change, DEFAULT_ADMIN_USERNAME @@ -121,15 +370,14 @@ def _bootstrap_password_pending() -> bool: def start_cloudflare_tunnel(port: int) -> "str | None": """Open a shareable Cloudflare quick tunnel to localhost:*port*, or None. - run_server suppresses the tunnel on Colab by design, so we start it directly. - Refused while the bootstrap password is pending; any failure collapses to None - and the Colab proxy still works. + run_server suppresses the tunnel on Colab, so we start it directly. Refused while the + bootstrap password is pending; any failure collapses to None (Colab proxy still works). """ if _bootstrap_password_pending(): logger.warning( "Cloudflare link not started: the admin account still has its temporary " "bootstrap password, which is exposed to anyone who can load the page. " - "Open Studio in this tab, log in and change the admin password, then re-run " + "Open Unsloth in this tab, log in and change the admin password, then re-run " "start(cloudflare=True) to get the shareable link." ) return None @@ -152,9 +400,9 @@ def start_cloudflare_tunnel(port: int) -> "str | None": def _publish_cloudflare_url(cloudflare_url: "str | None") -> None: """Publish a directly-started tunnel URL onto app.state so /api/health advertises it. - run_server only sets this when it opens the tunnel itself, which it skips on Colab, - so we set it here. Otherwise the frontend's API examples fall back to an - unreachable server_url. Best-effort. + run_server sets this only when it opens the tunnel itself (skipped on Colab), so we + set it here; otherwise the frontend's API examples fall back to an unreachable + server_url. Best-effort. """ if not cloudflare_url: return @@ -183,8 +431,7 @@ def _stop_cloudflare_tunnel() -> None: def _is_studio_healthy(port: int, timeout: float = 2.0) -> bool: """True only if Unsloth Studio (not some other app) answers /api/health on *port*. - The service-marker check stops the reuse path reusing or tunneling a foreign - process that merely serves /api/health. + The service-marker check stops the reuse path reusing or tunneling a foreign process. """ import json, urllib.request try: @@ -203,7 +450,7 @@ def _shareable_link_html(cloudflare_url: str) -> str: display: flex; align-items: center; gap: 12px;"> - Shareable Studio Link is Ready! + Shareable Unsloth Link is Ready! bool: + """Embed Studio via Colab's native kernel-port iframe helper. + + Only trusted on a real Colab runtime: colabtools can import ``google.colab`` and + queue browser-side JS without appending an iframe, so callers outside Colab must use + the HTML iframe path instead. + """ + if not _is_colab_runtime(): + return False + try: + from google.colab import output as colab_output + except ImportError: + return False + try: + colab_output.serve_kernel_port_as_iframe( + port, + height = _COLAB_IFRAME_HEIGHT, + width = "100%", + ) + return True + except Exception as e: + logger.info(f"serve_kernel_port_as_iframe failed ({e}); trying HTML iframe.") + return False + + +def _embed_html_iframe(url: str, port: int) -> bool: + """Fallback embed: raw HTML iframe when the Colab helper is unavailable.""" try: from IPython.display import HTML, display + except ImportError: + return False - iframe_id = f"unsloth-studio-{port}" - - # Truncated header URL — best-effort, falls back to full URL. - try: - port_prefix = f"{port}-" - idx = url.index(port_prefix) - next_dash = url.index("-", idx + len(port_prefix)) - short_url = url[: next_dash + 1] + "..." - except (ValueError, IndexError): - short_url = url - - if cloudflare_url: - display(HTML(_shareable_link_html(cloudflare_url))) - + short_url = _short_colab_url(url, port) + iframe_id = f"unsloth-studio-{port}" + try: display( HTML(f"""
""") ) - except Exception: - # Fallback: Colab's built-in helper. + return True + except Exception as e: + logger.info(f"HTML iframe embed failed ({e}).") + return False + + +def _show_and_embed( + port: int, + *, + cloudflare_url: "str | None" = None, + colab_login: "tuple[str, str] | None" = None, + cloudflare_requested: bool = False, +): + """Render the Unsloth ready card + iframe for *port*. + + Prefer Colab's ``serve_kernel_port_as_iframe`` on real Colab; raw HTML iframe is the + fallback. Cloudflare cards stay clickable. + """ + url = get_colab_url(port) + logger.info(f"🌐 Unsloth Studio URL: {url}") + if cloudflare_url: + logger.info(f"🔗 Shareable Cloudflare link: {cloudflare_url}") + + _warn_colab_cloudflare_missing( + use_cloudflare = cloudflare_requested, + cloudflare_url = cloudflare_url, + ) + + if cloudflare_url: try: - from google.colab import output as colab_output - colab_output.serve_kernel_port_as_iframe(port, height = 900, width = "100%") - except ImportError: - pass + from IPython.display import HTML, display + display(HTML(_shareable_link_html(cloudflare_url))) + except Exception as e: + logger.info(f"Could not render Cloudflare link card ({e}).") + + if colab_login: + try: + _show_colab_login_credentials(*colab_login) + except Exception as e: + logger.info(f"Could not render Colab login card ({e}).") + + try: + show_link( + port, + _url = url, + has_cloudflare_link = bool(cloudflare_url), + cloudflare_requested = cloudflare_requested, + ) + except Exception as e: + logger.info(f"Could not render Unsloth link card ({e}).") + + # On Colab with a working tunnel, skip the in-cell proxy embed (often blank). + if _is_colab_runtime() and cloudflare_url: + return + + # Real Colab: kernel helper needs only the port (works when eval_js failed). + if _is_colab_runtime(): + if _embed_kernel_port_iframe(port): + return + _embed_html_iframe(url, port) -def start(port: int = 8888, *, cloudflare: bool = False): +def start(port: int = 8888, *, cloudflare: "bool | None" = None): """Start Unsloth Studio in Colab and display the URL. Args: port: Port to bind/serve on. - cloudflare: Opt in to a shareable Cloudflare HTTPS link reachable from any - device (default OFF). It exposes Studio's login page beyond Colab, so it - stays an explicit opt-in; the default shows only the in-tab proxy iframe. + cloudflare: Shareable Cloudflare HTTPS link. ``None`` (default) auto-enables on + real Colab because the in-cell proxy embed is often blank; pass ``False`` to + skip the tunnel or ``True`` to force it on other runtimes. Usage: - start() # Colab-proxy iframe only (default) - start(cloudflare=True) # also open a shareable Cloudflare link + start() # Cloudflare link on Colab (auto); proxy iframe elsewhere + start(cloudflare=False) # Colab proxy iframe only (often blank on current Colab) + start(cloudflare=True) # force Cloudflare link on any runtime """ import time logger.info("🦥 Starting Unsloth Studio...") + use_cloudflare = _colab_wants_cloudflare(cloudflare) - # Fast path: Studio already running (cell re-run). Re-launching would collide on - # the port, so just re-show the link and iframe. + # Fast path: already running (cell re-run); re-show link/iframe instead of rebinding the port. if _is_studio_healthy(port): - logger.info(f" Studio is already running on port {port} — reusing existing server.") + logger.info(f" Unsloth is already running on port {port} — reusing existing server.") # try/finally: tear the tunnel down even if interrupted mid-start/render. try: - cf_url = start_cloudflare_tunnel(port) if cloudflare else None + colab_login = _finalize_colab_admin_password() if use_cloudflare else None + cf_url = start_cloudflare_tunnel(port) if use_cloudflare else None _publish_cloudflare_url(cf_url) - _show_and_embed(port, cloudflare_url = cf_url) + _show_and_embed( + port, + cloudflare_url = cf_url, + colab_login = colab_login, + cloudflare_requested = use_cloudflare, + ) for _ in range(10000): time.sleep(300) print("=", end = "", flush = True) @@ -313,7 +634,6 @@ def start(port: int = 8888, *, cloudflare: bool = False): logger.info(" Loading backend...") from run import run_server - # Auto-detect frontend path repo_root = Path(__file__).parent.parent frontend_path = repo_root / "frontend" / "dist" @@ -323,8 +643,7 @@ def start(port: int = 8888, *, cloudflare: bool = False): logger.info(" Starting server...") try: - # cloudflare=False: this helper owns the tunnel (Colab's own - # start(cloudflare=...) drives it), so pin it off explicitly. + # cloudflare=False: this helper owns the tunnel (via start(cloudflare=...)), so pin it off. app = run_server( host = "0.0.0.0", port = port, @@ -339,14 +658,12 @@ def start(port: int = 8888, *, cloudflare: bool = False): logger.error(f"❌ Unsloth Studio failed to start: {exc}") return - # run_server auto-increments the port if in use; read back the bound port so the - # proxy URL and iframe point at the right place. + # run_server may auto-increment the port; read back the bound port for the proxy URL/iframe. actual_port: int = getattr(getattr(app, "state", None), "server_port", None) or port logger.info(f" Server started on port {actual_port}!") - # Poll health endpoint before showing the link — avoids the race where ready_event - # fires but the process hasn't finished binding. + # Poll health before showing the link: avoids the race where ready_event fires pre-bind. import urllib.request server_ready = False @@ -365,12 +682,17 @@ def start(port: int = 8888, *, cloudflare: bool = False): ) return - # Open the tunnel now the server is healthy, publish its URL for /api/health, and - # tear it down on interrupt (try/finally) rather than orphan the process. + # Server healthy: finalize Colab auth, open the tunnel, publish URL, tear down on interrupt. try: - cf_url = start_cloudflare_tunnel(actual_port) if cloudflare else None + colab_login = _finalize_colab_admin_password() if use_cloudflare else None + cf_url = start_cloudflare_tunnel(actual_port) if use_cloudflare else None _publish_cloudflare_url(cf_url) - _show_and_embed(actual_port, cloudflare_url = cf_url) + _show_and_embed( + actual_port, + cloudflare_url = cf_url, + colab_login = colab_login, + cloudflare_requested = use_cloudflare, + ) # Keep kernel alive so the daemon server thread runs. for _ in range(10000): diff --git a/studio/backend/core/data_recipe/jobs/manager.py b/studio/backend/core/data_recipe/jobs/manager.py index 0e0044702e..135c9fccf6 100644 --- a/studio/backend/core/data_recipe/jobs/manager.py +++ b/studio/backend/core/data_recipe/jobs/manager.py @@ -27,7 +27,6 @@ from .constants import ( ) from .parse import apply_update, coerce_event, parse_log_message from .types import Job -from .worker import run_job_process from loggers import get_logger logger = get_logger(__name__) @@ -169,12 +168,18 @@ class JobManager: native_path_secret_removed_for_child_start, run_without_native_path_secret, ) + from utils.hf_cache_settings import child_environment_for_spawn, get_hf_cache_paths - with native_path_secret_removed_for_child_start(): + cache_env = get_hf_cache_paths().child_env({}) + + with ( + child_environment_for_spawn(cache_env), + native_path_secret_removed_for_child_start(), + ): mp_q = _CTX.Queue() proc = _CTX.Process( target = run_without_native_path_secret, - args = (run_job_process,), + args = ("core.data_recipe.jobs.worker", "run_job_process", cache_env), kwargs = {"event_queue": mp_q, "recipe": recipe, "run": run_payload}, daemon = True, ) diff --git a/studio/backend/core/data_recipe/jobs/parse.py b/studio/backend/core/data_recipe/jobs/parse.py index 3be830d0e4..8c2e8a4d55 100644 --- a/studio/backend/core/data_recipe/jobs/parse.py +++ b/studio/backend/core/data_recipe/jobs/parse.py @@ -133,7 +133,7 @@ def parse_log_message(msg: str) -> ParsedUpdate | None: source = "github", status = "rate_limited", retry_after_sec = seconds, - message = ("Waiting for GitHub rate limit. Studio will resume automatically."), + message = ("Waiting for GitHub rate limit. Unsloth will resume automatically."), ), ) @@ -147,7 +147,7 @@ def parse_log_message(msg: str) -> ParsedUpdate | None: status = "rate_limited", retry_after_sec = seconds, message = ( - "Waiting for GitHub secondary rate limit. Studio will resume automatically." + "Waiting for GitHub secondary rate limit. Unsloth will resume automatically." ), ), ) @@ -161,7 +161,7 @@ def parse_log_message(msg: str) -> ParsedUpdate | None: source = "github", status = "rate_limited", retry_after_sec = seconds, - message = ("Waiting for GitHub rate limit. Studio will resume automatically."), + message = ("Waiting for GitHub rate limit. Unsloth will resume automatically."), ), ) diff --git a/studio/backend/core/data_recipe/local_callable_validators.py b/studio/backend/core/data_recipe/local_callable_validators.py index ebb1d39dfb..ffc81669ae 100644 --- a/studio/backend/core/data_recipe/local_callable_validators.py +++ b/studio/backend/core/data_recipe/local_callable_validators.py @@ -238,7 +238,7 @@ def _run_oxc_batch( if not node_executable: return _fallback_results( len(code_values), - "Node.js not found (install Node >= 20.19, or re-run Studio setup to provision it).", + "Node.js not found (install Node >= 20.19, or re-run Unsloth setup to provision it).", ) try: tmp_dir = ensure_dir(oxc_validator_tmp_root()) diff --git a/studio/backend/core/data_recipe/service.py b/studio/backend/core/data_recipe/service.py index 4647dc098d..9770e88b7f 100644 --- a/studio/backend/core/data_recipe/service.py +++ b/studio/backend/core/data_recipe/service.py @@ -280,8 +280,8 @@ def create_data_designer(recipe: dict[str, Any], *, artifact_path: str | None = from data_designer.interface.data_designer import DataDesigner # pyright: ignore[reportMissingImports] if artifact_path is None: - # DataDesigner defaults to cwd/artifacts; packaged Studio can run with - # cwd=/, so keep default callers on Studio's writable recipe artifact root. + # DataDesigner defaults to cwd/artifacts; packaged Unsloth can run with + # cwd=/, so keep default callers on Unsloth's writable recipe artifact root. artifact_path = str(recipe_datasets_root()) recipe = _strip_frontend_model_config_metadata(recipe) diff --git a/studio/backend/core/export/export.py b/studio/backend/core/export/export.py index c8be50b08b..1b24c46e65 100644 --- a/studio/backend/core/export/export.py +++ b/studio/backend/core/export/export.py @@ -1048,6 +1048,21 @@ class ExportBackend: "Use the safetensors adapter instead.", None, ) + # llama.cpp's convert_lora_to_gguf.py has no concept of DoRA's + # lora_magnitude_vector tensors: it only reads the standard + # lora_A/lora_B delta, so exporting a DoRA adapter would silently + # drop the magnitude rescaling and produce a GGUF LoRA file that + # loads fine but no longer matches the trained model. + _peft_config = getattr(self.current_model, "peft_config", {}).get("default") + if getattr(_peft_config, "use_dora", False): + return ( + False, + "GGUF LoRA export is not supported for DoRA adapters: the GGUF LoRA " + "format has no way to represent DoRA's magnitude vectors, so the " + "exported file would silently lose the DoRA behavior. Use the " + "safetensors adapter instead, or merge to a full GGUF model.", + None, + ) outtype = str(gguf_outtype).lower() if outtype not in _GGUF_LORA_OUTTYPES: return ( diff --git a/studio/backend/core/export/orchestrator.py b/studio/backend/core/export/orchestrator.py index 6d1a928f2e..aaf48615f0 100644 --- a/studio/backend/core/export/orchestrator.py +++ b/studio/backend/core/export/orchestrator.py @@ -230,16 +230,20 @@ class ExportOrchestrator: native_path_secret_removed_for_child_start, run_without_native_path_secret, ) + from utils.hf_cache_settings import child_environment_for_spawn, get_hf_cache_paths - from .worker import run_export_process + cache_env = get_hf_cache_paths().child_env({}) - with native_path_secret_removed_for_child_start(): + with ( + child_environment_for_spawn(cache_env), + native_path_secret_removed_for_child_start(), + ): self._cmd_queue = _CTX.Queue() self._resp_queue = _CTX.Queue() self._proc = _CTX.Process( target = run_without_native_path_secret, - args = (run_export_process,), + args = ("core.export.worker", "run_export_process", cache_env), kwargs = { "cmd_queue": self._cmd_queue, "resp_queue": self._resp_queue, diff --git a/studio/backend/core/inference/__init__.py b/studio/backend/core/inference/__init__.py index ad78157418..1491dfa749 100644 --- a/studio/backend/core/inference/__init__.py +++ b/studio/backend/core/inference/__init__.py @@ -11,7 +11,7 @@ subprocess and can be imported directly from .inference when needed. Public names are resolved lazily (PEP 562): importing this package -- or a dependency-light leaf like ``core.inference.chat_eos`` -- must NOT eagerly pull the orchestrator / llama_cpp import chain (httpx, subprocess plumbing, the ML -backend and its Studio dependencies). Those load only when a public name is +backend and its Unsloth dependencies). Those load only when a public name is actually accessed, so standalone helpers stay unit-testable without the full inference stack. """ diff --git a/studio/backend/core/inference/anthropic_compat.py b/studio/backend/core/inference/anthropic_compat.py index 3c7a4cb182..34445cc58e 100644 --- a/studio/backend/core/inference/anthropic_compat.py +++ b/studio/backend/core/inference/anthropic_compat.py @@ -539,7 +539,7 @@ class AnthropicPassthroughEmitter: Only calls naming a tool in ``allowed_tools`` (the client's declared tools) are promoted; everything else streams as text exactly as before. - Never enabled for Studio's own tool loop. + Never enabled for Unsloth's own tool loop. """ from core.inference.passthrough_healing import StreamToolCallHealer diff --git a/studio/backend/core/inference/audio_codecs.py b/studio/backend/core/inference/audio_codecs.py index 93c7da72cb..b59f2bcce0 100644 --- a/studio/backend/core/inference/audio_codecs.py +++ b/studio/backend/core/inference/audio_codecs.py @@ -76,8 +76,14 @@ class AudioCodecManager: if self._snac_model is not None: return from snac import SNAC + from utils.hf_cache_settings import active_hf_hub_cache - self._snac_model = SNAC.from_pretrained("hubertsiuzdak/snac_24khz").to(device).eval() + # Route weights to the selected cache; this can run in the main process. + self._snac_model = ( + SNAC.from_pretrained("hubertsiuzdak/snac_24khz", cache_dir = active_hf_hub_cache()) + .to(device) + .eval() + ) logger.info("Loaded SNAC codec (24kHz)") def _load_bicodec( diff --git a/studio/backend/core/inference/chat_template_helpers.py b/studio/backend/core/inference/chat_template_helpers.py index 897db8262d..528c059fbc 100644 --- a/studio/backend/core/inference/chat_template_helpers.py +++ b/studio/backend/core/inference/chat_template_helpers.py @@ -10,10 +10,242 @@ native-chat-template fallback used by the transformers and MLX backends. import copy import json import logging +from dataclasses import dataclass from typing import Optional _THINK_OPEN = "" _THINK_CLOSE = "" +_GEMMA_CHANNEL_START = "<|channel>" +_GEMMA_THOUGHT_OPEN = "<|channel>thought" +_GEMMA_THOUGHT_CLOSE = "" +_GEMMA_TEMPLATE_OPENERS = ( + _GEMMA_THOUGHT_OPEN + "\n", + _GEMMA_THOUGHT_OPEN + "\\n", + _GEMMA_THOUGHT_OPEN + _GEMMA_THOUGHT_CLOSE, +) + + +def _tokenizer_objects(tokenizer) -> tuple: + """Return a processor/tokenizer and its distinct nested tokenizer.""" + if tokenizer is None: + return () + nested = getattr(tokenizer, "tokenizer", None) + return (tokenizer,) if nested is None or nested is tokenizer else (tokenizer, nested) + + +def _selected_template_strings_from_value( + template, + tools = None, + *, + prefer_tool_use: bool = True, +) -> tuple[str, ...]: + """Return the named chat template matching HF's default selection rules.""" + tools = tools or None + if isinstance(template, str): + return (template,) + if not isinstance(template, dict): + return () + if prefer_tool_use and tools and isinstance(template.get("tool_use"), str): + return (template["tool_use"],) + if isinstance(template.get("default"), str): + return (template["default"],) + values = tuple(value for value in template.values() if isinstance(value, str)) + return values if len(values) == 1 else () + + +def _selected_chat_template_strings(tokenizer, tools = None) -> tuple[str, ...]: + """Return the active chat template selected for this request.""" + tools = tools or None + getter = getattr(tokenizer, "get_chat_template", None) + if callable(getter): + for kwargs in ({"chat_template": None, "tools": tools}, {"tools": tools}, {}): + try: + selected = getter(**kwargs) + except Exception: + continue + if isinstance(selected, str): + return (selected,) + # ProcessorMixin.apply_chat_template does not switch to "tool_use" implicitly; + # it uses "default" unless chat_template= names another template. + is_processor = getattr(tokenizer, "tokenizer", None) is not None and callable( + getattr(tokenizer, "apply_chat_template", None) + ) + return _selected_template_strings_from_value( + getattr(tokenizer, "chat_template", None), + tools, + prefer_tool_use = not is_processor, + ) + + +def _detect_reasoning_channel_markers_from_templates( + templates: tuple[str, ...], +) -> Optional[tuple[str, str]]: + """Return Gemma native reasoning markers only when a template emits them.""" + if any(opener in template for template in templates for opener in _GEMMA_TEMPLATE_OPENERS): + return _GEMMA_THOUGHT_OPEN, _GEMMA_THOUGHT_CLOSE + return None + + +def detect_reasoning_channel_markers(tokenizer, tools = None) -> Optional[tuple[str, str]]: + """Return native Gemma thought-channel markers supported by a tokenizer. + + Detection uses the active chat template rather than model names or vocabulary + membership. Some models expose Gemma control tokens without using the native + thought-channel response protocol, and those must keep normal + ``skip_special_tokens`` streaming. + """ + for obj in _tokenizer_objects(tokenizer): + templates = _selected_chat_template_strings(obj, tools) + if templates: + return _detect_reasoning_channel_markers_from_templates(templates) + return None + + +def detect_reasoning_channel_markers_from_template( + template, tools = None +) -> Optional[tuple[str, str]]: + """Return native Gemma thought-channel markers from a raw template value.""" + return _detect_reasoning_channel_markers_from_templates( + _selected_template_strings_from_value(template, tools) + ) + + +def detect_reasoning_channel_markers_from_model_info( + tokenizer, + model_info: Optional[dict] = None, + tools = None, +) -> Optional[tuple[str, str]]: + """Return reasoning markers from the active or cached native template.""" + markers = detect_reasoning_channel_markers(tokenizer, tools = tools) + if markers is not None or not isinstance(model_info, dict): + return markers + + native_templates = ( + model_info.get("native_chat_template"), + (model_info.get("chat_template_info") or {}).get("template"), + ) + for template in native_templates: + markers = detect_reasoning_channel_markers_from_template(template, tools) + if markers is not None: + return markers + return None + + +@dataclass(frozen = True) +class ChatTemplateRenderResult: + """Prompt plus response-protocol metadata selected by the renderer.""" + + prompt: str + reasoning_channel_markers: Optional[tuple[str, str]] = None + + +def _split_partial_marker(text: str, marker: str) -> tuple[str, str]: + """Hold the longest suffix that may become ``marker`` in the next chunk.""" + for length in range(min(len(text), len(marker) - 1), 0, -1): + if text.endswith(marker[:length]): + return text[:-length], text[-length:] + return text, "" + + +class ReasoningChannelNormalizer: + """Incrementally convert one native reasoning channel to ````. + + The parser follows mlx-vlm's streaming boundary behavior but emits Unsloth's + established canonical text contract. Only the configured opening and + closing markers are consumed; tool-call and other control markers remain + available to downstream parsers. + """ + + def __init__(self, opening_marker: str, closing_marker: str): + self._opening_marker = opening_marker + self._closing_marker = closing_marker + self._buffer = "" + self._in_reasoning = False + self._reasoning_done = False + self._skip_opening_newline = False + + def feed(self, text: str) -> str: + """Consume a raw text delta and return the stable canonical delta.""" + self._buffer += text or "" + output: list[str] = [] + while self._buffer: + if self._reasoning_done: + output.append(self._buffer) + self._buffer = "" + break + + if self._in_reasoning and self._skip_opening_newline: + if self._buffer.startswith("\n"): + self._buffer = self._buffer[1:] + self._skip_opening_newline = False + if not self._buffer: + break + + marker = self._closing_marker if self._in_reasoning else self._opening_marker + index = self._buffer.find(marker) + if index < 0: + stable, self._buffer = _split_partial_marker(self._buffer, marker) + output.append(stable) + break + + output.append(self._buffer[:index]) + self._buffer = self._buffer[index + len(marker) :] + if self._in_reasoning: + output.append(_THINK_CLOSE) + self._in_reasoning = False + self._reasoning_done = True + else: + output.append(_THINK_OPEN) + self._in_reasoning = True + self._skip_opening_newline = True + return "".join(output) + + def finish(self) -> str: + """Flush a naturally completed stream and close an open think block.""" + output = self.drain() + if self._in_reasoning: + output += _THINK_CLOSE + self._in_reasoning = False + self._reasoning_done = True + return output + + def drain(self) -> str: + """Flush buffered literal text without synthesizing a closing tag.""" + output = self._buffer + self._buffer = "" + return output + + +def normalize_reasoning_snapshots( + stream, + tokenizer = None, + cancel_event = None, + markers: Optional[tuple[str, str]] = None, + tools = None, +): + """Normalize a prefix-monotonic cumulative text stream when supported.""" + markers = markers or detect_reasoning_channel_markers(tokenizer, tools = tools) + if markers is None: + yield from stream + return + + normalizer = ReasoningChannelNormalizer(*markers) + raw_output = "" + normalized_output = "" + for snapshot in stream: + if not snapshot.startswith(raw_output): + raise RuntimeError("Reasoning normalization requires cumulative text snapshots") + delta = normalizer.feed(snapshot[len(raw_output) :]) + raw_output = snapshot + if delta: + normalized_output += delta + yield normalized_output + + cancelled = cancel_event is not None and cancel_event.is_set() + tail = normalizer.drain() if cancelled else normalizer.finish() + if tail: + normalized_output += tail + yield normalized_output def detect_think_prefill(prompt: Optional[str], special_tokens = None) -> str: @@ -166,7 +398,8 @@ def render_native_template( preserve_thinking: Optional[bool] = None, apply_fn = None, hf_token: Optional[str] = None, -) -> Optional[str]: + return_metadata: bool = False, +): """Render ``messages`` + ``tools`` with the model's NATIVE chat template. Some Unsloth override templates (e.g. ``mistral``, ``gemma-4``) do not emit @@ -175,7 +408,9 @@ def render_native_template( tool-calling syntax. It is loaded straight from the repo (bypassing any override on the live tokenizer) and cached on ``model_info``. Returns the rendered prompt only if the native template actually emits the tools (render - differs with vs without tools); otherwise ``None``. + differs with vs without tools); otherwise ``None``. With ``return_metadata``, + returns ``ChatTemplateRenderResult`` so callers can stream with the response + protocol selected by this request's template. ``hf_token`` is the token the model was loaded with -- passed to the repo load so a gated/private model's native template can still be fetched (otherwise the @@ -261,7 +496,16 @@ def render_native_template( exc, ) return None - return with_tools if with_tools != no_tools else None + if with_tools == no_tools: + return None + if return_metadata: + return ChatTemplateRenderResult( + with_tools, + _detect_reasoning_channel_markers_from_templates( + _selected_template_strings_from_value(native_tpl, tools) + ), + ) + return with_tools def render_with_native_template_fallback( @@ -277,7 +521,8 @@ def render_with_native_template_fallback( preserve_thinking: Optional[bool] = None, apply_fn = None, hf_token: Optional[str] = None, -) -> str: + return_metadata: bool = False, +): """Return ``formatted_prompt``, swapping in a native-template render when an override template dropped the ``tools`` schema. @@ -285,9 +530,27 @@ def render_with_native_template_fallback( them (detected by comparison, robust against tool names in the system prompt), re-render with the model's native template. Shared by the transformers and MLX backends so both advertise tools consistently. ``hf_token`` is forwarded so a - gated/private model's native template can still be fetched.""" + gated/private model's native template can still be fetched. With + ``return_metadata``, returns the selected prompt plus reasoning-channel markers + for the exact template used by this request.""" + live_markers = detect_reasoning_channel_markers(tokenizer, tools = tools) + + def _result(prompt: str, markers = live_markers): + if return_metadata: + return ChatTemplateRenderResult(prompt, markers) + return prompt + if not tools: - return formatted_prompt + # Gemma 4 can emit its native reasoning protocol even when a generation-time + # Unsloth override rendered a marker-free prompt. Preserve the live-verified + # no-tools thinking behavior without letting cached native metadata describe + # unrelated tool prompts that kept the active override. + markers = live_markers + if markers is None: + markers = detect_reasoning_channel_markers_from_model_info( + tokenizer, model_info, tools = None + ) + return _result(formatted_prompt, markers) if apply_fn is None: apply_fn = apply_chat_template_for_generation # Probe whether the live template dropped the schema. A tools-requiring template @@ -307,9 +570,9 @@ def render_with_native_template_fallback( active_model_name, exc, ) - return formatted_prompt + return _result(formatted_prompt) if formatted_prompt != probe_no_tools: - return formatted_prompt # template already emits the tools schema + return _result(formatted_prompt) # template already emits the tools schema native_prompt = render_native_template( model_info = model_info, active_model_name = active_model_name, @@ -320,6 +583,7 @@ def render_with_native_template_fallback( preserve_thinking = preserve_thinking, apply_fn = apply_fn, hf_token = hf_token, + return_metadata = return_metadata, ) if native_prompt: logger.info( @@ -328,4 +592,4 @@ def render_with_native_template_fallback( active_model_name, ) return native_prompt - return formatted_prompt + return _result(formatted_prompt) diff --git a/studio/backend/core/inference/chat_templates.py b/studio/backend/core/inference/chat_templates.py index 58f63ff61b..04c0db6aae 100644 --- a/studio/backend/core/inference/chat_templates.py +++ b/studio/backend/core/inference/chat_templates.py @@ -4,13 +4,13 @@ """Bundled chat-template selection for GGUF inference. Some shipped GGUF quants embed an older chat template. Rather than re-cutting and -asking users to re-download every quant, Studio can override the embedded template +asking users to re-download every quant, Unsloth can override the embedded template at llama-server launch time with a bundled, up-to-date Jinja template for known model families. The override is wired through the existing ``chat_template_override`` -> ``--chat-template-file`` path in ``LlamaCppBackend.load_model``. Currently this covers ``unsloth/gemma-4-*-GGUF``, which gains the upstream PR #118 -``preserve_thinking`` flag (defaulted OFF here) so the Studio "Preserve thinking" +``preserve_thinking`` flag (defaulted OFF here) so the Unsloth "Preserve thinking" toggle appears while staying disabled by default. """ diff --git a/studio/backend/core/inference/external_provider.py b/studio/backend/core/inference/external_provider.py index 20312e067c..2debf946e9 100644 --- a/studio/backend/core/inference/external_provider.py +++ b/studio/backend/core/inference/external_provider.py @@ -473,7 +473,7 @@ def _apply_mistral_reasoning_controls( # handles every provider without storing credentials. def _create_shared_http_client() -> httpx.AsyncClient: # Unsupported env proxy schemes (socks:// etc) raise at construction and - # would crash Studio startup (#6090); retry ignoring env proxies instead. + # would crash Unsloth startup (#6090); retry ignoring env proxies instead. try: return httpx.AsyncClient() except (ImportError, ValueError) as exc: @@ -858,7 +858,7 @@ class ExternalProviderClient: if not self._is_openai_compatible(): # Gemini speaks its own native REST shape (contents/parts); # `_stream_gemini` translates request/response into the OpenAI - # Chat Completions chunk format the rest of Studio expects. + # Chat Completions chunk format the rest of Unsloth expects. # API ref: https://ai.google.dev/gemini-api/docs if self.provider_type == "gemini": async for line in self._stream_gemini( @@ -1706,7 +1706,7 @@ class ExternalProviderClient: # Translate OpenAI multimodal parts -> Anthropic native shapes. # - `image_url` -> `{type:"image", source:...}` # - `input_document` -> `{type:"document", source:...}` - # (Studio extension; mirrors Anthropic's document block, + # (Unsloth extension; mirrors Anthropic's document block, # which supports PDFs as base64 or URL per # https://platform.claude.com/docs/en/build-with-claude/vision) anthropic_parts: list[dict[str, Any]] = [] @@ -1749,7 +1749,7 @@ class ExternalProviderClient: } ) elif part.get("type") == "input_document": - # Studio's normalised PDF/doc type (file_data data-URI or + # Unsloth's normalised PDF/doc type (file_data data-URI or # file_url) -> Anthropic's native `document` block. url = part.get("file_url") or "" data_uri = part.get("file_data") or "" @@ -4704,7 +4704,7 @@ class ExternalProviderClient: {"type": "image_generation_call", "id": call_id} ) elif part_type == "input_document": - # Map Studio's `input_document` onto Responses' `input_file`. + # Map Unsloth's `input_document` onto Responses' `input_file`. # https://developers.openai.com/api/docs/guides/images-vision file_url = part.get("file_url") file_data = part.get("file_data") @@ -6010,7 +6010,7 @@ class ExternalProviderClient: if not models and self.provider_type == "ollama": models = await self._list_ollama_native_models() # Gemini's native /v1beta/models uses a different shape; repackage - # into the OpenAI-compatible one Studio expects. + # into the OpenAI-compatible one Unsloth expects. if not models and self.provider_type == "gemini": models = self._parse_gemini_models(data) return models @@ -6213,7 +6213,7 @@ def _friendly_provider_error_text( *, model: str | None = None, ) -> str: - """Rewrite common provider errors into actionable Studio copy.""" + """Rewrite common provider errors into actionable Unsloth copy.""" if status_code == 404 and model: lowered = raw_message.lower() if "not found" in lowered or "not_found" in lowered: diff --git a/studio/backend/core/inference/inference.py b/studio/backend/core/inference/inference.py index 172e9e5546..2f46470091 100644 --- a/studio/backend/core/inference/inference.py +++ b/studio/backend/core/inference/inference.py @@ -5,9 +5,10 @@ from unsloth import FastLanguageModel, FastVisionModel from unsloth.chat_templates import get_chat_template -from transformers import TextStreamer +from transformers import TextIteratorStreamer, TextStreamer from peft import PeftModel, PeftModelForCausalLM +import contextlib import json import sys import torch @@ -32,6 +33,11 @@ from core.inference.chat_eos import ( chat_eos_repair, resolve_chat_turn_end_eos_ids_using, ) +from core.inference.chat_template_helpers import ( + ReasoningChannelNormalizer, + detect_reasoning_channel_markers, + detect_think_prefill, +) from core.inference.presence_penalty import _make_presence_penalty_processor from io import StringIO import structlog @@ -187,6 +193,53 @@ class HarmonyTextStreamer: self._queue.put(new_content) +class ReasoningTextIteratorStreamer(TextIteratorStreamer): + """TextIteratorStreamer that preserves native channel tokens until parsed.""" + + def __init__( + self, + tokenizer, + *, + markers: tuple[str, str], + skip_prompt: bool = True, + timeout: float = 0.2, + cancel_event = None, + **decode_kwargs, + ): + decode_kwargs["skip_special_tokens"] = False + super().__init__(tokenizer, skip_prompt = skip_prompt, timeout = timeout, **decode_kwargs) + self._normalizer = ReasoningChannelNormalizer(*markers) + self._cancel_event = cancel_event + self._aborted = False + + def abort(self): + """Mark generation as failed so ``end`` drains without closing.""" + self._aborted = True + + def on_finalized_text( + self, + text: str, + stream_end: bool = False, + ): + """Queue canonical deltas, closing only on natural stream completion.""" + delta = self._normalizer.feed(text) + if delta: + self.text_queue.put(delta, timeout = self.timeout) + + if stream_end: + cancelled = self._aborted or ( + self._cancel_event is not None and self._cancel_event.is_set() + ) + tail = self._normalizer.drain() if cancelled else self._normalizer.finish() + if tail: + self.text_queue.put(tail, timeout = self.timeout) + self.text_queue.put(self.stop_signal, timeout = self.timeout) + + +class _GenerationThreadError(RuntimeError): + """Generation worker failures that should propagate through stream routes.""" + + class InferenceBackend: """Unified inference backend supporting text, vision, and LoRA models""" @@ -836,6 +889,7 @@ class InferenceBackend: thread_id: Optional[str] = None, rag_scope: Optional[dict] = None, presence_penalty: float = 0.0, + reasoning_prefilled: bool = False, ): """Run an agentic tool loop on top of ``generate_chat_response``. @@ -889,6 +943,7 @@ class InferenceBackend: session_id = session_id, thread_id = thread_id, rag_scope = rag_scope, + reasoning_prefilled = reasoning_prefilled, ) def generate_chat_response( @@ -960,8 +1015,7 @@ class InferenceBackend: thread can toggle adapters under the generation lock. """ if not self.active_model_name: - yield "Error: No active model" - return + raise RuntimeError("No active model") model_info = self.models[self.active_model_name] is_vision = model_info.get("is_vision", False) @@ -1049,6 +1103,7 @@ class InferenceBackend: template_messages = [{"role": "system", "content": system_prompt}] + messages else: template_messages = messages + reasoning_channel_markers_resolved = False try: if not (hasattr(tokenizer, "chat_template") and tokenizer.chat_template): raise ValueError( @@ -1058,6 +1113,7 @@ class InferenceBackend: f"Please use a model that includes a chat template, or manually set " f"one via tokenizer.chat_template before inference." ) + reasoning_channel_markers = None formatted_prompt = self._apply_chat_template_for_generation( tokenizer, template_messages, @@ -1073,7 +1129,7 @@ class InferenceBackend: render_with_native_template_fallback, ) - formatted_prompt = render_with_native_template_fallback( + render_result = render_with_native_template_fallback( formatted_prompt = formatted_prompt, tokenizer = tokenizer, model_info = model_info, @@ -1085,13 +1141,19 @@ class InferenceBackend: preserve_thinking = preserve_thinking, apply_fn = self._apply_chat_template_for_generation, hf_token = model_info.get("hf_token"), + return_metadata = True, ) + formatted_prompt = render_result.prompt + reasoning_channel_markers = render_result.reasoning_channel_markers + reasoning_channel_markers_resolved = True logger.debug(f"Formatted prompt: {formatted_prompt[:200]}...") except Exception as e: logger.error(f"Error applying chat template: {e}") # Fall back to manual formatting formatted_prompt = self.format_chat_prompt(messages, system_prompt) + reasoning_channel_markers = None + reasoning_channel_markers_resolved = True # Step 3: generate yield from self.generate_stream( @@ -1105,6 +1167,8 @@ class InferenceBackend: cancel_event = cancel_event, _adapter_state = _adapter_state, presence_penalty = presence_penalty, + reasoning_channel_markers = reasoning_channel_markers, + reasoning_channel_markers_resolved = reasoning_channel_markers_resolved, ) def _generate_vision_response( @@ -1190,21 +1254,27 @@ class InferenceBackend: # Stream with TextIteratorStreamer + background thread try: - from core.inference.chat_template_helpers import detect_think_prefill - # Re-emit an open prefill swallowed by skip_prompt (see # generate_stream). think_prefix = detect_think_prefill( prompt_text, getattr(raw_tokenizer, "all_special_tokens", None) ) - from transformers import TextIteratorStreamer import threading - streamer = TextIteratorStreamer( + streamer = self._make_text_streamer( raw_tokenizer, + protocol_source = processor, + # The text-only VLM fallback above did not render with the + # processor template, so its native markers do not describe + # this request's response protocol. + reasoning_channel_markers = detect_reasoning_channel_markers(processor) + if image + else None, + reasoning_channel_markers_resolved = True, skip_prompt = True, - skip_special_tokens = True, timeout = 0.2, + cancel_event = cancel_event, + use_harmony = self._is_gpt_oss_model(), ) generation_kwargs = dict( @@ -1226,6 +1296,10 @@ class InferenceBackend: ) if _pp is not None: generation_kwargs["logits_processor"] = _pp + stopping_criteria = self._cancel_stopping_criteria(cancel_event) + if stopping_criteria is not None: + generation_kwargs["stopping_criteria"] = stopping_criteria + active_stop_token_ids = self._generation_stop_token_ids(model, generation_kwargs) err: dict[str, str] = {} @@ -1235,6 +1309,8 @@ class InferenceBackend: model.generate(**generation_kwargs) except Exception as e: err["msg"] = str(e) + if hasattr(streamer, "abort"): + streamer.abort() logger.error(f"Vision generation error in thread: {e}") finally: try: @@ -1251,12 +1327,17 @@ class InferenceBackend: if think_prefix: yield think_prefix from queue import Empty + import time generation_complete = False + cancel_deadline = None try: while True: if cancel_event is not None and cancel_event.is_set(): - break + if cancel_deadline is None: + cancel_deadline = time.monotonic() + 10 + elif time.monotonic() >= cancel_deadline: + break try: new_token = next(streamer) except StopIteration: @@ -1265,27 +1346,48 @@ class InferenceBackend: except Empty: if not thread.is_alive(): generation_complete = True + output = yield from self._drain_streamer_tail( + streamer, output, active_stop_token_ids + ) + break + if cancel_deadline is not None: + remaining = cancel_deadline - time.monotonic() + if remaining <= 0: + break + thread.join(timeout = remaining) + if thread.is_alive(): + break + generation_complete = True + output = yield from self._drain_streamer_tail( + streamer, output, active_stop_token_ids + ) break continue if new_token: - output += new_token - cleaned = self._clean_generated_text(output) + output, cleaned = self._append_stream_delta( + output, new_token, active_stop_token_ids + ) yield cleaned finally: if cancel_event is not None and not generation_complete: cancel_event.set() - thread.join(timeout = 10) + join_timeout = 10 + if cancel_deadline is not None: + join_timeout = max(0, cancel_deadline - time.monotonic()) + thread.join(timeout = join_timeout) if thread.is_alive(): logger.warning( "Vision generation thread did not exit after cancel/join timeout" ) if err.get("msg"): - yield f"Error: {err['msg']}" + raise _GenerationThreadError(err["msg"]) + except _GenerationThreadError: + raise except Exception as e: logger.error(f"Vision generation error: {e}") - yield f"Error: {str(e)}" + raise def generate_audio_input_response( self, @@ -1410,11 +1512,13 @@ class InferenceBackend: ) if err.get("msg"): - yield f"Error: {err['msg']}" + raise _GenerationThreadError(err["msg"]) + except _GenerationThreadError: + raise except Exception as e: logger.error(f"Audio input generation error: {e}") - yield f"Error: {str(e)}" + raise def generate_whisper_response( self, @@ -1447,6 +1551,86 @@ class InferenceBackend: from utils.datasets import is_gpt_oss_model_name return is_gpt_oss_model_name(model_name or self.active_model_name or "") + def _make_text_streamer( + self, + tokenizer, + *, + protocol_source = None, + reasoning_channel_markers = None, + reasoning_channel_markers_resolved: bool = False, + skip_prompt: bool = True, + timeout: float = 0.2, + cancel_event = None, + use_harmony: bool = False, + ): + """Create the streamer matching this model's native response protocol.""" + if use_harmony: + try: + return HarmonyTextStreamer( + tokenizer, + skip_prompt = skip_prompt, + timeout = timeout, + ) + except Exception as e: + logger.warning(f"HarmonyTextStreamer init failed, falling back: {e}") + return TextIteratorStreamer( + tokenizer, + skip_prompt = skip_prompt, + skip_special_tokens = True, + timeout = timeout, + ) + + markers = ( + reasoning_channel_markers + if reasoning_channel_markers_resolved + else reasoning_channel_markers + or detect_reasoning_channel_markers(protocol_source or tokenizer) + ) + if markers is not None: + return ReasoningTextIteratorStreamer( + tokenizer, + markers = markers, + skip_prompt = skip_prompt, + timeout = timeout, + cancel_event = cancel_event, + ) + return TextIteratorStreamer( + tokenizer, + skip_prompt = skip_prompt, + skip_special_tokens = True, + timeout = timeout, + ) + + def _append_stream_delta( + self, + output: str, + new_token: str, + stop_token_ids = None, + ): + """Append a streamer delta and apply response-boundary cleanup.""" + output += new_token + return output, self._clean_generated_text(output, stop_token_ids = stop_token_ids) + + def _drain_streamer_tail( + self, + streamer, + output: str, + stop_token_ids = None, + ): + """Drain queued streamer text after the producer exits.""" + while True: + try: + new_token = next(streamer) + except StopIteration: + return output + except Exception: + return output + if new_token: + output, cleaned = self._append_stream_delta( + output, new_token, stop_token_ids = stop_token_ids + ) + yield cleaned + def generate_stream( self, prompt: str, @@ -1459,6 +1643,8 @@ class InferenceBackend: cancel_event = None, _adapter_state = None, presence_penalty: float = 0.0, + reasoning_channel_markers = None, + reasoning_channel_markers_resolved: bool = False, ) -> Generator[str, None, None]: """Generate a streaming text response (text models only). @@ -1467,8 +1653,7 @@ class InferenceBackend: ``presence_penalty`` matches the GGUF sampling path via a logits processor (0 disables it). """ if not self.active_model_name: - yield "Error: No active model" - return + raise RuntimeError("No active model") model_info = self.models[self.active_model_name] model = model_info["model"] @@ -1481,9 +1666,7 @@ class InferenceBackend: try: inputs = tokenizer(prompt, return_tensors = "pt").to(model.device) - from transformers import TextIteratorStreamer import threading - from core.inference.chat_template_helpers import detect_think_prefill # skip_prompt swallows an open prefilled by the template; # re-emit it so the frontend can render the thinking block. @@ -1494,30 +1677,16 @@ class InferenceBackend: else detect_think_prefill(prompt, getattr(tokenizer, "all_special_tokens", None)) ) - # gpt-oss models: HarmonyTextStreamer parses the multi-channel - # harmony protocol into tags - if self._is_gpt_oss_model(): - try: - streamer = HarmonyTextStreamer( - tokenizer, - skip_prompt = True, - timeout = 0.2, - ) - except Exception as e: - logger.warning(f"HarmonyTextStreamer init failed, falling back: {e}") - streamer = TextIteratorStreamer( - tokenizer, - skip_prompt = True, - skip_special_tokens = True, - timeout = 0.2, - ) - else: - streamer = TextIteratorStreamer( - tokenizer, - skip_prompt = True, - skip_special_tokens = True, - timeout = 0.2, - ) + streamer = self._make_text_streamer( + tokenizer, + protocol_source = model_info.get("tokenizer"), + reasoning_channel_markers = reasoning_channel_markers, + reasoning_channel_markers_resolved = reasoning_channel_markers_resolved, + skip_prompt = True, + timeout = 0.2, + cancel_event = cancel_event, + use_harmony = self._is_gpt_oss_model(), + ) generation_kwargs = dict( **inputs, @@ -1535,27 +1704,16 @@ class InferenceBackend: if tokenizer.pad_token_id is None else tokenizer.pad_token_id, ) + active_stop_token_ids = self._generation_stop_token_ids(model, generation_kwargs) # Presence penalty (GGUF parity); prompt_len excludes prompt tokens. _pp = _make_presence_penalty_processor( presence_penalty, int(inputs["input_ids"].shape[1]) ) if _pp is not None: generation_kwargs["logits_processor"] = _pp - if cancel_event is not None: - from transformers.generation.stopping_criteria import ( - StoppingCriteria, - StoppingCriteriaList, - ) - class _CancelCriteria(StoppingCriteria): - def __init__(self, ev): - self.ev = ev - - def __call__(self, input_ids, scores, **kwargs): - return self.ev.is_set() - - generation_kwargs["stopping_criteria"] = StoppingCriteriaList( - [_CancelCriteria(cancel_event)] - ) + stopping_criteria = self._cancel_stopping_criteria(cancel_event) + if stopping_criteria is not None: + generation_kwargs["stopping_criteria"] = stopping_criteria def generate_fn(): with self._generation_lock: @@ -1565,6 +1723,8 @@ class InferenceBackend: model.generate(**generation_kwargs) except Exception as e: err["msg"] = str(e) + if hasattr(streamer, "abort"): + streamer.abort() logger.error(f"Generation error: {e}") finally: try: @@ -1582,12 +1742,17 @@ class InferenceBackend: if think_prefix: yield think_prefix from queue import Empty + import time generation_complete = False + cancel_deadline = None try: while True: if cancel_event is not None and cancel_event.is_set(): - break + if cancel_deadline is None: + cancel_deadline = time.monotonic() + 10 + elif time.monotonic() >= cancel_deadline: + break try: new_token = next(streamer) except StopIteration: @@ -1596,11 +1761,27 @@ class InferenceBackend: except Empty: if not thread.is_alive(): generation_complete = True + output = yield from self._drain_streamer_tail( + streamer, output, active_stop_token_ids + ) + break + if cancel_deadline is not None: + remaining = cancel_deadline - time.monotonic() + if remaining <= 0: + break + thread.join(timeout = remaining) + if thread.is_alive(): + break + generation_complete = True + output = yield from self._drain_streamer_tail( + streamer, output, active_stop_token_ids + ) break continue if new_token: - output += new_token - cleaned = self._clean_generated_text(output) + output, cleaned = self._append_stream_delta( + output, new_token, active_stop_token_ids + ) yield cleaned finally: # Set cancel_event only on early exit (user cancel), NOT on @@ -1609,16 +1790,21 @@ class InferenceBackend: # disrupt the next serialized request (e.g. compare mode). if cancel_event is not None and not generation_complete: cancel_event.set() - thread.join(timeout = 10) + join_timeout = 10 + if cancel_deadline is not None: + join_timeout = max(0, cancel_deadline - time.monotonic()) + thread.join(timeout = join_timeout) if thread.is_alive(): logger.warning("Generation thread did not exit after cancel/join timeout") if err.get("msg"): - yield f"Error: {err['msg']}" + raise _GenerationThreadError(err["msg"]) + except _GenerationThreadError: + raise except Exception as e: logger.error(f"Error during generation: {e}") - yield f"Error: {str(e)}" + raise # ── Audio (TTS) Generation ──────────────────────────────────── @@ -1757,8 +1943,30 @@ class InferenceBackend: + text + "<|text_end|>\n<|audio_start|><|global_features_start|>\n" ) + with torch.inference_mode(): - with torch.amp.autocast("cuda", dtype = model.dtype): + # Derive the autocast device from the loaded model, not from the + # global backend: a CPU-fallback DAC on an XPU/CUDA host must not + # open a GPU autocast context around CPU tensors. + device_type = ( + model.device.type + if hasattr(model.device, "type") + else str(model.device).split(":", 1)[0] + ) + # Clamp to autocast-supported backends so exotic devices + # (e.g. "meta" during accelerate offloaded loading) do not raise. + # MPS is autocast-supported since torch 2.3, keep it in the set. + if device_type not in ("cuda", "xpu", "mps", "cpu"): + device_type = "cpu" + # CPU and XPU autocast only accept bfloat16/float16. For a + # float32 model, skip autocast entirely to avoid raising or + # producing a warning on every generate call. + autocast_dtype_supported = model.dtype in (torch.bfloat16, torch.float16) + if device_type in ("cpu", "xpu") and not autocast_dtype_supported: + autocast_ctx = contextlib.nullcontext() + else: + autocast_ctx = torch.amp.autocast(device_type, dtype = model.dtype) + with autocast_ctx: inputs = tokenizer([prompt], return_tensors = "pt").to(model.device) generated = model.generate( **inputs, @@ -2107,8 +2315,42 @@ class InferenceBackend: return img.resize(new_size, Image.Resampling.LANCZOS) return img - def _clean_generated_text(self, text: str) -> str: - """Strip leaked special tokens using the tokenizer's own token list.""" + def _generation_stop_token_ids(self, model, generation_kwargs: dict): + """Return the stop-token ids active for a ``generate`` call.""" + if "eos_token_id" in generation_kwargs: + return generation_kwargs.get("eos_token_id") + generation_config = getattr(model, "generation_config", None) + eos_token_id = getattr(generation_config, "eos_token_id", None) + if eos_token_id is not None: + return eos_token_id + config = getattr(model, "config", None) + return getattr(config, "eos_token_id", None) + + def _cancel_stopping_criteria(self, cancel_event): + """Build a Transformers stopping criteria list for user cancellation.""" + if cancel_event is None: + return None + from transformers.generation.stopping_criteria import ( + StoppingCriteria, + StoppingCriteriaList, + ) + + class _CancelCriteria(StoppingCriteria): + def __init__(self, ev): + self.ev = ev + + def __call__(self, input_ids, scores, **kwargs): + return self.ev.is_set() + + return StoppingCriteriaList([_CancelCriteria(cancel_event)]) + + def _clean_generated_text( + self, + text: str, + *, + stop_token_ids = None, + ) -> str: + """Strip leaked response-boundary tokens after streaming.""" if self._is_gpt_oss_model(): # HarmonyTextStreamer emits clean .... Strip any # harmony protocol tokens and other gpt-oss tokens (e.g. @@ -2118,10 +2360,28 @@ class InferenceBackend: return text.strip() tokenizer = self.models.get(self.active_model_name, {}).get("tokenizer") + tokenizer = getattr(tokenizer, "tokenizer", tokenizer) if tokenizer: - for token in getattr(tokenizer, "all_special_tokens", []): - if token in text: - text = text.replace(token, "") + if stop_token_ids is None: + stop_token_ids = self.models.get(self.active_model_name, {}).get( + "chat_turn_end_eos_ids" + ) + if isinstance(stop_token_ids, int): + stop_token_ids = (stop_token_ids,) + for token_id in stop_token_ids or (): + try: + token = tokenizer.convert_ids_to_tokens(int(token_id)) + except Exception: + token = None + if isinstance(token, str) and token and text.endswith(token): + text = text[: -len(token)] + elif ( + isinstance(token, str) + and token + and text.endswith("") + and text[: -len("")].endswith(token) + ): + text = text[: -len("") - len(token)] + "" return text.strip() def _load_chat_template_info(self, model_name: str): diff --git a/studio/backend/core/inference/llama_cpp.py b/studio/backend/core/inference/llama_cpp.py index 3e93a26787..1fe134c3f9 100644 --- a/studio/backend/core/inference/llama_cpp.py +++ b/studio/backend/core/inference/llama_cpp.py @@ -9,7 +9,9 @@ OpenAI-compatible /v1/chat/completions endpoint. import atexit import contextlib +import functools import json +import math import os import re import struct @@ -21,12 +23,24 @@ import subprocess import sys import threading import time +import uuid from pathlib import Path -from typing import Callable, Collection, Generator, Iterable, List, Mapping, Optional, Union +from typing import ( + Callable, + Collection, + Generator, + Iterable, + List, + Literal, + Mapping, + Optional, + Union, +) import httpx from core.inference.llama_server_args import ( + _LAYER_OFFLOAD_FLAGS, _effective_tensor_parallel, _tensor_parallel_matches_loaded, extra_args_disable_mmproj, @@ -115,7 +129,7 @@ LLAMA_SERVER_NOT_FOUND_DETAIL = ( # llama-server can serve HTTP 200 while running a model entirely on CPU when a # GPU backend fails to init (#5807 / #5106 / #5830). Classify the startup log so -# Studio can warn. Priority: explicit "offloaded N/M layers to GPU" counts +# Unsloth can warn. Priority: explicit "offloaded N/M layers to GPU" counts # (authoritative), then GPU "model buffer size" lines (host-pinned _Host # excluded), then the "device_info:" device table (disconfirm only). _GPU_OFFLOAD_MARKERS = ( @@ -233,8 +247,60 @@ def _wsl_system_rocm_lib_dirs() -> "list[str]": return out -# Plan-without-action re-prompt state (intent signal, caps, message) now lives -# in tool_call_parser, imported above under its old aliases. +def _bundled_hip_present(binary_dir: str) -> bool: + """True when a prebuilt bundle ships its own HIP backend library.""" + if not binary_dir: + return False + try: + # Glob the version suffix (libggml-hip.so, .so.0, .so.0.11.1) the same + # way the installer's runtime health check matches libggml-hip.so*. + return any(Path(str(binary_dir)).glob("libggml-hip.so*")) + except OSError: + return False + + +def _native_linux_system_rocm_lib_dirs(binary_dir: str = "") -> "list[str]": + """System ROCm lib dir(s) to prepend before a prebuilt's bundled HIP, on native Linux. + + The bundled bare-metal HIP runtime can mismatch the host amdkfd driver and crash + in hsa_init(); prepending the whole system ROCm lib dir loads a driver-matched, + version-consistent stack (libhsa-runtime64 / libamdhip64 / librocblas) ahead of it. + The whole dir is deliberate: mixing the bundle's rocBLAS with a different-version + system HIP/ROCR risks missing symbols. UNSLOTH_LLAMA_NO_SYSTEM_ROCM=1 keeps the pure + bundle (for a host whose system ROCm lacks this arch); no-op on WSL / non-Linux. + """ + if os.environ.get("UNSLOTH_LLAMA_NO_SYSTEM_ROCM") == "1": + return [] + if sys.platform != "linux" or os.path.exists("/dev/dxg"): + return [] + if not os.path.exists("/dev/kfd"): + return [] + if not _bundled_hip_present(binary_dir): + return [] + # Env-configured ROCm root first; /opt/rocm only as a fallback so a stale + # /opt/rocm doesn't shadow the driver-matching install these vars point at. + candidates = [] + for var in ("HIP_PATH", "HIP_PATH_57", "ROCM_PATH"): + val = os.environ.get(var) + if val: + candidates.append(val) + candidates.append("/opt/rocm") + out: "list[str]" = [] + seen: "set[str]" = set() + for base in candidates: + for lib_sub in ("lib", "lib64"): + d = os.path.join(base, lib_sub) + if d in seen: + continue + seen.add(d) + if os.path.exists(os.path.join(d, "libhsa-runtime64.so")) or os.path.exists( + os.path.join(d, "libhsa-runtime64.so.1") + ): + out.append(d) + return out + + +# Plan-without-action re-prompt state now lives in tool_call_parser (imported above). # Default max_tokens to the effective context when known. The floor is high # enough for reasoning-heavy GGUFs and max_tokens-omitting API clients. @@ -441,6 +507,23 @@ def _hf_offline_if_dns_dead(): os.environ.pop("TRANSFORMERS_OFFLINE", None) +try: + _SLOT_SAVE_MAX_BYTES = int(os.environ.get("UNSLOTH_SLOT_SAVE_MAX_BYTES") or (10 << 30)) +except ValueError: + _SLOT_SAVE_MAX_BYTES = 10 << 30 + +# The idle loop holds the lifecycle gate across a slot save, so a newly arriving +# request waits on the in-flight save's HTTP call. Bound it (was 120s) so a slow +# or stuck save can't stall the next request for minutes; best-effort save just +# falls back to a plain unload. Override with UNSLOTH_SLOT_SAVE_TIMEOUT (seconds). +try: + _SLOT_SAVE_HTTP_TIMEOUT = float(os.environ.get("UNSLOTH_SLOT_SAVE_TIMEOUT") or 30.0) +except ValueError: + _SLOT_SAVE_HTTP_TIMEOUT = 30.0 +if _SLOT_SAVE_HTTP_TIMEOUT <= 0: + _SLOT_SAVE_HTTP_TIMEOUT = 30.0 + + def _swa_cache_path() -> Path: home = os.environ.get("UNSLOTH_STUDIO_HOME") or os.environ.get("STUDIO_HOME") base = Path(home) if home else Path.home() / ".unsloth" / "studio" @@ -496,7 +579,14 @@ def _swa_entry_from_layer_types(lt) -> Optional[object]: def _fetch_swa_entry_from_hf(repo_id: str) -> Optional[object]: try: from huggingface_hub import hf_hub_download - cfg_path = hf_hub_download(repo_id, "config.json", repo_type = "model") + from utils.hf_cache_settings import active_hf_hub_cache + + cfg_path = hf_hub_download( + repo_id, + "config.json", + repo_type = "model", + cache_dir = active_hf_hub_cache(), + ) with open(cfg_path) as f: cfg = json.load(f) except Exception: @@ -898,6 +988,7 @@ def _cached_hf_snapshot_file( filename: str, *, expected_size: Optional[int] = None, + cache_dir: Optional[str] = None, ) -> Optional[str]: """Return a cached snapshot file even when HF's current-ref probe misses it.""" if not filename: @@ -906,8 +997,22 @@ def _cached_hf_snapshot_file( if not parts or any(part in (".", "..") for part in parts): return None try: - from utils.models.model_config import _iter_hf_cache_snapshots - for snap in _iter_hf_cache_snapshots(repo_id): + if cache_dir is None: + from utils.models.model_config import _iter_hf_cache_snapshots + snapshots = _iter_hf_cache_snapshots(repo_id) + else: + from hub.utils.hf_cache_state import iter_active_repo_cache_dirs + snapshots = ( + snapshot + for repo_dir in iter_active_repo_cache_dirs( + "model", + repo_id, + root = Path(cache_dir), + ) + for snapshot in (repo_dir / "snapshots").glob("*") + if snapshot.is_dir() + ) + for snap in snapshots: candidate = snap.joinpath(*parts) if not candidate.is_file(): continue @@ -999,6 +1104,293 @@ def _cached_colocated_split_main( return None +def _cached_variant_resolution(repo_id: str, hf_variant: str) -> tuple[Optional[str], list[str]]: + """Find a cached main GGUF and its shards for a variant.""" + candidate = next(_cached_variant_candidates(repo_id, hf_variant), None) + if candidate is None: + return None, [] + _, main, shards, _ = candidate + return main, shards + + +def _cached_variant_candidates( + repo_id: str, + hf_variant: str, + *, + require_mmproj: bool = False, +) -> Generator[tuple[str, str, list[str], Path], None, None]: + """Yield complete cached variant copies in snapshot preference order.""" + try: + from utils.models.model_config import _iter_hf_cache_snapshots + for snap in _iter_hf_cache_snapshots(repo_id): + cached_files = _gguf_snapshot_files(snap) + matches = _gguf_files_for_variant(cached_files, hf_variant) + if not matches: + continue + main = matches[0] + shards = _gguf_extra_shards(matches, main) + split = _SHARD_FULL_RE.match(main) + if split: + numbers = { + int(match.group(2)) + for path in [main, *shards] + if (match := _SHARD_FULL_RE.match(path)) + } + if numbers != set(range(1, int(split.group(3)) + 1)): + continue + main_path = snap.joinpath(*main.replace("\\", "/").split("/")) + if not main_path.is_file() or not _snapshot_has_all_shards( + str(main_path), main, shards, {} + ): + continue + if require_mmproj and not _pick_mmproj(cached_files): + continue + yield str(main_path), main, shards, snap + except Exception as e: + logger.debug(f"Cache lookup for variant failed: {e}") + + +def _cached_candidate_matches_revision_size( + repo_id: str, candidate: tuple[str, str, list[str], Path], hf_token: Optional[str] +) -> bool: + """Check cached byte sizes against the snapshot's own Hub revision. + + A snapshot pointer is normally published only after its blob is complete. + When the old revision is still queryable, also compare every weight file's + size so a manually truncated cache entry is not treated as reusable. If + metadata cannot be reached, retain the cache's normal offline semantics. + """ + main_path, main, shards, snap = candidate + paths = [main, *shards] + try: + from huggingface_hub import get_paths_info + infos = list( + get_paths_info( + repo_id, + paths, + revision = snap.name, + token = hf_token, + ) + ) + except Exception as e: + logger.debug( + "Could not size-check cached GGUF %s at revision %s: %s", + repo_id, + snap.name, + e, + ) + return True + + if not infos: + # The Hub answers an unknown (e.g. force-pushed away) revision with an + # empty result, not an error; treat it like unreachable metadata. + return True + expected_sizes = {info.path: info.size for info in infos if info.size is not None} + if any(path not in expected_sizes for path in paths): + return False + try: + if os.path.getsize(main_path) < expected_sizes[main]: + return False + except OSError: + return False + return _snapshot_has_all_shards(main_path, main, shards, expected_sizes) + + +def _cached_complete_candidate( + repo_id: str, gguf_filename: Optional[str], shards: list[str] +) -> Optional[tuple[str, str, list[str], Path]]: + """Return one complete exact-filename cache candidate with snapshot context.""" + if not gguf_filename: + return None + if shards: + main_path = _cached_colocated_split_main(repo_id, gguf_filename, shards, {}) + else: + m = _SHARD_FULL_RE.match(gguf_filename) + if m and int(m.group(3)) > 1: + return None + main_path = _cached_hf_snapshot_file(repo_id, gguf_filename) + if main_path is None: + return None + snap = _snapshot_dir_of(main_path) + if snap is None: + return None + return main_path, gguf_filename, shards, snap + + +def cached_gguf_for_load( + hf_repo: str, + hf_variant: Optional[str], + *, + require_mmproj: bool = False, + verify_sizes: bool = False, + hf_token: Optional[str] = None, +) -> Optional[str]: + """Return a cached GGUF that can be loaded without downloading.""" + if not hf_variant: + return None + hf_repo = _resolve_repo_id_casing(hf_repo) + for candidate in _cached_variant_candidates( + hf_repo, + hf_variant, + require_mmproj = require_mmproj, + ): + if verify_sizes and not _cached_candidate_matches_revision_size( + hf_repo, candidate, hf_token + ): + continue + return candidate[0] + return None + + +def _snapshot_dir_of(path: str) -> Optional[Path]: + """Return the HF cache snapshot containing path, if any.""" + try: + p = Path(os.path.abspath(path)) + except OSError: + return None + for ancestor in p.parents: + if ancestor.parent.name == "snapshots": + return ancestor + return None + + +def _hub_cache_dir_for_snapshot_path(path: Optional[str]) -> Optional[str]: + """Return the HF Hub cache root that owns a snapshot-contained path.""" + if not path: + return None + snapshot = _snapshot_dir_of(path) + if snapshot is None or snapshot.parent.name != "snapshots": + return None + return str(snapshot.parent.parent.parent) + + +def _companion_snapshot_sibling( + near_path: str, pick: Callable[[list[str]], Optional[str]] +) -> Optional[str]: + """Find a companion in the same snapshot as near_path.""" + snap = _snapshot_dir_of(near_path) + if snap is None: + return None + try: + sibling = pick(_gguf_snapshot_files(snap)) + except Exception: + return None + if not sibling: + return None + candidate = snap / sibling + return str(candidate) if candidate.is_file() else None + + +def _pick_mmproj(candidates: list[str]) -> Optional[str]: + mmproj_files = sorted( + f for f in candidates if f.lower().endswith(".gguf") and "mmproj" in Path(f).name.lower() + ) + if not mmproj_files: + return None + return next((f for f in mmproj_files if f.lower().endswith("-f16.gguf")), mmproj_files[0]) + + +def _hub_download_in_flight(hf_repo: str) -> bool: + try: + from hub.utils.download_registry import get_models_registry + return bool(get_models_registry().active_job_refs(hf_repo)) + except Exception: + return False + + +def _hub_download_blocks_gguf_load( + hf_repo: str, + hf_variant: Optional[str], + *, + require_mmproj: bool = False, + hf_token: Optional[str] = None, +) -> bool: + """Whether an active Hub job makes this GGUF load unsafe. + + Same-variant jobs can reclaim the stale snapshot a load would reuse, so + they always block. Other jobs block only when this load lacks a complete + cached copy and would write to the shared cache itself. + """ + try: + from hub.utils.download_registry import get_models_registry + + registry = get_models_registry() + if not registry.active_job_refs(hf_repo): + return False + if registry.has_active_variant(hf_repo, hf_variant): + return True + except Exception: + return False + return ( + cached_gguf_for_load( + hf_repo, + hf_variant, + require_mmproj = require_mmproj, + verify_sizes = True, + hf_token = hf_token, + ) + is None + ) + + +# Active GGUF loads by normalized repo ID. +_LOADS_IN_FLIGHT: dict[str, int] = {} +_LOADS_IN_FLIGHT_LOCK = threading.Lock() + + +@contextlib.contextmanager +def gguf_load_in_flight(hf_repo: Optional[str]): + """Track an HF GGUF load until the context exits.""" + key = (hf_repo or "").strip().lower() + if not key: + yield + return + with _LOADS_IN_FLIGHT_LOCK: + _LOADS_IN_FLIGHT[key] = _LOADS_IN_FLIGHT.get(key, 0) + 1 + try: + yield + finally: + with _LOADS_IN_FLIGHT_LOCK: + remaining = _LOADS_IN_FLIGHT.get(key, 1) - 1 + if remaining <= 0: + _LOADS_IN_FLIGHT.pop(key, None) + else: + _LOADS_IN_FLIGHT[key] = remaining + + +def hf_gguf_load_in_flight(hf_repo: str) -> bool: + """Return whether a GGUF load is active for hf_repo.""" + key = (hf_repo or "").strip().lower() + if not key: + return False + with _LOADS_IN_FLIGHT_LOCK: + return _LOADS_IN_FLIGHT.get(key, 0) > 0 + + +def _with_gguf_load_marker(load: Callable): + """Keep an HF repo marked for the full synchronous load call.""" + + @functools.wraps(load) + def wrapped(self, *args, **kwargs): + hf_repo = kwargs.get("hf_repo") + with gguf_load_in_flight(hf_repo): + if hf_repo and _hub_download_blocks_gguf_load( + hf_repo, + kwargs.get("hf_variant"), + require_mmproj = bool( + kwargs.get("is_vision") + and not extra_args_disable_mmproj(kwargs.get("extra_args")) + ), + hf_token = kwargs.get("hf_token"), + ): + raise RuntimeError( + f"'{hf_repo}' is currently being downloaded by the download manager" + ) + return load(self, *args, **kwargs) + + return wrapped + + def _gguf_extra_shards(files: Iterable[str], first_shard: str) -> list[str]: m = _SHARD_FULL_RE.match(first_shard) if not m: @@ -1085,7 +1477,7 @@ def _kv_bytes_per_elem(cache_type: Optional[str]) -> float: def _env_main_cache_type_for_budget(env: Optional[Mapping[str, str]] = None) -> Optional[str]: """Heavier of the inherited LLAMA_ARG_CACHE_TYPE_K/_V env types when it - exceeds the f16 default, else None. Studio emits --cache-type only for the + exceeds the f16 default, else None. Unsloth emits --cache-type only for the param/extras path, so a heavier env (f32) would otherwise reach the child unbudgeted; quantized env types stay over-reserved by f16 (-> None).""" e = os.environ if env is None else env @@ -1153,7 +1545,10 @@ def _extra_args_set_spec_type(extra_args: Optional[Iterable[str]]) -> bool: return _extra_args_set_any_flag(extra_args, {"--spec-type", "--spec-default"}) -_GPU_OFFLOAD_OVERRIDE_FLAGS = frozenset({"-ngl", "--gpu-layers", "--n-gpu-layers", "-fit", "--fit"}) +# Layer-offload override detection. Single-sourced from llama_server_args, which +# also strips these (plus the MoE flags) from inherited extras; sharing the layer +# set keeps detection and stripping from drifting. +_GPU_OFFLOAD_OVERRIDE_FLAGS = _LAYER_OFFLOAD_FLAGS _THREAD_OVERRIDE_FLAGS = frozenset({"-t", "--threads"}) @@ -1404,7 +1799,7 @@ def _build_ngram_mod_flags( return [] -# Canonical Speculative Decoding modes exposed by the Studio chat UI. +# Canonical Speculative Decoding modes exposed by the Unsloth chat UI. # Dropdown renders five (auto, mtp, ngram, mtp+ngram, off); the load API # also accepts legacy values the original Switch and external callers emit # (default, draft-mtp, ngram-mod, ngram-simple). @@ -1453,7 +1848,7 @@ def _backfill_usage_from_timings(usage, timings): """Synthesize ``usage`` from llama-server's ``timings`` when the OpenAI-style usage block is missing or reports zero tokens. - The Studio chat UI computes generation t/s from + The Unsloth chat UI computes generation t/s from ``meta.usage.completion_tokens / totalStreamTime``. llama-server always populates ``timings.predicted_n`` (true decoded count) and ``timings.prompt_n``, but the final SSE chunk's ``usage`` can be absent @@ -1526,7 +1921,7 @@ def _llama_lib_dir(binary: str) -> Path: def _is_external_link(path: Path) -> bool: """True when ``path`` is a --with-llama-cpp-dir local link: a POSIX symlink or a Windows directory junction / reparse point. Such a link resolves into - the user's own llama.cpp checkout, which Studio does not own.""" + the user's own llama.cpp checkout, which Unsloth does not own.""" try: if os.path.islink(path): return True @@ -1617,6 +2012,21 @@ class LlamaCppBackend: self._cache_type_kv: Optional[str] = None # Whether --split-mode tensor was applied on the active load. self._tensor_parallel: bool = False + # GPU memory strategy applied on the active load ("auto"/"manual"). + self._gpu_memory_mode: str = "auto" + # Manual-mode load options (echoed back so the UI round-trips them). + self._gpu_layers: int = -1 + # MoE expert layers to keep on CPU (--n-cpu-moe); 0 = none. + self._n_cpu_moe: int = 0 + # Relative model share per GPU (--tensor-split), in GPU order; None = + # default (llama.cpp splits by free VRAM). + self._tensor_split: Optional[List[float]] = None + # User-picked physical GPU indices (None = automatic selection). + self._gpu_ids: Optional[List[int]] = None + # RAW requested GPU pin, before the fit narrowed it. self._gpu_ids records the + # EFFECTIVE (fit-narrowed) pin for /status; dedupe compares this raw value so a + # [0, 1] narrowed to [0] and re-sent as [0, 1] still matches (#7239). + self._requested_gpu_ids: Optional[List[int]] = None # Layer load kept multi-GPU only to honor a downgraded tensor request, so a # later explicit tensor-off reloads instead of deduping to it (#6659). self._layer_preserves_tensor_intent: bool = False @@ -1631,6 +2041,11 @@ class LlamaCppBackend: self._spec_draft_n_max: Optional[int] = None # KV-cache estimation fields (populated by _read_gguf_metadata) self._n_layers: Optional[int] = None + # MoE metadata (populated by _read_gguf_metadata): expert count (>0 = + # MoE) and leading dense-layer count (offsets --n-cpu-moe, which counts + # from layer 0). See the n_moe_layers property. + self._n_experts: Optional[int] = None + self._leading_dense_block_count: Optional[int] = None self._n_kv_heads: Optional[int] = None self._n_kv_heads_by_layer: Optional[list[int]] = None self._n_heads: Optional[int] = None @@ -1682,7 +2097,7 @@ class LlamaCppBackend: # observes it (direct proxy endpoints, or nothing in flight). self._mtp_watchdog_thread: Optional[threading.Thread] = None self._mtp_watchdog_stop = threading.Event() - # True when the launch actually runs MTP+tensor (Studio- or user/env-driven); + # True when the launch actually runs MTP+tensor (Unsloth- or user/env-driven); # gates the probe, watchdog, and recovery so pass-through MTP is covered. self._mtp_runtime_fallback_active = False self._stdout_lines: list[str] = [] @@ -1692,6 +2107,12 @@ class LlamaCppBackend: self._llama_log_path: Optional[Path] = None self._cancel_event = threading.Event() self._api_key: Optional[str] = None + self._slot_save_dir: Optional[str] = None + self._slot_save_binary: Optional[tuple[str, int]] = None + # (gguf_identity, launch_fingerprint) snapshotted at load, so a later slot + # save can tell whether the model files were swapped on disk since load. + self._slot_loaded_identity: Optional[tuple] = None + self._prompt_cache_disabled: bool = False # True once a probe has completed; cleared on transient failure. self._is_audio: bool = False self._audio_type: Optional[str] = None @@ -2051,6 +2472,119 @@ class LlamaCppBackend: """Whether --split-mode tensor is active on the loaded server.""" return self._tensor_parallel + @property + def gpu_memory_mode(self) -> str: + """Active GPU memory strategy: 'auto' or 'manual' (gpu_layers < 0 = Auto/--fit, >= 0 = pinned).""" + return self._gpu_memory_mode + + @property + def gpu_layers(self) -> int: + """Requested --gpu-layers for manual mode (-1 when not manual).""" + return self._gpu_layers + + @property + def n_cpu_moe(self) -> int: + """MoE expert layers manual mode kept on CPU (--n-cpu-moe); 0 = none.""" + return self._n_cpu_moe + + @property + def tensor_split(self) -> Optional[List[float]]: + """Manual-mode relative model share per GPU (--tensor-split); None = + default (split by free VRAM).""" + return self._tensor_split + + @property + def gpu_ids(self) -> Optional[List[int]]: + """User-picked physical GPU indices, or None for automatic selection.""" + return self._gpu_ids + + @property + def requested_gpu_ids(self) -> Optional[List[int]]: + """RAW requested GPU pin (before the fit narrowed it), or None for auto. + gpu_ids echoes the EFFECTIVE pin for /status.""" + return self._requested_gpu_ids + + def matches_gpu_ids(self, gpu_ids: Optional[List[int]]) -> bool: + """Whether a requested pin is already satisfied by the active runner. + + A regular GGUF load may narrow the requested placement pool to the + smallest fitting subset. Accept both the original request and the + effective status-echoed subset so either can round-trip without a + needless reload. Diffusion drives one device and keeps its existing + lowest-device normalization. + """ + if self._is_diffusion: + requested = [sorted(int(x) for x in gpu_ids)[0]] if gpu_ids else None + return requested == (self._gpu_ids or None) + + requested = sorted(int(x) for x in gpu_ids) if gpu_ids else None + raw = self._requested_gpu_ids or None + effective = self._gpu_ids or None + return requested == raw or requested == effective + + def _record_matching_gpu_request(self, gpu_ids: Optional[List[int]]) -> None: + """Adopt the caller's explicit pool after a full already-loaded match. + + Matching an effective subset avoids a reload, but the incoming request + is still the user's latest placement intent. Record it so status and a + later reload do not restore GPUs the user just removed. + """ + if self._is_diffusion: + self._requested_gpu_ids = [sorted(int(x) for x in gpu_ids)[0]] if gpu_ids else None + else: + self._requested_gpu_ids = sorted(int(x) for x in gpu_ids) if gpu_ids else None + if self._last_load_kwargs is not None: + self._last_load_kwargs["gpu_ids"] = ( + list(self._requested_gpu_ids) if self._requested_gpu_ids else None + ) + + @property + def n_layers(self) -> Optional[int]: + """Model layer count (GGUF block_count), or None if unknown.""" + return self._n_layers + + @property + def n_moe_layers(self) -> int: + """Number of MoE expert layers (the --n-cpu-moe ceiling), 0 if not MoE. + + block_count minus the leading dense layers (which carry no experts): + --n-cpu-moe counts from layer 0, so those dense layers are no-ops. + """ + if not self._n_experts or not self._n_layers: + return 0 + return max(0, self._n_layers - (self._leading_dense_block_count or 0)) + + @staticmethod + def _resolve_cpu_moe_flag( + n_cpu_moe: int, n_moe_layers: int, leading_dense: int + ) -> Optional[int]: + """The --n-cpu-moe value (absolute first-N layers), or None to omit it. + + Clamps the requested count to the model's MoE layers, then offsets past + the leading dense layers (--n-cpu-moe counts from layer 0). Returns None + for nothing-to-offload (0 requested) or a non-MoE model. + """ + if n_cpu_moe <= 0 or n_moe_layers <= 0: + return None + return leading_dense + min(n_cpu_moe, n_moe_layers) + + @staticmethod + def _sanitize_tensor_split(tensor_split: Optional[List[float]]) -> List[float]: + """Per-GPU shares with negative and non-finite entries clamped to 0. + + A direct caller's negative entry would launch a placement different + from the ratio the UI showed, and inf would pass a plain ``> 0`` total + gate and emit ``--tensor-split inf,...``. Returns [] for input that + can't be read as floats (the length gate at the call site then drops + the split). + """ + try: + return [ + x if math.isfinite(x) and x > 0.0 else 0.0 for x in (float(v) for v in tensor_split) + ] + except (TypeError, ValueError, OverflowError): + return [] + @property def layer_preserves_tensor_intent(self) -> bool: """True when a downgraded tensor request kept this layer load multi-GPU.""" @@ -2075,7 +2609,7 @@ class LlamaCppBackend: @staticmethod def _resolved_studio_root_and_is_legacy() -> "tuple[Optional[Path], bool]": - """Resolve the Studio install root and classify it as the legacy + """Resolve the Unsloth install root and classify it as the legacy ~/.unsloth/studio root vs. a custom (env/venv-inferred) root. Returns (resolved_root, is_legacy). On any import/resolution failure the @@ -2247,15 +2781,18 @@ class LlamaCppBackend: "found": False, "mtp_token": None, "supports_mtp": False, + "mtp_probe_inconclusive": True, "ngram_mod_flavor": None, "supports_ngram_mod": False, "spec_draft_n_max_flag": None, "supports_kv_unified": False, "supports_fit_ctx": False, + "supports_fit_target": False, "supports_cache_ram": False, "supports_ctx_checkpoints": False, "supports_no_cache_prompt": False, "supports_metrics": False, + "supports_slot_save": False, } try: mtime = int(Path(bin_path).stat().st_mtime) @@ -2271,10 +2808,15 @@ class LlamaCppBackend: spec_draft_n_max_flag: Optional[str] = None supports_kv_unified = False supports_fit_ctx = False + supports_fit_target = False supports_cache_ram = False supports_ctx_checkpoints = False supports_no_cache_prompt = False supports_metrics = False + supports_slot_save = False + saw_spec_type = False + probe_ok = False + help_text = "" try: probe_env = cls._llama_server_env_for_binary(bin_path) result = subprocess.run( @@ -2286,6 +2828,7 @@ class LlamaCppBackend: check = False, env = probe_env, ) + probe_ok = result.returncode == 0 help_text = (result.stdout or "") + "\n" + (result.stderr or "") # Split into per-flag blocks (each --flag line + its indented # continuation), so the "argument has been removed" description @@ -2330,17 +2873,19 @@ class LlamaCppBackend: return False return "argument has been removed" not in desc - # MTP token from the --spec-type line. - spec_line = "" - for line in help_text.splitlines(): - if "--spec-type" in line: - spec_line = line - break - # PR #22673 used draft-mtp; later renamed to mtp. - if "draft-mtp" in spec_line: - mtp_token = "draft-mtp" - elif re.search(r"[|,\[]mtp[|,\]]", spec_line): - mtp_token = "mtp" + # MTP token from the full --spec-type help block (decl + indented + # continuation). First-line-only probing missed builds putting the + # enum on the next line (#7302). Prefer draft-mtp (PR #22673) over mtp. + spec_help = blocks.get("--spec-type") or "" + if not spec_help: + # Fallback: join --spec-type lines, avoiding incidental "mtp" in --help. + spec_help = "\n".join( + line for line in help_text.splitlines() if "--spec-type" in line + ) + mtp_token = cls._mtp_token_from_spec_help(spec_help) + # Only a resolved --spec-type block confirms missing MTP; empty/crash + # leaves saw_spec_type False so supports_mtp fails open. + saw_spec_type = bool(spec_help.strip()) and "--spec-type" in spec_help # ngram-mod flag flavor. Post-rename builds advertise both new # args (real) and legacy ones (stubs); pre-rename builds only @@ -2368,30 +2913,67 @@ class LlamaCppBackend: supports_kv_unified = _is_real("--kv-unified") supports_fit_ctx = _is_real("--fit-ctx") + supports_fit_target = _is_real("--fit-target") supports_cache_ram = _is_real("--cache-ram") supports_ctx_checkpoints = _is_real("--ctx-checkpoints") supports_no_cache_prompt = _is_real("--no-cache-prompt") supports_metrics = _is_real("--metrics") + supports_slot_save = _is_real("--slot-save-path") except (OSError, subprocess.SubprocessError) as exc: logger.debug(f"llama-server --help probe failed: {exc}") + saw_spec_type = False + probe_ok = False + help_text = "" + + help_nonempty = bool(help_text.strip()) + # Confirmed only when a successful --help lists a --spec-type block with + # mtp/draft-mtp; nonempty --help without it is a definitive pre-spec + # binary; failed/empty probes stay inconclusive (#7302). + if saw_spec_type and probe_ok: + supports_mtp = mtp_token is not None + mtp_probe_inconclusive = False + elif help_nonempty and probe_ok: + supports_mtp = False + mtp_probe_inconclusive = False + else: + supports_mtp = False + mtp_probe_inconclusive = True info = { "found": True, "mtp_token": mtp_token, - "supports_mtp": mtp_token is not None, + "supports_mtp": supports_mtp, + "mtp_probe_inconclusive": mtp_probe_inconclusive, "ngram_mod_flavor": ngram_mod_flavor, "supports_ngram_mod": ngram_mod_flavor is not None, "spec_draft_n_max_flag": spec_draft_n_max_flag, "supports_kv_unified": supports_kv_unified, "supports_fit_ctx": supports_fit_ctx, + "supports_fit_target": supports_fit_target, "supports_cache_ram": supports_cache_ram, "supports_ctx_checkpoints": supports_ctx_checkpoints, "supports_no_cache_prompt": supports_no_cache_prompt, "supports_metrics": supports_metrics, + "supports_slot_save": supports_slot_save, } cls._capability_cache[cache_key] = info return info + @staticmethod + def _mtp_token_from_spec_help(spec_help: str) -> Optional[str]: + """Extract ``draft-mtp`` / ``mtp`` from a ``--spec-type`` help snippet. + + Prefers ``draft-mtp`` (llama.cpp PR #22673) over the later bare ``mtp`` + rename. Returns ``None`` when neither token appears as an enum value. + """ + text = spec_help or "" + if "draft-mtp" in text: + return "draft-mtp" + # Bare `mtp` enum token (`|mtp|`, `,mtp,`, ...), not a substring. + if re.search(r"(?physical mapping.""" try: import torch - is_rocm = getattr(torch.version, "hip", None) is not None + + # Same ROCm detection as _emit_child_gpu_visibility: AMD SDK wheels + # leave version.hip unset but encode "rocm" in __version__. The two + # must agree, else an inherited ROCR mask reads back as "no mask", + # ordinal 0 is labelled physical 0, and the child's new ROCR pin + # re-exposes the GPU the inherited mask was hiding. + is_rocm = ( + getattr(torch.version, "hip", None) is not None + or "rocm" in getattr(torch, "__version__", "").lower() + ) except Exception: is_rocm = False if is_rocm: hip_v = os.environ.get("HIP_VISIBLE_DEVICES") - rocr_v = os.environ.get("ROCR_VISIBLE_DEVICES") + # ROCR_VISIBLE_DEVICES is a Linux ROCr variable; Windows HIP has no + # ROCr layer, so a stray ROCR var there does not mask the runtime and + # must not be read as the ordinal->physical mapping (mirrors the + # Windows gate in _emit_child_gpu_visibility). + rocr_v = None if sys.platform == "win32" else os.environ.get("ROCR_VISIBLE_DEVICES") cvd = ( hip_v if hip_v is not None @@ -2468,6 +3063,103 @@ class LlamaCppBackend: except ValueError: return None + @staticmethod + def _emit_child_gpu_visibility( + env: dict, + pinned: str, + *, + prefer_rocr: bool = False, + ) -> None: + """Write the child's GPU visibility mask: CUDA, plus a ROCm mirror on AMD + (masking only CUDA_VISIBLE_DEVICES leaves an AMD child seeing every GPU). + + Default: HIP_VISIBLE_DEVICES, clearing any inherited ROCR mask so the two + can't stack (ROCR re-indexes from 0, then a non-zero HIP pin points out of + range, HIP sees 0 devices, and llama.cpp falls back to CPU). + + prefer_rocr masks at the ROCr/HSA layer instead (clearing HIP). A HIP mask + filters only AFTER the HSA runtime enumerates every agent, and that + enumeration segfaults at startup on a GPU the build has no kernels for + (e.g. a gfx1103 iGPU under a gfx110X prebuilt), before llama-server logs a + line. ROCR drops the device at the driver layer, consuming physical ids. + The CPU-only sentinel ("-1") has no portable ROCR spelling, so it keeps + the HIP mask. Windows keeps the HIP mask too: ROCR_VISIBLE_DEVICES is a + Linux ROCr variable (Windows HIP has no ROCr layer), so the ROCR pin + would be dead there while the cleared HIP mask stops selecting.""" + env["CUDA_VISIBLE_DEVICES"] = pinned + try: + import torch as _torch + + # torch.version.hip is set on ROCm, None on CUDA; AMD SDK wheels may + # leave it unset but encode "rocm" in __version__ (mirrors detect_hardware). + if ( + getattr(_torch.version, "hip", None) is not None + or "rocm" in getattr(_torch, "__version__", "").lower() + ): + if prefer_rocr and pinned != "-1" and sys.platform != "win32": + env["ROCR_VISIBLE_DEVICES"] = pinned + env.pop("HIP_VISIBLE_DEVICES", None) + # ROCR re-indexes the visible agents from 0, and with HIP + # cleared HIP honours CUDA_VISIBLE_DEVICES -- so it must carry + # the post-ROCR ordinals (0..N-1), not the physical ids, else a + # non-zero pick points out of range and HIP sees 0 devices (the + # same stacking the default path avoids by clearing ROCR). + env["CUDA_VISIBLE_DEVICES"] = ",".join( + str(i) for i in range(len(pinned.split(","))) + ) + else: + env["HIP_VISIBLE_DEVICES"] = pinned + env.pop("ROCR_VISIBLE_DEVICES", None) + except Exception as e: + logger.debug("Failed to set ROCm visibility env vars for child: %s", e) + + @staticmethod + def _pin_visible_gpu_order_for_split(env: dict) -> None: + """Pin the child's GPU enumeration to the picker's order for a manual + ``--tensor-split`` across the whole visible set. CUDA's default + FASTEST_FIRST enumeration applies the shares to the wrong cards on + heterogeneous hosts (#5025), and CUDA_DEVICE_ORDER only fixes the + numbering base: an inherited numeric visibility mask ALSO defines + enumeration order, so a reordered parent mask (CUDA_VISIBLE_DEVICES=3,1) + would still hand the shares to the wrong cards. The UI built the split + positionally over get_backend_visible_gpu_info's device list (ascending + physical via nvidia-smi, inherited mask order on the torch fallback), so + re-emit the same set in that report order -- not an assumed ascending + sort. The visible set itself never changes. No mask, an empty mask, or a + UUID/MIG mask (which resolves to None) is left alone -- the multi-GPU + controls are hidden for the latter.""" + env["CUDA_DEVICE_ORDER"] = "PCI_BUS_ID" + inherited = LlamaCppBackend._resolve_visible_physical_ids() + if not inherited: + return + order = None + try: + from utils.hardware import get_backend_visible_gpu_info + info = get_backend_visible_gpu_info() + if info.get("available") and info.get("index_kind") == "physical": + reported = [d["index"] for d in info.get("devices", [])] + if sorted(reported) == sorted(inherited): + order = reported + except Exception as e: + logger.debug("Could not read reported GPU order for split pin: %s", e) + if order is None: + order = sorted(inherited) + # Re-emit at the layer that produced the mapping. A parent masked only + # via ROCR_VISIBLE_DEVICES hides agents at the driver layer, and the + # default HIP re-emission clears that mask -- HSA then enumerates every + # agent again and can segfault at startup on an unsupported GPU the + # parent was hiding (the crash prefer_rocr exists to avoid). Linux-only, + # mirroring _resolve_visible_physical_ids: on Windows a stray ROCR var + # is dead and was not the mapping's source. + prefer_rocr = ( + sys.platform != "win32" + and env.get("HIP_VISIBLE_DEVICES") is None + and env.get("ROCR_VISIBLE_DEVICES") is not None + ) + LlamaCppBackend._emit_child_gpu_visibility( + env, ",".join(str(i) for i in order), prefer_rocr = prefer_rocr + ) + @staticmethod def _amd_apu_wants_unified_memory(gpu_indices = None) -> bool: """True only for AMD unified-memory APUs (gfx1150/gfx1151), where @@ -2963,7 +3655,7 @@ class LlamaCppBackend: return prev = curr - # Free-VRAM fraction at which Studio pins the GPU directly instead of + # Free-VRAM fraction at which Unsloth pins the GPU directly instead of # deferring to ``--fit on``. 3% headroom: the compute buffer is now modelled in # the fit, so this only guards fragmentation + multi-GPU per-device CUDA context # (~2-3%); kept >= 3% as a floor (0.90 dropped 91-94% fits to CPU offload, #5106). @@ -2984,6 +3676,20 @@ class LlamaCppBackend: # aborts a --split-mode tensor load, so it's dropped for the tensor attempt. _TENSOR_PARALLEL_KV_TYPES = frozenset({"f16", "bf16", "f32"}) + # Main-model placement settings that Manual mode owns. They must not leak + # from Studio's parent environment into llama-server and silently override + # the command assembled from the current request. Draft-model placement is + # intentionally separate and remains available to speculative decoding. + _MANUAL_PLACEMENT_ENV_VARS = ( + "LLAMA_ARG_CPU_MOE", + "LLAMA_ARG_N_CPU_MOE", + "LLAMA_ARG_N_GPU_LAYERS", + "LLAMA_ARG_TENSOR_SPLIT", + "LLAMA_ARG_FIT", + "LLAMA_ARG_FIT_TARGET", + "LLAMA_ARG_FIT_CTX", + ) + # (binary, mtime, model) that aborted on --split-mode tensor this process (#6415 # geometry limit, e.g. MQA n_head_kv=1). Model-keyed so one model's abort doesn't # skip tensor for others; tensor is tried by default, recorded only on a real abort. @@ -3114,6 +3820,9 @@ class LlamaCppBackend: lib_dirs.extend(_wsl_system_rocm_lib_dirs()) if lib_dirs: env.setdefault("HSA_ENABLE_DXG_DETECTION", "1") + # Native Linux AMD: system ROCm libs before the bundle's HIP runtime, + # which can be incompatible with the host amdkfd driver. + lib_dirs.extend(_native_linux_system_rocm_lib_dirs(binary_dir)) lib_dirs.append(binary_dir) _arch = platform.machine() # x86_64, aarch64, etc. @@ -3148,6 +3857,12 @@ class LlamaCppBackend: return env + @classmethod + def _clear_manual_placement_env(cls, env: dict[str, str]) -> None: + """Remove inherited main-model placement owned by Manual mode.""" + for name in cls._MANUAL_PLACEMENT_ENV_VARS: + env.pop(name, None) + @staticmethod def _select_gpus( model_size_bytes: int, @@ -3522,7 +4237,7 @@ class LlamaCppBackend: return total if total > 0 else None return draft_kv + weights + target_ctx_copy - _DEFAULT_N_UBATCH = 512 # llama.cpp --ubatch default; Studio does not override it + _DEFAULT_N_UBATCH = 512 # llama.cpp --ubatch default; Unsloth does not override it _COMPUTE_BUFFER_SAFETY = 1.15 # upper-bound margin on the compute-buffer estimate # Soft VRAM the modeled terms omit; charged to the fit budget on tight tiers (#6682). _CUDA_CONTEXT_RESERVE_BYTES = 320 * 1024 * 1024 # CUDA ctx + cuBLAS workspace (~330 MiB) @@ -3662,7 +4377,7 @@ class LlamaCppBackend: n_ubatch: Optional[int] = None, ) -> tuple[Optional[list[int]], bool, int]: """Largest serving-slot count in [1, n_parallel) whose fully-on-GPU footprint fits, - so Studio keeps the model on GPU (-ngl -1) instead of --fit on, which offloads layers + so Unsloth keeps the model on GPU (-ngl -1) instead of --fit on, which offloads layers to host and collapses decode ~3x (oobabooga #6718). ``base_footprint_bytes`` is the slot-independent footprint (weights + soft overhead + MTP + context-linear compute, minus the folded compute buffer); each candidate re-adds the slot-sized compute buffer @@ -3806,13 +4521,13 @@ class LlamaCppBackend: hf_repo: str, free_bytes: int, hf_token: Optional[str] = None, - ) -> Optional[tuple[str, int]]: + ) -> Optional[tuple[str, int, list[str]]]: """Find the smallest GGUF variant (including all shards) that fits. Groups split shards by variant prefix and sums their sizes (e.g. UD-Q4_K_XL with 9 shards of 50 GB each = 450 GB total). - Returns (first_shard_filename, total_size_bytes) or None. + Returns (first_shard_filename, total_size_bytes, extra_shards) or None. """ try: from huggingface_hub import get_paths_info, list_repo_files @@ -3848,9 +4563,13 @@ class LlamaCppBackend: # Smallest that fits variant_sizes.sort(key = lambda x: x[1]) - for first_file, total_size, _ in variant_sizes: + for first_file, total_size, shard_files in variant_sizes: if total_size > 0 and total_size <= free_bytes: - return first_file, total_size + return ( + first_file, + total_size, + [path for path in sorted(shard_files) if path != first_file], + ) return None except Exception: @@ -3946,6 +4665,14 @@ class LlamaCppBackend: LlamaCppBackend._gguf_skip_value(f, atype) return None + @classmethod + def _gguf_path_is_diffusion(cls, gguf_path: str, model_identifier: str) -> bool: + """Classify a downloaded GGUF without mutating the active backend.""" + probe = object.__new__(cls) + probe._model_identifier = model_identifier + probe._read_gguf_metadata(gguf_path) + return probe._is_diffusion + def _read_gguf_metadata(self, gguf_path: str) -> None: """Read context_length, architecture params, and chat_template from a GGUF header. @@ -3964,6 +4691,8 @@ class LlamaCppBackend: self._supports_preserve_thinking = False self._supports_tools = False self._n_layers = None + self._n_experts = None + self._leading_dense_block_count = None self._n_kv_heads = None self._n_kv_heads_by_layer = None self._n_heads = None @@ -4053,6 +4782,8 @@ class LlamaCppBackend: arch_keys = { f"{arch}.context_length": "context_length", f"{arch}.block_count": "n_layers", + f"{arch}.expert_count": "n_experts", + f"{arch}.leading_dense_block_count": "leading_dense_block_count", f"{arch}.attention.head_count_kv": "n_kv_heads", f"{arch}.attention.head_count": "n_heads", f"{arch}.embedding_length": "embedding_length", @@ -4134,7 +4865,7 @@ class LlamaCppBackend: ] # Otherwise hand off to the resolver (cache / bootstrap / transformers / HF). Diffusion models - # skip it: they do not use Studio's SWA pattern and the resolver can raise for them. + # skip it: they do not use Unsloth's SWA pattern and the resolver can raise for them. if ( self._sliding_window_pattern is None and self._sliding_window @@ -4241,6 +4972,28 @@ class LlamaCppBackend: return None + @staticmethod + def _diffusion_gpu_arg(gpu_ids: Optional[List[int]], *, cpu_only: bool = False) -> str: + """Device token passed to the diffusion visual-server child. + + The visual engine replaces its child's CUDA visibility mask with this + token, so an unpinned load must carry forward the first token from the + parent's mask rather than turning a parent-relative ordinal into a new + physical selection. + """ + if gpu_ids: + return str(sorted(gpu_ids)[0]) + if cpu_only: + return "" + if "DG_GPU" in os.environ: + return os.environ["DG_GPU"] + parent_mask = os.environ.get("CUDA_VISIBLE_DEVICES") + if parent_mask: + first = next((token.strip() for token in parent_mask.split(",") if token.strip()), "") + if first and first != "-1": + return first + return "0" + def _start_diffusion_server( self, *, @@ -4251,10 +5004,11 @@ class LlamaCppBackend: model_identifier: str, n_ctx: int, extra_args: Optional[List[str]], + gpu_ids: Optional[List[int]] = None, ) -> bool: """Launch the OpenAI-compat diffusion shim (which drives the on-device visual decoder) and wait for health. Presents the same /v1 + /health - interface as llama-server, so the rest of Studio is unchanged. + interface as llama-server, so the rest of Unsloth is unchanged. """ assets = self._find_diffusion_assets() if assets is None: @@ -4276,7 +5030,11 @@ class LlamaCppBackend: # CUDA_VISIBLE_DEVICES="" to force CPU serving. Keep the visual-server child # CPU-masked (empty --gpu) so the shim does not re-expose GPU 0 via its default. cpu_only = self._effective_gpu_count() == 0 - gpu = "" if cpu_only else os.environ.get("DG_GPU", "0") + # Honor the GPU picker first: the diffusion runner takes a single device, + # so use the lowest selected GPU (matches the sorted set recorded below, so + # the device used == the echoed gpu_ids[0]). With no pick, fall back to the + # CPU-only mask, else DG_GPU / 0. + gpu = self._diffusion_gpu_arg(gpu_ids, cpu_only = cpu_only) cmd = list(shim_cmd) + [ "--gguf", @@ -4304,6 +5062,11 @@ class LlamaCppBackend: env.setdefault("UNSLOTH_ALLOW_CPU", "1") env["DG_VISUAL_BIN"] = visual_bin env["DG_GPU"] = gpu + if gpu_ids: + # The visual server remasks via CUDA_VISIBLE_DEVICES=; pin PCI + # order (as the llama-server path does) so the picked physical id maps + # to the GPU the picker showed, not CUDA's default fastest-first order. + env["CUDA_DEVICE_ORDER"] = "PCI_BUS_ID" # The file-override shim imports its sibling visual_engine; put its dir on PYTHONPATH. # (The zoo-package shim is an installed module and needs no PYTHONPATH change.) if extra_pythonpath: @@ -4326,7 +5089,7 @@ class LlamaCppBackend: logger.debug(f"Could not open diffusion runner log file: {e}") # The shim (and its visual server) die with this backend process, so a - # Studio crash/restart never orphans a GPU process. + # Unsloth crash/restart never orphans a GPU process. self._process = subprocess.Popen( cmd, stdout = subprocess.PIPE, @@ -4349,6 +5112,26 @@ class LlamaCppBackend: self._model_identifier = model_identifier self._cache_type_kv = None self._gpu_offload_active = True + # Diffusion doesn't use the llama.cpp GPU-memory knobs; reset them to + # defaults (the picked device is still recorded below) so /load, /status + # and reload dedup don't report a previous GGUF's manual settings. + self._gpu_memory_mode = "auto" + self._gpu_layers = -1 + self._n_cpu_moe = 0 + self._tensor_split = None + # Diffusion is never tensor-parallel; clear any state left by a prior TP + # chat load (load_model phase 1 only kills the process, it doesn't run + # the unload reset) so /status doesn't misreport TP and an identical + # re-Apply doesn't reload against stale tensor-parallel state. + self._tensor_parallel = False + # The single-device runner records only the lowest selected GPU (chosen + # above), not the whole pick, and clears any explicit pin from a prior + # chat load; a multi-GPU list would misreport placement and mis-dedup. + self._gpu_ids = [sorted(gpu_ids)[0]] if gpu_ids else None + # The frontend prefers requested_gpu_ids when hydrating the picker. + # Diffusion uses only one device, so echo the collapsed effective pin, + # not unused members of the original request. + self._requested_gpu_ids = list(self._gpu_ids) if self._gpu_ids else None if hf_variant: self._hf_variant = hf_variant elif gguf_path: @@ -4414,6 +5197,9 @@ class LlamaCppBackend: touching the shared one; defaults to the shared event. """ cancel_event = cancel_event if cancel_event is not None else self._cancel_event + from utils.hf_cache_settings import get_hf_cache_paths + + download_cache_dir = str(get_hf_cache_paths().hub_cache) try: import huggingface_hub # noqa: F401 -- presence check only except ImportError: @@ -4446,42 +5232,52 @@ class LlamaCppBackend: except Exception as e: logger.warning(f"Could not list repo files: {e}") - # Offline: resolve variant -> filename from the local HF cache. - # The heuristic below assumes filenames echo the repo name, which - # breaks for e.g. Qwen3.6-27B-MTP-GGUF (no "MTP" in file). Match - # against the rel path (not just basename) so subdir layouts like - # ``BF16/foo.gguf`` are findable. + # Fall back to the local cache when the repo listing is unavailable. if not gguf_filename: - try: - from utils.models.model_config import _iter_hf_cache_snapshots - for snap in _iter_hf_cache_snapshots(hf_repo): - cached_files = _gguf_snapshot_files(snap) - matches = _gguf_files_for_variant(cached_files, hf_variant) - if not matches: - continue - gguf_filename = matches[0] - gguf_extra_shards = _gguf_extra_shards(matches, gguf_filename) - logger.info( - "Resolved variant %s -> %s from local HF cache", - hf_variant, - gguf_filename, - ) - break - except Exception as e: - logger.debug(f"Offline cache lookup for variant failed: {e}") + cached_name, cached_shards = _cached_variant_resolution(hf_repo, hf_variant) + if cached_name: + gguf_filename = cached_name + gguf_extra_shards = cached_shards + logger.info( + "Resolved variant %s -> %s from local HF cache", + hf_variant, + gguf_filename, + ) if not gguf_filename: repo_name = hf_repo.split("/")[-1].replace("-GGUF", "") gguf_filename = f"{repo_name}-{hf_variant}.gguf" + # Prefer the existing model. Updates use force=True to fetch a new revision. + if not force: + if hf_variant: + # Resolve by variant so a newer revision's filename does not hide + # the complete older copy. Size-check against that older snapshot's + # own revision when its metadata remains available. + cached_main = cached_gguf_for_load( + hf_repo, + hf_variant, + verify_sizes = True, + hf_token = hf_token, + ) + else: + candidate = _cached_complete_candidate(hf_repo, gguf_filename, gguf_extra_shards) + cached_main = ( + candidate[0] + if candidate is not None + and _cached_candidate_matches_revision_size(hf_repo, candidate, hf_token) + else None + ) + if cached_main is not None: + logger.info(f"Reusing cached GGUF: {cached_main}") + return cached_main + # Check disk space; fall back to a smaller variant if needed all_gguf_files = [gguf_filename] + gguf_extra_shards - expected_sizes: dict[str, int] = {} try: from huggingface_hub import get_paths_info, try_to_load_from_cache path_infos = list(get_paths_info(hf_repo, all_gguf_files, token = hf_token)) - expected_sizes = {p.path: p.size for p in path_infos if p.size} total_bytes = sum((p.size or 0) for p in path_infos) # Subtract bytes already in the HF cache so we only preflight @@ -4490,31 +5286,20 @@ class LlamaCppBackend: # cold whenever free disk is below the full weight footprint, # even though nothing needs downloading. already_cached_bytes = 0 - # Cross-snapshot / case-variant cache reuse is offline-only (see the download - # path below); online, hf_hub_download fetches the current revision and - # resumes partials, so an old snapshot must not be counted as cached here or - # the preflight would under-count the download and skip the disk fallback. + # Count only files that can resume this download. offline = _hf_env_offline() - # A split GGUF whose shards are not co-located in a single snapshot is - # refetched as a whole set later, so it must not be counted as cached here. - split_needs_refetch = False - if offline and not force and gguf_extra_shards: - # Scan all snapshots for one that holds the whole set co-located, so a - # newer snapshot with only the first shard does not mask an older - # complete one and needlessly trip the disk fallback. - if ( - _cached_colocated_split_main( - hf_repo, gguf_filename, gguf_extra_shards, expected_sizes - ) - is None - ): - split_needs_refetch = True + # Offline split sets are reusable only when every shard shares a snapshot. + split_needs_refetch = bool(offline and not force and gguf_extra_shards) if not force and not split_needs_refetch: for p in path_infos: if not p.size: continue try: - cached_path = try_to_load_from_cache(hf_repo, p.path) + cached_path = try_to_load_from_cache( + hf_repo, + p.path, + cache_dir = download_cache_dir, + ) except Exception: cached_path = None if ( @@ -4525,6 +5310,7 @@ class LlamaCppBackend: hf_repo, p.path, expected_size = p.size, + cache_dir = download_cache_dir, ) if isinstance(cached_path, str) and os.path.exists(cached_path): try: @@ -4538,12 +5324,8 @@ class LlamaCppBackend: total_download_bytes = max(0, total_bytes - already_cached_bytes) if total_download_bytes > 0: - cache_dir = os.environ.get( - "HF_HUB_CACHE", - str(Path.home() / ".cache" / "huggingface" / "hub"), - ) - Path(cache_dir).mkdir(parents = True, exist_ok = True) - free_bytes = shutil.disk_usage(cache_dir).free + Path(download_cache_dir).mkdir(parents = True, exist_ok = True) + free_bytes = shutil.disk_usage(download_cache_dir).free total_gb = total_download_bytes / (1024**3) free_gb = free_bytes / (1024**3) @@ -4561,7 +5343,7 @@ class LlamaCppBackend: # surface the disk shortfall for the requested variant. raise RuntimeError( f"Not enough disk space to download {gguf_filename}. " - f"Only {free_gb:.1f} GB free in {cache_dir}" + f"Only {free_gb:.1f} GB free in {download_cache_dir}" ) smaller = self._find_smallest_fitting_variant( hf_repo, @@ -4569,36 +5351,30 @@ class LlamaCppBackend: hf_token, ) if smaller: - fallback_file, fallback_size = smaller + fallback_file, fallback_size, fallback_shards = smaller logger.info( f"Selected variant too large ({total_gb:.1f} GB), " f"falling back to {fallback_file} ({fallback_size / (1024**3):.1f} GB)" ) gguf_filename = fallback_file - _m = _SHARD_RE.match(gguf_filename) - _prefix = _m.group(1) if _m else None - if _prefix: - prefix_lower = _prefix.lower() - gguf_extra_shards = sorted( - f - for f in all_gguf_files - if f.lower().startswith(prefix_lower) - and f != gguf_filename - and not _is_companion_gguf_path(f) + gguf_extra_shards = fallback_shards + + # The selected fallback is a new load target. Apply the + # same any-revision reuse policy before starting a fetch. + fallback_candidate = _cached_complete_candidate( + hf_repo, gguf_filename, gguf_extra_shards + ) + if fallback_candidate is not None and ( + _cached_candidate_matches_revision_size( + hf_repo, fallback_candidate, hf_token ) - else: - gguf_extra_shards = [] - # Record the fallback's size so the later cache-reuse probe can - # size-verify it; only for a single-file fallback, since - # _find_smallest_fitting_variant returns the whole-variant size - # and using that as the first shard's expected size would reject - # a valid cached first shard of a split fallback. - if not gguf_extra_shards: - expected_sizes[fallback_file] = fallback_size + ): + logger.info(f"Reusing cached fallback GGUF: {fallback_candidate[0]}") + return fallback_candidate[0] else: raise RuntimeError( f"Not enough disk space to download any variant. " - f"Only {free_gb:.1f} GB free in {cache_dir}" + f"Only {free_gb:.1f} GB free in {download_cache_dir}" ) except RuntimeError: raise @@ -4614,45 +5390,27 @@ class LlamaCppBackend: raise RuntimeError("Cancelled") dl_start = time.monotonic() # Xet primary, HTTP fallback on stall; per-file so finished shards stay cached. - local_path = None - # Reuse a cached copy from another snapshot / case-variant repo dir only when - # offline. Online, fall through to hf_hub_download so its revision/etag check - # fetches the current file (and resumes a partial) instead of serving a stale - # same-name blob from an older revision. - if not force and _hf_env_offline(): - if gguf_extra_shards: - # A split GGUF must load every shard from one snapshot; reuse only a - # snapshot that holds the whole set co-located, scanning past a newer - # snapshot that has just the first shard while an older one is complete. - local_path = _cached_colocated_split_main( - hf_repo, gguf_filename, gguf_extra_shards, expected_sizes - ) - else: - local_path = _cached_hf_snapshot_file( - hf_repo, - gguf_filename, - expected_size = expected_sizes.get(gguf_filename), - ) - if local_path is None: - local_path = hf_hub_download_with_xet_fallback( + local_path = hf_hub_download_with_xet_fallback( + hf_repo, + gguf_filename, + hf_token, + cancel_event = cancel_event, + on_status = lambda m: logger.info(m), + force_download = force, + cache_dir = download_cache_dir, + ) + for shard in gguf_extra_shards: + if cancel_event.is_set(): + raise RuntimeError("Cancelled") + logger.info(f"Resolving GGUF shard: {shard}") + hf_hub_download_with_xet_fallback( hf_repo, - gguf_filename, + shard, hf_token, cancel_event = cancel_event, - on_status = lambda m: logger.info(m), force_download = force, + cache_dir = download_cache_dir, ) - for shard in gguf_extra_shards: - if cancel_event.is_set(): - raise RuntimeError("Cancelled") - logger.info(f"Resolving GGUF shard: {shard}") - hf_hub_download_with_xet_fallback( - hf_repo, - shard, - hf_token, - cancel_event = cancel_event, - force_download = force, - ) except Exception as e: if isinstance(e, RuntimeError) and "Cancelled" in str(e): raise @@ -4675,10 +5433,12 @@ class LlamaCppBackend: pick: Callable[[list[str]], Optional[str]], label: str, cancel_event: Optional[threading.Event] = None, + near_path: Optional[str] = None, ) -> Optional[str]: """Resolve and fetch a companion GGUF (mmproj / MTP drafter) by name. - Tries the live repo file list, then the local HF cache snapshots + Prefers a companion co-located with ``near_path``'s cache snapshot, + then tries the live repo file list, then the local HF cache snapshots (offline, same fallback as _download_gguf), then hf_hub_download. Runs WITHOUT self._lock (like _download_gguf); honors _cancel_event so an /unload between the main download and here skips the fetch. @@ -4688,6 +5448,23 @@ class LlamaCppBackend: if cancel_event.is_set(): return None + # Keep companion files in the main GGUF's snapshot. + if near_path: + cached = _companion_snapshot_sibling(near_path, pick) + if cached: + logger.info("Reusing cached %s: %s", label, cached) + return cached + + from utils.hf_cache_settings import get_hf_cache_paths + + companion_cache_dir = _hub_cache_dir_for_snapshot_path(near_path) or str( + get_hf_cache_paths().hub_cache + ) + + if _hub_download_in_flight(hf_repo): + logger.info("Skipping %s download while a hub download is active", label) + return None + target: Optional[str] = None from huggingface_hub import list_repo_files @@ -4718,7 +5495,7 @@ class LlamaCppBackend: if target is None: try: from utils.models.model_config import _iter_hf_cache_snapshots - for snap in _iter_hf_cache_snapshots(hf_repo): + for snap in _iter_hf_cache_snapshots(hf_repo, companion_cache_dir): rel_files = _gguf_snapshot_files(snap) target = pick(rel_files) if target is not None: @@ -4736,7 +5513,11 @@ class LlamaCppBackend: # hf_hub_download with hf_repo would miss the canonical file and silently # drop the companion. _cached_hf_snapshot_file scans every case variant. if _hf_env_offline(): - cached = _cached_hf_snapshot_file(hf_repo, target) + cached = _cached_hf_snapshot_file( + hf_repo, + target, + cache_dir = companion_cache_dir, + ) if cached: logger.info("Resolved %s from local HF cache: %s", label, cached) return cached @@ -4749,6 +5530,7 @@ class LlamaCppBackend: target, hf_token, cancel_event = cancel_event, + cache_dir = companion_cache_dir, ) except Exception as e: logger.warning(f"Could not download {label}: {e}") @@ -4760,36 +5542,31 @@ class LlamaCppBackend: hf_repo: str, hf_token: Optional[str] = None, cancel_event: Optional[threading.Event] = None, + near_path: Optional[str] = None, ) -> Optional[str]: """Download the mmproj (vision projection) file from a GGUF repo. Prefers mmproj-F16.gguf, else any mmproj*.gguf. Returns the local path, or None if none exists. ``cancel_event`` overrides - ``self._cancel_event`` (defaults to it). + ``self._cancel_event`` (defaults to it). ``near_path`` prefers a + copy co-located with the main GGUF's cache snapshot. """ - def _pick_mmproj(candidates: list[str]) -> Optional[str]: - mmproj_files = sorted( - f - for f in candidates - if f.lower().endswith(".gguf") and "mmproj" in Path(f).name.lower() - ) - if not mmproj_files: - return None - for f in mmproj_files: - if f.lower().endswith("-f16.gguf"): - return f - return mmproj_files[0] - return self._download_companion_gguf( hf_repo = hf_repo, hf_token = hf_token, pick = _pick_mmproj, label = "mmproj", cancel_event = cancel_event, + near_path = near_path, ) - def _cached_repo_mtp_drafter(self, hf_repo: str) -> Optional[str]: + def _cached_repo_mtp_drafter( + self, + hf_repo: str, + *, + cache_dir: Optional[str] = None, + ) -> Optional[str]: """A drafter already in this repo's local HF cache, reused offline when a fresh copy can't be fetched. Prefers a repo-root ``mtp-*.gguf`` across all cached snapshots; else an existing ``MTP/`` copy (any precision -- the @@ -4799,7 +5576,12 @@ class LlamaCppBackend: roots: list[Path] = [] subdirs: list[Path] = [] - for snap in _iter_hf_cache_snapshots(hf_repo): # newest first + snapshots = ( + _iter_hf_cache_snapshots(hf_repo) + if cache_dir is None + else _iter_hf_cache_snapshots(hf_repo, cache_dir) + ) + for snap in snapshots: # newest first for f in sorted(_gguf_snapshot_files(snap)): if _is_companion_gguf_path(f) and "mmproj" not in f.lower(): (roots if "/" not in f else subdirs).append(snap / f) @@ -4817,6 +5599,7 @@ class LlamaCppBackend: *, hf_repo: str, hf_token: Optional[str] = None, + near_path: Optional[str] = None, ) -> Optional[str]: """Download the separate MTP drafter (speculative head) from a GGUF repo. @@ -4828,16 +5611,6 @@ class LlamaCppBackend: are intentionally skipped. Returns the local path, or None. """ - # Offline, reuse any drafter already on disk (a fresh copy can't be - # fetched). Online, _download_companion_gguf/hf_hub_download reuse the - # current cached file and refetch a changed one, so skip the probe here - # rather than pair new weights with a stale draft. - if _hf_env_offline(): - cached = self._cached_repo_mtp_drafter(hf_repo) - if cached: - logger.info(f"Reusing cached MTP drafter (offline): {cached}") - return cached - def _pick_mtp(candidates: list[str]) -> Optional[str]: # Root-level only: MTP/ subdir copies now share the mtp- prefix but # are explicit-selection, not auto-fetch (they'd sort ahead of root). @@ -4850,11 +5623,31 @@ class LlamaCppBackend: ) return mtp_files[0] if mtp_files else None + if near_path: + cached = _companion_snapshot_sibling(near_path, _pick_mtp) + if cached: + logger.info("Reusing cached MTP drafter: %s", cached) + return cached + + # Offline, reuse any drafter already on disk (a fresh copy can't be + # fetched). Online, _download_companion_gguf/hf_hub_download reuse the + # current cached file and refetch a changed one, so skip the probe here + # rather than pair new weights with a stale draft. + if _hf_env_offline(): + cached = self._cached_repo_mtp_drafter( + hf_repo, + cache_dir = _hub_cache_dir_for_snapshot_path(near_path), + ) + if cached: + logger.info(f"Reusing cached MTP drafter (offline): {cached}") + return cached + return self._download_companion_gguf( hf_repo = hf_repo, hf_token = hf_token, pick = _pick_mtp, label = "MTP drafter", + near_path = near_path, ) def _resolve_launch_mmproj_path( @@ -4980,7 +5773,7 @@ class LlamaCppBackend: return ( f"'{arch}' is a diffusion (image-generation) GGUF, which " "llama-server cannot run as a chat/completion model. Use " - "Studio's Images page to generate with local diffusion " + "Unsloth's Images page to generate with local diffusion " "GGUFs such as FLUX and Qwen-Image." ) if is_ollama: @@ -5262,6 +6055,24 @@ class LlamaCppBackend: and ("unknown" in text or "unsupported" in text or "not supported" in text) ) + @staticmethod + def _mmproj_retry_failure_message(*, projector_confirmed: bool, detail: str) -> str: + """User-facing error when the text-only --mmproj strip retry also fails. + + Confirmed projector-format mismatches keep the historical wording. + Bare signal crashes (common on some ROCm/driver paths) must not be + reported as "Vision projector incompatible" — that misled #7302. + """ + if projector_confirmed: + return ( + "Vision projector incompatible with this llama.cpp " + "build, and the text-only retry also failed: " + detail + ) + return ( + "Vision model failed to start (llama-server crashed with " + "--mmproj), and the text-only retry also failed: " + detail + ) + @staticmethod def _output_has_nonprojector_diagnostic(output: str) -> bool: """True when the output already names a concrete non-projector cause (out @@ -5436,6 +6247,7 @@ class LlamaCppBackend: ) self._stdout_thread.start() + @_with_gguf_load_marker def load_model( self, *, @@ -5458,6 +6270,13 @@ class LlamaCppBackend: speculative_type: Optional[str] = None, spec_draft_n_max: Optional[int] = None, tensor_parallel: bool = False, + gpu_memory_mode: Literal["auto", "manual"] = "auto", + gpu_layers: int = -1, + n_cpu_moe: int = 0, + tensor_split: Optional[List[float]] = None, + # Explicit GPU placement pool (issue #7164). None/[] = auto-select; + # the fitter may pin the smallest subset of this pool that fits. + gpu_ids: Optional[List[int]] = None, n_threads: Optional[int] = None, n_gpu_layers: Optional[int] = None, # caller compat, unused n_parallel: int = 1, @@ -5490,6 +6309,14 @@ class LlamaCppBackend: "speculative_type": speculative_type, "spec_draft_n_max": spec_draft_n_max, "tensor_parallel": tensor_parallel, + # GPU-memory placement: replayed on respawn so a server SIGKILL'd by + # GPU/RAM pressure reloads onto the same devices with the same + # offload, not the auto defaults. + "gpu_memory_mode": gpu_memory_mode, + "gpu_layers": gpu_layers, + "n_cpu_moe": n_cpu_moe, + "tensor_split": list(tensor_split) if tensor_split is not None else None, + "gpu_ids": list(gpu_ids) if gpu_ids is not None else None, "n_threads": n_threads, "n_gpu_layers": n_gpu_layers, "n_parallel": n_parallel, @@ -5516,6 +6343,11 @@ class LlamaCppBackend: speculative_type = speculative_type, spec_draft_n_max = spec_draft_n_max, tensor_parallel = tensor_parallel, + gpu_memory_mode = gpu_memory_mode, + gpu_layers = gpu_layers, + n_cpu_moe = n_cpu_moe, + tensor_split = tensor_split, + gpu_ids = gpu_ids, chat_template_override = chat_template_override, extra_args = extra_args, is_vision = is_vision, @@ -5541,15 +6373,63 @@ class LlamaCppBackend: self._cancel_event.clear() - # ── Phase 1: kill old process (under lock, fast) ────────── - with self._lock: - self._kill_process() - # Resolve llama-server now but defer a not-found error: a block-diffusion # GGUF uses the diffusion runner, and its arch is only known after the header. binary = self._find_llama_server_binary() is_vulkan_backend = self._is_vulkan_backend(binary) + # ── Vulkan-ordinal preflight (BEFORE the Phase 1 kill) ──────── + # An explicit Vulkan pin the ggml probe never enumerated cannot be honored. + # Validate it ABOVE the kill so an invalid selection leaves the live model + # untouched: CUDA ids are range-checked at the route, but Vulkan ordinals are + # not, so a stale gpu_ids=[99] used to kill the server then 400, leaving + # nothing running (#7239). _get_gpu_memory needs only the binary (safe pre- + # download) and reuses the later fit's issubset logic. Guarded on a found + # Vulkan build + a pin so a deferred not-found stays deferred for diffusion. + if is_vulkan_backend and gpu_ids and binary: + _pf_wanted = {int(x) for x in gpu_ids} + _pf_probed = {g[0] for g in self._get_gpu_memory(binary)} + if not _pf_wanted.issubset(_pf_probed): + raise ValueError( + f"Requested Vulkan GPU ordinal(s) {sorted(_pf_wanted)} not " + f"present. Available Vulkan devices: {sorted(_pf_probed)}." + ) + + # A remote uncached GGUF may only reveal that it needs the + # single-device diffusion runner after download. On Vulkan, an + # explicit gpu_ids request cannot be mapped from ggml ordinals to + # that runner's CUDA physical index. Download and classify the main + # file before killing the healthy server so this late rejection is + # non-destructive. The Phase 2 call below reuses this cached path. + _preflight_model_path = None + if is_vulkan_backend and gpu_ids and hf_repo: + _resolved_repo = _resolve_repo_id_casing(hf_repo) + if _resolved_repo != hf_repo: + logger.info( + "Using cached repo_id casing '%s' for requested '%s'", + _resolved_repo, + hf_repo, + ) + hf_repo = _resolved_repo + with _hf_offline_if_dns_dead(): + _preflight_model_path = self._download_gguf( + hf_repo = hf_repo, + hf_variant = hf_variant, + hf_token = hf_token, + ) + if self._gguf_path_is_diffusion(_preflight_model_path, model_identifier): + raise ValueError( + "GPU selection (gpu_ids) is not supported for a DiffusionGemma " + "GGUF on a Vulkan llama.cpp build: the diffusion runner selects " + "its device by CUDA physical index, which has no defined mapping " + "to ggml Vulkan device ordinals. Omit gpu_ids to use the default " + "device." + ) + + # ── Phase 1: kill old process (under lock, fast) ────────── + with self._lock: + self._kill_process() + # ── Phase 2: download (NO lock held, so cancel can proceed) ── # mtp_draft_path arrives set for local Gemma loads (detected # sibling); for -hf loads it's None here and resolved just below. @@ -5571,7 +6451,7 @@ class LlamaCppBackend: ) hf_repo = _resolved_repo with _hf_offline_if_dns_dead(): - model_path = self._download_gguf( + model_path = _preflight_model_path or self._download_gguf( hf_repo = hf_repo, hf_variant = hf_variant, hf_token = hf_token, @@ -5581,6 +6461,7 @@ class LlamaCppBackend: mmproj_path = self._download_mmproj( hf_repo = hf_repo, hf_token = hf_token, + near_path = model_path, ) # Auto-download the separate MTP drafter (e.g. Gemma) when # the requested spec mode can use it. Repos with the head @@ -5598,6 +6479,7 @@ class LlamaCppBackend: mtp_draft_path = self._download_mtp( hf_repo = hf_repo, hf_token = hf_token, + near_path = model_path, ) elif gguf_path: if not Path(gguf_path).is_file(): @@ -5619,6 +6501,18 @@ class LlamaCppBackend: # Block-diffusion GGUFs (DiffusionGemma) cannot run on llama-server; # serve them with the diffusion runner (same OpenAI-compat interface). if self._is_diffusion: + # The diffusion runner pins its child by CUDA visibility mask, so a + # ggml Vulkan ordinal cannot be honored (wrong GPU / CPU fallback). + # Route and remote-download preflights reject before teardown; keep + # this as a final defense if classification ever disagrees. + if is_vulkan_backend and gpu_ids: + raise ValueError( + "GPU selection (gpu_ids) is not supported for a DiffusionGemma " + "GGUF on a Vulkan llama.cpp build: the diffusion runner selects " + "its device by CUDA physical index, which has no defined mapping " + "to ggml Vulkan device ordinals. Omit gpu_ids to use the default " + "device." + ) # Not a tensor/layer GGUF: clear any preserved-fallback flag from a # prior load (this path skips the command builder that clears it). self._layer_preserves_tensor_intent = False @@ -5634,6 +6528,7 @@ class LlamaCppBackend: model_identifier = model_identifier, n_ctx = n_ctx, extra_args = extra_args, + gpu_ids = gpu_ids, ) if not binary: @@ -5695,6 +6590,59 @@ class LlamaCppBackend: # use the same helper so a healthy env-driven tensor server matches. split_mode_override = parse_split_mode_override(extra_args) tensor_parallel = _effective_tensor_parallel(extra_args, tensor_parallel) + # gpu_layers=0 leaves nothing to split, yet --split-mode tensor or + # a per-GPU ratio still launches tensor mode -- and under the + # CPU-only mask below (no visible devices) that aborts the server + # instead of loading on CPU. Drop both here (nothing to split). + if gpu_memory_mode == "manual" and gpu_layers == 0: + if tensor_parallel or tensor_split: + logger.info( + "Manual gpu_layers=0: dropping tensor split/parallel " + "flags (nothing to split on the GPU)" + ) + tensor_parallel = False + tensor_split = None + # Record the requested strategy for /status and the load + # response. 'manual' has no fallback, so the request value is the + # value actually applied. + self._gpu_memory_mode = gpu_memory_mode + # The layer/MoE/split knobs apply only with an explicit offload + # (manual + gpu_layers >= 0); else record defaults so /status and + # /load don't report knobs the server never applied. + if gpu_memory_mode == "manual" and gpu_layers >= 0: + self._gpu_layers = gpu_layers + self._n_cpu_moe = n_cpu_moe + self._tensor_split = tensor_split + else: + self._gpu_layers = -1 + self._n_cpu_moe = 0 + self._tensor_split = None + self._gpu_ids = sorted(gpu_ids) if gpu_ids else None + # Manual offload skips the TP planner but still emits --split-mode + # tensor at launch; drop it when fewer than 2 GPUs are in use -- + # tensor split is a no-op there and aborts on some architectures. + # Done before the cache-drop below so a quantized KV survives. + if ( + tensor_parallel + and gpu_memory_mode == "manual" + and gpu_layers >= 0 + and self._effective_gpu_count(sorted(gpu_ids) if gpu_ids else None) < 2 + ): + logger.info( + "Tensor parallelism requested in manual mode but fewer " + "than 2 GPUs are in use; ignoring (needs >= 2)." + ) + tensor_parallel = False + # Drop TP for manual + Auto layers before the cache-drop below (like + # the <2-GPU guard above), so a requested quantized KV survives into + # the --fit load rather than being stripped for a tensor attempt. + if tensor_parallel and gpu_memory_mode == "manual" and gpu_layers < 0: + logger.info( + "Manual mode with Auto layers hands memory management to " + "llama.cpp --fit, which is incompatible with tensor " + "parallelism; ignoring the tensor split." + ) + tensor_parallel = False # Tensor mode aborts on a quantized KV cache, so drop it for the # tensor attempt (and strip any inherited/explicit --cache-type # that would re-impose it when appended last). Layer split does @@ -5775,10 +6723,22 @@ class LlamaCppBackend: "Vision-capable GGUF loaded without a usable mmproj; " "image input will be disabled for this session" ) + # Seed before the try: the except (GPU-selection failure -> + # --fit on) falls through to the launch which reads this, and the + # probe that assigns it may throw first. Captured before manual + # empty `gpus` so the speculative defaults stay GPU-aware and the + # CPU-fallback check still knows GPUs were present. + _detected_gpus: list[tuple[int, int]] = [] model_size = None # set in the fit try; used by the APU RAM guard # Layer-fallback min GPUs; raised below on a tensor downgrade. Bound # before the try so the --fit-on except path still has it (no UnboundLocal). _layer_min_gpus = 1 + # An explicit Vulkan ordinal absent from the ggml probe cannot be + # honored; flag it in the fit and reject after the try (raising inside + # would be swallowed into the --fit-on fallback). Bound before the try. + _vulkan_explicit_unmatched = False + _vulkan_requested_ids: list[int] = [] + _vulkan_available_ordinals: list[int] = [] try: gguf_size = self._get_gguf_size_bytes(model_path) # Include GPU-loaded mmproj in the fit budget (#5825). @@ -5791,7 +6751,41 @@ class LlamaCppBackend: # Pass binary so a Vulkan build probes ggml's Vulkan ordinals. _gpu_mem = self._get_gpu_memory(binary) gpus = [(idx, free) for idx, free, _t in _gpu_mem] + # Restrict the fit (and thus the layer plan + pin env) to the + # selected GPUs; fail-open if none match so a stale UI choice + # can't strand the load on CPU (issue #7164). + if gpu_ids: + # A Vulkan build indexes by ggml ordinal. An explicit ordinal + # absent from the probe can't be pinned, so reject after the try + # rather than fail-open onto a device the user didn't pick. + _wanted_ids = {int(x) for x in gpu_ids} + # Reject if ANY requested ordinal is absent, not only when none + # match: [0, 99] against {0, 1} silently drops 99. Comparing the + # full requested set (before filter narrows) still lets the fitter + # pick a valid subset later -- that is narrowing, not absence. + _probed_ordinals = {g[0] for g in gpus} + if is_vulkan_backend and not _wanted_ids.issubset(_probed_ordinals): + _vulkan_explicit_unmatched = True + _vulkan_requested_ids = sorted(_wanted_ids) + _vulkan_available_ordinals = sorted(_probed_ordinals) + # Restrict the probed pool to the selection; fail-open (keep the + # full pool) if none match so a stale UI choice can't strand the + # load on CPU (issue #7164). + _sel_gpus = [g for g in gpus if g[0] in _wanted_ids] + gpus = _sel_gpus if _sel_gpus else gpus total_by_idx = {idx: total for idx, _f, total in _gpu_mem} + # GPU picker: restrict every mode to the chosen devices, so + # auto selection only considers them and manual mask to + # them (the env block below pins CUDA/HIP_VISIBLE_DEVICES). + if gpu_ids: + _picked = set(gpu_ids) + gpus = [g for g in gpus if g[0] in _picked] + + # GPUs the model will run on -- captured before manual + # empty `gpus` to bypass the planner. bool() drives the + # GPU-aware speculative defaults; the list feeds the + # CPU-fallback check. + _detected_gpus = list(gpus) def _gpu_usable(g, frac = _CTX_FIT_VRAM_FRACTION): # Per-GPU usable budget for ranking: free - (1-frac)*total. @@ -5823,6 +6817,44 @@ class LlamaCppBackend: # GPU/VRAM-fit logic below may shrink it on limited HW. max_available_ctx = self._context_length or effective_ctx + # Manual + Auto layers (the Manual default): hand memory + # management to llama.cpp's --fit. Emptying the probed GPU set + # no-ops the selection/TP planning below, leaving gpu_indices + # None (an explicit gpu_ids pick still pins below) and use_fit + # True. An explicit context is honored (--fit optimizes around + # it); 0 lets --fit size it. + if gpu_memory_mode == "manual" and gpu_layers < 0: + # Tensor parallelism was already dropped above (before the + # cache-drop), so a quantized KV survives into this --fit load. + gpus = [] + effective_ctx = requested_ctx if requested_ctx > 0 else 0 + original_ctx = effective_ctx + # --fit aborts under --split-mode tensor; a raw extras + # --split-mode/--tensor-split (appended last) would + # otherwise reach llama-server. Strip it like the TP + # downgrade does. + extra_args = strip_split_mode_only(extra_args) + elif gpu_memory_mode == "manual": + # Manual offload (--gpu-layers + --fit off): no automatic + # device masking (a gpu_ids pick still pins below) or + # context cap -- the user owns both. tensor_parallel is + # honored but skips the memory-based planner (gpus = []); + # the toggle just emits --split-mode tensor (split by free + # VRAM, or by the Split ratio if set). + gpus = [] + effective_ctx = ( + requested_ctx if requested_ctx > 0 else (self._context_length or 0) + ) + original_ctx = effective_ctx + # Strip the user --split-mode when the toggle owns the split + # (TP engaged -> Studio emits --split-mode tensor) or when the + # user asked for tensor (which aborts on a single GPU even if + # the manual <2-GPU guard downgraded TP). Otherwise keep their + # non-tensor mode (row/none/layer) -- the toggle can't express + # those. + if tensor_parallel or split_mode_override == "tensor": + extra_args = strip_split_mode_only(extra_args) + # Will MTP engage? If so, auto-fit reserves draft-model VRAM. # Mirrors _build_speculative_flags: forced mtp/mtp+ngram always # engage; auto only on an MTP model >= 3B; ngram/off never. A @@ -5838,7 +6870,7 @@ class LlamaCppBackend: and not bool(mtp_draft_path) ) # LLAMA_ARG_SPEC_TYPE only reaches the child when neither extras - # nor Studio emit a spec flag (mode "off", no user --spec-type), + # nor Unsloth emit a spec flag (mode "off", no user --spec-type), # since _build_speculative_flags emits one for every other mode. # Consult the env for the reserve only then, else a stale MTP env # would over-reserve. @@ -5847,7 +6879,7 @@ class LlamaCppBackend: if (not _extra_args_set_spec_type(extra_args) and _mtp_canonical == "off") else {} ) - # Extras can run MTP even when Studio suppresses its own emission. + # Extras can run MTP even when Unsloth suppresses its own emission. _user_mtp_via_extras = _extra_args_requests_mtp(extra_args, env = _spec_env) # A non-MTP model-based draft mode (draft-simple/draft-eagle3) in # extras also loads a separate draft model that needs reserving; @@ -5910,10 +6942,13 @@ class LlamaCppBackend: _extra_n_max = _extra_args_spec_draft_n_max(extra_args) _mtp_eff_n_max = _extra_n_max if _extra_n_max is not None else spec_draft_n_max if _mtp_eff_n_max is None: - _mtp_eff_n_max = 2 if gpus else 3 + # _detected_gpus (not gpus) so manual -- which empty + # gpus to bypass the planner -- keep the GPU draft depth the + # launch flags also use, instead of the CPU default. + _mtp_eff_n_max = 2 if _detected_gpus else 3 # Separate-drafter weights live on GPU (an embedded head is # already in model_size). Size the drafter the launch loads, by - # precedence: extras --model-draft (last-wins), else Studio's + # precedence: extras --model-draft (last-wins), else Unsloth's # emitted mtp_draft_path, else the env drafter. Sizing the wrong # one would under-reserve and OOM. _cli_draft_for_budget = _extra_args_mtp_draft_path(extra_args, env = {}) @@ -6048,7 +7083,8 @@ class LlamaCppBackend: # honor it, cap only if it fits no combination. Auto (native): # prefer fewer GPUs with reduced context (multi-GPU is slower). gpu_indices, use_fit = None, True - # Per-GPU weight proportions for tensor mode (None = even). + # Per-GPU weight proportions for tensor mode (None lets + # llama.cpp split by free VRAM). tp_tensor_split: Optional[list[int]] = None explicit_ctx = requested_ctx > 0 # Flat MTP reserve fraction: used only as the fallback when the @@ -6123,7 +7159,12 @@ class LlamaCppBackend: # GPUs below that reserve from the set up front (gpu_indices # becomes the CUDA_VISIBLE_DEVICES mask, fully excluding them). tp_gpus = gpus - if tensor_parallel: + # Manual mode owns the layer count and context, so it skips + # the memory-based planner; its toggle still emits + # --split-mode tensor below (split by free VRAM, or by the + # Split ratio if set). auto plans here. + plan_tp = tensor_parallel and gpu_memory_mode != "manual" + if plan_tp: # Deterministic per-device compute buffer (replicated on # every device in tensor mode); flat fallback when dims # are unavailable. _plan_tensor_parallel uses the same. @@ -6142,7 +7183,7 @@ class LlamaCppBackend: # free yet have no budget left. tp_gpus = [g for g in gpus if _gpu_usable(g) >= reserve_mib] - if tensor_parallel and len(tp_gpus) < 2: + if plan_tp and len(tp_gpus) < 2: # Tensor parallelism needs >= 2 usable GPUs. On a single # GPU --split-mode tensor is a no-op; with 0 GPUs (CPU-only # or probe failed) it must not reach llama-server; and a @@ -6558,6 +7599,23 @@ class LlamaCppBackend: tp_tensor_split = None effective_ctx = requested_ctx # fall back to original + # An unenumerated explicit Vulkan ordinal can't be pinned; fail loudly + # instead of fitting onto an unselected device. Clear the raw selection + # the early state-publish recorded so it never leaks into gpu_ids (#7239). + if _vulkan_explicit_unmatched: + self._gpu_ids = None + self._requested_gpu_ids = None + raise ValueError( + f"Requested Vulkan GPU ordinal(s) {_vulkan_requested_ids} not " + f"present. Available Vulkan devices: {_vulkan_available_ordinals}." + ) + + # GPU picker: when no narrower subset was chosen (manual, or + # a failed/file-size selection), pin the whole picked set so the + # model can't spill onto an unpicked GPU. + if gpu_ids and gpu_indices is None: + gpu_indices = sorted(gpu_ids) + # Unified-memory APUs load weights into system RAM (under WSL the VM # cap, not the ROCm-reported VRAM, is the real ceiling); refuse an # oversize load the OS would otherwise kill mid-flight. Base model @@ -6594,8 +7652,6 @@ class LlamaCppBackend: model_path, "--port", str(self._port), - "-c", - str(effective_ctx) if effective_ctx > 0 else "0", "--parallel", str(n_parallel), "--flash-attn", @@ -6603,6 +7659,17 @@ class LlamaCppBackend: # Error out at n_ctx instead of silently rotating the KV cache; frontend catches it and points the user at "Context Length". "--no-context-shift", ] + # A positive context is always passed (in auto-fit, --fit then + # optimizes the gpu-layer offload around it). When auto-fit has + # no explicit context, omit -c so --fit sizes it to fit VRAM: + # "-c 0" would instead pin the FULL native context (llama.cpp's + # -c handler sets fit_params_min_ctx = UINT32_MAX on value 0, + # disabling --fit's reduction). See gpu_memory_mode. + auto_fit = gpu_memory_mode == "manual" and gpu_layers < 0 + if effective_ctx > 0: + cmd.extend(["-c", str(effective_ctx)]) + elif not auto_fit: + cmd.extend(["-c", "0"]) # Report a clean public model id (matching GET /v1/models) rather # than the raw -m path in llama-server's own /v1/models and the @@ -6614,7 +7681,63 @@ class LlamaCppBackend: cmd.extend(["--alias", _alias]) fully_gpu_offloaded = False - if use_fit: + # Set when a positional --tensor-split is emitted, so the env block + # can pin CUDA to PCI order even without a GPU subset (see below). + manual_tensor_split_emitted = False + if gpu_memory_mode == "manual" and gpu_layers >= 0: + # Pin the user's layer count and disable auto-fit. --fit off + # also means _ctx_integrity_flags must not add --fit-ctx. + use_fit = False + cmd.extend(["--gpu-layers", str(gpu_layers), "--fit", "off"]) + # Keep the first n_cpu_moe MoE layers' experts on CPU. + moe_flag = self._resolve_cpu_moe_flag( + n_cpu_moe, + self.n_moe_layers, + self._leading_dense_block_count or 0, + ) + if moe_flag is not None: + cmd.extend(["--n-cpu-moe", str(moe_flag)]) + elif n_cpu_moe: + # Requested on a dense model: nothing was emitted, so + # don't report a count llama-server never received. + self._n_cpu_moe = 0 + # Distribute the model across GPUs by the user's per-GPU shares + # (--tensor-split). Works with layer split and tensor + # parallelism; --fit off means no fit/tensor abort. Only emit + # when >1 GPU is in use AND the list length matches that count: + # the field is hidden (not cleared) when the picker narrows to + # one, and a direct caller can send a stale ratio for a different + # GPU set. Studio drops any mismatch to the free-VRAM default + # (llama.cpp would silently zero-pad a short list, or abort past + # its 16-device cap). + _split_gpus = self._effective_gpu_count(gpu_indices) + if tensor_split and _split_gpus > 1: + # An all-zero/non-positive sanitized split assigns nothing + # anywhere, so fall through to the free-VRAM default in + # that case. + _sanitized_split = self._sanitize_tensor_split(tensor_split) + _split_total = sum(_sanitized_split) + if len(_sanitized_split) == _split_gpus and _split_total > 0: + cmd.extend( + ["--tensor-split", ",".join(f"{x:g}" for x in _sanitized_split)] + ) + self._tensor_split = _sanitized_split + manual_tensor_split_emitted = True + else: + logger.warning( + "Dropping manual --tensor-split (%d entries for " + "%d GPUs, sanitized total %s); llama.cpp's " + "free-VRAM split applies instead", + len(tensor_split), + _split_gpus, + _split_total, + ) + self._tensor_split = None + elif tensor_split: + # Single effective GPU: the split is never emitted, so + # don't report it as active via /status and /load. + self._tensor_split = None + elif use_fit: cmd.extend(["--fit", "on"]) elif gpu_indices is not None: # Fits on selected GPU(s) -- force all layers on GPU. --fit off is @@ -6628,10 +7751,31 @@ class LlamaCppBackend: # when the binary advertises it (older/custom binaries may not). if server_caps.get("supports_metrics"): cmd.append("--metrics") + self._slot_save_dir = None + self._slot_save_binary = None + self._prompt_cache_disabled = False + if server_caps.get("supports_slot_save"): + try: + from utils.paths.storage_roots import ( # noqa: WPS433 + llama_slot_cache_root, + ) + + slot_dir = llama_slot_cache_root() + slot_dir.mkdir(parents = True, exist_ok = True) + # Saved KV encodes chat content; keep it from other local users. + with contextlib.suppress(OSError): + os.chmod(slot_dir, 0o700) + cmd.extend(["--slot-save-path", str(slot_dir)]) + self._slot_save_dir = str(slot_dir) + self._slot_save_binary = (binary, Path(binary).stat().st_mtime_ns) + except OSError: + self._slot_save_dir = None + self._slot_save_binary = None cmd.extend( self._ctx_integrity_flags( n_parallel, use_fit, + auto_fit, requested_ctx, effective_ctx, server_caps, @@ -6695,9 +7839,11 @@ class LlamaCppBackend: self._cache_type_kv = None # Tensor parallelism: split the model across GPUs by tensor - # rather than by layer. Multi-GPU only -- a no-op on a single - # GPU. Default (layer split) is left implicit by omitting the - # flag. See llama.cpp --split-mode. + # rather than by layer. The UI only offers it on multi-GPU; a + # direct single-GPU caller is redundant (supported archs no-op, + # unsupported ones abort and the /load path retries layer split). + # Default (layer split) is left implicit by omitting the flag. + # See llama.cpp --split-mode. if tensor_parallel: cmd.extend(["--split-mode", "tensor"]) if tp_tensor_split and len(tp_tensor_split) > 1: @@ -6729,7 +7875,7 @@ class LlamaCppBackend: extra_args = extra_args, model_identifier = model_identifier, model_path = model_path, - gpus = bool(gpus), + gpus = bool(_detected_gpus), binary = binary, mtp_draft_path = launch_mtp_draft_path, ) @@ -6808,8 +7954,9 @@ class LlamaCppBackend: else: self._api_key = None - # Windows + full offload: disable KV checkpoints (WDDM/PCI-E - # overhead). CPU/partial offload keeps prompt caching. #5692. + # Windows + full offload: drop the host-RAM KV checkpoints that cause + # WDDM/PCI-E overhead, but keep prompt caching (in-VRAM prefix reuse) so + # a repeated prompt is not re-prefilled on every request. #5692. if sys.platform == "win32" and full_offload_tuning_active: unsupported_cache_flags: list[str] = [] if server_caps.get("supports_cache_ram"): @@ -6820,24 +7967,54 @@ class LlamaCppBackend: cmd.extend(["--ctx-checkpoints", "0"]) else: unsupported_cache_flags.append("--ctx-checkpoints") - if server_caps.get("supports_no_cache_prompt"): - cmd.append("--no-cache-prompt") - else: - unsupported_cache_flags.append("--no-cache-prompt") if unsupported_cache_flags: logger.info( "Skipping unsupported Windows cache flags for llama-server: %s", ", ".join(unsupported_cache_flags), ) - # Vulkan pins via --device (a cmd arg, unlike the env-based - # CUDA/ROCm pin below), emitted BEFORE user extras so llama.cpp's - # last-wins parsing lets a user --device override Studio's pick. - if is_vulkan_backend and gpu_indices is not None: - cmd += LlamaCppBackend._vulkan_pin_args(gpu_indices) + # Vulkan pins via --device (a cmd arg), before user extras so a user + # --device wins. Fall back to raw ids when the fit did not narrow. + _vulkan_pin_ids = gpu_indices if gpu_indices is not None else (gpu_ids or None) + + # Record the pin actually applied (fit-narrowed gpu_indices, else the raw + # request) for the keep-warm loop, dedupe, and /status, so an explicit + # [0, 1] narrowed to [0] records [0] and /status never echoes an ordinal + # the child never saw. Auto selection (no gpu_ids) stays None (#7239). + if is_vulkan_backend: + # Only record an EXPLICIT Vulkan pin: an auto pick still narrows + + # pins below, but recording it would misreport an explicit pin and + # make dedupe miss the loaded server; mirrors the CUDA/ROCm branch. + self._gpu_ids = ( + sorted(int(x) for x in _vulkan_pin_ids) + if (gpu_ids and _vulkan_pin_ids) + else None + ) + elif gpu_ids: + # Physical pin: the fit-selected subset when the fit ran, else the raw + # user selection so an explicit choice is honoured even when the fit + # could not size the model. + _effective_pin_ids = ( + [int(x) for x in gpu_indices] + if gpu_indices is not None + else [int(x) for x in gpu_ids] + ) + self._gpu_ids = ( + sorted(int(x) for x in _effective_pin_ids) if _effective_pin_ids else None + ) + else: + self._gpu_ids = None + + # Also record the RAW requested pin (before the fit narrowed it). Load + # dedupe compares this so a [0, 1] narrowed to [0] and re-sent as [0, 1] + # still matches, while /status keeps echoing the effective pin (#7239). + self._requested_gpu_ids = sorted(int(x) for x in gpu_ids) if gpu_ids else None + + if is_vulkan_backend and _vulkan_pin_ids is not None: + cmd += LlamaCppBackend._vulkan_pin_args(_vulkan_pin_ids) # User pass-through args go last so llama.cpp's last-wins parsing - # lets the user override Studio's auto-set flags. Already + # lets the user override Unsloth's auto-set flags. Already # validated by the route via validate_extra_args(). if extra_args: cmd.extend(str(a) for a in extra_args) @@ -6847,15 +8024,17 @@ class LlamaCppBackend: # Library paths so llama-server finds its shared libs and CUDA DLLs. env = self._llama_server_env_for_binary(binary) + if gpu_memory_mode == "manual": + self._clear_manual_placement_env(env) # Omitting --threads relies on llama.cpp's physical-core default, so # drop an inherited LLAMA_ARG_THREADS that would otherwise feed the # arg handler and silently force hardware_concurrency(). #5692 if "--threads" not in cmd: env.pop("LLAMA_ARG_THREADS", None) - # Reconcile the inherited LLAMA_ARG_* env with Studio's final + # Reconcile the inherited LLAMA_ARG_* env with Unsloth's final # decision: stripping CLI extras on a tensor->layer downgrade - # can't remove env vars, so the child could run a mode/KV Studio + # can't remove env vars, so the child could run a mode/KV Unsloth # didn't budget. if not tensor_parallel: # Layer split: clear a non-layer inherited split mode (and any @@ -6865,7 +8044,7 @@ class LlamaCppBackend: env.pop("LLAMA_ARG_SPLIT_MODE", None) env.pop("LLAMA_ARG_TENSOR_SPLIT", None) else: - # Studio owns the tensor split: it emits --tensor-split when it + # Unsloth owns the tensor split: it emits --tensor-split when it # picks an uneven one (CLI wins) and nothing when an even split # is safe. Clear any inherited LLAMA_ARG_TENSOR_SPLIT so the even # case can't be overridden by a stale env (the layer branch above @@ -6901,32 +8080,48 @@ class LlamaCppBackend: f"Data-center GPU detected: applied DC llama.cpp env tuning (multi_gpu={multi_gpu})" ) - # Pin to selected GPU(s). On ROCm, narrowing only - # CUDA_VISIBLE_DEVICES leaves an AMD child seeing the full set, so - # set HIP_VISIBLE_DEVICES too. Vulkan is pinned via --device - # (above), not here. - if gpu_indices is not None and not is_vulkan_backend: - pinned = ",".join(str(i) for i in gpu_indices) - env["CUDA_VISIBLE_DEVICES"] = pinned - try: - import torch as _torch - if getattr(_torch.version, "hip", None) is not None: - env["HIP_VISIBLE_DEVICES"] = pinned - # Do NOT also set ROCR_VISIBLE_DEVICES to the same - # value. ROCR_VISIBLE_DEVICES filters at the HSA/ROCr - # layer and HIP_VISIBLE_DEVICES at the HIP layer, so - # setting both with the same physical indices applies - # the mask twice: ROCR reduces the visible set and - # re-indexes it from 0, then HIP indexes into the - # already-reduced set. A single non-zero pin (e.g. - # "1") then points out of range at the HIP layer, HIP - # enumerates 0 devices, and llama.cpp falls back to - # CPU ("ggml_cuda_init: no ROCm-capable device is - # detected"). The HIP mask alone narrows correctly; - # clear any inherited ROCR mask so it can't double up. - env.pop("ROCR_VISIBLE_DEVICES", None) - except Exception as e: - logger.debug("Failed to set ROCm visibility env vars for child: %s", e) + # Pin to selected GPU(s) (issue #7164; resolved above into gpu_indices). + # On ROCm, narrowing only CUDA_VISIBLE_DEVICES leaves the AMD child + # seeing the full set, so set HIP_VISIBLE_DEVICES too. Vulkan is pinned + # via --device (above), not here. + # A deliberate zero-offload load with no GPU companions runs + # entirely on CPU, yet a visible CUDA device still costs the child + # ~0.5 GB (context + compute scratch) that the CPU-only + # classification below reports as free. Hide the GPUs so the load + # is exactly what it claims: zero VRAM (verified: GPU stays at idle + # baseline and generation runs). Companion loads keep the normal + # masking, and a user device pin (in extras or an inherited + # LLAMA_ARG_DEVICE) keeps control of its own devices -- the child + # aborts on a pin it can't see. The draft-device forms count too: + # llama-server parses them even with no drafter loaded. + _cpu_only_zero_offload = ( + gpu_memory_mode == "manual" + and gpu_layers == 0 + and not is_vulkan_backend + and not self._zero_offload_keeps_gpu_visible(cmd, env) + ) + if _cpu_only_zero_offload: + self._emit_child_gpu_visibility(env, "-1") + elif gpu_indices is not None and not is_vulkan_backend: + # When the user picked GPUs by index, align CUDA's ordering + # with the PCI-bus order the picker enumerated (nvidia-smi), + # so "GPU 1" in the UI is GPU 1 to llama.cpp -- not CUDA's + # default FASTEST_FIRST order (#5025). + if gpu_ids: + env["CUDA_DEVICE_ORDER"] = "PCI_BUS_ID" + # Mask on AMD at the ROCr/HSA layer: HIP-only masking still + # enumerates every agent first, which segfaults on a deselected + # unsupported GPU (e.g. gfx1103 iGPU under a gfx110X prebuilt). + self._emit_child_gpu_visibility( + env, ",".join(str(i) for i in gpu_indices), prefer_rocr = True + ) + elif manual_tensor_split_emitted and not is_vulkan_backend: + # A manual per-GPU ratio across ALL GPUs (no explicit pick, so + # no CUDA_VISIBLE_DEVICES mask above): the UI built the + # --tensor-split list in ascending physical/PCI index order, + # so pin the child's enumeration to that order too. The whole + # visible set stays in use; only its ordering is fixed. + self._pin_visible_gpu_order_for_split(env) # Captured before any text-only fallback strips it from cmd. launched_with_mmproj = "--mmproj" in cmd @@ -6936,7 +8131,7 @@ class LlamaCppBackend: # 'on') even when -ngl is explicit. That step has aborted on # some ROCm hosts (ggml-cuda.cu ROCm error during worst-case # estimation, e.g. MTP + mmproj models on gfx1151). When - # Studio's own VRAM math already placed the model + # Unsloth's own VRAM math already placed the model # (use_fit=False), the step is redundant second-guessing -- # retry once with --fit off before declaring the load failed. # Never retry when fit was requested (use_fit) or the caller @@ -7019,7 +8214,7 @@ class LlamaCppBackend: and _startup_crashed and not _split_axis_crash ): - # We forced --fit off because Studio's (conservative) VRAM + # We forced --fit off because Unsloth's (conservative) VRAM # math placed the model fully on GPU. A startup crash here # means that estimate was optimistic, so fall back to --fit # on and let llama.cpp offload rather than fail the load. @@ -7031,7 +8226,7 @@ class LlamaCppBackend: self._process.returncode, self._llama_log_path, ) - # Flip Studio's own --fit off (added first, before any + # Flip Unsloth's own --fit off (added first, before any # user extra args) to on; a user's later --fit still wins # by last-arg. Defensive: if absent, the default is already # --fit on, so leave it. @@ -7048,7 +8243,7 @@ class LlamaCppBackend: ): logger.warning( "llama-server crashed during startup (exit code %s) " - "with the default memory-fit step enabled; Studio " + "with the default memory-fit step enabled; Unsloth " "already verified the model fits, retrying once " "with --fit off. Crash log: %s", self._process.returncode, @@ -7085,7 +8280,6 @@ class LlamaCppBackend: self._effective_context_length = ( effective_ctx if effective_ctx > 0 else self._context_length ) - self._reconcile_effective_ctx_with_server() self._max_context_length = ( max_available_ctx if max_available_ctx > 0 else self._effective_context_length ) @@ -7128,7 +8322,7 @@ class LlamaCppBackend: cmd = _fa_cmd healthy = _spawn_and_wait(_fa_cmd, label = "-noflash") - # MTP from Studio's spec flags or the user's (extra_args + # MTP from Unsloth's spec flags or the user's (extra_args # --spec-type / LLAMA_ARG_SPEC_TYPE). The env reaches the child # only when neither emits a spec flag, so consult it only then. _launch_spec_env: Mapping[str, str] = ( @@ -7252,24 +8446,34 @@ class LlamaCppBackend: self._kill_process() # The #6415 split-axis abort is latched earlier (first spawn). # Skip if a cancel/unload is pending (mirrors the MTP guard). + _projector_msg = self._is_projector_incompatibility(out) + _signal_mmproj_guess = self._is_signal_crash( + _crash_rc + ) and not self._output_has_nonprojector_diagnostic(out) if ( launched_with_mmproj and not self._cancel_event.is_set() - and ( - self._is_projector_incompatibility(out) - or ( - self._is_signal_crash(_crash_rc) - and not self._output_has_nonprojector_diagnostic(out) - ) - ) + and (_projector_msg or _signal_mmproj_guess) ): - logger.warning( - "llama-server could not load this model's vision " - "projector (--mmproj). The installed llama.cpp build is " - "likely too old for it. Loading text-only for this " - "session; run 'unsloth studio update' to enable vision." - ) + if _projector_msg: + logger.warning( + "llama-server could not load this model's vision " + "projector (--mmproj). The installed llama.cpp build is " + "likely too old for it. Loading text-only for this " + "session; run 'unsloth studio update' to enable vision." + ) + else: + logger.warning( + "llama-server crashed while loading this model's vision " + "projector (--mmproj). Retrying text-only for this " + "session; if this persists, run 'unsloth studio update' " + "or check GPU/driver logs." + ) cmd = self._strip_mmproj_args(_last_spawn_cmd) + # This retry bypasses _spawn_and_wait, so refresh the + # launched-argv snapshot itself -- the zero-offload + # classification below must not see the stripped --mmproj. + _last_spawn_cmd = list(cmd) self._is_vision = False self._mmproj_has_audio = False self._start_llama_process(cmd, env) @@ -7278,14 +8482,30 @@ class LlamaCppBackend: # an OS-killed text-only retry still gets the OOM message. _retry_rc = self._process.poll() if self._process is not None else None self._kill_process() + # If the text-only retry ALSO hard-crashed (a signal, not + # OOM/timeout), the vision projector was never the cause: + # llama-server is faulting during GPU/driver init. Say so + # -- with the ROCm fix -- instead of blaming the mmproj. + if self._is_signal_crash(_retry_rc): + raise RuntimeError( + "llama-server crashed at startup on both the vision " + "and text-only attempts -- a GPU driver/runtime " + "initialization crash, not a model or vision-projector " + "problem. This often means an unsupported secondary " + "GPU; on AMD/ROCm, hide it with ROCR_VISIBLE_DEVICES " + "(e.g. ROCR_VISIBLE_DEVICES=0 exposes only the first " + "GPU) before launching Unsloth Studio." + ) + _retry_detail = self._classify_llama_start_failure( + "\n".join(self._stdout_lines[-50:]), + gguf_path, + self._model_identifier, + _retry_rc, + ) raise RuntimeError( - "Vision projector incompatible with this llama.cpp " - "build, and the text-only retry also failed: " - + self._classify_llama_start_failure( - "\n".join(self._stdout_lines[-50:]), - gguf_path, - self._model_identifier, - _retry_rc, + self._mmproj_retry_failure_message( + projector_confirmed = _projector_msg, + detail = _retry_detail, ) ) else: @@ -7301,6 +8521,13 @@ class LlamaCppBackend: self._healthy = True self._commit_effective_parallel_slots(n_parallel) + # Server is up: adopt the real per-request context it allocated + # -- the length --fit chose, or a --parallel slot split -- so the + # reported context_length matches reality. (Querying /props + # before the spawn above always failed; the seeded value was the + # requested/native length.) + self._reconcile_effective_ctx_with_server() + # Commit caller intent only after _healthy=True so a failed start # can't poison the next inheritance check. None keeps prior, [] # clears, list sets. Source records hf_variant for the route's @@ -7315,18 +8542,31 @@ class LlamaCppBackend: self._mtp_runtime_fallback_active = _mtp_active_for_launched_server self._start_mtp_crash_watchdog() - # Catch silent CPU fallback when GPU was intended (#5106). - self._gpu_offload_active = self._classify_gpu_offload( - gpu_indices is not None or use_fit, gpus or [] - ) - if self._gpu_offload_active is False: + # Catch silent CPU fallback when GPU was intended (#5106). Manual + # offload (no picker) leaves gpu_indices None and use_fit False, so + # include its GPU-layer intent; use the preserved probe since + # auto-layers/manual empty `gpus`. A deliberate zero-offload load + # classifies by its launched argv instead: the main model is + # CPU-only by construction and must read False (not None), or + # training needlessly unloads a server holding no VRAM. + _deliberate_cpu_only = gpu_memory_mode == "manual" and gpu_layers == 0 + if _deliberate_cpu_only: + self._gpu_offload_active = self._zero_offload_gpu_flag( + _last_spawn_cmd, _detected_gpus, env + ) + else: + self._gpu_offload_active = self._classify_gpu_offload( + gpu_indices is not None or use_fit or gpu_memory_mode == "manual", + _detected_gpus, + ) + if self._gpu_offload_active is False and not _deliberate_cpu_only: logger.warning( "llama-server appears to have loaded the model entirely " - "on CPU even though Studio detected at least one GPU. " + "on CPU even though Unsloth detected at least one GPU. " "This usually means the prebuilt binary's GPU backend " "failed to load -- on Windows, cudart64_X.dll / " "cublas64_X.dll could not be resolved. Reinstall the " - "Studio llama.cpp prebuilt or install a matching CUDA " + "Unsloth llama.cpp prebuilt or install a matching CUDA " "toolkit (issue unslothai/unsloth#5106).", ) @@ -7362,6 +8602,15 @@ class LlamaCppBackend: if not self._healthy: return False + # Snapshot the files the server actually loaded. If a GGUF shard or a + # LoRA/control-vector sidecar is swapped on disk afterwards while the + # old weights stay mapped, save_slots_for_resume() compares against + # this and refuses to persist KV that a reload could misapply. + if self._slot_save_dir: + self._slot_loaded_identity = ( + self._gguf_file_identity(self._gguf_path), + self._slot_launch_fingerprint(), + ) return True def _build_speculative_flags( @@ -7486,18 +8735,29 @@ class LlamaCppBackend: caps = self.probe_server_capabilities(binary) mtp_token = caps.get("mtp_token") if caps else None if not mtp_token: - logger.warning( - "Requested MTP speculative decoding but " - "llama-server lacks --spec-type mtp/draft-mtp; " - "run `unsloth studio update`. Loading without " - "speculative decoding." - ) + inconclusive = bool(caps.get("mtp_probe_inconclusive")) if caps else True + if inconclusive: + logger.info( + "Requested MTP speculative decoding but llama-server MTP " + "capability probe was inconclusive; loading without " + "speculative decoding." + ) + else: + logger.warning( + "Requested MTP speculative decoding but " + "llama-server lacks --spec-type mtp/draft-mtp; " + "run `unsloth studio update`. Loading without " + "speculative decoding." + ) # Override an inherited LLAMA_ARG_SPEC_TYPE=draft-mtp (CLI wins # over env) so the child matches the binary-capability gate and # the no-MTP budget, like the sibling no-head/non-MTP fallbacks. flags.append("--spec-default") self._speculative_type = "default" - self._spec_fallback_reason = "binary_no_mtp" + if inconclusive: + self._spec_fallback_reason = None + else: + self._spec_fallback_reason = "binary_no_mtp" return False draft_n_max = _resolved_draft_n_max() n_max_flag = caps.get("spec_draft_n_max_flag") or "--spec-draft-n-max" @@ -7623,7 +8883,7 @@ class LlamaCppBackend: logger.info( "Auto: MLA embedded-MTP model detected; llama.cpp's MLA/DSA " "MTP path is slower than no speculation, so using ngram-mod " - "instead. Override via the Studio Speculative Decoding " + "instead. Override via the Unsloth Speculative Decoding " "dropdown or UNSLOTH_MLA_MTP_ENABLED=1." ) _emit_ngram_mod() @@ -7651,7 +8911,7 @@ class LlamaCppBackend: f"MTP GGUF detected but model size {_mtp_size_b:.1f}B " "is below the 3B speedup threshold; using ngram-mod " "only (zero-VRAM, no draft head). Override via " - "--spec-type or the Studio Speculative Decoding " + "--spec-type or the Unsloth Speculative Decoding " "dropdown." ) _emit_ngram_mod() @@ -7682,6 +8942,11 @@ class LlamaCppBackend: gguf_path: Optional[str] = None, spec_draft_n_max: Optional[int] = None, tensor_parallel: bool = False, + gpu_memory_mode: Literal["auto", "manual"] = "auto", + gpu_layers: int = -1, + n_cpu_moe: int = 0, + tensor_split: Optional[List[float]] = None, + gpu_ids: Optional[List[int]] = None, mtp_draft_path: Optional[str] = None, preserve_multi_gpu_on_layer: bool = False, ) -> bool: @@ -7738,6 +9003,32 @@ class LlamaCppBackend: ): return False + # The diffusion runner is mode-agnostic (always "auto", ignores the + # layer/MoE/split knobs), so a standing manual preference in the + # request must not force a needless reload -- only the GPU pick matters. + if not self._is_diffusion: + # A GPU-memory-mode flip (Unsloth / manual) must always reload. + if self._gpu_memory_mode != gpu_memory_mode: + return False + # Manual: a layer-count change always reloads (covers Auto(-1) <-> a + # pinned count); MoE/split only matter with an explicit offload. + if gpu_memory_mode == "manual" and ( + self._gpu_layers != gpu_layers + or ( + gpu_layers >= 0 + and ( + self._n_cpu_moe != n_cpu_moe + or (self._tensor_split or None) != (tensor_split or None) + ) + ) + ): + return False + # A changed GPU pick must reload. Regular GGUF accepts either the raw + # requested placement pool or the effective status-echoed subset; + # diffusion compares its normalized single-device pick. + if not self.matches_gpu_ids(gpu_ids): + return False + # Compare on the canonical requested mode. With --spec-type in # extra_args the backend stores None; mirror that here. if _extra_args_set_spec_type(extra_args): @@ -7793,6 +9084,7 @@ class LlamaCppBackend: current = list(self._extra_args) if self._extra_args is not None else [] if list(extra_args) != current: return False + self._record_matching_gpu_request(gpu_ids) return True def _classify_gpu_offload( @@ -7806,6 +9098,78 @@ class LlamaCppBackend: return None return classify_gpu_offload_lines(self._stdout_lines) + @staticmethod + def _cmd_has_gpu_companion(cmd: list, env: Optional[Mapping[str, str]] = None) -> bool: + """True when the argv/env carries a GPU companion: any --mmproj form, or + a drafter (Studio's --model-draft, the extras aliases, or the + LLAMA_ARG_SPEC_DRAFT_* env) -- these offload to the GPU regardless of + the main ``--gpu-layers``. A drafter explicitly forced to CPU + (--spec-draft-ngl 0 / --spec-draft-device cpu) doesn't count.""" + if any(str(a).startswith("--mmproj") for a in cmd): + return True + if _extra_args_mtp_draft_path(cmd, env) is None: + return False + return not _extra_args_draft_offloaded_to_cpu(cmd, env) + + @staticmethod + def _zero_offload_keeps_gpu_visible(cmd: list, env: Optional[Mapping[str, str]] = None) -> bool: + """Whether a zero-layer launch still has a reason to use visible GPUs. + + Keep this shared by child masking and post-launch residency bookkeeping: + a device pin, surviving tensor mode, mmproj, or GPU drafter prevents the + launch from being a confirmed zero-VRAM server. + """ + return ( + LlamaCppBackend._cmd_has_gpu_device_pin(cmd, env) + or _effective_tensor_parallel(cmd, False, env) + or LlamaCppBackend._cmd_has_gpu_companion(cmd, env) + ) + + @staticmethod + def _cmd_has_gpu_device_pin(cmd: list, env: Optional[Mapping[str, str]] = None) -> bool: + """True when the effective main or draft ``--device`` pin names a GPU.""" + main_flags = {"--device", "-dev"} + draft_flags = {"--spec-draft-device", "-devd", "--device-draft"} + last_main: Optional[str] = None + last_draft: Optional[str] = None + args = [str(arg) for arg in cmd] + for index, raw in enumerate(args): + flag, equals, inline = raw.partition("=") + if flag not in main_flags and flag not in draft_flags: + continue + value = inline if equals else (args[index + 1] if index + 1 < len(args) else "") + if flag in main_flags: + last_main = value + else: + last_draft = value + if last_main is None: + last_main = (env or {}).get("LLAMA_ARG_DEVICE") + + def _names_gpu(value: Optional[str]) -> bool: + if value is None: + return False + devices = [item.strip().lower() for item in value.split(",") if item.strip()] + return not devices or any(item not in ("cpu", "none") for item in devices) + + return _names_gpu(last_main) or _names_gpu(last_draft) + + @staticmethod + def _zero_offload_gpu_flag( + spawn_cmd: list, + detected_gpus: list, + env: Optional[Mapping[str, str]] = None, + ) -> Optional[bool]: + """GPU-residency flag for a deliberate manual zero-offload load. The + main model is CPU-only by construction, but device pins, tensor mode, + mmproj, and GPU drafters can still make the server hold VRAM. The counted + offload classifier cannot see those allocations. This uses the same + predicate as the launch-time zero-VRAM mask; None means no GPU signal.""" + if not detected_gpus: + return None + if LlamaCppBackend._is_vulkan_backend(): + return True + return LlamaCppBackend._zero_offload_keeps_gpu_visible(spawn_cmd, env) + def load_cancelled(self) -> bool: """True if a load was cancelled (e.g. via unload/_cancel_event) and not yet consumed by the next load_model. Lets the tensor->layer fallback @@ -7838,6 +9202,10 @@ class LlamaCppBackend: self._effective_context_length = None self._max_context_length = None self._reset_effective_parallel_slots() + self._slot_save_dir = None + self._slot_save_binary = None + self._slot_loaded_identity = None + self._prompt_cache_disabled = False self._chat_template = None self._chat_template_override = None self._supports_reasoning = False @@ -7848,12 +9216,22 @@ class LlamaCppBackend: self._supports_preserve_thinking = False self._supports_tools = False self._cache_type_kv = None + # GPU-pin state describes the active runner only; clear it so an explicit + # pin never leaks into the next (or diffusion) runner. + self._gpu_ids = None + self._requested_gpu_ids = None self._tensor_parallel = False + self._gpu_memory_mode = "auto" + self._gpu_layers = -1 + self._n_cpu_moe = 0 + self._tensor_split = None self._layer_preserves_tensor_intent = False self._speculative_type = None self._requested_spec_mode = None self._spec_draft_n_max = None self._n_layers = None + self._n_experts = None + self._leading_dense_block_count = None self._n_kv_heads = None self._n_kv_heads_by_layer = None self._n_heads = None @@ -7916,6 +9294,10 @@ class LlamaCppBackend: # Clear healthy so a /load during the replacement's warm-up can't # short-circuit against the previous server's health (#5401). self._healthy = False + # Reset to unknown so the training guard treats the next (still + # loading) server as VRAM-resident rather than reading the killed + # server's stale zero-offload flag until the health probe reclassifies. + self._gpu_offload_active = None # Drives _wait_for_vram_settle in the next load_model; set in finally # so both in-process and frontend Apply paths record the kill. self._last_kill_monotonic = time.monotonic() @@ -8029,7 +9411,7 @@ class LlamaCppBackend: def _pid_parent_is_alive(pid: int) -> bool: """True if the recorded server's parent is still running, i.e. the server is NOT orphaned. Lets the cross-session reap kill only a true orphan (parent - gone) and never a live server owned by a running Studio, regardless of which + gone) and never a live server owned by a running Unsloth, regardless of which process performs the sweep. Biased toward "alive" on uncertainty so a live server is never mistakenly reaped.""" try: @@ -8069,9 +9451,9 @@ class LlamaCppBackend: @classmethod def _reap_recorded_pid(cls) -> int: """Kill the exact llama-server PID recorded at spawn, but only when it is a - genuine orphan -- its parent (the Studio that spawned it) is gone. This is + genuine orphan -- its parent (the Unsloth that spawned it) is gone. This is the cross-session backstop the parent-death reaper (Job Object / - PR_SET_PDEATHSIG) cannot cover: an orphan left by an already-dead Studio + PR_SET_PDEATHSIG) cannot cover: an orphan left by an already-dead Unsloth (macOS, a best-effort failure, or a pre-existing orphan). Path-independent, so it also catches an orphan the install-root match would miss. @@ -8128,7 +9510,7 @@ class LlamaCppBackend: """Kill orphaned llama-server processes started by studio. Only kills processes whose resolved binary lives under a known - Studio install dir (or matches an exact env-var override), to avoid + Unsloth install dir (or matches an exact env-var override), to avoid terminating unrelated llama-server instances. Mirrors every location _find_llama_server_binary() can return, so orphans from any supported install path are cleaned up. @@ -8148,7 +9530,7 @@ class LlamaCppBackend: try: # -- Build the ownership allowlist -------------------------------- # exact_binaries -- env var overrides (exact path match). - # install_roots -- Studio-owned dir trees (binary must be under one). + # install_roots -- Unsloth-owned dir trees (binary must be under one). install_roots: list[Path] = [] # Env-mode custom root (mirrors _find_llama_server_binary). @@ -8158,7 +9540,7 @@ class LlamaCppBackend: install_roots.append(_resolved_sr / "llama.cpp") # Primary install dir (default mode only). Env-mode skips this so a - # custom-root Studio can't kill a default-install Studio's server. + # custom-root Unsloth can't kill a default-install Unsloth's server. if not _is_custom_root: install_roots.append(Path.home() / ".unsloth" / "llama.cpp") @@ -8232,7 +9614,7 @@ class LlamaCppBackend: if not is_ours: continue - # A live parent means a running Studio (or the user's + # A live parent means a running Unsloth (or the user's # shell) still owns it -- not an orphan. if LlamaCppBackend._pid_parent_is_alive(proc.info["pid"]): continue @@ -8312,7 +9694,7 @@ class LlamaCppBackend: def _fit_off_retry_eligible(cmd: "list[str]", use_fit: bool) -> bool: """Whether a llama-server startup crash may be retried with --fit off. - Only when Studio's own VRAM math placed the model (use_fit=False) + Only when Unsloth's own VRAM math placed the model (use_fit=False) and nothing on the command line set the fit mode explicitly (-fit / --fit, space- or equals-form). --fit-ctx / --fit-target / -fitc / -fitt tune the fit step but do not select the mode, so @@ -8353,6 +9735,237 @@ class LlamaCppBackend: return False return True + def _slot_launch_fingerprint(self) -> tuple: + # KV validity keys on extra args, stat'd sidecar weights, effective ctx. + sidecars = [] + for path in self._sidecar_weight_files(): + try: + st = os.stat(path) + sidecars.append((path, st.st_size, st.st_mtime_ns)) + except OSError: + sidecars.append((path, None, None)) + return ( + tuple(self._extra_args or ()), + tuple(sidecars), + self._requested_n_ctx, + self._effective_context_length, + getattr(self, "_cache_type_kv", None), + self.effective_parallel_slots, + ) + + def _gguf_file_identity(self, path) -> Optional[tuple]: + # (size, mtime_ns) per shard: a split GGUF keys KV validity on every sibling. + p = Path(path) + paths = [p] + m = _SHARD_FULL_RE.match(p.name) + if m: + prefix, _first, total = m.groups() + paths = [ + p.with_name(f"{prefix}-{i:05d}-of-{total}{p.suffix}") + for i in range(1, int(total) + 1) + ] + try: + return tuple((sp.stat().st_size, sp.stat().st_mtime_ns) for sp in paths) + except OSError: + return None + + _SIDECAR_WEIGHT_FLAGS = ( + "--lora", + "--lora-scaled", + "--control-vector", + "--control-vector-scaled", + ) + + def _sidecar_weight_files(self) -> list[str]: + # llama.cpp: comma-separated paths, FNAME:SCALE on -scaled (older builds: FNAME SCALE). + args = [str(a).strip() for a in (self._extra_args or ())] + files: list[str] = [] + for i, arg in enumerate(args): + flag, sep, inline = arg.partition("=") + if flag not in self._SIDECAR_WEIGHT_FLAGS: + continue + operand = inline if sep else (args[i + 1] if i + 1 < len(args) else "") + if not operand: + continue + candidates = [operand] + pieces = [p for p in operand.split(",") if p] + if len(pieces) > 1: + candidates.extend(pieces) + if flag.endswith("-scaled"): + for item in list(candidates): + # ":" tail is a scale; rpartition spares drive letters. + head, colon, tail = item.rpartition(":") + if not (colon and head): + continue + try: + float(tail) + except ValueError: + continue + candidates.append(head) + for cand in candidates: + if cand not in files: + files.append(cand) + return files + + def _prompt_cache_off(self) -> bool: + # Caching off makes restores useless; last prompt-cache flag wins, env only when unset. + last = None + for arg in self._extra_args or (): + flag = arg.strip().split("=", 1)[0] + if flag in ("--cache-prompt", "--no-cache-prompt"): + last = flag + if last is not None: + return last == "--no-cache-prompt" + if self._prompt_cache_disabled: + return True + if os.environ.get("LLAMA_ARG_NO_CACHE_PROMPT") is not None: + return True + env = (os.environ.get("LLAMA_ARG_CACHE_PROMPT") or "").strip().lower() + return env in {"off", "disabled", "false", "0"} + + def save_slots_for_resume( + self, should_abort: Optional[Callable[[], bool]] = None + ) -> Optional[dict]: + if ( + not self.is_loaded + or not self._slot_save_dir + or not self._gguf_path + or self._prompt_cache_off() + ): + return None + save_dir = Path(self._slot_save_dir) + gguf_stat = self._gguf_file_identity(self._gguf_path) + if gguf_stat is None: + return None + launch = self._slot_launch_fingerprint() + # If the GGUF or a sidecar was swapped on disk while the original weights + # stayed mapped, the live KV belongs to the old weights but a reload would + # load the new file. Persisting it would let restore misapply stale KV. + if self._slot_loaded_identity is not None and self._slot_loaded_identity != ( + gguf_stat, + launch, + ): + logger.debug("Skipping slot save: model files changed on disk since load") + return None + try: + estimate = self._estimate_kv_cache_bytes( + self._effective_context_length or self._context_length or 0, + self._cache_type_kv, + n_parallel = self.effective_parallel_slots, + ) + # Skip before writing anything when the estimate alone blows the cap, + # rather than fully writing a slot and discarding it afterwards. + if estimate > _SLOT_SAVE_MAX_BYTES: + logger.debug( + "Skipping slot save: estimated %d bytes exceeds cap %d", + estimate, + _SLOT_SAVE_MAX_BYTES, + ) + return None + # A 0 estimate means metadata was insufficient, not a zero-byte cache: + # a slot can still be many GiB, so demand room for the whole cap before + # trusting the post-write check. + required = (estimate if estimate > 0 else _SLOT_SAVE_MAX_BYTES) + (1 << 30) + if shutil.disk_usage(save_dir).free < required: + logger.debug("Skipping slot save: insufficient free disk") + return None + except Exception: + pass + token = uuid.uuid4().hex[:8] + entries: list[dict] = [] + total_bytes = 0 + for slot in range(self.effective_parallel_slots): + # A request pending mid-save waits on the gate; stop wasting its time. + if should_abort is not None and should_abort(): + break + filename = f"resume-{token}-slot{slot}.bin" + path = save_dir / filename + try: + resp = httpx.post( + f"{self.base_url}/slots/{slot}", + params = {"action": "save"}, + json = {"filename": filename}, + headers = self._auth_headers, + timeout = _SLOT_SAVE_HTTP_TIMEOUT, + trust_env = False, + ) + except Exception as e: + logger.debug(f"slot {slot} save failed: {e}") + with contextlib.suppress(OSError): + path.unlink() + break + if resp.status_code != 200: + logger.debug(f"slot {slot} save returned HTTP {resp.status_code}") + with contextlib.suppress(OSError): + path.unlink() + continue + try: + body = resp.json() + if not isinstance(body, dict): + raise ValueError("slot save response was not a JSON object") + n_saved = int(body.get("n_saved") or 0) + except Exception as e: + # A 200 that still wrote a file but returns a malformed body must + # clean up like the transport/HTTP error paths above, or the file + # (which holds chat KV) is orphaned until the next startup sweep. + logger.debug(f"slot {slot} save returned an invalid response: {e}") + with contextlib.suppress(OSError): + path.unlink() + continue + if n_saved <= 0: + with contextlib.suppress(OSError): + path.unlink() + continue + # Account by the bytes actually on disk, not the server-reported + # count, so the cap holds even if a custom binary under-reports. + try: + n_written = path.stat().st_size + except OSError: + n_written = 0 + total_bytes += n_written + entries.append({"id": slot, "filename": filename, "n_saved": n_saved}) + if total_bytes > _SLOT_SAVE_MAX_BYTES: + break # already over the cap; the discard below cleans up + if not entries: + return None + if total_bytes > _SLOT_SAVE_MAX_BYTES: + logger.debug( + "Discarding slot save: %d bytes exceeds cap %d", + total_bytes, + _SLOT_SAVE_MAX_BYTES, + ) + for entry in entries: + with contextlib.suppress(OSError): + (save_dir / entry["filename"]).unlink() + return None + return { + "dir": self._slot_save_dir, + "binary": self._slot_save_binary, + "gguf": str(self._gguf_path), + "gguf_stat": gguf_stat, + "launch": launch, + "slots": entries, + } + + def restore_slots_for_resume(self, manifest: dict) -> None: + if not self.is_loaded or not self._slot_save_dir: + return + for entry in manifest.get("slots") or []: + try: + resp = httpx.post( + f"{self.base_url}/slots/{int(entry['id'])}", + params = {"action": "restore"}, + json = {"filename": str(entry["filename"])}, + headers = self._auth_headers, + timeout = _SLOT_SAVE_HTTP_TIMEOUT, + trust_env = False, + ) + except Exception as e: + logger.debug(f"slot restore failed: {e}") + break + if resp.status_code != 200: + logger.debug(f"slot {entry.get('id')} restore returned HTTP {resp.status_code}") + def _maybe_recover_from_mtp_crash(self, exc: Optional[BaseException] = None) -> bool: """Schedule one background reload without MTP after a mid-generation death. @@ -8520,7 +10133,12 @@ class LlamaCppBackend: @staticmethod def _ctx_integrity_flags( - n_parallel: int, use_fit: bool, requested_ctx: int, effective_ctx: int, caps: dict + n_parallel: int, + use_fit: bool, + auto_fit: bool, + requested_ctx: int, + effective_ctx: int, + caps: dict, ) -> list[str]: """Flags that keep the per-request window equal to the advertised ctx. @@ -8528,14 +10146,28 @@ class LlamaCppBackend: ``--kv-unified`` default, silently splitting ``-c`` into per-slot windows of ``-c / N``; restore the shared pool so one request can use the full context. With ``--fit on``, ``--fit-ctx`` floors the fit step - at an explicitly requested ctx (default floor is 4096) so it offloads - or fails instead of silently shrinking the window. + at an explicitly requested ctx so it offloads or fails instead of + silently shrinking the window. The 8192 auto-floor and the tighter + ``--fit-target`` margin apply only under Manual + Auto (``auto_fit``), + which omits ``-c``: on the legacy auto path ``-c 0`` already pins the + native window and ``--fit-ctx 8192`` would override it down to 8192. """ flags: list[str] = [] if n_parallel > 1 and caps.get("supports_kv_unified"): flags.append("--kv-unified") - if use_fit and requested_ctx > 0 and effective_ctx > 0 and caps.get("supports_fit_ctx"): - flags.extend(["--fit-ctx", str(effective_ctx)]) + if use_fit and caps.get("supports_fit_ctx"): + if requested_ctx > 0 and effective_ctx > 0: + # Floor the fit step at the explicitly requested ctx. + flags.extend(["--fit-ctx", str(effective_ctx)]) + elif auto_fit: + # Manual + Auto omits -c, so floor at 8192 so --fit doesn't + # shrink the window below a usable size. + flags.extend(["--fit-ctx", "8192"]) + if use_fit and auto_fit and caps.get("supports_fit_target"): + # llama.cpp's --fit leaves 1 GiB free per device by default; + # tighten that to 512 MiB so it packs more of the model onto + # the GPU before spilling to system RAM. + flags.extend(["--fit-target", "512"]) return flags def _query_server_n_ctx(self) -> Optional[int]: @@ -8556,7 +10188,7 @@ class LlamaCppBackend: return None def _reconcile_effective_ctx_with_server(self) -> None: - """Adopt the server's real ``n_ctx`` when it is below Studio's value. + """Adopt the server's real ``n_ctx`` when it is below Unsloth's value. Keeps ``context_length`` (load response, status route, passthrough ``max_tokens`` ceiling) honest; clients sized to the requested value @@ -8732,6 +10364,75 @@ class LlamaCppBackend: except Exception: logger.debug("Could not close httpx client", exc_info = True) + @staticmethod + def _install_cancel_aware_read( + client: "httpx.Client", + cancel_event: threading.Event, + response: Optional["httpx.Response"] = None, + poll_s: float = 0.2, + ) -> None: + """Wrap the httpcore stream so the reader interrupts its own blocked recv() on cancel. + + A cross-thread socket shutdown wakes a parked recv() on POSIX but not on + Windows (Winsock), so read in short slices and poll cancel_event between them + (plain or TLS); slice timeouts are swallowed so a slow-but-alive stream survives. + httpcore snapshots request.extensions["timeout"]["read"] once at body start, so + given ``response`` we re-read the live value per call to honor the post-first-token + stall timeout instead of the long prefill timeout.""" + import httpcore + + def _live_read_timeout() -> Optional[float]: + if response is None: + return None + try: + ext = response.request.extensions.get("timeout") + if isinstance(ext, dict): + value = ext.get("read") + if isinstance(value, (int, float)): + return float(value) + except Exception: + pass + return None + + try: + pool = getattr(getattr(client, "_transport", None), "_pool", None) + for connection in list(getattr(pool, "_connections", []) or []): + inner = getattr(connection, "_connection", None) + stream = getattr(inner, "_network_stream", None) + if stream is None or getattr(stream, "_unsloth_cancel_wrapped", False): + continue + orig_read = stream.read + + def read( + max_bytes, + timeout = None, + _orig = orig_read, + ): + live = _live_read_timeout() + effective = live if live is not None else timeout + deadline = None if effective is None else time.monotonic() + effective + while True: + if cancel_event.is_set(): + raise httpcore.ReadError("stream cancelled by user") + if deadline is None: + step = poll_s + else: + remaining = deadline - time.monotonic() + if remaining <= 0: + raise httpcore.ReadTimeout("read operation timed out") + step = min(poll_s, remaining) + try: + return _orig(max_bytes, timeout = step) + except httpcore.ReadTimeout: + if deadline is not None and time.monotonic() >= deadline: + raise + continue # slow but alive: keep reading + + stream.read = read + stream._unsloth_cancel_wrapped = True + except Exception: + logger.debug("Could not install cancel-aware read", exc_info = True) + @staticmethod @contextlib.contextmanager def _stream_with_retry( @@ -8789,6 +10490,11 @@ class LlamaCppBackend: headers = headers, ) as response: _response_ref[0] = response + if cancel_event is not None: + # Portable mid-stream cancel: the reader polls cancel itself, so + # Stop interrupts a stalled read where the watcher's Windows socket + # shutdown does not. Pass response to honor the live stall timeout. + LlamaCppBackend._install_cancel_aware_read(client, cancel_event, response) if cancel_event is not None and cancel_event.is_set(): raise _LlamaStreamCancelled yield response diff --git a/studio/backend/core/inference/llama_keepwarm.py b/studio/backend/core/inference/llama_keepwarm.py index 4ce663c3ce..3380ebf5f5 100644 --- a/studio/backend/core/inference/llama_keepwarm.py +++ b/studio/backend/core/inference/llama_keepwarm.py @@ -15,6 +15,7 @@ import asyncio import contextlib import threading import time +from pathlib import Path from loggers import get_logger @@ -30,6 +31,8 @@ _last_active = time.monotonic() # otherwise 503 against an empty backend can reload it (set on unload, cleared on # reload). Storing the quant means the reload restores the exact freed variant. _last_unloaded_model = None +# Slot KV manifest saved by the idle unload; whoever pops it owns deleting its files. +_kv_resume = None # Guards inflight bumps against the idle-check-then-unload race, and blocks new # inference from starting mid-swap. Process-wide, not per-loop: the backend slot is # shared across every event loop in the process, so a per-loop gate would let a @@ -59,7 +62,7 @@ _INFERENCE_SUFFIXES = ( "/messages/count_tokens", # counts via the loaded tokenizer; protect like /messages "/embeddings", "/responses", - "/generate/stream", # Studio's own streaming route on the same llama-server + "/generate/stream", # Unsloth's own streaming route on the same llama-server "/audio/generate", # direct GGUF TTS; can outlive the idle TTL ) @@ -161,11 +164,17 @@ def inference_lifecycle_gate(): return _unload_gate() -def note_model_loaded() -> None: - """Record a successful GGUF load: stamp activity and drop any reload stash so - a manual load clears it synchronously, not only on the next idle poll.""" +def note_model_loaded(backend = None) -> None: + """Stamp activity and synchronously drop any reload stash.""" _note_activity() + resume = take_kv_resume() _set_last_unloaded(None) + if resume is None: + return + if backend is not None: + restore_kv_resume(backend, resume) + else: + _delete_resume_files(resume) def note_model_unloaded() -> None: @@ -182,9 +191,81 @@ def get_last_unloaded_model(): def _set_last_unloaded(value) -> None: - global _last_unloaded_model + global _last_unloaded_model, _kv_resume + stale = None with _lock: _last_unloaded_model = value + if value is None and _kv_resume is not None: + stale, _kv_resume = _kv_resume, None + if stale: + _delete_resume_files(stale) + + +def _delete_resume_files(manifest) -> None: + try: + base = Path(manifest.get("dir") or "") + for entry in manifest.get("slots") or []: + with contextlib.suppress(OSError): + (base / str(entry.get("filename"))).unlink() + except Exception: + pass + + +def _set_kv_resume(value) -> None: + global _kv_resume + stale = None + with _lock: + if _kv_resume is not None and _kv_resume is not value: + stale = _kv_resume + _kv_resume = value + if stale: + _delete_resume_files(stale) + + +def take_kv_resume(): + global _kv_resume + with _lock: + manifest, _kv_resume = _kv_resume, None + return manifest + + +def purge_kv_resume() -> None: + resume = take_kv_resume() + if resume: + _delete_resume_files(resume) + + +def restore_kv_resume(backend, manifest) -> None: + try: + gguf = manifest.get("gguf") + binary = manifest.get("binary") + current = getattr(backend, "_gguf_path", None) + same_gguf = bool(gguf and current) and Path(current).resolve() == Path(gguf).resolve() + if same_gguf: + # Same path is not enough: shards may have been rewritten meanwhile. + identity = getattr(backend, "_gguf_file_identity", None) + same_gguf = callable(identity) and identity(current) == manifest.get("gguf_stat") + if same_gguf: + # Nor the same file: launch overrides can invalidate KV numerics. + fingerprint = getattr(backend, "_slot_launch_fingerprint", None) + same_gguf = callable(fingerprint) and manifest.get("launch") == fingerprint() + if same_gguf and binary and binary == getattr(backend, "_slot_save_binary", None): + logger.info("Restoring saved slot KV onto the reloaded model") + backend.restore_slots_for_resume(manifest) + except Exception as exc: + logger.debug("slot restore after reload failed: %s", exc) + finally: + _delete_resume_files(manifest) + + +def sweep_slot_save_dir() -> None: + try: + from utils.paths.storage_roots import llama_slot_cache_root + for path in llama_slot_cache_root().glob("resume-*.bin"): + with contextlib.suppress(OSError): + path.unlink() + except Exception: + pass class LlamaKeepWarmMiddleware: @@ -266,7 +347,10 @@ def _loaded_identity(backend): async def idle_unload_loop(poll_seconds: float = 15.0) -> None: """Unload the loaded GGUF once idle past the configured TTL. Inert when off.""" - from utils.openai_auto_switch_settings import get_auto_unload_idle_seconds + from utils.openai_auto_switch_settings import ( + get_auto_unload_idle_seconds, + get_auto_unload_keep_kv, + ) seen_model = None while True: @@ -281,17 +365,47 @@ async def idle_unload_loop(poll_seconds: float = 15.0) -> None: # Track by (id, variant): a (re)loaded model -- including the same repo # at a different quant -- counts as activity so it survives one TTL # before its first request (loads bypass the activity middleware). - current = _loaded_identity(backend) - if current != seen_model: - seen_model = current - if current is not None: - _note_activity() - _set_last_unloaded(None) # a model is loaded; drop stale stash async with _unload_gate(): + # Purging the stash mid-reload would race the restore. + current = _loaded_identity(backend) + if current != seen_model: + seen_model = current + if current is not None: + _note_activity() + _set_last_unloaded(None) # a model is loaded; drop stale stash if backend.is_loaded and _is_idle(ttl): freed = _loaded_identity(backend) - await asyncio.to_thread(backend.unload_model) + manifest = None + if get_auto_unload_keep_kv(): + try: + manifest = await asyncio.to_thread( + backend.save_slots_for_resume, + lambda: not _is_idle(ttl), + ) + except Exception as exc: + logger.debug("slot save before idle unload failed: %s", exc) + # Re-read settings: the save can outlive a settings change. + ttl = get_auto_unload_idle_seconds() + if ttl <= 0 or not _is_idle(ttl): + if manifest: + _delete_resume_files(manifest) + continue + if manifest and not get_auto_unload_keep_kv(): + _delete_resume_files(manifest) + manifest = None + try: + await asyncio.to_thread(backend.unload_model) + except Exception: + # Failed unload means nothing will stash the manifest. + if manifest: + _delete_resume_files(manifest) + raise _set_last_unloaded(freed) # let an alias request reload it + if manifest and freed: + _set_kv_resume({"identity": freed, **manifest}) + logger.info("Idle auto-unload: saved slot KV for restore on reload") + elif manifest: + _delete_resume_files(manifest) logger.info("Idle auto-unload: freed GGUF after %ss idle", ttl) seen_model = None except Exception as exc: diff --git a/studio/backend/core/inference/llama_server_args.py b/studio/backend/core/inference/llama_server_args.py index f400d2ae40..7b42d2f40d 100644 --- a/studio/backend/core/inference/llama_server_args.py +++ b/studio/backend/core/inference/llama_server_args.py @@ -3,10 +3,10 @@ """Boundary validator for user-supplied llama-server pass-through args. -Reject only flags Studio manages (model identity, auth, network, parallel +Reject only flags Unsloth manages (model identity, auth, network, parallel slots). Everything else (sampling, ``-c``, ``-ngl``, ``--flash-attn``, ``--cache-type-*``, ``--spec-*``, ``--jinja``, ...) is appended after -Studio's auto-set flags so llama.cpp's last-wins parser lets the user override. +Unsloth's auto-set flags so llama.cpp's last-wins parser lets the user override. Ref: https://github.com/ggml-org/llama.cpp/blob/master/tools/server/README.md """ @@ -22,12 +22,12 @@ _DENYLIST_GROUPS: tuple[frozenset[str], ...] = ( # Parallel slots: owned by typer --parallel; a pass-through would desync # app.state.llama_parallel_slots from llama-server. frozenset({"-np", "--parallel", "--n-parallel"}), - # Model identity: Studio resolves it from LoadRequest; a second -m would - # load a different model than Studio thinks it loaded. + # Model identity: Unsloth resolves it from LoadRequest; a second -m would + # load a different model than Unsloth thinks it loaded. frozenset({"-m", "--model"}), - # Public model id: Studio sets a sanitized --alias so the OpenAI API never + # Public model id: Unsloth sets a sanitized --alias so the OpenAI API never # exposes the local .gguf path. A user-supplied alias is appended after - # Studio's and, with llama.cpp's last-wins parsing, would reintroduce the + # Unsloth's and, with llama.cpp's last-wins parsing, would reintroduce the # path leak this is meant to prevent. frozenset({"-a", "--alias"}), frozenset({"-mu", "--model-url"}), @@ -39,14 +39,14 @@ _DENYLIST_GROUPS: tuple[frozenset[str], ...] = ( frozenset({"-hft", "--hf-token"}), frozenset({"-mm", "--mmproj"}), frozenset({"-mmu", "--mmproj-url"}), - # Networking: Studio binds + proxies; retargeting orphans the proxy. + # Networking: Unsloth binds + proxies; retargeting orphans the proxy. frozenset({"--host"}), frozenset({"--port"}), frozenset({"--path"}), frozenset({"--api-prefix"}), frozenset({"--reuse-port"}), - # Auth / TLS: Studio terminates auth; upstream --api-key / TLS shadows - # Studio's key and breaks the proxy hop. + # Auth / TLS: Unsloth terminates auth; upstream --api-key / TLS shadows + # Unsloth's key and breaks the proxy hop. frozenset({"--api-key"}), frozenset({"--api-key-file"}), frozenset({"--ssl-key-file"}), @@ -64,12 +64,14 @@ _DENYLIST_GROUPS: tuple[frozenset[str], ...] = ( frozenset({"--models-max"}), frozenset({"--models-autoload", "--no-models-autoload"}), # Server-mode flips: --embedding / --rerank restrict llama-server to - # those endpoints, breaking Studio's /v1/chat/completions hop. + # those endpoints, breaking Unsloth's /v1/chat/completions hop. frozenset({"--embedding", "--embeddings"}), frozenset({"--rerank", "--reranking"}), # llama-server's own built-in tools flag would silently stack on top of - # Studio's --enable-tools / --disable-tools policy resolver. + # Unsloth's --enable-tools / --disable-tools policy resolver. frozenset({"--tools"}), + # Slot-state dir: Studio owns it for KV persistence across idle unload. + frozenset({"--slot-save-path"}), ) _DENYLIST: frozenset[str] = frozenset().union(*_DENYLIST_GROUPS) @@ -120,7 +122,7 @@ def validate_extra_args(args: Optional[Iterable[str]]) -> list[str]: def is_managed_flag(flag: str) -> bool: - """True if ``flag`` is Studio-managed. Normalises via ``_flag_name`` so + """True if ``flag`` is Unsloth-managed. Normalises via ``_flag_name`` so `-np8` / `--parallel=8` classify like the canonical tokens.""" normalised = _flag_name(flag) return normalised is not None and normalised in _DENYLIST @@ -142,7 +144,7 @@ _SPEC_FLAGS: frozenset[str] = frozenset( "--draft-min", "--draft-max", # MTP path (llama.cpp #22673). The drafter selectors (local --model-draft - # and HF --spec-draft-hf aliases) are Studio-managed since the separate- + # and HF --spec-draft-hf aliases) are Unsloth-managed since the separate- # drafter support (Gemma 4): an inherited copy must not last-wins-override # the auto-detected drafter. Explicit extras for the current load are never # stripped. The per-drafter tuning knobs (--spec-draft-type-*, -ngld, @@ -179,25 +181,38 @@ _TEMPLATE_FLAGS: frozenset[str] = frozenset( # (--split-mode tensor). Pass-through stays allowed so users keep the # row/none/layer modes the toggle doesn't expose, but it's stripped on # inherit and reconciled into the round-tripped tensor_parallel state. -# --tensor-split is coupled to the split mode and is stripped with it: Studio +# --tensor-split is coupled to the split mode and is stripped with it: Unsloth # owns the tensor-mode split ratios, so an inherited/stale --tensor-split must -# not last-wins-override Studio's computed asymmetric split. +# not last-wins-override Unsloth's computed asymmetric split. _SPLIT_MODE_FLAGS: frozenset[str] = frozenset({"-sm", "--split-mode"}) _TENSOR_SPLIT_FLAGS: frozenset[str] = frozenset({"-ts", "--tensor-split"}) _SPLIT_SHADOWING_FLAGS: frozenset[str] = _SPLIT_MODE_FLAGS | _TENSOR_SPLIT_FLAGS +# GPU-offload flags. Stripped only when the GPU Memory mode owns offload +# (manual emits --fit / --gpu-layers / --n-cpu-moe); in auto, a user's +# inherited -ngl is respected (the offload_overridden path), so this group is +# opt-in, not default. Layer flags are shared with llama_cpp's override +# detection; the MoE flags are strip-only (manual's --n-cpu-moe slider owns them). +_LAYER_OFFLOAD_FLAGS: frozenset[str] = frozenset( + {"-ngl", "--gpu-layers", "--n-gpu-layers", "-fit", "--fit"} +) +_MOE_OFFLOAD_FLAGS: frozenset[str] = frozenset({"-ncmoe", "--n-cpu-moe", "-cmoe", "--cpu-moe"}) +_OFFLOAD_SHADOWING_FLAGS: frozenset[str] = _LAYER_OFFLOAD_FLAGS | _MOE_OFFLOAD_FLAGS + _SHADOWING_FLAGS: frozenset[str] = ( _CONTEXT_FLAGS | _CACHE_FLAGS | _SPEC_FLAGS | _TEMPLATE_FLAGS | _SPLIT_SHADOWING_FLAGS ) # Shadowing flags that take no value -- strip the flag only, not the next token. -_BOOLEAN_SHADOWING_FLAGS: frozenset[str] = frozenset({"--spec-default", "--jinja", "--no-jinja"}) +_BOOLEAN_SHADOWING_FLAGS: frozenset[str] = frozenset( + {"--spec-default", "--jinja", "--no-jinja", "-cmoe", "--cpu-moe"} +) def parse_ctx_override(args: Optional[Iterable[str]]) -> Optional[int]: """Return the last user-supplied ``-c`` / ``--ctx-size`` value. - Mirrors llama.cpp's last-wins parsing for the one numeric knob Studio's + Mirrors llama.cpp's last-wins parsing for the one numeric knob Unsloth's load-time fit logic needs. """ if not args: @@ -286,7 +301,7 @@ def parse_cache_override(args: Optional[Iterable[str]]) -> Optional[str]: Mirrors parse_ctx_override but for cache type. Recognises both -ctk (key) and -ctv (value). When both flags appear, returns the last-wins value, treating key and value cache flags as the same setting because - Studio's KV estimate has a single cache_type_kv knob. + Unsloth's KV estimate has a single cache_type_kv knob. """ return _last_flag_value(args, _CACHE_FLAGS) @@ -341,7 +356,7 @@ def resolve_tensor_parallel(args: Optional[Iterable[str]], fallback_tensor_paral def _env_split_mode_is_tensor(env: Optional[Mapping[str, str]] = None) -> bool: - """True when the inherited LLAMA_ARG_SPLIT_MODE env selects tensor. Studio + """True when the inherited LLAMA_ARG_SPLIT_MODE env selects tensor. Unsloth emits --split-mode only on its tensor branch, so a tensor env on the layer path would run the child tensor-parallel unbudgeted; this flips the budget to tensor. Only tensor is heavier, so other modes are ignored.""" @@ -424,14 +439,22 @@ def strip_shadowing_flags( strip_spec: bool = True, strip_template: bool = True, strip_split_mode: bool = True, + strip_tensor_split: bool = False, + strip_offload: bool = False, ) -> list[str]: - """Strip flags that shadow first-class Studio settings. + """Strip flags that shadow first-class Unsloth settings. Used when inheriting a previous load's ``llama_extra_args`` so an inherited `-c 4096` can't override the current `max_seq_length` (same for cache / spec / template / split-mode). Each ``strip_*`` toggle controls one group; the route only strips groups whose first-class field the caller actually supplied. + + ``strip_split_mode`` removes both ``--split-mode`` and the coupled + ``--tensor-split`` (the Tensor Parallelism toggle owns the whole split). + ``strip_tensor_split`` removes ``--tensor-split`` *alone*, so manual mode can + replace an inherited per-GPU ratio while leaving the user's ``--split-mode`` + row/none/layer choice intact. """ shadowing: set[str] = set() if strip_context: @@ -444,6 +467,10 @@ def strip_shadowing_flags( shadowing |= _TEMPLATE_FLAGS if strip_split_mode: shadowing |= _SPLIT_SHADOWING_FLAGS + if strip_tensor_split: + shadowing |= _TENSOR_SPLIT_FLAGS + if strip_offload: + shadowing |= _OFFLOAD_SHADOWING_FLAGS tokens = [str(a) for a in (args or [])] out: list[str] = [] diff --git a/studio/backend/core/inference/llama_stats.py b/studio/backend/core/inference/llama_stats.py index 6047aedbc0..ab0d287e8c 100644 --- a/studio/backend/core/inference/llama_stats.py +++ b/studio/backend/core/inference/llama_stats.py @@ -5,7 +5,7 @@ engine-stats log line (generation/prompt throughput, requests in flight). llama-server already computes these (it needs `--metrics`); this lifts them -into Studio's structured log so the terminal shows serving health, not just +into Unsloth's structured log so the terminal shows serving health, not just per-request access lines. Emitted only while there is activity. """ diff --git a/studio/backend/core/inference/local_model_resolver.py b/studio/backend/core/inference/local_model_resolver.py index 002cafe2c8..9e3eaeda3f 100644 --- a/studio/backend/core/inference/local_model_resolver.py +++ b/studio/backend/core/inference/local_model_resolver.py @@ -130,7 +130,7 @@ def info_has_local_gguf(info) -> bool: def _build_index() -> dict[str, _LocalGgufEntry]: """Map normalized id/model_id/display_name -> local GGUF entry. - Scans the same roots Studio's model picker lists (./models, the active plus + Scans the same roots Unsloth's model picker lists (./models, the active plus legacy/default HF caches, LM Studio dirs, and user scan folders) so a named local model is never missed and silently served as the loaded one. Ollama's scanner is skipped: it creates symlinks as a side effect and this runs on the @@ -146,6 +146,7 @@ def _build_index() -> dict[str, _LocalGgufEntry]: _is_hidden_model, ) from utils.paths import legacy_hf_cache_dir, hf_default_cache_dir, lmstudio_model_dirs + from utils.hf_cache_settings import known_hf_hub_caches index: dict[str, _LocalGgufEntry] = {} seen_hf: set[str] = set() @@ -174,7 +175,12 @@ def _build_index() -> dict[str, _LocalGgufEntry]: except Exception as exc: logger.debug("auto-switch: ./models scan failed: %s", exc) try: - for hf_dir in (_resolve_hf_cache_dir(), legacy_hf_cache_dir(), hf_default_cache_dir()): + for hf_dir in ( + *known_hf_hub_caches(), + _resolve_hf_cache_dir(), + legacy_hf_cache_dir(), + hf_default_cache_dir(), + ): found += _scan_hf_once(hf_dir) except Exception as exc: logger.debug("auto-switch: HF cache scan failed: %s", exc) @@ -199,9 +205,13 @@ def _build_index() -> dict[str, _LocalGgufEntry]: raw_id = getattr(info, "id", None) if not raw_id: continue - # Skip what Studio hides from its pickers (validation probe, RAG embed + # Skip what Unsloth hides from its pickers (validation probe, RAG embed # weights): not chat models, so never an auto-switch target. - if _is_hidden_model(raw_id, getattr(info, "path", None)): + if _is_hidden_model( + raw_id, + getattr(info, "model_id", None), + getattr(info, "path", None), + ): continue # Advertise a client-facing alias, not an absolute filesystem path. loader_id = _advertised_loader_id(info) diff --git a/studio/backend/core/inference/mcp_client.py b/studio/backend/core/inference/mcp_client.py index 6b5ce02216..0256df944e 100644 --- a/studio/backend/core/inference/mcp_client.py +++ b/studio/backend/core/inference/mcp_client.py @@ -906,7 +906,7 @@ def _call_stdio_tool( def _remaining() -> Optional[float]: return None if deadline is None else max(0.0, deadline - time.monotonic()) - # Callers without a Studio session id must retain the former one-shot + # Callers without an Unsloth session id must retain the former one-shot # behavior: no browser/cookie/tool state can leak into another request. # Use an ephemeral key (and close it below) rather than the shared empty # scope that the persistent-session cache used previously. diff --git a/studio/backend/core/inference/mlx_inference.py b/studio/backend/core/inference/mlx_inference.py index 163ade10c4..d19c67a01a 100644 --- a/studio/backend/core/inference/mlx_inference.py +++ b/studio/backend/core/inference/mlx_inference.py @@ -8,14 +8,76 @@ instead of torch/transformers for model loading and generation. import json import os import threading +from contextlib import contextmanager from typing import Optional, Generator from core.inference.message_content import content_to_text from core.inference.runtime_context import runtime_context_length +from core.inference.chat_template_helpers import ( + ReasoningChannelNormalizer, + normalize_reasoning_snapshots, +) from loggers import get_logger logger = get_logger(__name__) +def _mlx_adapter_modules(model): + """Return bypassable adapter entries and unsupported wrapper paths.""" + adapters = [] + unsupported = [] + for path, module in model.named_modules(): + if not path or not (hasattr(module, "lora_a") and hasattr(module, "lora_b")): + continue + base = getattr(module, "linear", None) + if base is None: + base = getattr(module, "embedding", None) + if base is None: + unsupported.append(path) + else: + adapters.append((path, module, base)) + return adapters, unsupported + + +@contextmanager +def _temporary_mlx_adapter_state(model, use_adapter): + """Select base or adapter modules for one request, then restore the tree.""" + if use_adapter is None: + yield + return + if isinstance(use_adapter, str): + raise NotImplementedError( + "Unsloth MLX: named adapter selection is not supported; use True for " + "the loaded adapter or False for the base model." + ) + if use_adapter is not True and use_adapter is not False: + raise TypeError("Unsloth MLX: use_adapter must be None, True, False, or a string.") + + adapters, unsupported = _mlx_adapter_modules(model) + if use_adapter is True: + if not adapters and not unsupported: + logger.warning("MLX adapter requested, but the active model has no adapter layers") + yield + return + if unsupported: + raise RuntimeError( + "Unsloth MLX: cannot disable adapter layers without their base modules: " + + ", ".join(unsupported[:5]) + ) + if not adapters: + yield + return + + from mlx.utils import tree_unflatten + + base_modules = tree_unflatten([(path, base) for path, _, base in adapters]) + adapter_modules = tree_unflatten([(path, wrapper) for path, wrapper, _ in adapters]) + try: + model.update_modules(base_modules) + yield + finally: + model.update_modules(adapter_modules) + + def _mlx_vlm_model_config(model): """Return the loaded MLX model config and its type, preferring whichever of config / _config actually carries a model_type.""" @@ -119,19 +181,27 @@ def _vlm_messages_have_tool_history(messages): ) -def _build_generation_stats(prompt_n, prompt_tps, gen_n, gen_tps): +def _build_generation_stats( + prompt_n, + prompt_tps, + gen_n, + gen_tps, + cached_n = 0, +): """Map mlx stream stats onto the usage/timings shape llama-server emits.""" prompt_n = int(prompt_n or 0) gen_n = int(gen_n or 0) + cached_n = int(cached_n or 0) prompt_tps = float(prompt_tps or 0.0) gen_tps = float(gen_tps or 0.0) prompt_ms = (prompt_n / prompt_tps * 1000.0) if prompt_tps > 0 else 0.0 predicted_ms = (gen_n / gen_tps * 1000.0) if gen_tps > 0 else 0.0 + total_prompt_n = prompt_n + cached_n return { "usage": { - "prompt_tokens": prompt_n, + "prompt_tokens": total_prompt_n, "completion_tokens": gen_n, - "total_tokens": prompt_n + gen_n, + "total_tokens": total_prompt_n + gen_n, }, "timings": { "prompt_n": prompt_n, @@ -142,11 +212,123 @@ def _build_generation_stats(prompt_n, prompt_tps, gen_n, gen_tps): "predicted_ms": predicted_ms, "predicted_per_token_ms": (predicted_ms / gen_n) if gen_n > 0 else 0.0, "predicted_per_second": gen_tps, - "cache_n": 0, + "cache_n": cached_n, }, } +PROMPT_CACHE_ENTRIES = 6 +PROMPT_CACHE_MEMORY_FRACTION = 0.15 +PROMPT_CACHE_FALLBACK_BYTES = 2 * 1024**3 + + +def _mlx_prompt_cache_api(): + try: + from mlx_lm.models.cache import ( + LRUPromptCache, + can_trim_prompt_cache, + make_prompt_cache, + trim_prompt_cache, + ) + except ImportError: + return None + return LRUPromptCache, make_prompt_cache, can_trim_prompt_cache, trim_prompt_cache + + +def _prompt_cache_max_bytes(recommended_gb = None): + override = os.environ.get("UNSLOTH_MLX_PROMPT_CACHE_BYTES") + if override: + try: + return max(int(override), 0) + except ValueError: + logger.warning("Ignoring non-integer UNSLOTH_MLX_PROMPT_CACHE_BYTES=%r", override) + if recommended_gb: + return int(recommended_gb * 1e9 * PROMPT_CACHE_MEMORY_FRACTION) + return PROMPT_CACHE_FALLBACK_BYTES + + +def _flatten_kv_entries(cache): + for entry in cache: + nested = getattr(entry, "caches", None) + if nested is None: + yield entry + else: + yield from _flatten_kv_entries(nested) + + +def _kv_prefix_coverage(cache): + covered = None + for entry in _flatten_kv_entries(cache): + offset = getattr(entry, "offset", None) + if offset is None: + return None + if getattr(entry, "start_position", 0): + return None + window = getattr(entry, "max_size", None) + if window is not None and offset > window: + return None + if covered is None: + covered = offset + elif covered != offset: + return None + return covered + + +class _MLXPromptCacheHistory: + def __init__(self, max_entries, max_bytes): + api = _mlx_prompt_cache_api() + if api is None: + raise RuntimeError("mlx-lm is too old for LRUPromptCache") + lru_cls, make, can_trim, trim = api + self._make_prompt_cache = make + self._can_trim = can_trim + self._trim = trim + self._max_bytes = max_bytes + self._lru = lru_cls(max_size = max_entries, max_bytes = max_bytes) + + def fetch(self, model, key, tokens): + cache, rest = self._lru.fetch_nearest_cache(key, list(tokens)) + if cache is not None: + if rest: + return cache, list(rest) + if self._can_trim(cache) and self._trim(cache, 1) == 1: + return cache, list(tokens[-1:]) + if len(tokens) > 1: + head = list(tokens[:-1]) + cache, rest = self._lru.fetch_nearest_cache(key, head) + if cache is not None: + covered = len(head) - len(rest) + return cache, list(tokens[covered:]) + return self._make_prompt_cache(model), list(tokens) + + def insert(self, key, tokens, cache): + # An over-budget entry evicts itself and every other conversation. + nbytes = sum(getattr(entry, "nbytes", 0) for entry in cache) + if nbytes > self._max_bytes: + logger.debug( + "MLX prompt cache: skipping %.2f GB entry over the %.2f GB budget", + nbytes / 1e9, + self._max_bytes / 1e9, + ) + return + covered = _kv_prefix_coverage(cache) + if covered is None: + logger.debug("MLX prompt cache: skipping cache with unverifiable prefix coverage") + return + tokens = list(tokens) + if covered > len(tokens): + logger.debug( + "MLX prompt cache: cache covers %d tokens but only %d were tracked", + covered, + len(tokens), + ) + return + tokens = tokens[:covered] + if not tokens: + return + self._lru.insert_cache(key, tokens, cache) + + def _mlx_distributed_rank_size(group = None): """Return ``(rank, world_size)`` for an optional MLX distributed group.""" if group is None: @@ -251,6 +433,55 @@ class MLXInferenceBackend: # Recorded for unload to release pinned memory back to the OS. self._memory_limits_applied = {} + self._prompt_cache_history = None + self._prompt_cache_unavailable = False + + def _prompt_cache(self): + if self._prompt_cache_history is not None or self._prompt_cache_unavailable: + return self._prompt_cache_history + max_bytes = _prompt_cache_max_bytes(self._memory_limits_applied.get("recommended_gb")) + if max_bytes <= 0: + self._prompt_cache_unavailable = True + logger.info("MLX prompt cache disabled by budget") + return None + try: + self._prompt_cache_history = _MLXPromptCacheHistory( + PROMPT_CACHE_ENTRIES, + max_bytes, + ) + except Exception as exc: + self._prompt_cache_unavailable = True + logger.info("MLX prompt cache unavailable (%s); prefilling every request", exc) + return None + logger.info( + "MLX prompt cache: %d entries, %.2f GB budget", + PROMPT_CACHE_ENTRIES, + max_bytes / 1e9, + ) + return self._prompt_cache_history + + def _clear_prompt_cache(self): + self._prompt_cache_history = None + self._prompt_cache_unavailable = False + + def _prepare_prompt_cache(self, prompt, adapter_state): + history = self._prompt_cache() + if history is None: + return prompt, None, None, None, 0 + try: + tokenizer = self._tokenizer + bos = getattr(tokenizer, "bos_token", None) + add_special_tokens = bos is None or not prompt.startswith(bos) + tokens = list(tokenizer.encode(prompt, add_special_tokens = add_special_tokens)) + if not tokens: + return prompt, None, None, None, 0 + key = f"{self.active_model_name}|{adapter_state!r}" + cache, rest = history.fetch(self._model, key, tokens) + except Exception as exc: + logger.debug("MLX prompt cache lookup failed: %s", exc) + return prompt, None, None, None, 0 + return rest, cache, key, tokens, len(tokens) - len(rest) + def _configure_memory_limits(self): """Apply Metal memory caps before loading a model. @@ -473,6 +704,7 @@ class MLXInferenceBackend: self._distributed_world_size = 1 if self.active_model_name == model_name: self.active_model_name = None + self._clear_prompt_cache() gc.collect() mx.clear_cache() @@ -504,6 +736,7 @@ class MLXInferenceBackend: reasoning_effort = None, preserve_thinking = None, presence_penalty = 0.0, + _adapter_state = None, ) -> Generator[str, None, None]: if self._model is None: raise RuntimeError("No model loaded") @@ -533,7 +766,7 @@ class MLXInferenceBackend: break if self._is_vlm: - yield from self._generate_vlm( + stream = self._generate_vlm( full_messages, image, temperature, @@ -548,9 +781,10 @@ class MLXInferenceBackend: reasoning_effort = reasoning_effort, preserve_thinking = preserve_thinking, presence_penalty = presence_penalty, + _adapter_state = _adapter_state, ) else: - yield from self._generate_text( + stream = self._generate_text( full_messages, temperature, top_p, @@ -564,7 +798,9 @@ class MLXInferenceBackend: reasoning_effort = reasoning_effort, preserve_thinking = preserve_thinking, presence_penalty = presence_penalty, + _adapter_state = _adapter_state, ) + yield from stream def _generate_text( self, @@ -582,6 +818,7 @@ class MLXInferenceBackend: reasoning_effort = None, preserve_thinking = None, presence_penalty = 0.0, + _adapter_state = None, ): from mlx_lm import stream_generate from mlx_lm.sample_utils import make_sampler, make_logits_processors @@ -609,7 +846,7 @@ class MLXInferenceBackend: # probe and native render share a renderer. (VLM renders via the # processor for image tokens and is not wired here.) model_info = self.models.get(self.active_model_name, {}) - prompt = render_with_native_template_fallback( + render_result = render_with_native_template_fallback( formatted_prompt = prompt, tokenizer = self._tokenizer, model_info = model_info, @@ -620,17 +857,16 @@ class MLXInferenceBackend: reasoning_effort = reasoning_effort, preserve_thinking = preserve_thinking, hf_token = model_info.get("hf_token"), + return_metadata = True, ) + prompt = render_result.prompt + reasoning_channel_markers = render_result.reasoning_channel_markers # An open prefilled by the template lives in the prompt, not # the generated tokens; re-emit it so the frontend renders the block. think_prefix = detect_think_prefill( prompt, getattr(self._tokenizer, "all_special_tokens", None) ) - # Emit it before the first token so the block renders during prefill. - if think_prefix: - yield think_prefix - sampler = make_sampler( temp = temperature, top_p = top_p, @@ -654,22 +890,45 @@ class MLXInferenceBackend: if not logits_processors: logits_processors = None + preserve_native_channels = reasoning_channel_markers is not None token_ids = [] - logger.info( - "Generating: prompt_len=%d, max_tokens=%d, model=%s, tokenizer=%s", - len(prompt), - max_new_tokens, - type(self._model).__name__, - type(self._tokenizer).__name__, + normalizer = ( + ReasoningChannelNormalizer(*reasoning_channel_markers) + if reasoning_channel_markers is not None + else None ) - with self._generation_lock: + # MLX consumers diff cumulative snapshots. Keep a prompt-prefilled + # prefix on every native-protocol snapshot just as the normal + # decoding path does below. + normalized_output = think_prefix + with self._generation_lock, _temporary_mlx_adapter_state(self._model, _adapter_state): + ( + gen_prompt, + prompt_cache, + cache_key, + prompt_tokens, + cached_n, + ) = self._prepare_prompt_cache(prompt, _adapter_state) + logger.info( + "Generating: prompt_len=%d, cached=%d, max_tokens=%d, model=%s, tokenizer=%s", + len(prompt), + cached_n, + max_new_tokens, + type(self._model).__name__, + type(self._tokenizer).__name__, + ) final_response = None try: + # Enter request-scoped model state before yielding any response. + if think_prefix: + yield think_prefix gen_kwargs = dict( - prompt = prompt, + prompt = gen_prompt, max_tokens = max_new_tokens, sampler = sampler, ) + if prompt_cache is not None: + gen_kwargs["prompt_cache"] = prompt_cache if logits_processors is not None: gen_kwargs["logits_processors"] = logits_processors for response in stream_generate( @@ -679,14 +938,28 @@ class MLXInferenceBackend: ): final_response = response token_ids.append(response.token) - cumulative = self._tokenizer.decode( - token_ids, - skip_special_tokens = True, - ) - yield think_prefix + cumulative + if preserve_native_channels: + piece = getattr(response, "text", None) or "" + delta = normalizer.feed(piece) + if delta: + normalized_output += delta + yield normalized_output + else: + cumulative = self._tokenizer.decode( + token_ids, + skip_special_tokens = True, + ) + yield think_prefix + cumulative if cancel_event and cancel_event.is_set(): break + if prompt_cache is not None and prompt_tokens is not None: + history = self._prompt_cache_history + if history is not None: + try: + history.insert(cache_key, prompt_tokens + token_ids, prompt_cache) + except Exception as exc: + logger.debug("MLX prompt cache insert failed: %s", exc) except Exception as e: import traceback logger.error("stream_generate failed:\n%s", traceback.format_exc()) @@ -699,7 +972,14 @@ class MLXInferenceBackend: getattr(final_response, "prompt_tps", 0.0), getattr(final_response, "generation_tokens", 0), getattr(final_response, "generation_tps", 0.0), + cached_n, ) + if normalizer is not None: + cancelled = cancel_event is not None and cancel_event.is_set() + tail = normalizer.drain() if cancelled else normalizer.finish() + if tail: + normalized_output += tail + yield normalized_output def _generate_vlm( self, @@ -718,6 +998,7 @@ class MLXInferenceBackend: reasoning_effort = None, preserve_thinking = None, presence_penalty = 0.0, + _adapter_state = None, ): from mlx_vlm import stream_generate as vlm_stream @@ -821,9 +1102,6 @@ class MLXInferenceBackend: # Re-emit an open prefill from the prompt (see _generate_text). cumulative = detect_think_prefill(prompt, getattr(chat_target, "all_special_tokens", None)) - # Emit it before the first token so the block renders during prefill. - if cumulative: - yield cumulative logger.info( "VLM generating: prompt_len=%d, has_image=%s", len(prompt), @@ -858,31 +1136,46 @@ class MLXInferenceBackend: elif _rep_active: vlm_kwargs["repetition_penalty"] = float(repetition_penalty) - with self._generation_lock: - final_response = None - try: - for response in vlm_stream( - self._model, - self._processor, - prompt, - images, - **vlm_kwargs, - ): - final_response = response - token_text = response.text if hasattr(response, "text") else str(response) - cumulative += token_text - yield cumulative - if cancel_event and cancel_event.is_set(): - break - finally: - # mlx_vlm exposes the same stats fields as mlx_lm. - if final_response is not None: - self.last_generation_stats = _build_generation_stats( - getattr(final_response, "prompt_tokens", 0), - getattr(final_response, "prompt_tps", 0.0), - getattr(final_response, "generation_tokens", 0), - getattr(final_response, "generation_tps", 0.0), - ) + def _stream_vlm_snapshots(): + nonlocal cumulative + # Hold the generation lock AND the request-scoped adapter state for the + # whole stream so Base-vs-LoRA compare mode honors use_adapter and the + # wrapper tree is restored on completion, cancellation, or close. + with self._generation_lock, _temporary_mlx_adapter_state(self._model, _adapter_state): + final_response = None + try: + # Emit any prefilled block before the first token so the + # UI renders it during prefill, matching _generate_text. Done + # inside the adapter context so an unsupported request raises + # before any output escapes. + if cumulative: + yield cumulative + for response in vlm_stream( + self._model, + self._processor, + prompt, + images, + **vlm_kwargs, + ): + final_response = response + token_text = response.text if hasattr(response, "text") else str(response) + cumulative += token_text + yield cumulative + if cancel_event and cancel_event.is_set(): + break + finally: + # mlx_vlm exposes the same stats fields as mlx_lm. + if final_response is not None: + self.last_generation_stats = _build_generation_stats( + getattr(final_response, "prompt_tokens", 0), + getattr(final_response, "prompt_tps", 0.0), + getattr(final_response, "generation_tokens", 0), + getattr(final_response, "generation_tps", 0.0), + ) + + yield from normalize_reasoning_snapshots( + _stream_vlm_snapshots(), chat_target, cancel_event, tools = tools + ) def generate_with_adapter_control( self, @@ -890,8 +1183,11 @@ class MLXInferenceBackend: cancel_event = None, **gen_kwargs, ) -> Generator[str, None, None]: - # MLX LoRA adapter toggling not yet supported; generate normally - yield from self.generate_chat_response(cancel_event = cancel_event, **gen_kwargs) + yield from self.generate_chat_response( + cancel_event = cancel_event, + _adapter_state = use_adapter, + **gen_kwargs, + ) def reset_generation_state(self): import mlx.core as mx diff --git a/studio/backend/core/inference/orchestrator.py b/studio/backend/core/inference/orchestrator.py index c2082bc198..616384386d 100644 --- a/studio/backend/core/inference/orchestrator.py +++ b/studio/backend/core/inference/orchestrator.py @@ -27,7 +27,7 @@ import uuid from io import BytesIO from pathlib import Path from typing import Any, Generator, Optional, Tuple, Union -from utils.hardware import prepare_gpu_selection +from utils.hardware import get_device, prepare_gpu_selection # Re-exported from the shared helper so GGUF, training, and inference share one # type; kept importable here for backwards compatibility. @@ -54,12 +54,36 @@ class GenStreamError(str): """A stream chunk carrying a real backend/generation error, not model text. Subclasses str so existing display/logging consumers are unaffected, while - callers that must abort a distributed run on error (raise_on_streamed_error) - can distinguish a real error from model output whose visible text starts with - "Error:" by checking isinstance(chunk, GenStreamError). + callers can distinguish a real error from model output whose visible text + starts with "Error:" by checking isinstance(chunk, GenStreamError). """ - __slots__ = () + __slots__ = ("public",) + + def __new__( + cls, + value, + *, + public: bool = False, + ): + obj = str.__new__(cls, value) + obj.public = bool(public) + return obj + + +class GenStreamErrorRaised(RuntimeError): + """Internal exception form of ``GenStreamError`` for generator boundaries.""" + + __slots__ = ("public",) + + def __init__( + self, + value, + *, + public: bool = False, + ): + super().__init__(value) + self.public = bool(public) class InferenceOrchestrator: @@ -193,10 +217,14 @@ class InferenceOrchestrator: native_path_secret_removed_for_child_start, run_without_native_path_secret, ) + from utils.hf_cache_settings import child_environment_for_spawn, get_hf_cache_paths - from .worker import run_inference_process + cache_env = get_hf_cache_paths().child_env({}) - with native_path_secret_removed_for_child_start(): + with ( + child_environment_for_spawn(cache_env), + native_path_secret_removed_for_child_start(), + ): self._cmd_queue = _CTX.Queue() self._resp_queue = _CTX.Queue() self._cancel_event = _CTX.Event() @@ -204,7 +232,7 @@ class InferenceOrchestrator: self._proc = _CTX.Process( target = run_without_native_path_secret, - args = (run_inference_process,), + args = ("core.inference.worker", "run_inference_process", cache_env), kwargs = { "cmd_queue": self._cmd_queue, "resp_queue": self._resp_queue, @@ -531,13 +559,19 @@ class InferenceOrchestrator: initial_resp_queue = self._resp_queue while True: if self._proc is not initial_proc or self._resp_queue is not initial_resp_queue: - yield GenStreamError(f"Error: {self._subprocess_crash_message(crash_context)}") + yield GenStreamError( + f"Error: {self._subprocess_crash_message(crash_context)}", + public = True, + ) return resp = read_one(read_timeout) if resp is None: # Check subprocess health if not self._ensure_subprocess_alive(): - yield GenStreamError(f"Error: {self._subprocess_crash_message(crash_context)}") + yield GenStreamError( + f"Error: {self._subprocess_crash_message(crash_context)}", + public = True, + ) return continue @@ -689,11 +723,11 @@ class InferenceOrchestrator: GPU work stays serialized; this only avoids orchestrator lock contention. """ if not self._ensure_subprocess_alive(): - yield GenStreamError("Error: Inference subprocess is not running") + yield GenStreamError("Error: Inference subprocess is not running", public = True) return if not self.active_model_name: - yield GenStreamError("Error: No active model") + yield GenStreamError("Error: No active model", public = True) return # Latch the target model so the recheck below can detect a switch that completed # between _start_dispatcher and mailbox registration (mirrors the locked path's @@ -704,7 +738,7 @@ class InferenceOrchestrator: # so without this early-out a compare request would enqueue a generate on the # outgoing model and delay the switch. if self._unload_pending: - yield GenStreamError("Error: model is being unloaded") + yield GenStreamError("Error: model is being unloaded", public = True) return # Ensure the dispatcher runs. _start_dispatcher serializes concurrent starters under @@ -776,7 +810,7 @@ class InferenceOrchestrator: # _stop_dispatcher joins the dispatcher, which itself takes that lock. if orphaned_dispatcher: self._stop_dispatcher() - yield GenStreamError("Error: model is being unloaded") + yield GenStreamError("Error: model is being unloaded", public = True) return try: @@ -978,6 +1012,8 @@ class InferenceOrchestrator: ) sub_config["resolved_gpu_ids"] = resolved_gpu_ids sub_config["gpu_selection"] = gpu_selection + # Parent-detected backend for the worker's apply_gpu_ids(). + sub_config["device_backend"] = get_device().value # Recheck the sidecar reservation BEFORE tearing the old worker down, # for REPAIRS only: an install holds this same lifecycle gate, so it @@ -1376,6 +1412,7 @@ class InferenceOrchestrator: use_adapter: Optional[Union[bool, str]] = None, stats_holder: Optional[dict] = None, presence_penalty: float = 0.0, + reasoning_prefilled: bool = False, **_unused, ): """Run the safetensors agentic tool loop in the parent process, @@ -1414,12 +1451,27 @@ class InferenceOrchestrator: presence_penalty = presence_penalty, ) if use_adapter is not None: - yield from self.generate_with_adapter_control( + stream = self.generate_with_adapter_control( use_adapter = use_adapter, **common_kwargs, ) else: - yield from self.generate_chat_response(**common_kwargs) + stream = self.generate_chat_response(**common_kwargs) + close_stream = False + try: + for chunk in stream: + if isinstance(chunk, GenStreamError): + close_stream = True + raise GenStreamErrorRaised(str(chunk), public = chunk.public) + yield chunk + finally: + if close_stream: + close = getattr(stream, "close", None) + if callable(close): + try: + close() + except Exception: + logger.debug("failed to close errored generation stream", exc_info = True) initial = list(messages) if system_prompt: @@ -1441,6 +1493,7 @@ class InferenceOrchestrator: confirm_tool_calls = confirm_tool_calls, bypass_permissions = bypass_permissions, permission_mode = permission_mode, + reasoning_prefilled = reasoning_prefilled, ) def generate_with_adapter_control( @@ -1454,14 +1507,27 @@ class InferenceOrchestrator: Uses the dispatcher path (no _gen_lock) so compare-mode requests don't block each other; the subprocess serializes them via its - sequential command loop. + sequential command loop. Backend failures raise instead of becoming + assistant text. """ - yield from self._generate_dispatched( + stream = self._generate_dispatched( use_adapter = use_adapter, cancel_event = cancel_event, stats_holder = stats_holder, **gen_kwargs, ) + try: + for chunk in stream: + if isinstance(chunk, GenStreamError): + # Preserve the public/operational flag so the route can surface + # the real message (e.g. "model is being unloaded") instead of a + # generic error. Mirrors the safetensors tool loop's _single_turn. + raise GenStreamErrorRaised(str(chunk), public = chunk.public) + yield chunk + finally: + close = getattr(stream, "close", None) + if callable(close): + close() def _generate_inner( self, @@ -1489,11 +1555,11 @@ class InferenceOrchestrator: readers don't consume each other's tokens off the shared resp_queue. """ if not self._ensure_subprocess_alive(): - yield GenStreamError("Error: Inference subprocess is not running") + yield GenStreamError("Error: Inference subprocess is not running", public = True) return if not self.active_model_name: - yield GenStreamError("Error: No active model") + yield GenStreamError("Error: No active model", public = True) return expected_model = self.active_model_name @@ -1510,7 +1576,7 @@ class InferenceOrchestrator: # so we never generate on the wrong one. if self._unload_pending or self.active_model_name != expected_model: # Won the lock handoff during a switch; don't start on the outgoing model. - yield GenStreamError("Error: model is being unloaded") + yield GenStreamError("Error: model is being unloaded", public = True) return request_id = str(uuid.uuid4()) image_b64 = self._pil_to_base64(image) if image is not None else None @@ -1695,10 +1761,10 @@ class InferenceOrchestrator: ) -> Generator[str, None, None]: """Shared inner logic for audio input generation (Whisper + ASR).""" if not self._ensure_subprocess_alive(): - yield GenStreamError("Error: Inference subprocess is not running") + yield GenStreamError("Error: Inference subprocess is not running", public = True) return if not self.active_model_name: - yield GenStreamError("Error: No active model") + yield GenStreamError("Error: No active model", public = True) return expected_model = self.active_model_name @@ -1707,7 +1773,7 @@ class InferenceOrchestrator: # cleared or swapped the model while we waited. if self._unload_pending or self.active_model_name != expected_model: # Won the lock handoff during a switch; don't start on the outgoing model. - yield GenStreamError("Error: model is being unloaded") + yield GenStreamError("Error: model is being unloaded", public = True) return request_id = str(uuid.uuid4()) diff --git a/studio/backend/core/inference/passthrough_healing.py b/studio/backend/core/inference/passthrough_healing.py index ed7c7ecfcf..e6da0a22b0 100644 --- a/studio/backend/core/inference/passthrough_healing.py +++ b/studio/backend/core/inference/passthrough_healing.py @@ -5,7 +5,7 @@ With server-side tools disabled (``unsloth run --disable-tools``, every ``unsloth start`` coding agent), requests carrying the client's own ``tools`` -bypass Studio's tool loop and are relayed to/from llama-server verbatim. Small +bypass Unsloth's tool loop and are relayed to/from llama-server verbatim. Small GGUF models often emit their tool calls as TEXT (``{...}``, Gemma ``<|tool_call>...``, ```` XML) instead of structured ``tool_calls`` -- on the passthrough that text reaches the agent as prose and @@ -18,7 +18,7 @@ promotes calls whose function name exactly matches a declared tool. Promotion removes EXACTLY the promoted calls' markup spans (the parser reports them): undeclared calls, unparseable blocks, and suppressed alternate formats keep every byte and relay as text, so healing can never silently delete model -output. Responses without a tool signal, requests without tools, and Studio's +output. Responses without a tool signal, requests without tools, and Unsloth's own enable-tools loop are untouched. Per-request opt-out: ``auto_heal_tool_calls: false``. Process kill-switch: ``UNSLOTH_DISABLE_TOOL_CALL_HEALING=1``. diff --git a/studio/backend/core/inference/pricing.py b/studio/backend/core/inference/pricing.py index 3b611d3596..30fec47723 100644 --- a/studio/backend/core/inference/pricing.py +++ b/studio/backend/core/inference/pricing.py @@ -122,12 +122,12 @@ def calculate_cost(provider: str, model: str, usage: dict[str, Any]) -> dict[str "priced": bool(prices), } - # Accept raw (input_tokens/output_tokens) and Studio chat-style + # Accept raw (input_tokens/output_tokens) and Unsloth chat-style # (prompt_tokens/completion_tokens) envelopes. Cache buckets differ: # raw Anthropic: input_tokens EXCLUDES cache buckets # raw OpenAI: input_tokens INCLUDES cache_read - # Studio Anthropic: prompt_tokens INCLUDES cache_creation + cache_read - # Studio OpenAI: prompt_tokens == raw input_tokens + # Unsloth Anthropic: prompt_tokens INCLUDES cache_creation + cache_read + # Unsloth OpenAI: prompt_tokens == raw input_tokens # Clamp >=0 so corrupted payloads can't produce a negative bill. cache_creation = max(0, int(usage.get("cache_creation_input_tokens") or 0)) cache_read_native_present = ( @@ -160,7 +160,7 @@ def calculate_cost(provider: str, model: str, usage: dict[str, Any]) -> dict[str output_tokens = max(0, int(usage.get("completion_tokens") or 0)) if provider == "openai": # Cached tokens land on input_tokens_details (raw Responses) or - # prompt_tokens_details (Studio chat-style). + # prompt_tokens_details (Unsloth chat-style). for key in ("input_tokens_details", "prompt_tokens_details"): details = usage.get(key) or {} if isinstance(details, dict): diff --git a/studio/backend/core/inference/providers.py b/studio/backend/core/inference/providers.py index 5b72373c03..d3bffc2f3d 100644 --- a/studio/backend/core/inference/providers.py +++ b/studio/backend/core/inference/providers.py @@ -276,8 +276,9 @@ PROVIDER_REGISTRY: dict[str, dict[str, Any]] = { "auth_header": "Authorization", "auth_prefix": "Bearer ", "notes": ( - "Local Ollama server. OpenAI-compatible /v1/chat/completions; " - "no API key. Surfaced via CUSTOM_PROVIDER_PRESETS in the frontend." + "Ollama server (local or cloud). OpenAI-compatible " + "/v1/chat/completions; API key optional (required by Ollama " + "cloud). Surfaced via CUSTOM_PROVIDER_PRESETS in the frontend." ), "hidden": True, }, diff --git a/studio/backend/core/inference/safetensors_agentic.py b/studio/backend/core/inference/safetensors_agentic.py index 43b72110ff..40731de57b 100644 --- a/studio/backend/core/inference/safetensors_agentic.py +++ b/studio/backend/core/inference/safetensors_agentic.py @@ -50,6 +50,7 @@ from core.inference.tool_call_parser import ( # pattern lists, so the safetensors streaming strip stays aligned with the parser. from core.tool_healing import ( _REHEARSAL_TAIL_STRIP_RE, + _THINK_CLOSE_RE, _strip_bracket_tag_calls, _think_spans_outside_tool_markup, apply_tool_strip_patterns, @@ -304,6 +305,45 @@ def _status_for_tool(tool_name: str, arguments: dict) -> str: return status_for_tool(tool_name, arguments) +def _reprompt_intent_text(text: str, *, reasoning_prefilled: bool = False) -> str: + """Return visible answer text for the plan-without-action classifier. + + Safetensors reasoning shares the cumulative text channel with the answer. + Forward-looking phrases inside ```` / ``[THINK]`` are private + planning, not a user-visible promise to call a tool. Match GGUF's behavior: + classify visible content when present and fall back to reasoning only for a + reasoning-only stall. + """ + prefilled_reasoning = "" + if reasoning_prefilled: + close = _THINK_CLOSE_RE.search(text) + if close is None: + return text.strip() + prefilled_reasoning = text[: close.end()].strip() + text = text[close.end() :].strip() + if not text: + return prefilled_reasoning + + spans = _think_spans_outside_tool_markup(text) + if not spans: + return text.strip() + + visible: list[str] = [] + reasoning: list[str] = [] + cursor = 0 + for start, end in spans: + visible.append(text[cursor:start]) + reasoning.append(text[start:end]) + cursor = end + visible.append(text[cursor:]) + + visible_text = "".join(visible).strip() + reasoning_text = "".join(reasoning).strip() + if visible_text: + return visible_text + return "\n".join(part for part in (prefilled_reasoning, reasoning_text) if part).strip() + + def _looks_like_enabled_bare_json(text: str, enabled_tool_names: Optional[set]) -> bool: """True when ``text`` opens with an ENABLED markerless bare-JSON call; an ordinary JSON answer returns False.""" probe = strip_llama3_leading_sentinels(text.lstrip()) @@ -448,6 +488,7 @@ def run_safetensors_tool_loop( confirm_tool_calls: bool = False, bypass_permissions: bool = False, permission_mode: Optional[str] = None, + reasoning_prefilled: bool = False, ) -> Generator[dict, None, None]: """Drive an agentic tool loop on top of a cumulative-text generator. @@ -954,9 +995,12 @@ def run_safetensors_tool_loop( if not safety_tc: # Re-prompt once on plan-without-action, before any tool runs # (GGUF loop parity). The retry is gated on nudge_tool_calls so - # Studio callers (which send True) always nudge, while API callers + # Unsloth callers (which send True) always nudge, while API callers # who omit the flag keep today's no-reprompt behavior (opt-in). - stripped_answer = content_accum.strip() + intent_text = _reprompt_intent_text( + content_accum, + reasoning_prefilled = reasoning_prefilled, + ) if ( auto_heal_tool_calls and nudge_tool_calls @@ -965,7 +1009,7 @@ def run_safetensors_tool_loop( and not rag_autoinjected and not tool_denied and not any(record.executed for record in tool_controller.history) - and is_short_intent_without_action(stripped_answer) + and is_short_intent_without_action(intent_text) ): reprompt_count += 1 logger.info( @@ -973,9 +1017,9 @@ def run_safetensors_tool_loop( "calling tools (%d chars)", reprompt_count, MAX_ACT_REPROMPTS, - len(stripped_answer), + len(intent_text), ) - conversation.append({"role": "assistant", "content": stripped_answer}) + conversation.append({"role": "assistant", "content": intent_text}) tool_hint = " or ".join(_active_tool_names(active_tools)) or "an available tool" conversation.append( { diff --git a/studio/backend/core/inference/sandbox_site/sitecustomize.py b/studio/backend/core/inference/sandbox_site/sitecustomize.py index d655e8e35a..244fa95145 100644 --- a/studio/backend/core/inference/sandbox_site/sitecustomize.py +++ b/studio/backend/core/inference/sandbox_site/sitecustomize.py @@ -4,7 +4,7 @@ """Sandbox-side compatibility shim for ChatGPT code-interpreter paths. Models habitually write to /mnt/data (or /mnt/outputs, /home/sandbox, -/workspace), none of which exist in the Studio sandbox. This module sits on the +/workspace), none of which exist in the Unsloth sandbox. This module sits on the sandbox subprocess PYTHONPATH (see ``tools._build_safe_env``), so it loads at interpreter startup in every sandboxed ``python`` run and any Python the ``terminal`` tool launches. diff --git a/studio/backend/core/inference/stt_ggml_sidecar.py b/studio/backend/core/inference/stt_ggml_sidecar.py new file mode 100644 index 0000000000..02b376dea5 --- /dev/null +++ b/studio/backend/core/inference/stt_ggml_sidecar.py @@ -0,0 +1,876 @@ +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 + +"""whisper.cpp (GGML/GGUF) speech-to-text sidecar for Studio dictation. + +Runs the same curated Whisper checkpoints as the Transformers sidecar +(stt_sidecar.py) through whisper.cpp's `whisper-server`, ~2.5x faster at +identical quality on Apple Silicon and CPU because its Metal/CPU kernels run +the weights in f16 where PyTorch MPS requires fp32. + +Owns a single `whisper-server` subprocess bound to 127.0.0.1 on an ephemeral +port; the model loads on demand, stays warm between dictations, and unloads +after the same keep-alive as the Transformers sidecar. Curated GGML checkpoints +are single files from `unslothai/whisper-*-GGUF`, downloaded directly rather +than through the Model Hub (whose variant planner only handles `.gguf` chat +layouts). + +Binary discovery mirrors `_find_llama_server_binary`: env override, then managed +Studio home, then PATH. With no binary the engine is unavailable and dictation +falls back to the Transformers sidecar; `scripts/build_whisper_cpp.sh` installs +the binary. +""" + +from __future__ import annotations + +import io +import json +import os +import re +import shutil +import socket +import subprocess +import sys +import threading +import time +import urllib.request +import uuid +import wave +from contextlib import contextmanager +from pathlib import Path +from typing import Iterator, Optional + +from loggers import get_logger + +from core.inference.stt_sidecar import ( + STT_KEEP_ALIVE_SECONDS, + SttAudioDecodeError, + SttLanguageError, + SttLoadCancelledError, + SttModelIdError, + SttModelNotDownloadedError, + SttUnavailableError, + _decode_audio_bounded, + _known_whisper_languages, + _TARGET_SAMPLE_RATE, + _training_active, + normalize_whisper_language, +) +from utils.prebuilt.child_env import isolate_home, scrub_env, wsl_system_rocm_lib_dirs +from utils.prebuilt.runtime_libs import dedupe_existing_dirs +from utils.prebuilt.whisper_layout import lookup_marker +from utils.process_lifetime import adopt_pid, child_popen_kwargs, forget_pid + +logger = get_logger(__name__) + +# Curated GGML checkpoints, one repo per model. Keys match the Transformers +# sidecar's ids so the frontend reuses one picker; values are the single file +# inside each repo. +GGML_STT_REPOS: dict[str, str] = { + "tiny": "unslothai/whisper-tiny-GGUF", + "base": "unslothai/whisper-base-GGUF", + "small": "unslothai/whisper-small-GGUF", + "large-v3-turbo": "unslothai/whisper-large-v3-turbo-GGUF", + "large-v3": "unslothai/whisper-large-v3-GGUF", +} +GGML_STT_MODELS: dict[str, str] = { + "tiny": "whisper-tiny.bin", + "base": "whisper-base.bin", + "small": "whisper-small.bin", + "large-v3-turbo": "whisper-large-v3-turbo.bin", + "large-v3": "whisper-large-v3.bin", +} +DEFAULT_GGML_STT_MODEL = "small" + +_SERVER_START_TIMEOUT_SECONDS = 120.0 +_TRANSCRIBE_TIMEOUT_SECONDS = 600.0 + + +class SttEngineUnavailableError(SttUnavailableError): + """whisper-server is not installed; the GGUF dictation engine is off.""" + + +def resolve_ggml_model_id(model: Optional[str]) -> str: + """Validate a curated GGML model id. Custom repos are not supported here.""" + if model is None or not str(model).strip(): + return DEFAULT_GGML_STT_MODEL + normalized = str(model).strip() + if normalized in GGML_STT_MODELS: + return normalized + raise SttModelIdError( + f"STT model '{model}' is not a curated GGUF dictation model. " + f"Choose one of: {', '.join(GGML_STT_MODELS)}." + ) + + +def _managed_whisper_cpp_dir() -> Path: + """`/whisper.cpp` in custom mode, else `~/.unsloth/whisper.cpp`. + + Mirrors `managed_node_dir` / `_find_llama_server_binary` so managed runtimes + share one parent directory. + """ + legacy = Path.home() / ".unsloth" / "whisper.cpp" + try: + from utils.paths.storage_roots import studio_root + + resolved = studio_root() + legacy_studio = Path.home() / ".unsloth" / "studio" + try: + is_legacy = resolved.resolve() == legacy_studio.resolve() + except (OSError, ValueError): + is_legacy = resolved == legacy_studio + return legacy if is_legacy else (resolved / "whisper.cpp") + except (ImportError, OSError, ValueError): + override = ( + os.environ.get("UNSLOTH_STUDIO_HOME") or os.environ.get("STUDIO_HOME") or "" + ).strip() + if override: + try: + return Path(override).expanduser().resolve() / "whisper.cpp" + except (OSError, ValueError): + return Path(override).expanduser() / "whisper.cpp" + return legacy + + +def find_whisper_server_binary() -> Optional[str]: + """Locate the whisper-server binary. + + Search order: + 1. WHISPER_SERVER_PATH environment variable (direct path to binary) + 2. UNSLOTH_WHISPER_CPP_PATH env var (custom whisper.cpp install dir) + 3. managed dir: /whisper.cpp/{,build/bin/}whisper-server + 4. whisper-server on PATH + """ + binary_name = "whisper-server.exe" if sys.platform == "win32" else "whisper-server" + + def _layout_candidates(d: Path) -> list[Path]: + cands = [d / binary_name, d / "build" / "bin" / binary_name] + if sys.platform == "win32": + cands.append(d / "build" / "bin" / "Release" / binary_name) + return cands + + env_path = os.environ.get("WHISPER_SERVER_PATH") + if env_path: + p = Path(env_path) + if _is_runnable(p): + return str(p) + + custom_dir = os.environ.get("UNSLOTH_WHISPER_CPP_PATH") + if custom_dir: + for p in _layout_candidates(Path(custom_dir)): + if _is_runnable(p): + return str(p) + + for p in _layout_candidates(_managed_whisper_cpp_dir()): + if _is_runnable(p): + return str(p) + + return shutil.which(binary_name) + + +def _is_runnable(p: Path) -> bool: + """A real whisper-server is an executable file. On Windows os.access(X_OK) is + effectively an existence check; on Unix it rejects a non-executable stub so a + half-written or wrong-mode file isn't mistaken for the server.""" + return p.is_file() and (sys.platform == "win32" or os.access(p, os.X_OK)) + + +def _whisper_install_marker(binary: str) -> Optional[dict]: + """The prebuilt install marker above ``binary``, or None (source/custom builds).""" + return lookup_marker(binary).marker + + +def slim_runtime_intact(binary: str) -> bool: + """True unless the marker says slim and the linked ggml runtime is missing + beside the server. New markers record the exact wired filenames + (linked_libraries), all of which must be present; legacy markers without the + field fall back to the per-OS core ggml name globs. A broken slim install + reads as engine-unavailable (reinstall via `unsloth studio update`), never a + crash at load.""" + lookup = lookup_marker(binary) + marker = lookup.marker + if lookup.invalid or marker is None: + return not lookup.slim_collision + if not marker or marker.get("install_kind") != "slim": + return True + if lookup.authoritative: + valid = marker.get("component") == "whisper.cpp" + valid = valid and isinstance(marker.get("schema_version"), int) + valid = valid and all( + isinstance(marker.get(key), str) and marker[key] + for key in ("release_tag", "backend", "paired_llama_tag") + ) + valid = valid and isinstance(marker.get("linked_libraries"), list) + valid = valid and bool(marker.get("linked_libraries")) + valid = valid and all( + isinstance(name, str) and name and Path(name).name == name + for name in marker["linked_libraries"] + ) + if not valid: + return False + bin_dir = Path(binary).parent + linked = marker.get("linked_libraries") + if isinstance(linked, list) and linked and all(isinstance(name, str) for name in linked): + intact = all((bin_dir / name).is_file() for name in linked) + else: + if sys.platform == "win32": + required = ("ggml.dll", "ggml-base.dll") + elif sys.platform == "darwin": + required = ("libggml*.dylib", "libggml-base*.dylib") + else: + required = ("libggml.so*", "libggml-base.so*") + intact = all(any(p.is_file() for p in bin_dir.glob(pattern)) for pattern in required) + runtime_dirs = marker.get("linked_runtime_directories") + if intact and isinstance(runtime_dirs, list) and runtime_dirs: + intact = all( + isinstance(name, str) + and name + and (bin_dir / name).is_dir() + and any(path.is_file() for path in (bin_dir / name).rglob("*")) + for name in runtime_dirs + ) + if intact and marker.get("backend") == "rocm": + expected_runtime_dirs = set() if sys.platform == "win32" else {"hipblaslt", "rocblas"} + intact = ( + marker.get("runtime_wiring_version") == 2 + and isinstance(runtime_dirs, list) + and set(runtime_dirs) == expected_runtime_dirs + ) + if not intact: + logger.warning( + "slim whisper install is missing its linked ggml runtime at " + f"{bin_dir}; run `unsloth studio update` to reinstall it" + ) + return intact + + +def is_available() -> bool: + binary = find_whisper_server_binary() + if binary is None: + return False + if not slim_runtime_intact(binary): + return False + try: + import av # noqa: F401 + except Exception: + # No PyAV means every transcription 501s on decode. + return False + return True + + +def ensure_engine_available() -> str: + binary = find_whisper_server_binary() + if binary is None: + raise SttEngineUnavailableError( + "The local transcription runtime is not installed. Run " + "`unsloth studio update` to install it." + ) + if not slim_runtime_intact(binary): + raise SttEngineUnavailableError( + "The local transcription runtime is missing its paired ggml " + "libraries. Run `unsloth studio update` to reinstall it." + ) + return binary + + +# --------------------------------------------------------------------------- +# whisper-server child-process environment +# --------------------------------------------------------------------------- +# Build the whisper-server env: prepend the binary dir (co-located libs win, and +# a backstop where the loader ignores the rpath) and scrub secret-bearing vars the +# binary never needs. On WSL2 ROCm the system HIP libs go first, since a bundle's +# bare-metal HIP cannot drive /dev/dxg. A CUDA bundle ships libggml-cuda.so but not +# libcudart/libcublas (paired with the user's PyTorch), so add the +# CUDA-from-PyTorch runtime dirs the selection gated on, else the backend cannot +# resolve a runtime that lives only in wheels. Mirrors llama's binary_env(); the +# scrub/WSL/dedupe helpers live in utils.prebuilt. + +# Module-level aliases keep the historical patch points for tests and callers. +_wsl_system_rocm_lib_dirs = wsl_system_rocm_lib_dirs +_dedupe_existing_dirs = dedupe_existing_dirs + + +def _whisper_server_child_env(binary: str) -> dict[str, str]: + """Env for the whisper-server subprocess: secrets scrubbed, home/profile vars + repointed at a managed scratch dir (a downloaded binary must not see the real + home's token caches), co-located libs on the loader path, WSL system HIP first + on WSL2 ROCm.""" + env = scrub_env(os.environ) + isolate_home(env, str(_managed_whisper_cpp_dir() / ".child_home")) + bin_dir = str(Path(binary).parent) + # A CUDA bundle needs the CUDA-from-PyTorch wheel dirs so libcudart/libcublas + # resolve at launch when they live only in site-packages/nvidia/*/lib. Placed + # after bin_dir so co-located libs still win; empty for other bundles. + cuda_runtime_dirs: list[str] = [] + bundle_dir = Path(bin_dir) + has_cuda_module = any( + path.is_file() + for pattern in ("libggml-cuda.so*", "ggml-cuda*.dll") + for path in bundle_dir.glob(pattern) + ) + if has_cuda_module: + try: + from utils.prebuilt.runtime_libs import python_runtime_dirs + cuda_runtime_dirs = python_runtime_dirs() + except Exception: + cuda_runtime_dirs = [] + if sys.platform == "win32": + var, lead = "PATH", [bin_dir, *cuda_runtime_dirs] + elif sys.platform == "darwin": + var, lead = "DYLD_LIBRARY_PATH", [bin_dir] + else: + var, lead = "LD_LIBRARY_PATH", [bin_dir, *cuda_runtime_dirs] + wsl_rocm = _wsl_system_rocm_lib_dirs() + if wsl_rocm: + lead = [*wsl_rocm, bin_dir, *cuda_runtime_dirs] + env.setdefault("HSA_ENABLE_DXG_DETECTION", "1") + existing = [p for p in env.get(var, "").split(os.pathsep) if p] + env[var] = os.pathsep.join(_dedupe_existing_dirs([*lead, *existing])) + return env + + +# --------------------------------------------------------------------------- +# Model file download (single files; deliberately outside the Model Hub flow) +# --------------------------------------------------------------------------- + + +def _cached_model_path(model_id: str) -> Optional[str]: + """Path of a fully downloaded GGML file in the shared HF cache, else None.""" + from huggingface_hub import hf_hub_download + try: + return hf_hub_download( + repo_id = GGML_STT_REPOS[model_id], + filename = GGML_STT_MODELS[model_id], + local_files_only = True, + ) + except Exception: + return None + + +class _GgmlDownloadState: + """Tracks one background hf_hub_download of a curated GGML file.""" + + def __init__(self) -> None: + self._lock = threading.Lock() + self._thread: Optional[threading.Thread] = None + self._model_id: Optional[str] = None + self._error: Optional[str] = None + self._total_bytes: Optional[int] = None + self._etag: Optional[str] = None + + def status(self) -> dict: + with self._lock: + downloading = self._thread is not None and self._thread.is_alive() + return { + "downloading": downloading, + "model": self._model_id if downloading else None, + "error": self._error, + "bytes_total": self._total_bytes if downloading else None, + "bytes_done": self._incomplete_bytes() if downloading else None, + } + + def _incomplete_bytes(self) -> Optional[int]: + """Best-effort progress: size of the in-flight blob in the HF cache. + + hf_hub_download writes ``blobs/.incomplete``; prefer this file's + etag, else the largest in-flight blob. + """ + try: + from huggingface_hub.constants import HF_HUB_CACHE + + # Caller may hold the non-reentrant self._lock; bare reads are safe. + model_id = self._model_id + if not model_id: + return None + repo_dir = ( + Path(HF_HUB_CACHE) + / f"models--{GGML_STT_REPOS[model_id].replace('/', '--')}" + / "blobs" + ) + if not repo_dir.is_dir(): + return None + etag = self._etag + if etag: + target = repo_dir / f"{etag}.incomplete" + if target.is_file(): + return target.stat().st_size + sizes = [p.stat().st_size for p in repo_dir.glob("*.incomplete") if p.is_file()] + return max(sizes) if sizes else None + except Exception: + return None + + def start( + self, + model_id: str, + hf_token: Optional[str] = None, + ) -> None: + model_id = resolve_ggml_model_id(model_id) + with self._lock: + if self._thread is not None and self._thread.is_alive(): + if self._model_id == model_id: + return + raise SttModelIdError( + f"Another GGUF dictation model ('{self._model_id}') is still " + "downloading; wait for it to finish." + ) + self._model_id = model_id + self._error = None + self._total_bytes = None + self._etag = None + thread = threading.Thread(target = self._run, args = (model_id, hf_token), daemon = True) + self._thread = thread + thread.start() + + def _run(self, model_id: str, hf_token: Optional[str]) -> None: + repo_id = GGML_STT_REPOS[model_id] + filename = GGML_STT_MODELS[model_id] + try: + from huggingface_hub import ( + get_hf_file_metadata, + hf_hub_download, + hf_hub_url, + ) + try: + # One HEAD request for the total and etag. + meta = get_hf_file_metadata(hf_hub_url(repo_id, filename), token = hf_token or None) + with self._lock: + self._total_bytes = meta.size + self._etag = meta.etag + except Exception: + pass + hf_hub_download( + repo_id = repo_id, + filename = filename, + token = hf_token or None, + ) + except Exception as exc: + logger.warning("GGUF STT download failed for %s: %s", model_id, exc) + with self._lock: + self._error = f"Download failed for '{model_id}'." + + +_download_state = _GgmlDownloadState() + + +def start_model_download(model: Optional[str], hf_token: Optional[str] = None) -> None: + _download_state.start(resolve_ggml_model_id(model), hf_token) + + +def download_status() -> dict: + return _download_state.status() + + +# --------------------------------------------------------------------------- +# WAV packaging +# --------------------------------------------------------------------------- + + +def _pcm_to_wav_bytes(decoded_audio) -> bytes: + """Wrap decoded float32 mono 16 kHz PCM into an in-memory 16-bit WAV.""" + import numpy as np + + clipped = np.clip(decoded_audio, -1.0, 1.0) + pcm16 = (clipped * 32767.0).astype(" None: + self._lock = threading.RLock() + self._process: Optional[subprocess.Popen] = None + self._port: Optional[int] = None + self._model_id: Optional[str] = None + self._idle_timer: Optional[threading.Timer] = None + self._idle_generation = 0 + self._keep_alive_seconds = keep_alive_seconds + # Set while whisper-server starts so training admission can account for + # the accelerator memory it is about to bind. Read without the lock. + self._loading = False + # A still-starting whisper-server is cancellable so training can preempt + # it before it binds accelerator memory. Assigned inside self._lock but + # acted on without it: cancel_pending_load() runs while load() holds the + # lock, so the event is the source of truth and terminating the process + # is a best-effort fast path. + self._load_cancel_event: Optional[threading.Event] = None + self._starting_process: Optional[subprocess.Popen] = None + # Set before the updater waits for _lock, then kept set while it owns + # the lock and atomically replaces the managed install tree. New loads + # fail fast instead of starting a process from files being swapped. + self._update_in_progress = False + + @property + def loaded_model(self) -> Optional[str]: + # Lock-free status read (like stt_sidecar.py): transcribe() holds + # self._lock for the whole inference call (up to + # _TRANSCRIBE_TIMEOUT_SECONDS), and status polls plus training admission + # must not block behind it. _process_alive() snapshots self._process + # before poll(), which subprocess guards with _waitpid_lock, so a + # concurrent unload is safe. + return self._model_id if self._process_alive() else None + + @property + def device(self) -> Optional[str]: + return "whisper.cpp" if self._process_alive() else None + + def is_loading(self) -> bool: + # True only while whisper-server is starting (seconds to bind its GPU + # backend); load() sets and clears the flag around that window. + return self._loading + + @property + def keep_alive_seconds(self) -> float: + return self._keep_alive_seconds + + def _process_alive(self) -> bool: + # Snapshot self._process once: a concurrent unload() nulls it under the + # lock, so lock-free readers would otherwise re-read None between the + # truthiness check and .poll(). + process = self._process + return process is not None and process.poll() is None + + # -- idle unload ------------------------------------------------------ + + def _cancel_idle_unload_locked(self) -> None: + self._idle_generation += 1 + if self._idle_timer is not None: + self._idle_timer.cancel() + self._idle_timer = None + + def _schedule_idle_unload_locked(self) -> None: + self._cancel_idle_unload_locked() + if not self._process_alive(): + return + generation = self._idle_generation + timer = threading.Timer(self._keep_alive_seconds, self._idle_unload, args = (generation,)) + timer.daemon = True + self._idle_timer = timer + timer.start() + + def _idle_unload(self, generation: int) -> None: + with self._lock: + if generation != self._idle_generation: + return + logger.info("Unloading idle GGUF STT model %s", self._model_id) + self._release_locked() + + # -- process lifecycle ------------------------------------------------- + + def _release_locked(self) -> None: + self._cancel_idle_unload_locked() + process = self._process + self._process = None + self._port = None + self._model_id = None + if process is not None and process.poll() is None: + process.terminate() + try: + process.wait(timeout = 10) + except subprocess.TimeoutExpired: + process.kill() + process.wait(timeout = 10) + if process is not None: + forget_pid(process.pid) + + def unload(self) -> None: + with self._lock: + self._release_locked() + + def _raise_if_update_in_progress(self) -> None: + if self._update_in_progress: + raise SttEngineUnavailableError( + "The local transcription runtime is being updated. Try dictation again shortly." + ) + + @contextmanager + def update_maintenance(self) -> Iterator[bool]: + """Block new loads while the managed whisper.cpp tree is replaced. + + The flag is published before waiting for an existing transcription to + release ``_lock``. Holding that lock across the yielded installer phase + prevents Windows from relocking the executable and prevents every host + from starting a process against a partially swapped tree. The yielded + value records whether a warm model had to be unloaded. + """ + self._update_in_progress = True + try: + with self._lock: + model_was_active = self._process_alive() + self._release_locked() + yield model_was_active + finally: + self._update_in_progress = False + + def cancel_pending_load(self) -> bool: + # Preempt a starting whisper-server so training does not launch while it + # binds accelerator memory. load() holds self._lock for the whole startup, + # so act without the lock: signal abort and terminate the starting + # process. _wait_for_server observes the event and raises, then load() + # reaps the process and releases the lock. + if not self._loading: + return False + event = self._load_cancel_event + if event is None: + return False + event.set() + process = self._starting_process + if process is not None and process.poll() is None: + try: + process.terminate() + except Exception: + pass + return True + + def wait_for_load_to_settle(self) -> None: + # load() holds self._lock across startup and cancel cleanup, so acquiring + # it blocks until a cancelled server is killed, reaped, and its + # accelerator memory released. + with self._lock: + pass + + @staticmethod + def _reserve_free_port() -> tuple[socket.socket, int]: + """Bind an ephemeral port and keep the socket held. + + The caller closes the reservation immediately before spawning + whisper-server, shrinking the window in which another local process + could bind the port. SO_REUSEADDR lets the child rebind right after. + """ + s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + s.bind(("127.0.0.1", 0)) + return s, s.getsockname()[1] + + def _ensure_model_downloaded(self, model_id: str) -> str: + path = _cached_model_path(model_id) + if path is None: + raise SttModelNotDownloadedError( + f"STT model '{model_id}' (GGUF) is not downloaded. " + "Download it in Settings, then Voice, before loading it." + ) + return path + + def load(self, model: Optional[str] = None) -> None: + """Start (or switch) whisper-server for the requested curated model.""" + self._raise_if_update_in_progress() + model_id = resolve_ggml_model_id(model) + with self._lock: + self._raise_if_update_in_progress() + binary = ensure_engine_available() + if self._process_alive() and self._model_id == model_id: + self._schedule_idle_unload_locked() + return + model_path = self._ensure_model_downloaded(model_id) + self._release_locked() + reservation, port = self._reserve_free_port() + command = [binary, "-m", model_path, "--host", "127.0.0.1", "--port", str(port)] + marker = _whisper_install_marker(binary) + if _training_active(): + # Keep whisper.cpp off the accelerator during training (like the + # Transformers sidecar's CPU choice) so a mid-training dictation + # cannot reclaim the VRAM training just freed. + command.append("--no-gpu") + elif marker is not None and marker.get("backend") == "cpu": + # A deliberate CPU install must stay CPU: the slim wiring links + # every llama ggml backend (including CUDA/ROCm), so without + # this flag a cpu-selected install would still grab the GPU. + command.append("--no-gpu") + logger.info( + "Starting whisper-server for STT model %s on 127.0.0.1:%s", + model_id, + port, + ) + cancel_event = threading.Event() + self._load_cancel_event = cancel_event + self._loading = True + try: + # Release the reservation as late as possible: whisper-server + # binds the port moments after this close. + reservation.close() + process = subprocess.Popen( + command, + stdout = subprocess.DEVNULL, + stderr = subprocess.DEVNULL, + stdin = subprocess.DEVNULL, + # Co-located GPU libs on the loader path (WSL system HIP first), + # secrets scrubbed from the downloaded binary's env. + env = _whisper_server_child_env(binary), + # Die with Studio (Linux PDEATHSIG, Windows job) so a crash + # never orphans a server holding the model. + **child_popen_kwargs(), + ) + self._starting_process = process + adopt_pid(process.pid) # terminate_all backstop for graceful exits + try: + self._wait_for_server(process, port, cancel_event) + except Exception: + if process.poll() is None: + process.kill() + process.wait(timeout = 10) + forget_pid(process.pid) + raise + self._process = process + self._port = port + self._model_id = model_id + self._schedule_idle_unload_locked() + finally: + reservation.close() # no-op when already released before spawn + self._loading = False + self._load_cancel_event = None + self._starting_process = None + + @staticmethod + def _wait_for_server( + process: subprocess.Popen, + port: int, + cancel_event: Optional[threading.Event] = None, + ) -> None: + deadline = time.monotonic() + _SERVER_START_TIMEOUT_SECONDS + while time.monotonic() < deadline: + if cancel_event is not None and cancel_event.is_set(): + raise SttLoadCancelledError( + "GGUF STT model loading was cancelled so training could start." + ) + if process.poll() is not None: + raise SttEngineUnavailableError( + "The local transcription runtime exited before becoming " + "ready; the model file may be corrupt or unsupported." + ) + # Require a whisper-server-specific response twice, with the managed + # child alive around each probe. An arbitrary local process that won + # the bind race would otherwise be mistaken for the sidecar and + # receive the user's microphone audio. + if GgmlSttSidecar._probe_is_whisper_server(process, port) and ( + GgmlSttSidecar._probe_is_whisper_server(process, port) + ): + return + time.sleep(0.2) + raise SttEngineUnavailableError("The local transcription runtime did not start in time.") + + @staticmethod + def _probe_is_whisper_server(process: subprocess.Popen, port: int) -> bool: + """One readiness probe: our child is alive and the responder looks like + whisper.cpp's server (its index page and errors identify whisper).""" + if process.poll() is not None: + return False + try: + req = urllib.request.Request(f"http://127.0.0.1:{port}/", method = "GET") + with urllib.request.urlopen(req, timeout = 2) as response: + body = response.read(65536) + except Exception: + return False + if process.poll() is not None: + return False + return b"whisper" in body.lower() + + # -- transcription ------------------------------------------------------ + + def transcribe( + self, + audio: bytes, + model: Optional[str] = None, + language: Optional[str] = None, + fast: bool = False, + ) -> dict: + """Transcribe encoded audio bytes via whisper-server. + + Accepts any container PyAV can decode (same validation and caps as the + Transformers sidecar). Returns {text, language, duration, model}. + """ + self._raise_if_update_in_progress() + ensure_engine_available() + model_id = resolve_ggml_model_id(model) + lang = normalize_whisper_language(language) + known_languages = _known_whisper_languages() + if lang is not None and known_languages is not None and lang not in known_languages: + raise SttLanguageError( + f"Language '{language}' is not supported by STT model '{model_id}'." + ) + # Reject a missing model before decoding so a long clip does not burn CPU + # only to 409 (matches the Transformers sidecar's preflight). + self._ensure_model_downloaded(model_id) + decoded_audio = _decode_audio_bounded(audio) + wav_bytes = _pcm_to_wav_bytes(decoded_audio) + with self._lock: + try: + self.load(model_id) + text = self._post_inference(wav_bytes, lang, fast) + finally: + self._schedule_idle_unload_locked() + duration = (len(decoded_audio) / _TARGET_SAMPLE_RATE) if len(decoded_audio) else None + return { + "text": text, + "language": lang, + "duration": duration, + "model": model_id, + } + + def _post_inference(self, wav_bytes: bytes, lang: Optional[str], fast: bool) -> str: + boundary = uuid.uuid4().hex + fields = { + "temperature": "0.0", + "response_format": "json", + # Match the Transformers sidecar: 5-way beam search, greedy for fast. + "beam_size": "1" if fast else "5", + "language": lang or "auto", + } + parts: list[bytes] = [] + for name, value in fields.items(): + parts.append( + ( + f"--{boundary}\r\nContent-Disposition: form-data; " + f'name="{name}"\r\n\r\n{value}\r\n' + ).encode() + ) + parts.append( + ( + f"--{boundary}\r\nContent-Disposition: form-data; " + 'name="file"; filename="dictation.wav"\r\n' + "Content-Type: audio/wav\r\n\r\n" + ).encode() + + wav_bytes + + b"\r\n" + ) + parts.append(f"--{boundary}--\r\n".encode()) + body = b"".join(parts) + req = urllib.request.Request( + f"http://127.0.0.1:{self._port}/inference", + data = body, + headers = {"Content-Type": f"multipart/form-data; boundary={boundary}"}, + ) + try: + with urllib.request.urlopen(req, timeout = _TRANSCRIBE_TIMEOUT_SECONDS) as resp: + payload = json.load(resp) + except SttAudioDecodeError: + raise + except Exception as exc: + raise SttEngineUnavailableError( + "The local transcription runtime did not answer the request." + ) from exc + text = payload.get("text") + if not isinstance(text, str): + raise SttAudioDecodeError("Could not decode the audio.") + # whisper.cpp joins segments with newlines; dictation wants one line. + return " ".join(part.strip() for part in text.splitlines() if part.strip()).strip() + + +_sidecar: Optional[GgmlSttSidecar] = None + + +def get_ggml_stt_sidecar() -> GgmlSttSidecar: + global _sidecar + if _sidecar is None: + _sidecar = GgmlSttSidecar() + return _sidecar diff --git a/studio/backend/core/inference/stt_sidecar.py b/studio/backend/core/inference/stt_sidecar.py new file mode 100644 index 0000000000..edf57c16e3 --- /dev/null +++ b/studio/backend/core/inference/stt_sidecar.py @@ -0,0 +1,1142 @@ +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 + +""" +Standalone speech-to-text (STT) sidecar for dictation. + +Loads a Whisper model (via Transformers) in the backend process, separate from +the chat model's inference subprocess, so dictation works with any chat model +without evicting it. Curated defaults plus any Transformers-compatible Whisper +repo; weights come through Studio's Model Hub and stay warm briefly between +dictations. CUDA runs float16; MPS and CPU run float32. +""" + +from __future__ import annotations + +import gc +import hashlib +import io +import json +import os +import re +import threading +import uuid +from dataclasses import dataclass +from pathlib import Path +from typing import Optional + +from loggers import get_logger + +logger = get_logger(__name__) + +# Multilingual Whisper defaults: stable API/UI id -> Hub repository. A request +# may instead pass a validated Hugging Face `owner/model` id. +STT_MODELS: dict[str, str] = { + "tiny": "unsloth/whisper-tiny", + "base": "unsloth/whisper-base", + "small": "unsloth/whisper-small", + "large-v3-turbo": "unsloth/whisper-large-v3-turbo", + "large-v3": "unsloth/whisper-large-v3", +} +DEFAULT_STT_MODEL = "small" +STT_KEEP_ALIVE_SECONDS = 5 * 60 +_HF_REPO_ID = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]{0,95}/[A-Za-z0-9][A-Za-z0-9._-]{0,95}$") +_HF_COMMIT_SHA = re.compile(r"^[0-9a-f]{40}$") + +# Bound decoded PCM length so a crafted upload cannot exhaust memory (callers +# also cap the encoded bytes). +_MAX_AUDIO_SECONDS = 30 * 60 +_TARGET_SAMPLE_RATE = 16000 + +# Non-weight files WhisperProcessor/WhisperForConditionalGeneration may load. +# Weight selection is built from pinned Hub metadata. A custom repo id is +# attacker-controllable, so only safetensors weights are accepted: a +# pytorch_model.bin is a pickle and executes code while Transformers +# deserializes it (see utils/security/file_security.py), and this path skips +# the malware gate the normal model loader applies. +_STT_SNAPSHOT_SUPPORT_FILES = ( + "config.json", + "generation_config.json", + "preprocessor_config.json", + "processor_config.json", + "tokenizer.json", + "tokenizer_config.json", + "vocab.json", + "merges.txt", + "normalizer.json", + "special_tokens_map.json", + "added_tokens.json", +) +_STT_SAFETENSORS_INDEX = "model.safetensors.index.json" +_STT_SAFETENSORS_WEIGHTS = "model.safetensors" +_STT_REVISION_RECORD_VERSION = 1 + + +@dataclass(frozen = True) +class _SelectedHubFile: + path: str + size: int + blob_key: Optional[str] + + +@dataclass(frozen = True) +class _CachedSttSnapshot: + path: Optional[Path] + is_multilingual: Optional[bool] + + +class SttUnavailableError(RuntimeError): + """The STT backend (PyTorch/Transformers or PyAV) is not installed.""" + + +class SttLoadCancelledError(RuntimeError): + """An in-flight STT model load was cancelled for training.""" + + +class SttModelNotDownloadedError(RuntimeError): + """The selected model is not complete in the shared Hub cache.""" + + +class SttModelIdError(ValueError): + """The requested custom model is not a valid Hugging Face repository id.""" + + +class SttModelCompatibilityError(ValueError): + """The requested repository is not a Transformers Whisper checkpoint.""" + + +class SttAudioDecodeError(ValueError): + """The uploaded bytes could not be decoded as audio.""" + + +class SttAudioTooLongError(ValueError): + """The decoded audio exceeds the bounded transcription duration.""" + + +class SttLanguageError(ValueError): + """The requested language is not supported by the selected STT model.""" + + +_WHISPER_LANGUAGE_ALIASES = { + # Legacy/browser BCP-47 primaries whose Whisper code differs. + "cmn": "zh", + "fil": "tl", + "in": "id", + "iw": "he", + "ji": "yi", + "nb": "no", + "nn": "no", +} + + +def normalize_whisper_language(language: Optional[str]) -> Optional[str]: + """Convert a BCP-47 locale into the short code Whisper expects.""" + if not language: + return None + normalized = language.strip().replace("_", "-").lower() + if not normalized or normalized == "auto": + return None + primary = normalized.split("-", 1)[0] + return _WHISPER_LANGUAGE_ALIASES.get(primary, primary) + + +def _known_whisper_languages() -> Optional[frozenset[str]]: + """Return Whisper's language codes without constructing/loading a model.""" + try: + from transformers.models.whisper.tokenization_whisper import LANGUAGES + except Exception: + # Transformers unavailable or the constant moved: skip the check. + return None + return frozenset(LANGUAGES) + + +def ensure_stt_available() -> None: + """Raise when the complete local Whisper backend cannot be imported.""" + try: + import av # noqa: F401 + import torch # noqa: F401 + import transformers # noqa: F401 + except Exception as exc: + raise SttUnavailableError( + "Speech-to-text needs PyTorch, Transformers, and PyAV. " + "Run `unsloth studio update` to install them." + ) from exc + + +def is_available() -> bool: + """True when the complete local Whisper backend can be imported.""" + try: + ensure_stt_available() + except SttUnavailableError: + return False + return True + + +def resolve_model_id(model: Optional[str]) -> str: + """Resolve a curated id or validate a custom Hugging Face repository.""" + if not model: + return DEFAULT_STT_MODEL + normalized = model.strip() + if normalized in STT_MODELS: + return normalized + if _HF_REPO_ID.fullmatch(normalized): + return normalized + raise SttModelIdError( + "STT model must be one of Studio's defaults or a Hugging Face " + "repository in 'owner/model' form." + ) + + +def resolve_model_repo(model_id: str) -> str: + """Return the Hub repository for a curated or custom model id.""" + resolved = resolve_model_id(model_id) + return STT_MODELS.get(resolved, resolved) + + +def _is_whisper_config(config: object) -> bool: + """True when Hub/local config metadata identifies a Whisper ASR model.""" + if not isinstance(config, dict): + return False + model_type = config.get("model_type") + if isinstance(model_type, str) and model_type.strip().lower() == "whisper": + return True + architectures = config.get("architectures") + return isinstance(architectures, list) and any( + isinstance(name, str) and name == "WhisperForConditionalGeneration" + for name in architectures + ) + + +def _read_json_object(path: Path) -> dict: + try: + with open(path, "r", encoding = "utf-8") as file: + value = json.load(file) + return value if isinstance(value, dict) else {} + except Exception: + return {} + + +def _active_hf_hub_cache() -> Path: + """Return the active Hub cache while respecting runtime test overrides.""" + explicit = (os.environ.get("HF_HUB_CACHE") or "").strip() + if explicit: + return Path(explicit).expanduser() + hf_home = (os.environ.get("HF_HOME") or "").strip() + if hf_home: + return Path(hf_home).expanduser() / "hub" + from huggingface_hub.constants import HF_HUB_CACHE + + return Path(HF_HUB_CACHE) + + +def _repo_cache_dir(repo: str) -> Path: + return _active_hf_hub_cache() / f"models--{repo.replace('/', '--')}" + + +def _revision_record_path(repo: str) -> Path: + from utils.paths.storage_roots import cache_root + digest = hashlib.sha256(repo.encode("utf-8")).hexdigest() + return cache_root() / "stt-revisions" / f"{digest}.json" + + +def _write_revision_record(repo: str, revision: str) -> None: + """Persist immutable identity only, never an HF-cache absolute path.""" + if not _HF_COMMIT_SHA.fullmatch(revision): + return + path = _revision_record_path(repo) + tmp = path.with_name(f".{path.name}.tmp-{uuid.uuid4().hex[:8]}") + try: + path.parent.mkdir(parents = True, exist_ok = True) + with tmp.open("w", encoding = "utf-8") as handle: + json.dump( + { + "version": _STT_REVISION_RECORD_VERSION, + "repo": repo, + "revision": revision, + }, + handle, + ) + handle.flush() + os.fsync(handle.fileno()) + os.replace(tmp, path) + except OSError as exc: + logger.debug("Could not persist STT revision for %s: %s", repo, exc) + try: + tmp.unlink(missing_ok = True) + except OSError: + pass + + +def _read_revision_record(repo: str) -> Optional[str]: + payload = _read_json_object(_revision_record_path(repo)) + if payload.get("version") != _STT_REVISION_RECORD_VERSION or payload.get("repo") != repo: + return None + revision = payload.get("revision") + return revision if isinstance(revision, str) and _HF_COMMIT_SHA.fullmatch(revision) else None + + +def _safe_snapshot_for_revision(repo: str, revision: str) -> Optional[Path]: + """Resolve a canonical SHA below this repository's active snapshots dir.""" + if not _HF_COMMIT_SHA.fullmatch(revision): + return None + snapshots = _repo_cache_dir(repo) / "snapshots" + candidate = snapshots / revision + try: + snapshots_resolved = snapshots.resolve() + candidate_resolved = candidate.resolve() + except (OSError, RuntimeError): + return None + if snapshots_resolved not in candidate_resolved.parents or not candidate_resolved.is_dir(): + return None + return candidate_resolved + + +def _snapshot_usable(model_id: str, snapshot: Path) -> bool: + if not _snapshot_is_complete(snapshot): + return False + if model_id not in STT_MODELS: + return _is_whisper_config(_read_json_object(snapshot / "config.json")) + return True + + +def _find_complete_cached_snapshot(model: Optional[str]) -> Optional[Path]: + """Find one complete local snapshot without contacting the Hub.""" + model_id = resolve_model_id(model) + repo = resolve_model_repo(model_id) + + recorded = _read_revision_record(repo) + if recorded: + snapshot = _safe_snapshot_for_revision(repo, recorded) + if snapshot is not None and _snapshot_usable(model_id, snapshot): + return snapshot + + ref = _repo_cache_dir(repo) / "refs" / "main" + try: + revision = ref.read_text(encoding = "utf-8").strip() + except OSError: + revision = "" + snapshot = _safe_snapshot_for_revision(repo, revision) + if snapshot is not None and _snapshot_usable(model_id, snapshot): + _write_revision_record(repo, revision) + return snapshot + + snapshots = _repo_cache_dir(repo) / "snapshots" + try: + revisions = sorted( + ( + (path.stat().st_mtime_ns, path.name) + for path in snapshots.iterdir() + if path.is_dir() and _HF_COMMIT_SHA.fullmatch(path.name) + ), + reverse = True, + ) + except OSError: + return None + for _mtime, revision in revisions: + snapshot = _safe_snapshot_for_revision(repo, revision) + if snapshot is not None and _snapshot_usable(model_id, snapshot): + _write_revision_record(repo, revision) + return snapshot + return None + + +def _selected_file_from_sibling(sibling) -> _SelectedHubFile: + lfs = getattr(sibling, "lfs", None) + blob_key = getattr(lfs, "sha256", None) or getattr(sibling, "blob_id", None) + return _SelectedHubFile( + path = sibling.rfilename, + size = max(0, int(getattr(sibling, "size", 0) or 0)), + blob_key = blob_key if isinstance(blob_key, str) and blob_key else None, + ) + + +def _select_snapshot_files(info, load_index) -> tuple[_SelectedHubFile, ...]: + """Select support files and one complete safetensors weight set. Pickle + (pytorch_model.bin) weights are never selected: they are an RCE sink on a + custom repo id (see _STT_SNAPSHOT_SUPPORT_FILES).""" + siblings = { + sibling.rfilename: sibling + for sibling in (getattr(info, "siblings", None) or []) + if isinstance(getattr(sibling, "rfilename", None), str) + } + selected = {name for name in _STT_SNAPSHOT_SUPPORT_FILES if name in siblings} + + index_name: Optional[str] = None + if _STT_SAFETENSORS_INDEX in siblings: + index_name = _STT_SAFETENSORS_INDEX + elif _STT_SAFETENSORS_WEIGHTS in siblings: + selected.add(_STT_SAFETENSORS_WEIGHTS) + else: + raise SttModelCompatibilityError( + "The STT repository has no safetensors model weights. Only safetensors " + "checkpoints are supported; convert the model with save_pretrained(safe_serialization=True)." + ) + + if index_name is not None: + weight_map = load_index(index_name).get("weight_map") + if not isinstance(weight_map, dict) or not weight_map: + raise SttModelCompatibilityError(f"Invalid checkpoint index '{index_name}'.") + shards = set(weight_map.values()) + if not all(isinstance(shard, str) and shard in siblings for shard in shards): + raise SttModelCompatibilityError(f"Checkpoint index '{index_name}' has missing shards.") + # The index JSON is attacker-controlled: a safetensors index can name + # pytorch_model-*.bin shards, which Transformers still loads through + # torch.load (pickle) since it dispatches per shard by file extension. + # Require every shard to be safetensors so no pickle file is selected. + if not all(shard.endswith(".safetensors") for shard in shards): + raise SttModelCompatibilityError( + f"Checkpoint index '{index_name}' references non-safetensors shards." + ) + selected.add(index_name) + selected.update(shards) + + return tuple(_selected_file_from_sibling(siblings[name]) for name in sorted(selected)) + + +def validate_remote_model(model: Optional[str], hf_token: Optional[str] = None) -> dict: + """Verify a custom Hub repository is Whisper-compatible without downloading weights.""" + model_id = resolve_model_id(model) + repo = resolve_model_repo(model_id) + if model_id in STT_MODELS: + return {"model": model_id, "repo": repo} + + try: + from huggingface_hub import HfApi + info = HfApi(token = hf_token or False).model_info( + repo, + expand = ["config", "sha"], + timeout = 10, + ) + except Exception as exc: + raise SttModelCompatibilityError( + f"Could not verify STT model '{model_id}'. " + "Check that the repository exists and your Hugging Face token can access it." + ) from exc + + if not _is_whisper_config(getattr(info, "config", None)): + raise SttModelCompatibilityError( + f"STT model '{model_id}' is not a compatible Transformers Whisper model." + ) + revision = getattr(info, "sha", None) + if not isinstance(revision, str) or not _HF_COMMIT_SHA.fullmatch(revision): + raise SttModelCompatibilityError( + f"Could not resolve an immutable revision for STT model '{model_id}'." + ) + # The commit that was validated; the download pins to it so the repo cannot + # be swapped between validation and snapshot_download (TOCTOU). + return {"model": model_id, "repo": repo, "revision": revision} + + +def _is_missing_local_model_error(exc: BaseException) -> bool: + """Recognize a local-cache-only miss by name/message, without importing HF + internals (tolerates huggingface_hub/Transformers moving the exception).""" + current: Optional[BaseException] = exc + seen: set[int] = set() + while current is not None and id(current) not in seen: + seen.add(id(current)) + if type(current).__name__ in ("LocalEntryNotFoundError", "EntryNotFoundError"): + return True + message = str(current).lower() + if "local_files_only" in message or "does not appear to have a file" in message: + return True + current = current.__cause__ or current.__context__ + return False + + +def _snapshot_is_complete(snapshot: Path) -> bool: + """True when a cached snapshot holds every file loading needs. + + An aborted download can leave only metadata behind, and an offline lookup + cannot know the repo's full file list, so verify config, preprocessor, + tokenizer, and weights directly. is_file() follows cache symlinks, so a + link from an interrupted blob download does not count. + """ + # Safetensors only: a cached pytorch_model.bin is a pickle load path and is + # never treated as a usable snapshot (a repo shipping only pickle weights + # re-resolves and fails closed in _select_snapshot_files). + index = snapshot / _STT_SAFETENSORS_INDEX + if index.is_file(): + # Sharded safetensors checkpoint: every shard must exist and be + # safetensors (a safe index naming .bin shards would still pickle-load + # them, matching the _select_snapshot_files guard). + weight_map = _read_json_object(index).get("weight_map") + if not isinstance(weight_map, dict) or not weight_map: + return False + shards = set(weight_map.values()) + if not all(isinstance(shard, str) and shard.endswith(".safetensors") for shard in shards): + return False + has_weights = all((snapshot / shard).is_file() for shard in shards) + else: + has_weights = (snapshot / _STT_SAFETENSORS_WEIGHTS).is_file() + # WhisperProcessor needs the tokenizer: either the fast tokenizer.json or + # the slow vocab.json + merges.txt pair. + has_tokenizer = (snapshot / "tokenizer.json").is_file() or ( + (snapshot / "vocab.json").is_file() and (snapshot / "merges.txt").is_file() + ) + return ( + has_weights + and has_tokenizer + and (snapshot / "config.json").is_file() + and (snapshot / "preprocessor_config.json").is_file() + ) + + +def is_model_downloaded(model: Optional[str]) -> bool: + """True when a usable Whisper snapshot exists in the local HF cache.""" + try: + return _find_complete_cached_snapshot(model) is not None + except Exception: + return False + + +class _SnapshotDownloadState: + """Tracks one background snapshot_download of a dictation repository. + + Like stt_ggml_sidecar's tracker, but a Transformers checkpoint is a whole + repo, so progress is the byte count of its cache blobs. + """ + + def __init__(self) -> None: + self._lock = threading.Lock() + self._thread: Optional[threading.Thread] = None + self._model_id: Optional[str] = None + self._repo: Optional[str] = None + self._error: Optional[str] = None + self._total_bytes: Optional[int] = None + self._selected_files: tuple[_SelectedHubFile, ...] = () + self._complete = False + + def status(self) -> dict: + with self._lock: + downloading = self._thread is not None and self._thread.is_alive() + show_progress = downloading or self._complete + return { + "downloading": downloading, + "model": self._model_id if downloading else None, + "error": self._error, + "bytes_total": self._total_bytes if show_progress else None, + "bytes_done": self._blob_bytes() if show_progress else None, + } + + def _blob_bytes(self) -> Optional[int]: + """Best-effort progress: bytes in the repo's HF cache blobs. + + Counts only the selected support files and one selected weight format, + including in-progress ``.incomplete`` blobs. + """ + try: + # Caller may hold the non-reentrant self._lock; a bare read is safe. + repo = self._repo + selected_files = self._selected_files + if not repo or not selected_files: + return None + blobs = _repo_cache_dir(repo) / "blobs" + if not blobs.is_dir(): + return 0 + done = 0 + for selected in selected_files: + if not selected.blob_key: + continue + complete = blobs / selected.blob_key + incomplete = blobs / f"{selected.blob_key}.incomplete" + candidate = complete if complete.is_file() else incomplete + if candidate.is_file(): + done += min(candidate.stat().st_size, selected.size) + total = self._total_bytes + return min(done, total) if total is not None else done + except Exception: + return None + + def start( + self, + model_id: str, + hf_token: Optional[str] = None, + revision: Optional[str] = None, + ) -> None: + model_id = resolve_model_id(model_id) + with self._lock: + if self._thread is not None and self._thread.is_alive(): + if self._model_id == model_id: + return + raise SttModelIdError( + f"Another dictation model ('{self._model_id}') is still " + "downloading; wait for it to finish." + ) + self._model_id = model_id + self._repo = resolve_model_repo(model_id) + self._error = None + self._total_bytes = None + self._selected_files = () + self._complete = False + thread = threading.Thread( + target = self._run, args = (self._repo, hf_token, revision), daemon = True + ) + self._thread = thread + thread.start() + + def _run( + self, + repo: str, + hf_token: Optional[str], + revision: Optional[str] = None, + ) -> None: + try: + from huggingface_hub import HfApi, hf_hub_download, snapshot_download + + info = HfApi(token = hf_token or None).model_info( + repo, + revision = revision, + files_metadata = True, + timeout = 30, + ) + if not revision: + revision = getattr(info, "sha", None) + if not isinstance(revision, str) or not _HF_COMMIT_SHA.fullmatch(revision): + raise SttModelCompatibilityError( + f"Could not resolve an immutable revision for STT model '{repo}'." + ) + + def load_index(filename: str) -> dict: + path = hf_hub_download( + repo_id = repo, + filename = filename, + revision = revision, + token = hf_token or None, + ) + return _read_json_object(Path(path)) + + selected_files = _select_snapshot_files(info, load_index) + total = sum(selected.size for selected in selected_files) + with self._lock: + self._selected_files = selected_files + self._total_bytes = total or None + snapshot = Path( + snapshot_download( + repo_id = repo, + revision = revision, + allow_patterns = [selected.path for selected in selected_files], + token = hf_token or None, + ) + ) + if not _snapshot_is_complete(snapshot): + raise SttModelCompatibilityError( + f"Downloaded STT snapshot for '{repo}' is incomplete." + ) + _write_revision_record(repo, revision) + with self._lock: + self._complete = True + except Exception as exc: + logger.warning("STT snapshot download failed for %s: %s", repo, exc) + with self._lock: + self._error = f"Download failed for '{repo}'." + + +_download_state = _SnapshotDownloadState() + + +def start_model_download( + model: Optional[str], + hf_token: Optional[str] = None, + revision: Optional[str] = None, +) -> None: + _download_state.start(resolve_model_id(model), hf_token, revision = revision) + + +def download_status() -> dict: + return _download_state.status() + + +def _training_active() -> bool: + try: + from core.training import get_training_backend + return bool(get_training_backend().is_training_active()) + except Exception: + return False + + +def _clear_device_cache(device: Optional[str]) -> None: + gc.collect() + try: + import torch + if device == "cuda": + torch.cuda.empty_cache() + elif device == "mps": + torch.mps.empty_cache() + except Exception: + pass + + +def _pick_device(): + """Return (device, torch_dtype) for the Whisper model. + + CUDA uses float16. MPS and CPU use float32: Whisper's decoder is unstable in + float16 on MPS and degenerates into repeated tokens. + """ + try: + import torch + + # New loads use CPU during training; a resident GPU model may stay put + # when the training admission check confirms enough headroom. + training_active = _training_active() + if not training_active and torch.cuda.is_available(): + return "cuda", torch.float16 + if ( + not training_active + and getattr(torch.backends, "mps", None) is not None + and torch.backends.mps.is_available() + ): + return "mps", torch.float32 + return "cpu", torch.float32 + except Exception as exc: + logger.debug("STT device detection failed, using CPU: %s", exc) + import torch + return "cpu", torch.float32 + + +def _decode_audio_bounded(audio: bytes): + """Decode to 16 kHz mono PCM without buffering unbounded audio. + + A small, highly-compressed upload can expand far past the encoded request + limit once decoded, so decode frame-by-frame and enforce the sample cap as + frames arrive, then hand the array straight to Whisper. + """ + try: + import av + import numpy as np + from av.error import FFmpegError, InvalidDataError + except ImportError as exc: + raise SttUnavailableError( + "Speech-to-text needs the PyAV package to decode audio. " + "Run `unsloth studio update` to install it." + ) from exc + + max_samples = _MAX_AUDIO_SECONDS * _TARGET_SAMPLE_RATE + sample_count = 0 + raw_buffer = io.BytesIO() + resampler = av.audio.resampler.AudioResampler( + format = "s16", + layout = "mono", + rate = _TARGET_SAMPLE_RATE, + ) + # Group frames before resampling so short clips need one resampler call + # rather than one per codec frame. + fifo = av.audio.fifo.AudioFifo() + + def write_frame(frame) -> None: + nonlocal sample_count + array = frame.to_ndarray() + sample_count += array.size + if sample_count > max_samples: + max_minutes = _MAX_AUDIO_SECONDS // 60 + unit = "minute" if max_minutes == 1 else "minutes" + raise SttAudioTooLongError(f"Audio must be {max_minutes} {unit} or shorter.") + raw_buffer.write(array) + + try: + with av.open(io.BytesIO(audio), mode = "r", metadata_errors = "ignore") as container: + if not container.streams.audio: + raise SttAudioDecodeError("Could not decode the audio.") + frames = iter(container.decode(audio = 0)) + while True: + try: + frame = next(frames) + except StopIteration: + break + except InvalidDataError: + # Skip a corrupt frame rather than fail the whole transcription. + continue + frame.pts = None + fifo.write(frame) + if fifo.samples >= 500000: + for resampled in resampler.resample(fifo.read()): + write_frame(resampled) + if fifo.samples > 0: + for resampled in resampler.resample(fifo.read()): + write_frame(resampled) + for resampled in resampler.resample(None): + write_frame(resampled) + except (SttAudioDecodeError, SttAudioTooLongError): + raise + except (FFmpegError, ValueError, RuntimeError) as exc: + raise SttAudioDecodeError("Could not decode the audio.") from exc + finally: + del fifo, resampler + + if sample_count == 0: + raise SttAudioDecodeError("Could not decode the audio.") + decoded = np.frombuffer(raw_buffer.getbuffer(), dtype = np.int16).astype(np.float32) + decoded /= 32768.0 + return decoded + + +class WhisperSttSidecar: + """Lazily loaded Whisper model with idle eviction. Thread-safe.""" + + def __init__(self, keep_alive_seconds: float = STT_KEEP_ALIVE_SECONDS) -> None: + self._engine = None + self._model_id: Optional[str] = None + self._device: Optional[str] = None + self._lock = threading.RLock() + self._load_state_lock = threading.Lock() + self._loading = False + self._load_cancel_event: Optional[threading.Event] = None + self._keep_alive_seconds = max(0.0, keep_alive_seconds) + self._idle_timer: Optional[threading.Timer] = None + self._idle_generation = 0 + + @property + def loaded_model(self) -> Optional[str]: + return self._model_id + + @property + def device(self) -> Optional[str]: + return self._device + + def is_loading(self) -> bool: + with self._load_state_lock: + return self._loading + + def cancel_pending_load(self) -> bool: + """Cancel a model load without waiting for the model lock.""" + with self._load_state_lock: + event = self._load_cancel_event + if not self._loading or event is None: + return False + event.set() + return True + + def wait_for_load_to_settle(self) -> None: + """Block until any in-flight load() has exited and freed its memory. + + load() holds self._lock throughout, including the from_pretrained()/ + .to(device) allocation and cancel cleanup, so acquiring the lock here + waits for that memory to be freed. + """ + with self._lock: + pass + + def _begin_load(self) -> threading.Event: + event = threading.Event() + with self._load_state_lock: + self._load_cancel_event = event + self._loading = True + return event + + def _end_load(self, event: threading.Event) -> None: + with self._load_state_lock: + if self._load_cancel_event is event: + self._load_cancel_event = None + self._loading = False + + @staticmethod + def _raise_if_load_cancelled(event: threading.Event) -> None: + if event.is_set(): + raise SttLoadCancelledError("STT model loading was cancelled so training could start.") + + @property + def keep_alive_seconds(self) -> float: + return self._keep_alive_seconds + + def _cancel_idle_unload_locked(self) -> None: + self._idle_generation += 1 + timer = self._idle_timer + self._idle_timer = None + if timer is not None: + timer.cancel() + + def _schedule_idle_unload_locked(self) -> None: + self._cancel_idle_unload_locked() + if self._engine is None or self._keep_alive_seconds <= 0: + return + generation = self._idle_generation + timer = threading.Timer( + self._keep_alive_seconds, + self._idle_unload, + args = (generation,), + ) + timer.daemon = True + self._idle_timer = timer + timer.start() + + def _idle_unload(self, generation: int) -> None: + with self._lock: + if generation != self._idle_generation or self._engine is None: + return + logger.info("Unloading idle STT model %s", self._model_id) + self._release_engine_locked() + + def _release_engine_locked(self) -> None: + self._cancel_idle_unload_locked() + engine = self._engine + device = self._device + self._engine = None + self._model_id = None + self._device = None + del engine + _clear_device_cache(device) + + def _build_model(self, snapshot_path: str, device: str, dtype, cancel_event: threading.Event): + """Load a Whisper model + processor from the local Hub cache. + + local_files_only keeps the Model Hub the only download path; a cache + miss raises so the caller can surface SttModelNotDownloadedError. + """ + import torch + from transformers import WhisperForConditionalGeneration, WhisperProcessor + + processor = None + model = None + try: + processor = WhisperProcessor.from_pretrained(snapshot_path, local_files_only = True) + self._raise_if_load_cancelled(cancel_event) + # use_safetensors forces the pickle-free load path even if a + # pytorch_model.bin somehow reached the cache; the selector and the + # completeness check already exclude pickle weights upstream. + model = WhisperForConditionalGeneration.from_pretrained( + snapshot_path, torch_dtype = dtype, local_files_only = True, use_safetensors = True + ) + self._raise_if_load_cancelled(cancel_event) + model.to(torch.device(device)) + self._raise_if_load_cancelled(cancel_event) + model.eval() + return model, processor + except SttLoadCancelledError: + model = None + processor = None + _clear_device_cache(device) + raise + + def _ensure_model_downloaded(self, model_id: str) -> _CachedSttSnapshot: + """Validate the local snapshot before decode or model replacement. + + Returns the checkpoint's multilingual flag when local metadata provides + it. Curated defaults are known multilingual. + """ + model_id = resolve_model_id(model_id) + with self._lock: + if self._engine is not None and self._model_id == model_id: + resident_model = ( + self._engine[0] if isinstance(self._engine, (tuple, list)) else self._engine + ) + generation_config = getattr(resident_model, "generation_config", None) + is_multilingual = getattr(generation_config, "is_multilingual", None) + return _CachedSttSnapshot( + path = None, + is_multilingual = is_multilingual if isinstance(is_multilingual, bool) else None, + ) + snapshot_path = _find_complete_cached_snapshot(model_id) + if snapshot_path is None: + raise SttModelNotDownloadedError( + f"STT model '{model_id}' is not downloaded. " + "Download it in Settings, then Voice, before loading it." + ) + + if model_id in STT_MODELS: + return _CachedSttSnapshot(path = snapshot_path, is_multilingual = True) + + if not _is_whisper_config(_read_json_object(snapshot_path / "config.json")): + raise SttModelCompatibilityError( + f"STT model '{model_id}' is not a compatible Transformers Whisper model." + ) + generation_config = _read_json_object(snapshot_path / "generation_config.json") + is_multilingual = generation_config.get("is_multilingual") + if isinstance(is_multilingual, bool): + return _CachedSttSnapshot(path = snapshot_path, is_multilingual = is_multilingual) + if resolve_model_repo(model_id).lower().endswith(".en"): + return _CachedSttSnapshot(path = snapshot_path, is_multilingual = False) + return _CachedSttSnapshot(path = snapshot_path, is_multilingual = None) + + def load(self, model: Optional[str] = None): + """Load (or switch to) a model, reusing it if already resident. + + Returns a ``(model, processor)`` pair. + """ + model_id = resolve_model_id(model) + with self._lock: + ensure_stt_available() + if self._engine is not None and self._model_id == model_id: + self._schedule_idle_unload_locked() + return self._engine + import torch + + cancel_event = self._begin_load() + candidate = None + device: Optional[str] = None + try: + cached = self._ensure_model_downloaded(model_id) + snapshot_path = cached.path + if snapshot_path is None: + raise SttModelNotDownloadedError( + f"STT model '{model_id}' is not downloaded. " + "Download it in Settings, then Voice, before loading it." + ) + self._raise_if_load_cancelled(cancel_event) + device, dtype = _pick_device() + self._release_engine_locked() + logger.info("Loading STT model %s (%s) on %s", model_id, snapshot_path, device) + + def not_downloaded(cause: BaseException) -> SttModelNotDownloadedError: + return SttModelNotDownloadedError( + f"STT model '{model_id}' is not downloaded. " + "Download it in Settings, then Voice, before loading it." + ) + + retry_on_cpu = False + try: + candidate = self._build_model(str(snapshot_path), device, dtype, cancel_event) + self._raise_if_load_cancelled(cancel_event) + except SttLoadCancelledError: + raise + except Exception as exc: + if _is_missing_local_model_error(exc): + raise not_downloaded(exc) from exc + if device == "cpu": + raise + logger.warning("STT load on %s failed (%s); retrying on CPU", device, exc) + retry_on_cpu = True + if retry_on_cpu: + # Retry outside the handler: live exception state pins frames + # referencing the partly loaded model, so leave it before + # clearing the cache to release that memory. + _clear_device_cache(device) + try: + candidate = self._build_model( + str(snapshot_path), + "cpu", + torch.float32, + cancel_event, + ) + self._raise_if_load_cancelled(cancel_event) + except SttLoadCancelledError: + raise + except Exception as cpu_exc: + if _is_missing_local_model_error(cpu_exc): + raise not_downloaded(cpu_exc) from cpu_exc + raise + device = "cpu" + with self._load_state_lock: + self._raise_if_load_cancelled(cancel_event) + self._engine = candidate + self._model_id = model_id + self._device = device + self._load_cancel_event = None + self._loading = False + self._schedule_idle_unload_locked() + logger.info("STT model %s ready on %s", model_id, device) + return self._engine + except SttLoadCancelledError: + candidate = None + self._release_engine_locked() + _clear_device_cache(device) + raise + finally: + self._end_load(cancel_event) + + def _transcribe_decoded(self, model_id: str, decoded_audio, generate_kwargs: dict) -> str: + """Run Whisper on already-decoded 16 kHz mono PCM and return text. + + Feeds a pre-decoded array so nothing here touches the Transformers audio + path (torchcodec/ffmpeg). Splits into 30s windows (Whisper's receptive + field); short clips take one pass. + """ + import torch + + model, processor = self.load(model_id) + effective_generate_kwargs = dict(generate_kwargs) + generation_config = getattr(model, "generation_config", None) + if getattr(generation_config, "is_multilingual", None) is False: + # English-only checkpoints fix language and task in their generation + # config, and Transformers rejects passing them here. + effective_generate_kwargs.pop("task", None) + effective_generate_kwargs.pop("language", None) + window = 30 * _TARGET_SAMPLE_RATE + target_dtype = getattr(model, "dtype", None) + parts: list[str] = [] + with torch.no_grad(): + for start in range(0, max(len(decoded_audio), 1), window): + segment = decoded_audio[start : start + window] + if segment.size == 0: + continue + inputs = processor( + segment, + sampling_rate = _TARGET_SAMPLE_RATE, + return_tensors = "pt", + ) + features = inputs.input_features.to(model.device) + if target_dtype is not None: + features = features.to(target_dtype) + generated = model.generate(features, **effective_generate_kwargs) + text = processor.batch_decode(generated, skip_special_tokens = True) + parts.append(text[0] if text else "") + return " ".join(part.strip() for part in parts if part.strip()).strip() + + def transcribe( + self, + audio: bytes, + model: Optional[str] = None, + language: Optional[str] = None, + fast: bool = False, + ) -> dict: + """Transcribe encoded audio bytes to text. + + Accepts any container PyAV can decode: wav, mp3, opus/webm, ogg, + m4a/aac. Returns {text, language, duration, model}. + """ + # Reject a missing runtime up front, before the cache and bounded decode. + ensure_stt_available() + # A set language beats auto-detect. API takes BCP-47; Whisper wants short + # codes like en or fr. + lang = normalize_whisper_language(language) + # Pin the requested id: another request may switch the resident model + # mid-transcription, so sidecar state is not this request's identity. + model_id = resolve_model_id(model) + known_languages = _known_whisper_languages() + if lang is not None and known_languages is not None and lang not in known_languages: + raise SttLanguageError( + f"Language '{language}' is not supported by STT model '{model_id}'." + ) + cached = self._ensure_model_downloaded(model_id) + if cached.is_multilingual is False and lang not in (None, "en"): + raise SttLanguageError( + f"Language '{language}' is not supported by English-only STT model '{model_id}'." + ) + decoded_audio = _decode_audio_bounded(audio) + # condition_on_prev_tokens=False stops a fresh clip inheriting prior + # context, which causes runaway repeats. + generate_kwargs = { + "task": "transcribe", + "condition_on_prev_tokens": False, + "num_beams": 5, + } + if lang is not None: + generate_kwargs["language"] = lang + if fast: + # Short voiced clips: greedy decoding drops beam search for latency. + generate_kwargs["num_beams"] = 1 + # Serialize inference with model switches and unloads. + with self._lock: + try: + text = self._transcribe_decoded(model_id, decoded_audio, generate_kwargs) + finally: + self._schedule_idle_unload_locked() + duration = (len(decoded_audio) / _TARGET_SAMPLE_RATE) if len(decoded_audio) else None + return { + "text": text, + "language": lang, + "duration": duration, + "model": model_id, + } + + def unload(self) -> None: + with self._lock: + self._release_engine_locked() + + +_sidecar: Optional[WhisperSttSidecar] = None + + +def get_stt_sidecar() -> WhisperSttSidecar: + global _sidecar + if _sidecar is None: + _sidecar = WhisperSttSidecar() + return _sidecar diff --git a/studio/backend/core/inference/tool_loop_controller.py b/studio/backend/core/inference/tool_loop_controller.py index f7ed450d11..61643b5795 100644 --- a/studio/backend/core/inference/tool_loop_controller.py +++ b/studio/backend/core/inference/tool_loop_controller.py @@ -1,7 +1,7 @@ # SPDX-License-Identifier: AGPL-3.0-only # Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 -"""Shared controller state for Studio local agentic tool loops. +"""Shared controller state for Unsloth local agentic tool loops. This module is intentionally dependency-light: it owns only per-response ledger state and value objects used by the GGUF and safetensors loops. diff --git a/studio/backend/core/inference/tools.py b/studio/backend/core/inference/tools.py index 5fd57e1b2c..0ef6dd46cf 100644 --- a/studio/backend/core/inference/tools.py +++ b/studio/backend/core/inference/tools.py @@ -18,6 +18,7 @@ import queue import random import re import shlex +import shutil import ssl import subprocess import sys @@ -328,6 +329,7 @@ def _find_blocked_commands(command: str) -> set[str]: # Directory holding the sandbox ``sitecustomize.py`` shim (code-interpreter # path remap); placed on the sandboxed child's PYTHONPATH in _build_safe_env. _SANDBOX_SITE_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)), "sandbox_site") + # ── "Approve for me" (permission_mode="auto") safety detection ────────────── # Auto mode pauses only calls classified here as potentially unsafe. The sandbox # and hard blocks (blocklist, rlimits) still apply at run time; this gate only @@ -2491,18 +2493,127 @@ def is_potentially_unsafe_tool_call(name: str, arguments: dict) -> bool: return True +def _canon_win_path(p: str) -> str: + """Canonical form for trust comparison: realpath (expands 8.3 aliases and + resolves junctions/symlinks) + normcase/normpath.""" + return os.path.normcase(os.path.normpath(os.path.realpath(p))) + + +def _augment_native_program_roots(roots: list[str]) -> list[str]: + """Add the native Program Files sibling for any x86 root by stripping the + `` (x86)`` suffix, so a 32-bit process (whose known-folder ids map only to + the x86 root) still trusts a 64-bit Git install.""" + out = list(roots) + for root in roots: + base = root.rstrip("\\/") + if base.lower().endswith(" (x86)"): + native = base[: -len(" (x86)")] + if native and native not in out: + out.append(native) + return out + + +def _windows_program_roots() -> list[str]: + """Program Files install roots, resolved ONLY from the Windows known-folder + API (SHGetKnownFolderPath). Fails closed (returns ``[]``) if the API is + unavailable: env vars (%ProgramFiles%, even %SystemDrive%) are caller- + overrideable and could relocate the trust boundary, so we never derive a + trusted root from them. On any real Windows host shell32 is present, so + this only returns empty in a broken/non-Windows environment where the + sandbox git-PATH feature is not needed anyway (#7317). + """ + roots: list[str] = [] + try: + import ctypes + from ctypes import wintypes + + # FOLDERID_ProgramFiles, _ProgramFilesX86, _ProgramFilesX64. The X64 + # id (Win10 1703+) yields the native root even from a 32-bit process, + # where the first two both map to Program Files (x86). + folder_ids = ( + "{905e63b6-c1bf-494e-b29c-65b732d3d21a}", + "{7C5A40EF-A0FB-4BFC-874A-C0F2E0B9FA8E}", + "{6D809377-6AF0-444b-8957-A3773F02200E}", + ) + _SHGet = ctypes.windll.shell32.SHGetKnownFolderPath + _CoTaskMemFree = ctypes.windll.ole32.CoTaskMemFree + for fid in folder_ids: + guid = ctypes.create_string_buffer(16) + ctypes.windll.ole32.CLSIDFromString(wintypes.LPCWSTR(fid), ctypes.byref(guid)) + ptr = ctypes.c_wchar_p() + if _SHGet(ctypes.byref(guid), 0, None, ctypes.byref(ptr)) == 0: + if ptr.value: + roots.append(ptr.value) + _CoTaskMemFree(ptr) + except Exception: + return [] + return _augment_native_program_roots(roots) + + +def _resolve_trusted_windows_git() -> tuple[str, str]: + """Find a git launcher in a TRUSTED Program Files dir. Returns + ``(canonical_dir, ext)`` or ``("", "")``. + + ``shutil.which`` returns only the first PATH match, which may be an + untrusted user shim; scan the remaining PATH entries for a later trusted + Git so bare ``git`` still resolves (#7317). + """ + exts = [e for e in (os.environ.get("PATHEXT") or ".EXE;.CMD;.BAT;.COM").split(os.pathsep)] + candidates: list[str] = [] + primary = shutil.which("git") + if primary: + candidates.append(primary) + for entry in (os.environ.get("PATH") or "").split(os.pathsep): + entry = entry.strip().strip('"') + if not entry or not os.path.isabs(entry): + continue + for ext in exts: + cand = os.path.join(entry, "git" + ext) + if os.path.isfile(cand): + candidates.append(cand) + for git_exe in candidates: + git_dir = os.path.dirname(git_exe) + if os.path.isabs(git_dir) and _is_trusted_windows_program_dir(git_dir): + return os.path.realpath(git_dir), os.path.splitext(git_exe)[1].upper() + return "", "" + + +def _is_trusted_windows_program_dir(path: str) -> bool: + """True when ``path`` sits under a system-managed Program Files root. + + Only the Program Files roots are trusted (admin-writable only), resolved + via the known-folder API so an overridden env var cannot relocate them, + never ``%SystemRoot%`` (Git does not install there and it holds + world-writable subdirs like ``Windows\\Temp``). Per-user managers + (Scoop/Choco shims under the profile) are refused. Paths are canonicalized + so 8.3 aliases and junctions still resolve to their real root (#7317). + """ + norm = _canon_win_path(path) + for root in _windows_program_roots(): + root_norm = _canon_win_path(root) + if norm == root_norm or norm.startswith(root_norm + os.sep): + return True + return False + + def _build_safe_env(workdir: str) -> dict[str, str]: """Build a minimal, credential-free environment for sandboxed subprocesses. Whitelist-built from scratch (parent env NOT inherited): only PATH/HOME/ TMPDIR/LANG/TERM/PYTHONIOENCODING/PYTHONPATH (+VIRTUAL_ENV or Windows - SystemRoot) reach the child; all credential vars (HF_TOKEN, AWS_*, etc.) - are absent. HOME points at the sandbox workdir so SDKs can't read the + SystemRoot and a minimal PATHEXT) reach the child; all credential vars + (HF_TOKEN, AWS_*, etc.) are absent. HOME points at the sandbox workdir so SDKs can't read the operator's cached creds. PYTHONPATH carries only the sandbox sitecustomize shim directory. + + PATH starts with the Studio interpreter / venv and OS system dirs so + ``python``/``pip`` stay pinned. On Windows only, Git-for-Windows install + dirs from the host PATH are appended so bare ``git`` resolves (#7317). + User-writable host PATH entries (venv, ``node_modules/.bin``, etc.) are + never inherited — they could shadow auto-safe terminal commands. """ # Start from the running interpreter's dir so 'python'/'pip' resolve to the - # same environment the Studio server runs in. + # same environment the Unsloth server runs in. exe_dir = os.path.dirname(sys.executable) path_entries = [exe_dir] if exe_dir else [] @@ -2519,6 +2630,20 @@ def _build_safe_env(workdir: str) -> dict[str, str]: else: path_entries.extend(["/usr/local/bin", "/usr/bin", "/bin"]) + # Windows Git installs live outside System32; inherit the dir of the git + # the HOST shell resolves, but ONLY when it sits under a system install + # root (Program Files, windir). A user-writable dir (Scoop/Choco shims) + # is refused: it would let an attacker drop rg.exe/jq.exe beside git and + # have an auto-approved bare command execute it (#7317). + git_ext = "" + if sys.platform == "win32": + # Append the CANONICAL (realpath) trusted git dir, scanning past any + # untrusted user shim that sorts first on PATH; the canonical path + # cannot be retargeted via a junction after the trust check. + _trusted_git_dir, git_ext = _resolve_trusted_windows_git() + if _trusted_git_dir: + path_entries.append(_trusted_git_dir) + # Deduplicate, preserving order. deduped = list(dict.fromkeys(p for p in path_entries if p)) @@ -2538,6 +2663,15 @@ def _build_safe_env(workdir: str) -> dict[str, str]: # Windows needs SystemRoot for Python/subprocess to work. if sys.platform == "win32": env["SystemRoot"] = os.environ.get("SystemRoot", r"C:\Windows") + # Restrict PATHEXT so cwd .BAT/.CMD cannot hijack bare names (#7317). + pathext = ".EXE;.COM" + if git_ext and git_ext not in (".EXE", ".COM"): + # Keep the host git launcher (e.g. a .CMD shim) resolvable. + pathext += ";" + git_ext + env["PATHEXT"] = pathext + # cmd/CreateProcess search cwd before PATH for bare names; disable so + # a workdir rg.exe/git.exe cannot shadow auto-approved commands. + env["NoDefaultCurrentDirectoryInExePath"] = "1" return env @@ -2792,7 +2926,7 @@ def _bypass_preexec(): """Minimal pre-exec for bypass exec: os.setsid() only. Required, not a restriction: _kill_process_tree does killpg(getpgid(child)), - so without a new session a timeout/cancel would kill the Studio server too. + so without a new session a timeout/cancel would kill the Unsloth server too. """ try: os.setsid() @@ -2800,13 +2934,13 @@ def _bypass_preexec(): pass -# Hardening the Studio parent is done once (PR_SET_DUMPABLE is process-global +# Hardening the Unsloth parent is done once (PR_SET_DUMPABLE is process-global # and sticky); guarded so repeated bypass calls do not re-issue the prctl. _parent_proc_hardened = False def _harden_parent_against_proc_env_leak() -> bool: - """Make the Studio process's /proc//environ unreadable to its children. + """Make the Unsloth process's /proc//environ unreadable to its children. Stripping the child env is not enough on Linux: a bypassed same-UID child can read /proc//environ to recover the parent's unfiltered @@ -5482,7 +5616,7 @@ def _truncate(text: str, limit: int = _MAX_OUTPUT_CHARS) -> str: # ChatGPT code-interpreter path conventions models write out of habit; none -# exist in the Studio sandbox, so a failure on one earns the retry hint. +# exist in the Unsloth sandbox, so a failure on one earns the retry hint. _MISSING_PATH_PREFIXES = ( "/mnt/data", "/mnt/outputs", @@ -5688,7 +5822,7 @@ def _python_exec( # Close the /proc//environ secret-recovery path first; if it # cannot be applied, fail closed rather than leak the parent environ. return ( - "Execution error: could not harden the Studio process against " + "Execution error: could not harden the Unsloth process against " "/proc environment reads; refusing bypass execution." ) @@ -5833,7 +5967,7 @@ def _bash_exec( # Close the /proc//environ secret-recovery path first; if it # cannot be applied, fail closed rather than leak the parent environ. return ( - "Execution error: could not harden the Studio process against " + "Execution error: could not harden the Unsloth process against " "/proc environment reads; refusing bypass execution." ) diff --git a/studio/backend/core/inference/worker.py b/studio/backend/core/inference/worker.py index e4628dcea8..367de196f7 100644 --- a/studio/backend/core/inference/worker.py +++ b/studio/backend/core/inference/worker.py @@ -513,20 +513,25 @@ def _handle_generate(backend, cmd: dict, resp_queue: Any, cancel_event) -> None: logger.info("Starting text generation for request_id=%s", request_id) - for cumulative_text in generator: - # cancel_event is an mp.Event — checked instantly, no queue polling. - if cancel_event.is_set(): - logger.info("Generation cancelled for request %s", request_id) - break + try: + for cumulative_text in generator: + # cancel_event is an mp.Event — checked instantly, no queue polling. + if cancel_event.is_set(): + logger.info("Generation cancelled for request %s", request_id) + break - _send_response( - resp_queue, - { - "type": "token", - "request_id": request_id, - "text": cumulative_text, - }, - ) + _send_response( + resp_queue, + { + "type": "token", + "request_id": request_id, + "text": cumulative_text, + }, + ) + finally: + close = getattr(generator, "close", None) + if callable(close): + close() _send_response( resp_queue, @@ -789,7 +794,7 @@ def run_inference_process( env = os.getenv("ENVIRONMENT_TYPE", "production"), ) - apply_gpu_ids(config.get("resolved_gpu_ids")) + apply_gpu_ids(config.get("resolved_gpu_ids"), backend = config.get("device_backend")) model_name = config["model_name"] diff --git a/studio/backend/core/rag/captioner.py b/studio/backend/core/rag/captioner.py index 6d1512a770..8398506f21 100644 --- a/studio/backend/core/rag/captioner.py +++ b/studio/backend/core/rag/captioner.py @@ -6,7 +6,7 @@ Both turn pixels into indexable text and are a no-op (never raise) without a loaded vision model. They reuse the chat model's vision endpoint, so it must be served with ``--ubatch-size`` >= one image's tokens (some encoders, e.g. Gemma, attend -non-causally and abort otherwise); Studio's vision chat already requires this.""" +non-causally and abort otherwise); Unsloth's vision chat already requires this.""" from __future__ import annotations diff --git a/studio/backend/core/rag/config.py b/studio/backend/core/rag/config.py index 2de32a68e4..f54d795731 100644 --- a/studio/backend/core/rag/config.py +++ b/studio/backend/core/rag/config.py @@ -87,6 +87,22 @@ def _names_gguf(model: str) -> bool: return "gguf" in re.split(r"[^a-z0-9]+", model.lower()) +def gguf_repo_for_embedding_model(model: str) -> str: + """GGUF repo for ``model``, honoring an explicit companion override.""" + if "RAG_EMBED_GGUF_REPO" in os.environ: + return EMBED_GGUF_REPO + if model == DEFAULT_EMBEDDING_MODEL: + return EMBED_GGUF_REPO + if _names_gguf(model): + return model + return f"{model}-GGUF" + + +def default_gguf_repo() -> str: + """GGUF companion for the env/default embedding model.""" + return gguf_repo_for_embedding_model(EMBEDDING_MODEL) + + def effective_gguf_repo() -> str: """GGUF repo for the llama-server backend, tracking the effective model. @@ -95,14 +111,7 @@ def effective_gguf_repo() -> str: ``-GGUF`` companion repo (the unsloth convention the default pair follows), or is used as-is when it already names a GGUF repo. """ - if "RAG_EMBED_GGUF_REPO" in os.environ: - return EMBED_GGUF_REPO - model = effective_embedding_model() - if model == DEFAULT_EMBEDDING_MODEL: - return EMBED_GGUF_REPO - if _names_gguf(model): - return model - return f"{model}-GGUF" + return gguf_repo_for_embedding_model(effective_embedding_model()) # llama-server backend only. F16 over Q8_0: faster (no per-block dequant for this diff --git a/studio/backend/core/rag/embed_llama_server.py b/studio/backend/core/rag/embed_llama_server.py index 46a282c939..facd989b27 100644 --- a/studio/backend/core/rag/embed_llama_server.py +++ b/studio/backend/core/rag/embed_llama_server.py @@ -10,7 +10,7 @@ Opt-in (``RAG_EMBED_BACKEND=llama-server``). Runs a dedicated Device is ``auto`` (GPU when present, else CPU, falling back to CPU if a GPU start fails); ``RAG_EMBED_DEVICE`` forces it. We call only llama_cpp's *static* helpers (no torch), copying the instance-coupled bits locally, since constructing a -``LlamaCppBackend`` runs an ``__init__`` reaper that kills any Studio llama-server +``LlamaCppBackend`` runs an ``__init__`` reaper that kills any Unsloth llama-server -- so each request re-spawns ours if it died (self-heal). """ @@ -188,7 +188,14 @@ class LlamaServerBackend: match = [f for f in files if variant in f.lower()] or files filename = sorted(match, key = len)[0] logger.info("resolving GGUF embedder %s/%s", repo, filename) - self._model_path = hf_hub_download(repo_id = repo, filename = filename, token = token) + from utils.hf_cache_settings import active_hf_hub_cache + + self._model_path = hf_hub_download( + repo_id = repo, + filename = filename, + token = token, + cache_dir = active_hf_hub_cache(), + ) self._model_repo = desired self._dim = None return self._model_path diff --git a/studio/backend/core/rag/embeddings.py b/studio/backend/core/rag/embeddings.py index b0ecedd593..3354585d2a 100644 --- a/studio/backend/core/rag/embeddings.py +++ b/studio/backend/core/rag/embeddings.py @@ -22,6 +22,7 @@ from typing import Callable from utils.hardware.hardware import DeviceType, get_device from utils.transformers_dtype import dtype_kwargs +from utils.utils import hf_env_offline from . import config @@ -39,7 +40,7 @@ _model = None _name: str | None = None -# Studio device -> torch device string. Apple has no torch device -> CPU. +# Unsloth device -> torch device string. Apple has no torch device -> CPU. _TORCH_DEVICE = {DeviceType.CUDA: "cuda", DeviceType.XPU: "xpu"} @@ -103,9 +104,15 @@ def _st_module_subdirs(name: str, token: str | None) -> tuple[str, ...]: else: from huggingface_hub import hf_hub_download from huggingface_hub.utils import EntryNotFoundError + from utils.hf_cache_settings import active_hf_hub_cache try: - local = hf_hub_download(name, "modules.json", token = token or None) + local = hf_hub_download( + name, + "modules.json", + token = token or None, + cache_dir = active_hf_hub_cache(), + ) except EntryNotFoundError: return () data = json.loads(open(local).read()) @@ -119,30 +126,55 @@ def _st_module_subdirs(name: str, token: str | None) -> tuple[str, ...]: return () -def _guard_model_security(name: str) -> None: +def _guard_model_security(name: str, local_only: bool = False) -> None: """Refuse to load a repo HF flagged as unsafe: a poisoned pickle deserializes inside SentenceTransformer regardless of trust_remote_code. Defense in depth behind the /settings gate (a name can also arrive via env/default); local paths and unreachable scans fail open inside evaluate_file_security. Never bricks the embedder on a gate error. + + ``local_only`` (offline) inspects the local cache; subdir probes are skipped (they'd hit the + network and hang, and the offline gate walks the whole snapshot anyway). """ try: from utils.security import evaluate_file_security, security_load_subdirs token = _ambient_hf_token() - # Union the audio-model load roots with the ST module dirs so a flagged pickle - # directly under a Transformer module dir (0_Transformer/) blocks instead of - # passing as an unreferenced nested shard. - load_subdirs = tuple( - dict.fromkeys((*security_load_subdirs(name, token), *_st_module_subdirs(name, token))) - ) - blocked = evaluate_file_security(name, hf_token = token, load_subdirs = load_subdirs).blocked + if local_only: + load_subdirs = () + else: + # Union audio-model load roots with ST module dirs so a flagged pickle under a + # Transformer module dir blocks instead of passing as an unreferenced nested shard. + load_subdirs = tuple( + dict.fromkeys( + (*security_load_subdirs(name, token), *_st_module_subdirs(name, token)) + ) + ) + blocked = evaluate_file_security( + name, hf_token = token, load_subdirs = load_subdirs, local_only_load = local_only + ).blocked except Exception: return if blocked: - raise UnsafeEmbeddingModelError( - f"Embedding model {name!r} is flagged as unsafe by Hugging Face's security " - "scan; refusing to load. Set a different RAG embedding model." + reason = ( + "has cached pickle weights that cannot be security-scanned offline and no " + "safetensors alternative" + if local_only + else "is flagged as unsafe by Hugging Face's security scan" ) + raise UnsafeEmbeddingModelError( + f"Embedding model {name!r} {reason}; refusing to load. " + "Set a different RAG embedding model." + ) + + +def _st_accepts_local_files_only(st_cls) -> bool: + """Whether this SentenceTransformer version accepts local_files_only; passing it to an + older constructor raises, so gate on the signature.""" + try: + import inspect + return "local_files_only" in inspect.signature(st_cls.__init__).parameters + except Exception: + return False def _get(model_name: str | None = None): @@ -150,15 +182,35 @@ def _get(model_name: str | None = None): for a ~1.5x speedup at negligible accuracy loss.""" global _model, _name name = model_name or config.effective_embedding_model() + # Capture offline state once so the gate and the load agree (no window where the gate is + # skipped as offline but the constructor then reaches the network). + local_only = hf_env_offline() with _lock: if _model is None or _name != name: _install_torchao_stub_once() from sentence_transformers import SentenceTransformer + from utils.hf_cache_settings import active_hf_hub_cache device = _device() logger.info("loading embedding model %s on %s", name, device) - _guard_model_security(name) - _model = SentenceTransformer(name, device = device, model_kwargs = dtype_kwargs("float16")) + _guard_model_security(name, local_only) + st_kwargs = dict( + device = device, + cache_folder = active_hf_hub_cache(), + model_kwargs = dtype_kwargs("float16"), + ) + load_target = name + if local_only: + from utils.utils import hf_cache_snapshot_dir + snapshot = hf_cache_snapshot_dir(name) + if snapshot is not None: + # Load from the local snapshot dir: a local path never touches the Hub, so + # this is offline-safe on ANY sentence-transformers version (even ones + # predating local_files_only). + load_target = str(snapshot) + elif _st_accepts_local_files_only(SentenceTransformer): + st_kwargs["local_files_only"] = True + _model = SentenceTransformer(load_target, **st_kwargs) _name = name return _model diff --git a/studio/backend/core/rag/store.py b/studio/backend/core/rag/store.py index f9128d1715..1165b6bb0e 100644 --- a/studio/backend/core/rag/store.py +++ b/studio/backend/core/rag/store.py @@ -158,6 +158,16 @@ def list_documents(conn: sqlite3.Connection, scope: str) -> list[dict]: return [dict(r) for r in rows] +def list_all_documents(conn: sqlite3.Connection) -> list[dict]: + """Every uploaded document across all scopes (KBs, threads, projects).""" + rows = conn.execute( + "SELECT id, scope, kb_id, thread_id, project_id, filename, sha256, status, error, " + "num_chunks, stored_path, created_at " + "FROM documents ORDER BY created_at DESC" + ).fetchall() + return [dict(r) for r in rows] + + def get_document(conn: sqlite3.Connection, document_id: str) -> dict | None: row = conn.execute("SELECT * FROM documents WHERE id=?", (document_id,)).fetchone() return dict(row) if row else None diff --git a/studio/backend/core/training/resume.py b/studio/backend/core/training/resume.py index 2a4a198610..17183484c5 100644 --- a/studio/backend/core/training/resume.py +++ b/studio/backend/core/training/resume.py @@ -4,6 +4,8 @@ """Helpers for validating resumable training outputs.""" import json +import pickletools +import zipfile from pathlib import Path from typing import Optional @@ -33,27 +35,164 @@ def _checkpoint_step(path: Path) -> int: return -1 -def get_resume_checkpoint_path(path_value: str) -> Optional[str]: +_MODEL_FILES = ( + "adapter_model.safetensors", + "adapter_model.bin", + "model.safetensors", + "pytorch_model.bin", +) +_MODEL_INDEXES = ("model.safetensors.index.json", "pytorch_model.bin.index.json") + + +def _valid_state_file(path: Path, require_tensor: bool = True) -> bool: + try: + if not path.is_file() or path.stat().st_size == 0: + return False + if path.suffix == ".safetensors": + try: + from safetensors import SafetensorError, safe_open + except ImportError: + return False + try: + with safe_open(str(path), framework = "np") as state: + return bool(state.keys()) + except SafetensorError: + return False + if path.suffix in {".bin", ".pt"}: + with zipfile.ZipFile(path) as state: + infos = state.infolist() + names = [info.filename for info in infos] + data_name = next( + (name for name in names if name == "data.pkl" or name.endswith("/data.pkl")), + None, + ) + if data_name is None: + return False + data_prefix = data_name.removesuffix("data.pkl") + "data/" + operations = list(pickletools.genops(state.read(data_name))) + if not operations or operations[-1][0].name != "STOP": + return False + if not require_tensor: + return True + # Require a non-empty tensor record; a zero-byte one fails torch.load. + return any( + info.filename.startswith(data_prefix) + and not info.is_dir() + and info.file_size > 0 + for info in infos + ) + # Unrecognized state-file formats are not usable resume state. + return False + except (OSError, ValueError, zipfile.BadZipFile): + return False + + +def _checkpoint_state(path: Path) -> Optional[int]: + try: + state = json.loads((path / "trainer_state.json").read_text(encoding = "utf-8")) + step = state.get("global_step") if isinstance(state, dict) else None + except (OSError, UnicodeDecodeError, json.JSONDecodeError): + return None + if isinstance(step, bool) or not isinstance(step, int) or step < 0: + return None + directory_step = _checkpoint_step(path) + return step if directory_step < 0 or step == directory_step else None + + +_INDEX_SHARD_SUFFIX = { + "model.safetensors.index.json": ".safetensors", + "pytorch_model.bin.index.json": ".bin", +} + + +def _valid_indexed_shard(checkpoint: Path, shard: object, expected_suffix: str) -> bool: + # Shard must be a relative, in-format path contained in the checkpoint dir. + if not isinstance(shard, str) or not shard: + return False + if Path(shard).is_absolute() or Path(shard).suffix != expected_suffix: + return False + try: + root = checkpoint.resolve(strict = True) + candidate = (checkpoint / shard).resolve(strict = True) + candidate.relative_to(root) + except (OSError, ValueError): + return False + return _valid_state_file(candidate) + + +def _has_model_state(path: Path) -> bool: + if any(_valid_state_file(path / name) for name in _MODEL_FILES): + return True + for name in _MODEL_INDEXES: + try: + index = json.loads((path / name).read_text(encoding = "utf-8")) + shards = set(index["weight_map"].values()) + except ( + AttributeError, + OSError, + KeyError, + TypeError, + UnicodeDecodeError, + json.JSONDecodeError, + ): + continue + expected_suffix = _INDEX_SHARD_SUFFIX[name] + if shards and all(_valid_indexed_shard(path, shard, expected_suffix) for shard in shards): + return True + return False + + +def is_resume_checkpoint_valid( + path: Path, + expected_step: Optional[int] = None, + backend: Optional[str] = None, +) -> bool: + step = _checkpoint_state(path) if path.is_dir() else None + step_valid = step is not None and (expected_step is None or step == expected_step) + if backend == "mlx": + valid_bundle = _valid_state_file(path / "adapters.safetensors") and _valid_state_file( + path / "optimizer_state.safetensors" + ) + else: + valid_bundle = ( + _has_model_state(path) + # optimizer/scheduler state can be validly tensor-free (e.g. SGD without + # momentum); _has_model_state still requires real model tensors. + and _valid_state_file(path / "optimizer.pt", require_tensor = False) + and _valid_state_file(path / "scheduler.pt", require_tensor = False) + ) + if backend is None and not valid_bundle: + valid_bundle = _valid_state_file(path / "adapters.safetensors") and _valid_state_file( + path / "optimizer_state.safetensors" + ) + return step_valid and valid_bundle + + +def get_resume_checkpoint_path( + path_value: str, expected_step: Optional[int] = None +) -> Optional[str]: path = resolve_output_dir(path_value) if not _is_under_outputs(path) or not path.is_dir(): return None - if (path / "trainer_state.json").is_file(): + if is_resume_checkpoint_valid(path, expected_step): return str(path) - checkpoints = [ - child - for child in path.glob("checkpoint-*") - if child.is_dir() and (child / "trainer_state.json").is_file() - ] - if not checkpoints: - return None - return str(max(checkpoints, key = _checkpoint_step)) + checkpoints = sorted(path.glob("checkpoint-*"), key = _checkpoint_step, reverse = True) + return next( + ( + str(checkpoint) + for checkpoint in checkpoints + if _checkpoint_step(checkpoint) >= 0 + and is_resume_checkpoint_valid(checkpoint, expected_step) + ), + None, + ) def normalize_resume_output_dir(path_value: str) -> str: path = resolve_output_dir(path_value) if not _is_under_outputs(path): - raise ValueError("Resume checkpoint must be inside Studio outputs.") + raise ValueError("Resume checkpoint must be inside Unsloth outputs.") return str(path) @@ -78,9 +217,17 @@ def _uses_s3_dataset(run: dict) -> bool: def can_resume_run(run: dict) -> bool: if run.get("resumed_later"): return False + # Set when a stop-and-save failed to write a current-step checkpoint. + if run.get("resume_blocked"): + return False if _uses_s3_dataset(run): return False + status = run.get("status") + if status == "error": + # A save-time crash can report final_step == total_steps with no artifacts; checkpoint state alone decides resumability. + return has_resume_state(run.get("output_dir")) + final_step = run.get("final_step") total_steps = run.get("total_steps") has_remaining_steps = ( @@ -89,8 +236,4 @@ def can_resume_run(run: dict) -> bool: or total_steps <= 0 or final_step < total_steps ) - return ( - run.get("status") == "stopped" - and has_remaining_steps - and has_resume_state(run.get("output_dir")) - ) + return status == "stopped" and has_remaining_steps and has_resume_state(run.get("output_dir")) diff --git a/studio/backend/core/training/trainer.py b/studio/backend/core/training/trainer.py index 883a535a89..b858fe6f17 100644 --- a/studio/backend/core/training/trainer.py +++ b/studio/backend/core/training/trainer.py @@ -797,7 +797,7 @@ class UnslothTrainer: ) logger.info("Loaded text model") - raise_if_offloaded(self.model, device_map, "Studio training") + raise_if_offloaded(self.model, device_map, "Unsloth training") if self.should_stop: return False @@ -891,6 +891,7 @@ class UnslothTrainer: use_gradient_checkpointing: str = "unsloth", use_rslora: bool = False, use_loftq: bool = False, + use_dora: bool = False, modules_to_save: list = None, ) -> bool: """ @@ -993,6 +994,7 @@ class UnslothTrainer: use_gradient_checkpointing = use_gradient_checkpointing, random_state = 3407, use_rslora = use_rslora, + use_dora = use_dora, loftq_config = {"loftq_bits": 4, "loftq_iter": 1} if use_loftq else None, ) # Audio VLM models support VLM-style layer selection @@ -1023,6 +1025,7 @@ class UnslothTrainer: use_gradient_checkpointing = use_gradient_checkpointing, random_state = 3407, use_rslora = use_rslora, + use_dora = use_dora, loftq_config = {"loftq_bits": 4, "loftq_iter": 1} if use_loftq else None, task_type = None, ) @@ -1042,6 +1045,7 @@ class UnslothTrainer: use_gradient_checkpointing = use_gradient_checkpointing, random_state = 3407, use_rslora = use_rslora, + use_dora = use_dora, loftq_config = {"loftq_bits": 4, "loftq_iter": 1} if use_loftq else None, ) @@ -1067,6 +1071,7 @@ class UnslothTrainer: use_gradient_checkpointing = use_gradient_checkpointing, random_state = 3407, use_rslora = use_rslora, + use_dora = use_dora, loftq_config = {"loftq_bits": 4, "loftq_iter": 1} if use_loftq else None, modules_to_save = modules_to_save, ) @@ -1087,6 +1092,7 @@ class UnslothTrainer: use_gradient_checkpointing = use_gradient_checkpointing, random_state = 3407, use_rslora = use_rslora, + use_dora = use_dora, loftq_config = {"loftq_bits": 4, "loftq_iter": 1} if use_loftq else None, modules_to_save = modules_to_save, ) @@ -1481,6 +1487,9 @@ class UnslothTrainer: SNAC_MODEL_NAME = "hubertsiuzdak/snac_24khz" SNAC_SAMPLE_RATE = 24000 + + # SNAC codec unvalidated on Intel XPU; keep the pre-PR CPU + # fallback for non-CUDA hosts. device = "cuda" if torch.cuda.is_available() else "cpu" max_length = self.max_seq_length or 2048 tokenizer = self.tokenizer @@ -1642,7 +1651,8 @@ class UnslothTrainer: del snac_model gc.collect() - torch.cuda.empty_cache() + + clear_gpu_cache() self._cuda_audio_used = True if not processed_examples: @@ -1669,6 +1679,8 @@ class UnslothTrainer: import numpy as np import torchaudio.transforms as T + # Spark-TTS BiCodec unvalidated on Intel XPU; keep the pre-PR CPU + # fallback for non-CUDA hosts. device = "cuda" if torch.cuda.is_available() else "cpu" # sparktts lives in the SparkAudio/Spark-TTS GitHub repo, not the HF model @@ -1857,7 +1869,8 @@ class UnslothTrainer: del audio_tokenizer gc.collect() - torch.cuda.empty_cache() + + clear_gpu_cache() self._cuda_audio_used = True if not processed_examples: @@ -1894,6 +1907,8 @@ class UnslothTrainer: from datasets import Dataset as HFDataset from utils.paths import ensure_dir, tmp_root + # OuteTTS DAC/Whisper preprocess unvalidated on Intel XPU; keep the + # pre-PR CPU fallback for non-CUDA hosts. device = "cuda" if torch.cuda.is_available() else "cpu" # Clone OuteTTS repo (same as audio_codecs._load_dac) @@ -2065,7 +2080,8 @@ class UnslothTrainer: del prompt_processor gc.collect() - torch.cuda.empty_cache() + + clear_gpu_cache() self._cuda_audio_used = True if not processed_examples: @@ -3425,15 +3441,19 @@ class UnslothTrainer: logger.info( f"CPT: using UnslothTrainer with embedding_learning_rate={embedding_lr}\n" ) + cpt_args = _UnslothTrainingArguments( + embedding_learning_rate = embedding_lr, + **config_args, + ) + if config_args.get("packing", False): + cpt_args.packing_strategy = "wrapped" + logger.info("CPT packing strategy: wrapped\n") trainer_kwargs = { "model": self.model, "tokenizer": sft_tokenizer, "train_dataset": dataset["dataset"], "data_collator": data_collator, - "args": _UnslothTrainingArguments( - embedding_learning_rate = embedding_lr, - **config_args, - ), + "args": cpt_args, } if eval_dataset is not None: trainer_kwargs["eval_dataset"] = eval_dataset diff --git a/studio/backend/core/training/training.py b/studio/backend/core/training/training.py index 38f6b92f6d..8592dabbfe 100644 --- a/studio/backend/core/training/training.py +++ b/studio/backend/core/training/training.py @@ -30,7 +30,7 @@ from typing import Optional, Tuple, Any, Callable, Union, TYPE_CHECKING if TYPE_CHECKING: import matplotlib.pyplot as plt -from utils.hardware import prepare_gpu_selection +from utils.hardware import get_device, prepare_gpu_selection from utils.native_path_leases import ( native_path_secret_removed_for_child_start, run_without_native_path_secret, @@ -140,7 +140,7 @@ def should_use_mlx_training_backend(*, device: Optional[Any] = None) -> bool: def _build_training_worker_config(values: dict[str, Any]) -> dict[str, Any]: - """Build the normalized worker config shared by Studio and the CLI adapter.""" + """Build the normalized worker config shared by Unsloth and the CLI adapter.""" config = { "model_name": values["model_name"], "project_name": values.get("project_name"), @@ -196,6 +196,7 @@ def _build_training_worker_config(values: dict[str, Any]) -> dict[str, Any]: "gradient_checkpointing": values.get("gradient_checkpointing", "unsloth"), "use_rslora": values.get("use_rslora", False), "use_loftq": values.get("use_loftq", False), + "use_dora": values.get("use_dora", False), "train_on_completions": values.get("train_on_completions", False), "finetune_vision_layers": values.get("finetune_vision_layers", True), "finetune_language_layers": values.get("finetune_language_layers", True), @@ -219,6 +220,9 @@ def _build_training_worker_config(values: dict[str, Any]) -> dict[str, Any]: config[key] = values.get(key) if config["training_type"] == "Full Finetuning": config["load_in_4bit"] = False + # The parent's detected backend: the worker's apply_gpu_ids() targets the + # right visibility env var from this, without probing torch pre-mask. + config["device_backend"] = get_device().value return config @@ -307,7 +311,7 @@ PLOT_HEIGHT = 3.5 @dataclass class TrainingProgress: - """Shared training progress payload for Studio and backend-aware trainers.""" + """Shared training progress payload for Unsloth and backend-aware trainers.""" epoch: float = 0 step: int = 0 @@ -328,7 +332,7 @@ class TrainingProgress: class _MLXTrainerAdapter: - """Adapts the legacy UnslothTrainer API to the shared Studio MLX worker path.""" + """Adapts the legacy UnslothTrainer API to the shared Unsloth MLX worker path.""" def __init__(self): self.model = None @@ -452,6 +456,7 @@ class _MLXTrainerAdapter: use_gradient_checkpointing: Union[str, bool] = "unsloth", use_rslora: bool = False, use_loftq: bool = False, + use_dora: bool = False, ) -> bool: self._peft_config = { "use_lora": bool(use_lora), @@ -462,6 +467,7 @@ class _MLXTrainerAdapter: "gradient_checkpointing": use_gradient_checkpointing, "use_rslora": bool(use_rslora), "use_loftq": bool(use_loftq), + "use_dora": bool(use_dora), "finetune_vision_layers": bool(finetune_vision_layers), "finetune_language_layers": bool(finetune_language_layers), "finetune_attention_modules": bool(finetune_attention_modules), @@ -569,6 +575,7 @@ class _MLXTrainerAdapter: "gradient_checkpointing": "unsloth", "use_rslora": False, "use_loftq": False, + "use_dora": False, "finetune_vision_layers": True, "finetune_language_layers": True, "finetune_attention_modules": True, @@ -754,6 +761,9 @@ class TrainingBackend: def __init__(self): # Subprocess state self._proc: Optional[mp.Process] = None + # True from the sidecar-swap handshake until the worker is recorded, so + # installs and STT loads treat the startup window as active. + self._spawn_in_progress: bool = False self._event_queue: Any = None self._stop_queue: Any = None self._pump_thread: Optional[threading.Thread] = None @@ -761,6 +771,7 @@ class TrainingBackend: # Left True after an abnormal death so _ensure_pump_alive spots a crash. self._pump_running: bool = False self._lock = threading.Lock() + self._run_intent_lock = threading.RLock() # Stop watchdog: after a stop is requested, escalates to force_terminate() # if the worker does not exit on its own within a bounded time. The watched @@ -773,6 +784,7 @@ class TrainingBackend: self._progress = TrainingProgress() self._should_stop = False self._cancel_requested = False # True only for stop(save=False) + self._cancel_cleanup_output_dir: Optional[str] = None # Throttled training-status logging to the server log (not one line/step). self._last_progress_log_ts: float = 0.0 @@ -792,6 +804,8 @@ class TrainingBackend: # Job metadata self.current_job_id: Optional[str] = None self._output_dir: Optional[str] = None + self._resume_source_run_id: Optional[str] = None + self._terminal_finalize_payload: Optional[dict] = None # DB persistence self._metric_buffer: list[dict] = [] @@ -819,6 +833,7 @@ class TrainingBackend: job_id: str, *, before_spawn = None, + resume_source_run_id: Optional[str] = None, **kwargs, ) -> bool: """Spawn a subprocess to run the full training pipeline. @@ -924,16 +939,21 @@ class TrainingBackend: config["resolved_gpu_ids"] = resolved_gpu_ids config["gpu_selection"] = gpu_selection - from .worker import run_training_process + from utils.hf_cache_settings import child_environment_for_spawn, get_hf_cache_paths + + cache_env = get_hf_cache_paths().child_env({}) try: - with native_path_secret_removed_for_child_start(): + with ( + child_environment_for_spawn(cache_env), + native_path_secret_removed_for_child_start(), + ): event_queue = _CTX.Queue() stop_queue = _CTX.Queue() proc = _CTX.Process( target = run_without_native_path_secret, - args = (run_training_process,), + args = ("core.training.worker", "run_training_process", cache_env), kwargs = { "event_queue": event_queue, "stop_queue": stop_queue, @@ -956,6 +976,7 @@ class TrainingBackend: self.current_job_id = job_id self._should_stop = False self._cancel_requested = False + self._cancel_cleanup_output_dir = None self._complete_seen.clear() self._progress = TrainingProgress( is_training = True, status_message = "Initializing training..." @@ -972,7 +993,10 @@ class TrainingBackend: self.eval_loss_history.clear() self.eval_step_history.clear() self.eval_enabled = False - self._output_dir = None + self._output_dir = config.get("output_dir") if resume_source_run_id else None + self._progress.output_dir = self._output_dir + self._resume_source_run_id = resume_source_run_id + self._terminal_finalize_payload = None self._metric_buffer.clear() self._run_finalized = False self._db_run_created = False @@ -982,6 +1006,7 @@ class TrainingBackend: self._db_started_at = datetime.now(timezone.utc).isoformat() # Start each job Xet-first; keep config so a stall can respawn over HTTP. self._last_full_config = config + self._last_hf_cache_env = cache_env self._in_model_load = False self._xet_fallback_used = False self._needs_xet_respawn = False @@ -990,6 +1015,17 @@ class TrainingBackend: # in history during model loading and a fast terminal worker can't race the # pump into a duplicate create/finalize. From here the pump only finalizes. self._ensure_db_run_created() + if resume_source_run_id and not self._db_run_created: + if proc.is_alive(): + proc.terminate() + proc.join(timeout = 5.0) + if proc.is_alive(): + proc.kill() + proc.join(timeout = 2.0) + self._progress.is_training = False + self._progress.error = "Resume checkpoint is no longer available." + self._spawn_in_progress = False + return False # Assign handles and start the pump together under the lock so a concurrent # poll can't see a live _proc with no pump and spawn a duplicate. @@ -1011,28 +1047,75 @@ class TrainingBackend: def stop_training(self, save: bool = True) -> bool: """Send stop signal to the training subprocess.""" - self._should_stop = True - if not save: - self._cancel_requested = True - with self._lock: - if self._stop_queue is not None: - try: - self._stop_queue.put({"type": "stop", "save": save}) - except (OSError, ValueError): - pass - # Update progress immediately for responsive UI. - self._progress.status_message = ( - "Stopping training and saving checkpoint..." if save else "Cancelling training..." - ) - # Guarantee the run finalizes even if the worker wedges after saving. - self._start_stop_watchdog(cancel = not save) + with self._run_intent_lock: + with self._lock: + run_id = self.current_job_id + if not save and run_id: + persist_error: Optional[Exception] = None + for attempt in range(_DB_FINALIZE_RETRIES): + try: + from storage.studio_db import mark_run_cancel_requested + + self._ensure_db_run_created() + with self._lock: + terminal_payload = self._terminal_finalize_payload + if ( + terminal_payload + and terminal_payload.get("expected_job_id") == run_id + ): + return False + if not mark_run_cancel_requested(run_id): + if self._db_run_created: + return False + raise RuntimeError( + "Training run disappeared before cancellation persisted" + ) + if self.current_job_id != run_id: + return False + self._should_stop = self._cancel_requested = True + self._cancel_cleanup_output_dir = self._output_dir + self._output_dir = self._progress.output_dir = None + persist_error = None + break + except Exception as exc: + persist_error = exc + if attempt + 1 < _DB_FINALIZE_RETRIES: + time.sleep(_DB_FINALIZE_RETRY_S) + if persist_error is not None: + raise RuntimeError("Failed to persist Stop-without-Save") from persist_error + with self._lock: + if self.current_job_id != run_id: + return False + if save or not run_id: + self._should_stop = True + if not save and not run_id: + self._cancel_requested = True + self._cancel_cleanup_output_dir = self._output_dir + self._output_dir = self._progress.output_dir = None + if self._stop_queue is not None: + try: + self._stop_queue.put({"type": "stop", "save": save}) + except (OSError, ValueError): + pass + self._progress.status_message = ( + "Stopping training and saving checkpoint..." + if save + else "Cancelling training..." + ) + self._start_stop_watchdog(cancel = not save, expected_job_id = run_id) return True - def _start_stop_watchdog(self, cancel: bool) -> None: + def _start_stop_watchdog( + self, + cancel: bool, + expected_job_id: Optional[str] = None, + ) -> None: """Start a daemon that force-terminates the worker if a requested stop does not exit on its own. No-op if no worker is alive or a live watchdog already watches this proc (a stale watchdog on an old proc never blocks a new run's watcher).""" with self._lock: + if expected_job_id is not None and self.current_job_id != expected_job_id: + return proc = self._proc if proc is None or not proc.is_alive(): return @@ -1113,8 +1196,9 @@ class TrainingBackend: watched_job_id: Optional[str] = None, ) -> None: """Finalize parent state after a force-terminate so the UI leaves "Stopping..." - even if the worker is wedged in driver teardown; preserves output_dir so a saved - checkpoint is kept. No-ops if a new run already replaced the watched worker, so a + even if the worker is wedged in driver teardown; preserves output_dir on a save so + the checkpoint is kept, and clears it on a cancel (Stop without saving must not + offer resume/export). No-ops if a new run already replaced the watched worker, so a stale watchdog never marks a fresh run stopped or drops its handle. Supersession is checked on both the watched proc and job id: start_training sets @@ -1134,7 +1218,18 @@ class TrainingBackend: return # a new run is already starting up; leave its state alone run_id = self.current_job_id # == watched_job_id self._progress.is_training = False - self._progress.status_message = "Training stopped." + terminal_payload = self._terminal_finalize_kwargs() + status = terminal_payload["status"] + error_message = terminal_payload.get("error_message") + output_dir = terminal_payload["output_dir"] + clear_output_dir = terminal_payload["clear_output_dir"] + resume_blocked = bool(terminal_payload.get("resume_blocked")) + with self._lock: + if self.current_job_id != run_id: + return + self._progress.status_message = error_message or "Training stopped." + if error_message: + self._progress.error = error_message # Create the row if a start-time create failed (no-op otherwise; skips when the pump # is mid-create, in which case its create-then-finalize records the run instead). self._ensure_db_run_created() @@ -1148,7 +1243,8 @@ class TrainingBackend: batch: list = [] final_step = final_loss = duration = None loss_history: list = [] - output_dir = self._output_dir + if clear_output_dir: + self._output_dir = self._progress.output_dir = None if claim: self._run_finalized = True # claim this run's finalize batch = list(self._metric_buffer) @@ -1161,7 +1257,17 @@ class TrainingBackend: loss_history = list(self.loss_history) if claim: self._finish_stopped_run( - run_id, output_dir, batch, final_step, final_loss, duration, loss_history + run_id, + output_dir, + batch, + final_step, + final_loss, + duration, + loss_history, + status = status, + error_message = error_message, + clear_output_dir = clear_output_dir, + resume_blocked = resume_blocked, ) with self._lock: if target_proc is None or self._proc is target_proc: @@ -1176,6 +1282,10 @@ class TrainingBackend: final_loss: Optional[float], duration: Optional[float], loss_history: list, + status: str = "stopped", + error_message: Optional[str] = None, + clear_output_dir: bool = False, + resume_blocked: bool = False, ) -> None: """Record a force-stopped run finished by its captured id, from state snapshotted under the lock. insert_metrics_batch upserts and finish_run is an idempotent UPDATE, @@ -1194,14 +1304,16 @@ class TrainingBackend: sparkline = downsample(loss_history, 50) finish_run( id = run_id, - status = "stopped", + status = status, ended_at = datetime.now(timezone.utc).isoformat(), final_step = final_step, final_loss = final_loss, duration_seconds = duration, loss_sparkline = _json.dumps(sparkline), output_dir = output_dir, - error_message = None, + error_message = error_message, + clear_output_dir = clear_output_dir, + resume_blocked = resume_blocked, ) return except Exception: @@ -1231,7 +1343,7 @@ class TrainingBackend: logger.info("Force-terminating training subprocess (pid=%s)", proc.pid) proc.terminate() cancelled = self._cancel_requested - output_dir = self._output_dir + output_dir = self._cancel_cleanup_output_dir or self._output_dir if proc is not None: proc.join(timeout = 5.0) @@ -1304,7 +1416,11 @@ class TrainingBackend: self._last_full_config = config logger.warning("Respawning training worker with HF_HUB_DISABLE_XET=1 after Xet stall") - from .worker import run_training_process + cache_env = getattr(self, "_last_hf_cache_env", None) + if not cache_env: + from utils.hf_cache_settings import get_hf_cache_paths + cache_env = get_hf_cache_paths().child_env({}) + from utils.hf_cache_settings import child_environment_for_spawn # This run is active, so an install request 409s rather than proceeds: a reservation seen here # is transient (an aborting install or short lazy repair). Wait it out instead of stranding the @@ -1336,12 +1452,15 @@ class TrainingBackend: # crashed respawn cannot wedge is_training_active until restart. try: try: - with native_path_secret_removed_for_child_start(): + with ( + child_environment_for_spawn(cache_env), + native_path_secret_removed_for_child_start(), + ): event_queue = _CTX.Queue() stop_queue = _CTX.Queue() new_proc = _CTX.Process( target = run_without_native_path_secret, - args = (run_training_process,), + args = ("core.training.worker", "run_training_process", cache_env), kwargs = { "event_queue": event_queue, "stop_queue": stop_queue, @@ -1595,17 +1714,60 @@ class TrainingBackend: ) self._ensure_db_run_created() - self._finalize_run_in_db( - status = "stopped" if self._should_stop else "error", - error_message = None - if self._should_stop - else "Training process terminated unexpectedly", - ) + terminal_payload = self._terminal_finalize_kwargs() + with self._lock: + if terminal_payload["clear_output_dir"]: + self._output_dir = self._progress.output_dir = None + if terminal_payload.get("error_message"): + self._progress.error = terminal_payload["error_message"] + self._progress.status_message = terminal_payload["error_message"] + self._finalize_run_in_db(**terminal_payload) except Exception: logger.exception("Training event pump: finalization after worker exit failed") self._pump_running = False return + def _has_current_resume_checkpoint(self, output_dir, step) -> bool: + # A valid checkpoint at the current step means the stop-and-save landed on + # disk even if the worker died before confirming it. + if not output_dir or not isinstance(step, int) or step <= 0: + return False + from core.training.resume import get_resume_checkpoint_path + return get_resume_checkpoint_path(output_dir, expected_step = step) is not None + + def _terminal_finalize_kwargs(self) -> dict: + with self._lock: + job_id = self.current_job_id + payload = self._terminal_finalize_payload + if payload and payload.get("expected_job_id") == job_id: + return dict(payload) + cancel, stopped = self._cancel_requested, self._should_stop + output_dir = None if cancel else self._output_dir + step = self._progress.step + existing_error = self._progress.error + status, error, blocked = ( + ("stopped", None, cancel) + if stopped + else ( + "error", + existing_error or "Training process terminated unexpectedly", + False, + ) + ) + # Block only when no valid current-step checkpoint actually landed. + if stopped and not cancel and not self._has_current_resume_checkpoint(output_dir, step): + status = "error" + error = "Stop and Save ended before a valid current-step checkpoint was written." + blocked = True + return { + "status": status, + "error_message": error, + "output_dir": output_dir, + "clear_output_dir": cancel, + "resume_blocked": blocked, + "expected_job_id": job_id, + } + def _handle_event(self, event: dict) -> None: """Apply a subprocess event to local state. @@ -1764,6 +1926,15 @@ class TrainingBackend: elif etype == "eval_configured": self.eval_enabled = True + elif etype == "output_dir": + event_output_dir = event.get("output_dir") + if self._cancel_requested: + self._cancel_cleanup_output_dir = event_output_dir + self._output_dir = self._progress.output_dir = None + else: + self._output_dir = event_output_dir + db_action = "persist_output_dir" + elif etype == "status": self._progress.status_message = event.get("message", "") self._progress.is_training = True @@ -1778,7 +1949,12 @@ class TrainingBackend: self._complete_seen.set() self._progress.is_training = False self._progress.is_completed = not stopped - self._output_dir = event.get("output_dir") + event_output_dir = event.get("output_dir") + if self._cancel_requested: + self._cancel_cleanup_output_dir = event_output_dir + self._output_dir = None + else: + self._output_dir = event_output_dir self._progress.output_dir = self._output_dir self._progress.status_message = msg if not self._db_run_created and self.current_job_id and self._db_config: @@ -1788,11 +1964,16 @@ class TrainingBackend: db_action_kwargs = { "status": "stopped" if stopped else "completed", "output_dir": self._output_dir, + "clear_output_dir": self._cancel_requested, + "expected_job_id": self.current_job_id, } + self._terminal_finalize_payload = dict(db_action_kwargs) elif etype == "error": self._progress.is_training = False self._progress.error = event.get("error", "Unknown error") + if self._cancel_requested: + self._output_dir = self._progress.output_dir = None logger.error("Training error: %s", event.get("error")) stack = event.get("stack", "") if stack: @@ -1801,29 +1982,36 @@ class TrainingBackend: db_action = "create_and_finalize" else: db_action = "finalize" + stop_save_failed = ( + self._should_stop + and not self._cancel_requested + and not self._has_current_resume_checkpoint( + self._output_dir, self._progress.step + ) + ) db_action_kwargs = { - "status": "stopped" if self._should_stop else "error", + "status": "stopped" + if self._should_stop + and not stop_save_failed + and not event.get("keep_error_status") + else "error", "error_message": event.get("error", "Unknown error"), + "output_dir": self._output_dir, + "clear_output_dir": self._cancel_requested, + "resume_blocked": stop_save_failed or bool(event.get("resume_blocked")), + "expected_job_id": self.current_job_id, } + self._terminal_finalize_payload = dict(db_action_kwargs) # --- DB I/O outside the lock --- if db_action == "create_run": - try: - from storage.studio_db import create_run - - create_run( - id = db_action_kwargs["job_id"], - model_name = db_action_kwargs["model_name"], - dataset_name = db_action_kwargs["dataset_name"], - config_json = db_action_kwargs["config_json"], - started_at = db_action_kwargs["started_at"], - total_steps = db_action_kwargs["total_steps"], - ) - self._db_run_created = True + self._ensure_db_run_created() + if self._db_run_created: if db_action_kwargs["total_steps"]: self._db_total_steps_set = True - except Exception: - logger.warning("Failed to create DB run record", exc_info = True) + self._persist_output_dir() + elif db_action == "persist_output_dir": + self._persist_output_dir() elif db_action == "create_and_finalize": self._ensure_db_run_created() self._finalize_run_in_db(**db_action_kwargs) @@ -1842,6 +2030,22 @@ class TrainingBackend: if etype == "progress": self._log_training_progress() + def _persist_output_dir(self) -> None: + with self._lock: + if ( + not self._output_dir + or not self.current_job_id + or not self._db_run_created + or self._cancel_requested + ): + return + run_id, output_dir = self.current_job_id, self._output_dir + try: + from storage.studio_db import update_run_output_dir + update_run_output_dir(run_id, output_dir) + except Exception: + logger.warning("Failed to persist output_dir", exc_info = True) + def _log_training_progress(self) -> None: """One throttled training-status line to the server log (the per-step stream still goes to the UI via SSE): first step, then at most every 30s, plus the @@ -1875,6 +2079,7 @@ class TrainingBackend: caller create at a time, and ``_db_run_created`` is published only after ``create_run`` commits, so a concurrent finalize never runs ``finish_run`` against a not-yet-inserted row (a zero-row UPDATE that would leave the run stuck as running).""" + self._run_intent_lock.acquire() with self._lock: if ( self._db_run_created @@ -1882,6 +2087,7 @@ class TrainingBackend: or not self.current_job_id or not self._db_config ): + self._run_intent_lock.release() return self._db_create_in_progress = True # only one caller creates job_id = self.current_job_id @@ -1898,6 +2104,12 @@ class TrainingBackend: or _s3_dataset_name(db_config.get("s3_dataset")) or "unknown" ) + with self._lock: + if self.current_job_id != job_id: + return + output_dir = self._output_dir + cancel_requested = self._cancel_requested + resumed_from_run_id = self._resume_source_run_id create_run( id = job_id, model_name = db_config["model_name"], @@ -1905,6 +2117,9 @@ class TrainingBackend: config_json = _json.dumps(db_config), started_at = started_at, total_steps = total_steps, + output_dir = output_dir, + cancel_requested = cancel_requested, + resumed_from_run_id = resumed_from_run_id, ) created = True except Exception: @@ -1919,12 +2134,15 @@ class TrainingBackend: if created: self._db_run_created = True # publish only after the insert commits self._db_create_in_progress = False + self._run_intent_lock.release() def _finalize_run_in_db( self, status: str, error_message: Optional[str] = None, output_dir: Optional[str] = None, + clear_output_dir: bool = False, + resume_blocked: bool = False, expected_job_id: Optional[str] = None, ) -> None: """Flush remaining metrics and mark a run finished in the DB. Claims the finalize @@ -1947,26 +2165,33 @@ class TrainingBackend: duration = self._progress.elapsed_seconds loss_history = list(self.loss_history) self._flush_metrics_to_db(run_id = run_id) - try: - from storage.studio_db import finish_run - from utils.downsample import downsample + for attempt in range(_DB_FINALIZE_RETRIES): + try: + from storage.studio_db import finish_run + from utils.downsample import downsample - sparkline = downsample(loss_history, 50) - finish_run( - id = run_id, - status = status, - ended_at = datetime.now(timezone.utc).isoformat(), - final_step = final_step, - final_loss = final_loss, - duration_seconds = duration, - loss_sparkline = _json.dumps(sparkline), - output_dir = output_dir, - error_message = error_message, - ) - except Exception: - with self._lock: - self._run_finalized = False # unclaim so a later flush can retry - logger.warning("Failed to finalize run in DB (status=%s)", status, exc_info = True) + finish_run( + id = run_id, + status = status, + ended_at = datetime.now(timezone.utc).isoformat(), + final_step = final_step, + final_loss = final_loss, + duration_seconds = duration, + loss_sparkline = _json.dumps(downsample(loss_history, 50)), + output_dir = output_dir, + error_message = error_message, + clear_output_dir = clear_output_dir, + resume_blocked = resume_blocked, + ) + return + except Exception: + if attempt + 1 < _DB_FINALIZE_RETRIES: + time.sleep(_DB_FINALIZE_RETRY_S) + continue + with self._lock: + if self.current_job_id == run_id: + self._run_finalized = False + logger.warning("Failed to finalize run in DB (status=%s)", status, exc_info = True) def _flush_metrics_to_db(self, run_id: Optional[str] = None) -> None: """Flush buffered metrics to the DB and update live progress. The target run id, diff --git a/studio/backend/core/training/worker.py b/studio/backend/core/training/worker.py index c52adbe8fa..03327d3320 100644 --- a/studio/backend/core/training/worker.py +++ b/studio/backend/core/training/worker.py @@ -90,6 +90,79 @@ _FAST_PATH_HOOKS_SKIP_ENV = "UNSLOTH_STUDIO_SKIP_FAST_PATH_HOOKS" # run_training_process() and isn't GC'd mid-run. _WINDOWS_ROCM_GROUPED_MM_LIB = None + +def _install_grouped_mm_cpu_fallback(torch_mod, logger, label): + """Register a Python mm/bmm fallback for torch._grouped_mm and return the Library. + + RDNA4 (gfx1200/gfx1201) ships a null HIP _grouped_mm kernel on ROCm <= 7.12 + (fixed in 7.13; ROCm/TheRock #5284). JitDecomp dispatches _grouped_mm to the + null kernel and crashes; overriding the CUDA dispatch key bypasses it. Shared + by the Windows and Linux ROCm guards. Keep the returned Library referenced so + the registration outlives the caller. + """ + import warnings as _warnings + + _gm_lib = torch_mod.library.Library("aten", "IMPL") + + def _grouped_mm_safe_impl( + self, + mat2, + offs = None, + bias = None, + out_dtype = None, + ): + """Python mm/bmm fallback for _grouped_mm on gfx120X (null HIP kernel, ROCm <= 7.12).""" + _t = torch_mod + if offs is None: + # No offsets: 2-D -> mm, 3-D batched -> bmm (unconditional mm broke 3-D MoE). + if self.dim() == 3 and mat2.dim() == 3: + result = _t.bmm(self.contiguous(), mat2.contiguous()) + elif self.dim() == 3 and mat2.dim() == 2: + result = _t.matmul(self.contiguous(), mat2.contiguous()) + elif self.dim() == 2 and mat2.dim() == 3: + result = _t.matmul(self.contiguous(), mat2.contiguous()) + else: + result = _t.mm(self.contiguous(), mat2.contiguous()) + else: + # Grouped: offs[i] is the exclusive end-row of group i. + offs_list = offs.tolist() + pieces = [] + prev = 0 + for idx, end in enumerate(offs_list): + end = int(end) + a_part = self[prev:end].contiguous() + b_part = mat2[idx].contiguous() if mat2.dim() == 3 else mat2.contiguous() + pieces.append(_t.mm(a_part, b_part)) + prev = end + # Include trailing rows not covered by offs. + if prev < self.shape[0]: + a_tail = self[prev:].contiguous() + b_tail = mat2[-1].contiguous() if mat2.dim() == 3 else mat2.contiguous() + pieces.append(_t.mm(a_tail, b_tail)) + result = ( + _t.cat(pieces, dim = 0) + if pieces + else _t.zeros(0, mat2.shape[-1], device = self.device, dtype = self.dtype) + ) + if bias is not None: + result = result + bias + if out_dtype is not None: + result = result.to(out_dtype) + elif result.dtype != self.dtype: + result = result.to(self.dtype) + return result + + with _warnings.catch_warnings(): + _warnings.simplefilter("ignore") + _gm_lib.impl("_grouped_mm", _grouped_mm_safe_impl, "CUDA") + logger.info( + "%s: patched _grouped_mm CUDA dispatch (null HIP kernel on gfx120X, " + "ROCm <= 7.12 -- bypassed with Python mm fallback)", + label, + ) + return _gm_lib + + # Subprocesses don't inherit os.add_dll_directory registrations. Replicate # main.py's Windows ROCm DLL setup so the first `import torch` finds # amdhip64.dll. Handles retained at module scope so they aren't GC'd. @@ -702,8 +775,9 @@ def _rocm_classify_unified_memory(props: Any) -> tuple[str, bool]: 3. Device-name substring match (last resort when all arch attrs absent; AMD SDK / Radeon wheels may not populate them): - gfx1150 Strix Point: ``Radeon 890M``, ``Radeon 880M`` - - gfx1151 Strix Halo: ``Radeon 8060S`` (Ryzen AI MAX+ 395), - ``Radeon 8050S`` (cut-down SKU) + - gfx1151 Strix Halo / Gorgon Halo: ``Radeon 8065S`` (Ryzen AI + Max+ 495), ``Radeon 8060S`` (Ryzen AI MAX+ + 395), ``Radeon 8050S`` (cut-down SKU) """ gcn_arch = "" for _attr in ("gcnArchName", "gcn_arch_name", "arch_name", "gfx_arch_name"): @@ -728,7 +802,11 @@ def _rocm_classify_unified_memory(props: Any) -> tuple[str, bool]: # Arch attrs absent — fall back to device-name matching. dev_lower = (getattr(props, "name", "") or "").lower() is_unified = ( - "890m" in dev_lower or "880m" in dev_lower or "8060s" in dev_lower or "8050s" in dev_lower + "890m" in dev_lower + or "880m" in dev_lower + or "8065s" in dev_lower + or "8060s" in dev_lower + or "8050s" in dev_lower ) return gcn_arch, is_unified @@ -1100,7 +1178,7 @@ _MLX_VLM_RESIZED_IMAGE_LAYOUT_CACHE = {} def _mlx_vlm_resized_image_layout(processor = None) -> str | None: - """Return the numpy image layout expected after Studio-side VLM resizing.""" + """Return the numpy image layout expected after Unsloth-side VLM resizing.""" image_processor = getattr(processor, "image_processor", None) if image_processor is None: return None @@ -1257,7 +1335,7 @@ _MLX_STUDIO_LR_SCHEDULERS = {"linear", "cosine", "constant"} # Fallback alias map mirroring unsloth_zoo._normalize_mlx_optimizer_name, used -# only when mlx (Apple Silicon) is not importable so Studio config validation +# only when mlx (Apple Silicon) is not importable so Unsloth config validation # still works on non-MLX hosts. The zoo function stays the source of truth. _MLX_STUDIO_ADAMW_ALIASES = frozenset( ( @@ -1309,7 +1387,7 @@ def _normalize_mlx_studio_scheduler(value): def _resolve_mlx_local_dataset_files(file_paths: list) -> list[str]: - """Resolve CLI paths and Studio local dataset uploads without importing the GPU trainer.""" + """Resolve CLI paths and Unsloth local dataset uploads without importing the GPU trainer.""" from utils.paths import resolve_dataset_path all_files: list[str] = [] @@ -1469,6 +1547,10 @@ def _run_mlx_training(event_queue, stop_queue, config): message = "LoftQ is not supported for MLX training yet." _send("error", error = message) raise NotImplementedError(message) + if config.get("use_dora"): + message = "DoRA is not supported for MLX training yet." + _send("error", error = message) + raise NotImplementedError(message) if config.get("is_embedding"): message = "Embedding model training is not supported for MLX training yet." _send("error", error = message) @@ -1840,8 +1922,15 @@ def _run_mlx_training(event_queue, stop_queue, config): # Resolve to ~/.unsloth/studio/outputs/ so the export page finds it from utils.paths import ensure_dir - output_dir = _resolve_mlx_output_dir(config, model_name) + # Resume must land in the original run dir even when config lacks output_dir. + resume_dir = config.get("output_dir", "") or _output_dir_from_resume_checkpoint( + resume_from_checkpoint + ) + output_dir = _resolve_mlx_output_dir( + {**config, "output_dir": resume_dir} if resume_dir else config, model_name + ) ensure_dir(Path(output_dir)) + _emit_output_dir(event_queue, output_dir) # ── 6. Create trainer ── eval_steps_val = config.get("eval_steps", 0) or 0 @@ -1912,7 +2001,7 @@ def _run_mlx_training(event_queue, stop_queue, config): if "max_grad_leaf_norm" in _supported_fields: mlx_config_kwargs["max_grad_leaf_norm"] = max_grad_leaf_norm if "append_eos" in _supported_fields: - # Studio SFT formatting owns rendered examples; raw/CPT text still + # Unsloth SFT formatting owns rendered examples; raw/CPT text still # needs MLX to append EOS like the CUDA raw-text path. mlx_config_kwargs["append_eos"] = bool(raw_text_mode) @@ -2067,6 +2156,17 @@ def _run_mlx_training(event_queue, stop_queue, config): trainer.add_eval_callback(_on_eval) + _opt_ref = [None] + _orig_build_optimizer = getattr(trainer, "_build_optimizer", None) + + if callable(_orig_build_optimizer): + + def _capture_optimizer(total_steps): + _opt_ref[0] = _orig_build_optimizer(total_steps) + return _opt_ref[0] + + trainer._build_optimizer = _capture_optimizer + # ── 11. Run training ── gc.collect() mx.synchronize() @@ -2082,31 +2182,58 @@ def _run_mlx_training(event_queue, stop_queue, config): trainer.save_model = _save_model # ── 12. Save and finalize ── - if trainer.stop_requested: - if not _stop_save[0]: - # Cancel (save=False): skip saving. - _send("complete", output_dir = None, status_message = "Training cancelled") + def _finish_tracking() -> None: + # Runs on every save/finalize exit so TB/W&B never leak on early return. + if tb_writer is not None: + try: + tb_writer.close() + except Exception: + pass + if wandb_run is not None: + try: + wandb_run.finish() + except Exception: + pass + + def _stop_checkpoint_ok() -> bool: + if _write_mlx_stop_checkpoint(trainer, _opt_ref[0], output_dir): + return True + _send( + "error", + error = ( + "Failed to save a resumable checkpoint after stop. " + "Model files were saved, but this run cannot be resumed." + ), + # A user stop finalizes as 'stopped'; keep this failure's error status so history explains it. + keep_error_status = True, + # Older checkpoints are stale; resuming would roll back past this stop. + resume_blocked = True, + ) + return False + + try: + if trainer.stop_requested: + if not _stop_save[0]: + # Cancel (save=False): skip saving. + _send("complete", output_dir = None, status_message = "Training cancelled") + else: + _send("status", status_message = "Saving stopped model...") + mx.synchronize() + trainer.save_model(output_dir) + # Stop-and-save promises a resumable checkpoint, not just model files. + if not _stop_checkpoint_ok(): + return + _send("complete", output_dir = output_dir, status_message = "Training stopped") else: - _send("status", status_message = "Saving stopped model...") + _send("status", status_message = "Saving model...") mx.synchronize() trainer.save_model(output_dir) - _send("complete", output_dir = output_dir, status_message = "Training stopped") - else: - _send("status", status_message = "Saving model...") - mx.synchronize() - trainer.save_model(output_dir) - _send("complete", output_dir = output_dir, status_message = "Training completed") - - if tb_writer is not None: - try: - tb_writer.close() - except Exception: - pass - if wandb_run is not None: - try: - wandb_run.finish() - except Exception: - pass + # A save-stop can race the natural final save; it made the same promise. + if trainer.stop_requested and _stop_save[0] and not _stop_checkpoint_ok(): + return + _send("complete", output_dir = output_dir, status_message = "Training completed") + finally: + _finish_tracking() def _is_current_process_apple_silicon() -> bool: @@ -2121,7 +2248,7 @@ def run_mlx_training_process( config: dict, transformers_activated: bool = False, ) -> None: - """MLX worker entrypoint shared by Studio subprocesses and the CLI adapter.""" + """MLX worker entrypoint shared by Unsloth subprocesses and the CLI adapter.""" model_name = config["model_name"] backend_path = str(Path(__file__).resolve().parent.parent.parent) @@ -2250,7 +2377,7 @@ def run_training_process(*, event_queue: Any, stop_queue: Any, config: dict) -> env = os.getenv("ENVIRONMENT_TYPE", "production"), ) - apply_gpu_ids(config.get("resolved_gpu_ids")) + apply_gpu_ids(config.get("resolved_gpu_ids"), backend = config.get("device_backend")) model_name = config["model_name"] @@ -2644,80 +2771,8 @@ def run_training_process(*, event_queue: Any, stop_queue: Any, config: dict) -> # so 7.13+ uses the real GPU kernel. if not _hip_ver_at_least(7, 13): try: - import warnings as _warnings - - _gm_lib = _torch_for_rocm.library.Library("aten", "IMPL") - - def _grouped_mm_safe_impl( - self, - mat2, - offs = None, - bias = None, - out_dtype = None, - ): - """Python mm/bmm fallback for _grouped_mm on gfx1200 (null HIP kernel, ROCm ≤ 7.12).""" - _t = _torch_for_rocm - if offs is None: - # No offsets: 2-D -> mm, 3-D batched -> bmm - # (unconditional mm broke 3-D MoE). - if self.dim() == 3 and mat2.dim() == 3: - result = _t.bmm(self.contiguous(), mat2.contiguous()) - elif self.dim() == 3 and mat2.dim() == 2: - # Broadcast 2-D mat2 across the batch dim. - result = _t.matmul(self.contiguous(), mat2.contiguous()) - elif self.dim() == 2 and mat2.dim() == 3: - # Broadcast 2-D self across batch via matmul. - result = _t.matmul(self.contiguous(), mat2.contiguous()) - else: - result = _t.mm(self.contiguous(), mat2.contiguous()) - else: - # Grouped: offs[i] is the exclusive end-row of group i. - offs_list = offs.tolist() - pieces = [] - prev = 0 - for idx, end in enumerate(offs_list): - end = int(end) - a_part = self[prev:end].contiguous() - if mat2.dim() == 3: - b_part = mat2[idx].contiguous() - else: - b_part = mat2.contiguous() - pieces.append(_t.mm(a_part, b_part)) - prev = end - # Include trailing rows not covered by offs. - if prev < self.shape[0]: - a_tail = self[prev:].contiguous() - b_tail = ( - mat2[-1].contiguous() if mat2.dim() == 3 else mat2.contiguous() - ) - pieces.append(_t.mm(a_tail, b_tail)) - result = ( - _t.cat(pieces, dim = 0) - if pieces - else _t.zeros( - 0, - mat2.shape[-1], - device = self.device, - dtype = self.dtype, - ) - ) - if bias is not None: - result = result + bias - if out_dtype is not None: - result = result.to(out_dtype) - elif result.dtype != self.dtype: - result = result.to(self.dtype) - return result - - with _warnings.catch_warnings(): - _warnings.simplefilter("ignore") - _gm_lib.impl("_grouped_mm", _grouped_mm_safe_impl, "CUDA") - - _WINDOWS_ROCM_GROUPED_MM_LIB = _gm_lib # prevent GC - logger.info( - "Windows ROCm: patched _grouped_mm CUDA dispatch " - "(null HIP kernel on gfx1200, ROCm ≤ 7.12 — " - "bypassed with Python mm fallback)" + _WINDOWS_ROCM_GROUPED_MM_LIB = _install_grouped_mm_cpu_fallback( + _torch_for_rocm, logger, "Windows ROCm" ) except Exception as _patch_exc: logger.warning( @@ -2731,6 +2786,44 @@ def run_training_process(*, event_queue: Any, stop_queue: Any, config: dict) -> "skipping Python fallback (AMD fixed gfx1200 null kernel in ROCm 7.13)" ) + # ── 1f-linux. Linux ROCm RDNA4 _grouped_mm null kernel ── + # The win32 guard above misses Linux: RDNA4 (gfx1200/gfx1201) hits the same null + # HIP _grouped_mm kernel at ROCm <= 7.12 (fixed 7.13, ROCm/TheRock #5284). Gate on + # arch + HIP < 7.13 so NVIDIA/CUDA and non-RDNA4 AMD are untouched; no-op if fixed. + if sys.platform.startswith("linux") and _hw.IS_ROCM: + try: + _torch_lin = sys.modules.get("torch") + if _torch_lin is not None and _torch_lin.cuda.is_available(): + # Prefer torch.version.hip, else rocmX.Y from torch.__version__ (AMD + # SDK / Radeon wheels leave version.hip unset). Unknown version on a + # gfx120X build -> assume affected unless it is a post-fix rocmsdk wheel. + _hip_str = str(getattr(getattr(_torch_lin, "version", None), "hip", "") or "") + _ver = getattr(_torch_lin, "__version__", "").lower() + _m = re.match(r"(\d+)\.(\d+)", _hip_str) or re.search(r"rocm(\d+)\.(\d+)", _ver) + if _m: + _hip_lt_713 = (int(_m.group(1)), int(_m.group(2))) < (7, 13) + else: + _hip_lt_713 = "rocmsdk" not in _ver + # Scan every visible GPU (device_map="balanced" can place layers on a + # later RDNA4 card, so device 0 is not enough). Match gfx120X by arch, + # or by RX 9000 / R9700 name when the wheel omits gcnArchName. + _rdna4 = False + for _i in range(_torch_lin.cuda.device_count()): + _props = _torch_lin.cuda.get_device_properties(_i) + _lin_arch, _ = _rocm_classify_unified_memory(_props) + _lin_name = (getattr(_props, "name", "") or "").lower() + if _lin_arch.lower() in ("gfx1200", "gfx1201") or ( + not _lin_arch and re.search(r"rx\s*90[0-9]0|r9700", _lin_name) + ): + _rdna4 = True + break + if _rdna4 and _hip_lt_713: + _WINDOWS_ROCM_GROUPED_MM_LIB = _install_grouped_mm_cpu_fallback( + _torch_lin, logger, "Linux ROCm gfx120X" + ) + except Exception as _gm_lin_exc: + logger.warning("Linux ROCm gfx120X: could not patch _grouped_mm: %s", _gm_lin_exc) + # ── 1g. ROCm OOM guard ── # On ROCm, exhausting VRAM can hang the HIP driver instead of raising. # set_per_process_memory_fraction caps the allocator so PyTorch raises @@ -2780,7 +2873,7 @@ def run_training_process(*, event_queue: Any, stop_queue: Any, config: dict) -> ) # Unified Windows APUs: the WDDM budget is user-raisable, but # nothing on the box says so -- users see "48 GB VRAM" on a - # 96 GB machine and assume a Studio bug. Say where the limit + # 96 GB machine and assume an Unsloth bug. Say where the limit # comes from and how to raise it. if _is_unified and sys.platform == "win32": try: @@ -3097,6 +3190,7 @@ def run_training_process(*, event_queue: Any, stop_queue: Any, config: dict) -> use_gradient_checkpointing = config.get("gradient_checkpointing", "unsloth"), use_rslora = config.get("use_rslora", False), use_loftq = config.get("use_loftq", False), + use_dora = config.get("use_dora", False), ) elif use_lora: _send_status(event_queue, "Configuring LoRA adapters...") @@ -3113,6 +3207,7 @@ def run_training_process(*, event_queue: Any, stop_queue: Any, config: dict) -> use_gradient_checkpointing = config.get("gradient_checkpointing", "unsloth"), use_rslora = config.get("use_rslora", False), use_loftq = config.get("use_loftq", False), + use_dora = config.get("use_dora", False), ) else: _send_status(event_queue, "Preparing model for full finetuning...") @@ -3177,6 +3272,7 @@ def run_training_process(*, event_queue: Any, stop_queue: Any, config: dict) -> ) output_dir = str(resolve_output_dir(output_dir)) ensure_dir(Path(output_dir)) + _emit_output_dir(event_queue, output_dir) tensorboard_dir = config.get("tensorboard_dir") if config.get("enable_tensorboard", False): @@ -3296,6 +3392,61 @@ def _send_status(event_queue: Any, message: str) -> None: ) +def _emit_output_dir(event_queue: Any, output_dir: str) -> None: + try: + event_queue.put({"type": "output_dir", "output_dir": output_dir, "ts": time.time()}) + except Exception: + pass + + +def _mlx_has_checkpoint_at_step(output_dir, step: int) -> bool: + if step <= 0: + return False + from core.training.resume import is_resume_checkpoint_valid + return is_resume_checkpoint_valid( + Path(output_dir) / f"checkpoint-{step}", expected_step = step, backend = "mlx" + ) + + +def _write_mlx_stop_checkpoint(trainer, optimizer, output_dir) -> bool: + """Write a full resume checkpoint for a stopped MLX run. + + Returns True when a checkpoint for the current training step exists. + """ + step = int(getattr(trainer, "_global_step", 0) or 0) + # A periodic save or a resumed run may already cover the current step. + if _mlx_has_checkpoint_at_step(output_dir, step): + return True + if step <= 0 or optimizer is None: + return False + ckpt_dir = Path(output_dir) / f"checkpoint-{step}" + if ckpt_dir.is_symlink(): + # Refuse a symlinked dir: it could redirect writes outside output_dir. + logger.error("Refusing to write MLX stop checkpoint through symlink: %s", ckpt_dir) + return False + try: + ckpt_dir.mkdir(parents = True, exist_ok = True) + from unsloth_zoo.mlx.utils import ( + save_optimizer_state, + save_trainable_adapters, + save_trainer_state, + ) + + save_trainable_adapters(trainer.model, str(ckpt_dir)) + save_optimizer_state(optimizer, str(ckpt_dir)) + save_trainer_state( + { + "global_step": step, + "train_loss_history": list(getattr(trainer, "_train_loss_history", [])), + }, + str(ckpt_dir), + ) + logger.info("Saved stop checkpoint to %s", ckpt_dir) + except Exception: + logger.exception("Failed to write stop checkpoint under %s", output_dir) + return _mlx_has_checkpoint_at_step(output_dir, step) + + def _run_embedding_training(event_queue: Any, stop_queue: Any, config: dict) -> None: """Self-contained embedding model training pipeline. @@ -3485,6 +3636,7 @@ def _run_embedding_training(event_queue: Any, stop_queue: Any, config: dict) -> use_gradient_checkpointing = gradient_checkpointing, random_state = config.get("random_seed", 3407), use_rslora = config.get("use_rslora", False), + use_dora = config.get("use_dora", False), loftq_config = {"loftq_bits": 4, "loftq_iter": 1} if config.get("use_loftq") else None, @@ -3660,6 +3812,7 @@ def _run_embedding_training(event_queue: Any, stop_queue: Any, config: dict) -> config.get("project_name"), ) output_dir = str(resolve_output_dir(output_dir)) + _emit_output_dir(event_queue, output_dir) num_epochs = config.get("num_epochs", 2) batch_size = config.get("batch_size", 256) diff --git a/studio/backend/hub/routes/__init__.py b/studio/backend/hub/routes/__init__.py index e9579635b0..7c5cfb9b3c 100644 --- a/studio/backend/hub/routes/__init__.py +++ b/studio/backend/hub/routes/__init__.py @@ -5,8 +5,10 @@ from hub.routes.inventory import router as inventory_router from hub.routes.datasets import router as datasets_router +from hub.routes.token import router as token_router __all__ = [ "inventory_router", "datasets_router", + "token_router", ] diff --git a/studio/backend/hub/routes/datasets.py b/studio/backend/hub/routes/datasets.py index edf4f36ac0..7c7cc274d3 100644 --- a/studio/backend/hub/routes/datasets.py +++ b/studio/backend/hub/routes/datasets.py @@ -61,9 +61,11 @@ async def list_cached_datasets(current_subject: str = Depends(get_current_subjec @router.delete("/cached", response_model = DeleteCachedDatasetResponse) async def delete_cached_dataset( - repo_id: str = Body(..., embed = True), current_subject: str = Depends(get_current_subject) + repo_id: str = Body(..., embed = True), + cache_path: Optional[str] = Body(None, embed = True), + current_subject: str = Depends(get_current_subject), ): - return await cache_inventory.delete_cached_dataset_response(repo_id) + return await cache_inventory.delete_cached_dataset_response(repo_id, cache_path) @router.get("/download-progress", response_model = DownloadProgressResponse) diff --git a/studio/backend/hub/routes/inventory.py b/studio/backend/hub/routes/inventory.py index 4b6c179a2b..dc3e3641bc 100644 --- a/studio/backend/hub/routes/inventory.py +++ b/studio/backend/hub/routes/inventory.py @@ -28,6 +28,7 @@ from hub.schemas.inventory import ( CachedModelsResponse, DeleteCachedModelResponse, GgufVariantsResponse, + HiddenModelsResponse, LocalModelListResponse, ModelsFolderResponse, RecommendedFoldersResponse, @@ -214,6 +215,16 @@ async def list_cached_models( return await cache_inventory.list_cached_models_response(hf_token) +@router.get("/hidden-models", response_model = HiddenModelsResponse) +async def list_hidden_models(current_subject: str = Depends(get_current_subject)): + import asyncio + + from routes.models import hidden_model_matchers + + needles, exact_ids, exact_paths = await asyncio.to_thread(hidden_model_matchers) + return HiddenModelsResponse(needles = needles, exact_ids = exact_ids, exact_paths = exact_paths) + + @router.delete( "/delete-cached", response_model = DeleteCachedModelResponse, @@ -222,7 +233,8 @@ async def list_cached_models( async def delete_cached_model( repo_id: str = Body(...), variant: Optional[str] = Body(None), + cache_path: Optional[str] = Body(None), hf_token: Optional[str] = Depends(get_hf_token), current_subject: str = Depends(get_current_subject), ): - return await deletion.delete_cached_model_response(repo_id, variant, hf_token) + return await deletion.delete_cached_model_response(repo_id, variant, hf_token, cache_path) diff --git a/studio/backend/hub/routes/token.py b/studio/backend/hub/routes/token.py new file mode 100644 index 0000000000..1b7ad733a2 --- /dev/null +++ b/studio/backend/hub/routes/token.py @@ -0,0 +1,44 @@ +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 + +"""Hugging Face token validation endpoint.""" + +from __future__ import annotations + +import asyncio +from typing import Literal, Optional + +from fastapi import APIRouter, Depends, Request +from pydantic import BaseModel + +from auth.authentication import get_current_subject +from hub.dependencies import get_hf_token +from utils.client_ip import client_ip +from utils.hf_token_validation import validate_hf_token + + +router = APIRouter() + + +class HfTokenValidationResponse(BaseModel): + status: Literal["missing", "valid", "invalid", "rate_limited", "unavailable"] + retry_after_seconds: Optional[int] = None + + +@router.post("/token/validate", response_model = HfTokenValidationResponse) +async def validate_token( + request: Request, + hf_token: Optional[str] = Depends(get_hf_token), + current_subject: str = Depends(get_current_subject), +): + if not hf_token: + return HfTokenValidationResponse(status = "missing") + result = await asyncio.to_thread( + validate_hf_token, + hf_token, + rate_key = f"{current_subject}:{client_ip(request)}", + ) + return HfTokenValidationResponse( + status = result.status, + retry_after_seconds = result.retry_after_seconds, + ) diff --git a/studio/backend/hub/schemas/inventory.py b/studio/backend/hub/schemas/inventory.py index ef95efe2f2..ca0f4658a3 100644 --- a/studio/backend/hub/schemas/inventory.py +++ b/studio/backend/hub/schemas/inventory.py @@ -99,6 +99,10 @@ class LocalModelInfo(BaseModel): None, description = "HF repo id for cached models, e.g. org/model", ) + active_cache: Optional[bool] = Field( + None, + description = "Whether this HF entry belongs to the current download cache.", + ) base_model: Optional[str] = Field( None, description = "Base model from adapter_config.json when this is an adapter", @@ -160,6 +164,7 @@ class CachedRepoBase(BaseModel): repo_id: str size_bytes: int = 0 cache_path: Optional[str] = None + last_modified: Optional[float] = None partial: bool = False partial_transport: Optional[str] = None inventory_id: Optional[str] = None @@ -189,6 +194,12 @@ class CachedModelsResponse(BaseModel): cached: List[CachedModelRepo] = Field(default_factory = list) +class HiddenModelsResponse(BaseModel): + needles: List[str] = Field(default_factory = list) + exact_ids: List[str] = Field(default_factory = list) + exact_paths: List[str] = Field(default_factory = list) + + class AddScanFolderRequest(BaseModel): """Request body for adding a custom scan folder.""" diff --git a/studio/backend/hub/services/datasets/cache_inventory.py b/studio/backend/hub/services/datasets/cache_inventory.py index a180c9df58..c106b62ab7 100644 --- a/studio/backend/hub/services/datasets/cache_inventory.py +++ b/studio/backend/hub/services/datasets/cache_inventory.py @@ -20,12 +20,11 @@ from hub.utils import inventory_scan as hf_cache_scan from hub.utils.hf_cache_state import ( purge_partial_repo, purge_repo_cache_dirs, + resolve_delete_target_root, resolve_destructive_case_matches, ) from hub.utils.paths import ( - hf_default_cache_dir, is_valid_repo_id as _is_valid_repo_id, - legacy_hf_cache_dir, resolve_cached_repo_id_case, ) @@ -43,38 +42,8 @@ def _collect_hf_cache_scans() -> tuple[list, set[str]]: def _hf_hub_cache_roots() -> list[Path]: - roots: list[Path] = [] - seen: set[str] = set() - - def _add(path: Optional[Path]) -> None: - if path is None or not path.is_dir(): - return - try: - resolved = str(path.resolve()) - except OSError: - return - if resolved in seen: - return - seen.add(resolved) - roots.append(path) - - try: - from huggingface_hub.constants import HF_HUB_CACHE - _add(Path(HF_HUB_CACHE)) - except Exception: - pass - - hf_hub_cache = os.environ.get("HF_HUB_CACHE") - if hf_hub_cache: - _add(Path(hf_hub_cache).expanduser()) - - hf_home = os.environ.get("HF_HOME") - if hf_home: - _add(Path(hf_home).expanduser() / "hub") - - _add(legacy_hf_cache_dir()) - _add(hf_default_cache_dir()) - return roots + from hub.utils.hf_cache_state import hf_cache_roots + return hf_cache_roots() def _repo_id_from_hub_dataset_dir(name: str) -> str | None: @@ -207,6 +176,21 @@ def _repo_id_from_datasets_cache_dir(name: str) -> str | None: return repo_id if _is_valid_repo_id(repo_id) else None +def _is_processed_dataset_cache_path(repo_id: str, cache_path: str) -> bool: + """True when *cache_path* is this repo's processed Arrow cache dir + (``___`` directly under an HF_DATASETS_CACHE root). Such rows + have no Hub ``datasets--`` layout, so they are deleted via the processed + path and must not be rejected as an invalid cache_path.""" + try: + resolved = Path(cache_path).expanduser().resolve(strict = False) + except (OSError, RuntimeError, ValueError): + return False + if resolved.name.lower() != repo_id.replace("/", "___").lower(): + return False + roots = {r.resolve(strict = False) for r in _hf_datasets_cache_roots()} + return resolved.parent.resolve(strict = False) in roots + + def _processed_dataset_cache_size(path: Path) -> int: total = 0 try: @@ -361,7 +345,7 @@ async def list_cached_datasets_response() -> dict: ) from exc -async def delete_cached_dataset_response(repo_id: str) -> dict: +async def delete_cached_dataset_response(repo_id: str, cache_path: Optional[str] = None) -> dict: """Remove a cached dataset repo from the HF cache.""" if not _is_valid_repo_id(repo_id): raise HTTPException(status_code = 400, detail = "Invalid repo_id format") @@ -373,22 +357,40 @@ async def delete_cached_dataset_response(repo_id: str) -> dict: detail = "Cancel the active download before deleting.", ) try: - return await asyncio.to_thread(_delete_cached_dataset_blocking, repo_key) + return await asyncio.to_thread(_delete_cached_dataset_blocking, repo_key, cache_path) finally: downloads.registry.end_delete(repo_key) hf_cache_scan.invalidate_hf_cache_scans() -def _delete_cached_dataset_blocking(repo_id: str) -> dict: +def _delete_cached_dataset_blocking(repo_id: str, cache_path: Optional[str] = None) -> dict: scans, _seen_roots = _collect_hf_cache_scans() - candidate_entries = [] + # Group this dataset's copies by owning cache root, then target exactly one + # cache so a delete never removes copies in other, previously selected caches. + owners: dict = {} for hf_cache in scans: for repo_info in hf_cache.repos: if str(repo_info.repo_type) != "dataset": continue - if repo_info.repo_id.lower() == repo_id.lower(): - candidate_entries.append((hf_cache, repo_info)) + if repo_info.repo_id.lower() != repo_id.lower(): + continue + try: + owner = Path(repo_info.repo_path).parent.resolve(strict = False) + except (OSError, RuntimeError, ValueError): + continue + owners.setdefault(owner, []).append((hf_cache, repo_info)) + + target_root = resolve_delete_target_root("dataset", repo_id, cache_path, owners.keys()) + # A processed-only dataset row sends its Arrow cache path (___ + # under HF_DATASETS_CACHE), which is not a Hub datasets-- dir, so + # resolve_delete_target_root returns None. Accept it and fall through to the + # processed-cache delete rather than rejecting a legitimate row. + if target_root is None and not ( + cache_path and _is_processed_dataset_cache_path(repo_id, cache_path) + ): + raise HTTPException(status_code = 400, detail = "Invalid cache_path") + candidate_entries = owners.get(target_root, []) if target_root is not None else [] matched_repo_ids = resolve_destructive_repo_ids( repo_id, [str(repo_info.repo_id) for _hf_cache, repo_info in candidate_entries], @@ -414,7 +416,26 @@ def _delete_cached_dataset_blocking(repo_id: str) -> dict: exc_info = True, ) - processed_deleted, processed_failures = _delete_processed_dataset_cache(repo_id) + # Restrict the processed Arrow-cache delete to the selected cache's datasets + # root so it never removes copies under other cache homes. A processed + # cache_path scopes to its own root; a Hub target scopes to the datasets root + # sharing its cache home; an unspecified cache_path stays global (legacy). + processed_roots: Optional[set[Path]] + if not cache_path: + processed_roots = None + elif _is_processed_dataset_cache_path(repo_id, cache_path): + processed_roots = {Path(cache_path).expanduser().resolve(strict = False).parent} + else: + home = target_root.parent if target_root is not None else None + processed_roots = { + root.resolve(strict = False) + for root in _hf_datasets_cache_roots() + if home is not None and root.resolve(strict = False).parent == home + } + + processed_deleted, processed_failures = _delete_processed_dataset_cache( + repo_id, only_roots = processed_roots + ) failures.extend(processed_failures) if failures: raise HTTPException( @@ -427,15 +448,23 @@ def _delete_cached_dataset_blocking(repo_id: str) -> dict: # ``scan_cache_dir()`` skips blob-only/corrupt repos the revision delete # can't touch, yet the fallback scanner shows them; purge the whole dir. - cache_purged = purge_repo_cache_dirs("dataset", repo_id) - partial_purged = purge_partial_repo("dataset", repo_id) - state_purged = download_manifest.purge_all_state_for_repo("dataset", repo_id) > 0 + # Only for a Hub cache target; a processed-only path has no Hub dir/state. + cache_purged = partial_purged = state_purged = False + if target_root is not None: + cache_purged = purge_repo_cache_dirs("dataset", repo_id, root = target_root) + partial_purged = purge_partial_repo("dataset", repo_id, root = target_root) + state_purged = ( + download_manifest.purge_all_state_for_repo("dataset", repo_id, hub_cache = target_root) + > 0 + ) if not (deleted or processed_deleted or cache_purged or partial_purged or state_purged): raise HTTPException(status_code = 404, detail = "Dataset not found in cache") return {"status": "deleted", "repo_id": repo_id} -def _delete_processed_dataset_cache(repo_id: str) -> tuple[bool, list[str]]: +def _delete_processed_dataset_cache( + repo_id: str, only_roots: Optional[set[Path]] = None +) -> tuple[bool, list[str]]: import shutil target = repo_id.replace("/", "___") @@ -443,6 +472,10 @@ def _delete_processed_dataset_cache(repo_id: str) -> tuple[bool, list[str]]: deleted = False failures: list[str] = [] for root in _hf_datasets_cache_roots(): + # Scope to the selected cache's datasets root(s): a delete must not remove + # processed copies living under other, previously selected cache homes. + if only_roots is not None and root.resolve(strict = False) not in only_roots: + continue try: entries = [ entry diff --git a/studio/backend/hub/services/datasets/downloads.py b/studio/backend/hub/services/datasets/downloads.py index 5efac562fa..b412a339e9 100644 --- a/studio/backend/hub/services/datasets/downloads.py +++ b/studio/backend/hub/services/datasets/downloads.py @@ -159,12 +159,18 @@ async def download_dataset_response( use_xet = download_lifecycle.resolve_effective_use_xet(body.use_xet) transport = download_lifecycle.resolve_transport(use_xet) + from utils.hf_cache_settings import get_hf_cache_paths + + cache_paths = get_hf_cache_paths() + cache_env = cache_paths.child_env({}) claimed, claim_state = _registry.claim( key, transport, repo_type = "dataset", repo_id = repo_id, + hub_cache = str(cache_paths.hub_cache), + xet_cache = str(cache_paths.xet_cache), ) generation = _registry.current_generation(key) if not claimed: @@ -176,7 +182,12 @@ async def download_dataset_response( "accepted": _registry.adoptable(key), "generation": generation, } - download_manifest.clear_cancel_marker("dataset", repo_id, None) + download_manifest.clear_cancel_marker( + "dataset", + repo_id, + None, + hub_cache = cache_paths.hub_cache, + ) state = download_lifecycle.launch_worker( _registry, @@ -185,6 +196,7 @@ async def download_dataset_response( ["--repo-id", repo_id, "--dataset"], hf_token, use_xet = use_xet, + cache_env = cache_env, ), hf_token = hf_token, label = repo_id, diff --git a/studio/backend/hub/services/download_lifecycle.py b/studio/backend/hub/services/download_lifecycle.py index e5d48872c1..8e14427a56 100644 --- a/studio/backend/hub/services/download_lifecycle.py +++ b/studio/backend/hub/services/download_lifecycle.py @@ -11,7 +11,7 @@ import sys import time import threading from pathlib import Path -from typing import Callable, Optional +from typing import Callable, Mapping, Optional from fastapi import HTTPException @@ -57,6 +57,7 @@ def spawn_worker( *, use_xet: bool, protected_blob_hashes: Optional[frozenset[str]] = None, + cache_env: Optional[Mapping[str, str]] = None, ) -> subprocess.Popen: """Spawn the download worker. @@ -68,7 +69,11 @@ def spawn_worker( """ cwd = backend_dir() mode = download_registry.TRANSPORT_XET if use_xet else download_registry.TRANSPORT_HTTP - env = os.environ.copy() + from utils.hf_cache_settings import get_hf_cache_paths + + env = get_hf_cache_paths().child_env() + if cache_env is not None: + env.update(cache_env) if protected_blob_hashes: env["UNSLOTH_PROTECTED_BLOB_HASHES"] = ",".join(sorted(protected_blob_hashes)) else: @@ -76,7 +81,7 @@ def spawn_worker( env["HF_HUB_DISABLE_PROGRESS_BARS"] = "1" env["HF_HUB_DISABLE_TELEMETRY"] = "1" env["HF_HUB_DISABLE_XET"] = "0" if use_xet else "1" - # No token in Studio settings: fall back to the backend's own HF_TOKEN so + # No token in Unsloth settings: fall back to the backend's own HF_TOKEN so # private repos stay downloadable (needed while inkling repos are private). if not hf_token: hf_token = os.environ.get("HF_TOKEN") or None @@ -230,6 +235,7 @@ def finalize_worker_exit( (stderr_data or b"").decode("utf-8", "replace").strip(), hf_token = hf_token, ) + metadata = registry.get_job_metadata(key) state = classify_exit(rc, cancel_requested = cancel_requested) if state == "complete": registry.set_job(key, "complete") @@ -252,13 +258,13 @@ def finalize_worker_exit( repo_type, repo_id, download_registry.variant_from_key(key), + hub_cache = metadata.hub_cache if metadata is not None else None, ) except Exception as exc: logger.debug(f"clear_cancel_marker failed for {repo_id} (rc=0): {exc}") elif state == "cancelled": # Read metadata before the terminal set_job so a concurrent eviction # can't drop it; the job key is the fallback variant label. - metadata = registry.get_job_metadata(key) registry.set_job(key, "cancelled") logger.info(f"{log_prefix} cancelled: {label} (rc={rc})") download_registry.persist_cancel_marker( @@ -268,6 +274,7 @@ def finalize_worker_exit( if metadata is not None and metadata.variant else download_registry.variant_from_key(key), cancel_marker_transport or transport, + hub_cache = metadata.hub_cache if metadata is not None else None, logger = logger, ) else: @@ -303,6 +310,7 @@ def _set_retry_failure_state( metadata.transport if metadata is not None and metadata.transport else fallback_transport, + hub_cache = metadata.hub_cache if metadata is not None else None, logger = logger, ) return state @@ -371,6 +379,7 @@ def _try_http_retry( repo_type, repo_id, progress_blob_hashes, + root = Path(original_metadata.hub_cache) if original_metadata.hub_cache else None, ) if progress_blob_hashes else 0 @@ -403,6 +412,8 @@ def _try_http_retry( generation = generation, replace_active = True, cancel_marker_transport = original_metadata.transport, + hub_cache = original_metadata.hub_cache, + xet_cache = original_metadata.xet_cache, ) if claimed: break @@ -446,11 +457,24 @@ def _try_http_retry( label, ) try: + cache_env = ( + { + "HF_HUB_CACHE": original_metadata.hub_cache, + "HF_XET_CACHE": original_metadata.xet_cache, + } + if original_metadata.hub_cache and original_metadata.xet_cache + else None + ) + spawn_kwargs = { + "use_xet": False, + "protected_blob_hashes": peer_hashes or None, + } + if cache_env is not None: + spawn_kwargs["cache_env"] = cache_env proc = spawn_worker( args, hf_token, - use_xet = False, - protected_blob_hashes = peer_hashes or None, + **spawn_kwargs, ) except Exception as exc: scrubbed = download_registry.scrub_secrets(str(exc), hf_token = hf_token) diff --git a/studio/backend/hub/services/models/cache_inventory.py b/studio/backend/hub/services/models/cache_inventory.py index 1f38af9381..807ec70991 100644 --- a/studio/backend/hub/services/models/cache_inventory.py +++ b/studio/backend/hub/services/models/cache_inventory.py @@ -31,12 +31,19 @@ from hub.services.models.common import ( _is_checkpoint_weight_name, _is_gguf_filename, _is_main_gguf_filename, + _is_mmproj_filename, _is_transformers_safetensors_weight_name, _local_inventory_id, - _prefer_complete_larger, _runtime_for_format, ) +# Imported at module scope (not inside the per-repo scan loop) so a broken +# import surfaces at startup instead of silently emptying the inventory: the +# scan loop swallows per-repo exceptions and would drop every repo. Lives under +# ``utils`` (not ``utils.models``) to avoid the eager model-config/checkpoint +# imports in ``utils/models/__init__.py``. +from utils.hidden_models import is_hidden_model + logger = get_logger(__name__) _repo_size_cache: "OrderedDict[tuple[str, str, str], tuple[int, frozenset[str], float]]" = ( @@ -125,6 +132,39 @@ def _repo_has_gguf_files(repo_info) -> bool: return _repo_gguf_size_bytes(repo_info) > 0 +def _blob_mtime(file_obj) -> float: + ts = getattr(file_obj, "blob_last_modified", None) + if isinstance(ts, (int, float)) and ts > 0: + return float(ts) + blob_path = getattr(file_obj, "blob_path", None) + if blob_path: + try: + return float(Path(blob_path).stat().st_mtime) + except OSError: + pass + return 0.0 + + +def _repo_gguf_last_modified(repo_info) -> float: + latest = 0.0 + for revision in repo_info.revisions: + for f in revision.files: + if _is_main_gguf_filename(f.file_name): + latest = max(latest, _blob_mtime(f)) + return latest + + +def _repo_has_mmproj(repo_info) -> bool: + # An mmproj file only makes a repo vision-capable when it is an actual GGUF + # projector; a non-GGUF sidecar (e.g. mmproj_config.json) does not, and the + # runtime's projector detection is GGUF-only. + return any( + _is_gguf_filename(f.file_name) and _is_mmproj_filename(f.file_name) + for revision in repo_info.revisions + for f in revision.files + ) + + def _cached_repo_file_name(file_obj) -> str: file_path = getattr(file_obj, "file_path", None) if file_path: @@ -209,24 +249,46 @@ def _repo_gguf_blob_map(repo_info, *, include_companions: bool = False) -> dict[ def _prefer_cache_row(candidate: dict, existing: Optional[dict]) -> bool: if existing is None: return True - return _prefer_complete_larger( - bool(candidate.get("partial")), - int(candidate.get("size_bytes") or 0), - bool(existing.get("partial")), - int(existing.get("size_bytes") or 0), - ) + candidate_partial = bool(candidate.get("partial")) + existing_partial = bool(existing.get("partial")) + if candidate_partial != existing_partial: + return not candidate_partial + candidate_active = bool(candidate.get("active_cache")) + existing_active = bool(existing.get("active_cache")) + if candidate_active != existing_active: + return candidate_active + return int(candidate.get("size_bytes") or 0) > int(existing.get("size_bytes") or 0) def _cache_inventory_fields( repo_id: str, model_format: ModelFormat, *, + repo_path: Optional[Path] = None, + snapshot_path: Optional[Path] = None, + active_hub_cache: Optional[Path] = None, partial: bool = False, requires_variant: bool = False, ) -> dict: + load_id = repo_id + active_cache = True + if repo_path is not None: + try: + if active_hub_cache is None: + from utils.hf_cache_settings import get_hf_cache_paths + active_hub_cache = get_hf_cache_paths().hub_cache + active_root = active_hub_cache.resolve(strict = False) + cached_root = repo_path.parent.resolve(strict = False) + if cached_root != active_root: + active_cache = False + load_id = str(snapshot_path or repo_path.resolve(strict = False)) + except (OSError, RuntimeError, ValueError): + active_cache = False + load_id = str(snapshot_path or repo_path) return { "inventory_id": _local_inventory_id("cache", model_format, repo_id), - "load_id": repo_id, + "load_id": load_id, + "active_cache": active_cache, "model_format": model_format, "runtime": _runtime_for_format(model_format), "format_variant": None, @@ -243,9 +305,19 @@ def invalidate_hf_cache_scans() -> None: hf_cache_scan.invalidate_hf_cache_scans() +def _is_hidden_infra_repo(*values: str | None) -> bool: + """True for infra-only repos (the RAG embedder and the llama.cpp install + validation probe) that are cached as a side effect of Studio itself and are + not usable chat models.""" + return is_hidden_model(*values) + + def _scan_cached_gguf() -> list[dict]: """Synchronous HF-cache disk walk for GGUF repos; runs in a worker thread.""" cache_scans = all_hf_cache_scans() + from utils.hf_cache_settings import get_hf_cache_paths + + active_hub_cache = get_hf_cache_paths().hub_cache seen_lower: dict[str, dict] = {} for hf_cache in cache_scans: @@ -254,18 +326,33 @@ def _scan_cached_gguf() -> list[dict]: if str(repo_info.repo_type) != "model": continue repo_id = repo_info.repo_id + repo_path = Path(repo_info.repo_path) + snapshot_path = _cached_model_snapshot_path(repo_path) total_size = _repo_gguf_size_bytes(repo_info) - has_variant_state, variant_state_size = _gguf_variant_state_summary(repo_id) + has_variant_state, variant_state_size = _gguf_variant_state_summary( + repo_id, + hub_cache = repo_path.parent, + ) + is_hidden_infra = _is_hidden_infra_repo( + repo_id, + str(repo_path), + str(snapshot_path) if snapshot_path is not None else None, + ) + # Hide infra repos unless the user downloaded a variant via + # the Hub; variant state only exists for user downloads. + if is_hidden_infra and not has_variant_state: + continue if total_size == 0 and not has_variant_state: continue partial = hf_cache_scan.is_gguf_repo_partial( repo_id, - Path(repo_info.repo_path), + repo_path, ) if total_size == 0 and not partial: continue key = repo_id.lower() existing = seen_lower.get(key) + last_modified = _repo_gguf_last_modified(repo_info) row = { "repo_id": repo_id, "size_bytes": max(total_size, variant_state_size), @@ -275,16 +362,34 @@ def _scan_cached_gguf() -> list[dict]: # per-variant detail lives on GgufVariantDetail. "partial_transport": None, } + last_modified = max(last_modified, (existing or {}).get("last_modified", 0.0)) + if last_modified > 0: + row["last_modified"] = last_modified row.update( _cache_inventory_fields( repo_id, "gguf", + repo_path = repo_path, + snapshot_path = snapshot_path, + active_hub_cache = active_hub_cache, partial = bool(row["partial"]), requires_variant = True, ) ) + if _repo_has_mmproj(repo_info): + row["capabilities"]["supports_vision"] = True + # Visible infra variants remain management-only. + if is_hidden_infra: + row["capabilities"]["can_chat"] = False if _prefer_cache_row(row, existing): + if existing and existing["capabilities"].get("supports_vision"): + row["capabilities"]["supports_vision"] = True seen_lower[key] = row + else: + if last_modified > existing.get("last_modified", 0.0): + existing["last_modified"] = last_modified + if row["capabilities"].get("supports_vision"): + existing["capabilities"]["supports_vision"] = True except Exception as e: repo_label = getattr(repo_info, "repo_id", "") logger.warning(f"Skipping cached GGUF repo {repo_label}: {e}") @@ -312,13 +417,14 @@ class _CachedNonGgufPayload(NamedTuple): size_bytes: int has_runnable_weights: bool model_format: ModelFormat + last_modified: float def _repo_non_gguf_model_payload(repo_info) -> _CachedNonGgufPayload: - all_weight_blobs: dict[str, int] = {} - adapter_blobs: dict[str, int] = {} - safetensors_blobs: dict[str, int] = {} - checkpoint_blobs: dict[str, int] = {} + all_weight_blobs: dict[str, tuple[int, float]] = {} + adapter_blobs: dict[str, tuple[int, float]] = {} + safetensors_blobs: dict[str, tuple[int, float]] = {} + checkpoint_blobs: dict[str, tuple[int, float]] = {} has_config = False has_adapter_config = False has_adapter_weights = False @@ -326,12 +432,15 @@ def _repo_non_gguf_model_payload(repo_info) -> _CachedNonGgufPayload: has_transformers_safetensors = False has_checkpoint = False - def _record_blob(target: dict[str, int], file_obj, rev_id: str, file_name: str) -> None: + def _record_blob( + target: dict[str, tuple[int, float]], file_obj, rev_id: str, file_name: str + ) -> None: blob_path = getattr(file_obj, "blob_path", None) size = int(file_obj.size_on_disk or 0) key = str(blob_path) if blob_path else f"{rev_id}:{file_name}" - target[key] = size - all_weight_blobs[key] = size + value = (size, _blob_mtime(file_obj)) + target[key] = value + all_weight_blobs[key] = value for revision in repo_info.revisions: rev_id = getattr(revision, "commit_hash", None) or str(id(revision)) @@ -375,18 +484,19 @@ def _repo_non_gguf_model_payload(repo_info) -> _CachedNonGgufPayload: or "unknown" ) if model_format == "adapter": - size_bytes = sum(adapter_blobs.values()) + selected_blobs = adapter_blobs elif model_format == "safetensors": - size_bytes = sum(safetensors_blobs.values()) + selected_blobs = safetensors_blobs elif model_format == "checkpoint": - size_bytes = sum(checkpoint_blobs.values()) + selected_blobs = checkpoint_blobs else: - size_bytes = sum(all_weight_blobs.values()) + selected_blobs = all_weight_blobs return _CachedNonGgufPayload( - size_bytes = size_bytes, + size_bytes = sum(size for size, _mtime in selected_blobs.values()), has_runnable_weights = model_format != "unknown", model_format = model_format, + last_modified = max((mtime for _size, mtime in selected_blobs.values()), default = 0.0), ) @@ -407,6 +517,19 @@ def _read_json_object(path: Path) -> dict: return {} +def _is_whisper_model_config(config: object) -> bool: + if not isinstance(config, dict): + return False + model_type = config.get("model_type") + if isinstance(model_type, str) and model_type.strip().lower() == "whisper": + return True + architectures = config.get("architectures") + return isinstance(architectures, list) and any( + isinstance(name, str) and name == "WhisperForConditionalGeneration" + for name in architectures + ) + + def _read_model_card_frontmatter(path: Path) -> dict: try: text = path.read_text(encoding = "utf-8") @@ -437,6 +560,8 @@ def _cached_model_local_metadata(repo_path: Path) -> dict: result: dict = {} config = _read_json_object(snapshot / "config.json") + if _is_whisper_model_config(config): + result["_hidden_stt"] = True quant_method = ( config.get("quantization_config", {}).get("quant_method") if isinstance(config.get("quantization_config"), dict) @@ -463,11 +588,15 @@ def _cached_model_local_metadata(repo_path: Path) -> dict: def _scan_cached_models() -> list[dict]: """Synchronous HF-cache disk walk for non-GGUF model repos; runs in a worker thread.""" cache_scans = all_hf_cache_scans() + from utils.hf_cache_settings import get_hf_cache_paths + + active_hub_cache = get_hf_cache_paths().hub_cache seen_lower: dict[str, dict] = {} inspected = 0 skipped_gguf = 0 skipped_no_weights = 0 + skipped_stt = 0 for hf_cache in cache_scans: for repo_info in hf_cache.repos: inspected += 1 @@ -475,6 +604,15 @@ def _scan_cached_models() -> list[dict]: if str(repo_info.repo_type) != "model": continue repo_id = repo_info.repo_id + repo_path = Path(repo_info.repo_path) + snapshot_path = _cached_model_snapshot_path(repo_path) + # The non-GGUF embedder has no variant downloads; always hide. + if _is_hidden_infra_repo( + repo_id, + str(repo_path), + str(snapshot_path) if snapshot_path is not None else None, + ): + continue has_main_gguf = _repo_has_gguf_files(repo_info) payload = _repo_non_gguf_model_payload(repo_info) if payload.size_bytes == 0: @@ -486,7 +624,10 @@ def _scan_cached_models() -> list[dict]: continue key = repo_id.lower() existing = seen_lower.get(key) - repo_path = Path(repo_info.repo_path) + local_metadata = _cached_model_local_metadata(repo_path) + if local_metadata.pop("_hidden_stt", False): + skipped_stt += 1 + continue snapshot_partial = hf_cache_scan.is_snapshot_partial( "model", repo_id, @@ -506,27 +647,40 @@ def _scan_cached_models() -> list[dict]: if snapshot_partial else None ), - **_cached_model_local_metadata(repo_path), + **local_metadata, } + last_modified = max( + payload.last_modified, + (existing or {}).get("last_modified", 0.0), + ) + if last_modified > 0: + row["last_modified"] = last_modified row.update( _cache_inventory_fields( repo_id, payload.model_format, + repo_path = repo_path, + snapshot_path = snapshot_path, + active_hub_cache = active_hub_cache, partial = bool(row["partial"]), ) ) if _prefer_cache_row(row, existing): seen_lower[key] = row + elif last_modified > existing.get("last_modified", 0.0): + existing["last_modified"] = last_modified except Exception as e: repo_label = getattr(repo_info, "repo_id", "") logger.warning(f"Skipping cached model repo {repo_label}: {e}") continue cached = sorted(seen_lower.values(), key = lambda c: c["repo_id"]) logger.info( - "Cached model scan: inspected=%d skipped_gguf=%d skipped_no_weights=%d returned=%d", + "Cached model scan: inspected=%d skipped_gguf=%d skipped_no_weights=%d " + "skipped_stt=%d returned=%d", inspected, skipped_gguf, skipped_no_weights, + skipped_stt, len(cached), ) return cached diff --git a/studio/backend/hub/services/models/common.py b/studio/backend/hub/services/models/common.py index f381bffe9c..4c0e296fdc 100644 --- a/studio/backend/hub/services/models/common.py +++ b/studio/backend/hub/services/models/common.py @@ -150,7 +150,9 @@ def _prefer_complete_larger( return candidate_size_bytes > existing_size_bytes -def _gguf_variant_state_summary(repo_id: str) -> tuple[bool, int]: +def _gguf_variant_state_summary( + repo_id: str, *, hub_cache: Optional[str | Path] = None +) -> tuple[bool, int]: """Whether GGUF variant-scoped state exists and its expected size; a cancelled/in-progress variant may have only manifests/markers/`.incomplete` blobs, which inventory needs to avoid a generic fallback row.""" from hub.utils import download_manifest @@ -159,10 +161,16 @@ def _gguf_variant_state_summary(repo_id: str) -> tuple[bool, int]: for variant, _path in download_manifest.iter_variant_manifests( "model", repo_id, + hub_cache = hub_cache, ): key = variant.lower() variant_keys.add(key) - manifest = download_manifest.read_manifest("model", repo_id, variant) + manifest = download_manifest.read_manifest( + "model", + repo_id, + variant, + hub_cache = hub_cache, + ) if manifest is None: continue size_by_variant[key] = max( @@ -172,6 +180,7 @@ def _gguf_variant_state_summary(repo_id: str) -> tuple[bool, int]: for variant, _path in download_manifest.iter_variant_markers( "model", repo_id, + hub_cache = hub_cache, ): variant_keys.add(variant.lower()) return bool(variant_keys), sum(size_by_variant.values()) @@ -432,8 +441,13 @@ def _local_model_info( base_model_source: Optional[str] = None, adapter_type: Optional[str] = None, training_method: Optional[str] = None, + active_cache: Optional[bool] = None, ) -> LocalModelInfo: - load_id = model_id if source == "hf_cache" and model_id else str(load_path) + load_id = ( + model_id + if source == "hf_cache" and model_id and active_cache is not False + else str(load_path) + ) semantic_id = model_id or str(load_path) return LocalModelInfo( id = load_id, @@ -445,6 +459,7 @@ def _local_model_info( ), load_id = load_id, model_id = model_id, + active_cache = active_cache if source == "hf_cache" else None, display_name = display_name or (scan_path.stem if scan_path.is_file() else scan_path.name), path = str(load_path), size_bytes = max(0, int(size_bytes or 0)), @@ -476,6 +491,7 @@ def _classify_local_path( model_id: Optional[str] = None, updated_at: Optional[float] = None, partial: bool = False, + active_cache: Optional[bool] = None, ) -> list[LocalModelInfo]: load_path = load_path or scan_path files = ( @@ -512,6 +528,7 @@ def _classify_local_path( requires_variant = scan_path.is_dir(), format_variant = variant, size_bytes = gguf_size_bytes, + active_cache = active_cache, ) ) @@ -574,6 +591,7 @@ def _classify_local_path( ), adapter_type = adapter_type if model_format == "adapter" else None, training_method = training_method if model_format == "adapter" else None, + active_cache = active_cache, ) ) elif not rows: @@ -592,6 +610,7 @@ def _classify_local_path( updated_at = updated_at, partial = partial or trusted_hf_cache_repo, size_bytes = size_bytes, + active_cache = active_cache, ) ) diff --git a/studio/backend/hub/services/models/deletion.py b/studio/backend/hub/services/models/deletion.py index 636a223d4e..c736908058 100644 --- a/studio/backend/hub/services/models/deletion.py +++ b/studio/backend/hub/services/models/deletion.py @@ -19,8 +19,10 @@ from hub.utils import inventory_scan as hf_cache_scan from hub.utils.gguf import extract_quant_label, extract_quant_token from hub.utils.hf_cache_state import ( INCOMPLETE_SUFFIX, + iter_repo_cache_dirs, purge_partial_repo, purge_repo_cache_dirs, + resolve_delete_target_root, ) from hub.utils.paths import ( is_valid_gguf_variant as _is_valid_gguf_variant, @@ -184,6 +186,7 @@ def _delete_gguf_variant_from_repos( hf_token: Optional[str], *, sibling_active: bool = False, + root: Optional[Path] = None, ) -> dict: failures: list[str] = [] removed_snapshots = 0 @@ -265,6 +268,7 @@ def _delete_gguf_variant_from_repos( hf_token, extra_hashes = frozenset(completed_hashes), companions = not sibling_active, + root = root, ) if incomplete_result.unresolved: raise HTTPException( @@ -276,7 +280,7 @@ def _delete_gguf_variant_from_repos( ), ) - state_purged = download_manifest.purge_state("model", repo_id, variant) + state_purged = download_manifest.purge_state("model", repo_id, variant, hub_cache = root) # Reclaim the empty quant folder so it stops 404ing on delete. removed_dirs, dir_failures = _remove_empty_variant_dirs(target_repos, variant) removed_snap_dirs, snap_dir_failures = _remove_empty_snapshot_dirs(target_repos) @@ -316,6 +320,8 @@ def reclaim_replaced_gguf_variant( variant: str, keep_main_hashes: frozenset[str], hf_token: Optional[str] = None, + *, + hub_cache: Optional[str | Path] = None, ) -> dict: """Prune stale main-GGUF files for a variant after a replacement verified. @@ -366,12 +372,22 @@ def reclaim_replaced_gguf_variant( "reason": "scan_failed", } + if hub_cache is None: + from utils.hf_cache_settings import get_hf_cache_paths + hub_cache = get_hf_cache_paths().hub_cache + try: + target_hub_cache = Path(hub_cache).expanduser().resolve(strict = False) + except (OSError, RuntimeError, ValueError): + target_hub_cache = Path(hub_cache).expanduser() + candidate_repos = [ repo_info for hf_cache in cache_scans for repo_info in hf_cache.repos if str(getattr(repo_info, "repo_type", "")) == "model" and str(getattr(repo_info, "repo_id", "")).lower() == repo_id.lower() + and getattr(repo_info, "repo_path", None) + and Path(repo_info.repo_path).parent.resolve(strict = False) == target_hub_cache ] try: matched_repo_ids = resolve_destructive_repo_ids( @@ -493,10 +509,24 @@ def reclaim_replaced_gguf_variant( def _loaded_id_matches_repo(loaded_id: str, repo_id: str) -> bool: - """True when *loaded_id* is *repo_id* or a file within it; ``/``-boundary aware so ``org/model`` doesn't match sibling ``org/model-v2``.""" + """Match a loaded repo ID or an on-disk path inside any copy of the repo.""" rid = repo_id.lower() lid = loaded_id.lower() - return lid == rid or lid.startswith(f"{rid}/") + if lid == rid or lid.startswith(f"{rid}/"): + return True + + try: + loaded_path = Path(loaded_id).expanduser().resolve(strict = False) + except (OSError, RuntimeError, ValueError): + return False + for repo_dir in iter_repo_cache_dirs("model", repo_id): + try: + resolved_repo = repo_dir.resolve(strict = False) + if loaded_path == resolved_repo or loaded_path.is_relative_to(resolved_repo): + return True + except (OSError, RuntimeError, ValueError): + continue + return False def _loaded_repo_variant_blocks_delete( @@ -560,6 +590,7 @@ async def delete_cached_model_response( repo_id: str, variant: Optional[str] = None, hf_token: Optional[str] = None, + cache_path: Optional[str] = None, ): """Delete a cached model repo (or a specific GGUF variant) from the HF cache. @@ -603,14 +634,19 @@ async def delete_cached_model_response( ) raise HTTPException(status_code = 400, detail = detail) try: - return await asyncio.to_thread(_delete_cached_model_blocking, repo_id, variant, hf_token) + return await asyncio.to_thread( + _delete_cached_model_blocking, repo_id, variant, hf_token, cache_path + ) finally: downloads.registry.end_delete(repo_key, variant) cache_inventory.invalidate_hf_cache_scans() def _delete_cached_model_blocking( - repo_id: str, variant: Optional[str], hf_token: Optional[str] + repo_id: str, + variant: Optional[str], + hf_token: Optional[str], + cache_path: Optional[str] = None, ) -> dict: try: # If a sibling quant is downloading concurrently, restrict this delete to @@ -621,13 +657,26 @@ def _delete_cached_model_blocking( cache_scans = cache_inventory.all_hf_cache_scans() - candidate_entries = [] + # A repo can live in several remembered caches. Group its copies by the + # cache root that owns each, then target exactly one cache so a delete + # never removes copies in other, previously selected caches. + owners: dict = {} for hf_cache in cache_scans: for repo_info in hf_cache.repos: if str(repo_info.repo_type) != "model": continue - if repo_info.repo_id.lower() == repo_id.lower(): - candidate_entries.append((hf_cache, repo_info)) + if repo_info.repo_id.lower() != repo_id.lower(): + continue + try: + owner = Path(repo_info.repo_path).parent.resolve(strict = False) + except (OSError, RuntimeError, ValueError): + continue + owners.setdefault(owner, []).append((hf_cache, repo_info)) + + target_root = resolve_delete_target_root("model", repo_id, cache_path, owners.keys()) + if target_root is None: + raise HTTPException(status_code = 400, detail = "Invalid cache_path") + candidate_entries = owners.get(target_root, []) matched_repo_ids = resolve_destructive_repo_ids( repo_id, @@ -642,10 +691,15 @@ def _delete_cached_model_blocking( if not target_entries: if variant is None: - cache_purged = purge_repo_cache_dirs("model", repo_id) or purge_partial_repo( - "model", repo_id + cache_purged = purge_repo_cache_dirs( + "model", repo_id, root = target_root + ) or purge_partial_repo("model", repo_id, root = target_root) + state_purged = ( + download_manifest.purge_all_state_for_repo( + "model", repo_id, hub_cache = target_root + ) + > 0 ) - state_purged = download_manifest.purge_all_state_for_repo("model", repo_id) > 0 if cache_purged or state_purged: return {"status": "deleted", "repo_id": repo_id} if variant: @@ -654,6 +708,7 @@ def _delete_cached_model_blocking( variant, hf_token, companions = not sibling_active, + root = target_root, ) if incomplete_result.unresolved: raise HTTPException( @@ -668,6 +723,7 @@ def _delete_cached_model_blocking( "model", repo_id, variant, + hub_cache = target_root, ) if incomplete_result.deleted > 0 or state_purged: return { @@ -684,6 +740,7 @@ def _delete_cached_model_blocking( [repo for _cache, repo in target_entries], hf_token, sibling_active = sibling_active, + root = target_root, ) deleted_revisions = False @@ -702,9 +759,11 @@ def _delete_cached_model_blocking( delete_strategy.execute() deleted_revisions = True - cache_purged = purge_repo_cache_dirs("model", repo_id) - partial_purged = purge_partial_repo("model", repo_id) - state_purged = download_manifest.purge_all_state_for_repo("model", repo_id) > 0 + cache_purged = purge_repo_cache_dirs("model", repo_id, root = target_root) + partial_purged = purge_partial_repo("model", repo_id, root = target_root) + state_purged = ( + download_manifest.purge_all_state_for_repo("model", repo_id, hub_cache = target_root) > 0 + ) if not (deleted_revisions or cache_purged or partial_purged or state_purged): raise HTTPException(status_code = 404, detail = "No revisions found for model") diff --git a/studio/backend/hub/services/models/downloads.py b/studio/backend/hub/services/models/downloads.py index c2ffe7bffc..c93b21c082 100644 --- a/studio/backend/hub/services/models/downloads.py +++ b/studio/backend/hub/services/models/downloads.py @@ -60,12 +60,37 @@ def _job_status( return DownloadJobStatus(state = state, error = error, generation = generation) +def _load_in_flight(repo_id: str) -> bool: + try: + from core.inference.llama_cpp import hf_gguf_load_in_flight + return hf_gguf_load_in_flight(repo_id) + except Exception: + return False + + +def _load_in_flight_error(repo_id: str) -> HTTPException: + return HTTPException( + status_code = 409, + detail = ( + f"A model load for '{repo_id}' is in progress and may be " + "downloading it. Wait for the load to finish (or cancel it), " + "then start the download." + ), + ) + + +def _reject_if_load_in_flight(repo_id: str) -> None: + if _load_in_flight(repo_id): + raise _load_in_flight_error(repo_id) + + def _spawn_download_worker( repo_id: str, variant: Optional[str], hf_token: Optional[str], use_xet: bool = True, protected_blob_hashes: Optional[frozenset[str]] = None, + cache_env: Optional[dict[str, str]] = None, ) -> subprocess.Popen: args = ["--repo-id", repo_id] if variant: @@ -75,6 +100,7 @@ def _spawn_download_worker( hf_token, use_xet = use_xet, protected_blob_hashes = protected_blob_hashes, + cache_env = cache_env, ) @@ -89,6 +115,9 @@ async def download_model_response(body: DownloadModelRequest, hf_token: Optional # Canonicalize so two different-cased paste-ins share one job + cache dir. repo_id = await asyncio.to_thread(resolve_cached_repo_id_case, repo_id, repo_type = "model") + # Avoid concurrent writers to the same HF cache files. + _reject_if_load_in_flight(repo_id) + variant = (body.gguf_variant or "").strip() or None if variant is not None and not _is_valid_gguf_variant(variant): raise HTTPException( @@ -98,6 +127,10 @@ async def download_model_response(body: DownloadModelRequest, hf_token: Optional key = _download_job_key(repo_id, variant) use_xet = download_lifecycle.resolve_effective_use_xet(body.use_xet) transport = download_lifecycle.resolve_transport(use_xet) + from utils.hf_cache_settings import get_hf_cache_paths + + cache_paths = get_hf_cache_paths() + cache_env = cache_paths.child_env({}) variant_blob_hashes = frozenset() variant_progress_blob_hashes = frozenset() completed_baseline_bytes = 0 @@ -147,9 +180,14 @@ async def download_model_response(body: DownloadModelRequest, hf_token: Optional blob_hashes = variant_blob_hashes, progress_blob_hashes = variant_progress_blob_hashes, completed_baseline_bytes = completed_baseline_bytes, + admission_check = lambda: not _load_in_flight(repo_id), + hub_cache = str(cache_paths.hub_cache), + xet_cache = str(cache_paths.xet_cache), ) generation = _registry.current_generation(key) if not claimed: + if claim_state == "admission_blocked": + raise _load_in_flight_error(repo_id) # claim_state is the blocking job's state. The client can attach only # when the blocker is this key's own in-flight job (adoptable); a # cross-variant conflict or in-progress delete is not accepted. @@ -159,7 +197,12 @@ async def download_model_response(body: DownloadModelRequest, hf_token: Optional "accepted": _registry.adoptable(key), "generation": generation, } - download_manifest.clear_cancel_marker("model", repo_id, variant) + download_manifest.clear_cancel_marker( + "model", + repo_id, + variant, + hub_cache = cache_paths.hub_cache, + ) # Blobs a concurrent same-repo variant is already writing (e.g. a shared # mmproj). The worker must not purge these during cache preparation. protected_blob_hashes = _registry.peer_blob_hashes(key) if variant else frozenset() @@ -174,6 +217,7 @@ async def download_model_response(body: DownloadModelRequest, hf_token: Optional hf_token, use_xet = use_xet, protected_blob_hashes = protected_blob_hashes, + cache_env = cache_env, ), hf_token = hf_token, label = label, diff --git a/studio/backend/hub/services/models/folder_browser.py b/studio/backend/hub/services/models/folder_browser.py index d56b62c318..effb6a32ae 100644 --- a/studio/backend/hub/services/models/folder_browser.py +++ b/studio/backend/hub/services/models/folder_browser.py @@ -30,6 +30,7 @@ from hub.utils.paths import ( ) from utils.paths.external_media import ( linux_run_media_mount_roots, + macos_volume_roots, windows_drive_roots, ) from hub.services.models.common import _safe_is_dir @@ -165,7 +166,7 @@ def _looks_like_model_dir(directory: Path) -> bool: def _build_browse_allowlist( media_roots: Optional[list[Path]] = None, drive_roots: Optional[list[Path]] = None ) -> list[Path]: - """Root directories the browser may walk (also seeds the suggestion chips): HOME, resolved HF cache dirs, Studio outputs/exports/root, registered scan folders, and well-known local-LLM dirs. Each is added only if it resolves to a real directory so the sandbox has no dead boundary. + """Root directories the browser may walk (also seeds the suggestion chips): HOME, resolved HF cache dirs, Unsloth outputs/exports/root, registered scan folders, and well-known local-LLM dirs. Each is added only if it resolves to a real directory so the sandbox has no dead boundary. *media_roots* / *drive_roots* let the caller pass already-probed removable-media and Windows drive roots so they aren't scanned again (a @@ -187,7 +188,7 @@ def _build_browse_allowlist( _add(Path.home()) if media_roots is None: - media_roots = linux_run_media_mount_roots() + media_roots = [*linux_run_media_mount_roots(), *macos_volume_roots()] if drive_roots is None: drive_roots = windows_drive_roots() for p in media_roots: @@ -195,6 +196,12 @@ def _build_browse_allowlist( for p in drive_roots: _add(p) _add(_resolve_hf_cache_dir()) + try: + from utils.hf_cache_settings import known_hf_cache_homes + for cache_home in known_hf_cache_homes(): + _add(cache_home) + except Exception: # noqa: BLE001 -- best-effort + pass try: _add(hf_default_cache_dir()) except Exception: # noqa: BLE001 -- best-effort @@ -431,7 +438,7 @@ def browse_folders_response( # Probe removable-media and Windows drive roots once; the allowlist and # chips reuse the result so a disconnected mapped drive isn't scanned twice. - media_roots = linux_run_media_mount_roots() + media_roots = [*linux_run_media_mount_roots(), *macos_volume_roots()] drive_roots = windows_drive_roots() # Build the allowlist once -- the sandbox check and suggestion chips share # it so chips are always navigable. diff --git a/studio/backend/hub/services/models/gguf_variants.py b/studio/backend/hub/services/models/gguf_variants.py index 33f0297ff5..533fd2ca5a 100644 --- a/studio/backend/hub/services/models/gguf_variants.py +++ b/studio/backend/hub/services/models/gguf_variants.py @@ -9,6 +9,7 @@ import asyncio import threading import time from collections import OrderedDict +from pathlib import Path from typing import NamedTuple, Optional from fastapi import HTTPException @@ -22,6 +23,7 @@ from hub.utils.hf_errors import hf_error_status from hub.utils.hf_cache_state import ( INCOMPLETE_SUFFIX, iter_destructive_repo_cache_dirs, + repo_cache_dir_name, ) from hub.utils.gguf import ( extract_quant_label, @@ -233,8 +235,14 @@ def _manifest_variant_blob_hashes( variant: str, *, include_companions: bool = True, + repo_cache_dir: Optional[Path] = None, ) -> frozenset[str]: - manifest = download_manifest.read_manifest("model", repo_id, variant) + manifest = download_manifest.read_manifest( + "model", + repo_id, + variant, + hub_cache = repo_cache_dir.parent if repo_cache_dir is not None else None, + ) if manifest is None: return frozenset() variant_key = variant.lower() @@ -257,6 +265,7 @@ def gguf_variant_blob_hashes( *, include_companions: bool = True, allow_remote: bool = True, + repo_cache_dir: Optional[Path] = None, ) -> frozenset[str]: key = _variant_blob_hash_cache_key( repo_id, @@ -271,9 +280,9 @@ def gguf_variant_blob_hashes( repo_id, variant, include_companions = include_companions, + repo_cache_dir = repo_cache_dir, ) if hashes: - _variant_hash_cache_set(key, hashes) return hashes requirement_key = _variant_hash_cache_key(repo_id, variant, hf_token) requirement = _variant_requirement_cache_get(requirement_key) @@ -287,11 +296,22 @@ def gguf_variant_blob_hashes( return frozenset() -def _partial_transport_for_variant(repo_id: str, variant: str) -> Optional[str]: - return hf_cache_scan.partial_transport_for("model", repo_id, variant) +def _partial_transport_for_variant( + repo_id: str, + variant: str, + repo_cache_dir: Optional[Path] = None, +) -> Optional[str]: + return hf_cache_scan.partial_transport_for( + "model", + repo_id, + variant, + repo_cache_dir, + ) -def _local_main_gguf_blobs_by_quant(repo_id: str) -> dict[str, dict[str, set[str]]]: +def _local_main_gguf_blobs_by_quant( + repo_id: str, repo_cache_dir: Optional[Path] = None +) -> dict[str, dict[str, set[str]]]: """Map quant -> repo-relative expected GGUF filename -> cached blob hashes. Shared companions are copied into each main-quant bucket so update checks can @@ -313,6 +333,14 @@ def _local_main_gguf_blobs_by_quant(repo_id: str) -> dict[str, dict[str, set[str continue if str(getattr(repo_info, "repo_id", "")).lower() != target_lower: continue + if repo_cache_dir is not None: + try: + if Path(repo_info.repo_path).resolve(strict = False) != repo_cache_dir.resolve( + strict = False + ): + continue + except (AttributeError, OSError, RuntimeError, ValueError): + continue for path, hashes in cache_inventory._repo_gguf_blob_map( repo_info, include_companions = True, @@ -388,6 +416,7 @@ def delete_variant_incomplete_blobs_result( *, extra_hashes: frozenset[str] = frozenset(), companions: bool = True, + root: Optional[Path] = None, ) -> VariantIncompleteDeleteResult: # With a sibling still downloading, ``companions=False`` keeps a shared mmproj # from being unlinked out from under it; the repo's last delete reclaims it. @@ -409,8 +438,9 @@ def delete_variant_incomplete_blobs_result( ) deleted = 0 # Destructive iterator: only the exact-case match (or abort if ambiguous), - # so a case-variant sibling repo's partials are never unlinked. - for entry in iter_destructive_repo_cache_dirs("model", repo_id): + # so a case-variant sibling repo's partials are never unlinked. ``root`` scopes + # the purge to one cache so a delete never touches another cache's partials. + for entry in iter_destructive_repo_cache_dirs("model", repo_id, root = root): blobs_dir = entry / "blobs" if not blobs_dir.is_dir(): continue @@ -425,15 +455,37 @@ def delete_variant_incomplete_blobs_result( return VariantIncompleteDeleteResult(deleted = deleted, unresolved = False) +def _repo_cache_dir_for_request(repo_id: str, local_path: Optional[str]) -> Path: + """Resolve the one Hub repo cache represented by this variant request.""" + expected_name = repo_cache_dir_name("model", repo_id).lower() + if local_path: + try: + local = Path(local_path).expanduser().resolve(strict = False) + for candidate in (local, *local.parents): + if candidate.name.lower() == expected_name: + return candidate + except (OSError, RuntimeError, ValueError): + pass + from utils.hf_cache_settings import get_hf_cache_paths + + return get_hf_cache_paths().hub_cache / repo_cache_dir_name("model", repo_id) + + def _mark_empty_dir_cleanables( - repo_id: str, response: GgufVariantsResponse + repo_id: str, + response: GgufVariantsResponse, + repo_cache_dir: Optional[Path] = None, ) -> GgufVariantsResponse: """Surface empty leftover ``/`` folders (interrupted downloads) as partial so the UI can delete them -- on local/offline paths too, not just a remote listing. A listed quant is flipped to partial; an unlisted one is appended as a zero-byte cleanable entry.""" try: - empty_labels = list_empty_gguf_variant_dirs(repo_id) + empty_labels = ( + list_empty_gguf_variant_dirs(repo_id, root = repo_cache_dir.parent) + if repo_cache_dir is not None + else list_empty_gguf_variant_dirs(repo_id) + ) except Exception as e: logger.warning(f"Failed to scan empty GGUF variant folders for {repo_id}: {e}") return response @@ -468,6 +520,11 @@ async def get_gguf_variants_response( """ def _compute() -> GgufVariantsResponse: + repo_cache_dir = ( + None if is_local_path(repo_id) else _repo_cache_dir_for_request(repo_id, local_path) + ) + hub_cache = repo_cache_dir.parent if repo_cache_dir is not None else None + def _local_response( response_repo_id: str, variants, has_vision: bool ) -> GgufVariantsResponse: @@ -511,6 +568,7 @@ async def get_gguf_variants_response( partial_transport = _partial_transport_for_variant( response_repo_id, v.quant, + repo_cache_dir, ), ) for v in variants @@ -532,7 +590,7 @@ async def get_gguf_variants_response( local_only = prefer_local_cache or offline if local_only: - cached = list_gguf_variants_from_hf_cache(repo_id) + cached = list_gguf_variants_from_hf_cache(repo_id, root = hub_cache) if cached is not None: variants, has_vision = cached return _local_response(repo_id, variants, has_vision) @@ -540,7 +598,7 @@ async def get_gguf_variants_response( variants, has_vision = list_local_gguf_variants(local_path) if variants or has_vision: return _local_response(repo_id, variants, has_vision) - partial = list_partial_gguf_variants_from_state(repo_id) + partial = list_partial_gguf_variants_from_state(repo_id, hub_cache = hub_cache) if partial is not None: variants, has_vision = partial return _partial_local_response(repo_id, variants, has_vision) @@ -560,11 +618,11 @@ async def get_gguf_variants_response( try: variants, has_vision, siblings = list_gguf_variants(repo_id, hf_token = hf_token) except Exception: - cached = list_gguf_variants_from_hf_cache(repo_id) + cached = list_gguf_variants_from_hf_cache(repo_id, root = hub_cache) if cached is not None: variants, has_vision = cached return _local_response(repo_id, variants, has_vision) - partial = list_partial_gguf_variants_from_state(repo_id) + partial = list_partial_gguf_variants_from_state(repo_id, hub_cache = hub_cache) if partial is not None: variants, has_vision = partial return _partial_local_response(repo_id, variants, has_vision) @@ -581,7 +639,7 @@ async def get_gguf_variants_response( cached_filenames_by_snapshot: list[dict[str, int]] = [] cached_quant_bytes_by_snapshot: list[dict[str, int]] = [] if _is_valid_repo_id(repo_id): - for snap in iter_hf_cache_snapshots(repo_id): + for snap in iter_hf_cache_snapshots(repo_id, root = hub_cache): try: gguf_paths = list(_iter_gguf_paths(snap)) except (OSError, RuntimeError, ValueError) as e: @@ -694,11 +752,20 @@ async def get_gguf_variants_response( partial_quants: set[str] = set() partial_quant_transports: dict[str, Optional[str]] = {} try: - incomplete_hashes = download_registry.incomplete_blob_hashes("model", repo_id) + incomplete_hashes = download_registry.incomplete_blob_hashes( + "model", + repo_id, + active_only = True, + root = hub_cache, + ) except Exception as e: logger.warning(f"Failed to compute partial GGUF variants for {repo_id}: {e}") incomplete_hashes = set() - scan_snapshot_dir = hf_cache_scan.resolve_snapshot_dir_for_scan("model", repo_id) + scan_snapshot_dir = hf_cache_scan.resolve_snapshot_dir_for_scan( + "model", + repo_id, + repo_cache_dir, + ) # Manifest + marker + main incomplete-blob check: catches variants whose # download was cancelled or whose expected shards are missing/undersized. for variant in variants: @@ -711,6 +778,7 @@ async def get_gguf_variants_response( variant.quant, hf_token, include_companions = False, + repo_cache_dir = repo_cache_dir, ) if hf_cache_scan.is_variant_partial( repo_id, @@ -718,11 +786,13 @@ async def get_gguf_variants_response( scan_snapshot_dir, incomplete_blob_hashes = incomplete_hashes, variant_blob_hashes = variant_hashes, + repo_cache_dir = repo_cache_dir, ): partial_quants.add(variant.quant) partial_quant_transports[variant.quant] = _partial_transport_for_variant( repo_id, variant.quant, + repo_cache_dir, ) except Exception as e: logger.warning( @@ -744,10 +814,14 @@ async def get_gguf_variants_response( partial_quants.add(variant.quant) partial_quant_transports.setdefault( variant.quant, - _partial_transport_for_variant(repo_id, variant.quant), + _partial_transport_for_variant( + repo_id, + variant.quant, + repo_cache_dir, + ), ) - local_blobs_by_quant = _local_main_gguf_blobs_by_quant(repo_id) + local_blobs_by_quant = _local_main_gguf_blobs_by_quant(repo_id, repo_cache_dir) def _variant_detail(v) -> GgufVariantDetail: is_partial = v.quant in partial_quants @@ -790,14 +864,20 @@ async def get_gguf_variants_response( if skip: raise enriched = _mark_empty_dir_cleanables( - repo_id, GgufVariantsResponse(repo_id = repo_id, variants = []) + repo_id, + GgufVariantsResponse(repo_id = repo_id, variants = []), + _repo_cache_dir_for_request(repo_id, local_path), ) if enriched.variants: return enriched raise if skip: return response - return _mark_empty_dir_cleanables(repo_id, response) + return _mark_empty_dir_cleanables( + repo_id, + response, + _repo_cache_dir_for_request(repo_id, local_path), + ) try: return await asyncio.to_thread(_compute_with_cleanables) diff --git a/studio/backend/hub/services/models/local_inventory.py b/studio/backend/hub/services/models/local_inventory.py index a3782efead..9cf260b157 100644 --- a/studio/backend/hub/services/models/local_inventory.py +++ b/studio/backend/hub/services/models/local_inventory.py @@ -36,6 +36,7 @@ from hub.utils.paths import ( ) from hub.services.models import common as model_common from hub.services.models.ollama import scan_ollama_dir +from utils.hidden_models import is_hidden_model logger = get_logger(__name__) _MAX_MODELS_PER_CUSTOM_FOLDER = 200 @@ -105,11 +106,8 @@ def _is_model_directory_for_scan(path: Path, *, entry_limit: int | None) -> bool def _resolve_hf_cache_dir() -> Path: - try: - from huggingface_hub.constants import HF_HUB_CACHE - return Path(HF_HUB_CACHE) - except Exception: - return Path.home() / ".cache" / "huggingface" / "hub" + from utils.hf_cache_settings import get_hf_cache_paths + return get_hf_cache_paths().hub_cache def _scan_models_dir( @@ -201,7 +199,12 @@ def _hf_repo_dir_has_content(repo_dir: Path) -> bool: return False -def _scan_hf_cache(cache_dir: Path, *, entry_limit: int | None = None) -> List[LocalModelInfo]: +def _scan_hf_cache( + cache_dir: Path, + *, + entry_limit: int | None = None, + active_cache: bool = True, +) -> List[LocalModelInfo]: if not _safe_is_dir(cache_dir): return [] @@ -239,7 +242,10 @@ def _scan_hf_cache(cache_dir: Path, *, entry_limit: int | None = None) -> List[L repo_dir, ) gguf_partial = hf_cache_scan.is_gguf_repo_partial(model_id, repo_dir) - has_gguf_variant_state, gguf_variant_state_size = _gguf_variant_state_summary(model_id) + has_gguf_variant_state, gguf_variant_state_size = _gguf_variant_state_summary( + model_id, + hub_cache = cache_dir, + ) snapshot_partial_transport = ( hf_cache_scan.partial_transport_for( "model", @@ -251,23 +257,25 @@ def _scan_hf_cache(cache_dir: Path, *, entry_limit: int | None = None) -> List[L ) resolved = hf_cache_scan.resolve_hf_cache_realpath(repo_dir) scan_path = Path(resolved) if resolved else repo_dir + load_path = repo_dir if active_cache else scan_path # partial=False here; _apply_format_aware_partial below rewrites per-row # so a hybrid repo's gguf row doesn't taint its safetensors row. rows = _classify_local_path( scan_path, "hf_cache", - load_path = repo_dir, + load_path = load_path, display_name = model_id.split("/")[-1], model_id = model_id, updated_at = updated_at, partial = False, + active_cache = active_cache, ) if not rows: if has_gguf_variant_state and gguf_partial: rows = [ _local_model_info( scan_path = repo_dir, - load_path = repo_dir, + load_path = load_path, source = "hf_cache", model_format = "gguf", display_name = model_id.split("/")[-1], @@ -276,6 +284,7 @@ def _scan_hf_cache(cache_dir: Path, *, entry_limit: int | None = None) -> List[L partial = True, requires_variant = True, size_bytes = gguf_variant_state_size, + active_cache = active_cache, ) ] else: @@ -284,13 +293,14 @@ def _scan_hf_cache(cache_dir: Path, *, entry_limit: int | None = None) -> List[L rows = [ _local_model_info( scan_path = repo_dir, - load_path = repo_dir, + load_path = load_path, source = "hf_cache", model_format = "unknown", display_name = model_id.split("/")[-1], model_id = model_id, updated_at = updated_at, partial = snapshot_partial or gguf_partial, + active_cache = active_cache, ) ] elif ( @@ -301,7 +311,7 @@ def _scan_hf_cache(cache_dir: Path, *, entry_limit: int | None = None) -> List[L rows.append( _local_model_info( scan_path = repo_dir, - load_path = repo_dir, + load_path = load_path, source = "hf_cache", model_format = "gguf", display_name = model_id.split("/")[-1], @@ -310,6 +320,7 @@ def _scan_hf_cache(cache_dir: Path, *, entry_limit: int | None = None) -> List[L partial = True, requires_variant = True, size_bytes = gguf_variant_state_size, + active_cache = active_cache, ) ) rows = _apply_format_aware_partial( @@ -514,14 +525,39 @@ async def _collect_models_from_default_sources( local_models += await _scan_source("HF cache", _scan_hf_cache, hf_cache_dir) if _safe_is_dir(legacy_hf) and legacy_hf.resolve() != hf_cache_dir.resolve(): - local_models += await _scan_source("legacy HF cache", _scan_hf_cache, legacy_hf) + local_models += await _scan_source( + "legacy HF cache", + lambda path: _scan_hf_cache(path, active_cache = False), + legacy_hf, + ) if ( _safe_is_dir(hf_default) and hf_default.resolve() != hf_cache_dir.resolve() and hf_default.resolve() != legacy_hf.resolve() ): - local_models += await _scan_source("default HF cache", _scan_hf_cache, hf_default) + local_models += await _scan_source( + "default HF cache", + lambda path: _scan_hf_cache(path, active_cache = False), + hf_default, + ) + + from utils.hf_cache_settings import known_hf_hub_caches + + seen_hf = { + os.path.normcase(str(path.resolve(strict = False))) + for path in (hf_cache_dir, legacy_hf, hf_default) + } + for previous_cache in known_hf_hub_caches(): + key = os.path.normcase(str(previous_cache.resolve(strict = False))) + if key in seen_hf: + continue + seen_hf.add(key) + local_models += await _scan_source( + "previous HF cache", + lambda path: _scan_hf_cache(path, active_cache = False), + previous_cache, + ) for lm_dir in lm_dirs: local_models += await _scan_source("LM Studio", _scan_lmstudio_dir, lm_dir) @@ -542,7 +578,11 @@ def _scan_custom_folder(folder_path: Path) -> List[LocalModelInfo]: limit = _MAX_MODELS_PER_CUSTOM_FOLDER, entry_limit = _MAX_CUSTOM_FOLDER_ENTRIES, ) - + _scan_hf_cache(folder_path, entry_limit = _MAX_CUSTOM_FOLDER_ENTRIES) + + _scan_hf_cache( + folder_path, + entry_limit = _MAX_CUSTOM_FOLDER_ENTRIES, + active_cache = False, + ) + _scan_lmstudio_dir(folder_path, entry_limit = _MAX_CUSTOM_FOLDER_ENTRIES) ) if m.model_format in supported_formats @@ -609,12 +649,20 @@ def _dedupe_local_models(local_models: List[LocalModelInfo]) -> list[LocalModelI row_key = model.inventory_id or model.id key = f"{row_key}\x00custom" if model.source == "custom" else row_key existing = deduped.get(key) - if existing is None or _prefer_complete_larger( - model.partial, - model.size_bytes, - existing.partial, - existing.size_bytes, - ): + prefer_candidate = existing is None + if existing is not None: + if model.partial != existing.partial: + prefer_candidate = not model.partial + elif (model.active_cache is True) != (existing.active_cache is True): + prefer_candidate = model.active_cache is True + else: + prefer_candidate = _prefer_complete_larger( + model.partial, + model.size_bytes, + existing.partial, + existing.size_bytes, + ) + if prefer_candidate: deduped[key] = model return sorted( deduped.values(), @@ -623,6 +671,20 @@ def _dedupe_local_models(local_models: List[LocalModelInfo]) -> list[LocalModelI ) +def _filter_hidden_models(local_models: List[LocalModelInfo]) -> list[LocalModelInfo]: + """Remove infrastructure-only models from the shared local inventory.""" + visible: list[LocalModelInfo] = [] + for model in local_models: + resolved_cache_path = ( + hf_cache_scan.resolve_hf_cache_realpath(Path(model.path)) + if model.source == "hf_cache" + else None + ) + if not is_hidden_model(model.id, model.model_id, model.path, resolved_cache_path): + visible.append(model) + return visible + + async def list_local_models_response(models_dir: str = "./models") -> LocalModelListResponse: """List local model candidates from every supported on-device source.""" hf_cache_dir = _resolve_hf_cache_dir() @@ -653,7 +715,7 @@ async def list_local_models_response(models_dir: str = "./models") -> LocalModel ollama_dirs, ) local_models += await _collect_models_from_custom_folders() - models = _dedupe_local_models(local_models) + models = _dedupe_local_models(_filter_hidden_models(local_models)) return LocalModelListResponse( models_dir = str(models_root), diff --git a/studio/backend/hub/services/models/ollama.py b/studio/backend/hub/services/models/ollama.py index 96a4114620..2ccdbb44f1 100644 --- a/studio/backend/hub/services/models/ollama.py +++ b/studio/backend/hub/services/models/ollama.py @@ -85,7 +85,7 @@ def _contained_link_path(link_dir: Path, link_name: str) -> Optional[Path]: def _ollama_links_dir(ollama_dir: Path) -> Optional[Path]: - """Writable directory for Ollama ``.gguf`` symlinks. Prefers ``/.studio_links/`` next to the blobs; falls back to Studio's cache (read-only system installs), then the temp dir (sandboxed installs).""" + """Writable directory for Ollama ``.gguf`` symlinks. Prefers ``/.studio_links/`` next to the blobs; falls back to Unsloth's cache (read-only system installs), then the temp dir (sandboxed installs).""" def _ensure_writable_dir(path: Path) -> Optional[Path]: try: diff --git a/studio/backend/hub/services/snapshot_progress.py b/studio/backend/hub/services/snapshot_progress.py index 1fdf05e2e5..c3db6fed7a 100644 --- a/studio/backend/hub/services/snapshot_progress.py +++ b/studio/backend/hub/services/snapshot_progress.py @@ -86,9 +86,20 @@ def _snapshot_complete_on_disk( return False if variant is None and hf_cache_scan.repo_cache_dir_has_incomplete_blobs(entry): return False - if download_manifest.has_cancel_marker(repo_type, repo_id, variant): + hub_cache = entry.parent + if download_manifest.has_cancel_marker( + repo_type, + repo_id, + variant, + hub_cache = hub_cache, + ): return False - manifest = download_manifest.read_manifest(repo_type, repo_id, variant) + manifest = download_manifest.read_manifest( + repo_type, + repo_id, + variant, + hub_cache = hub_cache, + ) if manifest is None: return False return download_manifest.verify_against_disk(manifest, snapshot_dir).ok @@ -118,6 +129,8 @@ def compute_snapshot_progress( 0, int(getattr(metadata, "completed_baseline_bytes", 0) or 0), ) + metadata_hub_cache = getattr(metadata, "hub_cache", None) + active_root = Path(metadata_hub_cache) if metadata_hub_cache else None expected_total = max(expected_bytes, 0) # Always resolve the revision's blob hashes so stale blobs from a superseded @@ -134,11 +147,17 @@ def compute_snapshot_progress( count_finalized_unscoped = variant is None readings: list[tuple[int, int, Optional[str], bool]] = [] - for entry in preferred_repo_cache_dirs( - repo_type, - repo_id, - force_active = force_active, - ): + cache_dirs = ( + preferred_repo_cache_dirs( + repo_type, + repo_id, + force_active = force_active, + active_root = active_root, + ) + if active_root is not None + else preferred_repo_cache_dirs(repo_type, repo_id, force_active = force_active) + ) + for entry in cache_dirs: completed_bytes = 0 in_progress_bytes = 0 cache_path = hf_cache_scan.resolve_hf_cache_realpath(entry) diff --git a/studio/backend/hub/tests/test_dataset_services.py b/studio/backend/hub/tests/test_dataset_services.py index 4890714cd0..6aab07cc46 100644 --- a/studio/backend/hub/tests/test_dataset_services.py +++ b/studio/backend/hub/tests/test_dataset_services.py @@ -72,57 +72,115 @@ def test_dataset_cache_scan_merges_raw_and_processed_rows(monkeypatch): assert rows[0]["partial"] is False -def test_delete_cached_dataset_attempts_all_roots_before_raising(monkeypatch): +def test_delete_cached_dataset_scopes_delete_to_selected_root(monkeypatch, tmp_path): + """A dataset present in the active cache and a previously selected cache is + deleted only from the selected root, so the other cache's copy survives.""" calls = [] - purged_state = [] + target_hub = tmp_path / "active" / "hub" + other_hub = tmp_path / "previous" / "hub" + for hub in (target_hub, other_hub): + (hub / "datasets--Org--Data").mkdir(parents = True) class _DeleteStrategy: - def __init__(self, label: str, fail: bool): + def __init__(self, label: str): self.label = label - self.fail = fail def execute(self): calls.append(self.label) - if self.fail: - raise RuntimeError(f"{self.label} failed") - class _Cache: - def __init__(self, label: str, fail: bool): - self.cache_dir = label - self.repos = [ + def _cache(label: str, hub): + return SimpleNamespace( + cache_dir = label, + repos = [ SimpleNamespace( repo_type = "dataset", repo_id = "Org/Data", + repo_path = str(hub / "datasets--Org--Data"), revisions = [SimpleNamespace(commit_hash = f"{label}-rev")], ) - ] - self.fail = fail - - def delete_revisions(self, *_revisions): - return _DeleteStrategy(self.cache_dir, self.fail) + ], + delete_revisions = lambda *_revs, _label = label: _DeleteStrategy(_label), + ) monkeypatch.setattr( cache_inventory, "_collect_hf_cache_scans", - lambda: ([_Cache("first", True), _Cache("second", False)], set()), + lambda: ([_cache("active", target_hub), _cache("previous", other_hub)], set()), ) monkeypatch.setattr( cache_inventory, "_delete_processed_dataset_cache", - lambda _repo_id: (True, []), + lambda _repo_id, **_kwargs: (False, []), ) monkeypatch.setattr( cache_inventory.download_manifest, "purge_all_state_for_repo", - lambda *_args: purged_state.append(True) or 1, + lambda *_args, **_kwargs: 0, + ) + monkeypatch.setattr( + "utils.hf_cache_settings.get_hf_cache_paths", + lambda: SimpleNamespace(hub_cache = target_hub), + ) + monkeypatch.setattr( + "hub.utils.hf_cache_state.hf_cache_roots", + lambda: [target_hub, other_hub], ) - with pytest.raises(HTTPException) as exc_info: - cache_inventory._delete_cached_dataset_blocking("Org/Data") + result = cache_inventory._delete_cached_dataset_blocking("Org/Data") - assert exc_info.value.status_code == 500 - assert calls == ["first", "second"] - assert purged_state == [] + assert result == {"status": "deleted", "repo_id": "Org/Data"} + # Only the selected (active) cache's revision is deleted; the previous + # cache's copy is never touched. + assert calls == ["active"] + assert not (target_hub / "datasets--Org--Data").exists() + assert (other_hub / "datasets--Org--Data").exists() + + +def test_delete_processed_only_dataset_accepts_processed_cache_path(monkeypatch, tmp_path): + """A processed-only dataset row sends its Arrow cache path (___ + under HF_DATASETS_CACHE), which is not a Hub datasets-- dir. The delete must + accept it and run the processed-cache delete instead of raising 400.""" + datasets_root = tmp_path / "datasets" + processed_dir = datasets_root / "Org___Data" + processed_dir.mkdir(parents = True) + + # No Hub-cache copy exists; only the processed Arrow cache holds this repo. + monkeypatch.setattr(cache_inventory, "_collect_hf_cache_scans", lambda: ([], set())) + monkeypatch.setattr(cache_inventory, "_hf_datasets_cache_roots", lambda: [datasets_root]) + processed_calls: list[str] = [] + monkeypatch.setattr( + cache_inventory, + "_delete_processed_dataset_cache", + lambda repo_id, **_kwargs: (processed_calls.append(repo_id) or True, []), + ) + + result = cache_inventory._delete_cached_dataset_blocking("Org/Data", str(processed_dir)) + + assert result == {"status": "deleted", "repo_id": "Org/Data"} + assert processed_calls == ["Org/Data"] + + +def test_delete_processed_dataset_scopes_to_selected_root(monkeypatch, tmp_path): + """A dataset processed under two HF_DATASETS_CACHE roots is deleted only from + the selected root; the copy under the other cache home survives (real delete, + not stubbed).""" + selected_root = tmp_path / "selected" / "datasets" + other_root = tmp_path / "other" / "datasets" + for root in (selected_root, other_root): + (root / "Org___Data").mkdir(parents = True) + + monkeypatch.setattr(cache_inventory, "_collect_hf_cache_scans", lambda: ([], set())) + monkeypatch.setattr( + cache_inventory, "_hf_datasets_cache_roots", lambda: [selected_root, other_root] + ) + + result = cache_inventory._delete_cached_dataset_blocking( + "Org/Data", str(selected_root / "Org___Data") + ) + + assert result == {"status": "deleted", "repo_id": "Org/Data"} + assert not (selected_root / "Org___Data").exists() # the selected copy is deleted + assert (other_root / "Org___Data").exists() # the other cache home is untouched def test_delete_cached_dataset_purges_blob_only_repo_dir(monkeypatch): @@ -139,22 +197,22 @@ def test_delete_cached_dataset_purges_blob_only_repo_dir(monkeypatch): monkeypatch.setattr( cache_inventory, "_delete_processed_dataset_cache", - lambda _repo_id: (False, []), + lambda _repo_id, **_kwargs: (False, []), ) monkeypatch.setattr( cache_inventory, "purge_repo_cache_dirs", - lambda _repo_type, repo_id: purged_dirs.append(repo_id) or True, + lambda _repo_type, repo_id, **_kwargs: purged_dirs.append(repo_id) or True, ) monkeypatch.setattr( cache_inventory, "purge_partial_repo", - lambda *_args: False, + lambda *_args, **_kwargs: False, ) monkeypatch.setattr( cache_inventory.download_manifest, "purge_all_state_for_repo", - lambda *_args: 0, + lambda *_args, **_kwargs: 0, ) result = cache_inventory._delete_cached_dataset_blocking("Org/Data") @@ -172,22 +230,22 @@ def test_delete_cached_dataset_absent_everywhere_raises_404(monkeypatch): monkeypatch.setattr( cache_inventory, "_delete_processed_dataset_cache", - lambda _repo_id: (False, []), + lambda _repo_id, **_kwargs: (False, []), ) monkeypatch.setattr( cache_inventory, "purge_repo_cache_dirs", - lambda *_args: False, + lambda *_args, **_kwargs: False, ) monkeypatch.setattr( cache_inventory, "purge_partial_repo", - lambda *_args: False, + lambda *_args, **_kwargs: False, ) monkeypatch.setattr( cache_inventory.download_manifest, "purge_all_state_for_repo", - lambda *_args: 0, + lambda *_args, **_kwargs: 0, ) with pytest.raises(HTTPException) as exc_info: diff --git a/studio/backend/hub/tests/test_download_manifest_scoping.py b/studio/backend/hub/tests/test_download_manifest_scoping.py new file mode 100644 index 0000000000..966eeaf0c2 --- /dev/null +++ b/studio/backend/hub/tests/test_download_manifest_scoping.py @@ -0,0 +1,62 @@ +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 + +import json +from types import SimpleNamespace + +from hub.utils import download_manifest, state_dir + + +def _write_manifest(path, payload): + path.parent.mkdir(parents = True, exist_ok = True) + path.write_text(json.dumps(payload), encoding = "utf-8") + + +def test_purge_state_preserves_active_legacy_when_deleting_inactive_cache(monkeypatch, tmp_path): + """A scoped delete of an inactive cache must not erase the unscoped legacy + state, which _legacy_state_applies attributes to the active cache.""" + active = tmp_path / "active" / "hub" + previous = tmp_path / "previous" / "hub" + for path in (active, previous): + path.mkdir(parents = True) + + monkeypatch.setattr(state_dir, "cache_root", lambda: tmp_path / "state") + monkeypatch.setattr( + "utils.hf_cache_settings.get_hf_cache_paths", + lambda: SimpleNamespace(hub_cache = str(active)), + ) + + # Unowned legacy manifest -> belongs to the active cache. + legacy = state_dir.manifest_path("model", "Org/Model") + _write_manifest(legacy, {"version": 1}) + # The inactive cache's own scoped copy is the one being deleted. + scoped = state_dir.manifest_path("model", "Org/Model", hub_cache = str(previous)) + _write_manifest(scoped, {"version": 1, "hub_cache": str(previous)}) + + removed = download_manifest.purge_state("model", "Org/Model", hub_cache = str(previous)) + + assert removed is True + assert not scoped.is_file() # the inactive cache's copy is gone + assert legacy.is_file() # the active cache's legacy state survives + + +def test_purge_state_removes_legacy_owned_by_the_deleted_cache(monkeypatch, tmp_path): + """A legacy file that recorded the deleted cache as its owner is purged.""" + active = tmp_path / "active" / "hub" + previous = tmp_path / "previous" / "hub" + for path in (active, previous): + path.mkdir(parents = True) + + monkeypatch.setattr(state_dir, "cache_root", lambda: tmp_path / "state") + monkeypatch.setattr( + "utils.hf_cache_settings.get_hf_cache_paths", + lambda: SimpleNamespace(hub_cache = str(active)), + ) + + legacy = state_dir.manifest_path("model", "Org/Model") + _write_manifest(legacy, {"version": 1, "hub_cache": str(previous)}) + + removed = download_manifest.purge_state("model", "Org/Model", hub_cache = str(previous)) + + assert removed is True + assert not legacy.is_file() # owned by the deleted cache -> purged diff --git a/studio/backend/hub/tests/test_empty_variant_folder.py b/studio/backend/hub/tests/test_empty_variant_folder.py index 33bf6c6819..3ed8e69e0d 100644 --- a/studio/backend/hub/tests/test_empty_variant_folder.py +++ b/studio/backend/hub/tests/test_empty_variant_folder.py @@ -120,10 +120,16 @@ def _force_compute_to_raise(monkeypatch): monkeypatch.setattr(gguf_variants, "list_gguf_variants", _boom, raising = False) monkeypatch.setattr( - gguf_variants, "list_gguf_variants_from_hf_cache", lambda repo_id: None, raising = False + gguf_variants, + "list_gguf_variants_from_hf_cache", + lambda repo_id, root = None: None, + raising = False, ) monkeypatch.setattr( - gguf_variants, "list_partial_gguf_variants_from_state", lambda repo_id: None, raising = False + gguf_variants, + "list_partial_gguf_variants_from_state", + lambda repo_id, hub_cache = None: None, + raising = False, ) @@ -133,7 +139,11 @@ def test_get_variants_surfaces_cleanable_when_metadata_fails(monkeypatch): import asyncio _force_compute_to_raise(monkeypatch) - monkeypatch.setattr(gguf_variants, "list_empty_gguf_variant_dirs", lambda repo_id: {"UD-IQ1_S"}) + monkeypatch.setattr( + gguf_variants, + "list_empty_gguf_variant_dirs", + lambda repo_id, root = None: {"UD-IQ1_S"}, + ) resp = asyncio.run( gguf_variants.get_gguf_variants_response( @@ -152,7 +162,11 @@ def test_get_variants_reraises_when_no_cleanable(monkeypatch): from fastapi import HTTPException _force_compute_to_raise(monkeypatch) - monkeypatch.setattr(gguf_variants, "list_empty_gguf_variant_dirs", lambda repo_id: set()) + monkeypatch.setattr( + gguf_variants, + "list_empty_gguf_variant_dirs", + lambda repo_id, root = None: set(), + ) try: asyncio.run( diff --git a/studio/backend/hub/tests/test_model_services.py b/studio/backend/hub/tests/test_model_services.py index 2c33e09b2b..fa5862a13c 100644 --- a/studio/backend/hub/tests/test_model_services.py +++ b/studio/backend/hub/tests/test_model_services.py @@ -2,6 +2,7 @@ # Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 import asyncio +import json import sys from pathlib import Path from types import SimpleNamespace @@ -102,6 +103,236 @@ def test_big_endian_detection_ignores_model_name_be_token(): ) +def _cached_model_row(tmp_path: Path, *, partial: bool, active_cache: bool | None, size_bytes: int): + path = tmp_path / f"cache-{active_cache}-{partial}-{size_bytes}" + return model_common._local_model_info( + scan_path = path, + load_path = path, + source = "hf_cache", + model_format = "safetensors", + model_id = "Org/Model", + partial = partial, + active_cache = active_cache, + size_bytes = size_bytes, + ) + + +@pytest.mark.parametrize("reverse", [False, True]) +def test_local_inventory_prefers_complete_previous_cache_copy(tmp_path, reverse): + active_partial = _cached_model_row( + tmp_path, + partial = True, + active_cache = True, + size_bytes = 20, + ) + previous_complete = _cached_model_row( + tmp_path, + partial = False, + active_cache = False, + size_bytes = 10, + ) + rows = [active_partial, previous_complete] + if reverse: + rows.reverse() + + result = local_inventory._dedupe_local_models(rows) + + assert result == [previous_complete] + + +def test_local_inventory_compares_all_non_active_cache_copies(tmp_path): + inactive_partial = _cached_model_row( + tmp_path, + partial = True, + active_cache = False, + size_bytes = 20, + ) + custom_complete = _cached_model_row( + tmp_path, + partial = False, + active_cache = None, + size_bytes = 10, + ) + + assert local_inventory._dedupe_local_models([inactive_partial, custom_complete]) == [ + custom_complete + ] + + +def test_local_inventory_prefers_active_cache_when_copies_are_equally_complete(tmp_path): + previous = _cached_model_row( + tmp_path, + partial = False, + active_cache = False, + size_bytes = 20, + ) + active = _cached_model_row( + tmp_path, + partial = False, + active_cache = True, + size_bytes = 10, + ) + + assert local_inventory._dedupe_local_models([previous, active]) == [active] + + +def test_loaded_repo_match_accepts_previous_cache_snapshot_path(monkeypatch, tmp_path): + repo_dir = tmp_path / "old-hub" / "models--Org--Model" + snapshot = repo_dir / "snapshots" / "revision" + snapshot.mkdir(parents = True) + monkeypatch.setattr(deletion, "iter_repo_cache_dirs", lambda *_args: iter([repo_dir])) + + assert deletion._loaded_id_matches_repo(str(snapshot), "Org/Model") is True + assert deletion._loaded_id_matches_repo(str(snapshot / "model.gguf"), "Org/Model") is True + assert deletion._loaded_id_matches_repo(str(tmp_path / "other"), "Org/Model") is False + + +def test_cached_inventory_loads_previous_cache_copy_by_snapshot(monkeypatch, tmp_path): + active_hub = tmp_path / "active-hub" + previous_repo = tmp_path / "previous-hub" / "models--Org--Model" + snapshot = previous_repo / "snapshots" / "revision" + snapshot.mkdir(parents = True) + monkeypatch.setattr( + "utils.hf_cache_settings.get_hf_cache_paths", + lambda: SimpleNamespace(hub_cache = active_hub), + ) + + fields = cache_inventory._cache_inventory_fields( + "Org/Model", + "safetensors", + repo_path = previous_repo, + snapshot_path = snapshot, + ) + + assert fields["load_id"] == str(snapshot) + + +def test_cached_inventory_keeps_repo_id_for_active_cache(monkeypatch, tmp_path): + active_hub = tmp_path / "active-hub" + active_repo = active_hub / "models--Org--Model" + monkeypatch.setattr( + "utils.hf_cache_settings.get_hf_cache_paths", + lambda: SimpleNamespace(hub_cache = active_hub), + ) + + fields = cache_inventory._cache_inventory_fields( + "Org/Model", + "safetensors", + repo_path = active_repo, + ) + + assert fields["load_id"] == "Org/Model" + + +def test_cached_inventory_prefers_active_copy_when_completeness_matches(): + previous = {"partial": False, "active_cache": False, "size_bytes": 200} + active = {"partial": False, "active_cache": True, "size_bytes": 100} + + assert cache_inventory._prefer_cache_row(active, previous) is True + assert cache_inventory._prefer_cache_row(previous, active) is False + + +def test_cached_inventory_prefers_complete_copy_before_active_cache(): + previous = {"partial": False, "active_cache": False, "size_bytes": 100} + active_partial = {"partial": True, "active_cache": True, "size_bytes": 200} + + assert cache_inventory._prefer_cache_row(previous, active_partial) is True + assert cache_inventory._prefer_cache_row(active_partial, previous) is False + + +def test_inventory_scans_every_dynamic_cache_root(monkeypatch, tmp_path): + first = tmp_path / "first-hub" + second = tmp_path / "second-hub" + unreadable = tmp_path / "unreadable-hub" + first.mkdir() + second.mkdir() + unreadable.mkdir() + scanned = [] + + monkeypatch.setattr( + inventory_scan, + "hf_cache_roots", + lambda: [first, unreadable, second], + ) + + def scan_cache(cache_dir): + path = Path(cache_dir) + scanned.append(path) + if path == unreadable: + raise PermissionError("unreadable") + return SimpleNamespace(cache_dir = cache_dir) + + monkeypatch.setattr("huggingface_hub.scan_cache_dir", scan_cache) + + result = inventory_scan._compute_all_hf_cache_scans() + + assert scanned == [first, unreadable, second] + assert [Path(scan.cache_dir) for scan in result] == [first, second] + + +def test_inventory_applies_download_state_to_its_owning_cache(monkeypatch, tmp_path): + state_root = tmp_path / "state" + cache_a = tmp_path / "cache-a" + cache_b = tmp_path / "cache-b" + repo_id = "Org/Model" + repo_name = "models--Org--Model" + repo_a = cache_a / repo_name + repo_b = cache_b / repo_name + snapshot_a = repo_a / "snapshots" / "revision" + snapshot_b = repo_b / "snapshots" / "revision" + snapshot_a.mkdir(parents = True) + snapshot_b.mkdir(parents = True) + (snapshot_a / "config.json").write_bytes(b"x") + (snapshot_b / "config.json").write_bytes(b"xx") + + monkeypatch.setattr(state_dir, "cache_root", lambda: state_root) + monkeypatch.setattr( + "utils.hf_cache_settings.get_hf_cache_paths", + lambda: SimpleNamespace(hub_cache = cache_b), + ) + assert download_manifest.write_manifest( + "model", + repo_id, + None, + [download_manifest.ExpectedFile(path = "config.json", size = 2)], + "http", + hub_cache = cache_a, + ) + + assert inventory_scan.is_snapshot_partial("model", repo_id, repo_a) is True + assert inventory_scan.is_snapshot_partial("model", repo_id, repo_b) is False + assert inventory_scan.partial_transport_for("model", repo_id, None, repo_a) == "http" + assert inventory_scan.partial_transport_for("model", repo_id, None, repo_b) is None + + +def test_inventory_scopes_cancel_markers_to_their_owning_cache(monkeypatch, tmp_path): + state_root = tmp_path / "state" + cache_a = tmp_path / "cache-a" + cache_b = tmp_path / "cache-b" + repo_id = "Org/Model" + repo_name = "models--Org--Model" + repo_a = cache_a / repo_name + repo_b = cache_b / repo_name + repo_a.mkdir(parents = True) + repo_b.mkdir(parents = True) + + monkeypatch.setattr(state_dir, "cache_root", lambda: state_root) + monkeypatch.setattr( + "utils.hf_cache_settings.get_hf_cache_paths", + lambda: SimpleNamespace(hub_cache = cache_b), + ) + assert download_manifest.write_cancel_marker( + "model", + repo_id, + "Q4_K_M", + "xet", + hub_cache = cache_a, + ) + + assert inventory_scan.is_variant_partial(repo_id, "Q4_K_M", repo_cache_dir = repo_a) is True + assert inventory_scan.is_variant_partial(repo_id, "Q4_K_M", repo_cache_dir = repo_b) is False + + def test_list_local_gguf_variants_skips_big_endian_sibling(tmp_path): (tmp_path / "model-Q4_K_M-be.gguf").write_bytes(b"x" * 100) (tmp_path / "model-Q4_K_M.gguf").write_bytes(b"y" * 10) @@ -163,8 +394,19 @@ def test_download_state_bounds_long_repo_variant_filenames(monkeypatch, tmp_path "http", ) - marker_path = state_dir.marker_path("model", repo_id, variant) - manifest_path = state_dir.manifest_path("model", repo_id, variant) + hub_cache = download_manifest._canonical_hub_cache() + marker_path = state_dir.marker_path( + "model", + repo_id, + variant, + hub_cache = hub_cache, + ) + manifest_path = state_dir.manifest_path( + "model", + repo_id, + variant, + hub_cache = hub_cache, + ) assert marker_path is not None assert manifest_path is not None @@ -181,6 +423,97 @@ def test_download_state_bounds_long_repo_variant_filenames(monkeypatch, tmp_path ] +def test_download_state_isolated_across_hub_cache_switches(monkeypatch, tmp_path): + monkeypatch.setattr(state_dir, "cache_root", lambda: tmp_path) + cache_a = tmp_path / "cache-a" + cache_b = tmp_path / "cache-b" + selected = SimpleNamespace(hub_cache = cache_a) + + from utils import hf_cache_settings + + monkeypatch.setattr(hf_cache_settings, "get_hf_cache_paths", lambda: selected) + expected_a = [download_manifest.ExpectedFile(path = "a.gguf", size = 1)] + expected_b = [download_manifest.ExpectedFile(path = "b.gguf", size = 2)] + + assert download_manifest.write_manifest("model", "Owner/Repo", "Q4_K_M", expected_a) + assert download_manifest.write_cancel_marker("model", "Owner/Repo", "Q4_K_M", "http") + + selected.hub_cache = cache_b + assert download_manifest.write_manifest("model", "Owner/Repo", "Q4_K_M", expected_b) + + manifest_b = download_manifest.read_manifest("model", "Owner/Repo", "Q4_K_M") + manifest_a = download_manifest.read_manifest( + "model", + "Owner/Repo", + "Q4_K_M", + hub_cache = cache_a, + ) + + assert manifest_b is not None and manifest_b.expected_files == tuple(expected_b) + assert manifest_a is not None and manifest_a.expected_files == tuple(expected_a) + assert not download_manifest.has_cancel_marker("model", "Owner/Repo", "Q4_K_M") + assert download_manifest.has_cancel_marker( + "model", + "Owner/Repo", + "Q4_K_M", + hub_cache = cache_a, + ) + assert len(list((tmp_path / "hub-state" / "manifests").rglob("*.json"))) == 2 + + +def test_legacy_unscoped_download_state_falls_back_only_for_selected_cache(monkeypatch, tmp_path): + monkeypatch.setattr(state_dir, "cache_root", lambda: tmp_path) + cache_a = tmp_path / "cache-a" + cache_b = tmp_path / "cache-b" + monkeypatch.setattr( + "utils.hf_cache_settings.get_hf_cache_paths", + lambda: SimpleNamespace(hub_cache = cache_a), + ) + manifest = state_dir.manifest_path("model", "Owner/Repo", "Q4_K_M") + marker = state_dir.marker_path("model", "Owner/Repo", "Q4_K_M") + assert manifest is not None and marker is not None + manifest.write_text( + json.dumps( + { + "version": 1, + "repo_id": "Owner/Repo", + "variant": "Q4_K_M", + "expected_files": [{"path": "model.gguf", "size": 10}], + "transport": "http", + } + ), + encoding = "utf-8", + ) + marker.write_text( + json.dumps({"version": 1, "repo_id": "Owner/Repo", "variant": "Q4_K_M"}), + encoding = "utf-8", + ) + + assert download_manifest.read_manifest("model", "Owner/Repo", "Q4_K_M") is not None + assert download_manifest.has_cancel_marker("model", "Owner/Repo", "Q4_K_M") + assert list(download_manifest.iter_variant_manifests("model", "Owner/Repo")) == [ + ("Q4_K_M", manifest) + ] + assert list(download_manifest.iter_variant_markers("model", "Owner/Repo")) == [ + ("Q4_K_M", marker) + ] + assert ( + download_manifest.read_manifest( + "model", + "Owner/Repo", + "Q4_K_M", + hub_cache = cache_b, + ) + is None + ) + assert not download_manifest.has_cancel_marker( + "model", + "Owner/Repo", + "Q4_K_M", + hub_cache = cache_b, + ) + + class _RecordingLogger: def __init__(self): self.warnings = [] @@ -416,8 +749,15 @@ def test_cached_gguf_scan_includes_variant_state_without_completed_gguf(monkeypa "Q4_K_M", [download_manifest.ExpectedFile(path = "model-Q4_K_M.gguf", size = 4096)], "http", + hub_cache = repo_path.parent, + ) + assert download_manifest.write_cancel_marker( + "model", + "Org/PartialGguf", + "Q4_K_M", + "http", + hub_cache = repo_path.parent, ) - assert download_manifest.write_cancel_marker("model", "Org/PartialGguf", "Q4_K_M", "http") monkeypatch.setattr( cache_inventory, "all_hf_cache_scans", @@ -439,6 +779,288 @@ def test_cached_gguf_scan_includes_variant_state_without_completed_gguf(monkeypa assert row["capabilities"]["requires_variant"] is True +def test_cached_gguf_scan_hides_infra_repos_without_user_downloads(monkeypatch, tmp_path): + probe = _repo( + "ggml-org/models", + [_file("tinyllamas/stories260K.gguf", 1_200_000)], + tmp_path / "probe", + ) + embedder = _repo( + "unsloth/bge-small-en-v1.5-GGUF", + [_file("bge-small-en-v1.5-f16.gguf", 60_000_000)], + tmp_path / "embedder", + ) + chat = _repo("Org/Chat-GGUF", [_file("Q4_K_M.gguf", 100)], tmp_path / "chat") + monkeypatch.setattr( + cache_inventory, + "all_hf_cache_scans", + lambda: [SimpleNamespace(repos = [probe, embedder, chat])], + ) + monkeypatch.setattr( + cache_inventory.hf_cache_scan, + "is_gguf_repo_partial", + lambda _repo_id, _path: False, + ) + + result = {"cached": cache_inventory._scan_cached_gguf()} + + assert [row["repo_id"] for row in result["cached"]] == ["Org/Chat-GGUF"] + + +def test_cached_gguf_scan_keeps_infra_repo_with_user_downloaded_variant(monkeypatch, tmp_path): + monkeypatch.setattr(state_dir, "cache_root", lambda: tmp_path / "state") + embedder = _repo( + "unsloth/bge-small-en-v1.5-GGUF", + [ + _file("bge-small-en-v1.5-f16.gguf", 60_000_000), + _file("bge-small-en-v1.5-Q8_0.gguf", 35_000_000), + ], + tmp_path / "embedder", + ) + # Variant manifests only exist for user Hub downloads, not auto-downloads. + assert download_manifest.write_manifest( + "model", + "unsloth/bge-small-en-v1.5-GGUF", + "Q8_0", + [download_manifest.ExpectedFile(path = "bge-small-en-v1.5-Q8_0.gguf", size = 35_000_000)], + "http", + hub_cache = Path(embedder.repo_path).parent, + ) + monkeypatch.setattr( + cache_inventory, + "all_hf_cache_scans", + lambda: [SimpleNamespace(repos = [embedder])], + ) + monkeypatch.setattr( + cache_inventory.hf_cache_scan, + "is_gguf_repo_partial", + lambda _repo_id, _path: False, + ) + + result = {"cached": cache_inventory._scan_cached_gguf()} + + assert [row["repo_id"] for row in result["cached"]] == ["unsloth/bge-small-en-v1.5-GGUF"] + assert result["cached"][0]["capabilities"]["can_chat"] is False + + +def test_cached_models_scan_hides_non_gguf_embedder(monkeypatch, tmp_path): + embedder_path = tmp_path / "hub" / "models--unsloth--bge-small-en-v1.5" + embedder_path.mkdir(parents = True) + embedder = _repo( + "unsloth/bge-small-en-v1.5", + [_file("config.json", 12), _file("model.safetensors", 130_000_000)], + embedder_path, + ) + chat_path = tmp_path / "hub" / "models--Org--Chat" + chat_path.mkdir(parents = True) + chat = _repo( + "Org/Chat", + [_file("config.json", 12), _file("model.safetensors", 100)], + chat_path, + ) + monkeypatch.setattr( + cache_inventory, + "all_hf_cache_scans", + lambda: [SimpleNamespace(repos = [embedder, chat])], + ) + monkeypatch.setattr( + cache_inventory.hf_cache_scan, + "is_snapshot_partial", + lambda _kind, _repo_id, _path: False, + ) + + result = {"cached": cache_inventory._scan_cached_models()} + + assert [row["repo_id"] for row in result["cached"]] == ["Org/Chat"] + + +def test_cached_scans_hide_embedders_configured_by_cache_path(monkeypatch, tmp_path): + from core.rag import config as rag_config + + gguf_path = tmp_path / "hub" / "models--Org--PathEmbedder-GGUF" + gguf_path.mkdir(parents = True) + gguf = _repo( + "Org/PathEmbedder-GGUF", + [_file("model-F16.gguf", 60_000_000)], + gguf_path, + ) + model_path = tmp_path / "hub" / "models--Org--PathEmbedder" + model_path.mkdir(parents = True) + model = _repo( + "Org/PathEmbedder", + [_file("config.json", 12), _file("model.safetensors", 130_000_000)], + model_path, + ) + monkeypatch.setattr( + rag_config, + "effective_embedding_model", + lambda: str(model_path), + ) + monkeypatch.setattr( + rag_config, + "effective_gguf_repo", + lambda: str(gguf_path), + ) + monkeypatch.setattr( + cache_inventory, + "all_hf_cache_scans", + lambda: [SimpleNamespace(repos = [gguf, model])], + ) + monkeypatch.setattr( + cache_inventory.hf_cache_scan, + "is_gguf_repo_partial", + lambda _repo_id, _path: False, + ) + monkeypatch.setattr( + cache_inventory.hf_cache_scan, + "is_snapshot_partial", + lambda _kind, _repo_id, _path: False, + ) + + assert cache_inventory._scan_cached_gguf() == [] + assert cache_inventory._scan_cached_models() == [] + + +def test_cached_scans_hide_embedders_configured_by_snapshot_path(monkeypatch, tmp_path): + from core.rag import config as rag_config + + gguf_path = tmp_path / "hub" / "models--Org--SnapshotEmbedder-GGUF" + gguf_snapshot = gguf_path / "snapshots" / "gguf-revision" + gguf_snapshot.mkdir(parents = True) + gguf = _repo( + "Org/SnapshotEmbedder-GGUF", + [_file("model-F16.gguf", 60_000_000)], + gguf_path, + ) + model_path = tmp_path / "hub" / "models--Org--SnapshotEmbedder" + model_snapshot = model_path / "snapshots" / "model-revision" + model_snapshot.mkdir(parents = True) + model = _repo( + "Org/SnapshotEmbedder", + [_file("config.json", 12), _file("model.safetensors", 130_000_000)], + model_path, + ) + monkeypatch.setattr( + rag_config, + "effective_embedding_model", + lambda: str(model_snapshot), + ) + monkeypatch.setattr( + rag_config, + "effective_gguf_repo", + lambda: str(gguf_snapshot), + ) + monkeypatch.setattr( + cache_inventory, + "all_hf_cache_scans", + lambda: [SimpleNamespace(repos = [gguf, model])], + ) + + def _resolve_snapshot(repo_path): + return str( + { + gguf_path: gguf_snapshot, + model_path: model_snapshot, + }.get(Path(repo_path), Path(repo_path)) + ) + + monkeypatch.setattr( + cache_inventory.hf_cache_scan, + "resolve_hf_cache_realpath", + _resolve_snapshot, + ) + monkeypatch.setattr( + cache_inventory.hf_cache_scan, + "is_gguf_repo_partial", + lambda _repo_id, _path: False, + ) + monkeypatch.setattr( + cache_inventory.hf_cache_scan, + "is_snapshot_partial", + lambda _kind, _repo_id, _path: False, + ) + + assert cache_inventory._scan_cached_gguf() == [] + assert cache_inventory._scan_cached_models() == [] + + +def test_cached_models_scan_keeps_unrelated_repo_with_custom_generic_embedder( + monkeypatch, tmp_path +): + # A custom embedder with a generic basename ("org/model") must be hidden by + # EXACT repo-id match only. An unrelated cached chat model whose id merely + # contains "model" (e.g. "user/model-chat") must stay on device: substring + # basename matching used to drop real chat models from the inventory. + from core.rag import config as rag_config + + monkeypatch.setattr(rag_config, "effective_embedding_model", lambda: "org/model") + monkeypatch.setattr(rag_config, "effective_gguf_repo", lambda: "org/model-GGUF") + + def _model_repo(repo_id: str): + path = tmp_path / "hub" / f"models--{repo_id.replace('/', '--')}" + path.mkdir(parents = True) + return _repo( + repo_id, + [_file("config.json", 12), _file("model.safetensors", 100)], + path, + ) + + embedder = _model_repo("org/model") + chat = _model_repo("user/model-chat") + monkeypatch.setattr( + cache_inventory, + "all_hf_cache_scans", + lambda: [SimpleNamespace(repos = [embedder, chat])], + ) + monkeypatch.setattr( + cache_inventory.hf_cache_scan, + "is_snapshot_partial", + lambda _kind, _repo_id, _path: False, + ) + + result = {"cached": cache_inventory._scan_cached_models()} + + assert [row["repo_id"] for row in result["cached"]] == ["user/model-chat"] + + +def test_cached_scans_hide_stale_default_embedder_after_custom_setting(monkeypatch, tmp_path): + from core.rag import config as rag_config + + monkeypatch.setattr(rag_config, "effective_embedding_model", lambda: "org/custom") + monkeypatch.setattr(rag_config, "effective_gguf_repo", lambda: "org/custom-GGUF") + + gguf = _repo( + "unsloth/bge-small-en-v1.5-GGUF", + [_file("bge-small-en-v1.5-f16.gguf", 60_000_000)], + tmp_path / "default-gguf", + ) + weights_path = tmp_path / "hub" / "models--unsloth--bge-small-en-v1.5" + weights_path.mkdir(parents = True) + weights = _repo( + "unsloth/bge-small-en-v1.5", + [_file("config.json", 12), _file("model.safetensors", 130_000_000)], + weights_path, + ) + monkeypatch.setattr( + cache_inventory, + "all_hf_cache_scans", + lambda: [SimpleNamespace(repos = [gguf, weights])], + ) + monkeypatch.setattr( + cache_inventory.hf_cache_scan, + "is_gguf_repo_partial", + lambda _repo_id, _path: False, + ) + monkeypatch.setattr( + cache_inventory.hf_cache_scan, + "is_snapshot_partial", + lambda _kind, _repo_id, _path: False, + ) + + assert cache_inventory._scan_cached_gguf() == [] + assert cache_inventory._scan_cached_models() == [] + + def test_gguf_variant_requirements_include_split_files_and_preferred_mmproj(): requirements = gguf_variants._build_gguf_variant_requirements( [ @@ -925,6 +1547,7 @@ def test_gguf_progress_counts_completed_mmproj_with_expected_bytes(monkeypatch, ), ], "http", + hub_cache = entry.parent, ) requirement = gguf_variants._GgufVariantRequirement( @@ -1011,6 +1634,7 @@ def test_gguf_progress_subtracts_new_job_completed_baseline(monkeypatch, tmp_pat ), ], "http", + hub_cache = entry.parent, ) requirement = gguf_variants._GgufVariantRequirement( @@ -1181,6 +1805,7 @@ def test_gguf_progress_complete_on_disk_ignores_full_baseline(monkeypatch, tmp_p ), ], "http", + hub_cache = entry.parent, ) requirement = gguf_variants._GgufVariantRequirement( @@ -1580,8 +2205,15 @@ def test_hf_cache_scan_uses_gguf_partial_row_for_variant_state(monkeypatch, tmp_ "Q4_K_M", [download_manifest.ExpectedFile(path = "model-Q4_K_M.gguf", size = 8192)], "http", + hub_cache = cache_dir, + ) + assert download_manifest.write_cancel_marker( + "model", + "Org/PartialGguf", + "Q4_K_M", + "http", + hub_cache = cache_dir, ) - assert download_manifest.write_cancel_marker("model", "Org/PartialGguf", "Q4_K_M", "http") monkeypatch.setattr(local_inventory, "_classify_local_path", lambda *_args, **_kwargs: []) monkeypatch.setattr( local_inventory.hf_cache_scan, @@ -1610,6 +2242,63 @@ def test_hf_cache_scan_uses_gguf_partial_row_for_variant_state(monkeypatch, tmp_ assert rows[0].capabilities.requires_variant is True +def test_local_inventory_filters_custom_embedder_hf_cache_row(monkeypatch, tmp_path): + from core.rag import config as rag_config + + monkeypatch.setattr(rag_config, "effective_embedding_model", lambda: "org/embedder") + monkeypatch.setattr(rag_config, "effective_gguf_repo", lambda: "org/embedder-GGUF") + + def _row(repo_id: str): + repo_path = tmp_path / f"models--{repo_id.replace('/', '--')}" + return model_common._local_model_info( + scan_path = repo_path, + load_path = repo_path, + source = "hf_cache", + model_format = "safetensors", + model_id = repo_id, + ) + + rows = local_inventory._filter_hidden_models([_row("org/embedder"), _row("org/chat-model")]) + + assert [row.model_id for row in rows] == ["org/chat-model"] + + +def test_local_inventory_filters_embedder_configured_by_snapshot_path(monkeypatch, tmp_path): + from core.rag import config as rag_config + + embedder_path = tmp_path / "hub" / "models--org--embedder" + embedder_snapshot = embedder_path / "snapshots" / "revision" + embedder_snapshot.mkdir(parents = True) + chat_path = tmp_path / "hub" / "models--org--chat-model" + chat_path.mkdir(parents = True) + monkeypatch.setattr( + rag_config, + "effective_embedding_model", + lambda: str(embedder_snapshot), + ) + monkeypatch.setattr(rag_config, "effective_gguf_repo", lambda: "org/embedder-GGUF") + monkeypatch.setattr( + local_inventory.hf_cache_scan, + "resolve_hf_cache_realpath", + lambda path: str(embedder_snapshot) if Path(path) == embedder_path else str(path), + ) + + def _row(repo_id: str, repo_path: Path): + return model_common._local_model_info( + scan_path = repo_path, + load_path = repo_path, + source = "hf_cache", + model_format = "safetensors", + model_id = repo_id, + ) + + rows = local_inventory._filter_hidden_models( + [_row("org/embedder", embedder_path), _row("org/chat-model", chat_path)] + ) + + assert [row.model_id for row in rows] == ["org/chat-model"] + + def test_model_download_job_helpers_preserve_idle_shape(): key = downloads._download_job_key("Org/Model", None) status = downloads._job_status(key) @@ -1779,7 +2468,7 @@ def test_gguf_variants_partial_marker_overrides_size_only_downloaded(monkeypatch monkeypatch.setattr( gguf_variants, "iter_hf_cache_snapshots", - lambda _repo_id: [snapshot], + lambda _repo_id, root = None: [snapshot], ) monkeypatch.setattr( gguf_variants, @@ -1798,6 +2487,70 @@ def test_gguf_variants_partial_marker_overrides_size_only_downloaded(monkeypatch assert result.variants[0].partial is True +def test_gguf_variants_scopes_partial_state_to_requested_cache(monkeypatch, tmp_path): + async def _run_inline(fn, *args, **kwargs): + return fn(*args, **kwargs) + + repo_id = "Org/SharedRepo" + repo_name = "models--Org--SharedRepo" + cache_a = tmp_path / "cache-a" + cache_b = tmp_path / "cache-b" + repo_a = cache_a / repo_name + snapshot_a = repo_a / "snapshots" / "revision" + snapshot_a.mkdir(parents = True) + (snapshot_a / "model-Q8_0.gguf").write_bytes(b"complete") + blobs_b = cache_b / repo_name / "blobs" + blobs_b.mkdir(parents = True) + (blobs_b / "q8-hash.incomplete").write_bytes(b"partial") + + monkeypatch.setattr(state_dir, "cache_root", lambda: tmp_path / "state") + monkeypatch.setattr(gguf_variants.asyncio, "to_thread", _run_inline) + monkeypatch.setattr( + "utils.hf_cache_settings.get_hf_cache_paths", + lambda: SimpleNamespace(hub_cache = cache_b), + ) + assert download_manifest.write_cancel_marker( + "model", + repo_id, + "Q8_0", + "http", + hub_cache = cache_b, + ) + monkeypatch.setattr( + gguf_variants, + "list_gguf_variants", + lambda *_args, **_kwargs: ( + [ + SimpleNamespace( + filename = "model-Q8_0.gguf", + quant = "Q8_0", + display_label = None, + size_bytes = 8, + ) + ], + False, + [ + SimpleNamespace( + rfilename = "model-Q8_0.gguf", + size = 8, + lfs = SimpleNamespace(sha256 = "q8-hash"), + ) + ], + ), + ) + monkeypatch.setattr(cache_inventory, "all_hf_cache_scans", lambda: []) + + result = asyncio.run( + gguf_variants.get_gguf_variants_response( + repo_id, + local_path = str(repo_a), + ) + ) + + assert result.variants[0].downloaded is True + assert result.variants[0].partial is False + + def test_download_registry_repo_keys_are_case_insensitive(): registry = download_registry.DownloadRegistry() @@ -2106,6 +2859,34 @@ def test_prepare_cache_for_transport_purges_only_requested_hashes(monkeypatch, t assert (blobs / "shared-mmproj.incomplete").exists() +def test_prepare_cache_for_transport_uses_captured_root(monkeypatch, tmp_path): + cache_a = tmp_path / "cache-a" + cache_b = tmp_path / "cache-b" + repo_name = "models--Org--Repo" + partial_a = cache_a / repo_name / "blobs" / "blob.incomplete" + partial_b = cache_b / repo_name / "blobs" / "blob.incomplete" + partial_a.parent.mkdir(parents = True) + partial_b.parent.mkdir(parents = True) + partial_a.write_bytes(b"a") + partial_b.write_bytes(b"b") + monkeypatch.setattr( + download_registry, + "hf_cache_root", + lambda create = False, root = None: root or cache_b, + ) + + purged = download_registry.prepare_cache_for_transport( + "model", + "Org/Repo", + download_registry.TRANSPORT_HTTP, + root = cache_a, + ) + + assert purged == 1 + assert not partial_a.exists() + assert partial_b.exists() + + def _vision_cache_root(monkeypatch, tmp_path): root = tmp_path / "hub" blobs = root / "models--Org--Vision" / "blobs" @@ -2464,6 +3245,47 @@ def test_shutdown_skips_marker_for_worker_that_exits_cleanly(monkeypatch): assert markers == ["Org/Cut"] +def test_orphan_reaper_uses_worker_cache_root_after_setting_changes(monkeypatch, tmp_path): + workers = tmp_path / "workers" + workers.mkdir() + cache_a = tmp_path / "cache-a" / "hub" + cache_b = tmp_path / "cache-b" / "hub" + partial = cache_a / "models--Org--Model" / "blobs" / "abc.incomplete" + partial.parent.mkdir(parents = True) + partial.write_bytes(b"partial") + cache_b.mkdir(parents = True) + monkeypatch.setattr(state_dir, "workers_dir", lambda: workers) + monkeypatch.setattr(download_registry, "_process_alive", lambda _pid: False) + monkeypatch.setattr( + "utils.hf_cache_settings.get_hf_cache_paths", + lambda: SimpleNamespace(hub_cache = cache_b), + ) + markers = [] + monkeypatch.setattr( + download_registry, + "persist_cancel_marker", + lambda *args, **kwargs: markers.append(args), + ) + metadata = download_registry.DownloadMetadata( + repo_type = "model", + repo_id = "Org/Model", + variant = None, + transport = download_registry.TRANSPORT_HTTP, + hub_cache = str(cache_a), + xet_cache = str(tmp_path / "cache-a" / "xet"), + ) + download_registry.write_worker_breadcrumb("org/model", 1234, metadata) + [breadcrumb] = list(workers.iterdir()) + payload = json.loads(breadcrumb.read_text(encoding = "utf-8")) + assert payload["hub_cache"] == str(cache_a) + assert payload["xet_cache"] == str(tmp_path / "cache-a" / "xet") + + download_registry.reap_orphan_workers() + + assert markers == [("model", "Org/Model", None, "http")] + assert list(workers.iterdir()) == [] + + def test_model_claim_register_cancel_uses_registry_marker_owner(monkeypatch): killed = [] @@ -2787,12 +3609,19 @@ def _build_variant_cache_repo(repo_dir, blob_specs, snapshot_links): return repo -def _patch_variant_delete_side_effects(monkeypatch): +def _patch_variant_delete_side_effects(monkeypatch, hub_cache = None): monkeypatch.setattr( deletion.download_manifest, "purge_state", lambda *_args, **_kwargs: False, ) + # The repo under test lives in this cache; make it the active one so the + # delete scopes to it (default target root is the active hub cache). + if hub_cache is not None: + monkeypatch.setattr( + "utils.hf_cache_settings.get_hf_cache_paths", + lambda: SimpleNamespace(hub_cache = hub_cache), + ) def test_snapshot_progress_filters_stale_blobs(monkeypatch, tmp_path): @@ -2970,7 +3799,7 @@ def test_delete_variant_keeps_blob_shared_with_other_snapshot(monkeypatch, tmp_p "all_hf_cache_scans", lambda: [SimpleNamespace(repos = [repo])], ) - _patch_variant_delete_side_effects(monkeypatch) + _patch_variant_delete_side_effects(monkeypatch, tmp_path) result = deletion._delete_cached_model_blocking("Org/Repo-GGUF", "Q4_K_M", None) @@ -2997,7 +3826,7 @@ def test_delete_variant_unlinks_unshared_blob(monkeypatch, tmp_path): "all_hf_cache_scans", lambda: [SimpleNamespace(repos = [repo])], ) - _patch_variant_delete_side_effects(monkeypatch) + _patch_variant_delete_side_effects(monkeypatch, tmp_path) result = deletion._delete_cached_model_blocking("Org/Repo-GGUF", "Q4_K_M", None) @@ -3023,7 +3852,7 @@ def test_delete_variant_surfaces_locked_file_as_conflict(monkeypatch, tmp_path): "all_hf_cache_scans", lambda: [SimpleNamespace(repos = [repo])], ) - _patch_variant_delete_side_effects(monkeypatch) + _patch_variant_delete_side_effects(monkeypatch, tmp_path) real_unlink = Path.unlink diff --git a/studio/backend/hub/utils/download_manifest.py b/studio/backend/hub/utils/download_manifest.py index 5366689296..ac0ccb5490 100644 --- a/studio/backend/hub/utils/download_manifest.py +++ b/studio/backend/hub/utils/download_manifest.py @@ -77,6 +77,7 @@ class Manifest: started_at: str expected_files: tuple[ExpectedFile, ...] transport: Optional[str] = None + hub_cache: Optional[str] = None @dataclass(frozen = True) @@ -86,6 +87,78 @@ class VerifyResult: size_mismatched: tuple[str, ...] +def _canonical_hub_cache(hub_cache: Optional[str | Path] = None) -> Optional[str]: + if hub_cache is None: + try: + from utils.hf_cache_settings import get_hf_cache_paths + hub_cache = get_hf_cache_paths().hub_cache + except Exception: + return None + try: + return str(Path(hub_cache).expanduser().resolve(strict = False)) + except (OSError, RuntimeError, ValueError): + return str(hub_cache) + + +def _read_state_payload(path: Path) -> Optional[dict]: + try: + data = json.loads(path.read_text(encoding = "utf-8")) + except (OSError, ValueError) as exc: + logger.debug("Could not read Hub state %s: %s", path, exc) + return None + return data if isinstance(data, dict) else None + + +def _legacy_state_applies( + path: Path, + requested_hub_cache: Optional[str], + *, + fail_closed: bool = False, +) -> bool: + """Whether an old unscoped state file belongs to the requested cache. + + Transitional files that recorded their cache keep that ownership. Older + files with no ownership can only be attributed to the currently selected + cache, which matches the single-cache behavior under which they were + written without leaking them into remembered inactive caches. + """ + data = _read_state_payload(path) + if data is not None: + recorded = data.get("hub_cache") + if isinstance(recorded, str) and recorded: + return _canonical_hub_cache(recorded) == requested_hub_cache + elif not fail_closed: + return False + return requested_hub_cache == _canonical_hub_cache() + + +def _state_read_path( + path_factory, + repo_type: RepoType, + repo_id: str, + variant: Optional[str], + hub_cache: Optional[str | Path], + *, + fail_closed: bool = False, +) -> Optional[Path]: + requested = _canonical_hub_cache(hub_cache) + scoped = path_factory(repo_type, repo_id, variant, hub_cache = requested) + try: + if scoped is not None and scoped.is_file(): + return scoped + except OSError: + pass + legacy = path_factory(repo_type, repo_id, variant) + if legacy is None or legacy == scoped: + return None + try: + if not legacy.is_file(): + return None + except OSError: + return None + return legacy if _legacy_state_applies(legacy, requested, fail_closed = fail_closed) else None + + def _atomic_write_json(path: Path, payload: dict) -> bool: # Per-write uuid suffix so a concurrent caller or a stale tmp from a # previous crash cannot collide with the in-flight write. @@ -124,6 +197,8 @@ def write_manifest( variant: Optional[str], expected_files: Sequence[ExpectedFile], transport: Optional[str] = None, + *, + hub_cache: Optional[str | Path] = None, ) -> bool: """Write/overwrite the manifest for this triple. Best-effort. @@ -131,7 +206,13 @@ def write_manifest( worst-case fallback is the pre-fix scanner behavior (one missed partial detection), which is no regression. """ - path = manifest_path(repo_type, repo_id, variant) + recorded_hub_cache = _canonical_hub_cache(hub_cache) + path = manifest_path( + repo_type, + repo_id, + variant, + hub_cache = recorded_hub_cache, + ) if path is None: return False payload = { @@ -149,6 +230,7 @@ def write_manifest( for f in expected_files ], "transport": transport, + "hub_cache": recorded_hub_cache, } return _atomic_write_json(path, payload) @@ -157,6 +239,8 @@ def read_manifest( repo_type: RepoType, repo_id: str, variant: Optional[str] = None, + *, + hub_cache: Optional[str | Path] = None, ) -> Optional[Manifest]: """Return the manifest if present and parseable; ``None`` otherwise. @@ -171,15 +255,17 @@ def read_manifest( ``_MANIFEST_VERSION`` and widen this check) or live under a different filename, so an incompatible payload can never mis-classify rows. """ - path = manifest_path(repo_type, repo_id, variant) + path = _state_read_path( + manifest_path, + repo_type, + repo_id, + variant, + hub_cache, + ) if path is None or not path.is_file(): return None - try: - data = json.loads(path.read_text(encoding = "utf-8")) - except (OSError, ValueError) as exc: - logger.debug("Could not read manifest %s: %s", path, exc) - return None - if not isinstance(data, dict): + data = _read_state_payload(path) + if data is None: return None if data.get("version") != _MANIFEST_VERSION: logger.debug( @@ -216,6 +302,7 @@ def read_manifest( started_at = str(data.get("started_at", "")), expected_files = tuple(expected), transport = transport if transport in ("http", "xet") else None, + hub_cache = data.get("hub_cache") if isinstance(data.get("hub_cache"), str) else None, ) @@ -289,6 +376,8 @@ def write_cancel_marker( repo_id: str, variant: Optional[str] = None, transport: Optional[str] = None, + *, + hub_cache: Optional[str | Path] = None, ) -> bool: """Record that this triple was cancelled. Idempotent across repeated cancels. @@ -296,7 +385,13 @@ def write_cancel_marker( inventory rows so the UI labels HTTP retries as continuable and XET retries as full redownloads. None is accepted for forward-compat. """ - path = marker_path(repo_type, repo_id, variant) + recorded_hub_cache = _canonical_hub_cache(hub_cache) + path = marker_path( + repo_type, + repo_id, + variant, + hub_cache = recorded_hub_cache, + ) if path is None: return False payload = { @@ -306,6 +401,7 @@ def write_cancel_marker( "variant": variant, "transport": transport, "cancelled_at": datetime.now(timezone.utc).isoformat(), + "hub_cache": recorded_hub_cache, } return _atomic_write_json(path, payload) @@ -314,6 +410,8 @@ def read_cancel_marker_transport( repo_type: RepoType, repo_id: str, variant: Optional[str] = None, + *, + hub_cache: Optional[str | Path] = None, ) -> Optional[str]: """Return the transport recorded in the cancel marker, or ``None`` if no marker exists or it is unreadable. @@ -330,15 +428,17 @@ def read_cancel_marker_transport( ``None`` keeps the neutral "Retry" label. * Unknown future versions → ``None`` (unknown layout, unknown transport). """ - path = marker_path(repo_type, repo_id, variant) + path = _state_read_path( + marker_path, + repo_type, + repo_id, + variant, + hub_cache, + ) if path is None or not path.is_file(): return None - try: - data = json.loads(path.read_text(encoding = "utf-8")) - except (OSError, ValueError) as exc: - logger.debug("Could not read cancel marker %s: %s", path, exc) - return None - if not isinstance(data, dict): + data = _read_state_payload(path) + if data is None: return None version = data.get("version") if version == _LEGACY_MARKER_VERSION: @@ -351,10 +451,30 @@ def read_cancel_marker_transport( return None +def _all_matching_state_paths( + parent: Optional[Path], repo_type: RepoType, repo_id: str, variant: Optional[str] +) -> tuple[Path, ...]: + if parent is None: + return () + legacy_path = ( + manifest_path(repo_type, repo_id, variant) + if parent.name == "manifests" + else marker_path(repo_type, repo_id, variant) + ) + if legacy_path is None: + return () + try: + return tuple(path for path in parent.rglob(legacy_path.name) if path.is_file()) + except OSError: + return () + + def clear_cancel_marker( repo_type: RepoType, repo_id: str, variant: Optional[str] = None, + *, + hub_cache: Optional[str | Path] = None, ) -> None: """Remove the cancel marker for this triple if present. @@ -362,31 +482,48 @@ def clear_cancel_marker( download-start (a fresh attempt supersedes prior cancel state) and again at successful completion (cleans up if the start clear failed). """ - path = marker_path(repo_type, repo_id, variant) - if path is None: - return - try: - path.unlink(missing_ok = True) - except OSError as exc: - logger.debug("Could not clear cancel marker %s: %s", path, exc) + requested = _canonical_hub_cache(hub_cache) + path = marker_path( + repo_type, + repo_id, + variant, + hub_cache = requested, + ) + legacy = marker_path(repo_type, repo_id, variant) + paths = [path] + if ( + legacy is not None + and legacy != path + and _legacy_state_applies(legacy, requested, fail_closed = True) + ): + paths.append(legacy) + for target in paths: + if target is None: + continue + try: + target.unlink(missing_ok = True) + except OSError as exc: + logger.debug("Could not clear cancel marker %s: %s", target, exc) def has_cancel_marker( repo_type: RepoType, repo_id: str, variant: Optional[str] = None, + *, + hub_cache: Optional[str | Path] = None, ) -> bool: - """File-existence check only. Body is never read. - - Fail-closed: a corrupt marker still returns ``True`` because the - file's existence is the signal (the user once cancelled this - triple, even if the body is unreadable). - """ - path = marker_path(repo_type, repo_id, variant) - if path is None: - return False + """Return whether a cancel marker applies to the selected cache.""" + path = _state_read_path( + marker_path, + repo_type, + repo_id, + variant, + hub_cache, + fail_closed = True, + ) try: - return path.is_file() + return path is not None and path.is_file() except OSError: return False @@ -395,48 +532,124 @@ def delete_manifest( repo_type: RepoType, repo_id: str, variant: Optional[str] = None, + *, + hub_cache: Optional[str | Path] = None, ) -> bool: - path = manifest_path(repo_type, repo_id, variant) - if path is None: - return False - try: - if not path.is_file(): - return False - path.unlink() - return True - except OSError as exc: - logger.debug("Could not delete manifest %s: %s", path, exc) - return False + requested = _canonical_hub_cache(hub_cache) + path = manifest_path( + repo_type, + repo_id, + variant, + hub_cache = requested, + ) + legacy = manifest_path(repo_type, repo_id, variant) + paths = [path] + if legacy is not None and legacy != path and _legacy_state_applies(legacy, requested): + paths.append(legacy) + removed = False + for target in paths: + if target is None: + continue + try: + if target.is_file(): + target.unlink() + removed = True + except OSError as exc: + logger.debug("Could not delete manifest %s: %s", target, exc) + return removed def purge_state( repo_type: RepoType, repo_id: str, variant: Optional[str] = None, + *, + hub_cache: Optional[str | Path] = None, ) -> bool: """Remove manifest + cancel marker for this triple. Returns ``True`` - when anything was present on disk before the call. Idempotent.""" - marker_existed = has_cancel_marker(repo_type, repo_id, variant) - manifest_removed = delete_manifest(repo_type, repo_id, variant) - clear_cancel_marker(repo_type, repo_id, variant) - return marker_existed or manifest_removed + when anything was present on disk before the call. Idempotent. + + With ``hub_cache`` set, only that cache's scoped state (plus any legacy + unscoped file that belongs to it) is removed, so purging one cache's copy + never clears another cache's resumable/cancel state.""" + if hub_cache is None: + paths = ( + *_all_matching_state_paths(manifests_dir(), repo_type, repo_id, variant), + *_all_matching_state_paths(cancelled_dir(), repo_type, repo_id, variant), + ) + else: + requested = _canonical_hub_cache(hub_cache) + candidates = [ + manifest_path(repo_type, repo_id, variant, hub_cache = hub_cache), + marker_path(repo_type, repo_id, variant, hub_cache = hub_cache), + ] + # Legacy unscoped state is shared: an unowned file belongs to the active + # cache (per _legacy_state_applies), so only purge it when it belongs to + # the cache being deleted -- else deleting an inactive cache would erase + # the active cache's resume/cancel state. + for path_factory in (manifest_path, marker_path): + legacy = path_factory(repo_type, repo_id, variant) + if legacy is not None and _legacy_state_applies(legacy, requested): + candidates.append(legacy) + paths = tuple(p for p in candidates if p is not None) + removed = False + for path in paths: + try: + if path.is_file(): + path.unlink() + removed = True + except OSError as exc: + logger.debug("Could not purge Hub state %s: %s", path, exc) + return removed -def purge_all_state_for_repo(repo_type: RepoType, repo_id: str) -> int: +def purge_all_state_for_repo( + repo_type: RepoType, + repo_id: str, + *, + hub_cache: Optional[str | Path] = None, +) -> int: """Remove the snapshot-level manifest + marker AND every variant-keyed manifest + marker for this repo. Used by the route delete handlers so scanner state never outlives the cache it described. Returns the count - of (repo, variant) triples that had any state on disk.""" + of (repo, variant) triples that had any state on disk. + + With ``hub_cache`` set, only that cache's scoped state (plus any legacy + unscoped file) is enumerated and removed, so deleting one cache's copy does + not clear another cache's resumable/cancel state.""" removed = 0 - if purge_state(repo_type, repo_id, None): + if purge_state(repo_type, repo_id, None, hub_cache = hub_cache): removed += 1 variants: set[str] = set() - for variant, _ in iter_variant_manifests(repo_type, repo_id): - variants.add(variant) - for variant, _ in iter_variant_markers(repo_type, repo_id): - variants.add(variant) + prefix = variant_filename_prefix(repo_type, repo_id) + if hub_cache is None: + search = [(p, True) for p in (manifests_dir(), cancelled_dir()) if p is not None] + else: + # This cache's scoped dir (parent of its scoped path) plus the legacy + # unscoped base; glob (not rglob) so other caches' dirs are not swept. + search = [] + for scoped, base in ( + (manifest_path(repo_type, repo_id, None, hub_cache = hub_cache), manifests_dir()), + (marker_path(repo_type, repo_id, None, hub_cache = hub_cache), cancelled_dir()), + ): + if scoped is not None: + search.append((scoped.parent, False)) + if base is not None: + search.append((base, False)) + for parent, recursive in search: + try: + entries = tuple( + parent.rglob(f"{prefix}*.json") if recursive else parent.glob(f"{prefix}*.json") + ) + except OSError: + continue + for entry in entries: + if not entry.is_file(): + continue + fallback = entry.stem[len(prefix) :] + variants.add(_variant_from_state_file(entry, fallback)) for variant in variants: - if purge_state(repo_type, repo_id, variant): + if purge_state(repo_type, repo_id, variant, hub_cache = hub_cache): removed += 1 return removed @@ -453,35 +666,83 @@ def _variant_from_state_file(path: Path, fallback: str) -> str: def _iter_variant_state_files( - parent: Optional[Path], repo_type: RepoType, repo_id: str + parent: Optional[Path], + repo_type: RepoType, + repo_id: str, + hub_cache: Optional[str | Path], + *, + cancel_markers: bool, ) -> Iterator[tuple[str, Path]]: if parent is None: return - prefix = variant_filename_prefix(repo_type, repo_id) - try: - entries = list(parent.iterdir()) - except OSError: + path_factory = marker_path if cancel_markers else manifest_path + requested = _canonical_hub_cache(hub_cache) + scoped_probe = path_factory( + repo_type, + repo_id, + None, + hub_cache = requested, + ) + if scoped_probe is None: return - for entry in entries: - if not entry.is_file() or not entry.name.endswith(".json"): + prefix = variant_filename_prefix(repo_type, repo_id) + seen: set[str] = set() + for directory, legacy in ((scoped_probe.parent, False), (parent, True)): + if legacy and directory == scoped_probe.parent: continue - stem = entry.name[: -len(".json")] - if not stem.lower().startswith(prefix): + try: + entries = list(directory.iterdir()) + except OSError: continue - variant = stem[len(prefix) :] - if variant: - yield _variant_from_state_file(entry, variant), entry + for entry in entries: + if not entry.is_file() or not entry.name.endswith(".json"): + continue + stem = entry.name[: -len(".json")] + if not stem.lower().startswith(prefix) or entry.name in seen: + continue + if legacy and not _legacy_state_applies( + entry, + requested, + fail_closed = cancel_markers, + ): + continue + fallback = stem[len(prefix) :] + if fallback: + seen.add(entry.name) + yield _variant_from_state_file(entry, fallback), entry -def iter_variant_manifests(repo_type: RepoType, repo_id: str) -> Iterator[tuple[str, Path]]: +def iter_variant_manifests( + repo_type: RepoType, + repo_id: str, + *, + hub_cache: Optional[str | Path] = None, +) -> Iterator[tuple[str, Path]]: """Yield (variant, manifest_path) for every variant-keyed manifest written for this repo. Used by is_gguf_repo_partial to enumerate all variants present on disk so the all-variants-broken gate can run.""" - yield from _iter_variant_state_files(manifests_dir(), repo_type, repo_id) + yield from _iter_variant_state_files( + manifests_dir(), + repo_type, + repo_id, + hub_cache, + cancel_markers = False, + ) -def iter_variant_markers(repo_type: RepoType, repo_id: str) -> Iterator[tuple[str, Path]]: +def iter_variant_markers( + repo_type: RepoType, + repo_id: str, + *, + hub_cache: Optional[str | Path] = None, +) -> Iterator[tuple[str, Path]]: """Yield (variant, marker_path) for every variant-keyed cancel marker. Companion to iter_variant_manifests: catches variants cancelled before download-start ever wrote a manifest (very early failures).""" - yield from _iter_variant_state_files(cancelled_dir(), repo_type, repo_id) + yield from _iter_variant_state_files( + cancelled_dir(), + repo_type, + repo_id, + hub_cache, + cancel_markers = True, + ) diff --git a/studio/backend/hub/utils/download_registry.py b/studio/backend/hub/utils/download_registry.py index 274038b292..9e2b7d1a6d 100644 --- a/studio/backend/hub/utils/download_registry.py +++ b/studio/backend/hub/utils/download_registry.py @@ -47,7 +47,7 @@ import time import weakref from dataclasses import dataclass, field, replace from pathlib import Path -from typing import Iterator, Literal, Optional +from typing import Callable, Iterator, Literal, Optional from loggers import get_logger @@ -129,6 +129,8 @@ def write_worker_breadcrumb(key: str, pid: int, metadata: Optional["DownloadMeta "cancel_marker_transport": metadata.cancel_marker_transport if metadata is not None else None, + "hub_cache": metadata.hub_cache if metadata is not None else None, + "xet_cache": metadata.xet_cache if metadata is not None else None, } tmp = path.with_name(f".{path.name}.tmp-{pid}") try: @@ -236,6 +238,7 @@ def _settle_orphaned_download( repo_id: Optional[str], variant: Optional[str], transport: Optional[str], + hub_cache: Optional[str] = None, ) -> None: """Persist a cancel marker for a reaped orphan still mid-download so the next launch settles it to a resumable "cancelled" state instead of a phantom-running @@ -251,18 +254,42 @@ def _settle_orphaned_download( return from hub.utils import download_manifest - manifest = download_manifest.read_manifest(repo_type, repo_id, variant) + cache_root = Path(hub_cache) if isinstance(hub_cache, str) and hub_cache else None + + manifest = download_manifest.read_manifest( + repo_type, + repo_id, + variant, + hub_cache = cache_root, + ) if repo_type == "model" and variant and manifest is None: return if manifest is None: - if not has_active_incomplete_blobs(repo_type, repo_id): + if not has_active_incomplete_blobs(repo_type, repo_id, root = cache_root): return else: - if _manifest_verifies_against_active_cache(repo_type, repo_id, manifest): + if _manifest_verifies_against_active_cache( + repo_type, + repo_id, + manifest, + root = cache_root, + ): return - if not _manifest_has_active_incomplete_blobs(repo_type, repo_id, manifest): + if not _manifest_has_active_incomplete_blobs( + repo_type, + repo_id, + manifest, + root = cache_root, + ): return - persist_cancel_marker(repo_type, repo_id, variant, transport, logger = logger) + persist_cancel_marker( + repo_type, + repo_id, + variant, + transport, + hub_cache = hub_cache, + logger = logger, + ) def reap_orphan_workers() -> None: @@ -309,6 +336,7 @@ def reap_orphan_workers() -> None: repo_id, data.get("variant"), data.get("cancel_marker_transport") or data.get("transport"), + data.get("hub_cache"), ) except Exception as exc: logger.debug("Reaper failed for breadcrumb %s: %s", entry, exc) @@ -355,8 +383,13 @@ def _purge_incomplete_blobs( return removed -def _iter_active_snapshot_dirs(repo_type: str, repo_id: str) -> Iterator[Path]: - for entry in iter_active_repo_cache_dirs(repo_type, repo_id): +def _iter_active_snapshot_dirs( + repo_type: str, + repo_id: str, + *, + root: Optional[Path] = None, +) -> Iterator[Path]: + for entry in iter_active_repo_cache_dirs(repo_type, repo_id, root = root): snapshots_dir = entry / "snapshots" if not snapshots_dir.is_dir(): continue @@ -369,24 +402,41 @@ def _iter_active_snapshot_dirs(repo_type: str, repo_id: str) -> Iterator[Path]: yield snapshot -def _manifest_verifies_against_active_cache(repo_type: str, repo_id: str, manifest) -> bool: +def _manifest_verifies_against_active_cache( + repo_type: str, + repo_id: str, + manifest, + *, + root: Optional[Path] = None, +) -> bool: from hub.utils import download_manifest - for snapshot_dir in _iter_active_snapshot_dirs(repo_type, repo_id): + for snapshot_dir in _iter_active_snapshot_dirs(repo_type, repo_id, root = root): if download_manifest.verify_against_disk(manifest, snapshot_dir).ok: return True return False -def _manifest_has_active_incomplete_blobs(repo_type: str, repo_id: str, manifest) -> bool: +def _manifest_has_active_incomplete_blobs( + repo_type: str, + repo_id: str, + manifest, + *, + root: Optional[Path] = None, +) -> bool: if not getattr(manifest, "variant", None): - return has_active_incomplete_blobs(repo_type, repo_id) + return has_active_incomplete_blobs(repo_type, repo_id, root = root) expected_hashes = frozenset( expected.sha256 for expected in manifest.expected_files if expected.sha256 ) if not expected_hashes: - return has_active_incomplete_blobs(repo_type, repo_id) + return has_active_incomplete_blobs(repo_type, repo_id, root = root) return bool( - incomplete_blob_hashes(repo_type, repo_id, active_only = True).intersection(expected_hashes) + incomplete_blob_hashes( + repo_type, + repo_id, + active_only = True, + root = root, + ).intersection(expected_hashes) ) @@ -459,6 +509,7 @@ def prepare_cache_for_transport( only_blob_hashes: Optional[frozenset[str]] = None, companion_blob_hashes: Optional[frozenset[str]] = None, protected_blob_hashes: Optional[frozenset[str]] = None, + root: Optional[Path] = None, ) -> int: """Guarantee any pre-existing ``.incomplete`` blobs are SAFE to resume under *mode*. Returns the number of partial blobs purged for untrusted provenance. @@ -485,14 +536,13 @@ def prepare_cache_for_transport( they are excluded from every purge so a shared companion is never deleted mid-write. - Scope: only the active ``HF_HUB_CACHE`` root is inspected. That suffices for - resume safety because ``snapshot_download`` runs without a ``cache_dir`` - override and so can only read or resume a ``.incomplete`` under this same - active root. Markers are written for the new mode before returning. + Scope: ``root`` selects the cache captured by the caller. It defaults to the + active ``HF_HUB_CACHE`` root for workers that inherit their cache through + the environment. Markers are written for the new mode before returning. """ if mode not in VALID_TRANSPORTS: raise ValueError(f"Invalid transport mode: {mode!r}") - root = hf_cache_root(create = True) + root = hf_cache_root(create = True) if root is None else hf_cache_root(create = True, root = root) if root is None: return 0 target = target_dir_name(repo_type, repo_id) @@ -618,10 +668,11 @@ def incomplete_blob_hashes( repo_id: str, *, active_only: bool = False, + root: Optional[Path] = None, ) -> set[str]: out: set[str] = set() entries = ( - iter_active_repo_cache_dirs(repo_type, repo_id) + iter_active_repo_cache_dirs(repo_type, repo_id, root = root) if active_only else iter_repo_cache_dirs(repo_type, repo_id) ) @@ -638,16 +689,24 @@ def incomplete_blob_hashes( return out -def completed_blob_bytes(repo_type: str, repo_id: str, blob_hashes: frozenset[str]) -> int: - """Sum finalized blob bytes for *blob_hashes* in the active HF cache root. +def completed_blob_bytes( + repo_type: str, + repo_id: str, + blob_hashes: frozenset[str], + *, + root: Optional[Path] = None, +) -> int: + """Sum finalized blob bytes for *blob_hashes* in a single HF cache root. - A worker only writes to the active ``HF_HUB_CACHE`` root, so a baseline must - ignore legacy/default roots that ``snapshot_download`` won't reuse this run. + A worker only writes to its captured ``HF_HUB_CACHE`` root, so a baseline + must be scoped to that root (``root``), not re-resolved to whatever cache is + active now; otherwise a runtime cache switch makes the retry baseline count + bytes from the wrong disk. """ if not blob_hashes: return 0 total = 0 - for entry in iter_active_repo_cache_dirs(repo_type, repo_id): + for entry in iter_active_repo_cache_dirs(repo_type, repo_id, root = root): blobs_dir = entry / "blobs" if not blobs_dir.is_dir(): continue @@ -712,6 +771,8 @@ class DownloadMetadata: # Bytes already complete before this job started; not counted as this run's # progress. completed_baseline_bytes: int = 0 + hub_cache: Optional[str] = None + xet_cache: Optional[str] = None @dataclass(frozen = True) @@ -752,6 +813,7 @@ def persist_cancel_marker( variant: Optional[str], transport: Optional[str], *, + hub_cache: Optional[str] = None, logger = logger, ) -> None: if not repo_type or not repo_id: @@ -763,6 +825,7 @@ def persist_cancel_marker( repo_id, variant, transport = transport, + hub_cache = hub_cache, ): logger.debug("write_cancel_marker returned False for %s", repo_id) except Exception as exc: @@ -971,6 +1034,7 @@ class DownloadRegistry: metadata_to_persist.repo_id, metadata_to_persist.variant, metadata_to_persist.transport, + hub_cache = metadata_to_persist.hub_cache, ) return False @@ -1028,16 +1092,26 @@ class DownloadRegistry: blob_hashes: Optional[frozenset[str]] = None, progress_blob_hashes: Optional[frozenset[str]] = None, completed_baseline_bytes: int = 0, + admission_check: Optional[Callable[[], bool]] = None, generation: Optional[int] = None, replace_active: bool = False, metadata_transport: Optional[str] = None, cancel_marker_transport: Optional[str] = None, + hub_cache: Optional[str] = None, + xet_cache: Optional[str] = None, ) -> tuple[bool, str]: key = normalize_job_key(key) repo = _repo_of_key(key) requested_hashes = blob_hashes or frozenset() requested_progress_hashes = progress_blob_hashes or frozenset() with self._lock: + # Run the final external admission check while the registry lock is + # held, immediately before inspecting and publishing active state. + # The GGUF load path establishes its marker before calling + # its active-job probe, so either this claim observes that marker + # or the load's later probe observes this claim. + if admission_check is not None and not admission_check(): + return False, "admission_blocked" deleting_scopes = self._deleting.get(repo) if deleting_scopes is not None and ( None in deleting_scopes or variant_from_key(key) in deleting_scopes @@ -1098,6 +1172,8 @@ class DownloadRegistry: 0, int(completed_baseline_bytes or 0), ), + hub_cache = hub_cache, + xet_cache = xet_cache, ) if cancel_marker_transport is not None: self._cancel_marker_transports[key] = cancel_marker_transport @@ -1222,6 +1298,23 @@ class DownloadRegistry: ) return refs + def has_active_variant(self, repo_id: str, variant: Optional[str]) -> bool: + """Whether an active model job targets this exact GGUF variant. + + Scans the job table rather than only ``_repo_active`` so an XET-to-HTTP + retry handoff remains visible while it has temporarily released its + active slot. + """ + repo_key = normalize_repo_key(repo_id) + target = (variant or "").strip().lower() or None + with self._lock: + for key, job in self._jobs.items(): + if _repo_of_key(key) != repo_key or job.state not in _ACTIVE_STATES: + continue + if self._active_job_variant_locked(key) == target: + return True + return False + def begin_delete( self, repo_id: str, @@ -1361,6 +1454,7 @@ class DownloadRegistry: metadata.repo_id, metadata.variant, metadata.cancel_marker_transport or metadata.transport, + hub_cache = metadata.hub_cache, ) reaped: list[tuple[str, subprocess.Popen, Optional[DownloadMetadata]]] = [] for key, proc, metadata in live: @@ -1376,6 +1470,7 @@ class DownloadRegistry: metadata.repo_id, metadata.variant, metadata.cancel_marker_transport or metadata.transport, + hub_cache = metadata.hub_cache, ) continue reaped.append((key, proc, metadata)) @@ -1396,6 +1491,7 @@ class DownloadRegistry: metadata.repo_id, metadata.variant, metadata.cancel_marker_transport or metadata.transport, + hub_cache = metadata.hub_cache, ) diff --git a/studio/backend/hub/utils/gguf.py b/studio/backend/hub/utils/gguf.py index 2e3de125f1..eb768db5d6 100644 --- a/studio/backend/hub/utils/gguf.py +++ b/studio/backend/hub/utils/gguf.py @@ -253,11 +253,16 @@ def _env_offline() -> bool: ) or os.environ.get("TRANSFORMERS_OFFLINE", "").lower() in ("1", "true", "yes") -def iter_hf_cache_snapshots(repo_id: str): - from hub.utils.hf_cache_state import iter_repo_cache_dirs +def iter_hf_cache_snapshots(repo_id: str, root: Optional[Path] = None): + from hub.utils.hf_cache_state import iter_active_repo_cache_dirs, iter_repo_cache_dirs snapshots: list[Path] = [] - for repo_dir in iter_repo_cache_dirs("model", repo_id): + repo_dirs = ( + iter_active_repo_cache_dirs("model", repo_id, root = root) + if root is not None + else iter_repo_cache_dirs("model", repo_id) + ) + for repo_dir in repo_dirs: snapshots_dir = repo_dir / "snapshots" if not snapshots_dir.is_dir(): continue @@ -276,12 +281,17 @@ def iter_hf_cache_snapshots(repo_id: str): yield from snapshots -def list_empty_gguf_variant_dirs(repo_id: str) -> set[str]: +def list_empty_gguf_variant_dirs(repo_id: str, root: Optional[Path] = None) -> set[str]: """Quant labels present only as an EMPTY snapshot ``/`` folder (an interrupted split download); a quant with shards in any snapshot is excluded.""" empty: dict[str, str] = {} nonempty: set[str] = set() - for snapshot in iter_hf_cache_snapshots(repo_id): + snapshots = ( + iter_hf_cache_snapshots(repo_id, root = root) + if root is not None + else iter_hf_cache_snapshots(repo_id) + ) + for snapshot in snapshots: try: entries = list(snapshot.iterdir()) except OSError: @@ -303,8 +313,15 @@ def list_empty_gguf_variant_dirs(repo_id: str) -> set[str]: return {label for key, label in empty.items() if key not in nonempty} -def list_gguf_variants_from_hf_cache(repo_id: str) -> Optional[tuple[list[GgufVariantInfo], bool]]: - for snapshot in iter_hf_cache_snapshots(repo_id): +def list_gguf_variants_from_hf_cache( + repo_id: str, root: Optional[Path] = None +) -> Optional[tuple[list[GgufVariantInfo], bool]]: + snapshots = ( + iter_hf_cache_snapshots(repo_id, root = root) + if root is not None + else iter_hf_cache_snapshots(repo_id) + ) + for snapshot in snapshots: variants, has_vision = list_local_gguf_variants(str(snapshot)) if variants or has_vision: return variants, has_vision @@ -312,7 +329,7 @@ def list_gguf_variants_from_hf_cache(repo_id: str) -> Optional[tuple[list[GgufVa def list_partial_gguf_variants_from_state( - repo_id: str, + repo_id: str, hub_cache: Optional[Path] = None ) -> Optional[tuple[list[GgufVariantInfo], bool]]: """Reconstruct GGUF variants from download manifests/markers alone. @@ -328,10 +345,26 @@ def list_partial_gguf_variants_from_state( # original-casing label over a lowercased cancel marker for the same variant. seen: set[str] = set() ordered: list[str] = [] - for source in ( - download_manifest.iter_variant_manifests("model", repo_id), - download_manifest.iter_variant_markers("model", repo_id), - ): + sources = ( + ( + download_manifest.iter_variant_manifests("model", repo_id), + download_manifest.iter_variant_markers("model", repo_id), + ) + if hub_cache is None + else ( + download_manifest.iter_variant_manifests( + "model", + repo_id, + hub_cache = hub_cache, + ), + download_manifest.iter_variant_markers( + "model", + repo_id, + hub_cache = hub_cache, + ), + ) + ) + for source in sources: for variant, _path in source: key = variant.lower() if key not in seen: @@ -343,7 +376,16 @@ def list_partial_gguf_variants_from_state( variants: list[GgufVariantInfo] = [] has_vision = False for variant in ordered: - manifest = download_manifest.read_manifest("model", repo_id, variant) + manifest = ( + download_manifest.read_manifest("model", repo_id, variant) + if hub_cache is None + else download_manifest.read_manifest( + "model", + repo_id, + variant, + hub_cache = hub_cache, + ) + ) main_filename: Optional[str] = None size_bytes = 0 companion_bytes = 0 diff --git a/studio/backend/hub/utils/hf_cache_state.py b/studio/backend/hub/utils/hf_cache_state.py index 22c948b683..49a28c813c 100644 --- a/studio/backend/hub/utils/hf_cache_state.py +++ b/studio/backend/hub/utils/hf_cache_state.py @@ -29,12 +29,10 @@ def _safe_is_dir(path: Path) -> bool: return False -def hf_cache_root(*, create: bool = False) -> Optional[Path]: - try: - from huggingface_hub import constants as hf_constants - except ImportError: - return None - root = Path(hf_constants.HF_HUB_CACHE) +def hf_cache_root(*, create: bool = False, root: Optional[Path] = None) -> Optional[Path]: + from utils.hf_cache_settings import get_hf_cache_paths + + root = root or get_hf_cache_paths().hub_cache if create: try: root.mkdir(parents = True, exist_ok = True) @@ -46,6 +44,7 @@ def hf_cache_root(*, create: bool = False) -> Optional[Path]: def hf_cache_roots() -> list[Path]: from hub.utils.paths import hf_default_cache_dir, legacy_hf_cache_dir + from utils.hf_cache_settings import known_hf_hub_caches roots: list[Path] = [] seen: set[str] = set() @@ -62,7 +61,8 @@ def hf_cache_roots() -> list[Path]: seen.add(key) roots.append(path) - _add(hf_cache_root()) + for configured in known_hf_hub_caches(): + _add(configured) _add(legacy_hf_cache_dir()) _add(hf_default_cache_dir()) return roots @@ -181,12 +181,22 @@ def iter_repo_cache_dirs(repo_type: str, repo_id: str) -> Iterator[Path]: continue -def iter_destructive_repo_cache_dirs(repo_type: str, repo_id: str) -> Iterator[Path]: +def iter_destructive_repo_cache_dirs( + repo_type: str, + repo_id: str, + *, + root: Optional[Path] = None, +) -> Iterator[Path]: target = repo_cache_dir_name(repo_type, repo_id) folded_target = target.lower() - for root in hf_cache_roots(): + if root is not None: + scoped = hf_cache_root(root = root) + bases = [scoped] if scoped is not None else [] + else: + bases = hf_cache_roots() + for base in bases: try: - entries = [entry for entry in root.iterdir() if entry.name.lower() == folded_target] + entries = [entry for entry in base.iterdir() if entry.name.lower() == folded_target] except OSError: continue matched_names = resolve_destructive_case_matches( @@ -200,8 +210,13 @@ def iter_destructive_repo_cache_dirs(repo_type: str, repo_id: str) -> Iterator[P yield entry -def iter_active_repo_cache_dirs(repo_type: str, repo_id: str) -> Iterator[Path]: - root = hf_cache_root() +def iter_active_repo_cache_dirs( + repo_type: str, + repo_id: str, + *, + root: Optional[Path] = None, +) -> Iterator[Path]: + root = hf_cache_root(root = root) if root is None: return target = target_dir_name(repo_type, repo_id) @@ -218,12 +233,13 @@ def preferred_repo_cache_dirs( repo_id: str, *, force_active: bool = False, + active_root: Optional[Path] = None, ) -> list[Path]: - active_entries = list(iter_active_repo_cache_dirs(repo_type, repo_id)) + active_entries = list(iter_active_repo_cache_dirs(repo_type, repo_id, root = active_root)) if active_entries: return active_entries if force_active: - root = hf_cache_root() + root = hf_cache_root(root = active_root) if root is not None: canonical = repo_cache_dir_name(repo_type, repo_id) return [root / canonical] @@ -237,8 +253,13 @@ def has_incomplete_blobs(repo_type: str, repo_id: str) -> bool: return False -def has_active_incomplete_blobs(repo_type: str, repo_id: str) -> bool: - for entry in iter_active_repo_cache_dirs(repo_type, repo_id): +def has_active_incomplete_blobs( + repo_type: str, + repo_id: str, + *, + root: Optional[Path] = None, +) -> bool: + for entry in iter_active_repo_cache_dirs(repo_type, repo_id, root = root): if repo_cache_dir_has_incomplete_blobs(entry): return True return False @@ -273,9 +294,14 @@ def _prune_empty_dirs(root: Path) -> bool: return removed -def purge_partial_repo(repo_type: str, repo_id: str) -> bool: +def purge_partial_repo( + repo_type: str, + repo_id: str, + *, + root: Optional[Path] = None, +) -> bool: removed = False - for entry in iter_destructive_repo_cache_dirs(repo_type, repo_id): + for entry in iter_destructive_repo_cache_dirs(repo_type, repo_id, root = root): blobs_dir = entry / "blobs" if blobs_dir.is_dir(): for blob in blobs_dir.iterdir(): @@ -290,9 +316,14 @@ def purge_partial_repo(repo_type: str, repo_id: str) -> bool: return removed -def purge_repo_cache_dirs(repo_type: str, repo_id: str) -> bool: +def purge_repo_cache_dirs( + repo_type: str, + repo_id: str, + *, + root: Optional[Path] = None, +) -> bool: removed = False - for entry in iter_destructive_repo_cache_dirs(repo_type, repo_id): + for entry in iter_destructive_repo_cache_dirs(repo_type, repo_id, root = root): try: if entry.is_symlink() or not entry.is_dir(): continue @@ -301,3 +332,59 @@ def purge_repo_cache_dirs(repo_type: str, repo_id: str) -> bool: except FileNotFoundError: continue return removed + + +def scoped_delete_root(repo_type: str, repo_id: str, cache_path: Optional[str]) -> Optional[Path]: + """Resolve the single cache root a delete of this repo may touch. + + Returns the active hub cache when *cache_path* is falsy, the owning cache + root when *cache_path* points inside a known cache, or ``None`` when + *cache_path* is set but not inside any known cache (caller should reject). + This keeps a delete of one inventory row from removing copies in other, + previously selected caches. + """ + from utils.hf_cache_settings import get_hf_cache_paths + + if not cache_path: + return Path(get_hf_cache_paths().hub_cache).resolve(strict = False) + try: + resolved = Path(cache_path).expanduser().resolve(strict = False) + except (OSError, RuntimeError, ValueError): + return None + expected = repo_cache_dir_name(repo_type, repo_id).lower() + repo_dir = next( + ( + candidate + for candidate in (resolved, *resolved.parents) + if candidate.name.lower() == expected + ), + None, + ) + if repo_dir is None: + return None + allowed = {r.resolve(strict = False) for r in hf_cache_roots()} + root = repo_dir.parent.resolve(strict = False) + return root if root in allowed else None + + +def resolve_delete_target_root( + repo_type: str, repo_id: str, cache_path: Optional[str], owner_roots +) -> Optional[Path]: + """Pick the single cache root a delete of this repo should target. + + An explicit *cache_path* wins (``None`` when it is not a known cache, so the + caller can reject it). Otherwise prefer the active cache when it holds a + copy, else the sole cache that does -- so a model that lives only in a + previously selected cache stays deletable while other caches are untouched. + """ + if cache_path: + return scoped_delete_root(repo_type, repo_id, cache_path) + from utils.hf_cache_settings import get_hf_cache_paths + + active = Path(get_hf_cache_paths().hub_cache).resolve(strict = False) + roots = list(owner_roots) + if active in roots: + return active + if len(roots) == 1: + return roots[0] + return active diff --git a/studio/backend/hub/utils/inventory_scan.py b/studio/backend/hub/utils/inventory_scan.py index 57ad7f6655..058fdf9b65 100644 --- a/studio/backend/hub/utils/inventory_scan.py +++ b/studio/backend/hub/utils/inventory_scan.py @@ -36,7 +36,7 @@ from hub.utils.state_dir import RepoType from hub.utils.hf_cache_state import ( INCOMPLETE_SUFFIX, has_incomplete_blobs, - hf_cache_root, + hf_cache_roots, iter_repo_cache_dirs, latest_snapshot_dir, repo_cache_dir_has_incomplete_blobs, @@ -127,33 +127,13 @@ def all_hf_cache_scans() -> list: def _compute_all_hf_cache_scans() -> list: from huggingface_hub import scan_cache_dir - from hub.utils.paths import legacy_hf_cache_dir, hf_default_cache_dir scans: list = [] - seen: set[str] = set() - try: - from huggingface_hub.constants import HF_HUB_CACHE - - active = Path(HF_HUB_CACHE).resolve() - seen.add(str(active)) - if active.is_dir(): - scans.append(scan_cache_dir()) - except Exception as exc: - logger.warning("Could not scan active HF cache: %s", exc) - - for extra_fn in (legacy_hf_cache_dir, hf_default_cache_dir): + for cache_root in hf_cache_roots(): try: - extra = extra_fn() - # is_dir()/resolve() can raise on an inaccessible path; skip it. - if not extra.is_dir(): - continue - resolved = str(extra.resolve()) - if resolved in seen: - continue - seen.add(resolved) - scans.append(scan_cache_dir(cache_dir = str(extra))) + scans.append(scan_cache_dir(cache_dir = str(cache_root))) except Exception as exc: - logger.warning("Could not scan HF cache %s: %s", extra_fn.__name__, exc) + logger.warning("Could not scan HF cache %s: %s", cache_root, exc) return scans @@ -224,16 +204,8 @@ def _compose_partial(*signals: Callable[[], bool]) -> bool: return any(signal() for signal in signals) -def _state_applies_to_repo_cache_dir(repo_cache_dir: Optional[Path]) -> bool: - if repo_cache_dir is None: - return True - root = hf_cache_root() - if root is None: - return False - try: - return repo_cache_dir.resolve().parent == root.resolve() - except OSError: - return False +def _hub_cache_for_repo_dir(repo_cache_dir: Optional[Path]) -> Optional[Path]: + return repo_cache_dir.parent if repo_cache_dir is not None else None def _legacy_partial( @@ -285,12 +257,24 @@ def _repo_cache_dir_has_non_gguf_broken_snapshot_symlinks(repo_cache_dir: Path) return False -def _gguf_variant_manifest_blob_hashes(repo_id: str) -> frozenset[str]: +def _gguf_variant_manifest_blob_hashes( + repo_id: str, repo_cache_dir: Optional[Path] = None +) -> frozenset[str]: from hub.utils import download_manifest hashes: set[str] = set() - for variant, _path in download_manifest.iter_variant_manifests("model", repo_id): - manifest = download_manifest.read_manifest("model", repo_id, variant) + hub_cache = _hub_cache_for_repo_dir(repo_cache_dir) + for variant, _path in download_manifest.iter_variant_manifests( + "model", + repo_id, + hub_cache = hub_cache, + ): + manifest = download_manifest.read_manifest( + "model", + repo_id, + variant, + hub_cache = hub_cache, + ) if manifest is None: continue for expected in manifest.expected_files: @@ -315,7 +299,7 @@ def _snapshot_legacy_partial( ) -> bool: if repo_type != "model": return _legacy_partial(repo_type, repo_id, repo_cache_dir) - ignored_hashes = _gguf_variant_manifest_blob_hashes(repo_id) + ignored_hashes = _gguf_variant_manifest_blob_hashes(repo_id, repo_cache_dir) if repo_cache_dir is not None: return _repo_cache_dir_has_snapshot_legacy_partial( repo_cache_dir, @@ -375,9 +359,12 @@ def _manifest_partial( ) -> bool: from hub.utils import download_manifest - if not _state_applies_to_repo_cache_dir(repo_cache_dir): - return False - manifest = download_manifest.read_manifest(repo_type, repo_id, variant) + manifest = download_manifest.read_manifest( + repo_type, + repo_id, + variant, + hub_cache = _hub_cache_for_repo_dir(repo_cache_dir), + ) if manifest is None: return False resolved = ( @@ -452,10 +439,13 @@ def is_snapshot_partial( A manifest without a resolvable snapshot is partial: the worker got far enough to record expectations but did not leave a usable snapshot.""" from hub.utils import download_manifest - - state_applies = _state_applies_to_repo_cache_dir(repo_cache_dir) return _compose_partial( - lambda: state_applies and download_manifest.has_cancel_marker(repo_type, repo_id, None), + lambda: download_manifest.has_cancel_marker( + repo_type, + repo_id, + None, + hub_cache = _hub_cache_for_repo_dir(repo_cache_dir), + ), lambda: _snapshot_legacy_partial(repo_type, repo_id, repo_cache_dir), lambda: _manifest_partial( repo_type, @@ -484,10 +474,13 @@ def is_variant_partial( caller is checking many variants of the same repo (see is_gguf_repo_partial for that usage).""" from hub.utils import download_manifest - - state_applies = _state_applies_to_repo_cache_dir(repo_cache_dir) return _compose_partial( - lambda: state_applies and download_manifest.has_cancel_marker("model", repo_id, variant), + lambda: download_manifest.has_cancel_marker( + "model", + repo_id, + variant, + hub_cache = _hub_cache_for_repo_dir(repo_cache_dir), + ), lambda: bool( incomplete_blob_hashes and variant_blob_hashes @@ -526,22 +519,38 @@ def is_gguf_repo_partial(repo_id: str, repo_cache_dir: Optional[Path] = None) -> from hub.utils import download_manifest has_legacy_partial = _legacy_partial("model", repo_id, repo_cache_dir) - state_applies = _state_applies_to_repo_cache_dir(repo_cache_dir) snapshot_dir = resolve_snapshot_dir_for_scan( "model", repo_id, repo_cache_dir, ) variants: set[str] = set(_completed_gguf_variants(snapshot_dir)) - if state_applies: - for variant, _path in download_manifest.iter_variant_manifests( - "model", - repo_id, + hub_cache = _hub_cache_for_repo_dir(repo_cache_dir) + for variant, _path in download_manifest.iter_variant_manifests( + "model", + repo_id, + hub_cache = hub_cache, + ): + if ( + download_manifest.read_manifest( + "model", + repo_id, + variant, + hub_cache = hub_cache, + ) + is not None ): variants.add(variant) - for variant, _path in download_manifest.iter_variant_markers( + for variant, _path in download_manifest.iter_variant_markers( + "model", + repo_id, + hub_cache = hub_cache, + ): + if download_manifest.has_cancel_marker( "model", repo_id, + variant, + hub_cache = hub_cache, ): variants.add(variant) if not variants: @@ -576,14 +585,19 @@ def partial_transport_for( available.""" from hub.utils import download_manifest - if not _state_applies_to_repo_cache_dir(repo_cache_dir): - return None + hub_cache = _hub_cache_for_repo_dir(repo_cache_dir) marker_transport = download_manifest.read_cancel_marker_transport( repo_type, repo_id, variant, + hub_cache = hub_cache, ) if marker_transport is not None: return marker_transport - manifest = download_manifest.read_manifest(repo_type, repo_id, variant) + manifest = download_manifest.read_manifest( + repo_type, + repo_id, + variant, + hub_cache = hub_cache, + ) return manifest.transport if manifest is not None else None diff --git a/studio/backend/hub/utils/paths.py b/studio/backend/hub/utils/paths.py index 5435202565..81621edcf9 100644 --- a/studio/backend/hub/utils/paths.py +++ b/studio/backend/hub/utils/paths.py @@ -277,12 +277,8 @@ def _memo_drop(memo_key: tuple[str, str]) -> None: def _hf_hub_cache_dir() -> Path: - try: - from huggingface_hub.constants import HF_HUB_CACHE - return Path(HF_HUB_CACHE) - except Exception as exc: - logger.debug("Could not read huggingface_hub HF_HUB_CACHE, using default: %s", exc) - return Path.home() / ".cache" / "huggingface" / "hub" + from utils.hf_cache_settings import get_hf_cache_paths + return get_hf_cache_paths().hub_cache def _hf_hub_cache_dirs() -> list[Path]: @@ -300,7 +296,10 @@ def _hf_hub_cache_dirs() -> list[Path]: seen.add(key) roots.append(resolved) - _add(_hf_hub_cache_dir()) + from utils.hf_cache_settings import known_hf_hub_caches + + for configured in known_hf_hub_caches(): + _add(configured) try: _add(legacy_hf_cache_dir()) _add(hf_default_cache_dir()) diff --git a/studio/backend/hub/utils/state_dir.py b/studio/backend/hub/utils/state_dir.py index 183e934724..4650b97381 100644 --- a/studio/backend/hub/utils/state_dir.py +++ b/studio/backend/hub/utils/state_dir.py @@ -3,14 +3,15 @@ """Filesystem layout for Hub download state. -State directory sits beside HF's cache (under Studio's own cache root) +State directory sits beside HF's cache (under Unsloth's own cache root) so it survives ``huggingface-cli delete-cache`` and any other HF-side cache lifecycle. Two subdirectories: /hub-state/ - manifests/ .json per-download expected-files manifest - cancelled/ .json per-download cancel marker + manifests/cache-/.json expected-files manifest + cancelled/cache-/.json cancel marker +The cache digest isolates state for the same repo across selectable Hub caches. The ```` mirrors HF's cache dir naming while the resulting manifest, cancel-marker, and atomic-write temp filenames fit common filesystem basename limits. Very long repo IDs use a stable hash in the state key: @@ -29,6 +30,7 @@ configuration failure. from __future__ import annotations import hashlib +import os import re from pathlib import Path from typing import Literal, Optional, get_args @@ -55,6 +57,7 @@ _STATE_EXTENSION = ".json" # _atomic_write_json writes "..tmp-<8hex>" beside the final file. _ATOMIC_WRITE_TMP_OVERHEAD = len(".") + len(".tmp-") + 8 _MAX_VARIANT_FRAGMENT_LENGTH = 64 +_CACHE_SCOPE_DIGEST_LENGTH = 32 def state_root() -> Optional[Path]: @@ -130,13 +133,32 @@ def _entry_key(repo_type: RepoType, repo_id: str, variant: Optional[str]) -> str return f"{variant_filename_prefix(repo_type, repo_id)}{variant_fragment}" +def _cache_scope(parent: Path, hub_cache: Optional[str | Path]) -> Optional[Path]: + if hub_cache is None: + return parent + normalized = os.path.normcase(str(Path(hub_cache).expanduser())) + digest = hashlib.sha256(normalized.encode("utf-8")).hexdigest()[:_CACHE_SCOPE_DIGEST_LENGTH] + scoped = parent / f"cache-{digest}" + try: + scoped.mkdir(parents = True, exist_ok = True) + except OSError as exc: + logger.debug("Could not create cache-scoped Hub state dir %s: %s", scoped, exc) + return None + return scoped + + def manifest_path( repo_type: RepoType, repo_id: str, variant: Optional[str] = None, + *, + hub_cache: Optional[str | Path] = None, ) -> Optional[Path]: """Path to the manifest file for this triple. May or may not exist.""" parent = _subdir(_MANIFESTS_SUBDIR) + if parent is None: + return None + parent = _cache_scope(parent, hub_cache) if parent is None: return None return parent / f"{_entry_key(repo_type, repo_id, variant)}.json" @@ -146,9 +168,14 @@ def marker_path( repo_type: RepoType, repo_id: str, variant: Optional[str] = None, + *, + hub_cache: Optional[str | Path] = None, ) -> Optional[Path]: """Path to the cancel-marker file for this triple. May or may not exist.""" parent = _subdir(_CANCELLED_SUBDIR) + if parent is None: + return None + parent = _cache_scope(parent, hub_cache) if parent is None: return None return parent / f"{_entry_key(repo_type, repo_id, variant)}.json" diff --git a/studio/backend/hub/workers/hf_download.py b/studio/backend/hub/workers/hf_download.py index e45357d311..9ff394b009 100644 --- a/studio/backend/hub/workers/hf_download.py +++ b/studio/backend/hub/workers/hf_download.py @@ -661,6 +661,7 @@ def _download_gguf_variant(repo_id: str, variant: str, hf_token: str | None, mod variant, plan.main_hashes, hf_token, + hub_cache = Path(snapshot_path).parents[2], ) except Exception as e: print( diff --git a/studio/backend/main.py b/studio/backend/main.py index e64048dc00..5af25efa74 100644 --- a/studio/backend/main.py +++ b/studio/backend/main.py @@ -19,7 +19,7 @@ os.environ["PYTHONWARNINGS"] = "ignore" # Pin GPU index ordering to PCI bus id before any torch import creates a CUDA # context. Without this, torch/CUDA default to FASTEST_FIRST while nvidia-smi -# (and Studio's VRAM probes) use PCI-bus order, so a GPU index chosen from +# (and Unsloth's VRAM probes) use PCI-bus order, so a GPU index chosen from # nvidia-smi data can resolve to a different physical card via # CUDA_VISIBLE_DEVICES. setdefault so an explicit user override wins. See # utils/hardware/hardware.py for the full rationale; set here too so the entry @@ -93,7 +93,7 @@ if sys.platform == "win32": # ── Windows AMD ROCm: make hipInfo.exe resolvable for subprocess probes ── # bitsandbytes' get_rocm_gpu_arch() runs `hipinfo.exe` via PATH at import # time; the AMD torch wheel ships it in the venv Scripts dir, which is on - # PATH only when the venv is activated -- Studio launches python directly. + # PATH only when the venv is activated -- Unsloth launches python directly. # Without this, every bitsandbytes import logs a scary (but harmless) # "Could not detect ROCm GPU architecture: [WinError 2]" ERROR + WARNING. # Gated on the file existing: only AMD ROCm wheels ship hipInfo.exe, so @@ -252,7 +252,7 @@ def _read_studio_install_id() -> str: Returns "" when absent or not a 64-char lowercase-hex token; then /api/health emits "" and the launcher accepts any healthy backend. - Carries no install-path info (matters when Studio runs -H 0.0.0.0).""" + Carries no install-path info (matters when Unsloth runs -H 0.0.0.0).""" try: token = (_STUDIO_ROOT_RESOLVED / "share" / "studio_install_id").read_text().strip() except (OSError, ValueError): @@ -289,6 +289,7 @@ from fastapi import Depends, FastAPI, HTTPException, Query, Request from fastapi.middleware.cors import CORSMiddleware from fastapi.staticfiles import StaticFiles from fastapi.responses import FileResponse, HTMLResponse, Response +from starlette.middleware.gzip import GZipMiddleware from pathlib import Path from datetime import datetime @@ -308,11 +309,14 @@ from routes import ( training_router, ) from routes.llama import router as llama_router +from routes.whisper import router as whisper_router from routes.preview import router as preview_router from hub.routes import ( inventory_router as hub_inventory_router, datasets_router as hub_datasets_router, + token_router as hub_token_router, ) +from picker.routes import templates_router as picker_templates_router from hub.schemas.downloads import TransportCapabilities from hub.utils.download_registry import ( get_download_transport_capabilities, @@ -434,7 +438,11 @@ def _run_llama_cpp_startup_probes(app: FastAPI) -> None: import structlog as _structlog _log = _structlog.get_logger(__name__) - if _caps.get("found") and not _caps.get("supports_mtp"): + if ( + _caps.get("found") + and not _caps.get("supports_mtp") + and not _caps.get("mtp_probe_inconclusive") + ): _msg = ( "llama.cpp prebuilt lacks MTP support " "(--spec-type mtp/draft-mtp). Run `unsloth studio update`. " @@ -547,8 +555,9 @@ async def lifespan(app: FastAPI): threading.Thread(target = _warm_rag_embedder, daemon = True, name = "rag-embedder-warm").start() # Idle auto-unload loop (no-op unless the OpenAI auto-unload TTL is set). - from core.inference.llama_keepwarm import idle_unload_loop + from core.inference.llama_keepwarm import idle_unload_loop, sweep_slot_save_dir + sweep_slot_save_dir() app.state.idle_unload_task = asyncio.create_task(idle_unload_loop()) # Initialize RSA key pair for API key encryption (external providers). @@ -573,7 +582,7 @@ async def lifespan(app: FastAPI): print("DEFAULT ADMIN ACCOUNT CREATED") print(f" username: {storage.DEFAULT_ADMIN_USERNAME}") print(f" password saved to: {bootstrap_path}") - print(" Open the Studio UI to sign in and change it.") + print(" Open the Unsloth UI to sign in and change it.") print("=" * 60 + "\n") else: app.state.bootstrap_password = ( @@ -612,6 +621,22 @@ app = FastAPI( lifespan = lifespan, ) +# The MCP surface is opt-in because it can start GPU jobs and write model +# artifacts. Mount it only when explicitly enabled by the Unsloth process. +if os.environ.get("UNSLOTH_STUDIO_ENABLE_MCP") == "1": + from fastmcp.utilities.lifespan import combine_lifespans + + from mcp_server import BearerTokenMiddleware, create_studio_mcp + + _studio_mcp_app = create_studio_mcp().http_app(path = "/") + _studio_mcp_lifespan = _studio_mcp_app.lifespan + _mcp_token = os.environ.get("UNSLOTH_STUDIO_MCP_TOKEN") + if not _mcp_token: + raise RuntimeError("UNSLOTH_STUDIO_MCP_TOKEN is required when MCP is enabled") + _studio_mcp_app = BearerTokenMiddleware(_studio_mcp_app, _mcp_token) + app.router.lifespan_context = combine_lifespans(lifespan, _studio_mcp_lifespan) + app.mount("/mcp", _studio_mcp_app) + from loggers.config import LogConfig from loggers.handlers import LoggingMiddleware @@ -735,6 +760,8 @@ app.add_middleware(SecurityHeadersMiddleware) # headroom; non-upload routes keep the default body cap. import json as _json_for_413 # noqa: E402 from utils.upload_limits import ( # noqa: E402 + STT_AUDIO_JSON_MAX_BYTES, + STT_AUDIO_RAW_MAX_BYTES, UNSTRUCTURED_RECIPE_UPLOAD_MAX_BYTES, default_request_body_limit_bytes, upload_request_limit_bytes, @@ -745,6 +772,7 @@ _BODY_PROTECTED_PREFIXES = ( "/v1/completions", "/p/", "/api/inference", + "/api/picker", "/api/data-recipe", "/api/datasets", "/api/hub", @@ -752,6 +780,7 @@ _BODY_PROTECTED_PREFIXES = ( "/api/settings", "/api/train", "/api/export", + "/mcp", ) _DATASET_UPLOAD_PASSTHROUGH_PREFIX = "/api/datasets/upload" _DATA_RECIPE_UNSTRUCTURED_UPLOAD_PASSTHROUGH_PREFIX = ( @@ -771,6 +800,14 @@ def _get_upload_passthrough_request_max_bytes(path: str) -> int: return default_request_body_limit_bytes() +def _get_request_body_max_bytes(path: str) -> int: + if path.startswith("/api/inference/audio/transcribe/raw"): + return STT_AUDIO_RAW_MAX_BYTES + if path.startswith("/api/inference/audio/transcribe"): + return STT_AUDIO_JSON_MAX_BYTES + return default_request_body_limit_bytes() + + async def _send_411(send) -> None: payload = _json_for_413.dumps( {"detail": "Content-Length required for upload requests."}, @@ -813,12 +850,14 @@ class MaxBodyMiddleware: app, max_bytes_getter, protected_prefixes: tuple, + request_max_bytes_getter = None, upload_passthrough_prefixes: tuple = (), upload_passthrough_max_bytes_getter = None, ): self.app = app self.max_bytes_getter = max_bytes_getter self.protected_prefixes = protected_prefixes + self.request_max_bytes_getter = request_max_bytes_getter self.upload_passthrough_prefixes = upload_passthrough_prefixes self.upload_passthrough_max_bytes_getter = upload_passthrough_max_bytes_getter @@ -835,6 +874,14 @@ class MaxBodyMiddleware: except Exception: return int(self.max_bytes_getter()) + def _request_max_bytes(self, path: str) -> int: + if self.request_max_bytes_getter is None: + return int(self.max_bytes_getter()) + try: + return int(self.request_max_bytes_getter(path)) + except Exception: + return int(self.max_bytes_getter()) + async def __call__(self, scope, receive, send): if scope["type"] != "http": await self.app(scope, receive, send) @@ -847,7 +894,7 @@ class MaxBodyMiddleware: await self.app(scope, receive, send) return - max_bytes = int(self.max_bytes_getter()) + max_bytes = self._request_max_bytes(path) declared = None for name, value in scope.get("headers", []): if name == b"content-length": @@ -912,6 +959,7 @@ app.add_middleware( MaxBodyMiddleware, max_bytes_getter = default_request_body_limit_bytes, protected_prefixes = _BODY_PROTECTED_PREFIXES, + request_max_bytes_getter = _get_request_body_max_bytes, upload_passthrough_prefixes = _BODY_UPLOAD_PASSTHROUGH_PREFIXES, upload_passthrough_max_bytes_getter = _get_upload_passthrough_request_max_bytes, ) @@ -956,7 +1004,7 @@ app.include_router(training_router, prefix = "/api/train", tags = ["training"]) app.include_router(models_router, prefix = "/api/models", tags = ["models"]) app.include_router(chat_history_router, prefix = "/api/chat", tags = ["chat"]) app.include_router(inference_router, prefix = "/api/inference", tags = ["inference"]) -# Studio-only inference endpoints (cancel, etc.) are NOT exposed on the /v1 +# Unsloth-only inference endpoints (cancel, etc.) are NOT exposed on the /v1 # OpenAI-compat prefix below. app.include_router(inference_studio_router, prefix = "/api/inference", tags = ["inference"]) @@ -970,11 +1018,14 @@ app.include_router(prompts_router, prefix = "/api/prompts", tags = ["prompts"]) app.include_router(datasets_router, prefix = "/api/datasets", tags = ["datasets"]) app.include_router(data_recipe_router, prefix = "/api/data-recipe", tags = ["data-recipe"]) app.include_router(llama_router, prefix = "/api/llama", tags = ["llama"]) +app.include_router(whisper_router, prefix = "/api/whisper", tags = ["whisper"]) app.include_router(export_router, prefix = "/api/export", tags = ["export"]) app.include_router(rag_router, prefix = "/api/rag", tags = ["rag"]) app.include_router(training_history_router, prefix = "/api/train", tags = ["training-history"]) app.include_router(hub_inventory_router, prefix = "/api/hub", tags = ["hub"]) app.include_router(hub_datasets_router, prefix = "/api/hub/datasets", tags = ["hub"]) +app.include_router(picker_templates_router, prefix = "/api/picker", tags = ["picker"]) +app.include_router(hub_token_router, prefix = "/api/hub", tags = ["hub"]) # Re-wrap client-error responses on the /v1/* surface into OpenAI/Anthropic # error envelopes; non-/v1 paths keep FastAPI's default {"detail": ...} shape. @@ -1063,7 +1114,7 @@ def studio_install_source(_current_subject: str = Depends(get_current_subject)): @app.get("/api/studio/update-status") def studio_update_status(_current_subject: str = Depends(get_current_subject)): - """Return source-aware manual update status for browser-served Studio.""" + """Return source-aware manual update status for browser-served Unsloth.""" return get_studio_update_status(UNSLOTH_VERSION) @@ -1131,17 +1182,35 @@ def _get_cached_system_gpu_info(logger) -> dict[str, Any]: util = util_devices.get(idx, {}) total_vram = util.get("vram_total_gb") or dev.get("memory_total_gb") or 0 - used_vram = util.get("vram_used_gb") or 0 + # Keep None (usage unknown, e.g. Windows ROCm perf counter) so the UI + # shows unknown, not a fabricated 0 used / full free. + used_vram = util.get("vram_used_gb") enriched_dev = dict(dev) enriched_dev["vram_used_gb"] = used_vram - enriched_dev["vram_free_gb"] = round(total_vram - used_vram, 2) if total_vram else 0 + enriched_dev["vram_free_gb"] = ( + round(total_vram - used_vram, 2) if total_vram and used_vram is not None else None + ) enriched_dev["vram_utilization_pct"] = util.get("vram_utilization_pct") enriched_devices.append(enriched_dev) + # Whether GGUF loads accept an explicit gpu_ids pick: /load and + # /validate 400 picks on XPU hosts (no visibility mask speaks torch-xpu + # ordinals) and on Vulkan-only builds (--device pins ggml's own + # ordinals), so the picker must not offer them. + try: + from core.inference.llama_cpp import LlamaCppBackend + from utils.hardware import DeviceType, get_device + gpu_ids_supported = ( + get_device() != DeviceType.XPU and not LlamaCppBackend._is_vulkan_backend() + ) + except Exception as e: + logger.debug(f"Could not resolve gpu_ids support: {e}") + gpu_ids_supported = True gpu_info = { "available": visibility_info.get("available", False), "devices": enriched_devices, + "gguf_gpu_ids_supported": gpu_ids_supported, } _system_gpu_cache = (time.monotonic(), gpu_info) return gpu_info @@ -1471,6 +1540,34 @@ def _should_inject_bootstrap(request: Request) -> bool: return _is_local_bootstrap_request(request) +_IMMUTABLE_ASSET_CACHE_CONTROL = "public, max-age=31536000, immutable" + + +class ImmutableStaticFiles(StaticFiles): + """Serve Vite's content-hashed assets without browser revalidation.""" + + def file_response( + self, + full_path, + stat_result, + scope, + status_code = 200, + ): + response = super().file_response(full_path, stat_result, scope, status_code) + response.headers["Cache-Control"] = _IMMUTABLE_ASSET_CACHE_CONTROL + return response + + +class _AssetGZipMiddleware(GZipMiddleware): + """Serve range requests uncompressed; gzip + 206 mislabels Content-Range.""" + + async def __call__(self, scope, receive, send): + if scope["type"] == "http" and any(key == b"range" for key, _ in scope["headers"]): + await self.app(scope, receive, send) + return + await super().__call__(scope, receive, send) + + def setup_frontend(app: FastAPI, build_path: Path): """Mount frontend static files (optional)""" if not build_path.exists(): @@ -1478,7 +1575,12 @@ def setup_frontend(app: FastAPI, build_path: Path): assets_dir = build_path / "assets" if assets_dir.exists(): - app.mount("/assets", StaticFiles(directory = assets_dir), name = "assets") + assets_app = _AssetGZipMiddleware( + ImmutableStaticFiles(directory = assets_dir), + minimum_size = 1024, + compresslevel = 6, + ) + app.mount("/assets", assets_app, name = "assets") def _build_index_response(request: Request) -> Response: content = (build_path / "index.html").read_bytes() diff --git a/studio/backend/mcp_server.py b/studio/backend/mcp_server.py new file mode 100644 index 0000000000..e93490411d --- /dev/null +++ b/studio/backend/mcp_server.py @@ -0,0 +1,259 @@ +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 + +"""Curated MCP tools for driving an Unsloth Studio instance. + +The MCP surface deliberately wraps the existing Unsloth services instead of +duplicating training or export logic. It is opt-in because several tools can +start GPU work or write model artifacts. +""" + +from __future__ import annotations + +import hmac +from typing import Any + +from fastmcp import FastMCP + + +class BearerTokenMiddleware: + """Require an exact bearer token when Unsloth MCP is exposed remotely.""" + + def __init__(self, app: Any, token: str) -> None: + if not token or not token.strip(): + raise ValueError("Unsloth MCP bearer token must be a non-empty value") + if not token.isascii(): + # A non-ASCII token cannot be sent in an HTTP header; reject it here. + raise ValueError("Unsloth MCP bearer token must contain ASCII characters only") + self.app = app + # Compare on raw header bytes: str hmac.compare_digest raises on non-ASCII + # input, which would surface as a 500 instead of a clean 401. + self.expected = token.encode("utf-8") + + async def __call__(self, scope: dict[str, Any], receive: Any, send: Any) -> None: + scope_type = scope.get("type") + if scope_type not in ("http", "websocket"): + await self.app(scope, receive, send) + return + + headers = dict(scope.get("headers", [])) + raw_auth = headers.get(b"authorization", b"") + scheme, _, supplied = raw_auth.partition(b" ") + if scheme.lower() != b"bearer" or not hmac.compare_digest(supplied, self.expected): + await _send_unauthorized(send, scope_type) + return + + await self.app(scope, receive, send) + + +async def _send_unauthorized(send: Any, scope_type: str) -> None: + if scope_type == "websocket": + await send({"type": "websocket.close", "code": 4401}) + return + + await send( + { + "type": "http.response.start", + "status": 401, + "headers": [(b"content-type", b"application/json"), (b"www-authenticate", b"Bearer")], + } + ) + await send( + { + "type": "http.response.body", + "body": b'{"detail":"MCP bearer token required"}', + } + ) + + +def _dump(value: Any) -> Any: + """Convert Pydantic responses to plain JSON values for MCP clients.""" + if hasattr(value, "model_dump"): + return value.model_dump(mode = "json") + return value + + +def _clamp(value: int, low: int, high: int) -> int: + """Clamp an MCP-supplied integer into an inclusive range. + + MCP tools call the Unsloth route functions directly, which skips FastAPI's + Query(ge=, le=) validation, so we re-apply the same bounds here. + """ + return max(low, min(value, high)) + + +def create_studio_mcp() -> FastMCP: + """Create the Unsloth MCP server and register the high-value tools.""" + mcp = FastMCP( + "Unsloth Studio", + instructions = ( + "Use read tools to inspect the local Unsloth state before starting GPU work. " + "Training and export tools can consume substantial VRAM and write files. " + "Never expose tokens or local paths from tool results unless the user asks." + ), + ) + + @mcp.tool + async def studio_status() -> dict[str, Any]: + """Return the current training, export, inference, and GPU state.""" + from routes.export import get_export_status + from routes.inference import get_status as get_inference_status + from routes.training import get_training_status + + from utils.hardware import get_gpu_utilization + + training, export, inference = await _gather_status( + get_training_status(current_subject = "mcp"), + get_export_status(current_subject = "mcp"), + get_inference_status(current_subject = "mcp"), + ) + return { + "training": _dump(training), + "export": _dump(export), + "inference": _dump(inference), + "hardware": get_gpu_utilization(), + } + + @mcp.tool + async def list_local_models(models_dir: str = "./models") -> dict[str, Any]: + """List local and cached models available to Unsloth.""" + from routes.models import list_local_models as list_models + return _dump(await list_models(models_dir = models_dir, current_subject = "mcp")) + + @mcp.tool + async def get_training_status() -> dict[str, Any]: + """Read the active training job, phase, progress, and recent metrics.""" + from routes.training import get_training_status as get_status + return _dump(await get_status(current_subject = "mcp")) + + @mcp.tool + async def start_training(config: dict[str, Any]) -> dict[str, Any]: + """Start a validated Unsloth training job from a TrainingStartRequest-shaped object. + + The config is validated by the same Pydantic model used by the Unsloth UI. + Call get_training_status first and do not start work while another job runs. + """ + from models import TrainingStartRequest + from routes.training import start_training as start + + request = TrainingStartRequest.model_validate(config) + # Pass via_api_key explicitly (a direct call leaves it a Depends object). + # MCP drives Unsloth like the UI session, so it coexists and frees VRAM. + return _dump(await start(request, current_subject = "mcp", via_api_key = False)) + + @mcp.tool + async def stop_training(save: bool = True) -> dict[str, Any]: + """Ask the active training process to stop at its next safe checkpoint.""" + from routes.training import TrainingStopRequest, stop_training as stop + return _dump(await stop(TrainingStopRequest(save = save), current_subject = "mcp")) + + @mcp.tool + async def list_training_runs(limit: int = 50, offset: int = 0) -> dict[str, Any]: + """List completed and stopped training runs, newest first.""" + from routes.training_history import list_training_runs as list_runs + + # Clamp here (direct call skips Query bounds); a negative LIMIT = no limit. + limit = _clamp(limit, 1, 200) + offset = max(0, offset) + return _dump(await list_runs(limit = limit, offset = offset, current_subject = "mcp")) + + @mcp.tool + def validate_recipe(recipe: dict[str, Any]) -> dict[str, Any]: + """Validate a Data Recipe with the same validator used by Unsloth.""" + from models.data_recipe import RecipePayload + from routes.data_recipe.validate import validate + + return _dump(validate(RecipePayload(recipe = recipe))) + + @mcp.tool + def get_recipe_job_status(job_id: str) -> dict[str, Any]: + """Read the status of a Data Recipe job.""" + from routes.data_recipe.jobs import job_status + return _dump(job_status(job_id)) + + @mcp.tool + def get_recipe_job_dataset( + job_id: str, + limit: int = 20, + offset: int = 0, + ) -> dict[str, Any]: + """Read a bounded page of generated Data Recipe rows.""" + from routes.data_recipe.jobs import job_dataset + + # Clamp here (direct call skips FastAPI's Query bounds). + limit = _clamp(limit, 1, 500) + offset = max(0, offset) + return _dump(job_dataset(job_id, limit = limit, offset = offset)) + + @mcp.tool + async def load_checkpoint( + checkpoint_path: str, + max_seq_length: int = 2048, + load_in_4bit: bool = True, + trust_remote_code: bool = False, + approved_remote_code_fingerprint: str | None = None, + hf_token: str | None = None, + ) -> dict[str, Any]: + """Load a checkpoint into the export backend. + + Export runs in its own subprocess and coexists with training and + inference; it does not unload them, so a load can fail with a clear + out-of-memory error if the GPU is already full. Pass hf_token to load a + gated checkpoint, and approved_remote_code_fingerprint to retry a + trust_remote_code load that was blocked pending review. + """ + from models import LoadCheckpointRequest + from routes.export import load_checkpoint as load + + request = LoadCheckpointRequest( + checkpoint_path = checkpoint_path, + max_seq_length = max_seq_length, + load_in_4bit = load_in_4bit, + trust_remote_code = trust_remote_code, + approved_remote_code_fingerprint = approved_remote_code_fingerprint, + hf_token = hf_token, + ) + return _dump(await load(request, current_subject = "mcp")) + + @mcp.tool + async def export_gguf( + save_directory: str, + quantization_method: str | list[str] = "Q4_K_M", + push_to_hub: bool = False, + repo_id: str | None = None, + hf_token: str | None = None, + imatrix: bool = False, + imatrix_path: str | None = None, + ) -> dict[str, Any]: + """Export the loaded model to GGUF using Unsloth's existing path validation. + + quantization_method may be a single method or a list to produce several + GGUFs from one load. Pass hf_token when push_to_hub is set (the backend + rejects a Hub upload without it). Set imatrix (or imatrix_path) for the + IQ low-bit quants that require an importance matrix. + """ + from models import ExportGGUFRequest + from routes.export import export_gguf as export + + request = ExportGGUFRequest( + save_directory = save_directory, + quantization_method = quantization_method, + push_to_hub = push_to_hub, + repo_id = repo_id, + hf_token = hf_token, + imatrix = imatrix, + imatrix_path = imatrix_path, + ) + return _dump(await export(request, current_subject = "mcp")) + + return mcp + + +async def _gather_status(*coroutines: Any) -> tuple[Any, ...]: + """Gather independent status calls without letting one optional backend fail all state.""" + import asyncio + + results = await asyncio.gather(*coroutines, return_exceptions = True) + return tuple( + {"error": str(result)} if isinstance(result, Exception) else result for result in results + ) diff --git a/studio/backend/models/inference.py b/studio/backend/models/inference.py index 3ae974448e..1758efe515 100644 --- a/studio/backend/models/inference.py +++ b/studio/backend/models/inference.py @@ -18,6 +18,8 @@ from pydantic import ( model_validator, ) +from picker.schemas import MAX_CHAT_TEMPLATE_BYTES + class LoadRequest(BaseModel): """Request to load a model for inference""" @@ -54,17 +56,37 @@ class LoadRequest(BaseModel): @field_validator("chat_template_override") @classmethod def normalize_blank_chat_template_override(cls, value: Optional[str]) -> Optional[str]: - if value is not None and value.strip() == "": + if value is None: return None + # Char count is a lower bound on UTF-8 byte length: reject an oversized + # template before spending work encoding it. + if len(value) > MAX_CHAT_TEMPLATE_BYTES: + raise ValueError(f"Chat template exceeds the {MAX_CHAT_TEMPLATE_BYTES}-byte limit.") + if value.strip() == "": + return None + if len(value.encode("utf-8")) > MAX_CHAT_TEMPLATE_BYTES: + raise ValueError(f"Chat template exceeds the {MAX_CHAT_TEMPLATE_BYTES}-byte limit.") return value cache_type_kv: Optional[str] = Field( None, - description = "KV cache data type for both K and V (e.g. 'f16', 'bf16', 'q8_0', 'q4_1', 'q5_1')", + description = ( + "KV cache data type for both K and V " + "(e.g. 'f16', 'bf16', 'q8_0', 'q4_0', 'q4_1', 'q5_0', 'q5_1', 'iq4_nl', 'f32')" + ), ) gpu_ids: Optional[List[int]] = Field( None, - description = "Physical GPU indices to use, for example [0, 1]. Omit or pass [] to use automatic selection. Explicit gpu_ids are unsupported when the parent CUDA_VISIBLE_DEVICES uses UUID/MIG entries. Not supported for GGUF models.", + description = ( + "GPU placement pool, for example [0, 1]. Omit or pass [] to use " + "automatic selection. CUDA/ROCm and Intel XPU values are physical " + "GPU indices; Vulkan values are ggml device ordinals. Explicit " + "physical IDs are unsupported when the parent visibility mask uses " + "non-numeric or subdevice entries, including CUDA_VISIBLE_DEVICES " + "with UUID/MIG entries and ZE_AFFINITY_MASK with subdevice tokens " + "(for example '0.0,0.1') or FLAT-hierarchy tile handles. For GGUF " + "models the fitter may pin the smallest subset of this pool that fits." + ), ) speculative_type: Optional[str] = Field( None, @@ -100,12 +122,72 @@ class LoadRequest(BaseModel): "No effect on a single GPU. Ignored for non-GGUF models." ), ) + gpu_memory_mode: Literal["auto", "manual"] = Field( + "auto", + description = ( + "GPU memory strategy for GGUF models. 'auto' (default): Unsloth " + "selects GPUs and caps context to fit VRAM. 'manual': you own the " + "offload. Leave gpu_layers at -1 (Auto) to hand memory management to " + "llama.cpp's --fit (no device masking, no context auto-reduce, no " + "gpu-layer/tensor-split planning); set gpu_layers >= 0 to pin layers " + "and n_cpu_moe yourself (--fit off), with tensor_parallel still " + "applying (split by free VRAM unless tensor_split is set, no planner). " + "Ignored for non-GGUF." + ), + ) + gpu_layers: int = Field( + -1, + ge = -1, + description = ( + "Manual mode only: number of layers to offload to the GPU " + "(--gpu-layers, with --fit off). A value >= the model's layer count " + "offloads all of them. -1 = Auto: hand layer + context sizing to " + "llama.cpp's --fit. Ignored unless gpu_memory_mode is 'manual'." + ), + ) + n_cpu_moe: int = Field( + 0, + ge = 0, + description = ( + "Manual mode only: keep the first N MoE expert layers on the CPU " + "(--n-cpu-moe) to save VRAM on MoE models. 0 = none, N = number of " + "MoE layers offloaded (the backend offsets past any leading dense " + "layers). Ignored unless gpu_memory_mode is 'manual' with gpu_layers >= 0." + ), + ) + tensor_split: Optional[List[float]] = Field( + None, + description = ( + "Manual mode only: relative share of the model per GPU (--tensor-split), " + "in the order of the GPUs in use, e.g. [2, 1] for 2:1. Omit it to let " + "llama.cpp use its default, which splits by free VRAM. Any list given is " + "passed through as-is, so send [1, 1] to force an even split. Ignored " + "unless gpu_memory_mode is 'manual' with gpu_layers >= 0." + ), + ) + + @field_validator("tensor_split") + @classmethod + def _reject_degenerate_tensor_split(cls, value: Optional[List[float]]) -> Optional[List[float]]: + # A negative / non-finite / all-zero split is silently dropped at launch + # (stored as None) yet still compared raw in the reload dedupe, so an + # identical Apply reloads forever. Reject it up front; [] = no split. + if not value: + return value + import math + + if any((not math.isfinite(v)) or v < 0 for v in value): + raise ValueError("tensor_split entries must be finite and non-negative") + if sum(value) <= 0: + raise ValueError("tensor_split must have a positive total") + return value + llama_extra_args: Optional[List[str]] = Field( None, description = ( "Extra arguments forwarded verbatim to llama-server for GGUF models. " "One token per list entry, e.g. ['--top-k', '20', '--seed', '42']. " - "Studio-managed flags (model identity, port, context length, GPU placement, " + "Unsloth-managed flags (model identity, port, context length, GPU placement, " "auth, UI/server mode) are rejected. Ignored for non-GGUF models." ), ) @@ -117,6 +199,32 @@ class UnloadRequest(BaseModel): model_path: str = Field(..., description = "Model identifier to unload") +class TranscribeRequest(BaseModel): + """Speech-to-text request for the dictation STT sidecar.""" + + audio: str = Field(..., description = "Base64-encoded audio (any common format)") + model: Optional[str] = Field(None, description = "STT model id; defaults server-side") + language: Optional[str] = Field(None, description = "BCP-47 language, or 'auto'/None to detect") + fast: bool = Field( + False, + description = "Use low-latency single-candidate decoding for dictation", + ) + engine: Optional[str] = Field( + None, + description = "STT engine: 'transformers' (default) or 'gguf' (whisper.cpp)", + ) + + +class SttLoadRequest(BaseModel): + """Warm the STT sidecar with a model without transcribing.""" + + model: Optional[str] = Field(None, description = "STT model id; defaults server-side") + engine: Optional[str] = Field( + None, + description = "STT engine: 'transformers' (default) or 'gguf' (whisper.cpp)", + ) + + class ValidateModelRequest(BaseModel): """Check whether an identifier resolves to a ModelConfig; does NOT load weights.""" @@ -133,11 +241,26 @@ class ValidateModelRequest(BaseModel): max_seq_length: int = Field(0, ge = 0, le = 1048576) load_in_4bit: bool = Field(True) gpu_ids: Optional[List[int]] = Field(None) + gpu_memory_mode: Literal["auto", "manual"] = Field( + "auto", + description = ( + "GGUF GPU-memory strategy intended for the follow-up load. Manual " + "placement bypasses the training coexistence estimate: Auto layers " + "delegate fitting to llama.cpp, while explicit layers are user-owned." + ), + ) include_context_length: bool = Field( False, description = "Also read the native context length from the local GGUF header. " "Opt-in so the normal load preflight doesn't pay for a cache scan it doesn't need.", ) + include_chat_template: bool = Field( + False, + description = "Also read the embedded chat template from the local GGUF header, so a " + "native (picked / drag-drop) file's default template can be shown before it is loaded. " + "Opt-in and, like include_context_length, a metadata-only probe that skips the training " + "guard. Only the leased file's own embedded template is read, never sibling sidecars.", + ) class TransformersUpgradeInfo(BaseModel): @@ -151,13 +274,13 @@ class TransformersUpgradeInfo(BaseModel): ) supported_in_pypi: bool = Field( False, - description = "True if the latest PyPI release ships this model_type; Studio can " + description = "True if the latest PyPI release ships this model_type; Unsloth can " "install it into a persistent sidecar after user consent.", ) supported_in_main: bool = Field( False, description = "True if transformers GitHub main ships this model_type (dev-only; " - "not installable through Studio yet).", + "not installable through Unsloth yet).", ) @@ -188,6 +311,21 @@ class ValidateModelResponse(BaseModel): description = "Native training context length, read from the GGUF header when the file " "is already downloaded locally; None for non-GGUF, gated, or not-yet-downloaded models.", ) + layer_count: Optional[int] = Field( + None, + description = "Total layer count (GGUF block_count), the manual gpu-layers ceiling, read " + "from the header alongside context_length; None when not read.", + ) + moe_layer_count: Optional[int] = Field( + None, + description = "MoE expert-layer count (the manual --n-cpu-moe ceiling), read from the GGUF " + "header alongside context_length; 0 for dense models, None when not read.", + ) + chat_template: Optional[str] = Field( + None, + description = "Embedded GGUF chat template, read from the header when include_chat_template " + "is set (native lease-backed picks); None for non-GGUF, over-cap, or not-read templates.", + ) # Additive fields; the consuming consent dialog ships in a follow-up frontend PR. requires_transformers_upgrade: bool = Field( False, @@ -307,7 +445,10 @@ class LoadResponse(BaseModel): ) cache_type_kv: Optional[str] = Field( None, - description = "KV cache data type for K and V (e.g. 'f16', 'bf16', 'q8_0')", + description = ( + "KV cache data type for K and V " + "(e.g. 'f16', 'bf16', 'q8_0', 'q4_0', 'q4_1', 'q5_0', 'q5_1', 'iq4_nl', 'f32')" + ), ) chat_template: Optional[str] = Field( None, @@ -333,6 +474,41 @@ class LoadResponse(BaseModel): False, description = "Whether tensor-parallel split (--split-mode tensor) is active.", ) + gpu_memory_mode: Literal["auto", "manual"] = Field( + "auto", + description = "Active GPU memory strategy ('auto' or 'manual').", + ) + gpu_layers: int = Field( + -1, + description = "Manual mode: requested --gpu-layers value (-1 = Auto/--fit, or when not manual).", + ) + n_cpu_moe: int = Field( + 0, + description = "Manual mode: MoE expert layers pinned to CPU (--n-cpu-moe); 0 = none.", + ) + tensor_split: Optional[List[float]] = Field( + None, + description = "Manual mode: relative model share per GPU (--tensor-split); None = default (split by free VRAM).", + ) + n_layers: Optional[int] = Field( + None, + description = "Model's layer count (GGUF block_count), for the manual gpu-layers ceiling.", + ) + n_moe_layers: int = Field( + 0, + description = "Model's MoE expert-layer count (the n_cpu_moe ceiling); 0 if not an MoE model.", + ) + gpu_ids: Optional[List[int]] = Field( + None, + description = "Effective GPU indices the model is using after fit-time narrowing, or None for automatic selection.", + ) + requested_gpu_ids: Optional[List[int]] = Field( + None, + description = ( + "GPU placement pool requested by the user before fit-time narrowing, " + "or None for automatic selection." + ), + ) class UnloadResponse(BaseModel): @@ -432,7 +608,11 @@ class InferenceStatusResponse(BaseModel): ) cache_type_kv: Optional[str] = Field( None, - description = "KV cache quantization dtype (e.g. 'q8_0'), or None for default", + description = ( + "KV cache quantization dtype " + "(e.g. 'f16', 'bf16', 'q8_0', 'q4_0', 'q4_1', 'q5_0', 'q5_1', 'iq4_nl', 'f32'), " + "or None for default" + ), ) chat_template: Optional[str] = Field( None, description = "Model's default chat template (Jinja2 source), if any" @@ -461,6 +641,49 @@ class InferenceStatusResponse(BaseModel): False, description = "Whether tensor-parallel split (--split-mode tensor) is active.", ) + gpu_memory_mode: Literal["auto", "manual"] = Field( + "auto", + description = "Active GPU memory strategy ('auto' or 'manual').", + ) + gpu_layers: int = Field( + -1, + description = "Manual mode: requested --gpu-layers value (-1 = Auto/--fit, or when not manual).", + ) + n_cpu_moe: int = Field( + 0, + description = "Manual mode: MoE expert layers pinned to CPU (--n-cpu-moe); 0 = none.", + ) + tensor_split: Optional[List[float]] = Field( + None, + description = "Manual mode: relative model share per GPU (--tensor-split); None = default (split by free VRAM).", + ) + requested_context_length: Optional[int] = Field( + None, + description = ( + "The n_ctx the active GGUF load was invoked with (0 = Auto). Lets the " + "UI re-seed a Manual + Auto-layers context pin on hydration, where " + "context_length only exposes the resolved value. None for non-GGUF." + ), + ) + n_layers: Optional[int] = Field( + None, + description = "Model's layer count (GGUF block_count), for the manual gpu-layers ceiling.", + ) + n_moe_layers: int = Field( + 0, + description = "Model's MoE expert-layer count (the n_cpu_moe ceiling); 0 if not an MoE model.", + ) + gpu_ids: Optional[List[int]] = Field( + None, + description = "Effective GPU indices the model is using after fit-time narrowing, or None for automatic selection.", + ) + requested_gpu_ids: Optional[List[int]] = Field( + None, + description = ( + "GPU placement pool requested by the user before fit-time narrowing, " + "or None for automatic selection." + ), + ) llama_cpp_supports_mtp: bool = Field( True, description = ( @@ -533,7 +756,7 @@ class ImageContentPart(BaseModel): class InputDocumentContentPart(BaseModel): """Document (PDF / file) content part in a multimodal message. - Studio-normalised shape (file_data or file_url, plus optional filename/media_type). + Unsloth-normalised shape (file_data or file_url, plus optional filename/media_type). Mapped onto Anthropic ``document`` / OpenAI ``input_file`` for vision providers; dropped for non-vision providers. """ @@ -689,7 +912,7 @@ class ThinkingConfig(BaseModel): """Anthropic-compatible thinking/reasoning configuration. Use type='disabled' to turn off thinking, or type='enabled' to turn it on. Only type is read; extra fields (e.g. budget_tokens) are ignored, since - Studio sets provider thinking budgets itself. + Unsloth sets provider thinking budgets itself. """ type: Literal["disabled", "enabled"] = "disabled" @@ -748,7 +971,7 @@ class ChatCompletionRequest(BaseModel): None, description = ( "OpenAI function-tool definitions. When provided without `enable_tools=true`, " - "Studio forwards the tools to the backend so the model returns structured " + "Unsloth forwards the tools to the backend so the model returns structured " "tool_calls for the client to execute (standard OpenAI function calling)." ), ) @@ -1160,7 +1383,7 @@ class ChatCompletionRequest(BaseModel): and (self.enable_tools is True or bool(self.mcp_enabled)) ): # "Ask" gates every call, so a direct API caller that omits the legacy - # confirm flag must still hit the confirmation gate for Studio's own + # confirm flag must still hit the confirmation gate for Unsloth's own # tool loop. An explicit confirm_tool_calls=False wins over the mode # (mirrors _permission_mode_confirm and the Anthropic pre-switch guard), # so only self-enable when the flag is unset. Only self-enable when that @@ -1168,7 +1391,7 @@ class ChatCompletionRequest(BaseModel): # (enable_tools / mcp_enabled) -- the router enters the loop on those # signals, not on enabled_tools alone (which merely filters which tools # run). A plain client-tool passthrough (client-supplied `tools` that - # Studio does not execute) must route verbatim, and external-provider + # Unsloth does not execute) must route verbatim, and external-provider # routing rejects confirm_tool_calls with tools, so skip the fold there. # # "auto" is deliberately NOT folded: it only prompts for a call the diff --git a/studio/backend/models/models.py b/studio/backend/models/models.py index 54e88fed58..df6725c9c9 100644 --- a/studio/backend/models/models.py +++ b/studio/backend/models/models.py @@ -178,6 +178,14 @@ class LocalModelInfo(BaseModel): None, description = "HF repo id for cached models, e.g. org/model", ) + active_cache: Optional[bool] = Field( + None, + description = "Whether an HF model belongs to the current download cache.", + ) + partial: bool = Field( + False, + description = "Whether the cached model has an incomplete download.", + ) model_format: Optional[str] = Field( None, description = "Detected weights format ('gguf' when known). Lets the UI " diff --git a/studio/backend/models/providers.py b/studio/backend/models/providers.py index 5a75246c07..4238403e00 100644 --- a/studio/backend/models/providers.py +++ b/studio/backend/models/providers.py @@ -47,6 +47,14 @@ class ProviderCreate(BaseModel): None, description = "Custom base URL (overrides registry default). Omit to use the default.", ) + models: list[str] = Field( + default_factory = list, + description = "Enabled model IDs for this connection", + ) + available_models: list[str] = Field( + default_factory = list, + description = "Discovered catalog model IDs last fetched for this connection", + ) class ProviderUpdate(BaseModel): @@ -55,6 +63,11 @@ class ProviderUpdate(BaseModel): display_name: Optional[str] = Field(None, description = "New display name") base_url: Optional[str] = Field(None, description = "New base URL") is_enabled: Optional[bool] = Field(None, description = "Enable or disable this provider") + models: Optional[list[str]] = Field(None, description = "Enabled model IDs for this connection") + available_models: Optional[list[str]] = Field( + None, + description = "Discovered catalog model IDs last fetched for this connection", + ) class ProviderResponse(BaseModel): @@ -65,6 +78,14 @@ class ProviderResponse(BaseModel): display_name: str = Field(..., description = "User-chosen label") base_url: str = Field(..., description = "API base URL") is_enabled: bool = Field(True, description = "Whether this provider is enabled") + models: list[str] = Field( + default_factory = list, + description = "Enabled model IDs for this connection", + ) + available_models: list[str] = Field( + default_factory = list, + description = "Discovered catalog model IDs last fetched for this connection", + ) created_at: str = Field(..., description = "ISO 8601 creation timestamp") updated_at: str = Field(..., description = "ISO 8601 last-update timestamp") diff --git a/studio/backend/models/training.py b/studio/backend/models/training.py index ff815a2fa9..0aca5da72c 100644 --- a/studio/backend/models/training.py +++ b/studio/backend/models/training.py @@ -446,7 +446,7 @@ class TrainingStartRequest(BaseModel): random_seed: int = Field( 3407, description = ( - "Random seed; matches the Studio backend / MLX worker default " + "Random seed; matches the Unsloth backend / MLX worker default " "and unsloth's historical recommended value." ), ) @@ -470,6 +470,7 @@ class TrainingStartRequest(BaseModel): gradient_checkpointing: str = Field("", description = "Gradient checkpointing setting") use_rslora: bool = Field(False, description = "Use RSLoRA") use_loftq: bool = Field(False, description = "Use LoftQ") + use_dora: bool = Field(False, description = "Use DoRA") train_on_completions: bool = Field(False, description = "Train on completions only") # Vision-specific LoRA parameters @@ -496,7 +497,15 @@ class TrainingStartRequest(BaseModel): # GPU selection gpu_ids: Optional[List[int]] = Field( None, - description = "Physical GPU indices to use, for example [0, 1]. Omit or pass [] to use automatic selection. Explicit gpu_ids are unsupported when the parent CUDA_VISIBLE_DEVICES uses UUID/MIG entries.", + description = ( + "Physical GPU indices to use, for example [0, 1]. Omit or pass " + "[] to use automatic selection. Explicit gpu_ids are unsupported " + "when the parent visibility mask uses non-numeric or subdevice " + "entries -- this includes CUDA_VISIBLE_DEVICES with UUID/MIG " + "entries on NVIDIA, and ZE_AFFINITY_MASK with subdevice tokens " + "(e.g. '0.0,0.1') or FLAT-hierarchy (default) tile handles on " + "Intel XPU." + ), ) # S3 dataset source configuration @@ -505,6 +514,13 @@ class TrainingStartRequest(BaseModel): description = "S3 bucket configuration for loading datasets from AWS S3. Requires boto3 to be installed.", ) + @field_validator("target_modules", mode = "before") + @classmethod + def _normalize_target_modules(cls, value: Any) -> Any: + # Sanitized non-LoRA history stores the unused value as null; treat it as a + # fresh request's omitted/default empty list on resume. + return [] if value is None else value + @model_validator(mode = "after") def _validate_streaming_splits(self) -> "TrainingStartRequest": # Streaming load_dataset does not accept HF slice syntax (e.g. "train[:50%]" @@ -530,6 +546,37 @@ class TrainingStartRequest(BaseModel): raise ValueError("Either num_epochs or max_steps must be > 0; both cannot be 0.") return self + @model_validator(mode = "after") + def _validate_lora_variant_flags(self) -> "TrainingStartRequest": + # The frontend only ever sends one of these and never under Full + # Finetuning, but a direct API/YAML/CLI caller can bypass that. Nothing + # downstream breaks (full finetune ignores them, MLX rejects use_dora/ + # use_loftq outright), but reject early here for a clear error instead + # of a silently-ignored flag. + active = [ + name + for name, enabled in ( + ("use_rslora", self.use_rslora), + ("use_loftq", self.use_loftq), + ("use_dora", self.use_dora), + ) + if enabled + ] + if len(active) > 1: + raise ValueError( + f"Only one LoRA variant may be enabled at a time; got {active}. " + "use_rslora, use_loftq, and use_dora are mutually exclusive." + ) + # getattr, not self.training_type: model_construct() (used by tests that + # validate a single field in isolation) leaves required fields unset, and + # this is a mode="after" validator so it still runs on that partial instance. + if getattr(self, "training_type", None) == "Full Finetuning" and active: + raise ValueError( + f"{active[0]} requires an adapter method (LoRA/QLoRA or " + "Continued Pretraining); it has no effect under Full Finetuning." + ) + return self + class TrainingJobResponse(BaseModel): """Immediate response when training is initiated""" diff --git a/studio/backend/picker/__init__.py b/studio/backend/picker/__init__.py new file mode 100644 index 0000000000..32014236c6 --- /dev/null +++ b/studio/backend/picker/__init__.py @@ -0,0 +1,2 @@ +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 diff --git a/studio/backend/picker/routes/__init__.py b/studio/backend/picker/routes/__init__.py new file mode 100644 index 0000000000..c0e988c8bb --- /dev/null +++ b/studio/backend/picker/routes/__init__.py @@ -0,0 +1,6 @@ +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 + +from .templates import router as templates_router + +__all__ = ["templates_router"] diff --git a/studio/backend/picker/routes/templates.py b/studio/backend/picker/routes/templates.py new file mode 100644 index 0000000000..02b8bf7184 --- /dev/null +++ b/studio/backend/picker/routes/templates.py @@ -0,0 +1,45 @@ +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 + +from __future__ import annotations + +import asyncio +from typing import Optional + +from fastapi import APIRouter, Body, Depends, Query + +from auth.authentication import get_current_subject +from hub.dependencies import get_hf_token + +from ..schemas import ( + MAX_CHAT_TEMPLATE_BYTES, + ModelTemplateResponse, + ValidateChatTemplateRequest, + ValidateChatTemplateResponse, +) +from ..service import read_default_chat_template, validate_chat_template + +router = APIRouter() + + +@router.post("/validate-chat-template", response_model = ValidateChatTemplateResponse) +async def validate_chat_template_route( + body: ValidateChatTemplateRequest = Body(...), + current_subject: str = Depends(get_current_subject), +) -> ValidateChatTemplateResponse: + return await asyncio.to_thread(validate_chat_template, body.template) + + +@router.get("/chat-template/{model_name:path}", response_model = ModelTemplateResponse) +async def get_default_chat_template_route( + model_name: str, + gguf_variant: Optional[str] = Query(None), + hf_token: Optional[str] = Depends(get_hf_token), + current_subject: str = Depends(get_current_subject), +) -> ModelTemplateResponse: + template = await asyncio.to_thread( + read_default_chat_template, model_name, hf_token, gguf_variant + ) + if template is not None and len(template.encode("utf-8")) > MAX_CHAT_TEMPLATE_BYTES: + template = None + return ModelTemplateResponse(model_name = model_name, chat_template = template) diff --git a/studio/backend/picker/schemas.py b/studio/backend/picker/schemas.py new file mode 100644 index 0000000000..b4f956188f --- /dev/null +++ b/studio/backend/picker/schemas.py @@ -0,0 +1,32 @@ +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 + +from typing import Optional + +from pydantic import BaseModel, Field, field_validator + +# Mirror the frontend's 64 KiB chat-template contract (per-model-config.ts) at +# the API boundary so a direct caller cannot make Jinja parse an oversized +# template. MaxBodyMiddleware only caps the whole request body, not this field. +MAX_CHAT_TEMPLATE_BYTES = 65_536 + + +class ValidateChatTemplateRequest(BaseModel): + template: str = Field(default = "") + + @field_validator("template") + @classmethod + def _enforce_template_size(cls, value: str) -> str: + if len(value.encode("utf-8")) > MAX_CHAT_TEMPLATE_BYTES: + raise ValueError(f"Chat template exceeds the {MAX_CHAT_TEMPLATE_BYTES}-byte limit.") + return value + + +class ValidateChatTemplateResponse(BaseModel): + valid: bool + error: Optional[str] = None + + +class ModelTemplateResponse(BaseModel): + model_name: str + chat_template: Optional[str] = None diff --git a/studio/backend/picker/service.py b/studio/backend/picker/service.py new file mode 100644 index 0000000000..ccf9c3e152 --- /dev/null +++ b/studio/backend/picker/service.py @@ -0,0 +1,432 @@ +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 + +from __future__ import annotations + +import json +import logging +import os +import re +from pathlib import Path +from typing import Optional + +from hub.services.models.folder_browser import ( + _build_browse_allowlist, + _is_path_inside_allowlist, +) +from hub.utils.gguf import extract_quant_label, iter_hf_cache_snapshots +from utils.models.gguf_metadata import read_gguf_chat_template +from utils.models.model_config import ( + _extract_quant_label, + _is_big_endian_gguf_path, + _is_mmproj, + _is_mtp_drafter, +) +from utils.hf_cache_settings import active_hf_hub_cache +from utils.paths.path_utils import ( + is_local_path, + normalize_path, + resolve_cached_repo_id_case, +) + +from .schemas import MAX_CHAT_TEMPLATE_BYTES, ValidateChatTemplateResponse + +logger = logging.getLogger(__name__) + +_VALID_REPO_ID = re.compile(r"^[A-Za-z0-9._-]+/[A-Za-z0-9._-]+$") + + +def _is_valid_repo_id(repo_id: str) -> bool: + return bool(_VALID_REPO_ID.fullmatch(repo_id)) + + +_TOKENIZER_CONFIG_PATHS = ("tokenizer_config.json", "LLM/tokenizer_config.json") +_JINJA_TEMPLATE_PATHS = ("chat_template.jinja", "LLM/chat_template.jinja") +_PROCESSOR_TEMPLATE_PATHS = ("chat_template.json", "LLM/chat_template.json") + +# Cap sidecar reads so a malformed or hostile metadata file cannot exhaust memory +# before its template is size-checked. The JSON envelope may exceed a bare template +# (it carries other tokenizer metadata); the extracted template is still bounded by +# MAX_CHAT_TEMPLATE_BYTES downstream. +MAX_TEMPLATE_METADATA_BYTES = 4 * 1024 * 1024 + + +def _read_bounded_text(path: Path, limit: int) -> Optional[str]: + """Read at most `limit` bytes of UTF-8 text; None if larger or unreadable.""" + try: + with path.open("rb") as f: + data = f.read(limit + 1) + except OSError: + return None + if len(data) > limit: + return None + try: + return data.decode("utf-8") + except UnicodeError: + return None + + +def _leaf_inside_allowlist(path: Path, allow_roots: Optional[list[Path]]) -> bool: + # Block symlinked children from escaping the validated directory (realpath-checked). + # None = trusted caller (HF cache / remote download). + return allow_roots is None or _is_path_inside_allowlist(path, allow_roots) + + +def validate_chat_template(template: str) -> ValidateChatTemplateResponse: + text = (template or "").strip() + if not text: + return ValidateChatTemplateResponse(valid = True, error = None) + # Import Jinja lazily: optional at runtime (e.g. GGUF-only installs), so a + # missing dependency must not crash API startup. + try: + from jinja2 import TemplateError + from jinja2.ext import Extension + from jinja2.sandbox import ImmutableSandboxedEnvironment + except ImportError: + return ValidateChatTemplateResponse(valid = True, error = None) + + class _GenerationTag(Extension): + # Accept Transformers' {% generation %} assistant-mask tag so a pasted HF + # chat template validates (we only parse it). + tags = {"generation"} + + def parse(self, parser): + next(parser.stream) + return parser.parse_statements(["name:endgeneration"], drop_needle = True) + + try: + env = ImmutableSandboxedEnvironment( + trim_blocks = True, + lstrip_blocks = True, + extensions = ["jinja2.ext.loopcontrols", _GenerationTag], + ) + env.parse(text) + return ValidateChatTemplateResponse(valid = True, error = None) + except TemplateError as exc: + message = getattr(exc, "message", None) or str(exc) + lineno = getattr(exc, "lineno", None) + if lineno: + message = f"Line {lineno}: {message}" + return ValidateChatTemplateResponse(valid = False, error = message) + except Exception as exc: + return ValidateChatTemplateResponse(valid = False, error = str(exc)) + + +def _chat_template_from_tokenizer_config(config: dict) -> Optional[str]: + if not isinstance(config, dict): + return None + raw = config.get("chat_template") + if isinstance(raw, str) and raw.strip(): + return raw + if isinstance(raw, list): + fallback: Optional[str] = None + for entry in raw: + if not isinstance(entry, dict): + continue + template = entry.get("template") + if not isinstance(template, str): + continue + if entry.get("name") == "default": + return template + if fallback is None: + fallback = template + return fallback + return None + + +def _chat_template_from_jinja_file( + dir_path: Path, allow_roots: Optional[list[Path]] = None +) -> Optional[str]: + for rel in _JINJA_TEMPLATE_PATHS: + template_file = dir_path / rel + if not template_file.exists() or not _leaf_inside_allowlist(template_file, allow_roots): + continue + try: + if template_file.stat().st_size > MAX_CHAT_TEMPLATE_BYTES: + continue + template = template_file.read_text(encoding = "utf-8") + except Exception: + continue + if template.strip(): + return template + return None + + +def _chat_template_from_processor_payload(payload: object) -> Optional[str]: + # processor chat_template.json may be the template string itself or a + # {name: template} map, not only a tokenizer_config-shaped object. + if isinstance(payload, str): + return payload if payload.strip() else None + template = _chat_template_from_tokenizer_config(payload) # type: ignore[arg-type] + if template: + return template + if isinstance(payload, dict): + # Named-template map: prefer "default", else the first non-empty entry + # (mirrors the tokenizer-config list fallback). + default = payload.get("default") + if isinstance(default, str) and default.strip(): + return default + for value in payload.values(): + if isinstance(value, str) and value.strip(): + return value + return None + + +def _chat_template_from_processor_json( + dir_path: Path, allow_roots: Optional[list[Path]] = None +) -> Optional[str]: + for rel in _PROCESSOR_TEMPLATE_PATHS: + config_file = dir_path / rel + if not config_file.exists() or not _leaf_inside_allowlist(config_file, allow_roots): + continue + raw = _read_bounded_text(config_file, MAX_TEMPLATE_METADATA_BYTES) + if raw is None: + continue + try: + payload = json.loads(raw) + except Exception: + continue + template = _chat_template_from_processor_payload(payload) + if template: + return template + return None + + +def _chat_template_from_tokenizer_dir( + dir_path: Path, allow_roots: Optional[list[Path]] = None +) -> Optional[str]: + jinja = _chat_template_from_jinja_file(dir_path, allow_roots) + if jinja: + return jinja + for rel in _TOKENIZER_CONFIG_PATHS: + config_file = dir_path / rel + if not config_file.exists() or not _leaf_inside_allowlist(config_file, allow_roots): + continue + raw = _read_bounded_text(config_file, MAX_TEMPLATE_METADATA_BYTES) + if raw is None: + continue + try: + config = json.loads(raw) + except Exception: + continue + template = _chat_template_from_tokenizer_config(config) + if template: + return template + return _chat_template_from_processor_json(dir_path, allow_roots) + + +_GGUF_SCAN_MAX_DEPTH = 2 + + +def _iter_ggufs(dir_path: Path) -> list[Path]: + if dir_path == dir_path.parent: + return [] + root = str(dir_path) + found: list[Path] = [] + for current, dirs, files in os.walk(root, followlinks = False): + rel = os.path.relpath(current, root) + depth = 0 if rel == os.curdir else rel.count(os.sep) + 1 + if depth >= _GGUF_SCAN_MAX_DEPTH: + dirs[:] = [] + for name in files: + if not name.lower().endswith(".gguf") or _is_mmproj(name): + continue + path = Path(current) / name + try: + rel = path.relative_to(dir_path).as_posix() + except ValueError: + rel = name + quant = _extract_quant_label(rel) + if _is_mtp_drafter(rel) or _is_big_endian_gguf_path(rel, quant): + continue + found.append(path) + return found + + +def _variant_matches(relative_path: str, needle: str) -> bool: + quant = _extract_quant_label(relative_path).lower() + if quant == needle: + return True + if extract_quant_label(relative_path).lower() == needle: + return True + prefix = f"{needle}-" + if not quant.startswith(prefix): + return False + suffix = quant[len(prefix) :] + if not suffix.endswith("bpw"): + return False + value = suffix[:-3] + return bool(value) and value.replace(".", "", 1).isdigit() + + +_GGUF_SPLIT_INDEX_RE = re.compile(r"-(\d{3,})-of-\d{3,}$", re.IGNORECASE) + + +def _is_nonfirst_gguf_split(path: Path) -> bool: + match = _GGUF_SPLIT_INDEX_RE.search(path.stem) + return match is not None and int(match.group(1)) != 1 + + +def _find_gguf_in_dir(dir_path: Path, gguf_variant: Optional[str]) -> Optional[Path]: + try: + ggufs = sorted(_iter_ggufs(dir_path)) + except OSError: + return None + if not ggufs: + return None + needle = (gguf_variant or "").strip().lower() + if needle: + for path in ggufs: + try: + relative = path.relative_to(dir_path).as_posix() + except ValueError: + relative = path.name + if _variant_matches(relative, needle): + return path + return None + candidates = [path for path in ggufs if not _is_nonfirst_gguf_split(path)] or ggufs + try: + return max(candidates, key = lambda path: path.stat().st_size) + except OSError: + return candidates[0] + + +def _chat_template_from_dir( + dir_path: Path, + gguf_variant: Optional[str] = None, + allow_roots: Optional[list[Path]] = None, +) -> Optional[str]: + def from_gguf() -> Optional[str]: + gguf = _find_gguf_in_dir(dir_path, gguf_variant) + if gguf is None or not _leaf_inside_allowlist(gguf, allow_roots): + return None + return read_gguf_chat_template(str(gguf)) + + # Sidecar tokenizer files (chat_template.jinja / tokenizer_config.json) are the + # author's maintained template and supersede the GGUF's possibly-stale embedded + # copy. The variant only picks the GGUF fallback, so tokenizer-first precedence + # holds whether or not a variant is given. + return _chat_template_from_tokenizer_dir(dir_path, allow_roots) or from_gguf() + + +def read_default_chat_template( + model_name: str, + hf_token: Optional[str] = None, + gguf_variant: Optional[str] = None, +) -> Optional[str]: + if not isinstance(model_name, str) or not model_name.strip(): + return None + name = model_name.strip() + + if is_local_path(name): + try: + target = Path(normalize_path(name)).expanduser() + allow_roots = _build_browse_allowlist() + if not _is_path_inside_allowlist(target, allow_roots): + logger.debug("Refused chat template read outside allowed folders: %s", name) + return None + if name.lower().endswith(".gguf"): + # Prefer a maintained sidecar next to the file over the GGUF's + # embedded copy (tokenizer-first precedence, as elsewhere). + sidecar = _chat_template_from_tokenizer_dir(target.parent, allow_roots) + if sidecar: + return sidecar + return read_gguf_chat_template(str(target)) + return _chat_template_from_dir(target, gguf_variant, allow_roots) + except Exception as exc: + logger.debug("Could not read local chat template for %s: %s", name, exc) + return None + + if not _is_valid_repo_id(name): + return None + + resolved = resolve_cached_repo_id_case(name) + + try: + # Resolve within each cached revision, newest first. A revision's sidecar + # supersedes its own embedded GGUF copy, but must not override a newer + # revision, so precedence stays per-snapshot rather than global. + for snapshot in iter_hf_cache_snapshots(resolved): + template = _chat_template_from_dir(snapshot, gguf_variant) + if template: + return template + except Exception as exc: + logger.debug("Could not read cached chat template for %s: %s", resolved, exc) + + try: + from huggingface_hub import HfApi, hf_hub_download + + _api = HfApi() + + def _remote_exceeds_cap(rel: str) -> bool: + # Best-effort: skip the download when the remote's advertised size + # exceeds the cap, so a maliciously large sidecar is never fetched. + try: + infos = _api.get_paths_info(resolved, [rel], repo_type = "model", token = hf_token) + except Exception: + return False + for info in infos: + size = getattr(info, "size", None) + if ( + getattr(info, "path", None) == rel + and isinstance(size, int) + and size > MAX_TEMPLATE_METADATA_BYTES + ): + return True + return False + + def _download_text(rel: str) -> Optional[str]: + if _remote_exceeds_cap(rel): + return None + try: + path = hf_hub_download( + resolved, + rel, + token = hf_token, + cache_dir = active_hf_hub_cache(), + ) + return _read_bounded_text(Path(path), MAX_TEMPLATE_METADATA_BYTES) + except Exception: + return None + + for rel in _JINJA_TEMPLATE_PATHS: + template = _download_text(rel) + if not template or not template.strip(): + continue + # A raw Jinja sidecar is the whole template, so it must fit the route's + # response cap (the local path skips oversized .jinja too). Download stays + # bounded at MAX_TEMPLATE_METADATA_BYTES so a large JSON embedding a small + # template still extracts below, but an over-cap Jinja is dropped so the + # search falls through to the tokenizer/processor template. + if len(template.encode("utf-8")) > MAX_CHAT_TEMPLATE_BYTES: + continue + return template + + for rel in _TOKENIZER_CONFIG_PATHS: + raw = _download_text(rel) + if not raw: + continue + try: + config = json.loads(raw) + except Exception: + continue + template = _chat_template_from_tokenizer_config(config) + if template: + return template + + for rel in _PROCESSOR_TEMPLATE_PATHS: + raw = _download_text(rel) + if not raw: + continue + try: + payload = json.loads(raw) + except Exception: + continue + template = _chat_template_from_processor_payload(payload) + if template: + return template + + return None + except Exception as exc: + logger.debug("Could not fetch chat template for %s: %s", resolved, exc) + return None diff --git a/studio/backend/plugins/data-designer-github-repo-seed/README.md b/studio/backend/plugins/data-designer-github-repo-seed/README.md index 346d94b305..44519496f5 100644 --- a/studio/backend/plugins/data-designer-github-repo-seed/README.md +++ b/studio/backend/plugins/data-designer-github-repo-seed/README.md @@ -4,7 +4,7 @@ A Data Designer seed-reader plugin for **Unsloth Studio** that scrapes real GitHub data (issues, pull requests, commits) from one or more repositories and hands it to the recipe pipeline as a seed dataset. -Designed to ship with Studio as a default seed source so any user with a +Designed to ship with Unsloth as a default seed source so any user with a GitHub token can build training datasets straight from live repos. ## What it does @@ -64,7 +64,7 @@ sleeps until reset when the budget drops below a safety threshold. ## Install -Shipped as a default Studio plugin. For development: +Shipped as a default Unsloth plugin. For development: ```bash pip install -e . diff --git a/studio/backend/plugins/data-designer-github-repo-seed/src/data_designer_github_repo_seed/__init__.py b/studio/backend/plugins/data-designer-github-repo-seed/src/data_designer_github_repo_seed/__init__.py index 62ecb2e280..d4d46da370 100644 --- a/studio/backend/plugins/data-designer-github-repo-seed/src/data_designer_github_repo_seed/__init__.py +++ b/studio/backend/plugins/data-designer-github-repo-seed/src/data_designer_github_repo_seed/__init__.py @@ -3,4 +3,4 @@ # Intentionally empty. Data-designer loads submodules lazily via qualified names # in plugin.py, so importing this package must not touch data_designer.engine.* -# during Studio bootstrap (circular import). +# during Unsloth bootstrap (circular import). diff --git a/studio/backend/plugins/data-designer-github-repo-seed/src/data_designer_github_repo_seed/scraper.py b/studio/backend/plugins/data-designer-github-repo-seed/src/data_designer_github_repo_seed/scraper.py index 637193e8b3..1af8133cc5 100644 --- a/studio/backend/plugins/data-designer-github-repo-seed/src/data_designer_github_repo_seed/scraper.py +++ b/studio/backend/plugins/data-designer-github-repo-seed/src/data_designer_github_repo_seed/scraper.py @@ -1,7 +1,7 @@ # SPDX-License-Identifier: AGPL-3.0-only # Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 -"""Multi-repo GitHub scraper for the Studio seed plugin. +"""Multi-repo GitHub scraper for the Unsloth seed plugin. Drives the GraphQL scraper in `scraper_impl/` per repo, capped via trial_limits to stop at `limit` items per resource. Then reads the per-resource JSONL shards diff --git a/studio/backend/requirements/extras-no-deps.txt b/studio/backend/requirements/extras-no-deps.txt index 5830a47789..3361af50dd 100644 --- a/studio/backend/requirements/extras-no-deps.txt +++ b/studio/backend/requirements/extras-no-deps.txt @@ -5,7 +5,7 @@ julius torchcodec==0.10.0 snac -# peft 0.19.0 causes export subprocess shutdown issues in Studio; +# peft 0.19.0 causes export subprocess shutdown issues in Unsloth; # installing with --no-deps to avoid pulling in torch>=0.11.0 peft==0.18.1 diff --git a/studio/backend/requirements/extras.txt b/studio/backend/requirements/extras.txt index 1baf2b6f2d..1601ccbfae 100644 --- a/studio/backend/requirements/extras.txt +++ b/studio/backend/requirements/extras.txt @@ -10,6 +10,7 @@ omegaconf einx pyloudnorm openai-whisper +av # PyAV: decode dictation audio (webm/opus/mp3/…) for the Whisper STT sidecar uroman # 4.0 MB - used for Outetts. MeCab # 19.9 MB - used for Outetts. inflect # number-to-words, required by OuteTTS diff --git a/studio/backend/requirements/no-torch-runtime.txt b/studio/backend/requirements/no-torch-runtime.txt index de321f80ed..847e89823b 100644 --- a/studio/backend/requirements/no-torch-runtime.txt +++ b/studio/backend/requirements/no-torch-runtime.txt @@ -7,7 +7,7 @@ # (current PyPI metadata still declares torch as a hard dep). # unsloth direct deps (from pyproject.toml [project].dependencies) -typer +typer>=0.12.0 # typer's full runtime dep tree. Required explicitly because this # file is installed with --no-deps. On Linux/Mac CI runners these # are often cached transitively; on a fresh windows-latest venv they @@ -70,7 +70,7 @@ cut_cross_entropy pillow # RAG store + document parsing, mirroring studio.txt. Pinned here because -# this file installs --no-deps; without them Studio runs with RAG disabled. +# this file installs --no-deps; without them Unsloth runs with RAG disabled. sqlite-vec==0.1.9 pymupdf==1.27.2.3 # 0.3.x keeps pymupdf-layout (which pulls onnxruntime) an optional extra; the diff --git a/studio/backend/requirements/single-env/constraints.txt b/studio/backend/requirements/single-env/constraints.txt index 0ed2bf8b26..0a5619924a 100644 --- a/studio/backend/requirements/single-env/constraints.txt +++ b/studio/backend/requirements/single-env/constraints.txt @@ -4,7 +4,7 @@ transformers==4.57.6 trl==0.23.1 huggingface-hub==0.36.2 -# Studio stack +# Unsloth stack datasets==4.3.0 pyarrow==23.0.1 diff --git a/studio/backend/requirements/studio.txt b/studio/backend/requirements/studio.txt index 6f4a5c3292..0c7503a5ca 100644 --- a/studio/backend/requirements/studio.txt +++ b/studio/backend/requirements/studio.txt @@ -1,4 +1,4 @@ -# Studio UI backend dependencies +# Unsloth UI backend dependencies typer fastapi uvicorn @@ -9,7 +9,7 @@ pandas nest_asyncio datasets==4.3.0 pyjwt -# gradio>=4.0.0 # 148 MB - Studio uses React + FastAPI, not Gradio +# gradio>=4.0.0 # 148 MB - Unsloth uses React + FastAPI, not Gradio huggingface-hub==0.36.2 structlog>=24.1.0 diceware diff --git a/studio/backend/routes/auth.py b/studio/backend/routes/auth.py index c61c1a16e4..1acc48e3a3 100644 --- a/studio/backend/routes/auth.py +++ b/studio/backend/routes/auth.py @@ -338,11 +338,11 @@ def _clear_login_bucket(key: tuple[str, str]) -> None: # so FastAPI runs it in the threadpool rather than blocking the event loop. @router.get("/identity") def identity(nonce: str, request: Request) -> dict: - """Challenge-response proof this is the real local Studio: caller sends a nonce, + """Challenge-response proof this is the real local Unsloth: caller sends a nonce, gets HMAC(install identity secret, nonce, connection address + port). Unauthenticated and side-effect free; a process that can't read the same-user secret can't forge a proof, and binding to the address/port the connection - landed on stops a squatter relaying a proof from the real Studio elsewhere.""" + landed on stops a squatter relaying a proof from the real Unsloth elsewhere.""" try: raw = base64.urlsafe_b64decode(nonce) except Exception: @@ -494,6 +494,11 @@ async def change_password( status_code = status.HTTP_401_UNAUTHORIZED, detail = "Current password is incorrect", ) + if any(ch.isspace() for ch in payload.new_password): + raise HTTPException( + status_code = status.HTTP_400_BAD_REQUEST, + detail = "New password cannot contain spaces", + ) if payload.current_password == payload.new_password: raise HTTPException( status_code = status.HTTP_400_BAD_REQUEST, diff --git a/studio/backend/routes/chat_history.py b/studio/backend/routes/chat_history.py index 7a27a58a52..6a0d49b47d 100644 --- a/studio/backend/routes/chat_history.py +++ b/studio/backend/routes/chat_history.py @@ -5,7 +5,7 @@ Chat history API routes backed by studio.db. """ -from typing import Any, Literal, Optional +from typing import Annotated, Any, Literal, Optional from fastapi import APIRouter, Depends, HTTPException, Query from pydantic import BaseModel, ConfigDict, Field, ValidationError @@ -19,13 +19,16 @@ from storage.studio_db import ( clear_chat_history, count_chat_threads, count_forks_for_message, + delete_chat_attachment, delete_chat_threads, delete_chat_project, ensure_chat_project_workspace, fork_chat_thread, + get_chat_attachment, get_chat_project, get_chat_thread, get_chat_message, + list_chat_attachments_page, list_chat_projects, list_chat_legacy_imports, list_chat_settings, @@ -157,11 +160,26 @@ class ChatInferenceSettings(BaseModel): fastMode: Optional[bool] = None +class ChatPresetLoadConfig(BaseModel): + model_config = ConfigDict(extra = "forbid") + + customContextLength: Optional[int] = Field(default = None, gt = 0) + maxSeqLength: Optional[float] = None + kvCacheDtype: Optional[str] = None + speculativeType: Optional[str] = None + specDraftNMax: Optional[int] = Field(default = None, ge = 1, le = 16) + tensorParallel: Optional[bool] = None + gpuMemoryMode: Optional[Literal["manual"]] = None + gpuLayers: Optional[int] = None + nCpuMoe: Optional[int] = Field(default = None, ge = 0) + + class ChatPreset(BaseModel): model_config = ConfigDict(extra = "forbid") name: str params: ChatInferenceSettings + loadConfig: Optional[ChatPresetLoadConfig] = None class ChatSettingsPayload(BaseModel): @@ -279,6 +297,131 @@ async def delete_threads( return {"status": "deleted"} +@router.get("/attachments") +def list_attachments( + limit: Annotated[int, Query(ge = 1, le = 100)] = 50, + offset: Annotated[int, Query(ge = 0)] = 0, + current_subject: str = Depends(get_current_subject), +) -> dict: + """One bounded page of chat uploads for the settings Data tab.""" + attachments, next_offset = list_chat_attachments_page(limit = limit, offset = offset) + return {"attachments": attachments, "nextOffset": next_offset} + + +def _decode_attachment_base64(payload: str) -> bytes: + """Strict base64 decode of a stored payload. + + Normalizes first: strips whitespace, fixes padding, accepts the URL-safe + alphabet. validate=False would silently drop bad characters and serve + corrupted bytes instead of failing, so raise 422 on anything else. + """ + import base64 + + normalized = "".join(payload.split()) + altchars = b"-_" if ("-" in normalized or "_" in normalized) else None + normalized += "=" * (-len(normalized) % 4) + try: + return base64.b64decode(normalized, altchars = altchars, validate = True) + except Exception as exc: # noqa: BLE001 - corrupt stored payload + raise HTTPException(status_code = 422, detail = "Attachment data is corrupt") from exc + + +_AUDIO_FORMAT_MEDIA_TYPES = { + "mp3": "audio/mpeg", + "wav": "audio/wav", + "ogg": "audio/ogg", + "flac": "audio/flac", +} + + +def _safe_image_media_type(media_type: str) -> str: + """Clamp a data-URL media type to something inert to render. + + Imported chats store image parts verbatim, so the embedded type can be + text/html or image/svg+xml; echoing those would execute markup with the + app origin when opened. Anything not a plain raster type downloads as + bytes instead. + """ + lowered = media_type.strip().lower() + if lowered.startswith("image/") and lowered != "image/svg+xml": + return lowered + return "application/octet-stream" + + +@router.get("/attachments/{message_id}/{attachment_id}/file") +def get_attachment_file( + message_id: str, + attachment_id: str, + current_subject: str = Depends(get_current_subject), +): + """Serve one attachment's stored content: image or audio bytes, or + extracted text.""" + import urllib.parse + + from fastapi.responses import Response + + attachment = get_chat_attachment(message_id, attachment_id) + if attachment is None: + raise HTTPException(status_code = 404, detail = "Attachment not found") + + attachment_content_type = attachment.get("contentType") + texts: list[str] = [] + for part in attachment.get("content") or []: + if not isinstance(part, dict): + continue + image = part.get("image") + if isinstance(image, str) and image[:5].lower() == "data:": + header, _, payload = image.partition(",") + media_type = _safe_image_media_type( + header[5:].split(";", 1)[0] or "application/octet-stream" + ) + if "base64" not in header.lower(): + # RFC 2397 non-base64 form stores percent-encoded bytes. + data = urllib.parse.unquote_to_bytes(payload) + return Response(content = data, media_type = media_type) + data = _decode_attachment_base64(payload) + return Response(content = data, media_type = media_type) + # Audio parts: the attachment adapter stores {data, format} with raw + # base64; compare chats store a bare base64 string. + audio = part.get("audio") + if isinstance(audio, dict) or (isinstance(audio, str) and audio): + if isinstance(audio, dict): + payload = audio.get("data") + audio_format = audio.get("format") + else: + payload = audio.rsplit(",", 1)[-1] + audio_format = None + if isinstance(payload, str) and payload: + data = _decode_attachment_base64(payload) + media_type = ( + attachment_content_type + if isinstance(attachment_content_type, str) + and attachment_content_type.startswith("audio/") + else _AUDIO_FORMAT_MEDIA_TYPES.get( + str(audio_format or "").lower(), "application/octet-stream" + ) + ) + return Response(content = data, media_type = media_type) + text = part.get("text") + if isinstance(text, str) and text: + texts.append(text) + if texts: + return Response(content = "\n".join(texts), media_type = "text/plain; charset=utf-8") + raise HTTPException(status_code = 404, detail = "Attachment has no stored content") + + +@router.delete("/attachments/{message_id}/{attachment_id}") +def delete_attachment( + message_id: str, + attachment_id: str, + current_subject: str = Depends(get_current_subject), +) -> dict: + """Remove one attachment from its chat message.""" + if not delete_chat_attachment(message_id, attachment_id): + raise HTTPException(status_code = 404, detail = "Attachment not found") + return {"ok": True} + + @router.get("/projects", response_model = ChatProjectListResponse) async def list_projects( include_archived: bool = Query(False), current_subject: str = Depends(get_current_subject) @@ -409,7 +552,7 @@ async def get_thread_message( @router.put("/threads/{thread_id}/messages/{message_id}", response_model = ChatMessage) -async def save_thread_message( +def save_thread_message( thread_id: str, message_id: str, payload: ChatMessage, @@ -432,7 +575,7 @@ async def save_thread_message( @router.put("/threads/{thread_id}/messages", response_model = ChatMessageListResponse) -async def replace_thread_messages( +def replace_thread_messages( thread_id: str, payload: ChatMessageSyncRequest, current_subject: str = Depends(get_current_subject), diff --git a/studio/backend/routes/data_recipe/jobs.py b/studio/backend/routes/data_recipe/jobs.py index 59714380da..e870e8855e 100644 --- a/studio/backend/routes/data_recipe/jobs.py +++ b/studio/backend/routes/data_recipe/jobs.py @@ -37,7 +37,7 @@ def _resolve_local_v1_endpoint(request: Request) -> str: Resolution order: 1. ``app.state.server_port`` (run.py, post-bind) - survives proxies/tunnels. - 2. ``request.scope["server"]`` - when Studio starts outside ``run_server``. + 2. ``request.scope["server"]`` - when Unsloth starts outside ``run_server``. 3. parsed ``request.base_url`` - last resort for test fixtures. """ port: Any = getattr(request.app.state, "server_port", None) diff --git a/studio/backend/routes/datasets.py b/studio/backend/routes/datasets.py index 46319ca2ba..331eb5e1e0 100644 --- a/studio/backend/routes/datasets.py +++ b/studio/backend/routes/datasets.py @@ -23,40 +23,6 @@ def _is_valid_repo_id(repo_id: str) -> bool: return bool(_VALID_REPO_ID.fullmatch(repo_id)) -_dataset_size_cache: dict[str, int] = {} - - -def _get_dataset_size_cached(repo_id: str) -> int: - if repo_id in _dataset_size_cache: - return _dataset_size_cache[repo_id] - try: - from huggingface_hub import dataset_info as hf_dataset_info - - info = hf_dataset_info(repo_id, token = None, files_metadata = True) - total = sum(s.size for s in info.siblings if getattr(s, "size", None)) - _dataset_size_cache[repo_id] = total - return total - except Exception: - return 0 - - -def _resolve_hf_cache_realpath(repo_dir: Path) -> Optional[str]: - """Resolved realpath for a HF cache repo dir: most-recent snapshot, else cache root. - - Mirrors routes/models.py; duplicated here to keep this module self-contained. - """ - try: - snapshots_dir = repo_dir / "snapshots" - if snapshots_dir.is_dir(): - snaps = [s for s in snapshots_dir.iterdir() if s.is_dir()] - if snaps: - latest = max(snaps, key = lambda s: s.stat().st_mtime) - return str(latest.resolve()) - return str(repo_dir.resolve()) - except Exception: - return None - - backend_path = Path(__file__).parent.parent.parent if str(backend_path) not in sys.path: sys.path.insert(0, str(backend_path)) @@ -64,6 +30,7 @@ if str(backend_path) not in sys.path: from utils.datasets import check_dataset_format from utils.upload_limits import get_upload_limit_bytes, get_upload_limit_label from auth.authentication import get_current_subject +from hub.dependencies import get_hf_token router = APIRouter() logger = get_logger(__name__) @@ -292,11 +259,13 @@ def _download_hf_metadata(*, repo_id: str, repo_files: list[str], token: str | N try: from huggingface_hub import hf_hub_download + from utils.hf_cache_settings import active_hf_hub_cache local_path = hf_hub_download( repo_id = repo_id, filename = metadata_file, repo_type = "dataset", token = token, + cache_dir = active_hf_hub_cache(), ) except Exception as exc: logger.warning(f"Could not read HF dataset metadata for {repo_id}: {exc}") @@ -485,7 +454,7 @@ async def upload_dataset( # Stream to disk in chunks to avoid holding the whole file in memory. The # route-level cap gives a clear training-dataset error and avoids leaving - # oversized partial files in the Studio uploads directory. + # oversized partial files in the Unsloth uploads directory. upload_limit_bytes = get_upload_limit_bytes() total_bytes = 0 upload_complete = False @@ -525,77 +494,15 @@ def list_local_datasets( @router.get("/download-progress") async def get_dataset_download_progress( repo_id: str = Query(..., description = "HuggingFace dataset repo ID, e.g. 'unsloth/LaTeX_OCR'"), + hf_token: Optional[str] = Depends(get_hf_token), current_subject: str = Depends(get_current_subject), ): - """Return download progress for a HuggingFace dataset repo. - - Mirrors ``GET /api/models/download-progress`` but scans the - ``datasets--owner--name`` cache dir under HF_HUB_CACHE, where in-progress - download bytes are visible. Returns ``cache_path`` so the UI can show it. - """ - _empty = { - "downloaded_bytes": 0, - "expected_bytes": 0, - "progress": 0, - "cache_path": None, - } - try: - if not _is_valid_repo_id(repo_id): - return _empty - - from huggingface_hub import constants as hf_constants - - cache_dir = Path(hf_constants.HF_HUB_CACHE) - target = f"datasets--{repo_id.replace('/', '--')}".lower() - completed_bytes = 0 - in_progress_bytes = 0 - cache_path: Optional[str] = None - - if cache_dir.is_dir(): - for entry in cache_dir.iterdir(): - if entry.name.lower() != target: - continue - cache_path = _resolve_hf_cache_realpath(entry) - blobs_dir = entry / "blobs" - if not blobs_dir.is_dir(): - break - for f in blobs_dir.iterdir(): - if not f.is_file(): - continue - if f.name.endswith(".incomplete"): - in_progress_bytes += f.stat().st_size - else: - completed_bytes += f.stat().st_size - break - - downloaded_bytes = completed_bytes + in_progress_bytes - if downloaded_bytes == 0: - return {**_empty, "cache_path": cache_path} - - expected_bytes = _get_dataset_size_cached(repo_id) - if expected_bytes <= 0: - return { - "downloaded_bytes": downloaded_bytes, - "expected_bytes": 0, - "progress": 0, - "cache_path": cache_path, - } - - # 95% threshold (as in the model endpoint): HF blob dedup makes - # completed_bytes drift under expected_bytes; inter-file gaps look "done". - if completed_bytes >= expected_bytes * 0.95: - progress = 1.0 - else: - progress = min(downloaded_bytes / expected_bytes, 0.99) - return { - "downloaded_bytes": downloaded_bytes, - "expected_bytes": expected_bytes, - "progress": round(progress, 3), - "cache_path": cache_path, - } - except Exception as e: - logger.warning(f"Error checking dataset download progress for {repo_id}: {e}") - return _empty + """Compatibility route backed by the shared multi-cache progress service.""" + from hub.services.datasets import downloads + return await downloads.get_dataset_download_progress_response( + repo_id, + hf_token = hf_token, + ) @router.post("/check-format", response_model = CheckFormatResponse) diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.py index 0541a81ff2..445a26f04d 100644 --- a/studio/backend/routes/inference.py +++ b/studio/backend/routes/inference.py @@ -13,13 +13,14 @@ from pathlib import Path from fastapi import APIRouter, Depends, HTTPException, Request, status from fastapi.responses import StreamingResponse, JSONResponse, Response from starlette.requests import ClientDisconnect -from typing import Any, Callable, List, Optional, Union +from typing import Any, Callable, List, Literal, Optional, Union import json import httpx from loggers import get_logger import asyncio import threading import weakref +from contextlib import ExitStack import re as _re @@ -28,6 +29,9 @@ import re as _re from utils.models import extract_model_size_b as _extract_model_size_b from utils.api_errors import openai_error_body, anthropic_error_body +from utils.upload_limits import STT_AUDIO_B64_MAX_CHARS, STT_AUDIO_RAW_MAX_BYTES +from hub.dependencies import get_hf_token +from core.inference.orchestrator import GenStreamError, GenStreamErrorRaised from core.inference.llama_admission import ( LlamaAdmissionCancelled, LlamaAdmissionConfig, @@ -90,7 +94,7 @@ def _mlx_distributed_launch_detected() -> bool: def _install_httpcore_asyncgen_silencer() -> None: """Silence benign httpx/httpcore asyncgen GC noise on Python 3.13. - When Studio proxies a llama-server stream via httpx, the innermost + When Unsloth proxies a llama-server stream via httpx, the innermost ``HTTP11ConnectionByteStream.__aiter__`` async generator is finalised by the asyncgen GC hook on a task different from the one that opened it. Its ``aclose`` calls ``anyio.Lock.acquire`` → ``cancel_shielded_checkpoint``, @@ -212,6 +216,14 @@ def _friendly_error(exc: Exception) -> str: return "An internal error occurred" +def _friendly_gen_stream_error(value) -> str: + """Return a client-safe message for typed local generation errors.""" + text = str(value) + if getattr(value, "public", False): + return text + return safe_error_detail(RuntimeError(text), fallback = "An internal error occurred.") + + def _friendly_upstream_error(text: str) -> str: """Rewrite a raw llama-server error body into an actionable message where we can. @@ -219,14 +231,14 @@ def _friendly_upstream_error(text: str) -> str: parse grammar" / "failed to initialize samplers"). This surfaces to coding agents as a hard 400 on every tool-bearing turn. It is a llama-server limitation with some model/quant + tool-schema combinations, and recent llama.cpp builds handle the common - coding-agent tools, so point the user at updating Studio rather than the raw body. + coding-agent tools, so point the user at updating Unsloth rather than the raw body. """ lowered = text.lower() if "failed to parse grammar" in lowered or "failed to initialize samplers" in lowered: return ( "The model couldn't compile a tool-calling grammar for this request. This is a " "llama-server limitation with some model/quant and tool-schema combinations. " - "Update Studio (it installs the latest llama.cpp, which handles the common " + "Update Unsloth (it installs the latest llama.cpp, which handles the common " "coding-agent tools) or try a different GGUF model." ) return f"llama-server error: {text}" @@ -721,7 +733,7 @@ def _openai_passthrough_sse_line_terminal_state(raw_line: str) -> Optional[str]: Some llama-server builds can emit the logical final chunk (``finish_reason``) and optional usage chunk, then keep the HTTP stream open without sending the - OpenAI ``data: [DONE]`` sentinel. Classifying those chunks lets Studio close + OpenAI ``data: [DONE]`` sentinel. Classifying those chunks lets Unsloth close the client stream promptly while preserving an optional trailing usage chunk. """ if not raw_line.startswith("data:"): @@ -998,6 +1010,7 @@ try: from core.inference.llama_server_args import ( _effective_tensor_parallel, _tensor_parallel_matches_loaded, + extra_args_disable_mmproj, parse_split_mode_override, resolve_tensor_parallel, strip_shadowing_flags, @@ -1035,6 +1048,7 @@ except ImportError: from core.inference.llama_server_args import ( _effective_tensor_parallel, _tensor_parallel_matches_loaded, + extra_args_disable_mmproj, parse_split_mode_override, resolve_tensor_parallel, strip_shadowing_flags, @@ -1680,6 +1694,8 @@ async def _aiter_llama_stream_items( from models.inference import ( LoadRequest, UnloadRequest, + TranscribeRequest, + SttLoadRequest, GenerateRequest, LoadResponse, LoadProgressResponse, @@ -1766,7 +1782,7 @@ from core.inference.providers import get_base_url from core.inference.external_provider import ExternalProviderClient from core.inference.chat_templates import resolve_effective_chat_template_override from storage import providers_db -from utils.utils import safe_error_detail, log_and_http_error +from utils.utils import is_hf_authentication_error, safe_error_detail, log_and_http_error import io import base64 @@ -1774,7 +1790,7 @@ import numpy as np from datetime import date as _date router = APIRouter() -# Studio-only router (not mounted on /v1 OpenAI-compat). +# Unsloth-only router (not mounted on /v1 OpenAI-compat). studio_router = APIRouter() @@ -1915,16 +1931,57 @@ async def artifact_preview_frame(allow_network: bool = False): _BARE_JSON_NAME_MARKER_RE = _re.compile(r'\{\s*\\?"(?:name|function)\\?"\s*:') -def _detect_safetensors_features(backend, chat_template: Optional[str]) -> dict: +def _detect_safetensors_features( + backend, + chat_template: Optional[str], + tools = None, +) -> dict: """Classify reasoning/tool capabilities via the GGUF classifier so flags match across backends. gpt-oss is overridden: Harmony routes reasoning and tools through tokenizer channels, not template markup.""" model_id = getattr(backend, "active_model_name", None) + feature_template = chat_template + try: + from core.inference.chat_template_helpers import _selected_template_strings_from_value + selected_templates = _selected_template_strings_from_value(chat_template, tools) + if selected_templates: + feature_template = selected_templates[0] + except Exception: + logger.debug("safetensors_named_template_selection_failed", exc_info = True) flags = detect_reasoning_flags( - chat_template, + feature_template, model_identifier = model_id, log_source = "safetensors", ) + if not flags.get("supports_reasoning"): + try: + from core.inference.chat_template_helpers import ( + detect_reasoning_channel_markers_from_template, + ) + + templates = [chat_template] + models = getattr(backend, "models", None) + model_info = ( + models.get(model_id, {}) + if isinstance(models, dict) and model_id is not None + else {} + ) + if isinstance(model_info, dict): + templates.extend( + ( + model_info.get("native_chat_template"), + (model_info.get("chat_template_info") or {}).get("template"), + ) + ) + if any( + detect_reasoning_channel_markers_from_template(template, tools = tools) is not None + for template in templates + ): + flags["supports_reasoning"] = True + flags["reasoning_always_on"] = True + logger.info("safetensors: model always reasons (native channel markers)") + except Exception: + logger.debug("safetensors_native_reasoning_marker_check_failed", exc_info = True) # Markers any supported parser recognises (template advertises tools but # uses none -> drop the pill). Reuse the parser's own signal list so this # gate never drifts (a hand-maintained copy lost the DeepSeek variants); @@ -1938,9 +1995,9 @@ def _detect_safetensors_features(backend, chat_template: Optional[str]) -> dict: ) if ( flags.get("supports_tools") - and chat_template - and not any(m in chat_template for m in _PARSER_MARKERS) - and not _BARE_JSON_NAME_MARKER_RE.search(chat_template) + and isinstance(feature_template, str) + and not any(m in feature_template for m in _PARSER_MARKERS) + and not _BARE_JSON_NAME_MARKER_RE.search(feature_template) ): logger.info( "safetensors: template advertises tools but uses an " @@ -2055,9 +2112,9 @@ def _effective_enable_tools(payload) -> Optional[bool]: def _explicit_studio_tool_loop_requested(payload) -> bool: - """True when the request itself asks Studio to execute local tools. + """True when the request itself asks Unsloth to execute local tools. - Process-wide CLI policy can default Studio's tool loop on for ordinary chat, + Process-wide CLI policy can default Unsloth's tool loop on for ordinary chat, but it must not steal OpenAI-compatible client tools or response_format requests from the llama-server passthrough path. A policy of ``False`` (--disable-tools) vetoes even an explicit ``enable_tools: true`` ask. @@ -2069,7 +2126,7 @@ def _explicit_studio_tool_loop_requested(payload) -> bool: def _permission_mode_confirm(payload) -> bool: - """Effective confirm-gate intent for Studio's own local tool loop. + """Effective confirm-gate intent for Unsloth's own local tool loop. Honors the documented default that an unset permission_mode behaves as "ask". An explicit confirm_tool_calls (True or False) wins; explicit @@ -2091,7 +2148,7 @@ def _permission_mode_confirm(payload) -> bool: def _confirm_gate_needs_stream(payload) -> bool: - """Whether Studio's local tool-loop confirm gate still requires stream=true. + """Whether Unsloth's local tool-loop confirm gate still requires stream=true. The gate can only prompt while streaming, so a non-streaming request that will prompt must 400 up front. auto ("Approve for me") only prompts for a call the @@ -3062,13 +3119,16 @@ def _normalise_settings_str(value: Optional[str]) -> Optional[str]: def _should_strip_split_mode(request: LoadRequest, backend_extra: Optional[list[str]]) -> bool: - """Whether an inherited --split-mode should be stripped on reload. + """Whether an inherited --split-mode (and its coupled --tensor-split) should + be stripped on reload. The binary Tensor Parallelism toggle can't carry --split-mode's row/none/ layer modes, so only strip when the toggle overrides it: tensor being turned on, or the inherited mode is tensor (toggle turning it off). Non-tensor modes - survive. Shared by the inheritance strip and the already-loaded stale check - so they agree on what reload would do. + survive. A manual per-GPU ratio is handled by _should_strip_tensor_split, + which strips only --tensor-split so the inherited mode is kept. Shared by the + inheritance strip and the already-loaded stale check so they agree on what + reload would do. """ fields_set = getattr(request, "model_fields_set", set()) return "tensor_parallel" in fields_set and ( @@ -3076,6 +3136,25 @@ def _should_strip_split_mode(request: LoadRequest, backend_extra: Optional[list[ ) +def _should_strip_tensor_split(request: LoadRequest) -> bool: + """Whether an inherited --tensor-split alone should be stripped on reload. + + Manual explicit offload (gpu_layers >= 0) owns the per-GPU split: with a ratio + it emits its own --tensor-split (an inherited one, appended last, would + override it), and with the ratio cleared it wants llama.cpp's default + free-VRAM split. Either way an inherited --tensor-split must go, else the + cleared case silently keeps the stale ratio while status reports None. + Unlike _should_strip_split_mode this leaves --split-mode untouched, so a + user's row/none/layer mode survives a Studio split-ratio edit. When the + Tensor Parallelism toggle IS overriding the mode, _should_strip_split_mode + (called alongside this at every site) strips --split-mode anyway. + """ + return ( + getattr(request, "gpu_memory_mode", "auto") == "manual" + and getattr(request, "gpu_layers", -1) >= 0 + ) + + def _carry_preserved_tensor_intent( *, preserved: bool, same_model: bool, explicit_drop: bool ) -> bool: @@ -3090,7 +3169,7 @@ def _is_explicit_tensor_drop(request: LoadRequest) -> bool: """True only when the request explicitly selects a non-tensor --split-mode (e.g. layer/row/none), a deliberate departure from a preserved tensor->layer fallback. - A bare tensor_parallel field is NOT a drop: the Studio UI always sends it and echoes + A bare tensor_parallel field is NOT a drop: the Unsloth UI always sends it and echoes the /load response's resolved value back, so after a fallback every reload carries tensor_parallel=false even though the user never changed it -- treating that as a drop would collapse the preserved multi-GPU placement on the next ctx/settings reload. An @@ -3134,12 +3213,39 @@ def _request_matches_loaded_settings( else strip_shadowing_flags( backend_extra, strip_split_mode = _should_strip_split_mode(request, backend_extra), + strip_tensor_split = _should_strip_tensor_split(request), + strip_offload = request.gpu_memory_mode == "manual", ) ) if not _tensor_parallel_matches_loaded( effective_extra, request.tensor_parallel, llama_backend.tensor_parallel ): return False + # The diffusion runner is mode-agnostic (it always reports "auto" and ignores + # the layer/MoE/split knobs), so a standing manual preference in the request + # must not force a needless reload -- only the GPU pick matters. + if not llama_backend.is_diffusion: + if request.gpu_memory_mode != llama_backend.gpu_memory_mode: + return False + # Manual: a layer-count change always reloads; MoE/split only matter with + # an explicit offload (gpu_layers >= 0), so a leftover value under Auto + # must not force one. Mirrors LlamaCppBackend._already_in_target_state. + if request.gpu_memory_mode == "manual" and ( + request.gpu_layers != llama_backend.gpu_layers + or ( + request.gpu_layers >= 0 + and ( + request.n_cpu_moe != llama_backend.n_cpu_moe + or (request.tensor_split or None) != (llama_backend.tensor_split or None) + ) + ) + ): + return False + # A regular GGUF may narrow the requested placement pool. Accept either the + # original request or the effective status-echoed subset; diffusion keeps + # its single-device normalization. + if not llama_backend.matches_gpu_ids(request.gpu_ids): + return False # Preserved tensor->layer fallback (both report tensor=off, so the check above # matches): if the user now explicitly drops tensor intent, reload so placement # re-selects instead of keeping the all-GPU mask (#6659). The effective check @@ -3182,14 +3288,17 @@ def _request_matches_loaded_settings( # contain any shadow flag, so the reload path strips them rather than # leaving a stale override in effect. (backend_extra computed above.) if request.llama_extra_args is None: - # Mirror the reload's conditional split-mode strip, so a preserved - # non-tensor mode (row/none/layer) isn't seen as stale and doesn't - # trigger a needless reload of a healthy server. + # Mirror the reload's conditional strips, so a preserved non-tensor mode + # (row/none/layer) isn't seen as stale and doesn't trigger a needless + # reload of a healthy server, while an inherited offload/ratio flag that + # the reload *would* strip is correctly seen as stale. if ( backend_extra and strip_shadowing_flags( backend_extra, strip_split_mode = _should_strip_split_mode(request, backend_extra), + strip_tensor_split = _should_strip_tensor_split(request), + strip_offload = request.gpu_memory_mode == "manual", ) != backend_extra ): @@ -3296,9 +3405,8 @@ async def _acquire_swap_gate() -> None: await asyncio.sleep(0.02) -# Counts in-flight auto-switch requests per (target, variant). The busy guard -# subtracts same-target waiters so concurrent requests for one model load once -# instead of each 409-ing the other. +# Counts auto-switch requests queued to load each (target, variant). They are not +# generating, so the drain wait below excludes them from the active inference count. _auto_switch_waiters: dict[tuple[str, str], int] = {} _auto_switch_waiters_guard = threading.Lock() @@ -3316,35 +3424,31 @@ def _note_switch_waiter(key: tuple[str, str], delta: int) -> None: _auto_switch_waiters.pop(key, None) -def _same_target_waiters(key: tuple[str, str]) -> int: +def _switch_waiter_count() -> int: with _auto_switch_waiters_guard: - return _auto_switch_waiters.get(key, 0) + return sum(max(0, count) for count in _auto_switch_waiters.values()) -# A second waiter map keyed by the raw requested model, registered before the -# (slow) resolve. The middleware counts a concurrent same-model request as -# in-flight before it resolves and joins _auto_switch_waiters, so without this -# the first request would see it as an unrelated request and 409. -_auto_switch_request_waiters: dict[str, int] = {} -_auto_switch_request_waiters_guard = threading.Lock() +async def _wait_for_model_switch_idle(*, current_request_counted: bool) -> None: + """Wait until a model replacement cannot interrupt active inference. - -def _request_waiter_key(requested_model: str) -> str: - return requested_model.strip().lower() - - -def _note_request_waiter(key: str, delta: int) -> None: - with _auto_switch_request_waiters_guard: - n = _auto_switch_request_waiters.get(key, 0) + delta - if n > 0: - _auto_switch_request_waiters[key] = n - else: - _auto_switch_request_waiters.pop(key, None) - - -def _same_request_waiters(key: str) -> int: - with _auto_switch_request_waiters_guard: - return _auto_switch_request_waiters.get(key, 0) + The caller holds ``inference_lifecycle_gate``, which prevents new inference + from starting while existing requests drain. Auto-switch requests that have + resolved their targets are scheduler waiters, not active generations, so + exclude them to avoid a queue deadlock. + """ + from core.inference.llama_keepwarm import other_inference_request_count + while True: + queued_switches = _switch_waiter_count() + if current_request_counted and queued_switches > 0: + queued_switches -= 1 + active_others = other_inference_request_count( + current_request_counted = current_request_counted, + include_pending = False, + ) + if active_others <= queued_switches: + return + await asyncio.sleep(0.02) def _llama_public_model_id(llama_backend, fallback: Optional[str] = None) -> Optional[str]: @@ -3472,7 +3576,6 @@ async def _maybe_auto_switch_model( from core.inference.local_model_resolver import resolve_local_gguf from core.inference.llama_keepwarm import ( get_last_unloaded_model, - other_inference_request_count, inference_lifecycle_gate, ) @@ -3493,12 +3596,7 @@ async def _maybe_auto_switch_model( if not auto_switch_on and get_auto_unload_idle_seconds() <= 0: return - # Register by the raw requested model before resolving (which can be slow): - # the middleware already counts a concurrent same-model request as in-flight, - # so the busy guard must know it shares this target even while it resolves. - request_key = _request_waiter_key(requested_model) - _note_request_waiter(request_key, 1) - try: + async def _resolve_and_switch() -> None: # Off the loop: a cold-cache rebuild walks several model dirs + HF caches. # With auto-switch off (or an omitted-model reload-only request), skip the # resolve so only the reload-stash path runs and no name is ever matched. @@ -3596,6 +3694,7 @@ async def _maybe_auto_switch_model( ) key = _switch_key(override_id, variant) _note_switch_waiter(key, 1) + waiter_noted = True try: async with _auto_switch_lock(): # The asyncio lock is per loop; add a process-wide gate so a swap on @@ -3608,31 +3707,6 @@ async def _maybe_auto_switch_model( if _already_serving(): _record_serving_alias() return - # Single slot: refuse a cross-model swap while another inference - # request is active rather than killing its response. Requests - # heading to this same target (by resolved id or raw name) are - # excluded, so concurrent requests for one model load once. A - # pending request is still in the middleware, not generating, so - # it is not counted here. - same_others = max( - _same_target_waiters(key) - 1, _same_request_waiters(request_key) - 1, 0 - ) - others = other_inference_request_count( - current_request_counted = True, include_pending = False - ) - # Not gated on the GGUF being loaded: _load_model_impl also - # tears down an active Unsloth backend before loading a GGUF, - # so refuse whenever any other inference request is in flight. - if others > same_others: - raise HTTPException( - status_code = 409, - detail = openai_error_body( - "Cannot switch models while another inference request is in progress.", - status = 409, - code = "model_switch_busy", - param = "model", - ), - ) # Apply this model's saved launch flags so the swap honors the config. override = get_model_override(override_id) load_kwargs = {"model_path": target_id, "gguf_variant": variant} @@ -3647,16 +3721,22 @@ async def _maybe_auto_switch_model( LoadRequest(**load_kwargs), fastapi_request, current_subject, + current_request_counted = True, ) # Advertise the repo id (not the concrete load path) as the loaded # model's public id and override key for /v1/models and idle stash. get_llama_cpp_backend()._openai_advertised_id = override_id finally: + # Deregister before releasing the gate: otherwise a swap on another + # loop counts this finished request as queued and unloads its model. + _note_switch_waiter(key, -1) + waiter_noted = False _auto_switch_process_lock.release() finally: - _note_switch_waiter(key, -1) - finally: - _note_request_waiter(request_key, -1) + if waiter_noted: + _note_switch_waiter(key, -1) + + await _resolve_and_switch() async def _auto_switch_from_request_body(request: Request, current_subject: str): @@ -3808,6 +3888,113 @@ def _estimate_gguf_required_gb( return None +def _classify_diffusion_gguf(config: ModelConfig) -> Optional[bool]: + """Classify a GGUF as diffusion, normal, or unknown before it is loaded. + + ``None`` is important here: a remote GGUF whose header is not cached can + still be routed to the single-GPU diffusion runner after download. Default + placement keeps that unknown case guarded until the header is available. + """ + identity = " ".join( + str(getattr(config, attr, "") or "") for attr in ("identifier", "gguf_hf_repo", "gguf_file") + ).lower() + # Name-only hint, used ONLY as a pre-download fallback, scoped to the + # DiffusionGemma runner family: a bare "diffusion" substring is common in + # ordinary text-model names/paths (e.g. "stable-diffusion-prompt"), and treating + # those as diffusion falsely rejects a valid Vulkan+gpu_ids GGUF (#7239). Normalize + # non-alphanumerics so "DiffusionGemma"/"diffusion-gemma" collapse to one token. + # The local header below stays authoritative. + name_says_diffusion = "diffusiongemma" in _re.sub(r"[^a-z0-9]+", "", identity) + + try: + main = getattr(config, "gguf_file", None) + if not (main and Path(main).is_file()): + repo = getattr(config, "gguf_hf_repo", None) + variant = getattr(config, "gguf_variant", None) + if repo and variant: + from hub.utils.gguf import resolve_local_gguf_path + main = resolve_local_gguf_path(repo, variant) + if main and Path(main).is_file(): + # The local GGUF header is authoritative (same probe the loader uses), so + # it can't be fooled by a "diffusion"-flavored name/path. + probe = LlamaCppBackend() + probe._read_gguf_metadata(str(main)) + if probe.is_diffusion: + return True + # A decoded architecture proves a normal llama-server GGUF; no architecture + # means the probe was inconclusive, so fall through to the name hint below. + if getattr(probe, "_architecture", None): + return False + except Exception as e: + logger.debug("Could not identify diffusion GGUF for training guard: %s", e) + + # Header unavailable (remote uncached) or inconclusive: True only for the + # DiffusionGemma name family; otherwise None keeps an unknown remote GGUF guarded + # as potentially diffusion until its header proves otherwise. + return True if name_says_diffusion else None + + +async def _resolve_gguf_gpu_ids_for_request( + config: ModelConfig, gpu_ids: Optional[List[int]] +) -> Optional[List[int]]: + """Resolve and fully validate an explicit GGUF GPU placement pool. + + CUDA and ROCm use physical IDs. Vulkan uses ggml ordinals, so its device + existence check comes from the same ggml probe used by the loader. Both + /load and /validate call this before their training guard or any teardown. + """ + if not gpu_ids: + return None + + from utils.hardware import DeviceType, get_device + from utils.hardware.hardware import resolve_requested_gpu_ids + + is_vulkan = LlamaCppBackend._is_vulkan_backend() + if get_device() == DeviceType.XPU and not is_vulkan: + raise HTTPException( + status_code = 400, + detail = ( + "GPU selection (gpu_ids) is not supported on Intel XPU. " + "Omit gpu_ids to use all devices." + ), + ) + + if is_vulkan and _classify_diffusion_gguf(config) is True: + raise HTTPException( + status_code = 400, + detail = ( + "GPU selection (gpu_ids) is not supported for a DiffusionGemma " + "GGUF on a Vulkan llama.cpp build: the diffusion runner selects " + "its device by CUDA physical index, which has no defined mapping " + "to ggml Vulkan device ordinals. Omit gpu_ids to use the default " + "device." + ), + ) + + try: + resolved = resolve_requested_gpu_ids(gpu_ids, is_vulkan = is_vulkan) + except ValueError as exc: + raise HTTPException(status_code = 400, detail = str(exc)) from exc + + if is_vulkan and resolved: + binary = LlamaCppBackend._find_llama_server_binary() + if binary: + probed = { + gpu[0] for gpu in await asyncio.to_thread(LlamaCppBackend._get_gpu_memory, binary) + } + wanted = {int(gpu_id) for gpu_id in resolved} + if not wanted.issubset(probed): + raise HTTPException( + status_code = 400, + detail = ( + f"Requested Vulkan GPU ordinal(s) {sorted(wanted)} not " + f"present. Available Vulkan devices: {sorted(probed)}." + ), + ) + + return resolved + + def _guard_chat_load_against_training( config: ModelConfig, *, @@ -3818,11 +4005,19 @@ def _guard_chat_load_against_training( requested_gpu_ids: Optional[List[int]], llama_extra_args: Optional[list[str]] = None, n_parallel: int = 1, + gpu_memory_mode: Literal["auto", "manual"] = "auto", ) -> None: - """Refuse loading a local chat model that would OOM an active training run. + """Protect active training from automatically placed chat-model loads. + No-op when training is inactive or unknown. `load_in_4bit` must be the - effective quantization (see _effective_load_in_4bit). Raises HTTP 409 when the - model would not fit alongside training.""" + effective quantization (see _effective_load_in_4bit). Manual chat-GGUF + placement is an explicit override: Auto layers delegate fitting to + llama.cpp's ``--fit`` and pinned layers are owned by the user, so neither is + estimated here. Diffusion is still guarded because its mode-agnostic runner + ignores those controls and uses one GPU. An unclassified GGUF is guarded as + potentially diffusion until its local header proves otherwise. Other loads + raise HTTP 409 when they would not fit beside training. + """ from core.training import get_training_backend from routes.training_vram import can_load_chat_during_training @@ -3834,6 +4029,29 @@ def _guard_chat_load_against_training( return is_gguf = bool(getattr(config, "is_gguf", False)) + diffusion_kind = _classify_diffusion_gguf(config) if is_gguf else False + if is_gguf and gpu_memory_mode == "manual" and diffusion_kind is False: + return + + # Vulkan GGUF pins are ggml ordinals, not CUDA physical IDs. Detect this + # before deriving a possible diffusion fallback device so an unknown remote + # GGUF never sends its ordinal through the CUDA single-device path. + is_vulkan = False + if is_gguf: + try: + is_vulkan = LlamaCppBackend._is_vulkan_backend() + except Exception as e: + logger.warning("Could not detect Vulkan backend for chat-load guard: %s", e) + + diffusion_gpu = None + if is_gguf and diffusion_kind is not False and not (is_vulkan and requested_gpu_ids): + # Use the same token selection as the runner: an explicit pick wins, + # followed by DG_GPU, the first parent-visible token, then GPU 0. + diffusion_gpu = LlamaCppBackend._diffusion_gpu_arg( + requested_gpu_ids, + cpu_only = LlamaCppBackend._effective_gpu_count() == 0, + ) + required_override_gb = ( _estimate_gguf_required_gb( config, @@ -3853,7 +4071,9 @@ def _guard_chat_load_against_training( max_seq_length = max_seq_length, requested_gpu_ids = requested_gpu_ids, is_gguf = is_gguf, + is_vulkan = is_vulkan, required_override_gb = required_override_gb, + single_device_gpu = diffusion_gpu, ) if ok: return @@ -3881,6 +4101,98 @@ def _guard_chat_load_against_training( raise HTTPException(status_code = 409, detail = detail) +def _resolve_inherited_extra_args( + request, + config: ModelConfig, + model_identifier: str, + extra_llama_args: Optional[list[str]], + effective_chat_template_override: Optional[str] = None, +) -> Optional[list[str]]: + """Effective pass-through extras for a GGUF request that omitted the field: + the previous same-model load's extras, shadow-stripped, so a settings-Apply + reload (which does not round-trip the extras field) keeps them (#5401).""" + if getattr(request, "llama_extra_args", None) is not None: + return extra_llama_args + if not getattr(config, "is_gguf", False): + return extra_llama_args + llama_backend = get_llama_cpp_backend() + if not llama_backend.extra_args: + return extra_llama_args + # Inherit the previous load's extras (the chat-settings Apply path doesn't + # round-trip them; an explicit [] still clears). Gated on (model_identifier, + # hf_variant) to refuse cross-model pickup, and shadowing flags are + # stripped so an inherited override can't win the last-wins CLI + # parse against a freshly-supplied first-class field. + source = llama_backend.extra_args_source + # Compare against the resolved variant, not the request field: callers + # commonly omit gguf_variant for local ``.gguf`` paths and HF auto-pick + # flows. ``config.gguf_variant`` is the variant load_model was actually + # invoked with, so both sides of the comparison key off the same string. + resolved_variant = (config.gguf_variant or "").lower() + request_variant = (request.gguf_variant or "").lower() + stored_variant = (source[1] or "").lower() if source else "" + same_model = bool(source and source[0] and source[0].lower() == model_identifier.lower()) + if request.gguf_variant: + variant_mismatch = request_variant != stored_variant + else: + variant_mismatch = bool(stored_variant and resolved_variant != stored_variant) + same_source = same_model and not variant_mismatch + if not same_source: + logger.info( + "Not inheriting llama_extra_args: stored args came from %s, loading %s", + source, + (model_identifier, resolved_variant), + ) + # Cross-model: clear explicitly so the backend doesn't + # inherit via "no opinion" semantics. + extra_llama_args = [] + else: + # Strip only the groups whose first-class field was set by the caller, so + # an inherited --chat-template-file survives an Apply that omits + # chat_template_override. A bundled family template (e.g. gemma-4) counts as + # a first-class template even when the request omits chat_template_override, + # so strip the inherited --chat-template-file then too -- else the stale arg + # (appended last) shadows the bundled template while Studio reports its caps. + fields_set = getattr(request, "model_fields_set", set()) + stripped = strip_shadowing_flags( + llama_backend.extra_args, + strip_context = "max_seq_length" in fields_set, + strip_cache = "cache_type_kv" in fields_set, + strip_spec = ("speculative_type" in fields_set or "spec_draft_n_max" in fields_set), + strip_template = ( + "chat_template_override" in fields_set + or effective_chat_template_override is not None + ), + strip_split_mode = _should_strip_split_mode(request, llama_backend.extra_args), + # manual + per-GPU ratio emits its own --tensor-split; drop + # an inherited one (appended last would override it) while + # keeping the user's --split-mode row/none/layer choice. + strip_tensor_split = _should_strip_tensor_split(request), + # manual emits its own --fit/--gpu-layers, so an inherited offload flag + # must not last-wins-override it. auto leaves a user's inherited -ngl + # alone. getattr: a validate request reuses this resolver, no offload fields. + strip_offload = getattr(request, "gpu_memory_mode", "auto") == "manual", + ) + try: + extra_llama_args = validate_extra_args(stripped) + except ValueError: + # Shouldn't happen on already-validated args; degrade to + # no-extras rather than 400 if managed flags changed. + logger.warning( + "Stored llama_extra_args failed revalidation; loading without them: %s", + stripped, + ) + extra_llama_args = [] + else: + if extra_llama_args: + logger.info( + "Inheriting llama_extra_args from previous " + "load (same model, shadow-stripped): %s", + extra_llama_args, + ) + return extra_llama_args + + def _model_json_response(model, status_code: int = 200) -> Response: """Serialize a pydantic response once via pydantic-core. @@ -3922,6 +4234,15 @@ def _maybe_unsupported_message(msg: str) -> str: return msg +def _raise_if_sidecar_swap_in_progress() -> None: + from utils.transformers_version import sidecar_swap_in_progress + if sidecar_swap_in_progress(): + raise HTTPException( + status_code = 409, + detail = "A transformers installation is in progress. Retry when it completes.", + ) + + @router.post("/load", response_model = LoadResponse) async def load_model( request: LoadRequest, @@ -3942,24 +4263,23 @@ async def load_model( # install can reserve while this request queues on the gate, so the pre-gate # check alone is only a fast path. from core.inference.llama_keepwarm import inference_lifecycle_gate - from utils.transformers_version import sidecar_swap_in_progress - _swap_409 = HTTPException( - status_code = 409, - detail = "A transformers installation is in progress. Retry when it completes.", - ) - if sidecar_swap_in_progress(): - raise _swap_409 + _raise_if_sidecar_swap_in_progress() # Hold the lifecycle gate across the load so idle auto-unload can't unload the # model mid-load. Auto-switch calls _load_model_impl directly since it already # holds this gate. async with inference_lifecycle_gate(): - if sidecar_swap_in_progress(): - raise _swap_409 + _raise_if_sidecar_swap_in_progress() return await _load_model_impl(request, fastapi_request, current_subject) -async def _load_model_impl(request: LoadRequest, fastapi_request: Request, current_subject: str): +async def _load_model_impl( + request: LoadRequest, + fastapi_request: Request, + current_subject: str, + *, + current_request_counted: bool = False, +): from core.inference.llama_cpp import LlamaServerNotFoundError # A new load starts here; arm the progress throttle so this load's first @@ -3968,6 +4288,7 @@ async def _load_model_impl(request: LoadRequest, fastapi_request: Request, curre native_grant_backed = False model_log_label = request.model_path + gguf_load_stack = ExitStack() try: # Validate user pass-through args up front so a managed-flag collision # returns 400 before any model work. @@ -3986,6 +4307,35 @@ async def _load_model_impl(request: LoadRequest, fastapi_request: Request, curre None if request.llama_extra_args is None else extra_llama_args ) + # Manual mode owns the offload flags: strip them from EXPLICIT extras + # too (the inherited path already does), or a last-wins --gpu-layers / + # --fit in extras re-enables GPU offload on a load status reports as + # CPU-only. Manual + per-GPU ratio owns --tensor-split the same way. + if request.gpu_memory_mode == "manual" and extra_llama_args: + _stripped_explicit = strip_shadowing_flags( + extra_llama_args, + strip_context = False, + strip_cache = False, + strip_spec = False, + strip_template = False, + strip_split_mode = False, + strip_tensor_split = _should_strip_tensor_split(request), + strip_offload = True, + ) + if _stripped_explicit != extra_llama_args: + logger.info( + "Manual GPU memory owns the offload flags; stripping them " + "from explicit llama_extra_args: %s -> %s", + extra_llama_args, + _stripped_explicit, + ) + extra_llama_args = _stripped_explicit + + # Keep every downstream consumer on the normalized explicit list. In + # particular, the already-loaded comparator must not compare the raw + # request's managed offload flags against the stripped launch state. + request = request.model_copy(update = {"llama_extra_args": extra_llama_args}) + model_identifier, model_log_label, native_grant_backed = ( _resolve_model_identifier_for_request(request, operation = "load-model") ) @@ -4028,6 +4378,7 @@ async def _load_model_impl(request: LoadRequest, fastapi_request: Request, curre # Skip if a prior audio probe failed -- let load_model retry. and getattr(llama_backend, "_audio_probed", True) ): + llama_backend._record_matching_gpu_request(request.gpu_ids) logger.info( "Model already loaded (GGUF): " f"{model_log_label} variant={request.gguf_variant or llama_backend.hf_variant}, skipping reload" @@ -4067,6 +4418,14 @@ async def _load_model_impl(request: LoadRequest, fastapi_request: Request, curre speculative_type = llama_backend.requested_spec_mode, spec_draft_n_max = llama_backend.spec_draft_n_max, tensor_parallel = llama_backend.tensor_parallel, + gpu_memory_mode = llama_backend.gpu_memory_mode, + gpu_layers = llama_backend.gpu_layers, + n_cpu_moe = llama_backend.n_cpu_moe, + tensor_split = llama_backend.tensor_split, + n_layers = llama_backend.n_layers, + n_moe_layers = llama_backend.n_moe_layers, + gpu_ids = llama_backend.gpu_ids, + requested_gpu_ids = llama_backend.requested_gpu_ids, ) else: if ( @@ -4133,18 +4492,18 @@ async def _load_model_impl(request: LoadRequest, fastapi_request: Request, curre # Normalize gpu_ids: empty list means auto-selection, same as None effective_gpu_ids = request.gpu_ids if request.gpu_ids else None - # Reject GGUF + gpu_ids first so the guard can't mask it with a VRAM 409. - if config.is_gguf and effective_gpu_ids is not None: - raise HTTPException( - status_code = 400, - detail = "gpu_ids is not supported for GGUF models yet.", - ) + # Validate the full GGUF placement pool before the training guard so an + # invalid physical ID or Vulkan ordinal is a clean 400, not a masked VRAM + # 409. The same helper is used by /validate. + gguf_gpu_ids: Optional[List[int]] = None + if config.is_gguf: + gguf_gpu_ids = await _resolve_gguf_gpu_ids_for_request(config, effective_gpu_ids) if not config.is_gguf and _mlx_distributed_launch_detected(): raise HTTPException( status_code = 400, detail = ( - "Studio does not support distributed MLX inference under " - "mlx.launch. Use `mlx.launch ... unsloth chat` or run Studio " + "Unsloth does not support distributed MLX inference under " + "mlx.launch. Use `mlx.launch ... unsloth chat` or run Unsloth " "without the distributed launcher." ), ) @@ -4168,8 +4527,20 @@ async def _load_model_impl(request: LoadRequest, fastapi_request: Request, curre "architectures)" ) - # Refuse a load that would OOM active training, before the unload step below - # frees the resident model. Off-loop: guard does sync nvidia-smi / HF work. + # Inherit the previous same-model load's pass-through extras when this + # request omits the field (a settings-Apply reload doesn't round-trip + # them); shadow-stripped so an inherited flag can't override a + # first-class field the caller did set (#5401). + extra_llama_args = _resolve_inherited_extra_args( + request, + config, + model_identifier, + extra_llama_args, + effective_chat_template_override, + ) + + # Apply the training coexistence policy before the unload step below + # frees the resident model. Off-loop: the default-mode guard does sync work. await asyncio.to_thread( _guard_chat_load_against_training, config, @@ -4180,6 +4551,7 @@ async def _load_model_impl(request: LoadRequest, fastapi_request: Request, curre requested_gpu_ids = effective_gpu_ids, llama_extra_args = extra_llama_args, n_parallel = getattr(fastapi_request.app.state, "llama_parallel_slots", 1), + gpu_memory_mode = request.gpu_memory_mode, ) # ── GGUF path: load via llama-server ────────────────────── @@ -4187,8 +4559,41 @@ async def _load_model_impl(request: LoadRequest, fastapi_request: Request, curre llama_backend = get_llama_cpp_backend() unsloth_backend = get_inference_backend() - # Unload any active Unsloth model to free VRAM (off the event loop: - # unload takes _gen_lock and can wait on an in-flight stream). + if config.gguf_hf_repo: + from core.inference.llama_cpp import gguf_load_in_flight + gguf_load_stack.enter_context(gguf_load_in_flight(config.gguf_hf_repo)) + + # Block cache writes that would race the download manager. This runs + # after pass-through argument inheritance so a carried --no-mmproj + # changes the companion requirement exactly as it does for the load. + if config.gguf_hf_repo: + from core.inference.llama_cpp import _hub_download_blocks_gguf_load + if await asyncio.to_thread( + _hub_download_blocks_gguf_load, + config.gguf_hf_repo, + config.gguf_variant, + require_mmproj = bool( + config.is_vision and not extra_args_disable_mmproj(extra_llama_args) + ), + hf_token = request.hf_token, + ): + raise HTTPException( + status_code = 409, + detail = ( + f"'{model_log_label}' is currently being downloaded " + "by the download manager. Wait for the download to " + "finish (or cancel it), then load the model." + ), + ) + + # Keep the resident model alive until every active generation finishes; + # the caller's lifecycle gate blocks new starts. + await _wait_for_model_switch_idle(current_request_counted = current_request_counted) + # A sidecar install can reserve the gate while inference drains, after the + # route-level checks above, so recheck before replacing either backend. + _raise_if_sidecar_swap_in_progress() + + # Unload any active Unsloth model only after every hub conflict check. if unsloth_backend.active_model_name: logger.info( f"Unloading Unsloth model '{unsloth_backend.active_model_name}' before loading GGUF" @@ -4197,84 +4602,6 @@ async def _load_model_impl(request: LoadRequest, fastapi_request: Request, curre unsloth_backend.unload_model, unsloth_backend.active_model_name ) - # Inherit llama_extra_args from the previous load when the request - # omits the field (the chat-settings Apply path doesn't round-trip - # them; explicit [] still clears). Gated on (model_identifier, - # hf_variant) to refuse cross-model pickup, and shadowing flags are - # stripped so an inherited override can't win the last-wins CLI - # parse against a freshly-supplied first-class field. - if request.llama_extra_args is None and llama_backend.extra_args: - source = llama_backend.extra_args_source - # Compare against the resolved variant, not the request - # field: callers commonly omit gguf_variant for local - # ``.gguf`` paths and HF auto-pick flows. ``config.gguf_ - # variant`` is the variant load_model was actually - # invoked with (see the HF / local branches below), so - # both sides of the comparison key off the same string. - resolved_variant = (config.gguf_variant or "").lower() - request_variant = (request.gguf_variant or "").lower() - stored_variant = (source[1] or "").lower() if source else "" - same_model = bool( - source and source[0] and source[0].lower() == model_identifier.lower() - ) - if request.gguf_variant: - variant_mismatch = request_variant != stored_variant - else: - variant_mismatch = bool(stored_variant and resolved_variant != stored_variant) - same_source = same_model and not variant_mismatch - if not same_source: - logger.info( - "Not inheriting llama_extra_args: stored args came from %s, loading %s", - source, - (model_identifier, resolved_variant), - ) - # Cross-model: clear explicitly so the backend doesn't - # inherit via "no opinion" semantics. - extra_llama_args = [] - else: - # Strip only the groups whose first-class field was set by - # the caller, so an inherited --chat-template-file survives - # an Apply that omits chat_template_override. A bundled family - # template (e.g. the gemma-4 override) is an effective - # first-class template setting even when the raw request - # omits chat_template_override, so strip the inherited - # --chat-template-file in that case too -- otherwise the stale - # extra arg (appended last) shadows the bundled template while - # Studio reports the bundled template's capabilities. - fields_set = getattr(request, "model_fields_set", set()) - stripped = strip_shadowing_flags( - llama_backend.extra_args, - strip_context = "max_seq_length" in fields_set, - strip_cache = "cache_type_kv" in fields_set, - strip_spec = ( - "speculative_type" in fields_set or "spec_draft_n_max" in fields_set - ), - strip_template = ( - "chat_template_override" in fields_set - or effective_chat_template_override is not None - ), - strip_split_mode = _should_strip_split_mode( - request, llama_backend.extra_args - ), - ) - try: - extra_llama_args = validate_extra_args(stripped) - except ValueError: - # Shouldn't happen on already-validated args; degrade to - # no-extras rather than 400 if managed flags changed. - logger.warning( - "Stored llama_extra_args failed revalidation; loading without them: %s", - stripped, - ) - extra_llama_args = [] - else: - if extra_llama_args: - logger.info( - "Inheriting llama_extra_args from previous " - "load (same model, shadow-stripped): %s", - extra_llama_args, - ) - # Route to HF or local mode based on config. Run in a thread so the # event loop stays free for progress polling and other requests # during the (potentially long) GGUF download + llama-server start. @@ -4290,7 +4617,13 @@ async def _load_model_impl(request: LoadRequest, fastapi_request: Request, curre cache_type_kv = request.cache_type_kv, speculative_type = request.speculative_type, spec_draft_n_max = request.spec_draft_n_max, + gpu_memory_mode = request.gpu_memory_mode, + gpu_layers = request.gpu_layers, + n_cpu_moe = request.n_cpu_moe, + tensor_split = request.tensor_split, n_parallel = _n_parallel, + # Issue #7164: explicit GPU pin resolved to physical ids above. + gpu_ids = gguf_gpu_ids, ) if config.gguf_hf_repo: # HF mode: download via huggingface_hub then start llama-server @@ -4414,7 +4747,7 @@ async def _load_model_impl(request: LoadRequest, fastapi_request: Request, curre # Clear any idle-unload reload stash now, not only on the next poll. from core.inference.llama_keepwarm import note_model_loaded - note_model_loaded() + await asyncio.to_thread(note_model_loaded, llama_backend) # A plain load advertises its own identifier; auto-switch overwrites # this with the repo id right after _load_model_impl returns. llama_backend._openai_advertised_id = None @@ -4457,6 +4790,14 @@ async def _load_model_impl(request: LoadRequest, fastapi_request: Request, curre speculative_type = llama_backend.requested_spec_mode, spec_draft_n_max = llama_backend.spec_draft_n_max, tensor_parallel = llama_backend.tensor_parallel, + gpu_memory_mode = llama_backend.gpu_memory_mode, + gpu_layers = llama_backend.gpu_layers, + n_cpu_moe = llama_backend.n_cpu_moe, + tensor_split = llama_backend.tensor_split, + n_layers = llama_backend.n_layers, + n_moe_layers = llama_backend.n_moe_layers, + gpu_ids = llama_backend.gpu_ids, + requested_gpu_ids = llama_backend.requested_gpu_ids, ) # ── Standard path: load via Unsloth/transformers ────────── @@ -4464,6 +4805,8 @@ async def _load_model_impl(request: LoadRequest, fastapi_request: Request, curre # Unload any active GGUF model first llama_backend = get_llama_cpp_backend() + await _wait_for_model_switch_idle(current_request_counted = current_request_counted) + _raise_if_sidecar_swap_in_progress() if llama_backend.is_loaded: logger.info("Unloading GGUF model before loading Unsloth model") llama_backend.unload_model() @@ -4639,13 +4982,15 @@ async def _load_model_impl(request: LoadRequest, fastapi_request: Request, curre logger.error(f"Error loading model: {e}", exc_info = True) msg = _maybe_unsupported_message(redacted_msg) raise HTTPException(status_code = 500, detail = f"Failed to load model: {msg}") + finally: + gguf_load_stack.close() def _requires_trust_remote_code_for_model( model_identifier: str, hf_token: Optional[str] = None ) -> bool: """Whether loading this model would execute custom repo code, so the consent - dialog must run first. True if the Studio YAML default enables + dialog must run first. True if the Unsloth YAML default enables ``trust_remote_code`` OR the raw config declares an ``auto_map`` (Hub/local, config.json or tokenizer_config.json). Reads raw JSON only; never imports model code.""" @@ -4713,7 +5058,9 @@ def _requires_security_review_for_model( @router.post("/validate", response_model = ValidateModelResponse) async def validate_model( - request: ValidateModelRequest, current_subject: str = Depends(get_current_subject) + request: ValidateModelRequest, + fastapi_request: Request = None, + current_subject: str = Depends(get_current_subject), ): """ Lightweight validation endpoint for model identifiers. @@ -4741,15 +5088,11 @@ async def validate_model( detail = f"Invalid model identifier: {model_log_label}", ) - # Refuse early (before the frontend unloads to load this) if it can't fit - # alongside training, using the same settings /load uses so they agree. + # Apply the same training coexistence policy as /load before the frontend + # unloads the current model. effective_gpu_ids = request.gpu_ids if request.gpu_ids else None - # Mirror /load: reject GGUF + gpu_ids before the guard so both return 400. - if config.is_gguf and effective_gpu_ids is not None: - raise HTTPException( - status_code = 400, - detail = "gpu_ids is not supported for GGUF models yet.", - ) + if config.is_gguf: + await _resolve_gguf_gpu_ids_for_request(config, effective_gpu_ids) effective_load_in_4bit = _effective_load_in_4bit(config, request.load_in_4bit) # Both checks cover the [adapter, base] set (matching the scan route and workers): @@ -4813,16 +5156,32 @@ async def validate_model( latest_tier_active_for, config.identifier, request.hf_token ): effective_load_in_4bit = False - # Off-loop: guard does sync nvidia-smi / HF work. - await asyncio.to_thread( - _guard_chat_load_against_training, - config, - model_identifier = model_identifier, - hf_token = request.hf_token, - load_in_4bit = effective_load_in_4bit, - max_seq_length = request.max_seq_length, - requested_gpu_ids = effective_gpu_ids, - ) + # A metadata-only probe reads the GGUF header and allocates no VRAM, so the + # training guard must not refuse it. Real loads omit include_context_length / + # include_chat_template, and /load applies the guard again. + if not (request.include_context_length or request.include_chat_template): + # Match /load's inherited llama.cpp extras and parallel slot count so + # validation cannot pass a smaller estimate than the subsequent load. + effective_extra_args = _resolve_inherited_extra_args( + request, config, model_identifier, None + ) + # Off-loop: guard does sync nvidia-smi / HF work. + await asyncio.to_thread( + _guard_chat_load_against_training, + config, + model_identifier = model_identifier, + hf_token = request.hf_token, + load_in_4bit = effective_load_in_4bit, + max_seq_length = request.max_seq_length, + requested_gpu_ids = effective_gpu_ids, + llama_extra_args = effective_extra_args, + n_parallel = ( + getattr(fastapi_request.app.state, "llama_parallel_slots", 1) + if fastapi_request is not None + else 1 + ), + gpu_memory_mode = request.gpu_memory_mode, + ) # A selected GGUF loads via llama.cpp: auto_map Python and root pickle weights in a # mixed repo are inert for this load, so gating on them is a false positive. Only @@ -4836,10 +5195,21 @@ async def validate_model( # Native context length, read from the local GGUF header when present. # Lets the staged ("Load on selection" off) flow populate the context # slider before the GPU load; None until the file is downloaded. + # Staged header dims (one read): native context, total layer count, and + # MoE expert-layer count -- let the staged flow size the context, GPU- + # layers and manual --n-cpu-moe sliders before the load. context_length: Optional[int] = None - if request.include_context_length and is_gguf: + layer_count: Optional[int] = None + moe_layer_count: Optional[int] = None + chat_template: Optional[str] = None + # Both header probes read the same local GGUF, so resolve it once. + if (request.include_context_length or request.include_chat_template) and is_gguf: from hub.utils.gguf import resolve_local_gguf_path - from utils.models.gguf_metadata import read_gguf_context_length + from picker.schemas import MAX_CHAT_TEMPLATE_BYTES + from utils.models.gguf_metadata import ( + read_gguf_chat_template, + read_gguf_staged_dims, + ) # Best-effort: a header-read failure must never fail validation of an # otherwise-valid model (the outer except turns it into a 400). @@ -4855,9 +5225,26 @@ async def validate_model( model_identifier, request.gguf_variant ) if local_gguf: - context_length = read_gguf_context_length(local_gguf) + if request.include_context_length: + # Header walk reads tokenizer arrays (tens of ms); keep it + # off the event loop. + dims = await asyncio.to_thread(read_gguf_staged_dims, local_gguf) + if dims: + context_length = dims["context_length"] + layer_count = dims["layer_count"] + moe_layer_count = dims["moe_layer_count"] + if request.include_chat_template: + # Read only the leased GGUF's own embedded template (the copy + # llama.cpp loads), never a sibling sidecar: the native grant + # authorizes just this path, so neighbours would be scope escalation. + raw_template = await asyncio.to_thread(read_gguf_chat_template, local_gguf) + if ( + raw_template is not None + and len(raw_template.encode("utf-8")) <= MAX_CHAT_TEMPLATE_BYTES + ): + chat_template = raw_template except Exception as e: - logger.debug("Context-length probe failed for %s: %s", model_log_label, e) + logger.debug("Header probe failed for %s: %s", model_log_label, e) return ValidateModelResponse( valid = True, @@ -4872,6 +5259,9 @@ async def validate_model( requires_trust_remote_code = requires_trust_remote_code, requires_security_review = requires_security_review, context_length = context_length, + layer_count = layer_count, + moe_layer_count = moe_layer_count, + chat_template = chat_template, requires_transformers_upgrade = transformers_upgrade is not None, transformers_upgrade = transformers_upgrade, ) @@ -4884,6 +5274,14 @@ async def validate_model( raise HTTPException(status_code = 400, detail = str(e)) except Exception as e: redacted_msg = redact_native_paths(str(e)) + if is_hf_authentication_error(e): + raise HTTPException( + status_code = 400, + detail = ( + "Hugging Face authentication failed. Check or clear the token " + "in Settings, and confirm access to this gated repository." + ), + ) if _is_unsupported_nvfp4_inference_error(redacted_msg): logger.warning( "NVFP4 inference is not supported yet while validating '%s'", @@ -5278,7 +5676,7 @@ async def confirm_tool_call( @studio_router.get("/monitor") async def get_api_monitor(current_subject: str = Depends(get_current_subject)): - """Return recent OpenAI-compatible API activity for Studio.""" + """Return recent OpenAI-compatible API activity for Unsloth.""" active_model = _monitor_active_model() active_requests = api_monitor.active_count(subject = current_subject) if active_requests: @@ -5392,6 +5790,10 @@ async def generate_stream( if chunk is _DONE: completed = True break + if isinstance(chunk, GenStreamError): + yield f"data: {json.dumps({'error': _friendly_gen_stream_error(chunk)})}\n\n" + yield "data: [DONE]\n\n" + return yield f"data: {json.dumps({'content': chunk})}\n\n" if completed: yield "data: [DONE]\n\n" @@ -5405,6 +5807,7 @@ async def generate_stream( backend.reset_generation_state() logger.error(f"Error during generation: {e}", exc_info = True) yield f"data: {json.dumps({'error': _friendly_error(e)})}\n\n" + yield "data: [DONE]\n\n" finally: await _stop_local_disconnect_cancel_watcher(disconnect_watcher) if not completed and not cancel_event.is_set(): @@ -5432,10 +5835,15 @@ async def get_status(current_subject: str = Depends(get_current_subject)): try: _bin = type(llama_backend)._find_llama_server_binary() _caps = type(llama_backend).probe_server_capabilities(_bin) - _supports_mtp = bool(_caps.get("supports_mtp", False)) + # Fail open on inconclusive probes: False means a definitive + # "binary lacks MTP" to API consumers. + _supports_mtp = bool( + _caps.get("supports_mtp", False) + or (_caps.get("found", False) and _caps.get("mtp_probe_inconclusive", False)) + ) except Exception: _bin = None - _supports_mtp = True # fail open + _supports_mtp = False # no usable binary: MTP genuinely unavailable try: from utils.llama_cpp_freshness import check_prebuilt_freshness _freshness = check_prebuilt_freshness(_bin) @@ -5461,7 +5869,7 @@ async def get_status(current_subject: str = Depends(get_current_subject)): _display_model_id = os.path.basename(_model_id) _inference_cfg = load_inference_config(_model_id) if _model_id else None _audio_type = getattr(llama_backend, "_audio_type", None) - # Don't surface Studio's auto-applied bundled family template (e.g. the + # Don't surface Unsloth's auto-applied bundled family template (e.g. the # gemma-4 override) as a user-authored override: the frontend adopts # status.chat_template_override as editable state and would otherwise # re-send it as an explicit override for a later, unrelated model. Only @@ -5506,6 +5914,15 @@ async def get_status(current_subject: str = Depends(get_current_subject)): speculative_type = llama_backend.requested_spec_mode, spec_draft_n_max = llama_backend.spec_draft_n_max, tensor_parallel = llama_backend.tensor_parallel, + gpu_memory_mode = llama_backend.gpu_memory_mode, + gpu_layers = llama_backend.gpu_layers, + n_cpu_moe = llama_backend.n_cpu_moe, + tensor_split = llama_backend.tensor_split, + requested_context_length = llama_backend.requested_n_ctx, + n_layers = llama_backend.n_layers, + n_moe_layers = llama_backend.n_moe_layers, + gpu_ids = llama_backend.gpu_ids, + requested_gpu_ids = llama_backend.requested_gpu_ids, llama_cpp_supports_mtp = _supports_mtp, spec_fallback_reason = llama_backend.spec_fallback_reason, llama_cpp_prebuilt_stale = _stale, @@ -5670,6 +6087,7 @@ async def generate_audio( # Advertised repo id after an auto-switch load, else a clean public id, # never the absolute .gguf path. model_name = _llama_public_model_id(llama_backend) + _audio_model_id = getattr(llama_backend, "model_identifier", None) or model_name gen = lambda: llama_backend.generate_audio_response( text = text, audio_type = llama_backend._audio_type, @@ -5688,6 +6106,7 @@ async def generate_audio( if not model_info.get("is_audio"): raise HTTPException(status_code = 400, detail = "Active model is not an audio model.") model_name = public_model_id(backend.active_model_name) + _audio_model_id = getattr(backend, "active_model_name", None) or model_name gen = lambda: backend.generate_audio_response( text = text, temperature = payload.temperature, @@ -5699,6 +6118,13 @@ async def generate_audio( use_adapter = payload.use_adapter, ) + # Apply per-model recommended sampling + any operator UNSLOTH_SAMPLING_* pin before + # generating, so `unsloth run --temperature` (and the other pins) and per-model + # recommendations reach audio (TTS) generation too, not just chat. The gen lambdas read + # payload.* lazily at call time, so filling here takes effect; this covers both the direct + # /audio/generate route and the chat-completions audio branches that delegate here. + _fill_recommended_sampling_openai(payload, _audio_model_id) + try: wav_bytes, sample_rate = await asyncio.to_thread(gen) except Exception as e: @@ -5726,6 +6152,342 @@ async def generate_audio( ) +# ===================================================================== +# Speech-to-text (STT) sidecar (/audio/transcribe, /audio/stt/*) +# ===================================================================== + + +def _resolve_stt_engine(engine: Optional[str]) -> str: + """Normalize the requested STT engine name; default is Transformers.""" + normalized = (engine or "transformers").strip().lower() + if normalized in ("", "transformers", "whisper"): + return "transformers" + if normalized in ("gguf", "ggml", "whisper_cpp", "whisper.cpp"): + return "gguf" + raise HTTPException( + status_code = 422, + detail = f"Unknown STT engine '{engine}'. Use 'transformers' or 'gguf'.", + ) + + +def _resolve_serving_stt_engine(engine: Optional[str]) -> str: + """Resolve the engine that will actually serve a model. + + whisper.cpp (gguf) only accepts curated ids, which Transformers serves too, + so when whisper-server is not installed (the common case: `unsloth studio + update` does not yet build it) fall back to Transformers instead of 501-ing + on every recording. Used for download/load/transcribe; unload targets a + specific engine via _resolve_stt_engine. + """ + resolved = _resolve_stt_engine(engine) + if resolved == "gguf": + from core.inference import stt_ggml_sidecar + if not stt_ggml_sidecar.is_available(): + return "transformers" + return resolved + + +def _stt_sidecar_for(engine: str): + if engine == "gguf": + from core.inference.stt_ggml_sidecar import get_ggml_stt_sidecar + return get_ggml_stt_sidecar() + from core.inference.stt_sidecar import get_stt_sidecar + return get_stt_sidecar() + + +@studio_router.get("/audio/stt/status") +async def stt_status( + model: Optional[str] = None, current_subject: str = Depends(get_current_subject) +): + """Report STT availability and which model, if any, is resident. + + ``model`` extends the Transformers ``downloaded_models`` check to a + custom Hugging Face repository beyond the curated defaults. + """ + from core.inference import stt_ggml_sidecar, stt_sidecar + from core.inference.stt_sidecar import ( + DEFAULT_STT_MODEL, + STT_MODELS, + get_stt_sidecar, + is_available, + ) + + sidecar = get_stt_sidecar() + ggml = stt_ggml_sidecar.get_ggml_stt_sidecar() + transformers_downloaded = [ + model_id for model_id in STT_MODELS if stt_sidecar.is_model_downloaded(model_id) + ] + if model and model not in STT_MODELS and stt_sidecar.is_model_downloaded(model): + transformers_downloaded.append(model) + return JSONResponse( + content = { + "available": is_available(), + "loaded_model": sidecar.loaded_model, + "loading": sidecar.is_loading(), + "device": sidecar.device, + "keep_alive_seconds": sidecar.keep_alive_seconds, + "default_model": DEFAULT_STT_MODEL, + "models": list(STT_MODELS.keys()), + # Transformers engine, same shape as "gguf" below so clients read + # either generically. Top-level fields above kept for old clients. + "transformers": { + "available": is_available(), + "loaded_model": sidecar.loaded_model, + "loading": sidecar.is_loading(), + "device": sidecar.device, + "keep_alive_seconds": sidecar.keep_alive_seconds, + "default_model": DEFAULT_STT_MODEL, + "models": list(STT_MODELS.keys()), + "downloaded_models": transformers_downloaded, + "download": stt_sidecar.download_status(), + }, + # whisper.cpp (GGUF) engine. + "gguf": { + "available": stt_ggml_sidecar.is_available(), + "loaded_model": ggml.loaded_model, + "loading": ggml.is_loading(), + "device": ggml.device, + "keep_alive_seconds": ggml.keep_alive_seconds, + "default_model": stt_ggml_sidecar.DEFAULT_GGML_STT_MODEL, + "models": list(stt_ggml_sidecar.GGML_STT_MODELS.keys()), + "downloaded_models": [ + model_id + for model_id in stt_ggml_sidecar.GGML_STT_MODELS + if stt_ggml_sidecar._cached_model_path(model_id) is not None + ], + "download": stt_ggml_sidecar.download_status(), + }, + } + ) + + +@studio_router.post("/audio/stt/download") +async def stt_download( + payload: SttLoadRequest, + current_subject: str = Depends(get_current_subject), + hf_token: Optional[str] = Depends(get_hf_token), +): + """Start a background download of a dictation model. + + Both engines download directly (a GGML checkpoint is a single file the Model + Hub's GGUF variant planner cannot express; a Transformers checkpoint is a + whole snapshot). Progress is reported by /audio/stt/status. + """ + from core.inference import stt_ggml_sidecar, stt_sidecar + from core.inference.stt_sidecar import ( + SttModelCompatibilityError, + SttModelIdError, + validate_remote_model, + ) + + engine = _resolve_serving_stt_engine(payload.engine) + module = stt_ggml_sidecar if engine == "gguf" else stt_sidecar + try: + # Transformers accepts custom `owner/model` repos, so confirm the repo is + # a Whisper checkpoint (metadata-only) before snapshot_download pulls a + # possibly-large non-STT repo into the shared cache. Curated ids + # short-circuit; GGUF only accepts curated ids, so it needs no check. + if engine != "gguf": + validated = await asyncio.to_thread(validate_remote_model, payload.model, hf_token) + # Pin the download to the commit that was just validated so the + # repo cannot be swapped between validation and snapshot_download. + await asyncio.to_thread( + module.start_model_download, + payload.model, + hf_token, + validated.get("revision"), + ) + else: + await asyncio.to_thread(module.start_model_download, payload.model, hf_token) + except SttModelIdError as e: + raise HTTPException(status_code = 422, detail = str(e)) + except SttModelCompatibilityError as e: + raise HTTPException(status_code = 422, detail = str(e)) + return JSONResponse(content = module.download_status()) + + +@studio_router.post("/audio/stt/load") +async def stt_load(payload: SttLoadRequest, current_subject: str = Depends(get_current_subject)): + """Load the selected STT model after the user starts local dictation.""" + from core.inference.stt_sidecar import ( + SttLoadCancelledError, + SttModelCompatibilityError, + SttModelIdError, + SttModelNotDownloadedError, + SttUnavailableError, + get_stt_sidecar, + ) + + sidecar = _stt_sidecar_for(_resolve_serving_stt_engine(payload.engine)) + try: + await asyncio.to_thread(sidecar.load, payload.model) + except SttModelNotDownloadedError as e: + raise HTTPException(status_code = 409, detail = str(e)) + except SttUnavailableError as e: + raise HTTPException(status_code = 501, detail = str(e)) + except SttLoadCancelledError as e: + raise HTTPException(status_code = 409, detail = str(e)) + except SttModelIdError as e: + raise HTTPException(status_code = 422, detail = str(e)) + except SttModelCompatibilityError as e: + raise HTTPException(status_code = 422, detail = str(e)) + except Exception as e: + logger.error(f"STT load error: {e}", exc_info = True) + raise HTTPException(status_code = 500, detail = safe_error_detail(e)) + return JSONResponse(content = {"loaded_model": sidecar.loaded_model, "device": sidecar.device}) + + +@studio_router.post("/audio/stt/validate") +async def stt_validate( + payload: SttLoadRequest, + current_subject: str = Depends(get_current_subject), + hf_token: Optional[str] = Depends(get_hf_token), +): + """Verify a Hub repository is a Whisper checkpoint before downloading it.""" + from core.inference.stt_sidecar import ( + SttModelCompatibilityError, + SttModelIdError, + validate_remote_model, + ) + + try: + result = await asyncio.to_thread(validate_remote_model, payload.model, hf_token) + except (SttModelIdError, SttModelCompatibilityError) as e: + raise HTTPException(status_code = 422, detail = str(e)) + return JSONResponse(content = result) + + +@studio_router.post("/audio/stt/unload") +async def stt_unload( + engine: Optional[str] = None, current_subject: str = Depends(get_current_subject) +): + """Release the local STT model when dictation is idle. + + Without an engine, both sidecars unload so an engine switch in Voice + settings always frees whichever backend was resident. + """ + if engine is None: + engines = ["transformers", "gguf"] + else: + # Use the serving resolver: a "gguf" pick without whisper-server is + # actually served by the Transformers fallback, so unload must target + # that same engine or the resident model is never freed. + engines = [_resolve_serving_stt_engine(engine)] + # Attempt every engine even if one raises, so failing to unload one never + # skips freeing the other (both can be resident after a switch). + failed: list[str] = [] + for name in engines: + try: + await asyncio.to_thread(_stt_sidecar_for(name).unload) + except Exception as exc: # noqa: BLE001 - report after attempting all engines + logger.warning("Failed to unload STT engine '%s': %s", name, exc) + failed.append(name) + if failed: + raise HTTPException( + status_code = 500, + detail = f"Failed to unload STT engine(s): {', '.join(failed)}", + ) + return JSONResponse(content = {"loaded_model": None, "device": None}) + + +async def _transcribe_audio_bytes( + raw: bytes, + model: Optional[str], + language: Optional[str], + fast: bool, + engine: Optional[str] = None, +) -> JSONResponse: + """Run STT for already-decoded request bytes.""" + from core.inference.stt_sidecar import ( + SttAudioDecodeError, + SttAudioTooLongError, + SttLanguageError, + SttLoadCancelledError, + SttModelCompatibilityError, + SttModelIdError, + SttModelNotDownloadedError, + SttUnavailableError, + ) + + if not raw: + raise HTTPException(status_code = 400, detail = "Audio is empty.") + if len(raw) > _MAX_AUDIO_RAW_BYTES: + raise HTTPException(status_code = 413, detail = "Audio is too large.") + + sidecar = _stt_sidecar_for(_resolve_serving_stt_engine(engine)) + try: + result = await asyncio.to_thread( + sidecar.transcribe, + raw, + model, + language, + fast, + ) + except SttUnavailableError as e: + raise HTTPException(status_code = 501, detail = str(e)) + except SttLoadCancelledError as e: + raise HTTPException(status_code = 409, detail = str(e)) + except SttModelNotDownloadedError as e: + raise HTTPException(status_code = 409, detail = str(e)) + except SttModelIdError as e: + raise HTTPException(status_code = 422, detail = str(e)) + except SttModelCompatibilityError as e: + raise HTTPException(status_code = 422, detail = str(e)) + except SttLanguageError as e: + raise HTTPException(status_code = 422, detail = str(e)) + except SttAudioTooLongError as e: + raise HTTPException(status_code = 413, detail = str(e)) + except SttAudioDecodeError as e: + raise HTTPException(status_code = 400, detail = str(e)) + except Exception as e: + logger.error(f"Transcription error: {e}", exc_info = True) + raise HTTPException(status_code = 500, detail = safe_error_detail(e)) + return JSONResponse(content = result) + + +@studio_router.post("/audio/transcribe") +async def transcribe_audio( + payload: TranscribeRequest, current_subject: str = Depends(get_current_subject) +): + """Transcribe dictation audio to text via the STT sidecar. + + Runs alongside the chat model without evicting it, so any model (including + text-only ones) can be driven by voice. + """ + b64 = payload.audio or "" + if not b64: + raise HTTPException(status_code = 400, detail = "No audio provided.") + if len(b64) > _MAX_AUDIO_B64_CHARS: + raise HTTPException(status_code = 413, detail = "Audio is too large.") + try: + raw = base64.b64decode(b64, validate = True) + except Exception: + raise HTTPException(status_code = 400, detail = "Audio is not valid base64.") + return await _transcribe_audio_bytes( + raw, payload.model, payload.language, payload.fast, payload.engine + ) + + +@studio_router.post("/audio/transcribe/raw") +async def transcribe_audio_raw( + request: Request, + model: Optional[str] = None, + language: Optional[str] = None, + fast: bool = False, + engine: Optional[str] = None, + current_subject: str = Depends(get_current_subject), +): + """Transcribe a raw audio body without base64 or JSON conversion overhead.""" + chunks: list[bytes] = [] + size = 0 + async for chunk in request.stream(): + size += len(chunk) + if size > _MAX_AUDIO_RAW_BYTES: + raise HTTPException(status_code = 413, detail = "Audio is too large.") + chunks.append(chunk) + return await _transcribe_audio_bytes(b"".join(chunks), model, language, fast, engine) + + # ===================================================================== # OpenAI-Compatible Chat Completions (/chat/completions) # ===================================================================== @@ -5767,8 +6529,8 @@ def _decode_audio_base64(b64: str) -> np.ndarray: # cap the encoded length to bound the upload. _MAX_AUDIO_SECONDS additionally # bounds the *decoded* length, since a small compressed file (opus/flac/etc.) # can expand to a far larger PCM array than the encoded-size cap implies. -_MAX_AUDIO_RAW_BYTES = 25 * 1024 * 1024 -_MAX_AUDIO_B64_CHARS = _MAX_AUDIO_RAW_BYTES * 4 // 3 +_MAX_AUDIO_RAW_BYTES = STT_AUDIO_RAW_MAX_BYTES +_MAX_AUDIO_B64_CHARS = STT_AUDIO_B64_MAX_CHARS _MAX_AUDIO_SECONDS = 30 * 60 _WAV_HEADER_BYTES = 44 _MIN_TRANSCODE_AUDIO_SAMPLE_RATE = 8000 @@ -6086,7 +6848,7 @@ def _build_external_messages( metadata; strip it for providers that can't parse the unknown key. 2. Marked server-side builtin cards (`_server_tool: true` on a canonical builtin name, or a Gemini `native_part` payload) are - Studio-internal tool cards from a prior native Gemini turn; + Unsloth-internal tool cards from a prior native Gemini turn; forwarding them to OpenAI / Anthropic / custom OAI-compat gateways sends an orphan `tool_calls` entry (no matching tool declaration, often no matching `role="tool"` reply) that can be rejected. We @@ -6654,6 +7416,51 @@ async def delete_openai_container( await client.close() +def _fill_recommended_sampling_openai(payload, model_id) -> None: + """Apply per-model recommended sampling (and any operator UNSLOTH_SAMPLING_* pin) to a + ChatCompletionRequest in place. + + Only the sampling fields the client did NOT explicitly send (tracked via + ``model_fields_set``) are overwritten, so a client that sets a field stays byte-identical + unless an operator pins it. Fields with neither a recommendation nor a pin keep their + existing (schema-default) value. + """ + from utils.inference.inference_config import resolve_effective_sampling, SAMPLING_FIELD_NAMES + + explicit = { + f: (getattr(payload, f) if f in payload.model_fields_set else None) + for f in SAMPLING_FIELD_NAMES + } + effective = resolve_effective_sampling(model_id, explicit) + for field, value in effective.items(): + setattr(payload, field, value) + + +# /v1/completions is proxied to llama-server verbatim; its repetition knob is "repeat_penalty", +# and every other sampling field keeps its name (mirrors _build_passthrough_payload). +_COMPLETIONS_SAMPLING_BODY_KEY = {"repetition_penalty": "repeat_penalty"} + + +def _fill_recommended_sampling_completions(body: dict, model_id) -> None: + """Apply per-model recommended sampling (and any operator UNSLOTH_SAMPLING_* pin) to a raw + ``/v1/completions`` body in place, so the legacy (non-chat) endpoint honors the same pins as + ``/v1/chat/completions``. + + Unlike :func:`_fill_recommended_sampling_openai`, which fills a ChatCompletionRequest whose + schema already carries per-field defaults, this body is proxied to llama-server as-is. A field + with no operator pin, client value, or per-model recommendation is therefore left untouched + (``fill_defaults = False``) so llama-server keeps its own default rather than being forced onto + this schema's value. llama-server names the repetition knob ``repeat_penalty``, so read and + write that alias for the client-sent value and any pin. + """ + from utils.inference.inference_config import resolve_effective_sampling, SAMPLING_FIELD_NAMES + + explicit = {f: body.get(_COMPLETIONS_SAMPLING_BODY_KEY.get(f, f)) for f in SAMPLING_FIELD_NAMES} + effective = resolve_effective_sampling(model_id, explicit, fill_defaults = False) + for field, value in effective.items(): + body[_COMPLETIONS_SAMPLING_BODY_KEY.get(field, field)] = value + + @router.post("/chat/completions") async def openai_chat_completions( payload: ChatCompletionRequest, @@ -6697,7 +7504,7 @@ async def openai_chat_completions( if payload.provider_id or payload.provider_type: # External provider: this request won't touch the local GGUF, so drop it # from the keep-warm count or its in-flight stream would falsely block a - # concurrent local auto-switch with model_switch_busy. + # concurrent local model switch from proceeding. from core.inference.llama_keepwarm import untrack_current_request untrack_current_request(request.scope) @@ -6772,7 +7579,7 @@ async def openai_chat_completions( # is invalid and must not evict the resident model first. # # Enter the local-loop arm exactly when the passthrough router below would - # run Studio's own tool loop. That gate is `_tools_on or _mcp_allowed` + # run Unsloth's own tool loop. That gate is `_tools_on or _mcp_allowed` # (see the use_tools block): _effective_enable_tools (which lets a # process-wide --enable-tools policy force the loop on) plus mcp_enabled # honoring --disable-tools, and tool_choice="none" disabling it unless the @@ -6797,7 +7604,7 @@ async def openai_chat_completions( or bool(payload.openai_code_exec_container_id) or bool(payload.anthropic_code_exec_container_id) # A JSON-schema response_format is guided-decoding structured output the - # router forwards to the llama-server passthrough, not Studio's tool + # router forwards to the llama-server passthrough, not Unsloth's tool # loop, so a --enable-tools policy must not 400 it as a local-confirm # request under ask/auto. or bool(_extract_response_format(payload)) @@ -6874,7 +7681,7 @@ async def openai_chat_completions( using_gguf = llama_backend.is_loaded # OpenAI-SDK clients send ``chat_template_kwargs`` via ``extra_body``, which - # the SDK spreads into the request body at the top level. Studio's + # the SDK spreads into the request body at the top level. Unsloth's # ChatCompletionRequest has ``extra="allow"`` so pydantic stashes them in # ``model_extra``, but downstream generators consume the typed # ``payload.enable_thinking``. Lift ``enable_thinking`` from the extra-body @@ -6983,6 +7790,13 @@ async def openai_chat_completions( completion_id = f"chatcmpl-{uuid.uuid4().hex[:12]}" created = int(time.time()) + # Apply recommended sampling + operator pins to the omitted fields before generating, + # so audio-input (non-whisper) generation honors `unsloth run --temperature` and + # per-model recommendations like chat does. Whisper (ASR) ignores these fields. + _fill_recommended_sampling_openai( + payload, getattr(backend, "active_model_name", None) or model_name + ) + def audio_input_generate(): if model_info.get("audio_type") == "whisper": return backend.generate_whisper_response( @@ -7028,6 +7842,13 @@ async def openai_chat_completions( chunk_text = await asyncio.to_thread(next, gen, _DONE) if chunk_text is _DONE: break + if isinstance(chunk_text, GenStreamError): + _msg = _friendly_gen_stream_error(chunk_text) + api_monitor.fail(monitor_id, _msg) + yield _openai_stream_error_sse( + {"error": {"message": _msg, "type": "server_error"}} + ) + return if chunk_text: api_monitor.append_reply(monitor_id, chunk_text) yield _chat_content_chunk( @@ -7043,8 +7864,11 @@ async def openai_chat_completions( raise except Exception as e: logger.error(f"Error during audio input streaming: {e}", exc_info = True) - api_monitor.fail(monitor_id, _friendly_error(e)) - yield f"data: {json.dumps({'error': {'message': _friendly_error(e), 'type': 'server_error'}})}\n\n" + _msg = _friendly_error(e) + api_monitor.fail(monitor_id, _msg) + yield _openai_stream_error_sse( + {"error": {"message": _msg, "type": "server_error"}} + ) finally: await _stop_local_disconnect_cancel_watcher(disconnect_watcher) _tracker.__exit__(None, None, None) @@ -7061,7 +7885,15 @@ async def openai_chat_completions( ) else: try: - full_text = "".join(audio_input_generate()) + full_text = "" + for chunk_text in audio_input_generate(): + if isinstance(chunk_text, GenStreamError): + _msg = _friendly_gen_stream_error(chunk_text) + api_monitor.fail(monitor_id, _msg) + raise HTTPException(status_code = 500, detail = _msg) + full_text += chunk_text + except HTTPException: + raise except Exception as e: api_monitor.fail(monitor_id, _friendly_error(e)) raise @@ -7108,9 +7940,21 @@ async def openai_chat_completions( ), ) + # Apply per-model recommended sampling (and any operator UNSLOTH_SAMPLING_* pin) to the + # fields the client omitted, so agents and API clients get the model's tuned defaults + # unless they set the field explicitly. Placed after external-provider routing (which + # returned above) so only local llama-server / transformers requests are touched, and it + # covers both the passthrough and non-passthrough branches below since both read payload.*. + _reco_model_id = ( + getattr(llama_backend, "model_identifier", None) + if using_gguf + else getattr(backend, "active_model_name", None) + ) or model_name + _fill_recommended_sampling_openai(payload, _reco_model_id) + # ── Standard OpenAI function-calling pass-through (GGUF only) ──── # When a client (opencode / Claude Code via OpenAI compat / Cursor / - # Continue / ...) sends standard OpenAI `tools` without Studio's + # Continue / ...) sends standard OpenAI `tools` without Unsloth's # `enable_tools` shorthand, forward the request to llama-server # verbatim so structured `tool_calls` flow back to the client. This # branch runs BEFORE `_extract_content_parts` because that helper is @@ -7133,7 +7977,7 @@ async def openai_chat_completions( _has_tool_catalog = bool(payload.tools and len(payload.tools) > 0) _has_active_tool_catalog = _has_tool_catalog and payload.tool_choice != "none" _has_client_tool_contract = _has_active_tool_catalog or _has_tool_messages - # The Studio tool loop needs a tool-capable backend, so a request that asks + # The Unsloth tool loop needs a tool-capable backend, so a request that asks # for it on a backend that can't run it (DiffusionGemma forces supports_tools # off) must not steal client tools from the passthrough (#6851). _studio_tool_loop_requested = ( @@ -7329,7 +8173,7 @@ async def openai_chat_completions( use_tools = False if use_tools: - # permission_mode ask/auto require the confirm gate for Studio's own + # permission_mode ask/auto require the confirm gate for Unsloth's own # tool loop. The request validator self-enables confirm only for # request-level tool signals (enable_tools/enabled_tools/mcp_enabled); # when a CLI policy (--enable-tools) forces the loop on without those, @@ -8605,19 +9449,33 @@ async def openai_chat_completions( # Classify capability flags from the loaded template. _sf_model_info = backend.models.get(backend.active_model_name, {}) _sf_tpl = (_sf_model_info.get("chat_template_info") or {}).get("template") - _sf_features = _detect_safetensors_features(backend, _sf_tpl) - - # GGUF parity: enable_thinking templates prefill an unclosed ; split into - # reasoning_content deltas so the UI renders the block for safetensors and MLX. - _sf_parse_think = bool( - _sf_features.get("supports_reasoning") or _sf_features.get("reasoning_always_on") + # Named templates may expose native reasoning only in their ``tool_use`` + # branch. Use a truthy placeholder for Unsloth-managed tools, whose concrete + # schemas are selected below, and the request schemas for client passthrough. + _sf_server_tool_intent = bool( + _effective_enable_tools(payload) or _explicit_studio_tool_loop_requested(payload) ) - # Prefilled-open only for prefill styles with thinking on; gpt-oss uses the normal mode. - _sf_reasoning_prefilled = _sf_reasoning_prefill_mode( - _sf_features, - payload.enable_thinking, - _sf_tpl, - reasoning_effort = payload.reasoning_effort, + _sf_template_tools = payload.tools if payload.tool_choice != "none" else None + if not _sf_template_tools and _sf_server_tool_intent: + _sf_template_tools = ({},) + + def _sf_response_protocol(tools = None): + features = _detect_safetensors_features(backend, _sf_tpl, tools = tools) + parse_think = bool( + features.get("supports_reasoning") or features.get("reasoning_always_on") + ) + reasoning_prefilled = _sf_reasoning_prefill_mode( + features, + payload.enable_thinking, + _sf_tpl, + reasoning_effort = payload.reasoning_effort, + ) + return features, parse_think, reasoning_prefilled + + # GGUF parity: split canonical output into reasoning_content. The + # selected template branch must match whether this request renders tools. + _sf_features, _sf_parse_think, _sf_reasoning_prefilled = _sf_response_protocol( + _sf_template_tools ) def _new_sf_reasoning_extractor(): @@ -8671,7 +9529,7 @@ async def openai_chat_completions( _sf_use_tools = False if _sf_use_tools: - # permission_mode ask/auto require the confirm gate for Studio's own tool + # permission_mode ask/auto require the confirm gate for Unsloth's own tool # loop; when a CLI policy (--enable-tools) forces the loop on without a # request-level tool signal, derive confirm here so the mode still gates # the call (matching the GGUF path). off/full never prompt. @@ -8767,6 +9625,7 @@ async def openai_chat_completions( permission_mode = payload.permission_mode, use_adapter = payload.use_adapter, stats_holder = _sf_stats_holder, + reasoning_prefilled = _sf_reasoning_prefilled, ) _sf_tool_sentinel = object() @@ -8826,6 +9685,18 @@ async def openai_chat_completions( _sf_next_task = None if event is _sf_tool_sentinel: break + if isinstance(event, GenStreamError): + backend.reset_generation_state() + _msg = _friendly_gen_stream_error(event) + api_monitor.fail(monitor_id, _msg) + yield _openai_stream_error_sse( + {"error": {"message": _msg, "type": "server_error"}} + ) + return + if not isinstance(event, dict): + raise RuntimeError( + f"Invalid safetensors tool event: {type(event).__name__}" + ) if event["type"] == "heartbeat": # Tool-execution wrapper heartbeat -> SSE keepalive. @@ -8913,6 +9784,11 @@ async def openai_chat_completions( backend.reset_generation_state() api_monitor.finish(monitor_id, "cancelled") raise + except GenStreamErrorRaised as exc: + backend.reset_generation_state() + _msg = _friendly_gen_stream_error(exc) + api_monitor.fail(monitor_id, _msg) + yield _openai_stream_error_sse({"error": {"message": _msg, "type": "server_error"}}) except Exception: backend.reset_generation_state() # Generic wire message; full trace stays in the log (CWE-209: @@ -8962,6 +9838,15 @@ async def openai_chat_completions( for event in gen: if cancel_event.is_set(): break + if isinstance(event, GenStreamError): + raise HTTPException( + status_code = 500, + detail = _friendly_gen_stream_error(event), + ) + if not isinstance(event, dict): + raise RuntimeError( + f"Invalid safetensors tool event: {type(event).__name__}" + ) if event.get("type") == "content": full_text = _strip_tool_xml_for_display( event.get("text", ""), @@ -9002,6 +9887,15 @@ async def openai_chat_completions( backend.reset_generation_state() api_monitor.finish(monitor_id, "cancelled") raise + except GenStreamErrorRaised as exc: + backend.reset_generation_state() + _msg = _friendly_gen_stream_error(exc) + api_monitor.fail(monitor_id, _msg) + raise HTTPException(status_code = 500, detail = _msg) + except HTTPException as exc: + backend.reset_generation_state() + api_monitor.fail(monitor_id, str(exc.detail)) + raise except Exception: backend.reset_generation_state() # CWE-209: generic detail; full trace in log. @@ -9088,6 +9982,12 @@ async def openai_chat_completions( else: gen_kwargs["tools"] = payload.tools + # The potential tool context above is needed before server/client routing is + # known. This standard path now has the exact schemas that will be rendered, + # so resolve reasoning parsing again to keep empty registries, forced-tool + # misses, and tool_choice="none" on the marker-free template branch. + _, _sf_parse_think, _sf_reasoning_prefilled = _sf_response_protocol(gen_kwargs.get("tools")) + # Request-scoped usage/timings receptacle (filled at gen_done). stats_holder: dict = {} @@ -9168,6 +10068,14 @@ async def openai_chat_completions( _next_task = None if cumulative is _DONE: break + if isinstance(cumulative, GenStreamError): + backend.reset_generation_state() + _msg = _friendly_gen_stream_error(cumulative) + api_monitor.fail(monitor_id, _msg) + yield _openai_stream_error_sse( + {"error": {"message": _msg, "type": "server_error"}} + ) + return if await request.is_disconnected(): cancel_event.set() backend.reset_generation_state() @@ -9274,6 +10182,13 @@ async def openai_chat_completions( backend.reset_generation_state() api_monitor.finish(monitor_id, "cancelled") raise + except GenStreamErrorRaised as exc: + # Adapter-controlled (compare-mode) backend failure. Honor the + # public flag so operational errors surface their real message. + backend.reset_generation_state() + _msg = _friendly_gen_stream_error(exc) + api_monitor.fail(monitor_id, _msg) + yield _openai_stream_error_sse({"error": {"message": _msg, "type": "server_error"}}) except Exception as e: backend.reset_generation_state() logger.error(f"Error during OpenAI streaming: {e}", exc_info = True) @@ -9317,6 +10232,11 @@ async def openai_chat_completions( try: full_text = "" for token in generate(): + if isinstance(token, GenStreamError): + backend.reset_generation_state() + _msg = _friendly_gen_stream_error(token) + api_monitor.fail(monitor_id, _msg) + raise HTTPException(status_code = 500, detail = _msg) full_text = token # Split prefilled reasoning (GGUF parity); also covers MLX via @@ -9415,6 +10335,15 @@ async def openai_chat_completions( api_monitor.finish(monitor_id) return _model_json_response(response) + except HTTPException: + raise + except GenStreamErrorRaised as exc: + # Adapter-controlled (compare-mode) backend failure. Honor the public + # flag so operational errors surface their real message. + backend.reset_generation_state() + _msg = _friendly_gen_stream_error(exc) + api_monitor.fail(monitor_id, _msg) + raise HTTPException(status_code = 500, detail = _msg) except Exception as e: backend.reset_generation_state() logger.error(f"Error during OpenAI completion: {e}", exc_info = True) @@ -9800,6 +10729,10 @@ async def openai_completions(request: Request, current_subject: str = Depends(ge if _resolved_max_tokens is not None else (llama_backend.context_length or _DEFAULT_MAX_TOKENS_FLOOR) ) + # Apply per-model recommended sampling and any operator UNSLOTH_SAMPLING_* pin to the raw + # body so /v1/completions honors the same pins as /v1/chat/completions; it is otherwise a + # verbatim proxy that would keep llama-server's defaults for every omitted sampling field. + _fill_recommended_sampling_completions(body, getattr(llama_backend, "model_identifier", None)) target_url = f"{llama_backend.base_url}/v1/completions" is_stream = body.get("stream", False) prompt_text = _flatten_monitor_prompt(body.get("prompt", "")) @@ -10758,6 +11691,9 @@ async def _responses_stream( detail = "Image provided but current GGUF model does not support vision.", ) + # Streaming /v1/responses builds the passthrough body directly (bypassing + # openai_chat_completions), so apply recommended sampling here too. + _fill_recommended_sampling_openai(chat_req, getattr(llama_backend, "model_identifier", None)) body = _build_openai_passthrough_body( chat_req, backend_ctx = llama_backend.context_length, llama_backend = llama_backend ) @@ -11860,7 +12796,7 @@ def _anthropic_requested_studio_tools(tools: Optional[list]) -> set[str]: def _select_anthropic_server_tools( all_tools: list[dict], requested_studio_tools: set[str], enabled_tools: Optional[list[str]] ) -> list[dict]: - """Select Studio tools requested through Anthropic tools and extensions.""" + """Select Unsloth tools requested through Anthropic tools and extensions.""" if not requested_studio_tools and enabled_tools is None: return all_tools @@ -12099,7 +13035,7 @@ async def anthropic_messages( ), ) - # Reject an unsupported confirm-gated permission mode for Studio's own + # Reject an unsupported confirm-gated permission mode for Unsloth's own # ("server") Anthropic tools before the switch, mirroring the malformed- and # mixed-tool checks above. ask always wants a per-call pause this passthrough # cannot offer, so it 400s whenever server tools are selected. auto only needs @@ -12189,14 +13125,28 @@ async def anthropic_messages( # endpoint matches /v1/chat/completions. _has_image = _normalize_anthropic_openai_images(openai_messages, llama_backend.is_vision) - temperature = payload.temperature if payload.temperature is not None else 0.6 - top_p = payload.top_p if payload.top_p is not None else 0.95 - top_k = payload.top_k if payload.top_k is not None else 20 - min_p = payload.min_p if payload.min_p is not None else 0.01 - repetition_penalty = ( - payload.repetition_penalty if payload.repetition_penalty is not None else 1.0 + # Fill omitted sampling fields with the per-model recommendation (or an operator + # UNSLOTH_SAMPLING_* pin); an explicit client value wins unless the operator pinned it. + # Anthropic sampling fields are Optional, so None already marks "client omitted". + from utils.inference.inference_config import resolve_effective_sampling + + _anthropic_sampling = resolve_effective_sampling( + getattr(llama_backend, "model_identifier", None) or model_name, + { + "temperature": payload.temperature, + "top_p": payload.top_p, + "top_k": payload.top_k, + "min_p": payload.min_p, + "repetition_penalty": payload.repetition_penalty, + "presence_penalty": payload.presence_penalty, + }, ) - presence_penalty = payload.presence_penalty if payload.presence_penalty is not None else 0.0 + temperature = _anthropic_sampling["temperature"] + top_p = _anthropic_sampling["top_p"] + top_k = _anthropic_sampling["top_k"] + min_p = _anthropic_sampling["min_p"] + repetition_penalty = _anthropic_sampling["repetition_penalty"] + presence_penalty = _anthropic_sampling["presence_penalty"] stop = payload.stop_sequences or None # Translate Anthropic tool_choice to OpenAI format for llama-server. Falls @@ -12588,11 +13538,11 @@ async def _anthropic_tool_stream( ends_on_tool_use = True elif etype == "tool_end": tool_blocks_emitted += 1 - # A tool_end means Studio executed the tool server-side, so + # A tool_end means Unsloth executed the tool server-side, so # the response no longer ends on a pending client action. # Without this, a server tool that produces no trailing text # would be mislabeled stop_reason "tool_use", telling the - # client to run a tool Studio already ran. + # client to run a tool Unsloth already ran. ends_on_tool_use = False elif etype == "content" and event.get("text"): ends_on_tool_use = False @@ -13508,7 +14458,7 @@ def _openai_messages_for_passthrough(payload) -> list[dict]: structured ``tool_calls``. Content-parts images already in the list are left untouched. - When a client uses Studio's legacy ``image_base64`` top-level field, the + When a client uses Unsloth's legacy ``image_base64`` top-level field, the image is re-encoded to PNG (llama-server's stb_image has limited format support) and spliced into the last user message as an OpenAI ``image_url`` content part so vision + function-calling requests work transparently. @@ -13642,7 +14592,7 @@ def _build_openai_passthrough_body( ) -> dict: """Assemble the llama-server request body from a ChatCompletionRequest. - Only known OpenAI / llama-server fields are forwarded, so Studio-specific + Only known OpenAI / llama-server fields are forwarded, so Unsloth-specific extensions (``enable_tools``, ``enabled_tools``, ``session_id``, ...) never leak to the backend. """ @@ -13892,7 +14842,7 @@ async def _openai_passthrough_stream_admitted( admission_lease: LlamaAdmissionLease, tracker, ): - """Streaming client-side pass-through after Studio granted an upstream slot. + """Streaming client-side pass-through after Unsloth granted an upstream slot. Forwards the client's OpenAI function-calling request to llama-server and relays the SSE stream back with minimal normalization (reasoning-only diff --git a/studio/backend/routes/llama.py b/studio/backend/routes/llama.py index 540647e3bc..84b89ad7d5 100644 --- a/studio/backend/routes/llama.py +++ b/studio/backend/routes/llama.py @@ -1,7 +1,7 @@ # SPDX-License-Identifier: AGPL-3.0-only # Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 -"""llama.cpp prebuilt update endpoints. +"""llama.cpp prebuilt update endpoints -- the single main update item. GET /api/llama/update-status -> is a newer prebuilt available + job state POST /api/llama/update -> download + atomically swap to the latest @@ -9,13 +9,19 @@ POST /api/llama/update -> download + atomically swap to the latest Detection reuses utils.llama_cpp_freshness; the swap reuses install_llama_prebuilt.py via utils.llama_cpp_update. Both fail open so the UI never blocks on a missing marker / offline GitHub. + +whisper.cpp updates piggyback here: the status payload carries a whisper +sub-status (update_available is the llama OR whisper union) and the apply job +chains a whisper phase after the llama phase when whisper is behind, with a +per-phase breakdown in job.phases. All pre-existing top-level fields keep +their shape, so older clients keep working unchanged. """ from __future__ import annotations import asyncio import threading -from typing import Optional +from typing import Literal, Optional from fastapi import APIRouter, Depends, Query from pydantic import BaseModel, Field @@ -38,6 +44,31 @@ class LlamaUpdateJob(BaseModel): progress: Optional[float] = Field(None, description = "0..1 while running, 1 on success.") started_at: Optional[str] = None finished_at: Optional[str] = None + phases: Optional[dict] = Field( + None, + description = ( + "Per-phase breakdown of a chained llama+whisper job " + "(name -> state/progress/to_tag/...); None for pre-chaining jobs." + ), + ) + + +class WhisperSubStatus(BaseModel): + """The whisper piggyback inside the llama update item.""" + + update_available: bool = Field( + False, description = "True when the chained apply would run a whisper phase." + ) + installed_tag: Optional[str] = None + latest_tag: Optional[str] = None + update_size_bytes: Optional[int] = None + skip_reason: Optional[str] = Field( + None, + description = ( + "Why the whisper phase would be skipped " + "(up_to_date | local_link | source_build | not_installed | ...)." + ), + ) class LlamaUpdateStatusResponse(BaseModel): @@ -46,7 +77,18 @@ class LlamaUpdateStatusResponse(BaseModel): description = "True when the install came from an Unsloth prebuilt (has a marker).", ) update_available: bool = Field( - False, description = "True when the latest release is genuinely newer than the install." + False, + description = ( + "True when an update would do something: llama.cpp is behind OR the " + "whisper piggyback is behind." + ), + ) + llama_update_available: bool = Field( + False, description = "True when the latest llama.cpp release is newer than the install." + ) + update_component: Optional[Literal["llama", "whisper"]] = Field( + None, + description = "Component whose versions the combined update banner should display.", ) stale: bool = Field( False, description = "Update available AND install older than the staleness threshold." @@ -62,6 +104,9 @@ class LlamaUpdateStatusResponse(BaseModel): update_size_bytes: Optional[int] = Field( None, description = "Download size of the prebuilt Update would fetch, in bytes." ) + whisper: Optional[WhisperSubStatus] = Field( + None, description = "Whisper piggyback sub-status; None when the probe is unavailable." + ) job: LlamaUpdateJob = Field(default_factory = LlamaUpdateJob) diff --git a/studio/backend/routes/mcp_servers.py b/studio/backend/routes/mcp_servers.py index 71f0fd2874..dc018d163a 100644 --- a/studio/backend/routes/mcp_servers.py +++ b/studio/backend/routes/mcp_servers.py @@ -82,7 +82,7 @@ def _validate_url(url: str) -> str: if _looks_like_command(trimmed): detail = ( "Local commands aren't enabled on this server. To allow them, " - "set UNSLOTH_STUDIO_ALLOW_STDIO_MCP=1 and restart Studio, or use " + "set UNSLOTH_STUDIO_ALLOW_STDIO_MCP=1 and restart Unsloth, or use " "an http:// or https:// URL instead." ) else: diff --git a/studio/backend/routes/models.py b/studio/backend/routes/models.py index b8526c75e7..ed83a12f48 100644 --- a/studio/backend/routes/models.py +++ b/studio/backend/routes/models.py @@ -59,59 +59,51 @@ def _safe_is_dir(path) -> bool: return False -# Hub repo id shape ("owner/name", no leading separator); anything else is -# treated as a local filesystem path. -_HF_REPO_ID_RE = re.compile(r"^[A-Za-z0-9][\w.\-]*/[\w.\-]+$") +# Shared with the hub inventory scans; keep the private aliases so existing +# importers stay valid. ``_HF_REPO_ID_RE`` is the Hub repo id shape ("owner/name"); +# anything else is treated as a local filesystem path. +from utils.hidden_models import ( + _HF_REPO_ID_RE, + _existing_resolved_path, + _safe_resolve, + is_hidden_model as _is_hidden_model, +) -def _is_hidden_model(*values: str | None) -> bool: - """True if any id/path is the RAG embedding model (EMBEDDING_MODEL or - EMBED_GGUF_REPO basename) or the llama.cpp install validation probe - (ggml-org/models / stories260K), so pickers hide them (GGUF and non-GGUF). - None are usable chat models; the probe can be cached as a side effect of - installing the prebuilt llama-server and otherwise sorts smallest, so it - would be auto-selected. A local-path embedder is matched by exact resolved - path only: a generic basename like "model" must not substring-hide - unrelated chat models.""" +def hidden_model_matchers() -> tuple[list[str], list[str], list[str]]: + """Substring needles, exact repo ids, and exact resolved paths identifying + infra models (the RAG embedder and the llama.cpp install validation probe) + that pickers hide. Served by the ``/api/hub/hidden-models`` endpoint. A + configured HF-repo embedder is published as its exact lowercased repo id + (mirroring ``utils.hidden_models.is_hidden_model``) and a local-path + embedder as its exact resolved path only: a generic basename like "model" + must not substring-hide unrelated chat models.""" from core.rag import config as rag_config needles = [ - # The validation probe's repo (matches the cached repo id) and its exact - # filename (matches the on-disk path). The filename carries the .gguf so - # it does not hide unrelated repos like ``user/stories260K-finetune-GGUF``. + # The validation probe's repo and its exact filename. The filename carries + # .gguf so it won't hide unrelated repos like ``user/stories260K-finetune-GGUF``. "ggml-org/models", "stories260k.gguf", ] + exact_ids: list[str] = [] exact_paths: list[str] = [] for model in ( rag_config.effective_embedding_model(), rag_config.effective_gguf_repo(), ): - if _HF_REPO_ID_RE.match(model): - needles.append(model.split("/")[-1].lower()) + # Resolve an existing local path before the repo-id regex: a local embedder + # shaped like "models/embedder" is an exact path, not a Hub repo id. + existing_path = _existing_resolved_path(model) + if existing_path: + exact_paths.append(existing_path.lower()) + elif _HF_REPO_ID_RE.match(model): + exact_ids.append(model.lower()) else: resolved = _safe_resolve(Path(model).expanduser()) if resolved: exact_paths.append(resolved.lower()) - for v in values: - if not v: - continue - low = v.lower() - if any(n in low for n in needles): - return True - if exact_paths: - resolved = _safe_resolve(Path(v).expanduser()) - if resolved and resolved.lower() in exact_paths: - return True - return False - - -def _safe_resolve(path: Path) -> Optional[str]: - """resolve() to a string, or None when the path is inaccessible.""" - try: - return str(path.resolve()) - except OSError: - return None + return needles, exact_ids, exact_paths backend_path = Path(__file__).parent.parent.parent @@ -138,6 +130,7 @@ try: _pick_best_gguf, _extract_quant_label, _is_big_endian_gguf_path, + _is_mtp_drafter, is_audio_input_type, ) from core.inference import get_inference_backend @@ -170,6 +163,7 @@ except ImportError: _pick_best_gguf, _extract_quant_label, _is_big_endian_gguf_path, + _is_mtp_drafter, is_audio_input_type, ) from core.inference import get_inference_backend @@ -230,11 +224,8 @@ def derive_model_type( def _resolve_hf_cache_dir() -> Path: """Resolve local HF cache root used by hub downloads.""" - try: - from huggingface_hub.constants import HF_HUB_CACHE - return Path(HF_HUB_CACHE) - except Exception: - return Path.home() / ".cache" / "huggingface" / "hub" + from utils.hf_cache_settings import get_hf_cache_paths + return get_hf_cache_paths().hub_cache def _is_model_directory(d: Path) -> bool: @@ -376,10 +367,12 @@ def _scan_models_dir(models_dir: Path, *, limit: int | None = None) -> List[Loca return found -def _scan_hf_cache(cache_dir: Path) -> List[LocalModelInfo]: +def _scan_hf_cache(cache_dir: Path, *, active_cache: bool = True) -> List[LocalModelInfo]: if not cache_dir.exists() or not cache_dir.is_dir(): return [] + from hub.utils import inventory_scan as hf_cache_scan + found: List[LocalModelInfo] = [] for repo_dir in cache_dir.glob("models--*"): if not repo_dir.is_dir(): @@ -395,13 +388,21 @@ def _scan_hf_cache(cache_dir: Path) -> List[LocalModelInfo]: except OSError: updated_at = None + partial = hf_cache_scan.is_snapshot_partial("model", model_id, repo_dir) + partial = partial or hf_cache_scan.is_gguf_repo_partial(model_id, repo_dir) + + load_id = model_id + if not active_cache: + load_id = _resolve_hf_cache_realpath(repo_dir) or str(repo_dir.resolve()) found.append( LocalModelInfo( - id = model_id, + id = load_id, model_id = model_id, display_name = model_id.split("/")[-1], - path = str(repo_dir), + path = load_id if not active_cache else str(repo_dir), source = "hf_cache", + active_cache = active_cache, + partial = partial, updated_at = updated_at, ), ) @@ -544,7 +545,7 @@ def _ollama_links_dir(ollama_dir: Path) -> Optional[Path]: """Return a writable directory for Ollama ``.gguf`` symlinks. Prefers ``/.studio_links/`` so links sit next to their - blobs; falls back to a per-ollama-dir namespace under Studio's cache + blobs; falls back to a per-ollama-dir namespace under Unsloth's cache when the models dir is read-only (common for system installs). """ from utils.paths.storage_roots import cache_root @@ -555,7 +556,7 @@ def _ollama_links_dir(ollama_dir: Path) -> Optional[Path]: return primary except OSError as e: logger.debug( - "Ollama dir %s not writable for .studio_links (%s); falling back to Studio cache", + "Ollama dir %s not writable for .studio_links (%s); falling back to Unsloth cache", ollama_dir, e, ) @@ -594,7 +595,7 @@ def _scan_ollama_dir(ollama_dir: Path, limit: Optional[int] = None) -> List[Loca model, keyed by a short hash of the manifest path, so ``detect_mmproj_file`` only sees that model's projector). Links are symlinks when possible, else hardlinks; the link dir is - ``.studio_links/`` when writable, else Studio's cache. + ``.studio_links/`` when writable, else Unsloth's cache. """ manifests_root = ollama_dir / "manifests" if not manifests_root.is_dir(): @@ -782,26 +783,34 @@ def collect_local_models(models_root: Path) -> List[LocalModelInfo]: legacy_hf_cache_dir, lmstudio_model_dirs, ) + from utils.hf_cache_settings import known_hf_hub_caches hf_cache_dir = _resolve_hf_cache_dir() legacy_hf = legacy_hf_cache_dir() hf_default = hf_default_cache_dir() lm_dirs = lmstudio_model_dirs() - local_models = _scan_models_dir(models_root) + _scan_hf_cache(hf_cache_dir) - - # Resolve once; an inaccessible aux cache must skip that scan, not 500. - hf_cache_real = _safe_resolve(hf_cache_dir) - legacy_real = _safe_resolve(legacy_hf) - default_real = _safe_resolve(hf_default) - - # Scan legacy Unsloth HF cache for backward compatibility. - if _safe_is_dir(legacy_hf) and legacy_real != hf_cache_real: - local_models += _scan_hf_cache(legacy_hf) - - # Scan HF system default cache (may differ under env overrides). - if _safe_is_dir(hf_default) and default_real != hf_cache_real and default_real != legacy_real: - local_models += _scan_hf_cache(hf_default) + local_models = _scan_models_dir(models_root) + active_cache_real = _safe_resolve(hf_cache_dir) + active_cache_key = os.path.normcase(active_cache_real) if active_cache_real else None + seen_hf: set[str] = set() + for cache_dir in ( + hf_cache_dir, + *known_hf_hub_caches(), + legacy_hf, + hf_default, + ): + cache_real = _safe_resolve(cache_dir) + if cache_real is None: + continue + cache_key = os.path.normcase(str(cache_real)) + if cache_key in seen_hf: + continue + seen_hf.add(cache_key) + local_models += _scan_hf_cache( + cache_dir, + active_cache = cache_key == active_cache_key, + ) # Scan LM Studio directories. for lm_dir in lm_dirs: @@ -823,7 +832,7 @@ def collect_local_models(models_root: Path) -> List[LocalModelInfo]: m for m in ( _scan_models_dir(folder_path, limit = _MAX_MODELS_PER_FOLDER) - + _scan_hf_cache(folder_path) + + _scan_hf_cache(folder_path, active_cache = False) + _scan_lmstudio_dir(folder_path) ) if not any(p in (".studio_links", "ollama_links") for p in Path(m.path).parts) @@ -844,16 +853,26 @@ def collect_local_models(models_root: Path) -> List[LocalModelInfo]: # even when the model is also in the HF cache. deduped: dict[str, LocalModelInfo] = {} for model in local_models: - key = f"{model.id}\x00custom" if model.source == "custom" else model.id - if key not in deduped: + semantic_id = model.model_id if model.source == "hf_cache" and model.model_id else model.id + key = f"{semantic_id}\x00custom" if model.source == "custom" else semantic_id + existing = deduped.get(key) + prefer_model = existing is None + if existing is not None and model.source == existing.source == "hf_cache": + if model.partial != existing.partial: + prefer_model = not model.partial + elif bool(model.active_cache) != bool(existing.active_cache): + prefer_model = bool(model.active_cache) + else: + prefer_model = (model.updated_at or 0) > (existing.updated_at or 0) + if prefer_model: deduped[key] = model models = sorted( deduped.values(), - key = lambda item: (item.updated_at or 0), + key = lambda item: item.updated_at or 0, reverse = True, ) - return [m for m in models if not _is_hidden_model(m.id, m.path)] + return [m for m in models if not _is_hidden_model(m.id, m.model_id, m.path)] @router.get("/local", response_model = LocalModelListResponse) @@ -1194,7 +1213,7 @@ def _build_browse_allowlist( """Return the root directories the folder browser may walk. The same list seeds the sidebar suggestion chips, so chip targets are - always reachable. Roots: HOME, resolved HF cache dirs, Studio's + always reachable. Roots: HOME, resolved HF cache dirs, Unsloth's outputs/exports/studio root, registered scan folders, and well-known local-LLM dirs (LM Studio, Ollama, ``~/models``); each added only if it resolves to a real directory. @@ -1208,10 +1227,7 @@ def _build_browse_allowlist( legacy_hf_cache_dir, well_known_model_dirs, ) - from utils.paths.external_media import ( - linux_run_media_mount_roots, - windows_drive_roots, - ) + from utils.paths import external_media from storage.studio_db import list_scan_folders candidates: list[Path] = [] @@ -1228,9 +1244,12 @@ def _build_browse_allowlist( _add(Path.home()) if media_roots is None: - media_roots = linux_run_media_mount_roots() + media_roots = [ + *external_media.linux_run_media_mount_roots(), + *external_media.macos_volume_roots(), + ] if drive_roots is None: - drive_roots = windows_drive_roots() + drive_roots = external_media.windows_drive_roots() for p in media_roots: _add(p) for p in drive_roots: @@ -1486,7 +1505,7 @@ def browse_folders( "Directory to list. If omitted, defaults to the current user's " "home directory. Tilde (`~`) and relative paths are expanded. " "Must resolve inside the allowlist of browseable roots (HOME, " - "HF cache, Studio dirs, registered scan folders, well-known " + "HF cache, Unsloth dirs, registered scan folders, well-known " "model dirs)." ), ), @@ -1508,10 +1527,7 @@ def browse_folders( then hidden (if ``show_hidden=true``). """ from utils.paths import hf_default_cache_dir, well_known_model_dirs - from utils.paths.external_media import ( - linux_run_media_mount_roots, - windows_drive_roots, - ) + from utils.paths import external_media from storage.studio_db import ( contains_sensitive_path_component, is_denied_system_path, @@ -1520,8 +1536,11 @@ def browse_folders( # Probe removable-media and Windows drive roots once; the allowlist and # chips reuse the result so a disconnected mapped drive isn't scanned twice. - media_roots = linux_run_media_mount_roots() - drive_roots = windows_drive_roots() + media_roots = [ + *external_media.linux_run_media_mount_roots(), + *external_media.macos_volume_roots(), + ] + drive_roots = external_media.windows_drive_roots() # Build once; the sandbox check and suggestion chips share it. allowed_roots = _build_browse_allowlist(media_roots, drive_roots) @@ -1797,9 +1816,11 @@ def _get_model_size_bytes(model_name: str, hf_token: Optional[str] = None) -> Op async def get_model_config( model_name: str, hf_token: Optional[str] = Query(None), + header_hf_token: Optional[str] = Depends(get_hf_token), current_subject: str = Depends(get_current_subject), ): """Get configuration for a specific model (wraps load_model_defaults).""" + hf_token = _normalize_hf_token(header_hf_token) or _normalize_hf_token(hf_token) try: if not is_local_path(model_name): resolved = resolve_cached_repo_id_case(model_name) @@ -2038,19 +2059,19 @@ async def discard_remote_code_download( # Never delete a model that is loaded for inference. try: + from hub.services.models.deletion import _loaded_id_matches_repo from routes.inference import get_llama_cpp_backend + llama_backend = get_llama_cpp_backend() if llama_backend.is_loaded and llama_backend.model_identifier: - loaded = llama_backend.model_identifier.lower() - if loaded == model_name.lower() or loaded.startswith(model_name.lower()): + if _loaded_id_matches_repo(llama_backend.model_identifier, model_name): return {"deleted": False, "reason": "loaded"} except Exception: pass try: inference_backend = get_inference_backend() if inference_backend.active_model_name: - active = inference_backend.active_model_name.lower() - if active == model_name.lower() or active.startswith(model_name.lower()): + if _loaded_id_matches_repo(inference_backend.active_model_name, model_name): return {"deleted": False, "reason": "loaded"} except Exception: pass @@ -2251,15 +2272,15 @@ async def delete_finetuned_model( gguf_variant: Optional[str] = Body(None), current_subject: str = Depends(get_current_subject), ): - """Delete a Studio-trained or exported model from disk. + """Delete an Unsloth-trained or exported model from disk. - Only paths under Studio's outputs/exports roots are accepted. + Only paths under Unsloth's outputs/exports roots are accepted. Exported GGUF entries can delete one quant variant at a time. """ if source not in {"training", "exported"}: raise HTTPException( status_code = 400, - detail = "Only trained or exported Studio models can be deleted", + detail = "Only trained or exported Unsloth models can be deleted", ) if not model_path or not model_path.strip(): @@ -2291,14 +2312,14 @@ async def delete_finetuned_model( if not _is_path_under_lexically(delete_path, allowed_root): raise HTTPException( status_code = 400, - detail = "Model path is outside Studio storage", + detail = "Model path is outside Unsloth storage", ) if export_type == "gguf" and gguf_variant: target_path = delete_path.resolve() if not _is_path_under(target_path, allowed_root): raise HTTPException( status_code = 400, - detail = "Model path is outside Studio storage", + detail = "Model path is outside Unsloth storage", ) else: target_path = delete_path @@ -2311,7 +2332,7 @@ async def delete_finetuned_model( if should_check_resolved_path and not _is_path_under(target_path, allowed_root): raise HTTPException( status_code = 400, - detail = "Model path is outside Studio storage", + detail = "Model path is outside Unsloth storage", ) if target_path == allowed_root: raise HTTPException( @@ -2518,6 +2539,7 @@ async def get_lora_base_model(lora_path: str, current_subject: str = Depends(get async def check_vision_model( model_name: str, hf_token: Optional[str] = Query(None), + header_hf_token: Optional[str] = Depends(get_hf_token), current_subject: str = Depends(get_current_subject), ): """ @@ -2525,6 +2547,7 @@ async def check_vision_model( This endpoint wraps the backend is_vision_model function. """ + hf_token = _normalize_hf_token(header_hf_token) or _normalize_hf_token(hf_token) try: logger.info(f"Checking if vision model: {model_name}") # Authenticate so a gated/private VLM classifies correctly (else 404 -> non-vision). @@ -2550,6 +2573,7 @@ async def check_vision_model( async def check_embedding_model( model_name: str, hf_token: Optional[str] = Query(None), + header_hf_token: Optional[str] = Depends(get_hf_token), current_subject: str = Depends(get_current_subject), ): """ @@ -2557,6 +2581,7 @@ async def check_embedding_model( This endpoint wraps the backend is_embedding_model function. """ + hf_token = _normalize_hf_token(header_hf_token) or _normalize_hf_token(hf_token) try: logger.info(f"Checking if embedding model: {model_name}") is_embedding = is_embedding_model(model_name, hf_token = hf_token) @@ -2588,13 +2613,10 @@ def _read_native_context_length(repo_id: str, is_local: bool) -> Optional[int]: if is_local: roots = [Path(repo_id)] else: - from huggingface_hub import constants as hf_constants - + from hub.utils.hf_cache_state import iter_repo_cache_dirs if not _is_valid_repo_id(repo_id): return None - cache_dir = Path(hf_constants.HF_HUB_CACHE) - target = f"models--{repo_id.replace('/', '--')}".lower() - roots = [e for e in cache_dir.iterdir() if e.name.lower() == target] + roots = list(iter_repo_cache_dirs("model", repo_id)) for root in roots: for f in _iter_gguf_paths(root): @@ -2620,47 +2642,32 @@ def _resolve_quant_gguf(repo_id: str, quant: str, is_local: bool) -> tuple[Optio Q8_0 weights). Never raises. """ try: - from utils.models.model_config import ( - _extract_quant_label, - _is_big_endian_gguf_path, - _is_mtp_drafter, - ) - if is_local: roots = [Path(repo_id)] else: - from huggingface_hub import constants as hf_constants + from hub.utils.hf_cache_state import iter_repo_cache_dirs if not _is_valid_repo_id(repo_id): return None, 0 - cache_dir = Path(hf_constants.HF_HUB_CACHE) - target = f"models--{repo_id.replace('/', '--')}".lower() roots = [] - for entry in cache_dir.iterdir(): - if entry.name.lower() == target: - snaps = entry / "snapshots" - if snaps.is_dir(): - roots.extend(s for s in snaps.iterdir() if s.is_dir()) + for entry in iter_repo_cache_dirs("model", repo_id): + snaps = entry / "snapshots" + if snaps.is_dir(): + roots.extend(s for s in snaps.iterdir() if s.is_dir()) - want = quant.lower().replace("-", "").replace("_", "") + want = _normalized_quant_label(quant) best_total = 0 best_first: Optional[str] = None for root in roots: matches: list[tuple[str, Path]] = [] total = 0 for f in _iter_gguf_paths(root): - if _is_mmproj_filename(f.name): - continue try: rel = f.relative_to(root).as_posix() except ValueError: rel = f.name - if _is_mtp_drafter(rel): - continue - q = _extract_quant_label(rel) - if _is_big_endian_gguf_path(rel, q): - continue - if q.lower().replace("-", "").replace("_", "") != want: + q = _main_variant_gguf_label(rel) + if q is None or _normalized_quant_label(q) != want: continue try: total += f.stat().st_size @@ -2685,7 +2692,10 @@ async def get_kv_cache_estimate( repo_id: str = Query(..., description = "HF repo ID or local path"), quant: str = Query(..., description = "Quantization label (e.g. Q4_K_M)"), n_ctx: int = Query(..., ge = 1, description = "Context length to size the KV cache for"), - cache_type_kv: Optional[str] = Query(None, description = "KV cache dtype (e.g. q8_0)"), + cache_type_kv: Optional[str] = Query( + None, + description = "KV cache dtype (e.g. q8_0, q4_0, q5_0, iq4_nl, f32)", + ), current_subject: str = Depends(get_current_subject), ): """Estimate KV cache + weight bytes for a downloaded GGUF at n_ctx. @@ -2746,6 +2756,8 @@ async def get_gguf_variants( repo_id: str = Query( ..., description = "HuggingFace repo ID (e.g. 'unsloth/gemma-3-4b-it-GGUF')" ), + prefer_local_cache: bool = False, + local_path: Optional[str] = None, hf_token: Optional[str] = Query(None, description = "HuggingFace token for private repos"), hf_token_header: Optional[str] = Depends(get_hf_token), current_subject: str = Depends(get_current_subject), @@ -2757,9 +2769,16 @@ async def get_gguf_variants( response = await hub_gguf_variants.get_gguf_variants_response( repo_id, + prefer_local_cache = prefer_local_cache, + local_path = local_path, hf_token = hf_token, ) - local = is_local_path(repo_id) + context_model = ( + local_path + if prefer_local_cache and local_path and is_local_path(local_path) + else repo_id + ) + local = is_local_path(context_model) return GgufVariantsResponse( repo_id = response.repo_id, @@ -2778,7 +2797,11 @@ async def get_gguf_variants( ], has_vision = response.has_vision, default_variant = response.default_variant, - context_length = _read_native_context_length(repo_id, is_local = local), + # The header walk reads tokenizer arrays on dense models (tens of + # ms per uncached file); keep it off the event loop. + context_length = await asyncio.to_thread( + _read_native_context_length, context_model, is_local = local + ), ) except HTTPException: raise @@ -2795,69 +2818,17 @@ async def get_gguf_download_progress( repo_id: str = Query(..., description = "HuggingFace repo ID"), variant: str = Query("", description = "Quantization variant (e.g. UD-TQ1_0)"), expected_bytes: int = Query(0, description = "Expected total download size in bytes"), + hf_token: Optional[str] = Depends(get_hf_token), current_subject: str = Depends(get_current_subject), ): - """Download progress from cached GGUF files for a specific variant. - - Tracks completed shards in snapshots and in-progress (.incomplete) - downloads in the blobs directory. - """ - try: - if not _is_valid_repo_id(repo_id): - return { - "downloaded_bytes": 0, - "expected_bytes": expected_bytes, - "progress": 0, - } - - from huggingface_hub import constants as hf_constants - - cache_dir = Path(hf_constants.HF_HUB_CACHE) - target = f"models--{repo_id.replace('/', '--')}".lower() - variant_lower = variant.lower().replace("-", "").replace("_", "") - downloaded_bytes = 0 - in_progress_bytes = 0 - for entry in cache_dir.iterdir(): - if entry.name.lower() == target: - # Completed .gguf files for this variant in snapshots. - # Exclude mmproj so a vision adapter can't satisfy a same-label - # main variant (e.g. mmproj-F16 vs an F16 weight). - for f in _iter_gguf_paths(entry): - if _is_mmproj_filename(f.name): - continue - rel = f.relative_to(entry).as_posix() - quant = _extract_quant_label(rel) - if _is_big_endian_gguf_path(rel, quant): - continue - rel_key = rel.lower().replace("-", "").replace("_", "") - if not variant_lower or variant_lower in rel_key: - try: - downloaded_bytes += f.stat().st_size - except OSError: - continue # broken symlink / unreadable: skip - # In-progress (.incomplete) downloads in blobs. - blobs_dir = entry / "blobs" - if blobs_dir.is_dir(): - for f in blobs_dir.iterdir(): - if f.is_file() and f.name.endswith(".incomplete"): - try: - in_progress_bytes += f.stat().st_size - except OSError: - continue - break - - total_progress_bytes = downloaded_bytes + in_progress_bytes - progress = min(total_progress_bytes / expected_bytes, 0.99) if expected_bytes > 0 else 0 - # Report 1.0 only when all bytes are in completed files. - if expected_bytes > 0 and downloaded_bytes >= expected_bytes: - progress = 1.0 - return { - "downloaded_bytes": total_progress_bytes, - "expected_bytes": expected_bytes, - "progress": round(progress, 3), - } - except Exception: - return {"downloaded_bytes": 0, "expected_bytes": expected_bytes, "progress": 0} + """Compatibility route backed by the shared multi-cache progress service.""" + from hub.services.models import downloads + return await downloads.get_gguf_download_progress_response( + repo_id, + variant = variant, + expected_bytes = expected_bytes, + hf_token = hf_token, + ) def _resolve_hf_cache_realpath(repo_dir: Path) -> Optional[str]: @@ -2882,98 +2853,12 @@ def _resolve_hf_cache_realpath(repo_dir: Path) -> Optional[str]: @router.get("/download-progress") async def get_download_progress( repo_id: str = Query(..., description = "HuggingFace repo ID"), + hf_token: Optional[str] = Depends(get_hf_token), current_subject: str = Depends(get_current_subject), ): - """Return download progress for any HuggingFace model repo. - - Checks the local HF cache for completed blobs and in-progress - (.incomplete) downloads. Gets the expected total size from the HF API - on the first call, then caches it for later polls. Also returns - ``cache_path``: the realpath of the snapshot dir (or cache repo root - if no snapshot yet) so the UI can show where weights live on disk. - """ - _empty = { - "downloaded_bytes": 0, - "expected_bytes": 0, - "progress": 0, - "cache_path": None, - } - try: - if not _is_valid_repo_id(repo_id): - return _empty - - from huggingface_hub import constants as hf_constants - - cache_dir = Path(hf_constants.HF_HUB_CACHE) - target = f"models--{repo_id.replace('/', '--')}".lower() - completed_bytes = 0 - in_progress_bytes = 0 - cache_path: Optional[str] = None - - for entry in cache_dir.iterdir(): - if entry.name.lower() != target: - continue - cache_path = _resolve_hf_cache_realpath(entry) - blobs_dir = entry / "blobs" - if not blobs_dir.is_dir(): - break - for f in blobs_dir.iterdir(): - if not f.is_file(): - continue - if f.name.endswith(".incomplete"): - in_progress_bytes += f.stat().st_size - else: - completed_bytes += f.stat().st_size - break - - downloaded_bytes = completed_bytes + in_progress_bytes - if downloaded_bytes == 0: - return {**_empty, "cache_path": cache_path} - - expected_bytes = _get_repo_size_cached(repo_id) - if expected_bytes <= 0: - # Total unknown; report bytes only, no percentage. - return { - "downloaded_bytes": downloaded_bytes, - "expected_bytes": 0, - "progress": 0, - "cache_path": cache_path, - } - - # 95% threshold (blob dedup can skew completed_bytes). Do NOT - # treat "no .incomplete files" as done: HF downloads sequentially, - # so none exist between files even when far from finished. - if completed_bytes >= expected_bytes * 0.95: - progress = 1.0 - else: - progress = min(downloaded_bytes / expected_bytes, 0.99) - return { - "downloaded_bytes": downloaded_bytes, - "expected_bytes": expected_bytes, - "progress": round(progress, 3), - "cache_path": cache_path, - } - except Exception as e: - logger.warning(f"Error checking download progress for {repo_id}: {e}") - return _empty - - -_repo_size_cache: dict[str, int] = {} - - -def _get_repo_size_cached(repo_id: str) -> int: - if repo_id in _repo_size_cache: - return _repo_size_cache[repo_id] - try: - from huggingface_hub import model_info as hf_model_info - - info = hf_model_info(repo_id, token = None, files_metadata = True) - total = sum(s.size for s in info.siblings if s.size) - _repo_size_cache[repo_id] = total - return total - except Exception as e: - logger.warning(f"Failed to get repo size for {repo_id}: {e}") - return 0 + """Compatibility route backed by the shared multi-cache progress service.""" + from hub.services.models import downloads + return await downloads.get_download_progress_response(repo_id, hf_token = hf_token) def _repo_in_any_hf_cache(model_name: str) -> bool: @@ -2986,25 +2871,13 @@ def _repo_in_any_hf_cache(model_name: str) -> bool: would delete a model they did not download via the scan. Mirrors the cache set in ``_all_hf_cache_scans`` but only probes for the one repo dir (cheap, no full scan). """ - from utils.paths import ( - hf_default_cache_dir, - legacy_hf_cache_dir, - resolve_cached_repo_id_case, - ) + from utils.paths import resolve_cached_repo_id_case dirname = f"models--{resolve_cached_repo_id_case(model_name).replace('/', '--')}" dirname_lower = dirname.lower() - candidates = [] - try: - from huggingface_hub.constants import HF_HUB_CACHE - candidates.append(Path(HF_HUB_CACHE)) - except Exception: - pass - for fn in (legacy_hf_cache_dir, hf_default_cache_dir): - try: - candidates.append(fn()) - except Exception: - continue + from hub.utils.hf_cache_state import hf_cache_roots + + candidates = hf_cache_roots() # resolve_cached_repo_id_case only normalizes the ACTIVE cache, but discard deletes # case-insensitively across all caches, so detect case-insensitively too -- else a # pre-existing case-variant repo is misreported as scan-created and deleted on decline. @@ -3028,38 +2901,8 @@ def _all_hf_cache_scans(): broken symlink, OS-redirected ~/.cache) is skipped, not fatal, so the Downloaded list never blanks out and downloads never leak into Recommended. """ - from huggingface_hub import scan_cache_dir - from utils.paths import legacy_hf_cache_dir, hf_default_cache_dir - - scans = [] - # Guard the active cache too: degrade to "no downloads" instead of raising. - try: - scans.append(scan_cache_dir()) - except Exception as exc: - logger.warning("Could not scan active HF cache: %s", exc) - - seen: set[str] = set() - try: - # Resolve the active cache dir for dedup. - from huggingface_hub.constants import HF_HUB_CACHE - seen.add(str(Path(HF_HUB_CACHE).resolve())) - except Exception: - pass - - for extra_fn in (legacy_hf_cache_dir, hf_default_cache_dir): - try: - extra = extra_fn() - # is_dir()/resolve() can raise on an inaccessible path; skip it. - if not extra.is_dir(): - continue - resolved = str(extra.resolve()) - if resolved in seen: - continue - seen.add(resolved) - scans.append(scan_cache_dir(cache_dir = str(extra))) - except Exception as exc: - logger.warning("Could not scan HF cache %s: %s", extra_fn.__name__, exc) - return scans + from hub.utils.inventory_scan import all_hf_cache_scans + return all_hf_cache_scans() def _is_gguf_filename(name: str) -> bool: @@ -3078,6 +2921,22 @@ def _is_main_gguf_filename(name: str) -> bool: return _is_gguf_filename(name) and not _is_mmproj_filename(name) +def _main_variant_gguf_label(rel_path: str) -> Optional[str]: + name = rel_path.rsplit("/", 1)[-1] + if not _is_main_gguf_filename(name): + return None + if _is_mtp_drafter(rel_path): + return None + label = _extract_quant_label(rel_path) + if _is_big_endian_gguf_path(rel_path, label): + return None + return label + + +def _normalized_quant_label(label: str) -> str: + return label.lower().replace("-", "").replace("_", "") + + def _repo_has_mmproj(repo_info) -> bool: """True if the repo ships a GGUF vision adapter (mmproj), so it can take image inputs. Cheap: scans already-listed file names only.""" @@ -3170,7 +3029,9 @@ async def list_cached_gguf(current_subject: str = Depends(get_current_subject)): if repo_info.repo_type != "model": continue repo_id = repo_info.repo_id - if _is_hidden_model(repo_id): + # Pass the snapshot path too so the config check also hides + # custom Whisper checkpoints, not just curated repo ids. + if _is_hidden_model(repo_id, str(repo_info.repo_path)): continue total_size = _repo_gguf_size_bytes(repo_info) if total_size == 0: @@ -3227,7 +3088,9 @@ async def list_cached_models( if repo_info.repo_type != "model": continue repo_id = repo_info.repo_id - if _is_hidden_model(repo_id): + # Pass the snapshot path too so the config check also hides + # custom Whisper checkpoints, not just curated repo ids. + if _is_hidden_model(repo_id, str(repo_info.repo_path)): continue if _repo_has_gguf_files(repo_info): continue @@ -3285,124 +3148,177 @@ async def list_cached_models( async def delete_cached_model( repo_id: str = Body(...), variant: Optional[str] = Body(None), + cache_path: Optional[str] = Body(None), + hf_token: Optional[str] = Depends(get_hf_token), current_subject: str = Depends(get_current_subject), ): - """Delete a cached model repo (or a specific GGUF variant) from the HF cache. + """Compatibility route backed by the shared multi-cache deletion service.""" + from hub.services.models import deletion + return await deletion.delete_cached_model_response(repo_id, variant, hf_token, cache_path) - With *variant*, only GGUF files matching that quant label are removed - (e.g. ``UD-Q4_K_XL``); otherwise the whole repo is deleted. Refuses - if the model is currently loaded for inference. - """ + +def _resolve_cached_model_path(repo_id: str, variant: Optional[str]) -> Path: + """Absolute path of a cached repo (newest snapshot dir) or, with *variant*, + that quant's main GGUF file (first split of a sharded quant). Paths come + from the HF cache scan only, so callers can't probe arbitrary paths.""" + cache_scans = _all_hf_cache_scans() + + matching_repos = [] + for hf_cache in cache_scans: + for repo_info in hf_cache.repos: + if repo_info.repo_type != "model": + continue + if repo_info.repo_id.lower() == repo_id.lower(): + matching_repos.append(repo_info) + if not matching_repos: + raise HTTPException(status_code = 404, detail = "Model not found in cache") + + if variant: + want = _normalized_quant_label(variant) + candidate_revisions = sorted( + (rev for repo_info in matching_repos for rev in repo_info.revisions), + key = lambda rev: getattr(rev, "last_modified", 0) or 0, + reverse = True, + ) + for rev in candidate_revisions: + snapshot = getattr(rev, "snapshot_path", None) + matches = [] + for f in rev.files: + p = Path(f.file_path) + rel = f.file_name + if snapshot: + try: + rel = p.relative_to(snapshot).as_posix() + except ValueError: + pass + label = _main_variant_gguf_label(rel) + if label is None or _normalized_quant_label(label) != want: + continue + if p.exists() or p.is_symlink(): + matches.append((rel, p)) + if matches: + # Path-sorted so a sharded quant deterministically yields its first split. + return sorted(matches, key = lambda m: m[0].lower())[0][1] + raise HTTPException( + status_code = 404, + detail = f"Variant {variant} not found in cache for {repo_id}", + ) + + def repo_size(repo_info) -> int: + gguf_size = _repo_gguf_size_bytes(repo_info) + if gguf_size > 0: + return gguf_size + return sum( + (getattr(f, "size_on_disk", None) or 0) + for rev in repo_info.revisions + for f in rev.files + ) + + def repo_last_modified(repo_info) -> float: + return max( + (getattr(rev, "last_modified", 0) or 0 for rev in repo_info.revisions), + default = 0, + ) + + target_repo = max( + matching_repos, + key = lambda repo_info: (repo_size(repo_info), repo_last_modified(repo_info)), + ) + + # Whole repo: the newest revision's snapshot dir holds the visible files. + revisions = sorted( + (rev for rev in target_repo.revisions if getattr(rev, "snapshot_path", None)), + key = lambda rev: getattr(rev, "last_modified", 0) or 0, + reverse = True, + ) + for rev in revisions: + p = Path(rev.snapshot_path) + if p.exists(): + return p + p = Path(target_repo.repo_path) + if p.exists(): + return p + raise HTTPException(status_code = 404, detail = "Cached model path not found") + + +def _wsl_reveal_in_explorer(path: Path) -> bool: + import subprocess + + from utils.paths.path_utils import _IS_WSL + + if not _IS_WSL: + return False + try: + windows_path = subprocess.run( + ["wslpath", "-w", str(path)], + capture_output = True, + text = True, + check = True, + timeout = 10, + ).stdout.strip() + if not windows_path: + return False + argument = f"/select,{windows_path}" if path.is_file() else windows_path + subprocess.Popen(["explorer.exe", argument]) + return True + except (OSError, subprocess.SubprocessError): + return False + + +def _reveal_in_file_manager(path: Path) -> None: + """Open the OS file manager with *path* selected (best effort per platform).""" + import subprocess + + target = str(path) + if sys.platform == "darwin": + cmd = ["open", "-R", target] if path.is_file() else ["open", target] + subprocess.Popen(cmd) + elif os.name == "nt": + if path.is_file(): + subprocess.Popen(["explorer", f"/select,{target}"]) + else: + os.startfile(target) # noqa: S606 - local user's own file manager + elif not _wsl_reveal_in_explorer(path): + # No cross-desktop "select file" standard on Linux; open the directory. + directory = target if path.is_dir() else str(path.parent) + subprocess.Popen(["xdg-open", directory]) + + +class CachedModelPathResponse(BaseModel): + path: str + is_dir: bool + + +@router.get("/cached-model-path", response_model = CachedModelPathResponse) +async def get_cached_model_path( + repo_id: str = Query(..., description = "HuggingFace repo ID"), + variant: str = Query("", description = "Quantization variant (empty for whole repo)"), + current_subject: str = Depends(get_current_subject), +): + """Absolute on-disk path of a cached repo or one of its GGUF variants.""" if not _is_valid_repo_id(repo_id): raise HTTPException(status_code = 400, detail = "Invalid repo_id format") + path = await asyncio.to_thread(_resolve_cached_model_path, repo_id, variant.strip() or None) + return {"path": str(path), "is_dir": path.is_dir()} - # Refuse if the model is currently loaded. + +@router.post("/reveal-cached-model") +async def reveal_cached_model( + repo_id: str = Body(...), + variant: Optional[str] = Body(None), + current_subject: str = Depends(get_current_subject), +): + """Reveal a cached repo (or one GGUF variant's file) in the OS file manager.""" + if not _is_valid_repo_id(repo_id): + raise HTTPException(status_code = 400, detail = "Invalid repo_id format") + variant = (variant or "").strip() or None + path = await asyncio.to_thread(_resolve_cached_model_path, repo_id, variant) try: - from routes.inference import get_llama_cpp_backend - llama_backend = get_llama_cpp_backend() - if llama_backend.is_loaded and llama_backend.model_identifier: - loaded_id = llama_backend.model_identifier.lower() - if loaded_id == repo_id.lower() or loaded_id.startswith(repo_id.lower()): - raise HTTPException( - status_code = 400, - detail = "Unload the model before deleting", - ) - except HTTPException: - raise - except Exception: - pass - - try: - inference_backend = get_inference_backend() - if inference_backend.active_model_name: - active = inference_backend.active_model_name.lower() - if active == repo_id.lower() or active.startswith(repo_id.lower()): - raise HTTPException( - status_code = 400, - detail = "Unload the model before deleting", - ) - except HTTPException: - raise - except Exception: - pass - - try: - cache_scans = _all_hf_cache_scans() - - target_repo = None - for hf_cache in cache_scans: - for repo_info in hf_cache.repos: - if repo_info.repo_type != "model": - continue - if repo_info.repo_id.lower() == repo_id.lower(): - target_repo = repo_info - break - if target_repo is not None: - break - - if target_repo is None: - raise HTTPException(status_code = 404, detail = "Model not found in cache") - - # ── Per-variant GGUF deletion ──────────────────────────── - if variant: - deleted_bytes = 0 - deleted_count = 0 - for rev in target_repo.revisions: - for f in rev.files: - if not _is_gguf_filename(f.file_name): - continue - quant = _extract_quant_label(f.file_name) - if quant.lower() != variant.lower(): - continue - # Delete the blob (data) and the snapshot symlink. - try: - blob = Path(f.blob_path) - snap = Path(f.file_path) - size = blob.stat().st_size if blob.exists() else 0 - if snap.exists() or snap.is_symlink(): - snap.unlink() - if blob.exists(): - blob.unlink() - deleted_bytes += size - deleted_count += 1 - except Exception as e: - logger.warning(f"Failed to delete {f.file_name}: {e}") - - if deleted_count == 0: - raise HTTPException( - status_code = 404, - detail = f"Variant {variant} not found in cache for {repo_id}", - ) - - freed_mb = deleted_bytes / (1024 * 1024) - logger.info( - f"Deleted {deleted_count} file(s) for {repo_id} variant {variant}: " - f"{freed_mb:.1f} MB freed" - ) - return {"status": "deleted", "repo_id": repo_id, "variant": variant} - - # ── Full repo deletion ─────────────────────────────────── - revision_hashes = [rev.commit_hash for rev in target_repo.revisions] - if not revision_hashes: - raise HTTPException(status_code = 404, detail = "No revisions found for model") - - delete_strategy = hf_cache.delete_revisions(*revision_hashes) - logger.info( - f"Deleting cached model {repo_id}: " - f"{delete_strategy.expected_freed_size_str} will be freed" - ) - delete_strategy.execute() - - return {"status": "deleted", "repo_id": repo_id} - - except HTTPException: - raise + await asyncio.to_thread(_reveal_in_file_manager, path) except Exception as e: - logger.error(f"Error deleting cached model {repo_id}: {e}", exc_info = True) - raise HTTPException( - status_code = 500, - detail = "Failed to delete cached model", - ) + logger.error(f"Failed to reveal {path}: {e}") + raise HTTPException(status_code = 500, detail = "Failed to open file manager") + return {"status": "ok", "path": str(path)} @router.get("/checkpoints", response_model = CheckpointListResponse) @@ -3456,7 +3372,7 @@ _EXPORT_SIZE_CACHE: dict[str, tuple[int, int, str]] = {} def _is_sizable_local_path(model: str) -> bool: - """True only for local paths under a Studio data root. + """True only for local paths under an Unsloth data root. Containment is decided lexically (no filesystem access) before the path is touched, then the path is symlink-resolved and re-checked so a symlink diff --git a/studio/backend/routes/providers.py b/studio/backend/routes/providers.py index 5a55c9b0bb..4e7e53f2f0 100644 --- a/studio/backend/routes/providers.py +++ b/studio/backend/routes/providers.py @@ -47,6 +47,20 @@ logger = structlog.get_logger(__name__) router = APIRouter() +def _provider_response(row: dict) -> ProviderResponse: + return ProviderResponse( + id = row["id"], + provider_type = row["provider_type"], + display_name = row["display_name"], + base_url = row["base_url"], + is_enabled = bool(row["is_enabled"]), + models = row.get("models") or [], + available_models = row.get("available_models") or [], + created_at = row["created_at"], + updated_at = row["updated_at"], + ) + + # ── Public key for API key encryption ───────────────────────────── @@ -89,18 +103,7 @@ async def get_pricing_snapshot(current_subject: str = Depends(get_current_subjec async def list_provider_configs(current_subject: str = Depends(get_current_subject)): """List all saved provider configurations.""" rows = providers_db.list_providers() - return [ - ProviderResponse( - id = row["id"], - provider_type = row["provider_type"], - display_name = row["display_name"], - base_url = row["base_url"], - is_enabled = bool(row["is_enabled"]), - created_at = row["created_at"], - updated_at = row["updated_at"], - ) - for row in rows - ] + return [_provider_response(row) for row in rows] @router.post("/", response_model = ProviderResponse, status_code = 201) @@ -124,18 +127,12 @@ async def create_provider_config( provider_type = payload.provider_type, display_name = payload.display_name, base_url = base_url, + models = payload.models, + available_models = payload.available_models, ) row = providers_db.get_provider(provider_id) - return ProviderResponse( - id = row["id"], - provider_type = row["provider_type"], - display_name = row["display_name"], - base_url = row["base_url"], - is_enabled = bool(row["is_enabled"]), - created_at = row["created_at"], - updated_at = row["updated_at"], - ) + return _provider_response(row) @router.put("/{provider_id}", response_model = ProviderResponse) @@ -154,20 +151,14 @@ async def update_provider_config( display_name = payload.display_name, base_url = payload.base_url, is_enabled = payload.is_enabled, + models = payload.models, + available_models = payload.available_models, ) if not updated: raise HTTPException(status_code = 400, detail = "No fields to update") row = providers_db.get_provider(provider_id) - return ProviderResponse( - id = row["id"], - provider_type = row["provider_type"], - display_name = row["display_name"], - base_url = row["base_url"], - is_enabled = bool(row["is_enabled"]), - created_at = row["created_at"], - updated_at = row["updated_at"], - ) + return _provider_response(row) @router.delete("/{provider_id}", status_code = 204) diff --git a/studio/backend/routes/rag.py b/studio/backend/routes/rag.py index e20fea74a3..392a4e0d02 100644 --- a/studio/backend/routes/rag.py +++ b/studio/backend/routes/rag.py @@ -318,6 +318,39 @@ def list_project_documents(project_id: str, subject: str = Depends(get_current_s conn.close() +@router.get("/documents") +def list_all_uploaded_documents(subject: str = Depends(get_current_subject)) -> dict: + """Every uploaded file across chats, projects, and knowledge bases (settings + Data tab).""" + _require_rag() + conn = rag_db.get_connection() + try: + docs = store.list_all_documents(conn) + kb_names = {kb["id"]: kb["name"] for kb in store.list_kbs(conn)} + finally: + conn.close() + + from storage.studio_db import list_chat_projects + + project_names = {p["id"]: p["name"] for p in list_chat_projects(include_archived = True)} + + out = [] + for doc in docs: + view = _doc_view(doc) + stored_path = doc.get("stored_path") + size = None + if stored_path: + try: + size = os.path.getsize(stored_path) + except OSError: + size = None + view["sizeBytes"] = size + view["kbName"] = kb_names.get(doc.get("kb_id")) + view["projectName"] = project_names.get(doc.get("project_id")) + out.append(view) + return {"documents": out} + + @router.delete("/documents/{document_id}") def delete_document(document_id: str, subject: str = Depends(get_current_subject)) -> dict: _require_rag() @@ -424,8 +457,10 @@ _CONTENT_TYPES = { ".txt": "text/plain; charset=utf-8", ".md": "text/markdown; charset=utf-8", ".markdown": "text/markdown; charset=utf-8", - ".html": "text/html; charset=utf-8", - ".htm": "text/html; charset=utf-8", + # Served as plain text, never text/html: an uploaded HTML document rendered + # same-origin would execute its scripts with access to the app's storage. + ".html": "text/plain; charset=utf-8", + ".htm": "text/plain; charset=utf-8", ".docx": "application/vnd.openxmlformats-officedocument.wordprocessingml.document", } diff --git a/studio/backend/routes/settings.py b/studio/backend/routes/settings.py index 1ddfc0eacb..fef18a9145 100644 --- a/studio/backend/routes/settings.py +++ b/studio/backend/routes/settings.py @@ -10,6 +10,7 @@ from pydantic import BaseModel, ConfigDict, Field, field_validator from auth.authentication import get_current_subject from auth.storage import rotate_preview_link_secret +from core.rag.config import default_gguf_repo, effective_gguf_repo from loggers import get_logger from utils.utils import safe_error_detail, log_and_http_error from utils.personalization_settings import ( @@ -35,9 +36,10 @@ from utils.helper_precache_settings import ( ) from utils.coding_agents import CODING_AGENTS, detect_installed_coding_agents from utils.openai_auto_switch_settings import ( - DEFAULT_AUTO_UNLOAD_IDLE_SECONDS, + DEFAULT_AUTO_UNLOAD_KEEP_KV, DEFAULT_OPENAI_AUTO_SWITCH_ENABLED, get_auto_unload_idle_seconds, + get_auto_unload_keep_kv, get_model_overrides, get_openai_auto_switch_enabled, get_stored_auto_unload_idle_seconds, @@ -58,6 +60,7 @@ from utils.embedding_model_settings import ( set_rag_embedding_model, validate_embedding_model, ) +from utils.hf_cache_settings import cache_status, get_hf_cache_paths, set_hf_cache_home router = APIRouter() @@ -87,9 +90,28 @@ class HelperPrecacheResponse(BaseModel): disabled_by_env: bool +class HuggingFaceCachePayload(BaseModel): + cache_home: Optional[str] = Field(default = None, max_length = 4096) + + +class HuggingFaceCacheResponse(BaseModel): + cache_home: str + hub_cache: str + xet_cache: str + source: Literal["default", "studio", "environment"] + editable: bool + is_custom: bool + available: bool + writable: bool + free_bytes: Optional[int] = None + environment_variable: Optional[str] = None + + class OpenAIAutoSwitchPayload(BaseModel): enabled: bool - auto_unload_idle_seconds: int = Field(default = DEFAULT_AUTO_UNLOAD_IDLE_SECONDS, ge = 0) + # None leaves the stored value untouched (partial updates can't clobber it). + auto_unload_idle_seconds: Optional[int] = Field(default = None, ge = 0) + auto_unload_keep_kv: Optional[bool] = None class OpenAIAutoSwitchResponse(BaseModel): @@ -100,6 +122,7 @@ class OpenAIAutoSwitchResponse(BaseModel): # UNSLOTH_MODEL_IDLE_TTL set and nothing stored, this is true even while enabled # is false, so the UI can show idle-unload as active instead of "needs enable". idle_unload_active: bool = False + auto_unload_keep_kv: bool = DEFAULT_AUTO_UNLOAD_KEEP_KV class ModelOverridePayload(BaseModel): @@ -130,6 +153,30 @@ def _helper_precache_response(enabled: bool | None = None) -> HelperPrecacheResp ) +def _hugging_face_cache_response() -> HuggingFaceCacheResponse: + return HuggingFaceCacheResponse(**cache_status(get_hf_cache_paths())) + + +@router.get("/hugging-face-cache", response_model = HuggingFaceCacheResponse) +def get_hugging_face_cache( + current_subject: str = Depends(get_current_subject), +) -> HuggingFaceCacheResponse: + return _hugging_face_cache_response() + + +@router.put("/hugging-face-cache", response_model = HuggingFaceCacheResponse) +def update_hugging_face_cache( + payload: HuggingFaceCachePayload, current_subject: str = Depends(get_current_subject) +) -> HuggingFaceCacheResponse: + try: + set_hf_cache_home(payload.cache_home) + except RuntimeError as exc: + raise HTTPException(status_code = 409, detail = str(exc)) from exc + except ValueError as exc: + raise HTTPException(status_code = 400, detail = str(exc)) from exc + return _hugging_face_cache_response() + + @router.get("/upload-limit", response_model = UploadLimitResponse) def get_upload_limit(current_subject: str = Depends(get_current_subject)) -> UploadLimitResponse: return _upload_limit_response(get_upload_limit_mb()) @@ -197,6 +244,7 @@ def get_openai_auto_switch( enabled = get_openai_auto_switch_enabled(), auto_unload_idle_seconds = get_stored_auto_unload_idle_seconds(), idle_unload_active = get_auto_unload_idle_seconds() > 0, + auto_unload_keep_kv = get_auto_unload_keep_kv(), ) @@ -205,8 +253,8 @@ def update_openai_auto_switch( payload: OpenAIAutoSwitchPayload, current_subject: str = Depends(get_current_subject) ) -> OpenAIAutoSwitchResponse: try: - enabled, idle_seconds = set_openai_auto_switch( - payload.enabled, payload.auto_unload_idle_seconds + enabled, idle_seconds, keep_kv = set_openai_auto_switch( + payload.enabled, payload.auto_unload_idle_seconds, payload.auto_unload_keep_kv ) except ValueError as exc: raise log_and_http_error( @@ -216,10 +264,16 @@ def update_openai_auto_switch( event = "settings.update_openai_auto_switch_failed", log = logger, ) from exc + idle_unload_active = get_auto_unload_idle_seconds() > 0 + if not keep_kv or not idle_unload_active: + # Keep-KV off or idle unload disabled: drop already-saved chat context too. + from core.inference.llama_keepwarm import purge_kv_resume + purge_kv_resume() return OpenAIAutoSwitchResponse( enabled = enabled, auto_unload_idle_seconds = idle_seconds, - idle_unload_active = get_auto_unload_idle_seconds() > 0, + idle_unload_active = idle_unload_active, + auto_unload_keep_kv = keep_kv, ) @@ -263,14 +317,18 @@ class EmbeddingModelPayload(BaseModel): class EmbeddingModelResponse(BaseModel): embedding_model: str + embedding_gguf_repo: str default_embedding_model: str + default_embedding_gguf_repo: str is_custom: bool def _embedding_model_response() -> EmbeddingModelResponse: return EmbeddingModelResponse( embedding_model = get_rag_embedding_model(), + embedding_gguf_repo = effective_gguf_repo(), default_embedding_model = default_embedding_model(), + default_embedding_gguf_repo = default_gguf_repo(), is_custom = get_stored_embedding_model() is not None, ) @@ -400,6 +458,11 @@ def update_embedding_model( log = logger, ) from exc hf_token = (payload.hf_token or "").strip() or None + from utils.utils import hf_env_offline + + # Offline, both the Hub malware scan and the is-embedding check are unreachable and degrade + # to the local cache below; capture the state once. + local_only_load = hf_env_offline() # The env/default model needs no verification; saving it is a no-op override. # A local GGUF on the llama-server backend is accepted as-is: it is exactly # what the backend loads, and HF metadata cannot verify a local path. @@ -423,26 +486,41 @@ def update_embedding_model( # Fall back to the loader's own token so a gated/private repo is actually scanned # (a token-less scan fails open for exactly the repo that would still load). scan_token = hf_token or _ambient_hf_token() - # Include the ST module dirs (0_Transformer/) so a flagged pickle directly under - # one blocks instead of passing as an unreferenced nested shard. - load_subdirs = tuple( - dict.fromkeys( - ( - *security_load_subdirs(model, scan_token), - *_st_module_subdirs(model, scan_token), + # Offline: subdir probes would hit the network and hang; the offline gate walks the + # whole cached snapshot, so no load-subdir hints are needed. + if local_only_load: + load_subdirs = () + else: + # Include ST module dirs (0_Transformer/) so a flagged pickle directly under one + # blocks instead of passing as an unreferenced nested shard. + load_subdirs = tuple( + dict.fromkeys( + ( + *security_load_subdirs(model, scan_token), + *_st_module_subdirs(model, scan_token), + ) ) ) - ) - if evaluate_file_security(model, hf_token = scan_token, load_subdirs = load_subdirs).blocked: + if evaluate_file_security( + model, + hf_token = scan_token, + load_subdirs = load_subdirs, + local_only_load = local_only_load, + ).blocked: # 403, not 409: the client routes every 409 into the forceable "save anyway" # flow, but this block is a hard, non-forceable security refusal. - raise HTTPException( - status_code = 403, + if local_only_load: + detail = ( + f"{model!r} has cached pickle weights that cannot be security-scanned " + "offline and no safetensors alternative, so it cannot be used as the " + "embedding model. Re-download it with safetensors weights while online." + ) + else: detail = ( f"{model!r} is flagged as unsafe by Hugging Face's security scan and " "cannot be used as the embedding model." - ), - ) + ) + raise HTTPException(status_code = 403, detail = detail) if model != default_embedding_model() and not payload.force and not is_local_gguf: from core.rag import config as rag_config @@ -452,15 +530,28 @@ def update_embedding_model( # which would wrongly 409 a valid online GGUF embedder. gguf_named = _llama_backend_active() and rag_config._names_gguf(model) if not gguf_named and not is_embedding_model(model, hf_token = hf_token): - raise HTTPException( - status_code = 409, - detail = ( - f"Could not verify {model!r} as an embedding model on " - "Hugging Face (it may be the wrong model type, gated, or " - "you may be offline)." - ), - ) - gguf_error = _local_gguf_backend_error(model) or _hf_gguf_backend_error(model, hf_token) + # Offline, is_embedding_model can only confirm the ST layout (modules.json); a + # transformers-native embedder (e.g. gte-modernbert) is unverifiable without Hub + # metadata. If already cached and loadable, accept it rather than raising a 409 that + # online would not (ST can load any cached encoder). Uncached -> 409. + from utils.utils import hf_cache_snapshot_is_loadable + + # Require a genuinely loadable cache (config + weights), not just a resolved refs/main, + # so a metadata-only partial cache still gets the forceable 409. + offline_cached = local_only_load and hf_cache_snapshot_is_loadable(model) + if not offline_cached: + raise HTTPException( + status_code = 409, + detail = ( + f"Could not verify {model!r} as an embedding model on " + "Hugging Face (it may be the wrong model type, gated, or " + "you may be offline)." + ), + ) + # The Hub GGUF probe (list_repo_files) can hang offline; skip it. Local check stays. + gguf_error = _local_gguf_backend_error(model) + if gguf_error is None and not local_only_load: + gguf_error = _hf_gguf_backend_error(model, hf_token) if gguf_error: raise HTTPException(status_code = 409, detail = gguf_error) set_rag_embedding_model(model) diff --git a/studio/backend/routes/training.py b/studio/backend/routes/training.py index d53e8f2bbc..8be4283415 100644 --- a/studio/backend/routes/training.py +++ b/studio/backend/routes/training.py @@ -109,7 +109,9 @@ async def get_hardware_utilization(current_subject: str = Depends(get_current_su @router.get("/hardware/visible") async def get_visible_hardware_utilization(current_subject: str = Depends(get_current_subject)): from utils.hardware import get_visible_gpu_utilization - return get_visible_gpu_utilization() + + # Off the event loop: the ROCm fallbacks shell out (Windows perf counters, sysfs) and the System view polls this route. + return await asyncio.to_thread(get_visible_gpu_utilization) @router.post("/start") @@ -127,9 +129,9 @@ async def start_training( try: logger.info(f"Starting training job with model: {request.model_name}") - # When Studio is driven as an inference API (API-key auth), refuse to start + # When Unsloth is driven as an inference API (API-key auth), refuse to start # training while a request is in flight: training frees VRAM by unloading - # the chat model, which would kill the stream. The Studio UI (session auth) + # the chat model, which would kill the stream. The Unsloth UI (session auth) # still starts training and coexists/frees VRAM as before. (A mixed UI+API # session is not yet special-cased.) if via_api_key is True: @@ -139,7 +141,7 @@ async def start_training( status_code = 409, detail = ( "Cannot start training over the API while an inference request is in " - "progress. Wait for it to finish, or start training from the Studio UI." + "progress. Wait for it to finish, or start training from the Unsloth UI." ), ) @@ -196,6 +198,7 @@ async def start_training( request.local_eval_datasets, "Local eval dataset" ) resume_output_dir: Optional[str] = None + resume_run: Optional[dict] = None if request.resume_from_checkpoint: try: resume_output_dir = normalize_resume_output_dir(request.resume_from_checkpoint) @@ -208,7 +211,7 @@ async def start_training( if not resume_run or not can_resume_run(resume_run): raise HTTPException( status_code = 400, - detail = "Resume checkpoint must belong to a stopped run with saved trainer state.", + detail = "Resume checkpoint must belong to a stopped or errored run with complete saved trainer state.", ) resume_checkpoint = get_resume_checkpoint_path(resume_output_dir) if not resume_checkpoint: @@ -329,6 +332,7 @@ async def start_training( else "unsloth", "use_rslora": request.use_rslora, "use_loftq": request.use_loftq, + "use_dora": request.use_dora, "train_on_completions": request.train_on_completions, "finetune_vision_layers": request.finetune_vision_layers, "finetune_language_layers": request.finetune_language_layers, @@ -412,53 +416,39 @@ async def start_training( try: from routes.training_vram import ( can_keep_chat_during_training, - free_chat_models_for_training, - summarize_resident_chat, + coordinate_models_for_training, ) - resident = summarize_resident_chat() - if not resident["any"]: - return - if resident.get("loading"): - # In-flight load can't be sized -> free rather than risk OOM. - freed = free_chat_models_for_training(reason = "chat model still loading") - logger.info("Freed in-flight chat load for training: %s", freed) - return - keep, info = can_keep_chat_during_training( - model_name = training_kwargs["model_name"], - hf_token = training_kwargs["hf_token"], - training_type = training_kwargs["training_type"], - load_in_4bit = training_kwargs["load_in_4bit"], - batch_size = training_kwargs["batch_size"], - max_seq_length = training_kwargs["max_seq_length"], - lora_rank = training_kwargs["lora_r"], - target_modules = training_kwargs["target_modules"], - gradient_checkpointing = training_kwargs["gradient_checkpointing"], - optimizer = training_kwargs["optim"], - gpu_ids = training_kwargs["gpu_ids"], - ) - if keep: - logger.info( - "Keeping chat model(s) loaded during training " - "(free ~%s GB, needs ~%s GB): %s", - info.get("usable_gb"), - info.get("required_gb"), - resident, + def _can_keep_resident_models(): + return can_keep_chat_during_training( + model_name = training_kwargs["model_name"], + hf_token = training_kwargs["hf_token"], + training_type = training_kwargs["training_type"], + load_in_4bit = training_kwargs["load_in_4bit"], + batch_size = training_kwargs["batch_size"], + max_seq_length = training_kwargs["max_seq_length"], + lora_rank = training_kwargs["lora_r"], + target_modules = training_kwargs["target_modules"], + gradient_checkpointing = training_kwargs["gradient_checkpointing"], + optimizer = training_kwargs["optim"], + gpu_ids = training_kwargs["gpu_ids"], ) - else: - freed = free_chat_models_for_training( - reason = "insufficient VRAM to run training alongside chat", - ) - logger.info("Freed chat model(s) for training: %s", freed) + + freed = coordinate_models_for_training(_can_keep_resident_models) + if freed: + logger.info("Freed models for training: %s", freed) except Exception as e: - logger.warning("Chat/training VRAM coordination failed; proceeding: %s", e) + logger.warning("Inference/training memory coordination failed; proceeding: %s", e) # The hook runs only once start guards pass -> VRAM freed iff training starts. from utils.transformers_version import SidecarSwapInProgress try: success = backend.start_training( - job_id = job_id, before_spawn = _free_vram_for_training, **training_kwargs + job_id = job_id, + before_spawn = _free_vram_for_training, + resume_source_run_id = resume_run["id"] if resume_run else None, + **training_kwargs, ) except SidecarSwapInProgress as exc: # Expected loss of the race against a sidecar install: a retryable @@ -521,7 +511,10 @@ async def stop_training( status = "idle", message = "No training job is currently running" ) - backend.stop_training(save = body.save) + if not backend.stop_training(save = body.save): + return TrainingStopResponse( + status = "idle", message = "No training job is currently running" + ) return TrainingStopResponse( status = "stopped", @@ -637,9 +630,9 @@ async def get_training_status(current_subject: str = Depends(get_current_subject "loss": getattr(progress, "loss", None), "learning_rate": getattr(progress, "learning_rate", None), } - output_dir = getattr(backend, "_output_dir", None) - if output_dir: - details["output_dir"] = output_dir + # Always present: an explicit null tells the client to drop a cached + # path (stop without save clears the run's output_dir). + details["output_dir"] = getattr(backend, "_output_dir", None) or None # Metric history for chart recovery after SSE reconnection. metric_history = None diff --git a/studio/backend/routes/training_vram.py b/studio/backend/routes/training_vram.py index fb361d3359..8ddda11b1e 100644 --- a/studio/backend/routes/training_vram.py +++ b/studio/backend/routes/training_vram.py @@ -1,15 +1,13 @@ # SPDX-License-Identifier: AGPL-3.0-only # Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 -"""VRAM coordination between chat/inference and training. +"""Memory coordination between inference and training. -Decides, from live free VRAM, whether a resident chat model can stay loaded -during training or must be unloaded, and unloads it across all backends -(HF/MLX orchestrator + llama.cpp GGUF server). In the route layer because the -GGUF accessor lives in routes/inference.py; backends are imported lazily. +Uses live free VRAM to keep resident chat and STT models when they fit. STT is +evicted before chat when training needs memory. """ -from typing import Any, Dict, List, Optional, Tuple +from typing import Any, Callable, Dict, List, Optional, Tuple from loggers import get_logger @@ -77,6 +75,37 @@ def summarize_resident_chat() -> Dict[str, Any]: } +def summarize_resident_stt() -> Dict[str, Any]: + """Report the resident dictation model (either engine). Never raises.""" + try: + from core.inference.stt_ggml_sidecar import get_ggml_stt_sidecar + from core.inference.stt_sidecar import get_stt_sidecar + + sidecar = get_stt_sidecar() + model = sidecar.loaded_model + device = sidecar.device + loading = sidecar.is_loading() + # whisper.cpp holds GPU memory via its subprocess, and both engines can be + # live at once (engine switch or direct /audio/stt/load). Always fold the + # GGUF sidecar in: a resident Transformers model must not mask a GGUF + # server still binding its backend, or admission lets training launch into + # that startup and OOM. + ggml = get_ggml_stt_sidecar() + if not model: + model = ggml.loaded_model + device = device or ggml.device + loading = loading or ggml.is_loading() + return { + "model": model, + "device": device, + "loading": loading, + "any": bool(model or loading), + } + except Exception as e: + logger.warning("Could not inspect STT sidecar: %s", e) + return {"model": None, "device": None, "loading": False, "any": False} + + def can_keep_chat_during_training( *, model_name: str, @@ -106,8 +135,8 @@ def can_keep_chat_during_training( resolve_requested_gpu_ids, ) - if get_device() != DeviceType.CUDA: - return False, {"mode": "non_cuda", "reason": "non_cuda"} + if get_device() not in (DeviceType.CUDA, DeviceType.XPU): + return False, {"mode": "non_accelerator", "reason": "non_accelerator"} # Full finetuning runs in 16-bit, so ignore the 4-bit request or we under-count. effective_4bit = False if training_type == "Full Finetuning" else load_in_4bit @@ -196,16 +225,24 @@ def can_load_chat_during_training( max_seq_length: int, requested_gpu_ids: Optional[List[int]], is_gguf: bool = False, + is_vulkan: bool = False, required_override_gb: Optional[float] = None, + single_device_gpu: Optional[str] = None, ) -> Tuple[bool, Dict[str, Any]]: """Decide if a NEW chat model can load without OOMing active training (inverse of can_keep_chat_during_training: training is already resident, so size the chat model against the free VRAM that remains). Sizes/places it the same way the loader will: HF auto reuses auto_select_gpu_ids; HF explicit requires an even-share per-GPU floor for device_map="balanced"; GGUF sizes from - required_override_gb over the visible pool. `load_in_4bit` must be effective - (LoRA can flip 4-bit -> 16-bit). Non-CUDA allows the load; default-deny on any - CUDA case it can't size, so a load never OOMs training.""" + required_override_gb over the visible pool. A Vulkan GGUF selection picks by ggml + Vulkan ordinal (separate index space from CUDA ids), so its requested_gpu_ids is + NOT resolved against the CUDA set (which would raise -> invalid_gpu_ids -> bypass + the OOM check); conservatively size an N-device request against the least-free + N visible GPUs instead. + ``single_device_gpu`` is the exact physical device token selected by a + single-device runner. `load_in_4bit` must be effective (LoRA can flip 4-bit + -> 16-bit). CPU/MLX allows the load; default-deny on any CUDA/XPU case it + can't size, so a load never OOMs training.""" try: from utils.hardware import ( DeviceType, @@ -216,8 +253,8 @@ def can_load_chat_during_training( resolve_requested_gpu_ids, ) - if get_device() != DeviceType.CUDA: - return True, {"mode": "non_cuda", "reason": "non_cuda"} + if get_device() not in (DeviceType.CUDA, DeviceType.XPU): + return True, {"mode": "non_accelerator", "reason": "non_accelerator"} est_kwargs = dict( hf_token = hf_token or None, @@ -226,6 +263,11 @@ def can_load_chat_during_training( max_seq_length = max_seq_length or 2048, ) + # A Vulkan GGUF selection uses ggml Vulkan ordinals, not CUDA physical ids; + # size it against the full visible pool (GGUF self-placement) rather than + # resolving ordinals against the CUDA parent-visible set. + vulkan_gguf = is_gguf and is_vulkan + # HF auto: reuse the loader's selector; fits iff its pick clears the margin. if not requested_gpu_ids and not is_gguf: _selected, meta = auto_select_gpu_ids(model_name, **est_kwargs) @@ -251,26 +293,62 @@ def can_load_chat_during_training( } # Explicit GPUs, or GGUF: size directly and check live free VRAM. + if requested_gpu_ids and vulkan_gguf: + mode = "gguf_vulkan" + elif single_device_gpu is not None: + mode = "single_device" + elif is_gguf: + mode = "gguf" + else: + mode = "explicit" required_gb = required_override_gb if required_gb is None: required_gb, _meta = estimate_required_model_memory_gb(model_name, **est_kwargs) if required_gb is None: - mode = "explicit" if requested_gpu_ids else "gguf" return False, {"mode": mode, "reason": "estimate_unavailable"} free_by_index = _free_vram_by_index(get_visible_gpu_utilization().get("devices", [])) - if requested_gpu_ids: + if requested_gpu_ids and vulkan_gguf: + # Vulkan ordinals cannot be mapped to CUDA physical indices. Budget + # the least-free N visible cards for an N-device request. If that + # conservative subset fits, any physical mapping of the ordinals + # fits, without collapsing a multi-GPU request to one card. + visible_free = list(free_by_index.values()) + if not visible_free: + return False, {"mode": "gguf_vulkan", "reason": "no_visible_gpus"} + n_pins = min(len(requested_gpu_ids), len(visible_free)) + free_vals = sorted(visible_free)[:n_pins] + elif single_device_gpu is not None: + token = str(single_device_gpu).strip() + if not token: + # Empty token = a CPU-only single-device runner (e.g. a CPU + # diffusion GGUF): it uses no GPU VRAM, so it never threatens + # active training and can always load. + return True, {"mode": "single_device", "reason": "cpu_only"} + try: + selected_gpu = int(token) + if selected_gpu < 0: + raise ValueError + except (TypeError, ValueError): + # A non-numeric device token (e.g. a CUDA UUID / MIG handle) + # can't be mapped to a free-VRAM index, but the runner still + # drives ONE device. Size against the worst-case visible device + # (min free), never the aggregate pool, so a single-device load + # is never OK'd on capacity it can't use and OOMs training. + free_vals = [min(free_by_index.values())] if free_by_index else [] + else: + free_vals = [free_by_index.get(selected_gpu, 0.0)] + elif requested_gpu_ids: # Invalid ids -> load_model 400s first, so don't block; missing id = 0. try: resolved = resolve_requested_gpu_ids(requested_gpu_ids) except ValueError: - return True, {"mode": "explicit", "reason": "invalid_gpu_ids"} + return True, {"mode": mode, "reason": "invalid_gpu_ids"} free_vals = [free_by_index.get(i, 0.0) for i in resolved] - mode = "explicit" else: - # GGUF: llama.cpp picks the GPU(s); any visible GPU is a candidate. + # GGUF self-placement / auto Vulkan (no requested ids): llama.cpp picks + # the GPU(s), so any visible GPU is a candidate -> size the whole pool. free_vals = list(free_by_index.values()) - mode = "gguf" if not free_vals: return False, {"mode": mode, "reason": "no_visible_gpus"} @@ -340,3 +418,110 @@ def free_chat_models_for_training(reason: str) -> List[str]: logger.warning("Could not unload GGUF chat model: %s", e) return freed + + +def free_stt_model_for_training(reason: str) -> List[str]: + """Unload the dictation model(s) before training. Never raises. + + The Transformers and GGUF sidecars are freed under independent exception + boundaries so a failure unloading one backend never skips freeing the other + (both can hold accelerator memory at once after an engine switch). + """ + freed: List[str] = [] + try: + from core.inference.stt_sidecar import get_stt_sidecar + sidecar = get_stt_sidecar() + if sidecar.is_loading() and sidecar.cancel_pending_load(): + logger.info("Cancelling STT model load for training (%s)", reason) + # The loader may still be in from_pretrained()/.to(device) holding + # VRAM; wait for it to observe the cancel and release first. + sidecar.wait_for_load_to_settle() + # A load that finished before seeing the cancel leaves a resident + # model; unload it so training gets the memory back. + if sidecar.loaded_model: + sidecar.unload() + freed.append("stt:loading") + else: + model = sidecar.loaded_model + if model: + logger.info("Unloading STT model '%s' for training (%s)", model, reason) + sidecar.unload() + freed.append(f"stt:{model}") + except Exception as e: + logger.warning("Could not unload Transformers STT model: %s", e) + + # Check the GGUF sidecar even after a cancelled/failed Transformers unload; + # both engines can hold memory at once (engine switch or direct load). + try: + from core.inference.stt_ggml_sidecar import get_ggml_stt_sidecar + ggml = get_ggml_stt_sidecar() + if ggml.is_loading() and ggml.cancel_pending_load(): + logger.info("Cancelling GGUF STT model load for training (%s)", reason) + # whisper-server may still be binding its backend; wait for the + # cancelled startup to be killed and reaped before training claims + # the memory (loaded_model stays unset until it is ready). + ggml.wait_for_load_to_settle() + if ggml.loaded_model: + ggml.unload() + freed.append("stt:gguf-loading") + else: + ggml_model = ggml.loaded_model + if ggml_model: + logger.info("Unloading GGUF STT model '%s' for training (%s)", ggml_model, reason) + ggml.unload() + freed.append(f"stt:{ggml_model}") + except Exception as e: + logger.warning("Could not unload GGUF STT model: %s", e) + + return freed + + +def coordinate_models_for_training( + can_keep: Callable[[], Tuple[bool, Dict[str, Any]]], +) -> List[str]: + """Keep resident models when they fit, evicting STT before chat.""" + resident_chat = summarize_resident_chat() + resident_stt = summarize_resident_stt() + if not resident_chat["any"] and not resident_stt["any"]: + return [] + + if resident_chat.get("loading"): + freed = free_stt_model_for_training(reason = "chat model still loading") + freed += free_chat_models_for_training(reason = "chat model still loading") + return freed + + freed: List[str] = [] + if resident_stt.get("loading"): + released_stt = free_stt_model_for_training(reason = "STT model still loading") + freed += released_stt + resident_stt = ( + {"model": None, "device": None, "loading": False, "any": False} + if released_stt + else summarize_resident_stt() + ) + if not resident_chat["any"] and not resident_stt["any"]: + return freed + + keep, info = can_keep() + if keep: + logger.info( + "Keeping resident models loaded during training (free ~%s GB, needs ~%s GB): %s", + info.get("usable_gb"), + info.get("required_gb"), + {"chat": resident_chat, "stt": resident_stt}, + ) + return freed + + if resident_stt["any"]: + freed += free_stt_model_for_training(reason = "insufficient training memory") + if not resident_chat["any"]: + return freed + keep, _info = can_keep() + if keep: + logger.info("Keeping chat model loaded after freeing STT: %s", resident_chat) + return freed + + freed += free_chat_models_for_training( + reason = "insufficient VRAM to run training alongside chat", + ) + return freed diff --git a/studio/backend/routes/whisper.py b/studio/backend/routes/whisper.py new file mode 100644 index 0000000000..08a8f269ec --- /dev/null +++ b/studio/backend/routes/whisper.py @@ -0,0 +1,74 @@ +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 + +"""whisper.cpp prebuilt status endpoint. + +GET /api/whisper/update-status -> is a newer prebuilt available + job state + +Detection reuses utils.whisper_cpp_freshness and fails open so the UI never +blocks on a missing marker / offline GitHub. There is no whisper-only update +trigger: whisper updates piggyback on the single main update item +(POST /api/llama/update chains a whisper phase when whisper is behind). +""" + +from __future__ import annotations + +import asyncio +from typing import Optional + +from fastapi import APIRouter, Depends, Query +from pydantic import BaseModel, Field + +from auth.authentication import get_current_subject +from utils.whisper_cpp_update import get_update_status + +router = APIRouter() + + +class WhisperUpdateJob(BaseModel): + state: str = Field("idle", description = "idle | running | success | error") + message: str = "" + from_tag: Optional[str] = None + to_tag: Optional[str] = None + reload_required: Optional[bool] = None + error: Optional[str] = None + progress: Optional[float] = Field(None, description = "0..1 while running, 1 on success.") + started_at: Optional[str] = None + finished_at: Optional[str] = None + + +class WhisperUpdateStatusResponse(BaseModel): + supported: bool = Field( + False, + description = "True when the install came from an Unsloth prebuilt (has a marker).", + ) + update_available: bool = Field( + False, description = "True when the latest release is genuinely newer than the install." + ) + stale: bool = Field( + False, description = "Update available AND install older than the staleness threshold." + ) + installed_tag: Optional[str] = None + latest_tag: Optional[str] = None + published_repo: Optional[str] = None + installed_at_utc: Optional[str] = None + age_days: Optional[int] = None + source_build: bool = Field( + False, description = "True when there is no marker (source build) but a prebuilt is offered." + ) + update_size_bytes: Optional[int] = Field( + None, description = "Download size of the prebuilt an update would fetch, in bytes." + ) + job: WhisperUpdateJob = Field(default_factory = WhisperUpdateJob) + + +@router.get("/update-status", response_model = WhisperUpdateStatusResponse) +async def whisper_update_status( + force_refresh: bool = Query( + False, description = "Bypass the 24h release cache for an explicit check." + ), + current_subject: str = Depends(get_current_subject), +) -> WhisperUpdateStatusResponse: + # Off the event loop: detection may probe the host and read GitHub. + status = await asyncio.to_thread(get_update_status, force_refresh = force_refresh) + return WhisperUpdateStatusResponse(**status) diff --git a/studio/backend/run.py b/studio/backend/run.py index 56b9c78343..d9569c46f6 100644 --- a/studio/backend/run.py +++ b/studio/backend/run.py @@ -232,7 +232,7 @@ def _working_local_url(port: int) -> "str | None": def _localhost_ipv6_mismatch_url(bind_host: str, port: int) -> "str | None": """Return the IPv4 loopback URL when localhost won't reach 127.0.0.1. - Local Studio binds to 127.0.0.1. Where localhost resolves to IPv6 only (::1), + Local Unsloth binds to 127.0.0.1. Where localhost resolves to IPv6 only (::1), http://localhost: fails (or hits a different process on ::1) even though http://127.0.0.1: works. Return the IPv4 URL for the caller to surface. """ @@ -243,7 +243,7 @@ def _localhost_ipv6_mismatch_url(bind_host: str, port: int) -> "str | None": ipv4_url = f"http://127.0.0.1:{port}" - # Only warn once Studio is confirmed answering on IPv4 loopback. + # Only warn once Unsloth is confirmed answering on IPv4 loopback. if _working_local_url(port) != ipv4_url: return None @@ -265,7 +265,7 @@ def _localhost_ipv6_mismatch_url(bind_host: str, port: int) -> "str | None": if host == "::1": has_ipv6_loopback = True - # A connection to ::1 is NOT evidence Studio is reachable there: Studio binds + # A connection to ::1 is NOT evidence Unsloth is reachable there: Unsloth binds # 127.0.0.1 only, so anything on ::1 is a different process. Dual-stack # localhost is fine (browsers fall back to 127.0.0.1), so only the IPv6-only # case strands the user. @@ -287,7 +287,7 @@ def _stdout_color_ok() -> bool: def _print_localhost_ipv6_mismatch_warning(local_url: str, port: int) -> None: - """Warn that localhost points at ::1 while Studio is bound to 127.0.0.1.""" + """Warn that localhost points at ::1 while Unsloth is bound to 127.0.0.1.""" use_color = _stdout_color_ok() warn_c = "\033[38;5;215;1m" if use_color else "" reset = "\033[0m" if use_color else "" @@ -303,7 +303,7 @@ def _print_localhost_ipv6_mismatch_warning(local_url: str, port: int) -> None: def _verify_global_reachability(display_host: str, port: int) -> None: """Probe check-host.net to confirm display_host:port is reachable from the public internet. Synchronous so output lands between the banner URLs and the - stop hint. Bounded at ~15s; failures swallowed (verifier failing != Studio + stop hint. Bounded at ~15s; failures swallowed (verifier failing != Unsloth failing). Only meaningful for a wildcard bind.""" global _public_reachable # Reset to "unknown" each run; set True/False only when the probe decides. @@ -563,15 +563,15 @@ def _print_cloudflare_line(secure: bool = False, loopback_host: str = "127.0.0.1 " Cloudflare tunnel: ON. This Cloudflare URL is PUBLIC, and the " "raw port is also publicly reachable. --no-cloudflare disables " f"only the Cloudflare URL; bind {loopback_host} or close firewall " - "access to keep Studio private.", + "access to keep Unsloth private.", warn, ) else: _emit( " Cloudflare tunnel: ON. This is a PUBLIC internet URL: anyone " - "who has it can reach this Studio. Relaunch with --no-cloudflare " + "who has it can reach this Unsloth. Relaunch with --no-cloudflare " f"to disable the Cloudflare URL; bind {loopback_host} or close " - "firewall access to keep Studio private.", + "firewall access to keep Unsloth private.", warn, ) return @@ -580,12 +580,12 @@ def _print_cloudflare_line(secure: bool = False, loopback_host: str = "127.0.0.1 _emit( " Cloudflare tunnel: requested but failed to start. The raw port is " "still reachable from the public internet (see the reachability check " - "above): anyone who can reach it can access this Studio.", + "above): anyone who can reach it can access this Unsloth.", warn, ) elif _public_reachable is False: _emit( - " Cloudflare tunnel: requested but failed to start. Studio is reachable " + " Cloudflare tunnel: requested but failed to start. Unsloth is reachable " "on your local network only (no public link).", warn, ) @@ -593,7 +593,7 @@ def _print_cloudflare_line(secure: bool = False, loopback_host: str = "127.0.0.1 _emit( " Cloudflare tunnel: requested but failed to start. There is no " "Cloudflare public link. Raw port reachability was not verified; " - f"bind {loopback_host} or close firewall access to keep Studio private.", + f"bind {loopback_host} or close firewall access to keep Unsloth private.", warn, ) elif _cloudflare_flag: @@ -601,19 +601,19 @@ def _print_cloudflare_line(secure: bool = False, loopback_host: str = "127.0.0.1 _emit( " Cloudflare tunnel: OFF for this mode. The raw port is still " "reachable from the public internet (see the reachability check above): " - "anyone who can reach it can access this Studio.", + "anyone who can reach it can access this Unsloth.", warn, ) elif _public_reachable is False: _emit( - " Cloudflare tunnel: OFF for this mode. Studio is reachable on your " + " Cloudflare tunnel: OFF for this mode. Unsloth is reachable on your " "local network only (no public link)." ) else: _emit( " Cloudflare tunnel: OFF for this mode. There is no Cloudflare public " "link. Raw port reachability was not verified; " - f"bind {loopback_host} or close firewall access to keep Studio private.", + f"bind {loopback_host} or close firewall access to keep Unsloth private.", warn, ) elif _cloudflare_flag is False or _cloudflare_flag is None: @@ -624,12 +624,12 @@ def _print_cloudflare_line(secure: bool = False, loopback_host: str = "127.0.0.1 f" Cloudflare tunnel: OFF ({_reason}). The raw port is still " "reachable from the public internet (see the reachability check above): " "pass --cloudflare to also expose a public Cloudflare HTTPS link, or " - f"bind {loopback_host} to keep Studio private.", + f"bind {loopback_host} to keep Unsloth private.", warn, ) elif _public_reachable is False: _emit( - f" Cloudflare tunnel: OFF ({_reason}). Studio is reachable on your " + f" Cloudflare tunnel: OFF ({_reason}). Unsloth is reachable on your " "local network only. Pass --cloudflare to expose a public " "Cloudflare HTTPS link." ) @@ -638,7 +638,7 @@ def _print_cloudflare_line(secure: bool = False, loopback_host: str = "127.0.0.1 f" Cloudflare tunnel: OFF ({_reason}). There is no Cloudflare " "public link. Raw port reachability was not verified; pass --cloudflare " "to expose a public Cloudflare HTTPS link, or " - f"bind {loopback_host} or close firewall access to keep Studio private.", + f"bind {loopback_host} or close firewall access to keep Unsloth private.", warn, ) @@ -674,7 +674,7 @@ def _is_port_free(host: str, port: int) -> bool: For a ``0.0.0.0`` wildcard host, also check whether anything is listening on ``127.0.0.1`` (and ``::1`` when IPv6 exists): an SSH tunnel may hold loopback - while the wildcard bind succeeds, making Studio unreachable via ``localhost``. + while the wildcard bind succeeds, making Unsloth unreachable via ``localhost``. """ import socket @@ -1087,7 +1087,7 @@ def _terminal_password_gate( ) -> Tuple[bool, bool]: """Force a terminal password change before the public tunnel goes up. - When the tunnel is about to publish Studio and the seeded admin password was + When the tunnel is about to publish Unsloth and the seeded admin password was never changed, ask for a new one (masked, confirmed) before any public URL exists. The CLI normally does this before re-exec'ing the backend; this is the backstop for direct `python run.py` launches and older-CLI installs. @@ -1147,7 +1147,7 @@ def _terminal_password_gate( ) if not deadline_arms: print( - "Refusing to publish Studio on a public Cloudflare URL: the " + "Refusing to publish Unsloth on a public Cloudflare URL: the " "default admin password was never changed, no terminal is " "attached to change it here, and the bootstrap shutdown " "deadline does not apply to this launch (api-only, or " @@ -1163,11 +1163,11 @@ def _terminal_password_gate( # terminal-attached run / reset-password instead of reading it from disk. print( " WARNING: the default admin password is still active while " - "Studio is about to be published on a public Cloudflare URL, and " + "Unsloth is about to be published on a public Cloudflare URL, and " "no terminal is attached to change it here. The public page will " "NOT auto-fill the bootstrap credential. Set a new password by " "running `unsloth studio` locally with a terminal attached, or " - "`unsloth studio reset-password`. Studio shuts down after the " + "`unsloth studio reset-password`. Unsloth shuts down after the " "bootstrap deadline (UNSLOTH_STUDIO_BOOTSTRAP_TIMEOUT, default 1h) " "unless the password is changed.", file = sys.stderr, @@ -1222,7 +1222,7 @@ def _apply_supplied_password(password_value: "Optional[str]") -> None: _auth_storage.ensure_default_admin() if not _auth_storage.requires_password_change(_admin): print( - "Error: a Studio admin password is already set; --password only sets " + "Error: an Unsloth admin password is already set; --password only sets " "the initial password. Run `unsloth studio reset-password` first.", file = sys.stderr, flush = True, @@ -1244,6 +1244,13 @@ def _apply_supplied_password(password_value: "Optional[str]") -> None: flush = True, ) sys.exit(1) + if any(ch.isspace() for ch in supplied): + print( + "Error: password cannot contain spaces; not starting.", + file = sys.stderr, + flush = True, + ) + sys.exit(1) if _is_current_password(supplied): print( "Error: the new password must differ from the current bootstrap " @@ -1337,7 +1344,7 @@ def run_server( pass # Persist a session log + native-crash stacks BEFORE importing main, so - # even import-time failures leave evidence on disk. Field report: Studio + # even import-time failures leave evidence on disk. Field report: Unsloth # "terminates without a warning" -- a native crash in the GPU runtime # kills the process with no Python traceback, and a desktop-shortcut # console closes before anything can be read. Console-only logging made @@ -1354,11 +1361,23 @@ def run_server( if secure: os.environ["UNSLOTH_SECURE"] = "1" - import nest_asyncio - - nest_asyncio.apply() - import asyncio + + # nest_asyncio is for Colab/IPython, where the main thread already runs a loop + # the blocking waits below would collide with. Apply it only with a loop running + # (a plain CLI start has nothing to nest) and only on Python <= 3.13: on 3.14+ + # its global Task patch leaves asyncio.current_task() None (tracking moved into + # C), which also breaks the background uvicorn loop and 500s every request. It + # is archived upstream, so no 3.14 fix is coming; skip it there. + if sys.version_info < (3, 14): + try: + asyncio.get_running_loop() + except RuntimeError: + pass + else: + import nest_asyncio + nest_asyncio.apply() + from threading import Thread, Event import uvicorn @@ -1394,7 +1413,7 @@ def run_server( ensure_studio_directories() logger.info( - "Ensured Studio directories in %.1fms", + "Ensured Unsloth directories in %.1fms", (time.perf_counter() - boot_started) * 1000, ) @@ -1443,7 +1462,7 @@ def run_server( installer_bin = home / "unsloth_studio" / "bin" / "unsloth" tried_lines = "\n".join(f" - {p}" for p in attempted) or " (none)" raise SystemExit( - "[ERROR] Studio frontend build not found.\n" + "[ERROR] Unsloth frontend build not found.\n" f"Tried:\n{tried_lines}\n" "\n" "Likely cause: another 'unsloth' on PATH is shadowing the " @@ -1545,7 +1564,7 @@ def run_server( ) if not _pw_proceed: print( - "Not starting Studio; set a new admin password first, or launch " + "Not starting Unsloth; set a new admin password first, or launch " "without --secure/--cloudflare.", file = sys.stderr, flush = True, @@ -1683,7 +1702,7 @@ def run_server( logger = logger, ) logger.info( - "Studio will shut down in %ds unless the default admin password is changed.", + "Unsloth will shut down in %ds unless the default admin password is changed.", _bootstrap_timeout, ) except Exception as e: # best-effort: never block startup on the timeout @@ -1741,11 +1760,11 @@ def _build_arg_parser(): "--cloudflare", action = argparse.BooleanOptionalAction, default = None, - help = "Expose Studio on a PUBLIC internet URL via a free Cloudflare HTTPS " + help = "Expose Unsloth on a PUBLIC internet URL via a free Cloudflare HTTPS " "tunnel, for non-api-only wildcard binds (0.0.0.0 or ::). Off by default; " "pass --cloudflare to enable it (--secure implies it), --no-cloudflare to " "force it off. It does not change a raw wildcard bind. If the admin " - "password was never changed, Studio asks for a new one in the terminal " + "password was never changed, Unsloth asks for a new one in the terminal " "before publishing the URL.", ) parser.add_argument( @@ -1755,7 +1774,7 @@ def _build_arg_parser(): help = "Expose ONLY a Cloudflare HTTPS link: bind localhost and fail closed " "if the tunnel can't start. Without it, --no-secure also serves the raw " "0.0.0.0 port, which is reachable from anywhere on the network. If the " - "admin password was never changed, Studio asks for a new one in the " + "admin password was never changed, Unsloth asks for a new one in the " "terminal before publishing the URL.", ) # Back-compat: accept --not-secure as a hidden alias for --no-secure. diff --git a/studio/backend/startup_banner.py b/studio/backend/startup_banner.py index ea951a4325..9ec7a4f91c 100644 --- a/studio/backend/startup_banner.py +++ b/studio/backend/startup_banner.py @@ -1,7 +1,7 @@ # SPDX-License-Identifier: AGPL-3.0-only # Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 -"""Terminal banner for Studio startup. +"""Terminal banner for Unsloth startup. Stdlib only -- safe to import without the rest of the backend. """ @@ -172,7 +172,7 @@ def print_studio_access_banner( secondary, ), style( - " Only on trusted networks -- anyone who reaches this machine can use Studio.", + " Only on trusted networks -- anyone who reaches this machine can use Unsloth.", secondary, ), ] diff --git a/studio/backend/storage/providers_db.py b/studio/backend/storage/providers_db.py index 07165cbe70..e6f40c5030 100644 --- a/studio/backend/storage/providers_db.py +++ b/studio/backend/storage/providers_db.py @@ -6,8 +6,12 @@ Same pattern as studio_db.py (module-level functions, raw sqlite3, WAL, per-function connections). API keys are NOT stored here: they live only in the browser (localStorage) and are sent encrypted per-request. + +Enabled model selections and discovered catalog IDs are stored server-side so +remote Studio clients see the same connection state (#7281). """ +import json import logging import sqlite3 import threading @@ -22,6 +26,33 @@ _schema_lock = threading.Lock() _schema_ready = False +def _encode_models_json(models: Optional[list[str]]) -> str: + if not models: + return "[]" + return json.dumps([str(model).strip() for model in models if str(model).strip()]) + + +def _decode_models_json(raw: Optional[str]) -> list[str]: + if not raw: + return [] + try: + parsed = json.loads(raw) + except json.JSONDecodeError: + return [] + if not isinstance(parsed, list): + return [] + return [str(model).strip() for model in parsed if str(model).strip()] + + +def _row_models(row: sqlite3.Row) -> tuple[list[str], list[str]]: + return ( + _decode_models_json(row["models_json"] if "models_json" in row.keys() else None), + _decode_models_json( + row["available_models_json"] if "available_models_json" in row.keys() else None + ), + ) + + def _ensure_schema(conn: sqlite3.Connection) -> None: """Create the llm_providers table if absent. Called once per process.""" conn.execute("PRAGMA journal_mode=WAL") @@ -38,6 +69,13 @@ def _ensure_schema(conn: sqlite3.Connection) -> None: ) """ ) + existing_cols = {row[1] for row in conn.execute("PRAGMA table_info(llm_providers)").fetchall()} + if "models_json" not in existing_cols: + conn.execute("ALTER TABLE llm_providers ADD COLUMN models_json TEXT NOT NULL DEFAULT '[]'") + if "available_models_json" not in existing_cols: + conn.execute( + "ALTER TABLE llm_providers ADD COLUMN available_models_json TEXT NOT NULL DEFAULT '[]'" + ) def get_connection() -> sqlite3.Connection: @@ -59,17 +97,37 @@ def get_connection() -> sqlite3.Connection: return conn -def create_provider(id: str, provider_type: str, display_name: str, base_url: str) -> None: +def create_provider( + id: str, + provider_type: str, + display_name: str, + base_url: str, + models: Optional[list[str]] = None, + available_models: Optional[list[str]] = None, +) -> None: """Insert a new provider configuration.""" now = datetime.now(timezone.utc).isoformat() conn = get_connection() try: conn.execute( """ - INSERT INTO llm_providers (id, provider_type, display_name, base_url, created_at, updated_at) - VALUES (?, ?, ?, ?, ?, ?) + INSERT INTO llm_providers ( + id, provider_type, display_name, base_url, + models_json, available_models_json, + created_at, updated_at + ) + VALUES (?, ?, ?, ?, ?, ?, ?, ?) """, - (id, provider_type, display_name, base_url, now, now), + ( + id, + provider_type, + display_name, + base_url, + _encode_models_json(models), + _encode_models_json(available_models), + now, + now, + ), ) conn.commit() finally: @@ -81,6 +139,8 @@ def update_provider( display_name: Optional[str] = None, base_url: Optional[str] = None, is_enabled: Optional[bool] = None, + models: Optional[list[str]] = None, + available_models: Optional[list[str]] = None, ) -> bool: """Update fields on an existing provider. Returns True if a row was updated.""" updates = [] @@ -94,6 +154,12 @@ def update_provider( if is_enabled is not None: updates.append("is_enabled = ?") params.append(1 if is_enabled else 0) + if models is not None: + updates.append("models_json = ?") + params.append(_encode_models_json(models)) + if available_models is not None: + updates.append("available_models_json = ?") + params.append(_encode_models_json(available_models)) if not updates: return False updates.append("updated_at = ?") @@ -128,7 +194,13 @@ def get_provider(id: str) -> Optional[dict]: conn = get_connection() try: row = conn.execute("SELECT * FROM llm_providers WHERE id = ?", (id,)).fetchone() - return dict(row) if row else None + if not row: + return None + data = dict(row) + models, available_models = _row_models(row) + data["models"] = models + data["available_models"] = available_models + return data finally: conn.close() @@ -138,6 +210,13 @@ def list_providers() -> list[dict]: conn = get_connection() try: rows = conn.execute("SELECT * FROM llm_providers ORDER BY created_at").fetchall() - return [dict(row) for row in rows] + providers: list[dict] = [] + for row in rows: + data = dict(row) + models, available_models = _row_models(row) + data["models"] = models + data["available_models"] = available_models + providers.append(data) + return providers finally: conn.close() diff --git a/studio/backend/storage/studio_db.py b/studio/backend/storage/studio_db.py index 4e0c711b69..6972e7b7ff 100644 --- a/studio/backend/storage/studio_db.py +++ b/studio/backend/storage/studio_db.py @@ -7,6 +7,7 @@ Like auth/storage.py (module-level functions, raw sqlite3, per-function connections) plus WAL mode and PRAGMA foreign_keys = ON for CASCADE deletes. """ +import hashlib import json import logging import os @@ -100,6 +101,7 @@ _schema_lock = threading.Lock() _schema_ready = False _SQLITE_IN_CHUNK_SIZE = 900 _PROJECT_WORKSPACE_SUBDIRS = ("sandbox",) +_CHAT_ATTACHMENT_INVENTORY_VERSION = 1 def _project_slug(name: str) -> str: @@ -190,13 +192,18 @@ def _ensure_schema(conn: sqlite3.Connection) -> None: error_message TEXT, duration_seconds REAL, loss_sparkline TEXT, - display_name TEXT + display_name TEXT, + resume_blocked INTEGER NOT NULL DEFAULT 0 ) """ ) existing_cols = {row[1] for row in conn.execute("PRAGMA table_info(training_runs)").fetchall()} if "display_name" not in existing_cols: conn.execute("ALTER TABLE training_runs ADD COLUMN display_name TEXT") + if "resume_blocked" not in existing_cols: + conn.execute( + "ALTER TABLE training_runs ADD COLUMN resume_blocked INTEGER NOT NULL DEFAULT 0" + ) conn.execute( """ CREATE TABLE IF NOT EXISTS training_metrics ( @@ -313,6 +320,141 @@ def _ensure_schema(conn: sqlite3.Connection) -> None: ) """ ) + tombstone_schema = """ + CREATE TABLE chat_attachment_tombstones ( + thread_id TEXT NOT NULL REFERENCES chat_threads(id) ON DELETE CASCADE, + message_id TEXT NOT NULL, + attachment_id TEXT NOT NULL, + deleted_at INTEGER NOT NULL, + PRIMARY KEY(thread_id, message_id, attachment_id) + ) WITHOUT ROWID + """ + tombstone_table = conn.execute( + """ + SELECT 1 FROM sqlite_master + WHERE type = 'table' AND name = 'chat_attachment_tombstones' + """ + ).fetchone() + if tombstone_table is None: + conn.execute(tombstone_schema) + else: + tombstone_columns = { + row[1] for row in conn.execute("PRAGMA table_info(chat_attachment_tombstones)") + } + tombstone_fk_targets = { + row[2] for row in conn.execute("PRAGMA foreign_key_list(chat_attachment_tombstones)") + } + if "thread_id" not in tombstone_columns or "chat_threads" not in tombstone_fk_targets: + # The first implementation cascaded through chat_messages, which + # erased deletion knowledge during pruneMissing. Rebuild once, + # retaining every tombstone whose owning thread still exists. + conn.execute("SAVEPOINT migrate_chat_attachment_tombstones") + try: + conn.execute( + "ALTER TABLE chat_attachment_tombstones " + "RENAME TO chat_attachment_tombstones_legacy" + ) + conn.execute(tombstone_schema) + if "thread_id" in tombstone_columns: + conn.execute( + """ + INSERT OR IGNORE INTO chat_attachment_tombstones + (thread_id, message_id, attachment_id, deleted_at) + SELECT legacy.thread_id, legacy.message_id, + legacy.attachment_id, legacy.deleted_at + FROM chat_attachment_tombstones_legacy legacy + JOIN chat_threads thread ON thread.id = legacy.thread_id + """ + ) + else: + conn.execute( + """ + INSERT OR IGNORE INTO chat_attachment_tombstones + (thread_id, message_id, attachment_id, deleted_at) + SELECT message.thread_id, legacy.message_id, + legacy.attachment_id, legacy.deleted_at + FROM chat_attachment_tombstones_legacy legacy + JOIN chat_messages message ON message.id = legacy.message_id + """ + ) + conn.execute("DROP TABLE chat_attachment_tombstones_legacy") + conn.execute("RELEASE SAVEPOINT migrate_chat_attachment_tombstones") + except Exception: + conn.execute("ROLLBACK TO SAVEPOINT migrate_chat_attachment_tombstones") + conn.execute("RELEASE SAVEPOINT migrate_chat_attachment_tombstones") + raise + conn.execute( + """ + CREATE TABLE IF NOT EXISTS chat_attachment_inventory ( + message_id TEXT NOT NULL REFERENCES chat_messages(id) ON DELETE CASCADE, + attachment_id TEXT NOT NULL, + name TEXT NOT NULL, + type TEXT, + content_type TEXT, + size_bytes INTEGER, + PRIMARY KEY(message_id, attachment_id) + ) WITHOUT ROWID + """ + ) + conn.execute( + """ + CREATE TABLE IF NOT EXISTS chat_attachment_inventory_state ( + singleton INTEGER NOT NULL PRIMARY KEY CHECK(singleton = 1), + inventory_version INTEGER NOT NULL DEFAULT 0, + dirty INTEGER NOT NULL DEFAULT 1, + backfilled_at INTEGER NOT NULL + ) + """ + ) + inventory_state_columns = { + row[1] for row in conn.execute("PRAGMA table_info(chat_attachment_inventory_state)") + } + if "inventory_version" not in inventory_state_columns: + conn.execute( + "ALTER TABLE chat_attachment_inventory_state " + "ADD COLUMN inventory_version INTEGER NOT NULL DEFAULT 0" + ) + if "dirty" not in inventory_state_columns: + conn.execute( + "ALTER TABLE chat_attachment_inventory_state " + "ADD COLUMN dirty INTEGER NOT NULL DEFAULT 1" + ) + conn.execute( + """ + CREATE TRIGGER IF NOT EXISTS chat_attachment_inventory_dirty_insert + AFTER INSERT ON chat_messages + BEGIN + INSERT INTO chat_attachment_inventory_state + (singleton, inventory_version, dirty, backfilled_at) + VALUES (1, 0, 1, 0) + ON CONFLICT(singleton) DO UPDATE SET dirty = 1; + END + """ + ) + conn.execute( + """ + CREATE TRIGGER IF NOT EXISTS chat_attachment_inventory_dirty_update + AFTER UPDATE ON chat_messages + BEGIN + INSERT INTO chat_attachment_inventory_state + (singleton, inventory_version, dirty, backfilled_at) + VALUES (1, 0, 1, 0) + ON CONFLICT(singleton) DO UPDATE SET dirty = 1; + END + """ + ) + conn.execute( + """ + CREATE TRIGGER IF NOT EXISTS chat_attachment_inventory_dirty_delete + AFTER DELETE ON chat_messages + BEGIN + INSERT INTO chat_attachment_inventory_state + (singleton, inventory_version, dirty, backfilled_at) + VALUES (1, 0, 1, 0) + ON CONFLICT(singleton) DO UPDATE SET dirty = 1; + END + """ + ) conn.execute( "CREATE INDEX IF NOT EXISTS idx_chat_threads_model_type_created_at ON chat_threads(model_type, created_at)" ) @@ -391,6 +533,21 @@ def _ensure_schema(conn: sqlite3.Connection) -> None: conn.execute( "CREATE INDEX IF NOT EXISTS idx_prompt_lists_created_at ON prompt_lists(created_at)" ) + inventory_state = conn.execute( + """ + SELECT inventory_version, dirty + FROM chat_attachment_inventory_state + WHERE singleton = 1 + """ + ).fetchone() + if ( + inventory_state is None + or inventory_state["inventory_version"] != _CHAT_ATTACHMENT_INVENTORY_VERSION + or inventory_state["dirty"] + ): + _rebuild_chat_attachment_inventory(conn) + _mark_chat_attachment_inventory_clean(conn) + conn.commit() def _prompt_entry_from_row(row: sqlite3.Row) -> dict: @@ -582,16 +739,43 @@ def create_run( config_json: str, started_at: str, total_steps: Optional[int], + *, + output_dir: Optional[str] = None, + cancel_requested: bool = False, + resumed_from_run_id: Optional[str] = None, ) -> None: conn = get_connection() try: conn.execute( """ - INSERT INTO training_runs (id, model_name, dataset_name, config_json, started_at, total_steps) - VALUES (?, ?, ?, ?, ?, ?) + INSERT INTO training_runs ( + id, model_name, dataset_name, config_json, started_at, total_steps, + output_dir, resume_blocked + ) + VALUES (?, ?, ?, ?, ?, ?, ?, ?) """, - (id, model_name, dataset_name, config_json, started_at, total_steps), + ( + id, + model_name, + dataset_name, + config_json, + started_at, + total_steps, + None if cancel_requested else output_dir, + int(cancel_requested), + ), ) + if resumed_from_run_id: + claimed = conn.execute( + """ + UPDATE training_runs SET resume_blocked = 1 + WHERE id = ? AND status IN ('stopped', 'error') + AND output_dir = ? AND resume_blocked = 0 + """, + (resumed_from_run_id, output_dir), + ) + if claimed.rowcount != 1: + raise RuntimeError("Resume source is no longer available") conn.commit() finally: conn.close() @@ -634,6 +818,8 @@ def finish_run( loss_sparkline: Optional[str] = None, output_dir: Optional[str] = None, error_message: Optional[str] = None, + clear_output_dir: bool = False, + resume_blocked: bool = False, ) -> None: conn = get_connection() try: @@ -641,9 +827,16 @@ def finish_run( """ UPDATE training_runs SET status = ?, ended_at = ?, final_step = ?, final_loss = ?, - duration_seconds = ?, loss_sparkline = ?, output_dir = ?, - error_message = ? - WHERE id = ? + duration_seconds = ?, loss_sparkline = ?, + output_dir = CASE + WHEN resume_blocked = 1 OR ? = 1 THEN NULL + WHEN ? IS NOT NULL THEN ? + WHEN ? IN ('error', 'stopped') THEN output_dir + ELSE NULL + END, + error_message = ?, + resume_blocked = CASE WHEN resume_blocked = 1 OR ? = 1 THEN 1 ELSE ? END + WHERE id = ? AND status = 'running' """, ( status, @@ -652,8 +845,13 @@ def finish_run( final_loss, duration_seconds, loss_sparkline, + int(clear_output_dir), output_dir, + output_dir, + status, error_message, + int(clear_output_dir), + int(resume_blocked), id, ), ) @@ -713,6 +911,38 @@ def update_run_display_name(id: str, display_name: Optional[str]) -> None: conn.close() +def update_run_output_dir(id: str, output_dir: Optional[str]) -> None: + conn = get_connection() + try: + conn.execute( + """ + UPDATE training_runs SET output_dir = ? + WHERE id = ? AND status = 'running' AND resume_blocked = 0 + """, + (output_dir, id), + ) + conn.commit() + finally: + conn.close() + + +def mark_run_cancel_requested(id: str) -> bool: + """Clear resume/export state only while the exact run is still active.""" + conn = get_connection() + try: + cursor = conn.execute( + """ + UPDATE training_runs SET output_dir = NULL, resume_blocked = 1 + WHERE id = ? AND status = 'running' + """, + (id,), + ) + conn.commit() + return cursor.rowcount > 0 + finally: + conn.close() + + def list_runs(limit: int = 50, offset: int = 0) -> dict: conn = get_connection() try: @@ -722,15 +952,15 @@ def list_runs(limit: int = 50, offset: int = 0) -> dict: SELECT r.id, r.status, r.model_name, r.dataset_name, r.started_at, r.ended_at, r.total_steps, r.final_step, r.final_loss, r.output_dir, r.duration_seconds, r.error_message, - r.loss_sparkline, r.display_name, r.config_json, + r.loss_sparkline, r.display_name, r.config_json, r.resume_blocked, CASE - WHEN r.status = 'stopped' + WHEN r.status IN ('stopped', 'error') AND r.output_dir IS NOT NULL AND EXISTS ( SELECT 1 FROM training_runs newer WHERE newer.output_dir = r.output_dir - AND newer.status IN ('stopped', 'completed') + AND newer.status IN ('stopped', 'completed', 'error', 'running') AND newer.started_at > r.started_at ) THEN 1 ELSE 0 @@ -765,13 +995,13 @@ def get_run(id: str) -> Optional[dict]: """ SELECT r.*, CASE - WHEN r.status = 'stopped' + WHEN r.status IN ('stopped', 'error') AND r.output_dir IS NOT NULL AND EXISTS ( SELECT 1 FROM training_runs newer WHERE newer.output_dir = r.output_dir - AND newer.status IN ('stopped', 'completed') + AND newer.status IN ('stopped', 'completed', 'error', 'running') AND newer.started_at > r.started_at ) THEN 1 ELSE 0 @@ -806,12 +1036,12 @@ def get_resumable_run_by_output_dir(output_dir: str) -> Optional[dict]: 0 AS resumed_later FROM training_runs r WHERE r.output_dir = ? - AND r.status = 'stopped' + AND r.status IN ('stopped', 'error') AND NOT EXISTS ( SELECT 1 FROM training_runs newer WHERE newer.output_dir = r.output_dir - AND newer.status IN ('stopped', 'completed') + AND newer.status IN ('stopped', 'completed', 'error', 'running') AND newer.started_at > r.started_at ) ORDER BY r.started_at DESC @@ -914,8 +1144,12 @@ def cleanup_orphaned_runs() -> None: conn.execute( """ UPDATE training_runs - SET status = 'error', - error_message = 'Server restarted during training', + SET status = CASE WHEN resume_blocked = 1 THEN 'stopped' ELSE 'error' END, + error_message = CASE + WHEN resume_blocked = 1 THEN NULL + ELSE 'Server restarted during training' + END, + output_dir = CASE WHEN resume_blocked = 1 THEN NULL ELSE output_dir END, ended_at = ? WHERE status = 'running' """, @@ -1219,7 +1453,14 @@ def delete_chat_threads(ids: list[str]) -> None: return conn = get_connection() try: + conn.execute("BEGIN IMMEDIATE") + _ensure_chat_attachment_inventory_current(conn) + conn.executemany( + "DELETE FROM chat_attachment_tombstones WHERE thread_id = ?", + [(id,) for id in ids], + ) conn.executemany("DELETE FROM chat_threads WHERE id = ?", [(id,) for id in ids]) + _mark_chat_attachment_inventory_clean(conn) conn.commit() finally: conn.close() @@ -1228,7 +1469,11 @@ def delete_chat_threads(ids: list[str]) -> None: def clear_chat_history() -> None: conn = get_connection() try: + conn.execute("BEGIN IMMEDIATE") + _ensure_chat_attachment_inventory_current(conn) + conn.execute("DELETE FROM chat_attachment_tombstones") conn.execute("DELETE FROM chat_threads") + _mark_chat_attachment_inventory_clean(conn) conn.commit() finally: conn.close() @@ -1354,6 +1599,7 @@ def delete_chat_project(id: str, delete_files: bool = False) -> Optional[dict]: conn = get_connection() try: conn.execute("BEGIN IMMEDIATE") + _ensure_chat_attachment_inventory_current(conn) row = conn.execute("SELECT * FROM chat_projects WHERE id = ?", (id,)).fetchone() if row is None: conn.rollback() @@ -1361,6 +1607,7 @@ def delete_chat_project(id: str, delete_files: bool = False) -> Optional[dict]: project = _chat_project_from_row(row) conn.execute("DELETE FROM chat_threads WHERE project_id = ?", (id,)) conn.execute("DELETE FROM chat_projects WHERE id = ?", (id,)) + _mark_chat_attachment_inventory_clean(conn) conn.commit() if delete_files: _delete_project_workspace(project) @@ -1483,15 +1730,285 @@ def _recompute_chat_thread_updated_at(conn: sqlite3.Connection, thread_id: str) ) +_CONTENT_PART_ID_PREFIX = "content-part-sha256-" +_URI_SCHEME_RE = re.compile(r"^[A-Za-z][A-Za-z0-9+.-]*:") + + +def _is_locally_stored_blob(value: str) -> bool: + """True for data URIs or bare base64, never external/blob URI references.""" + candidate = value.lstrip() + if not candidate: + return False + if candidate[:5].lower() == "data:": + return True + if candidate.startswith(("//", "\\\\")): + return False + return _URI_SCHEME_RE.match(candidate) is None + + +def _managed_content_part_payload(part: dict) -> Optional[tuple[str, Any]]: + """Return the locally stored blob payload used to identify a content part.""" + image = part.get("image") + if isinstance(image, str) and image[:5].lower() == "data:": + return "image", image + + audio = part.get("audio") + if isinstance(audio, str) and _is_locally_stored_blob(audio): + return "audio", audio + if isinstance(audio, dict): + data = audio.get("data") + if isinstance(data, str) and _is_locally_stored_blob(data): + return "audio", audio + return None + + +def _content_part_id(part: dict) -> Optional[str]: + """Stable managed id derived from blob data, without mutating inference content.""" + payload = _managed_content_part_payload(part) + if payload is None: + return None + canonical = json.dumps( + payload, + ensure_ascii = False, + separators = (",", ":"), + sort_keys = True, + ).encode("utf-8") + return f"{_CONTENT_PART_ID_PREFIX}{hashlib.sha256(canonical).hexdigest()}" + + +def _chat_attachment_tombstones_for_messages( + conn: sqlite3.Connection, thread_id: str, message_ids: list[str] +) -> dict[str, set[str]]: + tombstones = {message_id: set() for message_id in message_ids} + unique_ids = list(dict.fromkeys(message_ids)) + for start in range(0, len(unique_ids), _SQLITE_IN_CHUNK_SIZE): + chunk = unique_ids[start : start + _SQLITE_IN_CHUNK_SIZE] + placeholders = ",".join("?" for _ in chunk) + rows = conn.execute( + f""" + SELECT message_id, attachment_id + FROM chat_attachment_tombstones + WHERE thread_id = ? AND message_id IN ({placeholders}) + """, + (thread_id, *chunk), + ).fetchall() + for row in rows: + tombstones[row["message_id"]].add(row["attachment_id"]) + return tombstones + + +def _reconcile_chat_message_uploads(message: dict, tombstones: set[str]) -> dict: + """Strip uploads previously deleted through the Data tab from a stale write.""" + if not tombstones: + return message + + reconciled = dict(message) + attachments = message.get("attachments") + if isinstance(attachments, list): + reconciled["attachments"] = [ + attachment + for attachment in attachments + if not (isinstance(attachment, dict) and str(attachment.get("id") or "") in tombstones) + ] + + content = message.get("content") + if isinstance(content, list): + reconciled["content"] = [ + part + for part in content + if not (isinstance(part, dict) and (_content_part_id(part) or "") in tombstones) + ] + return reconciled + + +def _chat_attachment_metadata_text(value, fallback: Optional[str] = None) -> Optional[str]: + """Keep untyped legacy/import metadata safe for SQLite binding.""" + if value is None: + return fallback + if isinstance(value, str): + return value or fallback + if isinstance(value, (bool, int, float)): + return str(value) + # Objects and arrays are not useful display metadata and sqlite3 rejects + # binding them directly. + return fallback + + +def _chat_attachment_inventory_entries( + attachments_json: Optional[str], + content_json: Optional[str], + tombstones: Optional[set[str]] = None, +) -> list[dict]: + tombstones = tombstones or set() + attachments = _json_loads(attachments_json, None) + if not isinstance(attachments, list): + attachments = [] + attachments = [ + attachment + for attachment in attachments + if isinstance(attachment, dict) and attachment.get("id") + ] + attachments.extend(_content_part_attachments(content_json)) + + entries: list[dict] = [] + seen: set[str] = set() + for attachment in attachments: + attachment_id = str(attachment["id"]) + if attachment_id in seen or attachment_id in tombstones: + continue + seen.add(attachment_id) + entries.append( + { + "id": attachment_id, + "name": _chat_attachment_metadata_text(attachment.get("name"), "attachment"), + "type": _chat_attachment_metadata_text(attachment.get("type")), + "contentType": _chat_attachment_metadata_text(attachment.get("contentType")), + "sizeBytes": _chat_attachment_size_bytes(attachment), + } + ) + return entries + + +def _replace_chat_attachment_inventory( + conn: sqlite3.Connection, + message_id: str, + attachments_json: Optional[str], + content_json: Optional[str], + tombstones: Optional[set[str]] = None, +) -> None: + conn.execute("DELETE FROM chat_attachment_inventory WHERE message_id = ?", (message_id,)) + entries = _chat_attachment_inventory_entries( + attachments_json, + content_json, + tombstones, + ) + conn.executemany( + """ + INSERT INTO chat_attachment_inventory + (message_id, attachment_id, name, type, content_type, size_bytes) + VALUES (?, ?, ?, ?, ?, ?) + """, + [ + ( + message_id, + entry["id"], + entry["name"], + entry["type"], + entry["contentType"], + entry["sizeBytes"], + ) + for entry in entries + ], + ) + + +def _mark_chat_attachment_inventory_clean(conn: sqlite3.Connection) -> None: + conn.execute( + """ + INSERT INTO chat_attachment_inventory_state + (singleton, inventory_version, dirty, backfilled_at) + VALUES (1, ?, 0, ?) + ON CONFLICT(singleton) DO UPDATE SET + inventory_version = excluded.inventory_version, + dirty = 0, + backfilled_at = excluded.backfilled_at + """, + ( + _CHAT_ATTACHMENT_INVENTORY_VERSION, + int(datetime.now(timezone.utc).timestamp() * 1000), + ), + ) + + +def _rebuild_chat_attachment_inventory(conn: sqlite3.Connection) -> None: + """Rebuild after schema upgrade or a write from an older Studio build.""" + conn.execute("DELETE FROM chat_attachment_inventory") + tombstones: dict[tuple[str, str], set[str]] = {} + for row in conn.execute( + "SELECT thread_id, message_id, attachment_id FROM chat_attachment_tombstones" + ).fetchall(): + tombstones.setdefault((row["thread_id"], row["message_id"]), set()).add( + row["attachment_id"] + ) + rows = conn.execute( + "SELECT id, thread_id, attachments_json, content_json FROM chat_messages" + ).fetchall() + for row in rows: + _replace_chat_attachment_inventory( + conn, + row["id"], + row["attachments_json"], + row["content_json"], + tombstones.get((row["thread_id"], row["id"]), set()), + ) + + +def _ensure_chat_attachment_inventory_current(conn: sqlite3.Connection) -> None: + state = conn.execute( + """ + SELECT inventory_version, dirty + FROM chat_attachment_inventory_state + WHERE singleton = 1 + """ + ).fetchone() + if ( + state is not None + and state["inventory_version"] == _CHAT_ATTACHMENT_INVENTORY_VERSION + and not state["dirty"] + ): + return + + owns_transaction = not conn.in_transaction + if owns_transaction: + conn.execute("BEGIN IMMEDIATE") + try: + state = conn.execute( + """ + SELECT inventory_version, dirty + FROM chat_attachment_inventory_state + WHERE singleton = 1 + """ + ).fetchone() + if ( + state is None + or state["inventory_version"] != _CHAT_ATTACHMENT_INVENTORY_VERSION + or state["dirty"] + ): + _rebuild_chat_attachment_inventory(conn) + _mark_chat_attachment_inventory_clean(conn) + if owns_transaction: + conn.commit() + except Exception: + if owns_transaction: + conn.rollback() + raise + + def upsert_chat_message(message: dict) -> dict: conn = get_connection() try: conn.execute("BEGIN IMMEDIATE") + _ensure_chat_attachment_inventory_current(conn) _raise_if_chat_message_thread_conflicts( conn, message["threadId"], [message["id"]], ) + tombstones = _chat_attachment_tombstones_for_messages( + conn, + message["threadId"], + [message["id"]], + ) + reconciled = _reconcile_chat_message_uploads( + message, + tombstones.get(message["id"], set()), + ) + content_json = json.dumps(reconciled.get("content", [])) + attachments_json = ( + json.dumps(reconciled.get("attachments")) + if reconciled.get("attachments") is not None + else None + ) conn.execute( """ INSERT INTO chat_messages @@ -1507,23 +2024,32 @@ def upsert_chat_message(message: dict) -> dict: WHERE excluded.thread_id = chat_messages.thread_id """, ( - message["id"], - message["threadId"], - message.get("parentId"), - message["role"], - json.dumps(message.get("content", [])), - json.dumps(message.get("attachments")) - if message.get("attachments") is not None + reconciled["id"], + reconciled["threadId"], + reconciled.get("parentId"), + reconciled["role"], + content_json, + attachments_json, + json.dumps(reconciled.get("metadata")) + if reconciled.get("metadata") is not None else None, - json.dumps(message.get("metadata")) - if message.get("metadata") is not None - else None, - int(message["createdAt"]), + int(reconciled["createdAt"]), ), ) - _bump_chat_thread_updated_at(conn, message["threadId"], int(message["createdAt"])) + _replace_chat_attachment_inventory( + conn, + reconciled["id"], + attachments_json, + content_json, + ) + _bump_chat_thread_updated_at( + conn, + reconciled["threadId"], + int(reconciled["createdAt"]), + ) + _mark_chat_attachment_inventory_clean(conn) conn.commit() - return message + return reconciled except Exception: conn.rollback() raise @@ -1539,13 +2065,28 @@ def sync_chat_messages( conn = get_connection() try: conn.execute("BEGIN IMMEDIATE") + _ensure_chat_attachment_inventory_current(conn) _raise_if_chat_message_thread_conflicts( conn, thread_id, [m["id"] for m in messages], ) - if prune_missing: - conn.execute("DELETE FROM chat_messages WHERE thread_id = ?", (thread_id,)) + tombstones = _chat_attachment_tombstones_for_messages( + conn, + thread_id, + [m["id"] for m in messages], + ) + reconciled_messages = [ + _reconcile_chat_message_uploads(m, tombstones.get(m["id"], set())) for m in messages + ] + serialized_messages = [ + ( + m, + json.dumps(m.get("content", [])), + json.dumps(m.get("attachments")) if m.get("attachments") is not None else None, + ) + for m in reconciled_messages + ] conn.executemany( """ INSERT INTO chat_messages @@ -1566,20 +2107,46 @@ def sync_chat_messages( thread_id, m.get("parentId"), m["role"], - json.dumps(m.get("content", [])), - json.dumps(m.get("attachments")) if m.get("attachments") is not None else None, + content_json, + attachments_json, json.dumps(m.get("metadata")) if m.get("metadata") is not None else None, int(m["createdAt"]), ) - for m in messages + for m, content_json, attachments_json in serialized_messages ], ) - if prune_missing: - _recompute_chat_thread_updated_at(conn, thread_id) - elif messages: - _bump_chat_thread_updated_at( - conn, thread_id, max(int(m["createdAt"]) for m in messages) + for m, content_json, attachments_json in serialized_messages: + _replace_chat_attachment_inventory( + conn, + m["id"], + attachments_json, + content_json, ) + if prune_missing: + retained_ids = {m["id"] for m in reconciled_messages} + existing_ids = { + row["id"] + for row in conn.execute( + "SELECT id FROM chat_messages WHERE thread_id = ?", + (thread_id,), + ).fetchall() + } + missing_ids = sorted(existing_ids - retained_ids) + for start in range(0, len(missing_ids), _SQLITE_IN_CHUNK_SIZE): + chunk = missing_ids[start : start + _SQLITE_IN_CHUNK_SIZE] + placeholders = ",".join("?" for _ in chunk) + conn.execute( + f"DELETE FROM chat_messages WHERE thread_id = ? AND id IN ({placeholders})", + (thread_id, *chunk), + ) + _recompute_chat_thread_updated_at(conn, thread_id) + elif reconciled_messages: + _bump_chat_thread_updated_at( + conn, + thread_id, + max(int(m["createdAt"]) for m in reconciled_messages), + ) + _mark_chat_attachment_inventory_clean(conn) conn.commit() return list_chat_messages(thread_id) except ChatMessageConflictError: @@ -1613,6 +2180,7 @@ def fork_chat_thread( conn = get_connection() try: conn.execute("BEGIN IMMEDIATE") + _ensure_chat_attachment_inventory_current(conn) src = conn.execute( "SELECT * FROM chat_threads WHERE id = ?", (source_thread_id,) ).fetchone() @@ -1686,6 +2254,14 @@ def fork_chat_thread( for row in ancestry ], ) + for row in ancestry: + _replace_chat_attachment_inventory( + conn, + id_map[row["id"]], + row["attachments_json"], + row["content_json"], + ) + _mark_chat_attachment_inventory_clean(conn) conn.commit() thread_row = conn.execute( "SELECT * FROM chat_threads WHERE id = ?", (new_thread_id,) @@ -1744,6 +2320,279 @@ def get_chat_message(thread_id: str, message_id: str) -> Optional[dict]: conn.close() +def _blob_part_base64_len(part: dict) -> int: + """Base64 payload length of an image or audio content part, or 0.""" + image = part.get("image") + if isinstance(image, str) and image[:5].lower() == "data:": + return len(image.rsplit(",", 1)[-1]) + audio = part.get("audio") + if isinstance(audio, str) and _is_locally_stored_blob(audio): + return len(audio.rsplit(",", 1)[-1]) + if isinstance(audio, dict): + data = audio.get("data") + if isinstance(data, str) and _is_locally_stored_blob(data): + return len(data) + return 0 + + +def _chat_attachment_size_bytes(attachment: dict) -> Optional[int]: + """Approximate stored size of one attachment's content parts. + + Image and audio parts hold base64 payloads (decoded bytes ~= 3/4 of the + encoded length); text parts count their character length. None when there + is no sizable content (e.g. a stripped/legacy attachment). + """ + total = 0 + found = False + for part in attachment.get("content") or []: + if not isinstance(part, dict): + continue + blob_len = _blob_part_base64_len(part) + if blob_len > 0: + total += (blob_len * 3) // 4 + found = True + continue + text = part.get("text") + if isinstance(text, str) and text: + total += len(text.encode("utf-8", errors = "ignore")) + found = True + return total if found else None + + +def _content_part_attachments(content_json: Optional[str]) -> list[dict]: + """Managed local blobs stored in content_json, with stable payload ids. + + Exact duplicate blobs intentionally share one inventory id. Deleting that + id removes every identical copy, avoiding ambiguous index-based addressing. + """ + content = _json_loads(content_json, None) + if not isinstance(content, list): + return [] + out: list[dict] = [] + seen: set[str] = set() + for part in content: + if not isinstance(part, dict): + continue + attachment_id = _content_part_id(part) + payload = _managed_content_part_payload(part) + if attachment_id is None or payload is None or attachment_id in seen: + continue + seen.add(attachment_id) + kind, value = payload + content_type = None + if kind == "image" and isinstance(value, str): + content_type = value[5:].split(";", 1)[0].split(",", 1)[0] or None + out.append( + { + "id": attachment_id, + "type": kind, + "name": "Chat image" if kind == "image" else "Chat audio", + "contentType": content_type, + "content": [part], + } + ) + return out + + +def list_chat_attachments_page( + limit: int = 50, offset: int = 0 +) -> tuple[list[dict], Optional[int]]: + """One bounded page from the normalized attachment inventory.""" + if not 1 <= limit <= 100: + raise ValueError("limit must be between 1 and 100") + if offset < 0: + raise ValueError("offset must be non-negative") + + conn = get_connection() + try: + _ensure_chat_attachment_inventory_current(conn) + rows = conn.execute( + """ + SELECT i.attachment_id, i.name, i.type, i.content_type, + i.size_bytes, m.id AS message_id, m.thread_id, + m.created_at, t.title AS thread_title, t.pair_id + FROM chat_attachment_inventory i + JOIN chat_messages m ON m.id = i.message_id + LEFT JOIN chat_threads t ON t.id = m.thread_id + ORDER BY m.created_at DESC, m.id ASC, i.attachment_id ASC + LIMIT ? OFFSET ? + """, + (limit + 1, offset), + ).fetchall() + finally: + conn.close() + + has_more = len(rows) > limit + page_rows = rows[:limit] + attachments = [ + { + "id": row["attachment_id"], + "messageId": row["message_id"], + "threadId": row["thread_id"], + "pairId": row["pair_id"], + "threadTitle": row["thread_title"], + "name": row["name"], + "type": row["type"], + "contentType": row["content_type"], + "sizeBytes": row["size_bytes"], + "createdAt": row["created_at"], + } + for row in page_rows + ] + return attachments, offset + limit if has_more else None + + +def list_chat_attachments() -> list[dict]: + """Compatibility helper returning the full normalized inventory.""" + attachments: list[dict] = [] + offset = 0 + while True: + page, next_offset = list_chat_attachments_page(limit = 100, offset = offset) + attachments.extend(page) + if next_offset is None: + return attachments + offset = next_offset + + +def get_chat_attachment(message_id: str, attachment_id: str) -> Optional[dict]: + """One attachment record (full content) from a message, or None.""" + conn = get_connection() + try: + row = conn.execute( + """ + SELECT message.attachments_json, message.content_json, + EXISTS( + SELECT 1 FROM chat_attachment_tombstones tombstone + WHERE tombstone.thread_id = message.thread_id + AND tombstone.message_id = message.id + AND tombstone.attachment_id = ? + ) AS tombstoned + FROM chat_messages message + WHERE message.id = ? + """, + (attachment_id, message_id), + ).fetchone() + finally: + conn.close() + if row is None or row["tombstoned"]: + return None + attachments = _json_loads(row["attachments_json"], None) + if isinstance(attachments, list): + for attachment in attachments: + if isinstance(attachment, dict) and str(attachment.get("id") or "") == attachment_id: + return attachment + if attachment_id.startswith(_CONTENT_PART_ID_PREFIX): + for attachment in _content_part_attachments(row["content_json"]): + if attachment["id"] == attachment_id: + return attachment + return None + + +def _record_chat_attachment_tombstone( + conn: sqlite3.Connection, thread_id: str, message_id: str, attachment_id: str +) -> None: + conn.execute( + """ + INSERT INTO chat_attachment_tombstones + (thread_id, message_id, attachment_id, deleted_at) + VALUES (?, ?, ?, ?) + ON CONFLICT(thread_id, message_id, attachment_id) DO UPDATE SET + deleted_at = excluded.deleted_at + """, + ( + thread_id, + message_id, + attachment_id, + int(datetime.now(timezone.utc).timestamp() * 1000), + ), + ) + + +def delete_chat_attachment(message_id: str, attachment_id: str) -> bool: + """Remove one stored upload from a message. + + The tombstone is retained while the thread exists, so pruning and later + recreating the same message id cannot restore the deleted upload. If an + ordinary attachment id collides with a content-blob id, both are deleted as + one managed item. + """ + conn = get_connection() + try: + conn.execute("BEGIN IMMEDIATE") + _ensure_chat_attachment_inventory_current(conn) + row = conn.execute( + """ + SELECT thread_id, attachments_json, content_json + FROM chat_messages WHERE id = ? + """, + (message_id,), + ).fetchone() + if row is None: + conn.rollback() + return False + + attachments = _json_loads(row["attachments_json"], None) + updated_attachments_json = row["attachments_json"] + deleted_attachment = False + if isinstance(attachments, list): + remaining_attachments = [ + attachment + for attachment in attachments + if not ( + isinstance(attachment, dict) + and str(attachment.get("id") or "") == attachment_id + ) + ] + deleted_attachment = len(remaining_attachments) != len(attachments) + if deleted_attachment: + updated_attachments_json = json.dumps(remaining_attachments) + + content = _json_loads(row["content_json"], None) + updated_content_json = row["content_json"] + deleted_content = False + if attachment_id.startswith(_CONTENT_PART_ID_PREFIX) and isinstance(content, list): + remaining_content = [ + part + for part in content + if not (isinstance(part, dict) and _content_part_id(part) == attachment_id) + ] + deleted_content = len(remaining_content) != len(content) + if deleted_content: + updated_content_json = json.dumps(remaining_content) + + if not deleted_attachment and not deleted_content: + conn.rollback() + return False + conn.execute( + """ + UPDATE chat_messages + SET attachments_json = ?, content_json = ? + WHERE id = ? + """, + (updated_attachments_json, updated_content_json, message_id), + ) + _record_chat_attachment_tombstone( + conn, + row["thread_id"], + message_id, + attachment_id, + ) + _replace_chat_attachment_inventory( + conn, + message_id, + updated_attachments_json, + updated_content_json, + ) + _mark_chat_attachment_inventory_clean(conn) + conn.commit() + return True + except Exception: + conn.rollback() + raise + finally: + conn.close() + + def list_chat_messages_for_threads(thread_ids: list[str]) -> list[dict]: if not thread_ids: return [] diff --git a/studio/backend/tests/conftest.py b/studio/backend/tests/conftest.py index b0b9ee309c..c2216104a3 100644 --- a/studio/backend/tests/conftest.py +++ b/studio/backend/tests/conftest.py @@ -101,13 +101,13 @@ def studio_server(request): @pytest.fixture def base_url(studio_server): - """Base URL for the e2e Studio server (from ``studio_server``).""" + """Base URL for the e2e Unsloth server (from ``studio_server``).""" return studio_server[0] @pytest.fixture def api_key(studio_server): - """API key for the e2e Studio server (from ``studio_server``).""" + """API key for the e2e Unsloth server (from ``studio_server``).""" return studio_server[1] diff --git a/studio/backend/tests/test_amd_apu_unified_memory.py b/studio/backend/tests/test_amd_apu_unified_memory.py index 4df9e85b30..9fd8260bf2 100644 --- a/studio/backend/tests/test_amd_apu_unified_memory.py +++ b/studio/backend/tests/test_amd_apu_unified_memory.py @@ -91,7 +91,7 @@ class TestApuRamShortfall: """On a unified-memory APU the weights load into system RAM, so a model larger than available RAM (the field case: a 64.6 GB GGUF on a WSL VM capped well below the ROCm-reported APU budget) must be refused before spawning, - not left to OOM-kill the Studio process.""" + not left to OOM-kill the Unsloth process.""" def test_field_case_wsl_cap_refuses(self): # 64.6 GB weights, ~46 GB available (WSL VM): refuse with guidance. diff --git a/studio/backend/tests/test_anthropic_compaction.py b/studio/backend/tests/test_anthropic_compaction.py index 1528eebe8b..acc0acc2e0 100644 --- a/studio/backend/tests/test_anthropic_compaction.py +++ b/studio/backend/tests/test_anthropic_compaction.py @@ -4,7 +4,7 @@ """Unit tests for Anthropic server-side context compaction wiring. Compaction is a beta (header ``compact-2026-01-12``) gated to Opus 4.6/4.7, -Sonnet 4.6, and Mythos preview. When enabled, Studio attaches +Sonnet 4.6, and Mythos preview. When enabled, Unsloth attaches ``context_management.edits[{type:"compact_20260112", trigger:{type:"input_tokens", value:N}}]``; the 50k-token minimum is clamped up so the request doesn't 400. diff --git a/studio/backend/tests/test_anthropic_fast_mode_edge.py b/studio/backend/tests/test_anthropic_fast_mode_edge.py index dd69d77590..03f5d1c0eb 100644 --- a/studio/backend/tests/test_anthropic_fast_mode_edge.py +++ b/studio/backend/tests/test_anthropic_fast_mode_edge.py @@ -330,7 +330,7 @@ def test_refusal_chunk_is_proper_openai_delta_shape(monkeypatch): def test_refusal_tool_event_chunk_shape(monkeypatch): - """Drop signal rides a Studio `_toolEvent` envelope (delta={}, + """Drop signal rides an Unsloth `_toolEvent` envelope (delta={}, finish_reason=null); the frontend latches on `_toolEvent.type == "anthropic_refusal"`.""" _, lines = _capture(monkeypatch, sse = _refusal_sse(), model = "claude-opus-4-7") @@ -409,7 +409,7 @@ def _fast_speed_sse(model: str = "claude-opus-4-7", speed: str = "fast") -> byte def test_usage_speed_propagates_to_final_usage_chunk_fast(monkeypatch): - """``usage.speed == "fast"`` from upstream must reach the Studio usage chunk.""" + """``usage.speed == "fast"`` from upstream must reach the Unsloth usage chunk.""" _, lines = _capture(monkeypatch, sse = _fast_speed_sse(speed = "fast")) usage_lines = [l for l in lines if l.startswith("data: ") and '"usage"' in l] assert usage_lines, lines @@ -428,7 +428,7 @@ def test_usage_speed_propagates_to_final_usage_chunk_standard(monkeypatch): def test_usage_speed_absent_when_anthropic_does_not_report(monkeypatch): - """Studio must not invent ``usage.speed`` when upstream omits it.""" + """Unsloth must not invent ``usage.speed`` when upstream omits it.""" _, lines = _capture(monkeypatch) parsed = [ json.loads(l[len("data: ") :]) for l in lines if l.startswith("data: ") and '"usage"' in l diff --git a/studio/backend/tests/test_anthropic_messages.py b/studio/backend/tests/test_anthropic_messages.py index 3b0ea37372..9ccc3f44dd 100644 --- a/studio/backend/tests/test_anthropic_messages.py +++ b/studio/backend/tests/test_anthropic_messages.py @@ -1418,7 +1418,7 @@ class TestNormalizeAnthropicOpenAIImages: # ===================================================================== -# Studio-tool alias detection (/v1/messages tool routing) +# Unsloth-tool alias detection (/v1/messages tool routing) # ===================================================================== @@ -1436,7 +1436,7 @@ class TestAnthropicRequestedStudioTools: def test_client_tool_named_python_is_not_misclassified(self): # input_schema is the client-tool discriminator; its presence must - # prevent the name from being treated as a Studio alias. + # prevent the name from being treated as an Unsloth alias. tools = [ { "name": "python", @@ -1747,9 +1747,9 @@ class TestAnthropicMessagesToolRouting: assert "name" in exc.value.detail def test_alias_named_client_tool_without_schema_rejected_with_400(self, monkeypatch): - # Regression: a typo'd client tool whose name collides with a Studio + # Regression: a typo'd client tool whose name collides with an Unsloth # alias (e.g. a custom "python" tool missing input_schema) must - # surface a 400, not silently switch into Studio's built-in python + # surface a 400, not silently switch into Unsloth's built-in python # execution. _mock_backend(monkeypatch) payload = _basic_payload(tools = [{"name": "python"}]) @@ -1770,7 +1770,7 @@ class TestAnthropicMessagesToolRouting: def test_disable_tools_policy_overrides_server_tool_alias(self, monkeypatch): # CLI `unsloth run --disable-tools` sets policy=False. A request with - # a Studio server-tool alias must NOT enter the agentic loop then. + # an Unsloth server-tool alias must NOT enter the agentic loop then. backend = _mock_backend(monkeypatch) set_tool_policy(False) payload = _basic_payload( diff --git a/studio/backend/tests/test_audio_sampling_fill.py b/studio/backend/tests/test_audio_sampling_fill.py new file mode 100644 index 0000000000..efea18b83e --- /dev/null +++ b/studio/backend/tests/test_audio_sampling_fill.py @@ -0,0 +1,90 @@ +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 + +"""Audio (TTS) generation applies recommended sampling + operator pins, like chat. + +Regression guard for the fix that moved the sampling fill ahead of the audio generators: a +prior version resolved sampling only after the audio branches returned, so `unsloth run +--temperature` (UNSLOTH_SAMPLING_*) and per-model recommendations never reached audio +generation. These exercise the transformers TTS path of ``generate_audio`` (the direct +``/audio/generate`` route, which the chat-completions audio branches also delegate to). +""" + +import asyncio + +import pytest + +import routes.inference as inference_route +from models.inference import ChatCompletionRequest +from utils.inference import inference_config as ic + + +class _FakeLlama: + # is_loaded False forces the transformers (non-GGUF) TTS branch in generate_audio. + is_loaded = False + _is_audio = False + + +class _FakeTransformersBackend: + def __init__(self): + self.active_model_name = "some/custom-tts" + self.models = {"some/custom-tts": {"is_audio": True}} + self.captured = {} + + def generate_audio_response(self, **kwargs): + self.captured.update(kwargs) + return (b"RIFFfake", 24000) + + +@pytest.fixture(autouse = True) +def _isolate(monkeypatch): + ic._recommended_sampling.cache_clear() + for field in ic.SAMPLING_FIELD_NAMES: + monkeypatch.delenv(ic._SAMPLING_FIELDS[field][0], raising = False) + yield + ic._recommended_sampling.cache_clear() + + +def _run_generate_audio( + monkeypatch, + *, + recommended = None, + temperature = None, +): + backend = _FakeTransformersBackend() + monkeypatch.setattr(inference_route, "get_llama_cpp_backend", lambda: _FakeLlama()) + monkeypatch.setattr(inference_route, "get_inference_backend", lambda: backend) + + async def _noop_switch(*a, **k): + return None + + monkeypatch.setattr(inference_route, "_maybe_auto_switch_model", _noop_switch) + + # Recommendation source == the Chat UI's .inference block. + monkeypatch.setattr(ic, "load_inference_config", lambda mid: dict(recommended or {})) + ic._recommended_sampling.cache_clear() + + kwargs = {"model": "some/custom-tts", "messages": [{"role": "user", "content": "hi"}]} + if temperature is not None: + kwargs["temperature"] = temperature + payload = ChatCompletionRequest(**kwargs) + + asyncio.run(inference_route.generate_audio(payload, request = None, current_subject = "t")) + return backend.captured + + +def test_audio_uses_recommended_sampling_when_omitted(monkeypatch): + captured = _run_generate_audio(monkeypatch, recommended = {"temperature": 1.0, "top_k": 64}) + assert captured["temperature"] == 1.0 + assert captured["top_k"] == 64 + + +def test_audio_operator_pin_overrides_client(monkeypatch): + monkeypatch.setenv("UNSLOTH_SAMPLING_TEMPERATURE", "0.9") + captured = _run_generate_audio(monkeypatch, recommended = {"temperature": 1.0}, temperature = 0.2) + assert captured["temperature"] == 0.9 # operator pin wins even over an explicit client value + + +def test_audio_client_explicit_preserved(monkeypatch): + captured = _run_generate_audio(monkeypatch, recommended = {"temperature": 1.0}, temperature = 0.2) + assert captured["temperature"] == 0.2 # explicit client value preserved over recommendation diff --git a/studio/backend/tests/test_cached_gguf_routes.py b/studio/backend/tests/test_cached_gguf_routes.py index d4a7cae208..6f2c672002 100644 --- a/studio/backend/tests/test_cached_gguf_routes.py +++ b/studio/backend/tests/test_cached_gguf_routes.py @@ -66,6 +66,66 @@ def test_iter_gguf_paths_matches_extension_case_insensitively(tmp_path): assert result == ["Q4_K_M.gguf", "Q8_0.GGUF"] +def test_legacy_hf_scan_uses_snapshot_path_for_inactive_cache(tmp_path): + repo = tmp_path / "models--Org--Model" + snapshot = repo / "snapshots" / "revision" + snapshot.mkdir(parents = True) + + [row] = models_route._scan_hf_cache(tmp_path, active_cache = False) + + assert row.model_id == "Org/Model" + assert row.id == str(snapshot.resolve()) + assert row.path == str(snapshot.resolve()) + + +def test_collect_local_models_scans_previous_cache(monkeypatch, tmp_path): + active = tmp_path / "active" + previous = tmp_path / "previous" + active.mkdir() + snapshot = previous / "models--Org--Previous" / "snapshots" / "revision" + snapshot.mkdir(parents = True) + + monkeypatch.setattr(models_route, "_resolve_hf_cache_dir", lambda: active) + monkeypatch.setattr("utils.paths.legacy_hf_cache_dir", lambda: tmp_path / "legacy") + monkeypatch.setattr("utils.paths.hf_default_cache_dir", lambda: tmp_path / "default") + monkeypatch.setattr("utils.paths.lmstudio_model_dirs", lambda: []) + monkeypatch.setattr("utils.hf_cache_settings.known_hf_hub_caches", lambda: [active, previous]) + monkeypatch.setattr("storage.studio_db.list_scan_folders", lambda: []) + + rows = models_route.collect_local_models(tmp_path / "models") + + previous_row = next(row for row in rows if row.model_id == "Org/Previous") + assert previous_row.id == str(snapshot.resolve()) + + +def test_collect_local_models_prefers_complete_previous_copy(monkeypatch, tmp_path): + active = tmp_path / "active" + previous = tmp_path / "previous" + active_partial = active / "models--Org--Model" / "blobs" / "abc.incomplete" + active_partial.parent.mkdir(parents = True) + active_partial.write_bytes(b"partial") + snapshot = previous / "models--Org--Model" / "snapshots" / "revision" + snapshot.mkdir(parents = True) + (snapshot / "model.safetensors").write_bytes(b"complete") + + monkeypatch.setattr(models_route, "_resolve_hf_cache_dir", lambda: active) + monkeypatch.setattr("utils.paths.legacy_hf_cache_dir", lambda: tmp_path / "legacy") + monkeypatch.setattr("utils.paths.hf_default_cache_dir", lambda: tmp_path / "default") + monkeypatch.setattr("utils.paths.lmstudio_model_dirs", lambda: []) + monkeypatch.setattr( + "utils.hf_cache_settings.known_hf_hub_caches", + lambda: [active, previous], + ) + monkeypatch.setattr("storage.studio_db.list_scan_folders", lambda: []) + + rows = models_route.collect_local_models(tmp_path / "models") + + [row] = [row for row in rows if row.model_id == "Org/Model"] + assert row.id == str(snapshot.resolve()) + assert row.partial is False + assert row.active_cache is False + + def test_list_cached_gguf_includes_non_suffix_repo_when_cache_contains_gguf(monkeypatch, tmp_path): repo = _repo( "HauhauCS/Gemma-4-E4B-Uncensored-HauhauCS-Aggressive", @@ -120,12 +180,217 @@ def test_is_hidden_model_hides_validation_probe_everywhere(): assert models_route._is_hidden_model( None, "/hf/models--ggml-org--models/snapshots/abc/tinyllamas/stories260K.gguf" ) + # A Windows-style snapshot path must match too, even on a POSIX interpreter + # (the filename check splits on both separators). + assert models_route._is_hidden_model( + r"C:\Users\u\.cache\huggingface\hub\models--ggml-org--models\snapshots\abc\tinyllamas\stories260K.gguf" + ) assert not models_route._is_hidden_model("unsloth/gemma-3-270m-it-GGUF") # The exact-filename needle must not hide a real repo that merely # references stories260K in its name. assert not models_route._is_hidden_model("user/stories260K-finetune-GGUF") +def test_is_hidden_model_hides_dictation_models(tmp_path): + assert models_route._is_hidden_model("unsloth/whisper-tiny") + assert models_route._is_hidden_model("unsloth/whisper-base") + assert models_route._is_hidden_model("unsloth/whisper-small") + assert models_route._is_hidden_model("unsloth/whisper-large-v3-turbo") + assert models_route._is_hidden_model( + "/hf/models--unsloth--whisper-large-v3/snapshots/abc/model.safetensors" + ) + assert not models_route._is_hidden_model("user/whisper-finetune") + assert not models_route._is_hidden_model( + "C:\\cache\\models--unsloth--whisper-small-finetune\\model.safetensors" + ) + custom = tmp_path / "custom-whisper" + custom.mkdir() + (custom / "config.json").write_text( + '{"model_type": "whisper", "architectures": ["WhisperForConditionalGeneration"]}' + ) + (custom / "model.safetensors").write_bytes(b"weights") + assert models_route._is_hidden_model( + "user/custom-checkpoint", + str(custom / "model.safetensors"), + ) + named_only = tmp_path / "whisper-finetune" + named_only.mkdir() + (named_only / "config.json").write_text('{"model_type": "llama"}') + assert not models_route._is_hidden_model("user/whisper-finetune", str(named_only)) + + +def test_list_cached_models_hides_custom_whisper_by_config(monkeypatch, tmp_path): + # Regression: the legacy /cached-models picker must pass the snapshot path so + # the config check hides a custom (non-curated) Whisper checkpoint; a bare + # repo id cannot ("user/whisper-finetune" is not in the curated set). + repo_path = tmp_path / "models--user--whisper-finetune" + snap = repo_path / "snapshots" / "abc" + snap.mkdir(parents = True) + (snap / "config.json").write_text( + '{"model_type": "whisper", "architectures": ["WhisperForConditionalGeneration"]}' + ) + (snap / "model.safetensors").write_bytes(b"weights") + + captured: list = [] + real_hidden = models_route._is_hidden_model + + def spy(*values): + captured.append(values) + return real_hidden(*values) + + monkeypatch.setattr(models_route, "_is_hidden_model", spy) + repo = _repo( + "user/whisper-finetune", + [SimpleNamespace(file_name = "model.safetensors", size_on_disk = 10)], + repo_path, + ) + monkeypatch.setattr( + models_route, "_all_hf_cache_scans", lambda: [SimpleNamespace(repos = [repo])] + ) + + result = asyncio.run( + models_route.list_cached_models(current_subject = "test-user", hf_token = None) + ) + # The route passed the snapshot path (not just the repo id) ... + assert any(str(repo_path) in values for values in captured) + # ... so the custom Whisper checkpoint is hidden from the chat picker. + assert result["cached"] == [] + + +def test_is_hidden_model_matches_repo_ids_exactly(monkeypatch): + """A custom embedder with a generic basename is hidden by EXACT repo-id + match only, so unrelated cached repos that merely contain the basename stay + visible. Regression: substring basename matching hid real chat models like + ``user/model-chat`` from the On Device inventory.""" + from core.rag import config as rag_config + + monkeypatch.setattr(rag_config, "effective_embedding_model", lambda: "org/model") + monkeypatch.setattr(rag_config, "effective_gguf_repo", lambda: "org/model-GGUF") + + # The exact embedder repo and its GGUF companion are hidden. + assert models_route._is_hidden_model("org/model") + assert models_route._is_hidden_model("org/model-GGUF") + # Unrelated repos that merely contain "model" must NOT be hidden. + assert not models_route._is_hidden_model("user/model-chat") + assert not models_route._is_hidden_model("org/model-instruct") + assert not models_route._is_hidden_model("acme/remodelled-chat") + # The validation probe stays hidden regardless of embedder config. + assert models_route._is_hidden_model("ggml-org/models") + + +def test_is_hidden_model_matches_repo_derived_local_paths(monkeypatch): + """Match exact repo-derived cache and LM Studio paths.""" + from core.rag import config as rag_config + + monkeypatch.setattr(rag_config, "effective_embedding_model", lambda: "org/model") + monkeypatch.setattr(rag_config, "effective_gguf_repo", lambda: "org/model-GGUF") + + assert models_route._is_hidden_model( + "/cache/models--org--model/snapshots/abc/model.safetensors" + ) + assert models_route._is_hidden_model( + r"C:\Users\u\.cache\huggingface\hub\models--org--model-GGUF\snapshots\abc" + ) + assert models_route._is_hidden_model("/lm-studio/org/model-GGUF/model-Q8_0.gguf") + assert not models_route._is_hidden_model("/lm-studio/user/model-chat/model-Q8_0.gguf") + assert not models_route._is_hidden_model("/cache/models--org--model-instruct") + + +def test_is_hidden_model_prefers_existing_relative_path(monkeypatch, tmp_path): + """Prefer an existing relative path over repo-id syntax.""" + from core.rag import config as rag_config + + embedder = tmp_path / "models" / "embedder" + embedder.mkdir(parents = True) + monkeypatch.chdir(tmp_path) + monkeypatch.setattr(rag_config, "effective_embedding_model", lambda: "models/embedder") + monkeypatch.setattr(rag_config, "effective_gguf_repo", lambda: "org/embedder-GGUF") + + assert models_route._is_hidden_model(str(embedder)) + + +def test_is_hidden_model_keeps_stale_default_embedder_hidden(monkeypatch): + """Keep default embedders hidden after a settings change.""" + from core.rag import config as rag_config + + monkeypatch.setattr(rag_config, "effective_embedding_model", lambda: "org/custom") + monkeypatch.setattr(rag_config, "effective_gguf_repo", lambda: "org/custom-GGUF") + + assert models_route._is_hidden_model("unsloth/bge-small-en-v1.5") + assert models_route._is_hidden_model("unsloth/bge-small-en-v1.5-GGUF") + assert models_route._is_hidden_model("/models/bge-small-en-v1.5") + assert models_route._is_hidden_model("/models/bge-small-en-v1.5-F16.gguf") + assert models_route._is_hidden_model(r"C:\models\bge-small-en-v1.5-Q8_0.gguf") + # Repo IDs still use exact matching, and similar local basenames must have + # a real separator after the static default name. + assert not models_route._is_hidden_model("user/bge-small-en-v1.5-chat") + assert not models_route._is_hidden_model("/models/bge-small-en-v1.50") + + +def test_is_hidden_model_keeps_env_default_hidden_after_override(monkeypatch): + """A persisted override must not expose the deployment's env default.""" + from core.rag import config as rag_config + + monkeypatch.delenv("RAG_EMBED_GGUF_REPO", raising = False) + monkeypatch.setattr(rag_config, "EMBEDDING_MODEL", "org/env-default") + monkeypatch.setattr(rag_config, "effective_embedding_model", lambda: "org/custom") + monkeypatch.setattr(rag_config, "effective_gguf_repo", lambda: "org/custom-GGUF") + + assert models_route._is_hidden_model("org/env-default") + assert models_route._is_hidden_model("org/env-default-GGUF") + assert models_route._is_hidden_model("org/custom") + assert models_route._is_hidden_model("org/custom-GGUF") + assert not models_route._is_hidden_model("org/env-default-chat") + + +def test_hidden_models_importable_without_heavy_model_stack(): + """The hub cache scanner imports ``is_hidden_model`` at module scope, so it + must not drag in ``utils/models/__init__`` (the model-config + checkpoint + stack). Verify in a clean interpreter that importing the helper touches + neither ``utils.models`` nor those heavy submodules, and still classifies + the probe.""" + import os + import subprocess + import textwrap + + backend = Path(__file__).resolve().parents[1] + code = textwrap.dedent( + """ + import sys + + class _Blocker: + _blocked = ( + "utils.models", + "utils.models.model_config", + "utils.models.checkpoints", + ) + + def find_spec(self, name, path=None, target=None): + if name in self._blocked: + raise ImportError("blocked heavy import: " + name) + return None + + sys.meta_path.insert(0, _Blocker()) + from utils.hidden_models import is_hidden_model + + loaded = sorted(m for m in sys.modules if m.startswith("utils.models")) + assert not loaded, loaded + assert is_hidden_model("ggml-org/models") is True + assert is_hidden_model("unsloth/gemma-3-270m-it-GGUF") is False + print("HIDDEN_MODELS_IMPORT_OK") + """ + ) + env = dict(os.environ, PYTHONPATH = str(backend)) + proc = subprocess.run( + [sys.executable, "-c", code], + capture_output = True, + text = True, + env = env, + ) + assert proc.returncode == 0, proc.stderr + assert "HIDDEN_MODELS_IMPORT_OK" in proc.stdout + + def test_list_cached_gguf_hides_llama_validation_probe(monkeypatch, tmp_path): """The ggml-org/models / stories260K install validation probe can land in the HF cache as a side effect of installing the prebuilt llama-server. @@ -434,33 +699,14 @@ def _gfile(name: str, size: int, mtime: float) -> SimpleNamespace: ) -def test_all_hf_cache_scans_survives_inaccessible_aux_cache(monkeypatch, tmp_path): - """An unreadable auxiliary cache (e.g. an inaccessible - ``~/.cache/huggingface/hub``) must be skipped, not abort the scan. - Regression guard for ``extra.is_dir()`` raising and wiping the response. - """ - import huggingface_hub - import utils.paths as paths_mod +def test_all_hf_cache_scans_uses_shared_inventory(monkeypatch, tmp_path): + from hub.utils import inventory_scan active = SimpleNamespace( repos = [_repo("Org/Active", [_file("Q4_K_M.gguf", 5_000)], tmp_path / "active")] ) - def _fake_scan(cache_dir = None): - if cache_dir is None: - return active - raise AssertionError("auxiliary scan should have been skipped") - - class _Boom: - def is_dir(self): - raise PermissionError(13, "Permission denied") - - def resolve(self): - raise PermissionError(13, "Permission denied") - - monkeypatch.setattr(huggingface_hub, "scan_cache_dir", _fake_scan) - monkeypatch.setattr(paths_mod, "legacy_hf_cache_dir", lambda: _Boom()) - monkeypatch.setattr(paths_mod, "hf_default_cache_dir", lambda: _Boom()) + monkeypatch.setattr(inventory_scan, "all_hf_cache_scans", lambda: [active]) scans = models_route._all_hf_cache_scans() assert scans == [active] @@ -547,13 +793,17 @@ def test_gguf_variants_mmproj_does_not_mark_quant_downloaded(monkeypatch, tmp_pa "list_gguf_variants", lambda repo_id, hf_token = None: (variants, True, []), ) - monkeypatch.setattr(GV, "_local_main_gguf_blobs_by_quant", lambda _repo_id: {}) + monkeypatch.setattr( + GV, + "_local_main_gguf_blobs_by_quant", + lambda _repo_id, repo_cache_dir = None: {}, + ) snap = tmp_path / "models--org--repo" / "snapshots" / "rev" snap.mkdir(parents = True) (snap / "model-Q4_K_M.gguf").write_bytes(b"x" * 10_000) # real weight, fully present (snap / "mmproj-F16.gguf").write_bytes(b"y" * 20_000) # mmproj adapter, label "F16" - monkeypatch.setattr(GV, "iter_hf_cache_snapshots", lambda _repo_id: [snap]) + monkeypatch.setattr(GV, "iter_hf_cache_snapshots", lambda _repo_id, root = None: [snap]) result = asyncio.run( models_route.get_gguf_variants( @@ -566,6 +816,52 @@ def test_gguf_variants_mmproj_does_not_mark_quant_downloaded(monkeypatch, tmp_pa assert flags["F16"] is False +def test_gguf_variants_route_scopes_local_probe_to_selected_cache(monkeypatch, tmp_path): + snapshot = tmp_path / "inactive" / "models--org--repo" / "snapshots" / "rev" + snapshot.mkdir(parents = True) + calls = [] + + async def scoped_variants(repo_id, **kwargs): + calls.append((repo_id, kwargs)) + return SimpleNamespace( + repo_id = repo_id, + variants = [], + has_vision = False, + default_variant = None, + ) + + context_calls = [] + monkeypatch.setattr(GV, "get_gguf_variants_response", scoped_variants) + monkeypatch.setattr( + models_route, + "_read_native_context_length", + lambda model, *, is_local: context_calls.append((model, is_local)) or 8192, + ) + + result = asyncio.run( + models_route.get_gguf_variants( + repo_id = "org/repo", + prefer_local_cache = True, + local_path = str(snapshot), + hf_token = None, + current_subject = "test-user", + ) + ) + + assert calls == [ + ( + "org/repo", + { + "prefer_local_cache": True, + "local_path": str(snapshot), + "hf_token": None, + }, + ) + ] + assert context_calls == [(str(snapshot), True)] + assert result.context_length == 8192 + + def test_gguf_variants_ignore_big_endian_siblings(monkeypatch, tmp_path): siblings = [ SimpleNamespace(rfilename = "model-Q4_K_M-be.gguf", size = 100), @@ -587,12 +883,16 @@ def test_gguf_variants_ignore_big_endian_siblings(monkeypatch, tmp_path): siblings, ), ) - monkeypatch.setattr(GV, "_local_main_gguf_blobs_by_quant", lambda _repo_id: {}) + monkeypatch.setattr( + GV, + "_local_main_gguf_blobs_by_quant", + lambda _repo_id, repo_cache_dir = None: {}, + ) snap = tmp_path / "models--org--repo" / "snapshots" / "rev" snap.mkdir(parents = True) (snap / "model-Q4_K_M.gguf").write_bytes(b"x" * 10) - monkeypatch.setattr(GV, "iter_hf_cache_snapshots", lambda _repo_id: [snap]) + monkeypatch.setattr(GV, "iter_hf_cache_snapshots", lambda _repo_id, root = None: [snap]) result = asyncio.run( models_route.get_gguf_variants( @@ -619,12 +919,16 @@ def test_gguf_variants_cached_big_endian_does_not_satisfy_variant(monkeypatch, t "list_gguf_variants", lambda repo_id, hf_token = None: (variants, False, []), ) - monkeypatch.setattr(GV, "_local_main_gguf_blobs_by_quant", lambda _repo_id: {}) + monkeypatch.setattr( + GV, + "_local_main_gguf_blobs_by_quant", + lambda _repo_id, repo_cache_dir = None: {}, + ) snap = tmp_path / "models--org--repo" / "snapshots" / "rev" snap.mkdir(parents = True) (snap / "model-Q4_K_M-be.gguf").write_bytes(b"x" * 10) - monkeypatch.setattr(GV, "iter_hf_cache_snapshots", lambda _repo_id: [snap]) + monkeypatch.setattr(GV, "iter_hf_cache_snapshots", lambda _repo_id, root = None: [snap]) result = asyncio.run( models_route.get_gguf_variants( @@ -635,66 +939,82 @@ def test_gguf_variants_cached_big_endian_does_not_satisfy_variant(monkeypatch, t assert result.variants[0].downloaded is False -def test_gguf_download_progress_excludes_mmproj(monkeypatch, tmp_path): - """A cached mmproj adapter must not count toward a same-label main - variant's download progress (mmproj-F16 vs an F16 weight).""" - import huggingface_hub.constants as hf_constants +def test_legacy_gguf_progress_delegates_to_shared_service(monkeypatch): + calls = [] - monkeypatch.setattr(hf_constants, "HF_HUB_CACHE", str(tmp_path)) - snap = tmp_path / "models--org--repo" / "snapshots" / "rev" - snap.mkdir(parents = True) - (snap / "mmproj-F16.gguf").write_bytes(b"y" * 20_000) # only the adapter on disk + async def shared(repo_id, *, variant, expected_bytes, hf_token): + calls.append((repo_id, variant, expected_bytes, hf_token)) + return {"downloaded_bytes": 10, "expected_bytes": 20, "progress": 0.5} - result = asyncio.run( - models_route.get_gguf_download_progress( - repo_id = "org/repo", - variant = "F16", - expected_bytes = 20_000, - current_subject = "test-user", - ) + monkeypatch.setattr( + "hub.services.models.downloads.get_gguf_download_progress_response", + shared, ) - assert result["downloaded_bytes"] == 0 - assert result["progress"] == 0 - - -def test_gguf_download_progress_excludes_big_endian_sibling(monkeypatch, tmp_path): - import huggingface_hub.constants as hf_constants - - monkeypatch.setattr(hf_constants, "HF_HUB_CACHE", str(tmp_path)) - snap = tmp_path / "models--org--repo" / "snapshots" / "rev" - snap.mkdir(parents = True) - (snap / "model-Q4_K_M-be.gguf").write_bytes(b"y" * 20_000) - result = asyncio.run( models_route.get_gguf_download_progress( repo_id = "org/repo", variant = "Q4_K_M", - expected_bytes = 20_000, + expected_bytes = 20, + hf_token = "token", current_subject = "test-user", ) ) - assert result["downloaded_bytes"] == 0 - assert result["progress"] == 0 + assert result["progress"] == 0.5 + assert calls == [("org/repo", "Q4_K_M", 20, "token")] -def test_gguf_download_progress_counts_quant_subdir(monkeypatch, tmp_path): - import huggingface_hub.constants as hf_constants +def test_legacy_model_progress_delegates_to_shared_service(monkeypatch): + calls = [] - monkeypatch.setattr(hf_constants, "HF_HUB_CACHE", str(tmp_path)) - snap = tmp_path / "models--org--repo" / "snapshots" / "rev" / "Q4_K_M" - snap.mkdir(parents = True) - (snap / "foo.gguf").write_bytes(b"x" * 20_000) + async def shared(repo_id, *, hf_token): + calls.append((repo_id, hf_token)) + return {"downloaded_bytes": 10, "expected_bytes": 20, "progress": 0.5} + + monkeypatch.setattr( + "hub.services.models.downloads.get_download_progress_response", + shared, + ) result = asyncio.run( - models_route.get_gguf_download_progress( + models_route.get_download_progress( repo_id = "org/repo", - variant = "Q4_K_M", - expected_bytes = 20_000, + hf_token = "token", current_subject = "test-user", ) ) - assert result["downloaded_bytes"] == 20_000 - assert result["progress"] == 1.0 + assert result["progress"] == 0.5 + assert calls == [("org/repo", "token")] + + +def test_legacy_delete_delegates_to_shared_service(monkeypatch): + calls = [] + + async def shared( + repo_id, + variant, + hf_token, + cache_path = None, + ): + calls.append((repo_id, variant, hf_token, cache_path)) + return {"status": "deleted", "repo_id": repo_id} + + monkeypatch.setattr( + "hub.services.models.deletion.delete_cached_model_response", + shared, + ) + + result = asyncio.run( + models_route.delete_cached_model( + repo_id = "org/repo", + variant = None, + cache_path = "/data/hf/hub", + hf_token = "token", + current_subject = "test-user", + ) + ) + + assert result == {"status": "deleted", "repo_id": "org/repo"} + assert calls == [("org/repo", None, "token", "/data/hf/hub")] diff --git a/studio/backend/tests/test_change_password_policy.py b/studio/backend/tests/test_change_password_policy.py new file mode 100644 index 0000000000..c73e9ed839 --- /dev/null +++ b/studio/backend/tests/test_change_password_policy.py @@ -0,0 +1,75 @@ +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 + +import asyncio +import importlib.util +import sys +from pathlib import Path + +import pytest +from fastapi import HTTPException + +_BACKEND_ROOT = Path(__file__).resolve().parents[1] +if str(_BACKEND_ROOT) not in sys.path: + sys.path.insert(0, str(_BACKEND_ROOT)) + +from models.auth import ChangePasswordRequest # noqa: E402 + +# Load routes/auth.py directly so collection does not execute routes/__init__.py, +# which pulls in the heavy training/models/inference routers. +_route_path = _BACKEND_ROOT / "routes" / "auth.py" +_spec = importlib.util.spec_from_file_location("_change_password_route", _route_path) +assert _spec is not None and _spec.loader is not None +auth_routes = importlib.util.module_from_spec(_spec) +_spec.loader.exec_module(auth_routes) + + +@pytest.fixture +def _user(monkeypatch): + monkeypatch.setattr( + auth_routes.storage, + "get_user_and_secret", + lambda username: ("salt", "hash", "jwt-secret", False), + ) + monkeypatch.setattr( + auth_routes.hashing, + "verify_password", + lambda password, salt, pwd_hash: password == "bootstrap-pw", + ) + + +def _change(new_password): + payload = ChangePasswordRequest( + current_password = "bootstrap-pw", + new_password = new_password, + ) + return asyncio.run(auth_routes.change_password(payload, None, "unsloth")) + + +def test_rejects_whitespace_only_password(_user): + with pytest.raises(HTTPException) as excinfo: + _change(" " * 8) + assert excinfo.value.status_code == 400 + assert "spaces" in excinfo.value.detail + + +def test_rejects_tabs_and_spaces_password(_user): + with pytest.raises(HTTPException) as excinfo: + _change(" \t \t \t \t ") + assert excinfo.value.status_code == 400 + + +def test_rejects_password_containing_spaces(_user): + with pytest.raises(HTTPException) as excinfo: + _change("correct horse battery") + assert excinfo.value.status_code == 400 + assert "spaces" in excinfo.value.detail + + +def test_allows_password_without_spaces(_user, monkeypatch): + monkeypatch.setattr(auth_routes.storage, "update_password", lambda *args, **kwargs: True) + monkeypatch.setattr(auth_routes, "create_access_token", lambda subject: "at") + monkeypatch.setattr(auth_routes, "create_refresh_token", lambda subject: "rt") + token = _change("correct-horse-battery") + assert token.access_token == "at" + assert token.must_change_password is False diff --git a/studio/backend/tests/test_chat_attachments.py b/studio/backend/tests/test_chat_attachments.py new file mode 100644 index 0000000000..459587ca9e --- /dev/null +++ b/studio/backend/tests/test_chat_attachments.py @@ -0,0 +1,634 @@ +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 + +import base64 +import json +import os +import sqlite3 +import sys + +import pytest +from fastapi import HTTPException + +_backend = os.path.join(os.path.dirname(__file__), "..") +sys.path.insert(0, _backend) + +from routes import chat_history +from storage import studio_db +from utils.paths import studio_db_path + +PNG_BYTES = base64.b64decode( + "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8BQDwAEhQGAhKmMIQAAAABJRU5ErkJggg==" +) +PNG_DATA_URL = "data:image/png;base64," + base64.b64encode(PNG_BYTES).decode("ascii") + + +def _reset_studio_db(tmp_path, monkeypatch): + monkeypatch.setenv("UNSLOTH_STUDIO_HOME", str(tmp_path)) + monkeypatch.setenv("UNSLOTH_STUDIO_PROJECTS_HOME", str(tmp_path / "Projects")) + monkeypatch.setattr(studio_db, "_schema_ready", False) + + +def _thread( + thread_id: str = "thread-1", + title: str = "Test Chat", + pair_id: str | None = None, +) -> dict: + return { + "id": thread_id, + "title": title, + "modelType": "base", + "modelId": "test-model", + "pairId": pair_id, + "archived": False, + "createdAt": 1_700_000_000_000, + } + + +def _message( + message_id: str, + created_at: int = 1_700_000_000_000, + attachments = None, + thread_id: str = "thread-1", +) -> dict: + message = { + "id": message_id, + "threadId": thread_id, + "parentId": None, + "role": "user", + "content": [{"type": "text", "text": "hello"}], + "createdAt": created_at, + } + if attachments is not None: + message["attachments"] = attachments + return message + + +def _image_attachment(attachment_id: str = "att-1", name: str = "photo.png") -> dict: + return { + "id": attachment_id, + "type": "image", + "name": name, + "contentType": "image/png", + "content": [{"type": "image", "image": PNG_DATA_URL}], + "status": {"type": "complete"}, + } + + +def _seed( + tmp_path, + monkeypatch, + attachments, + message_id: str = "msg-1", +): + _reset_studio_db(tmp_path, monkeypatch) + studio_db.upsert_chat_thread(_thread()) + studio_db.upsert_chat_message(_message(message_id, attachments = attachments)) + + +def _set_raw_attachments_json(message_id: str, raw: str) -> None: + conn = sqlite3.connect(studio_db_path()) + try: + conn.execute( + "UPDATE chat_messages SET attachments_json = ? WHERE id = ?", + (raw, message_id), + ) + conn.commit() + finally: + conn.close() + + +def _raw_attachments_json(message_id: str): + conn = sqlite3.connect(studio_db_path()) + try: + row = conn.execute( + "SELECT attachments_json FROM chat_messages WHERE id = ?", + (message_id,), + ).fetchone() + return row[0] if row is not None else None + finally: + conn.close() + + +# --------------------------------------------------------------------------- +# Storage: list_chat_attachments +# --------------------------------------------------------------------------- + + +def test_list_chat_attachments_empty_db(tmp_path, monkeypatch): + _reset_studio_db(tmp_path, monkeypatch) + assert studio_db.list_chat_attachments() == [] + + +def test_list_chat_attachments_round_trip(tmp_path, monkeypatch): + _seed(tmp_path, monkeypatch, [_image_attachment()]) + records = studio_db.list_chat_attachments() + assert len(records) == 1 + record = records[0] + assert record["id"] == "att-1" + assert record["messageId"] == "msg-1" + assert record["threadId"] == "thread-1" + assert record["threadTitle"] == "Test Chat" + assert record["name"] == "photo.png" + assert record["type"] == "image" + assert record["contentType"] == "image/png" + assert record["createdAt"] == 1_700_000_000_000 + # Base64 length estimate is within padding error of the decoded size. + assert abs(record["sizeBytes"] - len(PNG_BYTES)) <= 2 + + +def test_list_chat_attachments_counts_text_utf8(tmp_path, monkeypatch): + text = "héllo wörld é世界" + attachment = { + "id": "att-txt", + "type": "document", + "name": "notes.txt", + "content": [{"type": "text", "text": text}], + } + _seed(tmp_path, monkeypatch, [attachment]) + records = studio_db.list_chat_attachments() + assert records[0]["sizeBytes"] == len(text.encode("utf-8")) + + +def test_list_chat_attachments_no_content_size_is_none(tmp_path, monkeypatch): + attachment = {"id": "att-empty", "name": "ghost.bin", "content": []} + _seed(tmp_path, monkeypatch, [attachment]) + records = studio_db.list_chat_attachments() + assert records[0]["sizeBytes"] is None + assert records[0]["name"] == "ghost.bin" + + +def test_list_chat_attachments_defaults_missing_name(tmp_path, monkeypatch): + attachment = {"id": "att-noname", "content": []} + _seed(tmp_path, monkeypatch, [attachment]) + assert studio_db.list_chat_attachments()[0]["name"] == "attachment" + + +def test_list_chat_attachments_sanitizes_structured_metadata(tmp_path, monkeypatch): + attachment = { + "id": "att-weird", + "name": {"nested": "name"}, + "type": ["image"], + "contentType": {"mime": "image/png"}, + "content": [], + } + _seed(tmp_path, monkeypatch, [attachment]) + record = studio_db.list_chat_attachments()[0] + assert record["name"] == "attachment" + assert record["type"] is None + assert record["contentType"] is None + + +def test_list_chat_attachments_skips_malformed_rows(tmp_path, monkeypatch): + _reset_studio_db(tmp_path, monkeypatch) + studio_db.upsert_chat_thread(_thread()) + for i, raw in enumerate( + [ + "not json at all", + '{"id": "att-obj"}', + "null", + "[]", + '[{"noid": true}, "just a string", 42]', + '[{"id": ""}]', + ] + ): + message_id = f"msg-bad-{i}" + studio_db.upsert_chat_message(_message(message_id)) + _set_raw_attachments_json(message_id, raw) + studio_db.upsert_chat_message(_message("msg-good", attachments = [_image_attachment("att-ok")])) + records = studio_db.list_chat_attachments() + assert [r["id"] for r in records] == ["att-ok"] + + +def test_list_chat_attachments_orders_newest_first(tmp_path, monkeypatch): + _reset_studio_db(tmp_path, monkeypatch) + studio_db.upsert_chat_thread(_thread()) + studio_db.upsert_chat_message( + _message("msg-old", 1_700_000_000_000, [_image_attachment("att-old")]) + ) + studio_db.upsert_chat_message( + _message("msg-new", 1_700_000_100_000, [_image_attachment("att-new")]) + ) + assert [r["id"] for r in studio_db.list_chat_attachments()] == ["att-new", "att-old"] + + +def test_list_chat_attachments_survives_missing_thread_row(tmp_path, monkeypatch): + _reset_studio_db(tmp_path, monkeypatch) + studio_db.upsert_chat_thread(_thread()) + studio_db.upsert_chat_message(_message("msg-1", attachments = [_image_attachment()])) + conn = sqlite3.connect(studio_db_path()) + try: + conn.execute("DELETE FROM chat_threads WHERE id = 'thread-1'") + conn.commit() + finally: + conn.close() + records = studio_db.list_chat_attachments() + assert len(records) == 1 + assert records[0]["threadTitle"] is None + + +def test_list_chat_attachments_includes_compare_pair_id(tmp_path, monkeypatch): + _reset_studio_db(tmp_path, monkeypatch) + studio_db.upsert_chat_thread(_thread(pair_id = "pair-1")) + studio_db.upsert_chat_message(_message("msg-compare", attachments = [_image_attachment()])) + record = studio_db.list_chat_attachments()[0] + assert record["threadId"] == "thread-1" + assert record["pairId"] == "pair-1" + + +def test_list_chat_attachments_gone_after_thread_delete(tmp_path, monkeypatch): + _seed(tmp_path, monkeypatch, [_image_attachment()]) + studio_db.delete_chat_threads(["thread-1"]) + assert studio_db.list_chat_attachments() == [] + + +# --------------------------------------------------------------------------- +# Storage: get_chat_attachment / delete_chat_attachment +# --------------------------------------------------------------------------- + + +def test_get_chat_attachment_found_and_missing(tmp_path, monkeypatch): + _seed(tmp_path, monkeypatch, [_image_attachment()]) + attachment = studio_db.get_chat_attachment("msg-1", "att-1") + assert attachment is not None + assert attachment["content"][0]["image"] == PNG_DATA_URL + assert studio_db.get_chat_attachment("msg-1", "att-missing") is None + assert studio_db.get_chat_attachment("msg-missing", "att-1") is None + + +def test_delete_chat_attachment_keeps_others(tmp_path, monkeypatch): + _seed( + tmp_path, + monkeypatch, + [_image_attachment("att-1"), _image_attachment("att-2", "other.png")], + ) + assert studio_db.delete_chat_attachment("msg-1", "att-1") is True + assert studio_db.get_chat_attachment("msg-1", "att-1") is None + assert studio_db.get_chat_attachment("msg-1", "att-2") is not None + assert [r["id"] for r in studio_db.list_chat_attachments()] == ["att-2"] + + +def test_delete_last_chat_attachment_stores_empty_list(tmp_path, monkeypatch): + _seed(tmp_path, monkeypatch, [_image_attachment()]) + assert studio_db.delete_chat_attachment("msg-1", "att-1") is True + # '[]' rather than NULL: a NULL attachments field reads back as missing + # and triggers the legacy IndexedDB backfill, resurrecting the deleted + # attachment on the next chat load. + assert _raw_attachments_json("msg-1") == "[]" + assert studio_db.list_chat_attachments() == [] + # The message itself must survive with its content intact. + message = studio_db.get_chat_message("thread-1", "msg-1") + assert message is not None + assert message["content"] == [{"type": "text", "text": "hello"}] + assert message["attachments"] == [] + + +def test_delete_chat_attachment_missing_targets(tmp_path, monkeypatch): + _seed(tmp_path, monkeypatch, [_image_attachment()]) + assert studio_db.delete_chat_attachment("msg-missing", "att-1") is False + assert studio_db.delete_chat_attachment("msg-1", "att-missing") is False + _set_raw_attachments_json("msg-1", "not json") + assert studio_db.delete_chat_attachment("msg-1", "att-1") is False + + +# --------------------------------------------------------------------------- +# Routes: /attachments endpoints (real storage, direct calls) +# --------------------------------------------------------------------------- + + +def test_list_attachments_route(tmp_path, monkeypatch): + _seed(tmp_path, monkeypatch, [_image_attachment()]) + result = chat_history.list_attachments(current_subject = "unsloth") + assert [a["id"] for a in result["attachments"]] == ["att-1"] + + +def test_attachment_file_serves_image_bytes(tmp_path, monkeypatch): + _seed(tmp_path, monkeypatch, [_image_attachment()]) + response = chat_history.get_attachment_file("msg-1", "att-1", current_subject = "unsloth") + assert response.body == PNG_BYTES + assert response.media_type == "image/png" + + +def test_attachment_file_tolerates_whitespace_in_base64(tmp_path, monkeypatch): + encoded = base64.b64encode(PNG_BYTES).decode("ascii") + wrapped = "\n".join(encoded[i : i + 8] for i in range(0, len(encoded), 8)) + attachment = _image_attachment() + attachment["content"] = [{"type": "image", "image": "data:image/png;base64," + wrapped}] + _seed(tmp_path, monkeypatch, [attachment]) + response = chat_history.get_attachment_file("msg-1", "att-1", current_subject = "unsloth") + assert response.body == PNG_BYTES + + +def test_attachment_file_corrupt_base64_is_422(tmp_path, monkeypatch): + attachment = _image_attachment() + attachment["content"] = [{"type": "image", "image": "data:image/png;base64,%%%"}] + _seed(tmp_path, monkeypatch, [attachment]) + with pytest.raises(HTTPException) as excinfo: + chat_history.get_attachment_file("msg-1", "att-1", current_subject = "unsloth") + assert excinfo.value.status_code == 422 + + +def test_attachment_file_accepts_urlsafe_base64(tmp_path, monkeypatch): + data = bytes(range(251, 256)) * 3 # encodes to characters remapped by urlsafe + payload = base64.urlsafe_b64encode(data).decode("ascii") + assert "-" in payload or "_" in payload + attachment = _image_attachment() + attachment["content"] = [{"type": "image", "image": "data:image/png;base64," + payload}] + _seed(tmp_path, monkeypatch, [attachment]) + response = chat_history.get_attachment_file("msg-1", "att-1", current_subject = "unsloth") + assert response.body == data + + +def test_attachment_file_accepts_missing_padding(tmp_path, monkeypatch): + payload = base64.b64encode(PNG_BYTES).decode("ascii").rstrip("=") + attachment = _image_attachment() + attachment["content"] = [{"type": "image", "image": "data:image/png;base64," + payload}] + _seed(tmp_path, monkeypatch, [attachment]) + response = chat_history.get_attachment_file("msg-1", "att-1", current_subject = "unsloth") + assert response.body == PNG_BYTES + + +def test_attachment_file_serves_percent_encoded_data_url(tmp_path, monkeypatch): + attachment = _image_attachment() + attachment["content"] = [{"type": "image", "image": "data:text/plain,hello%20world"}] + _seed(tmp_path, monkeypatch, [attachment]) + response = chat_history.get_attachment_file("msg-1", "att-1", current_subject = "unsloth") + assert response.body == b"hello world" + # Non-image data URL types are clamped so markup never renders same-origin. + assert response.media_type == "application/octet-stream" + + +def test_attachment_file_serves_text_parts(tmp_path, monkeypatch): + attachment = { + "id": "att-txt", + "type": "document", + "name": "notes.txt", + "content": [ + {"type": "text", "text": "first"}, + {"type": "text", "text": "second"}, + ], + } + _seed(tmp_path, monkeypatch, [attachment]) + response = chat_history.get_attachment_file("msg-1", "att-txt", current_subject = "unsloth") + assert response.body.decode("utf-8") == "first\nsecond" + assert response.media_type.startswith("text/plain") + + +def test_attachment_file_no_content_is_404(tmp_path, monkeypatch): + _seed(tmp_path, monkeypatch, [{"id": "att-empty", "name": "ghost", "content": []}]) + with pytest.raises(HTTPException) as excinfo: + chat_history.get_attachment_file("msg-1", "att-empty", current_subject = "unsloth") + assert excinfo.value.status_code == 404 + + +def test_attachment_file_missing_message_is_404(tmp_path, monkeypatch): + _reset_studio_db(tmp_path, monkeypatch) + with pytest.raises(HTTPException) as excinfo: + chat_history.get_attachment_file("nope", "att-1", current_subject = "unsloth") + assert excinfo.value.status_code == 404 + + +def test_attachment_file_non_data_url_image_is_404(tmp_path, monkeypatch): + attachment = _image_attachment() + attachment["content"] = [{"type": "image", "image": "https://example.com/a.png"}] + _seed(tmp_path, monkeypatch, [attachment]) + with pytest.raises(HTTPException) as excinfo: + chat_history.get_attachment_file("msg-1", "att-1", current_subject = "unsloth") + assert excinfo.value.status_code == 404 + + +def test_attachment_file_defaults_media_type(tmp_path, monkeypatch): + payload = base64.b64encode(b"raw-bytes").decode("ascii") + attachment = _image_attachment() + attachment["content"] = [{"type": "image", "image": "data:;base64," + payload}] + _seed(tmp_path, monkeypatch, [attachment]) + response = chat_history.get_attachment_file("msg-1", "att-1", current_subject = "unsloth") + assert response.body == b"raw-bytes" + assert response.media_type == "application/octet-stream" + + +def test_attachment_file_svg_media_type(tmp_path, monkeypatch): + svg = b"" + payload = base64.b64encode(svg).decode("ascii") + attachment = _image_attachment() + attachment["content"] = [{"type": "image", "image": "data:image/svg+xml;base64," + payload}] + _seed(tmp_path, monkeypatch, [attachment]) + response = chat_history.get_attachment_file("msg-1", "att-1", current_subject = "unsloth") + assert response.body == svg + # SVG can carry scripts, so it downloads as bytes instead of rendering. + assert response.media_type == "application/octet-stream" + + +def test_delete_attachment_route_then_404(tmp_path, monkeypatch): + _seed(tmp_path, monkeypatch, [_image_attachment()]) + result = chat_history.delete_attachment("msg-1", "att-1", current_subject = "unsloth") + assert result == {"ok": True} + with pytest.raises(HTTPException) as excinfo: + chat_history.delete_attachment("msg-1", "att-1", current_subject = "unsloth") + assert excinfo.value.status_code == 404 + + +# --------------------------------------------------------------------------- +# Audio attachments (adapter {data, format} and compare-chat bare base64) +# --------------------------------------------------------------------------- + +WAV_BYTES = b"RIFF$\x00\x00\x00WAVEfmt \x10\x00\x00\x00\x01\x00\x01\x00" +WAV_B64 = base64.b64encode(WAV_BYTES).decode("ascii") + + +def _audio_attachment(attachment_id: str = "att-audio") -> dict: + return { + "id": attachment_id, + "type": "file", + "name": "clip.wav", + "contentType": "audio/wav", + "content": [{"type": "audio", "audio": {"data": WAV_B64, "format": "wav"}}], + "status": {"type": "complete"}, + } + + +def test_audio_attachment_lists_with_size(tmp_path, monkeypatch): + _seed(tmp_path, monkeypatch, [_audio_attachment()]) + records = studio_db.list_chat_attachments() + assert len(records) == 1 + assert records[0]["id"] == "att-audio" + assert abs(records[0]["sizeBytes"] - len(WAV_BYTES)) <= 2 + + +def test_audio_attachment_file_serves_bytes(tmp_path, monkeypatch): + _seed(tmp_path, monkeypatch, [_audio_attachment()]) + response = chat_history.get_attachment_file("msg-1", "att-audio", current_subject = "unsloth") + assert response.body == WAV_BYTES + assert response.media_type == "audio/wav" + + +def test_audio_attachment_media_type_from_format(tmp_path, monkeypatch): + attachment = _audio_attachment() + attachment["contentType"] = None + attachment["content"] = [{"type": "audio", "audio": {"data": WAV_B64, "format": "mp3"}}] + _seed(tmp_path, monkeypatch, [attachment]) + response = chat_history.get_attachment_file("msg-1", "att-audio", current_subject = "unsloth") + assert response.media_type == "audio/mpeg" + + +def test_audio_attachment_corrupt_payload_is_422(tmp_path, monkeypatch): + attachment = _audio_attachment() + attachment["content"] = [{"type": "audio", "audio": {"data": "%%%", "format": "wav"}}] + _seed(tmp_path, monkeypatch, [attachment]) + with pytest.raises(HTTPException) as excinfo: + chat_history.get_attachment_file("msg-1", "att-audio", current_subject = "unsloth") + assert excinfo.value.status_code == 422 + + +# --------------------------------------------------------------------------- +# Compare-chat uploads stored as message content parts +# --------------------------------------------------------------------------- + + +def _compare_message(message_id: str = "msg-cmp") -> dict: + return { + "id": message_id, + "threadId": "thread-1", + "parentId": None, + "role": "user", + "content": [ + {"type": "image", "image": PNG_DATA_URL}, + {"type": "audio", "audio": WAV_B64}, + {"type": "text", "text": "compare these"}, + ], + "createdAt": 1_700_000_000_000, + } + + +def _seed_compare(tmp_path, monkeypatch): + _reset_studio_db(tmp_path, monkeypatch) + studio_db.upsert_chat_thread(_thread()) + studio_db.upsert_chat_message(_compare_message()) + + +_CONTENT_PART_PREFIX = "content-part-sha256-" + + +def _content_part_id_for(message_id: str, kind: str) -> str: + """Resolve the stable content-hash id for a message's stored blob. + + Content-part ids are SHA-256 hashes of the blob payload, not array + indices, so tests look them up from the listing instead of hardcoding an + index that would shift when an earlier part is deleted. + """ + for record in studio_db.list_chat_attachments(): + if record["messageId"] == message_id and record["type"] == kind: + return record["id"] + raise AssertionError(f"no {kind} content-part upload for {message_id}") + + +def test_content_part_uploads_are_listed(tmp_path, monkeypatch): + _seed_compare(tmp_path, monkeypatch) + records = studio_db.list_chat_attachments() + # Ids are stable content hashes, not array indices. + assert all(r["id"].startswith(_CONTENT_PART_PREFIX) for r in records) + assert {r["type"] for r in records} == {"image", "audio"} + image = next(r for r in records if r["type"] == "image") + assert image["contentType"] == "image/png" + assert abs(image["sizeBytes"] - len(PNG_BYTES)) <= 2 + audio = next(r for r in records if r["type"] == "audio") + assert audio["type"] == "audio" + + +def test_content_part_file_serves_image_bytes(tmp_path, monkeypatch): + _seed_compare(tmp_path, monkeypatch) + image_id = _content_part_id_for("msg-cmp", "image") + response = chat_history.get_attachment_file("msg-cmp", image_id, current_subject = "unsloth") + assert response.body == PNG_BYTES + assert response.media_type == "image/png" + + +def test_content_part_delete_keeps_text(tmp_path, monkeypatch): + _seed_compare(tmp_path, monkeypatch) + image_id = _content_part_id_for("msg-cmp", "image") + assert studio_db.delete_chat_attachment("msg-cmp", image_id) is True + message = studio_db.get_chat_message("thread-1", "msg-cmp") + types = [p["type"] for p in message["content"]] + assert types == ["audio", "text"] + # The surviving audio blob keeps its own stable hash id after the delete. + remaining = studio_db.list_chat_attachments() + assert [r["type"] for r in remaining] == ["audio"] + assert remaining[0]["id"].startswith(_CONTENT_PART_PREFIX) + assert remaining[0]["id"] != image_id + + +def test_content_part_delete_rejects_non_blob(tmp_path, monkeypatch): + _seed_compare(tmp_path, monkeypatch) + # The text part is not a stored upload, so it never gets an id: only the + # image and audio blobs are addressable. + assert len(studio_db.list_chat_attachments()) == 2 + # A well-formed but unknown content-hash id, and malformed ids, all no-op. + assert studio_db.delete_chat_attachment("msg-cmp", _CONTENT_PART_PREFIX + "0" * 64) is False + assert studio_db.delete_chat_attachment("msg-cmp", "content-part-99") is False + assert studio_db.delete_chat_attachment("msg-cmp", "content-part-x") is False + + +def test_text_only_messages_not_listed_as_uploads(tmp_path, monkeypatch): + _reset_studio_db(tmp_path, monkeypatch) + studio_db.upsert_chat_thread(_thread()) + # The word "image" inside text must not create phantom upload rows. + message = _message("msg-txt") + message["content"] = [{"type": "text", "text": 'discussing an "image" and "audio" here'}] + studio_db.upsert_chat_message(message) + assert studio_db.list_chat_attachments() == [] + + +def test_remote_image_urls_are_not_listed_as_uploads(tmp_path, monkeypatch): + _reset_studio_db(tmp_path, monkeypatch) + studio_db.upsert_chat_thread(_thread()) + message = _message("msg-remote") + message["content"] = [ + {"type": "image", "image": "https://example.com/cat.png"}, + {"type": "text", "text": "look at this"}, + ] + studio_db.upsert_chat_message(message) + # No stored bytes: nothing to list, open, or delete. + assert studio_db.list_chat_attachments() == [] + assert studio_db.get_chat_attachment("msg-remote", "content-part-0") is None + assert studio_db.delete_chat_attachment("msg-remote", "content-part-0") is False + stored = studio_db.get_chat_message("thread-1", "msg-remote") + assert [p["type"] for p in stored["content"]] == ["image", "text"] + + +def test_html_data_url_serves_as_octet_stream(tmp_path, monkeypatch): + _reset_studio_db(tmp_path, monkeypatch) + studio_db.upsert_chat_thread(_thread()) + html_b64 = base64.b64encode(b"").decode() + message = _message("msg-html") + message["content"] = [ + {"type": "image", "image": f"data:text/html;base64,{html_b64}"}, + ] + studio_db.upsert_chat_message(message) + attachment_id = _content_part_id_for("msg-html", "image") + response = chat_history.get_attachment_file( + "msg-html", attachment_id, current_subject = "unsloth" + ) + # Never echo a script-capable media type back under the app origin. + assert response.media_type == "application/octet-stream" + assert response.body == b"" + + +def test_svg_data_url_serves_as_octet_stream(tmp_path, monkeypatch): + _reset_studio_db(tmp_path, monkeypatch) + studio_db.upsert_chat_thread(_thread()) + svg_b64 = base64.b64encode(b"").decode() + message = _message("msg-svg") + message["content"] = [ + {"type": "image", "image": f"data:image/svg+xml;base64,{svg_b64}"}, + ] + studio_db.upsert_chat_message(message) + attachment_id = _content_part_id_for("msg-svg", "image") + response = chat_history.get_attachment_file("msg-svg", attachment_id, current_subject = "unsloth") + assert response.media_type == "application/octet-stream" + + +def test_png_data_url_keeps_its_media_type(tmp_path, monkeypatch): + _seed_compare(tmp_path, monkeypatch) + image_id = _content_part_id_for("msg-cmp", "image") + response = chat_history.get_attachment_file("msg-cmp", image_id, current_subject = "unsloth") + assert response.media_type == "image/png" diff --git a/studio/backend/tests/test_chat_history_routes.py b/studio/backend/tests/test_chat_history_routes.py index a60ac700bf..896bf1a6cd 100644 --- a/studio/backend/tests/test_chat_history_routes.py +++ b/studio/backend/tests/test_chat_history_routes.py @@ -91,6 +91,28 @@ def test_chat_settings_payload_accepts_fast_mode_presets(): assert dumped["customPresets"][0]["params"]["fastMode"] is True +def test_chat_settings_payload_accepts_preset_load_config(): + payload = chat_history.ChatSettingsPayload.model_validate( + { + "customPresets": [ + { + "name": "GGUF preset", + "params": {"temperature": 0.7, "maxTokens": 512}, + "loadConfig": { + "customContextLength": 256, + "kvCacheDtype": "q8_0", + "tensorParallel": False, + }, + }, + ], + } + ) + + dumped = payload.model_dump(exclude_unset = True) + assert dumped["customPresets"][0]["loadConfig"]["customContextLength"] == 256 + assert dumped["customPresets"][0]["loadConfig"]["kvCacheDtype"] == "q8_0" + + def test_chat_settings_payload_accepts_nudge_tool_calls(): # extra="forbid" 400s PUT /api/chat/settings on unknown keys, so the # frontend's persisted nudgeToolCalls needs a payload field (like diff --git a/studio/backend/tests/test_chat_load_during_training.py b/studio/backend/tests/test_chat_load_during_training.py index 63dba8579c..f1d973f004 100644 --- a/studio/backend/tests/test_chat_load_during_training.py +++ b/studio/backend/tests/test_chat_load_during_training.py @@ -168,11 +168,15 @@ class TestCanLoadGGUF(_GpuCacheResetMixin, unittest.TestCase): devices, required_override = None, estimate = None, + single_device_gpu = None, + gpu_ids = None, + is_vulkan = False, ): with ( patch("utils.hardware.get_device", return_value = DeviceType.CUDA), patch("utils.hardware.estimate_required_model_memory_gb", return_value = (estimate, {})), patch("utils.hardware.get_visible_gpu_utilization", return_value = {"devices": devices}), + patch("utils.hardware.resolve_requested_gpu_ids", return_value = gpu_ids), patch("utils.hardware.auto_select_gpu_ids") as auto_mock, ): ok, info = tv.can_load_chat_during_training( @@ -180,9 +184,11 @@ class TestCanLoadGGUF(_GpuCacheResetMixin, unittest.TestCase): hf_token = None, load_in_4bit = True, max_seq_length = 0, - requested_gpu_ids = None, + requested_gpu_ids = gpu_ids, is_gguf = True, + is_vulkan = is_vulkan, required_override_gb = required_override, + single_device_gpu = single_device_gpu, ) return ok, info, auto_mock @@ -198,6 +204,117 @@ class TestCanLoadGGUF(_GpuCacheResetMixin, unittest.TestCase): ok, _, _ = self._run(devices = _devices((0, 80, 35), (1, 80, 70)), required_override = 20.0) self.assertTrue(ok) + def test_no_per_gpu_floor_for_gguf_with_explicit_gpu_ids(self): + # gpu_ids narrows llama.cpp's candidate pool but does not turn its + # self-placement into HF device_map="balanced". The uneven selected + # pair therefore keeps the aggregate GGUF check without an even-share + # floor on the nearly-full card. + ok, info, _ = self._run( + devices = _devices((0, 80, 35), (1, 80, 70), (2, 80, 0)), + required_override = 20.0, + gpu_ids = [0, 1], + ) + self.assertTrue(ok) + self.assertEqual(info["mode"], "gguf") + + def test_single_device_uses_selected_gpu(self): + # The model needs 27 GB with headroom. GPU 0 has 45 GB free, while an + # unrelated training-heavy GPU 1 has only 10 GB free. + ok, info, _ = self._run( + devices = _devices((0, 80, 35), (1, 80, 70)), + required_override = 20.0, + single_device_gpu = "0", + ) + self.assertTrue(ok) + self.assertEqual(info["usable_gb"], 45.0) + + blocked, blocked_info, _ = self._run( + devices = _devices((0, 80, 35), (1, 80, 70)), + required_override = 20.0, + single_device_gpu = "1", + ) + self.assertFalse(blocked) + self.assertEqual(blocked_info["usable_gb"], 10.0) + + def test_vulkan_pin_takes_precedence_over_unknown_diffusion_fallback(self): + # An uncached GGUF can carry a speculative single-device fallback while + # its explicit pin is actually a ggml Vulkan ordinal. Never interpret + # that ordinal as the same-numbered CUDA physical device. + ok, info, _ = self._run( + devices = _devices((0, 80, 0), (1, 80, 78)), + required_override = 20.0, + single_device_gpu = "0", + gpu_ids = [0], + is_vulkan = True, + ) + self.assertFalse(ok) + self.assertEqual(info["mode"], "gguf_vulkan") + self.assertEqual(info["usable_gb"], 2.0) + + def test_vulkan_multi_gpu_guard_counts_requested_devices(self): + # The ordinal mapping is unknown, so use the least-free two visible + # cards for a two-device request. Their aggregate capacity is still + # available instead of collapsing the request to one card. + ok, info, _ = self._run( + devices = _devices((0, 80, 70), (1, 80, 70), (2, 80, 0)), + required_override = 10.0, + gpu_ids = [0, 1], + is_vulkan = True, + ) + self.assertTrue(ok) + self.assertEqual(info["mode"], "gguf_vulkan") + self.assertEqual(info["usable_gb"], 18.5) + + def test_single_device_unresolved_token_sizes_against_worst_device(self): + # A non-numeric device token (a CUDA UUID / MIG handle) can't map to a + # free-VRAM index. The runner still drives ONE device, so size against the + # worst-case visible device (min free), not the aggregate pool: one GPU + # with 80 GB free vs a 20 GB model -> allow. + ok, info, _ = self._run( + devices = _devices((0, 80, 0)), + required_override = 20.0, + single_device_gpu = "GPU-uuid", + ) + self.assertTrue(ok) + self.assertEqual(info["mode"], "single_device") + self.assertNotIn("reason", info) + + def test_single_device_unresolved_token_refuses_when_worst_device_full(self): + # Same UUID fallback, worst-case device nearly full (2 GB for a 20 GB + # model) -> refuse (default-deny), not on an unresolved-token technicality. + ok, info, _ = self._run( + devices = _devices((0, 80, 78)), + required_override = 20.0, + single_device_gpu = "GPU-uuid", + ) + self.assertFalse(ok) + self.assertNotEqual(info.get("reason"), "unresolved_gpu_id") + + def test_single_device_unresolved_token_uses_min_free_not_aggregate(self): + # The single-device runner uses ONE device but we can't tell which from a + # UUID token. Sizing against the aggregate pool would let a 20 GB model + # "fit" 160 GB of pooled free VRAM while landing on a 2 GB card and OOMing + # training. Min-free (2 GB) is the safe worst case -> refuse. + ok, info, _ = self._run( + devices = _devices((0, 80, 78), (1, 80, 0), (2, 80, 0)), + required_override = 20.0, + single_device_gpu = "GPU-uuid", + ) + self.assertFalse(ok) + self.assertEqual(info["mode"], "single_device") + + def test_single_device_cpu_token_allows(self): + # An empty device token = a CPU-only single-device runner (CPU diffusion + # GGUF): it uses no GPU VRAM, so it never threatens training -> allow + # regardless of how full the GPUs are. + ok, info, _ = self._run( + devices = _devices((0, 80, 78)), + required_override = 20.0, + single_device_gpu = "", + ) + self.assertTrue(ok) + self.assertEqual(info["reason"], "cpu_only") + def test_estimate_unavailable_refuses(self): # No override and the estimator can't size it -> default-deny. ok, info, _ = self._run(devices = _devices((0, 80, 0)), required_override = None, estimate = None) @@ -209,7 +326,7 @@ class TestCanLoadGGUF(_GpuCacheResetMixin, unittest.TestCase): class TestCanLoadMisc(_GpuCacheResetMixin, unittest.TestCase): - def test_non_cuda_allows(self): + def test_non_accelerator_allows(self): with patch("utils.hardware.get_device", return_value = DeviceType.MLX): ok, info = tv.can_load_chat_during_training( model_name = "m", @@ -219,7 +336,30 @@ class TestCanLoadMisc(_GpuCacheResetMixin, unittest.TestCase): requested_gpu_ids = None, ) self.assertTrue(ok) - self.assertEqual(info["mode"], "non_cuda") + self.assertEqual(info["mode"], "non_accelerator") + + def test_xpu_overcommit_is_refused(self): + # XPU must NOT get the blanket non-accelerator allow: an oversized + # chat model during resident training is refused, like CUDA. + with ( + patch("utils.hardware.get_device", return_value = DeviceType.XPU), + patch( + "utils.hardware.auto_select_gpu_ids", + return_value = ( + None, + {"selection_mode": "auto", "required_gb": 50.0, "usable_gb": 4.0}, + ), + ), + ): + ok, info = tv.can_load_chat_during_training( + model_name = "m", + hf_token = None, + load_in_4bit = True, + max_seq_length = 0, + requested_gpu_ids = None, + ) + self.assertFalse(ok) + self.assertNotEqual(info.get("mode"), "non_accelerator") def test_no_visible_gpus_refuses(self): # GGUF with an empty device list -> no candidate GPU -> default-deny. @@ -309,6 +449,8 @@ class TestChatLoadGuardRoute(unittest.TestCase): captured = None, training_active, decision, + gpu_memory_mode = "auto", + requested_gpu_ids = None, ): config = config or SimpleNamespace(is_gguf = False, is_lora = False, path = None) with _stub_guard_deps( @@ -320,7 +462,8 @@ class TestChatLoadGuardRoute(unittest.TestCase): hf_token = None, load_in_4bit = True, max_seq_length = 0, - requested_gpu_ids = None, + requested_gpu_ids = requested_gpu_ids, + gpu_memory_mode = gpu_memory_mode, ) def test_noop_when_training_inactive(self): @@ -332,6 +475,99 @@ class TestChatLoadGuardRoute(unittest.TestCase): def test_allows_when_fits(self): self._guard(training_active = True, decision = (True, {"mode": "auto"})) + def test_diffusion_detection_uses_name_before_download(self): + config = SimpleNamespace( + identifier = "unsloth/DiffusionGemma-GGUF", + gguf_hf_repo = "unsloth/DiffusionGemma-GGUF", + gguf_file = None, + ) + self.assertTrue(self.route._classify_diffusion_gguf(config)) + + def test_uncached_gguf_classification_remains_unknown(self): + config = SimpleNamespace( + identifier = "owner/renamed-model", + gguf_hf_repo = "owner/renamed-model", + gguf_variant = "Q4_K_M", + gguf_file = None, + ) + self.assertIsNone(self.route._classify_diffusion_gguf(config)) + + def test_diffusion_detection_reuses_loader_metadata_probe(self): + import tempfile + + seen = [] + + class _Probe: + is_diffusion = False + _architecture = None + + def _read_gguf_metadata(self, path): + seen.append(path) + self.is_diffusion = True + + with tempfile.TemporaryDirectory() as d: + model = Path(d) / "renamed.gguf" + model.write_bytes(b"GGUF") + config = SimpleNamespace(identifier = "local", gguf_file = str(model)) + with patch.object(self.route, "LlamaCppBackend", _Probe): + self.assertTrue(self.route._classify_diffusion_gguf(config)) + self.assertEqual(seen, [str(model)]) + + def test_local_chat_gguf_classification_is_definitive(self): + import tempfile + class _Probe: + is_diffusion = False + _architecture = "llama" + + def _read_gguf_metadata(self, _path): + pass + + with tempfile.TemporaryDirectory() as d: + model = Path(d) / "renamed.gguf" + model.write_bytes(b"GGUF") + config = SimpleNamespace(identifier = "local", gguf_file = str(model)) + with patch.object(self.route, "LlamaCppBackend", _Probe): + self.assertFalse(self.route._classify_diffusion_gguf(config)) + + def test_manual_known_normal_gguf_bypasses_training_estimate(self): + captured = [] + config = SimpleNamespace(is_gguf = True) + with patch.object(self.route, "_classify_diffusion_gguf", return_value = False) as classify: + self._guard( + config = config, + captured = captured, + training_active = True, + decision = (False, {"reason": "must not run"}), + gpu_memory_mode = "manual", + requested_gpu_ids = [1, 3], + ) + classify.assert_called_once_with(config) + self.assertEqual(captured, []) + + def test_manual_diffusion_keeps_single_device_training_guard(self): + captured = [] + config = SimpleNamespace(is_gguf = True) + with ( + patch.object(self.route, "_classify_diffusion_gguf", return_value = True), + patch.object(self.route, "_estimate_gguf_required_gb", return_value = 12.5), + patch.object( + self.route.LlamaCppBackend, + "_effective_gpu_count", + return_value = 2, + ), + ): + self._guard( + config = config, + captured = captured, + training_active = True, + decision = (True, {"mode": "single_device"}), + gpu_memory_mode = "manual", + requested_gpu_ids = [3, 1], + ) + self.assertEqual(len(captured), 1) + self.assertEqual(captured[0]["single_device_gpu"], "1") + self.assertEqual(captured[0]["requested_gpu_ids"], [3, 1]) + def test_refuses_with_headroom_number(self): info = {"required_gb": 30.0, "usable_gb": 6.0, "needed_gb": 39.0, "mode": "auto"} with self.assertRaises(HTTPException) as exc: @@ -467,36 +703,189 @@ class TestValidateRefusesDuringTraining(unittest.TestCase): self.assertEqual(captured[0]["load_in_4bit"], False) self.assertEqual(captured[0]["max_seq_length"], 4096) - def test_rejects_gguf_with_gpu_ids_before_guard(self): - # /validate must mirror /load's GGUF + gpu_ids 400, before the VRAM guard. + def test_validate_forwards_manual_gpu_memory_mode_to_guard(self): from models.inference import ValidateModelRequest - request = ValidateModelRequest(model_path = "x.gguf", gpu_ids = [0]) + request = ValidateModelRequest( + model_path = "unsloth/model-GGUF", + gguf_variant = "Q4_K_M", + gpu_memory_mode = "manual", + ) cfg = SimpleNamespace( - identifier = "x.gguf", - display_name = "x", + identifier = "unsloth/model-GGUF", + display_name = "model-GGUF", is_gguf = True, is_lora = False, is_vision = False, path = None, base_model = None, ) - captured = [] + captured = {} with ( patch.object( self.route, "_resolve_model_identifier_for_request", - return_value = ("x.gguf", "x.gguf", False), + return_value = ("unsloth/model-GGUF", "unsloth/model-GGUF", False), ), patch.object(self.route.ModelConfig, "from_identifier", return_value = cfg), patch.object(self.route, "load_inference_config", return_value = {}), - _stub_guard_deps(training_active = True, decision = (True, {}), captured = captured), + patch.object( + self.route, + "_guard_chat_load_against_training", + lambda config, **kw: captured.update(kw), + ), ): - with self.assertRaises(HTTPException) as exc: - asyncio.run(self.route.validate_model(request, current_subject = "u")) - self.assertEqual(exc.exception.status_code, 400) - self.assertIn("gpu_ids is not supported for GGUF", exc.exception.detail) - self.assertEqual(captured, []) # guard never reached + asyncio.run(self.route.validate_model(request, current_subject = "u")) + self.assertEqual(captured.get("gpu_memory_mode"), "manual") + + def test_validate_forwards_inherited_extras_and_parallel_to_guard(self): + # Regression: /load resolves inherited same-model extras and passes the + # real slot count to the guard; validate must do the same, else it sizes + # a smaller estimate (no inherited -c/--model-draft, n_parallel=1) and + # /load then 409s after the frontend has already unloaded. + from models.inference import ValidateModelRequest + + request = ValidateModelRequest(model_path = "unsloth/Qwen3-1.7B", max_seq_length = 4096) + cfg = SimpleNamespace( + identifier = "unsloth/Qwen3-1.7B", + display_name = "Qwen3-1.7B", + is_gguf = False, + is_lora = False, + is_vision = False, + path = None, + base_model = None, + ) + captured = {} + with ( + patch.object( + self.route, + "_resolve_model_identifier_for_request", + return_value = ("unsloth/Qwen3-1.7B", "unsloth/Qwen3-1.7B", False), + ), + patch.object(self.route.ModelConfig, "from_identifier", return_value = cfg), + patch.object(self.route, "load_inference_config", return_value = {}), + patch.object(self.route, "_resolve_inherited_extra_args", return_value = ["-c", "32768"]), + patch.object( + self.route, + "_guard_chat_load_against_training", + lambda config, **kw: captured.update(kw), + ), + ): + asyncio.run(self.route.validate_model(request, current_subject = "u")) + self.assertEqual(captured.get("llama_extra_args"), ["-c", "32768"]) + self.assertIn("n_parallel", captured) + + def test_metadata_probe_skips_training_guard(self): + # A header-only probe (include_context_length) allocates no VRAM, so the + # training guard must not run -- else the staging GPU-layers / MoE sliders + # it feeds are hidden exactly when a during-training user needs them. + from models.inference import ValidateModelRequest + + request = ValidateModelRequest( + model_path = "unsloth/Qwen3-1.7B", + max_seq_length = 4096, + include_context_length = True, + ) + cfg = SimpleNamespace( + identifier = "unsloth/Qwen3-1.7B", + display_name = "Qwen3-1.7B", + is_gguf = False, + is_lora = False, + is_vision = False, + path = None, + base_model = None, + ) + guard_called = [] + with ( + patch.object( + self.route, + "_resolve_model_identifier_for_request", + return_value = ("unsloth/Qwen3-1.7B", "unsloth/Qwen3-1.7B", False), + ), + patch.object(self.route.ModelConfig, "from_identifier", return_value = cfg), + patch.object(self.route, "load_inference_config", return_value = {}), + patch.object( + self.route, + "_guard_chat_load_against_training", + lambda *a, **kw: guard_called.append(True), + ), + ): + asyncio.run(self.route.validate_model(request, current_subject = "u")) + self.assertEqual(guard_called, []) + + def _validate_gguf_template( + self, + *, + template, + canonical_path = "/picked/model.gguf", + ): + # Drive validate_model for a native lease-backed GGUF template probe and + # capture what the embedded-template reader was called with. + from models.inference import ValidateModelRequest + + request = ValidateModelRequest( + model_path = "model.gguf", + gguf_variant = "Q4_K_M", + native_path_lease = "signed-lease", + include_chat_template = True, + ) + cfg = SimpleNamespace( + identifier = canonical_path, + display_name = "model.gguf", + is_gguf = True, + is_lora = False, + is_vision = False, + gguf_file = canonical_path, + path = None, + base_model = None, + ) + import utils.models.gguf_metadata as gguf_meta + + seen = {} + + def _fake_read(path): + seen["path"] = path + return template + + guard_called = [] + with ( + patch.object( + self.route, + "_resolve_model_identifier_for_request", + return_value = (canonical_path, "model.gguf", True), + ), + patch.object(self.route.ModelConfig, "from_identifier", return_value = cfg), + patch.object(self.route, "load_inference_config", return_value = {}), + patch.object(gguf_meta, "read_gguf_chat_template", _fake_read), + patch.object( + self.route, + "_guard_chat_load_against_training", + lambda *a, **kw: guard_called.append(True), + ), + ): + resp = asyncio.run(self.route.validate_model(request, current_subject = "u")) + return resp, seen, guard_called + + def test_include_chat_template_reads_leased_gguf_embedded_template(self): + # The picker chat-template GET has no lease plumbing, so a native picked + # GGUF surfaces its default template through this lease-aware probe: the + # embedded template is read from the granted canonical path and returned. + resp, seen, _ = self._validate_gguf_template(template = "{{ messages }}") + self.assertEqual(resp.chat_template, "{{ messages }}") + # Read strictly the leased file's own embedded template, never a sibling + # sidecar: the grant authorizes just this one path. + self.assertEqual(seen["path"], "/picked/model.gguf") + + def test_include_chat_template_skips_training_guard(self): + # A template-only probe allocates no VRAM, so like include_context_length + # it must not be refused by the training guard. + _, _, guard_called = self._validate_gguf_template(template = "{{ messages }}") + self.assertEqual(guard_called, []) + + def test_include_chat_template_over_cap_is_dropped(self): + from picker.schemas import MAX_CHAT_TEMPLATE_BYTES + resp, _, _ = self._validate_gguf_template(template = "a" * (MAX_CHAT_TEMPLATE_BYTES + 1)) + self.assertIsNone(resp.chat_template) # ── _estimate_gguf_required_gb (sizes the same weights the loader loads) ────── diff --git a/studio/backend/tests/test_cloudflare_tunnel.py b/studio/backend/tests/test_cloudflare_tunnel.py index bb51cabf76..2094d15066 100644 --- a/studio/backend/tests/test_cloudflare_tunnel.py +++ b/studio/backend/tests/test_cloudflare_tunnel.py @@ -403,11 +403,234 @@ def test_reader_ignores_api_endpoint_failure_line(): assert t.error == "cloudflared exited before emitting a tunnel URL" +# ── public reachability probe ──────────────────────────────────────── + + +class _FakeResponse: + def __init__(self, body): + self._body = body + + def read(self, size = -1): + return self._body + + def __enter__(self): + return self + + def __exit__(self, *exc): + return False + + +def _patch_urlopen(monkeypatch, handler): + import urllib.request + monkeypatch.setattr(urllib.request, "urlopen", lambda req, timeout = None: handler(req)) + + +@pytest.fixture(autouse = True) +def _stub_dns_wait(monkeypatch, request): + if request.node.name.startswith("test_verify_public_url"): + monkeypatch.setattr(ct, "_wait_for_dns", lambda *a, **kw: None) + + +def test_wait_for_dns_polls_until_answer(monkeypatch): + calls = [] + + def handler(req): + calls.append(req.full_url) + if len(calls) < 3: + return _FakeResponse(b'{"Status":3}') + return _FakeResponse(b'{"Status":0,"Answer":[{"data":"104.16.0.1"}]}') + + _patch_urlopen(monkeypatch, handler) + monkeypatch.setattr(ct.time, "sleep", lambda _s: None) + ct._wait_for_dns("words.trycloudflare.com", ct.time.monotonic() + 5) + assert len(calls) == 3 + assert "name=words.trycloudflare.com" in calls[0] + + +def test_wait_for_dns_gives_up_at_deadline(monkeypatch): + _patch_urlopen(monkeypatch, lambda req: _FakeResponse(b'{"Status":3}')) + monkeypatch.setattr(ct.time, "sleep", lambda _s: None) + ct._wait_for_dns("words.trycloudflare.com", ct.time.monotonic() + 0.05) + + +def test_wait_for_dns_retries_transient_doh_error(monkeypatch): + calls = [] + + def handler(req): + calls.append(req.full_url) + if len(calls) < 3: + raise OSError("transient") + return _FakeResponse(b'{"Status":0,"Answer":[{"data":"104.16.0.1"}]}') + + _patch_urlopen(monkeypatch, handler) + monkeypatch.setattr(ct.time, "sleep", lambda _s: None) + ct._wait_for_dns("words.trycloudflare.com", ct.time.monotonic() + 5) + assert len(calls) == 3 + + +def test_wait_for_dns_bails_on_persistent_doh_errors(monkeypatch): + calls = [] + + def handler(req): + calls.append(req.full_url) + raise OSError("blocked") + + _patch_urlopen(monkeypatch, handler) + monkeypatch.setattr(ct.time, "sleep", lambda _s: None) + ct._wait_for_dns("words.trycloudflare.com", ct.time.monotonic() + 5) + assert len(calls) == ct._DNS_MAX_DOH_ERRORS + + +def test_verify_public_url_accepts_studio_marker(monkeypatch): + seen = {} + + def handler(req): + seen["url"] = req.full_url + return _FakeResponse(b'{"status":"healthy","service":"Unsloth UI Backend"}') + + _patch_urlopen(monkeypatch, handler) + assert ct.verify_public_url("https://words.trycloudflare.com") is True + assert seen["url"] == "https://words.trycloudflare.com/api/health" + + +def test_verify_public_url_waits_for_dns_first(monkeypatch): + order = [] + monkeypatch.setattr(ct, "_wait_for_dns", lambda host, deadline: order.append(("dns", host))) + + def handler(req): + order.append(("probe", req.full_url)) + return _FakeResponse(b'{"service":"Unsloth UI Backend"}') + + _patch_urlopen(monkeypatch, handler) + assert ct.verify_public_url("https://words.trycloudflare.com") is True + assert order[0] == ("dns", "words.trycloudflare.com") + assert order[1][0] == "probe" + + +def test_verify_public_url_dns_wait_and_probe_share_deadline(monkeypatch): + # An exhausted DNS wait leaves the probe a single attempt, not a fresh window. + calls = [] + monkeypatch.setattr(ct, "_wait_for_dns", lambda host, deadline: None) + + def handler(req): + calls.append(req.full_url) + raise OSError("unreachable") + + _patch_urlopen(monkeypatch, handler) + assert ct.verify_public_url("https://words.trycloudflare.com", timeout = 0) is False + assert len(calls) == 1 + + +def test_verify_public_url_retries_then_succeeds(monkeypatch): + calls = [] + + def handler(req): + calls.append(req.full_url) + if len(calls) < 3: + raise OSError("Name or service not known") + return _FakeResponse(b'{"service":"Unsloth UI Backend"}') + + _patch_urlopen(monkeypatch, handler) + monkeypatch.setattr(ct.time, "sleep", lambda _s: None) + assert ct.verify_public_url("https://words.trycloudflare.com") is True + assert len(calls) == 3 + + +def test_verify_public_url_rejects_unreachable_host(monkeypatch): + def handler(req): + raise OSError("Name or service not known") + + _patch_urlopen(monkeypatch, handler) + monkeypatch.setattr(ct.time, "sleep", lambda _s: None) + assert ct.verify_public_url("https://words.trycloudflare.com", timeout = 0.05) is False + + +def test_verify_public_url_rejects_foreign_responder(monkeypatch): + # e.g. a Cloudflare error page: no service marker in the body. + _patch_urlopen(monkeypatch, lambda req: _FakeResponse(b"error 1033")) + monkeypatch.setattr(ct.time, "sleep", lambda _s: None) + assert ct.verify_public_url("https://words.trycloudflare.com", timeout = 0.05) is False + + +@pytest.fixture(autouse = True) +def _stub_public_probe(monkeypatch, request): + # start_studio_tunnel tests use fake hostnames; keep them off the network. + if not request.node.name.startswith("test_start_studio_tunnel"): + return + monkeypatch.setattr(ct, "verify_public_url", lambda url, **kw: True) + + def test_start_studio_tunnel_no_binary(monkeypatch): monkeypatch.setattr(ct, "ensure_cloudflared", lambda: None) assert ct.start_studio_tunnel(8080) is None +def test_start_studio_tunnel_drops_url_that_is_not_publicly_reachable(monkeypatch): + attempts = [] + + class _Stub: + def __init__( + self, + port, + binary, + protocol = None, + ): + self.url = None + attempts.append(protocol) + + def start(self): + self.url = "https://words.trycloudflare.com" + + def wait_for_ready(self, timeout): + return self.url + + def stop(self): + pass + + monkeypatch.setattr(ct, "ensure_cloudflared", lambda: "/bin/cloudflared") + monkeypatch.setattr(ct, "CloudflareTunnel", _Stub) + monkeypatch.setattr(ct, "verify_public_url", lambda url, **kw: False) + assert ct.start_studio_tunnel(8080) is None + assert attempts == [None] + assert ct._active_tunnel is None + + +def test_start_studio_tunnel_returns_url_once_probe_passes(monkeypatch): + probed = [] + + class _Stub: + def __init__( + self, + port, + binary, + protocol = None, + ): + self.url = None + self.protocol = protocol + + def start(self): + self.url = "https://words.trycloudflare.com" + + def wait_for_ready(self, timeout): + return self.url + + def stop(self): + pass + + def _probe(url, **kw): + probed.append(url) + return True + + monkeypatch.setattr(ct, "ensure_cloudflared", lambda: "/bin/cloudflared") + monkeypatch.setattr(ct, "CloudflareTunnel", _Stub) + monkeypatch.setattr(ct, "verify_public_url", _probe) + try: + assert ct.start_studio_tunnel(8080) == "https://words.trycloudflare.com" + assert probed == ["https://words.trycloudflare.com"] + finally: + ct.stop_studio_tunnel() + + def test_start_studio_tunnel_registers_before_wait(monkeypatch): # The tunnel must be visible to stop_studio_tunnel() during the readiness # wait, else a shutdown in that window orphans cloudflared. diff --git a/studio/backend/tests/test_colab_embed.py b/studio/backend/tests/test_colab_embed.py new file mode 100644 index 0000000000..dae0c7dae0 --- /dev/null +++ b/studio/backend/tests/test_colab_embed.py @@ -0,0 +1,479 @@ +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 + +"""Regression coverage for Colab iframe embedding (#7344).""" + +import sys +import types +from types import SimpleNamespace +from unittest.mock import MagicMock, patch + +import colab + + +def _mock_google_colab_modules(colab_mod): + """Mock ``google`` and ``google.colab`` for environments without Google packages.""" + google_mod = types.ModuleType("google") + google_mod.colab = colab_mod + return {"google": google_mod, "google.colab": colab_mod} + + +def test_short_colab_url_truncates_proxy_host(): + url = "https://8888-gpu-a100-s-kkb-usc1f0-9hzedjcxrlu8-f.us-central1-0.prod.colab.dev/" + assert colab._short_colab_url(url, 8888) == "https://8888-gpu-..." + + +def test_short_colab_url_falls_back_on_unexpected_shape(): + assert colab._short_colab_url("https://example.com", 8888) == "https://example.com" + + +def test_is_colab_proxy_url_requires_https_proxy(): + assert colab._is_colab_proxy_url("https://8888-test.prod.colab.dev/", 8888) is True + assert colab._is_colab_proxy_url("http://localhost:8888", 8888) is False + assert colab._is_colab_proxy_url("http://127.0.0.1:8888", 8888) is False + + +def test_ready_card_html_does_not_open_colab_proxy_in_new_tab(): + """Colab proxy hosts 404 as top-level tabs (#7349 reporter); never window.open them.""" + html = colab._ready_card_html("https://8888-test.prod.colab.dev/", 8888) + assert "window.open" not in html + assert 'href="https://8888-test.prod.colab.dev/"' not in html + assert "start(cloudflare=True)" in html + + +def test_ready_card_html_points_to_cloudflare_when_link_ready(monkeypatch): + monkeypatch.setattr(colab, "_is_colab_runtime", lambda: True) + html = colab._ready_card_html( + "https://8888-test.prod.colab.dev/", + 8888, + has_cloudflare_link = True, + ) + assert "Cloudflare link above" in html + + +def test_ready_card_html_warns_when_cloudflare_tunnel_missing(monkeypatch): + monkeypatch.setattr(colab, "_is_colab_runtime", lambda: True) + html = colab._ready_card_html( + "https://8888-test.prod.colab.dev/", + 8888, + cloudflare_requested = True, + ) + assert "Could not open a Cloudflare tunnel" in html + + +def test_warn_colab_cloudflare_missing_logs_on_colab_without_tunnel(monkeypatch): + warnings: list[str] = [] + monkeypatch.setattr(colab, "_is_colab_runtime", lambda: True) + monkeypatch.setattr(colab.logger, "warning", lambda msg, **kwargs: warnings.append(msg)) + colab._warn_colab_cloudflare_missing(use_cloudflare = True, cloudflare_url = None) + assert warnings + assert "Cloudflare tunnel unavailable" in warnings[0] + + +def test_warn_colab_cloudflare_missing_skips_when_tunnel_ready(monkeypatch, caplog): + import logging + + monkeypatch.setattr(colab, "_is_colab_runtime", lambda: True) + with caplog.at_level(logging.WARNING): + colab._warn_colab_cloudflare_missing( + use_cloudflare = True, + cloudflare_url = "https://share.trycloudflare.com", + ) + assert "Cloudflare tunnel unavailable" not in caplog.text + + +def test_is_colab_runtime_uses_backend_colab_detector(monkeypatch): + fake_main = types.ModuleType("main") + fake_main._IS_COLAB = True + monkeypatch.setitem(sys.modules, "main", fake_main) + assert colab._is_colab_runtime() is True + fake_main._IS_COLAB = False + assert colab._is_colab_runtime() is False + + +def test_ready_card_html_uses_cloudflare_hint_on_colab_runtime_localhost(monkeypatch): + monkeypatch.setattr(colab, "_is_colab_runtime", lambda: True) + html = colab._ready_card_html("http://localhost:8888", 8888) + assert "window.open" not in html + assert "start(cloudflare=True)" in html + + +def test_ready_card_html_keeps_open_button_for_localhost_outside_colab(monkeypatch): + monkeypatch.setattr(colab, "_is_colab_runtime", lambda: False) + html = colab._ready_card_html("http://localhost:8888", 8888) + assert "window.open" in html + assert 'href="http://localhost:8888"' in html + assert "Open Unsloth Studio" in html + + +def test_embed_kernel_port_iframe_uses_colab_helper(monkeypatch): + colab_output = MagicMock() + google_colab = SimpleNamespace(output = colab_output) + monkeypatch.setattr(colab, "_is_colab_runtime", lambda: True) + with patch.dict("sys.modules", _mock_google_colab_modules(google_colab)): + assert colab._embed_kernel_port_iframe(8888) is True + colab_output.serve_kernel_port_as_iframe.assert_called_once_with( + 8888, + height = colab._COLAB_IFRAME_HEIGHT, + width = "100%", + ) + + +def test_embed_kernel_port_iframe_returns_false_without_colab(): + with patch.dict("sys.modules", _mock_google_colab_modules(None)): + assert colab._embed_kernel_port_iframe(8888) is False + + +def test_embed_kernel_port_iframe_skips_colabtools_without_runtime(monkeypatch): + """colabtools can queue JS without appending an iframe; only trust the helper on Colab.""" + colab_output = MagicMock() + google_colab = SimpleNamespace(output = colab_output) + monkeypatch.setattr(colab, "_is_colab_runtime", lambda: False) + with patch.dict("sys.modules", _mock_google_colab_modules(google_colab)): + assert colab._embed_kernel_port_iframe(8888) is False + colab_output.serve_kernel_port_as_iframe.assert_not_called() + + +def test_show_and_embed_prefers_kernel_port_iframe(monkeypatch): + calls: list[str] = [] + + monkeypatch.setattr(colab, "get_colab_url", lambda port: f"https://{port}-test.prod.colab.dev/") + monkeypatch.setattr(colab, "_is_colab_runtime", lambda: True) + monkeypatch.setattr( + colab, + "show_link", + lambda port, + *, + _url = None, + has_cloudflare_link = False, + cloudflare_requested = False: calls.append("show_link"), + ) + monkeypatch.setattr( + colab, + "_embed_kernel_port_iframe", + lambda port: calls.append("kernel_iframe") or True, + ) + monkeypatch.setattr( + colab, + "_embed_html_iframe", + lambda url, port: calls.append("html_iframe") or True, + ) + + colab._show_and_embed(8888) + + assert calls == ["show_link", "kernel_iframe"] + + +def test_show_and_embed_falls_back_to_html_iframe(monkeypatch): + calls: list[str] = [] + + monkeypatch.setattr(colab, "get_colab_url", lambda port: f"https://{port}-test.prod.colab.dev/") + monkeypatch.setattr(colab, "_is_colab_runtime", lambda: False) + monkeypatch.setattr( + colab, + "show_link", + lambda port, *, _url = None, has_cloudflare_link = False: None, + ) + monkeypatch.setattr(colab, "_embed_kernel_port_iframe", lambda port: False) + monkeypatch.setattr( + colab, + "_embed_html_iframe", + lambda url, port: calls.append((url, port)) or True, + ) + + colab._show_and_embed(8888) + + assert calls == [("https://8888-test.prod.colab.dev/", 8888)] + + +def test_colab_wants_cloudflare_auto_enables_on_runtime(monkeypatch): + monkeypatch.setattr(colab, "_is_colab_runtime", lambda: True) + assert colab._colab_wants_cloudflare(None) is True + assert colab._colab_wants_cloudflare(True) is True + assert colab._colab_wants_cloudflare(False) is False + + +def test_colab_wants_cloudflare_defaults_off_outside_runtime(monkeypatch): + monkeypatch.setattr(colab, "_is_colab_runtime", lambda: False) + assert colab._colab_wants_cloudflare(None) is False + assert colab._colab_wants_cloudflare(True) is True + + +def test_finalize_colab_admin_password_skips_outside_runtime(monkeypatch): + monkeypatch.setattr(colab, "_is_colab_runtime", lambda: False) + assert colab._finalize_colab_admin_password() is None + + +def test_finalize_colab_admin_password_clears_bootstrap_gate(monkeypatch): + monkeypatch.setattr(colab, "_is_colab_runtime", lambda: True) + monkeypatch.setattr(colab, "_load_colab_login_credentials", lambda: None) + stored: list[tuple[str, str]] = [] + monkeypatch.setattr( + colab, + "_store_colab_login_credentials", + lambda username, password: stored.append((username, password)), + ) + + storage = SimpleNamespace( + DEFAULT_ADMIN_USERNAME = "unsloth", + ensure_default_admin = MagicMock(), + get_bootstrap_password = MagicMock(return_value = "alpha-beta-gamma"), + generate_bootstrap_password = MagicMock(return_value = "alpha-beta-gamma"), + requires_password_change = MagicMock(return_value = True), + update_password = MagicMock(return_value = True), + ) + auth_pkg = types.ModuleType("auth") + auth_pkg.storage = storage + with patch.dict("sys.modules", {"auth": auth_pkg, "auth.storage": storage}): + result = colab._finalize_colab_admin_password() + + assert result == ("unsloth", "alpha-beta-gamma") + storage.ensure_default_admin.assert_called_once() + storage.update_password.assert_called_once_with("unsloth", "alpha-beta-gamma") + assert stored == [("unsloth", "alpha-beta-gamma")] + + +def test_start_skips_finalize_when_cloudflare_disabled(monkeypatch): + import time + + finalize_calls: list[str] = [] + monkeypatch.setattr(colab, "_is_studio_healthy", lambda port: True) + monkeypatch.setattr(colab, "_is_colab_runtime", lambda: True) + monkeypatch.setattr( + colab, + "_finalize_colab_admin_password", + lambda: finalize_calls.append("finalize") or ("unsloth", "secret"), + ) + monkeypatch.setattr( + colab, "start_cloudflare_tunnel", lambda port: "https://share.trycloudflare.com" + ) + monkeypatch.setattr(colab, "_publish_cloudflare_url", lambda url: None) + monkeypatch.setattr(colab, "_show_and_embed", lambda port, **kwargs: None) + monkeypatch.setattr(colab, "_stop_cloudflare_tunnel", lambda: None) + monkeypatch.setattr(time, "sleep", lambda _: (_ for _ in ()).throw(KeyboardInterrupt)) + + colab.start(cloudflare = False) + + assert finalize_calls == [] + + +def test_finalize_colab_admin_password_redisplay_on_rerun(monkeypatch): + monkeypatch.setattr(colab, "_is_colab_runtime", lambda: True) + monkeypatch.setattr( + colab, + "_load_colab_login_credentials", + lambda: ("unsloth", "saved-pass"), + ) + monkeypatch.setattr(colab, "_colab_credentials_still_valid", lambda username, password: True) + + storage = SimpleNamespace( + DEFAULT_ADMIN_USERNAME = "unsloth", + ensure_default_admin = MagicMock(), + get_bootstrap_password = MagicMock(), + generate_bootstrap_password = MagicMock(), + requires_password_change = MagicMock(return_value = False), + update_password = MagicMock(), + ) + auth_pkg = types.ModuleType("auth") + auth_pkg.storage = storage + with patch.dict("sys.modules", {"auth": auth_pkg, "auth.storage": storage}): + result = colab._finalize_colab_admin_password() + + assert result == ("unsloth", "saved-pass") + storage.update_password.assert_not_called() + + +def test_finalize_colab_admin_password_drops_stale_cached_credentials(monkeypatch): + """After an in-app password change the cached first-run password no longer + authenticates, so it must not be redisplayed (#7349 Codex review).""" + monkeypatch.setattr(colab, "_is_colab_runtime", lambda: True) + monkeypatch.setattr( + colab, + "_load_colab_login_credentials", + lambda: ("unsloth", "stale-pass"), + ) + monkeypatch.setattr(colab, "_colab_credentials_still_valid", lambda username, password: False) + cleared: list[bool] = [] + monkeypatch.setattr(colab, "_clear_colab_login_credentials", lambda: cleared.append(True)) + + storage = SimpleNamespace( + DEFAULT_ADMIN_USERNAME = "unsloth", + ensure_default_admin = MagicMock(), + get_bootstrap_password = MagicMock(), + generate_bootstrap_password = MagicMock(), + requires_password_change = MagicMock(return_value = False), + update_password = MagicMock(), + ) + auth_pkg = types.ModuleType("auth") + auth_pkg.storage = storage + with patch.dict("sys.modules", {"auth": auth_pkg, "auth.storage": storage}): + result = colab._finalize_colab_admin_password() + + assert result is None + assert cleared == [True] + storage.update_password.assert_not_called() + + +def test_colab_credentials_still_valid_matches_stored_hash(monkeypatch): + from auth.hashing import hash_password + + salt, pwd_hash = hash_password("right-pass") + storage = SimpleNamespace( + get_user_and_secret = MagicMock(return_value = (salt, pwd_hash, "jwt", False)), + ) + with patch.dict("sys.modules", {"auth.storage": storage}): + assert colab._colab_credentials_still_valid("unsloth", "right-pass") is True + assert colab._colab_credentials_still_valid("unsloth", "wrong-pass") is False + + +def test_colab_credentials_still_valid_false_when_user_missing(monkeypatch): + storage = SimpleNamespace(get_user_and_secret = MagicMock(return_value = None)) + with patch.dict("sys.modules", {"auth.storage": storage}): + assert colab._colab_credentials_still_valid("unsloth", "any") is False + + +def test_colab_login_html_includes_credentials(): + html = colab._colab_login_html("unsloth", "alpha-beta-gamma-delta") + assert "unsloth" in html + assert "alpha-beta-gamma-delta" in html + + +def test_show_and_embed_renders_cloudflare_before_colab_login(monkeypatch): + displayed: list[str] = [] + ipython_display = SimpleNamespace( + HTML = lambda html: SimpleNamespace(html = html), + display = lambda html: displayed.append(html.html), + ) + + monkeypatch.setattr(colab, "get_colab_url", lambda port: "https://8888-test.prod.colab.dev/") + monkeypatch.setattr(colab, "_is_colab_runtime", lambda: True) + monkeypatch.setattr( + colab, + "show_link", + lambda port, *, _url = None, has_cloudflare_link = False, cloudflare_requested = False: None, + ) + monkeypatch.setattr(colab, "_embed_kernel_port_iframe", lambda port: True) + with patch.dict("sys.modules", {"IPython.display": ipython_display}): + colab._show_and_embed( + 8888, + cloudflare_url = "https://share.trycloudflare.com", + colab_login = ("unsloth", "secret-pass"), + ) + + assert len(displayed) == 2 + assert "share.trycloudflare.com" in displayed[0] + assert "secret-pass" in displayed[1] + + +def test_show_and_embed_skips_iframe_on_colab_when_cloudflare_ready(monkeypatch): + calls: list[str] = [] + + monkeypatch.setattr(colab, "get_colab_url", lambda port: f"https://{port}-test.prod.colab.dev/") + monkeypatch.setattr(colab, "_is_colab_runtime", lambda: True) + monkeypatch.setattr( + colab, + "show_link", + lambda port, *, _url = None, has_cloudflare_link = False, cloudflare_requested = False: None, + ) + monkeypatch.setattr( + colab, + "_embed_kernel_port_iframe", + lambda port: calls.append("kernel_iframe") or True, + ) + monkeypatch.setattr( + colab, + "_embed_html_iframe", + lambda url, port: calls.append("html_iframe") or True, + ) + + colab._show_and_embed(8888, cloudflare_url = "https://share.trycloudflare.com") + + assert calls == [] + + +def test_show_and_embed_uses_kernel_helper_on_colab_runtime_despite_localhost(monkeypatch): + calls: list[str] = [] + + monkeypatch.setattr(colab, "get_colab_url", lambda port: f"http://localhost:{port}") + monkeypatch.setattr(colab, "_is_colab_runtime", lambda: True) + + monkeypatch.setattr( + colab, + "show_link", + lambda port, + *, + _url = None, + has_cloudflare_link = False, + cloudflare_requested = False: calls.append("show_link"), + ) + monkeypatch.setattr( + colab, + "_embed_kernel_port_iframe", + lambda port: calls.append("kernel_iframe") or True, + ) + monkeypatch.setattr( + colab, + "_embed_html_iframe", + lambda url, port: calls.append("html_iframe") or True, + ) + + colab._show_and_embed(8888) + + assert calls == ["show_link", "kernel_iframe"] + + +def test_show_and_embed_skips_kernel_helper_for_localhost_outside_colab(monkeypatch): + calls: list[str] = [] + + monkeypatch.setattr(colab, "get_colab_url", lambda port: f"http://localhost:{port}") + monkeypatch.setattr(colab, "_is_colab_runtime", lambda: False) + + monkeypatch.setattr( + colab, + "show_link", + lambda port, + *, + _url = None, + has_cloudflare_link = False, + cloudflare_requested = False: calls.append("show_link"), + ) + monkeypatch.setattr( + colab, + "_embed_kernel_port_iframe", + lambda port: calls.append("kernel_iframe") or True, + ) + monkeypatch.setattr( + colab, + "_embed_html_iframe", + lambda url, port: calls.append("html_iframe") or True, + ) + + colab._show_and_embed(8888) + + assert calls == ["show_link", "html_iframe"] + + +def test_show_and_embed_still_embeds_when_show_link_fails(monkeypatch): + calls: list[str] = [] + + monkeypatch.setattr(colab, "get_colab_url", lambda port: f"https://{port}-test.prod.colab.dev/") + monkeypatch.setattr(colab, "_is_colab_runtime", lambda: True) + monkeypatch.setattr( + colab, + "show_link", + lambda port, *, _url = None: (_ for _ in ()).throw(RuntimeError("no display")), + ) + monkeypatch.setattr( + colab, + "_embed_kernel_port_iframe", + lambda port: calls.append("kernel_iframe") or True, + ) + monkeypatch.setattr( + colab, + "_embed_html_iframe", + lambda url, port: calls.append("html_iframe") or True, + ) + + colab._show_and_embed(8888) + + assert calls == ["kernel_iframe"] diff --git a/studio/backend/tests/test_combined_update.py b/studio/backend/tests/test_combined_update.py new file mode 100644 index 0000000000..b96d3d030c --- /dev/null +++ b/studio/backend/tests/test_combined_update.py @@ -0,0 +1,735 @@ +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 + +"""Hermetic tests for the combined llama+whisper update item. + +llama.cpp is the single main update item; whisper.cpp piggybacks on it. These +pin the union status (update_available = llama behind OR whisper behind), the +chained apply (llama phase first, whisper phase only when behind), the failure +policy (llama failure aborts; whisper failure keeps the llama partial success), +the silent whisper skips, and the backward-compatible payload shape. +""" + +from __future__ import annotations + +import json +import sys +import time +from pathlib import Path + +import pytest + +_BACKEND = Path(__file__).resolve().parents[1] +if str(_BACKEND) not in sys.path: + sys.path.insert(0, str(_BACKEND)) + +import utils.llama_cpp_freshness as freshness # noqa: E402 +import utils.llama_cpp_update as upd # noqa: E402 +import utils.whisper_cpp_freshness as wfresh # noqa: E402 +import utils.whisper_cpp_update as wupd # noqa: E402 + +MARKER = "UNSLOTH_PREBUILT_INFO.json" +WHISPER_MARKER = "UNSLOTH_WHISPER_PREBUILT_INFO.json" + +# The top-level status and job fields that predate the whisper piggyback; the +# combined payload must stay an exact superset so current UI code keeps working. +LEGACY_STATUS_FIELDS = { + "supported", + "update_available", + "stale", + "installed_tag", + "latest_tag", + "published_repo", + "installed_at_utc", + "age_days", + "source_build", + "update_size_bytes", + "job", +} +LEGACY_JOB_FIELDS = { + "state", + "message", + "from_tag", + "to_tag", + "reload_required", + "error", + "progress", + "started_at", + "finished_at", +} + + +class _FakeInstallerPopen: + """Stands in for the streamed llama installer process.""" + + def __init__( + self, + cmd, + *, + returncode = 0, + lines = None, + on_start = None, + **kwargs, + ): + if on_start is not None: + on_start(list(cmd)) + self.returncode = returncode + self.stdout = iter(lines or []) + + def wait(self): + return self.returncode + + def kill(self): + pass + + +def _patch_llama_installer( + monkeypatch, + *, + returncode = 0, + lines = None, + on_start = None, +): + # Only intercept the installer invocation: importing routes.inference inside + # the worker can Popen unrelated host probes (ldconfig etc). + def _popen(cmd, **kw): + is_installer = any("install_llama_prebuilt" in str(part) for part in cmd) + return _FakeInstallerPopen( + cmd, + returncode = returncode if is_installer else 0, + lines = lines if is_installer else None, + on_start = on_start if is_installer else None, + ) + + monkeypatch.setattr(upd.subprocess, "Popen", _popen) + + +def _write_llama_install(dir_: Path, tag: str) -> str: + """Create a fake llama prebuilt install and return the llama-server path.""" + bin_dir = dir_ / "build" / "bin" + bin_dir.mkdir(parents = True, exist_ok = True) + binary = bin_dir / "llama-server" + binary.write_text("stub") + (dir_ / MARKER).write_text( + json.dumps( + { + "tag": tag, + "release_tag": tag, + "published_repo": "unslothai/llama.cpp", + "installed_at_utc": "2020-01-01T00:00:00Z", + } + ) + ) + return str(binary) + + +def _write_whisper_install( + dir_: Path, + tag: str, + backend: str = "cpu", +) -> str: + """Create a fake whisper prebuilt install and return the whisper-server path.""" + bin_dir = dir_ / "build" / "bin" + bin_dir.mkdir(parents = True, exist_ok = True) + binary = bin_dir / "whisper-server" + binary.write_text("stub") + (dir_ / WHISPER_MARKER).write_text( + json.dumps( + { + "release_tag": tag, + "upstream_tag": tag.split("-")[0], + "published_repo": "unslothai/whisper.cpp", + "backend": backend, + "installed_at_utc": "2020-01-01T00:00:00Z", + } + ) + ) + return str(binary) + + +@pytest.fixture(autouse = True) +def _clean_state(monkeypatch, tmp_path): + freshness.reset_caches() + wfresh.reset_caches() + upd._reset_job_for_tests() + upd._resolve_memo.clear() + wupd._resolve_memo.clear() + monkeypatch.setattr(freshness, "_cache_dir", lambda: tmp_path / ".llama_cache") + monkeypatch.setattr(wfresh, "_cache_dir", lambda: tmp_path / ".whisper_cache") + for var in ( + "LLAMA_SERVER_PATH", + "UNSLOTH_LLAMA_CPP_PATH", + "WHISPER_SERVER_PATH", + "UNSLOTH_WHISPER_CPP_PATH", + ): + monkeypatch.delenv(var, raising = False) + # Never hit the network in these tests. + monkeypatch.setattr(freshness, "_fetch_latest_release_tag", lambda repo, timeout = 5.0: None) + monkeypatch.setattr(wfresh, "_fetch_latest_release_tag", lambda repo, timeout = 5.0: None) + yield + freshness.reset_caches() + wfresh.reset_caches() + upd._reset_job_for_tests() + upd._resolve_memo.clear() + wupd._resolve_memo.clear() + + +def _setup_llama( + monkeypatch, + tmp_path, + *, + installed = "b9493", + latest = "b9518", +): + """Marker-managed llama install; behind when installed != latest.""" + install_dir = tmp_path / "llama.cpp" + binary = _write_llama_install(install_dir, installed) + monkeypatch.setattr(upd, "_find_binary", lambda: binary) + monkeypatch.setattr(upd, "_installer_script", lambda: tmp_path / "install_llama_prebuilt.py") + monkeypatch.setattr(freshness, "_fetch_latest_release_tag", lambda repo, timeout = 5.0: latest) + return install_dir + + +def _setup_whisper( + monkeypatch, + tmp_path, + *, + installed = "v1.9.1-unsloth.1", + latest = "v1.9.2-unsloth.1", +): + """Marker-managed whisper install; behind when latest is newer.""" + install_dir = tmp_path / "whisper.cpp" + binary = _write_whisper_install(install_dir, installed) + monkeypatch.setattr(wupd, "_find_binary", lambda: binary) + monkeypatch.setattr(wupd, "_installer_script", lambda: tmp_path / "install_whisper_prebuilt.py") + monkeypatch.setattr(wfresh, "_fetch_latest_release_tag", lambda repo, timeout = 5.0: latest) + return install_dir + + +def _patch_whisper_phase( + monkeypatch, + events, + *, + to_tag = "v1.9.2-unsloth.1", + error = None, +): + """Record whisper phase runs without touching a real installer.""" + + def _run(phase, set_progress): + events.append("whisper") + if error is not None: + raise RuntimeError(error) + set_progress(0.5) + return { + "to_tag": to_tag, + "reload_required": False, + "message": f"Updated whisper.cpp to {to_tag}.", + } + + monkeypatch.setattr(wupd, "run_chained_phase", _run) + + +def _wait_for_job(): + deadline = time.time() + 10 + while time.time() < deadline: + with upd._job_lock: + job = dict(upd._job) + if job["state"] in ("success", "error"): + return job + time.sleep(0.05) + with upd._job_lock: + return dict(upd._job) + + +# --- status: the single item folds whisper in --- + + +def test_status_payload_is_exact_superset_of_legacy_fields(monkeypatch, tmp_path): + _setup_llama(monkeypatch, tmp_path) + _setup_whisper(monkeypatch, tmp_path) + st = upd.get_update_status(force_refresh = True) + assert LEGACY_STATUS_FIELDS <= set(st) + assert LEGACY_JOB_FIELDS <= set(st["job"]) + # The new fields ride alongside, never replacing the legacy ones. + assert st["llama_update_available"] is True + assert st["whisper"]["update_available"] is True + assert st["whisper"]["latest_tag"] == "v1.9.2-unsloth.1" + assert st["update_component"] == "llama" + + +def test_status_union_whisper_only_surfaces_update(monkeypatch, tmp_path): + # llama current, whisper behind: the single item still shows an update. + _setup_llama(monkeypatch, tmp_path, installed = "b9518", latest = "b9518") + _setup_whisper(monkeypatch, tmp_path) + st = upd.get_update_status(force_refresh = True) + assert st["llama_update_available"] is False + assert st["whisper"]["update_available"] is True + assert st["update_available"] is True + assert st["update_component"] == "whisper" + assert st["installed_tag"] == "b9518" + assert st["latest_tag"] == "b9518" + assert st["whisper"]["installed_tag"] == "v1.9.1-unsloth.1" + assert st["whisper"]["latest_tag"] == "v1.9.2-unsloth.1" + + +def test_status_whisper_current_does_not_flip_union(monkeypatch, tmp_path): + _setup_llama(monkeypatch, tmp_path, installed = "b9518", latest = "b9518") + _setup_whisper(monkeypatch, tmp_path, installed = "v1.9.2-unsloth.1", latest = "v1.9.2-unsloth.1") + st = upd.get_update_status(force_refresh = True) + assert st["update_available"] is False + assert st["whisper"]["skip_reason"] == "up_to_date" + assert st["update_component"] is None + + +def test_status_survives_whisper_probe_failure(monkeypatch, tmp_path): + # The piggyback fails open: llama status still works without a whisper probe. + _setup_llama(monkeypatch, tmp_path) + + def _boom(*, force_refresh = False): + raise RuntimeError("probe exploded") + + monkeypatch.setattr(wupd, "chained_phase_plan", _boom) + st = upd.get_update_status(force_refresh = True) + assert st["update_available"] is True + assert st["whisper"] is None + + +# --- whisper chained_phase_plan: silent skips --- + + +def test_whisper_plan_skips_local_link(monkeypatch, tmp_path): + monkeypatch.setattr(wupd, "_find_binary", lambda: str(tmp_path / "whisper-server")) + monkeypatch.setattr(wupd, "_active_install_is_local_link", lambda b: True) + plan = wupd.chained_phase_plan() + assert plan["update_available"] is False + assert plan["skip_reason"] == "local_link" + assert plan["phase"] is None + + +def test_whisper_plan_skips_source_build(monkeypatch, tmp_path): + binary = tmp_path / "whisper.cpp" / "build" / "bin" / "whisper-server" + binary.parent.mkdir(parents = True) + binary.write_text("stub") # no marker + monkeypatch.setattr(wupd, "_find_binary", lambda: str(binary)) + plan = wupd.chained_phase_plan() + assert plan["skip_reason"] == "source_build" + assert plan["phase"] is None + + +def test_whisper_update_targets_canonical_root_when_inner_marker_exists(tmp_path): + install_dir = tmp_path / "whisper.cpp" + binary = install_dir / "build" / "bin" / "whisper-server" + binary.parent.mkdir(parents = True) + binary.write_text("stub") + (install_dir / WHISPER_MARKER).write_text("{}") + (binary.parent / WHISPER_MARKER).write_text("{}") + assert wupd._install_dir_for(str(binary)) == install_dir + + +def test_whisper_plan_skips_when_not_installed(monkeypatch): + monkeypatch.setattr(wupd, "_find_binary", lambda: None) + plan = wupd.chained_phase_plan() + assert plan["skip_reason"] == "not_installed" + assert plan["phase"] is None + + +def test_whisper_plan_eligible_when_behind(monkeypatch, tmp_path): + install_dir = _setup_whisper(monkeypatch, tmp_path) + script = tmp_path / "install_whisper_prebuilt.py" + script.write_text("stub") + plan = wupd.chained_phase_plan(force_refresh = True) + assert plan["update_available"] is True + assert plan["skip_reason"] is None + assert plan["phase"]["install_dir"] == install_dir + assert plan["phase"]["repo"] == "unslothai/whisper.cpp" + assert plan["phase"]["backend"] == "cpu" + # Pin to the exact release the freshness check offered: unpinned, the + # installer's download-host /releases/latest pointer can lag published_at + # and reinstall an older build in a loop. + assert plan["phase"]["pin_release_tag"] == "v1.9.2-unsloth.1" + + +def test_whisper_plan_requires_a_repairable_pair_for_slim_installs(monkeypatch, tmp_path): + install_dir = _setup_whisper(monkeypatch, tmp_path) + marker_path = install_dir / WHISPER_MARKER + marker = json.loads(marker_path.read_text()) + marker["install_kind"] = "slim" + marker_path.write_text(json.dumps(marker)) + wfresh.reset_caches() + monkeypatch.setattr( + wupd, + "_resolve_prebuilt_for_host", + lambda **kwargs: {"prebuilt_available": False}, + ) + + plan = wupd.chained_phase_plan(force_refresh = True) + assert plan["update_available"] is False + assert plan["skip_reason"] == "paired_llama_unavailable" + + repaired = wupd.chained_phase_plan( + force_refresh = True, + paired_llama_will_update = True, + ) + assert repaired["update_available"] is True + assert repaired["phase"] is not None + + +def test_whisper_phase_pins_installer_to_checked_release(monkeypatch, tmp_path): + calls = [] + monkeypatch.setattr( + wupd._flow, + "stream_installer", + lambda cmd, env, **kw: calls.append(cmd), + ) + monkeypatch.setattr(wupd, "reset_caches", lambda **kw: None) + monkeypatch.setattr(wupd, "latest_published_release", lambda repo, **kw: "v9") + install_dir = tmp_path / "whisper.cpp" + binary = _write_whisper_install(install_dir, "v9") + monkeypatch.setattr(wupd, "_find_binary", lambda: binary) + wupd.run_chained_phase( + { + "install_dir": install_dir, + "repo": "unslothai/whisper.cpp", + "asset": None, + "backend": "cpu", + "script": tmp_path / "install_whisper_prebuilt.py", + "pin_release_tag": "v9", + }, + lambda f: None, + ) + cmd = calls[0] + assert "--published-release-tag" in cmd + assert cmd[cmd.index("--published-release-tag") + 1] == "v9" + + +def test_whisper_phase_exit_2_is_a_failed_phase(monkeypatch, tmp_path): + # No install occurred, so incompatibility must remain an actionable job + # error instead of producing a false success toast and hiding the banner. + def _raise_exit_2(cmd, env, **kw): + raise wupd._flow.InstallerExit(2, "installer exited 2: incompatible release") + + monkeypatch.setattr(wupd._flow, "stream_installer", _raise_exit_2) + install_dir = tmp_path / "whisper.cpp" + binary = _write_whisper_install(install_dir, "v1") + monkeypatch.setattr(wupd, "_find_binary", lambda: binary) + with pytest.raises(wupd._flow.InstallerExit) as exc_info: + wupd.run_chained_phase( + { + "install_dir": install_dir, + "repo": "unslothai/whisper.cpp", + "asset": None, + "backend": "cpu", + "script": tmp_path / "install_whisper_prebuilt.py", + "pin_release_tag": None, + }, + lambda f: None, + ) + assert exc_info.value.returncode == 2 + + +def test_llama_update_survives_unavailable_whisper_module(monkeypatch, tmp_path): + import builtins + + llama_dir = _setup_llama(monkeypatch, tmp_path) + monkeypatch.setattr(upd, "_whisper_chain_status", lambda **kw: None) + _patch_llama_installer( + monkeypatch, + on_start = lambda cmd: _write_llama_install(llama_dir, "b9518"), + ) + real_import = builtins.__import__ + + def guarded_import( + name, + globals = None, + locals = None, + fromlist = (), + level = 0, + ): + if name == "utils" and "whisper_cpp_update" in fromlist: + raise AssertionError("whisper module was re-imported after its failed probe") + return real_import(name, globals, locals, fromlist, level) + + monkeypatch.setattr(builtins, "__import__", guarded_import) + + # A failed optional whisper probe must not be followed by an unconditional + # import. The valid llama phase still starts and completes. + assert upd.start_update()["started"] is True + job = _wait_for_job() + assert job["state"] == "success", job + assert job["phases"]["llama"]["state"] == "success" + assert job["phases"]["whisper"]["state"] == "skipped" + assert job["phases"]["whisper"]["reason"] == "unavailable" + + +def test_macos_status_uses_compatible_resolver_release(monkeypatch, tmp_path): + _setup_whisper( + monkeypatch, + tmp_path, + installed = "v1.9.1-unsloth.1", + latest = "v1.9.2-unsloth.1", + ) + monkeypatch.setattr(wupd.sys, "platform", "darwin") + monkeypatch.setattr( + wupd, + "_resolve_prebuilt_for_host", + lambda **kw: { + "prebuilt_available": True, + "release_tag": "v1.9.1-unsloth.1", + }, + ) + + status = wupd.get_update_status(force_refresh = True) + assert status["latest_tag"] == "v1.9.1-unsloth.1" + assert status["update_available"] is False + assert status["stale"] is False + + +def test_whisper_phase_integrity_failure_is_not_swallowed(monkeypatch, tmp_path): + def _raise_exit_1(cmd, env, **kw): + raise wupd._flow.InstallerExit(1, "installer exited 1: checksum mismatch") + + monkeypatch.setattr(wupd._flow, "stream_installer", _raise_exit_1) + install_dir = tmp_path / "whisper.cpp" + binary = _write_whisper_install(install_dir, "v1") + monkeypatch.setattr(wupd, "_find_binary", lambda: binary) + with pytest.raises(wupd._flow.InstallerExit, match = "checksum mismatch"): + wupd.run_chained_phase( + { + "install_dir": install_dir, + "repo": "unslothai/whisper.cpp", + "asset": None, + "backend": "cpu", + "script": tmp_path / "install_whisper_prebuilt.py", + "pin_release_tag": None, + }, + lambda f: None, + ) + + +# --- apply: the chained job --- + + +def test_apply_runs_llama_then_whisper(monkeypatch, tmp_path): + llama_dir = _setup_llama(monkeypatch, tmp_path) + _setup_whisper(monkeypatch, tmp_path) + (tmp_path / "install_whisper_prebuilt.py").write_text("stub") + + events = [] + _patch_llama_installer( + monkeypatch, + on_start = lambda cmd: (events.append("llama"), _write_llama_install(llama_dir, "b9518")), + ) + _patch_whisper_phase(monkeypatch, events) + + res = upd.start_update() + assert res["started"] is True, res + job = _wait_for_job() + assert job["state"] == "success", job + assert events == ["llama", "whisper"] # llama phase strictly first + assert job["phases"]["llama"]["state"] == "success" + assert job["phases"]["llama"]["to_tag"] == "b9518" + assert job["phases"]["whisper"]["state"] == "success" + assert job["phases"]["whisper"]["to_tag"] == "v1.9.2-unsloth.1" + # Legacy top-level fields keep their llama meaning. + assert job["from_tag"] == "b9493" + assert job["to_tag"] == "b9518" + assert "Updated llama.cpp to b9518." in job["message"] + assert "Updated whisper.cpp to v1.9.2-unsloth.1." in job["message"] + assert job["progress"] == 1.0 + assert LEGACY_JOB_FIELDS <= set(job) + + +def test_apply_llama_only_when_whisper_current(monkeypatch, tmp_path): + llama_dir = _setup_llama(monkeypatch, tmp_path) + _setup_whisper(monkeypatch, tmp_path, installed = "v1.9.2-unsloth.1", latest = "v1.9.2-unsloth.1") + + events = [] + _patch_llama_installer( + monkeypatch, + on_start = lambda cmd: (events.append("llama"), _write_llama_install(llama_dir, "b9518")), + ) + _patch_whisper_phase(monkeypatch, events) + + assert upd.start_update()["started"] is True + job = _wait_for_job() + assert job["state"] == "success", job + assert events == ["llama"] + assert job["phases"]["whisper"]["state"] == "skipped" + assert job["phases"]["whisper"]["reason"] == "up_to_date" + + +def test_apply_whisper_only_noops_llama(monkeypatch, tmp_path): + # llama current + whisper behind: the same single apply runs, with the llama + # phase a cheap already-matches no-op and the whisper phase doing the work. + _setup_llama(monkeypatch, tmp_path, installed = "b9518", latest = "b9518") + _setup_whisper(monkeypatch, tmp_path) + (tmp_path / "install_whisper_prebuilt.py").write_text("stub") + + events = [] + _patch_llama_installer(monkeypatch, on_start = lambda cmd: events.append("llama")) + _patch_whisper_phase(monkeypatch, events) + + res = upd.start_update() + assert res["started"] is True, res + job = _wait_for_job() + assert job["state"] == "success", job + assert events == ["whisper"] # the llama installer never ran + # The legacy job-level to_tag means "llama tag"; a whisper-only round + # leaves it unset so the UI never reports a llama update that never ran. + assert job["to_tag"] is None + assert job["phases"]["llama"]["state"] == "skipped" + assert job["phases"]["llama"]["reason"] == "up_to_date" + assert job["phases"]["whisper"]["state"] == "success" + assert "Updated whisper.cpp to v1.9.2-unsloth.1." in job["message"] + + +def test_whisper_reload_never_raises_job_reload_flag(monkeypatch, tmp_path): + # A whisper-only update that had to unload a warm sidecar reports + # reload_required on its phase, but the JOB flag stays down: the chat + # frontend resyncs (and clears the local checkpoint) off the job flag, + # which must mean "the llama server changed", not "the sidecar restarted". + _setup_llama(monkeypatch, tmp_path, installed = "b9518", latest = "b9518") + _setup_whisper(monkeypatch, tmp_path) + (tmp_path / "install_whisper_prebuilt.py").write_text("stub") + + def _whisper_phase(phase, set_progress): + return { + "to_tag": "v1.9.2-unsloth.1", + "reload_required": True, + "message": "Updated whisper.cpp to v1.9.2-unsloth.1.", + } + + monkeypatch.setattr(wupd, "run_chained_phase", _whisper_phase) + assert upd.start_update()["started"] is True + job = _wait_for_job() + assert job["state"] == "success", job + assert job["phases"]["whisper"]["reload_required"] is True + assert not job["reload_required"] + + +def test_apply_refuses_when_both_current(monkeypatch, tmp_path): + _setup_llama(monkeypatch, tmp_path, installed = "b9518", latest = "b9518") + _setup_whisper(monkeypatch, tmp_path, installed = "v1.9.2-unsloth.1", latest = "v1.9.2-unsloth.1") + res = upd.start_update() + assert res["started"] is False + assert res["reason"] == "up_to_date" + + +def test_apply_llama_failure_aborts_before_whisper(monkeypatch, tmp_path): + _setup_llama(monkeypatch, tmp_path) + _setup_whisper(monkeypatch, tmp_path) + (tmp_path / "install_whisper_prebuilt.py").write_text("stub") + + events = [] + _patch_llama_installer(monkeypatch, returncode = 2, lines = ["boom: disk full\n"]) + _patch_whisper_phase(monkeypatch, events) + + assert upd.start_update()["started"] is True + job = _wait_for_job() + assert job["state"] == "error", job + assert "boom" in (job["error"] or "") + assert events == [] # whisper never attempted + assert job["phases"]["llama"]["state"] == "error" + assert job["phases"]["whisper"]["state"] == "skipped" + assert job["phases"]["whisper"]["reason"] == "aborted" + assert job["message"] == "llama.cpp update failed." + + +def test_apply_whisper_failure_keeps_llama_partial_success(monkeypatch, tmp_path): + llama_dir = _setup_llama(monkeypatch, tmp_path) + _setup_whisper(monkeypatch, tmp_path) + (tmp_path / "install_whisper_prebuilt.py").write_text("stub") + + # An active model makes the llama phase report reload_required. + import threading + from types import ModuleType + + class _FakeBackend: + def __init__(self): + self._serial_load_lock = threading.Lock() + self._llama_update_in_progress = False + self.is_active = True + + def unload_model(self): + self.is_active = False + + backend = _FakeBackend() + routes_pkg = ModuleType("routes") + routes_pkg.__path__ = [] + inference_mod = ModuleType("routes.inference") + inference_mod.get_llama_cpp_backend = lambda: backend + monkeypatch.setitem(sys.modules, "routes", routes_pkg) + monkeypatch.setitem(sys.modules, "routes.inference", inference_mod) + + events = [] + _patch_llama_installer( + monkeypatch, + on_start = lambda cmd: (events.append("llama"), _write_llama_install(llama_dir, "b9518")), + ) + _patch_whisper_phase(monkeypatch, events, error = "whisper installer exploded") + + assert upd.start_update()["started"] is True + job = _wait_for_job() + assert job["state"] == "error", job + assert events == ["llama", "whisper"] + # The message says both halves: llama landed, whisper did not. + assert "Updated llama.cpp to b9518." in job["message"] + assert "whisper.cpp update failed." in job["message"] + assert "whisper installer exploded" in (job["error"] or "") + # The llama phase's reload_required survives the whisper failure. + assert job["reload_required"] is True + assert job["to_tag"] == "b9518" + assert job["phases"]["llama"]["state"] == "success" + assert job["phases"]["whisper"]["state"] == "error" + + +def test_apply_skips_whisper_local_link_silently(monkeypatch, tmp_path): + llama_dir = _setup_llama(monkeypatch, tmp_path) + _setup_whisper(monkeypatch, tmp_path) + monkeypatch.setattr(wupd, "_active_install_is_local_link", lambda b: True) + + events = [] + _patch_llama_installer( + monkeypatch, + on_start = lambda cmd: (events.append("llama"), _write_llama_install(llama_dir, "b9518")), + ) + _patch_whisper_phase(monkeypatch, events) + + assert upd.start_update()["started"] is True + job = _wait_for_job() + assert job["state"] == "success", job + assert events == ["llama"] + assert job["phases"]["whisper"]["state"] == "skipped" + assert job["phases"]["whisper"]["reason"] == "local_link" + assert job["message"] == "Updated llama.cpp to b9518." + + +def test_chained_progress_windows(monkeypatch, tmp_path): + # The llama phase fills roughly the first 0.7 slice and whisper the rest. + llama_dir = _setup_llama(monkeypatch, tmp_path) + _setup_whisper(monkeypatch, tmp_path) + (tmp_path / "install_whisper_prebuilt.py").write_text("stub") + + seen = {} + + def _whisper_phase(phase, set_progress): + with upd._job_lock: + seen["at_whisper_start"] = upd._job["progress"] + set_progress(0.5) + with upd._job_lock: + seen["mid_whisper"] = upd._job["progress"] + return {"to_tag": "v1.9.2-unsloth.1", "reload_required": False, "message": "ok"} + + monkeypatch.setattr(wupd, "run_chained_phase", _whisper_phase) + _patch_llama_installer( + monkeypatch, + lines = ["Downloading app.tar.gz: 100.0% (35.0 MiB/35.0 MiB) at 9.0 MiB/s\n"], + on_start = lambda cmd: _write_llama_install(llama_dir, "b9518"), + ) + + assert upd.start_update()["started"] is True + job = _wait_for_job() + assert job["state"] == "success", job + assert seen["at_whisper_start"] == pytest.approx(0.7) + assert seen["mid_whisper"] == pytest.approx(0.7 + 0.5 * 0.3) + assert job["progress"] == 1.0 diff --git a/studio/backend/tests/test_compute_buffer.py b/studio/backend/tests/test_compute_buffer.py index 8408f8203d..3e95acc98d 100644 --- a/studio/backend/tests/test_compute_buffer.py +++ b/studio/backend/tests/test_compute_buffer.py @@ -152,7 +152,7 @@ class TestFallback: class TestParallel1Default: - """At Studio's default --parallel 1 the buffer is negligible in pipeline.""" + """At Unsloth's default --parallel 1 the buffer is negligible in pipeline.""" def test_default_n_parallel(self): est = _backend()._estimate_compute_buffer_bytes() / MIB diff --git a/studio/backend/tests/test_consent_gate.py b/studio/backend/tests/test_consent_gate.py index 804221ec7e..181e0c9fad 100644 --- a/studio/backend/tests/test_consent_gate.py +++ b/studio/backend/tests/test_consent_gate.py @@ -873,6 +873,7 @@ class TestScannerCoversAllExecutableCode: repo, fn, token = None, + cache_dir = None, ): if fn == "config.json": import json @@ -899,6 +900,7 @@ class TestScannerCoversAllExecutableCode: repo, fn, token = None, + cache_dir = None, ): import json import tempfile @@ -932,6 +934,7 @@ class TestScannerCoversAllExecutableCode: repo, fn, token = None, + cache_dir = None, ): import json import tempfile @@ -972,6 +975,7 @@ class TestScannerCoversAllExecutableCode: repo, fn, token = None, + cache_dir = None, ): import json import tempfile @@ -1008,6 +1012,7 @@ class TestScannerCoversAllExecutableCode: repo, fn, token = None, + cache_dir = None, ): import json import tempfile @@ -1037,6 +1042,7 @@ class TestScannerCoversAllExecutableCode: repo, fn, token = None, + cache_dir = None, ): import json import tempfile @@ -1079,6 +1085,7 @@ class TestScannerCoversAllExecutableCode: repo, fn, token = None, + cache_dir = None, ): import json import tempfile @@ -1120,6 +1127,7 @@ class TestScannerCoversAllExecutableCode: repo, fn, token = None, + cache_dir = None, ): import json import tempfile @@ -1182,6 +1190,7 @@ class TestScannerCoversAllExecutableCode: repo, fn, token = None, + cache_dir = None, ): import json import tempfile diff --git a/studio/backend/tests/test_cpu_threads.py b/studio/backend/tests/test_cpu_threads.py index 2930c9f081..9d8795b6c0 100644 --- a/studio/backend/tests/test_cpu_threads.py +++ b/studio/backend/tests/test_cpu_threads.py @@ -1,7 +1,7 @@ # SPDX-License-Identifier: AGPL-3.0-only # Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 -"""Tests for Studio's early CPU thread-pool configuration.""" +"""Tests for Unsloth's early CPU thread-pool configuration.""" import ast import os @@ -30,7 +30,7 @@ def test_cpu_thread_cap_seeds_native_pool_limits(): } -# Explicit per-library values win over the Studio knob via setdefault. +# Explicit per-library values win over the Unsloth knob via setdefault. def test_cpu_thread_cap_preserves_runtime_specific_override(): env = {"UNSLOTH_CPU_THREADS": "4", "OMP_NUM_THREADS": "2"} diff --git a/studio/backend/tests/test_cuda_torch_spec.py b/studio/backend/tests/test_cuda_torch_spec.py new file mode 100644 index 0000000000..928cef787e --- /dev/null +++ b/studio/backend/tests/test_cuda_torch_spec.py @@ -0,0 +1,73 @@ +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 + +"""Tests for _CUDA_TORCH_PKG_SPEC in install_python_stack.py. + +The CUDA repair path installs the torch trio from an exclusive --index-url (no +PyPI fallback), so these pinned ranges decide which torch the venv gets. The +upper bound is locked to the 2.11.x family to match the base image and rocm7.2 +spec and to keep the companions off a torch-2.12 wheel that would ABI-mismatch. +""" + +from __future__ import annotations + +import sys +from pathlib import Path + +import pytest +from packaging.requirements import Requirement + +# install_python_stack.py lives at repo_root/studio/install_python_stack.py +_INSTALL_SCRIPT = Path(__file__).resolve().parents[2] / "install_python_stack.py" + + +def _load_module(monkeypatch): + """(Re-)import and return install_python_stack (mirrors test_torchao_select).""" + sys.modules.pop("install_python_stack", None) + monkeypatch.syspath_prepend(str(_INSTALL_SCRIPT.parent)) + import install_python_stack + + return install_python_stack + + +def _spec_of(pkg_spec: str): + """Parse 'torch>=2.4,<2.12.0' into a packaging SpecifierSet.""" + return Requirement(pkg_spec).specifier + + +@pytest.mark.parametrize( + "index, allowed, rejected", + [ + # torch: 2.11.x allowed (matches base image); 2.12.x excluded. + (0, ["2.11.0", "2.11.2", "2.10.0", "2.4.0"], ["2.12.0", "2.3.0", "1.13.1"]), + # torchvision: 0.26.x (torch 2.11 companion) allowed; 0.27.x (torch 2.12) out. + (1, ["0.26.0", "0.26.1", "0.19.0"], ["0.27.0", "0.18.0"]), + # torchaudio: same 2.11.x window as torch. + (2, ["2.11.0", "2.10.0", "2.4.0"], ["2.12.0", "2.3.0"]), + ], +) +def test_cuda_spec_bounds(monkeypatch, index, allowed, rejected): + mod = _load_module(monkeypatch) + spec = _spec_of(mod._CUDA_TORCH_PKG_SPEC[index]) + for v in allowed: + assert spec.contains(v, prereleases = True), f"{v} should satisfy {spec}" + for v in rejected: + assert not spec.contains(v, prereleases = True), f"{v} should not satisfy {spec}" + + +def test_cuda_spec_matches_rocm72_upper_bound(monkeypatch): + """CUDA and rocm7.2 target the same torch 2.11.x family, so their upper + bounds must stay in lockstep (bump both together at 2.12.x).""" + mod = _load_module(monkeypatch) + rocm72 = mod._ROCM_TORCH_PKG_SPECS["rocm7.2"] + + def _upper(pkg_spec: str) -> str: + for clause in _spec_of(pkg_spec): + if clause.operator == "<": + return clause.version + raise AssertionError(f"no upper bound in {pkg_spec!r}") + + for cuda_pkg, rocm_pkg in zip(mod._CUDA_TORCH_PKG_SPEC, rocm72, strict = True): + assert _upper(cuda_pkg) == _upper( + rocm_pkg + ), f"CUDA {cuda_pkg!r} upper bound must match rocm7.2 {rocm_pkg!r}" diff --git a/studio/backend/tests/test_embedding_model_security_gate.py b/studio/backend/tests/test_embedding_model_security_gate.py index 940b35d7ba..a6c18bd8de 100644 --- a/studio/backend/tests/test_embedding_model_security_gate.py +++ b/studio/backend/tests/test_embedding_model_security_gate.py @@ -52,6 +52,16 @@ def client(monkeypatch): monkeypatch.setattr(settings, "_resolves_as_local_gguf", lambda m: False) monkeypatch.setattr(settings, "get_rag_embedding_model", lambda: saved.get("model", "")) monkeypatch.setattr(settings, "get_stored_embedding_model", lambda: saved.get("model")) + monkeypatch.setattr( + settings, + "effective_gguf_repo", + lambda: f"{saved.get('model', 'unsloth/default-embed')}-GGUF", + ) + monkeypatch.setattr( + settings, + "default_gguf_repo", + lambda: "unsloth/default-embed-GGUF", + ) app = FastAPI() app.include_router(settings.router) @@ -96,6 +106,56 @@ def test_hard_block_uses_non_forceable_status(client, monkeypatch): assert unverified.status_code == 409 +def test_offline_cached_non_st_model_is_accepted(client, monkeypatch): + # Offline, a cached transformers-native embedder (no modules.json) is unverifiable via HF + # metadata, but ST can load any cached encoder, so accept it (no 409). + c, saved = client + monkeypatch.setitem(sys.modules, "utils.security", _security_stub(blocked = False)) + monkeypatch.setenv("HF_HUB_OFFLINE", "1") + import utils.models as _models + import utils.utils as _uu + + monkeypatch.setattr(_models, "is_embedding_model", lambda *a, **k: False) + monkeypatch.setattr(_uu, "hf_cache_snapshot_is_loadable", lambda name: True) + r = c.put("/embedding-model", json = {"embedding_model": "acme/gte-modernbert"}) + assert r.status_code == 200 + assert saved.get("model") == "acme/gte-modernbert" + + +def test_offline_partial_or_uncached_model_still_409(client, monkeypatch): + # Offline but not loadable (uncached or metadata-only partial cache): keep the forceable + # 409, since the cache-only load would fail anyway. + c, _saved = client + monkeypatch.setitem(sys.modules, "utils.security", _security_stub(blocked = False)) + monkeypatch.setenv("HF_HUB_OFFLINE", "1") + import utils.models as _models + import utils.utils as _uu + + monkeypatch.setattr(_models, "is_embedding_model", lambda *a, **k: False) + monkeypatch.setattr(_uu, "hf_cache_snapshot_is_loadable", lambda name: False) + r = c.put("/embedding-model", json = {"embedding_model": "acme/uncached-embedder"}) + assert r.status_code == 409 + + +def test_offline_skips_remote_gguf_probe(client, monkeypatch): + # Offline + llama backend: the remote GGUF probe (list_repo_files) must be skipped so a + # dead-DNS session cannot hang. + c, _saved = client + monkeypatch.setenv("HF_HUB_OFFLINE", "1") + monkeypatch.setattr(settings, "_llama_backend_active", lambda: True) + monkeypatch.setattr(settings, "_local_gguf_backend_error", lambda model: None) + + def _boom(*a, **k): + raise AssertionError("hit the network for the GGUF probe") + + monkeypatch.setattr(settings, "_hf_gguf_backend_error", _boom) + import utils.models as _models + + monkeypatch.setattr(_models, "is_embedding_model", lambda *a, **k: True) + r = c.put("/embedding-model", json = {"embedding_model": "acme/embedder"}) + assert r.status_code == 200 + + def test_llama_backend_skips_the_st_pickle_scan(monkeypatch): # On the llama-server backend the embedder loads GGUF (inert), not the ST repo's # pickle, so a flagged ST repo with a clean GGUF companion must not be rejected here. @@ -257,6 +317,13 @@ def test_clean_repo_saves_under_force(client, monkeypatch): r = c.put("/embedding-model", json = {"embedding_model": "acme/clean-embed", "force": True}) assert r.status_code == 200 assert saved.get("model") == "acme/clean-embed" + assert r.json() == { + "embedding_model": "acme/clean-embed", + "embedding_gguf_repo": "acme/clean-embed-GGUF", + "default_embedding_model": "unsloth/default-embed", + "default_embedding_gguf_repo": "unsloth/default-embed-GGUF", + "is_custom": True, + } def test_load_sink_refuses_flagged_model(monkeypatch): diff --git a/studio/backend/tests/test_embedding_model_settings.py b/studio/backend/tests/test_embedding_model_settings.py index 3be4af0e32..bcf3ded71c 100644 --- a/studio/backend/tests/test_embedding_model_settings.py +++ b/studio/backend/tests/test_embedding_model_settings.py @@ -53,3 +53,10 @@ def test_custom_model_overrides_default_and_derives_gguf(settings_store, monkeyp assert ems.reset_rag_embedding_model() == rag_config.EMBEDDING_MODEL assert ems.get_stored_embedding_model() is None + + +def test_env_default_derives_its_gguf_companion(monkeypatch): + monkeypatch.delenv("RAG_EMBED_GGUF_REPO", raising = False) + monkeypatch.setattr(rag_config, "EMBEDDING_MODEL", "org/env-default-embedder") + + assert rag_config.default_gguf_repo() == "org/env-default-embedder-GGUF" diff --git a/studio/backend/tests/test_export_absolute_paths.py b/studio/backend/tests/test_export_absolute_paths.py index 761ea08e3f..5097f9f53a 100644 --- a/studio/backend/tests/test_export_absolute_paths.py +++ b/studio/backend/tests/test_export_absolute_paths.py @@ -158,6 +158,7 @@ def _install_lightweight_backend_stubs(monkeypatch): utils_model_config._pick_best_gguf = lambda variants: variants[0] if variants else None utils_model_config._extract_quant_label = lambda value: value utils_model_config._is_big_endian_gguf_path = lambda *args, **kwargs: False + utils_model_config._is_mtp_drafter = lambda *args, **kwargs: False utils_model_config.is_audio_input_type = lambda *args, **kwargs: None monkeypatch.setitem( sys.modules, diff --git a/studio/backend/tests/test_file_security.py b/studio/backend/tests/test_file_security.py index b4c8f5d242..e02c33a0f1 100644 --- a/studio/backend/tests/test_file_security.py +++ b/studio/backend/tests/test_file_security.py @@ -165,6 +165,23 @@ def test_skips_local_path(): assert "local" in d.reason +def test_scans_inactive_hf_cache_snapshot_path(tmp_path): + # An inactive HF cache loads by snapshot path; the gate must recover the repo id + + # commit from models--org--repo/snapshots/ and scan that exact commit, not exempt + # it and not fall back to the default branch (an older commit may hold a dropped pickle). + snapshot = tmp_path / "models--evil--repo" / "snapshots" / "deadbeef" + snapshot.mkdir(parents = True) + status = { + "scansDone": True, + "filesWithIssues": [{"path": "pytorch_model.bin", "level": "unsafe"}], + } + with _patch_status(status) as model_info: + d = evaluate_file_security(str(snapshot)) + assert d.blocked is True + assert model_info.call_args.args[0] == "evil/repo" + assert model_info.call_args.kwargs["revision"] == "deadbeef" + + def test_remote_gguf_named_repo_is_still_scanned(): # Only LOCAL paths skip the Hub scan, so a remote .gguf repo is still scanned and a # poisoned pickle smuggled into it is blocked. diff --git a/studio/backend/tests/test_frontend_resolution.py b/studio/backend/tests/test_frontend_resolution.py index c3e0524a30..7ac2717aae 100644 --- a/studio/backend/tests/test_frontend_resolution.py +++ b/studio/backend/tests/test_frontend_resolution.py @@ -218,7 +218,7 @@ def test_systemexit_message_contains_actionable_fixes(tmp_path, monkeypatch): installer_bin = home / "unsloth_studio" / "bin" / "unsloth" tried_lines = "\n".join(f" - {p}" for p in attempted) message = ( - "[ERROR] Studio frontend build not found.\n" + "[ERROR] Unsloth frontend build not found.\n" f"Tried:\n{tried_lines}\n" "\n" "Likely cause: another 'unsloth' on PATH is shadowing the " diff --git a/studio/backend/tests/test_gemini_provider.py b/studio/backend/tests/test_gemini_provider.py index 85ceb04d27..c6ffa798d0 100644 --- a/studio/backend/tests/test_gemini_provider.py +++ b/studio/backend/tests/test_gemini_provider.py @@ -768,7 +768,7 @@ def test_cached_content_pass_through(monkeypatch): def test_boolean_caching_does_not_set_cached_content(monkeypatch): - """Studio's existing True/False signals shouldn't fabricate a cache id.""" + """Unsloth's existing True/False signals shouldn't fabricate a cache id.""" captured = _capture_body(monkeypatch, enable_prompt_caching = True) assert "cachedContent" not in captured["body"] @@ -2613,7 +2613,7 @@ def test_gemini_native_skips_orphan_function_response_for_native_part_replay(mon def test_gemini_native_part_falls_back_to_args_google(monkeypatch): """Round 27: a direct OpenAI-compat API caller (or imported third-party - thread) cannot use Studio's non-standard `tool_calls[].extra_content` + thread) cannot use Unsloth's non-standard `tool_calls[].extra_content` field, so the native_part payload round-trips through `function.arguments` as `{"google": {"native_part": {...}}}`. The synthetic-builtin detector recognizes that location, but the replay branch was only reading from diff --git a/studio/backend/tests/test_gemma4_chat_template_override.py b/studio/backend/tests/test_gemma4_chat_template_override.py index f726741aa5..9fb24a4cf6 100644 --- a/studio/backend/tests/test_gemma4_chat_template_override.py +++ b/studio/backend/tests/test_gemma4_chat_template_override.py @@ -3,7 +3,7 @@ """Auto-override of the chat template for ``unsloth/gemma-4-*-GGUF``. -Studio ships a bundled ``gemma-4.jinja`` (PR #118 based, ``preserve_thinking`` +Unsloth ships a bundled ``gemma-4.jinja`` (PR #118 based, ``preserve_thinking`` defaulted off) and applies it to gemma-4 GGUF loads via the existing ``chat_template_override`` -> ``--chat-template-file`` path, so users do not need to re-download quants. Pins the family matcher, the resolver precedence, the diff --git a/studio/backend/tests/test_gguf_load_cache_reuse.py b/studio/backend/tests/test_gguf_load_cache_reuse.py new file mode 100644 index 0000000000..6d1fac980b --- /dev/null +++ b/studio/backend/tests/test_gguf_load_cache_reuse.py @@ -0,0 +1,803 @@ +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 + +"""Tests for cached GGUF reuse and load/download exclusion. + +No GPU, network, or subprocesses are required. +""" + +from __future__ import annotations + +import asyncio +import sys +import threading +import types as _types +from pathlib import Path +from unittest.mock import patch + +import pytest + + +_BACKEND_DIR = str(Path(__file__).resolve().parent.parent) +if _BACKEND_DIR not in sys.path: + sys.path.insert(0, _BACKEND_DIR) + +# Stub optional dependencies before importing the modules under test. +_loggers_stub = _types.ModuleType("loggers") +_loggers_stub.get_logger = lambda name: __import__("logging").getLogger(name) +sys.modules.setdefault("loggers", _loggers_stub) + +_structlog_stub = _types.ModuleType("structlog") +sys.modules.setdefault("structlog", _structlog_stub) + +try: + import httpx # noqa: F401 +except ImportError: + _httpx_stub = _types.ModuleType("httpx") + for _exc_name in ( + "ConnectError", + "TimeoutException", + "ReadTimeout", + "ReadError", + "RemoteProtocolError", + "CloseError", + "HTTPError", + "RequestError", + "HTTPStatusError", + ): + setattr(_httpx_stub, _exc_name, type(_exc_name, (Exception,), {})) + _httpx_stub.Response = type("Response", (), {}) + _httpx_stub.Request = type("Request", (), {}) + + class _FakeTimeout: + def __init__(self, *a, **kw): + pass + + _httpx_stub.Timeout = _FakeTimeout + _httpx_stub.Client = type( + "Client", + (), + { + "__init__": lambda self, **kw: None, + "__enter__": lambda self: self, + "__exit__": lambda self, *a: None, + }, + ) + sys.modules.setdefault("httpx", _httpx_stub) + + +from huggingface_hub import constants as hf_constants + +from core.inference.llama_cpp import ( + LlamaCppBackend, + cached_gguf_for_load, + gguf_load_in_flight, + hf_gguf_load_in_flight, +) + + +REPO = "unsloth/gemma-test-GGUF" +VARIANT = "UD-Q4_K_XL" +MAIN = f"gemma-test-{VARIANT}.gguf" + + +def _build_cache( + root: Path, + repo_id: str, + files: dict[str, int], + *, + snapshot_sha: str = "a" * 40, +) -> Path: + """Create ``$root/models--/snapshots//`` for each entry.""" + repo_dir = root / f"models--{repo_id.replace('/', '--')}" + (repo_dir / "blobs").mkdir(parents = True, exist_ok = True) + snap = repo_dir / "snapshots" / snapshot_sha + snap.mkdir(parents = True, exist_ok = True) + for rel, size in files.items(): + full = snap / rel + full.parent.mkdir(parents = True, exist_ok = True) + full.write_bytes(b"\0" * size) + return snap + + +@pytest.fixture +def hf_cache(tmp_path, monkeypatch): + monkeypatch.setattr(hf_constants, "HF_HUB_CACHE", str(tmp_path)) + monkeypatch.setattr( + "utils.hf_cache_settings.get_hf_cache_paths", + lambda: _types.SimpleNamespace(hub_cache = tmp_path), + ) + monkeypatch.delenv("HF_HUB_OFFLINE", raising = False) + monkeypatch.delenv("TRANSFORMERS_OFFLINE", raising = False) + return tmp_path + + +def _fail_download(*_args, **_kwargs): + raise AssertionError("must reuse the cached GGUF instead of downloading") + + +def _fail_get_paths_info(*_args, **_kwargs): + raise AssertionError("cached reuse must return before the sizing preflight") + + +class TestLoadReusesCachedCopy: + def test_download_uses_selected_cache_for_lookup_preflight_and_write( + self, tmp_path, monkeypatch + ): + backend = LlamaCppBackend() + selected = tmp_path / "selected" / "hub" + startup = tmp_path / "startup" / "hub" + monkeypatch.setattr(hf_constants, "HF_HUB_CACHE", str(startup)) + monkeypatch.setattr( + "utils.hf_cache_settings.get_hf_cache_paths", + lambda: _types.SimpleNamespace(hub_cache = selected), + ) + seen = {"lookups": [], "disk": [], "downloads": []} + + def cached_lookup( + repo_id, + filename, + *, + cache_dir = None, + **_kwargs, + ): + seen["lookups"].append((repo_id, filename, cache_dir)) + return None + + def disk_usage(path): + seen["disk"].append(str(path)) + return _types.SimpleNamespace(free = 1024) + + def download(repo_id, filename, _token, **kwargs): + seen["downloads"].append((repo_id, filename, kwargs.get("cache_dir"))) + return str(selected / filename) + + with ( + patch("huggingface_hub.list_repo_files", lambda *_a, **_k: [MAIN]), + patch( + "huggingface_hub.get_paths_info", + lambda _repo, paths, **_kwargs: [ + _types.SimpleNamespace(path = path, size = 4) for path in paths + ], + ), + patch("huggingface_hub.try_to_load_from_cache", cached_lookup), + patch("core.inference.llama_cpp.shutil.disk_usage", disk_usage), + patch( + "core.inference.llama_cpp.hf_hub_download_with_xet_fallback", + download, + ), + ): + out = backend._download_gguf(hf_repo = REPO, hf_variant = VARIANT) + + assert out == str(selected / MAIN) + assert seen == { + "lookups": [(REPO, MAIN, str(selected))], + "disk": [str(selected)], + "downloads": [(REPO, MAIN, str(selected))], + } + + def test_online_reuse_after_revision_bump(self, hf_cache): + """A new repo revision does not replace a complete cached model.""" + backend = LlamaCppBackend() + snap = _build_cache(hf_cache, REPO, {MAIN: 4}) + + with ( + patch("huggingface_hub.list_repo_files", lambda *_a, **_k: [MAIN]), + patch("huggingface_hub.get_paths_info", _fail_get_paths_info), + patch("core.inference.llama_cpp.hf_hub_download_with_xet_fallback", _fail_download), + ): + out = backend._download_gguf(hf_repo = REPO, hf_variant = VARIANT) + + assert out == str(snap / MAIN) + + def test_reuse_size_check_uses_cached_snapshot_revision(self, hf_cache): + """Current-revision size changes do not invalidate an older complete copy.""" + backend = LlamaCppBackend() + snap = _build_cache(hf_cache, REPO, {MAIN: 4}) + revisions: list[str | None] = [] + + def fake_get_paths_info( + _repo, + paths, + *, + revision = None, + token = None, + ): + revisions.append(revision) + size = 4 if revision == snap.name else 8 + return [_types.SimpleNamespace(path = path, size = size) for path in paths] + + with ( + patch("huggingface_hub.list_repo_files", lambda *_a, **_k: [MAIN]), + patch("huggingface_hub.get_paths_info", fake_get_paths_info), + patch("core.inference.llama_cpp.hf_hub_download_with_xet_fallback", _fail_download), + ): + out = backend._download_gguf(hf_repo = REPO, hf_variant = VARIANT) + + assert out == str(snap / MAIN) + assert revisions == [snap.name] + + def test_reuse_when_cached_revision_vanished_from_hub(self, hf_cache): + """The Hub answers an unknown revision with an empty result, not an error.""" + backend = LlamaCppBackend() + snap = _build_cache(hf_cache, REPO, {MAIN: 4}) + + with ( + patch("huggingface_hub.list_repo_files", lambda *_a, **_k: [MAIN]), + patch("huggingface_hub.get_paths_info", lambda *_a, **_k: []), + patch("core.inference.llama_cpp.hf_hub_download_with_xet_fallback", _fail_download), + ): + out = backend._download_gguf(hf_repo = REPO, hf_variant = VARIANT) + + assert out == str(snap / MAIN) + + def test_truncated_cached_file_is_not_reused(self, hf_cache): + backend = LlamaCppBackend() + _build_cache(hf_cache, REPO, {MAIN: 4}) + downloaded: list[str] = [] + + def fake_get_paths_info( + _repo, + paths, + *, + revision = None, + token = None, + ): + return [_types.SimpleNamespace(path = path, size = 8) for path in paths] + + def fake_download( + repo_id, + filename, + token = None, + **_kwargs, + ): + downloaded.append(filename) + return f"/fake/{repo_id}/{filename}" + + with ( + patch("huggingface_hub.list_repo_files", lambda *_a, **_k: [MAIN]), + patch("huggingface_hub.get_paths_info", fake_get_paths_info), + patch("huggingface_hub.try_to_load_from_cache", lambda *_a, **_k: None), + patch("core.inference.llama_cpp.hf_hub_download_with_xet_fallback", fake_download), + ): + out = backend._download_gguf(hf_repo = REPO, hf_variant = VARIANT) + + assert downloaded == [MAIN] + assert out == f"/fake/{REPO}/{MAIN}" + + def test_truncated_cached_split_shard_is_not_reused(self, hf_cache): + backend = LlamaCppBackend() + shard1 = f"gemma-test-{VARIANT}-00001-of-00002.gguf" + shard2 = f"gemma-test-{VARIANT}-00002-of-00002.gguf" + _build_cache(hf_cache, REPO, {shard1: 8, shard2: 4}) + downloaded: list[str] = [] + + def fake_get_paths_info( + _repo, + paths, + *, + revision = None, + token = None, + ): + return [_types.SimpleNamespace(path = path, size = 8) for path in paths] + + def fake_download( + repo_id, + filename, + token = None, + **_kwargs, + ): + downloaded.append(filename) + return f"/fake/{repo_id}/{filename}" + + with ( + patch("huggingface_hub.list_repo_files", lambda *_a, **_k: [shard1, shard2]), + patch("huggingface_hub.get_paths_info", fake_get_paths_info), + patch("huggingface_hub.try_to_load_from_cache", lambda *_a, **_k: None), + patch("core.inference.llama_cpp.hf_hub_download_with_xet_fallback", fake_download), + ): + out = backend._download_gguf(hf_repo = REPO, hf_variant = VARIANT) + + assert downloaded == [shard1, shard2] + assert out == f"/fake/{REPO}/{shard1}" + + def test_online_reuse_when_reupload_renamed_the_file(self, hf_cache): + """A renamed variant still reuses its cached file.""" + backend = LlamaCppBackend() + old_name = f"gemma-test-old-{VARIANT}.gguf" + snap = _build_cache(hf_cache, REPO, {old_name: 4}) + + with ( + patch("huggingface_hub.list_repo_files", lambda *_a, **_k: [MAIN]), + patch("huggingface_hub.get_paths_info", _fail_get_paths_info), + patch("core.inference.llama_cpp.hf_hub_download_with_xet_fallback", _fail_download), + ): + out = backend._download_gguf(hf_repo = REPO, hf_variant = VARIANT) + + assert out == str(snap / old_name) + + def test_downloads_when_nothing_cached(self, hf_cache): + backend = LlamaCppBackend() + downloaded: list[str] = [] + + def fake_download( + repo_id, + filename, + token = None, + **_kwargs, + ): + downloaded.append(filename) + return f"/fake/{repo_id}/{filename}" + + def fake_get_paths_info( + _repo_id, + paths, + token = None, + ): + return [_types.SimpleNamespace(path = p, size = 1) for p in paths if p is not None] + + with ( + patch("huggingface_hub.list_repo_files", lambda *_a, **_k: [MAIN]), + patch("huggingface_hub.get_paths_info", fake_get_paths_info), + patch("huggingface_hub.try_to_load_from_cache", lambda *_a, **_k: None), + patch("core.inference.llama_cpp.hf_hub_download_with_xet_fallback", fake_download), + ): + out = backend._download_gguf(hf_repo = REPO, hf_variant = VARIANT) + + assert downloaded == [MAIN] + assert out == f"/fake/{REPO}/{MAIN}" + + def test_force_redownloads_despite_cache(self, hf_cache): + """A forced download ignores a complete cached copy.""" + backend = LlamaCppBackend() + _build_cache(hf_cache, REPO, {MAIN: 4}) + downloaded: list[str] = [] + + def fake_download( + repo_id, + filename, + token = None, + **kwargs, + ): + assert kwargs.get("force_download") is True + downloaded.append(filename) + return f"/fake/{repo_id}/{filename}" + + def fake_get_paths_info( + _repo_id, + paths, + token = None, + ): + return [_types.SimpleNamespace(path = p, size = 1) for p in paths if p is not None] + + with ( + patch("huggingface_hub.list_repo_files", lambda *_a, **_k: [MAIN]), + patch("huggingface_hub.get_paths_info", fake_get_paths_info), + patch("huggingface_hub.try_to_load_from_cache", lambda *_a, **_k: None), + patch("core.inference.llama_cpp.hf_hub_download_with_xet_fallback", fake_download), + ): + out = backend._download_gguf(hf_repo = REPO, hf_variant = VARIANT, force = True) + + assert downloaded == [MAIN] + assert out == f"/fake/{REPO}/{MAIN}" + + def test_split_reused_only_when_colocated(self, hf_cache): + backend = LlamaCppBackend() + shard1 = f"gemma-test-{VARIANT}-00001-of-00002.gguf" + shard2 = f"gemma-test-{VARIANT}-00002-of-00002.gguf" + snap = _build_cache(hf_cache, REPO, {shard1: 4, shard2: 4}) + + with ( + patch("huggingface_hub.list_repo_files", lambda *_a, **_k: [shard1, shard2]), + patch("huggingface_hub.get_paths_info", _fail_get_paths_info), + patch("core.inference.llama_cpp.hf_hub_download_with_xet_fallback", _fail_download), + ): + out = backend._download_gguf(hf_repo = REPO, hf_variant = VARIANT) + + assert out == str(snap / shard1) + + def test_partial_split_set_downloads(self, hf_cache): + """A partial split set is not reused.""" + backend = LlamaCppBackend() + shard1 = f"gemma-test-{VARIANT}-00001-of-00002.gguf" + shard2 = f"gemma-test-{VARIANT}-00002-of-00002.gguf" + _build_cache(hf_cache, REPO, {shard1: 4}) + downloaded: list[str] = [] + + def fake_download( + repo_id, + filename, + token = None, + **_kwargs, + ): + downloaded.append(filename) + return f"/fake/{repo_id}/{filename}" + + def fake_get_paths_info( + _repo_id, + paths, + token = None, + ): + return [_types.SimpleNamespace(path = p, size = 4) for p in paths if p is not None] + + with ( + patch("huggingface_hub.list_repo_files", lambda *_a, **_k: [shard1, shard2]), + patch("huggingface_hub.get_paths_info", fake_get_paths_info), + patch("huggingface_hub.try_to_load_from_cache", lambda *_a, **_k: None), + patch("core.inference.llama_cpp.hf_hub_download_with_xet_fallback", fake_download), + ): + out = backend._download_gguf(hf_repo = REPO, hf_variant = VARIANT) + + assert downloaded == [shard1, shard2] + assert out == f"/fake/{REPO}/{shard1}" + + def test_reuse_prefers_newest_snapshot_after_update(self, hf_cache): + """Loads prefer the newest complete snapshot.""" + import os + + backend = LlamaCppBackend() + old_snap = _build_cache(hf_cache, REPO, {MAIN: 4}, snapshot_sha = "a" * 40) + new_snap = _build_cache(hf_cache, REPO, {MAIN: 6}, snapshot_sha = "b" * 40) + os.utime(old_snap, (1_000_000, 1_000_000)) + os.utime(new_snap, (2_000_000, 2_000_000)) + + with ( + patch("huggingface_hub.list_repo_files", lambda *_a, **_k: [MAIN]), + patch("huggingface_hub.get_paths_info", _fail_get_paths_info), + patch("core.inference.llama_cpp.hf_hub_download_with_xet_fallback", _fail_download), + ): + out = backend._download_gguf(hf_repo = REPO, hf_variant = VARIANT) + + assert out == str(new_snap / MAIN) + + def test_low_disk_fallback_reuses_cached_copy(self, hf_cache): + backend = LlamaCppBackend() + fallback = "gemma-test-Q2_K.gguf" + snap = _build_cache(hf_cache, REPO, {fallback: 4}) + + def fake_get_paths_info( + _repo, + paths, + *, + revision = None, + token = None, + ): + size = 4 if revision == snap.name else 100 + return [_types.SimpleNamespace(path = path, size = size) for path in paths] + + with ( + patch("huggingface_hub.list_repo_files", lambda *_a, **_k: [MAIN]), + patch("huggingface_hub.get_paths_info", fake_get_paths_info), + patch("huggingface_hub.try_to_load_from_cache", lambda *_a, **_k: None), + patch("shutil.disk_usage", lambda *_a, **_k: _types.SimpleNamespace(free = 10)), + patch.object( + backend, + "_find_smallest_fitting_variant", + lambda *_a, **_k: (fallback, 4, []), + ), + patch("core.inference.llama_cpp.hf_hub_download_with_xet_fallback", _fail_download), + ): + out = backend._download_gguf(hf_repo = REPO, hf_variant = VARIANT) + + assert out == str(snap / fallback) + + def test_companion_prefers_main_snapshot_sibling(self, hf_cache): + """A cached mmproj is reused from the main model's snapshot.""" + backend = LlamaCppBackend() + snap = _build_cache(hf_cache, REPO, {MAIN: 4, "mmproj-F16.gguf": 2}) + + def _fail_list(*_args, **_kwargs): + raise AssertionError("snapshot sibling must resolve without a repo listing") + + with patch("huggingface_hub.list_repo_files", _fail_list): + out = backend._download_mmproj(hf_repo = REPO, near_path = str(snap / MAIN)) + + assert out == str(snap / "mmproj-F16.gguf") + + def test_companion_finds_snapshot_through_hf_symlink(self, hf_cache): + backend = LlamaCppBackend() + snap = _build_cache(hf_cache, REPO, {}) + blobs = snap.parent.parent / "blobs" + main_blob = blobs / "main" + mmproj_blob = blobs / "mmproj" + main_blob.write_bytes(b"main") + mmproj_blob.write_bytes(b"mmproj") + try: + (snap / MAIN).symlink_to(main_blob) + (snap / "mmproj-F16.gguf").symlink_to(mmproj_blob) + except OSError as exc: + pytest.skip(f"symlinks unavailable: {exc}") + + with patch("huggingface_hub.list_repo_files", _fail_download): + out = backend._download_mmproj(hf_repo = REPO, near_path = str(snap / MAIN)) + + assert out == str(snap / "mmproj-F16.gguf") + + def test_companion_does_not_download_during_hub_job(self, hf_cache): + backend = LlamaCppBackend() + snap = _build_cache(hf_cache, REPO, {MAIN: 4}) + registry = _types.SimpleNamespace(active_job_refs = lambda _repo: [object()]) + + with ( + patch("huggingface_hub.list_repo_files", _fail_download), + patch("hub.utils.download_registry.get_models_registry", lambda: registry), + patch("core.inference.llama_cpp.hf_hub_download_with_xet_fallback", _fail_download), + ): + out = backend._download_mmproj(hf_repo = REPO, near_path = str(snap / MAIN)) + + assert out is None + + +class TestCachedGgufForLoadProbe: + def test_complete_copy_found(self, hf_cache): + snap = _build_cache(hf_cache, REPO, {MAIN: 4}) + assert cached_gguf_for_load(REPO, VARIANT) == str(snap / MAIN) + + def test_absent_copy_is_none(self, hf_cache): + assert cached_gguf_for_load(REPO, VARIANT) is None + + def test_partial_split_is_none(self, hf_cache): + shard1 = f"gemma-test-{VARIANT}-00001-of-00002.gguf" + _build_cache(hf_cache, REPO, {shard1: 4}) + assert cached_gguf_for_load(REPO, VARIANT) is None + + def test_partial_new_snapshot_does_not_hide_complete_split(self, hf_cache): + import os + + shard1 = f"gemma-test-{VARIANT}-00001-of-00002.gguf" + shard2 = f"gemma-test-{VARIANT}-00002-of-00002.gguf" + old = _build_cache( + hf_cache, + REPO, + {shard1: 4, shard2: 4}, + snapshot_sha = "a" * 40, + ) + new = _build_cache(hf_cache, REPO, {shard1: 4}, snapshot_sha = "b" * 40) + os.utime(old, (1_000_000, 1_000_000)) + os.utime(new, (2_000_000, 2_000_000)) + + assert cached_gguf_for_load(REPO, VARIANT) == str(old / shard1) + + def test_split_requires_every_declared_shard(self, hf_cache): + shard1 = f"gemma-test-{VARIANT}-00001-of-00003.gguf" + shard2 = f"gemma-test-{VARIANT}-00002-of-00003.gguf" + _build_cache(hf_cache, REPO, {shard1: 4, shard2: 4}) + + assert cached_gguf_for_load(REPO, VARIANT) is None + + def test_required_mmproj_must_share_main_snapshot(self, hf_cache): + snap = _build_cache(hf_cache, REPO, {MAIN: 4}) + assert cached_gguf_for_load(REPO, VARIANT) == str(snap / MAIN) + assert cached_gguf_for_load(REPO, VARIANT, require_mmproj = True) is None + + (snap / "mmproj-F16.gguf").write_bytes(b"mmproj") + assert cached_gguf_for_load(REPO, VARIANT, require_mmproj = True) == str(snap / MAIN) + + def test_required_mmproj_scans_past_newer_main_only_snapshot(self, hf_cache): + import os + + old = _build_cache( + hf_cache, + REPO, + {MAIN: 4, "mmproj-F16.gguf": 2}, + snapshot_sha = "a" * 40, + ) + new = _build_cache(hf_cache, REPO, {MAIN: 4}, snapshot_sha = "b" * 40) + os.utime(old, (1_000_000, 1_000_000)) + os.utime(new, (2_000_000, 2_000_000)) + + assert cached_gguf_for_load(REPO, VARIANT, require_mmproj = True) == str(old / MAIN) + + +class TestLoadHubDownloadExclusion: + def test_in_flight_marker_counts_and_normalizes_case(self): + assert not hf_gguf_load_in_flight(REPO) + with gguf_load_in_flight(REPO): + assert hf_gguf_load_in_flight(REPO.upper()) + with gguf_load_in_flight(REPO.lower()): + assert hf_gguf_load_in_flight(REPO) + assert hf_gguf_load_in_flight(REPO) + assert not hf_gguf_load_in_flight(REPO) + + def test_marker_noops_for_local_loads(self): + with gguf_load_in_flight(None): + assert not hf_gguf_load_in_flight("") + + def test_marker_cleared_on_exception(self): + with pytest.raises(RuntimeError): + with gguf_load_in_flight(REPO): + raise RuntimeError("boom") + assert not hf_gguf_load_in_flight(REPO) + + def test_hub_download_refused_while_load_in_flight(self): + from fastapi import HTTPException + + from hub.schemas.downloads import DownloadModelRequest + from hub.services.models import downloads as dl + + body = DownloadModelRequest(repo_id = REPO, gguf_variant = VARIANT) + with ( + patch.object(dl, "resolve_cached_repo_id_case", lambda repo_id, repo_type: repo_id), + gguf_load_in_flight(REPO), + ): + with pytest.raises(HTTPException) as exc_info: + asyncio.run(dl.download_model_response(body)) + + assert exc_info.value.status_code == 409 + assert "load" in exc_info.value.detail.lower() + + def test_hub_download_rechecks_marker_before_claim(self): + from fastapi import HTTPException + + from hub.schemas.downloads import DownloadModelRequest + from hub.services.models import downloads as dl + + scope = None + + def mark_load(*_args, **_kwargs): + nonlocal scope + if scope is None: + scope = gguf_load_in_flight(REPO) + scope.__enter__() + return frozenset() + + class _Registry: + def claim(self, *_args, admission_check, **_kwargs): + assert admission_check() is False + return False, "admission_blocked" + + def current_generation(self, _key): + return 0 + + registry = _Registry() + body = DownloadModelRequest(repo_id = REPO, gguf_variant = VARIANT) + try: + with ( + patch.object(dl, "resolve_cached_repo_id_case", lambda repo_id, repo_type: repo_id), + patch.object(dl.gguf_variants, "gguf_variant_blob_hashes", mark_load), + patch.object(dl, "_registry", registry), + ): + with pytest.raises(HTTPException) as exc_info: + asyncio.run(dl.download_model_response(body)) + finally: + if scope is not None: + scope.__exit__(None, None, None) + + assert exc_info.value.status_code == 409 + + def test_registry_admission_check_prevents_claim(self): + from hub.utils.download_registry import DownloadRegistry, TRANSPORT_HTTP + + registry = DownloadRegistry() + claimed, state = registry.claim( + f"{REPO}::{VARIANT}", + TRANSPORT_HTTP, + repo_type = "model", + repo_id = REPO, + variant = VARIANT, + admission_check = lambda: False, + ) + + assert claimed is False + assert state == "admission_blocked" + assert registry.active_jobs(REPO) == {} + + def test_same_variant_job_stays_visible_during_retry_handoff(self): + from hub.utils.download_registry import DownloadRegistry, TRANSPORT_XET + from core.inference.llama_cpp import _hub_download_blocks_gguf_load + + registry = DownloadRegistry() + key = f"{REPO}::{VARIANT}" + claimed, _ = registry.claim( + key, + TRANSPORT_XET, + repo_type = "model", + repo_id = REPO, + variant = VARIANT, + ) + assert claimed is True + assert registry.has_active_variant(REPO, VARIANT.lower()) is True + + registry.release_active_slot(key) + + assert registry.active_jobs(REPO) == {} + assert registry.active_job_refs(REPO) + assert registry.has_active_variant(REPO, VARIANT) is True + with ( + patch("hub.utils.download_registry.get_models_registry", lambda: registry), + patch( + "core.inference.llama_cpp.cached_gguf_for_load", + side_effect = AssertionError("same-variant jobs must block before cache reuse"), + ), + ): + assert _hub_download_blocks_gguf_load(REPO, VARIANT) is True + + registry.set_job(key, "complete") + assert registry.has_active_variant(REPO, VARIANT) is False + + def test_other_variant_job_still_allows_complete_cached_load(self): + from core.inference.llama_cpp import _hub_download_blocks_gguf_load + from hub.utils.download_registry import DownloadRegistry, TRANSPORT_HTTP + + registry = DownloadRegistry() + registry.claim( + f"{REPO}::Q8_0", + TRANSPORT_HTTP, + repo_type = "model", + repo_id = REPO, + variant = "Q8_0", + ) + with ( + patch("hub.utils.download_registry.get_models_registry", lambda: registry), + patch( + "core.inference.llama_cpp.cached_gguf_for_load", + return_value = "/cached/model.gguf", + ) as cached_probe, + ): + assert _hub_download_blocks_gguf_load(REPO, VARIANT) is False + + cached_probe.assert_called_once_with( + REPO, + VARIANT, + require_mmproj = False, + verify_sizes = True, + hf_token = None, + ) + + def test_cancelled_request_keeps_marker_until_load_thread_finishes(self): + from core.inference.llama_cpp import _with_gguf_load_marker + + started = threading.Event() + release = threading.Event() + finished = threading.Event() + + class FakeBackend: + @_with_gguf_load_marker + def load_model(self, *, hf_repo): + started.set() + release.wait(timeout = 2) + finished.set() + return True + + async def scenario(): + with patch( + "core.inference.llama_cpp._hub_download_blocks_gguf_load", + return_value = False, + ): + task = asyncio.create_task( + asyncio.to_thread(FakeBackend().load_model, hf_repo = REPO) + ) + assert await asyncio.to_thread(started.wait, 1) + task.cancel() + with pytest.raises(asyncio.CancelledError): + await task + assert hf_gguf_load_in_flight(REPO) + + release.set() + assert await asyncio.to_thread(finished.wait, 1) + for _ in range(100): + if not hf_gguf_load_in_flight(REPO): + break + await asyncio.sleep(0.001) + assert not hf_gguf_load_in_flight(REPO) + + asyncio.run(scenario()) + + def test_load_marker_precedes_hub_guard_and_unload(self): + source = (Path(__file__).resolve().parent.parent / "routes" / "inference.py").read_text() + gguf_branch = source[source.index("if config.is_gguf:") :] + + # The gguf_load_in_flight marker must be entered before the hub-download + # guard and the unload so a concurrent load can't race the download + # manager. The llama_extra_args inheritance that used to sit between the + # marker and the guard now runs in _guard_chat_load_against_training, ahead + # of the GGUF branch, so it is no longer a landmark inside this slice. + assert ( + gguf_branch.index("enter_context(gguf_load_in_flight") + < gguf_branch.index("_hub_download_blocks_gguf_load") + < gguf_branch.index("unsloth_backend.unload_model") + ) + llama_source = ( + Path(__file__).resolve().parent.parent / "core" / "inference" / "llama_cpp.py" + ).read_text() + assert "@_with_gguf_load_marker\n def load_model(" in llama_source diff --git a/studio/backend/tests/test_gguf_metadata.py b/studio/backend/tests/test_gguf_metadata.py index a5be07f8e3..ec0330ce05 100644 --- a/studio/backend/tests/test_gguf_metadata.py +++ b/studio/backend/tests/test_gguf_metadata.py @@ -15,6 +15,7 @@ from utils.models.gguf_metadata import ( pairing_score, read_gguf_context_length, read_gguf_general_metadata, + read_gguf_staged_dims, read_mmproj_audio_capability, ) @@ -153,6 +154,78 @@ def test_context_length_ignores_foreign_arch_key(tmp_path: Path): assert read_gguf_context_length(str(p)) is None +# --- read_gguf_staged_dims (one pass: context + layer + moe counts) ---- + + +def test_staged_dims_none_for_missing_or_non_gguf(tmp_path: Path): + assert read_gguf_staged_dims(str(tmp_path / "nope.gguf")) is None + p = tmp_path / "garbage.gguf" + p.write_bytes(b"not a gguf at all") + assert read_gguf_staged_dims(str(p)) is None + + +def test_staged_dims_moe_with_leading_dense(tmp_path: Path): + # GLM-4.7-Flash shape: context + total layers + MoE layers in one read. + p = _write_synthetic_gguf( + tmp_path / "glm.gguf", + {"general.architecture": "deepseek2"}, + extra_uint32 = { + "deepseek2.context_length": 202752, + "deepseek2.block_count": 47, + "deepseek2.expert_count": 64, + "deepseek2.leading_dense_block_count": 1, + }, + ) + assert read_gguf_staged_dims(str(p)) == { + "context_length": 202752, + "layer_count": 47, + "moe_layer_count": 46, + } + + +def test_staged_dims_dense_model(tmp_path: Path): + # Dense: layer_count present, moe_layer_count 0 (slider hidden). + p = _write_synthetic_gguf( + tmp_path / "dense.gguf", + {"general.architecture": "qwen3"}, + extra_uint32 = {"qwen3.context_length": 40960, "qwen3.block_count": 36}, + ) + assert read_gguf_staged_dims(str(p)) == { + "context_length": 40960, + "layer_count": 36, + "moe_layer_count": 0, + } + + +def test_staged_dims_all_moe_no_leading_dense(tmp_path: Path): + # Experts present, no leading_dense key -> every block is a MoE layer. + p = _write_synthetic_gguf( + tmp_path / "moe.gguf", + {"general.architecture": "qwen35moe"}, + extra_uint32 = {"qwen35moe.block_count": 40, "qwen35moe.expert_count": 256}, + ) + assert read_gguf_staged_dims(str(p)) == { + "context_length": None, + "layer_count": 40, + "moe_layer_count": 40, + } + + +def test_staged_dims_uint64_block_count(tmp_path: Path): + # block_count stored as uint64 (vtype 10) still parses; moe == block_count. + p = _write_synthetic_gguf( + tmp_path / "moe64.gguf", + {"general.architecture": "gpt-oss"}, + extra_uint32 = {"gpt-oss.expert_count": 32}, + extra_uint64 = {"gpt-oss.block_count": 24}, + ) + assert read_gguf_staged_dims(str(p)) == { + "context_length": None, + "layer_count": 24, + "moe_layer_count": 24, + } + + def test_context_length_read_from_uint64(tmp_path: Path): # Some models store context_length as a uint64 (vtype 10). p = _write_synthetic_gguf( diff --git a/studio/backend/tests/test_gpu_memory_mode.py b/studio/backend/tests/test_gpu_memory_mode.py new file mode 100644 index 0000000000..271a882b11 --- /dev/null +++ b/studio/backend/tests/test_gpu_memory_mode.py @@ -0,0 +1,1156 @@ +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 + +"""Backend contract for the GPU Memory mode dropdown. + +The dropdown threads a single ``gpu_memory_mode`` ("auto" | "manual") from the +chat UI through the load request. "manual" lets the user own the offload: with +``gpu_layers < 0`` (Auto, the default) it hands all memory management to +llama.cpp's ``--fit on`` (no CUDA/HIP device masking, no context auto-reduce, no +gpu-layer or tensor-split planning); with ``gpu_layers >= 0`` it pins the layers +and MoE offload itself (``--fit off``). These tests pin: + + * the pydantic request/response/status contract (snake_case key, default + "auto", unknown values rejected), + * the backend ``gpu_memory_mode`` property and its reset on unload, + * the ``_already_in_target_state`` reload-detection branch, and + * that the manual + Auto-layers branch in ``load_model`` empties the probed + GPU set and drops tensor parallelism so the selection below no-ops, while + the explicit-offload branch emits ``--gpu-layers`` / ``--fit off``. +""" + +from __future__ import annotations + +import inspect +import sys +import types as _types +from pathlib import Path + +import pytest + +_BACKEND_DIR = str(Path(__file__).resolve().parent.parent) +if _BACKEND_DIR not in sys.path: + sys.path.insert(0, _BACKEND_DIR) + +# Same external-dep stubs as the other llama_cpp unit tests so importing +# the backend doesn't drag in structlog / httpx / loggers. +_loggers_stub = _types.ModuleType("loggers") +_loggers_stub.get_logger = lambda name: __import__("logging").getLogger(name) +sys.modules.setdefault("loggers", _loggers_stub) + +_structlog_stub = _types.ModuleType("structlog") +_structlog_stub.get_logger = lambda *a, **k: __import__("logging").getLogger("stub") +sys.modules.setdefault("structlog", _structlog_stub) + +# httpx is a real, installed backend dependency: import it so the genuine module +# is in sys.modules. A hand-rolled stub here is inevitably incomplete and, since +# setdefault installs it before real httpx loads, would poison a combined pytest +# run -- routes/inference references httpx.Response (and other attrs) at def time. +import httpx # noqa: F401 + +from core.inference import llama_cpp as llama_cpp_module +from core.inference.llama_cpp import LlamaCppBackend +from models.inference import ( + InferenceStatusResponse, + LoadRequest, + LoadResponse, +) + + +# ── Pydantic contract (snake_case key, default "auto") ─────────────── + + +def test_load_request_defaults_gpu_memory_mode_auto(): + assert LoadRequest(model_path = "owner/repo").gpu_memory_mode == "auto" + + +def test_load_request_round_trips_json_key(): + req = LoadRequest.model_validate({"model_path": "owner/repo", "gpu_memory_mode": "manual"}) + assert req.gpu_memory_mode == "manual" + assert req.model_dump()["gpu_memory_mode"] == "manual" + + +def test_load_request_rejects_unknown_mode(): + with pytest.raises(ValueError): + LoadRequest(model_path = "owner/repo", gpu_memory_mode = "bogus") + + +@pytest.mark.parametrize("model_cls", [LoadResponse, InferenceStatusResponse]) +def test_response_models_emit_gpu_memory_mode(model_cls): + if model_cls is LoadResponse: + default = model_cls( + status = "loaded", + model = "owner/repo", + display_name = "repo", + inference = {}, + ) + manual = model_cls( + status = "loaded", + model = "owner/repo", + display_name = "repo", + inference = {}, + gpu_memory_mode = "manual", + ) + else: + default = model_cls() + manual = model_cls(gpu_memory_mode = "manual") + assert default.model_dump()["gpu_memory_mode"] == "auto" + assert manual.model_dump()["gpu_memory_mode"] == "manual" + + +# ── Backend property + reset ───────────────────────────────────────── + + +class _FakeProcess: + """Stand-in for subprocess.Popen so _kill_process is a no-op.""" + + def terminate(self): + pass + + def wait(self, timeout = None): + return 0 + + def kill(self): + pass + + def poll(self): + return 0 + + +def test_gpu_memory_mode_property_defaults_auto(): + assert LlamaCppBackend().gpu_memory_mode == "auto" + + +def test_gpu_memory_mode_property_reflects_field(): + backend = LlamaCppBackend() + backend._gpu_memory_mode = "manual" + assert backend.gpu_memory_mode == "manual" + + +def test_unload_resets_gpu_memory_mode(): + backend = LlamaCppBackend() + backend._process = _FakeProcess() + backend._gpu_memory_mode = "manual" + backend.unload_model() + assert backend.gpu_memory_mode == "auto" + + +# ── _already_in_target_state reload-detection branch ───────────────── + + +def _loaded_backend(gpu_memory_mode: str) -> LlamaCppBackend: + backend = LlamaCppBackend() + backend._process = _FakeProcess() # is_loaded only checks "is not None" + backend._healthy = True + backend._model_identifier = "owner/repo" + backend._hf_variant = "Q4_K_M" + backend._requested_n_ctx = 8192 + backend._cache_type_kv = None + backend._requested_spec_mode = "auto" + backend._chat_template_override = None + backend._is_vision = False + backend._extra_args = None + backend._gguf_path = None + backend._gpu_memory_mode = gpu_memory_mode + return backend + + +def _target_state(backend: LlamaCppBackend, gpu_memory_mode: str) -> bool: + return backend._already_in_target_state( + gguf_path = None, + model_identifier = "owner/repo", + hf_variant = "Q4_K_M", + n_ctx = 8192, + cache_type_kv = None, + speculative_type = "auto", + chat_template_override = None, + extra_args = None, + is_vision = False, + gpu_memory_mode = gpu_memory_mode, + ) + + +@pytest.mark.parametrize("mode", ["auto", "manual"]) +def test_already_in_target_state_matches_same_mode(mode): + assert _target_state(_loaded_backend(mode), mode) is True + + +@pytest.mark.parametrize("loaded,requested", [("auto", "manual"), ("manual", "auto")]) +def test_already_in_target_state_reloads_on_mode_change(loaded, requested): + # Flipping the dropdown either direction must force a reload so the command + # is rebuilt with/without the Unsloth GPU masking. + assert _target_state(_loaded_backend(loaded), requested) is False + + +def test_already_in_target_state_ignores_mode_for_diffusion(): + # The diffusion runner is mode-agnostic (always "auto"), so a standing manual + # preference must not force a needless reload. + backend = _loaded_backend("auto") + backend._is_diffusion = True + assert _target_state(backend, "manual") is True + + +# ── load_model: manual + Auto layers bypasses Unsloth GPU management ── + + +def _load_model_source() -> str: + return inspect.getsource(llama_cpp_module.LlamaCppBackend.load_model) + + +def test_auto_layers_branch_empties_gpus_and_drops_tensor_parallel(): + # Emptying the probed set makes the selection / TP planning below no-op, so + # gpu_indices stays None and use_fit True (--fit on). + src = _load_model_source() + gate = src.find('if gpu_memory_mode == "manual" and gpu_layers < 0:') + assert gate != -1, "load_model must branch on manual + Auto layers (gpu_layers < 0)" + block = src[gate : gate + 1400] + assert "gpus = []" in block, "Auto-layers branch must empty the probed GPU set" + # --fit aborts under --split-mode tensor, so a raw-extras split-mode is stripped. + assert "strip_split_mode_only(extra_args)" in block + assert "requested_ctx if requested_ctx > 0 else 0" in block + # The branch sits before GPU selection assigns gpu_indices; --fit on is its emission. + assert gate < src.find("gpu_indices, use_fit = None, True") + assert 'cmd.extend(["--fit", "on"])' in src + # TP drops for this path, but at a guard BEFORE the quantized-KV cache-drop, so + # a requested quantized cache survives into the --fit load. + tp_drop = src.find('if tensor_parallel and gpu_memory_mode == "manual" and gpu_layers < 0:') + assert tp_drop != -1, "manual + Auto layers must drop tensor_parallel" + assert "tensor_parallel = False" in src[tp_drop : tp_drop + 400] + cache_drop = src.find("Tensor parallelism requires a non-quantized KV cache") + assert cache_drop != -1 + assert ( + tp_drop < cache_drop + ), "TP must drop before the cache-drop so a quantized KV survives --fit" + + +def test_auto_layers_never_sends_ctx_size_zero(): + # Sending "-c 0" sets fit_params_min_ctx = UINT32_MAX in llama.cpp, pinning + # the full native context and disabling --fit's reduction. So the base cmd + # must never carry -c, "-c 0" is emitted only outside the Auto-layers (--fit) + # case, and a positive context is passed through (which --fit optimizes + # layers around). + src = _load_model_source() + base_start = src.find("cmd = [") + base_end = src.find("\n ]", base_start) + base_block = src[base_start:base_end] + assert '"-c"' not in base_block, "-c must be conditional, not in the base cmd list" + assert 'cmd.extend(["-c", str(effective_ctx)])' in src, "positive ctx must pass -c" + assert 'auto_fit = gpu_memory_mode == "manual" and gpu_layers < 0' in src + zero = src.find('cmd.extend(["-c", "0"])') + assert zero != -1, '"-c 0" emission must exist outside the Auto-layers case' + guard = src.rfind("elif not auto_fit:", 0, zero) + assert guard != -1 and zero - guard < 120, '"-c 0" must sit under the not-auto_fit guard' + + +def test_manual_mode_clears_inherited_main_model_placement_env(): + env = {name: "inherited" for name in LlamaCppBackend._MANUAL_PLACEMENT_ENV_VARS} + env["LLAMA_ARG_N_GPU_LAYERS_DRAFT"] = "7" + env["UNRELATED"] = "kept" + + LlamaCppBackend._clear_manual_placement_env(env) + + assert not (set(env) & set(LlamaCppBackend._MANUAL_PLACEMENT_ENV_VARS)) + assert env["LLAMA_ARG_N_GPU_LAYERS_DRAFT"] == "7" + assert env["UNRELATED"] == "kept" + + +def test_load_model_sanitizes_manual_env_after_building_child_env(): + src = _load_model_source() + env_build = src.find("env = self._llama_server_env_for_binary(binary)") + env_clear = src.find("self._clear_manual_placement_env(env)", env_build) + launch = src.find("subprocess.Popen", env_build) + assert env_build != -1 + assert env_build < env_clear < launch + + +# ── Manual offload (--gpu-layers + --fit off + --n-cpu-moe) ─────────── + + +def test_load_request_accepts_manual(): + req = LoadRequest( + model_path = "owner/repo", + gpu_memory_mode = "manual", + gpu_layers = 20, + n_cpu_moe = 8, + tensor_split = [2, 1], + ) + assert req.gpu_memory_mode == "manual" + assert req.gpu_layers == 20 + assert req.n_cpu_moe == 8 + assert req.tensor_split == [2, 1] + + +def test_load_request_manual_defaults(): + req = LoadRequest(model_path = "owner/repo") + assert req.gpu_layers == -1 + assert req.n_cpu_moe == 0 + assert req.tensor_split is None + + +@pytest.mark.parametrize("bad", [[0, 0], [-1, 2], [float("inf"), 1], [float("nan"), 1]]) +def test_load_request_rejects_degenerate_tensor_split(bad): + # A negative/non-finite/all-zero split is dropped at launch but compared raw + # in the reload dedupe, so it would reload forever -- reject it up front. + with pytest.raises(ValueError): + LoadRequest(model_path = "owner/repo", tensor_split = bad) + + +@pytest.mark.parametrize("good", [[2, 1], [1, 1], [], None]) +def test_load_request_accepts_valid_tensor_split(good): + assert LoadRequest(model_path = "owner/repo", tensor_split = good).tensor_split == good + + +def test_route_normalizes_explicit_extras_before_reload_dedupe(): + route_src = (Path(_BACKEND_DIR) / "routes" / "inference.py").read_text(encoding = "utf-8") + load_impl = route_src[route_src.index("async def _load_model_impl") :] + strip = load_impl.index("_stripped_explicit = strip_shadowing_flags") + normalize = load_impl.index( + 'request = request.model_copy(update = {"llama_extra_args": extra_llama_args})' + ) + dedupe = load_impl.index("and _request_matches_loaded_settings(") + assert strip < normalize < dedupe + + +@pytest.mark.parametrize("model_cls", [LoadResponse, InferenceStatusResponse]) +def test_response_models_emit_manual_fields(model_cls): + if model_cls is LoadResponse: + obj = model_cls( + status = "loaded", + model = "owner/repo", + display_name = "repo", + inference = {}, + gpu_memory_mode = "manual", + gpu_layers = 20, + n_cpu_moe = 8, + tensor_split = [2, 1], + n_layers = 32, + n_moe_layers = 32, + ) + else: + obj = model_cls( + gpu_memory_mode = "manual", + gpu_layers = 20, + n_cpu_moe = 8, + tensor_split = [2, 1], + n_layers = 32, + n_moe_layers = 32, + ) + dumped = obj.model_dump() + assert dumped["gpu_memory_mode"] == "manual" + assert dumped["gpu_layers"] == 20 + assert dumped["n_cpu_moe"] == 8 + assert dumped["tensor_split"] == [2, 1] + assert dumped["n_layers"] == 32 + assert dumped["n_moe_layers"] == 32 + + +def test_manual_properties_default_and_reflect_and_reset(): + backend = LlamaCppBackend() + assert backend.gpu_layers == -1 and backend.n_cpu_moe == 0 + assert backend.tensor_split is None + backend._gpu_layers = 20 + backend._n_cpu_moe = 8 + backend._tensor_split = [2, 1] + assert backend.gpu_layers == 20 and backend.n_cpu_moe == 8 + assert backend.tensor_split == [2, 1] + backend._process = _FakeProcess() + backend.unload_model() + assert backend.gpu_layers == -1 and backend.n_cpu_moe == 0 + assert backend.tensor_split is None + + +def test_n_moe_layers_property(): + # 0 for a dense model (hides the slider); block_count for all-MoE; + # block_count - leading_dense otherwise (GLM-4.7-Flash: 47 - 1 -> 46). + b = LlamaCppBackend() + b._n_layers = 36 + b._n_experts = None + assert b.n_moe_layers == 0 + b._n_experts = 128 + b._leading_dense_block_count = None + assert b.n_moe_layers == 36 + b._n_layers = 47 + b._leading_dense_block_count = 1 + assert b.n_moe_layers == 46 + + +def _target_state_manual( + backend, + *, + gpu_layers, + n_cpu_moe, + tensor_split = None, +): + return backend._already_in_target_state( + gguf_path = None, + model_identifier = "owner/repo", + hf_variant = "Q4_K_M", + n_ctx = 8192, + cache_type_kv = None, + speculative_type = "auto", + chat_template_override = None, + extra_args = None, + is_vision = False, + gpu_memory_mode = "manual", + gpu_layers = gpu_layers, + n_cpu_moe = n_cpu_moe, + tensor_split = tensor_split, + ) + + +def test_manual_reloads_on_gpu_layers_or_n_cpu_moe_or_split_change(): + backend = _loaded_backend("manual") + backend._gpu_layers = 20 + backend._n_cpu_moe = 0 + backend._tensor_split = None + # Same knobs -> no reload. + assert _target_state_manual(backend, gpu_layers = 20, n_cpu_moe = 0) is True + # Changed layer count -> reload. + assert _target_state_manual(backend, gpu_layers = 16, n_cpu_moe = 0) is False + # Changed MoE offload -> reload. + assert _target_state_manual(backend, gpu_layers = 20, n_cpu_moe = 8) is False + # Added a GPU split -> reload. + assert _target_state_manual(backend, gpu_layers = 20, n_cpu_moe = 0, tensor_split = [2, 1]) is False + # Same GPU split -> no reload. + backend._tensor_split = [2, 1] + assert _target_state_manual(backend, gpu_layers = 20, n_cpu_moe = 0, tensor_split = [2, 1]) is True + + +def test_auto_layers_reload_tracks_only_gpu_layers(): + # Under Auto (gpu_layers < 0) the MoE/split knobs don't apply, so a leftover + # request value must not reload -- only a gpu_layers change (Auto -> pinned) does. + backend = _loaded_backend("manual") + backend._gpu_layers = -1 + backend._n_cpu_moe = 0 + backend._tensor_split = None + # Same Auto, leftover MoE/split in the request -> still no reload. + assert _target_state_manual(backend, gpu_layers = -1, n_cpu_moe = 8, tensor_split = [2, 1]) is True + # Auto -> explicit offload reloads. + assert _target_state_manual(backend, gpu_layers = 20, n_cpu_moe = 0) is False + + +def test_manual_offload_emits_gpu_layers_fit_off_and_n_cpu_moe(): + src = _load_model_source() + gate = src.find('elif gpu_memory_mode == "manual":') + assert gate != -1, "load_model must have an explicit-offload manual branch" + block = src[gate : gate + 700] + # Empties the probed set (skips the planner) but keeps the user's TP choice + # (only the Auto-layers branch above drops TP). + assert "gpus = []" in block + assert "tensor_parallel = False" not in block + # The cmd emits the layer count with fit disabled, gated on gpu_layers >= 0. + assert 'if gpu_memory_mode == "manual" and gpu_layers >= 0:' in src + assert 'cmd.extend(["--gpu-layers", str(gpu_layers), "--fit", "off"])' in src + # MoE offload uses --n-cpu-moe via _resolve_cpu_moe_flag (tested behaviorally below). + assert "_resolve_cpu_moe_flag(" in src + assert 'cmd.extend(["--n-cpu-moe", str(moe_flag)])' in src + # A count requested on a dense model is never emitted, so it must also be + # dropped from the recorded state -- else /status and /load report a count + # llama-server never received (same rule as the tensor-split drop below). + moe_emit = src.find('cmd.extend(["--n-cpu-moe", str(moe_flag)])') + assert "elif n_cpu_moe:" in src[moe_emit : moe_emit + 300] + assert "self._n_cpu_moe = 0" in src[moe_emit : moe_emit + 300] + # The offload path forces use_fit False so --fit-ctx is never added under --fit off. + emit = src.find('cmd.extend(["--gpu-layers", str(gpu_layers), "--fit", "off"])') + assert "use_fit = False" in src[src.rfind("\n", 0, emit) - 200 : emit + 80] + + +def test_status_reports_requested_context_length(): + # The hydration path re-seeds a Manual+Auto context pin from the REQUESTED + # n_ctx (0 = Auto); context_length only exposes the resolved value. + assert "requested_context_length" in InferenceStatusResponse.model_fields + s = InferenceStatusResponse(requested_context_length = 8192) + assert s.model_dump()["requested_context_length"] == 8192 + assert InferenceStatusResponse().model_dump()["requested_context_length"] is None + # The /status route must actually wire it from the backend (a declared-but- + # never-populated field would leave hydration silently reverting the pin). + from pathlib import Path as _P + + route_src = (_P(_BACKEND_DIR) / "routes" / "inference.py").read_text(encoding = "utf-8") + assert "requested_context_length = llama_backend.requested_n_ctx" in route_src + + +def test_manual_offload_emits_tensor_split(): + # The offload path emits --tensor-split from the per-GPU shares, only when + # provided, with >1 GPU in use, AND matching that count (a stale ratio on a + # narrowed picker or a mismatched direct-API list must not emit -- llama- + # server aborts on a split/GPU-count mismatch). + src = _load_model_source() + assert "if tensor_split and _split_gpus > 1:" in src + # Emit only on a length match AND a positive sanitized total: a mismatched + # or all-zero split aborts llama-server / assigns nothing, so it's dropped. + # The emitted list is the sanitized one (clamping tested behaviorally below). + assert "_sanitized_split = self._sanitize_tensor_split(tensor_split)" in src + assert "if len(_sanitized_split) == _split_gpus and _split_total > 0:" in src + assert '"--tensor-split"' in src + # Joined as a comma list (e.g. "2,1") within the explicit-offload cmd branch. + gate = src.find('if gpu_memory_mode == "manual" and gpu_layers >= 0:') + nxt = src.find("elif use_fit:", gate) + assert '","' in src[gate:nxt] and "tensor_split" in src[gate:nxt] + # A split with a single effective GPU is never emitted, so it must also be + # dropped from the recorded state -- else /status and /load report a ratio + # llama-server never received and the dedupe baseline preserves it. + assert "elif tensor_split:" in src[gate:nxt] + drop = src.find("elif tensor_split:", gate, nxt) + assert "self._tensor_split = None" in src[drop : drop + 250] + + +def test_sanitize_tensor_split_clamps_negative_and_non_finite(): + # Negative entries would launch a placement different from the ratio the + # UI showed; inf passes a plain > 0 total gate and would emit + # "--tensor-split inf,..." (llama.cpp normalizes shares by the running + # total, so an inf poisons the shares from that entry on). Both clamp to 0. + sanitize = LlamaCppBackend._sanitize_tensor_split + assert sanitize([2, 1]) == [2.0, 1.0] + assert sanitize([-1, 2]) == [0.0, 2.0] + assert sanitize([float("inf"), 1]) == [0.0, 1.0] + assert sanitize([float("nan"), 1]) == [0.0, 1.0] + # All-zero survives sanitization; the call site's total gate drops it. + assert sanitize([0, 0]) == [0.0, 0.0] + # Unreadable input -> []; the call site's length gate drops it. + assert sanitize(["x", 1]) == [] + assert sanitize([10**400, 1]) == [] + + +def test_zero_offload_mask_honors_device_pin_spellings(): + # A user device pin must keep the GPUs visible: llama-server aborts on a + # pin it can't see ('error: invalid device'). The pin can arrive as + # --device or its -dev alias, as the draft forms (parsed even with no + # drafter loaded), or as an inherited LLAMA_ARG_DEVICE env var. + load_src = _load_model_source() + assert "self._zero_offload_keeps_gpu_visible(cmd, env)" in load_src + block = inspect.getsource(LlamaCppBackend._cmd_has_gpu_device_pin) + for flag in ( + '"--device"', + '"-dev"', + '"--spec-draft-device"', + '"-devd"', + '"--device-draft"', + ): + assert flag in block + assert '"LLAMA_ARG_DEVICE"' in block + + +def test_resolve_cpu_moe_flag(): + # Clamp the requested MoE-layer count to the model's MoE layers, then offset + # past leading dense layers (--n-cpu-moe counts from layer 0). + R = LlamaCppBackend._resolve_cpu_moe_flag + assert R(0, 40, 0) is None # nothing requested + assert R(8, 0, 0) is None # dense model (no MoE layers) + assert R(8, 40, 0) == 8 # all-MoE: direct + assert R(100, 40, 0) == 40 # clamp to the MoE layer count + # GLM-4.7-Flash (deepseek2): block_count 47, leading_dense 1, n_moe 46. + assert R(5, 46, 1) == 6 # offset past the 1 dense layer + assert R(46, 46, 1) == 47 # all MoE on CPU == block_count + + +def test_manual_allows_tensor_parallel_via_split_mode(): + # Manual offload keeps the user's TP choice but skips the memory-based planner + # (plan_tp excludes manual, so its empty gpu set can't downgrade TP). The + # --split-mode tensor emission gates on tensor_parallel alone, so manual + # reaches it -- with tp_tensor_split None it's an even split (no + # --tensor-split). --fit off means no fit/tensor abort. + src = _load_model_source() + assert 'plan_tp = tensor_parallel and gpu_memory_mode != "manual"' in src + assert "if plan_tp:" in src + assert "if plan_tp and len(tp_gpus) < 2:" in src + sm = src.find('cmd.extend(["--split-mode", "tensor"])') + assert sm != -1, "TP must emit --split-mode tensor" + guard = src.rfind("if tensor_parallel:", 0, sm) + assert guard != -1 and sm - guard < 200, "split-mode gates on tensor_parallel" + # The tensor-split is only emitted for a planned (non-even) split, which + # manual never produces, so manual stays an even split. + assert "if tp_tensor_split and len(tp_tensor_split) > 1:" in src + + +def test_fit_sets_target_margin(): + # Manual + Auto (auto_fit) tightens the per-device VRAM margin to 512 MiB. + caps = {"supports_fit_target": True} + flags = LlamaCppBackend._ctx_integrity_flags(1, True, True, 0, 0, caps) + assert flags[flags.index("--fit-target") + 1] == "512" + # Not emitted on the legacy auto path (fit on but not auto_fit): -c 0 pins + # native there, so the tighter margin must not ride along. + assert "--fit-target" not in LlamaCppBackend._ctx_integrity_flags(1, True, False, 0, 0, caps) + # Not emitted when fit is off. + assert "--fit-target" not in LlamaCppBackend._ctx_integrity_flags(1, False, False, 0, 0, caps) + # Not emitted when the binary lacks support. + assert "--fit-target" not in LlamaCppBackend._ctx_integrity_flags( + 1, True, True, 0, 0, {"supports_fit_target": False} + ) + + +# ── GPU picker (gpu_ids -> CUDA_VISIBLE_DEVICES) ───────────────────── + + +def test_load_request_accepts_gpu_ids(): + req = LoadRequest(model_path = "owner/repo", gpu_ids = [1, 0]) + assert req.gpu_ids == [1, 0] + assert LoadRequest(model_path = "owner/repo").gpu_ids is None + + +@pytest.mark.parametrize("model_cls", [LoadResponse, InferenceStatusResponse]) +def test_response_models_emit_gpu_ids(model_cls): + if model_cls is LoadResponse: + obj = model_cls( + status = "loaded", + model = "m", + display_name = "m", + inference = {}, + gpu_ids = [1], + requested_gpu_ids = [1, 2], + ) + else: + obj = model_cls(gpu_ids = [1], requested_gpu_ids = [1, 2]) + assert obj.model_dump()["gpu_ids"] == [1] + assert obj.model_dump()["requested_gpu_ids"] == [1, 2] + + +def test_gguf_load_and_status_responses_include_requested_gpu_pool(): + route_src = (Path(_BACKEND_DIR) / "routes" / "inference.py").read_text(encoding = "utf-8") + assert route_src.count("requested_gpu_ids = llama_backend.requested_gpu_ids") == 3 + + +def test_gpu_ids_property_default_and_reset(): + backend = LlamaCppBackend() + assert backend.gpu_ids is None + backend._gpu_ids = [0, 1] + assert backend.gpu_ids == [0, 1] + backend._process = _FakeProcess() + backend.unload_model() + assert backend.gpu_ids is None + + +def _target_state_gpu_ids(backend, gpu_ids): + return backend._already_in_target_state( + gguf_path = None, + model_identifier = "owner/repo", + hf_variant = "Q4_K_M", + n_ctx = 8192, + cache_type_kv = None, + speculative_type = "auto", + chat_template_override = None, + extra_args = None, + is_vision = False, + gpu_ids = gpu_ids, + ) + + +def test_gpu_ids_reload_detection_is_order_insensitive(): + backend = _loaded_backend("auto") + backend._gpu_ids = [0, 1] + # A real non-narrowed load records the raw request too; the non-diffusion + # dedupe now compares that raw pin (#7239). Set it to match the effective pin + # (no narrowing) so this exercises the order-insensitive comparison. + backend._requested_gpu_ids = [0, 1] + # Same set, different order -> no reload. + assert _target_state_gpu_ids(backend, [1, 0]) is True + # Different set -> reload. + assert _target_state_gpu_ids(backend, [0]) is False + # Dropping the pick (auto) -> reload. + assert _target_state_gpu_ids(backend, None) is False + + +def test_gpu_ids_reload_detection_accepts_raw_and_effective_pin(): + backend = _loaded_backend("auto") + backend._requested_gpu_ids = [0, 1] + backend._gpu_ids = [0] + backend._last_load_kwargs = {"gpu_ids": [0, 1], "model_identifier": "owner/repo"} + + # The original request still matches after the fitter narrows it. + assert _target_state_gpu_ids(backend, [1, 0]) is True + assert backend.requested_gpu_ids == [0, 1] + # The status response echoes the effective pin, which must also round-trip. + # Treat the incoming subset as the latest intent so status and a future + # reload do not restore GPU 1 after the user removed it. + assert _target_state_gpu_ids(backend, [0]) is True + assert backend.requested_gpu_ids == [0] + assert backend._last_load_kwargs == {"gpu_ids": [0], "model_identifier": "owner/repo"} + # A genuinely different placement pool still reloads. + assert _target_state_gpu_ids(backend, [1]) is False + assert _target_state_gpu_ids(backend, None) is False + + +def test_gpu_ids_reload_detection_collapses_diffusion_to_single_device(): + # The diffusion runner drives only its single lowest device, so the backend + # records [lowest]. A later multi-GPU request that still resolves to that + # same lowest device must dedupe (no needless reload); a request whose lowest + # device moves, or that drops the pick, must reload. + backend = _loaded_backend("auto") + backend._is_diffusion = True + backend._gpu_ids = [1] # loaded on the lowest of an earlier [3, 1] pick + assert _target_state_gpu_ids(backend, [3, 1]) is True + assert backend.requested_gpu_ids == [1] + assert _target_state_gpu_ids(backend, [1]) is True + # Lowest device changes (2, not 1) -> reload. + assert _target_state_gpu_ids(backend, [3, 2]) is False + # Dropping the pick (auto) -> reload. + assert _target_state_gpu_ids(backend, None) is False + + +def test_remote_vulkan_diffusion_preflight_runs_before_teardown(monkeypatch): + def _mark_diffusion(probe, path): + assert path == "/cache/model.gguf" + probe._is_diffusion = True + + monkeypatch.setattr(LlamaCppBackend, "_read_gguf_metadata", _mark_diffusion) + assert LlamaCppBackend._gguf_path_is_diffusion("/cache/model.gguf", "owner/model") is True + + src = inspect.getsource(llama_cpp_module.LlamaCppBackend.load_model) + preflight = src.index("_preflight_model_path = self._download_gguf(") + teardown = src.index("# ── Phase 1: kill old process") + assert preflight < teardown + assert "model_path = _preflight_model_path or self._download_gguf(" in src + + +def test_remote_vulkan_diffusion_rejection_keeps_active_server(monkeypatch): + backend = LlamaCppBackend() + killed = [] + monkeypatch.setattr(backend, "_find_llama_server_binary", lambda **_kwargs: "/bin/llama") + monkeypatch.setattr(backend, "_is_vulkan_backend", lambda _binary = None: True) + monkeypatch.setattr(backend, "_get_gpu_memory", lambda _binary = None: [(0, 1024, 2048)]) + monkeypatch.setattr( + backend, + "_download_gguf", + lambda **_kwargs: "/cache/diffusion.gguf", + ) + monkeypatch.setattr(backend, "_gguf_path_is_diffusion", lambda *_args: True) + monkeypatch.setattr(backend, "_kill_process", lambda: killed.append(True)) + monkeypatch.setattr( + llama_cpp_module, + "_resolve_repo_id_casing", + lambda repo: repo, + ) + monkeypatch.setattr( + llama_cpp_module, + "_hf_offline_if_dns_dead", + lambda: __import__("contextlib").nullcontext(), + ) + + with pytest.raises(ValueError, match = "DiffusionGemma"): + backend.load_model( + hf_repo = "owner/model", + hf_variant = "Q4_K_M", + model_identifier = "owner/model", + gpu_ids = [0], + ) + + assert killed == [] + + +def test_start_diffusion_server_resets_tensor_parallel(): + # A prior tensor-parallel chat load leaves self._tensor_parallel True (load_model + # phase 1 only kills the process, it skips the unload reset). Diffusion is never + # TP, so startup must clear it -- else /status misreports TP and an identical + # diffusion re-Apply reloads against stale tensor-parallel state. + src = inspect.getsource(llama_cpp_module.LlamaCppBackend._start_diffusion_server) + assert "self._tensor_parallel = False" in src + assert "self._requested_gpu_ids = list(self._gpu_ids) if self._gpu_ids else None" in src + + +def test_route_matches_loaded_settings_uses_shared_gpu_pin_matcher(): + # Route-level and backend race dedupe must share one normalization path so + # raw, effective, and diffusion pins cannot drift apart. + route_src = (Path(_BACKEND_DIR) / "routes" / "inference.py").read_text(encoding = "utf-8") + match_impl = route_src[route_src.index("def _request_matches_loaded_settings") :] + assert "if not llama_backend.matches_gpu_ids(request.gpu_ids):" in match_impl + assert "llama_backend._record_matching_gpu_request(request.gpu_ids)" in match_impl + + +# ── Manual tensor split: child enumeration pinned to the picker's order ────── + + +def _patch_split_pin_env(monkeypatch, *, inherited, reported): + """Point the pin helper at a fake inherited mask and picker report. + ``reported`` None = enumeration unavailable (falls back to ascending).""" + import utils.hardware as hw + + monkeypatch.setattr( + LlamaCppBackend, "_resolve_visible_physical_ids", staticmethod(lambda: inherited) + ) + info = ( + {"available": False} + if reported is None + else { + "available": True, + "index_kind": "physical", + "devices": [{"index": i} for i in reported], + } + ) + monkeypatch.setattr(hw, "get_backend_visible_gpu_info", lambda: info) + + +def test_split_pin_reorders_inherited_numeric_mask(monkeypatch): + # Parent CUDA_VISIBLE_DEVICES=3,1 makes the child enumerate dev0=phys3, but + # nvidia-smi reported the picker's list ascending -- the mask must be + # re-emitted in that order or the per-GPU shares land on the wrong cards. + _patch_split_pin_env(monkeypatch, inherited = [3, 1], reported = [1, 3]) + env = {"CUDA_VISIBLE_DEVICES": "3,1"} + LlamaCppBackend._pin_visible_gpu_order_for_split(env) + assert env["CUDA_DEVICE_ORDER"] == "PCI_BUS_ID" + assert env["CUDA_VISIBLE_DEVICES"] == "1,3" + + +def test_split_pin_keeps_mask_order_when_picker_reported_it(monkeypatch): + # Torch-fallback enumeration (no nvidia-smi) reports devices in inherited + # mask order, so the picker's split list follows the mask -- the pin must + # keep that order, not re-sort it into a mismatch. + _patch_split_pin_env(monkeypatch, inherited = [3, 1], reported = [3, 1]) + env = {"CUDA_VISIBLE_DEVICES": "3,1"} + LlamaCppBackend._pin_visible_gpu_order_for_split(env) + assert env["CUDA_VISIBLE_DEVICES"] == "3,1" + + +def test_split_pin_falls_back_to_ascending_without_report(monkeypatch): + # Enumeration unavailable: ascending physical is the best guess (it matches + # the dominant nvidia-smi report order). + _patch_split_pin_env(monkeypatch, inherited = [3, 1], reported = None) + env = {"CUDA_VISIBLE_DEVICES": "3,1"} + LlamaCppBackend._pin_visible_gpu_order_for_split(env) + assert env["CUDA_VISIBLE_DEVICES"] == "1,3" + + +def test_split_pin_without_mask_only_sets_pci_order(monkeypatch): + # No inherited mask (or a UUID/MIG one resolving to None): enumeration order + # is fully fixed by CUDA_DEVICE_ORDER, so no mask is written. + _patch_split_pin_env(monkeypatch, inherited = None, reported = None) + env = {} + LlamaCppBackend._pin_visible_gpu_order_for_split(env) + assert env == {"CUDA_DEVICE_ORDER": "PCI_BUS_ID"} + + +def test_split_pin_mirrors_hip_mask_on_rocm(monkeypatch): + # ROCm with the mask sourced from HIP: the pin must land in + # HIP_VISIBLE_DEVICES too, and an inherited ROCR mask is cleared so the + # mask can't apply twice (ROCR re-indexes, then HIP would index into the + # already-reduced set). + _patch_split_pin_env(monkeypatch, inherited = [3, 1], reported = [1, 3]) + _rocm_torch_stub(monkeypatch) + env = { + "CUDA_VISIBLE_DEVICES": "3,1", + "HIP_VISIBLE_DEVICES": "3,1", + "ROCR_VISIBLE_DEVICES": "3,1", + } + LlamaCppBackend._pin_visible_gpu_order_for_split(env) + assert env["CUDA_VISIBLE_DEVICES"] == "1,3" + assert env["HIP_VISIBLE_DEVICES"] == "1,3" + assert "ROCR_VISIBLE_DEVICES" not in env + + +def test_split_pin_preserves_inherited_rocr_mask(monkeypatch): + # Mask sourced from ROCR alone (e.g. an AMD SDK parent): the pin must + # re-emit at the ROCr layer, not swap to HIP -- clearing ROCR re-exposes + # every agent to HSA enumeration, which can segfault at startup on an + # unsupported GPU the parent mask was hiding (#7272 review). CUDA carries + # the post-ROCR ordinals, mirroring the prefer_rocr emission. + _patch_split_pin_env(monkeypatch, inherited = [3, 1], reported = [1, 3]) + _rocm_torch_stub(monkeypatch) + env = {"ROCR_VISIBLE_DEVICES": "3,1"} + LlamaCppBackend._pin_visible_gpu_order_for_split(env) + assert env["ROCR_VISIBLE_DEVICES"] == "1,3" + assert env["CUDA_VISIBLE_DEVICES"] == "0,1" + assert "HIP_VISIBLE_DEVICES" not in env + + +def test_split_pin_keeps_hip_on_windows_despite_stray_rocr(monkeypatch): + # On Windows the ROCR var is dead (no ROCr layer) and the resolver never + # reads it, so a stray value must not flip the pin to the ROCR emission: + # the HIP mask is the only effective selector there. + _patch_split_pin_env(monkeypatch, inherited = [3, 1], reported = [1, 3]) + torch_stub = _types.ModuleType("torch") + torch_stub.version = _types.SimpleNamespace(hip = "6.0") + monkeypatch.setitem(sys.modules, "torch", torch_stub) + monkeypatch.setattr(sys, "platform", "win32") + env = {"CUDA_VISIBLE_DEVICES": "3,1", "ROCR_VISIBLE_DEVICES": "9"} + LlamaCppBackend._pin_visible_gpu_order_for_split(env) + assert env["CUDA_VISIBLE_DEVICES"] == "1,3" + assert env["HIP_VISIBLE_DEVICES"] == "1,3" + assert "ROCR_VISIBLE_DEVICES" not in env + + +def _rocm_torch_stub(monkeypatch): + torch_stub = _types.ModuleType("torch") + torch_stub.version = _types.SimpleNamespace(hip = "6.0") + monkeypatch.setitem(sys.modules, "torch", torch_stub) + # prefer_rocr is Linux-only (ROCR is an ROCr variable); pin the platform so + # these Linux-behaviour tests also pass on a Windows dev box. + monkeypatch.setattr(sys, "platform", "linux") + + +def test_subset_pin_masks_via_rocr_on_rocm(monkeypatch): + # A GPU-subset pin must exclude the rest at the ROCr/HSA layer: HIP masking + # still enumerates every agent first, which segfaults the build on an + # unsupported deselected GPU (e.g. a gfx1103 iGPU under a gfx110X prebuilt). + # ROCR drops it at the driver layer; only one mask is set (HIP cleared). + _rocm_torch_stub(monkeypatch) + env = {"HIP_VISIBLE_DEVICES": "9"} # stale/inherited HIP mask must not survive + LlamaCppBackend._emit_child_gpu_visibility(env, "0", prefer_rocr = True) + assert env["ROCR_VISIBLE_DEVICES"] == "0" + assert env["CUDA_VISIBLE_DEVICES"] == "0" + assert "HIP_VISIBLE_DEVICES" not in env + + +def test_prefer_rocr_remaps_cuda_to_post_rocr_ordinals(monkeypatch): + # ROCR re-indexes the visible agents from 0, and HIP (cleared here) falls back + # to CUDA_VISIBLE_DEVICES -- so on the prefer_rocr path CUDA must carry the + # post-ROCR ordinals, not the physical ids, else a non-zero pick indexes out + # of range and the child sees no GPU and drops to CPU (#7272 review). + _rocm_torch_stub(monkeypatch) + # Single non-zero GPU: ROCR keeps the physical id, CUDA becomes ordinal 0. + env = {} + LlamaCppBackend._emit_child_gpu_visibility(env, "1", prefer_rocr = True) + assert env["ROCR_VISIBLE_DEVICES"] == "1" + assert env["CUDA_VISIBLE_DEVICES"] == "0" + assert "HIP_VISIBLE_DEVICES" not in env + # Multi-GPU subset: ROCR keeps the physical ids, CUDA is the 0-based ordinals. + env = {} + LlamaCppBackend._emit_child_gpu_visibility(env, "1,3", prefer_rocr = True) + assert env["ROCR_VISIBLE_DEVICES"] == "1,3" + assert env["CUDA_VISIBLE_DEVICES"] == "0,1" + assert "HIP_VISIBLE_DEVICES" not in env + + +def test_subset_pin_default_still_uses_hip_and_clears_rocr(monkeypatch): + # Without prefer_rocr the masking is unchanged: HIP narrows, inherited ROCR + # is cleared so the two can't double-mask. + _rocm_torch_stub(monkeypatch) + env = {"ROCR_VISIBLE_DEVICES": "0,1"} + LlamaCppBackend._emit_child_gpu_visibility(env, "1") + assert env["HIP_VISIBLE_DEVICES"] == "1" + assert "ROCR_VISIBLE_DEVICES" not in env + + +def test_cpu_only_pin_keeps_hip_even_with_prefer_rocr(monkeypatch): + # The CPU-only sentinel never routes through ROCR (no portable "hide all" + # spelling); it hides every GPU via HIP. + _rocm_torch_stub(monkeypatch) + env = {} + LlamaCppBackend._emit_child_gpu_visibility(env, "-1", prefer_rocr = True) + assert env["HIP_VISIBLE_DEVICES"] == "-1" + assert "ROCR_VISIBLE_DEVICES" not in env + + +def _amd_sdk_torch_stub(monkeypatch): + # AMD SDK wheel: torch.version.hip is None but __version__ encodes rocm. + torch_stub = _types.ModuleType("torch") + torch_stub.version = _types.SimpleNamespace(hip = None) + torch_stub.__version__ = "2.9.1+rocm7.2.1" + monkeypatch.setitem(sys.modules, "torch", torch_stub) + monkeypatch.setattr(sys, "platform", "linux") + + +def test_prefer_rocr_falls_back_to_hip_on_windows(monkeypatch): + # ROCR_VISIBLE_DEVICES is a Linux ROCr variable (Windows HIP has no ROCr + # layer), so on Windows ROCm prefer_rocr must keep the HIP mask or a nonzero + # pick loses its only effective selector (#7272 review). + torch_stub = _types.ModuleType("torch") + torch_stub.version = _types.SimpleNamespace(hip = "6.0") + monkeypatch.setitem(sys.modules, "torch", torch_stub) + monkeypatch.setattr(sys, "platform", "win32") + env = {"ROCR_VISIBLE_DEVICES": "9"} + LlamaCppBackend._emit_child_gpu_visibility(env, "1", prefer_rocr = True) + assert env["HIP_VISIBLE_DEVICES"] == "1" + assert env["CUDA_VISIBLE_DEVICES"] == "1" + assert "ROCR_VISIBLE_DEVICES" not in env + + +def test_amd_sdk_wheel_hip_none_still_masks_rocr(monkeypatch): + # An AMD SDK wheel leaves torch.version.hip unset but has "rocm" in __version__. + # It must still get the ROCR mask, else only CUDA_VISIBLE_DEVICES is set and an + # unsupported iGPU keeps enumerating and can crash llama-server. + _amd_sdk_torch_stub(monkeypatch) + env = {"HIP_VISIBLE_DEVICES": "9"} + LlamaCppBackend._emit_child_gpu_visibility(env, "0", prefer_rocr = True) + assert env["ROCR_VISIBLE_DEVICES"] == "0" + assert "HIP_VISIBLE_DEVICES" not in env + + +def test_cuda_wheel_hip_none_gets_no_rocm_mask(monkeypatch): + # A CUDA wheel (hip=None, no "rocm" in __version__) must NOT get a HIP/ROCR mask + # -- only CUDA_VISIBLE_DEVICES -- so the version-string check can't false-positive. + torch_stub = _types.ModuleType("torch") + torch_stub.version = _types.SimpleNamespace(hip = None) + torch_stub.__version__ = "2.9.1+cu124" + monkeypatch.setitem(sys.modules, "torch", torch_stub) + env = {} + LlamaCppBackend._emit_child_gpu_visibility(env, "0", prefer_rocr = True) + assert env["CUDA_VISIBLE_DEVICES"] == "0" + assert "ROCR_VISIBLE_DEVICES" not in env + assert "HIP_VISIBLE_DEVICES" not in env + + +def test_resolve_physical_ids_reads_rocr_on_amd_sdk_wheel(monkeypatch): + # _resolve_visible_physical_ids must use the same ROCm detection as + # _emit_child_gpu_visibility: on an AMD SDK wheel (hip=None, rocm in + # __version__) an inherited ROCR mask IS the ordinal->physical mapping. + # Reading it as "no mask" labels ordinal 0 as physical 0 and the child's + # ROCR pin then re-exposes the GPU the mask was hiding (#7272 review). + _amd_sdk_torch_stub(monkeypatch) + for var in ("HIP_VISIBLE_DEVICES", "CUDA_VISIBLE_DEVICES"): + monkeypatch.delenv(var, raising = False) + monkeypatch.setenv("ROCR_VISIBLE_DEVICES", "1") + assert LlamaCppBackend._resolve_visible_physical_ids() == [1] + + +def test_resolve_physical_ids_ignores_rocr_on_cuda_wheel(monkeypatch): + # A CUDA wheel (hip=None, no "rocm") keeps CUDA-only semantics: a stray + # ROCR var must not be read as the mask. + torch_stub = _types.ModuleType("torch") + torch_stub.version = _types.SimpleNamespace(hip = None) + torch_stub.__version__ = "2.9.1+cu124" + monkeypatch.setitem(sys.modules, "torch", torch_stub) + for var in ("HIP_VISIBLE_DEVICES", "CUDA_VISIBLE_DEVICES"): + monkeypatch.delenv(var, raising = False) + monkeypatch.setenv("ROCR_VISIBLE_DEVICES", "1") + assert LlamaCppBackend._resolve_visible_physical_ids() is None + + +def test_resolve_physical_ids_ignores_rocr_on_windows(monkeypatch): + # ROCR_VISIBLE_DEVICES is a Linux ROCr variable: Windows HIP has no ROCr + # layer, so a stray ROCR var there does not mask the runtime. Reading it as + # the ordinal->physical mapping would label ordinal 0 with a stale ROCR id + # while the runtime still enumerates every adapter, so auto-selection could + # budget one card and pin another (#7272 review). HIP must still be honoured. + torch_stub = _types.ModuleType("torch") + torch_stub.version = _types.SimpleNamespace(hip = None) + torch_stub.__version__ = "2.9.1+rocm7.2.1" # AMD SDK wheel + monkeypatch.setitem(sys.modules, "torch", torch_stub) + monkeypatch.setattr(sys, "platform", "win32") + for var in ("HIP_VISIBLE_DEVICES", "CUDA_VISIBLE_DEVICES"): + monkeypatch.delenv(var, raising = False) + monkeypatch.setenv("ROCR_VISIBLE_DEVICES", "1") + assert LlamaCppBackend._resolve_visible_physical_ids() is None + # HIP precedence is unchanged on Windows. + monkeypatch.setenv("HIP_VISIBLE_DEVICES", "1") + assert LlamaCppBackend._resolve_visible_physical_ids() == [1] + + +# ── Diffusion single-device selection ─────────────────────────────────────── + + +def test_diffusion_gpu_arg_uses_lowest_explicit_physical_id(monkeypatch): + monkeypatch.setenv("CUDA_VISIBLE_DEVICES", "3,1") + monkeypatch.setenv("DG_GPU", "7") + assert LlamaCppBackend._diffusion_gpu_arg([3, 1]) == "1" + + +def test_diffusion_gpu_arg_preserves_parent_mask_order(monkeypatch): + monkeypatch.delenv("DG_GPU", raising = False) + monkeypatch.setenv("CUDA_VISIBLE_DEVICES", "3,1") + assert LlamaCppBackend._diffusion_gpu_arg(None) == "3" + + +def test_diffusion_gpu_arg_honors_override_and_cpu_mask(monkeypatch): + monkeypatch.setenv("DG_GPU", "GPU-abc") + assert LlamaCppBackend._diffusion_gpu_arg(None) == "GPU-abc" + assert LlamaCppBackend._diffusion_gpu_arg(None, cpu_only = True) == "" + + +# ── Deliberate zero-offload (manual gpu_layers=0): training-skip flag ───────── + + +def test_zero_offload_flag_false_without_companions(): + # CPU-only by construction: False lets training skip unloading a server that + # holds no VRAM. + cmd = ["llama-server", "-m", "model.gguf", "--gpu-layers", "0", "--fit", "off"] + assert LlamaCppBackend._zero_offload_gpu_flag(cmd, [(0, 8000, 24000)], {}) is False + + +@pytest.mark.parametrize( + "companion", + ["--mmproj", "--model-draft", "-md", "--spec-draft-model", "-hfd"], +) +def test_zero_offload_flag_true_with_companion(companion): + # mmproj / a drafter offload to GPU regardless of --gpu-layers, so the + # server still holds VRAM and training must unload it. Drafter detection + # reuses the extras parser, so pass-through aliases count too. + cmd = ["llama-server", "-m", "model.gguf", "--gpu-layers", "0", companion, "x.gguf"] + assert LlamaCppBackend._zero_offload_gpu_flag(cmd, [(0, 8000, 24000)], {}) is True + + +def test_zero_offload_flag_true_with_inline_companion_forms(): + cmd = ["llama-server", "-m", "model.gguf", "--spec-draft-model=x.gguf"] + assert LlamaCppBackend._zero_offload_gpu_flag(cmd, [(0, 8000, 24000)], {}) is True + cmd = ["llama-server", "-m", "model.gguf", "--mmproj=proj.gguf"] + assert LlamaCppBackend._zero_offload_gpu_flag(cmd, [(0, 8000, 24000)], {}) is True + + +def test_zero_offload_flag_true_with_env_drafter(): + cmd = ["llama-server", "-m", "model.gguf", "--gpu-layers", "0"] + env = {"LLAMA_ARG_SPEC_DRAFT_MODEL": "x.gguf"} + assert LlamaCppBackend._zero_offload_gpu_flag(cmd, [(0, 8000, 24000)], env) is True + + +@pytest.mark.parametrize( + "device_args", + [ + ["--device", "CUDA0"], + ["--device=CUDA0"], + ["-dev", "CUDA0"], + ["--spec-draft-device", "CUDA0"], + ["--device-draft=CUDA0"], + ], +) +def test_zero_offload_flag_true_with_device_pin(device_args): + cmd = ["llama-server", "-m", "model.gguf", "--gpu-layers", "0", *device_args] + assert LlamaCppBackend._zero_offload_gpu_flag(cmd, [(0, 8000, 24000)], {}) is True + + +def test_zero_offload_flag_true_with_env_device_pin(): + cmd = ["llama-server", "-m", "model.gguf", "--gpu-layers", "0"] + env = {"LLAMA_ARG_DEVICE": "CUDA0"} + assert LlamaCppBackend._zero_offload_gpu_flag(cmd, [(0, 8000, 24000)], env) is True + + +@pytest.mark.parametrize( + ("device_args", "env"), + [ + (["--device", "cpu"], {}), + (["--device=none"], {}), + (["--spec-draft-device", "cpu"], {}), + ([], {"LLAMA_ARG_DEVICE": "none"}), + (["--device", "CUDA0", "--device", "cpu"], {}), + ], +) +def test_zero_offload_flag_false_with_cpu_device_pin(device_args, env): + cmd = ["llama-server", "-m", "model.gguf", "--gpu-layers", "0", *device_args] + assert LlamaCppBackend._zero_offload_gpu_flag(cmd, [(0, 8000, 24000)], env) is False + + +def test_zero_offload_flag_true_with_surviving_tensor_mode(): + cmd = ["llama-server", "-m", "model.gguf", "--gpu-layers", "0", "--split-mode", "tensor"] + assert LlamaCppBackend._zero_offload_gpu_flag(cmd, [(0, 8000, 24000)], {}) is True + + +def test_zero_offload_flag_true_for_unmasked_vulkan(monkeypatch): + monkeypatch.setattr(LlamaCppBackend, "_is_vulkan_backend", staticmethod(lambda: True)) + cmd = ["llama-server", "-m", "model.gguf", "--gpu-layers", "0"] + assert LlamaCppBackend._zero_offload_gpu_flag(cmd, [(0, 8000, 24000)], {}) is True + + +def test_zero_offload_flag_none_without_gpus(): + cmd = ["llama-server", "-m", "model.gguf", "--gpu-layers", "0"] + assert LlamaCppBackend._zero_offload_gpu_flag(cmd, [], {}) is None + + +def test_cmd_has_gpu_companion_detection(): + # The env mask for CPU-only zero-offload loads keys off this scan: any + # --mmproj form or a drafter (flag aliases / env) keeps the GPUs visible. + has = LlamaCppBackend._cmd_has_gpu_companion + assert has(["llama-server", "-m", "m.gguf"], {}) is False + assert has(["llama-server", "--mmproj", "p.gguf"], {}) is True + assert has(["llama-server", "--mmproj=p.gguf"], {}) is True + assert has(["llama-server", "-md", "d.gguf"], {}) is True + assert has(["llama-server"], {"LLAMA_ARG_SPEC_DRAFT_MODEL": "d.gguf"}) is True + + +def test_cmd_companion_ignores_cpu_forced_drafter(): + # A CPU-pinned drafter holds no VRAM: the zero-offload mask may hide the GPUs + # and training may leave the server alone. + has = LlamaCppBackend._cmd_has_gpu_companion + cmd = ["llama-server", "-md", "d.gguf", "--spec-draft-ngl", "0"] + assert has(cmd, {}) is False + cmd = ["llama-server", "-md", "d.gguf", "--spec-draft-device", "cpu"] + assert has(cmd, {}) is False + # mmproj still counts even alongside a CPU drafter. + cmd = ["llama-server", "-md", "d.gguf", "--spec-draft-ngl", "0", "--mmproj", "p.gguf"] + assert has(cmd, {}) is True diff --git a/studio/backend/tests/test_gpu_selection.py b/studio/backend/tests/test_gpu_selection.py index 69ad560788..3dab7ef368 100644 --- a/studio/backend/tests/test_gpu_selection.py +++ b/studio/backend/tests/test_gpu_selection.py @@ -119,7 +119,8 @@ class TestResolveRequestedGpuIds(_GpuCacheResetMixin, unittest.TestCase): patch("utils.hardware.hardware.get_physical_gpu_count", return_value = 8), ): with self.assertRaisesRegex( - ValueError, "unsupported when CUDA_VISIBLE_DEVICES uses UUID/MIG" + ValueError, + "unsupported when CUDA_VISIBLE_DEVICES uses non-numeric or subdevice", ): resolve_requested_gpu_ids([1]) @@ -130,6 +131,26 @@ class TestResolveRequestedGpuIds(_GpuCacheResetMixin, unittest.TestCase): ): self.assertEqual(resolve_requested_gpu_ids([]), [1, 3]) + def test_vulkan_ordinals_bypass_cuda_parent_visible_validation(self): + # Vulkan build on a CPU-only torch host: no CUDA parent-visible set and a + # zero physical count, yet a valid Vulkan ordinal must not be rejected as + # a CUDA physical id (issue #7239). + with ( + patch.dict(os.environ, {}, clear = True), + patch("utils.hardware.hardware.get_physical_gpu_count", return_value = 0), + ): + # As a CUDA physical id, [0] is outside the empty parent-visible set. + with self.assertRaises(ValueError): + resolve_requested_gpu_ids([0]) + # As Vulkan ordinals, [0] and [0, 1] pass through unchanged. + self.assertEqual(resolve_requested_gpu_ids([0], is_vulkan = True), [0]) + self.assertEqual(resolve_requested_gpu_ids([0, 1], is_vulkan = True), [0, 1]) + # Malformed ordinals are still rejected. + with self.assertRaisesRegex(ValueError, "duplicate GPU IDs"): + resolve_requested_gpu_ids([0, 0], is_vulkan = True) + with self.assertRaisesRegex(ValueError, "non-negative"): + resolve_requested_gpu_ids([-1], is_vulkan = True) + def test_apply_gpu_ids_only_updates_cuda_visible_devices(self): with patch.dict( os.environ, @@ -846,14 +867,20 @@ class TestPreSpawnGpuResolution(_GpuCacheResetMixin, unittest.TestCase): class TestRouteErrors(unittest.TestCase): - def test_prepare_gpu_selection_rejects_gpu_ids_on_non_cuda_backend(self): + def test_prepare_gpu_selection_rejects_gpu_ids_on_non_accelerator_backend(self): with patch("utils.hardware.hardware.get_device", return_value = DeviceType.CPU): with self.assertRaises(ValueError) as exc_info: prepare_gpu_selection([0], model_name = "unsloth/test") - self.assertIn("only supported on CUDA devices", str(exc_info.exception)) + self.assertIn("only supported on CUDA and Intel XPU", str(exc_info.exception)) + + def test_inference_route_resolves_gguf_gpu_ids(self): + # GGUF gpu_ids are now supported: /load routes them through the same + # resolution as non-GGUF loads (rejecting only genuinely invalid ids with + # the resolver's actionable message) rather than a blanket "not supported" + # reject, so /validate can stay consistent with /load (#7239). + import utils.hardware.hardware as hardware_mod - def test_inference_route_rejects_gpu_ids_for_gguf(self): inference_route = _load_route_module( "inference_route_module_for_gguf_gpu_ids_test", "routes/inference.py", @@ -874,12 +901,188 @@ class TestRouteErrors(unittest.TestCase): has_audio_input = False, ) + def _fake_resolve(ids, is_vulkan = False): + raise ValueError("SENTINEL requested GPUs are outside the parent-visible set") + + with ( + patch.object( + inference_route, + "ModelConfig", + SimpleNamespace(from_identifier = lambda **_kwargs: model_config), + ), + # Patch both the package re-export and the defining module so the stub + # fires no matter which import path the route uses. + patch("utils.hardware.resolve_requested_gpu_ids", _fake_resolve), + patch.object(hardware_mod, "resolve_requested_gpu_ids", _fake_resolve), + patch.object( + inference_route, + "_guard_chat_load_against_training", + return_value = None, + ), + patch.object(inference_route.asyncio, "to_thread", new = _inline_to_thread), + patch.object(inference_route, "_hf_offline_if_dns_dead", nullcontext), + ): + with self.assertRaises(HTTPException) as exc_info: + asyncio.run( + inference_route._load_model_impl( + request, + SimpleNamespace( + app = SimpleNamespace( + state = SimpleNamespace(llama_parallel_slots = 1), + ), + ), + current_subject = "test-user", + ) + ) + + # The selection was routed through resolution (not the old blanket reject). + self.assertEqual(exc_info.exception.status_code, 400) + self.assertIn("SENTINEL", exc_info.exception.detail) + self.assertNotIn("not supported for GGUF", exc_info.exception.detail) + + def test_load_rejects_unavailable_vulkan_ordinal_before_training_guard(self): + inference_route = _load_route_module( + "inference_route_module_for_vulkan_preflight_test", + "routes/inference.py", + ) + request = LoadRequest(model_path = "unsloth/test.gguf", gpu_ids = [99]) + model_config = SimpleNamespace( + is_gguf = True, + is_lora = False, + gguf_hf_repo = None, + gguf_file = "/tmp/test.gguf", + gguf_mmproj_file = None, + gguf_variant = None, + identifier = "unsloth/test.gguf", + display_name = "unsloth/test.gguf", + is_vision = False, + is_audio = False, + audio_type = None, + has_audio_input = False, + ) + + with ( + patch.object( + inference_route, + "ModelConfig", + SimpleNamespace(from_identifier = lambda **_kwargs: model_config), + ), + patch("utils.hardware.get_device", return_value = DeviceType.CUDA), + patch.object(inference_route, "_classify_diffusion_gguf", return_value = None), + patch.object( + inference_route.LlamaCppBackend, + "_is_vulkan_backend", + return_value = True, + ), + patch.object( + inference_route.LlamaCppBackend, + "_find_llama_server_binary", + return_value = "/tmp/llama-server", + ), + patch.object( + inference_route.LlamaCppBackend, + "_get_gpu_memory", + return_value = [(0, 8 * 1024**3, 16 * 1024**3)], + ), + patch.object( + inference_route, + "_guard_chat_load_against_training", + return_value = None, + ) as training_guard, + patch.object(inference_route.asyncio, "to_thread", new = _inline_to_thread), + patch.object(inference_route, "_hf_offline_if_dns_dead", nullcontext), + ): + with self.assertRaises(HTTPException) as exc_info: + asyncio.run( + inference_route._load_model_impl( + request, + SimpleNamespace( + app = SimpleNamespace( + state = SimpleNamespace(llama_parallel_slots = 1), + ), + ), + current_subject = "test-user", + ) + ) + + self.assertEqual(exc_info.exception.status_code, 400) + self.assertIn("Vulkan GPU ordinal(s) [99]", exc_info.exception.detail) + training_guard.assert_not_called() + + def test_vulkan_ordinals_are_allowed_on_xpu_hosts(self): + import utils.hardware.hardware as hardware_mod + + inference_route = _load_route_module( + "inference_route_module_for_xpu_vulkan_test", + "routes/inference.py", + ) + config = SimpleNamespace(is_gguf = True) + + with ( + patch("utils.hardware.get_device", return_value = DeviceType.XPU), + patch.object( + inference_route.LlamaCppBackend, + "_is_vulkan_backend", + return_value = True, + ), + patch.object(inference_route, "_classify_diffusion_gguf", return_value = False), + patch.object(hardware_mod, "resolve_requested_gpu_ids", return_value = [0, 1]), + patch.object( + inference_route.LlamaCppBackend, + "_find_llama_server_binary", + return_value = None, + ), + ): + resolved = asyncio.run( + inference_route._resolve_gguf_gpu_ids_for_request(config, [1, 0]) + ) + + self.assertEqual(resolved, [0, 1]) + + def test_inference_route_validates_gpu_ids_for_gguf(self): + # gpu_ids is now SUPPORTED for GGUF (the GPU picker), but still + # validated: a rejected pick surfaces as a clean 400, not the old + # "not supported for GGUF" rejection. Patch the validator so the test + # is deterministic regardless of the host's (or a prior test's) GPU env. + import utils.hardware.hardware as hardware_mod + + inference_route = _load_route_module( + "inference_route_module_for_gguf_gpu_ids_test2", + "routes/inference.py", + ) + request = LoadRequest(model_path = "unsloth/test.gguf", gpu_ids = [0, 1]) + model_config = SimpleNamespace( + is_gguf = True, + is_lora = False, + gguf_hf_repo = None, + gguf_file = "/tmp/test.gguf", + gguf_mmproj_file = None, + gguf_variant = None, + identifier = "unsloth/test.gguf", + display_name = "unsloth/test.gguf", + is_vision = False, + is_audio = False, + audio_type = None, + has_audio_input = False, + ) + with ( patch.object( inference_route, "ModelConfig", SimpleNamespace(from_identifier = lambda **_kwargs: model_config), ), + # Patch both the package re-export and the defining module so the stub + # fires no matter which import path the route uses. + patch( + "utils.hardware.resolve_requested_gpu_ids", + side_effect = ValueError("Invalid gpu_ids [0, 1]: rejected by test"), + ), + patch.object( + hardware_mod, + "resolve_requested_gpu_ids", + side_effect = ValueError("Invalid gpu_ids [0, 1]: rejected by test"), + ), patch.object( inference_route, "_guard_chat_load_against_training", @@ -901,8 +1104,11 @@ class TestRouteErrors(unittest.TestCase): ) ) + # The validator's ValueError becomes a clean 400 (not the removed + # "not supported for GGUF" rejection). self.assertEqual(exc_info.exception.status_code, 400) - self.assertIn("GGUF", exc_info.exception.detail) + self.assertIn("gpu_ids", exc_info.exception.detail.lower()) + self.assertNotIn("not supported", exc_info.exception.detail.lower()) def test_training_route_returns_400_for_invalid_gpu_ids(self): training_route = _load_route_module( @@ -1425,18 +1631,61 @@ class TestAutoSelectWithNoneRequired(_GpuCacheResetMixin, unittest.TestCase): self.assertEqual(metadata["selection_mode"], "fallback_all") -class TestXpuRejection(_GpuCacheResetMixin, unittest.TestCase): - def test_auto_select_returns_non_cuda_for_xpu(self): - with patch("utils.hardware.hardware.get_device", return_value = DeviceType.XPU): +class TestXpuSelection(_GpuCacheResetMixin, unittest.TestCase): + def test_auto_select_supports_xpu(self): + with ( + patch("utils.hardware.hardware.get_device", return_value = DeviceType.XPU), + patch( + "utils.hardware.hardware.estimate_required_model_memory_gb", + return_value = (1.0, {}), + ), + patch( + "utils.hardware.hardware.get_visible_gpu_utilization", + return_value = { + "devices": [ + {"index": 0, "vram_total_gb": 8, "vram_used_gb": 1}, + ] + }, + ), + patch( + "utils.hardware.hardware._get_parent_visible_gpu_spec", + return_value = { + "raw": None, + "numeric_ids": [0], + "supports_explicit_gpu_ids": True, + }, + ), + patch( + "utils.hardware.hardware.get_parent_visible_gpu_ids", + return_value = [0], + ), + ): selected, metadata = auto_select_gpu_ids("unsloth/test") - self.assertIsNone(selected) - self.assertEqual(metadata["selection_mode"], "non_cuda") + self.assertEqual(selected, [0]) + self.assertEqual(metadata["selection_mode"], "auto") - def test_prepare_gpu_selection_rejects_explicit_ids_on_xpu(self): - with patch("utils.hardware.hardware.get_device", return_value = DeviceType.XPU): - with self.assertRaisesRegex(ValueError, "only supported on CUDA"): - prepare_gpu_selection([0], model_name = "unsloth/test") + def test_prepare_gpu_selection_accepts_explicit_ids_on_xpu(self): + with ( + patch("utils.hardware.hardware.get_device", return_value = DeviceType.XPU), + patch( + "utils.hardware.hardware._get_parent_visible_gpu_spec", + return_value = { + "raw": "0", + "numeric_ids": [0], + "supports_explicit_gpu_ids": True, + }, + ), + patch( + "utils.hardware.hardware.get_parent_visible_gpu_ids", + return_value = [0], + ), + patch("utils.hardware.hardware.get_physical_gpu_count", return_value = 1), + ): + selected, metadata = prepare_gpu_selection([0], model_name = "unsloth/test") + + self.assertEqual(selected, [0]) + self.assertEqual(metadata["selection_mode"], "explicit") class TestEstimateFp16ModelSizeBytesPrefersLocalWeights(unittest.TestCase): diff --git a/studio/backend/tests/test_gpu_selection_sandbox.py b/studio/backend/tests/test_gpu_selection_sandbox.py index 733933271b..ba6d057123 100644 --- a/studio/backend/tests/test_gpu_selection_sandbox.py +++ b/studio/backend/tests/test_gpu_selection_sandbox.py @@ -294,13 +294,13 @@ class TestAutoSelectGpuIds(unittest.TestCase): # 35GB (first) + 30*0.85 (second) = 60.5GB > 50GB self.assertEqual(len(selected), 2) - def test_non_cuda_returns_none(self): + def test_non_accelerator_returns_none(self): from utils.hardware.hardware import auto_select_gpu_ids import utils.hardware.hardware as hw with patch.object(hw, "get_device", return_value = hw.DeviceType.CPU): selected, meta = auto_select_gpu_ids("test/model") self.assertIsNone(selected) - self.assertEqual(meta["selection_mode"], "non_cuda") + self.assertEqual(meta["selection_mode"], "non_accelerator") class TestGetDeviceMap(unittest.TestCase): diff --git a/studio/backend/tests/test_hf_cache_settings.py b/studio/backend/tests/test_hf_cache_settings.py new file mode 100644 index 0000000000..1875d61809 --- /dev/null +++ b/studio/backend/tests/test_hf_cache_settings.py @@ -0,0 +1,290 @@ +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 + +from __future__ import annotations + +import os +import sys +import threading +import time +from pathlib import Path + +import pytest + +_BACKEND_DIR = str(Path(__file__).resolve().parent.parent) +if _BACKEND_DIR not in sys.path: + sys.path.insert(0, _BACKEND_DIR) + +from hub.services.models.common import _local_model_info +from utils import hf_cache_settings +from utils import native_path_leases + + +@pytest.fixture() +def settings_store(monkeypatch, tmp_path): + store = {} + monkeypatch.setattr(hf_cache_settings, "_EXPLICIT_CACHE_ENV", {}) + monkeypatch.setenv("XDG_CACHE_HOME", str(tmp_path / "xdg")) + monkeypatch.setattr( + "storage.studio_db.get_app_setting", + lambda key, fallback = None: store.get(key, fallback), + ) + monkeypatch.setattr( + "storage.studio_db.upsert_app_settings", + lambda values: store.update(values) or values, + ) + return store + + +def test_studio_cache_switch_is_live_and_keeps_history(settings_store, tmp_path): + first = tmp_path / "external-a" / "huggingface" + second = tmp_path / "external-b" / "huggingface" + first.parent.mkdir() + second.parent.mkdir() + + selected = hf_cache_settings.set_hf_cache_home(str(first)) + assert selected.hub_cache == first / "hub" + assert selected.xet_cache == first / "xet" + assert selected.child_env({}) == { + "HF_HUB_CACHE": str(first / "hub"), + "HF_XET_CACHE": str(first / "xet"), + } + + hf_cache_settings.set_hf_cache_home(str(second)) + assert settings_store[hf_cache_settings.CACHE_HISTORY_SETTING_KEY] == [str(first)] + assert first / "hub" in hf_cache_settings.known_hf_hub_caches() + + reset = hf_cache_settings.set_hf_cache_home(None) + assert reset.source == "default" + assert second in hf_cache_settings.known_hf_cache_homes() + + +def test_environment_cache_is_read_only(monkeypatch, tmp_path): + custom = tmp_path / "managed" + monkeypatch.setattr( + hf_cache_settings, + "_EXPLICIT_CACHE_ENV", + {"HF_HOME": str(custom)}, + ) + paths = hf_cache_settings.get_hf_cache_paths() + assert paths.source == "environment" + assert paths.editable is False + assert paths.hub_cache == custom / "hub" + with pytest.raises(RuntimeError, match = "environment variable"): + hf_cache_settings.set_hf_cache_home(str(tmp_path / "other")) + + +def test_explicit_hub_cache_is_the_displayed_location(monkeypatch, tmp_path): + custom_hub = tmp_path / "models-cache" + custom_hub.mkdir() + monkeypatch.setattr( + hf_cache_settings, + "_EXPLICIT_CACHE_ENV", + {"HF_HUB_CACHE": str(custom_hub)}, + ) + + paths = hf_cache_settings.get_hf_cache_paths() + status = hf_cache_settings.cache_status(paths) + + assert paths.cache_home == custom_hub + assert paths.hub_cache == custom_hub + assert status["cache_home"] == str(custom_hub) + assert status["available"] is True + assert custom_hub / "hub" not in hf_cache_settings.known_hf_hub_caches() + + +def test_explicit_hub_cache_display_wins_over_hf_home(monkeypatch, tmp_path): + hf_home = tmp_path / "hf-home" + custom_hub = tmp_path / "other-disk" / "models-cache" + hf_home.mkdir() + custom_hub.mkdir(parents = True) + monkeypatch.setattr( + hf_cache_settings, + "_EXPLICIT_CACHE_ENV", + {"HF_HOME": str(hf_home), "HF_HUB_CACHE": str(custom_hub)}, + ) + + paths = hf_cache_settings.get_hf_cache_paths() + + assert paths.cache_home == custom_hub + assert paths.hub_cache == custom_hub + assert paths.xet_cache == hf_home / "xet" + assert custom_hub / "hub" not in hf_cache_settings.known_hf_hub_caches() + assert hf_home / "hub" in hf_cache_settings.known_hf_hub_caches() + + +def test_xet_only_override_keeps_model_cache_editable(settings_store, monkeypatch, tmp_path): + xet_cache = tmp_path / "chunks" + stored = tmp_path / "stored-cache" + settings_store[hf_cache_settings.CACHE_HOME_SETTING_KEY] = str(stored) + monkeypatch.setattr( + hf_cache_settings, + "_EXPLICIT_CACHE_ENV", + {"HF_XET_CACHE": str(xet_cache)}, + ) + + paths = hf_cache_settings.get_hf_cache_paths() + + assert paths.cache_home == stored + assert paths.hub_cache == stored / "hub" + assert paths.xet_cache == xet_cache + assert paths.editable is True + + selected = tmp_path / "selected-cache" + selected.parent.mkdir(exist_ok = True) + updated = hf_cache_settings.set_hf_cache_home(str(selected)) + assert updated.hub_cache == selected / "hub" + assert updated.xet_cache == xet_cache + + +def test_worker_environment_is_applied_before_import(monkeypatch, tmp_path): + hub = str(tmp_path / "hub") + xet = str(tmp_path / "xet") + observed = {} + + class Module: + @staticmethod + def run(): + import os + return os.environ["HF_HUB_CACHE"], os.environ["HF_XET_CACHE"] + + def fake_import(name): + import os + + observed["name"] = name + observed["hub"] = os.environ.get("HF_HUB_CACHE") + return Module + + monkeypatch.setattr(native_path_leases.importlib, "import_module", fake_import) + result = native_path_leases.run_without_native_path_secret( + "fake.worker", + "run", + {"HF_HUB_CACHE": hub, "HF_XET_CACHE": xet}, + ) + assert observed == {"name": "fake.worker", "hub": hub} + assert result == (hub, xet) + + +def test_spawn_environment_is_applied_then_restored(monkeypatch, tmp_path): + hub = str(tmp_path / "hub") + xet = str(tmp_path / "xet") + monkeypatch.setenv("HF_HUB_CACHE", "parent-hub") + monkeypatch.delenv("HF_XET_CACHE", raising = False) + + with hf_cache_settings.child_environment_for_spawn({"HF_HUB_CACHE": hub, "HF_XET_CACHE": xet}): + import os + assert os.environ["HF_HUB_CACHE"] == hub + assert os.environ["HF_XET_CACHE"] == xet + + assert os.environ["HF_HUB_CACHE"] == "parent-hub" + assert "HF_XET_CACHE" not in os.environ + + +def test_spawn_environment_supports_nested_contexts(monkeypatch): + monkeypatch.setenv("HF_HUB_CACHE", "parent") + + with hf_cache_settings.child_environment_for_spawn({"HF_HUB_CACHE": "outer"}): + assert os.environ["HF_HUB_CACHE"] == "outer" + with hf_cache_settings.child_environment_for_spawn({"HF_HUB_CACHE": "inner"}): + assert os.environ["HF_HUB_CACHE"] == "inner" + assert os.environ["HF_HUB_CACHE"] == "outer" + + assert os.environ["HF_HUB_CACHE"] == "parent" + + +def test_spawn_environment_serializes_threads(monkeypatch): + monkeypatch.setenv("HF_HUB_CACHE", "parent") + first_entered = threading.Event() + release_first = threading.Event() + observations: list[tuple[str, str]] = [] + + def first(): + with hf_cache_settings.child_environment_for_spawn({"HF_HUB_CACHE": "first"}): + observations.append(("first", os.environ["HF_HUB_CACHE"])) + first_entered.set() + assert release_first.wait(timeout = 2) + + def second(): + assert first_entered.wait(timeout = 2) + with hf_cache_settings.child_environment_for_spawn({"HF_HUB_CACHE": "second"}): + observations.append(("second", os.environ["HF_HUB_CACHE"])) + + first_thread = threading.Thread(target = first) + second_thread = threading.Thread(target = second) + first_thread.start() + second_thread.start() + assert first_entered.wait(timeout = 2) + time.sleep(0.02) + assert observations == [("first", "first")] + release_first.set() + first_thread.join(timeout = 2) + second_thread.join(timeout = 2) + + assert observations == [("first", "first"), ("second", "second")] + assert os.environ["HF_HUB_CACHE"] == "parent" + + +def test_cache_switch_invalidates_inventory(settings_store, tmp_path, monkeypatch): + invalidations = [] + monkeypatch.setattr( + "hub.utils.inventory_scan.invalidate_hf_cache_scans", + lambda: invalidations.append(True), + ) + selected = tmp_path / "external" / "huggingface" + selected.parent.mkdir() + + hf_cache_settings.set_hf_cache_home(str(selected)) + + assert invalidations == [True] + + +def test_cache_validation_write_tests_hub_and_xet(settings_store, tmp_path, monkeypatch): + selected = tmp_path / "external" / "huggingface" + selected.parent.mkdir() + tested = [] + real_named_temporary_file = hf_cache_settings.tempfile.NamedTemporaryFile + + def recording_write_test(*args, **kwargs): + tested.append(Path(kwargs["dir"])) + return real_named_temporary_file(*args, **kwargs) + + monkeypatch.setattr( + hf_cache_settings.tempfile, + "NamedTemporaryFile", + recording_write_test, + ) + + hf_cache_settings.set_hf_cache_home(str(selected)) + + assert tested == [selected / "hub", selected / "xet"] + + +def test_cache_validation_rejects_unwritable_child(settings_store, tmp_path, monkeypatch): + selected = tmp_path / "external" / "huggingface" + selected.parent.mkdir() + + def reject_hub(*args, **kwargs): + if Path(kwargs["dir"]).name == "hub": + raise PermissionError("read-only") + raise AssertionError("xet should not be tested after hub fails") + + monkeypatch.setattr(hf_cache_settings.tempfile, "NamedTemporaryFile", reject_hub) + + with pytest.raises(ValueError, match = "permission"): + hf_cache_settings.set_hf_cache_home(str(selected)) + + +def test_inactive_cache_model_loads_from_snapshot_path(tmp_path): + snapshot = tmp_path / "snapshots" / "revision" + snapshot.mkdir(parents = True) + row = _local_model_info( + scan_path = snapshot, + load_path = snapshot, + source = "hf_cache", + model_format = "safetensors", + model_id = "org/model", + active_cache = False, + ) + assert row.model_id == "org/model" + assert row.active_cache is False + assert row.load_id == str(snapshot) diff --git a/studio/backend/tests/test_hf_token_validation.py b/studio/backend/tests/test_hf_token_validation.py new file mode 100644 index 0000000000..31b30fc37d --- /dev/null +++ b/studio/backend/tests/test_hf_token_validation.py @@ -0,0 +1,165 @@ +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 + +"""Focused coverage for cached, rate-limited HF token validation.""" + +from __future__ import annotations + +from pathlib import Path +import sys + +import httpx +import pytest + + +_BACKEND_DIR = str(Path(__file__).resolve().parent.parent) +if _BACKEND_DIR not in sys.path: + sys.path.insert(0, _BACKEND_DIR) + +import utils.hf_token_validation as validation + + +@pytest.fixture(autouse = True) +def _reset_validation_state(): + validation.reset_hf_token_validation_state() + yield + validation.reset_hf_token_validation_state() + + +def test_cached_token_does_not_spend_another_attempt(monkeypatch): + calls = [] + + def _check(token): + calls.append(token) + return validation.TokenValidationResult(status = "valid") + + monkeypatch.setattr(validation, "_check_remote", _check) + first = validation.validate_hf_token("hf_valid", rate_key = "user:ip") + second = validation.validate_hf_token("hf_valid", rate_key = "user:ip") + + assert first.status == second.status == "valid" + assert calls == ["hf_valid"] + + +def test_three_uncached_attempts_per_hour(monkeypatch): + monkeypatch.setattr( + validation, + "_check_remote", + lambda _token: validation.TokenValidationResult(status = "invalid"), + ) + + for index in range(3): + result = validation.validate_hf_token(f"hf_bad_{index}", rate_key = "user:ip") + assert result.status == "invalid" + + limited = validation.validate_hf_token("hf_bad_4", rate_key = "user:ip") + assert limited.status == "rate_limited" + assert limited.retry_after_seconds is not None + assert limited.retry_after_seconds > 0 + + other_user = validation.validate_hf_token("hf_other", rate_key = "other:ip") + assert other_user.status == "invalid" + + +def test_window_rolls_forward(monkeypatch): + clock = {"now": 100.0} + monkeypatch.setattr(validation.time, "monotonic", lambda: clock["now"]) + monkeypatch.setattr(validation, "_MAX_ATTEMPTS", 1) + monkeypatch.setattr(validation, "_WINDOW_SECONDS", 10.0) + monkeypatch.setattr( + validation, + "_check_remote", + lambda _token: validation.TokenValidationResult(status = "invalid"), + ) + + assert validation.validate_hf_token("hf_a", rate_key = "user:ip").status == "invalid" + assert validation.validate_hf_token("hf_b", rate_key = "user:ip").status == "rate_limited" + clock["now"] += 11.0 + assert validation.validate_hf_token("hf_b", rate_key = "user:ip").status == "invalid" + + +@pytest.mark.parametrize( + ("status_code", "expected"), + [(200, "valid"), (401, "invalid"), (429, "rate_limited"), (500, "unavailable")], +) +def test_remote_status_classification(monkeypatch, status_code, expected): + response = httpx.Response( + status_code, + request = httpx.Request("GET", "https://huggingface.co/api/whoami-v2"), + headers = {"Retry-After": "42"} if status_code == 429 else None, + ) + + class _Session: + def get(self, url, *, headers, timeout): + assert url == "https://huggingface.co/api/whoami-v2" + assert headers["authorization"] == "Bearer hf_test" + assert timeout == validation._REMOTE_TIMEOUT_SECONDS + return response + + monkeypatch.setattr(validation, "get_session", lambda: _Session()) + result = validation._check_remote("hf_test") + assert result.status == expected + if status_code == 429: + assert result.retry_after_seconds == 42 + + +def test_wrapped_http_401_is_invalid(monkeypatch): + response = httpx.Response( + 401, + request = httpx.Request("GET", "https://huggingface.co/api/whoami-v2"), + ) + + class _Session: + def get(self, _url, **_kwargs): + error = RuntimeError("Invalid user token.") + error.response = response + raise error + + monkeypatch.setattr(validation, "get_session", lambda: _Session()) + assert validation._check_remote("hf_test").status == "invalid" + + +def test_remote_timeout_is_bounded_and_unavailable(monkeypatch): + class _Session: + def get(self, _url, *, headers, timeout): + assert headers["authorization"] == "Bearer hf_test" + assert timeout == validation._REMOTE_TIMEOUT_SECONDS + raise TimeoutError("timed out") + + monkeypatch.setattr(validation, "get_session", lambda: _Session()) + assert validation._check_remote("hf_test").status == "unavailable" + + +def test_raw_token_is_not_retained(monkeypatch): + monkeypatch.setattr( + validation, + "_check_remote", + lambda _token: validation.TokenValidationResult(status = "valid"), + ) + token = "hf_do_not_store_this_value" + validation.validate_hf_token(token, rate_key = "user:ip") + + assert token not in repr(validation._cache) + assert token not in repr(validation._attempts) + + +def test_unexpected_remote_exception_releases_singleflight(monkeypatch): + calls = 0 + monkeypatch.setattr(validation, "_INFLIGHT_WAIT_SECONDS", 0.0) + + def _check(_token): + nonlocal calls + calls += 1 + if calls == 1: + raise RuntimeError("unexpected failure") + return validation.TokenValidationResult(status = "valid") + + monkeypatch.setattr(validation, "_check_remote", _check) + + with pytest.raises(RuntimeError, match = "unexpected failure"): + validation.validate_hf_token("hf_test", rate_key = "user:ip") + + result = validation.validate_hf_token("hf_test", rate_key = "user:ip") + assert result.status == "valid" + assert calls == 2 + assert validation._inflight == {} diff --git a/studio/backend/tests/test_hf_xet_fallback.py b/studio/backend/tests/test_hf_xet_fallback.py index 2fff744b64..a037ea2579 100644 --- a/studio/backend/tests/test_hf_xet_fallback.py +++ b/studio/backend/tests/test_hf_xet_fallback.py @@ -1,10 +1,10 @@ # SPDX-License-Identifier: AGPL-3.0-only # Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 -"""Tests for the Studio shim over the shared unsloth_zoo Xet -> HTTP fallback. +"""Tests for the Unsloth shim over the shared unsloth_zoo Xet -> HTTP fallback. The transport-policy matrix is tested once in unsloth_zoo; here we assert only the -Studio seam: re-exporting the shared API and injecting the marker-aware +Unsloth seam: re-exporting the shared API and injecting the marker-aware prepare_cache_for_transport on the HTTP retry. CPU-only, no network, no real subprocess. """ @@ -69,7 +69,7 @@ def test_child_should_disable_xet_truth_table(): def test_shim_injects_studio_prepare_on_http_retry(monkeypatch): - """A Xet stall retries over HTTP and the shim runs Studio's marker-aware + """A Xet stall retries over HTTP and the shim runs Unsloth's marker-aware ``prepare_cache_for_transport(..., 'http')`` before the retry.""" _requires_shared() for var in ("UNSLOTH_DISABLE_XET", "UNSLOTH_STABLE_DOWNLOADS", "HF_HUB_DISABLE_XET"): @@ -101,17 +101,27 @@ def test_shim_injects_studio_prepare_on_http_retry(monkeypatch): prepared = [] monkeypatch.setattr( "hub.utils.download_registry.prepare_cache_for_transport", - lambda repo_type, repo_id, mode, *a, **k: prepared.append((repo_type, repo_id, mode)), + lambda repo_type, repo_id, mode, *a, **k: prepared.append( + (repo_type, repo_id, mode, k.get("root")) + ), ) - out = xf.hf_hub_download_with_xet_fallback(DL_REPO, FILE, None) + selected_cache = "/captured/hub" + out = xf.hf_hub_download_with_xet_fallback( + DL_REPO, + FILE, + None, + cache_dir = selected_cache, + ) assert out == "/cache/model.gguf" assert seen_disable_xet == [False, True] # Xet first, then HTTP - assert prepared == [("model", DL_REPO, "http")], "shim must run Studio's marker-aware prep" + assert prepared == [ + ("model", DL_REPO, "http", Path(selected_cache)) + ], "shim must prepare the cache captured by the download" def test_shim_snapshot_injects_studio_prepare(monkeypatch): - """The snapshot wrapper forwards Studio's marker-aware prep, like the file wrapper.""" + """The snapshot wrapper forwards Unsloth's marker-aware prep, like the file wrapper.""" captured = {} def fake_snapshot(repo_id, **kwargs): @@ -120,14 +130,26 @@ def test_shim_snapshot_injects_studio_prepare(monkeypatch): return "/tmp/snap-dir" monkeypatch.setattr(xf, "_shared_snapshot_download_with_xet_fallback", fake_snapshot) - out = xf.snapshot_download_with_xet_fallback("org/model") + selected_cache = "/captured/hub" + out = xf.snapshot_download_with_xet_fallback( + "org/model", + cache_dir = selected_cache, + ) assert out == "/tmp/snap-dir" assert captured["repo_id"] == "org/model" - assert captured["prepare_for_http_fn"] is xf._studio_prepare_for_http + prepared = [] + monkeypatch.setattr( + "hub.utils.download_registry.prepare_cache_for_transport", + lambda repo_type, repo_id, mode, *a, **k: prepared.append( + (repo_type, repo_id, mode, k.get("root")) + ), + ) + captured["prepare_for_http_fn"]("model", "org/model") + assert prepared == [("model", "org/model", "http", Path(selected_cache))] def test_degrades_gracefully_without_shared_helper(monkeypatch): - """On an older unsloth_zoo lacking the shared helper, the shim still imports (Studio + """On an older unsloth_zoo lacking the shared helper, the shim still imports (Unsloth boots) and exposes stub API doing plain HF downloads with the watchdog disabled.""" import importlib @@ -206,7 +228,7 @@ def test_degrades_gracefully_without_shared_helper(monkeypatch): def test_degrades_when_unsloth_zoo_entirely_absent(): """When unsloth_zoo is absent entirely, the import raises ModuleNotFoundError(name='unsloth_zoo') (top-level package). Guard that the shim still - degrades and does not re-raise, breaking every Studio import that pulls it in.""" + degrades and does not re-raise, breaking every Unsloth import that pulls it in.""" import importlib class _BlockZoo: @@ -248,7 +270,7 @@ def test_degrades_when_unsloth_zoo_entirely_absent(): def test_degrades_when_shared_helper_import_raises_importerror(): """unsloth_zoo can be installed yet fail to import when torch is missing (llama.cpp/GGUF-only - Studio), raising ImportError not ModuleNotFoundError. The shim must degrade for that too.""" + Unsloth), raising ImportError not ModuleNotFoundError. The shim must degrade for that too.""" import importlib class _BlockWithImportError: @@ -329,7 +351,7 @@ def test_retries_under_light_gpu_init_when_import_fails(monkeypatch): # with it set); accessing DownloadStallError drives it via __getattr__. stall_error = degraded.DownloadStallError assert seen_env == [None, "1"], seen_env - # Both attempts raised -> Studio still boots in degraded mode. + # Both attempts raised -> Unsloth still boots in degraded mode. assert issubclass(stall_error, RuntimeError) # The env override must not leak past the load. assert os.environ.get("UNSLOTH_ZOO_DISABLE_GPU_INIT") is None diff --git a/studio/backend/tests/test_identity.py b/studio/backend/tests/test_identity.py index 1e84ddef35..712348f7ca 100644 --- a/studio/backend/tests/test_identity.py +++ b/studio/backend/tests/test_identity.py @@ -3,7 +3,7 @@ """Tests for the server identity handshake (`GET /api/auth/identity`). -The endpoint lets a client confirm an endpoint is really this Studio install +The endpoint lets a client confirm an endpoint is really this Unsloth install before sending it a credential: the client sends a random nonce and checks the returned HMAC against one computed from the install identity secret. A process that cannot read this same-user secret cannot forge a matching proof. diff --git a/studio/backend/tests/test_index_bootstrap_origin_extra.py b/studio/backend/tests/test_index_bootstrap_origin_extra.py index feda88c14c..e1c52a653e 100644 --- a/studio/backend/tests/test_index_bootstrap_origin_extra.py +++ b/studio/backend/tests/test_index_bootstrap_origin_extra.py @@ -26,7 +26,7 @@ def _build_request( def test_is_same_origin_request_ipv6_loopback_same_origin(): - """Studio supports ``-H ::1`` binds; netloc is ``[::1]:8902``. Bare + """Unsloth supports ``-H ::1`` binds; netloc is ``[::1]:8902``. Bare ``partition(":")`` mis-parses the bracketed form and would refuse the bootstrap on legitimate same-origin navigation. """ diff --git a/studio/backend/tests/test_install_resolve_prebuilt.py b/studio/backend/tests/test_install_resolve_prebuilt.py index e97ca47717..02ccc68b11 100644 --- a/studio/backend/tests/test_install_resolve_prebuilt.py +++ b/studio/backend/tests/test_install_resolve_prebuilt.py @@ -200,14 +200,9 @@ def _gpu_linux_host(caps): ) -def test_host_is_blackwell_includes_datacenter_parts(): - assert ilp._host_is_blackwell(_gpu_linux_host(["10.0"])) is True # B200 sm_100 - assert ilp._host_is_blackwell(_gpu_linux_host(["10.3"])) is True # B300 sm_103 - assert ilp._host_is_blackwell(_gpu_linux_host(["12.0"])) is True # RTX 50 sm_120 - assert ilp._host_is_blackwell(_gpu_linux_host(["12.1"])) is True # DGX Spark sm_121 - assert ilp._host_is_blackwell(_gpu_linux_host(["9.0"])) is False # Hopper - assert ilp._host_is_blackwell(_gpu_linux_host(["8.0"])) is False # Ampere - assert ilp._host_is_blackwell(_gpu_linux_host(["9.0", "10.0"])) is True # highest cap wins +# _host_is_blackwell / _blackwell_min_toolkit_for_host are prebuilt_core +# re-exports; their value tables moved verbatim to +# tests/studio/install/test_prebuilt_core.py. def _linux_cuda_artifact(runtime_line, supported_sms, min_sm, max_sm, profile): @@ -285,16 +280,6 @@ def test_drop_blackwell_incapable_windows_cuda_applies_to_datacenter(): assert [a.name for a in kept] == [cuda13.name] -def test_blackwell_min_toolkit_is_sm_aware(): - # Family floor is 12.8; sm_103/sm_121 (no native target before 12.9) lift it. - f = ilp._blackwell_min_toolkit_for_host - assert f(_gpu_linux_host(["10.0"])) == (12, 8) # B200 - assert f(_gpu_linux_host(["12.0"])) == (12, 8) # RTX 50 - assert f(_gpu_linux_host(["10.3"])) == (12, 9) # B300 - assert f(_gpu_linux_host(["12.1"])) == (12, 9) # DGX Spark - assert f(_gpu_linux_host(["10.0", "10.3"])) == (12, 9) # max across SMs wins - - def test_sm103_host_drops_cuda128_windows_build(): # B300 (sm_103) needs cuda-12.9: a legacy win-cuda-12.8 build must be dropped. host = _host( @@ -445,6 +430,101 @@ def test_route_to_vulkan_prebuilt_cpu_fallback_wins(): assert routed is host +@pytest.mark.parametrize("cpu_flag", ["--cpu-fallback", "--force-cpu"]) +def test_resolve_prebuilt_cpu_fallback_overrides_intel_vulkan(monkeypatch, capsys, cpu_flag): + """Either CPU flag via CLI must suppress Vulkan even on an Intel GPU host: both + drop GPU detection (--force-cpu additionally persists, on the install path).""" + monkeypatch.setattr( + ilp, + "detect_host", + lambda: _host(is_linux = True, is_x86_64 = True, has_intel_gpu = True), + ) + seen = {} + + def _resolver(tag, host, repo, published_release_tag): + seen["host"] = host + seen["repo"] = repo + raise ilp.PrebuiltFallback("no asset") + + monkeypatch.setattr(ilp, "resolve_simple_install_release_plans", _resolver) + monkeypatch.setattr( + sys, + "argv", + [ + "install_llama_prebuilt.py", + "--resolve-prebuilt", + "latest", + cpu_flag, + "--output-format", + "json", + ], + ) + assert ilp.main() == ilp.EXIT_SUCCESS + # The CPU flag must suppress Intel GPU, route to fork (not upstream Vulkan) + assert seen["host"].has_intel_gpu is False + assert seen["repo"] == FORK + + +@pytest.mark.parametrize( + "flags, expect_force, expect_persist", + [ + ([], False, False), + # Automatic/transient last resort (arm64 GPU-build recovery): drops GPU but + # does NOT persist, so a later update heals to a GPU bundle (#6097). + (["--cpu-fallback"], True, False), + # Deliberate CPU-only (UNSLOTH_LLAMA_CPP_BACKEND=cpu): drops GPU AND persists so + # the updater re-asserts it and never revives the Intel iGPU crash (#7213). + (["--force-cpu"], True, True), + (["--cpu-fallback", "--force-cpu"], True, True), + ], +) +def test_cli_cpu_flags_thread_force_and_persist( + monkeypatch, tmp_path, flags, expect_force, expect_persist +): + captured = {} + monkeypatch.setattr(ilp, "install_prebuilt", lambda **kw: captured.update(kw)) + monkeypatch.setattr( + sys, + "argv", + ["install_llama_prebuilt.py", "--install-dir", str(tmp_path / "llama.cpp"), *flags], + ) + assert ilp.main() == ilp.EXIT_SUCCESS + assert captured["force_cpu"] is expect_force + assert captured["persist_force_cpu"] is expect_persist + + +@pytest.mark.parametrize( + "existing, requested, expected", + [ + # A deliberate --force-cpu on top of a naturally-installed CPU bundle (same + # asset, install skipped) must still flip the marker to true (#7213). + (False, True, True), + (None, True, True), + # No spurious writes when already in sync, and a released force syncs down. + (True, True, True), + (False, False, False), + (True, False, False), + ], +) +def test_sync_marker_force_cpu(tmp_path, existing, requested, expected): + marker = {"tag": "b9585", "asset": "llama-b9585-bin-ubuntu-x64.tar.gz"} + if existing is not None: + marker["force_cpu"] = existing + marker_path = tmp_path / "UNSLOTH_PREBUILT_INFO.json" + marker_path.write_text(json.dumps(marker)) + ilp.sync_marker_force_cpu(tmp_path, requested) + written = json.loads(marker_path.read_text()) + assert written["force_cpu"] is expected + # Unrelated fields are preserved. + assert written["asset"] == "llama-b9585-bin-ubuntu-x64.tar.gz" + + +def test_sync_marker_force_cpu_missing_marker_is_noop(tmp_path): + # No marker (or unreadable) must not crash the reuse path. + ilp.sync_marker_force_cpu(tmp_path, True) + assert not (tmp_path / "UNSLOTH_PREBUILT_INFO.json").exists() + + def test_route_to_vulkan_prebuilt_hidden_nvidia_not_rerouted(): # A mixed NVIDIA+Intel host that hid NVIDIA (CUDA_VISIBLE_DEVICES=""/-1): # physical NVIDIA present but not usable. Must NOT auto-route to Vulkan, or diff --git a/studio/backend/tests/test_install_whisper_prebuilt_checksums.py b/studio/backend/tests/test_install_whisper_prebuilt_checksums.py new file mode 100644 index 0000000000..19bece9d0c --- /dev/null +++ b/studio/backend/tests/test_install_whisper_prebuilt_checksums.py @@ -0,0 +1,231 @@ +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 + +"""Trust-anchor tests for install_whisper_prebuilt.py. + +Whisper verifies each download against the release's own +whisper-prebuilt-sha256.json checksum index (the same model as +install_llama_prebuilt.py), not a committed pins file. These pin the index +parser, the fail-closed behaviour when an asset is not covered, the +tampered-manifest guard, and the newest-release resolution. +""" + +from __future__ import annotations + +import importlib +import sys +from pathlib import Path + +import pytest + +_studio = Path(__file__).resolve().parent.parent.parent +if str(_studio) not in sys.path: + sys.path.insert(0, str(_studio)) + +iwp = importlib.import_module("install_whisper_prebuilt") + +if not hasattr(iwp, "parse_release_checksums"): + pytest.skip("checksum-model symbols not present - check branch", allow_module_level = True) + +_A = "0" * 64 +_B = "1" * 64 +_TAG = "v1.9.1-unsloth.1" +_REPO = "unslothai/whisper.cpp" + + +def _index(**overrides) -> dict: + payload = { + "schema_version": 1, + "component": "whisper.cpp", + "release_tag": _TAG, + "upstream_tag": "v1.9.1", + "artifacts": { + "whisper-v1.9.1-unsloth.1-linux-x64-cpu.tar.gz": {"sha256": _A}, + "whisper-v1.9.1-unsloth.1-linux-x64-cuda12-portable.tar.gz": {"sha256": _B}, + }, + } + payload.update(overrides) + return payload + + +# parse_release_checksums / expected_sha256_for are prebuilt_core re-exports; +# their valid/fail-closed matrix is asserted against the real whisper +# descriptor in tests/studio/install/test_prebuilt_core.py. The download-host +# fast-path tests below still route through this module's parse wrapper. + +# release tag resolution. + + +def test_resolve_release_tag_explicit_override_passthrough(): + assert iwp.resolve_release_tag(_REPO, published_release_tag = "v1.9.1-unsloth.2") == ( + "v1.9.1-unsloth.2" + ) + + +def test_resolve_release_tag_resolves_newest_when_no_override(monkeypatch): + monkeypatch.setattr(iwp, "resolve_newest_release_tag", lambda repo: "v9.9.9-unsloth.9") + assert iwp.resolve_release_tag(_REPO, published_release_tag = None) == "v9.9.9-unsloth.9" + + +def test_resolve_newest_release_tag_picks_latest_published(monkeypatch): + releases = [ + {"tag_name": "v1.9.1-unsloth.1", "published_at": "2026-01-01T00:00:00Z"}, + {"tag_name": "v1.9.1-unsloth.3", "published_at": "2026-03-01T00:00:00Z"}, + {"tag_name": "v1.9.1-unsloth.2", "published_at": "2026-02-01T00:00:00Z"}, + {"tag_name": "draft", "published_at": "2026-09-01T00:00:00Z", "draft": True}, + {"tag_name": "pre", "published_at": "2026-09-01T00:00:00Z", "prerelease": True}, + ] + monkeypatch.setattr(iwp, "fetch_json", lambda url: releases) + assert iwp.resolve_newest_release_tag(_REPO) == "v1.9.1-unsloth.3" + + +def test_resolve_newest_release_tag_none_published_fails_closed(monkeypatch): + monkeypatch.setattr(iwp, "fetch_json", lambda url: [{"tag_name": "d", "draft": True}]) + with pytest.raises(iwp.PrebuiltFallback): + iwp.resolve_newest_release_tag(_REPO) + + +def test_pins_symbols_are_gone(): + # The committed-pins trust model was removed in favour of llama's runtime index. + for gone in ("load_pins", "pins_path", "resolve_expected_sha256", "PINS_FILENAME"): + assert not hasattr(iwp, gone), f"{gone} should have been removed" + + +# Download-host fast path (resolve + fetch the JSON assets with no GitHub API). + +_CPU_ASSET = "whisper-v1.9.1-unsloth.1-linux-x64-cpu.tar.gz" + + +def _manifest() -> dict: + return { + "schema_version": 1, + "component": "whisper.cpp", + "upstream_tag": "v1.9.1", + "artifacts": [{"asset": _CPU_ASSET, "os": "linux", "arch": "x64", "backend": "cpu"}], + } + + +def _no_api(monkeypatch): + """Fail loudly if any code path touches api.github.com.""" + + def _boom(*a, **k): + raise AssertionError("api.github.com was used on the fast path") + + monkeypatch.setattr(iwp, "fetch_json", _boom) + monkeypatch.setattr(iwp, "github_release", _boom) + monkeypatch.setattr(iwp, "fetch_release_bundle", _boom) + + +def test_fetch_release_for_install_prefers_download_host(monkeypatch): + _no_api(monkeypatch) + monkeypatch.setattr(iwp, "_download_host_latest_release_tag", lambda repo: _TAG) + + def _dhj(url): + if url.endswith(iwp.SHA256_ASSET_NAME): + return _index() + if url.endswith(iwp.MANIFEST_ASSET_NAME): + return _manifest() + raise AssertionError(f"unexpected url {url}") + + monkeypatch.setattr(iwp, "_download_host_json", _dhj) + bundle, checks = iwp.fetch_release_for_install(_REPO, published_release_tag = None) + assert bundle.release_tag == _TAG + assert checks[_CPU_ASSET] == _A + # asset_urls point at the download host (github.com), not the API. + assert bundle.asset_urls[iwp.SHA256_ASSET_NAME].startswith( + f"https://github.com/{_REPO}/releases/" + ) + assert bundle.asset_urls[_CPU_ASSET].startswith( + f"https://github.com/{_REPO}/releases/download/" + ) + walked = iwp._fetch_release_candidate(_REPO, _TAG) + assert iwp.SHA256_ASSET_NAME in walked.asset_urls + assert _CPU_ASSET in walked.asset_urls + + +def test_fetch_release_for_install_explicit_tag_skips_the_head(monkeypatch): + # An explicit tag needs no /releases/latest HEAD: resolving it must not call it. + monkeypatch.setattr( + iwp, + "_download_host_latest_release_tag", + lambda repo: (_ for _ in ()).throw(AssertionError("HEAD used for an explicit tag")), + ) + monkeypatch.setattr( + iwp, + "_download_host_json", + lambda url: _index() if url.endswith(iwp.SHA256_ASSET_NAME) else _manifest(), + ) + _no_api(monkeypatch) + bundle, checks = iwp.fetch_release_for_install(_REPO, published_release_tag = _TAG) + assert bundle.release_tag == _TAG + + +def test_fetch_release_for_install_falls_back_to_api(monkeypatch): + # Fast path returns None (e.g. a 404) -> the API path resolves the release. + monkeypatch.setattr(iwp, "_resolve_release_via_download_host", lambda repo, tag: None) + sentinel = iwp.ReleaseBundle(repo = _REPO, release_tag = _TAG, manifest = _manifest(), asset_urls = {}) + monkeypatch.setattr(iwp, "resolve_release_tag", lambda repo, *, published_release_tag: _TAG) + monkeypatch.setattr(iwp, "fetch_release_bundle", lambda repo, tag: sentinel) + monkeypatch.setattr(iwp, "fetch_release_checksums", lambda bundle: {_CPU_ASSET: _A}) + bundle, checks = iwp.fetch_release_for_install(_REPO, published_release_tag = None) + assert bundle is sentinel + assert checks == {_CPU_ASSET: _A} + + +def test_resolve_via_download_host_sha_404_returns_none(monkeypatch): + import urllib.error + + monkeypatch.setattr(iwp, "_download_host_latest_release_tag", lambda repo: _TAG) + + def _dhj(url): + raise urllib.error.HTTPError(url, 404, "not found", {}, None) + + monkeypatch.setattr(iwp, "_download_host_json", _dhj) + assert iwp._resolve_release_via_download_host(_REPO, None) is None + + +def test_resolve_via_download_host_tag_mismatch_returns_none(monkeypatch): + # A checksum index whose self-reported release_tag disagrees is rejected (None). + monkeypatch.setattr(iwp, "_download_host_latest_release_tag", lambda repo: _TAG) + monkeypatch.setattr( + iwp, "_download_host_json", lambda url: _index(release_tag = "v1.9.1-unsloth.2") + ) + assert iwp._resolve_release_via_download_host(_REPO, None) is None + + +def test_download_host_latest_release_tag_parses_redirect(monkeypatch): + class _Resp: + def __enter__(self): + return self + + def __exit__(self, *a): + return False + + def geturl(self): + return f"https://github.com/{_REPO}/releases/tag/{_TAG}" + + class _Opener: + def open( + self, + req, + timeout = None, + ): + return _Resp() + + monkeypatch.setattr(iwp, "_URL_OPENER", _Opener()) + assert iwp._download_host_latest_release_tag(_REPO) == _TAG + + +def test_download_host_latest_release_tag_404_returns_none(monkeypatch): + import urllib.error + + class _Opener: + def open( + self, + req, + timeout = None, + ): + raise urllib.error.HTTPError(req.full_url, 404, "nf", {}, None) + + monkeypatch.setattr(iwp, "_URL_OPENER", _Opener()) + assert iwp._download_host_latest_release_tag(_REPO) is None diff --git a/studio/backend/tests/test_linux_external_media_paths.py b/studio/backend/tests/test_linux_external_media_paths.py index b735bd1132..8373cdd6bb 100644 --- a/studio/backend/tests/test_linux_external_media_paths.py +++ b/studio/backend/tests/test_linux_external_media_paths.py @@ -254,8 +254,10 @@ def test_legacy_browse_allowlist_includes_linux_run_media_mounts(monkeypatch, tm ) fake_external_media = SimpleNamespace( linux_run_media_mount_roots = lambda: [media_root], + macos_volume_roots = lambda: [], windows_drive_roots = lambda: [], ) + fake_paths.external_media = fake_external_media fake_studio_db = SimpleNamespace( list_scan_folders = lambda: [], contains_sensitive_path_component = studio_db.contains_sensitive_path_component, diff --git a/studio/backend/tests/test_llama_cpp_context_fit.py b/studio/backend/tests/test_llama_cpp_context_fit.py index d3a10df8ca..2a4f6d19d2 100644 --- a/studio/backend/tests/test_llama_cpp_context_fit.py +++ b/studio/backend/tests/test_llama_cpp_context_fit.py @@ -567,7 +567,7 @@ class TestClassifyGpuOffload: assert inst._classify_gpu_offload(False, []) is None def test_user_did_not_intend_gpu_returns_none(self): - # Studio called start_llama_server without expecting GPU; don't warn. + # Unsloth called start_llama_server without expecting GPU; don't warn. inst = self._backend( [ "load_tensors: CPU_Mapped model buffer size = 21000.0 MiB", diff --git a/studio/backend/tests/test_llama_cpp_mmproj_fallback.py b/studio/backend/tests/test_llama_cpp_mmproj_fallback.py index 04d4aac9e1..4332a440a5 100644 --- a/studio/backend/tests/test_llama_cpp_mmproj_fallback.py +++ b/studio/backend/tests/test_llama_cpp_mmproj_fallback.py @@ -222,7 +222,7 @@ class TestFlashAttnOff: assert _flash_off(["llama-server", "-fa=on"]) == ["llama-server", "-fa=off"] def test_flips_every_occurrence_last_wins(self): - # extra_args can re-enable FA after Studio's flag; llama.cpp is last-wins, + # extra_args can re-enable FA after Unsloth's flag; llama.cpp is last-wins, # so one leftover 'on' would re-crash the retry. Every enable must flip. cmd = ["llama-server", "--flash-attn", "on", "--mmproj", "/p", "--flash-attn", "on"] out = _flash_off(cmd) @@ -234,7 +234,7 @@ class TestFlashAttnOff: assert _flash_off(["llama-server", "--flash-attn=off"]) is None def test_none_when_user_off_wins_last(self): - # User appended 'off' after Studio's 'on'; effective (last-wins) is off, + # User appended 'off' after Unsloth's 'on'; effective (last-wins) is off, # so there is nothing to retry. assert _flash_off(["llama-server", "--flash-attn", "on", "--flash-attn", "off"]) is None @@ -335,3 +335,24 @@ class TestRetryContract: def test_external_kill_skips_flash_attn_retry(self): # SIGKILL (-9, OOM killer) is not a program fault: no FA-off retry. assert _signal_crash(-9) is False + + +class TestMmprojRetryFailureMessage: + """#7302: bare mmproj crashes must not be reported as projector-format.""" + + def test_confirmed_projector_keeps_historical_wording(self): + msg = LlamaCppBackend._mmproj_retry_failure_message( + projector_confirmed = True, + detail = "llama-server failed to start", + ) + assert msg.startswith("Vision projector incompatible with this llama.cpp") + assert "llama-server failed to start" in msg + + def test_bare_crash_does_not_claim_projector_incompatibility(self): + msg = LlamaCppBackend._mmproj_retry_failure_message( + projector_confirmed = False, + detail = "llama-server failed to start. Check that the GGUF file is valid", + ) + assert "Vision projector incompatible" not in msg + assert "crashed with --mmproj" in msg + assert "GGUF file is valid" in msg diff --git a/studio/backend/tests/test_llama_cpp_mtp_detection.py b/studio/backend/tests/test_llama_cpp_mtp_detection.py index 3f9d2a8f50..27c1b17a85 100644 --- a/studio/backend/tests/test_llama_cpp_mtp_detection.py +++ b/studio/backend/tests/test_llama_cpp_mtp_detection.py @@ -9,6 +9,7 @@ the _already_in_target_state mirror that prevents needless reloads. from __future__ import annotations +import ast import inspect import os import struct @@ -345,10 +346,62 @@ def test_windows_full_offload_flags_use_current_llama_server_args(): stale_checkpoint_flag = "--checkpoint-" + "every-n-tokens" assert '"--cache-ram"' in src assert '"--ctx-checkpoints"' in src - assert '"--no-cache-prompt"' in src + # Prompt caching stays on (in-VRAM prefix reuse); #5692 only needed the host-RAM + # checkpoints (--cache-ram / --ctx-checkpoints) disabled, not prompt reuse. + assert '"--no-cache-prompt"' not in src assert stale_checkpoint_flag not in src +# Backend-wide guard: Unsloth must never inject --no-cache-prompt into a llama-server +# command. It disables in-VRAM prompt-prefix reuse, re-prefilling every repeated prompt +# (#5692 only needed --cache-ram / --ctx-checkpoints off; #7260 dropped the stray flag). +# Detecting it (_is_real) or honouring a user-supplied one (_prompt_cache_off) is fine. +_NO_CACHE_PROMPT_FLAG = "--no-cache-prompt" +_LIST_MUTATORS = frozenset({"append", "extend", "insert"}) + + +def _has_flag_literal(node: ast.AST) -> bool: + return any( + isinstance(n, ast.Constant) and n.value == _NO_CACHE_PROMPT_FLAG for n in ast.walk(node) + ) + + +def _no_cache_prompt_injections(source: str, filename: str) -> list[tuple[str, int]]: + """(file, lineno) for each spot adding --no-cache-prompt to a list.""" + hits: list[tuple[str, int]] = [] + for node in ast.walk(ast.parse(source, filename = filename)): + # cmd.append/extend/insert(... flag ...) or cmd += [... flag ...] + if ( + isinstance(node, ast.Call) + and isinstance(node.func, ast.Attribute) + and node.func.attr in _LIST_MUTATORS + and any(_has_flag_literal(a) for a in node.args) + ) or ( + isinstance(node, ast.AugAssign) + and isinstance(node.op, ast.Add) + and _has_flag_literal(node.value) + ): + hits.append((filename, node.lineno)) + return hits + + +def test_unsloth_never_injects_no_cache_prompt_into_any_command(): + root = Path(_BACKEND_DIR) + files = [p for p in root.rglob("*.py") if "tests" not in p.relative_to(root).parts] + violations: list[tuple[str, int]] = [] + for path in files: + try: + violations += _no_cache_prompt_injections(path.read_text(encoding = "utf-8"), str(path)) + except (OSError, UnicodeDecodeError, SyntaxError): + continue + assert files, "no backend source files were scanned" + assert violations == [], ( + "Unsloth must never add --no-cache-prompt to a llama-server command " + "(it disables prompt-prefix reuse); detecting or honouring a user-supplied " + f"one is fine. Offending sites: {violations}" + ) + + def test_load_model_sets_threads_once(): src = inspect.getsource(LlamaCppBackend.load_model) assert src.count('cmd.extend(["--threads", str(') == 1 @@ -584,7 +637,9 @@ def test_probe_server_capabilities_uses_binary_library_env(tmp_path, monkeypatch def fake_run(cmd, **kwargs): captured["cmd"] = cmd captured["env"] = kwargs.get("env") - return _types.SimpleNamespace(stdout = "--spec-type none,mtp,ngram-simple\n", stderr = "") + return _types.SimpleNamespace( + stdout = "--spec-type none,mtp,ngram-simple\n", stderr = "", returncode = 0 + ) monkeypatch.setattr("core.inference.llama_cpp.subprocess.run", fake_run) @@ -625,6 +680,95 @@ def test_probe_server_capabilities_reports_outdated_binary(tmp_path): assert caps["found"] is True assert caps["mtp_token"] is None assert caps["supports_mtp"] is False + assert caps["mtp_probe_inconclusive"] is False + + +@_NEEDS_BASH +def test_probe_server_capabilities_reads_mtp_from_multiline_help(tmp_path): + # Enum on the indented line: first-line-only probing falsely reported + # "lacks MTP" (#7302). + fake = _make_fake_llama_server( + tmp_path / "llama-server", + "--spec-type TYPE\n" + " speculative decoding type\n" + " (none,draft-simple,draft-mtp,ngram-mod)\n", + ) + _clear_caps_cache() + caps = LlamaCppBackend.probe_server_capabilities(str(fake)) + assert caps["mtp_token"] == "draft-mtp" + assert caps["supports_mtp"] is True + assert caps["mtp_probe_inconclusive"] is False + + +@_NEEDS_BASH +def test_probe_server_capabilities_empty_help_fails_open(tmp_path): + # --help prints nothing: must not claim the prebuilt lacks MTP (#7302). + fake = tmp_path / "llama-server" + fake.write_text("#!/usr/bin/env bash\nexit 0\n") + fake.chmod(0o755) + _clear_caps_cache() + caps = LlamaCppBackend.probe_server_capabilities(str(fake)) + assert caps["found"] is True + assert caps["mtp_token"] is None + assert caps["supports_mtp"] is False + assert caps["mtp_probe_inconclusive"] is True + + +@_NEEDS_BASH +def test_probe_server_capabilities_no_spec_type_is_definitive(tmp_path): + # Nonempty --help without --spec-type: pre-spec binary, not inconclusive. + fake = _make_fake_llama_server( + tmp_path / "llama-server", + "--gpu-layers N\n GPU layers to offload\n", + ) + _clear_caps_cache() + caps = LlamaCppBackend.probe_server_capabilities(str(fake)) + assert caps["found"] is True + assert caps["mtp_token"] is None + assert caps["supports_mtp"] is False + assert caps["mtp_probe_inconclusive"] is False + + +@_NEEDS_BASH +def test_probe_server_capabilities_failed_help_with_output_is_inconclusive(tmp_path): + fake = tmp_path / "llama-server" + fake.write_text( + "#!/usr/bin/env bash\n" + 'if [ "$1" = "--help" ]; then\n' + " echo 'illegal instruction'\n" + " exit 1\n" + "fi\n" + ) + fake.chmod(0o755) + _clear_caps_cache() + caps = LlamaCppBackend.probe_server_capabilities(str(fake)) + assert caps["found"] is True + assert caps["supports_mtp"] is False + assert caps["mtp_probe_inconclusive"] is True + + +@_NEEDS_BASH +def test_probe_server_capabilities_crash_on_help_fails_open(tmp_path): + fake = tmp_path / "llama-server" + fake.write_text("#!/usr/bin/env bash\nkill -SEGV $$\n") + fake.chmod(0o755) + _clear_caps_cache() + caps = LlamaCppBackend.probe_server_capabilities(str(fake)) + assert caps["found"] is True + assert caps["mtp_token"] is None + assert caps["supports_mtp"] is False + assert caps["mtp_probe_inconclusive"] is True + + +def test_mtp_token_from_spec_help_prefers_draft_mtp(): + assert ( + LlamaCppBackend._mtp_token_from_spec_help("--spec-type none,draft-mtp,mtp,ngram-mod") + == "draft-mtp" + ) + assert LlamaCppBackend._mtp_token_from_spec_help("--spec-type [none|mtp|ngram-cache]") == "mtp" + assert LlamaCppBackend._mtp_token_from_spec_help("--spec-type none,ngram-mod") is None + # No incidental substring matches. + assert LlamaCppBackend._mtp_token_from_spec_help("prompt cache") is None def test_probe_server_capabilities_handles_missing_binary(): @@ -632,6 +776,7 @@ def test_probe_server_capabilities_handles_missing_binary(): caps = LlamaCppBackend.probe_server_capabilities("/no/such/llama-server") assert caps["found"] is False assert caps["supports_mtp"] is False + assert caps["mtp_probe_inconclusive"] is True assert caps["supports_cache_ram"] is False assert caps["supports_ctx_checkpoints"] is False assert caps["supports_no_cache_prompt"] is False @@ -741,6 +886,25 @@ def test_probe_reports_windows_cache_flags_absent_for_older_binary(tmp_path): assert caps["supports_no_cache_prompt"] is False +@_NEEDS_BASH +def test_probe_detects_slot_save_path(tmp_path): + fake = _make_fake_llama_server( + tmp_path / "llama-server", + "--slot-save-path PATH path to save slot kv cache\n--threads N\n", + ) + _clear_caps_cache() + caps = LlamaCppBackend.probe_server_capabilities(str(fake)) + assert caps["supports_slot_save"] is True + + +@_NEEDS_BASH +def test_probe_reports_slot_save_absent_for_older_binary(tmp_path): + fake = _make_fake_llama_server(tmp_path / "llama-server", "--threads N\n") + _clear_caps_cache() + caps = LlamaCppBackend.probe_server_capabilities(str(fake)) + assert caps["supports_slot_save"] is False + + def test_build_ngram_mod_flags_new(): flags = _build_ngram_mod_flags({"ngram_mod_flavor": "new"}) assert flags == [ @@ -1014,7 +1178,7 @@ def test_already_in_target_state_2b_falls_back_to_ngram_below_threshold(monkeypa ) -# usage backfill from timings (Studio UI t/s widget fix). +# usage backfill from timings (Unsloth UI t/s widget fix). def test_backfill_usage_from_timings_fills_when_completion_tokens_zero(): @@ -1104,12 +1268,14 @@ def _resolver_backend( *, ngram_supported = True, mtp_token = "draft-mtp", + mtp_probe_inconclusive = False, ): """Backend with a deterministic probe so the resolver is hermetic.""" fake = { "found": True, "mtp_token": mtp_token, "supports_mtp": bool(mtp_token), + "mtp_probe_inconclusive": mtp_probe_inconclusive, "ngram_mod_flavor": "new" if ngram_supported else None, "supports_ngram_mod": bool(ngram_supported), "spec_draft_n_max_flag": "--spec-draft-n-max", @@ -1606,7 +1772,7 @@ def test_reload_forced_mtp_bounces_auto_mla(): ) -# ── Full named-repo resolver matrix (the shipping Studio families) ───── +# ── Full named-repo resolver matrix (the shipping Unsloth families) ───── # # Locks auto / off / forced-mtp routing for every Qwen3.5 (MTP + plain) and # gemma-4 (regular + QAT) GGUF repo, including the giant MoEs that stay @@ -1807,6 +1973,24 @@ def test_spec_fallback_reason_set_when_binary_lacks_mtp(monkeypatch): assert backend.spec_fallback_reason == "binary_no_mtp" +def test_spec_fallback_reason_none_when_mtp_probe_inconclusive(monkeypatch): + backend = _resolver_backend( + monkeypatch, + mtp_token = None, + mtp_probe_inconclusive = True, + ) + backend._build_speculative_flags( + speculative_type = "mtp", + spec_draft_n_max = None, + extra_args = None, + model_identifier = _MTP_MODEL, + model_path = None, + gpus = True, + binary = "/fake/llama-server", + ) + assert backend.spec_fallback_reason is None + + def test_spec_fallback_reason_none_when_mtp_engages(monkeypatch): backend = _resolver_backend(monkeypatch) backend._build_speculative_flags( diff --git a/studio/backend/tests/test_llama_cpp_no_context_shift.py b/studio/backend/tests/test_llama_cpp_no_context_shift.py index 10b1dc7ff6..662c918305 100644 --- a/studio/backend/tests/test_llama_cpp_no_context_shift.py +++ b/studio/backend/tests/test_llama_cpp_no_context_shift.py @@ -5,7 +5,7 @@ With llama-server's default context-shift behavior, the UI cannot tell the user the KV cache was rotated -- earlier turns silently vanish from the conversation. -The Studio backend always passes ``--no-context-shift`` so the server returns a +The Unsloth backend always passes ``--no-context-shift`` so the server returns a clean error instead, and the chat adapter can point the user at the ``Context Length`` input in the settings panel. @@ -118,9 +118,17 @@ def test_flag_sits_inside_the_base_cmd_list(): "conditional branch -- otherwise some code paths would still " "run with silent context shift enabled." ) - # Pin that it sits next to -c / --ctx so the grouping makes sense. - assert '"-c"' in block assert '"--flash-attn"' in block + # -c is emitted in the conditional right after the base list, not inside + # it: auto-fit (--fit on with no pinned context) must omit -c entirely, + # because "-c 0" pins the full native context and disables --fit's + # VRAM-based sizing. Pin that it still sits next to the base block so the + # context grouping stays intact. + after = rest[end_rel : end_rel + 1000] + assert '"-c"' in after, ( + "-c must still be emitted in the conditional immediately after the " + "base cmd list (omitted only in auto-fit, where --fit sizes context)." + ) def _iter_lines_with_offset(text: str): diff --git a/studio/backend/tests/test_llama_cpp_props_readback.py b/studio/backend/tests/test_llama_cpp_props_readback.py index 316956325f..fe1e67edad 100644 --- a/studio/backend/tests/test_llama_cpp_props_readback.py +++ b/studio/backend/tests/test_llama_cpp_props_readback.py @@ -4,7 +4,7 @@ """Tests for the post-launch /props context readback. llama-server's memory-fit step or --parallel slot split can allocate less -context than the requested -c while Studio keeps advertising the requested +context than the requested -c while Unsloth keeps advertising the requested value; clients sized to it then die on exceed_context_size_error 400s. ``_reconcile_effective_ctx_with_server`` must adopt the server's real ``default_generation_settings.n_ctx`` whenever it is smaller. @@ -223,33 +223,48 @@ _CAPS_NONE = {"supports_kv_unified": False, "supports_fit_ctx": False} def test_kv_unified_added_for_multi_slot(): """Explicit --parallel N disables llama-server's auto-slots kv-unified - default, splitting -c into per-slot windows of -c/N; Studio must restore + default, splitting -c into per-slot windows of -c/N; Unsloth must restore the shared pool so one request can use the full advertised context.""" - flags = LlamaCppBackend._ctx_integrity_flags(4, False, 98304, 98304, _CAPS_ALL) + flags = LlamaCppBackend._ctx_integrity_flags(4, False, False, 98304, 98304, _CAPS_ALL) assert "--kv-unified" in flags def test_kv_unified_skipped_for_single_slot_or_old_build(): assert "--kv-unified" not in LlamaCppBackend._ctx_integrity_flags( - 1, False, 98304, 98304, _CAPS_ALL + 1, False, False, 98304, 98304, _CAPS_ALL ) assert "--kv-unified" not in LlamaCppBackend._ctx_integrity_flags( - 4, False, 98304, 98304, _CAPS_NONE + 4, False, False, 98304, 98304, _CAPS_NONE ) def test_fit_ctx_floors_explicit_request_under_fit(): - flags = LlamaCppBackend._ctx_integrity_flags(1, True, 98304, 98304, _CAPS_ALL) + # An explicit requested ctx floors --fit-ctx at that value on any --fit + # path, including legacy auto (auto_fit False). + flags = LlamaCppBackend._ctx_integrity_flags(1, True, False, 98304, 98304, _CAPS_ALL) assert flags[flags.index("--fit-ctx") + 1] == "98304" -def test_fit_ctx_skipped_without_fit_or_explicit_ctx_or_support(): +def test_fit_ctx_skipped_without_fit_or_support(): + # No --fit on -> no --fit-ctx. assert "--fit-ctx" not in LlamaCppBackend._ctx_integrity_flags( - 1, False, 98304, 98304, _CAPS_ALL + 1, False, False, 98304, 98304, _CAPS_ALL ) - assert "--fit-ctx" not in LlamaCppBackend._ctx_integrity_flags(1, True, 0, 262144, _CAPS_ALL) + # --fit on but the binary doesn't support --fit-ctx. assert "--fit-ctx" not in LlamaCppBackend._ctx_integrity_flags( - 1, True, 98304, 98304, _CAPS_NONE + 1, True, True, 98304, 98304, _CAPS_NONE + ) + + +def test_fit_ctx_floors_auto_request_at_8192_only_under_auto_fit(): + # Manual + Auto (auto_fit) floors the auto window at 8192 so --fit can't + # shrink it to a tiny size. + flags = LlamaCppBackend._ctx_integrity_flags(1, True, True, 0, 262144, _CAPS_ALL) + assert flags[flags.index("--fit-ctx") + 1] == "8192" + # Legacy auto (fit on but not auto_fit) emits -c 0 to pin native, so the + # 8192 floor must NOT ride along and override that pin. + assert "--fit-ctx" not in LlamaCppBackend._ctx_integrity_flags( + 1, True, False, 0, 262144, _CAPS_ALL ) diff --git a/studio/backend/tests/test_llama_cpp_slot_resume.py b/studio/backend/tests/test_llama_cpp_slot_resume.py new file mode 100644 index 0000000000..8b20c952c4 --- /dev/null +++ b/studio/backend/tests/test_llama_cpp_slot_resume.py @@ -0,0 +1,494 @@ +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 + +import os +from types import SimpleNamespace + +import core.inference.llama_cpp as llama_cpp +from core.inference.llama_cpp import LlamaCppBackend + + +def _resume_backend(tmp_path, n_slots = 1): + backend = LlamaCppBackend() + backend._healthy = True + # No-op lifecycle methods so the atexit cleanup can kill the fake quietly. + backend._process = SimpleNamespace( + poll = lambda: None, + terminate = lambda: None, + wait = lambda *a, **k: 0, + kill = lambda: None, + pid = 0, + ) + backend._port = 8081 + backend._slot_save_dir = str(tmp_path) + backend._slot_save_binary = ("/bin/llama-server", 1) + (tmp_path / "model.gguf").write_bytes(b"gguf") + backend._gguf_path = str(tmp_path / "model.gguf") + backend._effective_parallel_slots = n_slots + backend._estimate_kv_cache_bytes = lambda *a, **k: 0 + return backend + + +def _fake_disk(monkeypatch, free = 1 << 40): + monkeypatch.setattr(llama_cpp.shutil, "disk_usage", lambda _p: SimpleNamespace(free = free)) + + +class _Resp: + def __init__( + self, + status_code = 200, + body = None, + ): + self.status_code = status_code + self._body = body or {} + + def json(self): + return self._body + + +def test_save_returns_none_when_slot_save_disabled(monkeypatch, tmp_path): + backend = _resume_backend(tmp_path) + backend._slot_save_dir = None + monkeypatch.setattr( + llama_cpp.httpx, + "post", + lambda *a, **k: (_ for _ in ()).throw(AssertionError), + raising = False, + ) + assert backend.save_slots_for_resume() is None + + +def test_save_skipped_when_prompt_cache_disabled(monkeypatch, tmp_path): + backend = _resume_backend(tmp_path) + backend._prompt_cache_disabled = True + monkeypatch.setattr( + llama_cpp.httpx, + "post", + lambda *a, **k: (_ for _ in ()).throw(AssertionError), + raising = False, + ) + assert backend.save_slots_for_resume() is None + + +def test_save_skipped_when_insufficient_free_disk(monkeypatch, tmp_path): + backend = _resume_backend(tmp_path) + backend._estimate_kv_cache_bytes = lambda *a, **k: 1 << 40 + _fake_disk(monkeypatch, free = 1 << 20) + monkeypatch.setattr( + llama_cpp.httpx, + "post", + lambda *a, **k: (_ for _ in ()).throw(AssertionError), + raising = False, + ) + assert backend.save_slots_for_resume() is None + + +def test_save_collects_manifest_across_slots(monkeypatch, tmp_path): + backend = _resume_backend(tmp_path, n_slots = 2) + _fake_disk(monkeypatch) + calls = [] + + def fake_post(url, **kwargs): + calls.append((url, kwargs["params"], kwargs["json"])) + return _Resp(200, {"n_saved": 40, "n_written": 100}) + + monkeypatch.setattr(llama_cpp.httpx, "post", fake_post, raising = False) + manifest = backend.save_slots_for_resume() + assert manifest is not None + assert manifest["dir"] == str(tmp_path) + assert manifest["binary"] == ("/bin/llama-server", 1) + assert manifest["gguf"] == str(tmp_path / "model.gguf") + st = os.stat(manifest["gguf"]) + assert manifest["gguf_stat"] == ((st.st_size, st.st_mtime_ns),) + assert manifest["launch"] == backend._slot_launch_fingerprint() + assert [e["id"] for e in manifest["slots"]] == [0, 1] + assert all(e["n_saved"] == 40 for e in manifest["slots"]) + assert [c[1] for c in calls] == [{"action": "save"}] * 2 + assert "/slots/0" in calls[0][0] and "/slots/1" in calls[1][0] + + +def test_save_unlinks_empty_slot_and_returns_none(monkeypatch, tmp_path): + backend = _resume_backend(tmp_path) + _fake_disk(monkeypatch) + + def fake_post(url, **kwargs): + (tmp_path / kwargs["json"]["filename"]).write_bytes(b"") + return _Resp(200, {"n_saved": 0, "n_written": 0}) + + monkeypatch.setattr(llama_cpp.httpx, "post", fake_post, raising = False) + assert backend.save_slots_for_resume() is None + assert list(tmp_path.glob("resume-*.bin")) == [] # empty-slot file removed + + +def test_save_cap_breach_discards_all_files(monkeypatch, tmp_path): + backend = _resume_backend(tmp_path, n_slots = 2) + _fake_disk(monkeypatch) + monkeypatch.setattr(llama_cpp, "_SLOT_SAVE_MAX_BYTES", 150) + + def fake_post(url, **kwargs): + (tmp_path / kwargs["json"]["filename"]).write_bytes(b"x" * 100) + return _Resp(200, {"n_saved": 40, "n_written": 100}) + + monkeypatch.setattr(llama_cpp.httpx, "post", fake_post, raising = False) + assert backend.save_slots_for_resume() is None # 200 bytes > 150 cap + assert list(tmp_path.glob("resume-*.bin")) == [] + + +def test_save_transport_error_aborts_remaining_slots(monkeypatch, tmp_path): + backend = _resume_backend(tmp_path, n_slots = 3) + _fake_disk(monkeypatch) + calls = [] + + def fake_post(url, **kwargs): + calls.append(url) + raise OSError("connection refused") + + monkeypatch.setattr(llama_cpp.httpx, "post", fake_post, raising = False) + assert backend.save_slots_for_resume() is None + assert len(calls) == 1 # no retries against a dead server + + +def test_save_transport_error_unlinks_partial_file(monkeypatch, tmp_path): + backend = _resume_backend(tmp_path) + _fake_disk(monkeypatch) + + def fake_post(url, **kwargs): + (tmp_path / kwargs["json"]["filename"]).write_bytes(b"partial") + raise OSError("timed out") + + monkeypatch.setattr(llama_cpp.httpx, "post", fake_post, raising = False) + assert backend.save_slots_for_resume() is None + assert list(tmp_path.glob("resume-*.bin")) == [] + + +def test_fingerprint_tracks_lora_sidecar_rewrite(tmp_path): + backend = _resume_backend(tmp_path) + adapter = tmp_path / "adapter.gguf" + adapter.write_bytes(b"v1") + backend._extra_args = ["--lora", str(adapter)] + + before = backend._slot_launch_fingerprint() + adapter.write_bytes(b"v2-different") # re-exported adapter, same path + assert backend._slot_launch_fingerprint() != before + + backend._extra_args = [f"--lora={adapter}"] + assert backend._sidecar_weight_files() == [str(adapter)] + backend._extra_args = ["--lora-scaled", str(adapter), "0.5"] + assert backend._sidecar_weight_files() == [str(adapter)] + backend._extra_args = ["--control-vector", str(adapter), "--threads", "4"] + assert backend._sidecar_weight_files() == [str(adapter)] + + +def test_sidecar_files_parse_csv_and_colon_scale(tmp_path): + backend = _resume_backend(tmp_path) + a, b = tmp_path / "a.gguf", tmp_path / "b.gguf" + + backend._extra_args = ["--lora", f"{a},{b}"] + files = backend._sidecar_weight_files() + assert str(a) in files and str(b) in files + + backend._extra_args = ["--lora-scaled", f"{a}:0.5"] + assert str(a) in backend._sidecar_weight_files() + + backend._extra_args = ["--control-vector-scaled", f"{a}:1.0,{b}:2.0"] + files = backend._sidecar_weight_files() + assert str(a) in files and str(b) in files + + # Windows drive letter must not be mistaken for a scale separator. + backend._extra_args = ["--lora-scaled", "C:\\adapters\\a.gguf:0.75"] + assert "C:\\adapters\\a.gguf" in backend._sidecar_weight_files() + backend._extra_args = ["--lora", "C:\\adapters\\a.gguf"] + assert backend._sidecar_weight_files() == ["C:\\adapters\\a.gguf"] + + +def test_fingerprint_tracks_colon_scaled_adapter_rewrite(tmp_path): + backend = _resume_backend(tmp_path) + adapter = tmp_path / "adapter.gguf" + adapter.write_bytes(b"v1") + backend._extra_args = ["--lora-scaled", f"{adapter}:0.5"] + + before = backend._slot_launch_fingerprint() + adapter.write_bytes(b"v2-different") # re-exported adapter, same path + assert backend._slot_launch_fingerprint() != before + + +def test_fingerprint_tracks_effective_context_length(tmp_path): + backend = _resume_backend(tmp_path) + backend._effective_context_length = 8192 + + before = backend._slot_launch_fingerprint() + backend._effective_context_length = 4096 # auto-fit landed smaller on reload + assert backend._slot_launch_fingerprint() != before + + +def test_gguf_file_identity_covers_split_shards(tmp_path): + backend = _resume_backend(tmp_path) + first = tmp_path / "m-00001-of-00002.gguf" + second = tmp_path / "m-00002-of-00002.gguf" + first.write_bytes(b"a") + second.write_bytes(b"bb") + + before = backend._gguf_file_identity(str(first)) + st1, st2 = os.stat(first), os.stat(second) + assert before == ((st1.st_size, st1.st_mtime_ns), (st2.st_size, st2.st_mtime_ns)) + + second.write_bytes(b"rewritten") # sibling changes, primary untouched + after = backend._gguf_file_identity(str(first)) + assert after is not None and after != before + assert after[0] == before[0] # primary shard unchanged + + second.unlink() + assert backend._gguf_file_identity(str(first)) is None # missing shard + + +def test_save_skipped_when_user_disabled_prompt_cache(monkeypatch, tmp_path): + backend = _resume_backend(tmp_path) + backend._extra_args = ["--no-cache-prompt"] + monkeypatch.setattr( + llama_cpp.httpx, + "post", + lambda *a, **k: (_ for _ in ()).throw(AssertionError), + raising = False, + ) + assert backend.save_slots_for_resume() is None + + +def test_save_skipped_when_env_disables_prompt_cache(monkeypatch, tmp_path): + backend = _resume_backend(tmp_path) + monkeypatch.setenv("LLAMA_ARG_CACHE_PROMPT", "0") + monkeypatch.setattr( + llama_cpp.httpx, + "post", + lambda *a, **k: (_ for _ in ()).throw(AssertionError), + raising = False, + ) + assert backend.save_slots_for_resume() is None + monkeypatch.delenv("LLAMA_ARG_CACHE_PROMPT") + monkeypatch.setenv("LLAMA_ARG_NO_CACHE_PROMPT", "1") # legacy negative form + assert backend.save_slots_for_resume() is None + + +def test_explicit_cache_prompt_flag_overrides_env(monkeypatch, tmp_path): + backend = _resume_backend(tmp_path) + monkeypatch.setenv("LLAMA_ARG_CACHE_PROMPT", "0") + backend._extra_args = ["--cache-prompt"] # CLI wins over env in llama.cpp + _fake_disk(monkeypatch) + monkeypatch.setattr( + llama_cpp.httpx, + "post", + lambda *a, **k: _Resp(200, {"n_saved": 1, "n_written": 1}), + raising = False, + ) + assert backend.save_slots_for_resume() is not None + + +def test_user_cache_prompt_overrides_studio_no_cache_flag(monkeypatch, tmp_path): + # User extras follow Studio's flags, so an explicit --cache-prompt wins. + backend = _resume_backend(tmp_path) + backend._prompt_cache_disabled = True + backend._extra_args = ["--cache-prompt"] + _fake_disk(monkeypatch) + monkeypatch.setattr( + llama_cpp.httpx, + "post", + lambda *a, **k: _Resp(200, {"n_saved": 1, "n_written": 1}), + raising = False, + ) + assert backend.save_slots_for_resume() is not None + # Last flag wins when both appear in extras. + backend._extra_args = ["--cache-prompt", "--no-cache-prompt"] + assert backend.save_slots_for_resume() is None + + +def test_save_stops_writing_once_cap_exceeded(monkeypatch, tmp_path): + backend = _resume_backend(tmp_path, n_slots = 3) + _fake_disk(monkeypatch) + monkeypatch.setattr(llama_cpp, "_SLOT_SAVE_MAX_BYTES", 150) + calls = [] + + def fake_post(url, **kwargs): + calls.append(url) + (tmp_path / kwargs["json"]["filename"]).write_bytes(b"x" * 100) + return _Resp(200, {"n_saved": 1, "n_written": 100}) + + monkeypatch.setattr(llama_cpp.httpx, "post", fake_post, raising = False) + assert backend.save_slots_for_resume() is None + assert len(calls) == 2 # cap blown after slot 1; slot 2 never attempted + assert list(tmp_path.glob("resume-*.bin")) == [] + + +def test_save_aborts_between_slots_when_no_longer_idle(monkeypatch, tmp_path): + backend = _resume_backend(tmp_path, n_slots = 3) + _fake_disk(monkeypatch) + calls = [] + + def fake_post(url, **kwargs): + calls.append(url) + return _Resp(200, {"n_saved": 5, "n_written": 10}) + + monkeypatch.setattr(llama_cpp.httpx, "post", fake_post, raising = False) + aborts = iter([False, True, True]) + manifest = backend.save_slots_for_resume(should_abort = lambda: next(aborts)) + assert len(calls) == 1 # slots 1 and 2 skipped + assert manifest is not None + assert [e["id"] for e in manifest["slots"]] == [0] + + +def test_save_non_200_slot_is_skipped_but_others_kept(monkeypatch, tmp_path): + backend = _resume_backend(tmp_path, n_slots = 2) + _fake_disk(monkeypatch) + + def fake_post(url, **kwargs): + if "/slots/0" in url: + return _Resp(500) + return _Resp(200, {"n_saved": 5, "n_written": 10}) + + monkeypatch.setattr(llama_cpp.httpx, "post", fake_post, raising = False) + manifest = backend.save_slots_for_resume() + assert manifest is not None + assert [e["id"] for e in manifest["slots"]] == [1] + + +def test_restore_posts_each_slot_and_tolerates_failures(monkeypatch, tmp_path): + backend = _resume_backend(tmp_path) + calls = [] + + def fake_post(url, **kwargs): + calls.append((url, kwargs["params"], kwargs["json"])) + return _Resp(500 if "/slots/0" in url else 200, {"n_restored": 5}) + + monkeypatch.setattr(llama_cpp.httpx, "post", fake_post, raising = False) + backend.restore_slots_for_resume( + { + "slots": [ + {"id": 0, "filename": "resume-a-slot0.bin", "n_saved": 5}, + {"id": 1, "filename": "resume-a-slot1.bin", "n_saved": 5}, + ] + } + ) + assert [c[1] for c in calls] == [{"action": "restore"}] * 2 + assert calls[0][2] == {"filename": "resume-a-slot0.bin"} + + +def test_restore_transport_error_stops_early(monkeypatch, tmp_path): + backend = _resume_backend(tmp_path) + calls = [] + + def fake_post(url, **kwargs): + calls.append(url) + raise OSError("connection refused") + + monkeypatch.setattr(llama_cpp.httpx, "post", fake_post, raising = False) + backend.restore_slots_for_resume( + {"slots": [{"id": 0, "filename": "a.bin"}, {"id": 1, "filename": "b.bin"}]} + ) + assert len(calls) == 1 + + +def test_save_deletes_orphan_on_malformed_response(monkeypatch, tmp_path): + # A 200 that writes a file but returns a non-numeric counter must be cleaned + # up like any other save failure, not left orphaned holding chat KV. + backend = _resume_backend(tmp_path) + _fake_disk(monkeypatch) + + def fake_post(url, **kwargs): + (tmp_path / kwargs["json"]["filename"]).write_bytes(b"chat-kv") + return _Resp(200, {"n_saved": "not-an-int"}) + + monkeypatch.setattr(llama_cpp.httpx, "post", fake_post, raising = False) + assert backend.save_slots_for_resume() is None + assert list(tmp_path.glob("resume-*.bin")) == [] + + +def test_save_deletes_orphan_on_non_dict_response(monkeypatch, tmp_path): + backend = _resume_backend(tmp_path) + _fake_disk(monkeypatch) + + def fake_post(url, **kwargs): + (tmp_path / kwargs["json"]["filename"]).write_bytes(b"chat-kv") + return _Resp(200, ["unexpected", "list"]) + + monkeypatch.setattr(llama_cpp.httpx, "post", fake_post, raising = False) + assert backend.save_slots_for_resume() is None + assert list(tmp_path.glob("resume-*.bin")) == [] + + +def test_save_cap_uses_actual_file_size_not_reported_bytes(monkeypatch, tmp_path): + # A binary under-reporting n_written must not slip past the disk cap: the + # cap is enforced against the bytes actually on disk. + backend = _resume_backend(tmp_path) + _fake_disk(monkeypatch) + monkeypatch.setattr(llama_cpp, "_SLOT_SAVE_MAX_BYTES", 150) + + def fake_post(url, **kwargs): + (tmp_path / kwargs["json"]["filename"]).write_bytes(b"x" * 200) + return _Resp(200, {"n_saved": 5, "n_written": 1}) # under-reported + + monkeypatch.setattr(llama_cpp.httpx, "post", fake_post, raising = False) + assert backend.save_slots_for_resume() is None # 200 real bytes > 150 cap + assert list(tmp_path.glob("resume-*.bin")) == [] + + +def test_save_skipped_when_estimate_exceeds_cap(monkeypatch, tmp_path): + # An estimate over the cap skips before writing any slot at all. + backend = _resume_backend(tmp_path) + backend._estimate_kv_cache_bytes = lambda *a, **k: 1 << 40 + monkeypatch.setattr(llama_cpp, "_SLOT_SAVE_MAX_BYTES", 1 << 20) + _fake_disk(monkeypatch) + monkeypatch.setattr( + llama_cpp.httpx, + "post", + lambda *a, **k: (_ for _ in ()).throw(AssertionError), + raising = False, + ) + assert backend.save_slots_for_resume() is None + + +def test_save_skipped_when_model_file_changed_since_load(monkeypatch, tmp_path): + # The GGUF/sidecars were swapped on disk after the server loaded them, so the + # live KV belongs to the old weights: refuse to persist it (no POST at all). + backend = _resume_backend(tmp_path) + backend._slot_loaded_identity = ((("stale", 0),), ()) # != current identity + _fake_disk(monkeypatch) + monkeypatch.setattr( + llama_cpp.httpx, + "post", + lambda *a, **k: (_ for _ in ()).throw(AssertionError), + raising = False, + ) + assert backend.save_slots_for_resume() is None + + +def test_save_proceeds_when_load_identity_matches(monkeypatch, tmp_path): + # Matching load-time snapshot: the save runs normally. + backend = _resume_backend(tmp_path) + backend._slot_loaded_identity = ( + backend._gguf_file_identity(backend._gguf_path), + backend._slot_launch_fingerprint(), + ) + _fake_disk(monkeypatch) + + def fake_post(url, **kwargs): + (tmp_path / kwargs["json"]["filename"]).write_bytes(b"kv") + return _Resp(200, {"n_saved": 5, "n_written": 2}) + + monkeypatch.setattr(llama_cpp.httpx, "post", fake_post, raising = False) + manifest = backend.save_slots_for_resume() + assert manifest is not None + assert [e["id"] for e in manifest["slots"]] == [0] + + +def test_save_skipped_when_estimate_unavailable_and_low_disk(monkeypatch, tmp_path): + # A 0 estimate means metadata was insufficient, not a zero-byte cache: the save + # must demand room for the whole cap, not just 1 GiB, on a low-disk host. + backend = _resume_backend(tmp_path) + backend._estimate_kv_cache_bytes = lambda *a, **k: 0 # metadata unavailable + monkeypatch.setattr(llama_cpp, "_SLOT_SAVE_MAX_BYTES", 8 << 30) # 8 GiB cap + _fake_disk(monkeypatch, free = 2 << 30) # 2 GiB free < 8 + 1 GiB required + monkeypatch.setattr( + llama_cpp.httpx, + "post", + lambda *a, **k: (_ for _ in ()).throw(AssertionError), + raising = False, + ) + assert backend.save_slots_for_resume() is None diff --git a/studio/backend/tests/test_llama_cpp_stall_timeout.py b/studio/backend/tests/test_llama_cpp_stall_timeout.py new file mode 100644 index 0000000000..da36f75e8e --- /dev/null +++ b/studio/backend/tests/test_llama_cpp_stall_timeout.py @@ -0,0 +1,125 @@ +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 + +"""Regression test for the post-first-token stall timeout in the cancel-aware read. + +httpcore snapshots ``request.extensions["timeout"]["read"]`` once at body start, so +when ``_iter_text_cancellable`` lowers it after the first token, a one-token-then-silent +server hangs for the full prefill window. The fix re-reads the live extensions timeout +per call; a fake clock and always-silent stream check the read gives up after the live +stall timeout, not the stale prefill one. +""" + +from __future__ import annotations + +import inspect +import sys +import threading +import types as _types +from pathlib import Path + +import pytest + +_BACKEND_DIR = str(Path(__file__).resolve().parent.parent) +if _BACKEND_DIR not in sys.path: + sys.path.insert(0, _BACKEND_DIR) + +# Mirror sibling tests' stubbing so the module imports without fastapi. +_loggers_stub = _types.ModuleType("loggers") +_loggers_stub.get_logger = lambda name: __import__("logging").getLogger(name) +sys.modules.setdefault("loggers", _loggers_stub) +sys.modules.setdefault("structlog", _types.ModuleType("structlog")) + +import httpcore # noqa: E402 + +from core.inference import llama_cpp as llama_cpp_mod # noqa: E402 +from core.inference.llama_cpp import LlamaCppBackend # noqa: E402 + +_PREFILL_TIMEOUT = 1200.0 # what httpcore snapshots from the prefill timeout +_STALL_TIMEOUT = 120.0 # the post-first-token stall timeout the wrapper must honor + + +class _Obj: + pass + + +def _install(response, clock, silent_stream): + """Wire fake client/pool so _install_cancel_aware_read finds the stream; return the wrapped stream.read.""" + inner = _Obj() + inner._network_stream = silent_stream + connection = _Obj() + connection._connection = inner + pool = _Obj() + pool._connections = [connection] + transport = _Obj() + transport._pool = pool + client = _Obj() + client._transport = transport + + cancel_event = threading.Event() # never set: we test the stall path, not cancel + sig = inspect.signature(LlamaCppBackend._install_cancel_aware_read) + if "response" in sig.parameters: + # Fixed signature: wrapper reads the live extensions timeout. + LlamaCppBackend._install_cancel_aware_read(client, cancel_event, response) + else: + # Pre-fix signature: no response, so the stall assertion fails (proves the bug). + LlamaCppBackend._install_cancel_aware_read(client, cancel_event) + return silent_stream.read + + +def test_stall_timeout_honored_after_first_token(monkeypatch): + clock = {"t": 0.0} + monkeypatch.setattr(llama_cpp_mod.time, "monotonic", lambda: clock["t"]) + + # One token then silence: every read times out, advancing fake time by its timeout. + def silent_read(max_bytes, timeout = None): + clock["t"] += timeout if timeout is not None else 0.0 + raise httpcore.ReadTimeout("slice timed out on silence") + + stream = _Obj() + stream.read = silent_read + + # First token seen: the live read timeout is lowered to the stall timeout. + request = _Obj() + request.extensions = {"timeout": {"read": _STALL_TIMEOUT}} + response = _Obj() + response.request = request + + wrapped_read = _install(response, clock, stream) + + # httpcore still passes the stale prefill timeout it snapshotted at body start. + with pytest.raises(httpcore.ReadTimeout): + wrapped_read(65536, timeout = _PREFILL_TIMEOUT) + + # Must give up ~stall timeout after the last token, not the prefill window. + assert clock["t"] <= _STALL_TIMEOUT * 1.5, ( + f"stall timeout not honored: waited {clock['t']}s " + f"(expected ~{_STALL_TIMEOUT}s, not {_PREFILL_TIMEOUT}s)" + ) + assert clock["t"] >= _STALL_TIMEOUT * 0.5 + + +def test_prefill_timeout_used_when_no_live_override(monkeypatch): + """Without a lowered live timeout, the wrapper honors the passed prefill timeout, so the normal first-token wait is unchanged.""" + clock = {"t": 0.0} + monkeypatch.setattr(llama_cpp_mod.time, "monotonic", lambda: clock["t"]) + + def silent_read(max_bytes, timeout = None): + clock["t"] += timeout if timeout is not None else 0.0 + raise httpcore.ReadTimeout("slice timed out on silence") + + stream = _Obj() + stream.read = silent_read + + # No timeout extension: wrapper falls back to httpcore's passed timeout. + request = _Obj() + request.extensions = {} + response = _Obj() + response.request = request + + wrapped_read = _install(response, clock, stream) + + with pytest.raises(httpcore.ReadTimeout): + wrapped_read(65536, timeout = _PREFILL_TIMEOUT) + + assert clock["t"] >= _PREFILL_TIMEOUT * 0.9 diff --git a/studio/backend/tests/test_llama_cpp_tool_loop.py b/studio/backend/tests/test_llama_cpp_tool_loop.py index bd2c008589..e99e227d40 100644 --- a/studio/backend/tests/test_llama_cpp_tool_loop.py +++ b/studio/backend/tests/test_llama_cpp_tool_loop.py @@ -122,7 +122,7 @@ def _structured_tool_call(tool_name: str, arguments: dict, call_id: str) -> list def test_structured_tool_call_after_visible_preface_is_executed(monkeypatch): """llama-server may emit content first and then native delta.tool_calls. - Studio must not drop that tool call after it has streamed the preface. + Unsloth must not drop that tool call after it has streamed the preface. """ tool_call_id = "call_render_late" diff --git a/studio/backend/tests/test_llama_cpp_update.py b/studio/backend/tests/test_llama_cpp_update.py index 83ea07a066..9e23242b97 100644 --- a/studio/backend/tests/test_llama_cpp_update.py +++ b/studio/backend/tests/test_llama_cpp_update.py @@ -83,6 +83,7 @@ def _write_install( repo: str = "unslothai/llama.cpp", asset: str | None = None, release_tag: str | None = None, + force_cpu: bool | None = None, ) -> str: """Create a fake prebuilt install and return the llama-server path.""" bin_dir = dir_ / "build" / "bin" @@ -99,6 +100,8 @@ def _write_install( } if asset is not None: marker["asset"] = asset + if force_cpu is not None: + marker["force_cpu"] = force_cpu (dir_ / MARKER).write_text(json.dumps(marker)) return str(binary) @@ -116,6 +119,9 @@ def _clean_state(monkeypatch, tmp_path): monkeypatch.delenv("UNSLOTH_LLAMA_CPP_PATH", raising = False) # Never hit the network in these tests. monkeypatch.setattr(freshness, "_fetch_latest_release_tag", lambda repo, timeout = 5.0: None) + # Keep the whisper piggyback out of the llama-only tests: no host probe, no + # whisper phase (test_combined_update.py covers the chained flow). + monkeypatch.setattr(upd, "_whisper_chain_status", lambda **kwargs: None) yield freshness.reset_caches() upd._reset_job_for_tests() @@ -493,6 +499,47 @@ def test_start_update_preserves_vulkan_via_env(monkeypatch, tmp_path): assert popen_kwargs["env"]["UNSLOTH_FORCE_VULKAN"] == "1" +@pytest.mark.parametrize( + "force_cpu, expect_flag", + [ + # A deliberate CPU install (marker force_cpu=True) re-asserts --force-cpu on + # update so detect_host on a GPU host cannot re-route and revive the crash + # (#7213); --force-cpu also re-persists the flag for the next update. + (True, True), + # A transient fallback (or a legacy marker without the flag) stays free to + # heal to a GPU bundle (#6097). + (False, False), + (None, False), + ], +) +def test_start_update_cpu_fallback_preserved_by_flag(monkeypatch, tmp_path, force_cpu, expect_flag): + asset = "llama-b9493-bin-ubuntu-x64.tar.gz" + install_dir = tmp_path / "llama.cpp" + binary = _write_install(install_dir, "b9493", asset = asset, force_cpu = force_cpu) + monkeypatch.setattr(upd, "_find_binary", lambda: binary) + monkeypatch.setattr(upd, "_installer_script", lambda: tmp_path / "install_llama_prebuilt.py") + monkeypatch.setattr(freshness, "_fetch_latest_release_tag", lambda repo, timeout = 5.0: "b9518") + + captured: dict = {} + + def _on_start(cmd): + captured["cmd"] = cmd + _write_install(install_dir, "b9518", asset = asset, force_cpu = force_cpu) + + _patch_installer_popen(monkeypatch, lines = ["installed\n"], on_start = _on_start) + + assert upd.start_update()["started"] is True + deadline = time.time() + 10 + while time.time() < deadline: + job = upd.get_update_status()["job"] + if job["state"] in ("success", "error"): + break + time.sleep(0.05) + assert job["state"] == "success", job + assert ("--force-cpu" in captured["cmd"]) is expect_flag + assert "--cpu-fallback" not in captured["cmd"] + + def test_start_update_reports_full_release_tag(monkeypatch, tmp_path): install_dir = tmp_path / "llama.cpp" binary = _write_install(install_dir, "b9595") @@ -676,7 +723,7 @@ def test_install_cmd_rocm_marker_forwards_gfx(monkeypatch, tmp_path): assert "--rocm-gfx" in cmd assert cmd[cmd.index("--rocm-gfx") + 1] == "gfx110x" assert "--has-rocm" not in cmd - assert "--cpu-fallback" not in cmd + assert "--force-cpu" not in cmd assert "--simple-policy" not in cmd assert "--published-repo" in cmd and "unslothai/llama.cpp" in cmd @@ -690,17 +737,17 @@ def test_install_cmd_fork_rocm_marker_forwards_has_rocm(monkeypatch, tmp_path): def test_install_cmd_ggml_cpu_marker_has_no_cpu_fallback(monkeypatch, tmp_path): - # Legacy CPU installs recorded a ggml-org marker (new installs use the fork). - # Re-running into the same install-dir/repo reproduces the same CPU bundle; - # --cpu-fallback (which force-drops GPU detection) is reserved for setup.sh's - # arm64 rescue and must not appear here. + # Legacy CPU installs recorded a ggml-org marker (new installs use the fork) with + # no force_cpu field. Re-running into the same install-dir/repo reproduces the same + # CPU bundle; --force-cpu (the persisted-CPU re-assert) must not appear for a marker + # that never recorded a deliberate CPU choice, so it can still heal to GPU (#6097). cmd = _capture_install_cmd( monkeypatch, tmp_path, repo = "ggml-org/llama.cpp", asset = "llama-b9334-bin-ubuntu-x64.tar.gz", ) - assert "--cpu-fallback" not in cmd + assert "--force-cpu" not in cmd assert "--rocm-gfx" not in cmd assert "--has-rocm" not in cmd assert "--simple-policy" not in cmd @@ -714,7 +761,7 @@ def test_install_cmd_cuda_marker_minimal_and_backward_compatible(monkeypatch, tm assert "--simple-policy" not in cmd assert "--rocm-gfx" not in cmd assert "--has-rocm" not in cmd - assert "--cpu-fallback" not in cmd + assert "--force-cpu" not in cmd def test_install_cmd_pins_offered_release_tag(monkeypatch, tmp_path): diff --git a/studio/backend/tests/test_llama_cpp_wait_for_health.py b/studio/backend/tests/test_llama_cpp_wait_for_health.py index 82c5b4931a..423c3dd009 100644 --- a/studio/backend/tests/test_llama_cpp_wait_for_health.py +++ b/studio/backend/tests/test_llama_cpp_wait_for_health.py @@ -224,7 +224,7 @@ class TestRetryLogFilenameUnique: class TestFitOffRetryEligible: """Gate for the one-shot --fit off startup-crash retry. - Retry only when Studio's own VRAM math placed the model and nothing + Retry only when Unsloth's own VRAM math placed the model and nothing on the command line chose the fit mode explicitly.""" def test_eligible_for_plain_ngl_launch(self): diff --git a/studio/backend/tests/test_llama_cpp_wait_for_vram_settle.py b/studio/backend/tests/test_llama_cpp_wait_for_vram_settle.py index d0213f6079..b28df7ec3f 100644 --- a/studio/backend/tests/test_llama_cpp_wait_for_vram_settle.py +++ b/studio/backend/tests/test_llama_cpp_wait_for_vram_settle.py @@ -346,7 +346,7 @@ def test_helper_is_static_method_callable_off_class(): def test_kill_orphaned_servers_returns_count(): """The reaper reports how many owned orphans it killed, so __init__ can - arm the settle wait. Only Studio-owned llama-server procs count.""" + arm the settle wait. Only Unsloth-owned llama-server procs count.""" import os mypid = os.getpid() @@ -376,7 +376,7 @@ def test_kill_orphaned_servers_returns_count(): patch.object(LlamaCppBackend, "_pid_parent_is_alive", staticmethod(lambda pid: False)), ): n = LlamaCppBackend._kill_orphaned_servers() - assert n == 1, "only the Studio-owned orphan should be counted" + assert n == 1, "only the Unsloth-owned orphan should be counted" assert killed == [mypid + 1] # No owned orphans -> zero, so __init__ leaves the cold-start sentinel. @@ -392,8 +392,8 @@ def test_kill_orphaned_servers_returns_count(): def test_kill_orphaned_servers_spares_live_parent(): - """A Studio-owned llama-server whose parent is still running is not an - orphan (a live Studio or the user's shell owns it) and must never be + """An Unsloth-owned llama-server whose parent is still running is not an + orphan (a live Unsloth or the user's shell owns it) and must never be killed; only the true orphan (parent gone) is reaped.""" import os @@ -548,7 +548,7 @@ def test_record_then_reap_round_trip_identity_matches(tmp_path): def test_reap_recorded_pid_spares_live_server(tmp_path): - """A recorded server whose parent is still alive (the running Studio) is NEVER + """A recorded server whose parent is still alive (the running Unsloth) is NEVER reaped, and its pidfile is kept. This is the finding-3 guard: a helper backend constructed in-process must not kill the active chat server. Uses the REAL _pid_parent_is_alive (the child's parent is this live test process).""" diff --git a/studio/backend/tests/test_llama_cpp_windows_nvidia_path.py b/studio/backend/tests/test_llama_cpp_windows_nvidia_path.py index 957de4bad6..489d9eb8d1 100644 --- a/studio/backend/tests/test_llama_cpp_windows_nvidia_path.py +++ b/studio/backend/tests/test_llama_cpp_windows_nvidia_path.py @@ -3,7 +3,7 @@ """Tests for the Windows pip-nvidia DLL dir resolver. -Studio installs torch with bundled CUDA wheels (nvidia-cuda-runtime-cu13, +Unsloth installs torch with bundled CUDA wheels (nvidia-cuda-runtime-cu13, nvidia-cublas-cu13, etc.) and the prebuilt llama-server.exe must find those DLLs at runtime to load CUDA. Mirrors the Linux LD_LIBRARY_PATH block. See unslothai/unsloth#5106. diff --git a/studio/backend/tests/test_llama_route.py b/studio/backend/tests/test_llama_route.py index 0ecfeee018..cc450d55cc 100644 --- a/studio/backend/tests/test_llama_route.py +++ b/studio/backend/tests/test_llama_route.py @@ -100,6 +100,21 @@ def test_status_response_exposes_update_size_bytes(): assert rl.LlamaUpdateStatusResponse(**without).model_dump()["update_size_bytes"] is None +def test_status_response_exposes_update_component(): + model = rl.LlamaUpdateStatusResponse( + supported = True, + update_available = True, + llama_update_available = False, + update_component = "whisper", + whisper = { + "update_available": True, + "installed_tag": "v1", + "latest_tag": "v2", + }, + ) + assert model.model_dump()["update_component"] == "whisper" + + def test_status_handler_runs_off_event_loop(monkeypatch): seen = {} diff --git a/studio/backend/tests/test_llama_server_args.py b/studio/backend/tests/test_llama_server_args.py index deeb228026..fa4ba71791 100644 --- a/studio/backend/tests/test_llama_server_args.py +++ b/studio/backend/tests/test_llama_server_args.py @@ -75,7 +75,7 @@ validate_extra_args = _lsa.validate_extra_args # Reasoning controls ["--reasoning-format", "deepseek"], ["-rea", "auto"], - # Soft-managed: user flags last-wins over Studio's auto-set version. + # Soft-managed: user flags last-wins over Unsloth's auto-set version. # --parallel / -np / --n-parallel are hard-denied (KV-cache + slot # count would desync); use `unsloth studio run --parallel N` instead. ["-c", "131072"], @@ -150,7 +150,7 @@ def test_non_flag_token_passes_through(): "--mmproj", "-mmu", "--mmproj-url", - # Networking (Studio binds + proxies) + # Networking (Unsloth binds + proxies) "--host", "--port", "--path", @@ -176,13 +176,15 @@ def test_non_flag_token_passes_through(): "--models-autoload", "--no-models-autoload", # Server-mode flips: --embedding / --rerank restrict llama-server to - # those endpoints and break Studio's chat hop. + # those endpoints and break Unsloth's chat hop. "--embedding", "--embeddings", "--rerank", "--reranking", - # llama-server's own --tools clashes with Studio's tool policy. + # llama-server's own --tools clashes with Unsloth's tool policy. "--tools", + # Slot-state dir: Studio owns it for KV persistence across idle unload. + "--slot-save-path", ], ) def test_denylist_rejects_all_aliases(denied): @@ -194,7 +196,7 @@ def test_denylist_rejects_all_aliases(denied): "args,offending", [ # Pass-through --parallel would last-wins-override the real slot - # count while Studio's KV-cache fit + llama_parallel_slots stay at + # count while Unsloth's KV-cache fit + llama_parallel_slots stay at # the typer value -- plan vs. process disagree. (["--parallel", "8"], "--parallel"), (["--parallel=8"], "--parallel"), @@ -224,6 +226,16 @@ def test_denylist_rejects_equals_form(): validate_extra_args(["--port=9000"]) +def test_slot_save_path_is_managed_in_all_forms(): + for args in (["--slot-save-path", "/tmp/x"], ["--slot-save-path=/tmp/x"], ["--slot-save-path"]): + with pytest.raises(ValueError, match = "--slot-save-path"): + validate_extra_args(args) + assert is_managed_flag("--slot-save-path") is True + assert is_managed_flag("--slot-save-path=/tmp/x") is True + # --slots (read-only diagnostics endpoint) stays a user choice. + assert is_managed_flag("--slots") is False + + @pytest.mark.parametrize( "padded", [" --parallel", "--parallel ", "\t--parallel", " -np", "-np \n", "-np\t"], @@ -656,7 +668,7 @@ def test_extra_args_disable_mmproj_last_wins(): def test_strip_shadowing_flags_drops_model_draft_with_spec(): - # --model-draft (and aliases) are Studio-managed since the separate + # --model-draft (and aliases) are Unsloth-managed since the separate # MTP drafter support: an inherited copy must not last-wins-override # the auto-detected drafter. out = strip_shadowing_flags( @@ -681,7 +693,7 @@ def test_strip_shadowing_flags_drops_model_draft_with_spec(): ) def test_strip_shadowing_flags_drops_hf_drafter_selectors_with_spec(selector): # HF drafter selectors must reset on inherit like local --model-draft, or a - # stale inherited HF drafter last-wins over Studio's re-derived spec choice. + # stale inherited HF drafter last-wins over Unsloth's re-derived spec choice. out = strip_shadowing_flags( selector + ["--top-k", "20"], strip_context = False, @@ -747,6 +759,34 @@ def test_strip_shadowing_flags_defaults_strip_split_mode_too(): assert strip_shadowing_flags(["--split-mode", "tensor"]) == [] +def test_strip_offload_is_opt_in_and_covers_moe(): + base = dict( + strip_context = False, + strip_cache = False, + strip_spec = False, + strip_template = False, + strip_split_mode = False, + ) + # Default: offload (incl. MoE) flags are NOT stripped. + assert strip_shadowing_flags(["--n-cpu-moe", "8", "--top-k", "20"], **base) == [ + "--n-cpu-moe", + "8", + "--top-k", + "20", + ] + # Opt-in strips layer AND MoE offload flags (value-aware), keeps the rest. + assert strip_shadowing_flags( + ["--n-cpu-moe", "8", "--gpu-layers", "33", "--fit", "off", "--top-k", "20"], + **base, + strip_offload = True, + ) == ["--top-k", "20"] + # Boolean --cpu-moe drops the flag only, not the following value. + assert strip_shadowing_flags(["--cpu-moe", "--seed", "-1"], **base, strip_offload = True) == [ + "--seed", + "-1", + ] + + @pytest.mark.parametrize( "args", [ @@ -769,7 +809,7 @@ def test_strip_split_mode_only_preserves_none_and_empty(): def test_strip_shadowing_flags_drops_tensor_split_with_split_mode(): # --tensor-split is coupled to the split mode: stripped together so a stale - # ratio can't override Studio's computed tensor split. Other flags survive. + # ratio can't override Unsloth's computed tensor split. Other flags survive. out = strip_shadowing_flags( ["--split-mode", "row", "--tensor-split", "1,1", "--top-k", "20"], strip_context = False, @@ -796,6 +836,23 @@ def test_strip_split_mode_only_drops_tensor_split_too(): assert strip_split_mode_only(["-sm=tensor", "-ts=3,1"]) == [] +def test_strip_tensor_split_alone_preserves_split_mode(): + # Manual mode emits its own --tensor-split, so an inherited ratio is dropped + # -- but the user's --split-mode row/none/layer choice (which the manual + # ratio toggle can't express) must survive. strip_tensor_split removes only + # the ratio, unlike strip_split_mode which removes the whole group. + out = strip_shadowing_flags( + ["--split-mode", "row", "--tensor-split", "1,1", "--top-k", "20"], + strip_context = False, + strip_cache = False, + strip_spec = False, + strip_template = False, + strip_split_mode = False, + strip_tensor_split = True, + ) + assert out == ["--split-mode", "row", "--top-k", "20"] + + def test_strip_shadowing_flags_keeps_model_draft_without_spec(): out = strip_shadowing_flags( ["--model-draft", "/custom/mtp.gguf"], diff --git a/studio/backend/tests/test_local_llama_cpp_link.py b/studio/backend/tests/test_local_llama_cpp_link.py index c78c029d91..79c9977c84 100644 --- a/studio/backend/tests/test_local_llama_cpp_link.py +++ b/studio/backend/tests/test_local_llama_cpp_link.py @@ -4,7 +4,7 @@ """Behavioral tests for the --with-llama-cpp-dir 'unmanaged local link' contract. When the canonical llama.cpp dir is a symlink (POSIX) / junction (Windows) to a -user's own checkout, Studio must treat it as externally managed: +user's own checkout, Unsloth must treat it as externally managed: - the in-app updater must not offer or apply a prebuilt over the link - orphan cleanup must not kill a llama-server the user launched from that tree @@ -21,6 +21,13 @@ from utils import llama_cpp_update as u from core.inference.llama_cpp import LlamaCppBackend +@pytest.fixture(autouse = True) +def _no_whisper_piggyback(monkeypatch): + # Keep the whisper piggyback probe off the host: these tests exercise the + # llama local-link contract only. + monkeypatch.setattr(u, "_whisper_chain_status", lambda **kwargs: None) + + def _make_link(link: Path, target: Path) -> None: """Create a directory junction (Windows) / symlink (POSIX); neither needs elevation.""" @@ -67,7 +74,7 @@ def test_active_install_is_local_link(tmp_path: Path) -> None: binary = str(link / _server_subpath()) assert u._active_install_is_local_link(binary) is True - # A plain (non-link) llama.cpp dir is Studio-managed, not a local link. + # A plain (non-link) llama.cpp dir is Unsloth-managed, not a local link. plain = tmp_path / "plain" / "llama.cpp" plain.mkdir(parents = True) assert u._active_install_is_local_link(str(plain / _server_subpath())) is False diff --git a/studio/backend/tests/test_mcp_server.py b/studio/backend/tests/test_mcp_server.py new file mode 100644 index 0000000000..71792605ae --- /dev/null +++ b/studio/backend/tests/test_mcp_server.py @@ -0,0 +1,290 @@ +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 + +import asyncio +import sys +import types + +import pytest + +from mcp_server import BearerTokenMiddleware, _clamp, _dump, create_studio_mcp + + +def _get_tool(name): + tools = asyncio.run(create_studio_mcp().list_tools()) + return {tool.name: tool for tool in tools}[name] + + +def test_studio_mcp_registers_control_plane_tools(): + tools = asyncio.run(create_studio_mcp().list_tools()) + + assert {tool.name for tool in tools} == { + "studio_status", + "list_local_models", + "get_training_status", + "start_training", + "stop_training", + "list_training_runs", + "validate_recipe", + "get_recipe_job_status", + "get_recipe_job_dataset", + "load_checkpoint", + "export_gguf", + } + + +def test_dump_serializes_pydantic_values(): + class Response: + def model_dump(self, *, mode): + assert mode == "json" + return {"ok": True} + + assert _dump(Response()) == {"ok": True} + assert _dump({"already": "json"}) == {"already": "json"} + + +def test_bearer_token_middleware_rejects_wrong_token(): + events = [] + + async def app(scope, receive, send): + events.append("app") + + async def send(message): + events.append(message) + + middleware = BearerTokenMiddleware(app, "secret") + asyncio.run( + middleware( + {"type": "http", "headers": [(b"authorization", b"Bearer wrong")]}, + None, + send, + ) + ) + + assert events[0]["status"] == 401 + assert "app" not in events + + +def test_bearer_token_middleware_closes_unauthorized_websocket(): + events = [] + + async def app(scope, receive, send): + events.append("app") + + async def send(message): + events.append(message) + + middleware = BearerTokenMiddleware(app, "secret") + asyncio.run( + middleware( + {"type": "websocket", "headers": []}, + None, + send, + ) + ) + + assert events == [{"type": "websocket.close", "code": 4401}] + + +def test_bearer_token_middleware_rejects_non_ascii_authorization(): + # A non-ASCII bearer value must produce a clean 401, not a 500. Comparing on + # bytes avoids the str hmac.compare_digest TypeError on non-ASCII input. + events = [] + + async def app(scope, receive, send): + events.append("app") + + async def send(message): + events.append(message) + + middleware = BearerTokenMiddleware(app, "secret") + asyncio.run( + middleware( + {"type": "http", "headers": [(b"authorization", b"Bearer \xff\xff")]}, + None, + send, + ) + ) + + assert events[0]["status"] == 401 + assert "app" not in events + + +def test_bearer_token_middleware_accepts_correct_token(): + events = [] + + async def app(scope, receive, send): + events.append("app") + + async def send(message): + events.append(message) + + middleware = BearerTokenMiddleware(app, "secret") + asyncio.run( + middleware( + {"type": "http", "headers": [(b"authorization", b"Bearer secret")]}, + None, + send, + ) + ) + + assert events == ["app"] + + +def test_bearer_token_middleware_requires_non_empty_token(): + async def app(scope, receive, send): + pass + + for bad in ("", " "): + with pytest.raises(ValueError): + BearerTokenMiddleware(app, bad) + + +def test_bearer_token_middleware_rejects_non_ascii_token(): + async def app(scope, receive, send): + pass + + # non-ASCII tokens cannot be transmitted in an HTTP header by a standard + # client, so they are rejected at construction instead of locking out. + for bad in ("töken", "\U0001f600"): + with pytest.raises(ValueError): + BearerTokenMiddleware(app, bad) + + +def test_bearer_token_middleware_passes_through_non_http_scopes(): + events = [] + + async def app(scope, receive, send): + events.append("app") + + async def send(message): + events.append(message) + + middleware = BearerTokenMiddleware(app, "secret") + asyncio.run(middleware({"type": "lifespan"}, None, send)) + + assert events == ["app"] + + +def test_clamp_restricts_to_inclusive_bounds(): + assert _clamp(5, 1, 200) == 5 + assert _clamp(-10, 1, 200) == 1 + assert _clamp(10_000, 1, 200) == 200 + assert _clamp(0, 1, 500) == 1 + assert _clamp(1_000, 1, 500) == 500 + + +def test_export_and_checkpoint_tools_expose_forwarded_fields(): + export_props = set(_get_tool("export_gguf").parameters["properties"]) + assert {"hf_token", "imatrix", "imatrix_path"} <= export_props + + checkpoint_props = set(_get_tool("load_checkpoint").parameters["properties"]) + assert {"hf_token", "approved_remote_code_fingerprint"} <= checkpoint_props + + +def _stub_module(monkeypatch, name, **attrs): + module = types.ModuleType(name) + for key, value in attrs.items(): + setattr(module, key, value) + if "." in name: + module.__path__ = [] # mark package-like so submodule imports resolve + monkeypatch.setitem(sys.modules, name, module) + return module + + +def test_export_gguf_forwards_hf_token_and_imatrix(monkeypatch): + captured = {} + + class FakeExportGGUFRequest: + def __init__(self, **kwargs): + captured.update(kwargs) + + async def fake_export(request, current_subject): + return {"current_subject": current_subject} + + _stub_module(monkeypatch, "models", ExportGGUFRequest = FakeExportGGUFRequest) + _stub_module(monkeypatch, "routes") + _stub_module(monkeypatch, "routes.export", export_gguf = fake_export) + + tool = _get_tool("export_gguf") + result = asyncio.run( + tool.fn( + save_directory = "/tmp/out", + quantization_method = ["Q4_K_M", "Q8_0"], + push_to_hub = True, + repo_id = "me/model", + hf_token = "hf_secret", + imatrix = True, + imatrix_path = "/tmp/imatrix.dat", + ) + ) + + assert captured["hf_token"] == "hf_secret" + assert captured["imatrix"] is True + assert captured["imatrix_path"] == "/tmp/imatrix.dat" + assert captured["quantization_method"] == ["Q4_K_M", "Q8_0"] + assert result["current_subject"] == "mcp" + + +def test_load_checkpoint_forwards_token_and_fingerprint(monkeypatch): + captured = {} + + class FakeLoadCheckpointRequest: + def __init__(self, **kwargs): + captured.update(kwargs) + + async def fake_load(request, current_subject): + return {"current_subject": current_subject} + + _stub_module(monkeypatch, "models", LoadCheckpointRequest = FakeLoadCheckpointRequest) + _stub_module(monkeypatch, "routes") + _stub_module(monkeypatch, "routes.export", load_checkpoint = fake_load) + + tool = _get_tool("load_checkpoint") + asyncio.run( + tool.fn( + checkpoint_path = "/tmp/ckpt", + approved_remote_code_fingerprint = "sha256:abc", + hf_token = "hf_secret", + ) + ) + + assert captured["hf_token"] == "hf_secret" + assert captured["approved_remote_code_fingerprint"] == "sha256:abc" + + +def test_list_training_runs_clamps_pagination(monkeypatch): + captured = {} + + async def fake_list_runs(limit, offset, current_subject): + captured["limit"] = limit + captured["offset"] = offset + return {"ok": True} + + _stub_module(monkeypatch, "routes") + _stub_module(monkeypatch, "routes.training_history", list_training_runs = fake_list_runs) + + tool = _get_tool("list_training_runs") + asyncio.run(tool.fn(limit = 10_000, offset = -5)) + + assert captured["limit"] == 200 + assert captured["offset"] == 0 + + +def test_get_recipe_job_dataset_clamps_pagination(monkeypatch): + captured = {} + + def fake_job_dataset(job_id, limit, offset): + captured["limit"] = limit + captured["offset"] = offset + return {"ok": True} + + _stub_module(monkeypatch, "routes") + _stub_module(monkeypatch, "routes.data_recipe") + _stub_module(monkeypatch, "routes.data_recipe.jobs", job_dataset = fake_job_dataset) + + tool = _get_tool("get_recipe_job_dataset") # this tool is synchronous + tool.fn(job_id = "job-1", limit = -1, offset = -9) + + assert captured["limit"] == 1 + assert captured["offset"] == 0 diff --git a/studio/backend/tests/test_mcp_servers.py b/studio/backend/tests/test_mcp_servers.py index 6432ffb8e1..c5c37f098f 100644 --- a/studio/backend/tests/test_mcp_servers.py +++ b/studio/backend/tests/test_mcp_servers.py @@ -577,7 +577,7 @@ def test_clear_oauth_tokens_swallows_constructor_errors(tmp_path, monkeypatch): def test_tool_xml_parser_handles_hyphenated_function_names(): """Hyphenated tool names like `mcp__srv__list-issues` must parse, else the - model can call the tool but Studio can't dispatch.""" + model can call the tool but Unsloth can't dispatch.""" from core.inference.tool_call_parser import parse_tool_calls_from_text calls = parse_tool_calls_from_text( diff --git a/studio/backend/tests/test_mcp_stdio_improvements.py b/studio/backend/tests/test_mcp_stdio_improvements.py index b0bfd45135..745c2cc447 100644 --- a/studio/backend/tests/test_mcp_stdio_improvements.py +++ b/studio/backend/tests/test_mcp_stdio_improvements.py @@ -188,7 +188,7 @@ def test_validate_url_allows_url_in_argument(monkeypatch): # ── P6: Data Recipe stdio path obeys the same host gate ───────────── -# build_mcp_providers needs the Studio-only data_designer plugin; skip if absent. +# build_mcp_providers needs the Unsloth-only data_designer plugin; skip if absent. _STDIO_RECIPE = { "mcp_providers": [ diff --git a/studio/backend/tests/test_middleware.py b/studio/backend/tests/test_middleware.py index 11aeee6d77..36061b5375 100644 --- a/studio/backend/tests/test_middleware.py +++ b/studio/backend/tests/test_middleware.py @@ -14,6 +14,7 @@ import pytest from fastapi import FastAPI, HTTPException, Request from fastapi.responses import Response from fastapi.testclient import TestClient +from starlette.middleware.gzip import GZipMiddleware _BACKEND_ROOT = Path(__file__).resolve().parents[1] @@ -33,6 +34,7 @@ def main_module(): def _make_protected_app( max_bytes: int, main_module, + request_max_bytes_getter = None, upload_passthrough_prefixes: tuple = (), upload_passthrough_max_bytes_getter = None, ): @@ -40,7 +42,13 @@ def _make_protected_app( app.add_middleware( main_module.MaxBodyMiddleware, max_bytes_getter = lambda: max_bytes, - protected_prefixes = ("/v1/chat/completions", "/api/settings", "/api/train"), + protected_prefixes = ( + "/v1/chat/completions", + "/api/inference", + "/api/settings", + "/api/train", + ), + request_max_bytes_getter = request_max_bytes_getter, upload_passthrough_prefixes = upload_passthrough_prefixes, upload_passthrough_max_bytes_getter = upload_passthrough_max_bytes_getter, ) @@ -67,6 +75,10 @@ def _make_protected_app( total += len(chunk) return {"ok": True, "chunks": chunks, "total": total} + @app.post("/api/inference/audio/transcribe/raw") + async def transcribe_raw(request: Request): + return {"ok": True, "total": len(await request.body())} + @app.get("/api/train/status") async def status_get(): return {"ok": True, "get": True} @@ -96,6 +108,43 @@ class TestMaxBodyMiddleware: assert r.status_code == 200 assert r.json()["unprotected"] is True + def test_route_specific_cap_overrides_default(self, main_module): + app = _make_protected_app( + 4096, + main_module, + request_max_bytes_getter = lambda path: ( + 128 if path.endswith("/transcribe/raw") else 4096 + ), + ) + c = TestClient(app) + + rejected = c.post( + "/api/inference/audio/transcribe/raw", + content = b"x" * 129, + ) + accepted = c.post( + "/api/inference/audio/transcribe/raw", + content = b"x" * 128, + ) + + assert rejected.status_code == 413 + assert accepted.status_code == 200 + assert accepted.json()["total"] == 128 + + def test_stt_routes_use_audio_specific_caps(self, main_module): + from utils.upload_limits import ( + STT_AUDIO_JSON_MAX_BYTES, + STT_AUDIO_RAW_MAX_BYTES, + ) + assert ( + main_module._get_request_body_max_bytes("/api/inference/audio/transcribe/raw") + == STT_AUDIO_RAW_MAX_BYTES + ) + assert ( + main_module._get_request_body_max_bytes("/api/inference/audio/transcribe") + == STT_AUDIO_JSON_MAX_BYTES + ) + def test_settings_put_body_over_cap_rejected(self, main_module): app = _make_protected_app(1024, main_module) c = TestClient(app) @@ -471,6 +520,71 @@ class TestSecurityHeadersMiddleware: assert b"server" in names +class TestFrontendAssets: + def test_hashed_assets_are_compressed_and_cached(self, tmp_path, main_module): + content = b"export const value = 'responsive';\n" * 200 + (tmp_path / "page-abc123.js").write_bytes(content) + app = FastAPI() + assets_app = GZipMiddleware( + main_module.ImmutableStaticFiles(directory = tmp_path), + minimum_size = 1024, + compresslevel = 6, + ) + app.mount("/assets", assets_app, name = "assets") + + response = TestClient(app).get( + "/assets/page-abc123.js", + headers = {"Accept-Encoding": "gzip"}, + ) + + assert response.status_code == 200 + assert response.content == content + assert response.headers["content-encoding"] == "gzip" + assert response.headers["cache-control"] == (main_module._IMMUTABLE_ASSET_CACHE_CONTROL) + assert "accept-encoding" in response.headers["vary"].lower() + + def test_asset_revalidation_keeps_immutable_cache_header(self, tmp_path, main_module): + (tmp_path / "page-abc123.js").write_text("export {};", encoding = "utf-8") + app = FastAPI() + app.mount( + "/assets", + main_module.ImmutableStaticFiles(directory = tmp_path), + name = "assets", + ) + client = TestClient(app) + first = client.get("/assets/page-abc123.js") + + response = client.get( + "/assets/page-abc123.js", + headers = {"If-None-Match": first.headers["etag"]}, + ) + + assert response.status_code == 304 + assert response.headers["cache-control"] == (main_module._IMMUTABLE_ASSET_CACHE_CONTROL) + + def test_range_request_is_not_compressed(self, tmp_path, main_module): + content = b"export const value = 'responsive';\n" * 200 + (tmp_path / "page-abc123.js").write_bytes(content) + app = FastAPI() + assets_app = main_module._AssetGZipMiddleware( + main_module.ImmutableStaticFiles(directory = tmp_path), + minimum_size = 1024, + compresslevel = 6, + ) + app.mount("/assets", assets_app, name = "assets") + + response = TestClient(app).get( + "/assets/page-abc123.js", + headers = {"Accept-Encoding": "gzip", "Range": "bytes=0-99"}, + ) + + assert response.status_code == 206 + assert response.headers.get("content-encoding") != "gzip" + assert response.headers["content-range"] == f"bytes 0-99/{len(content)}" + assert response.content == content[:100] + assert response.headers["cache-control"] == (main_module._IMMUTABLE_ASSET_CACHE_CONTROL) + + # /api/health auth gate diff --git a/studio/backend/tests/test_mlx_inference_backend.py b/studio/backend/tests/test_mlx_inference_backend.py index 29fbb45158..d49a2281a0 100644 --- a/studio/backend/tests/test_mlx_inference_backend.py +++ b/studio/backend/tests/test_mlx_inference_backend.py @@ -2,6 +2,7 @@ import sys import types +from contextlib import contextmanager from types import SimpleNamespace import pytest @@ -40,12 +41,16 @@ class _DummyModel: def _install_fake_mlx(monkeypatch): mlx_pkg = types.ModuleType("mlx") mlx_core = types.ModuleType("mlx.core") + mlx_utils = types.ModuleType("mlx.utils") mlx_core.metal = _DummyMetal() mlx_core.set_wired_limit = _DummyMX.set_wired_limit mlx_core.device_info = _DummyMX.device_info + mlx_utils.tree_unflatten = dict mlx_pkg.core = mlx_core + mlx_pkg.utils = mlx_utils monkeypatch.setitem(sys.modules, "mlx", mlx_pkg) monkeypatch.setitem(sys.modules, "mlx.core", mlx_core) + monkeypatch.setitem(sys.modules, "mlx.utils", mlx_utils) def _install_fake_fast_mlx(monkeypatch, calls): @@ -68,6 +73,99 @@ def _install_fake_fast_mlx(monkeypatch, calls): monkeypatch.setitem(sys.modules, "unsloth_zoo.mlx.loader", mlx_loader) +class _AdapterTree: + def __init__(self, modules): + self.modules = dict(modules) + + def named_modules(self): + return list(self.modules.items()) + + def update_modules(self, modules): + self.modules.update(modules) + + +def test_temporary_mlx_adapter_state_bypasses_and_restores_wrappers(monkeypatch): + _install_fake_mlx(monkeypatch) + from core.inference.mlx_inference import _temporary_mlx_adapter_state + + base = object() + wrapper = SimpleNamespace(lora_a = object(), lora_b = object(), linear = base, m = object()) + model = _AdapterTree({"model.layers.0.proj": wrapper}) + + with pytest.raises(RuntimeError, match = "generation failed"): + with _temporary_mlx_adapter_state(model, False): + assert model.modules["model.layers.0.proj"] is base + raise RuntimeError("generation failed") + assert model.modules["model.layers.0.proj"] is wrapper + + +def test_temporary_mlx_adapter_state_validates_requests(): + from core.inference.mlx_inference import _temporary_mlx_adapter_state + + wrapper = SimpleNamespace(lora_a = object(), lora_b = object(), embedding = object()) + model = _AdapterTree({"embed_tokens": wrapper}) + with _temporary_mlx_adapter_state(model, True): + assert model.modules["embed_tokens"] is wrapper + with pytest.raises(NotImplementedError, match = "named adapter"): + with _temporary_mlx_adapter_state(model, "other"): + pass + + base_model = _AdapterTree({"proj": object()}) + with _temporary_mlx_adapter_state(base_model, None): + pass + with _temporary_mlx_adapter_state(base_model, True): + pass + + unsupported = _AdapterTree({"proj": SimpleNamespace(lora_a = object(), lora_b = object())}) + with _temporary_mlx_adapter_state(unsupported, True): + pass + with pytest.raises(RuntimeError, match = "without their base modules"): + with _temporary_mlx_adapter_state(unsupported, False): + pass + + +def test_temporary_mlx_adapter_state_uses_real_mlx_module_tree(): + nn = pytest.importorskip("mlx.nn") + pytest.importorskip("mlx_lm") + from mlx_lm.models.switch_layers import SwitchLinear + from mlx_lm.tuner.dora import DoRALinear + from mlx_lm.tuner.lora import LoRAEmbedding, LoRALinear, LoRASwitchLinear + + from core.inference.mlx_inference import _temporary_mlx_adapter_state + + class _Layer(nn.Module): + def __init__(self): + super().__init__() + quantized = nn.QuantizedLinear.from_linear(nn.Linear(32, 32), group_size = 32, bits = 4) + self.quantized_proj = LoRALinear.from_base(quantized) + self.dora_proj = DoRALinear.from_base(nn.Linear(4, 4)) + + class _Model(nn.Module): + def __init__(self): + super().__init__() + self.layers = [_Layer()] + self.embed_tokens = LoRAEmbedding.from_base(nn.Embedding(16, 4)) + self.experts = LoRASwitchLinear.from_base(SwitchLinear(4, 4, 2)) + + model = _Model() + wrappers = { + path: module + for path, module in model.named_modules() + if hasattr(module, "lora_a") and hasattr(module, "lora_b") + } + bases = { + path: getattr(module, "linear", getattr(module, "embedding", None)) + for path, module in wrappers.items() + } + + with _temporary_mlx_adapter_state(model, False): + live = dict(model.named_modules()) + assert all(live[path] is base for path, base in bases.items()) + + restored = dict(model.named_modules()) + assert all(restored[path] is wrapper for path, wrapper in wrappers.items()) + + def test_mlx_inference_text_load_forwards_studio_settings(monkeypatch): _install_fake_mlx(monkeypatch) calls = [] @@ -138,7 +236,7 @@ def test_mlx_inference_vlm_lora_uses_unsloth_loader_without_native_adapter_rewri _install_fake_fast_mlx(monkeypatch, calls) def _native_vlm_load(*_args, **_kwargs): - raise AssertionError("Studio MLX VLM inference must use FastMLXModel") + raise AssertionError("Unsloth MLX VLM inference must use FastMLXModel") mlx_vlm = types.ModuleType("mlx_vlm") mlx_vlm.load = _native_vlm_load @@ -333,10 +431,87 @@ def test_mlx_generate_chat_response_accepts_template_kwargs(): ), f"{name!r} must default to None so existing callers stay valid" +def test_mlx_vlm_reemits_think_prefill_inside_adapter_context(monkeypatch): + """A prefilled block must be re-emitted as the first VLM snapshot, + inside the adapter context (so unsupported requests still raise first), so + the UI renders the thinking block during prefill and a pre-first-token + cancel does not drop it. Mirrors _generate_text.""" + from core.inference import mlx_inference + + MLXInferenceBackend = mlx_inference.MLXInferenceBackend + + order = [] + + @contextmanager + def _adapter_state(_model, state): + assert backend._generation_lock.locked() + order.append("adapter_enter") + try: + yield + finally: + order.append("adapter_exit") + + monkeypatch.setattr(mlx_inference, "_temporary_mlx_adapter_state", _adapter_state) + monkeypatch.setattr( + "core.inference.chat_template_helpers.detect_think_prefill", + lambda *_a, **_k: "\n", + ) + + prompt_utils = SimpleNamespace( + MODEL_CONFIG = {"deepseek_vl_v2": object()}, + apply_chat_template = lambda *_a, **_k: " model-aware", + ) + mlx_vlm = types.ModuleType("mlx_vlm") + mlx_vlm.prompt_utils = prompt_utils + + def _vlm_stream(*_a, **_k): + # The prefill must have been emitted before any generated token. + assert order[-1] == "adapter_enter" + yield SimpleNamespace(text = "ok", prompt_tokens = 3, generation_tokens = 1) + + mlx_vlm.stream_generate = _vlm_stream + monkeypatch.setitem(sys.modules, "mlx_vlm", mlx_vlm) + monkeypatch.setattr( + "core.inference.chat_template_helpers.apply_chat_template_for_generation", + lambda _t, _m, **_k: " model-aware", + ) + + backend = MLXInferenceBackend() + backend._model = SimpleNamespace(config = {"model_type": "deepseek_vl_v2"}) + backend._processor = SimpleNamespace(tokenizer = SimpleNamespace()) + args = ([{"role": "user", "content": [{"type": "image"}]}], object(), 0, 1, 0, 0, 1, 1, None) + + gen = backend._generate_vlm(*args, _adapter_state = False) + # First snapshot is the prefill alone, emitted after entering the adapter context. + assert next(gen) == "\n" + assert order == ["adapter_enter"] + # Subsequent snapshots are cumulative (prefill + generated text). + assert next(gen) == "\nok" + gen.close() + assert order == ["adapter_enter", "adapter_exit"] + + def test_mlx_vlm_generation_selects_renderer_by_capability(monkeypatch): - from core.inference.mlx_inference import MLXInferenceBackend + from core.inference import mlx_inference + + MLXInferenceBackend = mlx_inference.MLXInferenceBackend calls = {"generic": [], "model": [], "stream": []} + adapter_events = [] + adapter_active = {"value": False} + + @contextmanager + def _adapter_state(_model, state): + assert backend._generation_lock.locked() + adapter_events.append(("enter", state)) + adapter_active["value"] = True + try: + yield + finally: + adapter_active["value"] = False + adapter_events.append(("exit", state)) + + monkeypatch.setattr(mlx_inference, "_temporary_mlx_adapter_state", _adapter_state) state = {"generic": "serialized", "model": " model-aware"} prompt_utils = SimpleNamespace( MODEL_CONFIG = {"deepseek_vl_v2": object()}, @@ -346,10 +521,13 @@ def test_mlx_vlm_generation_selects_renderer_by_capability(monkeypatch): ) mlx_vlm = types.ModuleType("mlx_vlm") mlx_vlm.prompt_utils = prompt_utils - mlx_vlm.stream_generate = lambda *_args, **kwargs: ( - calls["stream"].append((_args, kwargs)) - or iter([SimpleNamespace(text = "ok", prompt_tokens = 3, generation_tokens = 1)]) - ) + + def _vlm_stream(*args, **kwargs): + assert adapter_active["value"] + calls["stream"].append((args, kwargs)) + yield SimpleNamespace(text = "ok", prompt_tokens = 3, generation_tokens = 1) + + mlx_vlm.stream_generate = _vlm_stream monkeypatch.setitem(sys.modules, "mlx_vlm", mlx_vlm) def generic(_target, _messages, **kwargs): @@ -369,7 +547,11 @@ def test_mlx_vlm_generation_selects_renderer_by_capability(monkeypatch): backend._processor = SimpleNamespace(tokenizer = SimpleNamespace()) args = ([{"role": "user", "content": [{"type": "image"}]}], object(), 0, 1, 0, 0, 1, 1, None) tools = [{"function": {"name": "search"}}] - assert list(backend._generate_vlm(*args)) == ["ok"] + generator = backend._generate_vlm(*args, _adapter_state = False) + assert next(generator) == "ok" + assert adapter_active["value"] and backend._generation_lock.locked() + generator.close() + assert adapter_events == [("enter", False), ("exit", False)] assert calls["model"][0]["num_images"] == 1 assert calls["stream"][0][0][2] == " model-aware" with pytest.raises(RuntimeError, match = "dropping requested tools"): @@ -449,7 +631,10 @@ def test_mlx_generate_text_forwards_kwargs_into_template_helper(monkeypatch): """Mac text path must route through apply_chat_template_for_generation so reasoning / tool kwargs reach the tokenizer.""" _install_fake_mlx(monkeypatch) - from core.inference.mlx_inference import MLXInferenceBackend + from core.inference import mlx_inference + + MLXInferenceBackend = mlx_inference.MLXInferenceBackend + real_adapter_state = mlx_inference._temporary_mlx_adapter_state # The text path renders once with tools, then the native-template fallback makes a second no- # tools probe call (tools=None) to detect whether the template dropped the schema. @@ -474,11 +659,31 @@ def test_mlx_generate_text_forwards_kwargs_into_template_helper(monkeypatch): mlx_lm_sample.make_sampler = lambda **_kw: object() mlx_lm_sample.make_logits_processors = lambda **_kw: None + adapter_events = [] + adapter_active = {"value": False} + stream_state = {"fail": False} + + @contextmanager + def _adapter_state(_model, state): + assert backend._generation_lock.locked() + adapter_events.append(("enter", state)) + adapter_active["value"] = True + try: + yield + finally: + adapter_active["value"] = False + adapter_events.append(("exit", state)) + + monkeypatch.setattr(mlx_inference, "_temporary_mlx_adapter_state", _adapter_state) + class _Resp: def __init__(self, tok): self.token = tok def _stream_generate(_model, _tokenizer, **_kw): + assert adapter_active["value"] + if stream_state["fail"]: + raise RuntimeError("generation failed") yield _Resp(1) mlx_lm_pkg.stream_generate = _stream_generate @@ -500,17 +705,45 @@ def test_mlx_generate_text_forwards_kwargs_into_template_helper(monkeypatch): backend._tokenizer = _Tok() backend._is_vlm = False - out = list( - backend.generate_chat_response( - messages = [{"role": "user", "content": "ping"}], - tools = [{"function": {"name": "web_search"}}], - enable_thinking = True, - reasoning_effort = "medium", - preserve_thinking = True, - max_new_tokens = 1, - ) + generator = backend.generate_with_adapter_control( + use_adapter = False, + messages = [{"role": "user", "content": "ping"}], + tools = [{"function": {"name": "web_search"}}], + enable_thinking = True, + reasoning_effort = "medium", + preserve_thinking = True, + max_new_tokens = 1, ) - assert out == ["hi"] + assert next(generator) == "hi" + assert adapter_active["value"] and backend._generation_lock.locked() + generator.close() + assert adapter_events == [("enter", False), ("exit", False)] + stream_state["fail"] = True + with pytest.raises(RuntimeError, match = "generation failed"): + list( + backend.generate_with_adapter_control( + use_adapter = False, + messages = [{"role": "user", "content": "ping"}], + max_new_tokens = 1, + ) + ) + assert adapter_events[-2:] == [("enter", False), ("exit", False)] + assert not backend._generation_lock.locked() + + monkeypatch.setattr(mlx_inference, "_temporary_mlx_adapter_state", real_adapter_state) + monkeypatch.setattr( + "core.inference.chat_template_helpers.detect_think_prefill", + lambda *_args, **_kwargs: "", + ) + stream_state["fail"] = False + named = backend.generate_with_adapter_control( + use_adapter = "named", + messages = [{"role": "user", "content": "ping"}], + max_new_tokens = 1, + ) + with pytest.raises(NotImplementedError, match = "named adapter"): + next(named) + assert not adapter_active["value"] and not backend._generation_lock.locked() # The toggled kwargs must reach the chat-template helper on the real render # (one of the calls carries the tools; the fallback probe passes tools=None). tool_renders = [ @@ -523,3 +756,579 @@ def test_mlx_generate_text_forwards_kwargs_into_template_helper(monkeypatch): assert render["kwargs"]["enable_thinking"] is True assert render["kwargs"]["reasoning_effort"] == "medium" assert render["kwargs"]["preserve_thinking"] is True + + +def test_mlx_text_normalizes_native_reasoning_and_close_releases_lock(monkeypatch): + _install_fake_mlx(monkeypatch) + from core.inference.mlx_inference import MLXInferenceBackend + + monkeypatch.setattr( + "core.inference.chat_template_helpers.apply_chat_template_for_generation", + lambda *_args, **_kwargs: "prompt", + raising = True, + ) + monkeypatch.setattr( + "core.inference.chat_template_helpers.render_with_native_template_fallback", + lambda formatted_prompt, **_kwargs: SimpleNamespace( + prompt = formatted_prompt, + reasoning_channel_markers = ("<|channel>thought\n", ""), + ), + raising = True, + ) + + mlx_lm_pkg = types.ModuleType("mlx_lm") + mlx_lm_sample = types.ModuleType("mlx_lm.sample_utils") + mlx_lm_sample.make_sampler = lambda **_kw: object() + mlx_lm_sample.make_logits_processors = lambda **_kw: None + + class _Resp: + def __init__(self, text, tok): + self.text = text + self.token = tok + + def _stream_generate(_model, _tokenizer, **_kw): + yield _Resp("<|channel>thought\n", 10) + yield _Resp("r", 11) + yield _Resp("", 12) + yield _Resp("a", 13) + + mlx_lm_pkg.stream_generate = _stream_generate + monkeypatch.setitem(sys.modules, "mlx_lm", mlx_lm_pkg) + monkeypatch.setitem(sys.modules, "mlx_lm.sample_utils", mlx_lm_sample) + + backend = MLXInferenceBackend() + backend._model = object() + backend._tokenizer = SimpleNamespace(all_special_tokens = []) + backend._is_vlm = False + + assert list( + backend.generate_chat_response( + messages = [{"role": "user", "content": "ping"}], + max_new_tokens = 4, + ) + ) == ["", "r", "r", "ra"] + + gen = backend.generate_chat_response( + messages = [{"role": "user", "content": "ping"}], + max_new_tokens = 4, + ) + assert next(gen) == "" + assert backend._generation_lock.locked() + gen.close() + assert not backend._generation_lock.locked() + + +def test_mlx_text_native_metadata_preserves_prefilled_think_snapshots(monkeypatch): + _install_fake_mlx(monkeypatch) + from core.inference.mlx_inference import MLXInferenceBackend + + monkeypatch.setattr( + "core.inference.chat_template_helpers.apply_chat_template_for_generation", + lambda *_args, **_kwargs: "prompt\n", + raising = True, + ) + monkeypatch.setattr( + "core.inference.chat_template_helpers.render_with_native_template_fallback", + lambda formatted_prompt, **_kwargs: SimpleNamespace( + prompt = formatted_prompt, + reasoning_channel_markers = ("<|channel>thought", ""), + ), + raising = True, + ) + + mlx_lm_pkg = types.ModuleType("mlx_lm") + mlx_lm_sample = types.ModuleType("mlx_lm.sample_utils") + mlx_lm_sample.make_sampler = lambda **_kw: object() + mlx_lm_sample.make_logits_processors = lambda **_kw: None + + class _Resp: + def __init__(self, text, tok): + self.text = text + self.token = tok + + def _stream_generate(_model, _tokenizer, **_kw): + yield _Resp("reason", 10) + yield _Resp("", 11) + yield _Resp("answer", 12) + + mlx_lm_pkg.stream_generate = _stream_generate + monkeypatch.setitem(sys.modules, "mlx_lm", mlx_lm_pkg) + monkeypatch.setitem(sys.modules, "mlx_lm.sample_utils", mlx_lm_sample) + + backend = MLXInferenceBackend() + backend._model = object() + backend._tokenizer = SimpleNamespace(all_special_tokens = []) + backend._is_vlm = False + + snapshots = list( + backend.generate_chat_response( + messages = [{"role": "user", "content": "ping"}], + max_new_tokens = 3, + ) + ) + assert snapshots == [ + "\n", + "\nreason", + "\nreason", + "\nreasonanswer", + ] + assert all(current.startswith(previous) for previous, current in zip(snapshots, snapshots[1:])) + + +def test_mlx_vlm_normalizes_native_reasoning_channels(monkeypatch): + _install_fake_mlx(monkeypatch) + from core.inference.mlx_inference import MLXInferenceBackend + + monkeypatch.setattr( + "core.inference.chat_template_helpers.apply_chat_template_for_generation", + lambda *_args, **_kwargs: "prompt", + raising = True, + ) + + mlx_vlm_pkg = types.ModuleType("mlx_vlm") + + class _Resp: + def __init__(self, text, tok): + self.text = text + self.token = tok + + def _stream_generate(_model, _processor, _prompt, _images, **_kw): + yield _Resp("<|channel>thought\n", 10) + yield _Resp("vision", 11) + yield _Resp("", 12) + yield _Resp(" answer", 13) + + mlx_vlm_pkg.stream_generate = _stream_generate + monkeypatch.setitem(sys.modules, "mlx_vlm", mlx_vlm_pkg) + + backend = MLXInferenceBackend() + backend._model = SimpleNamespace(config = SimpleNamespace()) + backend._processor = SimpleNamespace( + chat_template = "<|channel>thought\n...", + all_special_tokens = [], + apply_chat_template = lambda *_args, **_kwargs: "prompt", + ) + backend._is_vlm = True + + assert list( + backend.generate_chat_response( + messages = [{"role": "user", "content": "describe"}], + image = object(), + max_new_tokens = 4, + ) + ) == [ + "", + "vision", + "vision", + "vision answer", + ] + + +class _FakeLRUPromptCache: + def __init__( + self, + max_size = 10, + max_bytes = 1 << 63, + ): + self.max_size = max_size + self.max_bytes = max_bytes + self.entries = {} + + def fetch_nearest_cache(self, key, tokens): + import copy + + stored = self.entries.get(key, {}) + exact = stored.get(tuple(tokens)) + if exact is not None: + return copy.deepcopy(exact), [] + best = None + for candidate, cache in stored.items(): + if len(candidate) < len(tokens) and tuple(tokens[: len(candidate)]) == candidate: + if best is None or len(candidate) > len(best[0]): + best = (candidate, cache) + if best is not None: + return copy.deepcopy(best[1]), list(tokens[len(best[0]) :]) + return None, list(tokens) + + def insert_cache( + self, + key, + tokens, + prompt_cache, + *, + cache_type = "assistant", + ): + import copy + self.entries.setdefault(key, {})[tuple(tokens)] = copy.deepcopy(prompt_cache) + + +class _FakeCacheEntry: + def __init__( + self, + offset = 0, + nbytes = 1, + ): + self.offset = offset + self.nbytes = nbytes + + +def _install_fake_prompt_cache_api(monkeypatch, trimmable = True): + from core.inference import mlx_inference + + def _make_prompt_cache(_model): + return [_FakeCacheEntry()] + + def _can_trim_prompt_cache(_cache): + return trimmable + + def _trim_prompt_cache(cache, num): + cache[0].offset = max(cache[0].offset - num, 0) + return num + + monkeypatch.setattr( + mlx_inference, + "_mlx_prompt_cache_api", + lambda: ( + _FakeLRUPromptCache, + _make_prompt_cache, + _can_trim_prompt_cache, + _trim_prompt_cache, + ), + ) + + +def test_mlx_prompt_cache_max_bytes_budget(monkeypatch): + from core.inference.mlx_inference import ( + PROMPT_CACHE_FALLBACK_BYTES, + PROMPT_CACHE_MEMORY_FRACTION, + _prompt_cache_max_bytes, + ) + + monkeypatch.delenv("UNSLOTH_MLX_PROMPT_CACHE_BYTES", raising = False) + assert _prompt_cache_max_bytes(None) == PROMPT_CACHE_FALLBACK_BYTES + assert _prompt_cache_max_bytes(20.0) == int(20.0 * 1e9 * PROMPT_CACHE_MEMORY_FRACTION) + + monkeypatch.setenv("UNSLOTH_MLX_PROMPT_CACHE_BYTES", "4096") + assert _prompt_cache_max_bytes(20.0) == 4096 + monkeypatch.setenv("UNSLOTH_MLX_PROMPT_CACHE_BYTES", "0") + assert _prompt_cache_max_bytes(20.0) == 0 + monkeypatch.setenv("UNSLOTH_MLX_PROMPT_CACHE_BYTES", "not-a-number") + assert _prompt_cache_max_bytes(20.0) == int(20.0 * 1e9 * PROMPT_CACHE_MEMORY_FRACTION) + + +def test_mlx_prompt_cache_never_returns_empty_remainder(monkeypatch): + _install_fake_prompt_cache_api(monkeypatch) + from core.inference.mlx_inference import _MLXPromptCacheHistory + + history = _MLXPromptCacheHistory(6, 1 << 30) + tokens = list(range(10)) + cache, rest = history.fetch(object(), "key", tokens) + assert len(rest) == 10 + cache[0].offset = len(tokens) + history.insert("key", tokens, cache) + + _cache, rest = history.fetch(object(), "key", tokens) + assert rest == tokens[-1:] + + longer = tokens + [99, 100] + _cache, rest = history.fetch(object(), "key", longer) + assert rest == [99, 100] + + _install_fake_prompt_cache_api(monkeypatch, trimmable = False) + history = _MLXPromptCacheHistory(6, 1 << 30) + cache, _rest = history.fetch(object(), "key", tokens) + cache[0].offset = len(tokens) + history.insert("key", tokens, cache) + _cache, rest = history.fetch(object(), "key", tokens) + assert rest == tokens, "untrimmable entry must not be reused" + + +def test_mlx_prompt_cache_key_isolates_adapter_state(monkeypatch): + _install_fake_prompt_cache_api(monkeypatch) + _install_fake_mlx(monkeypatch) + from core.inference.mlx_inference import MLXInferenceBackend + + class _Tok: + bos_token = None + + def encode( + self, + text, + add_special_tokens = True, + ): + return [ord(c) for c in text] + + backend = MLXInferenceBackend() + backend._model = object() + backend._tokenizer = _Tok() + backend.active_model_name = "model-a" + + prompt = "shared prefix" + _rest, cache, key, tokens, cached = backend._prepare_prompt_cache(prompt, True) + assert cached == 0 + cache[0].offset = len(tokens) + backend._prompt_cache_history.insert(key, tokens, cache) + + _rest, _cache, _key, _tokens, cached_same = backend._prepare_prompt_cache(prompt, True) + assert cached_same > 0 + _rest, _cache, _key, _tokens, cached_flipped = backend._prepare_prompt_cache(prompt, False) + assert cached_flipped == 0 + + +def _install_fake_text_stack( + monkeypatch, + token_map, + captured, + markers = None, +): + import types as _types + + from core.inference import mlx_inference + + _install_fake_mlx(monkeypatch) + monkeypatch.setattr( + mlx_inference, + "_temporary_mlx_adapter_state", + lambda _model, _state: __import__("contextlib").nullcontext(), + ) + monkeypatch.setattr( + "core.inference.chat_template_helpers.apply_chat_template_for_generation", + lambda _tok, messages, **_kw: messages[-1]["content"], + ) + monkeypatch.setattr( + "core.inference.chat_template_helpers.render_with_native_template_fallback", + lambda formatted_prompt, **_kw: SimpleNamespace( + prompt = formatted_prompt, + reasoning_channel_markers = markers, + ), + ) + monkeypatch.setattr( + "core.inference.chat_template_helpers.detect_think_prefill", + lambda *_a, **_kw: "", + ) + + class _Resp: + def __init__(self, token, processed): + self.token = token + self.text = f"<{token}>" + self.prompt_tokens = processed + self.prompt_tps = 10.0 + self.generation_tokens = 1 + self.generation_tps = 5.0 + + def _stream_generate(_model, _tokenizer, **kwargs): + captured.append(kwargs) + processed = len(kwargs["prompt"]) + cache = kwargs.get("prompt_cache") + if cache is not None: + cache[0].offset += processed + for token in token_map["generated"]: + if cache is not None: + cache[0].offset += 1 + yield _Resp(token, processed) + + mlx_lm_pkg = _types.ModuleType("mlx_lm") + mlx_lm_pkg.stream_generate = _stream_generate + mlx_lm_sample = _types.ModuleType("mlx_lm.sample_utils") + mlx_lm_sample.make_sampler = lambda **_kw: object() + mlx_lm_sample.make_logits_processors = lambda **_kw: [] + monkeypatch.setitem(sys.modules, "mlx_lm", mlx_lm_pkg) + monkeypatch.setitem(sys.modules, "mlx_lm.sample_utils", mlx_lm_sample) + + class _Tok: + bos_token = None + chat_template = "x" + + def encode( + self, + text, + add_special_tokens = True, + ): + return list(token_map[text]) + + def decode( + self, + ids, + skip_special_tokens = False, + ): + return "".join(str(i) for i in ids) + + from core.inference.mlx_inference import MLXInferenceBackend + + backend = MLXInferenceBackend() + backend._model = object() + backend._tokenizer = _Tok() + backend._is_vlm = False + backend.active_model_name = "model-a" + return backend + + +def _run_turn(backend, prompt): + list( + backend.generate_chat_response( + messages = [{"role": "user", "content": prompt}], + max_new_tokens = 4, + ) + ) + + +def test_mlx_text_reuses_prompt_cache_on_the_next_turn(monkeypatch): + _install_fake_prompt_cache_api(monkeypatch) + captured = [] + token_map = { + "P1": [1, 2, 3], + "P2": [1, 2, 3, 7, 8, 9, 10], + "generated": [7, 8], + } + backend = _install_fake_text_stack(monkeypatch, token_map, captured) + + _run_turn(backend, "P1") + assert captured[0]["prompt"] == [1, 2, 3] + assert "prompt_cache" in captured[0] + assert backend.last_generation_stats["timings"]["cache_n"] == 0 + + _run_turn(backend, "P2") + assert captured[1]["prompt"] == [9, 10], "turn two should prefill only the new tail" + + stats = backend.last_generation_stats + assert stats["timings"]["cache_n"] == 5 + assert stats["timings"]["prompt_n"] == 2 + assert stats["usage"]["prompt_tokens"] == 7 + + +def test_mlx_text_without_lru_prompt_cache_prefills_the_full_prompt(monkeypatch): + from core.inference import mlx_inference + + monkeypatch.setattr(mlx_inference, "_mlx_prompt_cache_api", lambda: None) + captured = [] + token_map = {"P1": [1, 2, 3], "generated": [7]} + backend = _install_fake_text_stack(monkeypatch, token_map, captured) + + _run_turn(backend, "P1") + assert captured[0]["prompt"] == "P1" + assert "prompt_cache" not in captured[0] + assert backend.last_generation_stats["timings"]["cache_n"] == 0 + + +def test_mlx_text_tracks_tokens_on_the_native_reasoning_path(monkeypatch): + _install_fake_prompt_cache_api(monkeypatch) + captured = [] + token_map = {"P1": [1, 2, 3], "P2": [1, 2, 3, 7, 8, 9], "generated": [7, 8]} + backend = _install_fake_text_stack(monkeypatch, token_map, captured, markers = ("", "")) + + _run_turn(backend, "P1") + _run_turn(backend, "P2") + assert captured[1]["prompt"] == [9] + + +def test_mlx_presence_penalty_latches_the_first_decode_step(): + mx = pytest.importorskip("mlx.core") + import numpy as np + + from core.inference.mlx_inference import _make_mlx_presence_penalty_processor + + processor = _make_mlx_presence_penalty_processor(2.0) + logits = mx.zeros((1, 5)) + out = processor(mx.array([3]), logits) + assert np.array_equal(np.array(out), np.zeros((1, 5))), "prompt must not be penalized" + out = processor(mx.array([3, 1]), mx.zeros((1, 5))) + penalized = np.array(out)[0] + assert penalized[1] == -2.0 + assert penalized[3] == 0.0 + + +def test_mlx_prompt_cache_survives_reset_but_not_unload(monkeypatch): + _install_fake_prompt_cache_api(monkeypatch) + _install_fake_mlx(monkeypatch) + sys.modules["mlx.core"].clear_cache = lambda: None + from core.inference.mlx_inference import MLXInferenceBackend + + backend = MLXInferenceBackend() + backend.active_model_name = "model-a" + history = backend._prompt_cache() + assert history is not None + + backend.reset_generation_state() + assert backend._prompt_cache_history is history + + backend.unload_model("model-a") + assert backend._prompt_cache_history is None + + +def test_mlx_prompt_cache_skips_entries_over_budget(monkeypatch): + _install_fake_prompt_cache_api(monkeypatch) + from core.inference.mlx_inference import _MLXPromptCacheHistory + + history = _MLXPromptCacheHistory(6, 1000) + history.insert("key", [1, 2, 3], [_FakeCacheEntry(offset = 3, nbytes = 400)]) + assert len(history._lru.entries.get("key", {})) == 1 + + history.insert("key", list(range(50)), [_FakeCacheEntry(offset = 50, nbytes = 5000)]) + stored = history._lru.entries.get("key", {}) + assert tuple([1, 2, 3]) in stored + assert tuple(range(50)) not in stored + + +def test_mlx_prompt_cache_keys_on_what_the_kv_covers(monkeypatch): + _install_fake_prompt_cache_api(monkeypatch) + from core.inference.mlx_inference import _MLXPromptCacheHistory + + class _Entry: + def __init__( + self, + offset, + nbytes = 1, + ): + self.offset = offset + self.nbytes = nbytes + + history = _MLXPromptCacheHistory(6, 1 << 30) + + history.insert("key", list(range(10)), [_Entry(offset = 8)]) + assert tuple(range(8)) in history._lru.entries["key"] + assert tuple(range(10)) not in history._lru.entries["key"] + + history.insert("other", list(range(4)), [_Entry(offset = 9)]) + assert "other" not in history._lru.entries + + +def test_mlx_prompt_cache_only_stores_verifiable_prefix_coverage(monkeypatch): + mx = pytest.importorskip("mlx.core") + from mlx_lm.models.cache import CacheList, ChunkedKVCache, KVCache, RotatingKVCache + + _install_fake_prompt_cache_api(monkeypatch) + from core.inference.mlx_inference import _kv_prefix_coverage, _MLXPromptCacheHistory + + def feed(entry, n): + for _ in range(n): + block = mx.zeros((1, 2, 1, 4), dtype = mx.float16) + entry.update_and_fetch(block, block) + mx.eval(entry.state) + return entry + + plain = feed(KVCache(), 30) + unwrapped = feed(RotatingKVCache(max_size = 100, keep = 2), 30) + wrapped = feed(RotatingKVCache(max_size = 10, keep = 2), 30) + chunked = feed(ChunkedKVCache(chunk_size = 8), 30) + slid = feed(ChunkedKVCache(chunk_size = 8), 30) + slid.maybe_trim_front() + + assert _kv_prefix_coverage([plain]) == 30 + assert _kv_prefix_coverage([unwrapped]) == 30 + assert _kv_prefix_coverage([chunked]) == 30 + assert wrapped.offset == 30 and wrapped.state[0].shape[2] == 10 + assert _kv_prefix_coverage([wrapped]) is None + assert slid.start_position > 0 + assert _kv_prefix_coverage([slid]) is None + assert _kv_prefix_coverage([CacheList(feed(KVCache(), 30), feed(KVCache(), 30))]) == 30 + assert _kv_prefix_coverage([CacheList(feed(KVCache(), 30), wrapped)]) is None + assert _kv_prefix_coverage([feed(KVCache(), 30), feed(KVCache(), 29)]) is None + assert _kv_prefix_coverage([]) is None + + history = _MLXPromptCacheHistory(6, 1 << 40) + for unsafe in (wrapped, slid): + history.insert("key", list(range(30)), [unsafe]) + assert "key" not in history._lru.entries + + history.insert("key", list(range(30)), [plain]) + assert tuple(range(30)) in history._lru.entries["key"] diff --git a/studio/backend/tests/test_mlx_repair.py b/studio/backend/tests/test_mlx_repair.py index 365cc46410..47a695ccbd 100644 --- a/studio/backend/tests/test_mlx_repair.py +++ b/studio/backend/tests/test_mlx_repair.py @@ -103,7 +103,7 @@ def test_repair_install_pins_transformers_and_cleans_up(monkeypatch): assert mr.attempt_mlx_repair() is True cmd = captured["cmd"] # transformers is pinned via a constraint file so the mlx install cannot - # upgrade it underneath Studio, and the temp constraint file is cleaned up. + # upgrade it underneath Unsloth, and the temp constraint file is cleaned up. assert "--constraint" in cmd assert "--upgrade" in cmd reinstall_pairs = set(zip(cmd, cmd[1:])) @@ -123,7 +123,7 @@ def test_install_requires_prebuilt_wheels(monkeypatch): # A source distribution's PEP 517 build backend runs arbitrary code at install # time, before the post-install stack check. The unattended self-heal must # require pre-built wheels so a malicious resolver-selected sdist cannot execute - # during ordinary Studio startup. mlx/mlx-metal ship wheels only and + # during ordinary Unsloth startup. mlx/mlx-metal ship wheels only and # mlx-lm/mlx-vlm publish py3-none-any wheels, so a healthy self-heal still works. pytest.importorskip("transformers") captured = {} @@ -143,7 +143,7 @@ def test_install_requires_prebuilt_wheels(monkeypatch): def test_install_env_drops_secrets_and_source_redirects(monkeypatch): - # The unattended self-heal must not hand resolver/build code the full Studio + # The unattended self-heal must not hand resolver/build code the full Unsloth # environment: secrets and package-source redirects are dropped, while the # variables uv genuinely needs are forwarded. monkeypatch.setenv("HF_TOKEN", "secret-hf") diff --git a/studio/backend/tests/test_mlx_stop_checkpoint.py b/studio/backend/tests/test_mlx_stop_checkpoint.py new file mode 100644 index 0000000000..d4a00cc6c8 --- /dev/null +++ b/studio/backend/tests/test_mlx_stop_checkpoint.py @@ -0,0 +1,137 @@ +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 + +"""Regression tests for MLX stop-and-save checkpoint handling.""" + +import importlib.util +import json +import sys +import types +from pathlib import Path + +import numpy as np +from safetensors.numpy import save_file + + +_BACKEND = Path(__file__).resolve().parents[1] + + +def _load_worker_module(): + spec = importlib.util.spec_from_file_location( + "training_worker_under_test", + _BACKEND / "core" / "training" / "worker.py", + ) + module = importlib.util.module_from_spec(spec) + assert spec.loader is not None + spec.loader.exec_module(module) + return module + + +worker = _load_worker_module() + + +class _FakeTrainer: + def __init__(self, step: int): + self._global_step = step + self._train_loss_history = [] + self.model = object() + + +def _write_checkpoint(out: Path, step: int) -> Path: + checkpoint = out / f"checkpoint-{step}" + checkpoint.mkdir(parents = True, exist_ok = True) + (checkpoint / "trainer_state.json").write_text( + json.dumps({"global_step": step}), encoding = "utf-8" + ) + save_file({"weight": np.ones(1, dtype = np.float32)}, checkpoint / "adapters.safetensors") + save_file( + {"state": np.ones(1, dtype = np.float32)}, + checkpoint / "optimizer_state.safetensors", + ) + return checkpoint + + +def test_mlx_has_checkpoint_at_step_requires_complete_state(tmp_path): + out = tmp_path / "outputs" / "run_x" + _write_checkpoint(out, 5) + + assert worker._mlx_has_checkpoint_at_step(out, 5) is True + + +def test_write_mlx_stop_checkpoint_returns_true_when_current_step_checkpoint_exists(tmp_path): + out = tmp_path / "outputs" / "run_x" + _write_checkpoint(out, 5) + + assert worker._write_mlx_stop_checkpoint(_FakeTrainer(step = 5), object(), out) is True + + +def test_write_mlx_stop_checkpoint_writes_current_step_when_only_older_checkpoint_exists( + tmp_path, monkeypatch +): + out = tmp_path / "outputs" / "run_x" + _write_checkpoint(out, 5) + + saved_steps: list[int] = [] + + def _save_state(_value, path, name): + save_file({"state": np.ones(1, dtype = np.float32)}, Path(path, name)) + + def _save_trainer_state(state, ckpt_dir, **_kwargs): + Path(ckpt_dir, "trainer_state.json").write_text(json.dumps(state), encoding = "utf-8") + saved_steps.append(int(state["global_step"])) + + fake_utils = types.SimpleNamespace( + save_trainable_adapters = lambda model, path: _save_state( + model, path, "adapters.safetensors" + ), + save_optimizer_state = lambda optimizer, path: _save_state( + optimizer, path, "optimizer_state.safetensors" + ), + save_trainer_state = _save_trainer_state, + ) + monkeypatch.setitem(sys.modules, "unsloth_zoo.mlx.utils", fake_utils) + + assert worker._write_mlx_stop_checkpoint(_FakeTrainer(step = 10), object(), out) is True + assert saved_steps == [10] + assert (out / "checkpoint-10" / "trainer_state.json").is_file() + + +def test_write_mlx_stop_checkpoint_returns_false_without_optimizer(tmp_path): + out = tmp_path / "outputs" / "run_x" + out.mkdir(parents = True) + + assert worker._write_mlx_stop_checkpoint(_FakeTrainer(step = 5), None, out) is False + + +def test_write_mlx_stop_checkpoint_rejects_incomplete_current_checkpoint(tmp_path): + out = tmp_path / "outputs" / "run_x" + ckpt = out / "checkpoint-5" + ckpt.mkdir(parents = True) + (ckpt / "trainer_state.json").write_text('{"global_step": 5}', encoding = "utf-8") + + assert worker._write_mlx_stop_checkpoint(_FakeTrainer(step = 5), None, out) is False + + +def test_write_mlx_stop_checkpoint_ignores_stale_checkpoint_without_optimizer(tmp_path): + # An older checkpoint does not cover the current step, so this still fails. + out = tmp_path / "outputs" / "run_x" + _write_checkpoint(out, 5) + + assert worker._write_mlx_stop_checkpoint(_FakeTrainer(step = 10), None, out) is False + + +def test_write_mlx_stop_checkpoint_returns_false_when_save_fails(tmp_path, monkeypatch): + out = tmp_path / "outputs" / "run_x" + out.mkdir(parents = True) + + def _boom(*_args, **_kwargs): + raise RuntimeError("save failed") + + fake_utils = types.SimpleNamespace( + save_trainable_adapters = _boom, + save_optimizer_state = lambda *_a, **_k: None, + save_trainer_state = lambda *_a, **_k: None, + ) + monkeypatch.setitem(sys.modules, "unsloth_zoo.mlx.utils", fake_utils) + + assert worker._write_mlx_stop_checkpoint(_FakeTrainer(step = 5), object(), out) is False diff --git a/studio/backend/tests/test_model_picker_regression.py b/studio/backend/tests/test_model_picker_regression.py new file mode 100644 index 0000000000..f38a4d0b8d --- /dev/null +++ b/studio/backend/tests/test_model_picker_regression.py @@ -0,0 +1,232 @@ +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 + +"""Regression guards for the model-picker per-model-config feature (the set of +bugs that got the predecessor PR reverted). Pure-function / validation checks +only, so they run on CPU in the backend pytest job with no model download. + +Covers, at the backend layer: + - infra-model hiding: the RAG embedder (bge-small-en-v1.5) and the llama.cpp + install-validation probe (ggml-org/models / stories260K) stay hidden, while + normal chat repos are not hidden; + - the HF token is honored from the dedicated header with the query string as a + fallback, never the other way around; + - the chat-template byte caps reject oversized overrides (both the char-count + fast path and the UTF-8 byte path) and the sidecar reader is size-bounded. +""" + +from __future__ import annotations + +import sys +import types + +import pytest + +# Keep this test runnable without the optional structlog dependency (mirrors +# tests/test_cached_gguf_routes.py), since importing routes.models pulls it in. +if "structlog" not in sys.modules: + + class _DummyLogger: + def __getattr__(self, _name): + return lambda *args, **kwargs: None + + sys.modules["structlog"] = types.SimpleNamespace( + BoundLogger = _DummyLogger, + get_logger = lambda *args, **kwargs: _DummyLogger(), + ) + +import routes.models as models_route +from core.rag import config as rag_config +from hub.dependencies import get_hf_token +from models.inference import LoadRequest +from picker.schemas import MAX_CHAT_TEMPLATE_BYTES +from picker.service import _read_bounded_text +from utils.hidden_models import is_hidden_model + + +@pytest.fixture(autouse = True) +def _pin_default_embedder(monkeypatch): + """Pin the effective embedder to Studio's static default so hiding is + deterministic and cannot depend on ambient RAG config / env.""" + default = "unsloth/bge-small-en-v1.5" + monkeypatch.setattr(rag_config, "EMBEDDING_MODEL", default, raising = False) + monkeypatch.setattr(rag_config, "effective_embedding_model", lambda: default) + monkeypatch.setattr(rag_config, "effective_gguf_repo", lambda: default) + monkeypatch.setattr(rag_config, "default_gguf_repo", lambda: default) + + +# --------------------------------------------------------------------------- # +# Infra-model hiding (the "infra models resurfaced in the picker" regression) # +# --------------------------------------------------------------------------- # + + +@pytest.mark.parametrize( + "value", + [ + "ggml-org/models", # the probe repo id + "unsloth/bge-small-en-v1.5", # the RAG embedder repo + "unsloth/bge-small-en-v1.5-GGUF", # its GGUF companion + "/root/.cache/huggingface/hub/x/stories260K.gguf", # probe on disk + "/root/.cache/x/Stories260K.GGUF", # case-insensitive + r"C:\\models\\stories260K.gguf", # windows-style path + "/opt/models/bge-small-en-v1.5", # embedder basename folder + "/opt/models/bge-small-en-v1.5-Q8_0.gguf", # suffixed local weight + ], +) +def test_infra_models_are_hidden(value): + assert is_hidden_model(value) is True + + +@pytest.mark.parametrize( + "value", + [ + "unsloth/gemma-3-270m-it-GGUF", # a normal small chat GGUF + "unsloth/Qwen3-0.6B", # a normal non-GGUF chat model + "user/stories260K-finetune-GGUF", # repo id merely contains "stories260k" + "user/model-chat", # generic repo must not be hidden + "meta-llama/Llama-3.1-8B-Instruct", + ], +) +def test_normal_models_are_not_hidden(value): + assert is_hidden_model(value) is False + + +def test_is_hidden_model_ignores_empty_values(): + assert is_hidden_model(None) is False + assert is_hidden_model("") is False + assert is_hidden_model(None, "", "unsloth/gemma-3-270m-it-GGUF") is False + + +def test_hidden_model_matchers_expose_probe_needles(): + needles, exact_ids, _exact_paths = models_route.hidden_model_matchers() + lowered = [n.lower() for n in needles] + assert "ggml-org/models" in lowered + assert "stories260k.gguf" in lowered + # The configured embedder is exposed as an exact repo id, never as a + # basename needle that would substring-hide unrelated chat models. + assert "bge-small-en-v1.5" not in lowered + assert "unsloth/bge-small-en-v1.5" in exact_ids + + +def test_hidden_model_matchers_custom_repo_publishes_exact_ids(monkeypatch): + monkeypatch.setattr(rag_config, "effective_embedding_model", lambda: "org/model") + monkeypatch.setattr(rag_config, "effective_gguf_repo", lambda: "org/model-GGUF") + needles, exact_ids, exact_paths = models_route.hidden_model_matchers() + assert needles == ["ggml-org/models", "stories260k.gguf"] + assert "org/model" in exact_ids + assert "org/model-gguf" in exact_ids + assert exact_paths == [] + + +def test_hidden_model_matchers_local_owner_name_path_is_exact_path(monkeypatch, tmp_path): + # A local embedder shaped like owner/name that exists on disk must be an + # exact resolved path, not a Hub repo id (mirroring is_hidden_model), so the + # local row stays hidden instead of showing as a chat model. + (tmp_path / "models" / "embedder").mkdir(parents = True) + monkeypatch.chdir(tmp_path) + monkeypatch.setattr(rag_config, "effective_embedding_model", lambda: "models/embedder") + monkeypatch.setattr(rag_config, "effective_gguf_repo", lambda: "ggml-org/models") + _needles, exact_ids, exact_paths = models_route.hidden_model_matchers() + resolved = str((tmp_path / "models" / "embedder").resolve()).lower() + assert resolved in exact_paths + assert "models/embedder" not in exact_ids + + +# --------------------------------------------------------------------------- # +# HF token via header, query string only as a fallback (the token-leak fix) # +# --------------------------------------------------------------------------- # + + +def test_get_hf_token_strips_and_returns(): + assert get_hf_token(" hf_abc ") == "hf_abc" + + +@pytest.mark.parametrize("value", [None, "", " ", "\n\t"]) +def test_get_hf_token_blank_is_none(value): + assert get_hf_token(value) is None + + +@pytest.mark.parametrize( + "value,expected", + [(" hf_x ", "hf_x"), ("", None), (" ", None), (None, None), (1234, None)], +) +def test_normalize_hf_token(value, expected): + assert models_route._normalize_hf_token(value) == expected + + +def test_header_token_wins_over_query(): + header, query = "hf_header", "hf_query" + resolved = models_route._normalize_hf_token(header) or models_route._normalize_hf_token(query) + assert resolved == "hf_header" + + +def test_query_token_is_fallback_when_header_absent(): + resolved = models_route._normalize_hf_token(None) or models_route._normalize_hf_token( + "hf_query" + ) + assert resolved == "hf_query" + + +# --------------------------------------------------------------------------- # +# Chat-template byte caps (the unbounded-template hardening) # +# --------------------------------------------------------------------------- # + + +def _load_request(**overrides): + data = {"model_path": "unsloth/test-model-GGUF", "gguf_variant": "Q4_K_M"} + data.update(overrides) + return LoadRequest.model_validate(data) + + +def test_blank_chat_template_override_normalizes_to_none(): + assert _load_request(chat_template_override = " \n\t").chat_template_override is None + + +def test_nonblank_chat_template_override_preserved_verbatim(): + template = " {{ messages }} " + assert _load_request(chat_template_override = template).chat_template_override == template + + +def test_chat_template_at_byte_limit_is_accepted(): + template = "a" * MAX_CHAT_TEMPLATE_BYTES # exactly the limit, 1 byte/char + assert ( + len(_load_request(chat_template_override = template).chat_template_override) + == MAX_CHAT_TEMPLATE_BYTES + ) + + +def test_chat_template_over_char_limit_is_rejected(): + with pytest.raises(Exception): # pydantic ValidationError wrapping ValueError + _load_request(chat_template_override = "a" * (MAX_CHAT_TEMPLATE_BYTES + 1)) + + +def test_chat_template_over_byte_limit_is_rejected(): + # Char count stays under the limit but UTF-8 bytes exceed it (3 bytes/char), + # so only the byte-count branch can catch this. + multibyte = "€" * (MAX_CHAT_TEMPLATE_BYTES // 2) # euro sign, 3 bytes each + assert len(multibyte) <= MAX_CHAT_TEMPLATE_BYTES + assert len(multibyte.encode("utf-8")) > MAX_CHAT_TEMPLATE_BYTES + with pytest.raises(Exception): + _load_request(chat_template_override = multibyte) + + +def test_read_bounded_text_reads_within_limit(tmp_path): + p = tmp_path / "t.json" + p.write_text("hello", encoding = "utf-8") + assert _read_bounded_text(p, 16) == "hello" + + +def test_read_bounded_text_rejects_over_limit(tmp_path): + p = tmp_path / "big.json" + p.write_bytes(b"x" * 100) + assert _read_bounded_text(p, 50) is None + + +def test_read_bounded_text_at_limit_is_read(tmp_path): + p = tmp_path / "exact.json" + p.write_bytes(b"x" * 50) + assert _read_bounded_text(p, 50) == "x" * 50 + + +def test_read_bounded_text_missing_file_is_none(tmp_path): + assert _read_bounded_text(tmp_path / "nope.json", 50) is None diff --git a/studio/backend/tests/test_model_update_robustness.py b/studio/backend/tests/test_model_update_robustness.py index edf55812e2..d84f8c94a7 100644 --- a/studio/backend/tests/test_model_update_robustness.py +++ b/studio/backend/tests/test_model_update_robustness.py @@ -112,13 +112,21 @@ def patch_hub_gguf(monkeypatch): blob_ids = [local_blob], gguf_files = {"model-Q4_K_M.gguf": 1000}, ) + monkeypatch.setattr( + "utils.hf_cache_settings.get_hf_cache_paths", + lambda: SimpleNamespace(hub_cache = tmp_path), + ) monkeypatch.setattr( GV, "list_gguf_variants", lambda r, hf_token = None: (_variants(), False, [remote_sibling]), raising = True, ) - monkeypatch.setattr(GV, "iter_hf_cache_snapshots", lambda _repo_id: [snap]) + monkeypatch.setattr( + GV, + "iter_hf_cache_snapshots", + lambda _repo_id, root = None: [snap], + ) monkeypatch.setattr( CI, "all_hf_cache_scans", @@ -217,6 +225,10 @@ def test_variant_update_check_detects_companion_only_update( companion_path: 100, }, ) + monkeypatch.setattr( + "utils.hf_cache_settings.get_hf_cache_paths", + lambda: SimpleNamespace(hub_cache = tmp_path), + ) siblings = [ patch_hub_gguf.sibling("model-Q4_K_M.gguf", 1000, "mainsha"), patch_hub_gguf.sibling(companion_path, 100, "new-companion"), @@ -227,7 +239,11 @@ def test_variant_update_check_detects_companion_only_update( lambda r, hf_token = None: (_variants(), has_vision, siblings), raising = True, ) - monkeypatch.setattr(GV, "iter_hf_cache_snapshots", lambda _repo_id: [snap]) + monkeypatch.setattr( + GV, + "iter_hf_cache_snapshots", + lambda _repo_id, root = None: [snap], + ) monkeypatch.setattr( CI, "all_hf_cache_scans", @@ -314,6 +330,7 @@ def test_cached_model_scan_keeps_local_safetensors_repo(monkeypatch, tmp_path): file_name = "model.safetensors", size_on_disk = 100, blob_path = str(repo_path / "blobs" / "modelsha"), + blob_last_modified = 3_000.0, ), ] ) @@ -336,6 +353,98 @@ def test_cached_model_scan_keeps_local_safetensors_repo(monkeypatch, tmp_path): assert rows[0]["repo_id"] == "Org/SafeTensorRepo" assert rows[0]["model_format"] == "safetensors" assert rows[0]["size_bytes"] == 100 + assert rows[0]["last_modified"] == 3_000.0 + + +def test_cached_gguf_scan_keeps_download_timestamp(monkeypatch, tmp_path): + repo_path = tmp_path / "models--Org--GgufRepo" + repo = SimpleNamespace( + repo_id = "Org/GgufRepo", + repo_type = "model", + repo_path = repo_path, + revisions = [ + SimpleNamespace( + files = [ + SimpleNamespace( + file_name = "model-Q4_K_M.gguf", + size_on_disk = 100, + blob_path = None, + blob_last_modified = 5_000.0, + ), + ] + ) + ], + ) + monkeypatch.setattr( + CI, + "all_hf_cache_scans", + lambda: [SimpleNamespace(repos = [repo])], + ) + monkeypatch.setattr( + CI.hf_cache_scan, + "is_gguf_repo_partial", + lambda *args, **kwargs: False, + ) + monkeypatch.setattr( + CI, + "_gguf_variant_state_summary", + lambda _repo_id, **_kwargs: (False, 0), + ) + + rows = CI._scan_cached_gguf() + + assert len(rows) == 1 + assert rows[0]["repo_id"] == "Org/GgufRepo" + assert rows[0]["model_format"] == "gguf" + assert rows[0]["size_bytes"] == 100 + assert rows[0]["last_modified"] == 5_000.0 + + +def test_cached_model_scan_hides_custom_whisper_repo(monkeypatch, tmp_path): + repo_path = tmp_path / "models--Org--CustomWhisper" + snapshot = repo_path / "snapshots" / ("a" * 40) + snapshot.mkdir(parents = True) + (snapshot / "config.json").write_text( + '{"model_type": "whisper", "architectures": ["WhisperForConditionalGeneration"]}' + ) + repo = SimpleNamespace( + repo_id = "Org/CustomWhisper", + repo_type = "model", + repo_path = repo_path, + revisions = [ + SimpleNamespace( + files = [ + SimpleNamespace( + file_name = "config.json", + size_on_disk = 10, + blob_path = None, + ), + SimpleNamespace( + file_name = "model.safetensors", + size_on_disk = 100, + blob_path = str(repo_path / "blobs" / "modelsha"), + ), + ] + ) + ], + ) + monkeypatch.setattr( + CI, + "all_hf_cache_scans", + lambda: [SimpleNamespace(repos = [repo])], + ) + monkeypatch.setattr( + CI, + "_cached_model_snapshot_path", + lambda _repo_path: snapshot, + ) + monkeypatch.setattr( + CI.hf_cache_scan, + "is_snapshot_partial", + lambda *args, **kwargs: False, + ) + + assert CI._scan_cached_models() == [] # ── hf_hub_download_with_xet_fallback force_download bypass (X2/F2) ─── @@ -584,7 +693,12 @@ def test_reclaim_replaced_gguf_variant_prunes_old_revision_only(monkeypatch, tmp invalidated = [] monkeypatch.setattr(CI, "invalidate_hf_cache_scans", lambda: invalidated.append(True)) - result = D.reclaim_replaced_gguf_variant(repo_id, "Q4_K_M", frozenset({"NEWsha"})) + result = D.reclaim_replaced_gguf_variant( + repo_id, + "Q4_K_M", + frozenset({"NEWsha"}), + hub_cache = tmp_path, + ) assert result["removed_snapshots"] == 1 assert result["deleted_blobs"] == 1 @@ -631,8 +745,85 @@ def test_reclaim_replaced_gguf_variant_keeps_no_symlink_current_file(monkeypatch monkeypatch.setattr(CI, "all_hf_cache_scans", lambda: [SimpleNamespace(repos = [repo_info])]) monkeypatch.setattr(CI, "invalidate_hf_cache_scans", lambda: None) - result = D.reclaim_replaced_gguf_variant(repo_id, "Q4_K_M", frozenset({"REMOTEsha256"})) + result = D.reclaim_replaced_gguf_variant( + repo_id, + "Q4_K_M", + frozenset({"REMOTEsha256"}), + hub_cache = tmp_path, + ) assert snap.exists() is True # the current file must survive assert result["removed_snapshots"] == 0 assert result["deleted_blobs"] == 0 + + +def test_reclaim_replaced_gguf_variant_only_mutates_worker_cache(monkeypatch, tmp_path): + repo_id = "org/repo-GGUF" + cache_a = tmp_path / "cache-a" + cache_b = tmp_path / "cache-b" + + def cached_repo(cache_dir, revision): + repo_path = cache_dir / "models--org--repo-GGUF" + snap = repo_path / "snapshots" / revision / "model-Q4_K_M.gguf" + blob = repo_path / "blobs" / "OLDsha" + snap.parent.mkdir(parents = True, exist_ok = True) + blob.parent.mkdir(parents = True, exist_ok = True) + blob.write_bytes(b"old") + snap.symlink_to(blob) + return ( + SimpleNamespace( + repo_id = repo_id, + repo_type = "model", + repo_path = repo_path, + revisions = [ + SimpleNamespace( + files = [ + SimpleNamespace( + file_name = snap.name, + file_path = str(snap), + blob_path = str(blob), + ) + ] + ) + ], + ), + snap, + blob, + ) + + repo_a, snap_a, blob_a = cached_repo(cache_a, "a" * 40) + repo_b, snap_b, blob_b = cached_repo(cache_b, "b" * 40) + monkeypatch.setattr( + CI, + "all_hf_cache_scans", + lambda: [SimpleNamespace(repos = [repo_a]), SimpleNamespace(repos = [repo_b])], + ) + monkeypatch.setattr(CI, "invalidate_hf_cache_scans", lambda: None) + + result = D.reclaim_replaced_gguf_variant( + repo_id, + "Q4_K_M", + frozenset({"NEWsha"}), + hub_cache = cache_b, + ) + + assert result["removed_snapshots"] == 1 + assert snap_b.exists() is False + assert blob_b.exists() is False + assert snap_a.exists() is True + assert blob_a.exists() is True + + +def _mmproj_repo(*file_names: str): + return SimpleNamespace( + revisions = [SimpleNamespace(files = [SimpleNamespace(file_name = n) for n in file_names])] + ) + + +def test_repo_has_mmproj_requires_gguf_projector(): + # A non-GGUF sidecar whose name merely contains "mmproj" must NOT mark the + # repo vision-capable; the runtime's projector detection is GGUF-only. + assert CI._repo_has_mmproj(_mmproj_repo("model-Q4_K_M.gguf", "mmproj_config.json")) is False + assert CI._repo_has_mmproj(_mmproj_repo("model-Q4_K_M.gguf", "README-mmproj.md")) is False + # A real GGUF projector still marks the repo vision-capable. + assert CI._repo_has_mmproj(_mmproj_repo("model-Q4_K_M.gguf", "mmproj-F16.gguf")) is True diff --git a/studio/backend/tests/test_models_get_model_config_case_resolution.py b/studio/backend/tests/test_models_get_model_config_case_resolution.py index 12f6c497ab..a50765898b 100644 --- a/studio/backend/tests/test_models_get_model_config_case_resolution.py +++ b/studio/backend/tests/test_models_get_model_config_case_resolution.py @@ -84,7 +84,7 @@ def test_repo_in_any_hf_cache_matches_case_variant_in_legacy_cache(tmp_path, mon # covers the active cache; discard deletes case-insensitively, so detection must too, # else a decline deletes a pre-existing user repo). import utils.paths as paths_pkg - import huggingface_hub.constants as hf_constants + import hub.utils.paths as hub_paths active = tmp_path / "active" legacy = tmp_path / "legacy" @@ -96,9 +96,12 @@ def test_repo_in_any_hf_cache_matches_case_variant_in_legacy_cache(tmp_path, mon # No active-cache variant; case resolution is a no-op here. monkeypatch.setattr(paths_pkg, "resolve_cached_repo_id_case", lambda name: name) - monkeypatch.setattr(paths_pkg, "legacy_hf_cache_dir", lambda: legacy) - monkeypatch.setattr(paths_pkg, "hf_default_cache_dir", lambda: default) - monkeypatch.setattr(hf_constants, "HF_HUB_CACHE", str(active)) + monkeypatch.setattr(hub_paths, "legacy_hf_cache_dir", lambda: legacy) + monkeypatch.setattr(hub_paths, "hf_default_cache_dir", lambda: default) + monkeypatch.setattr( + "utils.hf_cache_settings.known_hf_hub_caches", + lambda: [active], + ) assert models_route._repo_in_any_hf_cache("unsloth/foo") is True # Absent from every cache -> reported absent. diff --git a/studio/backend/tests/test_mtp_drafter_companion.py b/studio/backend/tests/test_mtp_drafter_companion.py index 86e528ae67..02230632b6 100644 --- a/studio/backend/tests/test_mtp_drafter_companion.py +++ b/studio/backend/tests/test_mtp_drafter_companion.py @@ -328,6 +328,7 @@ def test_download_mtp_prefers_root_over_new_scheme_copies(monkeypatch): pick, label, cancel_event = None, + near_path = None, ): captured["pick"] = pick return None @@ -428,6 +429,32 @@ def test_download_mtp_reuse_follows_snapshot_order_offline(tmp_path, monkeypatch assert got is not None and Path(got).parent.parent.name == "newest" +def test_download_mtp_prefers_main_snapshot_offline(tmp_path, monkeypatch): + import utils.models.model_config as mc + from core.inference.llama_cpp import LlamaCppBackend + + monkeypatch.setenv("HF_HUB_OFFLINE", "1") + snapshots = tmp_path / "models--unsloth--gemma" / "snapshots" + old = snapshots / "old" + new = snapshots / "new" + old.mkdir(parents = True) + new.mkdir(parents = True) + main = old / "gemma-UD-Q4_K_XL.gguf" + old_drafter = old / "mtp-gemma.gguf" + new_drafter = new / "mtp-gemma.gguf" + main.write_bytes(b"main") + old_drafter.write_bytes(b"old") + new_drafter.write_bytes(b"new") + monkeypatch.setattr(mc, "_iter_hf_cache_snapshots", lambda _repo: [new, old]) + + got = LlamaCppBackend()._download_mtp( + hf_repo = "unsloth/gemma-GGUF", + near_path = str(main), + ) + + assert got == str(old_drafter) + + def test_download_mtp_online_skips_cache_reuse(tmp_path, monkeypatch): # Online, do not reuse a cached copy: go to the download path so a changed # drafter is refetched (hf_hub_download checks the current revision). @@ -447,6 +474,7 @@ def test_download_mtp_online_skips_cache_reuse(tmp_path, monkeypatch): pick, label, cancel_event = None, + near_path = None, ): reached["hit"] = True return None diff --git a/studio/backend/tests/test_mtp_vram_budget.py b/studio/backend/tests/test_mtp_vram_budget.py index 0efbbf596d..694d60cfc6 100644 --- a/studio/backend/tests/test_mtp_vram_budget.py +++ b/studio/backend/tests/test_mtp_vram_budget.py @@ -502,7 +502,7 @@ class TestExtraArgsMtpDetection: assert _extra_args_mtp_draft_path([], env = dict(os.environ)) == "/large.gguf" def test_load_model_gates_env_spec_type_on_off_mode(self): - # LLAMA_ARG_SPEC_TYPE only reaches the child when Studio emits no spec + # LLAMA_ARG_SPEC_TYPE only reaches the child when Unsloth emits no spec # flag (UI mode "off", no user --spec-type); otherwise the emitted # --spec-type/--spec-default overrides the env, so the reserve must not # consult it or a stale MTP env over-reserves (Finding F3). Whitespace- @@ -530,8 +530,8 @@ class TestExtraArgsMtpDetection: def test_load_model_drafter_budget_precedence(self): # The budget sizes the drafter the launch actually loads: CLI extras win, - # then Studio's emitted mtp_draft_path (overrides LLAMA_ARG_SPEC_DRAFT_MODEL), - # then the env drafter -- not the env before Studio's (reviewer.py R3). + # then Unsloth's emitted mtp_draft_path (overrides LLAMA_ARG_SPEC_DRAFT_MODEL), + # then the env drafter -- not the env before Unsloth's (reviewer.py R3). compact = "".join(inspect.getsource(LlamaCppBackend.load_model).split()) assert "_cli_draft_for_budget=_extra_args_mtp_draft_path(extra_args,env={})" in compact assert "_env_draft_for_budget=_extra_args_mtp_draft_path([],env=os.environ)" in compact @@ -732,7 +732,7 @@ class TestExtraArgsMtpDetection: assert _extra_args_n_ubatch([], env = {"LLAMA_ARG_UBATCH": "notint"}) is None def test_env_main_cache_type_for_budget(self): - # The child inherits LLAMA_ARG_CACHE_TYPE_K/_V, but Studio emits no + # The child inherits LLAMA_ARG_CACHE_TYPE_K/_V, but Unsloth emits no # --cache-type when neither param nor extras set it -> a heavier env # main KV (f32) must be adopted so the reserve matches the child. assert _env_main_cache_type_for_budget(env = {}) is None @@ -765,7 +765,7 @@ class TestExtraArgsMtpDetection: assert "cache_type_kv=_env_main_cache_type_for_budget()" in compact def test_env_split_mode_is_tensor(self): - # The child inherits LLAMA_ARG_SPLIT_MODE, but Studio emits --split-mode + # The child inherits LLAMA_ARG_SPLIT_MODE, but Unsloth emits --split-mode # only on its tensor branch -> a tensor env must flip the budget so the # heavier per-device compute buffer is reserved (not layer overhead). assert _env_split_mode_is_tensor(env = {}) is False @@ -918,7 +918,7 @@ class TestExtraArgsMtpDetection: # Cluster A: when the final decision is layer split, an inherited # non-layer LLAMA_ARG_SPLIT_MODE (and paired LLAMA_ARG_TENSOR_SPLIT) must # be popped from the child env so the child cannot run tensor/row/none - # against Studio's layer budget. Whitespace-stripped for formatter. + # against Unsloth's layer budget. Whitespace-stripped for formatter. compact = "".join(inspect.getsource(LlamaCppBackend.load_model).split()) assert 'env.get("LLAMA_ARG_SPLIT_MODE")' in compact assert '_inherited_sm!="layer"' in compact @@ -936,10 +936,10 @@ class TestExtraArgsMtpDetection: assert "env.pop(_ct_var,None)" in compact def test_load_model_clears_tensor_split_env_in_tensor_mode(self): - # review run3 #2: Studio owns the tensor split. When it emits no + # review run3 #2: Unsloth owns the tensor split. When it emits no # --tensor-split (even split), a stale inherited LLAMA_ARG_TENSOR_SPLIT must # be cleared in the TENSOR branch too (not just the layer downgrade), or the - # child runs a split Studio didn't budget. The else (tensor) branch pops it. + # child runs a split Unsloth didn't budget. The else (tensor) branch pops it. src = inspect.getsource(LlamaCppBackend.load_model) compact = "".join(src.split()) # appears in both the layer branch and the tensor branch. @@ -1005,14 +1005,14 @@ def test_qwen36_class_regression_picks_lower_ctx_with_mtp(): def test_mtp_draft_budget_prefers_user_extras_drafter(): # A user --model-draft in extras is appended last and wins at launch, so the - # VRAM budget must size it first; then Studio's emitted mtp_draft_path (which + # VRAM budget must size it first; then Unsloth's emitted mtp_draft_path (which # overrides LLAMA_ARG_SPEC_DRAFT_MODEL), then the env drafter (load_model is too # entangled to drive end-to-end; assert the precedence at the source level). # Whitespace-stripped so the check survives any formatter line-wrapping. compact = "".join(inspect.getsource(LlamaCppBackend.load_model).split()) - # CLI extras sized first (env={} so the env doesn't pre-empt Studio's drafter). + # CLI extras sized first (env={} so the env doesn't pre-empt Unsloth's drafter). assert "_cli_draft_for_budget=_extra_args_mtp_draft_path(extra_args,env={})" in compact - # Order: CLI extras, then Studio's mtp_draft_path, then the env drafter. + # Order: CLI extras, then Unsloth's mtp_draft_path, then the env drafter. assert "_cli_draft_for_budgetor_studio_draft_for_budgetor_env_draft_for_budget" in compact - # The env must not be consulted before Studio's resolved drafter. + # The env must not be consulted before Unsloth's resolved drafter. assert "_extra_args_mtp_draft_path(extra_args)ormtp_draft_path" not in compact diff --git a/studio/backend/tests/test_multimodal_document.py b/studio/backend/tests/test_multimodal_document.py index 5cd7c876cc..b347c4aef8 100644 --- a/studio/backend/tests/test_multimodal_document.py +++ b/studio/backend/tests/test_multimodal_document.py @@ -3,7 +3,7 @@ """Tests for PDF / document attachment translation on external providers. -Studio adds a normalised `input_document` content part on +Unsloth adds a normalised `input_document` content part on ChatCompletionRequest so the frontend needn't know the per-provider attachment shape: diff --git a/studio/backend/tests/test_nudge_tool_calls_wiring.py b/studio/backend/tests/test_nudge_tool_calls_wiring.py index e03fd0c7d7..82a6543aeb 100644 --- a/studio/backend/tests/test_nudge_tool_calls_wiring.py +++ b/studio/backend/tests/test_nudge_tool_calls_wiring.py @@ -3,7 +3,7 @@ """Wiring guard for the plan-without-action ``nudge_tool_calls`` policy. -Decided policy: the re-prompt is ALWAYS ON for the Studio inference paths +Decided policy: the re-prompt is ALWAYS ON for the Unsloth inference paths (safetensors, GGUF/llama_cpp, MLX) and OPT-IN for the API (/v1 OpenAI-compat + Anthropic-compat, controlled by the request's ``nudge_tool_calls``, default off). @@ -16,7 +16,7 @@ Mechanism (verified here without loading a model): opt-in), while the GGUF loop keeps its pre-existing default-on behaviour (``None`` keeps nudging) so an omitted flag never disables GGUF; * the API request models default the flag to ``None`` (opt-in / off); - * the Studio-facing routes forward the request's flag, and the Studio frontend + * the Unsloth-facing routes forward the request's flag, and the Unsloth frontend sends ``nudge_tool_calls: true`` -- exercised behaviourally in ``test_safetensors_tool_loop.py`` and ``test_llama_cpp_tool_loop.py``. """ @@ -87,7 +87,7 @@ def test_api_request_models_default_the_flag_off(): def test_studio_routes_forward_the_request_flag(): - # The Studio chat frontend posts to /v1/chat/completions and /v1/messages + # The Unsloth chat frontend posts to /v1/chat/completions and /v1/messages # with nudge_tool_calls=true; the route handlers forward the request value # (external API clients that omit it fall back to the opt-in default). from routes import inference as routes_inference diff --git a/studio/backend/tests/test_offline_embedding_minimal.py b/studio/backend/tests/test_offline_embedding_minimal.py new file mode 100644 index 0000000000..ccc6b5f76a --- /dev/null +++ b/studio/backend/tests/test_offline_embedding_minimal.py @@ -0,0 +1,942 @@ +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 + +"""Offline RAG embedding-model handling (issue #6817). + +Offline the studio must never call the Hub (a DNS-dead session hangs on retries). Using a fake +HF cache under a temp HF_HUB_CACHE, assert that offline: is_embedding_model classifies from the +cached modules.json without the Hub; the file-security gate fails CLOSED on an unscanned pickle +weight with no safetensors alternative and allows an inert cache; the embedder threads +local_files_only into the load. Online behavior is unchanged (bounded timeout + cache fallback). +""" + +import sys +import types +from pathlib import Path +from types import SimpleNamespace +from unittest.mock import patch + +import pytest + +from utils.security import evaluate_file_security +from utils.utils import ( + hf_cache_snapshot_dir, + hf_cache_snapshot_is_loadable, + hf_env_offline, + st_repo_id_candidates, +) + +# Minimal sentence-transformers modules.json (the marker the gate keys on). +MODULES_JSON = ( + '[{"idx": 0, "name": "0", "path": "", "type": "sentence_transformers.models.Transformer"}]' +) + + +def _modules_json(*paths): + """modules.json listing one Transformer module per path (a load root).""" + import json + return json.dumps( + [ + { + "idx": i, + "name": str(i), + "path": p, + "type": "sentence_transformers.models.Transformer", + } + for i, p in enumerate(paths) + ] + ) + + +_COMMIT = "0123456789abcdef0123456789abcdef01234567" + + +def _fs_case_sensitive(root): + """Whether root's filesystem is case-sensitive (Linux yes; macOS/Windows usually no). The gate + mirrors the loader, whose file lookups follow the same rule, so some cases only exist on one.""" + probe = Path(root) / "_case_probe" + probe.write_text("x") + try: + return not (Path(root) / "_CASE_PROBE").exists() + finally: + probe.unlink() + + +def _requires_case_sensitive_fs(root): + if not _fs_case_sensitive(root): + pytest.skip("requires a case-sensitive filesystem") + + +def _requires_case_insensitive_fs(root): + if _fs_case_sensitive(root): + pytest.skip("requires a case-insensitive filesystem") + + +def _make_cache( + root, + repo_id, + files, + commit = _COMMIT, +): + """Build a canonical HF-cache snapshot (refs/main + snapshots//) for repo_id under + root from {relpath: contents}; returns the snapshot dir.""" + from huggingface_hub.file_download import repo_folder_name + + repo_dir = Path(root) / repo_folder_name(repo_id = repo_id, repo_type = "model") + (repo_dir / "refs").mkdir(parents = True, exist_ok = True) + (repo_dir / "refs" / "main").write_text(commit) + snapshot = repo_dir / "snapshots" / commit + snapshot.mkdir(parents = True, exist_ok = True) + for rel, contents in files.items(): + path = snapshot / rel + path.parent.mkdir(parents = True, exist_ok = True) + path.write_text(contents) + return snapshot + + +def _no_network(): + """Patch model_info to fail loudly if any offline path reaches the network.""" + return patch("huggingface_hub.model_info", side_effect = AssertionError("hit the network")) + + +def _is_embedding_model(*args, **kwargs): + from utils.models.model_config import is_embedding_model + return is_embedding_model(*args, **kwargs) + + +@pytest.fixture +def hf_cache(tmp_path, monkeypatch): + """Point the HF cache at a fresh temp dir. + + get_hf_cache_paths() reads an import-time env snapshot, not live os.environ, + so point it (and thus active_hf_hub_cache + the snapshot lookup's selected + root) at this temp cache too.""" + root = tmp_path / "hub" + root.mkdir() + monkeypatch.setenv("HF_HOME", str(tmp_path)) + monkeypatch.setenv("HF_HUB_CACHE", str(root)) + monkeypatch.setattr( + "utils.hf_cache_settings.get_hf_cache_paths", + lambda: SimpleNamespace(hub_cache = root), + ) + return root + + +@pytest.fixture(autouse = True) +def _clean_env(monkeypatch): + """Start each test online with an empty detection cache; offline tests opt in.""" + monkeypatch.delenv("HF_HUB_OFFLINE", raising = False) + monkeypatch.delenv("TRANSFORMERS_OFFLINE", raising = False) + from utils.models import model_config as mc + + mc._embedding_detection_cache.clear() + yield + mc._embedding_detection_cache.clear() + + +# ── hf_env_offline ─────────────────────────────────────────────── + + +@pytest.mark.parametrize("value", ["1", "true", "TRUE", "yes", "on", " On "]) +def test_hf_env_offline_true(monkeypatch, value): + monkeypatch.setenv("HF_HUB_OFFLINE", value) + assert hf_env_offline() is True + + +@pytest.mark.parametrize("value", ["0", "false", "no", "off", ""]) +def test_hf_env_offline_false(monkeypatch, value): + monkeypatch.setenv("HF_HUB_OFFLINE", value) + assert hf_env_offline() is False + + +def test_hf_env_offline_honors_transformers_flag(monkeypatch): + monkeypatch.setenv("TRANSFORMERS_OFFLINE", "1") + assert hf_env_offline() is True + + +def test_hf_env_offline_default_false(): + assert hf_env_offline() is False + + +# ── st_repo_id_candidates ──────────────────────────────────────── + + +def test_candidates_slashless_adds_st_alias(): + assert st_repo_id_candidates("all-MiniLM-L6-v2") == [ + "all-MiniLM-L6-v2", + "sentence-transformers/all-MiniLM-L6-v2", + ] + + +def test_candidates_with_org_is_verbatim(): + assert st_repo_id_candidates("org/model") == ["org/model"] + + +def test_candidates_empty_name(): + assert st_repo_id_candidates(" ") == [] + + +# ── hf_cache_snapshot_dir ──────────────────────────────────────── + + +def test_snapshot_dir_resolves_active_commit(hf_cache): + snapshot = _make_cache(hf_cache, "org/emb", {"modules.json": MODULES_JSON}) + assert hf_cache_snapshot_dir("org/emb") == snapshot + + +def test_snapshot_dir_none_when_uncached(hf_cache): + assert hf_cache_snapshot_dir("org/missing") is None + + +def test_snapshot_dir_uses_st_alias_for_slashless(hf_cache): + snapshot = _make_cache( + hf_cache, "sentence-transformers/all-MiniLM-L6-v2", {"modules.json": MODULES_JSON} + ) + assert hf_cache_snapshot_dir("all-MiniLM-L6-v2") == snapshot + + +def test_snapshot_dir_none_when_snapshot_missing(hf_cache): + from huggingface_hub.file_download import repo_folder_name + + repo_dir = hf_cache / repo_folder_name(repo_id = "org/broken", repo_type = "model") + (repo_dir / "refs").mkdir(parents = True) + (repo_dir / "refs" / "main").write_text("deadbeef") # no snapshots/deadbeef dir + assert hf_cache_snapshot_dir("org/broken") is None + + +def test_snapshot_dir_expands_env_vars_in_cache_path(tmp_path, monkeypatch): + # An unexpanded $VAR in HF_HUB_CACHE must resolve where the loader looks. + real = tmp_path / "hub" + real.mkdir() + monkeypatch.setenv("MY_HF_CACHE", str(real)) + monkeypatch.setenv("HF_HUB_CACHE", "$MY_HF_CACHE") + monkeypatch.delenv("HF_HOME", raising = False) + monkeypatch.delenv("SENTENCE_TRANSFORMERS_HOME", raising = False) + snapshot = _make_cache(real, "org/emb", {"modules.json": MODULES_JSON}) + assert hf_cache_snapshot_dir("org/emb") == snapshot + + +def test_snapshot_dir_uses_sentence_transformers_home(tmp_path, monkeypatch): + # ST uses SENTENCE_TRANSFORMERS_HOME as its cache_folder, so the gate must inspect it too. + st_home = tmp_path / "st_home" + st_home.mkdir() + monkeypatch.setenv("SENTENCE_TRANSFORMERS_HOME", str(st_home)) + monkeypatch.delenv("HF_HUB_CACHE", raising = False) + monkeypatch.delenv("HF_HOME", raising = False) + snapshot = _make_cache(st_home, "org/emb", {"modules.json": MODULES_JSON}) + assert hf_cache_snapshot_dir("org/emb") == snapshot + + +def test_snapshot_dir_prefers_selected_cache_over_st_home(tmp_path, monkeypatch): + # The RAG loader passes cache_folder=active_hf_hub_cache(), which overrides + # SENTENCE_TRANSFORMERS_HOME, so the snapshot + offline security lookup must + # search the selected cache even when ST_HOME points elsewhere. Otherwise the + # gate scans a cache the model never loads from and a pickle weight in the + # selected cache slips through. + st_home = tmp_path / "st_home" + st_home.mkdir() + selected = tmp_path / "hub" + selected.mkdir() + monkeypatch.setenv("SENTENCE_TRANSFORMERS_HOME", str(st_home)) + monkeypatch.delenv("HF_HUB_CACHE", raising = False) + monkeypatch.delenv("HF_HOME", raising = False) + monkeypatch.setattr( + "utils.hf_cache_settings.get_hf_cache_paths", + lambda: SimpleNamespace(hub_cache = selected), + ) + snapshot = _make_cache(selected, "org/emb", {"modules.json": MODULES_JSON}) # only in selected + assert hf_cache_snapshot_dir("org/emb") == snapshot + + +def test_snapshot_is_loadable_with_config_and_weights(hf_cache): + _make_cache(hf_cache, "org/emb", {"config.json": "{}", "model.safetensors": "x"}) + assert hf_cache_snapshot_is_loadable("org/emb") is True + + +def test_snapshot_is_not_loadable_when_metadata_only(hf_cache): + # A partial cache (refs/main resolves but no weights) is not loadable. + _make_cache(hf_cache, "org/partial", {"config.json": "{}", "modules.json": MODULES_JSON}) + assert hf_cache_snapshot_is_loadable("org/partial") is False + + +def test_snapshot_is_not_loadable_when_uncached(hf_cache): + assert hf_cache_snapshot_is_loadable("org/missing") is False + + +def test_gate_blocks_pickle_in_sentence_transformers_home(tmp_path, monkeypatch): + # A pickle under SENTENCE_TRANSFORMERS_HOME must still fail closed offline. + st_home = tmp_path / "st_home" + st_home.mkdir() + monkeypatch.setenv("SENTENCE_TRANSFORMERS_HOME", str(st_home)) + monkeypatch.delenv("HF_HUB_CACHE", raising = False) + monkeypatch.delenv("HF_HOME", raising = False) + _make_cache(st_home, "org/pk", {"config.json": "{}", "pytorch_model.bin": "x"}) + with _no_network(): + assert evaluate_file_security("org/pk", local_only_load = True).blocked is True + + +# ── is_embedding_model: offline (no network) ───────────────────── + + +def test_offline_true_for_cached_st_model(hf_cache, monkeypatch): + monkeypatch.setenv("HF_HUB_OFFLINE", "1") + _make_cache(hf_cache, "org/emb", {"modules.json": MODULES_JSON, "config.json": "{}"}) + with _no_network(): + assert _is_embedding_model("org/emb") is True + + +def test_offline_false_for_cached_non_st_model(hf_cache, monkeypatch): + monkeypatch.setenv("HF_HUB_OFFLINE", "1") + _make_cache(hf_cache, "org/plain", {"config.json": "{}", "model.safetensors": "x"}) + with _no_network(): + assert _is_embedding_model("org/plain") is False + + +def test_offline_false_when_uncached(hf_cache, monkeypatch): + monkeypatch.setenv("TRANSFORMERS_OFFLINE", "1") + with _no_network(): + assert _is_embedding_model("org/missing") is False + + +def test_offline_slashless_resolves_via_alias(hf_cache, monkeypatch): + monkeypatch.setenv("HF_HUB_OFFLINE", "1") + _make_cache(hf_cache, "sentence-transformers/all-MiniLM-L6-v2", {"modules.json": MODULES_JSON}) + with _no_network(): + assert _is_embedding_model("all-MiniLM-L6-v2") is True + + +def test_offline_ignores_stale_online_memo(hf_cache, monkeypatch): + # An online lookup memoizes True for an UNCACHED repo (tags say embedding, no weights). Once + # offline, is_embedding_model must reclassify from the empty cache and return False, not the + # stale online True that would make settings accept a repo _get() cannot load. + with patch( + "huggingface_hub.model_info", + side_effect = lambda *a, **k: SimpleNamespace( + tags = ["sentence-transformers"], pipeline_tag = None + ), + ): + assert _is_embedding_model("org/uncached-emb") is True # memoized True online + + monkeypatch.setenv("HF_HUB_OFFLINE", "1") + with _no_network(): + assert _is_embedding_model("org/uncached-emb") is False # recomputed from empty cache + + +def test_offline_recomputes_after_cache_materializes(hf_cache, monkeypatch): + # Because the offline branch never records a memo, once an uncached repo's snapshot + # materializes (another process populates the cache) the next call re-reports True. + monkeypatch.setenv("HF_HUB_OFFLINE", "1") + with _no_network(): + assert _is_embedding_model("org/later") is False # uncached + _make_cache(hf_cache, "org/later", {"modules.json": MODULES_JSON}) + assert _is_embedding_model("org/later") is True # cache now present, no stale negative + + +# ── is_embedding_model: online (bounded + fallback) ────────────── + + +def test_online_passes_bounded_timeout(hf_cache): + seen = {} + + def _mi( + name, + token = None, + timeout = None, + **kw, + ): + seen["timeout"] = timeout + return SimpleNamespace(tags = ["sentence-transformers"], pipeline_tag = None) + + with patch("huggingface_hub.model_info", side_effect = _mi): + assert _is_embedding_model("org/emb") is True + assert seen["timeout"] == 15.0 + + +def test_online_error_falls_back_to_cache_marker(hf_cache): + _make_cache(hf_cache, "org/emb", {"modules.json": MODULES_JSON}) + with patch("huggingface_hub.model_info", side_effect = RuntimeError("dns dead")): + assert _is_embedding_model("org/emb") is True + + +def test_online_error_without_cache_returns_false(hf_cache): + with patch("huggingface_hub.model_info", side_effect = RuntimeError("dns dead")): + assert _is_embedding_model("org/missing") is False + + +# ── evaluate_file_security: offline fail-closed gate ───────────── + + +def _offline_decision(name): + return evaluate_file_security(name, local_only_load = True) + + +def test_gate_allows_safetensors_only(hf_cache): + _make_cache(hf_cache, "org/st", {"modules.json": MODULES_JSON, "model.safetensors": "x"}) + with _no_network(): + assert _offline_decision("org/st").blocked is False + + +def test_gate_blocks_pickle_without_safetensors(hf_cache): + _make_cache(hf_cache, "org/pk", {"config.json": "{}", "pytorch_model.bin": "x"}) + with _no_network(): + decision = _offline_decision("org/pk") + assert decision.blocked is True + assert any(u["path"] == "pytorch_model.bin" for u in decision.unsafe_files) + + +def test_gate_allows_pickle_with_safetensors_sibling(hf_cache): + _make_cache(hf_cache, "org/both", {"pytorch_model.bin": "x", "model.safetensors": "y"}) + with _no_network(): + assert _offline_decision("org/both").blocked is False + + +def test_gate_blocks_sharded_pickle(hf_cache): + _make_cache( + hf_cache, + "org/shard", + { + "pytorch_model-00001-of-00002.bin": "a", + "pytorch_model-00002-of-00002.bin": "b", + }, + ) + with _no_network(): + assert _offline_decision("org/shard").blocked is True + + +def test_gate_blocks_indexed_pickle_shard_in_subdirectory(hf_cache): + # from_pretrained follows weight_map paths relative to the root index, so these nested shards + # are deserialized even though they are not direct children of the load root (iterdir misses + # them). The online gate blocks index-referenced subdir pickles; the offline gate must too. + _make_cache( + hf_cache, + "org/indexed-shard", + { + "pytorch_model.bin.index.json": ( + '{"weight_map": {"layer.weight": "shards/pytorch_model-00001-of-00001.bin"}}' + ), + "shards/pytorch_model-00001-of-00001.bin": "pickle", + }, + ) + with _no_network(): + decision = _offline_decision("org/indexed-shard") + assert decision.blocked is True + assert any( + u["path"] == "shards/pytorch_model-00001-of-00001.bin" for u in decision.unsafe_files + ) + + +def test_gate_blocks_indexed_pickle_shard_with_nonstandard_stem(hf_cache): + # The index tells the loader to deserialize this file, so a pickle EXTENSION is enough -- the + # shard's stem need not match the on-disk weight-name heuristic (which only guesses bare files). + _make_cache( + hf_cache, + "org/indexed-odd", + { + "pytorch_model.bin.index.json": '{"weight_map": {"w": "shards/evil-00001-of-00001.bin"}}', + "shards/evil-00001-of-00001.bin": "pickle", + }, + ) + with _no_network(): + decision = _offline_decision("org/indexed-odd") + assert decision.blocked is True + assert any(u["path"] == "shards/evil-00001-of-00001.bin" for u in decision.unsafe_files) + + +def test_gate_blocks_safetensors_index_pointing_to_pickle_shard(hf_cache): + # load_state_dict picks safetensors vs torch.load by each shard's own suffix, so a + # model.safetensors.index.json that maps a weight to a .bin shard still deserializes it. The + # index's own existence must not suppress the shard it names. + _make_cache( + hf_cache, + "org/st-index-pickle", + { + "model.safetensors.index.json": ( + '{"weight_map": {"w": "shards/pytorch_model-00001-of-00001.bin"}}' + ), + "shards/pytorch_model-00001-of-00001.bin": "pickle", + }, + ) + with _no_network(): + decision = _offline_decision("org/st-index-pickle") + assert decision.blocked is True + assert any( + u["path"] == "shards/pytorch_model-00001-of-00001.bin" for u in decision.unsafe_files + ) + + +def test_gate_blocks_indexed_shard_with_no_pickle_extension(hf_cache): + # Transformers torch.loads any indexed shard not ending in .safetensors, so an unconventional + # extensionless name is still a deserialization target. + _make_cache( + hf_cache, + "org/indexed-noext", + { + "pytorch_model.bin.index.json": '{"weight_map": {"w": "shards/payload"}}', + "shards/payload": "pickle", + }, + ) + with _no_network(): + decision = _offline_decision("org/indexed-noext") + assert decision.blocked is True + assert any(u["path"] == "shards/payload" for u in decision.unsafe_files) + + +_UPPER_INDEX_FILES = { + "PYTORCH_MODEL.BIN.INDEX.JSON": ( + '{"weight_map": {"w": "shards/pytorch_model-00001-of-00001.bin"}}' + ), + "shards/pytorch_model-00001-of-00001.bin": "pickle", +} + + +def test_gate_blocks_uppercase_index_on_case_insensitive_fs(hf_cache): + # On a case-insensitive volume (Windows/macOS) from_pretrained opens an oddly-cased index when it + # requests the canonical lowercase name, so the loader-mirror lookup resolves it and blocks. + _requires_case_insensitive_fs(hf_cache) + _make_cache(hf_cache, "org/upper-index", _UPPER_INDEX_FILES) + with _no_network(): + decision = _offline_decision("org/upper-index") + assert decision.blocked is True + assert any( + u["path"] == "shards/pytorch_model-00001-of-00001.bin" for u in decision.unsafe_files + ) + + +def test_gate_allows_uppercase_index_on_case_sensitive_fs(hf_cache): + # On a case-sensitive FS from_pretrained's os.path.isfile of the canonical lowercase name misses + # the uppercase artifact and never loads its shard, so the gate must not over-block it. + _requires_case_sensitive_fs(hf_cache) + _make_cache(hf_cache, "org/upper-index", _UPPER_INDEX_FILES) + with _no_network(): + assert _offline_decision("org/upper-index").blocked is False + + +def test_gate_blocks_indexed_shard_named_with_backslash(hf_cache): + # On POSIX a backslash is a literal filename char, so from_pretrained joins the raw weight_map + # value and deserializes a file actually named "dir\payload.bin"; the gate must probe it verbatim. + import os + + if os.sep != "/": + pytest.skip("backslash is a path separator off POSIX") + _make_cache( + hf_cache, + "org/backslash", + { + "pytorch_model.bin.index.json": '{"weight_map": {"w": "dir\\\\payload.bin"}}', + "dir\\payload.bin": "pickle", + }, + ) + with _no_network(): + decision = _offline_decision("org/backslash") + assert decision.blocked is True + assert any(u["path"] == "dir\\payload.bin" for u in decision.unsafe_files) + + +def test_gate_blocks_indexed_shard_with_uppercase_safetensors_suffix(hf_cache): + # load_state_dict's endswith(".safetensors") is case-sensitive, so a shard named payload.SAFETENSORS + # falls to torch.load. The gate must classify shard suffixes case-sensitively to match it. + _make_cache( + hf_cache, + "org/upper-suffix", + { + "pytorch_model.bin.index.json": '{"weight_map": {"w": "shards/payload.SAFETENSORS"}}', + "shards/payload.SAFETENSORS": "pickle", + }, + ) + with _no_network(): + decision = _offline_decision("org/upper-suffix") + assert decision.blocked is True + assert any(u["path"] == "shards/payload.SAFETENSORS" for u in decision.unsafe_files) + + +def test_gate_allows_stale_safetensors_index_beside_direct_safetensors(hf_cache): + # A complete direct model.safetensors is selected before either index, so a stale + # model.safetensors.index.json referencing a .bin shard never deserializes -> must not block. + _make_cache( + hf_cache, + "org/direct-plus-stale-index", + { + "model.safetensors": "tensors", + "model.safetensors.index.json": ( + '{"weight_map": {"w": "shards/pytorch_model-00001-of-00001.bin"}}' + ), + "shards/pytorch_model-00001-of-00001.bin": "pickle", + }, + ) + with _no_network(): + assert _offline_decision("org/direct-plus-stale-index").blocked is False + + +def test_gate_blocks_pytorch_index_with_uppercase_safetensors_decoy(hf_cache): + # On a case-sensitive FS, from_pretrained asks for the canonical lowercase model.safetensors, does + # not find an uppercase decoy, and selects the pytorch index instead. The decoy must not suppress. + _requires_case_sensitive_fs(hf_cache) + _make_cache( + hf_cache, + "org/upper-decoy", + { + "MODEL.SAFETENSORS": "decoy", + "pytorch_model.bin.index.json": ( + '{"weight_map": {"w": "shards/pytorch_model-00001-of-00001.bin"}}' + ), + "shards/pytorch_model-00001-of-00001.bin": "pickle", + }, + ) + with _no_network(): + decision = _offline_decision("org/upper-decoy") + assert decision.blocked is True + assert any( + u["path"] == "shards/pytorch_model-00001-of-00001.bin" for u in decision.unsafe_files + ) + + +def test_gate_blocks_direct_pickle_with_uppercase_safetensors_decoy(hf_cache): + # Same decoy against a direct pytorch_model.bin: the loader selects the pickle, so the uppercase + # safetensors must not suppress it on a case-sensitive FS. + _requires_case_sensitive_fs(hf_cache) + _make_cache( + hf_cache, + "org/upper-decoy-direct", + {"MODEL.SAFETENSORS": "decoy", "pytorch_model.bin": "pickle"}, + ) + with _no_network(): + decision = _offline_decision("org/upper-decoy-direct") + assert decision.blocked is True + assert any(u["path"] == "pytorch_model.bin" for u in decision.unsafe_files) + + +def test_gate_blocks_indexed_pickle_shard_in_module_subdir(hf_cache): + # A weight index inside a sentence-transformers module load root points at a nested pickle shard. + _make_cache( + hf_cache, + "org/mod-indexed", + { + "modules.json": _modules_json("0_Transformer"), + "0_Transformer/pytorch_model.bin.index.json": ( + '{"weight_map": {"w": "shards/pytorch_model-00001-of-00001.bin"}}' + ), + "0_Transformer/shards/pytorch_model-00001-of-00001.bin": "pickle", + }, + ) + with _no_network(): + decision = _offline_decision("org/mod-indexed") + assert decision.blocked is True + assert any( + u["path"] == "0_Transformer/shards/pytorch_model-00001-of-00001.bin" + for u in decision.unsafe_files + ) + + +def test_gate_allows_indexed_pickle_shard_with_safetensors_sibling(hf_cache): + # A base model.safetensors makes the loader ignore the pickle index entirely, so it must not + # block (mirrors the direct-file safetensors-sibling suppression). + _make_cache( + hf_cache, + "org/indexed-both", + { + "pytorch_model.bin.index.json": ( + '{"weight_map": {"w": "shards/pytorch_model-00001-of-00001.bin"}}' + ), + "shards/pytorch_model-00001-of-00001.bin": "pickle", + "model.safetensors": "y", + }, + ) + with _no_network(): + assert _offline_decision("org/indexed-both").blocked is False + + +def test_gate_allows_indexed_safetensors_shard_in_subdirectory(hf_cache): + # A safetensors index lists inert shards -- following it must never block (guards against a + # scanner that flags every indexed shard regardless of format). + _make_cache( + hf_cache, + "org/st-indexed", + { + "model.safetensors.index.json": ( + '{"weight_map": {"w": "shards/model-00001-of-00001.safetensors"}}' + ), + "shards/model-00001-of-00001.safetensors": "tensors", + }, + ) + with _no_network(): + assert _offline_decision("org/st-indexed").blocked is False + + +def test_gate_blocks_on_index_path_traversal(hf_cache): + # A weight_map entry escaping the snapshot via ".." is abnormal/hostile -> fail closed. + _make_cache( + hf_cache, + "org/escape", + {"pytorch_model.bin.index.json": '{"weight_map": {"w": "../../../../etc/evil.bin"}}'}, + ) + with _no_network(): + assert _offline_decision("org/escape").blocked is True + + +def test_gate_allows_symlinked_sharded_safetensors(tmp_path, monkeypatch): + # Real HF caches store snapshot files as symlinks into blobs/. A resolve()-based containment + # check would escape the snapshot and false-block every sharded model; the lexical gate must not. + import hashlib + import os + + from huggingface_hub.file_download import repo_folder_name + + root = tmp_path / "hub" + root.mkdir() + monkeypatch.setenv("HF_HOME", str(tmp_path)) + monkeypatch.setenv("HF_HUB_CACHE", str(root)) + monkeypatch.setattr( + "utils.hf_cache_settings.get_hf_cache_paths", + lambda: SimpleNamespace(hub_cache = root), + ) + repo_dir = root / repo_folder_name(repo_id = "org/sym", repo_type = "model") + (repo_dir / "refs").mkdir(parents = True) + (repo_dir / "refs" / "main").write_text(_COMMIT) + blobs = repo_dir / "blobs" + blobs.mkdir() + snapshot = repo_dir / "snapshots" / _COMMIT + (snapshot / "shards").mkdir(parents = True) + + def _blobbed(rel, content): + digest = hashlib.sha256(content.encode()).hexdigest() + (blobs / digest).write_text(content) + target = snapshot / rel + target.parent.mkdir(parents = True, exist_ok = True) + target.symlink_to(os.path.relpath(blobs / digest, target.parent)) + + _blobbed("config.json", "{}") + _blobbed( + "model.safetensors.index.json", + '{"weight_map": {"w": "shards/model-00001-of-00001.safetensors"}}', + ) + _blobbed("shards/model-00001-of-00001.safetensors", "tensors") + with _no_network(): + assert _offline_decision("org/sym").blocked is False + + +def test_gate_allows_index_without_weight_map(hf_cache): + # An index whose top-level JSON has no dict weight_map lets the loader resolve no shards, so it + # must not crash or block on its own (only inert safetensors are cached here). + _make_cache( + hf_cache, + "org/no-wm", + {"model.safetensors.index.json": "[]", "model.safetensors": "x"}, + ) + with _no_network(): + assert _offline_decision("org/no-wm").blocked is False + + +def test_gate_allows_nothing_cached(hf_cache): + with _no_network(): + assert _offline_decision("org/missing").blocked is False + + +def test_gate_allows_gguf_only(hf_cache): + _make_cache(hf_cache, "org/gg", {"model.gguf": "x"}) + with _no_network(): + assert _offline_decision("org/gg").blocked is False + + +def test_gate_blocks_pickle_in_module_subdir(hf_cache): + # 0_Transformer is a module load root (listed in modules.json), so its pickle blocks. + _make_cache( + hf_cache, + "org/mod", + {"modules.json": _modules_json("0_Transformer"), "0_Transformer/pytorch_model.bin": "x"}, + ) + with _no_network(): + assert _offline_decision("org/mod").blocked is True + + +def test_gate_allows_pickle_in_subdir_with_safetensors(hf_cache): + _make_cache( + hf_cache, + "org/mod2", + { + "modules.json": _modules_json("0_Transformer"), + "0_Transformer/pytorch_model.bin": "x", + "0_Transformer/model.safetensors": "y", + }, + ) + with _no_network(): + assert _offline_decision("org/mod2").blocked is False + + +def test_gate_allows_unreferenced_nested_pickle(hf_cache): + # A pickle in a dir NOT referenced by modules.json (e.g. nemo/) is never deserialized, so it + # must not block the offline load (matches the online gate). + _make_cache( + hf_cache, + "org/aux", + { + "modules.json": MODULES_JSON, # Transformer at the root only + "model.safetensors": "w", + "nemo/pytorch_model.bin": "x", + }, + ) + with _no_network(): + assert _offline_decision("org/aux").blocked is False + + +def test_gate_blocks_adapter_pickle_without_safetensors(hf_cache): + _make_cache(hf_cache, "org/ad", {"config.json": "{}", "adapter_model.bin": "x"}) + with _no_network(): + decision = _offline_decision("org/ad") + assert decision.blocked is True + assert any(u["path"] == "adapter_model.bin" for u in decision.unsafe_files) + + +def test_gate_allows_adapter_pickle_with_adapter_safetensors(hf_cache): + _make_cache(hf_cache, "org/ad2", {"adapter_model.bin": "x", "adapter_model.safetensors": "y"}) + with _no_network(): + assert _offline_decision("org/ad2").blocked is False + + +def test_gate_blocks_base_pickle_with_only_adapter_safetensors_decoy(hf_cache): + # A decoy adapter_model.safetensors must NOT suppress a base pytorch_model.bin (the base + # loader would still deserialize the unscanned pickle). + _make_cache(hf_cache, "org/decoy", {"pytorch_model.bin": "x", "adapter_model.safetensors": "y"}) + with _no_network(): + assert _offline_decision("org/decoy").blocked is True + + +def test_gate_blocks_adapter_pickle_with_only_base_safetensors_decoy(hf_cache): + # Symmetric: a base model.safetensors must NOT suppress an adapter_model.bin. + _make_cache(hf_cache, "org/decoy2", {"adapter_model.bin": "x", "model.safetensors": "y"}) + with _no_network(): + assert _offline_decision("org/decoy2").blocked is True + + +def test_gate_reports_snapshot_relative_path(hf_cache): + _make_cache( + hf_cache, + "org/mod3", + {"modules.json": _modules_json("0_Transformer"), "0_Transformer/pytorch_model.bin": "x"}, + ) + with _no_network(): + decision = _offline_decision("org/mod3") + assert decision.blocked is True + assert any(u["path"] == "0_Transformer/pytorch_model.bin" for u in decision.unsafe_files) + + +# ── evaluate_file_security: online path unchanged ──────────────── + + +def test_online_default_blocks_unsafe(): + status = { + "scansDone": True, + "filesWithIssues": [{"path": "pytorch_model.bin", "level": "unsafe"}], + } + with patch( + "huggingface_hub.model_info", + side_effect = lambda *a, **k: SimpleNamespace(security_repo_status = status), + ): + assert evaluate_file_security("org/x").blocked is True + + +def test_online_default_allows_clean(): + status = {"scansDone": True, "filesWithIssues": []} + with patch( + "huggingface_hub.model_info", + side_effect = lambda *a, **k: SimpleNamespace(security_repo_status = status), + ): + assert evaluate_file_security("org/x").blocked is False + + +# ── embeddings guard + loader ──────────────────────────────────── + + +def test_guard_offline_blocks_pickle_only(hf_cache): + from core.rag.embeddings import UnsafeEmbeddingModelError, _guard_model_security + _make_cache(hf_cache, "org/pk", {"config.json": "{}", "pytorch_model.bin": "x"}) + with _no_network(): + with pytest.raises(UnsafeEmbeddingModelError): + _guard_model_security("org/pk", local_only = True) + + +def test_guard_offline_allows_safetensors(hf_cache): + from core.rag.embeddings import _guard_model_security + _make_cache(hf_cache, "org/st", {"modules.json": MODULES_JSON, "model.safetensors": "x"}) + with _no_network(): + _guard_model_security("org/st", local_only = True) # must not raise + + +def _install_fake_sentence_transformers(monkeypatch, captured): + class FakeSentenceTransformer: + def __init__( + self, + name, + *, + device = None, + model_kwargs = None, + local_files_only = False, + **kw, + ): + captured["name"] = name + captured["device"] = device + captured["local_files_only"] = local_files_only + + module = types.ModuleType("sentence_transformers") + module.SentenceTransformer = FakeSentenceTransformer + monkeypatch.setitem(sys.modules, "sentence_transformers", module) + + +def test_get_offline_loads_from_local_snapshot(hf_cache, monkeypatch): + from core.rag import embeddings + + snapshot = _make_cache( + hf_cache, "org/st", {"modules.json": MODULES_JSON, "model.safetensors": "x"} + ) + # TRANSFORMERS_OFFLINE only: a cached model loads from its local snapshot dir (a local path, + # never the Hub), offline-safe on ANY sentence-transformers version. + monkeypatch.setenv("TRANSFORMERS_OFFLINE", "1") + monkeypatch.delenv("HF_HUB_OFFLINE", raising = False) + monkeypatch.setattr(embeddings, "_model", None, raising = False) + monkeypatch.setattr(embeddings, "_name", None, raising = False) + monkeypatch.setattr(embeddings, "_install_torchao_stub_once", lambda: None) + monkeypatch.setattr(embeddings, "_device", lambda: "cpu") + captured = {} + _install_fake_sentence_transformers(monkeypatch, captured) + with _no_network(): + embeddings._get("org/st") + assert captured["name"] == str(snapshot) + + +def test_get_offline_uncached_uses_local_files_only(tmp_path, monkeypatch): + from core.rag import embeddings + + empty = tmp_path / "hub" + empty.mkdir() + monkeypatch.setenv("HF_HUB_CACHE", str(empty)) + monkeypatch.delenv("HF_HOME", raising = False) + monkeypatch.delenv("SENTENCE_TRANSFORMERS_HOME", raising = False) + monkeypatch.setenv("TRANSFORMERS_OFFLINE", "1") + monkeypatch.delenv("HF_HUB_OFFLINE", raising = False) + monkeypatch.setattr(embeddings, "_model", None, raising = False) + monkeypatch.setattr(embeddings, "_name", None, raising = False) + monkeypatch.setattr(embeddings, "_install_torchao_stub_once", lambda: None) + monkeypatch.setattr(embeddings, "_device", lambda: "cpu") + # No cache -> repo-id load forced cache-only (fails fast offline, not a hang). + monkeypatch.setattr(embeddings, "_guard_model_security", lambda name, local_only = False: None) + captured = {} + _install_fake_sentence_transformers(monkeypatch, captured) + embeddings._get("org/uncached-xyz") + assert captured["name"] == "org/uncached-xyz" + assert captured["local_files_only"] is True + + +def test_get_online_omits_local_files_only(monkeypatch): + from core.rag import embeddings + + monkeypatch.delenv("HF_HUB_OFFLINE", raising = False) + monkeypatch.delenv("TRANSFORMERS_OFFLINE", raising = False) + monkeypatch.setattr(embeddings, "_model", None, raising = False) + monkeypatch.setattr(embeddings, "_name", None, raising = False) + monkeypatch.setattr(embeddings, "_install_torchao_stub_once", lambda: None) + monkeypatch.setattr(embeddings, "_device", lambda: "cpu") + # Isolate the loader wiring from the online guard's network calls. + monkeypatch.setattr(embeddings, "_guard_model_security", lambda name, local_only = False: None) + captured = {} + _install_fake_sentence_transformers(monkeypatch, captured) + embeddings._get("org/online") + assert captured["local_files_only"] is False diff --git a/studio/backend/tests/test_offline_gguf_cache_fallback.py b/studio/backend/tests/test_offline_gguf_cache_fallback.py index e24e2ca451..d1e61d0546 100644 --- a/studio/backend/tests/test_offline_gguf_cache_fallback.py +++ b/studio/backend/tests/test_offline_gguf_cache_fallback.py @@ -119,10 +119,21 @@ def _build_cache( return snap +def _symlink_or_skip(link: Path, target: Path) -> None: + try: + link.symlink_to(target) + except OSError as exc: + pytest.skip(f"symlinks unavailable: {exc}") + + @pytest.fixture def hf_cache(tmp_path, monkeypatch): """Point ``huggingface_hub.constants.HF_HUB_CACHE`` at a temp dir.""" monkeypatch.setattr(hf_constants, "HF_HUB_CACHE", str(tmp_path)) + monkeypatch.setattr( + "utils.hf_cache_settings.get_hf_cache_paths", + lambda: _types.SimpleNamespace(hub_cache = tmp_path), + ) return tmp_path @@ -220,6 +231,10 @@ class TestGgufVariantFileResolution: return f"/fake/{repo_id}/{filename}" monkeypatch.setattr(hf_constants, "HF_HUB_CACHE", str(tmp_path)) + monkeypatch.setattr( + "utils.hf_cache_settings.get_hf_cache_paths", + lambda: _types.SimpleNamespace(hub_cache = tmp_path), + ) with ( patch( "huggingface_hub.list_repo_files", @@ -244,9 +259,7 @@ class TestGgufVariantFileResolution: def test_download_reuses_older_snapshot_when_current_ref_snapshot_is_partial( self, monkeypatch, hf_cache ): - # Cross-snapshot reuse is an offline-resilience path: online, hf_hub_download - # resumes the partial current-ref download and revalidates the revision instead - # of serving an older snapshot's same-name blob. + # Keep coverage for offline reuse; online reuse is tested separately. monkeypatch.setenv("HF_HUB_OFFLINE", "1") backend = LlamaCppBackend() repo = "unsloth/vision-GGUF" @@ -292,8 +305,7 @@ class TestGgufVariantFileResolution: def test_download_reuses_cached_gguf_when_lowercase_partial_cache_shadows_it( self, monkeypatch, hf_cache ): - # Case-variant cross-dir reuse is offline-only; online the canonical repo id - # resolves up front and hf_hub_download fetches the current revision. + # Keep coverage for case-insensitive offline cache lookup. monkeypatch.setenv("HF_HUB_OFFLINE", "1") backend = LlamaCppBackend() canonical_repo = "unsloth/gemma-4-E2B-it-GGUF" @@ -348,45 +360,26 @@ class TestGgufVariantFileResolution: assert out == str(snap / gguf_file) assert seen_repos - def test_download_online_does_not_reuse_old_snapshot(self, monkeypatch, hf_cache): - # Online, an older same-name snapshot must not be served (it may be a stale - # revision); hf_hub_download is called so the current revision is fetched and - # its etag revalidated. + def test_download_online_reuses_complete_cached_snapshot(self, monkeypatch, hf_cache): + # Loads reuse complete cached models across repo revisions. monkeypatch.delenv("HF_HUB_OFFLINE", raising = False) backend = LlamaCppBackend() repo = "unsloth/vision-GGUF" - _build_cache(hf_cache, repo, {"model-UD-Q4_K_XL.gguf": 4}, snapshot_sha = "a" * 40) - downloaded: list[str] = [] + snap = _build_cache(hf_cache, repo, {"model-UD-Q4_K_XL.gguf": 4}, snapshot_sha = "a" * 40) - def fake_get_paths_info( - _repo_id, - paths, - token = None, - ): - return [_types.SimpleNamespace(path = p, size = 4) for p in paths if p] - - def fake_download( - repo_id, - filename, - token = None, - **kwargs, - ): - downloaded.append(filename) - return f"/fresh/{filename}" + def fail_download(*_args, **_kwargs): + raise AssertionError("must reuse the cached GGUF instead of downloading") with ( patch( "huggingface_hub.list_repo_files", lambda *_a, **_k: ["model-UD-Q4_K_XL.gguf"], ), - patch("huggingface_hub.get_paths_info", fake_get_paths_info), - patch("huggingface_hub.try_to_load_from_cache", lambda *_a, **_k: None), - patch("core.inference.llama_cpp.hf_hub_download_with_xet_fallback", fake_download), + patch("core.inference.llama_cpp.hf_hub_download_with_xet_fallback", fail_download), ): out = backend._download_gguf(hf_repo = repo, hf_variant = "UD-Q4_K_XL") - assert downloaded == ["model-UD-Q4_K_XL.gguf"] - assert out == "/fresh/model-UD-Q4_K_XL.gguf" + assert out == str(snap / "model-UD-Q4_K_XL.gguf") def test_download_reuses_older_snapshot_when_offline_env_is_true(self, monkeypatch, hf_cache): # HF_HUB_OFFLINE accepts truthy spellings beyond "1" (true/yes/on); the offline @@ -449,6 +442,40 @@ class TestGgufVariantFileResolution: assert out == str(snap / "mmproj-F16.gguf") + def test_download_companion_uses_selected_cache_not_import_time_default( + self, monkeypatch, tmp_path + ): + monkeypatch.setenv("HF_HUB_OFFLINE", "1") + import_time_cache = tmp_path / "import-time-cache" + selected_cache = tmp_path / "selected-cache" + monkeypatch.setattr(hf_constants, "HF_HUB_CACHE", str(import_time_cache)) + monkeypatch.setattr( + "utils.hf_cache_settings.get_hf_cache_paths", + lambda: _types.SimpleNamespace(hub_cache = selected_cache), + ) + repo = "unsloth/vision-GGUF" + snap = _build_cache(selected_cache, repo, {"mmproj-F16.gguf": 4}) + backend = LlamaCppBackend() + + offline_error = type("OfflineModeIsEnabled", (Exception,), {}) + + def fail_list(*_args, **_kwargs): + raise offline_error("offline") + + def fail_download(*_args, **_kwargs): + raise AssertionError("selected-cache companion must not download") + + with ( + patch("huggingface_hub.list_repo_files", fail_list), + patch( + "core.inference.llama_cpp.hf_hub_download_with_xet_fallback", + fail_download, + ), + ): + out = backend._download_mmproj(hf_repo = repo) + + assert out == str(snap / "mmproj-F16.gguf") + def test_download_includes_uppercase_split_gguf_shards(self, monkeypatch, tmp_path): backend = LlamaCppBackend() downloaded: list[str] = [] @@ -475,6 +502,10 @@ class TestGgufVariantFileResolution: return f"/fake/{repo_id}/{filename}" monkeypatch.setattr(hf_constants, "HF_HUB_CACHE", str(tmp_path)) + monkeypatch.setattr( + "utils.hf_cache_settings.get_hf_cache_paths", + lambda: _types.SimpleNamespace(hub_cache = tmp_path), + ) with ( patch("huggingface_hub.list_repo_files", lambda *_a, **_k: files), patch("huggingface_hub.get_paths_info", fake_get_paths_info), @@ -919,7 +950,7 @@ class TestHfOfflineIfDnsDead: assert "HF_HUB_OFFLINE" not in os.environ def test_user_set_hf_hub_offline_is_preserved(self, dns, clean_offline_env, monkeypatch): - # User explicitly set offline before launching Studio. + # User explicitly set offline before launching Unsloth. monkeypatch.setenv("HF_HUB_OFFLINE", "1") dns.fail() with _hf_offline_if_dns_dead() as did_set: @@ -1106,7 +1137,7 @@ class TestListLocalGgufVariantsSubdir: target.write_bytes(b"\0" * 20) out = _find_local_gguf_by_variant(str(tmp_path), "Q4_K_M") - assert out == str(target.resolve()) + assert out == str(target.absolute()) def test_find_local_gguf_by_variant_skips_big_endian_only_match(self, tmp_path): from utils.models.model_config import _find_local_gguf_by_variant @@ -1116,6 +1147,57 @@ class TestListLocalGgufVariantsSubdir: assert _find_local_gguf_by_variant(str(tmp_path), "Q4_K_M") is None + def test_find_local_gguf_by_variant_keeps_split_symlink_name(self, tmp_path): + from utils.models.model_config import _find_local_gguf_by_variant + + blobs = tmp_path / "blobs" + blobs.mkdir() + snap = tmp_path / "snapshots" / "rev" / "BF16" + snap.mkdir(parents = True) + (tmp_path / "snapshots" / "rev" / "config.json").write_text("{}") + for i, sha in enumerate(("aa" * 32, "bb" * 32), start = 1): + (blobs / sha).write_bytes(b"\0" * 10) + _symlink_or_skip(snap / f"model-BF16-0000{i}-of-00002.gguf", blobs / sha) + + out = _find_local_gguf_by_variant(str(tmp_path / "snapshots" / "rev"), "BF16") + assert out is not None + assert Path(out).name == "model-BF16-00001-of-00002.gguf" + + def test_detect_gguf_model_keeps_split_symlink_name(self, tmp_path): + from utils.models.model_config import detect_gguf_model + + blobs = tmp_path / "blobs" + blobs.mkdir() + snap = tmp_path / "snapshots" / "rev" + snap.mkdir(parents = True) + for i, (sha, size) in enumerate((("cc" * 32, 10), ("dd" * 32, 20)), start = 1): + (blobs / sha).write_bytes(b"\0" * size) + _symlink_or_skip(snap / f"model-BF16-0000{i}-of-00002.gguf", blobs / sha) + + out = detect_gguf_model(str(snap)) + assert out is not None + assert Path(out).name == "model-BF16-00001-of-00002.gguf" + + def test_lone_split_symlink_uses_colocated_target_shards(self, tmp_path): + from utils.models.model_config import _find_local_gguf_by_variant, detect_gguf_model + + target_dir = tmp_path / "external" / "BF16" + target_dir.mkdir(parents = True) + target = target_dir / "model-BF16-00001-of-00002.gguf" + target.write_bytes(b"\0" * 10) + (target_dir / "model-BF16-00002-of-00002.gguf").write_bytes(b"\0" * 10) + + local = tmp_path / "local" + local.mkdir() + (local / "config.json").write_text("{}") + link = local / target.name + _symlink_or_skip(link, target) + + expected = str(target.absolute()) + assert _find_local_gguf_by_variant(str(local), "BF16") == expected + assert detect_gguf_model(str(local)) == expected + assert detect_gguf_model(str(link)) == expected + def test_model_config_variant_ignores_big_endian_sibling(self, tmp_path): from utils.models.model_config import ModelConfig diff --git a/studio/backend/tests/test_offline_inference_parent.py b/studio/backend/tests/test_offline_inference_parent.py index 71331220d6..bd0014ea64 100644 --- a/studio/backend/tests/test_offline_inference_parent.py +++ b/studio/backend/tests/test_offline_inference_parent.py @@ -139,7 +139,7 @@ class TestLoraDetectOffline: monkeypatch.setenv("HF_HUB_OFFLINE", "1") - # Studio catches Exception broadly; pin that the call still happens + # Unsloth catches Exception broadly; pin that the call still happens # (so cached LoRAs aren't missed) and returns fast via the mock. class _OfflineModeIsEnabled(Exception): pass diff --git a/studio/backend/tests/test_openai_auto_switch.py b/studio/backend/tests/test_openai_auto_switch.py index d02a2a4f7e..9c6c20e6b6 100644 --- a/studio/backend/tests/test_openai_auto_switch.py +++ b/studio/backend/tests/test_openai_auto_switch.py @@ -8,6 +8,7 @@ tests/test_gguf_completion_usage.py. """ import asyncio +import os import pytest @@ -18,6 +19,10 @@ from utils import openai_auto_switch_settings as settings class _FakeBackend: + effective_parallel_slots = 1 + _slot_save_binary = None + _gguf_path = None + def __init__( self, loaded_id = None, @@ -29,6 +34,22 @@ class _FakeBackend: self.hf_variant = hf_variant self._openai_advertised_id = advertised_id + def save_slots_for_resume(self, should_abort = None): + return None + + def restore_slots_for_resume(self, manifest): + return None + + def _slot_launch_fingerprint(self): + return ((), None, None, 1) + + def _gguf_file_identity(self, path): + try: + st = os.stat(path) + except OSError: + return None + return ((st.st_size, st.st_mtime_ns),) + class _LoadRecorder: """Stand-in for the load route: records calls and simulates a load.""" @@ -47,16 +68,27 @@ class _LoadRecorder: request, fastapi_request, current_subject = None, + *, + current_request_counted = False, ): + # Mirror the production load boundary before recording any replacement. + await inference_route._wait_for_model_switch_idle( + current_request_counted = current_request_counted + ) self.calls.append(request) if self.fail: from fastapi import HTTPException raise HTTPException(status_code = 503, detail = "load failed") self.backend.model_identifier = request.model_path + self.backend.hf_variant = getattr(request, "gguf_variant", None) + self.backend._gguf_path = request.model_path self.backend.is_loaded = True # Mirror _load_model_impl: a load advertises its own id until the # auto-switch caller overwrites it with the repo id. self.backend._openai_advertised_id = None + from core.inference import llama_keepwarm as kw + + kw.note_model_loaded(self.backend) return None @@ -68,7 +100,6 @@ def _wire(monkeypatch, *, enabled, resolves_to, backend, recorder): # gate that auto-switch already owns, so it calls the impl directly). monkeypatch.setattr(inference_route, "_load_model_impl", recorder) monkeypatch.setattr(inference_route, "_auto_switch_waiters", {}) - monkeypatch.setattr(inference_route, "_auto_switch_request_waiters", {}) def _run_hook(model = "some/model"): @@ -446,6 +477,75 @@ def test_idle_loop_unloads_after_ttl_and_stashes_for_reload(monkeypatch): assert stash is not None and stash[0] == "unsloth/Idle-GGUF" and stash[1] == "Q4_K_M" +def test_idle_loop_deletes_saved_kv_when_unload_fails(monkeypatch, tmp_path): + import time + from core.inference import llama_keepwarm as kw + + monkeypatch.setattr(settings, "get_auto_unload_idle_seconds", lambda: 0.005) + monkeypatch.setattr(settings, "get_auto_unload_keep_kv", lambda: True) + kw._inflight = 0 + kw._pending = 0 + kw._last_active = time.monotonic() - 3600 + kw._last_unloaded_model = None + kw._kv_resume = None + + saved = tmp_path / "resume-abc-slot0.bin" + backend = _FakeBackend("unsloth/Idle-GGUF") + manifests = [] + + def _save(should_abort = None): + if manifests: + return None + saved.write_bytes(b"kv") + manifest = {"dir": str(tmp_path), "slots": [{"id": 0, "filename": saved.name}]} + manifests.append(manifest) + return manifest + + def _unload(): + raise RuntimeError("cuda teardown failed") + + backend.save_slots_for_resume = _save + backend.unload_model = _unload + monkeypatch.setattr(inference_route, "get_llama_cpp_backend", lambda: backend) + + async def _drive(): + task = asyncio.create_task(kw.idle_unload_loop(poll_seconds = 0.01)) + for _ in range(200): + await asyncio.sleep(0.01) + if manifests and not saved.exists(): + break + task.cancel() + try: + await task + except asyncio.CancelledError: + pass + + asyncio.run(_drive()) + assert manifests and not saved.exists() + assert kw._kv_resume is None + + +def test_disabling_idle_unload_purges_saved_kv(monkeypatch, tmp_path): + # PUT leaves keep-KV on but makes idle unload inactive: saved KV must go too. + import routes.settings as settings_route + from core.inference import llama_keepwarm as kw + + saved = tmp_path / "resume-abc-slot0.bin" + saved.write_bytes(b"kv") + kw._kv_resume = { + "identity": ("m", None, "m"), + "dir": str(tmp_path), + "slots": [{"id": 0, "filename": saved.name}], + } + monkeypatch.setattr(settings_route, "set_openai_auto_switch", lambda *a: (False, 300, True)) + monkeypatch.setattr(settings_route, "get_auto_unload_idle_seconds", lambda: 0) + + payload = settings_route.OpenAIAutoSwitchPayload(enabled = False) + resp = settings_route.update_openai_auto_switch(payload, "tester") + assert resp.idle_unload_active is False and resp.auto_unload_keep_kv is True + assert kw._kv_resume is None and not saved.exists() + + def test_audio_generate_is_tracked_as_inference_path(): # Direct GGUF TTS uses the llama backend and can outlive the idle TTL, so # the keep-warm middleware must count it as in-flight inference. @@ -689,7 +789,7 @@ def test_v1_models_retrieve_is_case_insensitive(monkeypatch): def test_index_excludes_hidden_models(tmp_path, monkeypatch): # The llama.cpp validation probe and RAG embedding weights are hidden from - # Studio's pickers; they must never become auto-switch targets. + # Unsloth's pickers; they must never become auto-switch targets. from types import SimpleNamespace import routes.models as models_route @@ -697,6 +797,10 @@ def test_index_excludes_hidden_models(tmp_path, monkeypatch): normal.write_bytes(b"x" * 32) probe = tmp_path / "stories260K.gguf" # llama.cpp install-validation probe probe.write_bytes(b"x" * 32) + embedder = tmp_path / "embedding-Q8_0.gguf" + embedder.write_bytes(b"x" * 32) + local_default_embedder = tmp_path / "bge-small-en-v1.5-F16.gguf" + local_default_embedder.write_bytes(b"x" * 32) def _info(mid, path): return SimpleNamespace(id = mid, path = str(path), model_id = mid, display_name = mid) @@ -704,7 +808,22 @@ def test_index_excludes_hidden_models(tmp_path, monkeypatch): monkeypatch.setattr( models_route, "_scan_models_dir", - lambda *a, **k: [_info("org/Normal-GGUF", normal), _info("ggml-org/models", probe)], + lambda *a, **k: [ + _info("org/Normal-GGUF", normal), + _info("ggml-org/models", probe), + SimpleNamespace( + id = str(embedder), + path = str(embedder), + model_id = "unsloth/bge-small-en-v1.5-GGUF", + display_name = "embedding-Q8_0", + ), + SimpleNamespace( + id = str(local_default_embedder), + path = str(local_default_embedder), + model_id = None, + display_name = local_default_embedder.name, + ), + ], ) monkeypatch.setattr(models_route, "_scan_hf_cache", lambda *a, **k: []) monkeypatch.setattr(models_route, "_resolve_hf_cache_dir", lambda: tmp_path) @@ -713,6 +832,8 @@ def test_index_excludes_hidden_models(tmp_path, monkeypatch): index = resolver._index() assert "org/normal-gguf" in index # keys are normalized to lowercase assert "ggml-org/models" not in index + assert "unsloth/bge-small-en-v1.5-gguf" not in index + assert str(local_default_embedder).lower() not in index # And the hidden probe cannot be auto-switched to by name. resolver._scan = (0.0, {}) assert resolver.resolve_local_gguf("ggml-org/models") is None @@ -975,6 +1096,7 @@ def test_build_index_covers_legacy_default_lmstudio_and_custom_roots(monkeypatch from pathlib import Path import routes.models as models_route from utils import paths as upaths + from utils import hf_cache_settings import storage.studio_db as studio_db scanned = [] @@ -995,13 +1117,18 @@ def test_build_index_covers_legacy_default_lmstudio_and_custom_roots(monkeypatch ) monkeypatch.setattr(models_route, "_resolve_hf_cache_dir", lambda: tmp_path / "active") monkeypatch.setattr(models_route, "_is_hidden_model", lambda *a, **k: False) + monkeypatch.setattr( + hf_cache_settings, + "known_hf_hub_caches", + lambda: [tmp_path / "active", tmp_path / "previous"], + ) monkeypatch.setattr(upaths, "legacy_hf_cache_dir", lambda: tmp_path / "legacy") monkeypatch.setattr(upaths, "hf_default_cache_dir", lambda: tmp_path / "default") monkeypatch.setattr(upaths, "lmstudio_model_dirs", lambda: [tmp_path / "lmstudio"]) monkeypatch.setattr( studio_db, "list_scan_folders", lambda: [{"path": str(tmp_path / "custom")}] ) - for sub in ("active", "legacy", "default", "lmstudio", "custom"): + for sub in ("active", "previous", "legacy", "default", "lmstudio", "custom"): (tmp_path / sub).mkdir() resolver._build_index() @@ -1010,6 +1137,7 @@ def test_build_index_covers_legacy_default_lmstudio_and_custom_roots(monkeypatch lm = {p for k, p in scanned if k == "lm"} assert str((tmp_path / "legacy").resolve()) in hf assert str((tmp_path / "default").resolve()) in hf + assert str((tmp_path / "previous").resolve()) in hf assert str((tmp_path / "custom").resolve()) in hf assert str((tmp_path / "lmstudio").resolve()) in lm @@ -1089,10 +1217,9 @@ def test_middleware_ignores_non_post(monkeypatch): # ── review round 4: swap guard, idle variant identity, load-by-path, stash clear ── -def test_auto_switch_refuses_when_another_inference_is_active(monkeypatch): - # A cross-model swap must 409 (not kill) while another inference request is in - # flight; the requesting call itself is excluded from the count. - from fastapi import HTTPException +def test_auto_switch_waits_for_another_inference_to_finish(monkeypatch): + # A cross-model swap queues while another request is generating, then loads + # after that request drains. The requesting call itself is excluded. from core.inference import llama_keepwarm as kw backend = _FakeBackend("org/A-GGUF", hf_variant = "Q4_K_M") @@ -1106,10 +1233,18 @@ def test_auto_switch_refuses_when_another_inference_is_active(monkeypatch): ) monkeypatch.setattr(kw, "_inflight", 2) # this request + another active one monkeypatch.setattr(kw, "_pending", 0) - with pytest.raises(HTTPException) as exc: - _run_hook("org/B-GGUF:Q8_0") - assert exc.value.status_code == 409 - assert rec.calls == [] + + async def _drive(): + task = asyncio.create_task( + inference_route._maybe_auto_switch_model("org/B-GGUF:Q8_0", object(), "tester") + ) + await asyncio.sleep(0.05) + assert rec.calls == [] + kw._note_end() # the other generation finishes; this request remains counted + await asyncio.wait_for(task, timeout = 1) + + asyncio.run(_drive()) + assert len(rec.calls) == 1 def test_auto_switch_swaps_when_only_caller_is_active(monkeypatch): @@ -1295,13 +1430,12 @@ def test_concurrent_same_target_requests_load_once(monkeypatch): monkeypatch.setattr(kw, "_pending", 0) inference_route._note_switch_waiter(inference_route._switch_key("org/B-GGUF", "Q8_0"), 1) _run_hook("org/B-GGUF:Q8_0") - assert len(rec.calls) == 1 # loads once, no 409 + assert len(rec.calls) == 1 -def test_swap_still_refused_when_other_request_targets_different_model(monkeypatch): - # A concurrent request heading to a different target still blocks the swap: the - # same-target exclusion must not swallow a genuinely conflicting request. - from fastapi import HTTPException +def test_queued_different_target_does_not_deadlock_current_swap(monkeypatch): + # A concurrent request already queued for another target is not generating, + # so it must not prevent the current serialized swap from proceeding. from core.inference import llama_keepwarm as kw backend = _FakeBackend("org/A-GGUF") @@ -1316,10 +1450,8 @@ def test_swap_still_refused_when_other_request_targets_different_model(monkeypat monkeypatch.setattr(kw, "_inflight", 2) monkeypatch.setattr(kw, "_pending", 0) inference_route._note_switch_waiter(inference_route._switch_key("org/C-GGUF", "Q4_K_M"), 1) - with pytest.raises(HTTPException) as exc: - _run_hook("org/B-GGUF:Q8_0") - assert exc.value.status_code == 409 - assert rec.calls == [] + _run_hook("org/B-GGUF:Q8_0") + assert len(rec.calls) == 1 def test_v1_models_advertises_repo_id_not_load_path(monkeypatch): @@ -1365,6 +1497,37 @@ def test_load_route_holds_lifecycle_gate(monkeypatch): assert "_load_model_impl" in src +def test_model_replacements_recheck_sidecar_swap_before_either_backend_is_unloaded(): + # Both replacement directions drain active inference, then recheck whether a + # sidecar install reserved the lifecycle gate during that wait. Exact-model + # reuse exits earlier, so an already-loaded model never waits on unrelated inference. + import inspect + + src = inspect.getsource(inference_route._load_model_impl) + gguf_wait = src.index("await _wait_for_model_switch_idle", src.index("if config.is_gguf:")) + gguf_sidecar_check = src.index("_raise_if_sidecar_swap_in_progress()", gguf_wait) + unload_unsloth = src.index("unsloth_backend.unload_model", gguf_wait) + standard_wait = src.index("await _wait_for_model_switch_idle", gguf_wait + 1) + standard_sidecar_check = src.index("_raise_if_sidecar_swap_in_progress()", standard_wait) + unload_gguf = src.index("llama_backend.unload_model()", standard_wait) + already_loaded = src.index('status = "already_loaded"') + + assert already_loaded < gguf_wait < gguf_sidecar_check < unload_unsloth + assert standard_wait < standard_sidecar_check < unload_gguf + + +def test_switch_waiter_deregisters_before_swap_gate_release(): + # A waiter left registered after the swap gate is released would let a swap on + # another event loop count the finished request as still queued, pass the drain + # early, and unload the model that request is about to generate against. + import inspect + + src = inspect.getsource(inference_route._maybe_auto_switch_model) + deregister = src.index("_note_switch_waiter(key, -1)") + release = src.index("_auto_switch_process_lock.release()") + assert deregister < release + + def _anthropic_payload(max_tokens = None): from models.inference import AnthropicMessagesRequest, AnthropicMessage return AnthropicMessagesRequest( @@ -1403,9 +1566,9 @@ def test_anthropic_400_when_auto_switch_on_and_max_tokens_missing(monkeypatch): # ── review round 6: concurrency ordering, external untrack, unload gate, ids ── -def test_pending_same_target_request_does_not_force_409(monkeypatch): +def test_pending_same_target_request_does_not_block_swap(monkeypatch): # A second same-target request blocked in the middleware (pending, not yet - # generating) must not make the first request 409: pending is excluded. + # generating) must not block the first request: pending is excluded. from core.inference import llama_keepwarm as kw backend = _FakeBackend("org/A-GGUF") @@ -1420,13 +1583,13 @@ def test_pending_same_target_request_does_not_force_409(monkeypatch): monkeypatch.setattr(kw, "_inflight", 1) # just the caller monkeypatch.setattr(kw, "_pending", 1) # second request blocked in middleware _run_hook("org/B-GGUF:Q8_0") - assert len(rec.calls) == 1 # loads once, no 409 + assert len(rec.calls) == 1 -def test_concurrent_same_target_loads_once_while_other_still_resolving(monkeypatch): +def test_swap_waits_until_concurrent_request_finishes_resolving(monkeypatch): # The real middleware counts a concurrent same-model request as in-flight - # before it resolves and registers a target waiter. The raw-request waiter, - # registered before resolve, must still exclude it so the first request loads. + # before it resolves and registers a target waiter. Treat it as active until + # its target is known, then recognize it as another queued switch request. from core.inference import llama_keepwarm as kw backend = _FakeBackend("org/A-GGUF") @@ -1440,10 +1603,20 @@ def test_concurrent_same_target_loads_once_while_other_still_resolving(monkeypat ) monkeypatch.setattr(kw, "_inflight", 2) # caller + a still-resolving twin monkeypatch.setattr(kw, "_pending", 0) - # The twin has only registered its raw requested model (not yet a target waiter). - inference_route._note_request_waiter(inference_route._request_waiter_key("org/B-GGUF:Q8_0"), 1) - _run_hook("org/B-GGUF:Q8_0") - assert len(rec.calls) == 1 # loads once, no 409 + # The twin is still resolving, so it is counted in-flight but has not joined + # the concrete target queue yet. + + async def _drive(): + task = asyncio.create_task( + inference_route._maybe_auto_switch_model("org/B-GGUF:Q8_0", object(), "tester") + ) + await asyncio.sleep(0.05) + assert rec.calls == [] + inference_route._note_switch_waiter(inference_route._switch_key("org/B-GGUF", "Q8_0"), 1) + await asyncio.wait_for(task, timeout = 1) + + asyncio.run(_drive()) + assert len(rec.calls) == 1 def test_external_untrack_decrements_inflight_and_is_idempotent(): @@ -1479,11 +1652,9 @@ def test_manual_unload_interrupts_even_while_inference_active(monkeypatch): assert not backend.is_loaded # torn down despite the active request -def test_auto_switch_refuses_when_unsloth_stream_active(monkeypatch): +def test_auto_switch_waits_when_unsloth_stream_active(monkeypatch): # The GGUF slot is empty but an Unsloth model is streaming (counted in-flight). - # _load_model_impl would unload it, so auto-switch must 409, not only when a - # GGUF is loaded. - from fastapi import HTTPException + # The replacement waits for it just as it does for a GGUF generation. from core.inference import llama_keepwarm as kw backend = _FakeBackend(None) # no GGUF loaded @@ -1497,10 +1668,18 @@ def test_auto_switch_refuses_when_unsloth_stream_active(monkeypatch): ) monkeypatch.setattr(kw, "_inflight", 2) # an Unsloth stream + this request monkeypatch.setattr(kw, "_pending", 0) - with pytest.raises(HTTPException) as exc: - _run_hook("org/B-GGUF:Q8_0") - assert exc.value.status_code == 409 - assert rec.calls == [] # the active Unsloth model is not torn down + + async def _drive(): + task = asyncio.create_task( + inference_route._maybe_auto_switch_model("org/B-GGUF:Q8_0", object(), "tester") + ) + await asyncio.sleep(0.05) + assert rec.calls == [] + kw._note_end() + await asyncio.wait_for(task, timeout = 1) + + asyncio.run(_drive()) + assert len(rec.calls) == 1 def test_public_model_id_prefers_advertised_over_path(): @@ -1609,11 +1788,11 @@ def test_env_idle_ttl_standalone_when_no_stored_value(monkeypatch): def test_stored_idle_value_overrides_env_and_stays_gated(monkeypatch): # An explicit stored value wins over the env default and remains gated on the # auto-switch toggle. - store = {settings.AUTO_UNLOAD_IDLE_SETTING_KEY: 30} + store = {settings.AUTO_UNLOAD_IDLE_SETTING_KEY: 90} monkeypatch.setattr(settings, "_cached_setting", lambda k, d = None: store.get(k, d)) monkeypatch.setenv("UNSLOTH_MODEL_IDLE_TTL", "600") monkeypatch.setattr(settings, "get_openai_auto_switch_enabled", lambda: True) - assert settings.get_auto_unload_idle_seconds() == 30 # stored wins, not env + assert settings.get_auto_unload_idle_seconds() == 90 # stored wins, not env monkeypatch.setattr(settings, "get_openai_auto_switch_enabled", lambda: False) assert settings.get_auto_unload_idle_seconds() == 0 # explicit value still gated off @@ -1729,6 +1908,8 @@ def test_index_advertises_alias_not_filesystem_path(tmp_path, monkeypatch): # host path in /v1/models, yet the model stays resolvable by that path too. from types import SimpleNamespace import routes.models as models_route + from storage import studio_db + import utils.paths as paths gguf = tmp_path / "model-Q4_K_M.gguf" gguf.write_bytes(b"x" * 32) @@ -1742,6 +1923,8 @@ def test_index_advertises_alias_not_filesystem_path(tmp_path, monkeypatch): monkeypatch.setattr(models_route, "_scan_hf_cache", lambda *a, **k: []) monkeypatch.setattr(models_route, "_resolve_hf_cache_dir", lambda: tmp_path) monkeypatch.setattr(models_route, "_is_hidden_model", lambda *a, **k: False) + monkeypatch.setattr(paths, "lmstudio_model_dirs", lambda: []) + monkeypatch.setattr(studio_db, "list_scan_folders", lambda: []) resolver._scan = (0.0, {}) # The advertised id is the alias, never the absolute path. @@ -2887,8 +3070,10 @@ def test_non_gguf_load_clears_reload_stash(): # A non-GGUF (Transformers/Unsloth) load must clear the stash like the GGUF # branch, so it never lingers until the idle poll (or forever, idle-unload off). import inspect + src = inspect.getsource(inference_route._load_model_impl) - assert src.count("note_model_loaded()") >= 2 + assert src.count("note_model_loaded()") >= 1 # non-GGUF branch + assert "to_thread(note_model_loaded, llama_backend)" in src # GGUF branch def test_chat_rejects_malformed_tool_choice_before_switch(monkeypatch): @@ -2975,6 +3160,8 @@ def test_auto_switch_serializes_across_event_loops(monkeypatch): request, fastapi_request, current_subject = None, + *, + current_request_counted = False, ): with slock: state["cur"] += 1 @@ -2992,7 +3179,6 @@ def test_auto_switch_serializes_across_event_loops(monkeypatch): monkeypatch.setattr(inference_route, "get_llama_cpp_backend", lambda: backend) monkeypatch.setattr(inference_route, "_load_model_impl", _slow_load) monkeypatch.setattr(inference_route, "_auto_switch_waiters", {}) - monkeypatch.setattr(inference_route, "_auto_switch_request_waiters", {}) barrier = threading.Barrier(2) @@ -3094,3 +3280,542 @@ def test_responses_stream_hint_matches_toggle_regardless_of_active_model(monkeyp monkeypatch, enabled = False, active_model_name = "unsloth/Llama-3.2-1B-Instruct" ) assert "Model auto-switch" in non_gguf_loaded + + +# ── idle-unload KV persistence (slot save/restore) ────────────────── + + +def _seed_kv_manifest( + tmp_path, + identity = ("unsloth/A-GGUF", "Q4_K_M", "unsloth/A-GGUF"), + gguf = None, +): + if gguf is None: + gguf_file = tmp_path / "model.gguf" + gguf_file.write_bytes(b"gguf") + gguf = str(gguf_file) + st = os.stat(gguf) + state_file = tmp_path / "resume-abc-slot0.bin" + state_file.write_bytes(b"kv") + return state_file, { + "identity": identity, + "dir": str(tmp_path), + "binary": ("/bin/llama-server", 111), + "gguf": gguf, + "gguf_stat": ((st.st_size, st.st_mtime_ns),), + "launch": ((), None, None, 1), + "slots": [{"id": 0, "filename": state_file.name, "n_saved": 42}], + } + + +def _drive_idle_loop( + kw, + poll_seconds = 0.02, + run_for = 0.2, +): + async def _drive(): + task = asyncio.create_task(kw.idle_unload_loop(poll_seconds = poll_seconds)) + await asyncio.sleep(run_for) + task.cancel() + try: + await task + except asyncio.CancelledError: + pass + + asyncio.run(_drive()) + + +def test_idle_unload_saves_slots_before_unload_and_stashes_manifest(monkeypatch, tmp_path): + import time + from core.inference import llama_keepwarm as kw + + monkeypatch.setattr(settings, "get_auto_unload_idle_seconds", lambda: 0.005) + monkeypatch.setattr(settings, "get_auto_unload_keep_kv", lambda: True) + kw._inflight = 0 + kw._pending = 0 + kw._last_active = time.monotonic() - 3600 + kw._last_unloaded_model = None + kw._kv_resume = None + + events = [] + backend = _FakeBackend("unsloth/Idle-GGUF", hf_variant = "Q4_K_M") + manifest = { + "dir": str(tmp_path), + "binary": ("bin", 1), + "slots": [{"id": 0, "filename": "f.bin", "n_saved": 42}], + } + + def _save(should_abort = None): + events.append("save") + return manifest + + def _unload(): + events.append("unload") + backend.is_loaded = False + + backend.save_slots_for_resume = _save + backend.unload_model = _unload + monkeypatch.setattr(inference_route, "get_llama_cpp_backend", lambda: backend) + + _drive_idle_loop(kw) + # KV must be saved while the server is still alive, then exactly one unload. + assert events == ["save", "unload"] + assert kw.get_last_unloaded_model()[:2] == ("unsloth/Idle-GGUF", "Q4_K_M") + resume = kw.take_kv_resume() + assert resume is not None + assert resume["identity"][:2] == ("unsloth/Idle-GGUF", "Q4_K_M") + assert resume["slots"][0]["filename"] == "f.bin" + + +def test_idle_save_failure_still_unloads_plain(monkeypatch): + import time + from core.inference import llama_keepwarm as kw + + monkeypatch.setattr(settings, "get_auto_unload_idle_seconds", lambda: 0.005) + monkeypatch.setattr(settings, "get_auto_unload_keep_kv", lambda: True) + kw._inflight = 0 + kw._pending = 0 + kw._last_active = time.monotonic() - 3600 + kw._last_unloaded_model = None + kw._kv_resume = None + + unloads = [] + backend = _FakeBackend("unsloth/Idle-GGUF", hf_variant = "Q4_K_M") + + def _save(should_abort = None): + raise RuntimeError("slot save exploded") + + def _unload(): + unloads.append(1) + backend.is_loaded = False + + backend.save_slots_for_resume = _save + backend.unload_model = _unload + monkeypatch.setattr(inference_route, "get_llama_cpp_backend", lambda: backend) + + _drive_idle_loop(kw) + assert unloads == [1] # the save failure must not skip the unload + assert kw.get_last_unloaded_model() is not None + assert kw.take_kv_resume() is None + + +def test_keep_kv_setting_off_skips_save(monkeypatch): + import time + from core.inference import llama_keepwarm as kw + + monkeypatch.setattr(settings, "get_auto_unload_idle_seconds", lambda: 0.005) + monkeypatch.setattr(settings, "get_auto_unload_keep_kv", lambda: False) + kw._inflight = 0 + kw._pending = 0 + kw._last_active = time.monotonic() - 3600 + kw._last_unloaded_model = None + kw._kv_resume = None + + saves, unloads = [], [] + backend = _FakeBackend("unsloth/Idle-GGUF") + + def _unload(): + unloads.append(1) + backend.is_loaded = False + + backend.save_slots_for_resume = lambda *a, **k: saves.append(1) + backend.unload_model = _unload + monkeypatch.setattr(inference_route, "get_llama_cpp_backend", lambda: backend) + + _drive_idle_loop(kw) + assert saves == [] + assert unloads == [1] + assert kw.take_kv_resume() is None + + +def test_keep_kv_disabled_mid_save_discards_manifest(monkeypatch, tmp_path): + import time + from core.inference import llama_keepwarm as kw + + keep = {"on": True} + monkeypatch.setattr(settings, "get_auto_unload_idle_seconds", lambda: 0.005) + monkeypatch.setattr(settings, "get_auto_unload_keep_kv", lambda: keep["on"]) + kw._inflight = 0 + kw._pending = 0 + kw._last_active = time.monotonic() - 3600 + kw._last_unloaded_model = None + kw._kv_resume = None + + unloads = [] + backend = _FakeBackend("unsloth/Idle-GGUF", hf_variant = "Q4_K_M") + state_file = tmp_path / "resume-mid-slot0.bin" + state_file.write_bytes(b"kv") + manifest = { + "dir": str(tmp_path), + "binary": ("bin", 1), + "slots": [{"id": 0, "filename": state_file.name, "n_saved": 1}], + } + + def _save(should_abort = None): + keep["on"] = False # user flips the toggle while the save runs + return manifest + + def _unload(): + unloads.append(1) + backend.is_loaded = False + + backend.save_slots_for_resume = _save + backend.unload_model = _unload + monkeypatch.setattr(inference_route, "get_llama_cpp_backend", lambda: backend) + + _drive_idle_loop(kw) + assert unloads == [1] # still unloads; only the stash is dropped + assert kw.take_kv_resume() is None + assert not state_file.exists() + + +def test_idle_ttl_disabled_mid_save_skips_unload(monkeypatch, tmp_path): + import time + from core.inference import llama_keepwarm as kw + + ttl = {"v": 0.005} + monkeypatch.setattr(settings, "get_auto_unload_idle_seconds", lambda: ttl["v"]) + monkeypatch.setattr(settings, "get_auto_unload_keep_kv", lambda: True) + kw._inflight = 0 + kw._pending = 0 + kw._last_active = time.monotonic() - 3600 + kw._last_unloaded_model = None + kw._kv_resume = None + + unloads = [] + backend = _FakeBackend("unsloth/Idle-GGUF", hf_variant = "Q4_K_M") + state_file = tmp_path / "resume-mid-slot0.bin" + state_file.write_bytes(b"kv") + manifest = { + "dir": str(tmp_path), + "binary": ("bin", 1), + "slots": [{"id": 0, "filename": state_file.name, "n_saved": 1}], + } + + def _save(should_abort = None): + ttl["v"] = 0 # user turns idle unload off while the save runs + return manifest + + backend.save_slots_for_resume = _save + backend.unload_model = lambda: unloads.append(1) + monkeypatch.setattr(inference_route, "get_llama_cpp_backend", lambda: backend) + + _drive_idle_loop(kw) + assert unloads == [] # the unload was cancelled by the setting change + assert kw.take_kv_resume() is None + assert not state_file.exists() + + +def test_alias_reload_restores_slots_and_deletes_files(monkeypatch, tmp_path): + from core.inference import llama_keepwarm as kw + + backend = _FakeBackend(None) # idle-unload emptied the backend + backend._slot_save_binary = ("/bin/llama-server", 111) + restored = [] + backend.restore_slots_for_resume = lambda manifest: restored.append(manifest) + + rec = _LoadRecorder(backend) + _wire(monkeypatch, enabled = True, resolves_to = None, backend = backend, recorder = rec) + monkeypatch.setattr(kw, "_inflight", 0) + state_file, manifest = _seed_kv_manifest(tmp_path) + monkeypatch.setattr(kw, "_last_unloaded_model", (manifest["gguf"], "Q4_K_M")) + monkeypatch.setattr(kw, "_kv_resume", manifest) + + _run_hook("gpt-4o-mini") + assert len(rec.calls) == 1 + assert len(restored) == 1 # same model + binary: restore ran + assert not state_file.exists() # state file deleted after the restore + assert kw._kv_resume is None + + +def test_no_restore_when_different_model_loads(monkeypatch, tmp_path): + from core.inference import llama_keepwarm as kw + + backend = _FakeBackend(None) + backend._slot_save_binary = ("/bin/llama-server", 111) + restored = [] + backend.restore_slots_for_resume = lambda manifest: restored.append(manifest) + rec = _LoadRecorder(backend) + _wire( + monkeypatch, + enabled = True, + resolves_to = ("unsloth/B-GGUF", None, "unsloth/B-GGUF"), + backend = backend, + recorder = rec, + ) + monkeypatch.setattr(kw, "_inflight", 0) + state_file, manifest = _seed_kv_manifest(tmp_path) # manifest is for model A + monkeypatch.setattr(kw, "_kv_resume", manifest) + + _run_hook("unsloth/B-GGUF") + assert len(rec.calls) == 1 + assert restored == [] # different model: never restored + assert not state_file.exists() # but the stale files are gone + assert kw._kv_resume is None + + +def test_restore_skipped_when_binary_changed(monkeypatch, tmp_path): + from core.inference import llama_keepwarm as kw + + state_file, manifest = _seed_kv_manifest(tmp_path) + backend = _FakeBackend("unsloth/A-GGUF", hf_variant = "Q4_K_M") + backend._gguf_path = manifest["gguf"] + backend._slot_save_binary = ("/bin/llama-server", 222) # newer mtime + restored = [] + backend.restore_slots_for_resume = lambda manifest: restored.append(manifest) + + kw.restore_kv_resume(backend, manifest) + assert restored == [] + assert not state_file.exists() + + +def test_restore_skipped_when_launch_config_changed(tmp_path): + from core.inference import llama_keepwarm as kw + + state_file, manifest = _seed_kv_manifest(tmp_path) + backend = _FakeBackend("unsloth/A-GGUF", hf_variant = "Q4_K_M") + backend._gguf_path = manifest["gguf"] + backend._slot_save_binary = ("/bin/llama-server", 111) + backend._slot_launch_fingerprint = lambda: (("--rope-freq-scale", "0.5"), None, None, 1) + restored = [] + backend.restore_slots_for_resume = lambda manifest: restored.append(manifest) + + kw.restore_kv_resume(backend, manifest) + assert restored == [] + assert not state_file.exists() + + +def test_restore_skipped_when_gguf_rewritten_in_place(tmp_path): + from core.inference import llama_keepwarm as kw + + state_file, manifest = _seed_kv_manifest(tmp_path) + with open(manifest["gguf"], "wb") as fh: + fh.write(b"different weights") # same path, new content + backend = _FakeBackend("unsloth/A-GGUF", hf_variant = "Q4_K_M") + backend._gguf_path = manifest["gguf"] + backend._slot_save_binary = ("/bin/llama-server", 111) + restored = [] + backend.restore_slots_for_resume = lambda manifest: restored.append(manifest) + + kw.restore_kv_resume(backend, manifest) + assert restored == [] + assert not state_file.exists() + + +def test_note_model_unloaded_purges_manifest_and_files(tmp_path): + from core.inference import llama_keepwarm as kw + + state_file, manifest = _seed_kv_manifest(tmp_path) + kw._set_last_unloaded(("org/A-GGUF", "Q4_K_M")) + kw._set_kv_resume(manifest) + kw.note_model_unloaded() + assert kw.get_last_unloaded_model() is None + assert kw.take_kv_resume() is None + assert not state_file.exists() + + +def test_note_model_loaded_purges_manifest_and_files(tmp_path): + from core.inference import llama_keepwarm as kw + + state_file, manifest = _seed_kv_manifest(tmp_path) + kw._set_last_unloaded(("org/A-GGUF", "Q4_K_M")) + kw._set_kv_resume(manifest) + kw.note_model_loaded() + assert kw.get_last_unloaded_model() is None + assert kw.take_kv_resume() is None + assert not state_file.exists() + + +def test_new_idle_save_purges_previous_manifest_files(tmp_path): + from core.inference import llama_keepwarm as kw + + old_file, old_manifest = _seed_kv_manifest(tmp_path) + kw._set_kv_resume(old_manifest) + new_file = tmp_path / "resume-def-slot0.bin" + new_file.write_bytes(b"kv2") + kw._set_kv_resume( + { + "identity": ("unsloth/B-GGUF", None, "unsloth/B-GGUF"), + "dir": str(tmp_path), + "binary": ("/bin/llama-server", 111), + "slots": [{"id": 0, "filename": new_file.name, "n_saved": 7}], + } + ) + assert not old_file.exists() # replaced manifest's files purged + assert new_file.exists() + assert kw.take_kv_resume()["slots"][0]["filename"] == new_file.name + + +def test_sweep_slot_save_dir_removes_only_resume_files(monkeypatch, tmp_path): + from core.inference import llama_keepwarm as kw + from utils.paths import storage_roots + + monkeypatch.setattr(storage_roots, "llama_slot_cache_root", lambda: tmp_path) + stale = tmp_path / "resume-old-slot0.bin" + stale.write_bytes(b"kv") + other = tmp_path / "unrelated.txt" + other.write_text("keep") + kw.sweep_slot_save_dir() + assert not stale.exists() + assert other.exists() + + +def test_keep_kv_setting_roundtrip_and_default(monkeypatch): + import storage.studio_db as db + + store = {} + monkeypatch.setattr(db, "upsert_app_settings", lambda m: store.update(m)) + monkeypatch.setattr(settings, "_cached_setting", lambda k, d = None: store.get(k, d)) + + assert settings.get_auto_unload_keep_kv() is True # default when never stored + assert settings.set_openai_auto_switch(True, 60, False)[2] is False + assert store[settings.AUTO_UNLOAD_KEEP_KV_SETTING_KEY] is False + assert settings.get_auto_unload_keep_kv() is False + # None leaves the stored value untouched (older clients can't reset it). + assert settings.set_openai_auto_switch(True, 60, None)[2] is False + assert store[settings.AUTO_UNLOAD_KEEP_KV_SETTING_KEY] is False + with pytest.raises(ValueError, match = "true or false"): + settings.set_openai_auto_switch(True, 60, "garbage") + + +def test_stale_stash_cleanup_waits_for_lifecycle_gate(monkeypatch, tmp_path): + # The loop's stale-stash purge must wait on the gate a mid-reload holds. + import time + from core.inference import llama_keepwarm as kw + + monkeypatch.setattr(settings, "get_auto_unload_idle_seconds", lambda: 3600) + kw._inflight = 0 + kw._pending = 0 + kw._last_active = time.monotonic() + backend = _FakeBackend("unsloth/New-GGUF") + monkeypatch.setattr(inference_route, "get_llama_cpp_backend", lambda: backend) + state_file, manifest = _seed_kv_manifest(tmp_path) + kw._kv_resume = manifest + kw._last_unloaded_model = ("unsloth/A-GGUF", "Q4_K_M") + + assert kw._lifecycle_lock.acquire(blocking = False) # simulate in-flight reload + try: + _drive_idle_loop(kw) + assert kw._kv_resume is manifest # purge deferred while the gate is held + assert state_file.exists() + finally: + kw._lifecycle_lock.release() + _drive_idle_loop(kw) + assert kw._kv_resume is None # gate freed: genuinely stale stash purged + assert not state_file.exists() + + +def test_put_route_disabling_keep_kv_purges_saved_state(monkeypatch, tmp_path): + import routes.settings as settings_route + import storage.studio_db as db + from core.inference import llama_keepwarm as kw + + store = {} + monkeypatch.setattr(db, "upsert_app_settings", lambda m: store.update(m)) + monkeypatch.setattr(settings, "_cached_setting", lambda k, d = None: store.get(k, d)) + state_file, manifest = _seed_kv_manifest(tmp_path) + monkeypatch.setattr(kw, "_kv_resume", manifest) + + payload = settings_route.OpenAIAutoSwitchPayload(enabled = True, auto_unload_keep_kv = False) + resp = settings_route.update_openai_auto_switch(payload, "tester") + assert resp.auto_unload_keep_kv is False + assert kw._kv_resume is None + assert not state_file.exists() + + +def test_keep_kv_only_update_leaves_env_idle_ttl_active(monkeypatch): + # A keep-KV-only update must not materialize the env TTL as a stored value. + import routes.settings as settings_route + import storage.studio_db as db + + store = {} + monkeypatch.setattr(db, "upsert_app_settings", lambda m: store.update(m)) + monkeypatch.setattr(settings, "_cached_setting", lambda k, d = None: store.get(k, d)) + monkeypatch.setenv(settings.MODEL_IDLE_TTL_ENV_VAR, "600") + + assert settings_route.OpenAIAutoSwitchPayload(enabled = False).auto_unload_idle_seconds is None + enabled, idle, keep_kv = settings.set_openai_auto_switch(False, None, False) + assert settings.AUTO_UNLOAD_IDLE_SETTING_KEY not in store # idle untouched + assert settings.get_auto_unload_idle_seconds() == 600 # env TTL still active + assert (enabled, idle, keep_kv) == (False, 600, False) + + +def test_load_impl_notes_loaded_with_backend_off_loop(): + import inspect + src = inspect.getsource(inference_route._load_model_impl) + assert "to_thread(note_model_loaded, llama_backend)" in src + + +def test_restore_matches_gguf_realpath_across_naming(tmp_path): + from core.inference import llama_keepwarm as kw + + blob = tmp_path / "blob.gguf" + blob.write_bytes(b"gguf") + link = tmp_path / "snapshot.gguf" + try: + link.symlink_to(blob) + except OSError: + pytest.skip("symlinks unsupported on this host") + + backend = _FakeBackend("/hf/snapshots/d7f5", hf_variant = None) + backend._gguf_path = str(link) # reload resolved the symlink spelling + backend._slot_save_binary = ("/bin/llama-server", 111) + restored = [] + backend.restore_slots_for_resume = lambda manifest: restored.append(manifest) + state_file, manifest = _seed_kv_manifest( + tmp_path, identity = ("unsloth/A-GGUF", None, "unsloth/A-GGUF"), gguf = str(blob) + ) + + kw.restore_kv_resume(backend, manifest) + assert len(restored) == 1 # names differ, file identical: restore ran + assert not state_file.exists() + + +def test_setter_rejects_idle_below_floor(monkeypatch): + import storage.studio_db as db + + writes = [] + monkeypatch.setattr(db, "upsert_app_settings", lambda m: writes.append(dict(m))) + settings._cache.clear() + + with pytest.raises(ValueError, match = "at least 60"): + settings.set_openai_auto_switch(True, 30) + assert writes == [] # rejected before any persist + # 0 (off) and >= 60 pass through unchanged. + assert settings.set_openai_auto_switch(True, 0)[1] == 0 + assert settings.set_openai_auto_switch(True, 60)[1] == 60 + assert settings.set_openai_auto_switch(True, 3600)[1] == 3600 + + +def test_put_route_rejects_idle_below_floor(): + import routes.settings as settings_route + from fastapi import HTTPException + + payload = settings_route.OpenAIAutoSwitchPayload(enabled = True, auto_unload_idle_seconds = 30) + with pytest.raises(HTTPException) as excinfo: + settings_route.update_openai_auto_switch(payload, "tester") + assert excinfo.value.status_code == 400 + + +def test_stored_legacy_idle_below_floor_is_clamped(monkeypatch): + # Values persisted before the floor existed are raised to it on read, for + # both the effective TTL and the value the settings UI displays. + store = {settings.AUTO_UNLOAD_IDLE_SETTING_KEY: 5} + monkeypatch.setattr(settings, "_cached_setting", lambda k, d = None: store.get(k, d)) + monkeypatch.setattr(settings, "get_openai_auto_switch_enabled", lambda: True) + assert settings.get_auto_unload_idle_seconds() == 60 + assert settings.get_stored_auto_unload_idle_seconds() == 60 + store[settings.AUTO_UNLOAD_IDLE_SETTING_KEY] = 90 + assert settings.get_auto_unload_idle_seconds() == 90 + + +def test_env_idle_below_floor_is_clamped(monkeypatch): + monkeypatch.setattr(settings, "_cached_setting", lambda k, d = None: d) + monkeypatch.setenv(settings.MODEL_IDLE_TTL_ENV_VAR, "5") + assert settings.get_auto_unload_idle_seconds() == 60 + monkeypatch.setenv(settings.MODEL_IDLE_TTL_ENV_VAR, "0") + assert settings.get_auto_unload_idle_seconds() == 0 + monkeypatch.setenv(settings.MODEL_IDLE_TTL_ENV_VAR, "600") + assert settings.get_auto_unload_idle_seconds() == 600 + monkeypatch.delenv(settings.MODEL_IDLE_TTL_ENV_VAR) + assert settings.get_auto_unload_idle_seconds() == 0 diff --git a/studio/backend/tests/test_openai_compaction.py b/studio/backend/tests/test_openai_compaction.py index c7de0a9aed..6fad2c5eaf 100644 --- a/studio/backend/tests/test_openai_compaction.py +++ b/studio/backend/tests/test_openai_compaction.py @@ -86,7 +86,7 @@ def test_cloud_openai_sets_compaction_block(monkeypatch): def test_cloud_openai_below_default_threshold_passes_through(monkeypatch): - # Studio doesn't clamp the OpenAI side -- the API accepts whatever the + # Unsloth doesn't clamp the OpenAI side -- the API accepts whatever the # caller sends, so a small probe like 60k still goes through. captured = _capture( monkeypatch, diff --git a/studio/backend/tests/test_openai_image_generation.py b/studio/backend/tests/test_openai_image_generation.py index ace57588d3..c2eef0381f 100644 --- a/studio/backend/tests/test_openai_image_generation.py +++ b/studio/backend/tests/test_openai_image_generation.py @@ -4,7 +4,7 @@ """Unit tests for OpenAI Responses API image_generation tool wiring. The tool is a server-side Responses-API tool (``{type: "image_generation"}``); -the result comes back as an ``image_generation_call`` output item, which Studio +the result comes back as an ``image_generation_call`` output item, which Unsloth translates into ``_toolEvent`` chunks so the chat adapter renders it inline. Tests pin: the tool is added to the body only on a cloud OpenAI base when asked for, the done event produces the expected chunks, and non-cloud bases drop it. diff --git a/studio/backend/tests/test_openai_tool_passthrough.py b/studio/backend/tests/test_openai_tool_passthrough.py index 5300a48557..161c8743c4 100644 --- a/studio/backend/tests/test_openai_tool_passthrough.py +++ b/studio/backend/tests/test_openai_tool_passthrough.py @@ -119,7 +119,7 @@ class TestFriendlyUpstreamError: raw = '{"error":{"code":400,"message":"Failed to initialize samplers: failed to parse grammar","type":"invalid_request_error"}}' msg = _friendly_upstream_error(raw) assert "failed to parse grammar" not in msg # raw body is not surfaced verbatim - assert "tool-calling grammar" in msg and "Update Studio" in msg + assert "tool-calling grammar" in msg and "Update Unsloth" in msg def test_failed_to_initialize_samplers_alone_matches(self): assert "tool-calling grammar" in _friendly_upstream_error("Failed to initialize samplers") @@ -262,7 +262,7 @@ class TestChatMessageToolRoles: def test_tool_empty_content_accepted(self): # Empty tool output (mkdir, git add, ...) is routine in agentic loops; - # OpenAI and llama-server both accept it, so Studio must not 400. + # OpenAI and llama-server both accept it, so Unsloth must not 400. msg = ChatMessage(role = "tool", tool_call_id = "call_1", content = "") assert msg.content == "" @@ -400,7 +400,7 @@ class TestChatCompletionRequestToolFields: assert req.session_id == "abc" def test_stream_defaults_false_matching_openai_spec(self): - # OpenAI defaults `stream` to false. Studio used to default true, + # OpenAI defaults `stream` to false. Unsloth used to default true, # breaking naive curl/.NET clients (#5047) that omit it. Pin the fix. req = self._make() assert req.stream is False @@ -664,7 +664,7 @@ class TestChatCompletionRequestToolFields: raise AssertionError("client tools must use passthrough") def generate_chat_completion_with_tools(self, **_kwargs): - raise AssertionError("Studio tool loop must stay disabled") + raise AssertionError("Unsloth tool loop must stay disabled") async def fake_passthrough(llama_backend, payload, model_name, **kwargs): captured["body"] = inference_route._build_openai_passthrough_body( @@ -707,11 +707,11 @@ class TestChatCompletionRequestToolFields: assert monitor.active_count() == 0 def test_permission_mode_does_not_reject_client_tool_passthrough(self, monkeypatch): - # A non-streaming client-tool passthrough (client tools, no Studio tool + # A non-streaming client-tool passthrough (client tools, no Unsloth tool # loop) that also carries permission_mode "ask"/"auto" must reach the # provider passthrough, not the confirm-without-stream guard: the # validator leaves confirm_tool_calls unset for passthrough, and a bare - # permission_mode only gates Studio's own local tool loop. An explicit + # permission_mode only gates Unsloth's own local tool loop. An explicit # confirm_tool_calls=True still forces the local-confirm rejection. # The pre-switch guard only runs when an automatic load may run, so force # that predicate on to exercise it against a resident passthrough backend. @@ -732,7 +732,7 @@ class TestChatCompletionRequestToolFields: raise AssertionError("client tools must use passthrough") def generate_chat_completion_with_tools(self, **_kwargs): - raise AssertionError("Studio tool loop must stay disabled") + raise AssertionError("Unsloth tool loop must stay disabled") async def fake_passthrough(llama_backend, payload, model_name, **kwargs): inference_route.api_monitor.finish(kwargs.get("monitor_id")) @@ -757,7 +757,7 @@ class TestChatCompletionRequestToolFields: return self._v1_client(monkeypatch, _GGUFBackend()) # A process --enable-tools policy must not turn a client-tool passthrough - # into a Studio local loop, so a policy of None or True both keep the + # into an Unsloth local loop, so a policy of None or True both keep the # passthrough (the guard mirrors _explicit_studio_tool_loop_requested). for policy in (None, True): for mode in ("ask", "auto"): @@ -810,7 +810,7 @@ class TestChatCompletionRequestToolFields: assert "requires stream=true" in resp.json()["error"]["message"] def test_permission_mode_policy_forced_local_loop_rejected_before_switch(self, monkeypatch): - # A process --enable-tools policy forces Studio's own tool loop on even + # A process --enable-tools policy forces Unsloth's own tool loop on even # when the request omits enable_tools and carries no client tools. A # non-streaming ask/auto request is then confirm-gated with no stream to # prompt on, so it must 400 at the pre-switch guard -- before @@ -863,7 +863,7 @@ class TestChatCompletionRequestToolFields: def test_enable_tools_on_non_tool_backend_keeps_client_tools_on_passthrough(self, monkeypatch): # DiffusionGemma forces supports_tools off while passthrough stays # available (#6851): enable_tools=True must not steal client tools - # from the passthrough into a Studio tool loop that cannot run. + # from the passthrough into an Unsloth tool loop that cannot run. import routes.inference as inference_route captured = {} @@ -883,7 +883,7 @@ class TestChatCompletionRequestToolFields: raise AssertionError("client tools must use passthrough") def generate_chat_completion_with_tools(self, **_kwargs): - raise AssertionError("Studio tool loop cannot run on a non-tool backend") + raise AssertionError("Unsloth tool loop cannot run on a non-tool backend") async def fake_passthrough(llama_backend, payload, model_name, **kwargs): captured["body"] = inference_route._build_openai_passthrough_body( @@ -1054,7 +1054,7 @@ class TestChatCompletionRequestToolFields: monkeypatch.setattr( inference_route, "_detect_safetensors_features", - lambda backend, chat_template: {"supports_tools": True}, + lambda backend, chat_template, tools = None: {"supports_tools": True}, ) monitor = ApiMonitor(max_entries = 3) monkeypatch.setattr(inference_route, "api_monitor", monitor) @@ -2581,7 +2581,7 @@ class TestGgufVisionToolRouting: raise AssertionError("plain GGUF path should not be used") def _tools(**_kwargs): - raise AssertionError("Studio tool loop should not steal response_format") + raise AssertionError("Unsloth tool loop should not steal response_format") backend = SimpleNamespace( is_loaded = True, @@ -2654,7 +2654,7 @@ class TestGgufVisionToolRouting: raise AssertionError("plain GGUF path should not be used") def _tools(**_kwargs): - raise AssertionError("Studio tool loop should not replace client tools") + raise AssertionError("Unsloth tool loop should not replace client tools") backend = SimpleNamespace( is_loaded = True, @@ -2726,7 +2726,7 @@ class TestGgufVisionToolRouting: yield "plain response" def _tools(**_kwargs): - raise AssertionError("tool_choice='none' must not start Studio's tool loop") + raise AssertionError("tool_choice='none' must not start Unsloth's tool loop") backend = SimpleNamespace( is_loaded = True, @@ -2780,7 +2780,7 @@ class TestGgufVisionToolRouting: raise AssertionError("plain GGUF path should not be used") def _tools(**_kwargs): - raise AssertionError("enabled_tools alone must not start Studio's tool loop") + raise AssertionError("enabled_tools alone must not start Unsloth's tool loop") backend = SimpleNamespace( is_loaded = True, @@ -2844,7 +2844,7 @@ class TestGgufVisionToolRouting: raise AssertionError("plain GGUF path should not be used") def _tools(**_kwargs): - raise AssertionError("enabled_tools alone must not start Studio's tool loop") + raise AssertionError("enabled_tools alone must not start Unsloth's tool loop") backend = SimpleNamespace( is_loaded = True, @@ -6620,6 +6620,29 @@ class TestApiMonitorAudioInput: assert entry["reply"] == "hello world" assert monitor.active_count() == 0 + def failing_chunks(): + yield "partial" + raise RuntimeError("generation failed") + + self._patch_audio_backend(monkeypatch, failing_chunks()) + error_monitor = ApiMonitor(max_entries = 3) + monkeypatch.setattr(inf_mod, "api_monitor", error_monitor) + error_response = await openai_chat_completions( + payload, + request = request, + current_subject = "test", + ) + error_chunks = [ + chunk.decode() if isinstance(chunk, bytes) else chunk + async for chunk in error_response.body_iterator + ] + + assert '"type": "server_error"' in error_chunks[-1] + assert error_chunks[-1].endswith("data: [DONE]\n\n") + [error_entry] = error_monitor.snapshot() + assert error_entry["status"] == "error" + assert error_monitor.active_count() == 0 + asyncio.run(_run()) def test_non_gguf_tts_auto_route_records_monitor(self, monkeypatch): diff --git a/studio/backend/tests/test_orchestrator_unload_cancel.py b/studio/backend/tests/test_orchestrator_unload_cancel.py index fb80b6d061..3a36500aee 100644 --- a/studio/backend/tests/test_orchestrator_unload_cancel.py +++ b/studio/backend/tests/test_orchestrator_unload_cancel.py @@ -34,6 +34,70 @@ def _bare_orchestrator(): return o +def test_adapter_control_raises_stream_errors(monkeypatch): + o = _bare_orchestrator() + monkeypatch.setattr( + o, + "_generate_dispatched", + lambda **_kwargs: iter([orch_mod.GenStreamError("Error: adapter failed")]), + ) + + with pytest.raises(RuntimeError, match = "adapter failed"): + list(o.generate_with_adapter_control(use_adapter = False)) + + closed = [] + + def _stream(**_kwargs): + try: + yield "token" + yield "late token" + finally: + closed.append(True) + + monkeypatch.setattr(o, "_generate_dispatched", _stream) + generator = o.generate_with_adapter_control(use_adapter = False) + assert next(generator) == "token" + generator.close() + assert closed == [True] + + +def test_worker_closes_cancelled_generator_before_gen_done(): + from core.inference.worker import _handle_generate + + events = [] + + class _Backend: + last_generation_stats = None + + def generate_with_adapter_control(self, **_kwargs): + try: + yield "token" + yield "late token" + finally: + events.append("closed") + + class _Responses: + def __init__(self): + self.items = [] + + def put(self, item): + if item["type"] == "gen_done": + assert events == ["closed"] + self.items.append(item) + + responses = _Responses() + cancel = threading.Event() + cancel.set() + _handle_generate( + _Backend(), + {"request_id": "r1", "messages": [], "use_adapter": False}, + responses, + cancel, + ) + + assert [item["type"] for item in responses.items] == ["gen_done"] + + def test_unload_cancels_inflight_generation_then_unloads(monkeypatch): o = _bare_orchestrator() monkeypatch.setattr(o, "_ensure_subprocess_alive", lambda: True) diff --git a/studio/backend/tests/test_password_prompt.py b/studio/backend/tests/test_password_prompt.py index 372d6a2aa4..1af8836065 100644 --- a/studio/backend/tests/test_password_prompt.py +++ b/studio/backend/tests/test_password_prompt.py @@ -183,6 +183,22 @@ def test_loop_short_password_reprompts(monkeypatch): assert "at least 8 characters" in out +def test_loop_whitespace_only_reprompts(monkeypatch): + ok, applied, out = _run_loop(monkeypatch, _keys(" " * 8, "long-enough-pw", "long-enough-pw")) + assert ok is True + assert applied == ["long-enough-pw"] + assert "contain spaces" in out + + +def test_loop_password_with_inner_space_reprompts(monkeypatch): + ok, applied, out = _run_loop( + monkeypatch, _keys("has space pw", "long-enough-pw", "long-enough-pw") + ) + assert ok is True + assert applied == ["long-enough-pw"] + assert "contain spaces" in out + + def test_loop_rejects_current_password(monkeypatch): ok, applied, out = _run_loop( monkeypatch, _keys("bootstrap-pw", "fresh-password", "fresh-password") diff --git a/studio/backend/tests/test_password_prompt_backstop.py b/studio/backend/tests/test_password_prompt_backstop.py index 597eac1625..3c2c1956f9 100644 --- a/studio/backend/tests/test_password_prompt_backstop.py +++ b/studio/backend/tests/test_password_prompt_backstop.py @@ -3,7 +3,7 @@ """Pre-tunnel terminal password gate: never publish a public Cloudflare URL while the seeded default admin password is active. Imports run.py directly, -so run under the Studio venv.""" +so run under the Unsloth venv.""" from __future__ import annotations diff --git a/studio/backend/tests/test_permission_mode.py b/studio/backend/tests/test_permission_mode.py index 3b7197fc49..4fc64a6291 100644 --- a/studio/backend/tests/test_permission_mode.py +++ b/studio/backend/tests/test_permission_mode.py @@ -1438,7 +1438,7 @@ def test_unknown_permission_mode_normalizes_to_ask_on_request_models(): def test_ask_auto_self_enable_confirm_on_chat_request(): # "Ask" gates every call, so a direct /chat/completions caller that requests - # ask but omits the legacy confirm flag self-enables it when Studio's own tool + # ask but omits the legacy confirm flag self-enables it when Unsloth's own tool # loop is requested. Only the router's loop-entry signals count (enable_tools / # mcp_enabled); enabled_tools alone never starts the loop. for loop in ({"enable_tools": True}, {"mcp_enabled": True}): @@ -1481,7 +1481,7 @@ def test_ask_auto_self_enable_confirm_on_chat_request(): confirm_tool_calls = False, ) assert req.confirm_tool_calls is False - # A plain client-tool passthrough (client-supplied tools that Studio does not + # A plain client-tool passthrough (client-supplied tools that Unsloth does not # execute) must NOT self-enable confirm, or the route rejects the passthrough. req = ChatCompletionRequest( messages = [{"role": "user", "content": "hi"}], diff --git a/studio/backend/tests/test_picker_service.py b/studio/backend/tests/test_picker_service.py new file mode 100644 index 0000000000..1bdfc135e3 --- /dev/null +++ b/studio/backend/tests/test_picker_service.py @@ -0,0 +1,272 @@ +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 + +import json +from types import SimpleNamespace + +from picker.service import ( + MAX_TEMPLATE_METADATA_BYTES, + _chat_template_from_dir, + _chat_template_from_processor_json, + _chat_template_from_tokenizer_config, + _chat_template_from_tokenizer_dir, + _find_gguf_in_dir, + _iter_ggufs, + read_default_chat_template, + validate_chat_template, +) + + +def test_iter_ggufs_skips_gguf_companions(tmp_path): + mtp_dir = tmp_path / "MTP" + mtp_dir.mkdir() + main = tmp_path / "model-Q8_0.gguf" + main.write_bytes(b"") + (tmp_path / "mmproj-F16.gguf").write_bytes(b"") + (tmp_path / "mtp-model-Q8_0.gguf").write_bytes(b"") + (mtp_dir / "model-Q8_0-MTP.gguf").write_bytes(b"") + (tmp_path / "model-Q8_0-be.gguf").write_bytes(b"") + + assert _iter_ggufs(tmp_path) == [main] + + +def test_find_gguf_in_dir_matches_quant_label(tmp_path): + mtp_dir = tmp_path / "MTP" + mtp_dir.mkdir() + main = tmp_path / "model-Q8_0.gguf" + main.write_bytes(b"") + (mtp_dir / "model-Q8_0-MTP.gguf").write_bytes(b"") + (tmp_path / "model-Q4_K_M.gguf").write_bytes(b"") + + assert _find_gguf_in_dir(tmp_path, "Q8_0") == main + assert _find_gguf_in_dir(tmp_path, "Q4_K") is None + + +def test_find_gguf_in_dir_without_variant_prefers_largest_model(tmp_path): + smaller = tmp_path / "a-model-Q4_K_M.gguf" + larger = tmp_path / "z-model-Q8_0.gguf" + smaller.write_bytes(b"0") + larger.write_bytes(b"00") + + assert _find_gguf_in_dir(tmp_path, None) == larger + + +def test_find_gguf_in_dir_without_variant_prefers_first_split(tmp_path): + first = tmp_path / "model-Q4_K_M-00001-of-00003.gguf" + second = tmp_path / "model-Q4_K_M-00002-of-00003.gguf" + third = tmp_path / "model-Q4_K_M-00003-of-00003.gguf" + first.write_bytes(b"0") + second.write_bytes(b"000") + third.write_bytes(b"00") + + assert _find_gguf_in_dir(tmp_path, None) == first + + first.unlink() + assert _find_gguf_in_dir(tmp_path, None) == second + + +def test_find_gguf_in_dir_matches_bpw_variant_base_label(tmp_path): + target = tmp_path / "model-IQ4_XS-3.53bpw.gguf" + target.write_bytes(b"") + (tmp_path / "model-Q4_K_M.gguf").write_bytes(b"") + + assert _find_gguf_in_dir(tmp_path, "IQ4_XS") == target + assert _find_gguf_in_dir(tmp_path, "IQ4_XS-3.53bpw") == target + assert _find_gguf_in_dir(tmp_path, "Q4_K") is None + + +def test_validate_chat_template_accepts_valid_and_empty(): + assert validate_chat_template("{{ messages[0].content }}").valid is True + assert validate_chat_template("").valid is True + assert validate_chat_template(" ").valid is True + + +def test_validate_chat_template_reports_syntax_error_with_line(): + result = validate_chat_template("{% if %}{% endif %}") + assert result.valid is False + assert result.error is not None + assert result.error.startswith("Line ") + + +def test_chat_template_from_tokenizer_config_reads_string(): + assert _chat_template_from_tokenizer_config({"chat_template": "HELLO"}) == "HELLO" + assert _chat_template_from_tokenizer_config({"chat_template": " "}) is None + assert _chat_template_from_tokenizer_config({}) is None + + +def test_chat_template_from_tokenizer_config_prefers_named_default(): + config = { + "chat_template": [ + {"name": "tool_use", "template": "TOOL"}, + {"name": "default", "template": "DEFAULT"}, + ] + } + assert _chat_template_from_tokenizer_config(config) == "DEFAULT" + + +def test_chat_template_from_tokenizer_config_falls_back_to_first_entry(): + config = { + "chat_template": [ + {"name": "tool_use", "template": "TOOL"}, + {"name": "other", "template": "OTHER"}, + ] + } + assert _chat_template_from_tokenizer_config(config) == "TOOL" + + +def test_chat_template_from_tokenizer_dir_prefers_jinja_file(tmp_path): + (tmp_path / "chat_template.jinja").write_text("FROM_JINJA", encoding = "utf-8") + (tmp_path / "tokenizer_config.json").write_text( + json.dumps({"chat_template": "FROM_CONFIG"}), encoding = "utf-8" + ) + assert _chat_template_from_tokenizer_dir(tmp_path) == "FROM_JINJA" + + +def test_chat_template_from_tokenizer_dir_reads_tokenizer_config(tmp_path): + (tmp_path / "tokenizer_config.json").write_text( + json.dumps({"chat_template": "FROM_CONFIG"}), encoding = "utf-8" + ) + assert _chat_template_from_tokenizer_dir(tmp_path) == "FROM_CONFIG" + + +def test_chat_template_from_dir_without_variant_prefers_tokenizer(tmp_path): + (tmp_path / "tokenizer_config.json").write_text( + json.dumps({"chat_template": "FROM_CONFIG"}), encoding = "utf-8" + ) + assert _chat_template_from_dir(tmp_path) == "FROM_CONFIG" + + +def test_chat_template_from_dir_with_variant_still_prefers_tokenizer(tmp_path, monkeypatch): + (tmp_path / "tokenizer_config.json").write_text( + json.dumps({"chat_template": "FROM_CONFIG"}), encoding = "utf-8" + ) + (tmp_path / "model-Q4_K_M.gguf").write_bytes(b"") + monkeypatch.setattr("picker.service.read_gguf_chat_template", lambda _path: "FROM_GGUF") + # Selecting a variant must not flip precedence to the embedded GGUF template. + assert _chat_template_from_dir(tmp_path, "Q4_K_M") == "FROM_CONFIG" + + +def test_chat_template_from_dir_with_variant_falls_back_to_gguf(tmp_path, monkeypatch): + (tmp_path / "model-Q4_K_M.gguf").write_bytes(b"") + monkeypatch.setattr("picker.service.read_gguf_chat_template", lambda _path: "FROM_GGUF") + # With no tokenizer sidecar, the embedded GGUF template is still the fallback. + assert _chat_template_from_dir(tmp_path, "Q4_K_M") == "FROM_GGUF" + + +def test_chat_template_from_dir_returns_none_when_absent(tmp_path): + assert _chat_template_from_dir(tmp_path) is None + + +def test_read_default_chat_template_direct_gguf_prefers_sidecar(tmp_path, monkeypatch): + gguf = tmp_path / "model-Q4_K_M.gguf" + gguf.write_bytes(b"") + (tmp_path / "tokenizer_config.json").write_text( + json.dumps({"chat_template": "FROM_CONFIG"}), encoding = "utf-8" + ) + monkeypatch.setattr("picker.service._build_browse_allowlist", lambda: [tmp_path]) + monkeypatch.setattr("picker.service.read_gguf_chat_template", lambda _path: "FROM_GGUF") + # A directly selected .gguf must prefer a maintained sidecar over its embedded copy. + assert read_default_chat_template(str(gguf)) == "FROM_CONFIG" + + +def test_read_default_chat_template_direct_gguf_falls_back_to_embedded(tmp_path, monkeypatch): + gguf = tmp_path / "model-Q4_K_M.gguf" + gguf.write_bytes(b"") + monkeypatch.setattr("picker.service._build_browse_allowlist", lambda: [tmp_path]) + monkeypatch.setattr("picker.service.read_gguf_chat_template", lambda _path: "FROM_GGUF") + # With no sidecar next to the file, the embedded GGUF template is the fallback. + assert read_default_chat_template(str(gguf)) == "FROM_GGUF" + + +def test_tokenizer_config_over_size_limit_is_skipped_not_parsed(tmp_path): + # An oversized tokenizer_config.json must be skipped before json.loads so a + # hostile sidecar cannot exhaust memory. + padding = "x" * (MAX_TEMPLATE_METADATA_BYTES + 1024) + (tmp_path / "tokenizer_config.json").write_text( + json.dumps({"chat_template": "HELLO", "_pad": padding}), encoding = "utf-8" + ) + assert _chat_template_from_tokenizer_dir(tmp_path) is None + + +def test_processor_json_over_size_limit_is_skipped_not_parsed(tmp_path): + padding = "x" * (MAX_TEMPLATE_METADATA_BYTES + 1024) + (tmp_path / "chat_template.json").write_text( + json.dumps({"default": "HELLO", "_pad": padding}), encoding = "utf-8" + ) + assert _chat_template_from_processor_json(tmp_path) is None + + +def test_tokenizer_config_at_size_limit_is_still_read(tmp_path): + # A normal-sized config is unaffected by the bound (regression guard). + (tmp_path / "tokenizer_config.json").write_text( + json.dumps({"chat_template": "FROM_CONFIG"}), encoding = "utf-8" + ) + assert _chat_template_from_tokenizer_dir(tmp_path) == "FROM_CONFIG" + + +def test_remote_template_over_size_limit_is_skipped_before_download(monkeypatch): + # An uncached Hub repo whose template exceeds the cap must be skipped via the + # remote size pre-check, never downloaded. + import huggingface_hub + + monkeypatch.setattr("picker.service.resolve_cached_repo_id_case", lambda name: name) + monkeypatch.setattr("picker.service.iter_hf_cache_snapshots", lambda resolved: []) + + def _fail_download(*args, **kwargs): + raise AssertionError("oversized remote template must not be downloaded") + + def _fake_get_paths_info(self, repo_id, paths, **kwargs): + return [SimpleNamespace(path = p, size = MAX_TEMPLATE_METADATA_BYTES + 1) for p in paths] + + monkeypatch.setattr(huggingface_hub, "hf_hub_download", _fail_download) + monkeypatch.setattr(huggingface_hub.HfApi, "get_paths_info", _fake_get_paths_info) + + assert read_default_chat_template("org/oversized-model") is None + + +def test_remote_oversized_jinja_falls_through_to_tokenizer_template(tmp_path, monkeypatch): + # A raw chat_template.jinja between the response cap (MAX_CHAT_TEMPLATE_BYTES) + # and the download bound (MAX_TEMPLATE_METADATA_BYTES) must not be returned: the + # route drops it, so the remote path must skip the oversized Jinja and fall + # through to the smaller tokenizer_config.json. + import huggingface_hub + from picker.schemas import MAX_CHAT_TEMPLATE_BYTES + + big_jinja = tmp_path / "chat_template.jinja" + big_jinja.write_text("{{ x }}" * (MAX_CHAT_TEMPLATE_BYTES // 4), encoding = "utf-8") + assert MAX_CHAT_TEMPLATE_BYTES < big_jinja.stat().st_size < MAX_TEMPLATE_METADATA_BYTES + tokenizer_config = tmp_path / "tokenizer_config.json" + tokenizer_config.write_text(json.dumps({"chat_template": "SMALL_TEMPLATE"}), encoding = "utf-8") + files = { + "chat_template.jinja": big_jinja, + "tokenizer_config.json": tokenizer_config, + } + selected_cache = tmp_path / "selected-cache" / "hub" + observed_cache_dirs = [] + + monkeypatch.setattr("picker.service.resolve_cached_repo_id_case", lambda name: name) + monkeypatch.setattr("picker.service.iter_hf_cache_snapshots", lambda resolved: []) + monkeypatch.setattr("picker.service.active_hf_hub_cache", lambda: str(selected_cache)) + + def _fake_download(repo_id, rel, **kwargs): + observed_cache_dirs.append(kwargs.get("cache_dir")) + target = files.get(rel) + if target is None: + raise FileNotFoundError(rel) + return str(target) + + def _fake_get_paths_info(self, repo_id, paths, **kwargs): + return [ + SimpleNamespace( + path = p, + size = files[p].stat().st_size if p in files else 0, + ) + for p in paths + ] + + monkeypatch.setattr(huggingface_hub, "hf_hub_download", _fake_download) + monkeypatch.setattr(huggingface_hub.HfApi, "get_paths_info", _fake_get_paths_info) + + assert read_default_chat_template("org/big-jinja-model") == "SMALL_TEMPLATE" + assert observed_cache_dirs + assert set(observed_cache_dirs) == {str(selected_cache)} diff --git a/studio/backend/tests/test_providers_api.py b/studio/backend/tests/test_providers_api.py index 5e24ed752d..7cac3a9e99 100644 --- a/studio/backend/tests/test_providers_api.py +++ b/studio/backend/tests/test_providers_api.py @@ -38,11 +38,11 @@ BASE_URL = os.getenv("STUDIO_TEST_URL", "http://localhost:8000") USERNAME = os.getenv("STUDIO_TEST_USER", "unsloth") PASSWORD = os.getenv("STUDIO_TEST_PASSWORD", "") -# Skip the whole module when no live Studio server / bootstrap password is +# Skip the whole module when no live Unsloth server / bootstrap password is # available (e.g. on CI) so pytest discovery does not error out. pytestmark = pytest.mark.skipif( not PASSWORD, - reason = "Integration test requires a running Studio server; set STUDIO_TEST_PASSWORD to enable.", + reason = "Integration test requires a running Unsloth server; set STUDIO_TEST_PASSWORD to enable.", ) # provider_type → (env var name, model for inference test) diff --git a/studio/backend/tests/test_providers_db_models.py b/studio/backend/tests/test_providers_db_models.py new file mode 100644 index 0000000000..ca9dffbd70 --- /dev/null +++ b/studio/backend/tests/test_providers_db_models.py @@ -0,0 +1,70 @@ +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 + +"""Unit tests for provider model persistence (unslothai/unsloth#7281).""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +import storage.providers_db as providers_db + + +@pytest.fixture() +def isolated_providers_db(tmp_path: Path, monkeypatch: pytest.MonkeyPatch): + db_path = tmp_path / "studio.db" + monkeypatch.setattr(providers_db, "studio_db_path", lambda: db_path) + monkeypatch.setattr(providers_db, "ensure_dir", lambda _path: None) + providers_db._schema_ready = False + yield db_path + providers_db._schema_ready = False + + +def test_create_and_list_provider_models(isolated_providers_db: Path): + providers_db.create_provider( + id = "ollama1", + provider_type = "ollama", + display_name = "Home Ollama", + base_url = "http://127.0.0.1:11434", + models = ["llama3.2", "qwen2.5"], + available_models = ["llama3.2", "qwen2.5", "mistral"], + ) + + row = providers_db.get_provider("ollama1") + assert row is not None + assert row["models"] == ["llama3.2", "qwen2.5"] + assert row["available_models"] == ["llama3.2", "qwen2.5", "mistral"] + + listed = providers_db.list_providers() + assert len(listed) == 1 + assert listed[0]["models"] == ["llama3.2", "qwen2.5"] + + +def test_update_provider_models(isolated_providers_db: Path): + providers_db.create_provider( + id = "vllm1", + provider_type = "vllm", + display_name = "Remote vLLM", + base_url = "http://studio-host:8000/v1", + models = ["meta-llama/Llama-3.2-1B-Instruct"], + available_models = ["meta-llama/Llama-3.2-1B-Instruct"], + ) + + assert providers_db.update_provider( + id = "vllm1", + models = ["meta-llama/Llama-3.2-3B-Instruct"], + available_models = [ + "meta-llama/Llama-3.2-1B-Instruct", + "meta-llama/Llama-3.2-3B-Instruct", + ], + ) + + row = providers_db.get_provider("vllm1") + assert row is not None + assert row["models"] == ["meta-llama/Llama-3.2-3B-Instruct"] + assert row["available_models"] == [ + "meta-llama/Llama-3.2-1B-Instruct", + "meta-llama/Llama-3.2-3B-Instruct", + ] diff --git a/studio/backend/tests/test_rag_embed_llama_server.py b/studio/backend/tests/test_rag_embed_llama_server.py index 0e1f74cefe..3a332ee19b 100644 --- a/studio/backend/tests/test_rag_embed_llama_server.py +++ b/studio/backend/tests/test_rag_embed_llama_server.py @@ -149,7 +149,7 @@ def test_build_env_gpu_inherits_devices(monkeypatch): monkeypatch.setenv("CUDA_VISIBLE_DEVICES", "0,1") b = LlamaServerBackend() env = b._build_env("/bin/llama-server", use_gpu = True) - assert env.get("CUDA_VISIBLE_DEVICES") == "0,1" # inherit Studio's selection + assert env.get("CUDA_VISIBLE_DEVICES") == "0,1" # inherit Unsloth's selection def test_use_gpu_explicit_modes(monkeypatch): diff --git a/studio/backend/tests/test_rag_embeddings.py b/studio/backend/tests/test_rag_embeddings.py index 28a2f69426..197ae4c495 100644 --- a/studio/backend/tests/test_rag_embeddings.py +++ b/studio/backend/tests/test_rag_embeddings.py @@ -5,8 +5,10 @@ and token counting must be serialized (else threads panic "Already borrowed").""" import os +import sys import threading import time +from types import SimpleNamespace import numpy as np import pytest @@ -130,6 +132,35 @@ def test_token_counter_enables_parallelism_only_during_call(monkeypatch): assert os.environ.get("TOKENIZERS_PARALLELISM") == "false" # restored after +def test_sentence_transformer_load_uses_live_cache(monkeypatch, tmp_path): + observed = {} + + class FakeSentenceTransformer: + def __init__(self, name, **kwargs): + observed["name"] = name + observed.update(kwargs) + + monkeypatch.setitem( + sys.modules, + "sentence_transformers", + SimpleNamespace(SentenceTransformer = FakeSentenceTransformer), + ) + monkeypatch.setattr(embeddings, "_install_torchao_stub_once", lambda: None) + monkeypatch.setattr(embeddings, "_guard_model_security", lambda *_a, **_k: None) + monkeypatch.setattr(embeddings, "_device", lambda: "cpu") + monkeypatch.setattr( + "utils.hf_cache_settings.active_hf_hub_cache", + lambda: str(tmp_path / "selected-hub"), + ) + embeddings._model = None + embeddings._name = None + + embeddings._get("Org/Embedder") + + assert observed["name"] == "Org/Embedder" + assert observed["cache_folder"] == str(tmp_path / "selected-hub") + + class _SentinelLlamaBackend: """Stand-in for LlamaServerBackend; never spawns a real server.""" diff --git a/studio/backend/tests/test_recommended_folders_permission.py b/studio/backend/tests/test_recommended_folders_permission.py index 33a457755e..b65695ad93 100644 --- a/studio/backend/tests/test_recommended_folders_permission.py +++ b/studio/backend/tests/test_recommended_folders_permission.py @@ -112,7 +112,7 @@ def test_path_under_unreadable_parent_returns_false_not_raises(tmp_path): ) def test_demonstrates_the_underlying_stdlib_regression(tmp_path): """Documents *why* _safe_is_dir exists: the old bare pattern raises on - the interpreters Studio ships on (3.12+).""" + the interpreters Unsloth ships on (3.12+).""" parent = tmp_path / "ollama" parent.mkdir() os.chmod(parent, 0o000) diff --git a/studio/backend/tests/test_resolve_quant_gguf.py b/studio/backend/tests/test_resolve_quant_gguf.py index 840c4d8d4c..a137237e80 100644 --- a/studio/backend/tests/test_resolve_quant_gguf.py +++ b/studio/backend/tests/test_resolve_quant_gguf.py @@ -68,8 +68,6 @@ def test_skips_mtp_drafter_for_main_weights(tmp_path): def test_prefers_the_complete_snapshot(tmp_path, monkeypatch): - from huggingface_hub import constants as hf_constants - cache = tmp_path / "hub" snaps = cache / "models--org--repo" / "snapshots" # Partial older snapshot: one small shard. @@ -78,7 +76,10 @@ def test_prefers_the_complete_snapshot(tmp_path, monkeypatch): complete_first = _write(snaps / "bbbb" / "model-00001-of-00002-Q4_K_M.gguf", 30) _write(snaps / "bbbb" / "model-00002-of-00002-Q4_K_M.gguf", 40) - monkeypatch.setattr(hf_constants, "HF_HUB_CACHE", str(cache)) + monkeypatch.setattr( + "utils.hf_cache_settings.known_hf_hub_caches", + lambda: [cache], + ) path, total = models_route._resolve_quant_gguf("org/repo", "Q4_K_M", is_local = False) diff --git a/studio/backend/tests/test_responses_tool_passthrough.py b/studio/backend/tests/test_responses_tool_passthrough.py index 46dd0d42e4..69715649b7 100644 --- a/studio/backend/tests/test_responses_tool_passthrough.py +++ b/studio/backend/tests/test_responses_tool_passthrough.py @@ -120,7 +120,7 @@ class TestResponsesRequestTools: def test_builtin_tool_type_passes_validation(self): """Non-function built-in tools (web_search, file_search, mcp, ...) must not raise at validation so SDKs that default to them don't - fail on Studio; they're filtered out during translation.""" + fail on Unsloth; they're filtered out during translation.""" req = ResponsesRequest( input = "hi", tools = [{"type": "web_search_preview"}], diff --git a/studio/backend/tests/test_rocm_multi_gpu_vram_system_wide.py b/studio/backend/tests/test_rocm_multi_gpu_vram_system_wide.py new file mode 100644 index 0000000000..bdafdeae9b --- /dev/null +++ b/studio/backend/tests/test_rocm_multi_gpu_vram_system_wide.py @@ -0,0 +1,554 @@ +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 + +"""The System tab's multi-GPU view must show system-wide VRAM on ROCm (#7072). + +When amd-smi is unavailable, get_visible_gpu_utilization fell back to torch, +whose readings are process-local: a model held by the separate llama-server +process read as ~0 VRAM used even with the GPU full. These tests cover the +per-GPU system-wide overlay the multi-device endpoint now applies, matched by +physical device identity. +""" + +from __future__ import annotations + +import importlib +import sys +import types +from pathlib import Path + +_BACKEND_DIR = Path(__file__).resolve().parent.parent +if str(_BACKEND_DIR) not in sys.path: + sys.path.insert(0, str(_BACKEND_DIR)) + + +def _maybe_stub(name: str, builder): + # Stub only if the real module is missing, so we never shadow it for later tests. + try: + importlib.import_module(name) + except ImportError: + sys.modules[name] = builder() + + +def _build_loggers_stub(): + m = types.ModuleType("loggers") + m.get_logger = lambda name: __import__("logging").getLogger(name) + return m + + +def _build_structlog_stub(): + m = types.ModuleType("structlog") + m.get_logger = lambda *a, **k: __import__("logging").getLogger("stub") + return m + + +_maybe_stub("loggers", _build_loggers_stub) +_maybe_stub("structlog", _build_structlog_stub) + +import utils.hardware.hardware as hw # noqa: E402 + + +def _device( + index, + used, + total, + *, + ordinal = None, +): + return { + "index": index, + "index_kind": "physical", + "visible_ordinal": index if ordinal is None else ordinal, + "gpu_utilization_pct": None, + "temperature_c": None, + "vram_used_gb": used, + "vram_total_gb": total, + "vram_utilization_pct": round((used / total) * 100, 1) if total > 0 else None, + "power_draw_w": None, + "power_limit_w": None, + "power_utilization_pct": None, + } + + +# ── Linux per-card sysfs ── + + +def _fake_drm(tmp_path, monkeypatch, cards): + """Fake /sys/class/drm tree; glob returns cards REVERSED so the PCI sort must order them. + + ``cards``: (card_no, pci_bdf, driver, vram) tuples; vram is (used_gb, total_gb) + or None for a device with no mem_info_vram_* files. + """ + drivers = tmp_path / "drivers" + card_paths = [] + for card_no, bdf, driver, vram in cards: + pci_dir = tmp_path / "pci" / bdf + pci_dir.mkdir(parents = True, exist_ok = True) + drv_dir = drivers / driver + drv_dir.mkdir(parents = True, exist_ok = True) + (pci_dir / "driver").symlink_to(drv_dir) + if vram is not None: + used, total = vram + (pci_dir / "mem_info_vram_used").write_text(str(int(used * 1024**3))) + (pci_dir / "mem_info_vram_total").write_text(str(int(total * 1024**3))) + card_dir = tmp_path / "drm" / f"card{card_no}" + card_dir.mkdir(parents = True, exist_ok = True) + (card_dir / "device").symlink_to(pci_dir) + card_paths.append(str(card_dir)) + monkeypatch.setattr(hw.glob, "glob", lambda pattern: list(reversed(card_paths))) + return card_paths + + +def test_linux_vram_keyed_by_pci_excludes_foreign_adapters(monkeypatch, tmp_path): + # Foreign (non-amdgpu) adapters contribute no entry, so they cannot shift ordinals. + monkeypatch.setattr(hw.platform, "system", lambda: "Linux") + _fake_drm( + tmp_path, + monkeypatch, + [ + (0, "0000:00:02.0", "i915", (0.5, 2.0)), # foreign adapter: excluded + (1, "0000:03:00.0", "amdgpu", (40, 48)), # AMD device 0 + (2, "0000:41:00.0", "amdgpu", (1, 8)), # AMD device 1 + ], + ) + assert hw._rocm_linux_sysfs_vram_by_pci_gb() == { + "0000:03:00.0": (40.0, 48.0), + "0000:41:00.0": (1.0, 8.0), + } + + +def test_linux_vram_omits_bad_cards_without_shifting(monkeypatch, tmp_path): + # A zero-total card has no entry; identity keying means its absence renumbers nothing. + monkeypatch.setattr(hw.platform, "system", lambda: "Linux") + _fake_drm( + tmp_path, + monkeypatch, + [ + (0, "0000:03:00.0", "amdgpu", (0, 0)), # zero total -> no entry + (1, "0000:41:00.0", "amdgpu", (2, 16)), + ], + ) + assert hw._rocm_linux_sysfs_vram_by_pci_gb() == {"0000:41:00.0": (2.0, 16.0)} + + +def test_linux_vram_omits_amd_card_without_vram_files(monkeypatch, tmp_path): + # An APU with no mem_info_vram_* files has no entry; the discrete card keeps its address. + monkeypatch.setattr(hw.platform, "system", lambda: "Linux") + _fake_drm( + tmp_path, + monkeypatch, + [ + (0, "0000:03:00.0", "amdgpu", None), # APU: no VRAM sysfs files + (1, "0000:41:00.0", "amdgpu", (2, 16)), + ], + ) + assert hw._rocm_linux_sysfs_vram_by_pci_gb() == {"0000:41:00.0": (2.0, 16.0)} + + +# ── KFD topology: the authoritative ROCm device order ── + + +_AMD = 4098 # 0x1002 +_NVIDIA = 4318 # 0x10DE -- the open kernel module also registers KFD nodes + + +def _fake_kfd(tmp_path, monkeypatch, nodes): + """Fake KFD topology nodes tree, returned out of node order so the sort must order it. + + ``nodes``: (node_id, simd_count, location_id, domain, vendor_id); simd_count 0 + marks a CPU node, location_id None omits the property. + """ + node_paths = [] + for node_id, simd_count, location_id, domain, vendor_id in nodes: + d = tmp_path / "kfd" / str(node_id) + d.mkdir(parents = True, exist_ok = True) + lines = [f"cpu_cores_count {0 if simd_count else 8}", f"simd_count {simd_count}"] + if location_id is not None: + lines.append(f"location_id {location_id}") + lines.append(f"domain {domain}") + if vendor_id is not None: + lines.append(f"vendor_id {vendor_id}") + (d / "properties").write_text("\n".join(lines) + "\n") + node_paths.append(str(d)) + monkeypatch.setattr(hw.glob, "glob", lambda pattern: list(reversed(node_paths))) + return node_paths + + +def test_kfd_lists_gpu_nodes_in_device_order(monkeypatch, tmp_path): + # The CPU node (simd_count 0) takes no ordinal; GPU nodes in node-id order are HIP's order. + monkeypatch.setattr(hw.platform, "system", lambda: "Linux") + _fake_kfd( + tmp_path, + monkeypatch, + [ + (0, 0, None, 0, None), # CPU node + (1, 304, (0x03 << 8) | (0x00 << 3) | 0, 0, _AMD), # 0000:03:00.0 -> dev 0 + (2, 304, (0x41 << 8) | (0x00 << 3) | 0, 0, _AMD), # 0000:41:00.0 -> dev 1 + ], + ) + assert hw._rocm_kfd_gpu_pci_ids() == ["0000:03:00.0", "0000:41:00.0"] + + +def test_kfd_decodes_domain_device_and_function(monkeypatch, tmp_path): + monkeypatch.setattr(hw.platform, "system", lambda: "Linux") + _fake_kfd(tmp_path, monkeypatch, [(1, 64, (0xC1 << 8) | (0x1F << 3) | 5, 0x1234, _AMD)]) + assert hw._rocm_kfd_gpu_pci_ids() == ["1234:c1:1f.5"] + + +def test_kfd_skips_non_amd_gpu_nodes(monkeypatch, tmp_path): + # An NVIDIA KFD node is not a HIP device: it must take no ordinal, else it + # shifts every AMD GPU and ROCm device 1 resolves to AMD GPU 0. + monkeypatch.setattr(hw.platform, "system", lambda: "Linux") + _fake_kfd( + tmp_path, + monkeypatch, + [ + (0, 0, None, 0, None), # CPU + (1, 128, (0x01 << 8) | 0, 0, _NVIDIA), # NVIDIA: no ordinal + (2, 304, (0x03 << 8) | 0, 0, _AMD), # AMD device 0 + (3, 304, (0x41 << 8) | 0, 0, _AMD), # AMD device 1 + ], + ) + assert hw._rocm_kfd_gpu_pci_ids() == ["0000:03:00.0", "0000:41:00.0"] + + +def test_kfd_fails_closed_when_a_gpu_has_no_location(monkeypatch, tmp_path): + # Dropping an unplaceable AMD GPU shifts later ordinals; fail closed for the whole map. + monkeypatch.setattr(hw.platform, "system", lambda: "Linux") + _fake_kfd( + tmp_path, + monkeypatch, + [ + (1, 304, None, 0, _AMD), # AMD GPU with no location_id + (2, 304, (0x41 << 8) | 0, 0, _AMD), + ], + ) + assert hw._rocm_kfd_gpu_pci_ids() == [] + + +def test_kfd_fails_closed_when_a_node_is_unreadable(monkeypatch, tmp_path): + # An unreadable node could be a GPU; assuming otherwise would shift ordinals. + monkeypatch.setattr(hw.platform, "system", lambda: "Linux") + paths = _fake_kfd( + tmp_path, + monkeypatch, + [ + (1, 304, (0x03 << 8) | 0, 0, _AMD), + (2, 304, (0x41 << 8) | 0, 0, _AMD), + ], + ) + (Path(paths[0]) / "properties").unlink() + assert hw._rocm_kfd_gpu_pci_ids() == [] + + +def test_kfd_absent_yields_no_device_order(monkeypatch): + monkeypatch.setattr(hw.glob, "glob", lambda pattern: []) + assert hw._rocm_kfd_gpu_pci_ids() == [] + + +# ── overlay ── + + +def _patch_pci_map(monkeypatch, bdfs): + """Declare the ROCm device order by PCI address (index N is device N) and clear + the visibility masks the overlay requires unset. + """ + for var in ( + "HIP_VISIBLE_DEVICES", + "ROCR_VISIBLE_DEVICES", + "CUDA_VISIBLE_DEVICES", + "GPU_DEVICE_ORDINAL", + ): + monkeypatch.delenv(var, raising = False) + monkeypatch.setattr(hw, "_rocm_kfd_gpu_pci_ids", lambda: list(bdfs)) + + +def _pci(n): + """A distinct, well-formed PCI address for card n.""" + return f"0000:{n:02x}:00.0" + + +def test_overlay_windows_is_noop_keeps_torch(monkeypatch): + # Windows is intentionally not overlaid (perf counters can't map to ROCm ordinals): keep torch. + monkeypatch.setattr(hw.platform, "system", lambda: "Windows") + monkeypatch.setattr( + hw, + "_rocm_linux_sysfs_vram_by_pci_gb", + lambda: (_ for _ in ()).throw(AssertionError("sysfs must not run on Windows")), + ) + devices = [_device(0, used = 0.02, total = 8.0)] + _patch_pci_map(monkeypatch, [_pci(0)]) + hw._overlay_system_wide_vram(devices) + assert devices[0]["vram_used_gb"] == 0.02 # untouched + + +def test_overlay_linux_matches_by_device_ordinal(monkeypatch): + # Devices arriving as [index 1, index 0] each get their own GPU's figures by ordinal. + monkeypatch.setattr(hw.platform, "system", lambda: "Linux") + monkeypatch.setattr( + hw, + "_rocm_linux_sysfs_vram_by_pci_gb", + lambda: {_pci(0): (30.0, 45.0), _pci(1): (0.5, 8.0)}, # dev 0 big, dev 1 small + ) + devices = [_device(1, used = 0.01, total = 8.0), _device(0, used = 0.02, total = 45.0)] + _patch_pci_map(monkeypatch, [_pci(0), _pci(1)]) + hw._overlay_system_wide_vram(devices) + assert devices[0]["vram_used_gb"] == 0.5 # index 1 -> device 1 (small) + assert devices[0]["vram_total_gb"] == 8.0 + assert devices[1]["vram_used_gb"] == 30.0 # index 0 -> device 0 (big) + assert devices[1]["vram_total_gb"] == 45.0 + + +def test_overlay_linux_ordinal_hole_does_not_shift(monkeypatch): + # Device 0's card dropped: index 0 keeps torch, index 1 still maps to ordinal 1 (no compaction). + monkeypatch.setattr(hw.platform, "system", lambda: "Linux") + monkeypatch.setattr(hw, "_rocm_linux_sysfs_vram_by_pci_gb", lambda: {_pci(1): (0.5, 8.0)}) + devices = [_device(0, used = 0.02, total = 45.0), _device(1, used = 0.01, total = 8.0)] + _patch_pci_map(monkeypatch, [_pci(0), _pci(1)]) + hw._overlay_system_wide_vram(devices) + assert devices[0]["vram_used_gb"] == 0.02 # no ordinal 0 -> torch kept + assert devices[1]["vram_used_gb"] == 0.5 # ordinal 1 -> device 1, not device 0 + + +def test_overlay_linux_skips_unified_memory_card(monkeypatch): + # Unified-memory APU: the smaller sysfs total must not shrink torch's GTT-backed pool. + monkeypatch.setattr(hw.platform, "system", lambda: "Linux") + monkeypatch.setattr(hw, "_rocm_linux_sysfs_vram_by_pci_gb", lambda: {_pci(0): (0.4, 1.0)}) + devices = [_device(0, used = 12.0, total = 96.0)] # torch's unified pool + _patch_pci_map(monkeypatch, [_pci(0)]) + hw._overlay_system_wide_vram(devices) + assert devices[0]["vram_used_gb"] == 12.0 + assert devices[0]["vram_total_gb"] == 96.0 + + +def test_overlay_linux_skips_partitioned_device(monkeypatch): + # Partitioned MI300: the whole-card sysfs total dwarfs the partition, so the overlay must not overwrite it. + monkeypatch.setattr(hw.platform, "system", lambda: "Linux") + monkeypatch.setattr(hw, "_rocm_linux_sysfs_vram_by_pci_gb", lambda: {_pci(0): (40.0, 192.0)}) + devices = [_device(0, used = 1.0, total = 24.0)] # torch partition + _patch_pci_map(monkeypatch, [_pci(0)]) + hw._overlay_system_wide_vram(devices) + assert devices[0]["vram_used_gb"] == 1.0 # partition figures kept + assert devices[0]["vram_total_gb"] == 24.0 + + +def test_overlay_linux_out_of_range_index_untouched(monkeypatch): + # A masked host exposing physical index 5 with no card 5: keep torch data. + monkeypatch.setattr(hw.platform, "system", lambda: "Linux") + monkeypatch.setattr( + hw, "_rocm_linux_sysfs_vram_by_pci_gb", lambda: {_pci(0): (30.0, 45.0), _pci(1): (0.5, 8.0)} + ) + devices = [_device(5, used = 0.02, total = 45.0)] + _patch_pci_map(monkeypatch, [_pci(0), _pci(1)]) + hw._overlay_system_wide_vram(devices) + assert devices[0]["vram_used_gb"] == 0.02 + + +def test_overlay_ignores_adapters_rocm_cannot_enumerate(monkeypatch): + # A HIP-unenumerable amdgpu adapter has no KFD node, so device 0 resolves to + # the supported GPU's own address, never the display card's. + monkeypatch.setattr(hw.platform, "system", lambda: "Linux") + monkeypatch.setattr( + hw, + "_rocm_linux_sysfs_vram_by_pci_gb", + # Both in DRM sysfs with similar capacity -- what the total-size guard can't separate. + lambda: {_pci(9): (30.0, 45.0), _pci(3): (12.0, 45.0)}, + ) + _patch_pci_map(monkeypatch, [_pci(3)]) # KFD lists only the supported GPU + devices = [_device(0, used = 0.02, total = 45.0)] # torch sees that one GPU + hw._overlay_system_wide_vram(devices) + assert devices[0]["vram_used_gb"] == 12.0 # the supported GPU's own figures + + +def test_overlay_skips_masked_subsets(monkeypatch): + # Under a mask the index is not verifiably a host ordinal, so keep torch's figures. + monkeypatch.setattr(hw.platform, "system", lambda: "Linux") + _patch_pci_map(monkeypatch, [_pci(0), _pci(1), _pci(2), _pci(3)]) + monkeypatch.setenv("HIP_VISIBLE_DEVICES", "1,3") + monkeypatch.setattr( + hw, + "_rocm_linux_sysfs_vram_by_pci_gb", + lambda: {_pci(1): (30.0, 48.0), _pci(3): (12.0, 48.0)}, + ) + devices = [_device(1, used = 0.02, total = 48.0), _device(3, used = 0.01, total = 48.0)] + hw._overlay_system_wide_vram(devices) + assert devices[0]["vram_used_gb"] == 0.02 # torch kept + assert devices[1]["vram_used_gb"] == 0.01 + + +def test_overlay_skips_device_cgroup_filtered_container(monkeypatch): + # A device-cgroup container sets no env var yet compacts torch's indices from + # zero while KFD/DRM list every GPU, so the count mismatch must disable the overlay. + monkeypatch.setattr(hw.platform, "system", lambda: "Linux") + _patch_pci_map(monkeypatch, [_pci(0), _pci(1), _pci(2), _pci(3)]) # host has 4 + monkeypatch.setattr( + hw, + "_rocm_linux_sysfs_vram_by_pci_gb", + lambda: {_pci(0): (30.0, 48.0), _pci(2): (12.0, 48.0)}, + ) + devices = [_device(0, used = 0.02, total = 48.0)] # container sees 1, as index 0 + hw._overlay_system_wide_vram(devices) + assert devices[0]["vram_used_gb"] == 0.02 # torch kept, not host GPU 0's 30.0 + + +def test_overlay_skips_without_kfd_topology(monkeypatch): + # No KFD means no identity to join on; fall back to torch rather than guess. + monkeypatch.setattr(hw.platform, "system", lambda: "Linux") + monkeypatch.setattr(hw, "_rocm_kfd_gpu_pci_ids", lambda: []) + monkeypatch.setattr( + hw, + "_rocm_linux_sysfs_vram_by_pci_gb", + lambda: (_ for _ in ()).throw(AssertionError("must not read sysfs without KFD")), + ) + devices = [_device(0, used = 0.02, total = 45.0)] + hw._overlay_system_wide_vram(devices) + assert devices[0]["vram_used_gb"] == 0.02 + + +def test_overlay_empty_devices_is_noop(monkeypatch): + monkeypatch.setattr(hw.platform, "system", lambda: "Linux") + hw._overlay_system_wide_vram([]) # must not raise + + +# ── integration: the ROCm torch fallback applies the overlay ── + + +def test_visible_utilization_rocm_fallback_overlays(monkeypatch): + for _var in ( + "HIP_VISIBLE_DEVICES", + "ROCR_VISIBLE_DEVICES", + "CUDA_VISIBLE_DEVICES", + "GPU_DEVICE_ORDINAL", + ): + monkeypatch.delenv(_var, raising = False) + monkeypatch.setattr(hw, "IS_ROCM", True) + monkeypatch.setattr(hw, "get_device", lambda: hw.DeviceType.CUDA) + monkeypatch.setattr(hw, "_smi_query", lambda *a, **k: None) # amd-smi unavailable + monkeypatch.setattr( + hw, + "_get_parent_visible_gpu_spec", + lambda: {"raw": None, "numeric_ids": [0, 1], "supports_explicit_gpu_ids": True}, + ) + monkeypatch.setattr(hw, "get_parent_visible_gpu_ids", lambda: [0, 1]) + monkeypatch.setattr( + hw, + "_torch_get_per_device_info", + lambda ids: [ + {"index": 0, "visible_ordinal": 0, "used_gb": 0.02, "total_gb": 45.0}, + {"index": 1, "visible_ordinal": 1, "used_gb": 0.01, "total_gb": 8.0}, + ], + ) + overlaid = [] + monkeypatch.setattr( + hw, "_overlay_system_wide_vram", lambda devices: overlaid.append(len(devices)) + ) + result = hw.get_visible_gpu_utilization() + assert result["available"] is True + assert overlaid == [2] + + +def test_visible_utilization_relative_index_skips_overlay(monkeypatch): + # UUID/MIG mask gives relative indices; the overlay matches physical index, so it must not run. + monkeypatch.setattr(hw, "IS_ROCM", True) + monkeypatch.setattr(hw, "get_device", lambda: hw.DeviceType.CUDA) + monkeypatch.setattr(hw, "_smi_query", lambda *a, **k: None) + monkeypatch.setattr( + hw, + "_get_parent_visible_gpu_spec", + lambda: {"raw": "GPU-uuid-a", "numeric_ids": None, "supports_explicit_gpu_ids": False}, + ) + monkeypatch.setattr(hw, "get_parent_visible_gpu_ids", lambda: []) # UUID mask + monkeypatch.setattr(hw, "_torch_get_physical_gpu_count", lambda: 1) + monkeypatch.setattr( + hw, + "_torch_get_per_device_info", + lambda ids: [{"index": 0, "visible_ordinal": 0, "used_gb": 0.02, "total_gb": 8.0}], + ) + called = [] + monkeypatch.setattr(hw, "_overlay_system_wide_vram", lambda devices: called.append(1)) + result = hw.get_visible_gpu_utilization() + assert result["index_kind"] == "relative" + assert called == [] + + +def test_visible_utilization_nvidia_fallback_skips_overlay(monkeypatch): + monkeypatch.setattr(hw, "IS_ROCM", False) + monkeypatch.setattr(hw, "get_device", lambda: hw.DeviceType.CUDA) + monkeypatch.setattr(hw, "_smi_query", lambda *a, **k: None) + monkeypatch.setattr( + hw, + "_get_parent_visible_gpu_spec", + lambda: {"raw": None, "numeric_ids": [0], "supports_explicit_gpu_ids": True}, + ) + monkeypatch.setattr(hw, "get_parent_visible_gpu_ids", lambda: [0]) + monkeypatch.setattr( + hw, + "_torch_get_per_device_info", + lambda ids: [{"index": 0, "visible_ordinal": 0, "used_gb": 1.0, "total_gb": 24.0}], + ) + called = [] + monkeypatch.setattr(hw, "_overlay_system_wide_vram", lambda devices: called.append(1)) + result = hw.get_visible_gpu_utilization() + assert result["available"] is True + assert called == [] + + +def test_any_visibility_mask_is_detected(monkeypatch): + # Any of these makes the index not a host-physical ordinal, so each must disable the overlay. + for var in ( + "HIP_VISIBLE_DEVICES", + "ROCR_VISIBLE_DEVICES", + "CUDA_VISIBLE_DEVICES", + "GPU_DEVICE_ORDINAL", + ): + monkeypatch.delenv(var, raising = False) + assert hw._rocm_visibility_mask_active() is False + for var in ( + "HIP_VISIBLE_DEVICES", + "ROCR_VISIBLE_DEVICES", + "CUDA_VISIBLE_DEVICES", + "GPU_DEVICE_ORDINAL", + ): + monkeypatch.setenv(var, "1") + assert hw._rocm_visibility_mask_active() is True, var + monkeypatch.setenv(var, " ") # empty is not an active filter + assert hw._rocm_visibility_mask_active() is False, var + monkeypatch.delenv(var, raising = False) + + +def test_overlay_skips_under_gpu_device_ordinal(monkeypatch): + # GPU_DEVICE_ORDINAL=1 surfaces GPU 1 as torch ordinal 0, so index 0 is not GPU 0; overlay must not run. + monkeypatch.setattr(hw.platform, "system", lambda: "Linux") + _patch_pci_map(monkeypatch, [_pci(0)]) + monkeypatch.setenv("GPU_DEVICE_ORDINAL", "1") + monkeypatch.setattr(hw, "_rocm_linux_sysfs_vram_by_pci_gb", lambda: {_pci(0): (30.0, 45.0)}) + devices = [_device(0, used = 0.02, total = 45.0)] + hw._overlay_system_wide_vram(devices) + assert devices[0]["vram_used_gb"] == 0.02 + + +def test_visible_utilization_delegates_gating_to_the_overlay(monkeypatch): + # The call site no longer pre-checks masks; the overlay gates itself, so a physical payload always reaches it. + monkeypatch.setenv("ROCR_VISIBLE_DEVICES", "2,3") + monkeypatch.setenv("HIP_VISIBLE_DEVICES", "1") + monkeypatch.setattr(hw, "IS_ROCM", True) + monkeypatch.setattr(hw, "get_device", lambda: hw.DeviceType.CUDA) + monkeypatch.setattr(hw, "_smi_query", lambda *a, **k: None) + monkeypatch.setattr( + hw, + "_get_parent_visible_gpu_spec", + lambda: {"raw": "1", "numeric_ids": [1], "supports_explicit_gpu_ids": True}, + ) + monkeypatch.setattr(hw, "get_parent_visible_gpu_ids", lambda: [1]) + monkeypatch.setattr( + hw, + "_torch_get_per_device_info", + lambda ids: [{"index": 1, "visible_ordinal": 0, "used_gb": 0.02, "total_gb": 8.0}], + ) + # Real overlay + gating: the layered mask must leave torch's figures. + monkeypatch.setattr(hw.platform, "system", lambda: "Linux") + monkeypatch.setattr(hw, "_rocm_kfd_gpu_pci_ids", lambda: [_pci(0), _pci(1)]) + monkeypatch.setattr(hw, "_rocm_linux_sysfs_vram_by_pci_gb", lambda: {_pci(1): (30.0, 8.0)}) + result = hw.get_visible_gpu_utilization() + assert result["index_kind"] == "physical" + assert result["devices"][0]["vram_used_gb"] == 0.02 # untouched diff --git a/studio/backend/tests/test_rocm_oom_guard.py b/studio/backend/tests/test_rocm_oom_guard.py index 6e70c7cde4..ad46f6ee41 100644 --- a/studio/backend/tests/test_rocm_oom_guard.py +++ b/studio/backend/tests/test_rocm_oom_guard.py @@ -36,7 +36,7 @@ class TestIsIntegratedSignal: """hipDeviceProp_t.integrated wins when truthy; 0/absent never downgrades. Same universal gate PR #5988's UMA safetensors fast-load uses -- keeps - Studio's two unified-memory consumers on one signal.""" + Unsloth's two unified-memory consumers on one signal.""" def test_integrated_upgrades_unknown_apu(self) -> None: # gfx1103 Phoenix iGPU: outside the hardcoded arch set, but the @@ -163,6 +163,9 @@ class TestDeviceNameFallback: "AMD Radeon 8060S", "Radeon 8050S Graphics", # cut-down Strix Halo SKU "AMD Radeon 8050S", + # gfx1151 Gorgon Halo (Ryzen AI Max 400 refresh) + "Radeon 8065S Graphics", # Ryzen AI Max+ 495 + "AMD Radeon 8065S", # case variants "RADEON 8060S GRAPHICS", "radeon 8050s", diff --git a/studio/backend/tests/test_rocm_windows_vram_7072.py b/studio/backend/tests/test_rocm_windows_vram_7072.py new file mode 100644 index 0000000000..b4079831b7 --- /dev/null +++ b/studio/backend/tests/test_rocm_windows_vram_7072.py @@ -0,0 +1,361 @@ +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 + +"""Regression tests for issue #7072 -- "VRAM Usage in System Tab is wrong". + +Reporter: dual AMD (Radeon PRO W7900 ~48GB + W7500 8GB), Windows 10, ROCm 7.13, +torch 2.11.0+rocm7.13. On Windows without a HIP SDK, amd-smi is permanently +disabled (avoids a UAC/DiskPart prompt) and hipMemGetInfo returns free==total +(used 0). Two symptoms followed: + + * System tab (/api/system -> get_visible_gpu_utilization) showed ~0 VRAM used + on every GPU (torch mem_get_info free==total quirk; ROCm/ROCm#1909). + * get_gpu_utilization()'s Windows fallback SUMMED "GPU Adapter Memory\\Dedicated + Usage" across all adapters into ONE fake device with only GPU 0's total, so + the second GPU never appeared. + +The fix reads the per-adapter (LUID-instanced) Dedicated Usage performance +counter -- Task Manager's source -- for per-GPU used, takes per-GPU total from +torch device properties, and guards the free==total mem_get_info quirk. CI has no +AMD GPU/Windows, so torch, the performance counter, and platform are all mocked. +""" + +from __future__ import annotations + +import subprocess +import sys +import types + +import pytest + +from utils.hardware import hardware as hw + +GB = 1024**3 +MiB = 1024**2 + + +# ----------------------------------------------------------------------------- # +# Fakes +# ----------------------------------------------------------------------------- # +def _fake_torch( + devices, + *, + free_equals_total = False, + used_per_device = None, +): + """Build a fake `torch` module. devices: list of (name, total_bytes).""" + dev = list(devices) + + class _Props: + def __init__(self, name, total): + self.name = name + self.total_memory = total + + def get_device_properties(i): + name, total = dev[i] + return _Props(name, total) + + def mem_get_info(i): + _, total = dev[i] + if free_equals_total: + return (total, total) + used = used_per_device[i] if used_per_device is not None else 0 + return (total - used, total) + + t = types.ModuleType("torch") + t.__version__ = "2.11.0+rocm7.13" + t.version = types.SimpleNamespace(hip = "7.13", cuda = None) + t.cuda = types.SimpleNamespace( + is_available = lambda: len(dev) > 0, + device_count = lambda: len(dev), + current_device = lambda: 0, + get_device_properties = get_device_properties, + mem_get_info = mem_get_info, + memory_allocated = lambda i: 0, + memory_reserved = lambda i: 0, + ) + return t + + +def _adapter_output(adapters): + if not adapters: + return "__NONE__\n" + return "".join(f"{name}|{int(used)}\n" for name, used in adapters) + + +def _subprocess_run(*, adapter_output = "__NONE__\n", util_output = "12.0\n"): + def fake_run(cmd, *a, **k): + joined = " ".join(cmd) if isinstance(cmd, list) else str(cmd) + if "GPU Adapter Memory" in joined and "InstanceName" in joined: + out = adapter_output + elif "engtype_3D" in joined or "GPU Engine" in joined: + out = util_output + else: + out = "-1\n" + return subprocess.CompletedProcess(args = cmd, returncode = 0, stdout = out, stderr = "") + + return fake_run + + +@pytest.fixture +def win_rocm(monkeypatch): + """Configure the hardware module as a Windows ROCm host with 2 visible GPUs.""" + monkeypatch.setattr(hw, "get_device", lambda: hw.DeviceType.CUDA) + monkeypatch.setattr(hw, "IS_ROCM", True) + monkeypatch.setattr(hw.platform, "system", lambda: "Windows") + monkeypatch.setattr(hw.sys, "platform", "win32") + monkeypatch.setattr(hw, "_smi_query", lambda *a, **k: None) # amd-smi disabled + # Visible set via HIP mask so we don't shell out to amd-smi for the count. + monkeypatch.setenv("HIP_VISIBLE_DEVICES", "0,1") + monkeypatch.delenv("CUDA_VISIBLE_DEVICES", raising = False) + monkeypatch.delenv("ROCR_VISIBLE_DEVICES", raising = False) + return monkeypatch + + +REPORTER_ADAPTERS = [ + ("luid_0x00000000_0x0000d1e2_phys_0", 40.0 * GB), # W7900, model loaded + ("luid_0x00000000_0x0000e34a_phys_0", 0.5 * GB), # W7500, idle + ("luid_0x00000000_0x0000f001_phys_0", 3 * MiB), # Basic Render Driver +] +DEVICES = [("AMD Radeon PRO W7900", 48 * GB), ("AMD Radeon PRO W7500", 8 * GB)] + + +# ----------------------------------------------------------------------------- # +# System tab (get_visible_gpu_utilization) -- the reporter's screenshot +# ----------------------------------------------------------------------------- # +def test_system_tab_shows_per_gpu_used(win_rocm, monkeypatch): + monkeypatch.setitem(sys.modules, "torch", _fake_torch(DEVICES, free_equals_total = True)) + monkeypatch.setattr( + hw.subprocess, "run", _subprocess_run(adapter_output = _adapter_output(REPORTER_ADAPTERS)) + ) + + devices = hw.get_visible_gpu_utilization()["devices"] + by_idx = {d["index"]: d for d in devices} + assert len(devices) == 2 + assert by_idx[0]["vram_total_gb"] == 48.0 + assert by_idx[0]["vram_used_gb"] == pytest.approx(40.0, abs = 0.01) # not 0 + assert by_idx[1]["vram_total_gb"] == 8.0 # own total + # The 3 MiB Basic Render Driver counter makes this a hidden-adapter case: only + # the 40 GiB is forced onto the 48 GiB card; the idle card reads Unknown. + assert by_idx[1]["vram_used_gb"] is None + assert by_idx[1]["vram_utilization_pct"] is None + assert all( + d["vram_used_gb"] <= d["vram_total_gb"] for d in devices if d["vram_used_gb"] is not None + ) + + +def test_gpu_utilization_does_not_collapse(win_rocm, monkeypatch): + monkeypatch.setitem(sys.modules, "torch", _fake_torch(DEVICES, free_equals_total = True)) + monkeypatch.setattr( + hw.subprocess, "run", _subprocess_run(adapter_output = _adapter_output(REPORTER_ADAPTERS)) + ) + + result = hw.get_gpu_utilization() + devices = result["devices"] + assert sorted(d["index"] for d in devices) == [0, 1] # both GPUs, no collapse + assert {d["vram_total_gb"] for d in devices} == {48.0, 8.0} + assert result["vram_total_gb"] == 48.0 # legacy primary mirror preserved + + +def test_localized_counter_reports_unknown_not_zero(win_rocm, monkeypatch): + monkeypatch.setitem(sys.modules, "torch", _fake_torch(DEVICES, free_equals_total = True)) + monkeypatch.setattr(hw.subprocess, "run", _subprocess_run(adapter_output = "__NONE__\n")) + + devices = hw.get_visible_gpu_utilization()["devices"] + assert len(devices) == 2 # both still shown with correct totals + assert {d["vram_total_gb"] for d in devices} == {48.0, 8.0} + assert all(d["vram_used_gb"] is None for d in devices) # unknown, not fake 0 + assert all(d["vram_utilization_pct"] is None for d in devices) + + +# ----------------------------------------------------------------------------- # +# mem_get_info free==total guard scoping +# ----------------------------------------------------------------------------- # +def test_mem_get_info_guard_scopes_to_windows_rocm(monkeypatch): + torch_mod = _fake_torch(DEVICES, free_equals_total = True) + monkeypatch.setattr(hw, "get_device", lambda: hw.DeviceType.CUDA) + monkeypatch.setitem(sys.modules, "torch", torch_mod) + + # Windows ROCm -> used unknown (None), total kept. + monkeypatch.setattr(hw, "IS_ROCM", True) + monkeypatch.setattr(hw.sys, "platform", "win32") + win = hw._torch_get_per_device_info([0, 1]) + assert [d["used_gb"] for d in win] == [None, None] + assert [d["total_gb"] for d in win] == [48.0, 8.0] + + # Linux ROCm -> unchanged numeric used. + monkeypatch.setattr(hw.sys, "platform", "linux") + assert [d["used_gb"] for d in hw._torch_get_per_device_info([0, 1])] == [0.0, 0.0] + + # Windows NVIDIA -> guard must not fire. + monkeypatch.setattr(hw, "IS_ROCM", False) + monkeypatch.setattr(hw.sys, "platform", "win32") + assert [d["used_gb"] for d in hw._torch_get_per_device_info([0, 1])] == [0.0, 0.0] + + +# ----------------------------------------------------------------------------- # +# Per-adapter attribution helpers (pure unit) +# ----------------------------------------------------------------------------- # +def test_match_adapter_pairs_and_clamps(): + assert hw._match_adapter_used_to_devices([40 * GB, 0.5 * GB], [48 * GB, 8 * GB]) == [ + 40 * GB, + 0.5 * GB, + ] + assert hw._match_adapter_used_to_devices([100 * GB], [48 * GB]) == [48 * GB] # clamp + assert hw._match_adapter_used_to_devices([40 * GB], [48 * GB, 8 * GB]) == [40 * GB, None] + + +def test_match_adapter_reports_unknown_when_more_active_than_visible(): + # More adapters actively using VRAM than are visible (a GPU outside the mask): + # attribution would fabricate a value, so report unknown for every device. + assert hw._match_adapter_used_to_devices([40 * GB, 0.5 * GB], [8 * GB]) == [None] + + +def test_match_adapter_reports_unknown_when_hidden_high_use_adapter_survives_filter(): + # Idle 8 GiB card (10 MiB noise) beside a hidden 48 GiB card at 40 GiB: the + # 40 GiB can't fit the 8 GiB device, so clamping there would fabricate. Unknown. + assert hw._match_adapter_used_to_devices([40 * GB, 10 * MiB], [8 * GB]) == [None] + # Order of the counters must not matter. + assert hw._match_adapter_used_to_devices([10 * MiB, 40 * GB], [8 * GB]) == [None] + + +def test_match_adapter_reports_unknown_for_placeholder_fallback(): + # Every counter below the 64 MiB floor plus a placeholder: no LUID-to-ordinal + # mapping tells placeholder from idle GPU, so report unknown, not fabricate. + # Single visible 8 GiB card idle (10 MiB) beside a 50 MiB placeholder counter. + assert hw._match_adapter_used_to_devices([50 * MiB, 10 * MiB], [8 * GB]) == [None] + # Order of the counters must not matter. + assert hw._match_adapter_used_to_devices([10 * MiB, 50 * MiB], [8 * GB]) == [None] + # Two idle visible GPUs plus a placeholder: all three counters below the floor. + assert hw._match_adapter_used_to_devices([50 * MiB, 10 * MiB, 5 * MiB], [48 * GB, 8 * GB]) == [ + None, + None, + ] + + +def test_match_adapter_reports_unknown_when_usage_not_capacity_ordered(): + # 8 GiB card at 7 GiB beside a 48 GiB card at 5 GiB: the bigger usage still fits + # the smaller card, so both pairings are feasible -> unknown. + assert hw._match_adapter_used_to_devices([7 * GB, 5 * GB], [8 * GB, 48 * GB]) == [None, None] + # Device order must not matter (same physical situation, ordinals flipped). + assert hw._match_adapter_used_to_devices([7 * GB, 5 * GB], [48 * GB, 8 * GB]) == [None, None] + # Same-capacity cards with unequal usage are equally unattributable. + assert hw._match_adapter_used_to_devices([12 * GB, 8 * GB], [24 * GB, 24 * GB]) == [None, None] + # A single usage that fits both cards can sit on either -> unknown. + assert hw._match_adapter_used_to_devices([5 * GB], [48 * GB, 8 * GB]) == [None, None] + # But a capacity-forced assignment (usage exceeds the smaller card) is kept: + # 40 GiB can only be the 48 GiB card, so it is not fabrication. + assert hw._match_adapter_used_to_devices([40 * GB], [48 * GB, 8 * GB]) == [40 * GB, None] + + +def test_match_adapter_reports_unknown_when_hidden_usage_fits_visible_card(): + # A survivor that merely *fits* a visible card must not be pinned onto it. Two + # cards (48/8 GiB) at 40 GiB / 10 MiB beside a hidden 6 GiB adapter: the 6 GiB + # fits the idle 8 GiB card but isn't forced -> Unknown; only 40 GiB is forced. + assert hw._match_adapter_used_to_devices([40 * GB, 10 * MiB, 6 * GB], [48 * GB, 8 * GB]) == [ + 40 * GB, + None, + ] + # Counter order must not matter. + assert hw._match_adapter_used_to_devices([6 * GB, 40 * GB, 10 * MiB], [48 * GB, 8 * GB]) == [ + 40 * GB, + None, + ] + # A single visible card with a hidden adapter is never attributable: a fitting + # survivor could be the hidden GPU's while the visible card is idle. + assert hw._match_adapter_used_to_devices([6 * GB, 10 * MiB], [8 * GB]) == [None] + + +def test_match_adapter_capacity_forced_matrix(): + """Exhaustive hidden-adapter matrix for the capacity-forced rule. + + A value is emitted only when the supra-threshold counters number exactly the + visible devices AND a device's ranked usage strictly exceeds every smaller + card's capacity. Otherwise (a visible card idle, a merely-fitting usage, or the + smallest card) every device reports unknown. + """ + m = hw._match_adapter_used_to_devices + # -- exactly-n supra-threshold counters, capacity-forced survivors are kept - # + # Both visible cards have a real reading (the 3 MiB is a placeholder): 40 GiB + # forced onto the 48 GiB card, 0.5 GiB not forced -> None. + assert m([40 * GB, 0.5 * GB, 3 * MiB], [48 * GB, 8 * GB]) == [40 * GB, None] + # Three visible cards all active (supra-threshold) + placeholder: 40 > 24 and + # 20 > 8, both forced; the 8 GiB card is not forced -> None. + assert m([40 * GB, 20 * GB, 5 * GB, 3 * MiB], [48 * GB, 24 * GB, 8 * GB]) == [ + 40 * GB, + 20 * GB, + None, + ] + # -- fewer supra-threshold counters than visible cards -> all unknown ------ # + # A visible card is idle, so even a "forced" 40 could be the hidden GPU's. + assert m([40 * GB, 3 * MiB, 3 * MiB], [48 * GB, 8 * GB]) == [None, None] + assert m([40 * GB, 10 * MiB, 10 * MiB], [48 * GB, 8 * GB]) == [None, None] + assert m([40 * GB, 20 * GB, 3 * MiB, 3 * MiB], [48 * GB, 24 * GB, 8 * GB]) == [ + None, + None, + None, + ] + # Middle usage (6 GiB) fits both the 24 and 8 GiB cards, and only two cards are + # active for three visible -> not a bijection -> all unknown. + assert m([40 * GB, 6 * GB, 3 * MiB, 3 * MiB], [48 * GB, 24 * GB, 8 * GB]) == [ + None, + None, + None, + ] + # -- hidden larger than every visible card -> all unknown ----------------- # + assert m([40 * GB, 10 * MiB], [8 * GB]) == [None] + assert m([48 * GB, 3 * MiB, 3 * MiB], [24 * GB, 8 * GB]) == [None, None] + # -- more active adapters than visible cards -> all unknown --------------- # + assert m([40 * GB, 7 * GB, 6 * GB, 3 * MiB], [48 * GB, 8 * GB]) == [None, None] + assert m([40 * GB, 7 * GB, 6 * GB, 3 * MiB, 3 * MiB], [48 * GB, 8 * GB]) == [None, None] + # -- every counter below the noise floor (placeholder fallback) -> unknown - # + assert m([50 * MiB, 10 * MiB], [8 * GB]) == [None] + assert m([50 * MiB, 10 * MiB, 5 * MiB], [48 * GB, 8 * GB]) == [None, None] + # -- equal-capacity cards with a hidden adapter: nothing is forced -------- # + assert m([40 * GB, 40 * GB, 3 * MiB], [48 * GB, 48 * GB]) == [None, None] + assert m([40 * GB, 30 * GB, 3 * MiB], [48 * GB, 48 * GB]) == [None, None] + + +def test_perf_counter_parser_and_sentinel(monkeypatch): + monkeypatch.setattr(hw.platform, "system", lambda: "Windows") + monkeypatch.setattr( + hw.subprocess, "run", _subprocess_run(adapter_output = _adapter_output(REPORTER_ADAPTERS)) + ) + parsed = hw._rocm_windows_perf_counter_vram_by_adapter() + assert parsed is not None and len(parsed) == 3 + assert parsed[0][0].startswith("luid_") + monkeypatch.setattr(hw.subprocess, "run", _subprocess_run(adapter_output = "__NONE__\n")) + assert hw._rocm_windows_perf_counter_vram_by_adapter() is None + + +# ----------------------------------------------------------------------------- # +# Unified-memory (Strix Halo APU) total reconciliation (Codex #7238) +# ----------------------------------------------------------------------------- # +def test_unified_memory_adopts_torch_total_even_when_used_unknown(): + """Windows ROCm unified-memory APU: torch's used is None but its total (the full + GTT pool) is authoritative. The correction must still adopt the larger total; + used stays at amd-smi's figure when torch's is unknown.""" + metrics = {"vram_total_gb": 8.0, "vram_used_gb": 2.0, "vram_utilization_pct": 25.0} + hw._apply_unified_memory_correction(metrics, {"total_gb": 124.0, "used_gb": None, "index": 0}) + assert metrics["vram_total_gb"] == 124.0 # full unified pool, not the 8 GB carve-out + assert metrics["vram_used_gb"] == 2.0 # amd-smi used preserved (torch's was None) + assert metrics["vram_utilization_pct"] == pytest.approx(round(2.0 / 124.0 * 100, 1)) + + +def test_unified_memory_overwrites_used_when_torch_used_known(): + """When torch reports both a larger total and a known used, both are adopted + and utilization is recomputed against the corrected total (unchanged path).""" + metrics = {"vram_total_gb": 8.0, "vram_used_gb": 2.0, "vram_utilization_pct": 25.0} + hw._apply_unified_memory_correction(metrics, {"total_gb": 124.0, "used_gb": 40.0, "index": 0}) + assert metrics["vram_total_gb"] == 124.0 + assert metrics["vram_used_gb"] == 40.0 + assert metrics["vram_utilization_pct"] == pytest.approx(round(40.0 / 124.0 * 100, 1)) + + +def test_unified_memory_no_op_when_torch_total_not_larger(): + """A discrete GPU where torch total does not exceed amd-smi's is left untouched.""" + metrics = {"vram_total_gb": 48.0, "vram_used_gb": 10.0, "vram_utilization_pct": 20.8} + hw._apply_unified_memory_correction(metrics, {"total_gb": 48.0, "used_gb": None, "index": 0}) + assert metrics["vram_total_gb"] == 48.0 + assert metrics["vram_used_gb"] == 10.0 + assert metrics["vram_utilization_pct"] == 20.8 diff --git a/studio/backend/tests/test_safetensors_capability_advertise.py b/studio/backend/tests/test_safetensors_capability_advertise.py index 0ed670ac01..bd3d8d16b9 100644 --- a/studio/backend/tests/test_safetensors_capability_advertise.py +++ b/studio/backend/tests/test_safetensors_capability_advertise.py @@ -417,6 +417,59 @@ def test_detect_safetensors_features_gemma_native_tool_call_keeps_tools_on(): assert flags["supports_tools"] is True +def test_detect_safetensors_features_gemma_native_reasoning_is_parseable_not_prefilled(): + """Native Gemma channels are normalized to , then split by the route.""" + from routes.inference import _detect_safetensors_features, _sf_reasoning_prefill_mode + + tpl_with_gemma_native = "{% if add_generation_prompt %}<|channel>thought\n{% endif %}" + backend = SimpleNamespace( + active_model_name = "unsloth/gemma-4-E2B-it", + models = { + "unsloth/gemma-4-E2B-it": { + "native_chat_template": tpl_with_gemma_native, + "chat_template_info": {"template": "override has no native markers"}, + } + }, + ) + flags = _detect_safetensors_features(backend, "override has no native markers") + missing_arg_flags = _detect_safetensors_features(backend, None) + + assert flags["supports_reasoning"] is True + assert flags["reasoning_always_on"] is True + assert missing_arg_flags["supports_reasoning"] is True + assert _sf_reasoning_prefill_mode(flags, None, tpl_with_gemma_native) is False + + +def test_detect_safetensors_features_selects_native_reasoning_from_tool_template(): + """Request tools select a marker-bearing named template without affecting default chat.""" + from routes.inference import _detect_safetensors_features + + named_template = { + "default": "plain default template", + "tool_use": "{% if tools %}<|channel>thought\n{% endif %}", + } + backend = SimpleNamespace( + active_model_name = "custom/named-native-reasoning", + models = { + "custom/named-native-reasoning": { + "native_chat_template": named_template, + "chat_template_info": {"template": "{% if tools %}{% endif %}"}, + } + }, + ) + + default_flags = _detect_safetensors_features(backend, "plain override") + tool_flags = _detect_safetensors_features( + backend, + "plain override", + tools = [{"type": "function"}], + ) + + assert default_flags["supports_reasoning"] is False + assert tool_flags["supports_reasoning"] is True + assert tool_flags["reasoning_always_on"] is True + + # Qwen3.5 family pin: the live GGUF + safetensors templates both wrap tool # calls as ``\n...``. Faithful slice so the # classifier never silently regresses for this family. diff --git a/studio/backend/tests/test_safetensors_reasoning_stream.py b/studio/backend/tests/test_safetensors_reasoning_stream.py index 4e708139b7..af5a05d266 100644 --- a/studio/backend/tests/test_safetensors_reasoning_stream.py +++ b/studio/backend/tests/test_safetensors_reasoning_stream.py @@ -215,3 +215,135 @@ def test_s6_reasoning_effort_none_disables_prefill_for_enable_thinking_effort(): swallowed = _replay_sf_reasoning_stream(events, prefilled = True) assert swallowed["visible"] == "" assert swallowed["reasoning"] == "The capital of France is Paris." + + +def test_native_reasoning_streamer_selected_and_errors_raise(): + import threading + import pytest + + torch = pytest.importorskip("torch") + inf = pytest.importorskip("core.inference.inference") + + class Batch(dict): + def to(self, _device): + return self + + class Tok: + chat_template = "<|channel>thought\n..." + all_special_tokens = [] + eos_token_id = 1 + pad_token_id = None + pieces = {10: "<|channel>thought\n", 11: "r", 12: "", 13: "a"} + + def __call__(self, *_args, **_kwargs): + return Batch({"input_ids": torch.zeros((1, 1), dtype = torch.long)}) + + def decode(self, ids, **_kwargs): + return "".join(self.pieces.get(int(token_id), "") for token_id in ids) + + class Model: + device = "cpu" + generation_config = type("Cfg", (), {"eos_token_id": 1})() + config = generation_config + + def __init__(self, fail = False): + self.fail = fail + self.kwargs = None + + def generate(self, **kwargs): + self.kwargs = kwargs + streamer = kwargs["streamer"] + streamer.put(torch.zeros((1, 1), dtype = torch.long)) + for token_id in [10, 11, 12, 13]: + streamer.put(torch.tensor([token_id])) + if self.fail: + raise RuntimeError("boom") + + backend = inf.InferenceBackend.__new__(inf.InferenceBackend) + backend.active_model_name = "gemma-test" + backend._generation_lock = threading.Lock() + backend.models = {"gemma-test": {"model": Model(), "tokenizer": Tok()}} + + assert list(backend.generate_stream("prompt", max_new_tokens = 4))[-1] == "ra" + + backend.models["gemma-test"]["model"] = Model(fail = True) + + with pytest.raises(inf._GenerationThreadError, match = "boom"): + list(backend.generate_stream("prompt", max_new_tokens = 4)) + + +def test_text_only_vlm_fallback_resolves_native_markers_off(): + import threading + import pytest + + torch = pytest.importorskip("torch") + inf = pytest.importorskip("core.inference.inference") + + class Batch(dict): + def to(self, _device): + return self + + class Tokenizer: + all_special_tokens = [] + eos_token_id = 1 + pad_token_id = None + + def __call__(self, *_args, **_kwargs): + return Batch({"input_ids": torch.zeros((1, 1), dtype = torch.long)}) + + class Processor: + chat_template = "<|channel>thought\n..." + tokenizer = Tokenizer() + + class Model: + device = "cpu" + generation_config = type("Cfg", (), {"eos_token_id": 1})() + config = generation_config + + def generate(self, **_kwargs): + return None + + class EmptyStreamer: + def __next__(self): + raise StopIteration + + def end(self): + return None + + captured = {} + backend = inf.InferenceBackend.__new__(inf.InferenceBackend) + backend.active_model_name = "vision-test" + backend._generation_lock = threading.Lock() + backend.models = { + "vision-test": { + "model": Model(), + "processor": Processor(), + "tokenizer": Processor(), + } + } + backend.format_chat_prompt = lambda *_args, **_kwargs: "manual text-only prompt" + + def make_streamer(*_args, **kwargs): + captured.update(kwargs) + return EmptyStreamer() + + backend._make_text_streamer = make_streamer + + assert ( + list( + backend._generate_vision_response( + messages = [{"role": "user", "content": "hello"}], + system_prompt = "", + image = None, + temperature = 0.7, + top_p = 0.9, + top_k = 40, + min_p = 0.0, + max_new_tokens = 1, + repetition_penalty = 1.0, + ) + ) + == [] + ) + assert captured["reasoning_channel_markers"] is None + assert captured["reasoning_channel_markers_resolved"] is True diff --git a/studio/backend/tests/test_safetensors_tool_loop.py b/studio/backend/tests/test_safetensors_tool_loop.py index e3633de289..31c728afca 100644 --- a/studio/backend/tests/test_safetensors_tool_loop.py +++ b/studio/backend/tests/test_safetensors_tool_loop.py @@ -2230,8 +2230,8 @@ def _reprompt_loop(*, auto_heal_tool_calls): tools = [{"type": "function", "function": {"name": "search_knowledge_base"}}], execute_tool = exec_fn, auto_heal_tool_calls = auto_heal_tool_calls, - # Studio always nudges (always-on for the Studio inference paths); the - # API opts in per request. Model the Studio caller here. + # Unsloth always nudges (always-on for the Unsloth inference paths); the + # API opts in per request. Model the Unsloth caller here. nudge_tool_calls = True, max_tool_iterations = 3, ) @@ -3203,7 +3203,197 @@ class TestLoopBehaviour: class TestLoopRePrompt: - """Plan-without-action re-prompt parity with GGUF: nudge instead of terminating, up to ``MAX_ACT_REPROMPTS`` extra slots. Studio always nudges, so these drive the loop with ``nudge_tool_calls=True``.""" + """Plan-without-action re-prompt parity with GGUF: nudge instead of terminating, up to ``MAX_ACT_REPROMPTS`` extra slots. Unsloth always nudges, so these drive the loop with ``nudge_tool_calls=True``.""" + + def test_reasoning_intent_does_not_reprompt_a_visible_answer(self): + generations = 0 + + def _gen(_messages, active_tools = None): + nonlocal generations + generations += 1 + yield ( + "Let me prepare the requested summary carefully." + "This is the final visible answer." + ) + + exec_fn = FakeExecuteTool([]) + events = _collect_events( + run_safetensors_tool_loop( + single_turn = _gen, + messages = [{"role": "user", "content": "summarize this"}], + tools = [{"type": "function", "function": {"name": "web_search"}}], + execute_tool = exec_fn, + nudge_tool_calls = True, + ) + ) + + assert generations == 1 + assert exec_fn.calls == [] + contents = [e["text"] for e in events if e["type"] == "content"] + assert contents[-1].endswith("This is the final visible answer.") + + def test_prefilled_reasoning_intent_does_not_reprompt_a_visible_answer(self): + generations = 0 + + def _gen(_messages, active_tools = None): + nonlocal generations + generations += 1 + yield "Let me prepare the requested summary carefully.This is the final visible answer." + + exec_fn = FakeExecuteTool([]) + events = _collect_events( + run_safetensors_tool_loop( + single_turn = _gen, + messages = [{"role": "user", "content": "summarize this"}], + tools = [{"type": "function", "function": {"name": "web_search"}}], + execute_tool = exec_fn, + nudge_tool_calls = True, + reasoning_prefilled = True, + ) + ) + + assert generations == 1 + assert exec_fn.calls == [] + contents = [e["text"] for e in events if e["type"] == "content"] + assert contents[-1].endswith("This is the final visible answer.") + + def test_prefilled_reasoning_with_reemitted_think_does_not_reprompt(self): + generations = 0 + + def _gen(_messages, active_tools = None): + nonlocal generations + generations += 1 + yield ( + "Let me prepare the requested summary carefully." + "more private planningThis is the final visible answer." + ) + + exec_fn = FakeExecuteTool([]) + events = _collect_events( + run_safetensors_tool_loop( + single_turn = _gen, + messages = [{"role": "user", "content": "summarize this"}], + tools = [{"type": "function", "function": {"name": "web_search"}}], + execute_tool = exec_fn, + nudge_tool_calls = True, + reasoning_prefilled = True, + ) + ) + + assert generations == 1 + assert exec_fn.calls == [] + contents = [e["text"] for e in events if e["type"] == "content"] + assert contents[-1].endswith("This is the final visible answer.") + + def test_prefilled_reasoning_with_later_think_does_not_reprompt(self): + generations = 0 + + def _gen(_messages, active_tools = None): + nonlocal generations + generations += 1 + yield ( + "private prefilled planning" + "Let me prepare the requested summary carefully." + "This is the final visible answer." + ) + + exec_fn = FakeExecuteTool([]) + events = _collect_events( + run_safetensors_tool_loop( + single_turn = _gen, + messages = [{"role": "user", "content": "summarize this"}], + tools = [{"type": "function", "function": {"name": "web_search"}}], + execute_tool = exec_fn, + nudge_tool_calls = True, + reasoning_prefilled = True, + ) + ) + + assert generations == 1 + assert exec_fn.calls == [] + contents = [e["text"] for e in events if e["type"] == "content"] + assert contents[-1].endswith("This is the final visible answer.") + + def test_reasoning_only_intent_still_reprompts_and_uses_a_tool(self): + loop, exec_fn = _make_loop( + turns = [ + ["Let me search for that."], + ['{"name":"web_search","arguments":{"query":"cats"}}'], + ["Here is the answer."], + ], + exec_results = ["result"], + nudge_tool_calls = True, + ) + + events = _collect_events(loop) + + assert exec_fn.calls == [("web_search", {"query": "cats"})] + contents = [e["text"] for e in events if e["type"] == "content"] + assert contents[-1] == "Here is the answer." + + def test_prefilled_no_close_reasoning_intent_still_reprompts(self): + loop, exec_fn = _make_loop( + turns = [ + ["I need more context.Let me search for that."], + ['{"name":"web_search","arguments":{"query":"cats"}}'], + ["Here is the answer."], + ], + exec_results = ["result"], + nudge_tool_calls = True, + reasoning_prefilled = True, + ) + + events = _collect_events(loop) + + assert exec_fn.calls == [("web_search", {"query": "cats"})] + contents = [e["text"] for e in events if e["type"] == "content"] + assert contents[-1] == "Here is the answer." + + def test_prefilled_reasoning_prefix_is_kept_for_reasoning_only_reprompt(self): + loop, exec_fn = _make_loop( + turns = [ + ["Let me search for that.checking details"], + ['{"name":"web_search","arguments":{"query":"cats"}}'], + ["Here is the answer."], + ], + exec_results = ["result"], + nudge_tool_calls = True, + reasoning_prefilled = True, + ) + + events = _collect_events(loop) + + assert exec_fn.calls == [("web_search", {"query": "cats"})] + contents = [e["text"] for e in events if e["type"] == "content"] + assert contents[-1] == "Here is the answer." + + def test_reprompt_history_uses_visible_intent_text(self): + captured: list[list[dict]] = [] + + def _gen(messages, active_tools = None): + captured.append([dict(message) for message in messages]) + if len(captured) == 1: + yield "private planning detailsLet me search for that." + elif len(captured) == 2: + yield '{"name":"web_search","arguments":{"query":"cats"}}' + else: + yield "Here is the answer." + + exec_fn = FakeExecuteTool(["result"]) + events = _collect_events( + run_safetensors_tool_loop( + single_turn = _gen, + messages = [{"role": "user", "content": "find cats"}], + tools = [{"type": "function", "function": {"name": "web_search"}}], + execute_tool = exec_fn, + nudge_tool_calls = True, + ) + ) + + assert exec_fn.calls == [("web_search", {"query": "cats"})] + assert captured[1][1] == {"role": "assistant", "content": "Let me search for that."} + contents = [e["text"] for e in events if e["type"] == "content"] + assert contents[-1] == "Here is the answer." def test_intent_signal_triggers_reprompt(self): # Turn 1: intent signal, no tool call. @@ -4068,7 +4258,7 @@ class TestPlanWithoutActionReprompt: def test_omitted_nudge_flag_is_not_reprompted(self): # The retry is new on this loop: API callers who do not send the flag - # must keep today's behavior. Studio opts in explicitly. + # must keep today's behavior. Unsloth opts in explicitly. loop, exec_fn = _make_loop( turns = [ ["I'll search the web for that."], diff --git a/studio/backend/tests/test_sampling_resolution.py b/studio/backend/tests/test_sampling_resolution.py new file mode 100644 index 0000000000..1ebbae2502 --- /dev/null +++ b/studio/backend/tests/test_sampling_resolution.py @@ -0,0 +1,270 @@ +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 + +"""Effective sampling resolution: per-model recommendation + operator pins. + +Precedence per field: operator UNSLOTH_SAMPLING_* pin -> client explicit value -> +per-model recommendation (load_inference_config) -> static schema default. +""" + +import pytest + +from utils.inference.inference_config import resolve_effective_sampling, SAMPLING_FIELD_NAMES +from utils.inference import inference_config as ic + +_SCHEMA_DEFAULTS = { + "temperature": 0.6, + "top_p": 0.95, + "top_k": 20, + "min_p": 0.01, + "repetition_penalty": 1.0, + "presence_penalty": 0.0, +} + + +@pytest.fixture(autouse = True) +def _isolate(monkeypatch): + # The recommended lookup is lru-cached; clear it so a patched config takes effect. + ic._recommended_sampling.cache_clear() + for field in SAMPLING_FIELD_NAMES: + monkeypatch.delenv(ic._SAMPLING_FIELDS[field][0], raising = False) + yield + ic._recommended_sampling.cache_clear() + + +def _all_omitted(): + return {f: None for f in SAMPLING_FIELD_NAMES} + + +def _set_recommended(monkeypatch, mapping): + # _recommended_sampling sources from load_inference_config -- the exact block the Chat UI + # seeds from -- so patch that directly. Fields absent from `mapping` fall to schema defaults. + monkeypatch.setattr(ic, "load_inference_config", lambda mid: dict(mapping)) + ic._recommended_sampling.cache_clear() + + +def test_recommended_applies_when_client_omits(monkeypatch): + _set_recommended(monkeypatch, {"temperature": 1.0, "top_k": 64, "min_p": 0.0}) + eff = resolve_effective_sampling("some/model", _all_omitted()) + assert eff["temperature"] == 1.0 + assert eff["top_k"] == 64 + assert eff["min_p"] == 0.0 + # A field with no recommendation keeps the static schema default. + assert eff["top_p"] == 0.95 + + +def test_client_explicit_beats_recommended(monkeypatch): + _set_recommended(monkeypatch, {"temperature": 1.0}) + eff = resolve_effective_sampling("some/model", {**_all_omitted(), "temperature": 0.2}) + assert eff["temperature"] == 0.2 + + +def test_operator_pin_beats_client_and_recommended(monkeypatch): + _set_recommended(monkeypatch, {"temperature": 1.0}) + monkeypatch.setenv("UNSLOTH_SAMPLING_TEMPERATURE", "0.9") + eff = resolve_effective_sampling("some/model", {**_all_omitted(), "temperature": 0.2}) + assert eff["temperature"] == 0.9 + + +def test_unknown_model_matches_ui_inference_block(monkeypatch): + # An unknown model gets the same values the Chat UI would seed (load_inference_config's + # default.yaml fallback: temp 0.7 / top_k -1), NOT the request schema defaults. + ui_block = { + "temperature": 0.7, + "top_p": 0.95, + "top_k": -1, + "min_p": 0.01, + "presence_penalty": 0.0, + "repetition_penalty": 1.0, + } + monkeypatch.setattr(ic, "load_inference_config", lambda mid: dict(ui_block)) + ic._recommended_sampling.cache_clear() + eff = resolve_effective_sampling("some/unknown-model", _all_omitted()) + assert eff["temperature"] == 0.7 + assert eff["top_k"] == -1 + assert eff["min_p"] == 0.01 + + +def test_empty_recommendation_falls_back_to_schema_defaults(monkeypatch): + # If load_inference_config yields nothing usable, the resolver falls back to the request + # schema defaults. + monkeypatch.setattr(ic, "load_inference_config", lambda mid: {}) + ic._recommended_sampling.cache_clear() + eff = resolve_effective_sampling("some/model", _all_omitted()) + assert eff == _SCHEMA_DEFAULTS + + +@pytest.mark.parametrize( + "model", + ["unsloth/gemma-4-E4B", "unsloth/Qwen3-4B", "unsloth/Qwen3.5-9B", "someorg/unknown-xyz"], +) +def test_recommendation_matches_ui_source(model): + # Parity guard: what the server recommends for omitted fields equals the Chat UI's source + # (load_inference_config) for every field the UI adopts (mergeBackendRecommendedInference). + ic._recommended_sampling.cache_clear() + ui = ic.load_inference_config(model) + rec = ic._recommended_sampling(model) + for f in ic._UI_RECOMMENDED_FIELDS: + cleaned = ic._clean_sampling_value(f, ui.get(f)) + if cleaned is not None: + assert rec.get(f) == cleaned, f"{model}:{f} rec={rec.get(f)} ui={ui.get(f)}" + + +def test_repetition_penalty_not_auto_recommended(monkeypatch): + # The Chat UI's mergeBackendRecommendedInference never adopts a backend repetition_penalty + # (e.g. lfm2's family value 1.05), so the server must not auto-apply one either. It stays at + # the schema default unless the client sends it or an operator pins it. + monkeypatch.setattr( + ic, "load_inference_config", lambda mid: {"temperature": 0.7, "repetition_penalty": 1.05} + ) + ic._recommended_sampling.cache_clear() + eff = resolve_effective_sampling("some/lfm2-model", _all_omitted()) + assert eff["temperature"] == 0.7 # a UI-adopted field is recommended + assert eff["repetition_penalty"] == 1.0 # rep is NOT auto-recommended (matches the UI) + # An operator can still pin it explicitly. + monkeypatch.setenv("UNSLOTH_SAMPLING_REPETITION_PENALTY", "1.05") + eff2 = resolve_effective_sampling("some/lfm2-model", _all_omitted()) + assert eff2["repetition_penalty"] == 1.05 + + +@pytest.mark.parametrize( + "raw, expected", + [ + ("0.5", 0.5), + ("abc", None), # unparseable + ("9.0", None), # above temperature max (2.0) + ("-1", None), # below temperature min (0.0) + (" ", None), # blank + ("nan", None), # NaN would pass a naive range check + ("inf", None), # non-finite + ("-inf", None), # non-finite + ], +) +def test_operator_override_parsing(monkeypatch, raw, expected): + monkeypatch.setenv("UNSLOTH_SAMPLING_TEMPERATURE", raw) + assert ic._operator_sampling_override("temperature") == expected + + +def test_out_of_range_recommendation_is_dropped(monkeypatch): + # A malformed model recommendation (out of range) is ignored, so the request keeps the + # schema default rather than forwarding a bad value to llama-server. + _set_recommended(monkeypatch, {"temperature": 5.0, "top_k": 64}) + eff = resolve_effective_sampling("some/model", _all_omitted()) + assert eff["temperature"] == 0.6 # 5.0 is outside [0, 2] -> schema default + assert eff["top_k"] == 64 # a valid recommendation is still applied + + +def test_operator_override_top_k_int_and_range(monkeypatch): + monkeypatch.setenv("UNSLOTH_SAMPLING_TOP_K", "40") + assert ic._operator_sampling_override("top_k") == 40 + monkeypatch.setenv("UNSLOTH_SAMPLING_TOP_K", "200") # above max 100 + assert ic._operator_sampling_override("top_k") is None + monkeypatch.setenv("UNSLOTH_SAMPLING_TOP_K", "-1") # min allowed + assert ic._operator_sampling_override("top_k") == -1 + + +@pytest.mark.parametrize( + "field, val", + [ + ("top_k", 10**400), # oversized int on an int field: int() ok, but math.isfinite raises + ("top_k", float("nan")), # NaN reaching an int field: int(nan) raises ValueError + ("top_k", float("inf")), # inf reaching an int field: int(inf) raises OverflowError + ( + "temperature", + 10**400, + ), # oversized int on a float field: float(huge_int) raises OverflowError + ], +) +def test_clean_sampling_value_rejects_unrepresentable(field, val): + # None of these may raise; each is unusable and must be dropped to None (regression: an + # oversized value used to raise OverflowError before the range check could drop it). + assert ic._clean_sampling_value(field, val) is None + + +def test_oversized_operator_override_ignored(monkeypatch): + # A huge integer string parses via int() but overflows float(); math.isfinite would raise + # OverflowError and 500 the request. It must be ignored like any other bad override and the + # field must fall back to the schema default -- no exception. + monkeypatch.setenv("UNSLOTH_SAMPLING_TOP_K", "9" * 400) + assert ic._operator_sampling_override("top_k") is None + _set_recommended(monkeypatch, {}) # no per-model recommendation -> schema default applies + eff = resolve_effective_sampling("some/model", _all_omitted()) + assert eff["top_k"] == 20 # schema default, resolved without raising + + +def test_oversized_recommendation_ignored(monkeypatch): + # A malformed per-model recommendation carrying an oversized int must not raise while + # resolving either; the field simply falls back to the schema default. + _set_recommended(monkeypatch, {"temperature": 10**400, "top_k": 64}) + eff = resolve_effective_sampling("some/model", _all_omitted()) + assert eff["temperature"] == 0.6 # oversized -> dropped -> schema default + assert eff["top_k"] == 64 # a valid recommendation is still applied + + +def test_fill_recommended_sampling_openai_payload(monkeypatch): + from models.inference import ChatCompletionRequest + from routes.inference import _fill_recommended_sampling_openai + + _set_recommended(monkeypatch, {"temperature": 1.0, "top_k": 64, "min_p": 0.0}) + + # Client sent only temperature; top_k / min_p were omitted. + payload = ChatCompletionRequest( + model = "m", messages = [{"role": "user", "content": "hi"}], temperature = 0.2 + ) + _fill_recommended_sampling_openai(payload, "some/model") + assert payload.temperature == 0.2 # explicit client value preserved + assert payload.top_k == 64 # recommended fills the omitted field + assert payload.min_p == 0.0 + assert payload.top_p == 0.95 # no recommendation -> schema default unchanged + + +def test_fill_recommended_sampling_openai_operator_pin_overrides_client(monkeypatch): + from models.inference import ChatCompletionRequest + from routes.inference import _fill_recommended_sampling_openai + + monkeypatch.setattr(ic, "load_model_defaults", lambda mid: {}) + monkeypatch.setattr(ic, "get_family_inference_params", lambda mid: {}) + ic._recommended_sampling.cache_clear() + monkeypatch.setenv("UNSLOTH_SAMPLING_TEMPERATURE", "0.9") + + payload = ChatCompletionRequest( + model = "m", messages = [{"role": "user", "content": "hi"}], temperature = 0.2 + ) + _fill_recommended_sampling_openai(payload, "some/model") + assert payload.temperature == 0.9 # operator pin wins even over an explicit client value + + +def test_fill_recommended_sampling_completions_body(monkeypatch): + # /v1/completions is a raw proxy: recommendations fill omitted fields, but a field with no + # recommendation and no pin is left absent so llama-server keeps its own default (unlike the + # chat schema, which carries per-field defaults). + from routes.inference import _fill_recommended_sampling_completions + + _set_recommended(monkeypatch, {"temperature": 1.0, "top_k": 64, "min_p": 0.0}) + + body = {"prompt": "hi", "temperature": 0.2} + _fill_recommended_sampling_completions(body, "some/model") + assert body["temperature"] == 0.2 # explicit client value preserved + assert body["top_k"] == 64 # recommendation fills the omitted field + assert body["min_p"] == 0.0 + # No recommendation and no pin -> NOT injected (llama-server keeps its default). + assert "top_p" not in body + assert "presence_penalty" not in body + assert "repeat_penalty" not in body + + +def test_fill_recommended_sampling_completions_operator_pin(monkeypatch): + # An operator pin overrides the client's raw-body value, and the repetition pin is written + # under llama-server's "repeat_penalty" key (the schema field is repetition_penalty). + from routes.inference import _fill_recommended_sampling_completions + + monkeypatch.setattr(ic, "load_inference_config", lambda mid: {}) + ic._recommended_sampling.cache_clear() + monkeypatch.setenv("UNSLOTH_SAMPLING_TEMPERATURE", "0.9") + monkeypatch.setenv("UNSLOTH_SAMPLING_REPETITION_PENALTY", "1.2") + + body = {"prompt": "hi", "temperature": 0.2, "repeat_penalty": 1.05} + _fill_recommended_sampling_completions(body, "some/model") + assert body["temperature"] == 0.9 # operator pin wins over the client's explicit value + assert body["repeat_penalty"] == 1.2 # repetition pin lands on llama-server's key + assert "repetition_penalty" not in body # never leak the schema field name into the body diff --git a/studio/backend/tests/test_sandbox_tools.py b/studio/backend/tests/test_sandbox_tools.py index 2970b1a6bb..64201477e3 100644 --- a/studio/backend/tests/test_sandbox_tools.py +++ b/studio/backend/tests/test_sandbox_tools.py @@ -297,6 +297,8 @@ class TestSandboxEnvIsolation: "PYTHONPATH", "VIRTUAL_ENV", "SystemRoot", + "PATHEXT", # Windows only; minimal list so cwd scripts cannot hijack + "NoDefaultCurrentDirectoryInExePath", # Windows only; no cwd-first lookup } extras = set(env.keys()) - allowed assert not extras, f"sandbox env added unexpected keys: {extras}" @@ -305,6 +307,220 @@ class TestSandboxEnvIsolation: assert env["PYTHONPATH"].endswith("sandbox_site") assert "leak-me" not in env["PYTHONPATH"] + def test_host_git_dir_appended_after_curated(self, monkeypatch, tmp_path): + # #7317: Windows Git lives under Program Files, not System32. Sandbox + # PATH resolves bare `git` by appending the dir of the git the HOST + # shell resolves (shutil.which), after the curated prefix. + import core.inference.tools as tools_mod + from core.inference.tools import _build_safe_env + + monkeypatch.setattr(sys, "platform", "win32") + prog = tmp_path / "Program Files" + monkeypatch.setattr(tools_mod, "_windows_program_roots", lambda: [str(prog)]) + git_dir = prog / "Git" / "cmd" + git_dir.mkdir(parents = True) + monkeypatch.setattr(tools_mod.shutil, "which", lambda name: str(git_dir / "git.exe")) + env = _build_safe_env(str(tmp_path)) + parts = env["PATH"].split(os.pathsep) + assert str(git_dir) in parts + # Curated prefix stays ahead of host Git so Studio python/pip win. + assert parts.index(str(git_dir)) > 0 + + def test_host_path_dirs_not_inherited(self, monkeypatch, tmp_path): + """Host PATH dirs (user-writable, git-lookalike) are never inherited; + only the resolved git dir is. No git resolved -> nothing appended.""" + import core.inference.tools as tools_mod + from core.inference.tools import _build_safe_env + + monkeypatch.setattr(sys, "platform", "win32") + venv_scripts = tmp_path / "venv" / "Scripts" + venv_scripts.mkdir(parents = True) + fake_git = tmp_path / "scratch" / "Git" / "cmd" + fake_git.mkdir(parents = True) + monkeypatch.setenv( + "PATH", + os.pathsep.join([str(venv_scripts), str(fake_git), os.environ.get("PATH", "")]), + ) + monkeypatch.setattr(tools_mod.shutil, "which", lambda name: None) + env = _build_safe_env(str(tmp_path)) + parts = env["PATH"].split(os.pathsep) + assert str(venv_scripts) not in parts + # A git-suffixed but unresolved (user-writable) dir is NOT trusted. + assert str(fake_git) not in parts + + def test_git_cmd_shim_extension_added_to_pathext(self, monkeypatch, tmp_path): + """A host git resolved as a .cmd shim under a trusted root stays + resolvable under the restricted PATHEXT (cwd lookup disabled).""" + import core.inference.tools as tools_mod + from core.inference.tools import _build_safe_env + + monkeypatch.setattr(sys, "platform", "win32") + prog = tmp_path / "Program Files" + monkeypatch.setattr(tools_mod, "_windows_program_roots", lambda: [str(prog)]) + git_dir = prog / "Git" / "cmd" + git_dir.mkdir(parents = True) + monkeypatch.setattr(tools_mod.shutil, "which", lambda name: str(git_dir / "git.cmd")) + env = _build_safe_env(str(tmp_path)) + assert str(git_dir) in env["PATH"].split(os.pathsep) + assert env["PATHEXT"] == ".EXE;.COM;.CMD" + + def test_user_writable_git_dir_refused(self, monkeypatch, tmp_path): + """Git resolved from a per-user manager (Scoop shims) is NOT trusted: + an attacker could drop rg.exe beside it and hit the auto-approve gate.""" + import core.inference.tools as tools_mod + from core.inference.tools import _build_safe_env + + monkeypatch.setattr(sys, "platform", "win32") + monkeypatch.setattr( + tools_mod, "_windows_program_roots", lambda: [str(tmp_path / "Program Files")] + ) + shim_dir = tmp_path / "users" / "alice" / "scoop" / "shims" + shim_dir.mkdir(parents = True) + monkeypatch.setattr(tools_mod.shutil, "which", lambda name: str(shim_dir / "git.exe")) + env = _build_safe_env(str(tmp_path)) + assert str(shim_dir) not in env["PATH"].split(os.pathsep) + # No trusted git launcher -> PATHEXT stays minimal. + assert env["PATHEXT"] == ".EXE;.COM" + + def test_trust_uses_known_folder_not_env_override(self, monkeypatch, tmp_path): + """Trust is driven by the resolved Program Files roots, so a git under + an attacker-overridden %ProgramFiles% env value is still refused.""" + import core.inference.tools as tools_mod + from core.inference.tools import _build_safe_env + + monkeypatch.setattr(sys, "platform", "win32") + real_prog = tmp_path / "RealProgramFiles" + (real_prog).mkdir() + evil = tmp_path / "attacker" + (evil / "Git" / "cmd").mkdir(parents = True) + # Resolver returns the genuine root; env is overridden to the evil dir. + monkeypatch.setattr(tools_mod, "_windows_program_roots", lambda: [str(real_prog)]) + monkeypatch.setenv("ProgramFiles", str(evil)) + monkeypatch.setattr( + tools_mod.shutil, "which", lambda name: str(evil / "Git" / "cmd" / "git.exe") + ) + env = _build_safe_env(str(tmp_path)) + assert str(evil / "Git" / "cmd") not in env["PATH"].split(os.pathsep) + + def test_canonical_git_dir_appended(self, monkeypatch, tmp_path): + """The PATH entry is the realpath of the trusted dir, not a junction + alias, so it cannot be retargeted after the trust check.""" + import core.inference.tools as tools_mod + from core.inference.tools import _build_safe_env + + monkeypatch.setattr(sys, "platform", "win32") + real_prog = tmp_path / "Program Files" + real_git = real_prog / "Git" / "cmd" + real_git.mkdir(parents = True) + link = tmp_path / "link" + try: + link.symlink_to(real_prog, target_is_directory = True) + except (OSError, NotImplementedError): + pytest.skip("symlink unsupported in this environment") + monkeypatch.setattr(tools_mod, "_windows_program_roots", lambda: [str(real_prog)]) + monkeypatch.setattr( + tools_mod.shutil, + "which", + lambda name: str(link / "Git" / "cmd" / "git.exe"), + ) + env = _build_safe_env(str(tmp_path)) + parts = env["PATH"].split(os.pathsep) + assert str(real_git) in parts # canonical, not the `link/...` alias + + def test_windows_temp_git_dir_refused(self, monkeypatch, tmp_path): + """A git under a world-writable %SystemRoot% subdir (Windows\\Temp) is + NOT trusted, even though it sits under the Windows root.""" + import core.inference.tools as tools_mod + from core.inference.tools import _build_safe_env + + monkeypatch.setattr(sys, "platform", "win32") + monkeypatch.setattr( + tools_mod, "_windows_program_roots", lambda: [str(tmp_path / "Program Files")] + ) + temp_git = tmp_path / "Windows" / "Temp" / "Git" / "cmd" + temp_git.mkdir(parents = True) + monkeypatch.setattr(tools_mod.shutil, "which", lambda name: str(temp_git / "git.exe")) + env = _build_safe_env(str(tmp_path)) + assert str(temp_git) not in env["PATH"].split(os.pathsep) + + def test_trusted_program_dir_matches_via_realpath(self, monkeypatch, tmp_path): + """The trust check canonicalizes paths, so a symlinked/short alias of + Program Files still matches (stand-in for 8.3 PROGRA~1 on Windows).""" + import core.inference.tools as tools_mod + from core.inference.tools import _build_safe_env + + monkeypatch.setattr(sys, "platform", "win32") + real_prog = tmp_path / "Program Files" + (real_prog / "Git" / "cmd").mkdir(parents = True) + alias = tmp_path / "PROGRA~1" + try: + alias.symlink_to(real_prog, target_is_directory = True) + except (OSError, NotImplementedError): + pytest.skip("symlink unsupported in this environment") + monkeypatch.setattr(tools_mod, "_windows_program_roots", lambda: [str(real_prog)]) + git_via_alias = alias / "Git" / "cmd" / "git.exe" + monkeypatch.setattr(tools_mod.shutil, "which", lambda name: str(git_via_alias)) + env = _build_safe_env(str(tmp_path)) + parts = [os.path.normcase(os.path.realpath(p)) for p in env["PATH"].split(os.pathsep)] + assert os.path.normcase(str(real_prog / "Git" / "cmd")) in parts + + def test_scan_past_untrusted_git_shim(self, monkeypatch, tmp_path): + """When an untrusted shim sorts first on PATH, the scan still finds a + later trusted Program Files git.""" + import core.inference.tools as tools_mod + from core.inference.tools import _build_safe_env + + monkeypatch.setattr(sys, "platform", "win32") + prog = tmp_path / "Program Files" + trusted_git = prog / "Git" / "cmd" + trusted_git.mkdir(parents = True) + (trusted_git / "git.EXE").write_text("") # match PATHEXT case on this FS + shim = tmp_path / "scoop" / "shims" + shim.mkdir(parents = True) + (shim / "git.EXE").write_text("") + monkeypatch.setattr(tools_mod, "_windows_program_roots", lambda: [str(prog)]) + # shutil.which returns the untrusted shim first. + monkeypatch.setattr(tools_mod.shutil, "which", lambda name: str(shim / "git.EXE")) + monkeypatch.setenv("PATH", os.pathsep.join([str(shim), str(trusted_git)])) + monkeypatch.setenv("PATHEXT", ".EXE") + env = _build_safe_env(str(tmp_path)) + parts = env["PATH"].split(os.pathsep) + assert str(trusted_git) in parts + assert str(shim) not in parts + + def test_program_roots_fails_closed_without_known_folder_api(self, monkeypatch): + """When the known-folder API is unavailable, no roots are trusted: env + vars (even %SystemDrive%) are caller-overrideable, so we never derive a + trusted root from them.""" + import core.inference.tools as tools_mod + + # ctypes fails on this Linux host, so the API path raises and we fail + # closed. Any attacker override of these env vars must be irrelevant. + monkeypatch.setenv("ProgramFiles", r"D:\attacker-writable") + monkeypatch.setenv("ProgramW6432", r"D:\attacker-writable") + monkeypatch.setenv("SystemDrive", "D:") + assert tools_mod._windows_program_roots() == [] + + def test_augment_native_program_roots_derives_native_sibling(self): + """A 32-bit process only sees the x86 root; the native sibling is + derived by stripping the ` (x86)` suffix.""" + import core.inference.tools as tools_mod + + roots = tools_mod._augment_native_program_roots([r"C:\Program Files (x86)"]) + lowered = [r.lower() for r in roots] + assert r"c:\program files (x86)" in lowered + assert r"c:\program files" in lowered + + def test_no_default_current_directory_in_exe_path_set_on_windows(self, monkeypatch, tmp_path): + """cmd/CreateProcess must not search cwd for bare names in the sandbox.""" + import core.inference.tools as tools_mod + from core.inference.tools import _build_safe_env + + monkeypatch.setattr(sys, "platform", "win32") + monkeypatch.setattr(tools_mod.shutil, "which", lambda name: None) + env = _build_safe_env(str(tmp_path)) + assert env["NoDefaultCurrentDirectoryInExePath"] == "1" + def test_home_points_at_sandbox_workdir(self, tmp_path): from core.inference.tools import _build_safe_env diff --git a/studio/backend/tests/test_secure_tunnel_gate.py b/studio/backend/tests/test_secure_tunnel_gate.py index 2c13e13bbb..a8c0c2305f 100644 --- a/studio/backend/tests/test_secure_tunnel_gate.py +++ b/studio/backend/tests/test_secure_tunnel_gate.py @@ -2,7 +2,7 @@ # Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 """Cloudflare tunnel start gate, incl. --secure on loopback. Imports run.py -directly, so run under the Studio venv.""" +directly, so run under the Unsloth venv.""" from __future__ import annotations diff --git a/studio/backend/tests/test_server_disk_logging.py b/studio/backend/tests/test_server_disk_logging.py index 05d03d869c..ce733c2aaa 100644 --- a/studio/backend/tests/test_server_disk_logging.py +++ b/studio/backend/tests/test_server_disk_logging.py @@ -3,7 +3,7 @@ """Tests for the server session log + native-crash capture in run.py. -Field regression: Studio "terminates without a warning" -- a native crash in +Field regression: Unsloth "terminates without a warning" -- a native crash in the GPU runtime kills the process with no Python traceback, and a desktop- shortcut console closes before anything can be read. The server must tee its console output to disk and aim faulthandler at the same file so even hard diff --git a/studio/backend/tests/test_setup_cache_env_hf_home.py b/studio/backend/tests/test_setup_cache_env_hf_home.py index 4520c93a51..c5d5f820a8 100644 --- a/studio/backend/tests/test_setup_cache_env_hf_home.py +++ b/studio/backend/tests/test_setup_cache_env_hf_home.py @@ -28,6 +28,9 @@ def _isolate_studio_home(monkeypatch, tmp_path): def _load_storage_roots(): + # Each test models a fresh backend process. The cache resolver intentionally + # snapshots explicit environment variables once per process. + sys.modules.pop("utils.hf_cache_settings", None) spec = importlib.util.spec_from_file_location("storage_roots_under_test", _STORAGE_ROOTS_PATH) module = importlib.util.module_from_spec(spec) spec.loader.exec_module(module) @@ -40,10 +43,10 @@ def _clear_hf_env(monkeypatch): def test_custom_hf_home_seeds_hub_and_xet(monkeypatch, tmp_path): - sr = _load_storage_roots() _clear_hf_env(monkeypatch) custom = tmp_path / "shared" / "huggingface" monkeypatch.setenv("HF_HOME", str(custom)) + sr = _load_storage_roots() sr._setup_cache_env() @@ -54,9 +57,9 @@ def test_custom_hf_home_seeds_hub_and_xet(monkeypatch, tmp_path): def test_default_when_hf_home_unset(monkeypatch, tmp_path): - sr = _load_storage_roots() _clear_hf_env(monkeypatch) monkeypatch.setenv("XDG_CACHE_HOME", str(tmp_path / "xdg")) + sr = _load_storage_roots() sr._setup_cache_env() @@ -67,11 +70,11 @@ def test_default_when_hf_home_unset(monkeypatch, tmp_path): def test_explicit_hub_cache_is_not_overridden(monkeypatch, tmp_path): - sr = _load_storage_roots() _clear_hf_env(monkeypatch) monkeypatch.setenv("HF_HOME", str(tmp_path / "home")) explicit = tmp_path / "explicit" / "hub" monkeypatch.setenv("HF_HUB_CACHE", str(explicit)) + sr = _load_storage_roots() sr._setup_cache_env() @@ -81,11 +84,11 @@ def test_explicit_hub_cache_is_not_overridden(monkeypatch, tmp_path): def test_legacy_huggingface_hub_cache_alias_is_honored(monkeypatch, tmp_path): - sr = _load_storage_roots() _clear_hf_env(monkeypatch) monkeypatch.setenv("HF_HOME", str(tmp_path / "home")) legacy = tmp_path / "legacy" / "hub" monkeypatch.setenv("HUGGINGFACE_HUB_CACHE", str(legacy)) + sr = _load_storage_roots() sr._setup_cache_env() @@ -96,15 +99,16 @@ def test_legacy_huggingface_hub_cache_alias_is_honored(monkeypatch, tmp_path): def test_whitespace_hf_home_falls_back_to_default(monkeypatch, tmp_path): # A blank/whitespace HF_HOME must not become " /hub"; fall back to default. - sr = _load_storage_roots() _clear_hf_env(monkeypatch) monkeypatch.setenv("HF_HOME", " ") monkeypatch.setenv("XDG_CACHE_HOME", str(tmp_path / "xdg")) + sr = _load_storage_roots() sr._setup_cache_env() import os + assert os.environ["HF_HOME"] == str(tmp_path / "xdg" / "huggingface") assert os.environ["HF_HUB_CACHE"] == str(tmp_path / "xdg" / "huggingface" / "hub") @@ -114,9 +118,9 @@ def test_unwritable_hf_home_does_not_crash(monkeypatch, tmp_path): blocker = tmp_path / "blocker" blocker.write_text("not a dir") unwritable = blocker / "hf" - sr = _load_storage_roots() _clear_hf_env(monkeypatch) monkeypatch.setenv("HF_HOME", str(unwritable)) + sr = _load_storage_roots() sr._setup_cache_env() # must not raise diff --git a/studio/backend/tests/test_setup_llama_cpp_backend.py b/studio/backend/tests/test_setup_llama_cpp_backend.py new file mode 100644 index 0000000000..36928c680c --- /dev/null +++ b/studio/backend/tests/test_setup_llama_cpp_backend.py @@ -0,0 +1,154 @@ +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 + +"""setup.sh and setup.ps1 must map UNSLOTH_LLAMA_CPP_BACKEND=cpu to +install_llama_prebuilt.py's --force-cpu so users can force the CPU-only prebuilt +on GPU hosts (#7213). The match is case-insensitive and whitespace-trimmed, an +unrecognized value warns instead of silently falling back, and macOS warns (no +CPU-only bundle). Runs the real block extracted from each script so the tests +track the shipped logic. +""" + +import os +import re +import shutil +import subprocess +from pathlib import Path + +import pytest + +_STUDIO = Path(__file__).resolve().parents[2] +_SETUP_SH = _STUDIO / "setup.sh" +_SETUP_PS1 = _STUDIO / "setup.ps1" +_SKIP_NO_BASH = pytest.mark.skipif(shutil.which("bash") is None, reason = "bash unavailable") +_SKIP_NO_PWSH = pytest.mark.skipif(shutil.which("pwsh") is None, reason = "pwsh unavailable") + + +def _backend_block() -> str: + text = _SETUP_SH.read_text(encoding = "utf-8") + m = re.search(r"_llama_backend=.*?esac", text, re.DOTALL) + assert m, "UNSLOTH_LLAMA_CPP_BACKEND block not found in setup.sh" + return m.group(0) + + +def _run(value: str | None, system: str = "Linux") -> tuple[list[str], str]: + # Pass the value through env (not the script text) so whitespace survives, and + # stub the setup.sh logging helpers the unknown-value branch calls. system sets + # _HOST_SYSTEM so the macOS (Darwin) no-op branch can be exercised. + env = {k: v for k, v in os.environ.items() if k != "UNSLOTH_LLAMA_CPP_BACKEND"} + if value is not None: + env["UNSLOTH_LLAMA_CPP_BACKEND"] = value + harness = ( + f'_PREBUILT_CMD=()\nC_WARN=""\n_HOST_SYSTEM="{system}"\n' + 'step() { printf "STEP: %s\\n" "$*" >&2; }\n' + f"{_backend_block()}\n" + 'printf "%s\\n" "${_PREBUILT_CMD[@]}"' + ) + out = subprocess.run( + ["bash", "-c", harness], capture_output = True, text = True, env = env, check = True + ) + return out.stdout.split(), out.stderr + + +@_SKIP_NO_BASH +@pytest.mark.parametrize("value", ["cpu", "CPU", "Cpu", " cpu ", "CPU\t"]) +def test_backend_cpu_appends_flag(value): + # A deliberate CPU choice persists, so it uses --force-cpu (not the transient + # --cpu-fallback the arm64 GPU-build recovery uses). + args, stderr = _run(value) + assert "--force-cpu" in args + assert "--cpu-fallback" not in args + assert "Ignoring" not in stderr + + +@_SKIP_NO_BASH +@pytest.mark.parametrize("value", ["cpu", "CPU", " cpu "]) +def test_backend_cpu_macos_warns_no_flag(value): + # macOS has no CPU-only bundle (the universal build already runs on CPU), so the + # override warns instead of writing a misleading forced-CPU marker. + args, stderr = _run(value, system = "Darwin") + assert "--force-cpu" not in args + assert "--cpu-fallback" not in args + assert "macOS" in stderr + + +@_SKIP_NO_BASH +@pytest.mark.parametrize("value", [None, "", "auto", "AUTO", " "]) +def test_backend_auto_no_flag_no_warn(value): + args, stderr = _run(value) + assert "--force-cpu" not in args + assert "Ignoring" not in stderr + + +@_SKIP_NO_BASH +@pytest.mark.parametrize("value", ["vulkan", "gpu", "cuda"]) +def test_backend_unknown_warns_and_no_flag(value): + args, stderr = _run(value) + assert "--force-cpu" not in args + assert "Ignoring" in stderr + + +@_SKIP_NO_BASH +def test_arm64_recovery_uses_transient_cpu_fallback(): + # The arm64 Linux GPU-build recovery must stay transient (--cpu-fallback), never + # the persisted --force-cpu, so a later update can still heal to a GPU bundle (#6097). + text = _SETUP_SH.read_text(encoding = "utf-8") + m = re.search(r"_ARM64_CPU_CMD=\((.*?)\)", text, re.DOTALL) + assert m, "arm64 CPU recovery command not found in setup.sh" + block = m.group(1) + assert "--cpu-fallback" in block + assert "--force-cpu" not in block + + +def _ps1_search(pattern: str, flags = 0) -> str: + m = re.search(pattern, _SETUP_PS1.read_text(encoding = "utf-8"), flags) + assert m, f"setup.ps1 block not found: {pattern}" + return m.group(0) + + +def _run_ps1(value: str | None) -> str: + # The override is normalized (assign + warn) at the top of the prebuilt block and + # applied to $prebuiltArgs lower down; compose both real snippets. + normalize = _ps1_search( + r'\$llamaBackend = "\$\(\$env:UNSLOTH_LLAMA_CPP_BACKEND\)".*?Write-Host.*?\n\s*\}', + re.DOTALL, + ) + apply_flag = _ps1_search( + r'if \(\$llamaBackend -eq "cpu"\) \{\s*\$prebuiltArgs \+= "--force-cpu"\s*\}' + ) + env = {k: v for k, v in os.environ.items() if k != "UNSLOTH_LLAMA_CPP_BACKEND"} + if value is not None: + env["UNSLOTH_LLAMA_CPP_BACKEND"] = value + harness = f'$prebuiltArgs = @()\n{normalize}\n{apply_flag}\n"ARGS:" + ($prebuiltArgs -join ",")' + out = subprocess.run( + ["pwsh", "-NoProfile", "-Command", harness], + capture_output = True, + text = True, + env = env, + check = True, + ) + return out.stdout + + +@_SKIP_NO_PWSH +@pytest.mark.parametrize("value", ["cpu", "CPU", "Cpu", " cpu ", "CPU\t"]) +def test_ps1_backend_cpu_appends_flag(value): + out = _run_ps1(value) + assert "--force-cpu" in out + assert "Ignoring" not in out + + +@_SKIP_NO_PWSH +@pytest.mark.parametrize("value", [None, "", "auto", "AUTO", " "]) +def test_ps1_backend_auto_no_flag_no_warn(value): + out = _run_ps1(value) + assert "--force-cpu" not in out + assert "Ignoring" not in out + + +@_SKIP_NO_PWSH +@pytest.mark.parametrize("value", ["vulkan", "gpu", "cuda"]) +def test_ps1_backend_unknown_warns_and_no_flag(value): + out = _run_ps1(value) + assert "--force-cpu" not in out + assert "Ignoring" in out diff --git a/studio/backend/tests/test_sf_client_tools_passthrough.py b/studio/backend/tests/test_sf_client_tools_passthrough.py index 01905b712c..f91eec9817 100644 --- a/studio/backend/tests/test_sf_client_tools_passthrough.py +++ b/studio/backend/tests/test_sf_client_tools_passthrough.py @@ -177,6 +177,15 @@ def _sse_objects(chunks): # ── Non-streaming ───────────────────────────────────────────────── +def test_non_reasoning_backend_keeps_literal_think_tags(monkeypatch): + backend = _ScriptedBackend(_fixed("show example tags")) + response = _call(_request(stream = False), monkeypatch, backend, supports_tools = False) + + message = _json_body(response)["choices"][0]["message"] + assert message["content"] == "show example tags" + assert message["reasoning_content"] is None + + def test_xml_healed_to_tool_calls_non_streaming(monkeypatch): backend = _ScriptedBackend(_fixed(_CALL_XML)) payload = _request(tools = [LOOKUP_TOOL], stream = False) @@ -485,6 +494,52 @@ def test_streaming_no_tools_verbatim(monkeypatch): assert finishes == ["stop"] +def test_streaming_gen_stream_error_is_not_model_text(monkeypatch): + from core.inference.orchestrator import GenStreamError + + class _ErrorAfterPartial(_ScriptedBackend): + def __init__(self): + super().__init__(_fixed()) + + def generate_chat_response(self, **_kwargs): + yield "partial" + yield GenStreamError("Error: /tmp/secret traceback") + + backend = _ErrorAfterPartial() + payload = _request(stream = True) + response = _call(payload, monkeypatch, backend, supports_tools = False) + chunks = _collect_sse(response) + objs = _sse_objects(chunks) + + deltas = [o.get("choices", [{}])[0].get("delta", {}) for o in objs if o.get("choices")] + assert any("partial" in json.dumps(delta) for delta in deltas) + assert not any("/tmp/secret" in json.dumps(delta) for delta in deltas) + errors = [o["error"]["message"] for o in objs if "error" in o] + assert errors == ["An internal error occurred."] + assert any( + "data: [DONE]" in (chunk.decode() if isinstance(chunk, bytes) else chunk) + for chunk in chunks + ) + + +def test_server_tool_streaming_invalid_event_is_error(monkeypatch): + class _InvalidEventBackend(_ScriptedBackend): + def __init__(self): + super().__init__(_fixed()) + + def generate_chat_completion_with_tools(self, **_kwargs): + yield {"type": "content", "text": "partial"} + yield "not-an-event" + + backend = _InvalidEventBackend() + payload = _request(tools = [LOOKUP_TOOL], enable_tools = True, stream = True) + response = _call(payload, monkeypatch, backend) + objs = _sse_objects(_collect_sse(response)) + + errors = [o["error"]["message"] for o in objs if "error" in o] + assert errors == ["An internal error occurred."] + + def test_streaming_repeated_snapshot_no_duplicate_call(monkeypatch): # Repeated then shrunk cumulative snapshots must not double-heal. backend = _ScriptedBackend(_fixed(_CALL_XML, _CALL_XML, _CALL_XML[:5], _CALL_XML)) diff --git a/studio/backend/tests/test_slot_offload_fit.py b/studio/backend/tests/test_slot_offload_fit.py index ac606e4627..d354c7e113 100644 --- a/studio/backend/tests/test_slot_offload_fit.py +++ b/studio/backend/tests/test_slot_offload_fit.py @@ -3,7 +3,7 @@ """Tests for the offload-avoidance serving-slot reduction (`_slots_that_fit_on_gpu`). -When a pinned context does not fit at the requested `--parallel` slot count, Studio would +When a pinned context does not fit at the requested `--parallel` slot count, Unsloth would flip to `--fit on` and llama-server offloads layers to host RAM, collapsing decode ~3x (oobabooga #6718). Instead the loader retries the on-GPU fit at fewer slots and keeps the largest count that stays fully on GPU (`-ngl -1`). These tests drive the real helper with diff --git a/studio/backend/tests/test_stt_download_validation.py b/studio/backend/tests/test_stt_download_validation.py new file mode 100644 index 0000000000..a612b14531 --- /dev/null +++ b/studio/backend/tests/test_stt_download_validation.py @@ -0,0 +1,168 @@ +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 + +"""The /audio/stt/download route must validate a custom Transformers repo before +snapshot_download pulls it into the shared HF cache. + +Regression for a Codex finding: the Transformers engine accepts arbitrary +`owner/model` repos, so an authenticated caller could make Studio download a +large non-STT repository before load-time validation ever ran. Whisper- +compatibility is now enforced (metadata-only, no weights) before the background +download starts. The GGUF engine only accepts curated ids, so it is not gated. +""" + +from __future__ import annotations + +import asyncio +import sys +from pathlib import Path + +import pytest +from fastapi import HTTPException + +_BACKEND_ROOT = Path(__file__).resolve().parents[1] +if str(_BACKEND_ROOT) not in sys.path: + sys.path.insert(0, str(_BACKEND_ROOT)) + +import core.inference.stt_ggml_sidecar as ggml_module # noqa: E402 +import core.inference.stt_sidecar as stt_module # noqa: E402 +import routes.inference as ri # noqa: E402 +from core.inference.stt_sidecar import SttModelCompatibilityError # noqa: E402 +from models.inference import SttLoadRequest # noqa: E402 + + +def _run(coro): + return asyncio.run(coro) + + +def test_custom_non_whisper_repo_is_rejected_before_download(monkeypatch): + started: list = [] + validated: list = [] + + def fake_validate(model, hf_token = None): + validated.append(model) + raise SttModelCompatibilityError( + f"STT model '{model}' is not a compatible Transformers Whisper model." + ) + + def fake_download(model, hf_token = None): + started.append(model) + + monkeypatch.setattr(stt_module, "validate_remote_model", fake_validate) + monkeypatch.setattr(stt_module, "start_model_download", fake_download) + + with pytest.raises(HTTPException) as excinfo: + _run( + ri.stt_download( + SttLoadRequest(model = "owner/chat-model", engine = "transformers"), + current_subject = "tester", + hf_token = None, + ) + ) + + assert excinfo.value.status_code == 422 + assert validated == ["owner/chat-model"] + # The download never starts for a repo that failed the Whisper check. + assert started == [] + + +def test_validated_transformers_repo_downloads(monkeypatch): + started: list = [] + revision = "a" * 40 + + monkeypatch.setattr( + stt_module, + "validate_remote_model", + lambda model, hf_token = None: {"model": model, "revision": revision}, + ) + monkeypatch.setattr( + stt_module, + "start_model_download", + lambda model, hf_token = None, revision = None: started.append((model, revision)), + ) + monkeypatch.setattr(stt_module, "download_status", lambda: {"downloading": True}) + + resp = _run( + ri.stt_download( + SttLoadRequest(model = "owner/real-whisper", engine = "transformers"), + current_subject = "tester", + hf_token = None, + ) + ) + + assert resp.status_code == 200 + assert started == [("owner/real-whisper", revision)] + + +def test_gguf_engine_skips_the_transformers_repo_check(monkeypatch): + started: list = [] + + def fail_if_called(model, hf_token = None): + raise AssertionError("GGUF downloads must not run the Transformers repo check") + + # whisper-server present, so the GGUF request stays on the GGUF engine. + monkeypatch.setattr(ggml_module, "is_available", lambda: True) + monkeypatch.setattr(stt_module, "validate_remote_model", fail_if_called) + monkeypatch.setattr( + ggml_module, "start_model_download", lambda model, hf_token = None: started.append(model) + ) + monkeypatch.setattr(ggml_module, "download_status", lambda: {"downloading": True}) + + resp = _run( + ri.stt_download( + SttLoadRequest(model = "small", engine = "gguf"), + current_subject = "tester", + hf_token = None, + ) + ) + + assert resp.status_code == 200 + assert started == ["small"] + + +def test_resolve_serving_stt_engine_falls_back_when_whisper_server_absent(monkeypatch): + # A curated GGUF request downgrades to Transformers when whisper-server is not + # installed (both engines serve curated ids), but stays GGUF when it is. + monkeypatch.setattr(ggml_module, "is_available", lambda: False) + assert ri._resolve_serving_stt_engine("gguf") == "transformers" + monkeypatch.setattr(ggml_module, "is_available", lambda: True) + assert ri._resolve_serving_stt_engine("gguf") == "gguf" + # Transformers is unaffected by whisper-server availability. + monkeypatch.setattr(ggml_module, "is_available", lambda: False) + assert ri._resolve_serving_stt_engine("transformers") == "transformers" + + +def test_gguf_download_falls_back_to_transformers_when_server_absent(monkeypatch): + """Selecting the default curated model on a host without whisper-server must + download through the Transformers engine, not 501/dead-end on GGUF.""" + gguf_started: list = [] + tf_started: list = [] + + monkeypatch.setattr(ggml_module, "is_available", lambda: False) # no whisper-server + # validate_remote_model no-ops curated ids in production; keep it a no-op here. + monkeypatch.setattr( + stt_module, "validate_remote_model", lambda model, hf_token = None: {"model": model} + ) + monkeypatch.setattr( + stt_module, + "start_model_download", + lambda model, hf_token = None, revision = None: tf_started.append(model), + ) + monkeypatch.setattr(stt_module, "download_status", lambda: {"downloading": True}) + monkeypatch.setattr( + ggml_module, + "start_model_download", + lambda model, hf_token = None: gguf_started.append(model), + ) + + resp = _run( + ri.stt_download( + SttLoadRequest(model = "small", engine = "gguf"), + current_subject = "tester", + hf_token = None, + ) + ) + + assert resp.status_code == 200 + assert tf_started == ["small"] # served by Transformers instead of dead-ending on GGUF + assert gguf_started == [] diff --git a/studio/backend/tests/test_stt_ggml_sidecar.py b/studio/backend/tests/test_stt_ggml_sidecar.py new file mode 100644 index 0000000000..686fd8f546 --- /dev/null +++ b/studio/backend/tests/test_stt_ggml_sidecar.py @@ -0,0 +1,780 @@ +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 + +import http.server +import io +import json +import os +import sys +import threading +import time +import wave +from pathlib import Path + +import numpy as np +import pytest + +import core.inference.stt_ggml_sidecar as ggml_module +from core.inference.stt_ggml_sidecar import ( + DEFAULT_GGML_STT_MODEL, + GGML_STT_MODELS, + GGML_STT_REPOS, + GgmlSttSidecar, + SttEngineUnavailableError, + find_whisper_server_binary, + resolve_ggml_model_id, +) +from core.inference.stt_sidecar import ( + SttLanguageError, + SttLoadCancelledError, + SttModelIdError, + SttModelNotDownloadedError, + SttUnavailableError, +) + + +@pytest.fixture(autouse = True) +def isolate_runtime_and_stub_audio_decoder(monkeypatch, tmp_path): + """Unit tests exercise orchestration, not PyAV container parsing.""" + monkeypatch.setenv("UNSLOTH_STUDIO_HOME", str(tmp_path / "studio")) + monkeypatch.delenv("WHISPER_SERVER_PATH", raising = False) + monkeypatch.delenv("UNSLOTH_WHISPER_CPP_PATH", raising = False) + monkeypatch.setenv("PATH", "") + monkeypatch.setattr( + ggml_module, + "_decode_audio_bounded", + lambda audio: np.zeros(16000, dtype = np.float32), + ) + + +# --------------------------------------------------------------------------- +# Model id resolution +# --------------------------------------------------------------------------- + + +def test_curated_ids_resolve(): + for model_id in GGML_STT_MODELS: + assert resolve_ggml_model_id(model_id) == model_id + + +def test_default_model_resolves_from_none_and_blank(): + assert resolve_ggml_model_id(None) == DEFAULT_GGML_STT_MODEL + assert resolve_ggml_model_id(" ") == DEFAULT_GGML_STT_MODEL + + +def test_custom_repo_ids_are_rejected(): + with pytest.raises(SttModelIdError): + resolve_ggml_model_id("owner/model") + with pytest.raises(SttModelIdError): + resolve_ggml_model_id("large-v2") + + +def test_curated_ids_mirror_transformers_sidecar(): + from core.inference.stt_sidecar import STT_MODELS + assert list(GGML_STT_MODELS.keys()) == list(STT_MODELS.keys()) + + +def test_curated_filenames_match_repo_naming(): + # unslothai/whisper--GGUF hosts whisper-.bin; keep the download + # filename in lockstep with the repo so it resolves instead of 404ing. + for model_id, repo in GGML_STT_REPOS.items(): + expected = repo.split("/", 1)[1].removesuffix("-GGUF") + ".bin" + assert GGML_STT_MODELS[model_id] == expected + + +# --------------------------------------------------------------------------- +# Binary discovery +# --------------------------------------------------------------------------- + + +def test_env_binary_override_wins(monkeypatch, tmp_path): + binary = tmp_path / "whisper-server" + binary.write_text("#!/bin/sh\n") + binary.chmod(0o755) # find_whisper_server_binary requires an executable + monkeypatch.setenv("WHISPER_SERVER_PATH", str(binary)) + assert find_whisper_server_binary() == str(binary) + + +def test_env_dir_override_scans_layouts(monkeypatch, tmp_path): + monkeypatch.delenv("WHISPER_SERVER_PATH", raising = False) + build_bin = tmp_path / "build" / "bin" + build_bin.mkdir(parents = True) + binary = build_bin / "whisper-server" + binary.write_text("#!/bin/sh\n") + binary.chmod(0o755) # find_whisper_server_binary requires an executable + monkeypatch.setenv("UNSLOTH_WHISPER_CPP_PATH", str(tmp_path)) + assert find_whisper_server_binary() == str(binary) + + +def test_missing_binary_reports_unavailable(monkeypatch, tmp_path): + monkeypatch.delenv("WHISPER_SERVER_PATH", raising = False) + monkeypatch.setenv("UNSLOTH_WHISPER_CPP_PATH", str(tmp_path / "nope")) + monkeypatch.setattr(ggml_module, "_managed_whisper_cpp_dir", lambda: tmp_path / "gone") + monkeypatch.setattr(ggml_module.shutil, "which", lambda name: None) + assert find_whisper_server_binary() is None + assert not ggml_module.is_available() + with pytest.raises(SttEngineUnavailableError): + ggml_module.ensure_engine_available() + + +def test_non_executable_binary_is_not_runnable(monkeypatch, tmp_path): + if sys.platform == "win32": + pytest.skip("X_OK is an existence check on Windows") + binary = tmp_path / "whisper-server" + binary.write_text("#!/bin/sh\n") # written but not chmod +x + monkeypatch.setenv("WHISPER_SERVER_PATH", str(binary)) + monkeypatch.setattr(ggml_module.shutil, "which", lambda name: None) + assert find_whisper_server_binary() is None + + +# --------------------------------------------------------------------------- +# Slim-install launch guard +# --------------------------------------------------------------------------- + + +def _slim_install( + tmp_path, + *, + install_kind = "slim", + with_ggml = True, + linked_libraries = None, + backend = "cpu", + linked_runtime_directories = None, + runtime_wiring_version = None, +) -> str: + """A managed-looking install tree: marker at the root, server in build/bin.""" + install_dir = tmp_path / "whisper.cpp" + bin_dir = install_dir / "build" / "bin" + bin_dir.mkdir(parents = True) + binary = bin_dir / "whisper-server" + binary.write_text("#!/bin/sh\n") + binary.chmod(0o755) + marker: dict = { + "schema_version": 1, + "component": "whisper.cpp", + "release_tag": "v1.9.1-unsloth.1", + "backend": backend, + "paired_llama_tag": "b10069-mix-fb3d4ca", + } + if install_kind is not None: + marker["install_kind"] = install_kind + if linked_libraries is not None: + marker["linked_libraries"] = linked_libraries + if linked_runtime_directories is not None: + marker["linked_runtime_directories"] = linked_runtime_directories + for name in linked_runtime_directories: + catalog = bin_dir / name + catalog.mkdir() + (catalog / "kernel.dat").write_bytes(b"kernel") + if runtime_wiring_version is not None: + marker["runtime_wiring_version"] = runtime_wiring_version + (install_dir / "UNSLOTH_WHISPER_PREBUILT_INFO.json").write_text(json.dumps(marker)) + if with_ggml: + names = ( + ("ggml.dll", "ggml-base.dll") + if sys.platform == "win32" + else ("libggml.so.0", "libggml-base.so.0") + ) + for name in names: + (bin_dir / name).write_bytes(b"ggml") + return str(binary) + + +def test_slim_guard_flags_missing_ggml_links(monkeypatch, tmp_path): + # A slim marker whose linked ggml runtime is gone must read as engine + # unavailable (reinstall), never crash into a server launch. + binary = _slim_install(tmp_path, with_ggml = False) + assert ggml_module.slim_runtime_intact(binary) is False + monkeypatch.setattr(ggml_module, "find_whisper_server_binary", lambda: binary) + assert not ggml_module.is_available() + with pytest.raises(SttEngineUnavailableError, match = "ggml"): + ggml_module.ensure_engine_available() + + +def test_slim_guard_passes_with_links_in_place(monkeypatch, tmp_path): + names = ["libggml.so.0", "libggml-base.so.0"] + binary = _slim_install(tmp_path, with_ggml = True, linked_libraries = names) + assert ggml_module.slim_runtime_intact(binary) is True + monkeypatch.setattr(ggml_module, "find_whisper_server_binary", lambda: binary) + assert ggml_module.ensure_engine_available() == binary + + +def test_slim_guard_verifies_the_marker_linked_libraries(monkeypatch, tmp_path): + # New markers record the exact wired filenames; one missing name flips the + # install to unavailable even when the legacy core ggml names are present. + names = ["libggml.dylib", "libggml-base.dylib", "libggml-metal.dylib"] + binary = _slim_install(tmp_path, with_ggml = True, linked_libraries = names) + bin_dir = Path(binary).parent + for name in names[:-1]: + (bin_dir / name).write_bytes(b"ggml") + assert ggml_module.slim_runtime_intact(binary) is False # metal dylib absent + (bin_dir / names[-1]).write_bytes(b"ggml") + assert ggml_module.slim_runtime_intact(binary) is True + monkeypatch.setattr(ggml_module, "find_whisper_server_binary", lambda: binary) + assert ggml_module.ensure_engine_available() == binary + + +def test_slim_guard_malformed_authoritative_marker_fails_closed(tmp_path): + for bad in ("not-a-list", [], [1, 2]): + root = tmp_path / f"case_{type(bad).__name__}_{len(str(bad))}" + root.mkdir() + binary = _slim_install(root, with_ggml = True, linked_libraries = bad) + assert ggml_module.slim_runtime_intact(binary) is False + + +def test_slim_guard_prefers_authoritative_root_marker(tmp_path): + names = ["libggml.so.0", "libggml-base.so.0"] + binary = _slim_install(tmp_path, with_ggml = True, linked_libraries = names) + packaging_marker = Path(binary).parent / "UNSLOTH_WHISPER_PREBUILT_INFO.json" + packaging_marker.write_text(json.dumps({"backend": "slim", "release_tag": "packaging"})) + assert ggml_module._whisper_install_marker(binary)["install_kind"] == "slim" + assert ggml_module.slim_runtime_intact(binary) is True + + +def test_slim_guard_rejects_invalid_root_even_with_inner_marker(tmp_path): + binary = _slim_install(tmp_path, with_ggml = True, linked_libraries = ["libggml.so.0"]) + root_marker = Path(binary).parents[2] / "UNSLOTH_WHISPER_PREBUILT_INFO.json" + root_marker.write_text("not json") + (Path(binary).parent / root_marker.name).write_text(json.dumps({"backend": "slim"})) + assert ggml_module.slim_runtime_intact(binary) is False + + +def test_slim_guard_rejects_missing_rocm_catalog(tmp_path): + names = ["libggml.so.0", "libggml-base.so.0", "libggml-hip.so"] + binary = _slim_install( + tmp_path, + linked_libraries = names, + backend = "rocm", + linked_runtime_directories = ["hipblaslt", "rocblas"], + runtime_wiring_version = 2, + ) + bin_dir = Path(binary).parent + (bin_dir / "libggml-hip.so").write_bytes(b"ggml") + assert ggml_module.slim_runtime_intact(binary) is True + (bin_dir / "rocblas" / "kernel.dat").unlink() + assert ggml_module.slim_runtime_intact(binary) is False + + +def test_slim_guard_accepts_windows_rocm_dll_overlay(monkeypatch, tmp_path): + monkeypatch.setattr(ggml_module.sys, "platform", "win32") + names = ["ggml.dll", "ggml-base.dll", "ggml-hip.dll", "amdhip64.dll"] + binary = _slim_install( + tmp_path, + linked_libraries = names, + backend = "rocm", + linked_runtime_directories = [], + runtime_wiring_version = 2, + ) + for name in names: + (Path(binary).parent / name).write_bytes(b"dll") + assert ggml_module.slim_runtime_intact(binary) is True + + +def test_slim_guard_ignores_fat_and_markerless_installs(tmp_path): + # Fat installs carry their own ggml; no marker means source/custom build. + fat = _slim_install(tmp_path / "fat", install_kind = None, with_ggml = False) + assert ggml_module.slim_runtime_intact(fat) is True + bare = tmp_path / "bare" / "whisper-server" + bare.parent.mkdir(parents = True) + bare.write_text("#!/bin/sh\n") + assert ggml_module.slim_runtime_intact(str(bare)) is True + + +# --------------------------------------------------------------------------- +# whisper-server child-process environment +# --------------------------------------------------------------------------- + + +def _loader_path_var() -> str: + return {"win32": "PATH", "darwin": "DYLD_LIBRARY_PATH"}.get(sys.platform, "LD_LIBRARY_PATH") + + +def test_child_env_scrubs_secrets_and_adds_lib_dir(monkeypatch, tmp_path): + monkeypatch.setenv("HF_TOKEN", "secret-token") # exact name + monkeypatch.setenv("MY_API_KEY", "nope") # marker substring + monkeypatch.setenv("HTTPS_PROXY", "http://u:p@px:8080") # url-name + monkeypatch.setenv("SOME_REMOTE", "https://u:pw@host/repo") # url-userinfo value + monkeypatch.setenv("STT_KEEPME", "keep") # benign + binary = tmp_path / "whisper-server" + binary.write_text("#!/bin/sh\n") + env = ggml_module._whisper_server_child_env(str(binary)) + for scrubbed in ("HF_TOKEN", "MY_API_KEY", "HTTPS_PROXY", "SOME_REMOTE"): + assert scrubbed not in env + assert env.get("STT_KEEPME") == "keep" + assert str(tmp_path.resolve()) in env[_loader_path_var()].split(os.pathsep) + + +def test_child_env_isolates_home_and_cred_locations(monkeypatch, tmp_path): + # The downloaded server must not see the real home (token caches live + # there) nor explicit cred-store pointers like HF_HOME / NETRC. + monkeypatch.setenv("HOME", "/real/home") + monkeypatch.setenv("HF_HOME", "/real/hf") + monkeypatch.setenv("NETRC", "/real/.netrc") + monkeypatch.setattr(ggml_module, "_managed_whisper_cpp_dir", lambda: tmp_path / "managed") + binary = tmp_path / "whisper-server" + binary.write_text("#!/bin/sh\n") + env = ggml_module._whisper_server_child_env(str(binary)) + assert env["HOME"] == str(tmp_path / "managed" / ".child_home") + assert "HF_HOME" not in env + assert "NETRC" not in env + assert (tmp_path / "managed" / ".child_home").is_dir() + + +def test_child_env_wsl_rocm_prepends_system_hip(monkeypatch, tmp_path): + if sys.platform != "linux": + pytest.skip("WSL ROCm library precedence is Linux-only") + rocm = tmp_path / "rocm-lib" + rocm.mkdir() + bindir = tmp_path / "bin" + bindir.mkdir() + binary = bindir / "whisper-server" + binary.write_text("#!/bin/sh\n") + monkeypatch.setattr(ggml_module, "_wsl_system_rocm_lib_dirs", lambda: [str(rocm)]) + env = ggml_module._whisper_server_child_env(str(binary)) + parts = env["LD_LIBRARY_PATH"].split(os.pathsep) + assert parts[0] == str(rocm.resolve()) # system HIP wins + assert str(bindir.resolve()) in parts # bundle libs still present + assert env.get("HSA_ENABLE_DXG_DETECTION") == "1" + + +def test_child_env_adds_cuda_runtime_dirs_for_cuda_bundle(monkeypatch, tmp_path): + # Versioned CUDA backend modules are valid too. They still need the + # CUDA-from-PyTorch wheel dirs for libcudart/libcublas at launch. + if sys.platform == "darwin": + pytest.skip("no CUDA on macOS") + import utils.prebuilt.runtime_libs as rl + + bindir = tmp_path / "bin" + bindir.mkdir() + (bindir / "whisper-server").write_text("#!/bin/sh\n") + module_name = "ggml-cuda.dll" if sys.platform == "win32" else "libggml-cuda.so.0" + (bindir / module_name).write_text("") + cuda_dir = tmp_path / "nvidia" / "cuda_runtime" / "lib" + cuda_dir.mkdir(parents = True) + monkeypatch.setattr(rl, "python_runtime_dirs", lambda: [str(cuda_dir)]) + env = ggml_module._whisper_server_child_env(str(bindir / "whisper-server")) + parts = env[_loader_path_var()].split(os.pathsep) + assert str(bindir.resolve()) in parts + assert str(cuda_dir.resolve()) in parts + assert parts.index(str(bindir.resolve())) < parts.index(str(cuda_dir.resolve())) + + +def test_child_env_omits_cuda_runtime_dirs_for_cpu_bundle(monkeypatch, tmp_path): + # No libggml-cuda.so beside the binary -> a static CPU/Metal bundle -> the CUDA + # wheel discovery must not run and must not touch the loader path. + if sys.platform == "darwin": + pytest.skip("no CUDA on macOS") + import utils.prebuilt.runtime_libs as rl + + bindir = tmp_path / "bin" + bindir.mkdir() + (bindir / "whisper-server").write_text("#!/bin/sh\n") + cuda_dir = tmp_path / "nvidia" / "cuda_runtime" / "lib" + cuda_dir.mkdir(parents = True) + called = {"n": 0} + + def _fake_dirs(): + called["n"] += 1 + return [str(cuda_dir)] + + monkeypatch.setattr(rl, "python_runtime_dirs", _fake_dirs) + env = ggml_module._whisper_server_child_env(str(bindir / "whisper-server")) + parts = env[_loader_path_var()].split(os.pathsep) + assert str(cuda_dir.resolve()) not in parts + assert called["n"] == 0 + + +def test_engine_unavailable_is_stt_unavailable(): + # Routes map SttUnavailableError to HTTP 501; the engine error must share it. + assert issubclass(SttEngineUnavailableError, SttUnavailableError) + + +# --------------------------------------------------------------------------- +# WAV packaging +# --------------------------------------------------------------------------- + + +def test_pcm_to_wav_bytes_shape_and_rate(): + pcm = np.zeros(3200, dtype = np.float32) + data = ggml_module._pcm_to_wav_bytes(pcm) + with wave.open(io.BytesIO(data)) as w: + assert w.getnchannels() == 1 + assert w.getsampwidth() == 2 + assert w.getframerate() == 16000 + assert w.getnframes() == 3200 + + +def test_pcm_to_wav_bytes_clips_out_of_range(): + pcm = np.array([2.0, -2.0], dtype = np.float32) + data = ggml_module._pcm_to_wav_bytes(pcm) + with wave.open(io.BytesIO(data)) as w: + frames = np.frombuffer(w.readframes(2), dtype = "= {"downloading", "model", "error"} diff --git a/studio/backend/tests/test_stt_review_fixes.py b/studio/backend/tests/test_stt_review_fixes.py new file mode 100644 index 0000000000..e4495506a3 --- /dev/null +++ b/studio/backend/tests/test_stt_review_fixes.py @@ -0,0 +1,219 @@ +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 + +"""Regressions for a fresh review pass on the local STT dictation feature: + +1. Curated GGUF dictation repos (unslothai/whisper-*-GGUF) must be hidden from + chat pickers, not just their Transformers safetensors companions. +2. The GGUF sidecar's loaded_model/device status accessors must be lock-free so + they never block behind an in-flight transcription (which holds self._lock). +3. A "gguf" unload on a host without whisper-server must target the Transformers + fallback that actually served it, and unload-all must attempt both backends + even if one raises. +4. free_stt_model_for_training must free the GGUF sidecar even when the + Transformers unload raises (independent exception boundaries). +""" + +from __future__ import annotations + +import asyncio +import sys +import threading +from pathlib import Path + +import pytest +from fastapi import HTTPException + +_BACKEND_ROOT = Path(__file__).resolve().parents[1] +if str(_BACKEND_ROOT) not in sys.path: + sys.path.insert(0, str(_BACKEND_ROOT)) + + +# 1. Hidden-model GGUF companions ------------------------------------------------ +def test_curated_gguf_dictation_repos_are_hidden(): + from utils.hidden_models import _HIDDEN_STT_REPO_IDS, is_hidden_model + for repo in ( + "unslothai/whisper-tiny-GGUF", + "unslothai/whisper-base-GGUF", + "unslothai/whisper-small-GGUF", + "unslothai/whisper-large-v3-turbo-GGUF", + "unslothai/whisper-large-v3-GGUF", + ): + assert repo in _HIDDEN_STT_REPO_IDS + assert is_hidden_model(repo) is True + # Case-insensitive, matching how the cache stores the repo id. + assert is_hidden_model(repo.lower()) is True + + # A same-prefix but genuinely different repo is NOT hidden. + assert is_hidden_model("unslothai/whisper-large-v3-GGUF-finetune") is False + + +# 2. GGUF status accessors are lock-free ---------------------------------------- +def test_gguf_status_accessors_do_not_block_on_the_inference_lock(): + from core.inference.stt_ggml_sidecar import GgmlSttSidecar + + sidecar = GgmlSttSidecar() + + class _AliveProc: + pid = 4321 + + def poll(self): + return None # still running + + sidecar._process = _AliveProc() + sidecar._model_id = "small" + + holder_has_lock = threading.Event() + release = threading.Event() + + def _hold_inference_lock(): + # Mimic transcribe() holding self._lock across the whole HTTP call. + with sidecar._lock: + holder_has_lock.set() + release.wait(timeout = 5) + + holder = threading.Thread(target = _hold_inference_lock) + holder.start() + assert holder_has_lock.wait(timeout = 5) + + result: dict = {} + + def _read_status(): + result["model"] = sidecar.loaded_model + result["device"] = sidecar.device + + reader = threading.Thread(target = _read_status) + reader.start() + reader.join(timeout = 2) + blocked = reader.is_alive() + + release.set() + holder.join(timeout = 5) + reader.join(timeout = 5) + + assert not blocked, "loaded_model/device blocked on self._lock (should be lock-free)" + assert result == {"model": "small", "device": "whisper.cpp"} + + +def test_process_alive_snapshots_process_against_concurrent_unload(): + # _process_alive() must read self._process exactly once. The lock-free + # readers (loaded_model/device) can run while unload() nulls self._process; + # the old `self._process is not None and self._process.poll() is None` read it + # twice, so a null landing between the two reads called None.poll(). A + # property that yields the live process on the first read and None afterwards + # reproduces that interleaving deterministically. + from core.inference.stt_ggml_sidecar import GgmlSttSidecar + + class _AliveProc: + def poll(self): + return None # still running + + live = _AliveProc() + reads = {"n": 0} + + class _RacingSidecar(GgmlSttSidecar): + @property + def _process(self): + reads["n"] += 1 + return live if reads["n"] == 1 else None + + @_process.setter + def _process(self, value): + pass # __init__ assigns None; the property drives the read + + sidecar = GgmlSttSidecar() + sidecar.__class__ = _RacingSidecar # data descriptor wins over the instance attr + + # Snapshot fix: exactly one read, no AttributeError from a second None read. + assert sidecar._process_alive() is True + assert reads["n"] == 1 + + +# 3. Unload resolves through the serving engine + attempts every backend --------- +def test_gguf_unload_targets_transformers_fallback_without_whisper_server(monkeypatch): + import core.inference.stt_ggml_sidecar as ggml_module + import routes.inference as ri + + monkeypatch.setattr(ggml_module, "is_available", lambda: False) # no whisper-server + + calls: list = [] + + class _Sidecar: + def __init__(self, name): + self.name = name + + def unload(self): + calls.append(self.name) + + monkeypatch.setattr(ri, "_stt_sidecar_for", lambda name: _Sidecar(name)) + + resp = asyncio.run(ri.stt_unload(engine = "gguf", current_subject = "tester")) + assert resp.status_code == 200 + # gguf is served by the Transformers fallback here, so that is what unloads. + assert calls == ["transformers"] + + +def test_unload_all_attempts_both_backends_even_when_one_fails(monkeypatch): + import routes.inference as ri + + attempted: list = [] + + class _Sidecar: + def __init__(self, name): + self.name = name + + def unload(self): + attempted.append(self.name) + if self.name == "transformers": + raise RuntimeError("boom") + + monkeypatch.setattr(ri, "_stt_sidecar_for", lambda name: _Sidecar(name)) + + with pytest.raises(HTTPException) as excinfo: + asyncio.run(ri.stt_unload(engine = None, current_subject = "tester")) + + assert excinfo.value.status_code == 500 + # gguf is still attempted after the transformers unload raised. + assert attempted == ["transformers", "gguf"] + + +# 4. free_stt_model_for_training isolates the two backends ----------------------- +def test_free_stt_frees_gguf_even_when_transformers_unload_raises(monkeypatch): + import routes.training_vram as tv + + class _TransformersSidecar: + def is_loading(self): + return False + + @property + def loaded_model(self): + return "whisper-small" + + def unload(self): + raise RuntimeError("transformers unload failed") + + class _GgmlSidecar: + def __init__(self): + self.unloaded = False + + def is_loading(self): + return False + + @property + def loaded_model(self): + return None if self.unloaded else "small" + + def unload(self): + self.unloaded = True + + ggml = _GgmlSidecar() + monkeypatch.setattr( + "core.inference.stt_sidecar.get_stt_sidecar", lambda: _TransformersSidecar() + ) + monkeypatch.setattr("core.inference.stt_ggml_sidecar.get_ggml_stt_sidecar", lambda: ggml) + + freed = tv.free_stt_model_for_training("test") + + # The Transformers failure must not skip GGUF eviction. + assert ggml.unloaded is True + assert any("small" in entry for entry in freed) diff --git a/studio/backend/tests/test_stt_review_fixes_2.py b/studio/backend/tests/test_stt_review_fixes_2.py new file mode 100644 index 0000000000..f0bdab42b5 --- /dev/null +++ b/studio/backend/tests/test_stt_review_fixes_2.py @@ -0,0 +1,350 @@ +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 + +"""Regressions for the second review pass on the local STT dictation feature: + +1. scripts/build_whisper_cpp.sh must not rm -rf a whisper.cpp/src tree under a + custom Studio home unless Studio itself created it (ownership marker), the + same policy studio/setup.sh applies before its destructive replacements. +2. _snapshot_is_complete must reject pickle (pytorch_model.bin) checkpoints + outright; only safetensors weights count as a usable snapshot. +3. _snapshot_is_complete must require tokenizer assets (tokenizer.json or + vocab.json + merges.txt); weights + config alone decode to blank text. +4. Custom-repo downloads must pin the revision validated beforehand and + restrict snapshot_download to the model/tokenizer/config/preprocessor file + classes (TOCTOU + unbounded-download hardening). +5. The GGML sidecar's readiness probe must not treat an arbitrary local HTTP + responder as whisper-server (mic audio would be posted to it), and the port + reservation must stay held until just before spawn. +""" + +from __future__ import annotations + +import http.server +import json +import os +import socket +import stat +import subprocess +import sys +import threading +from pathlib import Path +from types import SimpleNamespace + +import pytest + +_BACKEND_ROOT = Path(__file__).resolve().parents[1] +if str(_BACKEND_ROOT) not in sys.path: + sys.path.insert(0, str(_BACKEND_ROOT)) + +import core.inference.stt_ggml_sidecar as ggml_module +import core.inference.stt_sidecar as stt_sidecar_module +from core.inference.stt_ggml_sidecar import GgmlSttSidecar, SttEngineUnavailableError +from core.inference.stt_sidecar import validate_remote_model + +_BUILD_SCRIPT = _BACKEND_ROOT.parents[1] / "scripts" / "build_whisper_cpp.sh" + + +# 1. build_whisper_cpp.sh ownership gate ---------------------------------------- + + +def _stub_tools(tmp_path: Path) -> dict: + """PATH with git/cmake stubs so the script never reaches a real build.""" + bin_dir = tmp_path / "stub-bin" + bin_dir.mkdir(exist_ok = True) + for tool in ("git", "cmake"): + stub = bin_dir / tool + stub.write_text("#!/bin/sh\necho stub-%s-invoked >&2\nexit 1\n" % tool) + stub.chmod(stub.stat().st_mode | stat.S_IEXEC) + env = dict(os.environ) + env["PATH"] = f"{bin_dir}:{env['PATH']}" + return env + + +def _run_build_script(env: dict) -> subprocess.CompletedProcess: + return subprocess.run( + ["sh", str(_BUILD_SCRIPT)], + env = env, + capture_output = True, + text = True, + timeout = 60, + ) + + +def test_build_script_refuses_unowned_dir_in_custom_studio_home(tmp_path): + home = tmp_path / "studio-home" + src = home / "whisper.cpp" / "src" + src.mkdir(parents = True) + user_file = src / "user-data.txt" + user_file.write_text("precious") + + env = _stub_tools(tmp_path) + env["UNSLOTH_STUDIO_HOME"] = str(home) + result = _run_build_script(env) + + assert result.returncode != 0 + assert "not marked as an Unsloth-owned" in result.stderr + # The unowned tree, and the user's file inside it, survived untouched. + assert user_file.read_text() == "precious" + + +def test_build_script_proceeds_when_marker_present(tmp_path): + home = tmp_path / "studio-home" + install = home / "whisper.cpp" + (install / "src").mkdir(parents = True) + (install / ".unsloth-studio-owned").write_text("") + + env = _stub_tools(tmp_path) + env["UNSLOTH_STUDIO_HOME"] = str(home) + result = _run_build_script(env) + + # Past the guard: it fails later at the stubbed git clone, not the gate. + assert "not marked as an Unsloth-owned" not in result.stderr + assert "stub-git-invoked" in result.stderr + + +def test_build_script_marks_fresh_custom_install_dir(tmp_path): + home = tmp_path / "studio-home" + home.mkdir() + + env = _stub_tools(tmp_path) + env["UNSLOTH_STUDIO_HOME"] = str(home) + _run_build_script(env) + + # A directory the script creates is marked so re-runs stay allowed. + assert (home / "whisper.cpp" / ".unsloth-studio-owned").is_file() + + +def test_build_script_keeps_legacy_home_behavior(tmp_path): + fake_home = tmp_path / "user-home" + src = fake_home / ".unsloth" / "whisper.cpp" / "src" + src.mkdir(parents = True) + + env = _stub_tools(tmp_path) + env.pop("UNSLOTH_STUDIO_HOME", None) + env.pop("STUDIO_HOME", None) + env["HOME"] = str(fake_home) + result = _run_build_script(env) + + # The legacy managed dir is always Studio-owned; no gate, straight to git. + assert "not marked as an Unsloth-owned" not in result.stderr + assert "stub-git-invoked" in result.stderr + + +# 2 + 3. _snapshot_is_complete -------------------------------------------------- + + +def _base_snapshot(tmp_path: Path) -> Path: + snap = tmp_path / "snap" + snap.mkdir() + (snap / "config.json").write_text("{}") + (snap / "preprocessor_config.json").write_text("{}") + (snap / "tokenizer.json").write_text("{}") + return snap + + +def test_pickle_checkpoint_snapshot_is_never_complete(tmp_path): + # A cached pytorch_model.bin is a pickle RCE load path; the snapshot must + # read as incomplete no matter how many shards are present, so update + # re-resolves and _select_snapshot_files fails it closed. + snap = _base_snapshot(tmp_path) + index = { + "weight_map": { + "a": "pytorch_model-00001-of-00002.bin", + "b": "pytorch_model-00002-of-00002.bin", + } + } + (snap / "pytorch_model.bin.index.json").write_text(json.dumps(index)) + (snap / "pytorch_model-00001-of-00002.bin").write_bytes(b"w" * 8) + (snap / "pytorch_model-00002-of-00002.bin").write_bytes(b"w" * 8) + assert stt_sidecar_module._snapshot_is_complete(snap) is False + + # A single-file pickle checkpoint is likewise rejected; the safetensors + # equivalent in the same dir makes it complete. + (snap / "pytorch_model.bin").write_bytes(b"w" * 8) + assert stt_sidecar_module._snapshot_is_complete(snap) is False + (snap / "model.safetensors").write_bytes(b"w" * 8) + assert stt_sidecar_module._snapshot_is_complete(snap) is True + + +def test_safe_index_naming_pickle_shards_is_not_complete(tmp_path): + # A safetensors index that references .bin shards would still pickle-load + # via Transformers' per-shard dispatch; the cached snapshot must read as + # incomplete so it re-resolves and fails closed at selection. + snap = _base_snapshot(tmp_path) + (snap / "model.safetensors.index.json").write_text( + json.dumps({"weight_map": {"a": "pytorch_model-00001-of-00001.bin"}}) + ) + (snap / "pytorch_model-00001-of-00001.bin").write_bytes(b"w" * 8) + assert stt_sidecar_module._snapshot_is_complete(snap) is False + + +def test_snapshot_without_tokenizer_assets_is_incomplete(tmp_path): + snap = _base_snapshot(tmp_path) + (snap / "model.safetensors").write_bytes(b"w" * 8) + assert stt_sidecar_module._snapshot_is_complete(snap) is True + + # Weights + config but no tokenizer decodes to blank text; not complete. + (snap / "tokenizer.json").unlink() + assert stt_sidecar_module._snapshot_is_complete(snap) is False + + # The slow vocab.json + merges.txt pair is an accepted alternative. + (snap / "vocab.json").write_text("{}") + assert stt_sidecar_module._snapshot_is_complete(snap) is False + (snap / "merges.txt").write_text("") + assert stt_sidecar_module._snapshot_is_complete(snap) is True + + +# 4. Revision pinning and allow_patterns ---------------------------------------- + + +def test_validate_remote_model_returns_the_validated_revision(monkeypatch): + revision = "a" * 40 + + class _FakeApi: + def __init__(self, token = None): + pass + + def model_info( + self, + repo, + expand = None, + timeout = None, + ): + return SimpleNamespace(config = {"model_type": "whisper"}, sha = revision) + + import huggingface_hub + + monkeypatch.setattr(huggingface_hub, "HfApi", _FakeApi) + result = validate_remote_model("someone/custom-whisper") + assert result["revision"] == revision + + +def test_download_pins_revision_and_limits_patterns(monkeypatch): + captured = {} + validated_revision = "a" * 40 + head_revision = "b" * 40 + + def fake_snapshot_download(**kwargs): + captured.update(kwargs) + return "/cached" + + class _FakeApi: + def __init__(self, token = None): + pass + + def model_info( + self, + repo, + revision = None, + files_metadata = None, + timeout = None, + ): + names = ( + "config.json", + "preprocessor_config.json", + "tokenizer.json", + "model.safetensors", + ) + siblings = [ + SimpleNamespace(rfilename = name, size = 10, blob_id = name, lfs = None) for name in names + ] + return SimpleNamespace(siblings = siblings, sha = head_revision) + + import huggingface_hub + + monkeypatch.setattr(huggingface_hub, "HfApi", _FakeApi) + monkeypatch.setattr(huggingface_hub, "snapshot_download", fake_snapshot_download) + + state = stt_sidecar_module._SnapshotDownloadState() + # The revision resolved at validation time wins over the current head. + state._run("someone/custom-whisper", None, revision = validated_revision) + assert captured["revision"] == validated_revision + patterns = captured["allow_patterns"] + assert "model.safetensors" in patterns and "tokenizer.json" in patterns + # No wildcard that would admit arbitrary repo contents. + assert "*" not in patterns + + # Without a validated revision (curated repos), pin to the metadata head. + captured.clear() + state._run("someone/custom-whisper", None) + assert captured["revision"] == head_revision + assert captured["allow_patterns"] + + +# 5. GGML readiness must identify whisper-server -------------------------------- + + +class _CannedHandler(http.server.BaseHTTPRequestHandler): + body = b"" + + def do_GET(self): # noqa: N802 + payload = type(self).body + self.send_response(200) + self.send_header("Content-Length", str(len(payload))) + self.end_headers() + self.wfile.write(payload) + + def log_message(self, *args): + pass + + +def _serve(body: bytes): + handler = type("Handler", (_CannedHandler,), {"body": body}) + server = http.server.HTTPServer(("127.0.0.1", 0), handler) + thread = threading.Thread(target = server.serve_forever, daemon = True) + thread.start() + return server, server.server_address[1] + + +def _fake_alive_process(): + return SimpleNamespace(poll = lambda: None, pid = 999999) + + +def test_wait_for_server_rejects_a_foreign_http_responder(monkeypatch): + server, port = _serve(b"hello from some other local app") + try: + monkeypatch.setattr(ggml_module, "_SERVER_START_TIMEOUT_SECONDS", 1.0) + with pytest.raises(SttEngineUnavailableError, match = "did not start in time"): + GgmlSttSidecar._wait_for_server(_fake_alive_process(), port) + finally: + server.shutdown() + + +def test_wait_for_server_accepts_the_whisper_server_page(monkeypatch): + server, port = _serve(b"Whisper.cpp Server") + try: + monkeypatch.setattr(ggml_module, "_SERVER_START_TIMEOUT_SECONDS", 5.0) + GgmlSttSidecar._wait_for_server(_fake_alive_process(), port) + finally: + server.shutdown() + + +def test_probe_requires_the_managed_child_to_be_alive(): + server, port = _serve(b"whisper") + try: + dead = SimpleNamespace(poll = lambda: 0, pid = 999999) + assert GgmlSttSidecar._probe_is_whisper_server(dead, port) is False + assert GgmlSttSidecar._probe_is_whisper_server(_fake_alive_process(), port) is True + finally: + server.shutdown() + + +def test_port_reservation_is_held_until_released(): + reservation, port = GgmlSttSidecar._reserve_free_port() + try: + probe = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + try: + with pytest.raises(OSError): + probe.bind(("127.0.0.1", port)) + finally: + probe.close() + finally: + reservation.close() + # Released right before spawn: the port becomes bindable for the child. + child = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + child.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + try: + child.bind(("127.0.0.1", port)) + finally: + child.close() diff --git a/studio/backend/tests/test_stt_sidecar.py b/studio/backend/tests/test_stt_sidecar.py new file mode 100644 index 0000000000..b138f46331 --- /dev/null +++ b/studio/backend/tests/test_stt_sidecar.py @@ -0,0 +1,1302 @@ +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 + +import gc +import io +import json +import sys +import threading +import time +import wave +import weakref +from pathlib import Path +from types import SimpleNamespace + +import numpy as np +import pytest + +import core.inference.stt_sidecar as stt_sidecar_module +from core.inference.stt_sidecar import ( + DEFAULT_STT_MODEL, + STT_MODELS, + SttAudioDecodeError, + SttAudioTooLongError, + SttLanguageError, + SttLoadCancelledError, + SttModelCompatibilityError, + SttModelIdError, + SttModelNotDownloadedError, + SttUnavailableError, + WhisperSttSidecar, + normalize_whisper_language, + resolve_model_id, + resolve_model_repo, + validate_remote_model, +) + +_REAL_DECODE_AUDIO_BOUNDED = stt_sidecar_module._decode_audio_bounded +_REAL_ENSURE_STT_AVAILABLE = stt_sidecar_module.ensure_stt_available +_REAL_SNAPSHOT_IS_COMPLETE = stt_sidecar_module._snapshot_is_complete +_REAL_FIND_COMPLETE_CACHED_SNAPSHOT = stt_sidecar_module._find_complete_cached_snapshot + + +@pytest.fixture(autouse = True) +def stub_audio_decoder(monkeypatch): + """Unit tests below exercise orchestration, not PyAV container parsing.""" + monkeypatch.setattr( + stt_sidecar_module, + "_decode_audio_bounded", + lambda _audio: np.zeros(8000, dtype = np.float32), + ) + monkeypatch.setattr( + "huggingface_hub.snapshot_download", + lambda **_kwargs: "/cached/model", + ) + monkeypatch.setattr( + stt_sidecar_module, + "_find_complete_cached_snapshot", + lambda _model: Path("/cached/model"), + ) + # The stubbed snapshot path holds no files; snapshot-integrity tests + # restore the real check. + monkeypatch.setattr(stt_sidecar_module, "_snapshot_is_complete", lambda _snapshot: True) + # transcribe() gates on the runtime up front; treat it as present so these + # orchestration tests run without PyTorch/Transformers/PyAV installed. + # The runtime-specific tests restore the real check. + monkeypatch.setattr(stt_sidecar_module, "ensure_stt_available", lambda: None) + + +class _CaptureInference: + """Stand-in for the model inference step; records how it was called.""" + + def __init__( + self, + text = "hello", + mutate = None, + ) -> None: + self.text = text + self.mutate = mutate + self.generate_kwargs = None + + def __call__(self, model_id, decoded, generate_kwargs): + self.generate_kwargs = generate_kwargs + if self.mutate is not None: + self.mutate() + return self.text + + +def test_five_curated_whisper_models_are_offered(): + assert STT_MODELS == { + "tiny": "unsloth/whisper-tiny", + "base": "unsloth/whisper-base", + "small": "unsloth/whisper-small", + "large-v3-turbo": "unsloth/whisper-large-v3-turbo", + "large-v3": "unsloth/whisper-large-v3", + } + assert all(repo.startswith(("unsloth/", "unslothai/")) for repo in STT_MODELS.values()) + assert DEFAULT_STT_MODEL in STT_MODELS + + +def test_av_is_required_for_stt_availability(monkeypatch): + monkeypatch.setattr(stt_sidecar_module, "ensure_stt_available", _REAL_ENSURE_STT_AVAILABLE) + monkeypatch.setitem(sys.modules, "torch", SimpleNamespace()) + monkeypatch.setitem(sys.modules, "transformers", SimpleNamespace()) + monkeypatch.setitem(sys.modules, "av", None) + + assert stt_sidecar_module.is_available() is False + + +def test_transformers_is_required_for_stt_availability(monkeypatch): + monkeypatch.setattr(stt_sidecar_module, "ensure_stt_available", _REAL_ENSURE_STT_AVAILABLE) + monkeypatch.setitem(sys.modules, "torch", SimpleNamespace()) + monkeypatch.setitem(sys.modules, "av", SimpleNamespace()) + monkeypatch.setitem(sys.modules, "transformers", None) + + assert stt_sidecar_module.is_available() is False + + +@pytest.mark.parametrize("missing", ["transformers", "av"]) +def test_load_rejects_an_incomplete_stt_runtime(monkeypatch, missing): + sidecar = WhisperSttSidecar(keep_alive_seconds = 0) + monkeypatch.setattr(stt_sidecar_module, "ensure_stt_available", _REAL_ENSURE_STT_AVAILABLE) + for module in ("torch", "transformers", "av"): + monkeypatch.setitem(sys.modules, module, SimpleNamespace()) + monkeypatch.setitem(sys.modules, missing, None) + monkeypatch.setattr( + sidecar, + "_ensure_model_downloaded", + lambda _model: pytest.fail("runtime must be checked before the model cache"), + ) + + with pytest.raises(SttUnavailableError, match = "needs PyTorch, Transformers, and PyAV"): + sidecar.load("small") + + +def test_model_id_accepts_defaults_and_custom_hub_repositories(): + assert resolve_model_id("tiny") == "tiny" + assert resolve_model_id(None) == DEFAULT_STT_MODEL + assert resolve_model_id("large-v3") == "large-v3" + assert resolve_model_id("openai/whisper-medium") == "openai/whisper-medium" + assert resolve_model_repo("tiny") == "unsloth/whisper-tiny" + assert resolve_model_repo("openai/whisper-medium") == "openai/whisper-medium" + + +@pytest.mark.parametrize("model", ["tiny-ish", "owner/model/extra", "../model", "owner/"]) +def test_invalid_custom_model_id_is_rejected(model): + with pytest.raises(SttModelIdError, match = "owner/model"): + resolve_model_id(model) + + +def test_remote_custom_model_validation_requires_whisper_config(monkeypatch): + calls = [] + + class FakeApi: + def __init__(self, token): + calls.append(("token", token)) + + def model_info(self, repo, **kwargs): + calls.append(("model_info", repo, kwargs)) + return SimpleNamespace( + sha = "a" * 40, + config = { + "model_type": "whisper", + "architectures": ["WhisperForConditionalGeneration"], + }, + ) + + monkeypatch.setattr("huggingface_hub.HfApi", FakeApi) + + result = validate_remote_model("owner/custom-whisper", "hf_private") + + assert result == { + "model": "owner/custom-whisper", + "repo": "owner/custom-whisper", + "revision": "a" * 40, + } + assert calls == [ + ("token", "hf_private"), + ( + "model_info", + "owner/custom-whisper", + {"expand": ["config", "sha"], "timeout": 10}, + ), + ] + + +def test_remote_custom_model_validation_rejects_non_whisper(monkeypatch): + class FakeApi: + def __init__(self, token): + assert token is False + + def model_info(self, _repo, **_kwargs): + return SimpleNamespace( + config = { + "model_type": "llama", + "architectures": ["LlamaForCausalLM"], + } + ) + + monkeypatch.setattr("huggingface_hub.HfApi", FakeApi) + + with pytest.raises(SttModelCompatibilityError, match = "not a compatible"): + validate_remote_model("owner/chat-model") + + +def test_remote_custom_model_validation_requires_an_immutable_sha(monkeypatch): + class FakeApi: + def __init__(self, token): + pass + + def model_info(self, _repo, **_kwargs): + return SimpleNamespace(sha = None, config = {"model_type": "whisper"}) + + monkeypatch.setattr("huggingface_hub.HfApi", FakeApi) + + with pytest.raises(SttModelCompatibilityError, match = "immutable revision"): + validate_remote_model("owner/custom-whisper") + + +def test_fast_transcription_uses_greedy_decoding(monkeypatch): + sidecar = WhisperSttSidecar() + infer = _CaptureInference() + monkeypatch.setattr(sidecar, "_transcribe_decoded", infer) + + result = sidecar.transcribe(b"encoded audio", language = "en", fast = True) + + assert result["text"] == "hello" + assert result["duration"] == 0.5 + assert result["model"] == DEFAULT_STT_MODEL + assert infer.generate_kwargs == { + "task": "transcribe", + "condition_on_prev_tokens": False, + "num_beams": 1, + "language": "en", + } + + +def test_accurate_transcription_keeps_beam_search_default(monkeypatch): + sidecar = WhisperSttSidecar() + infer = _CaptureInference() + monkeypatch.setattr(sidecar, "_transcribe_decoded", infer) + + sidecar.transcribe(b"encoded audio") + + assert infer.generate_kwargs == { + "task": "transcribe", + "condition_on_prev_tokens": False, + "num_beams": 5, + } + + +@pytest.mark.parametrize( + ("language", "expected"), + [ + (None, None), + ("auto", None), + ("en-US", "en"), + ("en-GB", "en"), + ("zh-CN", "zh"), + ("ja-JP", "ja"), + ("ko-KR", "ko"), + ("es-ES", "es"), + ("fr-FR", "fr"), + ("de-DE", "de"), + ("it-IT", "it"), + ("pt_BR", "pt"), + ("ru-RU", "ru"), + ("hi-IN", "hi"), + ("ar-SA", "ar"), + ("iw-IL", "he"), + ("nb-NO", "no"), + ], +) +def test_normalize_whisper_language_accepts_bcp47(language, expected): + assert normalize_whisper_language(language) == expected + + +def test_transcription_normalizes_region_qualified_language(monkeypatch): + sidecar = WhisperSttSidecar() + infer = _CaptureInference() + monkeypatch.setattr(sidecar, "_transcribe_decoded", infer) + + sidecar.transcribe(b"encoded audio", language = "fr-FR") + + assert infer.generate_kwargs["language"] == "fr" + + +def test_english_only_model_rejects_non_english_before_decode(monkeypatch, tmp_path): + (tmp_path / "config.json").write_text('{"model_type": "whisper"}') + (tmp_path / "generation_config.json").write_text('{"is_multilingual": false}') + sidecar = WhisperSttSidecar() + + def should_not_decode(_audio): + pytest.fail("English-only language mismatch must be rejected before decode") + + monkeypatch.setattr( + stt_sidecar_module, + "_find_complete_cached_snapshot", + lambda _model: tmp_path, + ) + monkeypatch.setattr(stt_sidecar_module, "_decode_audio_bounded", should_not_decode) + + with pytest.raises(SttLanguageError, match = "English-only"): + sidecar.transcribe( + b"encoded audio", + model = "owner/whisper-small.en", + language = "fr-FR", + ) + + +def test_english_only_model_omits_forbidden_generation_controls(monkeypatch): + calls = [] + + class FakeTensor: + def to(self, *_args): + return self + + class FakeProcessor: + def __call__(self, *_args, **_kwargs): + return SimpleNamespace(input_features = FakeTensor()) + + def batch_decode(self, *_args, **_kwargs): + return ["hello"] + + class FakeModel: + dtype = None + device = "cpu" + generation_config = SimpleNamespace(is_multilingual = False) + + def generate(self, _features, **kwargs): + calls.append(kwargs) + return [[1]] + + class NoGrad: + def __enter__(self): + return None + + def __exit__(self, *_args): + return False + + monkeypatch.setitem(sys.modules, "torch", SimpleNamespace(no_grad = NoGrad)) + sidecar = WhisperSttSidecar() + monkeypatch.setattr(sidecar, "load", lambda _model: (FakeModel(), FakeProcessor())) + + text = sidecar._transcribe_decoded( + "owner/whisper-small.en", + np.zeros(160, dtype = np.float32), + { + "task": "transcribe", + "language": "en", + "condition_on_prev_tokens": False, + "num_beams": 1, + }, + ) + + assert text == "hello" + assert calls == [{"condition_on_prev_tokens": False, "num_beams": 1}] + + +def test_unknown_language_is_rejected_before_decode_or_model_load(monkeypatch): + sidecar = WhisperSttSidecar() + + def should_not_run(*_args, **_kwargs): + pytest.fail("unknown language must be rejected before expensive work") + + monkeypatch.setattr(stt_sidecar_module, "_known_whisper_languages", lambda: frozenset({"en"})) + monkeypatch.setattr(stt_sidecar_module, "_decode_audio_bounded", should_not_run) + monkeypatch.setattr(sidecar, "_transcribe_decoded", should_not_run) + + with pytest.raises(SttLanguageError, match = "is not supported"): + sidecar.transcribe(b"encoded audio", language = "xx-YY") + + +def test_unknown_language_is_not_reported_as_bad_audio(monkeypatch): + sidecar = WhisperSttSidecar() + infer = _CaptureInference() + monkeypatch.setattr(sidecar, "_transcribe_decoded", infer) + monkeypatch.setattr(stt_sidecar_module, "_known_whisper_languages", lambda: frozenset({"en"})) + + with pytest.raises(SttLanguageError, match = "is not supported"): + sidecar.transcribe(b"encoded audio", language = "xx-YY") + + +def test_transcription_result_keeps_requested_model_id_during_switch(monkeypatch): + sidecar = WhisperSttSidecar() + + # Simulate another request changing the mutable resident-model state after + # this request pinned its own model id. + infer = _CaptureInference(mutate = lambda: setattr(sidecar, "_model_id", "large-v3")) + monkeypatch.setattr(sidecar, "_transcribe_decoded", infer) + + result = sidecar.transcribe(b"encoded audio", model = "small") + + assert result["model"] == "small" + + +def test_inference_failure_propagates(monkeypatch): + sidecar = WhisperSttSidecar() + + def boom(*_args, **_kwargs): + raise RuntimeError("inference failed") + + monkeypatch.setattr(sidecar, "_transcribe_decoded", boom) + + with pytest.raises(RuntimeError, match = "inference failed"): + sidecar.transcribe(b"encoded audio") + + +class _FakeModel: + def to(self, *_args, **_kwargs): + return self + + def eval(self): + return self + + +class _FakeTimer: + def __init__( + self, + interval, + function, + args = (), + kwargs = None, + ): + self.interval = interval + self.function = function + self.args = args + self.kwargs = kwargs or {} + self.cancelled = False + self.daemon = False + self.started = False + + def start(self): + self.started = True + + def cancel(self): + self.cancelled = True + + def fire(self): + self.function(*self.args, **self.kwargs) + + +def _install_fake_torch(monkeypatch): + fake_torch = SimpleNamespace( + float16 = "float16", + float32 = "float32", + device = lambda value: value, + cuda = SimpleNamespace(is_available = lambda: False), + backends = SimpleNamespace(mps = SimpleNamespace(is_available = lambda: False)), + ) + monkeypatch.setitem(sys.modules, "torch", fake_torch) + monkeypatch.setitem(sys.modules, "av", SimpleNamespace()) + return fake_torch + + +def test_load_uses_model_hub_cache_without_implicit_download(monkeypatch): + calls = [] + _install_fake_torch(monkeypatch) + + class FakeWhisperForConditionalGeneration: + @classmethod + def from_pretrained(cls, repo, **kwargs): + calls.append(("model", repo, kwargs)) + return _FakeModel() + + class FakeWhisperProcessor: + @classmethod + def from_pretrained(cls, repo, **kwargs): + calls.append(("processor", repo, kwargs)) + return object() + + monkeypatch.setitem( + sys.modules, + "transformers", + SimpleNamespace( + WhisperForConditionalGeneration = FakeWhisperForConditionalGeneration, + WhisperProcessor = FakeWhisperProcessor, + ), + ) + monkeypatch.setattr(stt_sidecar_module, "_pick_device", lambda: ("cpu", "float32")) + + WhisperSttSidecar(keep_alive_seconds = 0).load("small") + + assert {(kind, repo) for kind, repo, _ in calls} == { + ("processor", "/cached/model"), + ("model", "/cached/model"), + } + # Never fetch weights implicitly; the Model Hub owns downloads. + assert all(kwargs.get("local_files_only") is True for _, _, kwargs in calls) + # The weight load forces safetensors so a pickle checkpoint cannot execute. + model_kwargs = next(kwargs for kind, _, kwargs in calls if kind == "model") + assert model_kwargs.get("use_safetensors") is True + + +def test_model_cache_preflight_uses_shared_offline_resolver(monkeypatch): + seen = [] + monkeypatch.setattr( + stt_sidecar_module, + "_find_complete_cached_snapshot", + lambda model: seen.append(model) or Path("/cached/model"), + ) + + WhisperSttSidecar(keep_alive_seconds = 0)._ensure_model_downloaded("small") + + assert seen == ["small"] + + +def test_model_cache_preflight_reports_missing_snapshot(monkeypatch): + monkeypatch.setattr(stt_sidecar_module, "_find_complete_cached_snapshot", lambda _model: None) + + with pytest.raises(SttModelNotDownloadedError, match = "not downloaded"): + WhisperSttSidecar(keep_alive_seconds = 0)._ensure_model_downloaded("large-v3") + + +def test_load_reports_model_hub_cache_miss(monkeypatch): + _install_fake_torch(monkeypatch) + + class LocalEntryNotFoundError(RuntimeError): + pass + + class MissingWhisperProcessor: + @classmethod + def from_pretrained(cls, *_args, **_kwargs): + raise LocalEntryNotFoundError("not cached") + + monkeypatch.setitem( + sys.modules, + "transformers", + SimpleNamespace( + WhisperForConditionalGeneration = object, + WhisperProcessor = MissingWhisperProcessor, + ), + ) + monkeypatch.setattr(stt_sidecar_module, "_pick_device", lambda: ("cpu", "float32")) + + with pytest.raises(SttModelNotDownloadedError, match = "not downloaded"): + WhisperSttSidecar(keep_alive_seconds = 0).load("large-v3") + + +def test_unavailable_runtime_is_rejected_before_audio_decode(monkeypatch): + sidecar = WhisperSttSidecar() + + def unavailable() -> None: + raise SttUnavailableError("needs PyTorch, Transformers, and PyAV") + + def should_not_decode(_audio): + pytest.fail("runtime must be checked before audio decode") + + monkeypatch.setattr(stt_sidecar_module, "ensure_stt_available", unavailable) + monkeypatch.setattr(stt_sidecar_module, "_decode_audio_bounded", should_not_decode) + + with pytest.raises(SttUnavailableError, match = "needs PyTorch"): + sidecar.transcribe(b"encoded audio", model = "small") + + +def test_missing_model_is_rejected_before_audio_decode(monkeypatch): + sidecar = WhisperSttSidecar(keep_alive_seconds = 0) + + def missing(_model_id): + raise SttModelNotDownloadedError("not downloaded") + + def should_not_decode(_audio): + pytest.fail("missing models must be rejected before audio decode") + + monkeypatch.setattr(sidecar, "_ensure_model_downloaded", missing, raising = False) + monkeypatch.setattr(stt_sidecar_module, "_decode_audio_bounded", should_not_decode) + + with pytest.raises(SttModelNotDownloadedError, match = "not downloaded"): + sidecar.transcribe(b"encoded audio", model = "large-v3") + + +def test_missing_model_switch_keeps_resident_model(monkeypatch): + sidecar = WhisperSttSidecar(keep_alive_seconds = 0) + resident = object() + sidecar._engine = resident + sidecar._model_id = "small" + sidecar._device = "cpu" + + def missing(_model_id): + raise SttModelNotDownloadedError("not downloaded") + + monkeypatch.setattr(sidecar, "_ensure_model_downloaded", missing, raising = False) + monkeypatch.setattr(stt_sidecar_module, "ensure_stt_available", lambda: None) + monkeypatch.setattr( + sidecar, + "_build_model", + lambda *_args: pytest.fail("cache miss must be detected before model replacement"), + ) + _install_fake_torch(monkeypatch) + + with pytest.raises(SttModelNotDownloadedError, match = "not downloaded"): + sidecar.load("large-v3") + + assert sidecar._engine is resident + assert sidecar.loaded_model == "small" + + +def test_incompatible_custom_model_switch_keeps_resident_model(monkeypatch, tmp_path): + (tmp_path / "config.json").write_text( + '{"model_type": "llama", "architectures": ["LlamaForCausalLM"]}' + ) + sidecar = WhisperSttSidecar(keep_alive_seconds = 0) + resident = (object(), object()) + sidecar._engine = resident + sidecar._model_id = "small" + sidecar._device = "cpu" + _install_fake_torch(monkeypatch) + monkeypatch.setattr( + stt_sidecar_module, + "_find_complete_cached_snapshot", + lambda _model: tmp_path, + ) + + with pytest.raises(SttModelCompatibilityError, match = "not a compatible"): + sidecar.load("owner/chat-model") + + assert sidecar._engine is resident + assert sidecar.loaded_model == "small" + + +def test_loaded_model_stays_warm_until_idle_timer_fires(monkeypatch): + timers = [] + _install_fake_torch(monkeypatch) + + def make_timer(*args, **kwargs): + timer = _FakeTimer(*args, **kwargs) + timers.append(timer) + return timer + + sidecar = WhisperSttSidecar(keep_alive_seconds = 300) + monkeypatch.setattr(stt_sidecar_module, "ensure_stt_available", lambda: None) + monkeypatch.setattr(stt_sidecar_module.threading, "Timer", make_timer) + monkeypatch.setattr(sidecar, "_build_model", lambda *_args: (object(), object())) + monkeypatch.setattr(stt_sidecar_module, "_pick_device", lambda: ("cpu", "float32")) + + sidecar.load("small") + + assert sidecar.loaded_model == "small" + assert timers[-1].interval == 300 + assert timers[-1].started + + timers[-1].fire() + + assert sidecar.loaded_model is None + + +def test_reusing_loaded_model_refreshes_idle_timer(monkeypatch): + timers = [] + _install_fake_torch(monkeypatch) + + def make_timer(*args, **kwargs): + timer = _FakeTimer(*args, **kwargs) + timers.append(timer) + return timer + + sidecar = WhisperSttSidecar(keep_alive_seconds = 300) + monkeypatch.setattr(stt_sidecar_module, "ensure_stt_available", lambda: None) + monkeypatch.setattr(stt_sidecar_module.threading, "Timer", make_timer) + monkeypatch.setattr(sidecar, "_build_model", lambda *_args: (object(), object())) + monkeypatch.setattr(stt_sidecar_module, "_pick_device", lambda: ("cpu", "float32")) + + sidecar.load("small") + first = timers[-1] + sidecar.load("small") + + assert first.cancelled + assert timers[-1] is not first + + first.fire() + + assert sidecar.loaded_model == "small" + + +def test_unload_waits_for_inflight_transcription(monkeypatch): + sidecar = WhisperSttSidecar(keep_alive_seconds = 0) + started = threading.Event() + release = threading.Event() + + def transcribe(*_args): + started.set() + assert release.wait(timeout = 2) + return "hello" + + monkeypatch.setattr(sidecar, "_transcribe_decoded", transcribe) + transcribe_thread = threading.Thread(target = lambda: sidecar.transcribe(b"audio")) + transcribe_thread.start() + assert started.wait(timeout = 2) + + unload_thread = threading.Thread(target = sidecar.unload) + unload_thread.start() + time.sleep(0.02) + assert unload_thread.is_alive() + + release.set() + transcribe_thread.join(timeout = 2) + unload_thread.join(timeout = 2) + + assert not transcribe_thread.is_alive() + assert not unload_thread.is_alive() + + +def test_new_stt_load_uses_cpu_while_training(monkeypatch): + fake_torch = SimpleNamespace( + float16 = "float16", + float32 = "float32", + cuda = SimpleNamespace(is_available = lambda: True), + backends = SimpleNamespace(mps = SimpleNamespace(is_available = lambda: True)), + ) + monkeypatch.setitem(sys.modules, "torch", fake_torch) + monkeypatch.setattr(stt_sidecar_module, "_training_active", lambda: True) + + assert stt_sidecar_module._pick_device() == ("cpu", "float32") + + +def test_new_stt_load_prefers_cuda_when_training_is_idle(monkeypatch): + fake_torch = SimpleNamespace( + float16 = "float16", + float32 = "float32", + cuda = SimpleNamespace(is_available = lambda: True), + backends = SimpleNamespace(mps = SimpleNamespace(is_available = lambda: False)), + ) + monkeypatch.setitem(sys.modules, "torch", fake_torch) + monkeypatch.setattr(stt_sidecar_module, "_training_active", lambda: False) + + assert stt_sidecar_module._pick_device() == ("cuda", "float16") + + +def test_new_stt_load_prefers_mps_when_cuda_is_unavailable(monkeypatch): + fake_torch = SimpleNamespace( + float16 = "float16", + float32 = "float32", + cuda = SimpleNamespace(is_available = lambda: False), + backends = SimpleNamespace(mps = SimpleNamespace(is_available = lambda: True)), + ) + monkeypatch.setitem(sys.modules, "torch", fake_torch) + monkeypatch.setattr(stt_sidecar_module, "_training_active", lambda: False) + + assert stt_sidecar_module._pick_device() == ("mps", "float32") + + +def test_new_stt_load_uses_cpu_without_accelerators(monkeypatch): + fake_torch = SimpleNamespace( + float16 = "float16", + float32 = "float32", + cuda = SimpleNamespace(is_available = lambda: False), + backends = SimpleNamespace(mps = SimpleNamespace(is_available = lambda: False)), + ) + monkeypatch.setitem(sys.modules, "torch", fake_torch) + monkeypatch.setattr(stt_sidecar_module, "_training_active", lambda: False) + + assert stt_sidecar_module._pick_device() == ("cpu", "float32") + + +def test_accelerator_load_failure_retries_on_cpu(monkeypatch): + fake_torch = _install_fake_torch(monkeypatch) + calls = [] + sidecar = WhisperSttSidecar(keep_alive_seconds = 0) + monkeypatch.setattr(stt_sidecar_module, "ensure_stt_available", lambda: None) + + def build(_repo, device, dtype, _cancel_event): + calls.append((device, dtype)) + if device == "cuda": + raise RuntimeError("accelerator allocation failed") + return object(), object() + + monkeypatch.setattr(stt_sidecar_module, "_pick_device", lambda: ("cuda", "float16")) + monkeypatch.setattr(sidecar, "_build_model", build) + + sidecar.load("small") + + assert calls == [("cuda", "float16"), ("cpu", fake_torch.float32)] + assert sidecar.device == "cpu" + + +def test_pending_load_can_be_cancelled_without_waiting_for_model_lock(monkeypatch): + _install_fake_torch(monkeypatch) + sidecar = WhisperSttSidecar(keep_alive_seconds = 0) + build_started = threading.Event() + release_build = threading.Event() + errors = [] + + def build(_repo, _device, _dtype, _cancel_event): + build_started.set() + assert release_build.wait(timeout = 2) + return object(), object() + + def run_load(): + try: + sidecar.load("small") + except Exception as exc: + errors.append(exc) + + monkeypatch.setattr(stt_sidecar_module, "ensure_stt_available", lambda: None) + monkeypatch.setattr(stt_sidecar_module, "_pick_device", lambda: ("cpu", "float32")) + monkeypatch.setattr(sidecar, "_build_model", build) + + load_thread = threading.Thread(target = run_load) + load_thread.start() + assert build_started.wait(timeout = 2) + + result = [] + cancel_thread = threading.Thread(target = lambda: result.append(sidecar.cancel_pending_load())) + cancel_thread.start() + cancel_thread.join(timeout = 2) + + assert not cancel_thread.is_alive() + assert result == [True] + assert load_thread.is_alive() + + release_build.set() + load_thread.join(timeout = 2) + + assert not load_thread.is_alive() + assert len(errors) == 1 + assert isinstance(errors[0], SttLoadCancelledError) + assert sidecar.loaded_model is None + assert sidecar.is_loading() is False + + +def _wav_bytes(sample_count: int, sample_rate: int = 16000) -> bytes: + output = io.BytesIO() + with wave.open(output, "wb") as wav: + wav.setnchannels(1) + wav.setsampwidth(2) + wav.setframerate(sample_rate) + wav.writeframes(np.zeros(sample_count, dtype = np.int16).tobytes()) + return output.getvalue() + + +def test_bounded_decoder_returns_16khz_float_pcm(): + pytest.importorskip("av") + + decoded = _REAL_DECODE_AUDIO_BOUNDED(_wav_bytes(1600)) + + assert decoded.dtype == np.float32 + assert decoded.shape == (1600,) + + +def test_bounded_decoder_rejects_audio_as_soon_as_sample_cap_is_crossed(monkeypatch): + pytest.importorskip("av") + monkeypatch.setattr(stt_sidecar_module, "_MAX_AUDIO_SECONDS", 1) + + with pytest.raises(SttAudioTooLongError, match = "Audio must"): + _REAL_DECODE_AUDIO_BOUNDED(_wav_bytes(16001)) + + +def test_bounded_decoder_resamples_stereo_48khz_to_mono_16khz(): + pytest.importorskip("av") + output = io.BytesIO() + frames = np.zeros((4800, 2), dtype = np.int16) + with wave.open(output, "wb") as wav: + wav.setnchannels(2) + wav.setsampwidth(2) + wav.setframerate(48000) + wav.writeframes(frames.tobytes()) + + decoded = _REAL_DECODE_AUDIO_BOUNDED(output.getvalue()) + + assert decoded.dtype == np.float32 + assert 1590 <= len(decoded) <= 1610 + + +@pytest.mark.parametrize("audio", [b"", b"not audio", b"RIFF\x00\x00"]) +def test_bounded_decoder_rejects_malformed_audio(audio): + pytest.importorskip("av") + + with pytest.raises(SttAudioDecodeError, match = "Could not decode"): + _REAL_DECODE_AUDIO_BOUNDED(audio) + + +def test_bounded_decoder_rejects_container_without_audio_stream(monkeypatch): + class FakeFFmpegError(Exception): + pass + + class FakeResampler: + def __init__(self, **_kwargs): + pass + + class FakeFifo: + samples = 0 + + class FakeContainer: + streams = SimpleNamespace(audio = []) + + def __enter__(self): + return self + + def __exit__(self, *_args): + return False + + fake_av = SimpleNamespace( + audio = SimpleNamespace( + resampler = SimpleNamespace(AudioResampler = FakeResampler), + fifo = SimpleNamespace(AudioFifo = FakeFifo), + ), + open = lambda *_args, **_kwargs: FakeContainer(), + ) + monkeypatch.setitem(sys.modules, "av", fake_av) + monkeypatch.setitem( + sys.modules, + "av.error", + SimpleNamespace( + FFmpegError = FakeFFmpegError, + InvalidDataError = FakeFFmpegError, + ), + ) + + with pytest.raises(SttAudioDecodeError, match = "Could not decode"): + _REAL_DECODE_AUDIO_BOUNDED(b"video-only") + + +def test_unload_releases_model_and_device(): + sidecar = WhisperSttSidecar() + sidecar._engine = object() + sidecar._model_id = "small" + sidecar._device = "cpu" + + sidecar.unload() + + assert sidecar.loaded_model is None + assert sidecar.device is None + + +# --------------------------------------------------------------------------- +# Snapshot download tracking +# --------------------------------------------------------------------------- + + +def _write_complete_snapshot(snapshot: Path, *, model_type: str = "whisper") -> None: + snapshot.mkdir(parents = True, exist_ok = True) + (snapshot / "config.json").write_text(json.dumps({"model_type": model_type})) + (snapshot / "preprocessor_config.json").write_text("{}") + (snapshot / "tokenizer.json").write_text("{}") + (snapshot / "model.safetensors").write_bytes(b"weights") + + +def _sibling(name: str, size: int, key: str): + return SimpleNamespace(rfilename = name, size = size, blob_id = key, lfs = None) + + +def test_sha_snapshot_without_main_ref_survives_restart_and_cache_relocation(monkeypatch, tmp_path): + repo = "openai/whisper-tiny.en" + revision = "c" * 40 + studio_home = tmp_path / "studio" + first_cache = tmp_path / "first-hub" + second_cache = tmp_path / "second-hub" + monkeypatch.setenv("UNSLOTH_STUDIO_HOME", str(studio_home)) + monkeypatch.setenv("HF_HUB_OFFLINE", "1") + monkeypatch.setattr(stt_sidecar_module, "_snapshot_is_complete", _REAL_SNAPSHOT_IS_COMPLETE) + monkeypatch.setattr( + stt_sidecar_module, + "_find_complete_cached_snapshot", + _REAL_FIND_COMPLETE_CACHED_SNAPSHOT, + ) + + first = first_cache / "models--openai--whisper-tiny.en" / "snapshots" / revision + _write_complete_snapshot(first) + monkeypatch.setenv("HF_HUB_CACHE", str(first_cache)) + stt_sidecar_module._write_revision_record(repo, revision) + assert stt_sidecar_module._find_complete_cached_snapshot(repo) == first.resolve() + + second = second_cache / "models--openai--whisper-tiny.en" / "snapshots" / revision + _write_complete_snapshot(second) + monkeypatch.setenv("HF_HUB_CACHE", str(second_cache)) + assert stt_sidecar_module._find_complete_cached_snapshot(repo) == second.resolve() + + +def test_corrupt_or_escaping_revision_record_is_ignored(monkeypatch, tmp_path): + repo = "openai/whisper-tiny.en" + monkeypatch.setenv("UNSLOTH_STUDIO_HOME", str(tmp_path / "studio")) + monkeypatch.setenv("HF_HUB_CACHE", str(tmp_path / "hub")) + monkeypatch.setattr( + stt_sidecar_module, + "_find_complete_cached_snapshot", + _REAL_FIND_COMPLETE_CACHED_SNAPSHOT, + ) + record = stt_sidecar_module._revision_record_path(repo) + record.parent.mkdir(parents = True) + record.write_text(json.dumps({"version": 1, "repo": repo, "revision": "../../outside"})) + assert stt_sidecar_module._find_complete_cached_snapshot(repo) is None + + outside = tmp_path / "outside" + _write_complete_snapshot(outside) + snapshots = tmp_path / "hub" / "models--openai--whisper-tiny.en" / "snapshots" + snapshots.mkdir(parents = True) + (snapshots / ("d" * 40)).symlink_to(outside, target_is_directory = True) + assert stt_sidecar_module._find_complete_cached_snapshot(repo) is None + + +def test_adapter_only_snapshot_is_not_complete(tmp_path): + (tmp_path / "config.json").write_text('{"model_type": "whisper"}') + (tmp_path / "preprocessor_config.json").write_text("{}") + (tmp_path / "tokenizer.json").write_text("{}") + (tmp_path / "adapter_model.safetensors").write_bytes(b"adapter") + + assert _REAL_SNAPSHOT_IS_COMPLETE(tmp_path) is False + + +def test_snapshot_selection_prefers_safetensors_and_excludes_unrelated_files(): + info = SimpleNamespace( + siblings = [ + _sibling("config.json", 10, "config"), + _sibling("preprocessor_config.json", 20, "preprocessor"), + _sibling("tokenizer.json", 30, "tokenizer"), + _sibling("model.safetensors", 100, "safe"), + _sibling("pytorch_model.bin", 110, "torch"), + _sibling("README.md", 1000, "readme"), + ] + ) + + selected = stt_sidecar_module._select_snapshot_files( + info, lambda _name: pytest.fail("unsharded selection must not load an index") + ) + + assert {item.path for item in selected} == { + "config.json", + "preprocessor_config.json", + "tokenizer.json", + "model.safetensors", + } + assert sum(item.size for item in selected) == 160 + + +def test_snapshot_selection_includes_every_indexed_shard(): + info = SimpleNamespace( + siblings = [ + _sibling("config.json", 10, "config"), + _sibling("model.safetensors.index.json", 5, "index"), + _sibling("model-00001-of-00002.safetensors", 50, "shard1"), + _sibling("model-00002-of-00002.safetensors", 60, "shard2"), + _sibling("pytorch_model.bin", 120, "torch"), + ] + ) + + selected = stt_sidecar_module._select_snapshot_files( + info, + lambda name: { + "weight_map": { + "a": "model-00001-of-00002.safetensors", + "b": "model-00002-of-00002.safetensors", + } + }, + ) + + assert {item.path for item in selected} == { + "config.json", + "model.safetensors.index.json", + "model-00001-of-00002.safetensors", + "model-00002-of-00002.safetensors", + } + + +def test_snapshot_selection_rejects_pickle_only_weights(): + # A custom repo shipping only pytorch_model.bin (pickle) must fail closed: + # selecting it would download a checkpoint that runs code at load time. + info = SimpleNamespace( + siblings = [ + _sibling("config.json", 10, "config"), + _sibling("preprocessor_config.json", 20, "preprocessor"), + _sibling("tokenizer.json", 30, "tokenizer"), + _sibling("pytorch_model.bin", 110, "torch"), + ] + ) + + with pytest.raises(SttModelCompatibilityError, match = "safetensors"): + stt_sidecar_module._select_snapshot_files( + info, lambda _name: pytest.fail("pickle weights must not be selected") + ) + + +def test_snapshot_selection_rejects_safe_index_pointing_at_pickle_shards(): + # A safetensors index can name .bin shards; Transformers dispatches shard + # loading by extension, so those shards would still pickle-load. The index + # is attacker-controlled, so a non-safetensors shard must fail closed. + info = SimpleNamespace( + siblings = [ + _sibling("config.json", 10, "config"), + _sibling("model.safetensors.index.json", 5, "index"), + _sibling("pytorch_model-00001-of-00001.bin", 90, "shard"), + ] + ) + + with pytest.raises(SttModelCompatibilityError, match = "non-safetensors shards"): + stt_sidecar_module._select_snapshot_files( + info, + lambda _name: {"weight_map": {"a": "pytorch_model-00001-of-00001.bin"}}, + ) + + +def test_progress_counts_only_selected_blobs_and_caps_incomplete_files(monkeypatch, tmp_path): + monkeypatch.setenv("HF_HUB_CACHE", str(tmp_path / "hub")) + blobs = tmp_path / "hub" / "models--owner--whisper" / "blobs" + blobs.mkdir(parents = True) + (blobs / "one").write_bytes(b"x" * 10) + (blobs / "two.incomplete").write_bytes(b"x" * 30) + (blobs / "unrelated").write_bytes(b"x" * 1000) + state = stt_sidecar_module._SnapshotDownloadState() + state._repo = "owner/whisper" + state._selected_files = ( + stt_sidecar_module._SelectedHubFile("config.json", 10, "one"), + stt_sidecar_module._SelectedHubFile("model.safetensors", 20, "two"), + ) + state._total_bytes = 30 + state._complete = True + + status = state.status() + + assert status["bytes_total"] == 30 + assert status["bytes_done"] == 30 + + +def test_download_metadata_and_snapshot_use_the_same_revision(monkeypatch, tmp_path): + revision = "e" * 40 + calls = [] + siblings = [ + _sibling("config.json", 10, "config"), + _sibling("preprocessor_config.json", 20, "preprocessor"), + _sibling("tokenizer.json", 30, "tokenizer"), + _sibling("model.safetensors", 100, "safe"), + _sibling("pytorch_model.bin", 110, "torch"), + ] + + class FakeApi: + def __init__(self, token): + pass + + def model_info(self, repo, **kwargs): + calls.append(("info", repo, kwargs)) + return SimpleNamespace(sha = revision, siblings = siblings) + + def fake_snapshot_download(**kwargs): + calls.append(("snapshot", kwargs)) + return str(tmp_path) + + monkeypatch.setattr("huggingface_hub.HfApi", FakeApi) + monkeypatch.setattr("huggingface_hub.snapshot_download", fake_snapshot_download) + monkeypatch.setattr( + "huggingface_hub.hf_hub_download", + lambda **_kwargs: pytest.fail("unsharded selection must not load an index"), + ) + monkeypatch.setattr(stt_sidecar_module, "_snapshot_is_complete", lambda _path: True) + monkeypatch.setattr(stt_sidecar_module, "_write_revision_record", lambda *_args: None) + state = stt_sidecar_module._SnapshotDownloadState() + + state._run("owner/whisper", None, revision) + + assert calls[0] == ( + "info", + "owner/whisper", + {"revision": revision, "files_metadata": True, "timeout": 30}, + ) + assert calls[1][0] == "snapshot" + assert calls[1][1]["revision"] == revision + assert "model.safetensors" in calls[1][1]["allow_patterns"] + assert "pytorch_model.bin" not in calls[1][1]["allow_patterns"] + + +def test_download_status_is_idle_before_any_download(): + state = stt_sidecar_module._SnapshotDownloadState() + + status = state.status() + + assert status == { + "downloading": False, + "model": None, + "error": None, + "bytes_total": None, + "bytes_done": None, + } + + +def test_download_rejects_a_second_model_while_one_is_in_flight(monkeypatch): + state = stt_sidecar_module._SnapshotDownloadState() + release = threading.Event() + monkeypatch.setattr( + state, + "_run", + lambda repo, token, revision: release.wait(timeout = 5), + ) + + state.start("small") + try: + # Re-requesting the in-flight model is a no-op, not an error. + state.start("small") + with pytest.raises(SttModelIdError, match = "still"): + state.start("tiny") + assert state.status()["downloading"] is True + assert state.status()["model"] == "small" + finally: + release.set() + + +def test_download_failure_is_reported_in_status(monkeypatch): + state = stt_sidecar_module._SnapshotDownloadState() + # Mask huggingface_hub so the import inside _run fails fast. + monkeypatch.setitem(sys.modules, "huggingface_hub", None) + + state.start("small") + state._thread.join(timeout = 5) + + status = state.status() + assert status["downloading"] is False + assert "Download failed" in (status["error"] or "") + + +def test_is_model_downloaded_is_false_for_a_cache_miss(monkeypatch): + monkeypatch.setattr( + stt_sidecar_module, + "_find_complete_cached_snapshot", + _REAL_FIND_COMPLETE_CACHED_SNAPSHOT, + ) + monkeypatch.setenv("HF_HUB_CACHE", "/nonexistent/stt-test-cache") + + assert stt_sidecar_module.is_model_downloaded("small") is False + + +def test_sharded_snapshot_with_missing_shard_is_not_downloaded(monkeypatch, tmp_path): + import json + + monkeypatch.setenv("HF_HUB_CACHE", str(tmp_path / "hub")) + monkeypatch.setenv("UNSLOTH_STUDIO_HOME", str(tmp_path / "studio")) + monkeypatch.setattr( + stt_sidecar_module, + "_find_complete_cached_snapshot", + _REAL_FIND_COMPLETE_CACHED_SNAPSHOT, + ) + monkeypatch.setattr(stt_sidecar_module, "_snapshot_is_complete", _REAL_SNAPSHOT_IS_COMPLETE) + snap = tmp_path / "hub" / "models--unsloth--whisper-small" / "snapshots" / ("a" * 40) + snap.mkdir(parents = True) + (snap / "config.json").write_bytes(b"{}") + (snap / "preprocessor_config.json").write_bytes(b"{}") + (snap / "tokenizer.json").write_bytes(b"{}") + index = { + "weight_map": { + "a": "model-00001-of-00002.safetensors", + "b": "model-00002-of-00002.safetensors", + } + } + (snap / "model.safetensors.index.json").write_text(json.dumps(index)) + (snap / "model-00001-of-00002.safetensors").write_bytes(b"w" * 8) + + assert stt_sidecar_module.is_model_downloaded("small") is False + + # Completing the second shard flips the verdict. + (snap / "model-00002-of-00002.safetensors").write_bytes(b"w" * 8) + assert stt_sidecar_module.is_model_downloaded("small") is True + + +@pytest.mark.parametrize("model_id", ["small", "openai/whisper-medium"]) +def test_preflight_rejects_partial_snapshot(monkeypatch, tmp_path, model_id): + # A resolvable snapshot with metadata but no weights must fail preflight, + # not survive until load() after the audio has already been decoded. + monkeypatch.setenv("HF_HUB_CACHE", str(tmp_path / "hub")) + monkeypatch.setenv("UNSLOTH_STUDIO_HOME", str(tmp_path / "studio")) + monkeypatch.setattr( + stt_sidecar_module, + "_find_complete_cached_snapshot", + _REAL_FIND_COMPLETE_CACHED_SNAPSHOT, + ) + monkeypatch.setattr(stt_sidecar_module, "_snapshot_is_complete", _REAL_SNAPSHOT_IS_COMPLETE) + repo = STT_MODELS.get(model_id, model_id) + snapshot = tmp_path / "hub" / f"models--{repo.replace('/', '--')}" / "snapshots" / ("b" * 40) + snapshot.mkdir(parents = True) + (snapshot / "config.json").write_text('{"model_type": "whisper"}') + + with pytest.raises(SttModelNotDownloadedError, match = "not downloaded"): + WhisperSttSidecar(keep_alive_seconds = 0)._ensure_model_downloaded(model_id) + + # Completing the snapshot clears the preflight. + (snapshot / "preprocessor_config.json").write_text("{}") + (snapshot / "tokenizer.json").write_text("{}") + (snapshot / "model.safetensors").write_bytes(b"w" * 8) + WhisperSttSidecar(keep_alive_seconds = 0)._ensure_model_downloaded(model_id) + + +def test_cpu_retry_releases_failed_accelerator_load(monkeypatch): + _install_fake_torch(monkeypatch) + monkeypatch.setattr(stt_sidecar_module, "_pick_device", lambda: ("mps", "float16")) + + class Marker: + pass + + seen = {} + + def fake_build(self, repo, device, dtype, cancel_event): + if device != "cpu": + # The frame local stands in for a partly loaded accelerator model + # kept alive only through the raised traceback. + marker = Marker() + seen["ref"] = weakref.ref(marker) + raise RuntimeError("accelerator load failed") + gc.collect() + seen["alive_during_retry"] = seen["ref"]() is not None + return (_FakeModel(), object()) + + monkeypatch.setattr(WhisperSttSidecar, "_build_model", fake_build) + sidecar = WhisperSttSidecar(keep_alive_seconds = 0) + sidecar.load("small") + + # The failed attempt must be collectable before the CPU model loads, or + # its accelerator memory stays stranded for the whole retry. + assert seen["alive_during_retry"] is False + assert sidecar.device == "cpu" diff --git a/studio/backend/tests/test_studio_api.py b/studio/backend/tests/test_studio_api.py index 928b636e3e..087c00b648 100644 --- a/studio/backend/tests/test_studio_api.py +++ b/studio/backend/tests/test_studio_api.py @@ -11,7 +11,7 @@ the CLI's ``--help`` output: 1. curl -- basic chat completions (non-streaming) 2. curl -- streaming chat completions 3. Python OpenAI SDK -- streaming completions - 4. curl -- Studio server-side tools (enable_tools=true) + 4. curl -- Unsloth server-side tools (enable_tools=true) 5. curl -- Standard OpenAI function calling (non-streaming) 6. curl -- Standard OpenAI function calling (streaming) 7. curl -- Standard OpenAI function calling (multi-turn tool loop) @@ -31,7 +31,7 @@ Usage: python tests/test_studio_api.py python tests/test_studio_api.py --model unsloth/... --gguf-variant ... - # Pytest mode, external server — start a Studio server yourself, + # Pytest mode, external server — start an Unsloth server yourself, # then point pytest at it. Fastest iteration loop. unsloth studio run --model unsloth/Qwen3-1.7B-GGUF --gguf-variant UD-Q4_K_XL & export UNSLOTH_E2E_BASE_URL=http://127.0.0.1:8080 @@ -341,7 +341,7 @@ def _final_finish_reason(chunks: list[dict]) -> str | None: def test_openai_tools_nonstream(base_url: str, api_key: str): """Standard OpenAI function calling, non-streaming, tool_choice='required'. - Regression: before the fix, Studio stripped `tools` and the model + Regression: before the fix, Unsloth stripped `tools` and the model returned plain text with finish_reason='stop'. After the fix, llama-server's response is forwarded verbatim so the client sees finish_reason='tool_calls' with a structured tool_calls array and diff --git a/studio/backend/tests/test_tensor_parallel.py b/studio/backend/tests/test_tensor_parallel.py index 0d71b89d87..00c7aeac69 100644 --- a/studio/backend/tests/test_tensor_parallel.py +++ b/studio/backend/tests/test_tensor_parallel.py @@ -262,9 +262,12 @@ def test_proportional_tensor_split_is_emitted_in_tensor_mode(): src = _load_model_source() assert '"--tensor-split"' in src gate = src.find("if tensor_parallel:") - ts = src.find('"--tensor-split"') + # Find the TP block's emission (after the gate); manual mode emits its own + # --tensor-split earlier in the source from the user's per-GPU shares. + ts = src.find('"--tensor-split"', gate) nxt_else = src.find("self._tensor_parallel = False") assert 0 <= gate < ts < nxt_else, "--tensor-split must be emitted under `if tensor_parallel:`" + assert "tp_tensor_split" in src[gate:nxt_else] def test_mtp_decode_probe_wired_under_tensor_parallel(): @@ -420,7 +423,7 @@ def test_runtime_recovery_fires_for_user_env_mtp(monkeypatch): # MTP driven by user extra_args / LLAMA_ARG_SPEC_TYPE leaves _speculative_type # unset, but the launch flag still gates recovery on (pass-through MTP). b = _recovery_backend() - b._speculative_type = None # Studio stepped back; user/env owns the spec + b._speculative_type = None # Unsloth stepped back; user/env owns the spec done = threading.Event() captured = {} diff --git a/studio/backend/tests/test_think_prefill_reemit.py b/studio/backend/tests/test_think_prefill_reemit.py index 300ff92776..346399c3b2 100644 --- a/studio/backend/tests/test_think_prefill_reemit.py +++ b/studio/backend/tests/test_think_prefill_reemit.py @@ -2,7 +2,7 @@ # Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 """ -Unit tests for detect_think_prefill. +Unit tests for local reasoning-stream helpers. Reasoning templates (Qwen3.6-style) end the generation prompt with an open ``\\n`` so the model starts reasoning immediately. skip_prompt @@ -16,7 +16,13 @@ import sys _backend = os.path.join(os.path.dirname(__file__), "..") sys.path.insert(0, _backend) -from core.inference.chat_template_helpers import detect_think_prefill +from core.inference.chat_template_helpers import ( + ReasoningChannelNormalizer, + detect_reasoning_channel_markers, + detect_reasoning_channel_markers_from_model_info, + detect_think_prefill, + render_with_native_template_fallback, +) QWEN_PROMPT = "<|im_start|>user\nHi!<|im_end|>\n<|im_start|>assistant\n" @@ -87,3 +93,170 @@ def test_guard_emits_when_think_not_special(): def test_guard_default_and_empty_keep_emitting(): assert detect_think_prefill(QWEN_PROMPT + "\n", None) == "\n" assert detect_think_prefill(QWEN_PROMPT + "\n", []) == "\n" + + +def test_gemma_channel_detection_uses_active_template_not_token_metadata(): + class TemplateTokenizer: + chat_template = {"default": "...<|channel>thought\\n{{ eoc_token }}"} + + class NamedTemplateTokenizer: + chat_template = { + "default": "plain assistant template", + "tool_use": "...<|channel>thought\\n{{ eoc_token }}", + } + + class TokenMetadataOnly: + chat_template = None + soc_token = "<|channel>" + eoc_token = "" + + class NamedTemplateProcessor: + chat_template = { + "default": "plain processor default", + "tool_use": "<|channel>thought\nprocessor tool template", + } + tokenizer = TokenMetadataOnly() + + def apply_chat_template(self, *_args, **_kwargs): + raise NotImplementedError + + expected = ("<|channel>thought", "") + assert detect_reasoning_channel_markers(TemplateTokenizer()) == expected + assert detect_reasoning_channel_markers(NamedTemplateTokenizer()) is None + assert ( + detect_reasoning_channel_markers( + NamedTemplateTokenizer(), tools = [{"function": {"name": "web_search"}}] + ) + == expected + ) + assert detect_reasoning_channel_markers(NamedTemplateTokenizer(), tools = []) is None + assert ( + detect_reasoning_channel_markers( + NamedTemplateProcessor(), tools = [{"function": {"name": "web_search"}}] + ) + is None + ) + assert detect_reasoning_channel_markers(TokenMetadataOnly()) is None + + +def test_gemma_channel_detection_tries_no_argument_getter_fallback(): + class FallbackTokenizer: + chat_template = "plain fallback template" + + def get_chat_template(self, **kwargs): + if kwargs: + raise ValueError("tools are not supported") + return "...<|channel>thought\n" + + assert detect_reasoning_channel_markers( + FallbackTokenizer(), tools = [{"function": {"name": "web_search"}}] + ) == ("<|channel>thought", "") + + +def test_native_template_fallback_returns_selected_reasoning_metadata(): + from types import SimpleNamespace + + messages = [{"role": "user", "content": "hi"}] + tools = [{"type": "function", "function": {"name": "web_search"}}] + + def render(tokenizer, msgs, *, tools, **_kw): + body = "".join(message["content"] for message in msgs) + suffix = "|TOOLS" if tools else "" + return body + suffix if tokenizer.chat_template == "NATIVE <|channel>thought\n" else body + + result = render_with_native_template_fallback( + formatted_prompt = "hi", + tokenizer = SimpleNamespace(chat_template = "OVERRIDE"), + model_info = { + "native_chat_template": "NATIVE <|channel>thought\n", + "tokenizer": SimpleNamespace(chat_template = "OVERRIDE"), + }, + active_model_name = "gemma-test", + messages = messages, + tools = tools, + apply_fn = render, + return_metadata = True, + ) + + assert result.prompt == "hi|TOOLS" + assert result.reasoning_channel_markers == ("<|channel>thought", "") + + +def test_cached_native_template_metadata_recovers_reasoning_markers_without_tools(): + from types import SimpleNamespace + + model_info = {"chat_template_info": {"template": "native <|channel>thought\n"}} + + assert detect_reasoning_channel_markers_from_model_info( + SimpleNamespace(chat_template = "override has no native markers"), + model_info, + tools = None, + ) == ("<|channel>thought", "") + result = render_with_native_template_fallback( + formatted_prompt = "prompt from override", + tokenizer = SimpleNamespace(chat_template = "override has no native markers"), + model_info = model_info, + active_model_name = "gemma-test", + messages = [{"role": "user", "content": "hi"}], + tools = None, + return_metadata = True, + ) + assert result.prompt == "prompt from override" + assert result.reasoning_channel_markers == ("<|channel>thought", "") + + +def test_cached_native_markers_do_not_describe_live_tool_template(): + from types import SimpleNamespace + + tools = [{"type": "function", "function": {"name": "web_search"}}] + + class LiveTokenizer: + chat_template = "live tool template without native markers" + + def render(_tokenizer, _messages, *, tools, **_kwargs): + return "prompt with tools" if tools else "prompt without tools" + + result = render_with_native_template_fallback( + formatted_prompt = "prompt with tools", + tokenizer = LiveTokenizer(), + model_info = { + "chat_template_info": {"template": "native <|channel>thought\n"}, + "tokenizer": SimpleNamespace(), + }, + active_model_name = "gemma-test", + messages = [{"role": "user", "content": "hi"}], + tools = tools, + apply_fn = render, + return_metadata = True, + ) + + assert result.prompt == "prompt with tools" + assert result.reasoning_channel_markers is None + + +def test_gemma_channel_normalization_is_prefix_monotonic_and_preserves_tools(): + parser = ReasoningChannelNormalizer("<|channel>thought", "") + output = "" + snapshots = [] + for chunk in ( + "<|chan", + "nel>thought", + "\nReason", + "<|tool_call>web_search", + ): + delta = parser.feed(chunk) + if delta: + output += delta + snapshots.append(output) + + assert snapshots == [ + "", + "Reason", + "Reason<|tool_call>web_search", + ] + assert snapshots[1].startswith(snapshots[0]) + compact = ReasoningChannelNormalizer("<|channel>thought", "") + assert compact.feed("<|channel>thoughtanswer") + compact.finish() == ( + "answer" + ) diff --git a/studio/backend/tests/test_tool_confirm_stream.py b/studio/backend/tests/test_tool_confirm_stream.py index b8e0472e12..0813f6b68d 100644 --- a/studio/backend/tests/test_tool_confirm_stream.py +++ b/studio/backend/tests/test_tool_confirm_stream.py @@ -3,12 +3,12 @@ """End-to-end handshake test for the tool-confirmation gate, no model. -The real Studio stream wrappers in ``routes/inference.py`` drive the +The real Unsloth stream wrappers in ``routes/inference.py`` drive the synchronous agentic generator with ``await asyncio.to_thread(next, gen, ...)`` so the blocking ``threading.Event`` wait runs off the event loop. This test rebuilds that exact pattern around the real ``state.tool_approvals`` functions, served by a real uvicorn process on -loopback (the same server Studio uses), and proves the load-bearing +loopback (the same server Unsloth uses), and proves the load-bearing property: * ``tool_start`` reaches the client before the gate blocks, and diff --git a/studio/backend/tests/test_tool_message_empty_content.py b/studio/backend/tests/test_tool_message_empty_content.py index d63b16ce80..636a35f5a9 100644 --- a/studio/backend/tests/test_tool_message_empty_content.py +++ b/studio/backend/tests/test_tool_message_empty_content.py @@ -4,7 +4,7 @@ """Empty ``role="tool"`` content must be accepted on the OpenAI-compat surface. Agentic clients send ``content: ""`` when a command produced no output; -OpenAI and llama-server both accept it. Studio used to 400, which standard +OpenAI and llama-server both accept it. Unsloth used to 400, which standard clients treat as non-retryable and kill the session. The validator must normalize empty/missing tool content to ``""`` instead of raising. """ diff --git a/studio/backend/tests/test_tp_vision_regression.py b/studio/backend/tests/test_tp_vision_regression.py index 09af876da6..d1372ca415 100644 --- a/studio/backend/tests/test_tp_vision_regression.py +++ b/studio/backend/tests/test_tp_vision_regression.py @@ -126,10 +126,21 @@ _ALLOWED_TP_DROP_GUARDS = { # Capability: --split-mode tensor aborted for this (binary, model) (#6415). # Self-healing -- tried by default, skipped only after a real abort (vs #6416). "tensor_parallel and self._tensor_split_aborts(binary, model_identifier)", - # Capacity: tensor needs >= 2 GPUs clearing the compute-buffer reserve. - "tensor_parallel and len(tp_gpus) < 2", + # Capacity: tensor needs >= 2 GPUs clearing the compute-buffer reserve. Gated + # on plan_tp (not raw tensor_parallel) so manual mode skips this planner (#6414). + "plan_tp and len(tp_gpus) < 2", # Capacity: pooled usable VRAM can't hold weights + MTP reserve -> layer split. "_tp_weight_budget_mib <= _tp_required_mib", + # Manual mode, Auto layers: --fit owns memory and is incompatible with a + # tensor split, so TP is dropped (surfaced via logger.info) before the + # cache-drop, so a quantized KV survives into the --fit load (#6414). + "tensor_parallel and gpu_memory_mode == 'manual' and (gpu_layers < 0)", + # Manual mode, explicit layers: a tensor split still needs >= 2 GPUs in use. + "tensor_parallel and gpu_memory_mode == 'manual' and (gpu_layers >= 0) and (self._effective_gpu_count(sorted(gpu_ids) if gpu_ids else None) < 2)", + # Manual mode, zero layers: nothing to split on the GPU, and a tensor-mode + # launch under the CPU-only GPU mask (no visible devices) aborts the server + # instead of the intended CPU-only load (#6414). + "gpu_memory_mode == 'manual' and gpu_layers == 0", } @@ -364,7 +375,7 @@ def test_compute_buffer_downgrade_preserves_multi_gpu_intent(): full GPU set too, so it is symmetric with the budget/geometry downgrades and doesn't collapse a multi-GPU layer load to one card (reviewer.py P1 on #6659).""" src = inspect.getsource(LlamaCppBackend.load_model) - gate = src.find("tensor_parallel and len(tp_gpus) < 2") + gate = src.find("plan_tp and len(tp_gpus) < 2") assert gate != -1 # Bound to exactly this block: from its gate to the next (budget) downgrade. nxt = src.find("_tp_weight_budget_mib <= _tp_required_mib", gate) @@ -625,7 +636,7 @@ def _fallback_loaded_backend(layer_preserves_tensor_intent: bool) -> LlamaCppBac def test_tensor_off_echo_preserves_multi_gpu_fallback(): - """The Studio UI always sends tensor_parallel and echoes the /load response's + """The Unsloth UI always sends tensor_parallel and echoes the /load response's resolved value, so after a fallback a ctx/settings reload carries tensor_parallel= false even though the user never changed it. That echo must NOT collapse the preserved multi-GPU placement -- it dedupes (Codex #6659).""" diff --git a/studio/backend/tests/test_trained_model_scan.py b/studio/backend/tests/test_trained_model_scan.py index 7bf572e214..64228cec3c 100644 --- a/studio/backend/tests/test_trained_model_scan.py +++ b/studio/backend/tests/test_trained_model_scan.py @@ -1,7 +1,7 @@ # SPDX-License-Identifier: AGPL-3.0-only # Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 -"""Tests for Studio trained-model discovery used by Chat.""" +"""Tests for Unsloth trained-model discovery used by Chat.""" import json from pathlib import Path @@ -97,6 +97,7 @@ def test_lora_identifier_resolves_remote_adapter_base(tmp_path: Path): repo, fn, token = None, + cache_dir = None, ): assert repo == "someone/my-remote-lora" assert fn == "adapter_config.json" @@ -128,6 +129,7 @@ def test_lora_identifier_retries_transient_then_resolves(tmp_path: Path): repo, fn, token = None, + cache_dir = None, ): calls["n"] += 1 if calls["n"] == 1: diff --git a/studio/backend/tests/test_training_config_popover_source.py b/studio/backend/tests/test_training_config_popover_source.py new file mode 100644 index 0000000000..452a3a1ea8 --- /dev/null +++ b/studio/backend/tests/test_training_config_popover_source.py @@ -0,0 +1,110 @@ +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 + +"""Source-level regression guards for the Training Config popover data source +(#6853). + +The live Training Progress popover used to read the editable form store +(useTrainingConfigStore) while a run was active, so it showed stale/static +values whenever the user touched the form after starting the run; only the +History view read the run's saved config snapshot. These guards pin the fixed +wiring: both views feed ProgressSection a config override mapped from +GET /api/train/runs/{id}, and ProgressSection prefers that override whenever +one is present -- not only for historical views. +""" + +from __future__ import annotations + +from pathlib import Path + +_STUDIO_FRONTEND = Path(__file__).resolve().parents[2] / "frontend" / "src" / "features" / "studio" + + +def _read(rel: str) -> str: + return (_STUDIO_FRONTEND / rel).read_text(encoding = "utf-8") + + +def test_progress_section_prefers_override_over_form_store(): + src = _read("sections/progress-section.tsx") + # Fields key on the override's presence, not isHistorical: a live view passing + # an override wins over the store; without one, live keeps the store while + # History shows blanks rather than unrelated live form values. + assert "const cfg = configOverride ?? (isHistorical ? undefined : config)" in src + assert "const cfgEpochs = cfg?.epochs" in src + assert "isHistorical ? configOverride?.epochs" not in src + + +def test_live_view_fetches_the_active_run_config(): + src = _read("live-training-view.tsx") + # Live view resolves the run's saved config snapshot by job id... + assert "getTrainingRun(" in src + assert "mapRunConfigToOverride(" in src + # ...and hands it to the popover. + assert "configOverride={runConfigOverride}" in src + + +def test_live_view_fetches_as_soon_as_the_job_id_exists(): + # start_training() inserts the run row BEFORE the pump consumes any event, so + # the saved config is available during configuring/loading/downloading. The + # job id is therefore the whole readiness condition: gating on a first step + # or a terminal phase would show the wrong config for the entire pre-step + # window of a long load, or for a run adopted from another client. + src = _read("live-training-view.tsx") + assert "if (!runtime.jobId) {" in src + assert "[runtime.jobId, fetchedRunConfig, fetchAttempt]" in src + # No step/phase readiness gate may creep back in. + assert "runRowReady" not in src + + +def test_live_view_retries_the_transient_row_miss(): + # start_training() creates the row before the pump, but a lookup racing that + # commit can still 404. Nothing else in the effect deps changes on failure, so + # the retry must be explicit and bounded, else a genuinely absent row would + # poll forever instead of falling back to the form store. + src = _read("live-training-view.tsx") + assert "RUN_CONFIG_FETCH_RETRIES" in src + assert "RUN_CONFIG_FETCH_RETRY_MS" in src + assert "setFetchAttempt(" in src + assert "attempts >= RUN_CONFIG_FETCH_RETRIES" in src + # The budget is keyed by job so a new run always starts fresh. + assert "fetchAttempt?.jobId === jobId ? fetchAttempt.count : 0" in src + # The pending retry must be cancelled with the effect. + assert "clearTimeout(retryTimer)" in src + + +def test_live_view_prefers_saved_training_method(): + # The method label / LoRA-row visibility must come from the run snapshot, + # not the editable form (which may have changed since the run started). + src = _read("live-training-view.tsx") + assert "runConfigOverride?.trainingMethod ?? config.trainingMethod" in src + + +def test_history_view_uses_the_shared_mapper(): + src = _read("historical-training-view.tsx") + # Shared mapper, not a re-inlined field-by-field copy that could drift. + assert "mapRunConfigToOverride(detail.config)" in src + assert "num_epochs" not in src + + +def test_shared_mapper_matches_backend_config_keys(): + src = _read("sections/run-config-override.ts") + # The mapper reads the run config JSON the backend snapshots at job start; + # keep the key set pinned so a silent rename breaks loudly here. + for key in ( + "training_type", + "load_in_4bit", + "num_epochs", + "batch_size", + "learning_rate", + "max_steps", + "max_seq_length", + "warmup_steps", + "optim", + "lora_r", + "lora_alpha", + "lora_dropout", + "use_rslora", + "use_loftq", + "use_dora", + ): + assert key in src, f"run-config mapper lost backend key {key}" diff --git a/studio/backend/tests/test_training_nan_loss_handling.py b/studio/backend/tests/test_training_nan_loss_handling.py index a2dc78bee2..5a477a084d 100644 --- a/studio/backend/tests/test_training_nan_loss_handling.py +++ b/studio/backend/tests/test_training_nan_loss_handling.py @@ -1,7 +1,7 @@ # SPDX-License-Identifier: AGPL-3.0-only # Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. -"""Pin Studio's behavior when a training event reports non-finite (NaN/Inf) loss. +"""Pin Unsloth's behavior when a training event reports non-finite (NaN/Inf) loss. The training event handler used to filter NaN/Inf to None silently while leaving the previous finite loss in progress.loss — so the API kept reporting diff --git a/studio/backend/tests/test_training_pump_resilience.py b/studio/backend/tests/test_training_pump_resilience.py index d75b205f35..e7e47478b5 100644 --- a/studio/backend/tests/test_training_pump_resilience.py +++ b/studio/backend/tests/test_training_pump_resilience.py @@ -310,6 +310,80 @@ def test_pump_finalizes_when_read_keeps_raising_on_dead_worker(monkeypatch): assert b._pump_running is False +def test_interrupted_cancel_clears_in_memory_output_dir(monkeypatch): + # Stop-without-save interrupted before its complete event: /status must not + # keep serving the cleared run's output_dir. + b = TrainingBackend() + finalized: dict = {} + monkeypatch.setattr(b, "_ensure_db_run_created", lambda: None) + monkeypatch.setattr(b, "_finalize_run_in_db", lambda **kw: finalized.update(kw)) + + b._proc = _FakeProc(alive = False) + b._event_queue = _IdleQueue() + b._progress.is_training = True + b._should_stop = True + b._cancel_requested = True + b._output_dir = "/out/x" + + b._pump_loop() + + assert b._output_dir is None + assert finalized.get("status") == "stopped" + assert finalized.get("output_dir") is None + assert finalized.get("clear_output_dir") is True + + +def test_worker_exit_reuses_terminal_stop_save_error(monkeypatch): + b = TrainingBackend() + finalized: dict = {} + monkeypatch.setattr(b, "_ensure_db_run_created", lambda: None) + monkeypatch.setattr(b, "_finalize_run_in_db", lambda **kw: finalized.update(kw)) + + b._proc = _FakeProc(alive = False) + b._event_queue = _IdleQueue() + b._progress.is_training = True + b._should_stop = True + b._cancel_requested = False + b._output_dir = "/out/x" + b.current_job_id = "job-x" + b._terminal_finalize_payload = { + "status": "error", + "error_message": "checkpoint failed", + "output_dir": "/out/x", + "clear_output_dir": False, + "resume_blocked": True, + "expected_job_id": "job-x", + } + + b._pump_loop() + + assert b._output_dir == "/out/x" + assert finalized.get("status") == "error" + assert finalized.get("output_dir") == "/out/x" + assert finalized.get("clear_output_dir") is False + assert finalized.get("resume_blocked") is True + + +def test_dead_worker_crash_preserves_output_dir(monkeypatch): + # A crash (no stop requested) after output_dir was emitted must keep the dir + # in the error finalize: checkpoints under it may still exist. + b = TrainingBackend() + finalized: dict = {} + monkeypatch.setattr(b, "_ensure_db_run_created", lambda: None) + monkeypatch.setattr(b, "_finalize_run_in_db", lambda **kw: finalized.update(kw)) + + b._proc = _FakeProc(alive = False) + b._event_queue = _IdleQueue() + b._progress.is_training = True + b._output_dir = "/out/x" + + b._pump_loop() + + assert finalized.get("status") == "error" + assert finalized.get("output_dir") == "/out/x" + assert finalized.get("clear_output_dir") is False + + def test_start_training_clears_stale_pump_running_flag(): # A prior pump that died abnormally leaves _pump_running True. The next # start_training must clear it during reset so the start-time watchdog can't @@ -492,3 +566,34 @@ def test_db_run_created_before_pump_consumes_events(monkeypatch): # The pump observed an already-created run; it would be False if the pump # were started before the eager create. assert seen["db_created"] is True + + +def test_startup_flag_reports_training_active_before_proc(): + # Between freeing VRAM and _proc going live, a concurrent STT load must see + # training as active so it does not grab the just-freed GPU. + b = TrainingBackend() + b._spawn_in_progress = True + assert b.is_training_active() is True + + +def test_before_spawn_runs_inside_active_window(monkeypatch): + # The VRAM-freeing hook must run while training already counts as active, or + # an STT load racing it would place Whisper back on the freed GPU. + b = TrainingBackend() + _stub_spawn(monkeypatch) + monkeypatch.setattr(b, "_ensure_db_run_created", lambda: None) + monkeypatch.setattr(b, "_pump_loop", lambda: setattr(b, "_pump_running", False)) + + active_during_free = {} + + def before_spawn(): + active_during_free["value"] = b.is_training_active() + + assert b.start_training("job_active_window", model_name = "m", before_spawn = before_spawn) is True + if b._pump_thread is not None: + b._pump_thread.join(timeout = 2.0) + + assert active_during_free["value"] is True + # The transient flag clears, but the live proc keeps training active. + assert b._spawn_in_progress is False + assert b.is_training_active() is True diff --git a/studio/backend/tests/test_training_resume.py b/studio/backend/tests/test_training_resume.py index 91fdac9961..51425b0428 100644 --- a/studio/backend/tests/test_training_resume.py +++ b/studio/backend/tests/test_training_resume.py @@ -7,6 +7,9 @@ import importlib.util import json from pathlib import Path +import pytest +import torch + _BACKEND = Path(__file__).resolve().parents[1] @@ -25,6 +28,30 @@ def _load_resume_module(): resume = _load_resume_module() +def test_resume_request_accepts_sanitized_null_target_modules(): + from models.training import TrainingStartRequest + request = TrainingStartRequest( + model_name = "unsloth/Qwen3-0.6B", + training_type = "Full Finetuning", + format_type = "alpaca", + target_modules = None, + ) + + assert request.target_modules == [] + + +def _write_checkpoint(out: Path, step: int) -> Path: + checkpoint = out / f"checkpoint-{step}" + checkpoint.mkdir(parents = True, exist_ok = True) + (checkpoint / "trainer_state.json").write_text( + json.dumps({"global_step": step}), encoding = "utf-8" + ) + torch.save({"weight": torch.ones(1)}, checkpoint / "adapter_model.bin") + torch.save({"state": {0: torch.ones(1)}}, checkpoint / "optimizer.pt") + torch.save({"last_epoch": step}, checkpoint / "scheduler.pt") + return checkpoint + + def _stopped_run(**overrides): run = { "status": "stopped", @@ -44,6 +71,36 @@ def test_can_resume_run_allows_checkpointed_non_s3_run(monkeypatch): assert resume.can_resume_run(_stopped_run()) is True +def test_can_resume_run_allows_errored_run_with_checkpoint(monkeypatch): + monkeypatch.setattr(resume, "has_resume_state", lambda _path: True) + + assert resume.can_resume_run(_stopped_run(status = "error")) is True + + +def test_can_resume_run_rejects_errored_run_without_checkpoint(monkeypatch): + monkeypatch.setattr(resume, "has_resume_state", lambda _path: False) + + assert resume.can_resume_run(_stopped_run(status = "error")) is False + + +def test_can_resume_run_allows_errored_run_at_final_step(monkeypatch): + # A save-time crash records final_step == total_steps; resuming re-runs the + # final-save path from the checkpoint. + monkeypatch.setattr(resume, "has_resume_state", lambda _path: True) + + run = _stopped_run(status = "error", final_step = 10, total_steps = 10) + + assert resume.can_resume_run(run) is True + + +def test_can_resume_run_rejects_stopped_run_at_final_step(monkeypatch): + monkeypatch.setattr(resume, "has_resume_state", lambda _path: True) + + run = _stopped_run(final_step = 10, total_steps = 10) + + assert resume.can_resume_run(run) is False + + def test_can_resume_run_rejects_s3_dataset_source(monkeypatch): monkeypatch.setattr(resume, "has_resume_state", lambda _path: True) @@ -91,3 +148,444 @@ def test_list_runs_includes_config_json_for_resume_policy(monkeypatch, tmp_path) result = studio_db.list_runs() assert result["runs"][0]["config_json"] == config_json + + +def test_crashed_run_with_persisted_output_dir_is_resumable(monkeypatch, tmp_path): + from storage import studio_db + + monkeypatch.setenv("UNSLOTH_STUDIO_HOME", str(tmp_path)) + monkeypatch.setattr(studio_db, "_schema_ready", False) + + out = tmp_path / "outputs" / "run_x" + _write_checkpoint(out, 10) + + studio_db.create_run( + id = "run-crash", + model_name = "m", + dataset_name = "d", + config_json = "{}", + started_at = "2026-01-01T00:00:00Z", + total_steps = 20, + ) + studio_db.update_run_output_dir("run-crash", str(out)) + conn = studio_db.get_connection() + conn.execute("UPDATE training_runs SET status = 'error' WHERE id = 'run-crash'") + conn.commit() + conn.close() + + run = studio_db.get_run("run-crash") + assert run["output_dir"] == str(out) + assert resume.can_resume_run(run) is True + + +def test_checkpoint_discovery_skips_malformed_newest(monkeypatch, tmp_path): + monkeypatch.setenv("UNSLOTH_STUDIO_HOME", str(tmp_path)) + out = tmp_path / "outputs" / "run_x" + valid = _write_checkpoint(out, 5) + (_write_checkpoint(out, 8) / "scheduler.pt").unlink() + malformed = out / "checkpoint-10" + malformed.mkdir() + (malformed / "trainer_state.json").write_text(json.dumps({"global_step": 10}), encoding = "utf-8") + (malformed / "adapter_model.bin").write_bytes(b"not a torch archive") + (malformed / "optimizer.pt").write_bytes(b"not a torch archive") + + assert resume.get_resume_checkpoint_path(str(out)) == str(valid) + + +def test_completed_run_keeps_output_dir_and_rejects_stale_cancel(monkeypatch, tmp_path): + from storage import studio_db + + monkeypatch.setenv("UNSLOTH_STUDIO_HOME", str(tmp_path)) + monkeypatch.setattr(studio_db, "_schema_ready", False) + + studio_db.create_run( + id = "r", + model_name = "m", + dataset_name = "d", + config_json = "{}", + started_at = "2026-01-01T00:00:00Z", + total_steps = 10, + ) + studio_db.update_run_output_dir("r", "/out/x") + studio_db.finish_run( + id = "r", + status = "completed", + ended_at = "t", + final_step = 2, + final_loss = None, + duration_seconds = 1, + loss_sparkline = "[]", + output_dir = "/out/x", + error_message = None, + ) + + assert studio_db.get_run("r")["output_dir"] == "/out/x" + assert studio_db.mark_run_cancel_requested("r") is False + assert studio_db.get_run("r")["output_dir"] == "/out/x" + assert studio_db.get_run("r")["resume_blocked"] == 0 + + +def test_finish_run_clears_output_dir_for_stop_without_save(monkeypatch, tmp_path): + from storage import studio_db + + monkeypatch.setenv("UNSLOTH_STUDIO_HOME", str(tmp_path)) + monkeypatch.setattr(studio_db, "_schema_ready", False) + + studio_db.create_run( + id = "r", + model_name = "m", + dataset_name = "d", + config_json = "{}", + started_at = "2026-01-01T00:00:00Z", + total_steps = 10, + ) + studio_db.update_run_output_dir("r", "/out/x") + studio_db.finish_run( + id = "r", + status = "stopped", + ended_at = "t", + final_step = 2, + final_loss = None, + duration_seconds = 1, + loss_sparkline = "[]", + output_dir = None, + error_message = None, + clear_output_dir = True, + ) + + assert studio_db.get_run("r")["output_dir"] is None + conn = studio_db.get_connection() + conn.execute( + "UPDATE training_runs SET status = 'running', output_dir = '/out/x', resume_blocked = 0 WHERE id = 'r'" + ) + conn.commit() + conn.close() + studio_db.mark_run_cancel_requested("r") + studio_db.cleanup_orphaned_runs() + assert studio_db.get_run("r")["status"] == "stopped" + assert studio_db.get_run("r")["output_dir"] is None + + +def test_finish_run_clears_output_dir_on_cancel_error_finalize(monkeypatch, tmp_path): + from storage import studio_db + + monkeypatch.setenv("UNSLOTH_STUDIO_HOME", str(tmp_path)) + monkeypatch.setattr(studio_db, "_schema_ready", False) + + studio_db.create_run( + id = "r", + model_name = "m", + dataset_name = "d", + config_json = "{}", + started_at = "2026-01-01T00:00:00Z", + total_steps = 10, + ) + studio_db.update_run_output_dir("r", "/out/x") + studio_db.finish_run( + id = "r", + status = "stopped", + ended_at = "t", + final_step = 2, + final_loss = None, + duration_seconds = 1, + loss_sparkline = "[]", + output_dir = "/out/x", + error_message = "worker failed during cancel", + clear_output_dir = True, + ) + + assert studio_db.get_run("r")["output_dir"] is None + + +def test_finish_run_preserves_output_dir_for_interrupted_stop_and_save(monkeypatch, tmp_path): + from storage import studio_db + + monkeypatch.setenv("UNSLOTH_STUDIO_HOME", str(tmp_path)) + monkeypatch.setattr(studio_db, "_schema_ready", False) + + studio_db.create_run( + id = "r", + model_name = "m", + dataset_name = "d", + config_json = "{}", + started_at = "2026-01-01T00:00:00Z", + total_steps = 10, + ) + studio_db.update_run_output_dir("r", "/out/x") + studio_db.finish_run( + id = "r", + status = "stopped", + ended_at = "t", + final_step = 2, + final_loss = None, + duration_seconds = 1, + loss_sparkline = "[]", + output_dir = None, + error_message = None, + ) + + assert studio_db.get_run("r")["output_dir"] == "/out/x" + + +def test_resumed_errored_run_is_not_offered_again(monkeypatch, tmp_path): + from storage import studio_db + + monkeypatch.setenv("UNSLOTH_STUDIO_HOME", str(tmp_path)) + monkeypatch.setattr(studio_db, "_schema_ready", False) + + out = tmp_path / "outputs" / "run_x" + _write_checkpoint(out, 10) + + studio_db.create_run( + id = "run-old", + model_name = "m", + dataset_name = "d", + config_json = "{}", + started_at = "2026-01-01T00:00:00Z", + total_steps = 20, + ) + studio_db.update_run_output_dir("run-old", str(out)) + studio_db.finish_run( + id = "run-old", + status = "error", + ended_at = "2026-01-01T00:05:00Z", + final_step = 10, + final_loss = None, + duration_seconds = 1, + loss_sparkline = "[]", + output_dir = None, + error_message = "killed", + ) + studio_db.create_run( + id = "run-new", + model_name = "m", + dataset_name = "d", + config_json = "{}", + started_at = "2026-01-02T00:00:00Z", + total_steps = 20, + output_dir = str(out), + resumed_from_run_id = "run-old", + ) + with pytest.raises(RuntimeError, match = "no longer available"): + studio_db.create_run( + id = "run-duplicate", + model_name = "m", + dataset_name = "d", + config_json = "{}", + started_at = "2026-01-02T00:00:01Z", + total_steps = 20, + output_dir = str(out), + resumed_from_run_id = "run-old", + ) + assert studio_db.get_run("run-duplicate") is None + studio_db.finish_run( + id = "run-new", + status = "error", + ended_at = "2026-01-02T00:05:00Z", + final_step = 15, + final_loss = None, + duration_seconds = 1, + loss_sparkline = "[]", + output_dir = None, + error_message = "killed again", + ) + + old_run = studio_db.get_run("run-old") + new_run = studio_db.get_run("run-new") + assert old_run["resumed_later"] == 1 + assert resume.can_resume_run(old_run) is False + assert new_run["resumed_later"] == 0 + assert resume.can_resume_run(new_run) is True + assert studio_db.get_resumable_run_by_output_dir(str(out))["id"] == "run-new" + + +def test_running_continuation_blocks_older_resume(monkeypatch, tmp_path): + from storage import studio_db + + monkeypatch.setenv("UNSLOTH_STUDIO_HOME", str(tmp_path)) + monkeypatch.setattr(studio_db, "_schema_ready", False) + + out = tmp_path / "outputs" / "run_x" + _write_checkpoint(out, 10) + + studio_db.create_run( + id = "run-old", + model_name = "m", + dataset_name = "d", + config_json = "{}", + started_at = "2026-01-01T00:00:00Z", + total_steps = 20, + ) + studio_db.update_run_output_dir("run-old", str(out)) + studio_db.finish_run( + id = "run-old", + status = "error", + ended_at = "2026-01-01T00:05:00Z", + final_step = 10, + final_loss = None, + duration_seconds = 1, + loss_sparkline = "[]", + output_dir = None, + error_message = "killed", + ) + studio_db.create_run( + id = "run-new", + model_name = "m", + dataset_name = "d", + config_json = "{}", + started_at = "2026-01-02T00:00:00Z", + total_steps = 20, + output_dir = str(out), + resumed_from_run_id = "run-old", + ) + + old_run = studio_db.get_run("run-old") + assert old_run["resumed_later"] == 1 + assert resume.can_resume_run(old_run) is False + assert studio_db.get_resumable_run_by_output_dir(str(out)) is None + + +def test_stop_save_checkpoint_failure_keeps_error_status(monkeypatch, tmp_path): + # A stop-and-save whose checkpoint write failed must finalize as an error so + # history explains the missing resume state (keep_error_status flag). + from core.training.training import TrainingBackend + from storage import studio_db + + monkeypatch.setenv("UNSLOTH_STUDIO_HOME", str(tmp_path)) + monkeypatch.setattr(studio_db, "_schema_ready", False) + + studio_db.create_run( + id = "run-failed-save", + model_name = "m", + dataset_name = "d", + config_json = "{}", + started_at = "2026-01-01T00:00:00Z", + total_steps = 10, + ) + backend = TrainingBackend() + backend.current_job_id = "run-failed-save" + backend._db_run_created = True + backend._should_stop = True + backend._handle_event( + { + "type": "error", + "error": "Failed to save a resumable checkpoint after stop.", + "keep_error_status": True, + } + ) + + run = studio_db.get_run("run-failed-save") + assert run["status"] == "error" + assert "resumable checkpoint" in run["error_message"] + + +def test_can_resume_run_rejects_resume_blocked_run(monkeypatch): + monkeypatch.setattr(resume, "has_resume_state", lambda _path: True) + + assert resume.can_resume_run(_stopped_run(status = "error", resume_blocked = 1)) is False + + +def test_stop_save_checkpoint_failure_with_stale_checkpoint_is_not_resumable(monkeypatch, tmp_path): + # A failed stop-and-save must not offer Resume from an older periodic checkpoint; + # that would roll back past the recorded final step. + from core.training.training import TrainingBackend + from storage import studio_db + + monkeypatch.setenv("UNSLOTH_STUDIO_HOME", str(tmp_path)) + monkeypatch.setattr(studio_db, "_schema_ready", False) + + out = tmp_path / "outputs" / "run_x" + _write_checkpoint(out, 10) + + studio_db.create_run( + id = "run-stale-ckpt", + model_name = "m", + dataset_name = "d", + config_json = "{}", + started_at = "2026-01-01T00:00:00Z", + total_steps = 20, + ) + studio_db.update_run_output_dir("run-stale-ckpt", str(out)) + backend = TrainingBackend() + backend.current_job_id = "run-stale-ckpt" + backend._db_run_created = True + backend._should_stop = True + backend._output_dir = str(out) + backend._handle_event( + { + "type": "error", + "error": "Failed to save a resumable checkpoint after stop.", + "keep_error_status": True, + "resume_blocked": True, + } + ) + + run = studio_db.get_run("run-stale-ckpt") + assert run["status"] == "error" + assert run["resume_blocked"] == 1 + assert run["output_dir"] == str(out) + assert resume.can_resume_run(run) is False + + +def test_user_stop_error_without_checkpoint_ack_is_blocked(monkeypatch, tmp_path): + from core.training.training import TrainingBackend + from storage import studio_db + + monkeypatch.setenv("UNSLOTH_STUDIO_HOME", str(tmp_path)) + monkeypatch.setattr(studio_db, "_schema_ready", False) + + studio_db.create_run( + id = "run-user-stop", + model_name = "m", + dataset_name = "d", + config_json = "{}", + started_at = "2026-01-01T00:00:00Z", + total_steps = 10, + ) + backend = TrainingBackend() + backend.current_job_id = "run-user-stop" + backend._db_run_created = True + backend._should_stop = True + backend._handle_event({"type": "error", "error": "interrupted"}) + + run = studio_db.get_run("run-user-stop") + assert run["status"] == "error" and run["resume_blocked"] == 1 + + +def test_terminal_fallback_keeps_resumable_when_current_checkpoint_landed(monkeypatch, tmp_path): + # Worker died before its terminal event, but a valid current-step checkpoint + # is on disk: the fallback must keep the run resumable, not block it. + from core.training.training import TrainingBackend + + monkeypatch.setenv("UNSLOTH_STUDIO_HOME", str(tmp_path)) + out = tmp_path / "outputs" / "run_ok" + _write_checkpoint(out, 7) + + backend = TrainingBackend() + backend.current_job_id = "run-ok" + backend._should_stop = True + backend._output_dir = str(out) + backend._progress.step = 7 + + kwargs = backend._terminal_finalize_kwargs() + assert kwargs["status"] == "stopped" + assert kwargs["resume_blocked"] is False + + +def test_terminal_fallback_blocks_when_no_current_checkpoint(monkeypatch, tmp_path): + # Same path, but only a stale (older-step) checkpoint exists: must block. + from core.training.training import TrainingBackend + + monkeypatch.setenv("UNSLOTH_STUDIO_HOME", str(tmp_path)) + out = tmp_path / "outputs" / "run_stale" + _write_checkpoint(out, 5) + + backend = TrainingBackend() + backend.current_job_id = "run-stale" + backend._should_stop = True + backend._output_dir = str(out) + backend._progress.step = 7 + + kwargs = backend._terminal_finalize_kwargs() + assert kwargs["status"] == "error" + assert kwargs["resume_blocked"] is True diff --git a/studio/backend/tests/test_training_stop_watchdog.py b/studio/backend/tests/test_training_stop_watchdog.py index 457dfc8ea2..cbe2082e82 100644 --- a/studio/backend/tests/test_training_stop_watchdog.py +++ b/studio/backend/tests/test_training_stop_watchdog.py @@ -258,15 +258,27 @@ def test_watchdog_no_op_when_worker_superseded(monkeypatch): def test_new_run_gets_its_own_watchdog(monkeypatch): # A stale watchdog sleeping on an old proc must not stop a new run's stop from # creating its own watcher. - monkeypatch.setitem(_G, "_STOP_GRACE_S", 100.0) - monkeypatch.setitem(_G, "_STOP_TIMEOUT_S", 100.0) b = TrainingBackend() - _record_force_terminate(monkeypatch, b) + started = [] + release = threading.Event() + + def _blocked_watchdog( + target_proc, + cancel, + watched_job_id = None, + ): + started.append(target_proc) + # No timeout: the finally always releases this, so a superseded watchdog stays + # alive through the assertions regardless of load; as a daemon it can't hang exit. + release.wait() + + monkeypatch.setattr(b, "_stop_watchdog_loop", _blocked_watchdog) old_proc = _FakeProc(alive = True) b._proc = old_proc b._start_stop_watchdog(cancel = False) first_wd = b._stop_watchdog + assert _wait_until(lambda: started == [old_proc]) # New run: fresh worker replaces the handle; its stop must get a new watcher # even though the old (superseded) watchdog is still alive. @@ -276,12 +288,12 @@ def test_new_run_gets_its_own_watchdog(monkeypatch): second_wd = b._stop_watchdog try: + assert _wait_until(lambda: started == [old_proc, new_proc]) assert first_wd.is_alive() assert second_wd is not first_wd, "a new run must get its own watchdog" assert b._stop_watchdog_proc is new_proc finally: - old_proc._alive = False - new_proc._alive = False + release.set() first_wd.join(timeout = 5) second_wd.join(timeout = 5) @@ -341,7 +353,7 @@ def test_finalize_after_escalation_clears_state(monkeypatch): # stopped so the UI leaves "Stopping..." and a new run can start. b = TrainingBackend() finstop: list = [] - monkeypatch.setattr(b, "_finish_stopped_run", lambda *a: finstop.append(a)) + monkeypatch.setattr(b, "_finish_stopped_run", lambda *a, **k: finstop.append(a)) b._proc = _FakeProc(alive = True) # wedged: still reports alive b._should_stop = True @@ -353,7 +365,7 @@ def test_finalize_after_escalation_clears_state(monkeypatch): assert b._proc is None, "the wedged handle must be dropped so is_training_active clears" assert b._progress.is_training is False - assert b._progress.status_message == "Training stopped." + assert "valid current-step checkpoint" in b._progress.status_message assert finstop and finstop[0][0] == "job_c", "the captured run must be finalized by id" assert b.is_training_active() is False @@ -363,7 +375,7 @@ def test_finalize_after_escalation_preserves_output_dir(monkeypatch): # must record it even if the watchdog wins the finalize race against the pump. b = TrainingBackend() finstop: list = [] - monkeypatch.setattr(b, "_finish_stopped_run", lambda *a: finstop.append(a)) + monkeypatch.setattr(b, "_finish_stopped_run", lambda *a, **k: finstop.append(a)) b._proc = _FakeProc(alive = True) b._should_stop = True @@ -378,6 +390,28 @@ def test_finalize_after_escalation_preserves_output_dir(monkeypatch): assert finstop[0][1] == "/tmp/outputs/run-123" +def test_finalize_after_escalation_clears_output_dir_on_cancel(monkeypatch): + # Stop-without-saving promises no resume: a cancel that escalates through the + # watchdog clears the persisted output_dir, not a checkpoint path. + b = TrainingBackend() + finstop: list = [] + monkeypatch.setattr(b, "_finish_stopped_run", lambda *a, **k: finstop.append((a, k))) + + b._proc = _FakeProc(alive = True) + b._should_stop = True + b._cancel_requested = True + b.current_job_id = "job_c" + b._db_run_created = True + b._output_dir = "/tmp/outputs/run-123" + + b._finalize_stopped_after_escalation(watched_job_id = "job_c") + + assert finstop and finstop[0][0][0] == "job_c" + assert finstop[0][0][1] is None, "a cancelled run must not record a checkpoint path" + assert finstop[0][1].get("clear_output_dir") is True + assert b._output_dir is None, "/status must stop exposing the cancelled run's dir" + + def test_stop_training_starts_watchdog_only_when_worker_alive(monkeypatch): # No worker -> nothing to escalate; the watchdog must not spawn. b = TrainingBackend() @@ -397,7 +431,7 @@ def test_finalize_after_escalation_no_ops_when_superseded(monkeypatch): # The escalation finalize must then leave the NEW run untouched, not drop its handle. b = TrainingBackend() finstop: list = [] - monkeypatch.setattr(b, "_finish_stopped_run", lambda *a: finstop.append(a)) + monkeypatch.setattr(b, "_finish_stopped_run", lambda *a, **k: finstop.append(a)) old_proc = _FakeProc(alive = False) # force-terminated worker we were watching new_proc = _FakeProc(alive = True) # a new run already took over @@ -418,7 +452,7 @@ def test_finalize_after_escalation_runs_for_its_own_worker(monkeypatch): # finalizes the captured run by id. b = TrainingBackend() finstop: list = [] - monkeypatch.setattr(b, "_finish_stopped_run", lambda *a: finstop.append(a)) + monkeypatch.setattr(b, "_finish_stopped_run", lambda *a, **k: finstop.append(a)) proc = _FakeProc(alive = False) b._proc = proc @@ -439,7 +473,7 @@ def test_finalize_after_escalation_no_ops_on_job_change_during_startup(monkeypat # catch this even though the proc-only guard would not. b = TrainingBackend() finstop: list = [] - monkeypatch.setattr(b, "_finish_stopped_run", lambda *a: finstop.append(a)) + monkeypatch.setattr(b, "_finish_stopped_run", lambda *a, **k: finstop.append(a)) old_proc = _FakeProc(alive = False) # old worker, dead; new _proc not installed yet b._proc = old_proc # still the old handle (== target), so proc guard would pass @@ -497,6 +531,7 @@ def _install_fake_db(monkeypatch): recs["insert_ids"].append(job_id), ) fake_db.update_run_progress = lambda **kw: recs["progress_ids"].append(kw.get("id")) + fake_db.mark_run_cancel_requested = lambda _run_id: True fake_storage.studio_db = fake_db monkeypatch.setitem(sys.modules, "storage", fake_storage) monkeypatch.setitem(sys.modules, "storage.studio_db", fake_db) @@ -506,9 +541,48 @@ def _install_fake_db(monkeypatch): return recs +def test_stop_without_save_creates_missing_row_before_signal(monkeypatch): + recs = _install_fake_db(monkeypatch) + b = TrainingBackend() + b.current_job_id, b._db_config = "job_missing", {"model_name": "m"} + b._stop_queue = queue.Queue() + assert b.stop_training(save = False) is True + assert [run["id"] for run in recs["created"]] == ["job_missing"] + assert b._stop_queue.get_nowait() == {"type": "stop", "save": False} + + b._cancel_requested = b._should_stop = False + sys.modules["storage.studio_db"].mark_run_cancel_requested = lambda _run_id: False + assert b.stop_training(save = False) is False + assert not b._cancel_requested and b._stop_queue.empty() + + new_queue = queue.Queue() + b.current_job_id, b._db_run_created = "job_old", True + b._cancel_requested = b._should_stop = False + + def _supersede(_run_id): + b.current_job_id = "job_new" + b._stop_queue = new_queue + return True + + sys.modules["storage.studio_db"].mark_run_cancel_requested = _supersede + assert b.stop_training(save = False) is False + assert not b._cancel_requested and new_queue.empty() + + def test_finalize_run_in_db_single_winner_under_concurrency(monkeypatch): # The watchdog and pump can both finalize; only one call may reach finish_run. recs = _install_fake_db(monkeypatch) + monkeypatch.setitem(_G, "_DB_FINALIZE_RETRY_S", 0.0) + attempts = 0 + + def flaky_finish(**kw): + nonlocal attempts + attempts += 1 + if attempts < 3: + raise RuntimeError("database is locked") + recs["finished"].append(kw) + + sys.modules["storage.studio_db"].finish_run = flaky_finish b = TrainingBackend() b.current_job_id = "job_x" b._db_run_created = True @@ -527,6 +601,7 @@ def test_finalize_run_in_db_single_winner_under_concurrency(monkeypatch): t.join(timeout = 5) assert len(recs["finished"]) == 1, f"finalize must run once, got {len(recs['finished'])}" + assert attempts == 3 assert b._run_finalized is True @@ -634,7 +709,13 @@ def test_ensure_db_run_created_publishes_only_after_insert(monkeypatch): monkeypatch.setitem(sys.modules, "storage", fake_storage) monkeypatch.setitem(sys.modules, "storage.studio_db", fake_db) - b._ensure_db_run_created() + b._run_intent_lock.acquire() + creator = threading.Thread(target = b._ensure_db_run_created) + creator.start() + time.sleep(0.02) + assert b._db_create_in_progress is False + b._run_intent_lock.release() + creator.join(timeout = 5) assert observed["flag_during_create"] is False, "flag must not be published before insert" assert observed["in_progress_during_create"] is True @@ -706,6 +787,7 @@ def test_escalation_finalizes_watched_run_by_id_end_to_end(monkeypatch): b = TrainingBackend() b.current_job_id = "job_old" b._db_run_created = True + b._should_stop = True b._proc = _FakeProc(alive = False) b._progress.is_training = True b._progress.step = 42 @@ -714,7 +796,8 @@ def test_escalation_finalizes_watched_run_by_id_end_to_end(monkeypatch): b._finalize_stopped_after_escalation(target_proc = b._proc, watched_job_id = "job_old") assert [f["id"] for f in recs["finished"]] == ["job_old"], "must finish the captured run by id" - assert recs["finished"][0]["status"] == "stopped" + assert recs["finished"][0]["status"] == "error" + assert recs["finished"][0]["resume_blocked"] is True assert recs["insert_ids"] == ["job_old"], "buffered metrics must land on the captured run" assert b._metric_buffer == [], "the captured batch must be drained" @@ -725,7 +808,7 @@ def test_escalation_defers_when_row_cannot_be_created_here(monkeypatch): # so the pump's create-then-finalize records the run. Parent state still clears. b = TrainingBackend() called: list = [] - monkeypatch.setattr(b, "_finish_stopped_run", lambda *a: called.append(a)) + monkeypatch.setattr(b, "_finish_stopped_run", lambda *a, **k: called.append(a)) b._proc = _FakeProc(alive = False) b.current_job_id = "job_q" @@ -773,7 +856,7 @@ def test_escalation_does_not_drop_a_new_runs_handle(monkeypatch): new_proc = _FakeProc(alive = True) b._proc = old_proc - def hijack(*a): + def hijack(*a, **k): b._proc = new_proc # a new run takes over during the finalize monkeypatch.setattr(b, "_finish_stopped_run", hijack) diff --git a/studio/backend/tests/test_training_vram_coexistence.py b/studio/backend/tests/test_training_vram_coexistence.py index 2bedc46d1f..217caaa4fb 100644 --- a/studio/backend/tests/test_training_vram_coexistence.py +++ b/studio/backend/tests/test_training_vram_coexistence.py @@ -82,6 +82,63 @@ def _patch_backends(inf, llama): return patch.dict(sys.modules, {"core.inference": core_inf, "routes.inference": routes_inf}) +def _fake_stt_sidecar( + *, + model = None, + device = None, + loading = False, +): + sidecar = SimpleNamespace( + loaded_model = model, + device = device, + is_loading = lambda: loading, + ) + sidecar.cancel_pending_load = MagicMock(return_value = loading) + sidecar.wait_for_load_to_settle = MagicMock() + sidecar.unload = MagicMock() + return sidecar + + +def _fake_ggml_sidecar( + *, + model = None, + device = None, + loading = False, +): + ggml = SimpleNamespace( + loaded_model = model, + device = device, + is_loading = lambda: loading, + ) + ggml.cancel_pending_load = MagicMock(return_value = loading) + ggml.wait_for_load_to_settle = MagicMock() + ggml.unload = MagicMock() + return ggml + + +def _patch_stt(sidecar): + stt_module = types.ModuleType("core.inference.stt_sidecar") + stt_module.get_stt_sidecar = lambda: sidecar + # A fresh import of the GGUF sidecar pulls names from the fake module + # above and fails; fake it too so test ordering cannot break that import. + ggml_module = types.ModuleType("core.inference.stt_ggml_sidecar") + empty_ggml = _fake_ggml_sidecar() + ggml_module.get_ggml_stt_sidecar = lambda: empty_ggml + return patch.dict( + sys.modules, + { + "core.inference.stt_sidecar": stt_module, + "core.inference.stt_ggml_sidecar": ggml_module, + }, + ) + + +def _patch_ggml_stt(sidecar): + ggml_module = types.ModuleType("core.inference.stt_ggml_sidecar") + ggml_module.get_ggml_stt_sidecar = lambda: sidecar + return patch.dict(sys.modules, {"core.inference.stt_ggml_sidecar": ggml_module}) + + # ── summarize_resident_chat ────────────────────────────────────────────────── @@ -169,6 +226,49 @@ class TestSummarizeResidentChat(_GpuCacheResetMixin, unittest.TestCase): self.assertTrue(out["any"]) # GGUF still detected +class TestSummarizeResidentStt(_GpuCacheResetMixin, unittest.TestCase): + def test_reports_resident_model(self): + sidecar = _fake_stt_sidecar(model = "small", device = "cuda") + with _patch_stt(sidecar): + out = tv.summarize_resident_stt() + self.assertEqual(out["model"], "small") + self.assertEqual(out["device"], "cuda") + self.assertTrue(out["any"]) + self.assertFalse(out["loading"]) + + def test_reports_inflight_load(self): + sidecar = _fake_stt_sidecar(loading = True) + with _patch_stt(sidecar): + out = tv.summarize_resident_stt() + self.assertTrue(out["any"]) + self.assertTrue(out["loading"]) + + def test_reports_empty_sidecar(self): + with _patch_stt(_fake_stt_sidecar()): + out = tv.summarize_resident_stt() + self.assertFalse(out["any"]) + + def test_reports_resident_gguf_when_transformers_idle(self): + ggml = _fake_ggml_sidecar(model = "small", device = "whisper.cpp") + with _patch_stt(_fake_stt_sidecar()), _patch_ggml_stt(ggml): + out = tv.summarize_resident_stt() + self.assertEqual(out["model"], "small") + self.assertEqual(out["device"], "whisper.cpp") + self.assertTrue(out["any"]) + + def test_resident_transformers_does_not_mask_loading_gguf(self): + # A Transformers model resident on CPU holds no VRAM, but a GGUF + # whisper-server still binding its accelerator backend does; the CPU + # model must not hide that in-flight startup from training admission. + sidecar = _fake_stt_sidecar(model = "small", device = "cpu") + ggml = _fake_ggml_sidecar(loading = True) + with _patch_stt(sidecar), _patch_ggml_stt(ggml): + out = tv.summarize_resident_stt() + self.assertEqual(out["model"], "small") + self.assertTrue(out["loading"]) + self.assertTrue(out["any"]) + + # ── can_keep_during_training (auto mode) ───────────────────────────────────── @@ -226,12 +326,21 @@ class TestCanKeepAuto(_GpuCacheResetMixin, unittest.TestCase): keep, _, _ = self._run((None, meta)) self.assertFalse(keep) - def test_unload_on_non_cuda(self): + def test_unload_on_non_accelerator(self): keep, info, auto_mock = self._run(([0], {}), device = DeviceType.CPU) self.assertFalse(keep) - self.assertEqual(info["mode"], "non_cuda") + self.assertEqual(info["mode"], "non_accelerator") auto_mock.assert_not_called() + def test_xpu_gets_sized_like_cuda(self): + # XPU is a first-class training backend: the keep-guard must size it, + # not blanket-unload it as a non-accelerator. + meta = {"selection_mode": "auto", "required_gb": 10.0, "usable_gb": 30.0} + keep, info, auto_mock = self._run(([0], meta), device = DeviceType.XPU) + self.assertTrue(keep) + self.assertNotEqual(info.get("mode"), "non_accelerator") + auto_mock.assert_called_once() + def test_full_finetuning_forces_16bit_in_estimate(self): meta = {"selection_mode": "auto", "required_gb": 10.0, "usable_gb": 30.0} _keep, _info, auto_mock = self._run( @@ -438,5 +547,151 @@ class TestFreeChatModels(_GpuCacheResetMixin, unittest.TestCase): self.assertEqual(freed, ["gguf:gemma.gguf"]) +class TestFreeSttModel(_GpuCacheResetMixin, unittest.TestCase): + def test_unloads_resident_model(self): + sidecar = _fake_stt_sidecar(model = "small", device = "cuda") + with _patch_stt(sidecar): + freed = tv.free_stt_model_for_training(reason = "test") + sidecar.unload.assert_called_once() + self.assertEqual(freed, ["stt:small"]) + + def test_cancels_inflight_load_and_waits_to_settle(self): + sidecar = _fake_stt_sidecar(loading = True) + with _patch_stt(sidecar): + freed = tv.free_stt_model_for_training(reason = "test") + sidecar.cancel_pending_load.assert_called_once() + # The cancelled loader may still hold VRAM; we wait for it to release. + sidecar.wait_for_load_to_settle.assert_called_once() + # No model surfaced after the wait, so nothing to unload. + sidecar.unload.assert_not_called() + self.assertEqual(freed, ["stt:loading"]) + + def test_cancels_inflight_load_then_unloads_settled_model(self): + # A load that finished before observing the cancel leaves a resident + # model behind; it must be unloaded so training reclaims the memory. + sidecar = _fake_stt_sidecar(model = "small", loading = True) + with _patch_stt(sidecar): + freed = tv.free_stt_model_for_training(reason = "test") + sidecar.cancel_pending_load.assert_called_once() + sidecar.wait_for_load_to_settle.assert_called_once() + sidecar.unload.assert_called_once() + self.assertEqual(freed, ["stt:loading"]) + + def test_cancelled_load_still_unloads_gguf_sidecar(self): + # Cancelling a Transformers load must not skip the GGUF sidecar; both + # engines can hold memory at once (engine switch or direct load calls). + sidecar = _fake_stt_sidecar(loading = True) + ggml = _fake_ggml_sidecar(model = "small") + with _patch_stt(sidecar), _patch_ggml_stt(ggml): + freed = tv.free_stt_model_for_training(reason = "test") + sidecar.cancel_pending_load.assert_called_once() + ggml.unload.assert_called_once() + self.assertEqual(freed, ["stt:loading", "stt:small"]) + + def test_leaves_empty_sidecar_alone(self): + sidecar = _fake_stt_sidecar() + with _patch_stt(sidecar): + freed = tv.free_stt_model_for_training(reason = "test") + sidecar.unload.assert_not_called() + self.assertEqual(freed, []) + + def test_cancels_inflight_gguf_load_and_waits_to_settle(self): + # A GGUF whisper-server still in startup has no loaded_model yet, so the + # coordinator must cancel and wait for it, not skip it, before training + # claims the accelerator memory it is binding. + sidecar = _fake_stt_sidecar() # Transformers idle + ggml = _fake_ggml_sidecar(loading = True) + with _patch_stt(sidecar), _patch_ggml_stt(ggml): + freed = tv.free_stt_model_for_training(reason = "test") + ggml.cancel_pending_load.assert_called_once() + ggml.wait_for_load_to_settle.assert_called_once() + ggml.unload.assert_not_called() # nothing surfaced after the wait + self.assertEqual(freed, ["stt:gguf-loading"]) + + +class TestCoordinateModels(_GpuCacheResetMixin, unittest.TestCase): + def _run(self, chat, stt, keep_results): + keep = MagicMock(side_effect = keep_results) + with ( + patch.object(tv, "summarize_resident_chat", return_value = chat), + patch.object(tv, "summarize_resident_stt", return_value = stt), + patch.object( + tv, + "free_stt_model_for_training", + return_value = ["stt:small"], + ) as free_stt, + patch.object( + tv, + "free_chat_models_for_training", + return_value = ["hf:chat"], + ) as free_chat, + ): + freed = tv.coordinate_models_for_training(keep) + return freed, keep, free_stt, free_chat + + def test_keeps_everything_when_training_fits(self): + chat = {"any": True, "loading": False} + stt = {"any": True, "loading": False} + freed, keep, free_stt, free_chat = self._run( + chat, + stt, + [(True, {"usable_gb": 40, "required_gb": 10})], + ) + self.assertEqual(freed, []) + keep.assert_called_once() + free_stt.assert_not_called() + free_chat.assert_not_called() + + def test_frees_stt_before_chat(self): + chat = {"any": True, "loading": False} + stt = {"any": True, "loading": False} + freed, keep, free_stt, free_chat = self._run( + chat, + stt, + [ + (False, {"usable_gb": 8, "required_gb": 10}), + (True, {"usable_gb": 12, "required_gb": 10}), + ], + ) + self.assertEqual(freed, ["stt:small"]) + self.assertEqual(keep.call_count, 2) + free_stt.assert_called_once() + free_chat.assert_not_called() + + def test_frees_chat_when_stt_is_not_enough(self): + chat = {"any": True, "loading": False} + stt = {"any": True, "loading": False} + freed, keep, free_stt, free_chat = self._run( + chat, + stt, + [ + (False, {"usable_gb": 8, "required_gb": 10}), + (False, {"usable_gb": 9, "required_gb": 10}), + ], + ) + self.assertEqual(freed, ["stt:small", "hf:chat"]) + self.assertEqual(keep.call_count, 2) + free_stt.assert_called_once() + free_chat.assert_called_once() + + def test_frees_loading_models_without_probe(self): + chat = {"any": True, "loading": True} + stt = {"any": True, "loading": True} + freed, keep, free_stt, free_chat = self._run(chat, stt, []) + self.assertEqual(freed, ["stt:small", "hf:chat"]) + keep.assert_not_called() + free_stt.assert_called_once() + free_chat.assert_called_once() + + def test_cancels_loading_stt_without_probe(self): + chat = {"any": False, "loading": False} + stt = {"any": True, "loading": True} + freed, keep, free_stt, free_chat = self._run(chat, stt, []) + self.assertEqual(freed, ["stt:small"]) + keep.assert_not_called() + free_stt.assert_called_once() + free_chat.assert_not_called() + + if __name__ == "__main__": unittest.main() diff --git a/studio/backend/tests/test_transformers_latest.py b/studio/backend/tests/test_transformers_latest.py index 20616dccba..af48d674cc 100644 --- a/studio/backend/tests/test_transformers_latest.py +++ b/studio/backend/tests/test_transformers_latest.py @@ -1036,7 +1036,7 @@ def test_upgrade_check_mixed_pypi_main_reports_dev_only(monkeypatch): def test_install_endpoint_not_mounted_on_v1(): - """The consented pip-install endpoint is a Studio admin action; it must live + """The consented pip-install endpoint is an Unsloth admin action; it must live on studio_router (kept off the OpenAI-compatible /v1 mount), not router.""" from routes import inference as ri diff --git a/studio/backend/tests/test_transformers_version.py b/studio/backend/tests/test_transformers_version.py index a6e6803a5c..acb2ec449b 100644 --- a/studio/backend/tests/test_transformers_version.py +++ b/studio/backend/tests/test_transformers_version.py @@ -160,6 +160,25 @@ class TestResolveBaseModel: class TestRemoteLoraBase: """_remote_lora_base reads a remote adapter's base from its Hub adapter_config.json.""" + @pytest.fixture(autouse = True) + def _selected_cache_follows_env(self, monkeypatch): + # The cache helpers now read the selected cache (get_hf_cache_paths), + # which snapshots env at import; make it follow the HF_HUB_CACHE these + # tests set so they keep driving the lookup via env. + monkeypatch.setattr( + "utils.transformers_version.get_hf_cache_paths", + lambda: _types.SimpleNamespace( + hub_cache = Path( + os.environ.get("HF_HUB_CACHE") + or os.environ.get("HUGGINGFACE_HUB_CACHE") + or os.path.join( + os.environ.get("HF_HOME") or os.path.expanduser("~/.cache/huggingface"), + "hub", + ) + ) + ), + ) + @staticmethod def _resp(cfg: dict): class _Resp: @@ -645,6 +664,24 @@ def _hf_response(cfg: dict): class TestConfigJsonHfCacheFallback: """HF hub cache is consulted only offline or after a failed fetch (never stale online).""" + @pytest.fixture(autouse = True) + def _selected_cache_follows_env(self, monkeypatch): + # As above: route the selected-cache lookup through the HF_HUB_CACHE env + # these tests set, since get_hf_cache_paths snapshots env at import. + monkeypatch.setattr( + "utils.transformers_version.get_hf_cache_paths", + lambda: _types.SimpleNamespace( + hub_cache = Path( + os.environ.get("HF_HUB_CACHE") + or os.environ.get("HUGGINGFACE_HUB_CACHE") + or os.path.join( + os.environ.get("HF_HOME") or os.path.expanduser("~/.cache/huggingface"), + "hub", + ) + ) + ), + ) + def setup_method(self): _config_json_cache.clear() diff --git a/studio/backend/tests/test_utils.py b/studio/backend/tests/test_utils.py index 64a3c62156..741f19c67a 100644 --- a/studio/backend/tests/test_utils.py +++ b/studio/backend/tests/test_utils.py @@ -38,7 +38,7 @@ from utils.hardware import ( DeviceType, ) import utils.hardware.hardware as _hw_module -from utils.utils import format_error_message +from utils.utils import format_error_message, is_hf_authentication_error # ========== Helpers ========== @@ -439,6 +439,20 @@ class TestFormatErrorMessage: msg = format_error_message(err, "any/model") assert "invalid" in msg.lower() + def test_hf_authentication_error_follows_wrapped_401(self): + response = type("Response", (), {"status_code": 401})() + auth_error = Exception("request failed") + auth_error.response = response + wrapper = RuntimeError("model validation failed") + wrapper.__cause__ = auth_error + assert is_hf_authentication_error(wrapper) is True + + def test_hf_authentication_error_does_not_treat_429_as_invalid(self): + response = type("Response", (), {"status_code": 429})() + rate_error = Exception("too many requests") + rate_error.response = response + assert is_hf_authentication_error(rate_error) is False + # --- OOM on CUDA --- @needs_torch diff --git a/studio/backend/tests/test_whisper_cpp_freshness.py b/studio/backend/tests/test_whisper_cpp_freshness.py new file mode 100644 index 0000000000..69f0c87cee --- /dev/null +++ b/studio/backend/tests/test_whisper_cpp_freshness.py @@ -0,0 +1,156 @@ +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 + +"""Tests for the whisper.cpp prebuilt freshness check. + +Pins the whisper-specific version policy: the release-tag parser, the +is_behind decision matrix (with its downgrade guard), and one end-to-end +wiring smoke through the shared freshness flow. The shared marker-walk and +fail-open mechanics are covered by test_llama_cpp_freshness.py. +""" + +from __future__ import annotations + +import json +import sys +import types as _types +from datetime import datetime, timedelta, timezone +from pathlib import Path + +_BACKEND_DIR = str(Path(__file__).resolve().parent.parent) +if _BACKEND_DIR not in sys.path: + sys.path.insert(0, _BACKEND_DIR) + + +class _NoopLogger: + """structlog-style logger: every method swallows positional + kwargs.""" + + def __getattr__(self, _name): + return lambda *a, **k: None + + +_loggers_stub = _types.ModuleType("loggers") +_loggers_stub.get_logger = lambda *a, **k: _NoopLogger() +sys.modules.setdefault("loggers", _loggers_stub) + +_structlog_stub = _types.ModuleType("structlog") +_structlog_stub.get_logger = lambda *a, **k: _NoopLogger() +sys.modules.setdefault("structlog", _structlog_stub) + +import pytest + +from utils import whisper_cpp_freshness as fr + + +# Helpers. + + +def _write_marker(install_dir: Path, **overrides) -> Path: + payload = { + "requested_tag": "latest", + "release_tag": "v1.9.1-unsloth.1", + "upstream_tag": "v1.9.1", + "published_repo": "unslothai/whisper.cpp", + "asset": "whisper-v1.9.1-unsloth.1-linux-x64-cpu.tar.gz", + "asset_sha256": None, + "source": "published", + "installed_at_utc": (datetime.now(tz = timezone.utc) - timedelta(days = 1)) + .isoformat() + .replace("+00:00", "Z"), + } + payload.update(overrides) + install_dir.mkdir(parents = True, exist_ok = True) + marker = install_dir / "UNSLOTH_WHISPER_PREBUILT_INFO.json" + marker.write_text(json.dumps(payload)) + return marker + + +def _fake_binary(install_dir: Path) -> Path: + """Stub whisper-server under the canonical cmake install layout.""" + bin_dir = install_dir / "build" / "bin" + bin_dir.mkdir(parents = True, exist_ok = True) + bin_path = bin_dir / "whisper-server" + bin_path.write_text("stub\n") + return bin_path + + +@pytest.fixture(autouse = True) +def _reset(monkeypatch, tmp_path): + # Isolate disk cache per-test; never touch the real cache. + monkeypatch.setattr(fr, "_cache_dir", lambda: tmp_path / ".freshness") + fr.reset_caches() + yield + fr.reset_caches() + + +# parse_release_version. + + +def test_parse_release_version(): + assert fr.parse_release_version("v1.9.1-unsloth.2") == (1, 9, 1, 2) + assert fr.parse_release_version("1.10.0") == (1, 10, 0, 0) # no v, no serial + assert fr.parse_release_version(" v2.0.0-unsloth.10 ") == (2, 0, 0, 10) + assert fr.parse_release_version("v1.9") == (1, 9, 0, 0) # padded + assert fr.parse_release_version("nightly") is None + assert fr.parse_release_version(None) is None + assert fr.parse_release_version("") is None + + +# is_behind decision matrix + downgrade guard. + + +def test_is_behind_serial_bump(): + assert fr.is_behind("v1.9.1-unsloth.1", "v1.9.1-unsloth.2") is True + + +def test_is_behind_downgrade_guard(): + # A lower serial or version is never "behind". + assert fr.is_behind("v1.9.1-unsloth.2", "v1.9.1-unsloth.1") is False + assert fr.is_behind("v1.10.0-unsloth.1", "v1.9.1-unsloth.9") is False + + +def test_is_behind_upstream_bump(): + assert fr.is_behind("v1.9.1-unsloth.1", "v1.10.0-unsloth.1") is True + + +def test_is_behind_identical_is_false(): + assert fr.is_behind("v1.9.1-unsloth.1", "v1.9.1-unsloth.1") is False + + +def test_is_behind_unparseable_differs_is_behind(): + assert fr.is_behind("v1.9.1-unsloth.1", "nightly") is True + + +def test_is_behind_missing_side_fails_open(): + assert fr.is_behind(None, "v1.9.1-unsloth.2") is False + assert fr.is_behind("v1.9.1-unsloth.1", None) is False + + +# check_prebuilt_freshness end-to-end. + + +def test_check_prebuilt_freshness_reports_stale_when_old_and_behind(monkeypatch, tmp_path): + _write_marker( + tmp_path, + release_tag = "v1.9.1-unsloth.1", + installed_at_utc = (datetime.now(tz = timezone.utc) - timedelta(days = 10)) + .isoformat() + .replace("+00:00", "Z"), + ) + bin_path = _fake_binary(tmp_path) + monkeypatch.setattr(fr, "latest_published_release", lambda *a, **k: "v1.9.1-unsloth.3") + info = fr.check_prebuilt_freshness(str(bin_path)) + assert info["has_marker"] is True + assert info["behind"] is True + assert info["stale"] is True + assert info["installed_tag"] == "v1.9.1-unsloth.1" + assert info["latest_tag"] == "v1.9.1-unsloth.3" + + +def test_marker_reader_prefers_install_root_over_packaging_marker(tmp_path): + root_marker = _write_marker(tmp_path, release_tag = "v1.9.1-unsloth.2") + binary = _fake_binary(tmp_path) + (binary.parent / root_marker.name).write_text( + json.dumps({"backend": "slim", "release_tag": "archive-metadata"}) + ) + assert fr.read_install_marker(str(binary))["release_tag"] == "v1.9.1-unsloth.2" diff --git a/studio/backend/tests/test_windows_external_drive_paths.py b/studio/backend/tests/test_windows_external_drive_paths.py index 9686d45c9f..5687612916 100644 --- a/studio/backend/tests/test_windows_external_drive_paths.py +++ b/studio/backend/tests/test_windows_external_drive_paths.py @@ -57,6 +57,18 @@ def test_windows_drive_roots_empty_off_windows(monkeypatch): assert external_media.windows_drive_roots() == [] +def test_macos_volume_roots_lists_readable_mounts(monkeypatch, tmp_path): + volumes = tmp_path / "Volumes" + external = volumes / "External SSD" + unreadable = volumes / "Unavailable" + external.mkdir(parents = True) + unreadable.mkdir() + monkeypatch.setattr(external_media.platform, "system", lambda: "Darwin") + monkeypatch.setattr(external_media.os, "access", lambda path, _mode: Path(path) == external) + + assert external_media.macos_volume_roots(volumes) == [external] + + def test_windows_drive_roots_lists_readable_drives(monkeypatch): _stub_windows(monkeypatch, {"C", "D", "E"}) @@ -204,8 +216,10 @@ def test_browse_allowlist_includes_windows_drive_roots(monkeypatch, tmp_path): ) fake_external_media = SimpleNamespace( linux_run_media_mount_roots = lambda: [], + macos_volume_roots = lambda: [], windows_drive_roots = lambda: [drive_root], ) + fake_paths.external_media = fake_external_media fake_studio_db = SimpleNamespace( list_scan_folders = lambda: [], contains_sensitive_path_component = lambda _p: False, @@ -270,8 +284,10 @@ def test_build_browse_allowlist_reuses_passed_roots(monkeypatch, tmp_path): ) fake_external_media = SimpleNamespace( linux_run_media_mount_roots = _media_roots, + macos_volume_roots = lambda: [], windows_drive_roots = _drive_roots, ) + fake_paths.external_media = fake_external_media fake_studio_db = SimpleNamespace(list_scan_folders = lambda: []) monkeypatch.setitem(sys.modules, "utils.paths", fake_paths) monkeypatch.setitem(sys.modules, "utils.paths.external_media", fake_external_media) diff --git a/studio/backend/utils/_studio_release_build.py b/studio/backend/utils/_studio_release_build.py index 267197a202..07ede36912 100644 --- a/studio/backend/utils/_studio_release_build.py +++ b/studio/backend/utils/_studio_release_build.py @@ -1,7 +1,7 @@ # SPDX-License-Identifier: AGPL-3.0-only # Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 -"""Build-stamped Studio release metadata. +"""Build-stamped Unsloth release metadata. Release builds may rewrite this module in the build workspace before creating Python artifacts. Keep the committed value neutral so source checkouts do not diff --git a/studio/backend/utils/api_errors.py b/studio/backend/utils/api_errors.py index cae8daf287..a3686c3a26 100644 --- a/studio/backend/utils/api_errors.py +++ b/studio/backend/utils/api_errors.py @@ -20,7 +20,7 @@ client-error responses on the ``/v1/*`` surface: CRITICAL: the exception handlers installed by :func:`install_api_error_handlers` are global, but they ONLY transform responses for paths that start with ``/v1/``. For every other path (``/api/...``, frontend routes) they reproduce FastAPI's -default behavior byte-for-byte, because the Studio frontend depends on the +default behavior byte-for-byte, because the Unsloth frontend depends on the ``{"detail": ...}`` shape for ``/api/*``. Public contract (other modules depend on these): @@ -107,7 +107,7 @@ def anthropic_error_body( Returns ``{"type": "error", "request_id": None, "error": {"type", "message"}}``. ``request_id`` is a required (nullable) field on the spec's ErrorResponse; - Studio has no request-id system, so it is null. ``err_type`` defaults to + Unsloth has no request-id system, so it is null. ``err_type`` defaults to :data:`ANTHROPIC_TYPE_BY_STATUS` for ``status`` (``"api_error"`` fallback). """ return { @@ -192,7 +192,7 @@ def install_api_error_handlers(app) -> None: Both handlers are global but only transform responses for OpenAI/Anthropic- compatible surfaces (see :func:`wants_api_error_envelope`: the ``/v1/*`` mount and the preview ``/p/.../v1/*`` mount). Every other path reproduces FastAPI's - default ``{"detail": ...}`` behavior exactly so the Studio frontend keeps working. + default ``{"detail": ...}`` behavior exactly so the Unsloth frontend keeps working. """ @app.exception_handler(RequestValidationError) diff --git a/studio/backend/utils/client_ip.py b/studio/backend/utils/client_ip.py index 94acbf1809..cc48a096d2 100644 --- a/studio/backend/utils/client_ip.py +++ b/studio/backend/utils/client_ip.py @@ -4,12 +4,12 @@ """Resolve the caller's IP for rate limiting. Trust model, in order: - 1. If the operator opts in via ``UNSLOTH_STUDIO_TRUST_FORWARDED`` (Studio behind + 1. If the operator opts in via ``UNSLOTH_STUDIO_TRUST_FORWARDED`` (Unsloth behind their own reverse proxy), honor the *rightmost* ``X-Forwarded-For`` hop -- the one the trusted proxy appended. The leftmost entry is client-controlled and spoofable, so this assumes a proxy that appends (or overwrites) the header; only enable the env var behind such a proxy. - 2. If the socket peer is loopback, honor ``CF-Connecting-IP``. Studio's managed + 2. If the socket peer is loopback, honor ``CF-Connecting-IP``. Unsloth's managed Cloudflare tunnel terminates at 127.0.0.1, so every tunneled visitor would otherwise collapse onto the same socket peer (the local cloudflared process) and share one rate-limit bucket. ``CF-Connecting-IP`` is set by Cloudflare's diff --git a/studio/backend/utils/cpu_threads.py b/studio/backend/utils/cpu_threads.py index 4ed0021054..91d577408d 100644 --- a/studio/backend/utils/cpu_threads.py +++ b/studio/backend/utils/cpu_threads.py @@ -1,7 +1,7 @@ # SPDX-License-Identifier: AGPL-3.0-only # Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 -"""Early CPU thread-pool configuration for Studio processes.""" +"""Early CPU thread-pool configuration for Unsloth processes.""" import os from typing import MutableMapping, Optional diff --git a/studio/backend/utils/datasets/cache_safe.py b/studio/backend/utils/datasets/cache_safe.py index e629210f33..d2dc7b737a 100644 --- a/studio/backend/utils/datasets/cache_safe.py +++ b/studio/backend/utils/datasets/cache_safe.py @@ -7,7 +7,7 @@ A shared HF datasets cache can contain subtrees owned by another user (for example populated by an earlier root-run job). datasets then raises "[Errno 13] Permission denied: ..._builder.lock" while locking the cached builder, killing the training run even though the dataset itself is fine. -Retry such loads in a Studio-owned cache so the run proceeds; the worst case +Retry such loads in an Unsloth-owned cache so the run proceeds; the worst case is one rebuild of the dataset in the fallback location. """ @@ -26,7 +26,7 @@ def studio_datasets_cache() -> str: def load_dataset_cache_safe(*args, **kwargs): - """datasets.load_dataset, retried in a Studio-owned cache on EACCES.""" + """datasets.load_dataset, retried in an Unsloth-owned cache on EACCES.""" from datasets import load_dataset try: return load_dataset(*args, **kwargs) diff --git a/studio/backend/utils/datasets/format_conversion.py b/studio/backend/utils/datasets/format_conversion.py index 95c9a00534..9035068c01 100644 --- a/studio/backend/utils/datasets/format_conversion.py +++ b/studio/backend/utils/datasets/format_conversion.py @@ -406,10 +406,13 @@ def convert_to_vlm_format( elif _image_lookup is not None and image_data in _image_lookup: # Bare filename → resolve via HF repo lookup from huggingface_hub import hf_hub_download + from utils.hf_cache_settings import active_hf_hub_cache + local_path = hf_hub_download( dataset_name, _image_lookup[image_data], repo_type = "dataset", + cache_dir = active_hf_hub_cache(), ) image_data = Image.open(local_path).convert("RGB") else: @@ -774,10 +777,13 @@ def convert_sharegpt_with_images_to_vlm_format( return Image.open(BytesIO(f.read())).convert("RGB") elif _image_lookup is not None and image_data in _image_lookup: from huggingface_hub import hf_hub_download + from utils.hf_cache_settings import active_hf_hub_cache + local_path = hf_hub_download( dataset_name, _image_lookup[image_data], repo_type = "dataset", + cache_dir = active_hf_hub_cache(), ) return Image.open(local_path).convert("RGB") else: diff --git a/studio/backend/utils/datasets/llm_assist.py b/studio/backend/utils/datasets/llm_assist.py index f7b35e2869..c594e883a8 100644 --- a/studio/backend/utils/datasets/llm_assist.py +++ b/studio/backend/utils/datasets/llm_assist.py @@ -58,6 +58,7 @@ def precache_helper_gguf(): try: from huggingface_hub import HfApi, hf_hub_download from huggingface_hub.utils import disable_progress_bars, enable_progress_bars + from utils.hf_cache_settings import active_hf_hub_cache disable_progress_bars() logging.getLogger("huggingface_hub").setLevel(logging.WARNING) @@ -76,7 +77,11 @@ def precache_helper_gguf(): + (f" (+{len(matching) - 1} shards)" if len(matching) > 1 else "") ) for target in matching: - hf_hub_download(repo_id = repo, filename = target) + hf_hub_download( + repo_id = repo, + filename = target, + cache_dir = active_hf_hub_cache(), + ) logger.info(f"Helper GGUF cached: {len(matching)} file(s)") else: logger.warning(f"No GGUF matching variant '{variant}' in {repo}") diff --git a/studio/backend/utils/hardware/VRAM_ESTIMATION.md b/studio/backend/utils/hardware/VRAM_ESTIMATION.md index a6b4de29d2..68ca1d5ffd 100644 --- a/studio/backend/utils/hardware/VRAM_ESTIMATION.md +++ b/studio/backend/utils/hardware/VRAM_ESTIMATION.md @@ -106,7 +106,7 @@ Non_flash_attention = B * num_attention_heads * S^2 * 2 * 12.0 * effective_layer Activations = max(Per_layer_with_gc, Non_flash_attention) ``` -Studio resolves the attention implementation with Unsloth's +Unsloth resolves the attention implementation with Unsloth's `resolve_attention_implementation` helper and uses that result directly. The estimator does not duplicate model-family attention policy. diff --git a/studio/backend/utils/hardware/__init__.py b/studio/backend/utils/hardware/__init__.py index 62b537fbac..138238533f 100644 --- a/studio/backend/utils/hardware/__init__.py +++ b/studio/backend/utils/hardware/__init__.py @@ -50,6 +50,11 @@ def export_capability() -> dict: return _hardware.export_capability() +def get_torch_device_str() -> str: + """Return the torch device string ("cuda", "xpu", "cpu") for the detected hardware.""" + return _hardware.get_torch_device_str() + + __all__ = [ "DeviceType", "DEVICE", @@ -75,6 +80,7 @@ __all__ = [ "estimate_required_model_memory_gb", "auto_select_gpu_ids", "prepare_gpu_selection", + "get_torch_device_str", "safe_num_proc", "safe_thread_num_proc", "dataset_map_num_proc", diff --git a/studio/backend/utils/hardware/amd.py b/studio/backend/utils/hardware/amd.py index f5b64c45d0..91a06c9a2a 100644 --- a/studio/backend/utils/hardware/amd.py +++ b/studio/backend/utils/hardware/amd.py @@ -125,7 +125,7 @@ def _run_amd_smi(*args: str, timeout: int = _AMD_SMI_DEFAULT_TIMEOUT) -> Optiona # amd-smi does not exist on Windows (neither Adrenalin nor the HIP SDK # ship a CLI) and can be absent on minimal Linux installs. Disable the # poller in one step instead of burning the 3-strike circuit breaker - # on guaranteed FileNotFoundError spawns. Studio's VRAM display falls + # on guaranteed FileNotFoundError spawns. Unsloth's VRAM display falls # back to torch mem_get_info. if not _amd_smi_disabled: logger.info( diff --git a/studio/backend/utils/hardware/hardware.py b/studio/backend/utils/hardware/hardware.py index 117ad7b780..38ebc0b6d4 100644 --- a/studio/backend/utils/hardware/hardware.py +++ b/studio/backend/utils/hardware/hardware.py @@ -37,7 +37,7 @@ logger = get_logger(__name__) # ── GPU index ordering ────────────────────────────────────────────────────── # CUDA defaults to CUDA_DEVICE_ORDER=FASTEST_FIRST, numbering GPUs by compute -# performance. nvidia-smi -- and every free-VRAM probe in Studio -- numbers GPUs +# performance. nvidia-smi -- and every free-VRAM probe in Unsloth -- numbers GPUs # by PCI bus id instead. On a mixed-GPU host (e.g. an RTX 5090 alongside an RTX # PRO 6000) the two orderings disagree, so an index picked from nvidia-smi data # ("the emptiest card is GPU 1") gets written into CUDA_VISIBLE_DEVICES and then @@ -49,7 +49,7 @@ logger = get_logger(__name__) # and spawn workers copy os.environ. setdefault so an explicit user override wins. os.environ.setdefault("CUDA_DEVICE_ORDER", "PCI_BUS_ID") -# Studio workers can import MLX without importing unsloth first, so mirror the +# Unsloth workers can import MLX without importing unsloth first, so mirror the # package bootstrap here. Keep an explicit user value authoritative. if platform.system() == "Darwin" and platform.machine() == "arm64": os.environ.setdefault("AGX_RELAX_CDM_CTXSTORE_TIMEOUT", "1") @@ -117,7 +117,7 @@ def _has_mlx() -> bool: def _has_usable_mlx_stack() -> bool: - """True only when the FULL Studio MLX training/export stack is usable + """True only when the FULL Unsloth MLX training/export stack is usable (mlx + mlx-lm + mlx-vlm at the minimum versions unsloth-zoo requires), not just a bare ``import mlx.core``. A backtracked/old mlx-vlm still imports but breaks VLM Train/Export, so the training gate must match the self-heal's own @@ -175,18 +175,64 @@ def detect_hardware() -> DeviceType: Call once at FastAPI lifespan startup; idempotent. Detection order: - 1. CUDA (NVIDIA GPU, requires torch) - 2. MLX (Apple Silicon via MLX framework) - 3. CPU (fallback) + 1. XPU-preferred hint: only on an unambiguous "prefer XPU" signal + (CUDA hidden via ``CUDA_VISIBLE_DEVICES="" / "-1"``, + ``UNSLOTH_FORCE_XPU=1``, or CUDA unavailable) AND a non-empty + ``ZE_AFFINITY_MASK`` AND ``torch.xpu`` reports a device. A stray + inherited mask is not enough: CUDA still wins on hybrid hosts. + 2. CUDA (NVIDIA GPU, requires torch) + 3. XPU (Intel GPU, requires torch with XPU support) + 4. MLX (Apple Silicon via MLX framework) + 5. CPU (fallback) """ global DEVICE, CHAT_ONLY, CHAT_ONLY_REASON, IS_ROCM CHAT_ONLY = True # reset -- only CUDA/ROCm/XPU/MLX sets it to False CHAT_ONLY_REASON = None IS_ROCM = False - # --- CUDA / ROCm: try PyTorch --- + # --- CUDA / ROCm / XPU: try PyTorch --- if _has_torch(): import torch + + # --- Explicit-XPU hint --- + # Prefer XPU on UNSLOTH_FORCE_XPU=1, or ZE_AFFINITY_MASK set + CUDA + # hidden/unavailable. A bare mask alone is NOT enough (can leak from + # unrelated Intel tooling); torch.xpu must report a device. + ze_mask = os.environ.get("ZE_AFFINITY_MASK") + cvd = os.environ.get("CUDA_VISIBLE_DEVICES") + cuda_hidden = cvd is not None and cvd.strip() in ("", "-1") + force_xpu = os.environ.get("UNSLOTH_FORCE_XPU") == "1" + try: + cuda_unavailable = not torch.cuda.is_available() + except Exception: + cuda_unavailable = True + + prefer_xpu = force_xpu or (bool(ze_mask) and (cuda_hidden or cuda_unavailable)) + if prefer_xpu: + try: + xpu_ok = hasattr(torch, "xpu") and torch.xpu.is_available() + except Exception: + xpu_ok = False + if xpu_ok: + # Forced XPU on a hybrid host: unsloth's device_type picks + # CUDA before XPU and ignores this Studio-only env var, so + # hide CUDA or spawned workers would silently train on CUDA. + if force_xpu and not cuda_hidden and not cuda_unavailable: + os.environ["CUDA_VISIBLE_DEVICES"] = "" + DEVICE = DeviceType.XPU + CHAT_ONLY = False + CHAT_ONLY_REASON = None + device_name = torch.xpu.get_device_name(0) + if force_xpu and not ze_mask: + reason = "UNSLOTH_FORCE_XPU=1" + elif force_xpu: + reason = "UNSLOTH_FORCE_XPU=1 + ZE_AFFINITY_MASK" + else: + reason = "ZE_AFFINITY_MASK hint honoured" + print(f"Hardware detected: XPU -- {device_name} ({reason})") + return DEVICE + + # --- CUDA: NVIDIA GPU --- if torch.cuda.is_available(): DEVICE = DeviceType.CUDA CHAT_ONLY = False @@ -327,9 +373,18 @@ def clear_gpu_cache(): torch.cuda.empty_cache() torch.cuda.ipc_collect() elif device == DeviceType.XPU: - import torch - torch.xpu.synchronize() - torch.xpu.empty_cache() + # Guard synchronize/empty_cache: older torch-xpu builds may lack + # them, and an unguarded AttributeError would propagate to callers. + # torch.xpu has no ipc_collect(), so do not call it here. + try: + import torch + if hasattr(torch, "xpu"): + if hasattr(torch.xpu, "synchronize"): + torch.xpu.synchronize() + if hasattr(torch.xpu, "empty_cache"): + torch.xpu.empty_cache() + except Exception as e: + logger.debug("Failed to clear XPU cache: %s", e) elif device == DeviceType.MLX: # MLX manages memory automatically; gc.collect() above is enough. pass @@ -500,14 +555,27 @@ def get_package_versions() -> Dict[str, Optional[str]]: except PackageNotFoundError: versions[name] = None - # GPU runtime version bundled with torch + # GPU runtime versions bundled with torch (CUDA, ROCm/HIP, Intel XPU) try: import torch + versions["cuda"] = getattr(torch.version, "cuda", None) versions["rocm"] = getattr(torch.version, "hip", None) + # Isolated probe: a broken Intel runtime raising in is_available() + # must not blank the already-read cuda/rocm versions. + try: + if hasattr(torch, "xpu") and torch.xpu.is_available(): + # torch.version.xpu may be None on modern builds; fall back to + # "available" so the UI distinguishes present-but-unknown from + # "package not found". + xpu_ver = getattr(torch.version, "xpu", None) + versions["xpu"] = xpu_ver if xpu_ver is not None else "available" + except Exception: + versions["xpu"] = None except Exception: versions["cuda"] = None versions["rocm"] = None + versions["xpu"] = None return versions @@ -538,21 +606,51 @@ def _torch_get_physical_gpu_count() -> Optional[int]: def _torch_get_per_device_info(device_indices: list[int]) -> list[Dict[str, Any]]: - """Query torch for per-GPU name, total VRAM, and used VRAM.""" + """Query torch for per-GPU name, total VRAM, and used VRAM. + + ``used_gb`` is ``None`` on Windows ROCm when ``hipMemGetInfo`` reports + ``free == total`` (ROCm/ROCm#1909): that 0 means unknown, not empty. + """ mod, _ = _torch_get_device_module() if mod is None: return [] + device = get_device() + # free==total is a Windows-ROCm-only quirk. + _win_rocm = sys.platform == "win32" and IS_ROCM devices = [] for ordinal, phys_idx in enumerate(device_indices): try: # torch ordinals are 0-based relative to CUDA_VISIBLE_DEVICES. props = mod.get_device_properties(ordinal) total_bytes = props.total_memory + used_bytes: Optional[int] # Prefer mem_get_info (system-wide) so auto-select sees other consumers. if hasattr(mod, "mem_get_info"): - free_bytes, total_bytes = mod.mem_get_info(ordinal) - used_bytes = total_bytes - free_bytes + try: + free_bytes, total_bytes = mod.mem_get_info(ordinal) + used_bytes = total_bytes - free_bytes + except Exception as e: + if device != DeviceType.XPU: + raise + # Arc B580 and Lunar Lake can report properties while + # rejecting free-memory queries. Preserve the usable + # device and its total memory with unknown utilization. + logger.debug( + "XPU free-memory query failed for ordinal %d: %s", + ordinal, + e, + ) + used_bytes = None + else: + # free==total is the broken-API sentinel, not an idle GPU. + if _win_rocm and free_bytes == total_bytes: + used_bytes = None + elif device == DeviceType.XPU: + # XPU without mem_get_info: memory_allocated() is process-local + # and misleading for placement, so return None for the + # selector's no-telemetry fallback. + used_bytes = None else: used_bytes = mod.memory_allocated(ordinal) devices.append( @@ -561,7 +659,9 @@ def _torch_get_per_device_info(device_indices: list[int]) -> list[Dict[str, Any] "visible_ordinal": ordinal, "name": props.name, "total_gb": round(total_bytes / (1024**3), 2), - "used_gb": round(used_bytes / (1024**3), 2), + "used_gb": ( + round(used_bytes / (1024**3), 2) if used_bytes is not None else None + ), } ) except Exception as e: @@ -572,6 +672,43 @@ def _torch_get_per_device_info(device_indices: list[int]) -> list[Dict[str, Any] # ========== Live GPU Utilization ========== +def _xpu_hierarchy_is_composite() -> bool: + """Return True iff Level Zero is running in COMPOSITE device hierarchy. + + COMPOSITE: numeric ``ZE_AFFINITY_MASK`` entries address root GPU IDs + (tiles use ``N.M``). FLAT (the oneAPI default; also assumed when + ``ZE_FLAT_DEVICE_HIERARCHY`` is unset): entries address tile/device + handles, so mapping them back to root GPU IDs is unsafe. Only COMPOSITE + gives stable root-ID semantics. + """ + hierarchy = (os.environ.get("ZE_FLAT_DEVICE_HIERARCHY") or "FLAT").strip().upper() + return hierarchy == "COMPOSITE" + + +def _parse_ze_mask_roots(mask: str) -> list[int]: + """Parse a ``ZE_AFFINITY_MASK`` value into an ordered list of root device IDs. + + One root ID per mask token, preserving order and duplicates so logical + ordinals map 1-to-1 to physical root IDs (e.g. ``"0.0,0.1"`` -> ``[0, 0]``, + ``"2.0,0.1,0.2"`` -> ``[2, 0, 0]``); empty list if no parseable digits. + Only meaningful in COMPOSITE hierarchy -- callers needing a stable + root-ID mapping must gate on ``_xpu_hierarchy_is_composite()``. + """ + roots: list[int] = [] + if not mask: + return roots + for token in mask.split(","): + token = token.strip() + if not token: + continue + root = token.split(".", 1)[0] + # isdecimal() (not isdigit()) rejects Unicode superscripts like + # "²"/"³", which pass isdigit() but crash int() with ValueError. + if root.isdecimal(): + roots.append(int(root)) + return roots + + def _smi_query(func_name: str, *args, **kwargs) -> Optional[Dict[str, Any]]: """Query the appropriate SMI backend (amd-smi or nvidia-smi). @@ -724,20 +861,165 @@ def _rocm_linux_sysfs_vram_gb() -> tuple[Optional[float], Optional[float]]: return None, None -def _rocm_windows_perf_counter_vram_gb() -> tuple[Optional[float], Optional[float]]: - """Query system-wide dedicated GPU VRAM via Windows Performance Counters. +# 0x1002. NVIDIA's open kernel module also registers KFD nodes (vendor_id 0x10DE); +# a non-AMD node is not a HIP device and must never take an ordinal. +_AMD_PCI_VENDOR_ID = 4098 - Same data source as Task Manager, so cross-process usage is accurate. - Works for any GPU vendor without amd-smi or nvidia-smi. - Returns (used_gb, total_gb) or (None, None) on failure. + +def _rocm_kfd_gpu_pci_ids() -> list[str]: + """PCI addresses of the GPUs ROCm enumerates, in HIP device order. + + Reads /sys/class/kfd/kfd/topology/nodes//properties, the topology ROCm + itself enumerates from: AMD GPU nodes (simd_count > 0 excludes CPUs, + vendor_id == AMD excludes NVIDIA) in node-id order are HIP's device order, so + position N is ROCm physical device N. Unlike DRM sysfs, an amdgpu adapter HIP + cannot enumerate has no node here, so it never consumes an ordinal. + + Returns [] (disabling the overlay) when KFD is absent, and FAILS CLOSED the + same way on any unreadable node or an AMD node with no location_id: dropping + one would shift every later ordinal and let a similar-capacity GPU pass the + total-size guard while showing another card's usage. + + location_id is the kernel's (bus << 8) | devfn; domain is separate. + """ + nodes: list[tuple[int, str]] = [] + try: + node_dirs = glob.glob("/sys/class/kfd/kfd/topology/nodes/*") + except Exception: + return [] + for node_dir in node_dirs: + m = re.fullmatch(r".*/(\d+)", node_dir) + if m is None: + continue + props: dict[str, int] = {} + try: + with open(os.path.join(node_dir, "properties")) as f: + for line in f: + parts = line.split() + if len(parts) == 2: + try: + props[parts[0]] = int(parts[1]) + except ValueError: + continue + except OSError: + return [] # unreadable node could be a GPU: fail closed, don't shift + if props.get("simd_count", 0) <= 0: + continue # CPU node, not a GPU + if props.get("vendor_id") != _AMD_PCI_VENDOR_ID: + continue # non-AMD GPU node (NVIDIA open driver): not a HIP device + location_id = props.get("location_id") + if location_id is None: + return [] # an AMD GPU we cannot place: fail closed for the whole map + domain = props.get("domain", 0) + bus = (location_id >> 8) & 0xFF + devfn = location_id & 0xFF + bdf = f"{domain:04x}:{bus:02x}:{(devfn >> 3) & 0x1F:02x}.{devfn & 0x7}" + nodes.append((int(m.group(1)), bdf)) + nodes.sort(key = lambda n: n[0]) + return [bdf for _node_id, bdf in nodes] + + +def _rocm_linux_amdgpu_cards() -> list[tuple[str, int, str]]: + """The amdgpu-bound DRM cards in PCI order: ``(pci_bdf, card_no, device_dir)``. + + Membership is by the BOUND DRIVER, not the VRAM sysfs files: an AMD device + with incomplete sysfs support (some APUs expose no mem_info_vram_*) still + consumes a ROCm ordinal, and dropping it would shift every later card down. + PCI order is HIP's default enumeration order, so list position is the ROCm + ordinal; card_no is a stable tiebreak when the BDF cannot be resolved. + + NOTE this is a superset of the ROCm-visible set (a HIP-unsupported amdgpu + adapter appears too), so callers must check the counts agree before assuming + a 1:1 mapping onto torch devices. + """ + if platform.system() != "Linux": + return [] + amd_cards: list[tuple[str, int, str]] = [] + try: + for card_path in glob.glob("/sys/class/drm/card*"): + # Match card exactly so connector nodes (card0-DP-1) are skipped. + m = re.fullmatch(r".*/card(\d+)", card_path) + if m is None: + continue + dev_dir = os.path.join(card_path, "device") + try: + driver = os.path.basename(os.path.realpath(os.path.join(dev_dir, "driver"))) + except OSError: + continue + if driver != "amdgpu": + continue # foreign adapter: not a ROCm device, takes no ordinal + try: + bdf = os.path.basename(os.path.realpath(dev_dir)) + except OSError: + bdf = "" + amd_cards.append((bdf, int(m.group(1)), dev_dir)) + except Exception: + return [] + amd_cards.sort(key = lambda c: (c[0], c[1])) + return amd_cards + + +def _rocm_linux_sysfs_vram_by_pci_gb() -> dict[str, tuple[float, float]]: + """System-wide AMD VRAM via Linux DRM sysfs, keyed by the card's PCI address. + + Reads each card's mem_info_vram_{used,total} (kernel-updated across all + processes) so every GPU gets its own figure, unlike _rocm_linux_sysfs_vram_gb + which sums the host. Keyed by PCI address, not an ordinal, so the caller can + join it to _rocm_kfd_gpu_pci_ids() by identity: DRM card numbers include + foreign adapters and this set includes cards HIP does not enumerate, so any + ordinal from this list alone can be shifted relative to ROCm's. A card with + missing/unreadable/zero-total figures simply has no entry. Empty off Linux. + """ + if platform.system() != "Linux": + return {} + + try: + by_pci: dict[str, tuple[float, float]] = {} + for bdf, _card_no, dev_dir in _rocm_linux_amdgpu_cards(): + if not bdf: + continue + try: + with open(os.path.join(dev_dir, "mem_info_vram_used")) as f: + used_bytes = int(f.read().strip()) + with open(os.path.join(dev_dir, "mem_info_vram_total")) as f: + total_bytes = int(f.read().strip()) + except (OSError, ValueError): + continue + if total_bytes <= 0: + continue + by_pci[bdf.lower()] = ( + round(used_bytes / (1024**3), 2), + round(total_bytes / (1024**3), 2), + ) + return by_pci + except Exception: + return {} + + +# ── Windows AMD/ROCm per-adapter VRAM (issue #7072) ────────────────────────── +# amd-smi is disabled and hipMemGetInfo reports free==total, so read used from the +# per-LUID "GPU Adapter Memory" perf counters and take each total from torch, so +# every GPU shows instead of one fake device with GPU 0's total. +# Placeholder adapters (Basic Render Driver / idle iGPU) drop only when they would +# outnumber the real torch devices. +_ROCM_WIN_ADAPTER_MIN_BYTES = 64 * 1024 * 1024 # 64 MiB + + +def _rocm_windows_perf_counter_vram_by_adapter() -> Optional[list[tuple[str, float]]]: + """Per-adapter dedicated VRAM usage on Windows via Performance Counters. + + Returns ``[(instance_name, used_bytes)]`` (one per LUID-named adapter), or + ``None`` when the counter is unavailable/localized/empty so callers fall back. """ if platform.system() != "Windows": - return None, None + return None try: + # Emit "|" per sample, or a __NONE__ sentinel. ps = ( "$s=(Get-Counter '\\GPU Adapter Memory(*)\\Dedicated Usage'" " -ErrorAction SilentlyContinue).CounterSamples;" - "if($s){($s|Measure-Object CookedValue -Sum).Sum}else{-1}" + "if($s){$s|ForEach-Object{'{0}|{1}' -f $_.InstanceName,[int64]$_.CookedValue}}" + "else{'__NONE__'}" ) r = subprocess.run( ["powershell", "-NoProfile", "-NonInteractive", "-Command", ps], @@ -746,16 +1028,167 @@ def _rocm_windows_perf_counter_vram_gb() -> tuple[Optional[float], Optional[floa timeout = 5, ) if r.returncode != 0 or not r.stdout.strip(): - return None, None - used_bytes = float(r.stdout.strip()) - if used_bytes < 0: - return None, None - import torch as _torch - - total_bytes = _torch.cuda.get_device_properties(0).total_memory - return round(used_bytes / (1024**3), 2), round(total_bytes / (1024**3), 2) + return None + adapters: list[tuple[str, float]] = [] + for line in r.stdout.splitlines(): + line = line.strip() + if not line or line == "__NONE__" or "|" not in line: + continue + instance, _, raw = line.rpartition("|") + try: + used = float(raw.strip()) + except (ValueError, TypeError): + continue + if used < 0: + continue + adapters.append((instance.strip(), used)) + return adapters or None except Exception: - return None, None + return None + + +def _match_adapter_used_to_devices( + adapter_useds: list[float], device_totals: list[float] +) -> list[Optional[float]]: + """Attribute per-adapter used bytes to torch devices by capacity ranking. + + Windows shares no key between LUID counters and torch ordinals, so usages are + ranked against device totals and each is trusted only when capacity *forces* it + (it exceeds every smaller device); an ambiguous ranking reports unknown + (``None``) rather than fabricate a per-index free. + + Extra counters mean a hidden/display adapter, and the noise filter may have + dropped a real reading, so values are emitted only when the supra-threshold + counters number EXACTLY the visible devices AND capacity forces the mapping; + otherwise every device is unknown. Best-effort but correct for the common + loaded-card case (#7072). Returns a list aligned to ``device_totals``. + """ + n = len(device_totals) + if n == 0: + return [] + useds = sorted(adapter_useds, reverse = True) + ranked_positions = sorted(range(n), key = lambda i: -device_totals[i]) + ranked_totals = [device_totals[pos] for pos in ranked_positions] + assigned: list[Optional[float]] + # More counters than devices -> a hidden/display adapter (check before noise filter). + if len(useds) > n: + non_trivial = [u for u in useds if u >= _ROCM_WIN_ADAPTER_MIN_BYTES] + if len(non_trivial) != n: + # Not a clean bijection (a masked GPU is busy or a visible card idle): + # no counter maps to a specific card, so report unknown. + return [None] * n + # Exactly n supra-threshold counters: extras were placeholders, so a + # capacity-ranked bijection is plausible. + useds = non_trivial + ranked_useds = [useds[rank] for rank in range(n)] + # A usage above its ranked capacity is a hidden larger GPU; clamping onto the + # smaller card would fabricate a fully-used reading. + for rank in range(n): + if ranked_useds[rank] > ranked_totals[rank]: + return [None] * n + # Capacity forces the mapping only when the usage exceeds the next-smaller + # capacity; the smallest card and merely-fitting usages stay unknown. + # Keeps 40 GiB over 48/8 GiB -> [40, None]. + assigned = [None] * n + for rank, pos in enumerate(ranked_positions): + if rank + 1 < n and ranked_useds[rank] > ranked_totals[rank + 1]: + assigned[pos] = min(ranked_useds[rank], device_totals[pos]) + return assigned + # No hidden adapters: every counter is a visible card, so ranking is a permutation. + ranked_useds = [useds[rank] if rank < len(useds) else 0.0 for rank in range(n)] + # Ambiguous if a strictly larger usage also fits the next smaller card: the two + # could be swapped without breaking capacity, so ranking can't tell them apart. + for rank in range(n - 1): + upper, lower = ranked_useds[rank], ranked_useds[rank + 1] + if upper > lower and upper <= ranked_totals[rank + 1]: + return [None] * n + assigned = [None] * n + for rank, pos in enumerate(ranked_positions): + if rank < len(useds): + assigned[pos] = min(useds[rank], device_totals[pos]) + return assigned + + +def _rocm_windows_per_device_vram(device_indices: list[int]) -> list[Dict[str, Any]]: + """Per-GPU VRAM on Windows AMD/ROCm: total from torch properties (reliable), + used from the per-adapter Dedicated Usage counter. + + Returns ``{index, visible_ordinal, name, used_gb, total_gb}`` per visible GPU + (``used_gb`` may be ``None`` when the counter is unavailable), or ``[]`` when + torch can't enumerate devices so callers fall through to the torch last resort. + """ + if platform.system() != "Windows": + return [] + mod, _ = _torch_get_device_module() + if mod is None: + return [] + # Totals/names from torch properties (mem_get_info's free==total quirk zeroes used). + dev_meta: list[Dict[str, Any]] = [] + for ordinal, phys_idx in enumerate(device_indices): + try: + props = mod.get_device_properties(ordinal) + dev_meta.append( + { + "index": phys_idx, + "visible_ordinal": ordinal, + "name": props.name, + "total_bytes": int(props.total_memory), + } + ) + except Exception as e: + logger.debug("torch property probe failed for ordinal %d: %s", ordinal, e) + if not dev_meta: + return [] + + adapters = _rocm_windows_perf_counter_vram_by_adapter() + if adapters: + assigned = _match_adapter_used_to_devices( + [used for _, used in adapters], + [d["total_bytes"] for d in dev_meta], + ) + else: + # Counter unavailable: show every GPU with a correct total, used unknown. + assigned = [None] * len(dev_meta) + + devices: list[Dict[str, Any]] = [] + for meta, used_bytes in zip(dev_meta, assigned): + total_gb = round(meta["total_bytes"] / (1024**3), 2) + used_gb = round(used_bytes / (1024**3), 2) if used_bytes is not None else None + devices.append( + { + "index": meta["index"], + "visible_ordinal": meta["visible_ordinal"], + "name": meta["name"], + "used_gb": used_gb, + "total_gb": total_gb, + } + ) + return devices + + +def _rocm_windows_device_payload_entry( + device: DeviceType, dev: Dict[str, Any], gpu_util_pct: Optional[float] +) -> Dict[str, Any]: + """Build a ``get_gpu_utilization`` device entry from a per-device VRAM dict.""" + total_gb = dev["total_gb"] + used_gb = dev["used_gb"] + return { + "available": True, + "backend": _backend_label(device), + "index": dev["index"], + "visible_ordinal": dev["visible_ordinal"], + "name": dev.get("name", "Unknown"), + "gpu_utilization_pct": gpu_util_pct, + "temperature_c": None, + "vram_used_gb": used_gb, + "vram_total_gb": total_gb, + "vram_utilization_pct": round((used_gb / total_gb) * 100, 1) + if total_gb and total_gb > 0 and used_gb is not None + else None, + "power_draw_w": None, + "power_limit_w": None, + "power_utilization_pct": None, + } def _gpu_utilization_payload( @@ -821,30 +1254,24 @@ def get_gpu_utilization() -> Dict[str, Any]: index_kind = result.get("index_kind"), ) - # Fallback Windows ROCm + # Fallback Windows ROCm: per-adapter VRAM attribution (issue #7072), so + # every visible GPU is shown instead of a sum collapsed onto one device. if IS_ROCM and platform.system() == "Windows": - _win_used, _win_total = _rocm_windows_perf_counter_vram_gb() - if _win_used is not None and _win_total is not None: - _win_util = _rocm_windows_perf_counter_gpu_util_pct() + _win_ids = _get_parent_visible_gpu_spec().get("numeric_ids") + if not _win_ids: + _win_ids = list(range(_torch_get_physical_gpu_count() or 0)) + _win_devices = _rocm_windows_per_device_vram(_win_ids) + if _win_devices: + # A single visible GPU can own the aggregate 3D-engine utilization; + # across several GPUs the sum isn't per-device, so leave it unset. + _win_util = ( + _rocm_windows_perf_counter_gpu_util_pct() if len(_win_devices) == 1 else None + ) return _gpu_utilization_payload( device, [ - { - "available": True, - "backend": _backend_label(device), - "index": 0, - "visible_ordinal": 0, - "gpu_utilization_pct": _win_util, - "temperature_c": None, - "vram_used_gb": _win_used, - "vram_total_gb": _win_total, - "vram_utilization_pct": round((_win_used / _win_total) * 100, 1) - if _win_total > 0 - else None, - "power_draw_w": None, - "power_limit_w": None, - "power_utilization_pct": None, - } + _rocm_windows_device_payload_entry(device, _wd, _win_util) + for _wd in _win_devices ], ) @@ -901,7 +1328,7 @@ def get_gpu_utilization() -> Dict[str, Any]: "vram_used_gb": _used, "vram_total_gb": _total, "vram_utilization_pct": round((_used / _total) * 100, 1) - if _total > 0 + if _total > 0 and _used is not None else None, "power_draw_w": None, "power_limit_w": None, @@ -995,19 +1422,27 @@ def _apply_unified_memory_correction( endpoints stay in sync on AMD iGPUs with unified memory. """ torch_total_gb = torch_info["total_gb"] + torch_used_gb = torch_info.get("used_gb") smi_total_gb = device_metrics.get("vram_total_gb") or 0.0 + # torch sees the full unified (GTT) pool; amd-smi only the dedicated carve-out. + # Adopt torch's larger total regardless of used: on Windows ROCm torch_used is + # None (free==total sentinel) but its total stays authoritative. Overwrite used + # only when torch's is known, then recompute utilization against whatever remains. if torch_total_gb > smi_total_gb: - torch_used_gb = torch_info["used_gb"] device_metrics["vram_total_gb"] = torch_total_gb - device_metrics["vram_used_gb"] = torch_used_gb + if torch_used_gb is not None: + device_metrics["vram_used_gb"] = torch_used_gb + _used_for_pct = device_metrics.get("vram_used_gb") device_metrics["vram_utilization_pct"] = ( - round((torch_used_gb / torch_total_gb) * 100, 1) if torch_total_gb > 0 else None + round((_used_for_pct / torch_total_gb) * 100, 1) + if torch_total_gb > 0 and _used_for_pct is not None + else None ) logger.debug( - "ROCm unified memory: replaced amd-smi VRAM (%.2f GB) with " - "torch mem_get_info total (%.2f GB) for device %s", - smi_total_gb, + "ROCm unified memory: adopted torch mem_get_info total (%.2f GB) over " + "amd-smi (%.2f GB) for device %s", torch_total_gb, + smi_total_gb, torch_info.get("index"), ) @@ -1049,6 +1484,75 @@ def _reconcile_primary_rocm_unified_memory( _apply_unified_memory_correction(utilization, torch_devices[0]) +def _rocm_visibility_mask_active() -> bool: + """True when any ROCm/CUDA visibility variable filters the device set.""" + for var in ( + "HIP_VISIBLE_DEVICES", + "ROCR_VISIBLE_DEVICES", + "CUDA_VISIBLE_DEVICES", + "GPU_DEVICE_ORDINAL", + ): + value = os.environ.get(var) + if value and value.strip(): + return True + return False + + +def _overlay_system_wide_vram(devices: list[Dict[str, Any]]) -> None: + """Replace process-local torch VRAM with system-wide Linux ROCm figures. + + The torch fallback is process-local, so a model served by the separate + llama-server process reads as ~0 used even with the GPU full (#7072). DRM + sysfs gives per-card figures the kernel updates across all processes. Sources + are matched by the device's PHYSICAL index (never list position), and only + when NO visibility mask is active and the device count equals the host GPU + count; under any mask the index is not a verifiable host ordinal, so torch's + figures are kept. Best-effort, in place: a device with no matching card, or a + unified-memory APU whose sysfs total is below torch's GTT-backed total, keeps + torch's (mirrors _apply_unified_memory_correction). + + Windows is intentionally not overlaid: its per-adapter perf counters cannot be + mapped to ROCm ordinals and miss WDDM shared memory, so the multi-GPU view + keeps torch there rather than risk misattributing another adapter's usage. + """ + if not devices or platform.system() != "Linux": + return + # Match by PCI identity, never list position: index N in KFD topology is ROCm + # physical device N and carries its PCI address, which DRM sysfs keys on too. + # The two gates below verify ``index`` really is a host-physical ordinal + # (torch exposes no PCI id to check directly): + # * No visibility mask -- any mask makes ``index`` container/ROCR-relative + # rather than a host ordinal. + # * Device count == host GPU count -- rules out a device-cgroup container + # that sets no env var yet compacts torch's indices from zero. + pci_by_ordinal = _rocm_kfd_gpu_pci_ids() + if not pci_by_ordinal: + return + if _rocm_visibility_mask_active() or len(devices) != len(pci_by_ordinal): + return + vram_by_pci = _rocm_linux_sysfs_vram_by_pci_gb() + for dev in devices: + index = dev.get("index") + if not isinstance(index, int) or not (0 <= index < len(pci_by_ordinal)): + continue + entry = vram_by_pci.get(pci_by_ordinal[index].lower()) + if entry is None: + continue + used, total = entry + dev_total = dev.get("vram_total_gb") or 0.0 + # Overlay only a device that maps 1:1 to the whole card: torch total must + # match sysfs total within ~10%. A mismatch either way means a different + # memory scope -- a unified-memory APU (sysfs sees only the dedicated + # slice, torch the GTT pool) or a partitioned MI300 (sysfs reports the + # whole card, dwarfing a partition) -- and overlaying would misstate free + # VRAM (a partition would look like it has the whole card free). + if dev_total <= 0 or abs(total - dev_total) > 0.1 * dev_total: + continue + dev["vram_used_gb"] = used + dev["vram_total_gb"] = total + dev["vram_utilization_pct"] = round((used / total) * 100, 1) if total > 0 else None + + def get_visible_gpu_utilization() -> Dict[str, Any]: device = get_device() @@ -1067,6 +1571,49 @@ def get_visible_gpu_utilization() -> Dict[str, Any]: _reconcile_rocm_unified_memory(result, numeric_ids) return result + # Windows AMD/ROCm (issue #7072): the System tab's VRAM source. The torch + # fallback below would report used==0 (free==total), so read per-adapter + # Dedicated Usage instead; total from torch properties. + if IS_ROCM and platform.system() == "Windows": + win_numeric_ids = parent_visible_spec.get("numeric_ids") + if win_numeric_ids: + win_ids = win_numeric_ids + win_index_kind = "physical" + else: + win_ids = list(range(_torch_get_physical_gpu_count() or 0)) + win_index_kind = "relative" + win_devices = _rocm_windows_per_device_vram(win_ids) + if win_devices: + devices = [] + for wd in win_devices: + total = wd["total_gb"] + used = wd["used_gb"] + devices.append( + { + "index": wd["index"], + "index_kind": win_index_kind, + "visible_ordinal": wd["visible_ordinal"], + "name": wd.get("name"), + "gpu_utilization_pct": None, + "temperature_c": None, + "vram_used_gb": used, + "vram_total_gb": total, + "vram_utilization_pct": round((used / total) * 100, 1) + if total and total > 0 and used is not None + else None, + "power_draw_w": None, + "power_limit_w": None, + "power_utilization_pct": None, + } + ) + return { + "available": True, + "backend": _backend_label(device), + "parent_visible_gpu_ids": win_numeric_ids or [], + "devices": devices, + "index_kind": win_index_kind, + } + # Torch-based fallback for CUDA (nvidia-smi unavailable, AMD ROCm) and XPU (Intel) if device in (DeviceType.CUDA, DeviceType.XPU): parent_ids = get_parent_visible_gpu_ids() @@ -1084,6 +1631,13 @@ def get_visible_gpu_utilization() -> Dict[str, Any]: for td in torch_devices: total = td["total_gb"] used = td["used_gb"] + # used=None is a deliberate "telemetry unavailable" signal + # from _torch_get_per_device_info (e.g. XPU without + # mem_get_info); propagate None instead of dividing by it. On + # CUDA/ROCm used is always an int, so this stays byte-identical. + vram_pct = ( + round((used / total) * 100, 1) if used is not None and total > 0 else None + ) devices.append( { "index": td["index"], @@ -1093,14 +1647,18 @@ def get_visible_gpu_utilization() -> Dict[str, Any]: "temperature_c": None, "vram_used_gb": used, "vram_total_gb": total, - "vram_utilization_pct": round((used / total) * 100, 1) - if total > 0 - else None, + "vram_utilization_pct": vram_pct, "power_draw_w": None, "power_limit_w": None, "power_utilization_pct": None, } ) + if IS_ROCM and index_kind == "physical": + # Swap process-local torch VRAM for system-wide sysfs so a model + # held by the separate llama-server process shows up (#7072). + # Physical-index only: a relative index (UUID/MIG mask) is not a + # host GPU id. The overlay verifies the rest itself. + _overlay_system_wide_vram(devices) return { "available": True, "backend": _backend_label(device), @@ -1157,6 +1715,82 @@ _visible_gpu_count: Optional[int] = None def _get_parent_visible_gpu_spec() -> Dict[str, Any]: + # On Intel XPU, visibility is controlled by ZE_AFFINITY_MASK (Level Zero), + # not CUDA_VISIBLE_DEVICES. + if get_device() == DeviceType.XPU: + xpu_mask_raw = os.environ.get("ZE_AFFINITY_MASK") + composite = _xpu_hierarchy_is_composite() + + if xpu_mask_raw is None: + # COMPOSITE: root GPU IDs are stable physical IDs. + if composite: + return { + "raw": None, + "numeric_ids": list(range(get_physical_gpu_count())), + "supports_explicit_gpu_ids": True, + } + # FLAT (oneAPI default): ordinals are tile/device handles, not + # physical GPU IDs. numeric_ids=None so telemetry uses relative + # ordinals; explicit selection needs ZE_FLAT_DEVICE_HIERARCHY=COMPOSITE. + return { + "raw": None, + "numeric_ids": None, + "supports_explicit_gpu_ids": False, + } + + xpu_mask = xpu_mask_raw.strip() + if xpu_mask == "": + return { + "raw": xpu_mask, + "numeric_ids": [], + "supports_explicit_gpu_ids": True, + } + + # Subdevice syntax ("N.M") expands one root into multiple + # logical devices -- not addressable by explicit root-ID selection. + has_subdevice = any("." in token.strip() for token in xpu_mask.split(",") if token.strip()) + if has_subdevice: + return { + "raw": xpu_mask, + "numeric_ids": None, + "supports_explicit_gpu_ids": False, + } + + # FLAT numeric entries are tile handles, not physical GPU IDs. Keep + # numeric_ids unresolved so every telemetry and picker consumer uses + # relative torch ordinals and cannot advertise them as pinnable roots. + if not composite: + tokens = [token.strip() for token in xpu_mask.split(",") if token.strip()] + if tokens and all(token.isdecimal() for token in tokens): + return { + "raw": xpu_mask, + "numeric_ids": None, + "supports_explicit_gpu_ids": False, + } + return { + "raw": xpu_mask, + "numeric_ids": None, + "supports_explicit_gpu_ids": False, + } + + # COMPOSITE + pure numeric (subdevice handled above). _parse_ze_mask_roots + # maps to root GPU IDs, dropping non-decimal tokens so "*"/"GPU-uuid" -> []. + roots_with_dupes = _parse_ze_mask_roots(xpu_mask) + if not roots_with_dupes: + # Unparseable mask (e.g. "*", "GPU-uuid") -- cannot map to + # physical root IDs. + return { + "raw": xpu_mask, + "numeric_ids": None, + "supports_explicit_gpu_ids": False, + } + + return { + "raw": xpu_mask, + "numeric_ids": roots_with_dupes, + "supports_explicit_gpu_ids": True, + } + # ROCm uses HIP/ROCR_VISIBLE_DEVICES on top of CUDA_VISIBLE_DEVICES; check # them first. Explicit None checks (not `or`) so "" reads as "no visible GPUs". cuda_visible = None @@ -1213,24 +1847,44 @@ def get_parent_visible_gpu_ids() -> list[int]: return list(parent_visible_ids) if parent_visible_ids is not None else [] -def resolve_requested_gpu_ids(gpu_ids: Optional[list[int]]) -> list[int]: +def resolve_requested_gpu_ids( + gpu_ids: Optional[list[int]], *, is_vulkan: bool = False +) -> list[int]: parent_visible_spec = _get_parent_visible_gpu_spec() parent_visible_ids = get_parent_visible_gpu_ids() physical_gpu_count = get_physical_gpu_count() if gpu_ids is None: - return parent_visible_ids + return [] if is_vulkan else parent_visible_ids requested_ids = list(gpu_ids) if len(requested_ids) == 0: - return parent_visible_ids + return [] if is_vulkan else parent_visible_ids + + if is_vulkan: + # A Vulkan build selects by ggml Vulkan ordinal (--device VulkanN), a separate + # index space from CUDA/ROCm ids that may be empty under CPU-only torch. The + # CUDA parent-visible / physical-count checks below do not apply; only reject + # malformed ordinals (issue #7239). + if len(set(requested_ids)) != len(requested_ids): + raise ValueError(f"Invalid gpu_ids {requested_ids}: duplicate GPU IDs are not allowed.") + negative_ids = [gpu_id for gpu_id in requested_ids if gpu_id < 0] + if negative_ids: + raise ValueError( + f"Invalid gpu_ids {requested_ids}: GPU IDs must be non-negative. " + f"Rejected IDs: {negative_ids}." + ) + return requested_ids if not parent_visible_spec["supports_explicit_gpu_ids"]: + env_var_name = ( + "ZE_AFFINITY_MASK" if get_device() == DeviceType.XPU else "CUDA_VISIBLE_DEVICES" + ) raise ValueError( f"Invalid gpu_ids {requested_ids}: explicit physical GPU IDs are " - f"unsupported when CUDA_VISIBLE_DEVICES uses UUID/MIG entries " - f"({parent_visible_spec['raw']!r}). Omit gpu_ids to use the " - "parent-visible devices." + f"unsupported when {env_var_name} uses non-numeric or subdevice " + f"entries ({parent_visible_spec['raw']!r}). Omit gpu_ids to use " + "the parent-visible devices." ) if len(set(requested_ids)) != len(requested_ids): @@ -1669,8 +2323,11 @@ def auto_select_gpu_ids( ) -> tuple[Optional[list[int]], Dict[str, Any]]: metadata: Dict[str, Any] = {"selection_mode": "auto"} - if get_device() != DeviceType.CUDA: - metadata["selection_mode"] = "non_cuda" + # Auto-selection needs per-device free-VRAM telemetry, available on CUDA + # (nvidia-smi) and XPU (torch.xpu) but not MLX/CPU, which fall + # through to inheriting parent visibility. + if get_device() not in (DeviceType.CUDA, DeviceType.XPU): + metadata["selection_mode"] = "non_accelerator" return None, metadata required_gb, estimate_metadata = estimate_required_model_memory_gb( @@ -1767,12 +2424,13 @@ def auto_select_gpu_ids( metadata["selection_mode"] = "auto" metadata["selected_gpu_ids"] = selected logger.debug( - "Selected GPUs automatically", - model_name = model_name, - selected_gpu_ids = selected, - usable_gb = metadata["usable_gb"], - required_gb = metadata.get("required_gb"), - multi_gpu_overhead = multi_gpu_overhead, + "Selected GPUs automatically: model=%s selected=%s usable_gb=%s " + "required_gb=%s multi_gpu_overhead=%s", + model_name, + selected, + metadata["usable_gb"], + metadata.get("required_gb"), + multi_gpu_overhead, ) return selected, metadata @@ -1788,12 +2446,13 @@ def auto_select_gpu_ids( metadata["usable_gb"] = round(fallback_usable, 3) metadata["selected_gpu_ids"] = fallback_all logger.warning( - "Falling back to all visible GPUs -- model may not fit", - model_name = model_name, - selected_gpu_ids = fallback_all, - usable_gb = metadata["usable_gb"], - required_gb = metadata.get("required_gb"), - multi_gpu_overhead = multi_gpu_overhead, + "Falling back to all visible GPUs; model may not fit: model=%s " + "selected=%s usable_gb=%s required_gb=%s multi_gpu_overhead=%s", + model_name, + fallback_all, + metadata["usable_gb"], + metadata.get("required_gb"), + multi_gpu_overhead, ) return fallback_all, metadata @@ -1827,10 +2486,10 @@ def prepare_gpu_selection( to a Hugging Face ``device_map`` string) and to ``apply_gpu_ids()`` in the worker subprocess (narrows ``CUDA_VISIBLE_DEVICES`` before torch/CUDA init). """ - if gpu_ids and get_device() != DeviceType.CUDA: + if gpu_ids and get_device() not in (DeviceType.CUDA, DeviceType.XPU): raise ValueError( - f"gpu_ids {list(gpu_ids)} is only supported on CUDA devices, " - f"but the current backend is '{get_device().value}'." + f"gpu_ids {list(gpu_ids)} is only supported on CUDA and Intel XPU " + f"devices, but the current backend is '{get_device().value}'." ) if gpu_ids: @@ -1903,11 +2562,14 @@ def get_physical_gpu_count() -> int: def _backend_visible_devices_env() -> Optional[str]: """Return the raw visibility env string that applies to this backend. - On ROCm, HIP_VISIBLE_DEVICES / ROCR_VISIBLE_DEVICES take precedence over - CUDA_VISIBLE_DEVICES; this mirrors ``_get_parent_visible_gpu_spec`` so + On XPU the control is ``ZE_AFFINITY_MASK`` (not ``CUDA_VISIBLE_DEVICES``); + on ROCm, HIP_VISIBLE_DEVICES / ROCR_VISIBLE_DEVICES take precedence over + CUDA_VISIBLE_DEVICES. Mirrors ``_get_parent_visible_gpu_spec`` so ``backend_cuda_visible_devices`` reports the value actually narrowing the - visible device set. + visible device set on the current backend. """ + if get_device() == DeviceType.XPU: + return os.environ.get("ZE_AFFINITY_MASK") if IS_ROCM: return _get_parent_visible_gpu_spec().get("raw") return os.environ.get("CUDA_VISIBLE_DEVICES") @@ -2022,6 +2684,43 @@ def get_visible_gpu_count() -> int: if _visible_gpu_count is not None: return _visible_gpu_count + # Prefer torch.xpu.device_count() on Intel XPU: the Level Zero runtime + # correctly interprets ZE_AFFINITY_MASK semantics (e.g. subdevice syntax + # "0.0,0.1" collapses onto one root GPU). Supersedes the torch fallback below. + if get_device() == DeviceType.XPU: + xpu_mask_raw = os.environ.get("ZE_AFFINITY_MASK") + xpu_mask_set = xpu_mask_raw is not None + xpu_visible = (xpu_mask_raw or "").strip() + if xpu_mask_set and xpu_visible == "": + _visible_gpu_count = 0 + return _visible_gpu_count + + try: + import torch + _visible_gpu_count = torch.xpu.device_count() + except Exception as e: + logger.debug( + "torch.xpu.device_count() failed, falling back to mask parsing: %s", + e, + ) + if xpu_visible: + # Fallback: count unique root device IDs from the mask. + # "device.subdevice" notation means "0.0,0.1" is 1 root, not 2. + # Without torch the hierarchy mode is unknown, so root-device + # counting is the conservative choice. + if xpu_visible == "*": + # Documented wildcard: all physical XPUs visible. + _visible_gpu_count = get_physical_gpu_count() + else: + roots = _parse_ze_mask_roots(xpu_visible) + # Non-parseable masks (",,,", "GPU-abc") yield an empty + # roots list, treated as 0 visible devices, not "all + # visible" -- no evidence the whole fleet was intended. + _visible_gpu_count = len(set(roots)) + else: + _visible_gpu_count = get_physical_gpu_count() + return _visible_gpu_count + # _get_parent_visible_gpu_spec() already handles HIP_VISIBLE_DEVICES / # ROCR_VISIBLE_DEVICES on ROCm. visible_spec = _get_parent_visible_gpu_spec() @@ -2035,20 +2734,18 @@ def get_visible_gpu_count() -> int: _visible_gpu_count = len([x for x in raw.split(",") if x.strip()]) return _visible_gpu_count - # No visibility env var set -- try torch, else physical count + # No visibility env var set -- try torch, else physical count. XPU is + # handled by the early return above, so only torch.cuda is needed here. try: import torch - if get_device() == DeviceType.XPU and hasattr(torch, "xpu"): - _visible_gpu_count = torch.xpu.device_count() - else: - _visible_gpu_count = torch.cuda.device_count() + _visible_gpu_count = torch.cuda.device_count() except Exception: _visible_gpu_count = get_physical_gpu_count() return _visible_gpu_count -def apply_gpu_ids(gpu_ids) -> None: +def apply_gpu_ids(gpu_ids, backend: Optional[str] = None) -> None: if gpu_ids is None: return @@ -2064,6 +2761,62 @@ def apply_gpu_ids(gpu_ids) -> None: else: value = str(gpu_ids) + # Intel XPU honors ZE_AFFINITY_MASK, not CUDA_VISIBLE_DEVICES; route XPU + # pinning through it so worker subprocesses are restricted to the intended GPU. + # Decide WITHOUT get_device(): workers call this before detect_hardware(), + # and a lazy detect would probe torch.cuda against the unmasked parent env, + # latching device enumeration before the mask below is written. Pre-detect, + # use env + torch BUILD attributes only (no runtime init, like the ROCm + # mirror below). + _is_xpu = DEVICE == DeviceType.XPU + if backend is not None: + # The spawning parent's detected backend (config["device_backend"]): + # exact and probe-free, so the mask target always matches what + # detect_hardware() decided in the parent, including its XPU + # availability check and CUDA fallback. + _is_xpu = backend == DeviceType.XPU.value + elif DEVICE is None: + # No parent backend passed (direct caller). version.xpu can be None + # on a working XPU build, so also accept torch.xpu._is_compiled() + # (a pure symbol-presence check, no runtime init). UNSLOTH_FORCE_XPU + # counts only on an XPU-capable build: detect_hardware() falls back + # to CUDA when XPU is missing, and the mask target must follow. + try: + import torch as _torch + + _ver = _torch.version + _is_comp = getattr(getattr(_torch, "xpu", None), "_is_compiled", None) + _xpu_build = (callable(_is_comp) and bool(_is_comp())) or ( + getattr(_ver, "xpu", None) is not None + ) + if os.environ.get("UNSLOTH_FORCE_XPU") == "1": + _is_xpu = _xpu_build + else: + # Mirror detect_hardware: hidden CUDA prefers XPU on an + # XPU-capable build (with or without a ZE mask -- detection + # falls through to XPU either way), where writing these ids + # to CUDA_VISIBLE_DEVICES would re-expose the deliberately + # hidden CUDA. + _cvd = os.environ.get("CUDA_VISIBLE_DEVICES") + _cuda_hidden = _cvd is not None and _cvd.strip() in ("", "-1") + _is_xpu = _xpu_build and ( + _cuda_hidden + or (getattr(_ver, "cuda", None) is None and getattr(_ver, "hip", None) is None) + ) + except Exception as e: + logger.debug( + "apply_gpu_ids: torch XPU probe skipped (%s: %s)", + type(e).__name__, + e, + ) + if _is_xpu: + os.environ["ZE_AFFINITY_MASK"] = value + # Leave inherited CUDA_VISIBLE_DEVICES alone -- clearing it could let + # the worker flip back to CUDA on hybrid hosts. + _visible_gpu_count = None + logger.info("Applied gpu_ids: ZE_AFFINITY_MASK='%s'", value) + return + os.environ["CUDA_VISIBLE_DEVICES"] = value # Keep ROCm visibility env vars in sync. Workers may call apply_gpu_ids() # before detect_hardware() (IS_ROCM still False), so also mirror when the @@ -2108,26 +2861,41 @@ def get_device_map(gpu_ids: Optional[list[int]] = None) -> str: Returns ``"balanced"`` (shard evenly across GPUs) when: - ``gpu_ids`` explicitly lists >1 GPU, **or** - - ``CUDA_VISIBLE_DEVICES`` uses UUID/MIG identifiers (non-numeric) and - >1 GPU is visible (fallback: numeric IDs unresolvable, so assume - multi-GPU is intended). + - ``CUDA_VISIBLE_DEVICES``/``ZE_AFFINITY_MASK`` uses non-numeric + identifiers (UUID/MIG/wildcard) and >1 GPU is visible (fallback: + numeric IDs unresolvable, so assume multi-GPU is intended). - Returns ``"sequential"`` (single device) otherwise, including non-CUDA - backends (CPU, MLX). + Returns ``"sequential"`` (single device) otherwise, including CPU/MLX + backends. Use ``prepare_gpu_selection()`` upstream to determine ``gpu_ids`` -- it handles auto-selecting the minimum GPUs needed for a model. """ device = get_device() - if device == DeviceType.CUDA: + if device in (DeviceType.CUDA, DeviceType.XPU): multi_gpu = gpu_ids is not None and len(gpu_ids) > 1 if not multi_gpu: - # UUID/MIG masks can't be split into numeric IDs; >1 visible GPU - # means multi-GPU sharding is intended. parent_visible_spec = _get_parent_visible_gpu_spec() - if parent_visible_spec["numeric_ids"] is None and get_visible_gpu_count() > 1: - multi_gpu = True + if device == DeviceType.CUDA: + # UUID/MIG masks can't be split into numeric IDs; >1 visible GPU + # means multi-GPU sharding is intended. + if parent_visible_spec["numeric_ids"] is None and get_visible_gpu_count() > 1: + multi_gpu = True + elif device == DeviceType.XPU and gpu_ids is None: + # Shard across visible XPU ordinals via HF (no mask rewrite), + # only when no gpu_ids were passed -- an explicit gpu_ids=[0] + # means "use exactly device 0" and must stay sequential. + supports_physical = parent_visible_spec["supports_explicit_gpu_ids"] + has_multiple_numeric = ( + parent_visible_spec["numeric_ids"] is not None + and len(parent_visible_spec["numeric_ids"]) > 1 + ) + has_multiple_unresolved = ( + parent_visible_spec["numeric_ids"] is None and get_visible_gpu_count() > 1 + ) + if has_multiple_unresolved or (not supports_physical and has_multiple_numeric): + multi_gpu = True if multi_gpu: return "balanced" @@ -2162,6 +2930,19 @@ def raise_if_offloaded( ) +def get_torch_device_str() -> str: + """ + Return the torch device string for the detected hardware. + E.g. "cuda", "xpu", or "cpu". + """ + device = get_device() + if device == DeviceType.CUDA: + return "cuda" + elif device == DeviceType.XPU: + return "xpu" + return "cpu" + + def safe_num_proc(desired: Optional[int] = None) -> int: """ Return a safe ``num_proc`` for ``dataset.map()`` calls. @@ -2229,7 +3010,32 @@ def dataset_map_num_proc(desired: Optional[int] = None) -> Optional[int]: Returns ``None`` on spawn platforms (Windows, macOS) because ``datasets`` treats ``num_proc=1`` as multiprocessing (creates ``Pool(1)``); only ``num_proc=None`` guarantees in-process execution. + + Also returns ``None`` on XPU once its runtime is initialized in this + process: ``os.fork()`` corrupts the Level-Zero context, making Triton + kernels fail with "Pointer argument doesn't reference XPU device memory". + Pre-init XPU hosts can still parallelize CPU-side preprocessing. """ if sys.platform in ("win32", "darwin"): return None + + if get_device() == DeviceType.XPU: + try: + import torch + except Exception: + # No torch means no active XPU runtime, so CPU-side dataset + # parallelism is still safe. + return safe_num_proc(desired) + + xpu = getattr(torch, "xpu", None) + is_initialized = getattr(xpu, "is_initialized", None) + if callable(is_initialized): + try: + if is_initialized(): + return None + except Exception as e: + # Treat a failing probe as "runtime not touched yet" so + # pre-init CPU preprocessing can still parallelize. + logger.debug("torch.xpu.is_initialized() probe failed: %s", e) + return safe_num_proc(desired) diff --git a/studio/backend/utils/helper_precache_settings.py b/studio/backend/utils/helper_precache_settings.py index db19a2d028..e7d3c0e6dd 100644 --- a/studio/backend/utils/helper_precache_settings.py +++ b/studio/backend/utils/helper_precache_settings.py @@ -32,7 +32,7 @@ def helper_model_disabled_by_env() -> bool: def get_helper_precache_enabled() -> bool: """Read the persisted startup pre-cache preference. - Missing or unreadable settings default to False so Studio startup never + Missing or unreadable settings default to False so Unsloth startup never performs optional network work unless the user explicitly opted in. """ try: @@ -45,7 +45,7 @@ def get_helper_precache_enabled() -> bool: def set_helper_precache_enabled(value: Any) -> bool: - """Persist whether Studio should pre-cache the Helper LLM at startup.""" + """Persist whether Unsloth should pre-cache the Helper LLM at startup.""" parsed = _coerce_bool(value) if parsed is None: raise ValueError("Helper LLM startup pre-cache must be true or false.") diff --git a/studio/backend/utils/hf_cache_settings.py b/studio/backend/utils/hf_cache_settings.py new file mode 100644 index 0000000000..07d901a3d2 --- /dev/null +++ b/studio/backend/utils/hf_cache_settings.py @@ -0,0 +1,362 @@ +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 + +"""Live, persisted Hugging Face cache routing for Unsloth Studio. + +Hugging Face reads cache environment variables at import time. Studio therefore +owns an explicit cache snapshot for each operation instead of trying to refresh +``huggingface_hub.constants`` in the long-running API process. +""" + +from __future__ import annotations + +import os +import shutil +import tempfile +import threading +from contextlib import contextmanager +from dataclasses import dataclass +from pathlib import Path +from typing import Iterator, Literal, Mapping, Optional + + +CACHE_HOME_SETTING_KEY = "hugging_face_cache_home" +CACHE_HISTORY_SETTING_KEY = "hugging_face_cache_history" +MAX_CACHE_HISTORY = 16 + +CacheSource = Literal["default", "studio", "environment"] + +_CACHE_ENV_KEYS = ( + "HF_HOME", + "HF_HUB_CACHE", + "HUGGINGFACE_HUB_CACHE", + "HF_XET_CACHE", +) +# Imported by storage_roots._setup_cache_env before Studio seeds defaults. +_EXPLICIT_CACHE_ENV = { + key: value.strip() + for key in _CACHE_ENV_KEYS + if (value := os.environ.get(key)) is not None and value.strip() +} +_settings_lock = threading.RLock() +_spawn_env_lock = threading.RLock() + + +@dataclass(frozen = True) +class HuggingFaceCachePaths: + cache_home: Path + hub_cache: Path + xet_cache: Path + source: CacheSource + environment_variable: Optional[str] = None + + @property + def editable(self) -> bool: + return self.source != "environment" + + @property + def is_custom(self) -> bool: + return self.source == "studio" + + def child_env(self, base: Optional[Mapping[str, str]] = None) -> dict[str, str]: + env = dict(os.environ if base is None else base) + # Do not rewrite HF_HOME. It also owns HF's token path, and credentials + # must not be moved onto a removable cache volume. + env["HF_HUB_CACHE"] = str(self.hub_cache) + env["HF_XET_CACHE"] = str(self.xet_cache) + env.pop("HUGGINGFACE_HUB_CACHE", None) + return env + + +def _default_cache_home() -> Path: + xdg = (os.environ.get("XDG_CACHE_HOME") or "").strip() + return (Path(xdg).expanduser() if xdg else Path.home() / ".cache") / "huggingface" + + +def _canonical(path: Path | str) -> Path: + return Path(path).expanduser().resolve(strict = False) + + +def _environment_paths() -> Optional[HuggingFaceCachePaths]: + explicit_home = _EXPLICIT_CACHE_ENV.get("HF_HOME") + explicit_hub = _EXPLICIT_CACHE_ENV.get("HF_HUB_CACHE") or _EXPLICIT_CACHE_ENV.get( + "HUGGINGFACE_HUB_CACHE" + ) + if not explicit_home and not explicit_hub: + return None + explicit_xet = _EXPLICIT_CACHE_ENV.get("HF_XET_CACHE") + default_home = _default_cache_home() + hf_home = _canonical(explicit_home) if explicit_home else default_home + hub = _canonical(explicit_hub) if explicit_hub else hf_home / "hub" + xet = _canonical(explicit_xet) if explicit_xet else hf_home / "xet" + controlling = next( + key + for key in ("HF_HUB_CACHE", "HUGGINGFACE_HUB_CACHE", "HF_HOME") + if key in _EXPLICIT_CACHE_ENV + ) + # Settings describes model downloads, so an explicit hub path is the + # displayed/opened location even when HF_HOME points somewhere else for + # credentials or XET data. + display_home = ( + (hub.parent if explicit_hub and hub.name.lower() == "hub" else hub) + if explicit_hub + else hf_home + ) + return HuggingFaceCachePaths(display_home, hub, xet, "environment", controlling) + + +def _stored_cache_home() -> Optional[Path]: + try: + from storage.studio_db import get_app_setting + value = get_app_setting(CACHE_HOME_SETTING_KEY, None) + except Exception: + return None + if not isinstance(value, str) or not value.strip(): + return None + try: + return _canonical(value.strip()) + except (OSError, RuntimeError, ValueError): + return None + + +def get_hf_cache_paths() -> HuggingFaceCachePaths: + env_paths = _environment_paths() + if env_paths is not None: + return env_paths + stored = _stored_cache_home() + if stored is not None: + xet = _EXPLICIT_CACHE_ENV.get("HF_XET_CACHE") + return HuggingFaceCachePaths( + stored, + stored / "hub", + _canonical(xet) if xet else stored / "xet", + "studio", + ) + home = _default_cache_home() + xet = _EXPLICIT_CACHE_ENV.get("HF_XET_CACHE") + return HuggingFaceCachePaths( + home, + home / "hub", + _canonical(xet) if xet else home / "xet", + "default", + ) + + +def active_hf_hub_cache() -> str: + """Return the current hub cache as a string for library call kwargs.""" + + return str(get_hf_cache_paths().hub_cache) + + +@contextmanager +def child_environment_for_spawn(environment: Mapping[str, str]) -> Iterator[None]: + """Apply captured env before spawn imports the child entrypoint. + + Applying variables only inside the multiprocessing target can be too late + for libraries that snapshot environment variables at import. The lock keeps + this short parent-process override atomic through ``Process.start()``. + """ + + with _spawn_env_lock: + missing = object() + saved_environment: dict[str, str | object] = {} + for key, value in environment.items(): + saved_environment[key] = os.environ.get(key, missing) + os.environ[key] = value + try: + yield + finally: + for key, previous in saved_environment.items(): + if previous is missing: + os.environ.pop(key, None) + else: + os.environ[key] = str(previous) + + +def initialize_hf_cache_environment() -> HuggingFaceCachePaths: + """Seed import-time HF variables once during backend startup.""" + + paths = get_hf_cache_paths() + # Preserve an explicit HF_HOME, otherwise keep credentials at the platform + # default while routing cache bytes through the selected home. + if not os.environ.get("HF_HOME", "").strip(): + os.environ["HF_HOME"] = str(_default_cache_home()) + os.environ["HF_HUB_CACHE"] = str(paths.hub_cache) + os.environ["HF_XET_CACHE"] = str(paths.xet_cache) + if "HUGGINGFACE_HUB_CACHE" not in _EXPLICIT_CACHE_ENV: + os.environ.pop("HUGGINGFACE_HUB_CACHE", None) + for directory in (paths.hub_cache, paths.xet_cache): + try: + directory.mkdir(parents = True, exist_ok = True) + except OSError: + pass + return paths + + +def _validate_cache_home(raw_path: str) -> Path: + value = raw_path.strip() + if not value: + raise ValueError("Choose a cache folder.") + candidate = Path(value).expanduser() + if not candidate.is_absolute(): + raise ValueError("The Hugging Face cache folder must be an absolute path.") + try: + resolved = candidate.resolve(strict = False) + except (OSError, RuntimeError, ValueError) as exc: + raise ValueError("The Hugging Face cache folder is invalid.") from exc + + if resolved.parent == resolved: + raise ValueError("Choose a folder inside the filesystem or drive root.") + try: + from hub.storage.scan_folders import ( + contains_sensitive_path_component, + is_denied_system_path, + ) + except ImportError: + contains_sensitive_path_component = is_denied_system_path = None + if is_denied_system_path is not None and is_denied_system_path(str(resolved)): + raise ValueError("System folders cannot be used for model downloads.") + if contains_sensitive_path_component is not None and contains_sensitive_path_component( + str(resolved) + ): + raise ValueError("Credential or config folders cannot be used for model downloads.") + + parent = resolved.parent + if not parent.exists() or not parent.is_dir(): + raise ValueError("The parent folder does not exist.") + try: + resolved.mkdir(exist_ok = True) + if not resolved.is_dir(): + raise ValueError("The selected cache location is not a folder.") + for child in (resolved / "hub", resolved / "xet"): + child.mkdir(exist_ok = True) + with tempfile.NamedTemporaryFile(prefix = ".unsloth-write-test-", dir = child): + pass + except PermissionError as exc: + raise ValueError("Studio does not have permission to write to this folder.") from exc + except OSError as exc: + raise ValueError(f"Studio cannot use this cache folder: {exc}") from exc + return resolved + + +def _stored_history() -> list[Path]: + try: + from storage.studio_db import get_app_setting + raw = get_app_setting(CACHE_HISTORY_SETTING_KEY, []) + except Exception: + raw = [] + if not isinstance(raw, list): + return [] + out: list[Path] = [] + seen: set[str] = set() + for value in raw: + if not isinstance(value, str) or not value.strip(): + continue + try: + path = _canonical(value) + except (OSError, RuntimeError, ValueError): + continue + key = os.path.normcase(str(path)) + if key in seen: + continue + seen.add(key) + out.append(path) + return out[:MAX_CACHE_HISTORY] + + +def set_hf_cache_home(cache_home: Optional[str]) -> HuggingFaceCachePaths: + if _environment_paths() is not None: + raise RuntimeError("The Hugging Face cache location is managed by an environment variable.") + with _settings_lock: + previous = _stored_cache_home() + next_home = _validate_cache_home(cache_home) if cache_home is not None else None + history = _stored_history() + if previous is not None and previous != next_home: + history.insert(0, previous) + deduped: list[str] = [] + seen: set[str] = set() + for path in history: + key = os.path.normcase(str(path)) + if key in seen or path == next_home: + continue + seen.add(key) + deduped.append(str(path)) + if len(deduped) >= MAX_CACHE_HISTORY: + break + from storage.studio_db import upsert_app_settings + + upsert_app_settings( + { + CACHE_HOME_SETTING_KEY: str(next_home) if next_home is not None else None, + CACHE_HISTORY_SETTING_KEY: deduped, + } + ) + # Inventory scans are cached independently from settings. Invalidate after + # persistence so the next request sees both the new active root and history. + from hub.utils.inventory_scan import invalidate_hf_cache_scans + + invalidate_hf_cache_scans() + return get_hf_cache_paths() + + +def known_hf_cache_homes() -> list[Path]: + paths = get_hf_cache_paths() + stored = _stored_cache_home() + candidates: list[Path] = [] + if paths.source != "environment": + candidates.append(paths.cache_home) + elif explicit_home := _EXPLICIT_CACHE_ENV.get("HF_HOME"): + candidates.append(_canonical(explicit_home)) + if stored is not None: + candidates.append(stored) + candidates.extend([*_stored_history(), _default_cache_home()]) + out: list[Path] = [] + seen: set[str] = set() + for candidate in candidates: + try: + canonical = _canonical(candidate) + except (OSError, RuntimeError, ValueError): + continue + key = os.path.normcase(str(canonical)) + if key in seen: + continue + seen.add(key) + out.append(canonical) + return out + + +def known_hf_hub_caches() -> list[Path]: + active = get_hf_cache_paths() + out = [active.hub_cache] + seen = {os.path.normcase(str(_canonical(active.hub_cache)))} + for home in known_hf_cache_homes(): + hub = _canonical(home / "hub") + key = os.path.normcase(str(hub)) + if key not in seen: + seen.add(key) + out.append(hub) + return out + + +def cache_status(paths: Optional[HuggingFaceCachePaths] = None) -> dict: + paths = paths or get_hf_cache_paths() + available = paths.cache_home.is_dir() + writable = available and os.access(paths.cache_home, os.W_OK | os.X_OK) + free_bytes: Optional[int] = None + if available: + try: + free_bytes = int(shutil.disk_usage(paths.cache_home).free) + except OSError: + pass + return { + "cache_home": str(paths.cache_home), + "hub_cache": str(paths.hub_cache), + "xet_cache": str(paths.xet_cache), + "source": paths.source, + "editable": paths.editable, + "is_custom": paths.is_custom, + "available": available, + "writable": writable, + "free_bytes": free_bytes, + "environment_variable": paths.environment_variable, + } diff --git a/studio/backend/utils/hf_token_validation.py b/studio/backend/utils/hf_token_validation.py new file mode 100644 index 0000000000..7247c6e756 --- /dev/null +++ b/studio/backend/utils/hf_token_validation.py @@ -0,0 +1,208 @@ +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 + +"""Cached, rate-limited Hugging Face token validation.""" + +from __future__ import annotations + +import hashlib +import threading +import time +from collections import deque +from dataclasses import dataclass +from typing import Literal + +from huggingface_hub import HfApi +from huggingface_hub.utils import build_hf_headers, get_session + + +TokenValidationStatus = Literal["valid", "invalid", "rate_limited", "unavailable"] + + +@dataclass(frozen = True) +class TokenValidationResult: + status: TokenValidationStatus + retry_after_seconds: int | None = None + + +_WINDOW_SECONDS = 3600.0 +_MAX_ATTEMPTS = 3 +_CACHE_TTL_SECONDS = 3600.0 +_TEMPORARY_CACHE_TTL_SECONDS = 15.0 +_MAX_BUCKETS = 4096 +_MAX_CACHE_ENTRIES = 4096 +_INFLIGHT_WAIT_SECONDS = 30.0 +_REMOTE_TIMEOUT_SECONDS = 10.0 + +_attempts: dict[str, deque[float]] = {} +_cache: dict[str, tuple[float, TokenValidationResult]] = {} +_inflight: dict[str, threading.Event] = {} +_lock = threading.Lock() + + +def _fingerprint(token: str) -> str: + return hashlib.sha256(token.encode("utf-8")).hexdigest() + + +def _prune_attempts(bucket: deque[float], now: float) -> None: + while bucket and now - bucket[0] >= _WINDOW_SECONDS: + bucket.popleft() + + +def _prune_locked(now: float) -> None: + for key in list(_attempts): + bucket = _attempts[key] + _prune_attempts(bucket, now) + if not bucket: + del _attempts[key] + for key, (expires_at, _result) in list(_cache.items()): + if expires_at <= now: + del _cache[key] + + +def _cached_locked(fingerprint: str, now: float) -> TokenValidationResult | None: + cached = _cache.get(fingerprint) + if cached is None: + return None + expires_at, result = cached + if expires_at <= now: + del _cache[fingerprint] + return None + return result + + +def _retry_after(bucket: deque[float], now: float) -> int: + return max(1, int(_WINDOW_SECONDS - (now - bucket[0])) + 1) + + +def _reserve_attempt_locked(rate_key: str, now: float) -> TokenValidationResult | None: + bucket = _attempts.get(rate_key) + if bucket is None: + if len(_attempts) >= _MAX_BUCKETS: + _prune_locked(now) + if len(_attempts) >= _MAX_BUCKETS: + return TokenValidationResult( + status = "rate_limited", + retry_after_seconds = max(1, int(_WINDOW_SECONDS)), + ) + bucket = _attempts[rate_key] = deque() + _prune_attempts(bucket, now) + if len(bucket) >= _MAX_ATTEMPTS: + return TokenValidationResult( + status = "rate_limited", + retry_after_seconds = _retry_after(bucket, now), + ) + bucket.append(now) + return None + + +def _http_status(response: object | None) -> int | None: + status = getattr(response, "status_code", None) + try: + return int(status) if status is not None else None + except (TypeError, ValueError): + return None + + +def _remote_retry_after(response: object | None) -> int | None: + headers = getattr(response, "headers", None) + if not headers: + return None + raw = headers.get("Retry-After") + try: + return max(1, int(float(raw))) if raw is not None else None + except (TypeError, ValueError): + return None + + +def _classify_response(response: object | None) -> TokenValidationResult: + status = _http_status(response) + if status is not None and 200 <= status < 300: + return TokenValidationResult(status = "valid") + if status == 401: + return TokenValidationResult(status = "invalid") + if status == 429: + return TokenValidationResult( + status = "rate_limited", + retry_after_seconds = _remote_retry_after(response), + ) + return TokenValidationResult(status = "unavailable") + + +def _check_remote(token: str) -> TokenValidationResult: + api = HfApi() + try: + # HfApi.whoami has no timeout parameter in the pinned Hub client. + # Use its session and headers against the same whoami endpoint. + response = get_session().get( + f"{api.endpoint}/api/whoami-v2", + headers = build_hf_headers(token = token), + timeout = _REMOTE_TIMEOUT_SECONDS, + ) + except Exception as exc: + # huggingface-hub 0.36.x can wrap a 401 as requests.HTTPError. + return _classify_response(getattr(exc, "response", None)) + return _classify_response(response) + + +def validate_hf_token(token: str, *, rate_key: str) -> TokenValidationResult: + """Validate ``token`` without retaining it, sharing results across callers. + + Cached checks do not consume the caller's three-per-hour network budget. A + single-flight event also prevents simultaneously mounted UI surfaces from + sending duplicate ``whoami`` requests for the same token. + """ + normalized = token.strip() + if not normalized: + return TokenValidationResult(status = "invalid") + token_fingerprint = _fingerprint(normalized) + owner_event: threading.Event | None = None + + try: + while True: + now = time.monotonic() + with _lock: + cached = _cached_locked(token_fingerprint, now) + if cached is not None: + return cached + waiting = _inflight.get(token_fingerprint) + if waiting is None: + limited = _reserve_attempt_locked(rate_key, now) + if limited is not None: + return limited + owner_event = threading.Event() + _inflight[token_fingerprint] = owner_event + break + if not waiting.wait(_INFLIGHT_WAIT_SECONDS): + return TokenValidationResult(status = "unavailable") + + result = _check_remote(normalized) + now = time.monotonic() + ttl = ( + _CACHE_TTL_SECONDS + if result.status in ("valid", "invalid") + else max(_TEMPORARY_CACHE_TTL_SECONDS, float(result.retry_after_seconds or 0)) + ) + with _lock: + if len(_cache) >= _MAX_CACHE_ENTRIES: + _prune_locked(now) + if len(_cache) < _MAX_CACHE_ENTRIES: + _cache[token_fingerprint] = (now + ttl, result) + return result + finally: + if owner_event is not None: + with _lock: + event = _inflight.get(token_fingerprint) + if event is owner_event: + _inflight.pop(token_fingerprint, None) + event.set() + + +def reset_hf_token_validation_state() -> None: + """Clear process state for test isolation.""" + with _lock: + for event in _inflight.values(): + event.set() + _inflight.clear() + _attempts.clear() + _cache.clear() diff --git a/studio/backend/utils/hf_xet_fallback.py b/studio/backend/utils/hf_xet_fallback.py index 9bc4a60fad..49872f371e 100644 --- a/studio/backend/utils/hf_xet_fallback.py +++ b/studio/backend/utils/hf_xet_fallback.py @@ -1,9 +1,9 @@ # SPDX-License-Identifier: AGPL-3.0-only # Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 -"""Studio shim over the shared ``unsloth_zoo.hf_xet_fallback`` Xet -> HTTP stall fallback. +"""Unsloth shim over the shared ``unsloth_zoo.hf_xet_fallback`` Xet -> HTTP stall fallback. -Re-exports the shared API and injects Studio's marker-aware cache purge +Re-exports the shared API and injects Unsloth's marker-aware cache purge (``prepare_cache_for_transport``) so the download manager keeps its ``.transport`` marker semantics on the HTTP retry. @@ -21,6 +21,8 @@ never triggers the heavy load. from __future__ import annotations import threading +from functools import partial +from pathlib import Path from typing import Any, Callable, Optional # Defaults mirror unsloth_zoo.hf_xet_fallback; plain literals so they resolve (including as @@ -68,7 +70,7 @@ def _load_shared() -> bool: _shared_available = True _shared_import_error = None return True - except Exception as exc2: # noqa: BLE001 - degrade so Studio still boots with plain HF + except Exception as exc2: # noqa: BLE001 - degrade so Unsloth still boots with plain HF _shared_import_error = exc2 _shared_available = False import logging as _logging @@ -262,18 +264,28 @@ __all__ = [ ] -def _studio_prepare_for_http(repo_type: str, repo_id: str) -> None: - """Studio's marker-aware purge before an HTTP resume, keeping the download manager's ``.transport`` +def _studio_prepare_for_http( + repo_type: str, + repo_id: str, + *, + cache_dir: Optional[str] = None, +) -> None: + """Unsloth's marker-aware purge before an HTTP resume, keeping the download manager's ``.transport`` accounting consistent (vs unsloth_zoo's generic default). Guarded: a purge failure is logged, not fatal to the retry.""" try: from hub.utils.download_registry import prepare_cache_for_transport - prepare_cache_for_transport(repo_type, repo_id, "http") + prepare_cache_for_transport( + repo_type, + repo_id, + "http", + root = Path(cache_dir) if cache_dir else None, + ) except Exception as exc: try: from loggers import get_logger get_logger(__name__).debug( - "Studio prepare_cache_for_transport failed for %s: %s", repo_id, exc + "Unsloth prepare_cache_for_transport failed for %s: %s", repo_id, exc ) except ModuleNotFoundError as logger_exc: if logger_exc.name != "loggers": @@ -293,9 +305,13 @@ def hf_hub_download_with_xet_fallback( grace_period: float = DEFAULT_GRACE_PERIOD, on_status: Optional[Callable[[str], None]] = None, force_download: bool = False, + cache_dir: Optional[str] = None, ) -> str: - """Single-file download via the shared fallback with Studio's marker-aware HTTP-retry prep. - ``force_download`` re-fetches a newer blob over a cached one (Studio's model-update path).""" + """Single-file download via the shared fallback with Unsloth's marker-aware HTTP-retry prep. + ``force_download`` re-fetches a newer blob over a cached one (Unsloth's model-update path).""" + if cache_dir is None: + from utils.hf_cache_settings import get_hf_cache_paths + cache_dir = str(get_hf_cache_paths().hub_cache) return _shared_hf_hub_download_with_xet_fallback( repo_id, filename, @@ -308,11 +324,18 @@ def hf_hub_download_with_xet_fallback( grace_period = grace_period, on_status = on_status, force_download = force_download, - prepare_for_http_fn = _studio_prepare_for_http, + cache_dir = cache_dir, + prepare_for_http_fn = partial(_studio_prepare_for_http, cache_dir = cache_dir), ) def snapshot_download_with_xet_fallback(repo_id: str, **kwargs: Any) -> str: - """Whole-repo download via the shared fallback with Studio's marker-aware HTTP-retry prep.""" - kwargs.setdefault("prepare_for_http_fn", _studio_prepare_for_http) + """Whole-repo download via the shared fallback with Unsloth's marker-aware HTTP-retry prep.""" + if kwargs.get("cache_dir") is None: + from utils.hf_cache_settings import get_hf_cache_paths + kwargs["cache_dir"] = str(get_hf_cache_paths().hub_cache) + kwargs.setdefault( + "prepare_for_http_fn", + partial(_studio_prepare_for_http, cache_dir = kwargs["cache_dir"]), + ) return _shared_snapshot_download_with_xet_fallback(repo_id, **kwargs) diff --git a/studio/backend/utils/hidden_models.py b/studio/backend/utils/hidden_models.py new file mode 100644 index 0000000000..e7c3181d71 --- /dev/null +++ b/studio/backend/utils/hidden_models.py @@ -0,0 +1,202 @@ +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 + +"""Infra-only model detection shared by the model routes and the hub +inventory. Lives directly under ``utils`` (not ``utils.models``) so the hub +cache scanner can import it without pulling in ``utils/models/__init__.py``, +which eagerly loads the model-config/checkpoint stack, and without importing +``routes.models`` (import-time side effects, would cycle).""" + +from __future__ import annotations + +import json +import re +from pathlib import Path +from typing import Optional + +# Hub repo id shape ("owner/name", no leading separator); anything else is +# treated as a local filesystem path. +_HF_REPO_ID_RE = re.compile(r"^[A-Za-z0-9][\w.\-]*/[\w.\-]+$") + +# The llama.cpp install-validation probe repo. Always hidden. +_PROBE_REPO_ID = "ggml-org/models" +# The probe's on-disk filename. Carries the ".gguf" so it stays specific and +# does not hide unrelated repos like ``user/stories260K-finetune-GGUF``. +_PROBE_FILENAME = "stories260k.gguf" +# Keep previously cached defaults hidden after settings changes. +_DEFAULT_EMBEDDING_REPO_IDS = { + "unsloth/bge-small-en-v1.5", + "unsloth/bge-small-en-v1.5-GGUF", +} +# Local copies do not always retain the repo id. Keep a narrow basename +# fallback for Studio's static default embedder only; configured custom repos +# remain exact-match-only. +_DEFAULT_EMBEDDING_PATH_BASENAMES = {"bge-small-en-v1.5"} +# Curated Whisper dictation checkpoints (STT, never chat), hidden from the chat +# inventory and pickers: Transformers safetensors repos (unsloth/whisper-*) and +# their GGUF companions (unslothai/whisper-*-GGUF). Custom checkpoints are caught +# by config below, but the GGUF companions carry a raw .bin (no config.json), so +# they must be listed here by id or they leak into chat pickers. +_HIDDEN_STT_REPO_IDS = frozenset( + { + "unsloth/whisper-tiny", + "unsloth/whisper-base", + "unsloth/whisper-small", + "unsloth/whisper-large-v3-turbo", + "unsloth/whisper-large-v3", + "unslothai/whisper-tiny-GGUF", + "unslothai/whisper-base-GGUF", + "unslothai/whisper-small-GGUF", + "unslothai/whisper-large-v3-turbo-GGUF", + "unslothai/whisper-large-v3-GGUF", + } +) + + +def _config_is_whisper(path: Path) -> bool: + """True if a config.json declares a Whisper model.""" + try: + with open(path, "r", encoding = "utf-8") as file: + config = json.load(file) + except Exception: + return False + if not isinstance(config, dict): + return False + model_type = config.get("model_type") + if isinstance(model_type, str) and model_type.strip().lower() == "whisper": + return True + architectures = config.get("architectures") + return isinstance(architectures, list) and any( + isinstance(name, str) and name == "WhisperForConditionalGeneration" + for name in architectures + ) + + +def _path_is_whisper_model(value: str) -> bool: + """Inspect an existing local model path's config; never hides name-only matches.""" + if _HF_REPO_ID_RE.fullmatch(value.strip()): + return False + path = Path(value).expanduser() + try: + if path.is_file(): + path = path.parent + candidates = [path / "config.json"] + snapshots = path / "snapshots" + if snapshots.is_dir(): + candidates.extend(child / "config.json" for child in snapshots.iterdir()) + except OSError: + return False + return any(_config_is_whisper(candidate) for candidate in candidates) + + +def _safe_resolve(path: Path) -> Optional[str]: + """resolve() to a string, or None when the path is inaccessible.""" + try: + return str(path.resolve()) + except OSError: + return None + + +def _existing_resolved_path(value: str) -> Optional[str]: + """Resolve an existing local path.""" + path = Path(value).expanduser() + try: + if not path.exists(): + return None + except OSError: + return None + return _safe_resolve(path) + + +def _path_contains_repo_id(value: str, repo_ids: set[str]) -> bool: + """Match exact repo-derived path segments.""" + parts = [part for part in value.lower().replace("\\", "/").split("/") if part] + for repo_id in repo_ids: + owner, name = repo_id.split("/", 1) + if f"models--{owner}--{name}" in parts: + return True + if any( + parts[index] == owner and parts[index + 1] == name for index in range(len(parts) - 1) + ): + return True + return False + + +def _path_basename_is_default_embedder(value: str) -> bool: + """Match a default embedder folder or a suffixed local weight filename.""" + normalized = value.lower().replace("\\", "/").rstrip("/") + basename = normalized.rsplit("/", 1)[-1] + return any( + basename == needle + or any(basename.startswith(f"{needle}{separator}") for separator in ("-", "_", ".")) + for needle in _DEFAULT_EMBEDDING_PATH_BASENAMES + ) + + +def is_hidden_model(*values: str | None) -> bool: + """True if any id/path is the RAG embedding model (the effective embedder + or its GGUF companion repo), the llama.cpp install validation probe + (ggml-org/models / stories260K), or a curated/custom Whisper dictation + model, so pickers hide them (GGUF and non-GGUF). None are usable chat + models; the probe can be cached as a side effect of installing the prebuilt + llama-server and otherwise sorts smallest, so it would be auto-selected. + + Hub repo ids are matched EXACTLY (case-insensitive full "owner/name"), so a + custom embedder with a generic basename like "org/model" cannot substring + hide unrelated cached repos such as "user/model-chat" or "org/model-GGUF". + Existing paths take precedence over the identical ``owner/name`` repo + shape. Cache and LM Studio paths use exact repo-derived segments. Local + copies of the static default embedder also use a boundary-aware basename + fallback; configured custom repos never do.""" + from core.rag import config as rag_config + + hidden_repo_ids = { + _PROBE_REPO_ID.lower(), + *(repo_id.lower() for repo_id in _DEFAULT_EMBEDDING_REPO_IDS), + *(repo_id.lower() for repo_id in _HIDDEN_STT_REPO_IDS), + } + exact_paths: list[str] = [] + for model in { + rag_config.EMBEDDING_MODEL, + rag_config.default_gguf_repo(), + rag_config.effective_embedding_model(), + rag_config.effective_gguf_repo(), + }: + existing_path = _existing_resolved_path(model) + if existing_path: + exact_paths.append(existing_path.lower()) + elif _HF_REPO_ID_RE.match(model): + hidden_repo_ids.add(model.lower()) + else: + resolved = _safe_resolve(Path(model).expanduser()) + if resolved: + exact_paths.append(resolved.lower()) + for v in values: + if not v: + continue + low = v.lower() + if _HF_REPO_ID_RE.match(v): + # A repo id ("owner/name"): match the hidden set exactly. It is + # never a filesystem path, so skip the path/filename checks. + if low in hidden_repo_ids: + return True + continue + # Anything else is treated as a filesystem path (the cached snapshot + # path, or a local model id). Match the probe by its exact filename and + # any configured local-path embedder by exact resolved path. Split on + # both separators so a Windows-style path ("...\\stories260K.gguf") is + # matched even when this runs on a POSIX interpreter (and vice versa). + if low.replace("\\", "/").rsplit("/", 1)[-1] == _PROBE_FILENAME: + return True + if _path_basename_is_default_embedder(v): + return True + if _path_contains_repo_id(v, hidden_repo_ids): + return True + # Custom Whisper checkpoints keep no curated repo id, so match by config. + if _path_is_whisper_model(v): + return True + if exact_paths: + resolved = _safe_resolve(Path(v).expanduser()) + if resolved and resolved.lower() in exact_paths: + return True + return False diff --git a/studio/backend/utils/host_policy.py b/studio/backend/utils/host_policy.py index f506eadc03..55565bb338 100644 --- a/studio/backend/utils/host_policy.py +++ b/studio/backend/utils/host_policy.py @@ -1,7 +1,7 @@ # SPDX-License-Identifier: AGPL-3.0-only # Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 -"""Bind-host trust policy for the Studio backend. +"""Bind-host trust policy for the Unsloth backend. Stdlib only -- safe to import without the rest of the backend. diff --git a/studio/backend/utils/inference/inference_config.py b/studio/backend/utils/inference/inference_config.py index 05eb08067c..a264e06c85 100644 --- a/studio/backend/utils/inference/inference_config.py +++ b/studio/backend/utils/inference/inference_config.py @@ -5,7 +5,10 @@ from pathlib import Path from typing import Dict, Any, Optional +from functools import lru_cache import json +import math +import os import yaml import structlog from loggers import get_logger @@ -160,3 +163,137 @@ def load_inference_config(model_identifier: str) -> Dict[str, Any]: } return inference_config + + +# ── Effective sampling resolution for `unsloth run` / `unsloth start` ────────── +# +# Per-model recommended sampling is applied to a request only for the fields the +# client omitted; an operator can pin a field from the CLI via UNSLOTH_SAMPLING_* +# (a hard override that wins even over an explicit client value). Precedence per +# field: operator pin -> client explicit -> per-model recommendation -> the static +# schema default (mirroring ChatCompletionRequest, so behavior is unchanged when +# nothing is recommended or pinned). + +# field -> (env var, static default, min, max, is_int) +_SAMPLING_FIELDS = { + "temperature": ("UNSLOTH_SAMPLING_TEMPERATURE", 0.6, 0.0, 2.0, False), + "top_p": ("UNSLOTH_SAMPLING_TOP_P", 0.95, 0.0, 1.0, False), + "top_k": ("UNSLOTH_SAMPLING_TOP_K", 20, -1, 100, True), + "min_p": ("UNSLOTH_SAMPLING_MIN_P", 0.01, 0.0, 1.0, False), + "repetition_penalty": ("UNSLOTH_SAMPLING_REPETITION_PENALTY", 1.0, 1.0, 2.0, False), + "presence_penalty": ("UNSLOTH_SAMPLING_PRESENCE_PENALTY", 0.0, 0.0, 2.0, False), +} + +# Public, ordered tuple of the sampling fields callers resolve. +SAMPLING_FIELD_NAMES = tuple(_SAMPLING_FIELDS) + +# Fields the Studio Chat UI adopts as *per-model recommendations* from the backend +# `.inference` block. Its frontend `mergeBackendRecommendedInference` +# (presets/preset-policy.ts) seeds exactly these five and never reads repetition_penalty, +# so the server auto-recommends the same five for request parity. repetition_penalty stays a +# manual-only knob (client-sent or an UNSLOTH_SAMPLING_REPETITION_PENALTY operator pin), +# matching the UI where it is never auto-filled per model. +_UI_RECOMMENDED_FIELDS = ("temperature", "top_p", "top_k", "min_p", "presence_penalty") + + +def _clean_sampling_value(field: str, val: Any): + """Coerce ``val`` to the field's numeric type when it is a finite, in-range number, else None. + + Rejects bool, non-numeric, NaN/inf, and out-of-range values so neither a bad operator env + var nor a malformed model recommendation can reach llama-server. NaN matters because + ``nan < lo`` and ``nan > hi`` are both False, so a plain range check would let it through. + Coerce before the finiteness check: ``math.isfinite`` and ``float()`` raise ``OverflowError`` + on an int too big for a C double (an oversized UNSLOTH_SAMPLING_TOP_K would otherwise 500 the + request), while an in-range int is range-checked exactly and ``int()`` rejects a NaN/inf that + reached an int field. + """ + if isinstance(val, bool) or not isinstance(val, (int, float)): + return None + _env, _default, lo, hi, is_int = _SAMPLING_FIELDS[field] + try: + val = int(val) if is_int else float(val) + except (ValueError, OverflowError): + # int(nan)/int(inf) and float(oversized_int) raise; treat them as unusable. + return None + # After coercion an int is always finite; only a float can still be NaN/inf. + if isinstance(val, float) and not math.isfinite(val): + return None + if val < lo or val > hi: + return None + return val + + +def _operator_sampling_override(field: str): + """Operator-pinned value for a sampling field from UNSLOTH_SAMPLING_*, or None. + + An unparseable, non-finite, or out-of-range value is ignored so a bad env var can never + reach llama-server; the field then falls back to the client / recommended value. + """ + _env, _default, _lo, _hi, is_int = _SAMPLING_FIELDS[field] + raw = os.environ.get(_env) + if raw is None or raw.strip() == "": + return None + try: + val = int(raw) if is_int else float(raw) + except (TypeError, ValueError): + return None + return _clean_sampling_value(field, val) + + +@lru_cache(maxsize = 128) +def _recommended_sampling(model_id: str) -> Dict[str, Any]: + """Per-model recommended sampling, resolved through the SAME path the Studio Chat UI uses. + + The Chat UI seeds its sampling from the ``.inference`` block of the load/status responses, + which is exactly :func:`load_inference_config` (model-specific YAML -> family defaults + (inference_defaults.json) -> default.yaml). Sourcing recommendations here keeps the values + the server applies to a request identical to what the UI shows for the same model. Only the + fields the UI actually adopts (:data:`_UI_RECOMMENDED_FIELDS`) are recommended; each value + is validated (finite + in range) before use. Cached by model id. + """ + if not model_id: + return {} + try: + cfg = load_inference_config(model_id) or {} + except Exception as e: + logger.debug(f"Could not load recommended sampling for '{model_id}': {e}") + return {} + recommended: Dict[str, Any] = {} + for field in _UI_RECOMMENDED_FIELDS: + cleaned = _clean_sampling_value(field, cfg.get(field)) + if cleaned is not None: + recommended[field] = cleaned + return recommended + + +def resolve_effective_sampling( + model_id: Optional[str], + explicit: Dict[str, Any], + *, + fill_defaults: bool = True, +) -> Dict[str, Any]: + """Resolve the effective sampling params for a request. + + ``explicit`` maps each field in :data:`SAMPLING_FIELD_NAMES` to the client-sent + value, or ``None`` when the client omitted it. Precedence (highest first): an + operator ``UNSLOTH_SAMPLING_*`` pin, then the client's explicit value, then the + per-model recommendation, then the static schema default. + + When ``fill_defaults`` is False a field with no operator pin, client value, or + per-model recommendation is omitted from the result instead of set to the static + schema default, so a raw proxy body (``/v1/completions``) keeps llama-server's own + default for that field rather than being forced onto this schema's value. + """ + recommended = _recommended_sampling(model_id or "") + effective: Dict[str, Any] = {} + for field, (_env, default, _lo, _hi, _int) in _SAMPLING_FIELDS.items(): + override = _operator_sampling_override(field) + if override is not None: + effective[field] = override + elif explicit.get(field) is not None: + effective[field] = explicit[field] + elif field in recommended: + effective[field] = recommended[field] + elif fill_defaults: + effective[field] = default + return effective diff --git a/studio/backend/utils/llama_cpp_freshness.py b/studio/backend/utils/llama_cpp_freshness.py index 7d077bfa3b..a184fdb3e9 100644 --- a/studio/backend/utils/llama_cpp_freshness.py +++ b/studio/backend/utils/llama_cpp_freshness.py @@ -7,28 +7,28 @@ Reads UNSLOTH_PREBUILT_INFO.json (written by install_llama_prebuilt.py) and compares the installed release tag against the latest on GitHub. Surfaced via main.py:lifespan() and /api/inference/status. Fails open on any missing data so we never show a misleading banner. + +The mechanics (marker walk-up, GitHub fetch, memo + disk cache, report +skeleton) live in utils.prebuilt.freshness_flow; this module keeps the +llama version policy and the per-module caches its tests patch. """ from __future__ import annotations -import json -import os import re -import time -from datetime import datetime, timezone +from datetime import datetime from pathlib import Path from typing import Optional import structlog +from utils.prebuilt import freshness_flow as _flow + logger = structlog.get_logger(__name__) # 3 days matches Unsloth's typical llama.cpp release cadence. STALENESS_THRESHOLD_DAYS = 3 -# 24h TTL keeps the GitHub call off the hot path and within rate limits. -_RELEASE_CACHE_TTL_SECONDS = 24 * 60 * 60 - _INSTALL_MARKER_NAME = "UNSLOTH_PREBUILT_INFO.json" _marker_cache: dict[str, Optional[dict]] = {} @@ -49,203 +49,60 @@ def _cache_dir() -> Path: def read_install_marker(binary_path: Optional[str]) -> Optional[dict]: """Walk up from binary_path to find UNSLOTH_PREBUILT_INFO.json. None = no marker (source build / custom path) or invalid JSON.""" - if not binary_path: - return None - cached = _marker_cache.get(binary_path) - if cached is not None or binary_path in _marker_cache: - return cached - p = Path(binary_path) - marker: Optional[dict] = None - # Cover all _find_llama_server_binary layouts (binary is 1-4 dirs deep): - for parent in p.parents[:5]: - candidate = parent / _INSTALL_MARKER_NAME - if candidate.is_file(): - try: - marker = json.loads(candidate.read_text(encoding = "utf-8")) - except (OSError, json.JSONDecodeError) as exc: - logger.debug( - "failed to parse install marker", - path = str(candidate), - error = str(exc), - ) - marker = None - break - _marker_cache[binary_path] = marker - return marker - - -def _cache_path_for(repo: str) -> Path: - safe = repo.replace("/", "__") - return _cache_dir() / f"{safe}.json" + return _flow.read_install_marker( + binary_path, + marker_name = _INSTALL_MARKER_NAME, + cache = _marker_cache, + log_message = "failed to parse install marker", + ) def _load_disk_cache(repo: str) -> Optional[tuple[float, Optional[str]]]: - path = _cache_path_for(repo) - try: - payload = json.loads(path.read_text(encoding = "utf-8")) - except (OSError, json.JSONDecodeError): - return None - ts = payload.get("fetched_at") - tag = payload.get("latest_tag") - if not isinstance(ts, (int, float)): - return None - return float(ts), tag if isinstance(tag, str) else None + return _flow.load_disk_cache(repo, _cache_dir()) def _save_disk_cache(repo: str, latest_tag: Optional[str]) -> None: - path = _cache_path_for(repo) - try: - path.parent.mkdir(parents = True, exist_ok = True) - tmp = path.with_suffix(".tmp") - tmp.write_text( - json.dumps({"fetched_at": time.time(), "latest_tag": latest_tag}), - encoding = "utf-8", - ) - tmp.replace(path) - except OSError as exc: - logger.debug("freshness cache write failed", repo = repo, error = str(exc)) + _flow.save_disk_cache( + repo, latest_tag, _cache_dir(), log_message = "freshness cache write failed" + ) def _fetch_latest_release_tag(repo: str, timeout: float = 5.0) -> Optional[str]: - """Newest published release tag for `repo`, by publish time. - - Resolves "latest" the way install_llama_prebuilt.py does (newest - non-draft/non-prerelease by ``published_at``), NOT via GitHub's - ``/releases/latest`` pointer. That pointer sorts by commit date and can lag - behind the build the installer actually installs, so detection and apply - disagreed -- the cause of the downgrade/sticky banner. None on any failure - (offline, rate-limited, etc).""" - import urllib.error - import urllib.request - - url = f"https://api.github.com/repos/{repo}/releases?per_page=30" - headers = { - "Accept": "application/vnd.github+json", - "User-Agent": "unsloth-studio-freshness-check", - } - token = os.environ.get("GITHUB_TOKEN") or os.environ.get("GH_TOKEN") - if token: - headers["Authorization"] = f"Bearer {token}" - req = urllib.request.Request(url, headers = headers) - try: - with urllib.request.urlopen(req, timeout = timeout) as resp: - data = json.loads(resp.read().decode("utf-8")) - except ( - urllib.error.URLError, - urllib.error.HTTPError, - OSError, - json.JSONDecodeError, - ) as exc: - logger.debug("freshness fetch failed", repo = repo, error = str(exc)) - return None - if not isinstance(data, list): - return None - published = [ - r - for r in data - if isinstance(r, dict) - and not r.get("draft") - and not r.get("prerelease") - and isinstance(r.get("tag_name"), str) - and r.get("tag_name") - ] - if not published: - return None - newest = max(published, key = lambda r: r.get("published_at") or "") - return newest["tag_name"] + """Newest published release tag for `repo`, by publish time (see + freshness_flow for why this is not GitHub's /releases/latest pointer).""" + return _flow.fetch_latest_release_tag(repo, timeout, log_message = "freshness fetch failed") def latest_published_release(repo: str, *, force_refresh: bool = False) -> Optional[str]: """Latest release tag for `repo`. Memo + disk-cached (24h TTL). None when offline and never previously cached.""" - if not repo: - return None - now = time.time() - if not force_refresh: - memo = _release_memo.get(repo) - if memo and now - memo[0] < _RELEASE_CACHE_TTL_SECONDS: - return memo[1] - disk = _load_disk_cache(repo) - if disk and now - disk[0] < _RELEASE_CACHE_TTL_SECONDS: - _release_memo[repo] = disk - return disk[1] - latest = _fetch_latest_release_tag(repo) - if latest is None: - # Keep last-good disk value rather than poisoning with None. - disk = _load_disk_cache(repo) - if disk: - _release_memo[repo] = disk - return disk[1] - return None - _release_memo[repo] = (now, latest) - _save_disk_cache(repo, latest) - return latest + return _flow.latest_published_release( + repo, + force_refresh = force_refresh, + memo = _release_memo, + cache_dir = lambda: _cache_dir(), + fetch = lambda r: _fetch_latest_release_tag(r), + save = lambda r, tag: _save_disk_cache(r, tag), + ) def _fetch_latest_release_assets(repo: str, timeout: float = 5.0) -> Optional[dict[str, int]]: """Asset name -> size (bytes) for the newest published release of `repo`, selected exactly like _fetch_latest_release_tag. None on any failure.""" - import urllib.error - import urllib.request - - url = f"https://api.github.com/repos/{repo}/releases?per_page=30" - headers = { - "Accept": "application/vnd.github+json", - "User-Agent": "unsloth-studio-freshness-check", - } - token = os.environ.get("GITHUB_TOKEN") or os.environ.get("GH_TOKEN") - if token: - headers["Authorization"] = f"Bearer {token}" - req = urllib.request.Request(url, headers = headers) - try: - with urllib.request.urlopen(req, timeout = timeout) as resp: - data = json.loads(resp.read().decode("utf-8")) - except ( - urllib.error.URLError, - urllib.error.HTTPError, - OSError, - json.JSONDecodeError, - ) as exc: - logger.debug("freshness asset fetch failed", repo = repo, error = str(exc)) - return None - if not isinstance(data, list): - return None - published = [ - r - for r in data - if isinstance(r, dict) - and not r.get("draft") - and not r.get("prerelease") - and isinstance(r.get("tag_name"), str) - and r.get("tag_name") - ] - if not published: - return None - newest = max(published, key = lambda r: r.get("published_at") or "") - assets: dict[str, int] = {} - for a in newest.get("assets") or []: - name, size = a.get("name"), a.get("size") - if isinstance(name, str) and isinstance(size, int): - assets[name] = size - return assets + return _flow.fetch_latest_release_assets( + repo, timeout, log_message = "freshness asset fetch failed" + ) def latest_release_assets(repo: str, *, force_refresh: bool = False) -> Optional[dict[str, int]]: """Newest-release asset sizes for `repo`, memoized (24h TTL). None when offline and never fetched. In-memory only -- a restart simply re-fetches.""" - if not repo: - return None - now = time.time() - if not force_refresh: - memo = _assets_memo.get(repo) - if memo and now - memo[0] < _RELEASE_CACHE_TTL_SECONDS: - return memo[1] - assets = _fetch_latest_release_assets(repo) - if assets is None: - memo = _assets_memo.get(repo) - return memo[1] if memo else None - _assets_memo[repo] = (now, assets) - return assets + return _flow.latest_release_assets( + repo, + force_refresh = force_refresh, + memo = _assets_memo, + fetch = lambda r: _fetch_latest_release_assets(r), + ) def update_download_size_bytes( @@ -290,16 +147,7 @@ def update_download_size_bytes( def _parse_installed_at(value: object) -> Optional[datetime]: - if not isinstance(value, str) or not value: - return None - s = value.replace("Z", "+00:00") if value.endswith("Z") else value - try: - dt = datetime.fromisoformat(s) - except ValueError: - return None - if dt.tzinfo is None: - dt = dt.replace(tzinfo = timezone.utc) - return dt + return _flow.parse_installed_at(value) def parse_base_build(tag: object) -> Optional[int]: @@ -350,64 +198,27 @@ def check_prebuilt_freshness( behind = installed genuinely older than latest (see is_behind). stale = behind AND age >= threshold. Fails open on missing data (behind/stale stay False).""" - out: dict = { - "has_marker": False, - "stale": False, - "behind": False, - "installed_tag": None, - "latest_tag": None, - "installed_at_utc": None, - "age_days": None, - "published_repo": None, - "threshold_days": int(threshold_days), - } - marker = read_install_marker(binary_path) - if not marker: - return out - out["has_marker"] = True - # Display prefers the normalized base ("tag"); comparison below prefers the - # full "release_tag" -- deliberately opposite fallbacks. - out["installed_tag"] = marker.get("tag") or marker.get("release_tag") - out["installed_at_utc"] = marker.get("installed_at_utc") - out["published_repo"] = marker.get("published_repo") - # The marker records both a normalized base tag ("tag", e.g. b9596) and the - # full release tag ("release_tag", e.g. b9596-mix-). Compare against the - # FULL identity, since GitHub /releases/latest returns the full tag_name -- - # comparing the normalized base against the full latest is what produced the - # permanent "downgrade" banner on every mix release. - installed_full = marker.get("release_tag") or marker.get("tag") - repo = out["published_repo"] - if not repo or not installed_full: - return out - latest = latest_published_release(repo) - out["latest_tag"] = latest - out["behind"] = is_behind(installed_full, latest) - if not out["behind"]: - return out - - installed_at = _parse_installed_at(out["installed_at_utc"]) - if installed_at is None: - return out - now = now or datetime.now(tz = timezone.utc) - age_seconds = (now - installed_at).total_seconds() - out["age_days"] = max(0, int(age_seconds // 86400)) - if age_seconds >= threshold_days * 86400: - out["stale"] = True - return out + # full release tag ("release_tag", e.g. b9596-mix-). Display prefers the + # normalized base; comparison uses the FULL identity, since GitHub + # /releases/latest returns the full tag_name -- comparing the normalized base + # against the full latest is what produced the permanent "downgrade" banner + # on every mix release. Deliberately opposite fallbacks. + return _flow.check_freshness( + binary_path, + threshold_days = threshold_days, + now = now, + read_marker = lambda p: read_install_marker(p), + latest_release = lambda repo: latest_published_release(repo), + behind = lambda installed, latest: is_behind(installed, latest), + display_tag = lambda marker: marker.get("tag") or marker.get("release_tag"), + compare_tag = lambda marker: marker.get("release_tag") or marker.get("tag"), + ) def format_stale_warning(info: dict) -> str: """Human-readable one-liner for stale prebuilt info.""" - age = info.get("age_days") - installed = info.get("installed_tag") or "unknown" - latest = info.get("latest_tag") or "unknown" - age_str = f"{age} day{'s' if age != 1 else ''}" if age is not None else "some time" - return ( - f"llama.cpp prebuilt is {age_str} behind: installed " - f"{installed}, latest {latest}. Run `unsloth studio update` " - f"to refresh." - ) + return _flow.format_stale_warning(info, component = "llama.cpp") def reset_caches(*, drop_disk: bool = False) -> None: @@ -420,13 +231,8 @@ def reset_caches(*, drop_disk: bool = False) -> None: (see its last-good fallback) and the banner could linger. Dropping the disk cache makes latest read as None in that offline case, so the banner fails open (off) instead of pointing at the just-replaced build.""" - _marker_cache.clear() - _release_memo.clear() - _assets_memo.clear() - if drop_disk: - import shutil - - # _cache_dir() is a dedicated freshness-only subdir; it is re-created on - # the next _save_disk_cache. ignore_errors so a missing/locked dir is a - # no-op rather than breaking an otherwise successful install. - shutil.rmtree(_cache_dir(), ignore_errors = True) + _flow.reset_caches( + (_marker_cache, _release_memo, _assets_memo), + drop_disk = drop_disk, + cache_dir = lambda: _cache_dir(), + ) diff --git a/studio/backend/utils/llama_cpp_update.py b/studio/backend/utils/llama_cpp_update.py index f6d3635301..174e6ef4dc 100644 --- a/studio/backend/utils/llama_cpp_update.py +++ b/studio/backend/utils/llama_cpp_update.py @@ -17,17 +17,22 @@ Design notes: thread; callers poll get_update_status() for the job state. - Everything fails open: a missing marker / offline GitHub / source build just reports update_available=False and never blocks the app. +- The mechanics (managed-root resolution, local-link detection, the resolve + probe, the streamed installer run) live in utils.prebuilt.update_flow; this + module keeps the llama policy and the job dict its callers poll. +- This is the single main update item: whisper.cpp piggybacks on it. Status + folds in a whisper sub-status (update_available becomes the union) and apply + chains a whisper phase after the llama phase when whisper is behind (see + update_flow.run_chained_update and whisper_cpp_update.chained_phase_plan). """ from __future__ import annotations -import json import os import re import subprocess import sys import threading -import time from pathlib import Path from typing import Optional @@ -43,7 +48,7 @@ from utils.llama_cpp_freshness import ( reset_caches, update_download_size_bytes, ) -from utils.process_lifetime import child_popen_kwargs +from utils.prebuilt import update_flow as _flow logger = structlog.get_logger(__name__) @@ -51,38 +56,23 @@ DEFAULT_PUBLISHED_REPO = "unslothai/llama.cpp" _INSTALL_TIMEOUT_SECONDS = 1800 # 30 min ceiling for download + build/validate # Background job state. Single in-flight update at a time, guarded by _job_lock. -_JOB_IDLE = "idle" -_JOB_RUNNING = "running" -_JOB_SUCCESS = "success" -_JOB_ERROR = "error" +_JOB_IDLE = _flow.JOB_IDLE +_JOB_RUNNING = _flow.JOB_RUNNING +_JOB_SUCCESS = _flow.JOB_SUCCESS +_JOB_ERROR = _flow.JOB_ERROR _job_lock = threading.Lock() -_job: dict = { - "state": _JOB_IDLE, - "message": "", - "from_tag": None, - "to_tag": None, - "reload_required": None, - "error": None, - "progress": None, - "started_at": None, - "finished_at": None, -} +_job: dict = _flow.new_job() -# Matches the installer's download progress lines, e.g. -# "Downloading x.zip: 35.0% (12.3 MiB/35.1 MiB) at 8.2 MiB/s". -_PROGRESS_LINE_RE = re.compile(r"(\d+(?:\.\d+)?)%\s*\(") -# The download dominates the update; extract/validate fill the last slice. -_DOWNLOAD_PROGRESS_CEILING = 0.95 - - -def _utcnow() -> str: - return time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()) +_utcnow = _flow.utcnow +_is_under = _flow.is_under +_is_external_link = _flow.is_external_link +_rocm_install_args = _flow.rocm_install_args def _find_binary() -> Optional[str]: """Locate the active llama-server binary via the inference backend's own - resolver, so update targets exactly what Studio runs. Lazy import keeps the + resolver, so update targets exactly what Unsloth runs. Lazy import keeps the heavy inference module off this module's import path.""" try: from core.inference.llama_cpp import LlamaCppBackend @@ -94,37 +84,19 @@ def _find_binary() -> Optional[str]: def _install_dir_for(binary_path: Optional[str]) -> Optional[Path]: """The directory holding UNSLOTH_PREBUILT_INFO.json -- i.e. the install root - install_llama_prebuilt.py wrote and the one we re-install into. Walks up from - the binary the same way read_install_marker() does.""" - if not binary_path: - return None - p = Path(binary_path) - for parent in p.parents[:5]: - if (parent / _INSTALL_MARKER_NAME).is_file(): - return parent - return None + install_llama_prebuilt.py wrote and the one we re-install into.""" + return _flow.install_dir_for(binary_path, marker_name = _INSTALL_MARKER_NAME) def _installer_script() -> Optional[Path]: - """Locate install_llama_prebuilt.py. Honours UNSLOTH_LLAMA_INSTALLER, then - searches up from this file for both ``/install_llama_prebuilt.py`` and - ``/studio/install_llama_prebuilt.py`` so it works in the dev tree and - in an installed Studio layout.""" - env = os.environ.get("UNSLOTH_LLAMA_INSTALLER") - if env and Path(env).is_file(): - return Path(env) - here = Path(__file__).resolve() - for up in here.parents: - for cand in (up / "install_llama_prebuilt.py", up / "studio" / "install_llama_prebuilt.py"): - if cand.is_file(): - return cand - return None + """Locate install_llama_prebuilt.py (UNSLOTH_LLAMA_INSTALLER wins).""" + return _flow.find_installer_script( + env_var = "UNSLOTH_LLAMA_INSTALLER", script_name = "install_llama_prebuilt.py" + ) # Markerless (source-build) installs have no UNSLOTH_PREBUILT_INFO.json, so we -# ask the installer whether an official prebuilt now exists for this host. Memo -# is 24h; only successful answers are cached so a network blip retries. -_RESOLVE_TTL_SECONDS = 24 * 60 * 60 +# ask the installer whether an official prebuilt now exists for this host. _resolve_memo: dict = {} @@ -132,39 +104,12 @@ def _resolve_prebuilt_for_host(*, force_refresh: bool = False) -> Optional[dict] """Run install_llama_prebuilt.py --resolve-prebuilt (no download) and return {prebuilt_available, repo, release_tag, llama_tag, asset, install_kind} or None. Fail-open: any error -> None so a source build never blocks the app.""" - now = time.time() - if not force_refresh and _resolve_memo: - if now - _resolve_memo.get("at", 0.0) < _RESOLVE_TTL_SECONDS: - return _resolve_memo.get("value") - script = _installer_script() - if script is None: - return None - value: Optional[dict] = None - try: - proc = subprocess.run( - [ - sys.executable, - str(script), - "--resolve-prebuilt", - "latest", - "--output-format", - "json", - ], - capture_output = True, - text = True, - timeout = 60, - ) - out = (proc.stdout or "").strip() - if proc.returncode == 0 and out: - parsed = json.loads(out.splitlines()[-1]) - if isinstance(parsed, dict): - value = parsed - except Exception as exc: # pragma: no cover - subprocess/json defensive - logger.debug("llama update: resolve-prebuilt failed", error = str(exc)) - value = None - if value is not None: # cache real answers; let failures retry next poll - _resolve_memo.update(at = now, value = value) - return value + return _flow.resolve_prebuilt_for_host( + force_refresh = force_refresh, + memo = _resolve_memo, + installer_script = lambda: _installer_script(), + log_message = "llama update: resolve-prebuilt failed", + ) def _installed_build_number(binary: Optional[str]) -> Optional[int]: @@ -218,38 +163,16 @@ def get_installed_llama_version() -> Optional[str]: return f"b{n}" if n is not None else None -def _is_under(path: Path, root: Path) -> bool: - try: - p, r = path.resolve(), root.resolve() - except (OSError, ValueError): - p, r = path, root - return p == r or r in p.parents - - def _llama_install_root(binary: Optional[str]) -> Optional[Path]: - """The Studio-managed llama.cpp root the active binary lives under, or None - when the binary is unmanaged. Installing anywhere the active binary is not - would not replace what _find_llama_server_binary runs (which prefers a pinned - LLAMA_SERVER_PATH, then UNSLOTH_LLAMA_CPP_PATH, then a llama.cpp tree), so we - refuse rather than silently install into an inactive or foreign tree.""" - marked = _install_dir_for(binary) - if marked is not None: - return marked - if not binary: - return None - # LLAMA_SERVER_PATH is an explicit user pin that always wins in discovery; - # never auto-replace its tree (even a user's own llama.cpp checkout). - if os.environ.get("LLAMA_SERVER_PATH"): - return None - p = Path(binary) - env = os.environ.get("UNSLOTH_LLAMA_CPP_PATH") - if env and _is_under(p, Path(env)): - return Path(env) - for parent in p.parents: - if parent.name == "llama.cpp": - return parent - # PATH / system / custom install: not a managed tree, so do not offer. - return None + """The Unsloth-managed llama.cpp root the active binary lives under, or None + when the binary is unmanaged (see update_flow.managed_install_root).""" + return _flow.managed_install_root( + binary, + marker_root = _install_dir_for(binary), + server_path_var = "LLAMA_SERVER_PATH", + cpp_path_var = "UNSLOTH_LLAMA_CPP_PATH", + dir_name = "llama.cpp", + ) def _source_build_status(binary: str, *, force_refresh: bool) -> Optional[dict]: @@ -324,69 +247,83 @@ def _source_build_status(binary: str, *, force_refresh: bool) -> Optional[dict]: } -def _is_external_link(path: Optional[Path]) -> bool: - """True when ``path`` is a --with-llama-cpp-dir local link: a POSIX symlink - or a Windows directory junction / reparse point. Such a link resolves into - the user's own llama.cpp checkout, so Studio must never auto-update it.""" - if path is None: - return False - try: - if os.path.islink(path): - return True - except OSError: - return False - if os.name == "nt": - try: - import stat - attrs = os.lstat(path).st_file_attributes # type: ignore[attr-defined] - return bool(attrs & stat.FILE_ATTRIBUTE_REPARSE_POINT) - except (OSError, AttributeError): - return False - return False - - def _active_install_is_local_link(binary: Optional[str]) -> bool: """True when the active llama-server resolves through a --with-llama-cpp-dir - local link at the canonical llama.cpp directory. An update would write - through that link into the user's own checkout (or fail), so the install is - treated as externally managed: no update is offered or applied. Checks only - up to and including the ``llama.cpp`` dir so a symlinked HOME / studio root - above it can't trip a false positive.""" - if not binary: - return False - for parent in Path(binary).parents: - if _is_external_link(parent): - return True - if parent.name == "llama.cpp": - break - return False + local link at the canonical llama.cpp directory (see + update_flow.active_install_is_local_link).""" + return _flow.active_install_is_local_link(binary, dir_name = "llama.cpp") def _local_link_status() -> dict: """Status payload for a local-link install: unmanaged, no update offered.""" - with _job_lock: - job = dict(_job) - return { - "supported": False, - "update_available": False, - "stale": False, - "installed_tag": None, - "latest_tag": None, - "published_repo": None, - "installed_at_utc": None, - "age_days": None, - "source_build": False, - "local_link": True, - "update_size_bytes": None, - "job": job, + return _flow.local_link_status(_job, _job_lock) + + +def _whisper_chain_status( + *, force_refresh: bool = False, paired_llama_will_update: bool = False +) -> Optional[dict]: + """Whisper's piggyback plan for the combined update item (see + whisper_cpp_update.chained_phase_plan). None disables the piggyback -- + fail-open so whisper can never break the llama status or apply.""" + try: + from utils import whisper_cpp_update + return whisper_cpp_update.chained_phase_plan( + force_refresh = force_refresh, + paired_llama_will_update = paired_llama_will_update, + ) + except Exception as exc: # pragma: no cover - defensive + logger.debug("llama update: whisper piggyback probe failed", error = str(exc)) + return None + + +def _merge_whisper_status(status: dict, *, force_refresh: bool = False) -> dict: + """Fold the whisper sub-status into the llama status payload: the llama + update item is the single UI surface, so update_available becomes the union + (llama behind OR whisper behind) while llama_update_available keeps the + llama-only flag. All pre-existing top-level fields are preserved.""" + status["llama_update_available"] = bool(status.get("update_available")) + plan = _whisper_chain_status( + force_refresh = force_refresh, + paired_llama_will_update = status["llama_update_available"], + ) + if plan is None: + status["whisper"] = None + status["update_component"] = "llama" if status["llama_update_available"] else None + return status + sub = plan.get("status") or {} + status["whisper"] = { + "update_available": bool(plan.get("update_available")), + "installed_tag": sub.get("installed_tag"), + "latest_tag": sub.get("latest_tag"), + "update_size_bytes": sub.get("update_size_bytes"), + "skip_reason": plan.get("skip_reason"), } + whisper_update_available = bool(plan.get("update_available")) + if whisper_update_available: + status["update_available"] = True + status["update_component"] = ( + "llama" + if status["llama_update_available"] + else "whisper" + if whisper_update_available + else None + ) + return status def get_update_status(*, force_refresh: bool = False) -> dict: - """Report whether a newer prebuilt exists plus the current job state. + """Report whether an update is available plus the current job state. - force_refresh bypasses the 24h release cache for an explicit "check now". + This is the single main update item: llama.cpp drives it and the whisper + piggyback is folded in (see _merge_whisper_status). force_refresh bypasses + the 24h release cache for an explicit "check now". """ + status = _llama_only_status(force_refresh = force_refresh) + return _merge_whisper_status(status, force_refresh = force_refresh) + + +def _llama_only_status(*, force_refresh: bool = False) -> dict: + """The llama.cpp half of get_update_status (no whisper sub-status).""" binary = _find_binary() # A --with-llama-cpp-dir local link is the user's own tree; never offer to # replace it. Bail before any network/freshness work. @@ -456,32 +393,19 @@ def get_update_status(*, force_refresh: bool = False) -> dict: } -def _rocm_install_args(asset: Optional[str]) -> list[str]: - """Forward --rocm-gfx/--has-rocm from the marker asset, mirroring setup.sh. - The installer probe can miss the gfx arch on amd-smi-only hosts; per-gfx - ROCm bundles carry the family in the name (rocm-gfx110X), version-tagged - bundles only rocm/hip.""" - if not asset: - return [] - low = asset.lower() - if "rocm" not in low and "hip" not in low: - return [] - gfx = re.search(r"-gfx[0-9a-z]+", low) - if gfx: - # _normalize_forwarded_gfx accepts the family form (gfx110x -> gfx110X). - return ["--rocm-gfx", gfx.group(0).lstrip("-")] - return ["--has-rocm"] - - -def _run_update( +def _run_llama_phase( install_dir: Path, repo: str, asset: Optional[str], script: Path, - pin_release_tag: Optional[str] = None, -) -> None: - """Worker: put the backend into a maintenance state, run the installer for - the latest prebuilt, then refresh caches so the next load uses the new build. + pin_release_tag: Optional[str], + set_progress, + force_cpu: bool = False, +) -> dict: + """The llama phase of a chained update: put the backend into a maintenance + state, run the installer for the latest prebuilt, then refresh caches so the + next load uses the new build. Returns {to_tag, reload_required, message}; + raises on failure. pin_release_tag pins the installer to that exact published release instead of letting it re-resolve "latest" itself (see start_update for why).""" @@ -522,8 +446,13 @@ def _run_update( if pin_release_tag: cmd.extend(["--published-release-tag", pin_release_tag]) cmd.extend(_rocm_install_args(asset)) + # Re-assert a deliberate CPU install (--force-cpu) so detect_host on a GPU host + # does not re-route to a GPU/Vulkan bundle and revive the crash (#7213). --force-cpu + # (not --cpu-fallback) also re-persists force_cpu, keeping the choice across future + # updates. A natural fallback (or a legacy marker without the flag) heals to GPU (#6097). + if force_cpu: + cmd.append("--force-cpu") logger.info("llama update: installing", cmd = " ".join(cmd)) - # Stream progress lines into job["progress"]. env = dict(os.environ, UNSLOTH_PROGRESS_PERCENT_STEP = "5") # Preserve a Vulkan install across updates: detect_host on a CUDA/ROCm # box would otherwise re-route and silently replace the Vulkan build. @@ -531,44 +460,12 @@ def _run_update( # _rocm_install_args). if asset and "vulkan" in asset.lower(): env["UNSLOTH_FORCE_VULKAN"] = "1" - proc = subprocess.Popen( + _flow.stream_installer( cmd, - stdout = subprocess.PIPE, - stderr = subprocess.STDOUT, - text = True, - env = env, - **child_popen_kwargs(), + env, + set_progress = set_progress, + timeout_seconds = _INSTALL_TIMEOUT_SECONDS, ) - timed_out = threading.Event() - - def _kill_on_timeout() -> None: - timed_out.set() - proc.kill() - - watchdog = threading.Timer(_INSTALL_TIMEOUT_SECONDS, _kill_on_timeout) - watchdog.daemon = True - watchdog.start() - tail_lines: list[str] = [] - try: - assert proc.stdout is not None - for line in proc.stdout: - tail_lines.append(line) - if len(tail_lines) > 80: - del tail_lines[0] - m = _PROGRESS_LINE_RE.search(line) - if m is None: - continue - fraction = min(float(m.group(1)) / 100.0, 1.0) * _DOWNLOAD_PROGRESS_CEILING - with _job_lock: - _job["progress"] = max(_job.get("progress") or 0.0, fraction) - returncode = proc.wait() - finally: - watchdog.cancel() - if timed_out.is_set(): - raise RuntimeError(f"installer timed out after {_INSTALL_TIMEOUT_SECONDS}s") - if returncode != 0: - tail = "".join(tail_lines).strip()[-1500:] - raise RuntimeError(f"installer exited {returncode}: {tail or 'no output'}") # Drop stale caches so the banner re-checks the swapped marker. # If GitHub is offline, latest stays unknown and the banner fails open. @@ -590,29 +487,18 @@ def _run_update( ): raise RuntimeError(f"pinned release {pin_release_tag} but installer produced {new_tag}") - with _job_lock: - _job.update( - state = _JOB_SUCCESS, - message = ( - f"Updated llama.cpp to {new_tag}." - + (" Reload your model to use it." if model_was_active else "") - ), - to_tag = new_tag, - reload_required = model_was_active, - error = None, - progress = 1.0, - finished_at = _utcnow(), - ) logger.info("llama update: success", to_tag = new_tag) + return { + "to_tag": new_tag, + "reload_required": model_was_active, + "message": ( + f"Updated llama.cpp to {new_tag}." + + (" Reload your model to use it." if model_was_active else "") + ), + } except Exception as exc: logger.warning("llama update: failed", error = str(exc)) - with _job_lock: - _job.update( - state = _JOB_ERROR, - message = "llama.cpp update failed.", - error = str(exc), - finished_at = _utcnow(), - ) + raise finally: # Always clear maintenance state. if backend is not None: @@ -622,55 +508,64 @@ def _run_update( pass -def start_update() -> dict: - """Kick off a background update. Idempotent: a second call while one is - running returns the in-flight job rather than starting another.""" +# Combined-job progress split when both phases run (download sizes: the llama +# bundle dwarfs the whisper one); normalized to 0..1 when a phase is skipped. +_LLAMA_PHASE_WEIGHT = 0.7 +_WHISPER_PHASE_WEIGHT = 0.3 + + +def _plan_llama_phase() -> dict: + """Decide how the llama phase of a combined update runs. Returns {"spec"} + when llama should install, else {"skip_reason", "refusal"}: skip_reason + marks the phase skipped inside a chained job, refusal is the started=False + response when the whisper phase has nothing to run either.""" binary = _find_binary() # Refuse to update a --with-llama-cpp-dir local link: installing a prebuilt # here would write through the link into the user's own checkout (or fail) # and silently drop the link the flag created. if _active_install_is_local_link(binary): return { - "started": False, - "reason": "local_link", - "message": ( - "llama.cpp is a local directory linked with --with-llama-cpp-dir; " - "Studio won't replace it. Update your own llama.cpp checkout instead." - ), - "job": get_update_status()["job"], + "skip_reason": "local_link", + "refusal": { + "started": False, + "reason": "local_link", + "message": ( + "llama.cpp is a local directory linked with --with-llama-cpp-dir; " + "Unsloth won't replace it. Update your own llama.cpp checkout instead." + ), + }, } marker = read_install_marker(binary) script = _installer_script() if script is None: return { - "started": False, - "reason": "installer_missing", - "message": "install_llama_prebuilt.py could not be located.", - "job": get_update_status()["job"], + "skip_reason": "installer_missing", + "refusal": { + "started": False, + "reason": "installer_missing", + "message": "install_llama_prebuilt.py could not be located.", + }, } - # A job already in flight wins over any freshness re-check below (and skips - # its network call). The final lock block re-checks to close the TOCTOU. - with _job_lock: - if _job["state"] == _JOB_RUNNING: - return {"started": False, "reason": "already_running", "job": dict(_job)} - if marker: # Mirror the detection guard: a direct POST or a stale banner must not # start an install when the latest is not actually newer (force a fresh # check so a stale 24h cache can't wrongly block a real update either). - status = get_update_status(force_refresh = True) + status = _llama_only_status(force_refresh = True) if not status.get("update_available"): return { - "started": False, - "reason": "up_to_date", - "message": "The installed llama.cpp build is already at the latest prebuilt.", - "job": status["job"], + "skip_reason": "up_to_date", + "refusal": { + "started": False, + "reason": "up_to_date", + "message": "The installed llama.cpp build is already at the latest prebuilt.", + }, } install_dir = _install_dir_for(binary) repo = marker.get("published_repo") or DEFAULT_PUBLISHED_REPO from_tag = marker.get("tag") or marker.get("release_tag") asset = marker.get("asset") + force_cpu = bool(marker.get("force_cpu")) # Install exactly the release the banner offered: the installer's own # "latest" is commit-date ordered and can lag the published_at pick # above, reinstalling the current build in a loop (the #6219 class). @@ -685,57 +580,151 @@ def start_update() -> dict: src = _source_build_status(binary, force_refresh = True) if binary else None if src is None: return { - "started": False, - "reason": "no_prebuilt_available", - "message": ( - "No official llama.cpp prebuilt is available for this host, " - "so the source build cannot be swapped automatically." - ), - "job": get_update_status()["job"], + "skip_reason": "no_prebuilt_available", + "refusal": { + "started": False, + "reason": "no_prebuilt_available", + "message": ( + "No official llama.cpp prebuilt is available for this host, " + "so the source build cannot be swapped automatically." + ), + }, } if not src.get("update_available"): return { - "started": False, - "reason": "up_to_date", - "message": "The installed llama.cpp build is already at or newer than the latest prebuilt.", - "job": get_update_status()["job"], + "skip_reason": "up_to_date", + "refusal": { + "started": False, + "reason": "up_to_date", + "message": ( + "The installed llama.cpp build is already at or newer than the " + "latest prebuilt." + ), + }, } res = _resolve_prebuilt_for_host() install_dir = _llama_install_root(binary) repo = (res or {}).get("repo") or DEFAULT_PUBLISHED_REPO from_tag = None asset = (res or {}).get("asset") + # Source builds carry no forced-CPU marker, so nothing to preserve here. + force_cpu = False # No pin: source-build detection resolves via --resolve-prebuilt latest, # the same resolver the unpinned apply uses, so the two already agree. pin_release_tag = None if install_dir is None: return { - "started": False, - "reason": "no_install_dir", - "message": "Could not determine the llama.cpp install directory.", - "job": get_update_status()["job"], + "skip_reason": "no_install_dir", + "refusal": { + "started": False, + "reason": "no_install_dir", + "message": "Could not determine the llama.cpp install directory.", + }, } + return { + "spec": { + "install_dir": install_dir, + "repo": repo, + "asset": asset, + "script": script, + "pin_release_tag": pin_release_tag, + "from_tag": from_tag, + "force_cpu": force_cpu, + } + } + + +def start_update() -> dict: + """Kick off a background update job. The job chains the llama phase (the + existing flow) with a whisper phase that runs only when whisper is actually + behind; either phase no-ops cleanly when its component is current or + unmanaged. Idempotent: a second call while one is running returns the + in-flight job rather than starting another.""" + # A job already in flight wins over any freshness re-check below (and skips + # its network calls). The final lock block re-checks to close the TOCTOU. + with _job_lock: + if _job["state"] == _JOB_RUNNING: + return {"started": False, "reason": "already_running", "job": dict(_job)} + + llama_plan = _plan_llama_phase() + llama_spec = llama_plan.get("spec") + whisper_plan = _whisper_chain_status( + force_refresh = True, + paired_llama_will_update = llama_spec is not None, + ) + whisper_spec = (whisper_plan or {}).get("phase") + if llama_spec is None and whisper_spec is None: + # Nothing to run in either phase: answer with the llama refusal so the + # existing reasons (local_link / up_to_date / ...) keep their meaning. + refusal = dict(llama_plan["refusal"]) + with _job_lock: + refusal["job"] = dict(_job) + return refusal + + whisper_run = None + if whisper_spec is not None: + from utils import whisper_cpp_update as _whisper + whisper_run = lambda set_progress: _whisper.run_chained_phase(whisper_spec, set_progress) + + phases = [ + { + "name": "llama", + "weight": _LLAMA_PHASE_WEIGHT, + "failure_message": "llama.cpp update failed.", + "skip_reason": llama_plan.get("skip_reason"), + "run": ( + ( + lambda set_progress: _run_llama_phase( + llama_spec["install_dir"], + llama_spec["repo"], + llama_spec["asset"], + llama_spec["script"], + llama_spec["pin_release_tag"], + set_progress, + force_cpu = llama_spec.get("force_cpu", False), + ) + ) + if llama_spec + else None + ), + }, + { + "name": "whisper", + "weight": _WHISPER_PHASE_WEIGHT, + "failure_message": "whisper.cpp update failed.", + # The sidecar reload is whisper-internal; it must not trip the + # job-level reload flag the chat frontend resyncs on. + "affects_job_reload": False, + "skip_reason": (whisper_plan or {}).get("skip_reason") or "unavailable", + "run": whisper_run, + }, + ] + running = " + ".join( + name for name, spec in (("llama.cpp", llama_spec), ("whisper.cpp", whisper_spec)) if spec + ) with _job_lock: if _job["state"] == _JOB_RUNNING: return {"started": False, "reason": "already_running", "job": dict(_job)} _job.update( state = _JOB_RUNNING, - message = "Downloading and installing the latest llama.cpp prebuilt...", - from_tag = from_tag, + message = f"Downloading and installing the latest {running} prebuilt...", + from_tag = (llama_spec or {}).get("from_tag"), to_tag = None, reload_required = None, error = None, progress = 0.0, started_at = _utcnow(), finished_at = None, + phases = None, ) job_snapshot = dict(_job) thread = threading.Thread( - target = _run_update, - args = (install_dir, repo, asset, script, pin_release_tag), + target = _flow.run_chained_update, + args = (phases,), + kwargs = {"job": _job, "job_lock": _job_lock}, name = "llama-cpp-update", daemon = True, ) @@ -745,15 +734,4 @@ def start_update() -> dict: def _reset_job_for_tests() -> None: """Test-only: return the job tracker to idle.""" - with _job_lock: - _job.update( - state = _JOB_IDLE, - message = "", - from_tag = None, - to_tag = None, - reload_required = None, - error = None, - progress = None, - started_at = None, - finished_at = None, - ) + _flow.reset_job(_job, _job_lock) diff --git a/studio/backend/utils/mlx_repair.py b/studio/backend/utils/mlx_repair.py index 7e1c9864c9..4ea1ec62f5 100644 --- a/studio/backend/utils/mlx_repair.py +++ b/studio/backend/utils/mlx_repair.py @@ -3,7 +3,7 @@ """Best-effort MLX self-heal for Apple Silicon. -On macOS, Studio enables Train/Export only when the MLX training/export stack is +On macOS, Unsloth enables Train/Export only when the MLX training/export stack is usable (see utils.hardware.hardware.detect_hardware -> CHAT_ONLY). MLX is pulled only transitively via unsloth-zoo, and a resolver backtrack (mlx-vlm -> transformers>=5 vs the single-env transformers pin) can silently drop it, leaving @@ -13,7 +13,7 @@ a background thread, then re-detects so the gate re-opens without a manual The install mirrors the main Apple Silicon installer (install_python_stack.py): it points UV_OVERRIDE at overrides-darwin-arm64.txt so the resolver keeps the -Studio transformers pin AND installs a current mlx-vlm, and it requires the same +Unsloth transformers pin AND installs a current mlx-vlm, and it requires the same minimum versions unsloth-zoo declares so a backtracked old mlx-vlm (which still imports but breaks VLM Train/Export) is never accepted as healthy. @@ -69,11 +69,11 @@ _MLX_REINSTALL_ARGS = tuple( # reject anything. mlx/mlx-metal ship wheels only (no sdist on PyPI) and # mlx-lm/mlx-vlm publish py3-none-any wheels, so requiring wheels does not break a # healthy self-heal; if a wheel is genuinely unavailable the install fails and -# Studio stays chat-only (the existing safe fallback) until `unsloth studio update`. +# Unsloth stays chat-only (the existing safe fallback) until `unsloth studio update`. _ONLY_BINARY_ARG = "--only-binary=:all:" # Allowlist of environment variables forwarded to the install subprocess. The # self-heal runs without confirmation on the default startup path, so it must not -# hand resolver/build code the full Studio environment. Everything outside this +# hand resolver/build code the full Unsloth environment. Everything outside this # set is dropped, which excludes three dangerous classes by construction: # * secrets (HF_TOKEN, AWS_*, WANDB_API_KEY, ...) that a malicious wheel/sdist # build hook would otherwise read straight out of os.environ; @@ -207,13 +207,13 @@ def _mlx_install_env() -> dict[str, str]: The self-heal runs without confirmation on the default startup path, so it forwards only the variables uv genuinely needs (see _MLX_ENV_ALLOWLIST) instead - of the full Studio environment: secrets and package-source redirects in + of the full Unsloth environment: secrets and package-source redirects in os.environ are dropped so a malicious resolver-selected artifact cannot read - Studio secrets or be steered to a hostile index. + Unsloth secrets or be steered to a hostile index. Mirror the main installer (install_python_stack.py) by pointing UV_OVERRIDE at overrides-darwin-arm64.txt, which relaxes mlx-vlm/mlx-lm's transformers>=5 - requirement to >=4.57.6. Without it, uv keeps the Studio transformers pin only + requirement to >=4.57.6. Without it, uv keeps the Unsloth transformers pin only by silently backtracking mlx-vlm to an old, unsupported version (uv honours UV_OVERRIDE; plain pip ignores it, so the transformers constraint below is the pip-path safety net). We set UV_OVERRIDE ourselves, so a poisoned one in the @@ -234,17 +234,17 @@ def _mlx_install_env() -> dict[str, str]: def _transformers_constraint_args() -> tuple[list[str], str | None]: """Pin transformers to the running version for the mlx install. - The install must never upgrade transformers underneath a running Studio + The install must never upgrade transformers underneath a running Unsloth (the single-env install pins transformers==4.57.6). With UV_OVERRIDE set this is belt-and-suspenders; on the plain-pip path (no UV_OVERRIDE support) it is the actual guard -- the resolver either finds an mlx build compatible with the - pin or fails, leaving us chat-only rather than breaking Studio. Returns + pin or fails, leaving us chat-only rather than breaking Unsloth. Returns (pip args, temp file path to clean up). Read the version from installed metadata rather than `import transformers`: transformers can have valid metadata yet fail to import (e.g. an incompatible huggingface_hub), and in that case we still want to pin it so the mlx install - cannot quietly upgrade it out from under Studio.""" + cannot quietly upgrade it out from under Unsloth.""" from importlib.metadata import PackageNotFoundError, version as _dist_version try: @@ -263,10 +263,10 @@ def attempt_mlx_repair(*, timeout: int = _REPAIR_TIMEOUT_S) -> bool: """Install a usable mlx/mlx-lm/mlx-vlm stack by name into the running venv. Best-effort; returns True iff the resulting stack meets unsloth-zoo's minimums (so a backtracked old mlx-vlm is rejected, not accepted). transformers is held - at its pinned version so the install can never upgrade it underneath Studio.""" + at its pinned version so the install can never upgrade it underneath Unsloth.""" # Prepare the constraint inside the try: this runs on a daemon thread, so an # exception here (e.g. tempfile.mkstemp failing on a full disk or bad TMPDIR) - # must leave Studio chat-only, not crash the background self-heal thread. + # must leave Unsloth chat-only, not crash the background self-heal thread. constraint_path = None try: constraint_args, constraint_path = _transformers_constraint_args() @@ -279,7 +279,7 @@ def attempt_mlx_repair(*, timeout: int = _REPAIR_TIMEOUT_S) -> bool: ) if cmd is None: logger.warning( - "MLX self-heal requires uv so Studio can apply dependency overrides; " + "MLX self-heal requires uv so Unsloth can apply dependency overrides; " "staying chat-only. Run `unsloth studio update` to restore uv." ) return False diff --git a/studio/backend/utils/models/checkpoints.py b/studio/backend/utils/models/checkpoints.py index b6b080b1c4..f2125ad034 100644 --- a/studio/backend/utils/models/checkpoints.py +++ b/studio/backend/utils/models/checkpoints.py @@ -37,7 +37,7 @@ def _checkpoint_sort_key(checkpoint_path: Path) -> tuple[int, int, str]: def _infer_base_model_from_history(checkpoint_dir: Path) -> Optional[str]: - """Best-effort base-model lookup using persisted Studio run metadata.""" + """Best-effort base-model lookup using persisted Unsloth run metadata.""" checkpoint_name = checkpoint_dir.name resolved_checkpoint_dir = str(checkpoint_dir.resolve()) diff --git a/studio/backend/utils/models/gguf_metadata.py b/studio/backend/utils/models/gguf_metadata.py index c24ec28e1d..749f2c9234 100644 --- a/studio/backend/utils/models/gguf_metadata.py +++ b/studio/backend/utils/models/gguf_metadata.py @@ -50,9 +50,15 @@ _CACHE_MAX_ENTRIES = 4096 # keyed by (file cache key, wanted key). None = key absent / file unreadable. _BOOL_CACHE: Dict[Tuple[_CacheKey, str], Optional[bool]] = {} -# Native training context length (``{arch}.context_length``). None = absent / -# unreadable. Lets the UI show the real context ceiling before a model loads. -_CONTEXT_CACHE: Dict[_CacheKey, Optional[int]] = {} +_STRING_CACHE: Dict[Tuple[_CacheKey, str], Optional[str]] = {} + +# GGUF header dims for the staged/deferred-load UI: context_length, layer_count +# (block_count), and moe_layer_count (block_count minus leading dense layers; 0 +# if not MoE). One cached pass fills all three so the staged sheet can size every +# slider before the model loads. None = unreadable / not a GGUF. The native +# training context length (``{arch}.context_length``) the UI shows before a model +# loads is read from here via read_gguf_context_length. +_DIMS_CACHE: Dict[_CacheKey, Optional[Dict[str, Optional[int]]]] = {} def _cache_key(path: str) -> Optional[_CacheKey]: @@ -142,32 +148,45 @@ def _parse_gguf_header(path: str) -> Optional[Dict[str, str]]: return out -def read_gguf_context_length(path: str) -> Optional[int]: - """Return the GGUF's native training context length (``{arch}.context_length``), - or ``None`` if missing/unreadable/not a GGUF. Cached by (path, mtime, size). - Lets the UI populate the context slider before the model is loaded.""" +def read_gguf_staged_dims(path: str) -> Optional[Dict[str, Optional[int]]]: + """GGUF header dims for the staged-load UI in one cached pass: + ``{"context_length", "layer_count", "moe_layer_count"}``. Each may be None + when absent (moe_layer_count is 0 for a dense model). Returns ``None`` if not + a GGUF / unreadable. Cached by (path, mtime, size). Lets the staged sheet size + the context, GPU-layers and MoE sliders before the model loads.""" key = _cache_key(path) if key is None: return None with _CACHE_LOCK: - if key in _CONTEXT_CACHE: - return _CONTEXT_CACHE[key] - result = _parse_gguf_context_length(path) + if key in _DIMS_CACHE: + return _DIMS_CACHE[key] + result = _parse_gguf_staged_dims(path) with _CACHE_LOCK: - while len(_CONTEXT_CACHE) >= _CACHE_MAX_ENTRIES: + while len(_DIMS_CACHE) >= _CACHE_MAX_ENTRIES: try: - _CONTEXT_CACHE.pop(next(iter(_CONTEXT_CACHE))) + _DIMS_CACHE.pop(next(iter(_DIMS_CACHE))) except StopIteration: break - _CONTEXT_CACHE[key] = result + _DIMS_CACHE[key] = result return result -def _parse_gguf_context_length(path: str) -> Optional[int]: - # The context key is architecture-namespaced (``llama.context_length`` etc.), - # so we learn the key only after reading ``general.architecture``. GGUF writes - # general.* before arch.* keys, matching the loader's own parser. - ctx_key: Optional[str] = None +def read_gguf_context_length(path: str) -> Optional[int]: + """Native training context length (``{arch}.context_length``), or ``None``. + Thin accessor over read_gguf_staged_dims.""" + dims = read_gguf_staged_dims(path) + return dims["context_length"] if dims else None + + +def _parse_gguf_arch_uints(path: str, wanted_suffixes: frozenset[str]) -> Optional[Dict[str, int]]: + """Walk a GGUF header once and return the requested architecture-namespaced + uint (vtype 4/10) keys, e.g. ``{"block_count": 32}``. Keys are + ``{arch}.``; the arch is learned from ``general.architecture`` (GGUF + writes general.* before arch.* keys, matching the loader's own parser). + Returns ``None`` if not a GGUF / unreadable, else a dict (possibly empty or + partial when some keys are absent).""" + arch: Optional[str] = None + found: Dict[str, int] = {} try: with open(path, "rb") as f: head = f.read(24) @@ -204,28 +223,68 @@ def _parse_gguf_context_length(path: str) -> Optional[int]: sbytes = f.read(slen) if len(sbytes) < slen: break - ctx_key = f"{sbytes.decode('utf-8', 'replace')}.context_length" - elif ctx_key is not None and key == ctx_key and vtype in (4, 10): + arch = sbytes.decode("utf-8", "replace") + elif ( + arch is not None + and vtype in (4, 10) + and key.startswith(f"{arch}.") + and key[len(arch) + 1 :] in wanted_suffixes + ): width = 4 if vtype == 4 else 8 n_bytes = f.read(width) if len(n_bytes) < width: break - value = struct.unpack(" 0 else None + found[key[len(arch) + 1 :]] = struct.unpack( + " Optional[Dict[str, Optional[int]]]: + vals = _parse_gguf_arch_uints( + path, + frozenset( + { + "context_length", + "block_count", + "expert_count", + "leading_dense_block_count", + } + ), + ) + if vals is None: + return None + ctx = vals.get("context_length") + block = vals.get("block_count") + # A real context/layer count is positive; treat 0/garbage as absent so the + # UI never builds a slider with max < min. + context_length = ctx if ctx and ctx > 0 else None + layer_count = block if block and block > 0 else None + # MoE layer count = block_count - leading dense layers, only when experts + # exist; else 0 (dense -> slider hidden). Mirrors n_moe_layers in + # core/inference/llama_cpp.py. + if not vals.get("expert_count") or not block: + moe_layer_count: Optional[int] = 0 + else: + moe_layer_count = max(0, block - (vals.get("leading_dense_block_count") or 0)) + return { + "context_length": context_length, + "layer_count": layer_count, + "moe_layer_count": moe_layer_count, + } # Strings (8) and arrays (9) are handled inline. @@ -353,6 +412,83 @@ def _read_gguf_bool(path: str, wanted_key: str) -> Optional[bool]: return result +def _parse_gguf_string(path: str, wanted_key: str) -> Optional[str]: + try: + with open(path, "rb") as f: + head = f.read(24) + if len(head) < 24: + return None + magic, _version, _tcount, kv_count = struct.unpack(" 1 << 20: + break + kbytes = f.read(klen) + if len(kbytes) < klen: + break + key = kbytes.decode("utf-8", "replace") + vt_bytes = f.read(4) + if len(vt_bytes) < 4: + break + vtype = struct.unpack(" 1 << 22: + break + sbytes = f.read(slen) + if len(sbytes) < slen: + break + return sbytes.decode("utf-8", "replace") + if not _skip_gguf_value(f, vtype): + break + except (struct.error, UnicodeDecodeError): + break + except OSError as e: + logger.debug(f"_parse_gguf_string: cannot open {path}: {e}") + return None + except Exception as e: + logger.debug(f"_parse_gguf_string: parse failure on {path}: {e}") + return None + return None + + +def _read_gguf_string(path: str, wanted_key: str) -> Optional[str]: + fkey = _cache_key(path) + if fkey is None: + return None + ckey = (fkey, wanted_key) + with _CACHE_LOCK: + if ckey in _STRING_CACHE: + return _STRING_CACHE[ckey] + result = _parse_gguf_string(path, wanted_key) + with _CACHE_LOCK: + while len(_STRING_CACHE) >= _CACHE_MAX_ENTRIES: + try: + _STRING_CACHE.pop(next(iter(_STRING_CACHE))) + except StopIteration: + break + _STRING_CACHE[ckey] = result + return result + + +def read_gguf_chat_template(path: str) -> Optional[str]: + template = _read_gguf_string(path, "tokenizer.chat_template") + if isinstance(template, str) and template.strip(): + return template + return None + + def read_mmproj_audio_capability(path: str) -> Optional[bool]: """``clip.has_audio_encoder`` from an mmproj GGUF (e.g. Gemma 4's gemma4ua): ``True``/``False`` if present, ``None`` if absent/unreadable. diff --git a/studio/backend/utils/models/model_config.py b/studio/backend/utils/models/model_config.py index 284bbb5745..4897f05ce4 100644 --- a/studio/backend/utils/models/model_config.py +++ b/studio/backend/utils/models/model_config.py @@ -37,6 +37,7 @@ import yaml from utils.native_path_leases import child_env_without_native_path_secret +from utils.hf_cache_settings import active_hf_hub_cache, get_hf_cache_paths from utils.subprocess_compat import ( windows_hidden_subprocess_kwargs as _windows_hidden_subprocess_kwargs, ) @@ -493,6 +494,7 @@ def load_model_config( trust_remote_code = trust_remote_code, token = token, local_files_only = local_files_only, + cache_dir = active_hf_hub_cache(), ) if not use_auth: @@ -503,6 +505,7 @@ def load_model_config( trust_remote_code = trust_remote_code, token = None, local_files_only = local_files_only, + cache_dir = active_hf_hub_cache(), ) # Default auth (cached tokens) @@ -510,6 +513,7 @@ def load_model_config( model_name, trust_remote_code = trust_remote_code, local_files_only = local_files_only, + cache_dir = active_hf_hub_cache(), ) @@ -624,6 +628,7 @@ def _raw_config_has_vision_config( filename = "config.json", token = hf_token, local_files_only = local_files_only, + cache_dir = active_hf_hub_cache(), ) ) config = json.loads(config_path.read_text()) @@ -770,7 +775,7 @@ def _is_vision_model_subprocess(model_name: str, hf_token: Optional[str] = None) capture_output = True, text = True, timeout = 60, - env = child_env_without_native_path_secret(), + env = get_hf_cache_paths().child_env(child_env_without_native_path_secret()), **_windows_hidden_subprocess_kwargs(), ) @@ -1249,6 +1254,77 @@ def _iter_gguf_files(directory: Path, recursive: bool = False): yield f +_GGUF_SPLIT_FILE_RE = re.compile( + r"^(?P.+)-(?P\d{5})-of-(?P\d{5})\.gguf$", + re.IGNORECASE, +) + + +def _colocated_first_split_shard(path: Path) -> tuple[Optional[Path], bool]: + """Return shard 1 and whether every shard is beside *path*.""" + match = _GGUF_SPLIT_FILE_RE.match(path.name) + if match is None: + return None, False + + prefix = match.group("prefix").casefold() + total_text = match.group("total") + total = int(total_text) + if total < 1: + return None, False + + first: Optional[Path] = None + indices: set[int] = set() + try: + siblings = path.parent.iterdir() + for sibling in siblings: + sibling_match = _GGUF_SPLIT_FILE_RE.match(sibling.name) + if ( + sibling_match is None + or sibling_match.group("prefix").casefold() != prefix + or sibling_match.group("total") != total_text + ): + continue + try: + if not sibling.is_file(): + continue + except OSError: + continue + index = int(sibling_match.group("index")) + if not 1 <= index <= total: + continue + indices.add(index) + if index == 1: + first = sibling + except OSError: + return None, False + + return first, first is not None and len(indices) == total + + +def _local_gguf_load_path(path: Path) -> Path: + """Choose a loadable local path while preserving complete symlink sets.""" + if _GGUF_SPLIT_FILE_RE.match(path.name) is None: + return path.absolute() + + first, complete = _colocated_first_split_shard(path) + if complete and first is not None: + return first.absolute() + + try: + is_symlink = path.is_symlink() + except OSError: + is_symlink = False + if is_symlink: + try: + target = path.resolve() + except OSError: + return (first or path).absolute() + target_first, _ = _colocated_first_split_shard(target) + return (target_first or target).absolute() + + return (first or path).absolute() + + def detect_mmproj_file(path: str, search_root: Optional[str] = None) -> Optional[str]: """Find the mmproj GGUF for a model. @@ -1434,7 +1510,7 @@ def detect_gguf_model(path: str) -> Optional[str]: except OSError: is_dir = False # stat() unavailable in the lock window if not is_dir: - return str(p.absolute()) # absolute() keeps symlink names readable + return str(_local_gguf_load_path(p)) # Directory named "*.gguf": fall through to the dir scan below. # Case 2: directory containing .gguf files (skip mmproj / MTP drafter) @@ -1452,7 +1528,7 @@ def detect_gguf_model(path: str) -> Optional[str]: gguf_files.append(f) gguf_files.sort(key = lambda f: f.stat().st_size, reverse = True) if gguf_files: - return str(gguf_files[0].resolve()) + return str(_local_gguf_load_path(gguf_files[0])) return None @@ -1643,19 +1719,20 @@ def _local_gguf_companion_search_root(selected_path: str, gguf_file: str) -> str return str(gguf_dir) -def _iter_hf_cache_snapshots(repo_id: str): +def _iter_hf_cache_snapshots(repo_id: str, cache_dir: Optional[str | Path] = None): """Yield HF cache snapshot dirs for *repo_id*, newest first. Empty if HF_HUB_CACHE is missing, the repo isn't cached, or has no snapshots. Repo name match is case-insensitive to handle casing drift between download time and lookup. """ - try: - from huggingface_hub import constants as hf_constants - except Exception: - return - - cache_dir = Path(hf_constants.HF_HUB_CACHE) + if cache_dir is None: + try: + from utils.hf_cache_settings import get_hf_cache_paths + cache_dir = get_hf_cache_paths().hub_cache + except Exception: + return + cache_dir = Path(cache_dir) target = f"models--{repo_id.replace('/', '--')}".lower() repo_dirs: list[Path] = [] try: @@ -1879,7 +1956,7 @@ def _find_local_gguf_by_variant(directory: str, variant: str) -> Optional[str]: For sharded GGUFs (multiple files sharing a quant label), returns the first shard (sorted by name), which is what ``llama-server -m`` expects. - Returns the resolved absolute path, or ``None`` if no match. + Returns the absolute path, or ``None`` if no match. """ p = _resolve_gguf_dir(Path(directory)) if p is None: @@ -1900,7 +1977,7 @@ def _find_local_gguf_by_variant(directory: str, variant: str) -> Optional[str]: matches.append(f) matches.sort() if matches: - return str(matches[0].resolve()) + return str(_local_gguf_load_path(matches[0])) return None @@ -1997,6 +2074,7 @@ def download_gguf_file( repo_id = repo_id, filename = filename, token = hf_token, + cache_dir = active_hf_hub_cache(), ) return local_path @@ -2005,6 +2083,24 @@ def download_gguf_file( _embedding_detection_cache: Dict[tuple, bool] = {} +# Bound the Hub lookup so a DNS-dead session fails fast to the cache instead of hanging on retries. +_HUB_MODEL_INFO_TIMEOUT = 15.0 + + +def _embedding_marker_in_hf_cache(model_name: str) -> bool: + """True when model_name's cached snapshot carries a modules.json (the ST marker). + Cache-only, no network; used offline and as a fallback when the Hub lookup times out.""" + from utils.utils import hf_cache_snapshot_dir + + snapshot = hf_cache_snapshot_dir(model_name) + if snapshot is None: + return False + try: + return (snapshot / "modules.json").is_file() + except OSError: + return False + + def is_embedding_model(model_name: str, hf_token: Optional[str] = None) -> bool: """Detect embedding/sentence-transformer models via HF metadata. @@ -2019,6 +2115,15 @@ def is_embedding_model(model_name: str, hf_token: Optional[str] = None) -> bool: Returns: True if embedding model, else False (default for local paths or errors). """ + from utils.utils import hf_env_offline + + # Offline (remote repo): reclassify from the local cache on every call, before/without the + # memo. An online lookup can memoize True from tags with no weights cached, so trusting it once + # the session goes offline would accept a repo _get() cannot load; a cached negative can also be + # invalidated by later cache materialization. The cache probe is local-only, so it's cheap. + if not is_local_path(model_name) and hf_env_offline(): + return _embedding_marker_in_hf_cache(model_name) + cache_key = (model_name, hf_token) if cache_key in _embedding_detection_cache: return _embedding_detection_cache[cache_key] @@ -2033,7 +2138,7 @@ def is_embedding_model(model_name: str, hf_token: Optional[str] = None) -> bool: try: from huggingface_hub import model_info as hf_model_info - info = hf_model_info(model_name, token = hf_token) + info = hf_model_info(model_name, token = hf_token, timeout = _HUB_MODEL_INFO_TIMEOUT) tags = set(info.tags or []) pipeline_tag = info.pipeline_tag or "" @@ -2054,9 +2159,11 @@ def is_embedding_model(model_name: str, hf_token: Optional[str] = None) -> bool: return is_emb except Exception as e: + # Timeout or transient network error: fall back to the local cache marker, don't hard-fail. logger.warning(f"Could not determine if {model_name} is embedding model: {e}") - _embedding_detection_cache[cache_key] = False - return False + is_emb = _embedding_marker_in_hf_cache(model_name) + _embedding_detection_cache[cache_key] = is_emb + return is_emb def _has_model_weight_files(model_dir: Path) -> bool: @@ -2083,7 +2190,7 @@ def _has_model_weight_files(model_dir: Path) -> bool: def _detect_training_output_type(model_dir: Path) -> Optional[str]: - """Classify a Studio training output as LoRA or full finetune.""" + """Classify an Unsloth training output as LoRA or full finetune.""" adapter_config = model_dir / "adapter_config.json" adapter_model = model_dir / "adapter_model.safetensors" if adapter_config.exists() or adapter_model.exists(): @@ -2105,7 +2212,7 @@ def _looks_like_lora_adapter(model_dir: Path) -> bool: def scan_trained_models(outputs_dir: str = str(outputs_root())) -> List[Tuple[str, str, str]]: - """Scan outputs folder for trained Studio models. + """Scan outputs folder for trained Unsloth models. Returns: List of (display_name, model_path, model_type), where model_type is @@ -2416,7 +2523,10 @@ def get_base_model_from_lora_identifier( for _attempt in range(2): # one retry: a transient blip must not skip the base try: cfg_path = hf_hub_download( - identifier, "adapter_config.json", token = hf_token if hf_token else None + identifier, + "adapter_config.json", + token = hf_token if hf_token else None, + cache_dir = active_hf_hub_cache(), ) except (EntryNotFoundError, RepositoryNotFoundError): # No adapter_config.json -> not a resolvable LoRA; caller scans the identifier. @@ -2796,7 +2906,12 @@ class ModelConfig: try: from huggingface_hub import hf_hub_download - config_path = hf_hub_download(identifier, "adapter_config.json", token = hf_token) + config_path = hf_hub_download( + identifier, + "adapter_config.json", + token = hf_token, + cache_dir = active_hf_hub_cache(), + ) with open(config_path, "r") as f: adapter_config = json.load(f) base_model = adapter_config.get("base_model_name_or_path") diff --git a/studio/backend/utils/native_path_leases.py b/studio/backend/utils/native_path_leases.py index 08671cfe39..3ed7faa7c2 100644 --- a/studio/backend/utils/native_path_leases.py +++ b/studio/backend/utils/native_path_leases.py @@ -15,6 +15,7 @@ import base64 import binascii import hashlib import hmac +import importlib import json import os import stat as _stat_module @@ -35,7 +36,7 @@ _USED_NONCES: dict[str, int] = {} _REDACTION_LOCK = threading.Lock() _NATIVE_PATH_REDACTIONS: list[str] = [] _NATIVE_PATH_LABELS: dict[str, str] = {} -_NATIVE_PATH_ENV_LOCK = threading.Lock() +_NATIVE_PATH_ENV_LOCK = threading.RLock() _SECRET_INIT_LOCK = threading.Lock() _CACHED_LEASE_SECRET: bytes | None = None _SCRUB_REFCOUNT = 0 @@ -80,7 +81,9 @@ def child_env_without_native_path_secret(env: Mapping[str, str] | None = None) - return cleaned -def run_without_native_path_secret(target: Callable[..., Any], *args: Any, **kwargs: Any) -> Any: +def run_without_native_path_secret( + target: Callable[..., Any] | str, *args: Any, **kwargs: Any +) -> Any: """Run a multiprocessing child target without the native path lease secret.""" # Runs in the spawned child: bind it to the parent's death (Linux), since @@ -96,6 +99,11 @@ def run_without_native_path_secret(target: Callable[..., Any], *args: Any, **kwa os.environ.pop(LEASE_SECRET_ENV, None) _CACHED_LEASE_SECRET = None _SCRUB_SAVED_SECRET = None + if isinstance(target, str): + function_name, environment, *args = args + for key, value in environment.items(): + os.environ[key] = value + target = getattr(importlib.import_module(target), function_name) return target(*args, **kwargs) @@ -107,10 +115,9 @@ def native_path_secret_removed_for_child_start() -> Iterator[None]: _SCRUB_SAVED_SECRET = os.environ.pop(LEASE_SECRET_ENV, None) _CACHED_LEASE_SECRET = None _SCRUB_REFCOUNT += 1 - try: - yield - finally: - with _NATIVE_PATH_ENV_LOCK: + try: + yield + finally: _SCRUB_REFCOUNT -= 1 if _SCRUB_REFCOUNT == 0 and _SCRUB_SAVED_SECRET is not None: os.environ[LEASE_SECRET_ENV] = _SCRUB_SAVED_SECRET diff --git a/studio/backend/utils/openai_auto_switch_settings.py b/studio/backend/utils/openai_auto_switch_settings.py index 1689395f40..7007440f4c 100644 --- a/studio/backend/utils/openai_auto_switch_settings.py +++ b/studio/backend/utils/openai_auto_switch_settings.py @@ -8,7 +8,9 @@ Two settings, both off by default so existing API behavior is unchanged: names a downloaded local GGUF different from the loaded one transparently loads it before serving (llama-swap-style). Unknown names pass through. - ``openai_api_auto_unload_idle_seconds``: when > 0, the loaded GGUF is - unloaded after this many idle seconds to free VRAM. + unloaded after this many idle seconds to free VRAM. Enabled values have a + 60s floor (0 stays "off"): a tiny TTL tears the model down between turns of + an active chat, forcing a full weight reload + prompt re-prefill per turn. The idle TTL can also be set at startup via the ``UNSLOTH_MODEL_IDLE_TTL`` env var. Unlike the stored setting (which stays gated on auto-switch), the env value @@ -28,11 +30,14 @@ from typing import Any, Optional OPENAI_AUTO_SWITCH_SETTING_KEY = "openai_api_auto_switch_model" AUTO_UNLOAD_IDLE_SETTING_KEY = "openai_api_auto_unload_idle_seconds" +AUTO_UNLOAD_KEEP_KV_SETTING_KEY = "openai_api_auto_unload_keep_kv" MODEL_OVERRIDES_SETTING_KEY = "openai_api_auto_switch_overrides" MODEL_IDLE_TTL_ENV_VAR = "UNSLOTH_MODEL_IDLE_TTL" DEFAULT_OPENAI_AUTO_SWITCH_ENABLED = False DEFAULT_AUTO_UNLOAD_IDLE_SECONDS = 0 +DEFAULT_AUTO_UNLOAD_KEEP_KV = True +MIN_AUTO_UNLOAD_IDLE_SECONDS = 60 _CACHE_TTL_S = 2.0 _cache_lock = threading.Lock() @@ -58,6 +63,10 @@ def _coerce_int(value: Any) -> int | None: return None +def _apply_idle_floor(seconds: int) -> int: + return 0 if seconds <= 0 else max(MIN_AUTO_UNLOAD_IDLE_SECONDS, seconds) + + def _cached_setting(key: str, default: Any) -> Any: """Read an app setting, memoized for _CACHE_TTL_S to spare the hot path.""" now = time.monotonic() @@ -91,12 +100,34 @@ def _stored_idle_seconds() -> Optional[int]: return _coerce_int(_cached_setting(AUTO_UNLOAD_IDLE_SETTING_KEY, None)) +_env_floor_warned = False + + def _env_idle_seconds() -> Optional[int]: - """UNSLOTH_MODEL_IDLE_TTL as a non-negative seconds value, or None if unset/invalid.""" + """UNSLOTH_MODEL_IDLE_TTL as a non-negative seconds value, or None if unset/invalid. + + Floored to MIN_AUTO_UNLOAD_IDLE_SECONDS here (with a one-time warning) since + headless/container deploys have no UI to surface a validation error.""" raw = os.environ.get(MODEL_IDLE_TTL_ENV_VAR) if raw is None or not raw.strip(): return None - return _coerce_int(raw) + parsed = _coerce_int(raw) + if parsed is None: + return None + floored = _apply_idle_floor(parsed) + if floored != parsed: + global _env_floor_warned + if not _env_floor_warned: + _env_floor_warned = True + from loggers import get_logger + get_logger(__name__).warning( + "%s=%s is below the %ss minimum; using %ss", + MODEL_IDLE_TTL_ENV_VAR, + parsed, + MIN_AUTO_UNLOAD_IDLE_SECONDS, + floored, + ) + return floored def get_stored_auto_unload_idle_seconds() -> int: @@ -108,7 +139,9 @@ def get_stored_auto_unload_idle_seconds() -> int: """ stored = _stored_idle_seconds() if stored is not None: - return stored + # Floor legacy values persisted before the minimum existed, so the UI + # displays the effective TTL and round-trips it cleanly. + return _apply_idle_floor(stored) env = _env_idle_seconds() return env if env is not None else DEFAULT_AUTO_UNLOAD_IDLE_SECONDS @@ -118,32 +151,63 @@ def get_auto_unload_idle_seconds() -> int: stored = _stored_idle_seconds() if stored is not None: # An explicit UI/API value stays gated on auto-switch: off reports 0 so the - # off state is identical to pre-feature. - return stored if get_openai_auto_switch_enabled() else 0 + # off state is identical to pre-feature. Floored to cover values persisted + # before the minimum existed. + return _apply_idle_floor(stored) if get_openai_auto_switch_enabled() else 0 # No stored value: UNSLOTH_MODEL_IDLE_TTL is a standalone startup default that # enables idle-unload even with auto-switch off (headless/container deploys). env = _env_idle_seconds() return env if env is not None else 0 -def set_openai_auto_switch(enabled: Any, idle_seconds: Any) -> tuple[bool, int]: - """Set both auto-switch flags in one transaction so a settings PUT can't leave - one key updated and the other stale. Both values are coerced before any write, - so an invalid value raises without persisting either.""" +def get_auto_unload_keep_kv() -> bool: + """Whether the idle unload persists slot KV to disk for restore on reload.""" + parsed = _coerce_bool(_cached_setting(AUTO_UNLOAD_KEEP_KV_SETTING_KEY, None)) + return parsed if parsed is not None else DEFAULT_AUTO_UNLOAD_KEEP_KV + + +def set_openai_auto_switch( + enabled: Any, + idle_seconds: Any, + keep_kv: Any = None, +) -> tuple[bool, int, bool]: + """One-transaction write; ``None`` leaves a stored value untouched.""" parsed_enabled = _coerce_bool(enabled) if parsed_enabled is None: raise ValueError("OpenAI auto-switch must be true or false.") - parsed_idle = _coerce_int(idle_seconds) - if parsed_idle is None: - raise ValueError("Auto-unload idle seconds must be a non-negative integer.") + parsed_idle = None + if idle_seconds is not None: + parsed_idle = _coerce_int(idle_seconds) + if parsed_idle is None: + raise ValueError("Auto-unload idle seconds must be a non-negative integer.") + if 0 < parsed_idle < MIN_AUTO_UNLOAD_IDLE_SECONDS: + raise ValueError( + f"Auto-unload idle seconds must be 0 (off) or at least " + f"{MIN_AUTO_UNLOAD_IDLE_SECONDS}." + ) + parsed_keep_kv = None + if keep_kv is not None: + parsed_keep_kv = _coerce_bool(keep_kv) + if parsed_keep_kv is None: + raise ValueError("Keep KV on idle unload must be true or false.") from storage.studio_db import upsert_app_settings - upsert_app_settings( - {OPENAI_AUTO_SWITCH_SETTING_KEY: parsed_enabled, AUTO_UNLOAD_IDLE_SETTING_KEY: parsed_idle} - ) + updates: dict[str, Any] = {OPENAI_AUTO_SWITCH_SETTING_KEY: parsed_enabled} + if parsed_idle is not None: + updates[AUTO_UNLOAD_IDLE_SETTING_KEY] = parsed_idle + if parsed_keep_kv is not None: + updates[AUTO_UNLOAD_KEEP_KV_SETTING_KEY] = parsed_keep_kv + upsert_app_settings(updates) _invalidate(OPENAI_AUTO_SWITCH_SETTING_KEY) - _invalidate(AUTO_UNLOAD_IDLE_SETTING_KEY) - return parsed_enabled, parsed_idle + if parsed_idle is not None: + _invalidate(AUTO_UNLOAD_IDLE_SETTING_KEY) + if parsed_keep_kv is not None: + _invalidate(AUTO_UNLOAD_KEEP_KV_SETTING_KEY) + return ( + parsed_enabled, + parsed_idle if parsed_idle is not None else get_stored_auto_unload_idle_seconds(), + parsed_keep_kv if parsed_keep_kv is not None else get_auto_unload_keep_kv(), + ) def get_model_overrides() -> dict[str, dict]: diff --git a/studio/backend/utils/paths/external_media.py b/studio/backend/utils/paths/external_media.py index 0ea0477cc7..1a0d2d2746 100644 --- a/studio/backend/utils/paths/external_media.py +++ b/studio/backend/utils/paths/external_media.py @@ -131,6 +131,29 @@ def linux_run_media_mount_roots( return roots +def macos_volume_roots(base: Path | str = "/Volumes") -> list[Path]: + """Readable mounted volumes for the macOS folder browser.""" + + if platform.system() != "Darwin": + return [] + base_path = Path(base) + try: + entries = list(base_path.iterdir()) + except OSError: + return [] + roots: list[Path] = [] + for entry in entries: + if is_sensitive_path_component(entry.name): + continue + try: + resolved = entry.resolve() + if resolved.is_dir() and os.access(resolved, os.R_OK | os.X_OK): + roots.append(resolved) + except (OSError, RuntimeError, ValueError): + continue + return roots + + def _active_windows_drive_bitmask() -> int: """Active-logical-drive bitmask from ``GetLogicalDrives`` (bit 0 = ``A:``), or ``0`` when unavailable. diff --git a/studio/backend/utils/paths/path_utils.py b/studio/backend/utils/paths/path_utils.py index e8dabc8954..65541661f1 100644 --- a/studio/backend/utils/paths/path_utils.py +++ b/studio/backend/utils/paths/path_utils.py @@ -122,15 +122,8 @@ def is_model_cached(model_name: str) -> bool: def _hf_hub_cache_dir() -> Path: """Return HF cache root honoring HF_HUB_CACHE when available.""" - try: - from huggingface_hub.constants import HF_HUB_CACHE - return Path(HF_HUB_CACHE) - except Exception as exc: - logger.debug( - "Could not read huggingface_hub HF_HUB_CACHE, using default hub path: %s", - exc, - ) - return Path.home() / ".cache" / "huggingface" / "hub" + from utils.hf_cache_settings import get_hf_cache_paths + return get_hf_cache_paths().hub_cache def resolve_cached_repo_id_case(model_name: str, use_memo: bool = True) -> str: diff --git a/studio/backend/utils/paths/storage_roots.py b/studio/backend/utils/paths/storage_roots.py index 759681da3f..cea3cc61e3 100644 --- a/studio/backend/utils/paths/storage_roots.py +++ b/studio/backend/utils/paths/storage_roots.py @@ -36,7 +36,7 @@ def _infer_studio_home_from_venv() -> Path | None: def studio_root() -> Path: - """Studio install root. + """Unsloth install root. Priority: UNSLOTH_STUDIO_HOME, then STUDIO_HOME alias, then sys.prefix inference, then legacy ~/.unsloth/studio. UNSLOTH_STUDIO_HOME wins if @@ -61,8 +61,13 @@ def cache_root() -> Path: return studio_root() / "cache" +def llama_slot_cache_root() -> Path: + """Dir llama-server saves/restores slot KV state in across idle unloads.""" + return cache_root() / "llama-slots" + + def studio_bin_root() -> Path: - """Dir for Studio-managed executables (the `unsloth` shim, downloaded tools like cloudflared).""" + """Dir for Unsloth-managed executables (the `unsloth` shim, downloaded tools like cloudflared).""" return studio_root() / "bin" @@ -272,27 +277,15 @@ def well_known_model_dirs() -> list[Path]: def _setup_cache_env() -> None: """Set cache env vars for HuggingFace, uv, and vLLM. - Respects the standard HF cache chain (explicit HF_HOME / HF_HUB_CACHE, - then XDG_CACHE_HOME, then ~/.cache/huggingface) and only sets vars the - user hasn't, so explicit overrides are honored. A user-set HF_HOME also - seeds HF_HUB_CACHE / HF_XET_CACHE (HF defaults them to $HF_HOME/hub and - $HF_HOME/xet); without this, models download to and load from the standard - cache even when HF_HOME points elsewhere, and both the Xet and HTTP-fallback - download paths inherit the same wrong root. + Explicit Hugging Face environment variables take precedence over Studio's + stored location. Studio seeds import-time variables once, while each later + worker receives its own captured cache location. """ root = cache_root() - xdg_cache = Path(os.environ.get("XDG_CACHE_HOME", Path.home() / ".cache")).expanduser() - # HUGGINGFACE_HUB_CACHE is HF's legacy alias for HF_HUB_CACHE; honor it. - if "HF_HUB_CACHE" not in os.environ and os.environ.get("HUGGINGFACE_HUB_CACHE"): - os.environ["HF_HUB_CACHE"] = os.environ["HUGGINGFACE_HUB_CACHE"] - # Seed the hub/xet caches from HF_HOME when set, else the platform default. - # Strip so a blank/whitespace HF_HOME falls back instead of making " /hub". - hf_home = (os.environ.get("HF_HOME") or "").strip() - hf_base = Path(hf_home).expanduser() if hf_home else xdg_cache / "huggingface" + from utils.hf_cache_settings import initialize_hf_cache_environment + + initialize_hf_cache_environment() defaults: dict[str, str] = { - "HF_HOME": str(hf_base), - "HF_HUB_CACHE": str(hf_base / "hub"), - "HF_XET_CACHE": str(hf_base / "xet"), "UV_CACHE_DIR": str(root / "uv"), "VLLM_CACHE_ROOT": str(root / "vllm"), } @@ -443,7 +436,7 @@ def resolve_export_write_dir(path_value: str | None = None) -> Path: Unlike :func:`resolve_export_dir`, this function passes absolute paths through as-is so users can target a different drive when - their Studio install lives on a constrained system volume + their Unsloth install lives on a constrained system volume (see :gh-issue:`6082`). Used only by the export write path. """ if not path_value or not str(path_value).strip(): diff --git a/studio/backend/utils/prebuilt/__init__.py b/studio/backend/utils/prebuilt/__init__.py new file mode 100644 index 0000000000..c41cd1150d --- /dev/null +++ b/studio/backend/utils/prebuilt/__init__.py @@ -0,0 +1,11 @@ +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 + +"""Backend-importable prebuilt helpers. + +The installers reuse install_llama_prebuilt.py directly; this package holds the +backend-side shapes the studio/ scripts cannot provide (the backend runs with +studio/backend as its sys.path root): runtime_libs (wheel CUDA dirs), child_env +(secret scrubbing + WSL ROCm dirs), freshness_flow and update_flow (the shared +mechanics behind the *_cpp_freshness / *_cpp_update twins). +""" diff --git a/studio/backend/utils/prebuilt/child_env.py b/studio/backend/utils/prebuilt/child_env.py new file mode 100644 index 0000000000..b6b7a40df7 --- /dev/null +++ b/studio/backend/utils/prebuilt/child_env.py @@ -0,0 +1,145 @@ +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 + +"""Child-process environment hygiene for the managed ggml servers. + +Secret-env scrubbing and the WSL2 ROCm library-dir probe, shared by the STT +sidecar (and any future launcher of a downloaded binary). Kept in sync with +install_llama_prebuilt.py's scrub_env / _wsl_system_rocm_lib_dirs; the backend +cannot import the studio/ installer scripts, so this copy stays importable with +only the backend root on sys.path. +""" + +from __future__ import annotations + +import os +import re +from typing import Mapping + +SECRET_ENV_EXACT = frozenset( + { + "HF_TOKEN", + "HUGGING_FACE_HUB_TOKEN", + "GH_TOKEN", + "GITHUB_TOKEN", + "WANDB_API_KEY", + "OPENAI_API_KEY", + "ANTHROPIC_API_KEY", + "AWS_ACCESS_KEY_ID", + "AWS_SECRET_ACCESS_KEY", + "AWS_SESSION_TOKEN", + "GOOGLE_APPLICATION_CREDENTIALS", + "AZURE_CLIENT_SECRET", + "KUBECONFIG", + "SSH_AUTH_SOCK", + } +) +# Case-insensitive substring markers for names we do not enumerate (no bare "KEY"). +SECRET_ENV_MARKERS = ( + "TOKEN", + "SECRET", + "PASSWORD", + "PASSWD", + "PASSPHRASE", + "CREDENTIAL", + "PRIVATE_KEY", + "API_KEY", +) +# Proxy / index URLs embed creds in their value; the offline server never needs them. +SECRET_ENV_URL_NAMES = frozenset( + { + "HTTP_PROXY", + "HTTPS_PROXY", + "ALL_PROXY", + "FTP_PROXY", + "RSYNC_PROXY", + "PIP_INDEX_URL", + "PIP_EXTRA_INDEX_URL", + "UV_INDEX_URL", + "UV_DEFAULT_INDEX", + "UV_EXTRA_INDEX_URL", + } +) +# Also drop values with URL userinfo creds (scheme://user:secret@host). +URL_USERINFO_RE = re.compile(r"://[^/@\s]+@") + + +def is_secret_env_name(name: str) -> bool: + upper = name.upper() + return ( + upper in SECRET_ENV_EXACT + or upper in SECRET_ENV_URL_NAMES + or any(marker in upper for marker in SECRET_ENV_MARKERS) + ) + + +def scrub_env(env: Mapping[str, str]) -> dict[str, str]: + """Copy of ``env`` without secret-bearing names or URL-userinfo values.""" + return { + k: v + for k, v in env.items() + if not is_secret_env_name(k) and not URL_USERINFO_RE.search(v or "") + } + + +# Filesystem pointers a downloaded binary could follow to on-disk credential +# stores (token caches under $HF_HOME, ~/.netrc, XDG config). Dropped, not +# repointed; the offline inference server needs none. Mirrors the cred-location +# list of the tools bypass env (core/inference/tools.py). +CRED_LOCATION_ENV_NAMES = frozenset( + { + "HF_HOME", + "HF_HUB_CACHE", + "HUGGINGFACE_HUB_CACHE", + "HF_XET_CACHE", + "TRANSFORMERS_CACHE", + "HF_DATASETS_CACHE", + "XDG_CONFIG_HOME", + "XDG_CACHE_HOME", + "XDG_DATA_HOME", + "NETRC", + "BASH_ENV", + "GIT_CONFIG_GLOBAL", + "GIT_CONFIG_SYSTEM", + "GIT_ASKPASS", + "SSH_ASKPASS", + "HOMEDRIVE", + "HOMEPATH", + } +) +# Home dirs are repointed (not dropped): loaders and SDKs expect them present, +# but they must not resolve to the user's real profile with its token caches. +HOME_ENV_NAMES = ("HOME", "USERPROFILE", "APPDATA", "LOCALAPPDATA") + + +def isolate_home(env: dict[str, str], scratch_dir: str) -> dict[str, str]: + """Repoint home/profile vars at ``scratch_dir`` and drop credential-store + pointers so a compromised downloaded server cannot read token caches or cred + files through the environment. Mutates and returns ``env``.""" + os.makedirs(scratch_dir, exist_ok = True) + for name in HOME_ENV_NAMES: + if name in env: + env[name] = scratch_dir + for name in CRED_LOCATION_ENV_NAMES: + env.pop(name, None) + return env + + +def wsl_system_rocm_lib_dirs() -> list[str]: + """System ROCm lib dir(s) to load before a bundle's HIP on WSL2. Strict no-op + off WSL (needs /dev/dxg, a "microsoft" /proc/version, and a librocdxg).""" + try: + if not os.path.exists("/dev/dxg"): + return [] + with open("/proc/version", encoding = "utf-8", errors = "replace") as fh: + if "microsoft" not in fh.read().lower(): + return [] + except OSError: + return [] + dirs: list[str] = [] + for d in ("/opt/rocm/lib", "/opt/rocm/lib64"): + if os.path.exists(os.path.join(d, "librocdxg.so")) or os.path.exists( + os.path.join(d, "librocdxg.so.1") + ): + dirs.append(d) + return dirs diff --git a/studio/backend/utils/prebuilt/freshness_flow.py b/studio/backend/utils/prebuilt/freshness_flow.py new file mode 100644 index 0000000000..b90ebf776c --- /dev/null +++ b/studio/backend/utils/prebuilt/freshness_flow.py @@ -0,0 +1,325 @@ +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 + +"""Shared mechanics of the llama.cpp / whisper.cpp prebuilt freshness checks. + +The component modules (utils.llama_cpp_freshness / utils.whisper_cpp_freshness) +keep their public names, per-module caches, and version-comparison policy; +everything mechanical (marker walk-up, GitHub release fetch, memo + disk cache, +the freshness report skeleton) lives here, parameterized by call-time callables +so the modules' monkeypatch seams keep working. +""" + +from __future__ import annotations + +import json +import time +from datetime import datetime, timezone +from pathlib import Path +from typing import Any, Callable, Optional + +import structlog + +logger = structlog.get_logger(__name__) + +# 24h TTL keeps the GitHub call off the hot path and within rate limits. +RELEASE_CACHE_TTL_SECONDS = 24 * 60 * 60 + + +def read_install_marker( + binary_path: Optional[str], + *, + marker_name: str, + cache: dict[str, Optional[dict]], + log_message: str, +) -> Optional[dict]: + """Walk up from binary_path to find the install marker JSON. + None = no marker (source build / custom path) or invalid JSON.""" + if not binary_path: + return None + cached = cache.get(binary_path) + if cached is not None or binary_path in cache: + return cached + p = Path(binary_path) + marker: Optional[dict] = None + # Cover all managed binary layouts (binary is 1-4 dirs deep). + for parent in p.parents[:5]: + candidate = parent / marker_name + if candidate.is_file(): + try: + marker = json.loads(candidate.read_text(encoding = "utf-8")) + except (OSError, json.JSONDecodeError) as exc: + logger.debug(log_message, path = str(candidate), error = str(exc)) + marker = None + break + cache[binary_path] = marker + return marker + + +def cache_path_for(repo: str, cache_dir: Path) -> Path: + safe = repo.replace("/", "__") + return cache_dir / f"{safe}.json" + + +def load_disk_cache(repo: str, cache_dir: Path) -> Optional[tuple[float, Optional[str]]]: + path = cache_path_for(repo, cache_dir) + try: + payload = json.loads(path.read_text(encoding = "utf-8")) + except (OSError, json.JSONDecodeError): + return None + ts = payload.get("fetched_at") + tag = payload.get("latest_tag") + if not isinstance(ts, (int, float)): + return None + return float(ts), tag if isinstance(tag, str) else None + + +def save_disk_cache( + repo: str, latest_tag: Optional[str], cache_dir: Path, *, log_message: str +) -> None: + path = cache_path_for(repo, cache_dir) + try: + path.parent.mkdir(parents = True, exist_ok = True) + tmp = path.with_suffix(".tmp") + tmp.write_text( + json.dumps({"fetched_at": time.time(), "latest_tag": latest_tag}), + encoding = "utf-8", + ) + tmp.replace(path) + except OSError as exc: + logger.debug(log_message, repo = repo, error = str(exc)) + + +def _fetch_newest_published_release( + repo: str, timeout: float, *, log_message: str +) -> Optional[dict]: + """Newest published (non-draft/non-prerelease) release object for `repo`, by + ``published_at``. + + Resolves "latest" the way the installers do, NOT via GitHub's + ``/releases/latest`` pointer, which sorts by commit date and can lag the + build the installer installs (detection and apply then disagree -- the + downgrade/sticky-banner bug). None on any failure (offline, rate-limited).""" + import os + import urllib.error + import urllib.request + + url = f"https://api.github.com/repos/{repo}/releases?per_page=30" + headers = { + "Accept": "application/vnd.github+json", + "User-Agent": "unsloth-studio-freshness-check", + } + token = os.environ.get("GITHUB_TOKEN") or os.environ.get("GH_TOKEN") + if token: + headers["Authorization"] = f"Bearer {token}" + req = urllib.request.Request(url, headers = headers) + try: + with urllib.request.urlopen(req, timeout = timeout) as resp: + data = json.loads(resp.read().decode("utf-8")) + except ( + urllib.error.URLError, + urllib.error.HTTPError, + OSError, + json.JSONDecodeError, + ) as exc: + logger.debug(log_message, repo = repo, error = str(exc)) + return None + if not isinstance(data, list): + return None + published = [ + r + for r in data + if isinstance(r, dict) + and not r.get("draft") + and not r.get("prerelease") + and isinstance(r.get("tag_name"), str) + and r.get("tag_name") + ] + if not published: + return None + return max(published, key = lambda r: r.get("published_at") or "") + + +def fetch_latest_release_tag( + repo: str, + timeout: float = 5.0, + *, + log_message: str, +) -> Optional[str]: + """Newest published release tag for `repo`, by publish time. None on failure.""" + newest = _fetch_newest_published_release(repo, timeout, log_message = log_message) + return newest["tag_name"] if newest else None + + +def fetch_latest_release_assets( + repo: str, + timeout: float = 5.0, + *, + log_message: str, +) -> Optional[dict[str, int]]: + """Asset name -> size (bytes) for the newest published release of `repo`, + selected exactly like fetch_latest_release_tag. None on any failure.""" + newest = _fetch_newest_published_release(repo, timeout, log_message = log_message) + if newest is None: + return None + assets: dict[str, int] = {} + for a in newest.get("assets") or []: + name, size = a.get("name"), a.get("size") + if isinstance(name, str) and isinstance(size, int): + assets[name] = size + return assets + + +def latest_published_release( + repo: str, + *, + force_refresh: bool, + memo: dict[str, tuple[float, Optional[str]]], + cache_dir: Callable[[], Path], + fetch: Callable[[str], Optional[str]], + save: Callable[[str, Optional[str]], None], +) -> Optional[str]: + """Latest release tag for `repo`. Memo + disk-cached (24h TTL). + None when offline and never previously cached.""" + if not repo: + return None + now = time.time() + if not force_refresh: + cached = memo.get(repo) + if cached and now - cached[0] < RELEASE_CACHE_TTL_SECONDS: + return cached[1] + disk = load_disk_cache(repo, cache_dir()) + if disk and now - disk[0] < RELEASE_CACHE_TTL_SECONDS: + memo[repo] = disk + return disk[1] + latest = fetch(repo) + if latest is None: + # Keep the last-good disk value rather than poison it with None. + disk = load_disk_cache(repo, cache_dir()) + if disk: + memo[repo] = disk + return disk[1] + return None + memo[repo] = (now, latest) + save(repo, latest) + return latest + + +def latest_release_assets( + repo: str, + *, + force_refresh: bool, + memo: dict[str, tuple[float, dict[str, int]]], + fetch: Callable[[str], Optional[dict[str, int]]], +) -> Optional[dict[str, int]]: + """Newest-release asset sizes for `repo`, memoized (24h TTL). None when + offline and never fetched. In-memory only -- a restart re-fetches.""" + if not repo: + return None + now = time.time() + if not force_refresh: + cached = memo.get(repo) + if cached and now - cached[0] < RELEASE_CACHE_TTL_SECONDS: + return cached[1] + assets = fetch(repo) + if assets is None: + cached = memo.get(repo) + return cached[1] if cached else None + memo[repo] = (now, assets) + return assets + + +def parse_installed_at(value: object) -> Optional[datetime]: + if not isinstance(value, str) or not value: + return None + s = value.replace("Z", "+00:00") if value.endswith("Z") else value + try: + dt = datetime.fromisoformat(s) + except ValueError: + return None + if dt.tzinfo is None: + dt = dt.replace(tzinfo = timezone.utc) + return dt + + +def check_freshness( + binary_path: Optional[str], + *, + threshold_days: int, + now: Optional[datetime], + read_marker: Callable[[Optional[str]], Optional[dict]], + latest_release: Callable[[str], Optional[str]], + behind: Callable[[Optional[str], Optional[str]], bool], + display_tag: Callable[[dict], Any], + compare_tag: Callable[[dict], Any], +) -> dict: + """Freshness report skeleton shared by both components; the component's + marker-tag choice and is_behind policy come in as callables. Fails open on + missing data (behind/stale stay False).""" + out: dict = { + "has_marker": False, + "stale": False, + "behind": False, + "installed_tag": None, + "latest_tag": None, + "installed_at_utc": None, + "age_days": None, + "published_repo": None, + "threshold_days": int(threshold_days), + } + marker = read_marker(binary_path) + if not marker: + return out + out["has_marker"] = True + out["installed_tag"] = display_tag(marker) + out["installed_at_utc"] = marker.get("installed_at_utc") + out["published_repo"] = marker.get("published_repo") + + installed_full = compare_tag(marker) + repo = out["published_repo"] + if not repo or not installed_full: + return out + latest = latest_release(repo) + out["latest_tag"] = latest + out["behind"] = behind(installed_full, latest) + if not out["behind"]: + return out + + installed_at = parse_installed_at(out["installed_at_utc"]) + if installed_at is None: + return out + now = now or datetime.now(tz = timezone.utc) + age_seconds = (now - installed_at).total_seconds() + out["age_days"] = max(0, int(age_seconds // 86400)) + if age_seconds >= threshold_days * 86400: + out["stale"] = True + return out + + +def format_stale_warning(info: dict, *, component: str) -> str: + """Human-readable one-liner for stale prebuilt info.""" + age = info.get("age_days") + installed = info.get("installed_tag") or "unknown" + latest = info.get("latest_tag") or "unknown" + age_str = f"{age} day{'s' if age != 1 else ''}" if age is not None else "some time" + return ( + f"{component} prebuilt is {age_str} behind: installed " + f"{installed}, latest {latest}. Run `unsloth studio update` " + f"to refresh." + ) + + +def reset_caches( + caches: tuple[dict, ...], *, drop_disk: bool, cache_dir: Callable[[], Path] +) -> None: + """Drop the in-memory freshness caches; with drop_disk also the on-disk 24h + release cache (see the component modules for why).""" + for cache in caches: + cache.clear() + if drop_disk: + import shutil + + # cache_dir() is a dedicated freshness-only subdir, re-created on the next + # save_disk_cache. ignore_errors so a missing/locked dir is a no-op rather + # than breaking an otherwise successful install. + shutil.rmtree(cache_dir(), ignore_errors = True) diff --git a/studio/backend/utils/prebuilt/runtime_libs.py b/studio/backend/utils/prebuilt/runtime_libs.py new file mode 100644 index 0000000000..6e51fb8246 --- /dev/null +++ b/studio/backend/utils/prebuilt/runtime_libs.py @@ -0,0 +1,65 @@ +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 + +"""CUDA runtime dirs shipped inside Python wheels, for the STT sidecar's child env. + +Kept in sync with install_llama_prebuilt.py's python_runtime_dirs; the backend +cannot import the studio/ installer scripts, so this small copy stays importable +with only the backend root on sys.path. +""" + +from __future__ import annotations + +import site +import sys +from pathlib import Path +from typing import Iterable + + +def dedupe_existing_dirs(paths: Iterable[str | Path]) -> list[str]: + unique: list[str] = [] + seen: set[str] = set() + for raw in paths: + if not raw: + continue + try: + path = Path(raw).expanduser() + if not path.is_dir(): + continue + resolved = str(path.resolve()) + except (OSError, ValueError): + continue + if resolved in seen: + continue + seen.add(resolved) + unique.append(resolved) + return unique + + +def python_runtime_dirs() -> list[str]: + """CUDA runtime dirs shipped inside Python wheels (torch + nvidia-* wheels).""" + candidates: list[Path] = [] + search_roots = [Path(entry) for entry in sys.path if entry] + try: + search_roots.extend(Path(path) for path in site.getsitepackages()) + except Exception: + pass + try: + user_site = site.getusersitepackages() + if user_site: + search_roots.append(Path(user_site)) + except Exception: + pass + + for root in search_roots: + if not root.is_dir(): + continue + candidates.extend(root.glob("nvidia/*/lib")) # Linux convention + candidates.extend(root.glob("nvidia/*/bin")) # legacy modular Windows wheels + candidates.extend(root.glob("nvidia/*/bin/x86_64")) # CUDA 13 Windows wheel layout + candidates.extend(root.glob("nvidia/*/bin/x64")) + candidates.extend(root.glob("nvidia/*/Library/bin")) # conda-style repacks + candidates.extend(root.glob("nvidia/*/Library/bin/x86_64")) + candidates.extend(root.glob("nvidia/*/Library/bin/x64")) + candidates.extend(root.glob("torch/lib")) + return dedupe_existing_dirs(candidates) diff --git a/studio/backend/utils/prebuilt/update_flow.py b/studio/backend/utils/prebuilt/update_flow.py new file mode 100644 index 0000000000..74af0c18f9 --- /dev/null +++ b/studio/backend/utils/prebuilt/update_flow.py @@ -0,0 +1,447 @@ +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 + +"""Shared mechanics of the llama.cpp / whisper.cpp in-app prebuilt updates. + +The component modules (utils.llama_cpp_update / utils.whisper_cpp_update) keep +their public names, job dicts, and update policy (version comparison, pinning, +pre/post install steps); everything mechanical (managed-root resolution, +local-link detection, the resolve probe, the streamed installer run) lives here, +parameterized so the modules' monkeypatch seams keep working. +""" + +from __future__ import annotations + +import json +import os +import re +import subprocess +import sys +import threading +import time +from pathlib import Path +from typing import Callable, Optional + +import structlog + +from utils.process_lifetime import child_popen_kwargs + +logger = structlog.get_logger(__name__) + +# Markerless (source-build) resolve answers are memoized for 24h; only +# successful answers are cached so a network blip retries. +RESOLVE_TTL_SECONDS = 24 * 60 * 60 + +# Matches the installer's download progress lines, e.g. +# "Downloading x.zip: 35.0% (12.3 MiB/35.1 MiB) at 8.2 MiB/s". +PROGRESS_LINE_RE = re.compile(r"(\d+(?:\.\d+)?)%\s*\(") +# The download dominates the update; extract/validate fill the last slice. +DOWNLOAD_PROGRESS_CEILING = 0.95 + + +class InstallerExit(RuntimeError): + """Installer subprocess exited nonzero; carries the exit code so phase + runners can special-case contractual codes (whisper's 2 = unavailable).""" + + def __init__(self, returncode: int, message: str) -> None: + super().__init__(message) + self.returncode = returncode + + +JOB_IDLE = "idle" +JOB_RUNNING = "running" +JOB_SUCCESS = "success" +JOB_ERROR = "error" + +# Per-phase states inside a chained job's "phases" breakdown. +PHASE_PENDING = "pending" +PHASE_RUNNING = "running" +PHASE_SUCCESS = "success" +PHASE_ERROR = "error" +PHASE_SKIPPED = "skipped" + +_IDLE_JOB_FIELDS = dict( + state = JOB_IDLE, + message = "", + from_tag = None, + to_tag = None, + reload_required = None, + error = None, + progress = None, + started_at = None, + finished_at = None, + phases = None, +) + + +def new_job() -> dict: + """A fresh idle job-state dict (one per component module).""" + return dict(_IDLE_JOB_FIELDS) + + +def reset_job(job: dict, job_lock: threading.Lock) -> None: + """Return a job tracker to idle (test seam).""" + with job_lock: + job.update(_IDLE_JOB_FIELDS) + + +def utcnow() -> str: + return time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()) + + +def is_under(path: Path, root: Path) -> bool: + try: + p, r = path.resolve(), root.resolve() + except (OSError, ValueError): + p, r = path, root + return p == r or r in p.parents + + +def install_dir_for(binary_path: Optional[str], *, marker_name: str) -> Optional[Path]: + """The directory holding the install marker: the install root the installer + wrote and the one we re-install into. Walks up from the binary like the + freshness marker reader does.""" + if not binary_path: + return None + p = Path(binary_path) + for parent in p.parents[:5]: + if (parent / marker_name).is_file(): + return parent + return None + + +def find_installer_script(*, env_var: str, script_name: str) -> Optional[Path]: + """Locate the installer script. Honours the env override, then searches up + from this file for both ``/