diff --git a/.github/scripts/run-studio-permission-browser.sh b/.github/scripts/run-studio-permission-browser.sh index 2007789035..e5a9a4c135 100755 --- a/.github/scripts/run-studio-permission-browser.sh +++ b/.github/scripts/run-studio-permission-browser.sh @@ -17,7 +17,8 @@ if [ -n "${STUDIO_PERMISSION_FRONTEND:-}" ]; then fi mkdir -p "$artifact_dir" -unsloth studio reset-password +# Wipe (not reset-password): the boot below must re-seed a fresh .bootstrap_password. +rm -rf "$studio_home/auth" UNSLOTH_API_ONLY=1 unsloth studio -H 127.0.0.1 -p "$port" "$@" \ >"$server_log" 2>&1 & studio_pid=$! diff --git a/.github/workflows/consolidated-tests-ci.yml b/.github/workflows/consolidated-tests-ci.yml index d1bea819eb..afad1b6c46 100644 --- a/.github/workflows/consolidated-tests-ci.yml +++ b/.github/workflows/consolidated-tests-ci.yml @@ -7,7 +7,7 @@ # # Why a separate workflow: # - studio-backend-ci.yml's "Repo tests (CPU)" job already auto-discovers -# tests/ minus tests/qlora, tests/saving, tests/utils, tests/sh. The 16 +# tests/ minus tests/qlora, tests/saving, tests/utils, tests/sh. The 17 # Bucket-A tests below live inside those --ignore dirs (CPU-runnable but # historically excluded with their GPU siblings); pulling them out into # a sibling job keeps the existing 760-passed baseline stable while we @@ -274,6 +274,7 @@ jobs: tests/saving/test_export_dispatch.py \ tests/saving/test_imatrix_export.py \ tests/saving/test_gguf_single_pass_export.py \ + tests/saving/test_offline_gguf_vlm_tokenizer_7481.py \ tests/utils/test_attention_masks.py \ tests/utils/test_trunc_normal_patch.py \ tests/python/test_fast_language_model_text_only.py @@ -365,17 +366,17 @@ jobs: tests/saving/test_export_dispatch.py \ tests/saving/test_imatrix_export.py \ tests/saving/test_gguf_single_pass_export.py \ + tests/saving/test_offline_gguf_vlm_tokenizer_7481.py \ tests/utils/test_attention_masks.py \ tests/utils/test_trunc_normal_patch.py \ tests/python/test_fast_language_model_text_only.py \ tests/test_bad_mappings_redirect.py \ tests/test_prefetch_snapshot_scope.py \ tests/test_gemma_2b_mapper_key.py \ - --deselect 'tests/utils/test_attention_masks.py::test_run_attention_flash_varlen_receives_window_and_softcap' - # The deselected test monkeypatches flash_attn_varlen_func, which is - # only bound on the module when `flash_attn` is importable. flash_attn - # requires CUDA + dev toolchain, which the CPU-only ubuntu-latest - # runner does not have. The other Bucket-A tests pass cleanly. + tests/test_raw_text_json_loading.py + # test_run_attention_flash_varlen_receives_window_and_softcap was deselected + # until attention_dispatch.py predefined flash_attn_varlen_func as None; it + # monkeypatches that name, so it no longer needs flash_attn on this runner. - name: unsloth_zoo @ ${{ env.UNSLOTH_ZOO_REF }} — full pytest (CPU) # 106 of 111 test_* in unsloth_zoo are CPU-only. The two CUDA-skip @@ -2129,7 +2130,7 @@ jobs: pip show unsloth_zoo echo "::endgroup::" echo "Consolidated job done. Coverage:" - echo " - 16 unsloth Bucket-A tests under tests/saving/ + tests/utils/" + echo " - 17 unsloth Bucket-A tests under tests/saving/ + tests/utils/" echo " - unsloth_zoo @ ${UNSLOTH_ZOO_REF} pytest tests/ (5 GPU cases deselected)" echo " - unsloth_zoo.compiler.test_apply_fused_lm_head" diff --git a/.github/workflows/local-agent-guides-ci.yml b/.github/workflows/local-agent-guides-ci.yml index c48328e90f..0dc0cc66d7 100644 --- a/.github/workflows/local-agent-guides-ci.yml +++ b/.github/workflows/local-agent-guides-ci.yml @@ -167,7 +167,9 @@ jobs: # ── boot the server under test (factored helper) ────────────────── - name: Serve unsloth run --disable-tools (gemma-4-E4B) run: | - unsloth studio reset-password + # Wipe, not reset-password: since #7573 the reset rotates in place and + # prints the new passphrase, which would land unmasked in the job log. + rm -rf ~/.unsloth/studio/auth bash .github/scripts/serve-unsloth-run.sh \ --gguf-file "$GITHUB_WORKSPACE/gguf-cache/${GGUF_FILE}" \ --port "$STUDIO_PORT" --log-dir logs \ @@ -371,7 +373,7 @@ jobs: - name: Serve unsloth run --disable-tools (gemma-4-E4B) run: | - unsloth studio reset-password + rm -rf ~/.unsloth/studio/auth bash .github/scripts/serve-unsloth-run.sh \ --gguf-file "$GITHUB_WORKSPACE/gguf-cache/${GGUF_FILE}" \ --port "$STUDIO_PORT" --log-dir logs \ @@ -554,7 +556,7 @@ jobs: - name: Serve unsloth run --disable-tools (gemma-4-E4B) run: | - unsloth studio reset-password + rm -rf ~/.unsloth/studio/auth bash .github/scripts/serve-unsloth-run.sh \ --gguf-file "$GITHUB_WORKSPACE/gguf-cache/${GGUF_FILE}" \ --port "$STUDIO_PORT" --log-dir logs \ @@ -718,7 +720,7 @@ jobs: - name: Serve unsloth run --disable-tools (gemma-3-270m) run: | - unsloth studio reset-password + rm -rf ~/.unsloth/studio/auth bash .github/scripts/serve-unsloth-run.sh \ --model "$GGUF_REPO" --gguf-variant "$GGUF_VARIANT" \ --port "$STUDIO_PORT" --log-dir logs \ diff --git a/.github/workflows/release-desktop.yml b/.github/workflows/release-desktop.yml index 081eda4e32..0a8d71610d 100644 --- a/.github/workflows/release-desktop.yml +++ b/.github/workflows/release-desktop.yml @@ -766,6 +766,7 @@ jobs: env: GH_REPO: ${{ github.repository }} APP_VERSION: ${{ needs.prepare-version.outputs.app_version }} + PYPI_VERSION: ${{ needs.prepare-version.outputs.pypi_version }} STUDIO_VERSION: ${{ needs.prepare-version.outputs.studio_version }} DESKTOP_RELEASE_TAG: ${{ needs.prepare-version.outputs.desktop_release_tag }} DESKTOP_PRERELEASE: ${{ needs.prepare-version.outputs.prerelease }} @@ -911,6 +912,8 @@ jobs: notes = pathlib.Path(os.environ['RUNNER_TEMP'], 'desktop-release-notes.md').read_text() metadata = { 'version': os.environ['APP_VERSION'], + # App version is SemVer; CHANGELOG.md is keyed by the backend release. + 'pypi_version': os.environ['PYPI_VERSION'], 'notes': notes, 'pub_date': datetime.datetime.now(datetime.timezone.utc).isoformat(timespec='milliseconds').replace('+00:00', 'Z'), 'platforms': { diff --git a/.github/workflows/startup-profile-ci.yml b/.github/workflows/startup-profile-ci.yml new file mode 100644 index 0000000000..fbde99836d --- /dev/null +++ b/.github/workflows/startup-profile-ci.yml @@ -0,0 +1,156 @@ +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. + +# Measures where Studio's startup time goes, on each platform. +# +# Nothing recorded a number before: main.py logs "lifespan startup completed in X ms" +# and studio_test_kit polls /healthz, but both throw the elapsed time away. A first +# local run (Linux, warm cache, 18-core server) put `import main` at 5.7-6.6s BEFORE +# the server can bind, dominated by eager module-level imports pulled in by routes: +# torch ~1.9s self, unsloth_zoo ~0.8s, routes ~0.6s, transformers ~0.5s. +# +# Not a gate yet: --max-healthz-seconds exists, but a budget should come from +# observed numbers rather than a guess. + +name: Startup profile + +on: + pull_request: + paths: + # The measured import graph is the whole backend tree: main.py imports auth, + # core, hub, loggers, models, picker, routes and utils at module scope. + - 'studio/backend/**' + - '!studio/backend/tests/**' + # The launch phase spawns `unsloth studio --api-only`, so the CLI counts too. + - 'unsloth_cli/**' + - 'studio/src-tauri/src/preflight**' + # The profiler hardcodes the desktop argv that process.rs::backend_args builds, + # so a change there must schedule a run or the two silently diverge. + - 'studio/src-tauri/src/process.rs' + - 'scripts/profile_startup.py' + - '.github/workflows/startup-profile-ci.yml' + # The job profiles whatever `install.sh --local` built: the installers pick the + # venv's Python and the dependency specs, and pyproject's include list is what + # makes --local overlay studio.backend*. + - 'install.sh' + - 'install.ps1' + - 'pyproject.toml' + # --local also runs the checkout's setup scripts (install.sh picks + # $_REPO_ROOT/studio/setup.sh, the editable install resolves setup.ps1 to the + # repo), and both call install_python_stack.py, which picks the dependencies. + - 'studio/setup.sh' + - 'studio/setup.ps1' + - 'studio/install_python_stack.py' + workflow_dispatch: + inputs: + repeats: + description: 'launch repeats per OS (median reported)' + type: string + default: '3' + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +permissions: + contents: read + +jobs: + profile: + name: startup ${{ matrix.os }} + runs-on: ${{ matrix.os }} + timeout-minutes: 60 + continue-on-error: true + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, macos-14, windows-latest] + + env: + UNSLOTH_STUDIO_HOME: ${{ github.workspace }}/.studio-home + # A wildcard bind calls ifconfig.me on the startup path; loopback times our code. + UNSLOTH_STUDIO_DISABLE_PUBLIC_CHECK: '1' + + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + + - name: Install Studio + shell: bash + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + set -o pipefail + mkdir -p logs + # --local is load-bearing: it overlays the checkout, so the profiled server + # is this diff. Without it install.sh resolves unsloth from PyPI. + if [ "${{ runner.os }}" = "Windows" ]; then + pwsh -NoProfile -File ./install.ps1 --local 2>&1 | tee logs/install.log + else + bash install.sh --local 2>&1 | tee logs/install.log + fi + + - name: Profile startup + shell: bash + run: | + BIN="$UNSLOTH_STUDIO_HOME/unsloth_studio/bin/unsloth" + [ -x "$BIN" ] || BIN="$UNSLOTH_STUDIO_HOME/unsloth_studio/Scripts/unsloth.exe" + [ -x "$BIN" ] || BIN="" + # Profile imports with the INSTALLED interpreter: that venv is what launches. + PY="$UNSLOTH_STUDIO_HOME/unsloth_studio/bin/python" + [ -x "$PY" ] || PY="$UNSLOTH_STUDIO_HOME/unsloth_studio/Scripts/python.exe" + [ -x "$PY" ] || PY="$(command -v python3 || command -v python)" + python3 scripts/profile_startup.py \ + --python "$PY" \ + ${BIN:+--bin "$BIN"} \ + --repeats "${{ inputs.repeats || '3' }}" \ + --json "startup-${{ matrix.os }}.json" 2>&1 | tee logs/profile.log + + - name: Summary + if: always() + shell: bash + run: | + f="startup-${{ matrix.os }}.json" + [ -f "$f" ] || { echo "no profile produced"; exit 0; } + python3 - "$f" >> "$GITHUB_STEP_SUMMARY" <<'PY' + import json, sys + d = json.load(open(sys.argv[1])) + print(f"### {d['platform']} / {d['machine']} (py {d['python']}, {d['cpu_count']} cpu)\n") + imp = d.get("imports", {}) + # Gate on ok: a failed `import main` still leaves rows, so a total can lie. + if imp.get("ok"): + print(f"**`import main`: {imp['total_seconds']}s**\n") + print("| package | self ms |") + print("|---|---:|") + for k, v in list(imp.get("self_by_package_ms", {}).items())[:8]: + print(f"| {k} | {v} |") + print() + else: + print("**`import main` failed - no valid import profile**\n") + print("```\n" + (imp.get("error") or "")[-1500:] + "\n```\n") + lau = d.get("launch") or {} + runs = len(lau.get("runs") or []) + failed = lau.get("failed_runs") or 0 + if lau.get("healthz_median_seconds") is not None: + # The aggregates cover only the runs that reached healthz, so flag the + # failures: bare numbers would read as a normal fast startup. + note = f" _({runs - failed} of {runs} launches; {failed} never became healthy)_" if failed else "" + print(f"**time to a healthy port: {lau['healthz_median_seconds']}s median, " + f"{lau['healthz_max_seconds']}s max**{note}\n") + elif lau.get("skipped"): + print(f"_launch phase skipped: {lau['skipped']}_\n") + elif runs: + print(f"**no launch measurement: all {runs} launches failed to become healthy**\n") + PY + + - name: Upload profile + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: startup-profile-${{ matrix.os }} + path: | + startup-*.json + logs/ + retention-days: 14 + if-no-files-found: warn diff --git a/.github/workflows/studio-api-smoke.yml b/.github/workflows/studio-api-smoke.yml index cdf1f6bf12..1cfa66fea4 100644 --- a/.github/workflows/studio-api-smoke.yml +++ b/.github/workflows/studio-api-smoke.yml @@ -113,7 +113,8 @@ jobs: - name: Reset auth + boot Unsloth (API-only) run: | - unsloth studio reset-password + # Wipe (not reset-password): the boot below must re-seed a fresh .bootstrap_password. + rm -rf ~/.unsloth/studio/auth mkdir -p logs UNSLOTH_API_ONLY=1 unsloth studio -H 127.0.0.1 -p "$STUDIO_PORT" \ > logs/studio.log 2>&1 & diff --git a/.github/workflows/studio-backend-ci.yml b/.github/workflows/studio-backend-ci.yml index ec437e0c32..dd5efbb299 100644 --- a/.github/workflows/studio-backend-ci.yml +++ b/.github/workflows/studio-backend-ci.yml @@ -223,6 +223,16 @@ jobs: tests/studio/test_is_mlx_dispatch_gate.py \ tests/studio/test_xpu_spoof_pipeline.py + - name: CLI tests (unsloth_cli) + # unsloth_cli/tests had no CI at all: `unsloth_cli/**` was only a paths + # trigger and a ruff target, so 673 tests covering the studio launcher, + # the pre-exposure gate and the auth secret writers ran nowhere, and + # four of them had been failing on main unnoticed. + # Own step, not folded into the tests/ discovery above: pyproject's + # testpaths is tests/, and this suite needs no PYTHONPATH or CUDA spoof + # (it self-bootstraps sys.path and imports neither unsloth nor torch). + run: python -m pytest unsloth_cli/tests -q --tb=short + - name: Shell installer tests # Auto-discovered rather than allowlisted. The old hardcoded list had # silently fallen seven files behind tests/run_all.sh, including diff --git a/.github/workflows/studio-frontend-ci.yml b/.github/workflows/studio-frontend-ci.yml index 3a9e373915..773e555c8b 100644 --- a/.github/workflows/studio-frontend-ci.yml +++ b/.github/workflows/studio-frontend-ci.yml @@ -133,6 +133,9 @@ jobs: - name: Typecheck run: npm run typecheck + - name: Unit tests + run: npm test + - name: Build run: npm run build diff --git a/.github/workflows/studio-inference-smoke.yml b/.github/workflows/studio-inference-smoke.yml index c2d52eac22..c37c9555bf 100644 --- a/.github/workflows/studio-inference-smoke.yml +++ b/.github/workflows/studio-inference-smoke.yml @@ -127,7 +127,8 @@ jobs: - name: Reset auth + boot Unsloth (API-only) run: | - unsloth studio reset-password + # Wipe (not reset-password): the boot below must re-seed a fresh .bootstrap_password. + rm -rf ~/.unsloth/studio/auth mkdir -p logs UNSLOTH_API_ONLY=1 unsloth studio -H 127.0.0.1 -p "$STUDIO_PORT" \ > logs/studio.log 2>&1 & @@ -400,7 +401,7 @@ jobs: # tool_policy=None so each request's `enable_tools` field is # honoured. run: | - unsloth studio reset-password + rm -rf ~/.unsloth/studio/auth mkdir -p logs UNSLOTH_API_ONLY=1 unsloth studio -H 127.0.0.1 -p "$STUDIO_PORT" \ > logs/studio.log 2>&1 & @@ -978,7 +979,7 @@ jobs: # response_format requests aren't routed through the agentic # tool loop. run: | - unsloth studio reset-password + rm -rf ~/.unsloth/studio/auth mkdir -p logs UNSLOTH_API_ONLY=1 unsloth studio -H 127.0.0.1 -p "$STUDIO_PORT" \ > logs/studio.log 2>&1 & diff --git a/.github/workflows/studio-mac-api-smoke.yml b/.github/workflows/studio-mac-api-smoke.yml index 1968885a1d..c2307f17a1 100644 --- a/.github/workflows/studio-mac-api-smoke.yml +++ b/.github/workflows/studio-mac-api-smoke.yml @@ -101,7 +101,8 @@ jobs: - name: Reset auth + boot Unsloth (API-only) run: | - unsloth studio reset-password + # Wipe (not reset-password): the boot below must re-seed a fresh .bootstrap_password. + rm -rf ~/.unsloth/studio/auth mkdir -p logs UNSLOTH_API_ONLY=1 unsloth studio -H 127.0.0.1 -p "$STUDIO_PORT" \ > logs/studio.log 2>&1 & diff --git a/.github/workflows/studio-mac-inference-smoke.yml b/.github/workflows/studio-mac-inference-smoke.yml index ce15eed5c8..1dbf86ae98 100644 --- a/.github/workflows/studio-mac-inference-smoke.yml +++ b/.github/workflows/studio-mac-inference-smoke.yml @@ -126,7 +126,8 @@ jobs: - name: Reset auth + boot Unsloth (API-only) run: | - unsloth studio reset-password + # Wipe (not reset-password): the boot below must re-seed a fresh .bootstrap_password. + rm -rf ~/.unsloth/studio/auth mkdir -p logs UNSLOTH_API_ONLY=1 unsloth studio -H 127.0.0.1 -p "$STUDIO_PORT" \ > logs/studio.log 2>&1 & @@ -386,7 +387,7 @@ jobs: # tool_policy=None so each request's `enable_tools` field is # honoured. run: | - unsloth studio reset-password + rm -rf ~/.unsloth/studio/auth mkdir -p logs UNSLOTH_API_ONLY=1 unsloth studio -H 127.0.0.1 -p "$STUDIO_PORT" \ > logs/studio.log 2>&1 & @@ -831,7 +832,7 @@ jobs: # response_format requests aren't routed through the agentic # tool loop. run: | - unsloth studio reset-password + rm -rf ~/.unsloth/studio/auth mkdir -p logs UNSLOTH_API_ONLY=1 unsloth studio -H 127.0.0.1 -p "$STUDIO_PORT" \ > logs/studio.log 2>&1 & diff --git a/.github/workflows/studio-mac-ui-smoke.yml b/.github/workflows/studio-mac-ui-smoke.yml index 7375e9bcbf..3bed2fcdff 100644 --- a/.github/workflows/studio-mac-ui-smoke.yml +++ b/.github/workflows/studio-mac-ui-smoke.yml @@ -146,7 +146,8 @@ jobs: - name: Reset auth + boot Unsloth run: | - unsloth studio reset-password + # Wipe (not reset-password): the boot below must re-seed a fresh .bootstrap_password. + rm -rf ~/.unsloth/studio/auth mkdir -p logs UNSLOTH_API_ONLY=1 unsloth studio -H 127.0.0.1 -p "$STUDIO_PORT" \ > logs/studio.log 2>&1 & @@ -190,7 +191,7 @@ jobs: # runner's kernel briefly runs out of socket buffers, and (3) a # goto 'interrupted by another navigation' when the SPA auth # guard redirects mid-navigation. The retry FULLY resets Unsloth - # (kill, reset-password, reboot, wait /api/health, re-export + # (kill, wipe auth, reboot, wait /api/health, re-export # bootstrap pw) before re-running the script. A real test failure # (assertion / timeout) does NOT match any pattern so it bypasses # retry and surfaces immediately. @@ -213,7 +214,7 @@ jobs: echo "::warning::Playwright flake on attempt ${attempt}; resetting Unsloth and retrying..." kill "${STUDIO_PID}" 2>/dev/null || true sleep 2 - unsloth studio reset-password + rm -rf ~/.unsloth/studio/auth UNSLOTH_API_ONLY=1 unsloth studio -H 127.0.0.1 -p "$STUDIO_PORT" \ > "logs/studio_retry_${attempt}.log" 2>&1 & STUDIO_PID=$! @@ -251,7 +252,7 @@ jobs: - name: Reset auth + boot Unsloth for extra UI tests (port 18897) run: | - unsloth studio reset-password + rm -rf ~/.unsloth/studio/auth mkdir -p logs UNSLOTH_API_ONLY=1 unsloth studio -H 127.0.0.1 -p 18897 \ > logs/studio_extra.log 2>&1 & @@ -308,7 +309,7 @@ jobs: echo "::warning::Playwright flake on attempt ${attempt}; resetting Unsloth and retrying..." kill "${STUDIO_EXTRA_PID}" 2>/dev/null || true sleep 2 - unsloth studio reset-password + rm -rf ~/.unsloth/studio/auth UNSLOTH_API_ONLY=1 unsloth studio -H 127.0.0.1 -p 18897 \ > "logs/studio_extra_retry_${attempt}.log" 2>&1 & STUDIO_EXTRA_PID=$! diff --git a/.github/workflows/studio-tauri-smoke.yml b/.github/workflows/studio-tauri-smoke.yml index 8e26b9fd0c..c6dad07f37 100644 --- a/.github/workflows/studio-tauri-smoke.yml +++ b/.github/workflows/studio-tauri-smoke.yml @@ -91,6 +91,16 @@ jobs: npm run build test -f dist/index.html + # The crate carries ~100 unit tests (native_file_dialogs, preflight, + # install, desktop_auth, ...) that nothing ran until now: this workflow + # only ever built. Run them here, where the toolchain and the WebKit dev + # packages are already installed, so a broken assertion fails the PR + # instead of sitting unnoticed. `--no-fail-fast` reports every failing + # test in one run rather than stopping at the first. + - name: Rust unit tests (studio/src-tauri) + working-directory: studio/src-tauri + run: cargo test --no-fail-fast + - name: Tauri debug build (Linux, no bundle, no codesign) # `--debug` + `--no-bundle` keeps this lean: compiles the Rust crate, # confirms the frontend dist is wired into Tauri, but skips the AppImage diff --git a/.github/workflows/studio-ui-smoke.yml b/.github/workflows/studio-ui-smoke.yml index 97eb07b2d8..3a0713f301 100644 --- a/.github/workflows/studio-ui-smoke.yml +++ b/.github/workflows/studio-ui-smoke.yml @@ -115,7 +115,8 @@ jobs: - name: Reset auth + boot Unsloth run: | - unsloth studio reset-password + # Wipe (not reset-password): the boot below must re-seed a fresh .bootstrap_password. + rm -rf ~/.unsloth/studio/auth mkdir -p logs UNSLOTH_API_ONLY=1 unsloth studio -H 127.0.0.1 -p "$STUDIO_PORT" \ > logs/studio.log 2>&1 & @@ -193,7 +194,7 @@ jobs: # warm install we already did) so this adds little wall time. - name: Reset auth + boot Unsloth for extra UI tests (port 18894) run: | - unsloth studio reset-password + rm -rf ~/.unsloth/studio/auth mkdir -p logs UNSLOTH_API_ONLY=1 unsloth studio -H 127.0.0.1 -p 18894 \ > logs/studio_extra.log 2>&1 & @@ -253,7 +254,7 @@ jobs: # (RAG embedder + llama.cpp probe) stay hidden from the picker. - name: Reset auth + boot Unsloth for model-config tests (port 18898) run: | - unsloth studio reset-password + rm -rf ~/.unsloth/studio/auth mkdir -p logs UNSLOTH_API_ONLY=1 unsloth studio -H 127.0.0.1 -p 18898 \ > logs/studio_modelcfg.log 2>&1 & @@ -299,7 +300,7 @@ jobs: # earlier UI tests. No GGUF -- the bug surface is the composer. - name: Reset auth + boot Unsloth for IME / i18n tests (port 18896) run: | - unsloth studio reset-password + rm -rf ~/.unsloth/studio/auth mkdir -p logs UNSLOTH_API_ONLY=1 unsloth studio -H 127.0.0.1 -p 18896 \ > logs/studio_ime.log 2>&1 & diff --git a/.github/workflows/studio-update-smoke.yml b/.github/workflows/studio-update-smoke.yml index 625c2c7811..047840e41c 100644 --- a/.github/workflows/studio-update-smoke.yml +++ b/.github/workflows/studio-update-smoke.yml @@ -146,6 +146,46 @@ jobs: kill "$PID" 2>/dev/null || true echo "post-update Unsloth /api/health OK" + - name: A complete install reports itself complete + run: | + set -o pipefail + unsloth studio verify-install + unsloth studio desktop-capabilities --json | tee /tmp/caps.json + jq -e '.studio_install_ok == true' /tmp/caps.json + jq -e '.desktop_manageability_version >= 2' /tmp/caps.json + + - name: An incomplete install must not report itself ready + # An installer killed part-way leaves a working CLI but no studio.txt + # deps, which the old preflight called ManagedReady. The manifest is + # written last, so removing it reproduces that state. + run: | + set -o pipefail + # install.sh's default root, resolved explicitly: `python` on PATH + # here is setup-python's, not the managed venv. + MANIFEST="$HOME/.unsloth/studio/unsloth_studio/unsloth_install_manifest.json" + test -f "$MANIFEST" || { echo "::error::installer never wrote $MANIFEST"; exit 1; } + rm -f "$MANIFEST" + unsloth studio desktop-capabilities --json | tee /tmp/caps_bad.json + jq -e '.studio_install_ok == false' /tmp/caps_bad.json + if unsloth studio verify-install; then + echo "::error::verify-install passed on an install with no manifest" + exit 1 + fi + echo "incomplete install correctly reported not-ready" + + - name: Update repairs an incomplete install + # `--local` bypasses setup.sh's PyPI version compare, so this asserts + # the repair OUTCOME. The non-local fast path the desktop Repair button + # uses is covered by tests/studio/install/test_setup_fast_path_guard.py. + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + set -o pipefail + unsloth studio update --local 2>&1 | tee logs/update_repair.log + unsloth studio verify-install + unsloth studio desktop-capabilities --json | jq -e '.studio_install_ok == true' + echo "update repaired the incomplete install" + - name: Uninstall and verify clean # Round-trip the installer through scripts/uninstall.sh: confirms the # uninstaller actually finds and removes everything install.sh + diff --git a/.github/workflows/studio-windows-api-smoke.yml b/.github/workflows/studio-windows-api-smoke.yml index 6dbcceebbd..b328939846 100644 --- a/.github/workflows/studio-windows-api-smoke.yml +++ b/.github/workflows/studio-windows-api-smoke.yml @@ -179,7 +179,8 @@ jobs: - name: Reset auth + boot Unsloth (API-only) run: | - unsloth studio reset-password + # Wipe (not reset-password): the boot below must re-seed a fresh .bootstrap_password. + rm -rf ~/.unsloth/studio/auth mkdir -p logs UNSLOTH_API_ONLY=1 unsloth studio -H 127.0.0.1 -p "$STUDIO_PORT" \ > logs/studio.log 2>&1 & diff --git a/.github/workflows/studio-windows-inference-smoke.yml b/.github/workflows/studio-windows-inference-smoke.yml index 3ebe442f52..d821664327 100644 --- a/.github/workflows/studio-windows-inference-smoke.yml +++ b/.github/workflows/studio-windows-inference-smoke.yml @@ -229,7 +229,8 @@ jobs: - name: Reset auth + boot Unsloth (API-only) run: | - unsloth studio reset-password + # Wipe (not reset-password): the boot below must re-seed a fresh .bootstrap_password. + rm -rf ~/.unsloth/studio/auth mkdir -p logs UNSLOTH_API_ONLY=1 unsloth studio -H 127.0.0.1 -p "$STUDIO_PORT" \ > logs/studio.log 2>&1 & @@ -573,7 +574,7 @@ jobs: - name: Reset auth + boot Unsloth (API-only, default tool policy) run: | - unsloth studio reset-password + rm -rf ~/.unsloth/studio/auth mkdir -p logs UNSLOTH_API_ONLY=1 unsloth studio -H 127.0.0.1 -p "$STUDIO_PORT" \ > logs/studio.log 2>&1 & @@ -1074,7 +1075,7 @@ jobs: - name: Reset auth + boot Unsloth (API-only) run: | - unsloth studio reset-password + rm -rf ~/.unsloth/studio/auth mkdir -p logs UNSLOTH_API_ONLY=1 unsloth studio -H 127.0.0.1 -p "$STUDIO_PORT" \ > logs/studio.log 2>&1 & @@ -1546,7 +1547,7 @@ jobs: - name: Reset auth + boot Unsloth (API-only) run: | - unsloth studio reset-password + rm -rf ~/.unsloth/studio/auth mkdir -p logs UNSLOTH_API_ONLY=1 unsloth studio -H 127.0.0.1 -p "$STUDIO_PORT" \ > logs/studio.log 2>&1 & @@ -1888,8 +1889,11 @@ jobs: # (step/substep -> Write-StudioStdoutMirror / Get-StudioAnsi). $script:StudioVtOk = $false $script:UnslothVerbose = $false + # Get-HostMachineArch is reached only on the absent path, where + # Test-VCRedistInstalled consults it before trusting the System32 DLL, so + # part A passes without it and only the clean-box part fails. foreach ($fn in @('Get-StudioAnsi', 'Write-StudioStdoutMirror', 'step', 'substep', - 'Invoke-SetupCommand', 'Refresh-Environment', + 'Invoke-SetupCommand', 'Refresh-Environment', 'Get-HostMachineArch', 'Test-VCRedistInstalled', 'Ensure-VCRedist')) { $src = Get-FunctionSource -Path $setup -Name $fn if (-not $src) { throw "Function '$fn' not found in setup.ps1" } diff --git a/.github/workflows/studio-windows-ui-smoke.yml b/.github/workflows/studio-windows-ui-smoke.yml index f401f7be44..d23cca323f 100644 --- a/.github/workflows/studio-windows-ui-smoke.yml +++ b/.github/workflows/studio-windows-ui-smoke.yml @@ -297,7 +297,8 @@ jobs: - name: Reset auth + boot Unsloth run: | - unsloth studio reset-password + # Wipe (not reset-password): the boot below must re-seed a fresh .bootstrap_password. + rm -rf ~/.unsloth/studio/auth mkdir -p logs UNSLOTH_API_ONLY=1 unsloth studio -H 127.0.0.1 -p "$STUDIO_PORT" \ > logs/studio.log 2>&1 & @@ -352,7 +353,7 @@ jobs: - name: Reset auth + boot Unsloth for extra UI tests (port 18897) run: | - unsloth studio reset-password + rm -rf ~/.unsloth/studio/auth mkdir -p logs UNSLOTH_API_ONLY=1 unsloth studio -H 127.0.0.1 -p 18897 \ > logs/studio_extra.log 2>&1 & diff --git a/.github/workflows/studio-windows-update-smoke.yml b/.github/workflows/studio-windows-update-smoke.yml index 42d74d47d2..0dcc828e6b 100644 --- a/.github/workflows/studio-windows-update-smoke.yml +++ b/.github/workflows/studio-windows-update-smoke.yml @@ -198,6 +198,31 @@ jobs: fi echo "update path took the prebuilt fast path" + - name: Update must keep the --no-torch install GGUF-only + run: | + # `unsloth studio update` exports no UNSLOTH_NO_TORCH, so setup.ps1 has + # to recover the mode from the install manifest. Without that it reads + # the missing torch as a stale venv and tries to delete the venv it is + # running out of, and the shared dependency pass pulls torch back in. + # The skip line only prints when the dependency pass actually runs, so + # don't demand it if the fast path short-circuited that pass. + if grep -q "running ordered dependency installation" logs/update.log \ + && ! grep -q "skipping direct PyTorch and Triton installation (no-torch mode)" logs/update.log; then + echo "::error::studio update left no-torch mode; it would reinstall PyTorch." + grep -iE "no-torch|stale venv|PyTorch" logs/update.log | tail -40 + exit 1 + fi + PY="$HOME/.unsloth/studio/unsloth_studio/Scripts/python.exe" + if [ ! -f "$PY" ]; then + echo "::error::studio venv interpreter missing at $PY" + exit 1 + fi + if "$PY" -c "import torch" 2>/dev/null; then + echo "::error::torch was reinstalled into the --no-torch venv." + exit 1 + fi + echo "update preserved no-torch mode" + - name: Second update must also be a no-op env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/wheel-smoke.yml b/.github/workflows/wheel-smoke.yml index cdad617027..f7a7511616 100644 --- a/.github/workflows/wheel-smoke.yml +++ b/.github/workflows/wheel-smoke.yml @@ -127,6 +127,31 @@ jobs: cd /tmp /tmp/v/bin/python -c "from studio.backend.main import app; print('Unsloth backend OK:', app.title)" + - name: CLI without the Studio stack guides instead of tracebacking + # The smoke above installs studio.txt first, so it cannot catch a wheel + # that ships studio/ without declaring what it imports (#4701, #5260, + # #7147). Drop only structlog to reuse that venv without a re-download. + run: | + set -eu + /tmp/v/bin/pip uninstall -y structlog >/dev/null + cd /tmp + status=0 + for args in "export ./nope ./out" "list-checkpoints"; do + echo "--- unsloth $args" + out=$(/tmp/v/bin/unsloth $args 2>&1 || true) + printf '%s\n' "$out" + case "$out" in + *Traceback*) + echo "FAIL: raw traceback instead of guidance"; status=1 ;; + esac + case "$out" in + *'unsloth studio update'*) ;; + *) echo "FAIL: no remediation in the message"; status=1 ;; + esac + done + /tmp/v/bin/pip install -q structlog >/dev/null + exit "$status" + - name: Upload wheel on failure if: failure() uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 diff --git a/.gitignore b/.gitignore index 39ca2226ca..fa6997cb06 100644 --- a/.gitignore +++ b/.gitignore @@ -208,6 +208,9 @@ tmp/ **/node_modules/ auth.db +# Packaging snapshot of the root CHANGELOG.md (written by build.sh) +studio/CHANGELOG.md + # Tauri local build/generated output studio/src-tauri/target/ studio/src-tauri/gen/ @@ -238,4 +241,5 @@ package-lock.json !studio/package-lock.json llama.cpp/ # Stray "~" dir some tools create from a literal ~ TMPDIR; never part of the repo. -/~/ +~/ +/temp/ diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000000..241e013cea --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,88 @@ +# Changelog + +Release notes for Unsloth and Unsloth Studio. + +Unsloth Studio reads this file to show release notes inside the "New Unsloth +version" update popup. Edit it here and the popup picks the change up on the +next update check, with no release or rebuild required. + +## Format + +Every release is a level-2 heading whose first token is the version, optionally +followed by a date: + +```md +## 2026.7.6 - 2026-07-22 +``` + +`## [2026.7.6] - 2026-07-22` and `## v2026.7.6` also work. Everything under a +heading, up to the next level-2 heading, is that release's notes and renders as +Markdown in the popup. + +Notes are matched to one exact version. When Studio offers an update to +`2026.7.6` it renders the `2026.7.6` section and nothing else. If that section +is missing, the popup links out to the online changelog rather than showing +notes from an unrelated release, so a new version needs its own section here +before its notes can appear. + +Keep the newest release at the top. Lead each bullet with the change itself: +the collapsed popup highlights the first sentence and dims the rest. +`## Unreleased` is ignored by the popup, so it is safe to stage notes there and +rename the heading at release time. + + + +## Unreleased + +## 2026.7.5 + +### What's Changed + +- AMD support is here. Train, run RL, chat with and deploy 500+ models on + Radeon, Instinct, Ryzen and data center GPUs across Windows, WSL and Linux, + up to 2x faster with 70% less VRAM and no accuracy loss. +- Intel XPU support lands in Studio, so Arc and Data Center GPUs run chat and + training alongside the NVIDIA, AMD and Apple paths. +- Local speech to text dictation runs fully offline, with slim Whisper bundles + and a picker for custom models. +- DoRA training is available in Studio, selectable next to LoRA and full + fine-tuning in the training tab. +- The update popup previews release notes inline, pulled from this file and + matched to the exact version being offered. + +### AMD, 23 July update + +Our AMD collaboration, custom Triton kernels and math algorithms bring local +training and inference to AMD hardware. The 23 July update builds on the +[AMD release](https://github.com/unslothai/unsloth/releases/tag/v0.1.501-beta): + +- RDNA2 and Gorgon Halo are supported, and the installer no longer fails to + detect GPUs on Strix Halo and other AMD cards. +- RDNA4 handling is better, and HIP and ROCm failures are caught and fixed + automatically instead of stopping the install. +- Unified memory safetensors loading is 2x faster, with much faster gradient + checkpointing on unified memory devices. +- Voice dictation through whisper.cpp has preliminary support. +- Rollback environments left by installs no longer eat 5GB of disk. They are + cleaned up automatically. + +Optimized ROCm builds cover GGUF and safetensors inference, and ROCm +compatibility is improved for MI300X and MI325X. Full guide: +[unsloth.ai/docs/basics/amd](https://unsloth.ai/docs/basics/amd). + +### Running larger models + +- Automatic GPU placement, or pick exactly which GPUs and layers to use. +- Move MoE expert layers into system memory so larger models fit. +- Split a model across several GPUs, or use tensor parallelism. +- Hardware settings are saved per model and quant. + +### Also in this release + +- Remote access with `unsloth studio --secure` over free HTTPS via Cloudflare. +- Web search reads PDF papers and manuals, and parallel tool calls, reasoning + output and tool retries are more reliable. +- The model download location is configurable, so weights can live on a second + drive instead of the default cache. +- Stalled Hugging Face XET downloads retry over standard HTTP, and existing + GGUF files are reused instead of downloaded again. diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 0000000000..7bce036343 --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1,2 @@ +include _changelog_build.py +include CHANGELOG.md diff --git a/_changelog_build.py b/_changelog_build.py new file mode 100644 index 0000000000..f5bcf2052c --- /dev/null +++ b/_changelog_build.py @@ -0,0 +1,36 @@ +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. + +"""Snapshot CHANGELOG.md into the studio package at build time. + +CHANGELOG.md at the repo root stays the one file to edit. Copying it here, +rather than in build.sh, means every packaging path ships it, so release notes +still render when the popup cannot reach GitHub.""" + +from __future__ import annotations + +import shutil +from pathlib import Path + +from setuptools.command.build_py import build_py as _build_py + +ROOT = Path(__file__).resolve().parent +SOURCE = ROOT / "CHANGELOG.md" +SNAPSHOT = ROOT / "studio" / "CHANGELOG.md" + + +class build_py(_build_py): + def run(self) -> None: + # Beside the sources only if writable (PEP 517 may build an immutable + # checkout); into the staging directory always. + if SOURCE.is_file(): + try: + shutil.copyfile(SOURCE, SNAPSHOT) + except OSError: + pass + super().run() + if not SOURCE.is_file(): + return + staged = Path(self.build_lib) / "studio" / "CHANGELOG.md" + staged.parent.mkdir(parents = True, exist_ok = True) + shutil.copyfile(SOURCE, staged) diff --git a/build.sh b/build.sh index 2a836e19d9..5b09a7791b 100644 --- a/build.sh +++ b/build.sh @@ -103,9 +103,13 @@ else STUDIO_STAMPED_VERSION="$(python scripts/stamp_studio_release.py)" fi -# 4. Build wheel/sdist +# 4. Build wheel/sdist. _changelog_build.py snapshots CHANGELOG.md into the studio +# package so release notes render offline. python -m build +# Drop the snapshot so a source checkout never serves a stale copy. +rm -f studio/CHANGELOG.md + if [ "${1:-}" = "publish" ]; then python scripts/stamp_studio_release.py --verify-dist dist --expected "$STUDIO_STAMPED_VERSION" fi diff --git a/install.ps1 b/install.ps1 index a2aff0b69a..5b205df96d 100644 --- a/install.ps1 +++ b/install.ps1 @@ -28,6 +28,14 @@ function Install-UnslothStudio { } } + function Clear-TauriInstallError { + param([string]$Message) + if ($TauriMode) { + Write-TauriLog "ERROR_CLEAR" $Message + [Console]::Error.WriteLine("[TAURI:ERROR_CLEAR] $Message") + } + } + function Format-TauriDiagBool { param([bool]$Value) if ($Value) { return "true" } @@ -49,6 +57,26 @@ function Install-UnslothStudio { } } + # Machine arch; Get-TauriDiagArch above reports the process. An emulated x64 shell on + # ARM64 reports AMD64, but PROCESSOR_ARCHITEW6432 is ARM64 in exactly that case. + function Get-HostMachineArch { + $osArch = "" + try { $osArch = [System.Runtime.InteropServices.RuntimeInformation]::OSArchitecture.ToString() } catch { $osArch = "" } + $signals = @([string]$env:PROCESSOR_ARCHITEW6432, [string]$env:PROCESSOR_ARCHITECTURE, $osArch) + foreach ($s in $signals) { + if ($s.ToLowerInvariant() -eq "arm64") { return "arm64" } + } + foreach ($s in $signals) { + if ([string]::IsNullOrWhiteSpace($s)) { continue } + switch ($s.ToLowerInvariant()) { + "amd64" { return "x86_64" } + "x64" { return "x86_64" } + "x86" { return "x86" } + } + } + return "unknown" + } + function Get-TauriTorchIndexFamily { param([string]$TorchIndexUrl) if ($SkipTorch) { return "none" } @@ -86,7 +114,7 @@ function Install-UnslothStudio { [int]$Code = 1 ) if ($Code -eq 0) { $Code = 1 } - Write-TauriLog "ERROR" $Message + Write-TauriLog "ERROR_DEFAULT" $Message if (Get-Command Restore-StudioVenvRollback -CommandType Function -ErrorAction SilentlyContinue) { Restore-StudioVenvRollback } @@ -485,7 +513,8 @@ function Install-UnslothStudio { # Full command output is shown only when --verbose / UNSLOTH_VERBOSE=1. function Invoke-InstallCommand { param( - [Parameter(Mandatory = $true)][ScriptBlock]$Command + [Parameter(Mandatory = $true)][ScriptBlock]$Command, + [string]$Label = "install command" ) # Installer-pinned index installs (torch) must beat an inherited uv mirror (#6898): # for --default-index, clear the uv index env vars (restore in finally) and set @@ -504,6 +533,7 @@ function Install-UnslothStudio { try { # Reset to avoid stale values from prior native commands. $global:LASTEXITCODE = 0 + Write-TauriLog "OUTPUT_CLEAR" $Label if ($script:UnslothVerbose) { # Merge stderr into stdout so progress/warning output stays visible # without flipping $? on successful native commands (PS 5.1 treats @@ -518,7 +548,13 @@ function Install-UnslothStudio { Write-Host (Redact-InstallOutput $output) -ForegroundColor Red } } - return [int]$LASTEXITCODE + $exitCode = [int]$LASTEXITCODE + if ($exitCode -eq 0) { + Clear-TauriInstallError "$Label recovered" + } else { + Write-TauriLog "ERROR_OUTPUT" "$Label failed (exit code $exitCode)" + } + return $exitCode } finally { $ErrorActionPreference = $prevEap if ($savedUvIndex) { @@ -549,7 +585,7 @@ function Install-UnslothStudio { } $attempt = 1 while ($true) { - $code = Invoke-InstallCommand $Command + $code = Invoke-InstallCommand -Command $Command -Label $Label if ($code -eq 0) { return 0 } if ($attempt -ge $maxAttempts) { return $code } substep ("retrying ""$Label"" after transient failure (attempt $($attempt + 1)/$maxAttempts, waiting ${delay}s)...") "Yellow" @@ -1108,10 +1144,27 @@ exit 0 return $false } + # The interpreter's own arch, asked of it: win-amd64|win-arm64|win32|"". + function Get-PythonPlatformTag { + param([string]$Exe) + try { + return (& $Exe -c "import sysconfig; print(sysconfig.get_platform())" 2>$null | Out-String).Trim().ToLowerInvariant() + } catch { return "" } + } + # Returns @{ Version = "3.13"; Path = "C:\...\python.exe" } or $null. # The resolved Path is passed to `uv venv --python` to prevent uv from # re-resolving the version string back to a conda interpreter. function Find-CompatiblePython { + # -X64Only: best installed x64 interpreter or $null, never ARM64. Last resort for + # Install-X64Python, where x64 of a lower-priority minor beats ARM64. + param([switch]$X64Only) + # Windows on ARM: prefer x64. pyarrow (via datasets) and hf-transfer ship no + # win_arm64 wheel, so a native ARM64 Python source-builds both and dies on CMake / + # Rust minutes in; x64 runs fine emulated. ARM64 is still returned when it is all + # there is, and the caller then bootstraps x64 or warns. + $preferX64 = $X64Only -or ((Get-HostMachineArch) -eq "arm64") + $candidates = @() # Try the Python Launcher first (most reliable on Windows) # py.exe resolves to the standard CPython install, not conda. # Prefer the requested $PythonVersion, then newest-first fallback. @@ -1129,7 +1182,8 @@ exit 0 # Resolve the actual executable path and verify it is not conda-based $resolvedExe = (& $pyLauncher.Source "-$minor" -c "import sys; print(sys.executable)" 2>$null | Out-String).Trim() if ($resolvedExe -and (Test-Path $resolvedExe) -and -not (Test-IsCondaPython $resolvedExe)) { - return @{ Version = $ver; Path = $resolvedExe } + if (-not $preferX64) { return @{ Version = $ver; Path = $resolvedExe; Arch = "" } } + $candidates += @{ Version = $ver; Path = $resolvedExe } } } } catch {} @@ -1150,11 +1204,53 @@ exit 0 try { $out = & $cmd.Source --version 2>&1 | Out-String if ($out -match "Python (3\.1[1-3])\.\d+") { - return @{ Version = $Matches[1]; Path = $cmd.Source } + if (-not $preferX64) { return @{ Version = $Matches[1]; Path = $cmd.Source; Arch = "" } } + $candidates += @{ Version = $Matches[1]; Path = $cmd.Source } } } catch {} } } + # `py -3.12` runs the launcher's preferred build, normally the native ARM64 one, so + # a same-minor x64 install that is neither preferred nor on PATH never becomes a + # candidate. `-3.12-64` cannot disambiguate (deprecated, it only means "not + # 32-bit"), so enumerate every registration with -0p and probe each path. + if ($preferX64) { + foreach ($pyLauncher in @(Get-Command py -All -CommandType Application -ErrorAction SilentlyContinue)) { + if ($pyLauncher.Source -match $script:CondaSkipPattern) { continue } + $listed = @() + try { $listed = @(& $pyLauncher.Source "-0p" 2>$null) } catch {} + foreach ($line in $listed) { + # " -V:3.12 * C:\...\python.exe": tag, optional default marker, path. + $m = [regex]::Match([string]$line, '(?i)^\s*-\S+\s+\*?\s*"?(?

\S.*?\.exe)"?\s*$') + if (-not $m.Success) { continue } + $exe = $m.Groups['p'].Value.Trim() + if ($candidates | Where-Object { $_.Path -eq $exe }) { continue } + if (-not (Test-Path -LiteralPath $exe)) { continue } + if (Test-IsCondaPython $exe) { continue } + try { + $out = & $exe --version 2>&1 | Out-String + if ($out -match "Python (3\.1[1-3])\.\d+") { + $candidates += @{ Version = $Matches[1]; Path = $exe } + } + } catch {} + } + } + } + # Prefer x64, but only within one minor: $minors is the caller's version preference, + # so ranking on arch alone would answer UNSLOTH_PYTHON=3.12 with an x64 3.13 and + # never bootstrap x64 3.12. Probing costs a subprocess, so non-ARM returned above. + foreach ($c in $candidates) { + $tag = Get-PythonPlatformTag $c.Path + $c.Arch = if ($tag -eq "win-amd64") { "x86_64" } elseif ($tag -eq "win-arm64") { "arm64" } else { "unknown" } + } + foreach ($minor in $minors) { + $sameMinor = @($candidates | Where-Object { $_.Version -eq $minor }) + if ($sameMinor.Count -eq 0) { continue } + $x64 = $sameMinor | Where-Object { $_.Arch -eq "x86_64" } | Select-Object -First 1 + if ($x64) { return $x64 } + if (-not $X64Only) { return $sameMinor[0] } + } + if (-not $X64Only -and $candidates.Count -gt 0) { return $candidates[0] } return $null } @@ -1165,8 +1261,11 @@ exit 0 # (no UAC), putting python.exe + the py launcher on PATH. Mirrors the uv -> # astral.sh fallback below. Returns @{ Version; Path } or $null. function Install-PythonFromPythonOrg { + # $Arch overrides the host arch, to pull x64 onto an ARM64 box. + param([string]$Arch = "") # python.org ships one installer per architecture. - $archSuffix = switch (Get-TauriDiagArch) { + $targetArch = if ($Arch) { $Arch } else { Get-TauriDiagArch } + $archSuffix = switch ($targetArch) { "x86_64" { "-amd64" } "arm64" { "-arm64" } "x86" { "" } @@ -1231,6 +1330,28 @@ exit 0 return (Find-CompatiblePython) } + # ── Windows on ARM: get an x64 CPython ── + # --architecture x64 forces winget off the ARM64 build; python.org takes the same override. + function Install-X64Python { + if ($script:WingetAvailable) { + $prevEAP = $ErrorActionPreference + $ErrorActionPreference = "Continue" + try { + winget install -e --id "Python.Python.$PythonVersion" --source winget --architecture x64 --accept-package-agreements --accept-source-agreements + } catch { } + $ErrorActionPreference = $prevEAP + Refresh-SessionPath + $found = Find-CompatiblePython + if ($found -and $found.Arch -eq "x86_64") { return $found } + substep "winget could not provide an x64 Python -- trying python.org..." "Yellow" + } + $found = Install-PythonFromPythonOrg -Arch "x86_64" + if ($found -and $found.Arch -eq "x86_64") { return $found } + # Nothing installable (offline / no winget): an x64 build of another supported minor + # still runs the wheels ARM64 cannot, so take it over the native interpreter. + return (Find-CompatiblePython -X64Only) + } + # ── Install Python if no compatible version (3.11-3.13) found ── # Find-CompatiblePython returns @{ Version = "3.13"; Path = "C:\...\python.exe" } or $null. Write-TauriLog "STEP" "Installing Python" @@ -1302,6 +1423,26 @@ exit 0 return (Exit-InstallFailure "Python installation failed") } } + # ── Windows on ARM: swap a native ARM64 interpreter for x64 ── + # pyarrow and hf-transfer publish no win_arm64 wheel, so an ARM64 Python source-builds + # both and fails deep into the run. Warn up front if x64 is unobtainable. + if ($DetectedPython -and (Get-HostMachineArch) -eq "arm64" -and $DetectedPython.Arch -ne "x86_64") { + substep "windows on arm: only a native ARM64 Python $($DetectedPython.Version) was found." "Yellow" + substep "pyarrow and hf-transfer publish no win_arm64 wheels, so installing x64 Python..." "Yellow" + $X64Python = Install-X64Python + if ($X64Python) { + $DetectedPython = $X64Python + step "python" "using x64 Python $($DetectedPython.Version) under emulation" + } else { + Write-Host "[WARN] Could not install an x64 Python on this ARM64 machine." -ForegroundColor Yellow + Write-Host " Continuing with ARM64 Python $($DetectedPython.Version), but the install is likely to fail:" -ForegroundColor Yellow + Write-Host " pyarrow (via datasets) and hf-transfer ship no win_arm64 wheels and will be" -ForegroundColor Yellow + Write-Host " built from source, which needs CMake plus the MSVC and Rust toolchains." -ForegroundColor Yellow + Write-Host " Fix: install x64 Python from https://www.python.org/downloads/windows/" -ForegroundColor Yellow + Write-Host " (choose 'Windows installer (64-bit)', not ARM64), then re-run this installer." -ForegroundColor Yellow + } + } + $DiagPythonVersion = $PythonVersion if ($DetectedPython) { $DiagPythonVersion = $DetectedPython.Version } $InitialGpuBranch = "unknown" @@ -1603,7 +1744,7 @@ exit 0 if (-not (Test-Path -LiteralPath $VenvPython)) { step "venv" "creating Python $($DetectedPython.Version) virtual environment" substep "$VenvDir" - $venvExit = Invoke-InstallCommand { uv venv $VenvDir --python "$($DetectedPython.Path)" } + $venvExit = Invoke-InstallCommand -Label "create virtual environment" { uv venv $VenvDir --python "$($DetectedPython.Path)" } if ($venvExit -ne 0) { Write-Host "[ERROR] Failed to create virtual environment (exit code $venvExit)" -ForegroundColor Red return (Exit-InstallFailure "Failed to create virtual environment (exit code $venvExit)" $venvExit) @@ -2375,7 +2516,7 @@ exit 0 } if ($StudioLocalInstall) { substep "overlaying local repo (editable)..." - $overlayExit = Invoke-InstallCommand { uv pip install --python $VenvPython -e $RepoRoot --no-deps } + $overlayExit = Invoke-InstallCommand -Label "overlay local repo" { uv pip install --python $VenvPython -e $RepoRoot --no-deps } if ($overlayExit -ne 0) { Write-Host "[ERROR] Failed to overlay local repo (exit code $overlayExit)" -ForegroundColor Red return (Exit-InstallFailure "Failed to overlay local repo (exit code $overlayExit)" $overlayExit) @@ -2422,6 +2563,13 @@ exit 0 } } else { Write-TauriLog "STEP" "Installing PyTorch" + # Windows on ARM lacks only torchaudio (whl/cpu win_arm64: torch 42, + # torchvision 60, torchaudio 0), so drop that pin instead of aborting. Ask the + # interpreter, not PROCESSOR_ARCHITECTURE; reached when no x64 Python exists. + $VenvPlatform = "" + try { + $VenvPlatform = (& $VenvPython -c "import sysconfig; print(sysconfig.get_platform())" 2>$null | Out-String).Trim().ToLowerInvariant() + } catch { $VenvPlatform = "" } substep "installing PyTorch ($(Remove-IndexUrlCredentials $TorchIndexUrl))..." # Bound the companions to the capped torch on EVERY index, cu # families included: torchaudio 2.11 dropped its exact torch pin from @@ -2429,7 +2577,13 @@ exit 0 # resolve a mismatched 2.11.0 build. Mirrors install.sh. $_pinVisionSpec = "torchvision>=0.19,<0.26.0" $_pinAudioSpec = "torchaudio>=2.4,<2.11.0" - $torchInstallExit = Invoke-InstallCommandRetry -Label "install PyTorch" { uv pip install --python $VenvPython "torch>=2.4,<2.11.0" $_pinVisionSpec $_pinAudioSpec --default-index $TorchIndexUrl } + $_torchSpecs = @("torch>=2.4,<2.11.0", $_pinVisionSpec, $_pinAudioSpec) + if ($VenvPlatform -eq "win-arm64") { + substep "windows on arm: skipping torchaudio (upstream publishes no" + substep "win_arm64 wheel); torch and torchvision install normally." + $_torchSpecs = @("torch>=2.4,<2.11.0", $_pinVisionSpec) + } + $torchInstallExit = Invoke-InstallCommandRetry -Label "install PyTorch" { uv pip install --python $VenvPython @_torchSpecs --default-index $TorchIndexUrl } if ($torchInstallExit -ne 0) { Write-Host "[ERROR] Failed to install PyTorch (exit code $torchInstallExit)" -ForegroundColor Red return (Exit-InstallFailure "Failed to install PyTorch (exit code $torchInstallExit)" $torchInstallExit) @@ -2464,7 +2618,7 @@ exit 0 if ($StudioLocalInstall) { substep "overlaying local repo (editable)..." - $overlayExit = Invoke-InstallCommand { uv pip install --python $VenvPython -e $RepoRoot --no-deps } + $overlayExit = Invoke-InstallCommand -Label "overlay local repo" { uv pip install --python $VenvPython -e $RepoRoot --no-deps } if ($overlayExit -ne 0) { Write-Host "[ERROR] Failed to overlay local repo (exit code $overlayExit)" -ForegroundColor Red return (Exit-InstallFailure "Failed to overlay local repo (exit code $overlayExit)" $overlayExit) @@ -2487,7 +2641,7 @@ exit 0 return (Exit-InstallFailure "Failed to install unsloth (exit code $baseInstallExit)" $baseInstallExit) } substep "overlaying local repo (editable)..." - $overlayExit = Invoke-InstallCommand { uv pip install --python $VenvPython -e $RepoRoot --no-deps } + $overlayExit = Invoke-InstallCommand -Label "overlay local repo" { uv pip install --python $VenvPython -e $RepoRoot --no-deps } if ($overlayExit -ne 0) { Write-Host "[ERROR] Failed to overlay local repo (exit code $overlayExit)" -ForegroundColor Red return (Exit-InstallFailure "Failed to overlay local repo (exit code $overlayExit)" $overlayExit) @@ -2535,7 +2689,7 @@ exit 0 $visionSpec = if ($PinnedRocmVisionSpec) { $PinnedRocmVisionSpec } elseif ($ROCmGfxArch -and $torchvisionFloorMap -and $torchvisionFloorMap.ContainsKey($ROCmGfxArch)) { $torchvisionFloorMap[$ROCmGfxArch] } else { "torchvision" } $audioSpec = if ($PinnedRocmAudioSpec) { $PinnedRocmAudioSpec } elseif ($ROCmGfxArch -and $torchaudioFloorMap -and $torchaudioFloorMap.ContainsKey($ROCmGfxArch)) { $torchaudioFloorMap[$ROCmGfxArch] } else { "torchaudio" } substep "PyTorch flavor mismatch (installed $installedTorchTag, need ROCm) -- reinstalling correct build..." "Yellow" - $torchFixExit = Invoke-InstallCommand { uv pip install --python $VenvPython --force-reinstall --default-index $ROCmIndexUrl $rocmSpec $visionSpec $audioSpec } + $torchFixExit = Invoke-InstallCommand -Label "reinstall PyTorch (ROCm)" { uv pip install --python $VenvPython --force-reinstall --default-index $ROCmIndexUrl $rocmSpec $visionSpec $audioSpec } if ($torchFixExit -ne 0) { Write-Host "[ERROR] Failed to reinstall PyTorch with the correct ROCm build (exit code $torchFixExit)" -ForegroundColor Red return (Exit-InstallFailure "Failed to reinstall PyTorch (ROCm) (exit code $torchFixExit)" $torchFixExit) @@ -2544,7 +2698,7 @@ exit 0 } elseif ($expectedTorchTag -ne 'rocm') { # CUDA: stale +cpu (or wrong cuXXX) against a CUDA index -> reinstall triplet. substep "PyTorch flavor mismatch (installed $installedTorchTag, need $expectedTorchTag) -- reinstalling correct build..." "Yellow" - $torchFixExit = Invoke-InstallCommand { uv pip install --python $VenvPython "torch>=2.4,<2.11.0" "torchvision>=0.19,<0.26.0" "torchaudio>=2.4,<2.11.0" --default-index $TorchIndexUrl --reinstall-package torch --reinstall-package torchvision --reinstall-package torchaudio } + $torchFixExit = Invoke-InstallCommand -Label "reinstall PyTorch ($expectedTorchTag)" { uv pip install --python $VenvPython "torch>=2.4,<2.11.0" "torchvision>=0.19,<0.26.0" "torchaudio>=2.4,<2.11.0" --default-index $TorchIndexUrl --reinstall-package torch --reinstall-package torchvision --reinstall-package torchaudio } if ($torchFixExit -ne 0) { Write-Host "[ERROR] Failed to reinstall PyTorch with the correct CUDA build (exit code $torchFixExit)" -ForegroundColor Red return (Exit-InstallFailure "Failed to reinstall PyTorch ($expectedTorchTag) (exit code $torchFixExit)" $torchFixExit) @@ -2645,6 +2799,9 @@ exit 0 # an inherited value would put llama.cpp in the wrong place. $previousUnslothStudioHome = $env:UNSLOTH_STUDIO_HOME $hadPreviousUnslothStudioHome = ($null -ne $previousUnslothStudioHome) + $previousTauriMode = $env:UNSLOTH_TAURI_MODE + $hadPreviousTauriMode = ($null -ne $previousTauriMode) + $env:UNSLOTH_TAURI_MODE = if ($TauriMode) { "1" } else { "0" } if ($StudioRedirectMode -eq 'env') { $env:UNSLOTH_STUDIO_HOME = $StudioHome } else { @@ -2674,14 +2831,22 @@ exit 0 } else { Remove-Item Env:UNSLOTH_STUDIO_HOME -ErrorAction SilentlyContinue } + if ($hadPreviousTauriMode) { + $env:UNSLOTH_TAURI_MODE = $previousTauriMode + } else { + Remove-Item Env:UNSLOTH_TAURI_MODE -ErrorAction SilentlyContinue + } Remove-Item Env:UNSLOTH_LOCAL_LLAMA_CPP_DIR -ErrorAction SilentlyContinue Remove-Item Env:UNSLOTH_INSTALL_ROLLBACK_MANAGED -ErrorAction SilentlyContinue Remove-Item Env:UNSLOTH_SETUP_PYTHON -ErrorAction SilentlyContinue } if ($setupExit -ne 0) { - Write-Host "[ERROR] unsloth studio setup failed (exit code $setupExit)" -ForegroundColor Red + if (-not $TauriMode) { + Write-Host "[ERROR] unsloth studio setup failed (exit code $setupExit)" -ForegroundColor Red + } return (Exit-InstallFailure "unsloth studio setup failed (exit code $setupExit)" $setupExit) } + Clear-TauriInstallError "studio setup completed" # ── Expose `unsloth` via a shim dir containing only unsloth.exe ── # We do NOT add the venv Scripts dir to PATH (it also holds python.exe diff --git a/install.sh b/install.sh index 146a64e692..166beeb52c 100755 --- a/install.sh +++ b/install.sh @@ -19,6 +19,17 @@ # SPDX-License-Identifier: AGPL-3.0-only # Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 set -e +# ── Why the installer lives in a function ── +# Under `curl ... | sh`, sh is the pipe READER. This file is ~150KB, so a top-level +# `exit` left most of it unread, the write end failed, and curl tacked +# "(56) Failure writing output to destination" onto our own error message. Wrapping +# the body forces sh to parse to the closing brace first, so the pipe always drains +# (install.ps1 has always had this shape). +# +# Body is deliberately NOT reindented: reflowing 4000+ lines would bury the change, +# and `exit` still exits the shell from inside a function. Do not add +# `exec < /dev/null`: for a piped shell that closes the script's own source. +_unsloth_main() { # ── Output style (aligned with studio/setup.sh) ── RULE="" @@ -207,18 +218,37 @@ run_install_cmd() { # command's exit code across the pipe without relying on pipefail # (this script runs under plain sh). _rcf=$(mktemp) - { "$@" 2>&1; printf '%s' "$?" > "$_rcf"; } | _redact_install_output + tauri_stream_log stdout "OUTPUT_CLEAR" "$_label" + { + if "$@" 2>&1; then + _cmd_rc=0 + else + _cmd_rc=$? + fi + printf '%s' "$_cmd_rc" > "$_rcf" + } | _redact_install_output _rc=$(cat "$_rcf" 2>/dev/null || echo 1) rm -f "$_rcf" - [ "${_rc:-1}" -eq 0 ] 2>/dev/null && return 0 + _rc=${_rc:-1} + if [ "$_rc" -eq 0 ] 2>/dev/null; then + tauri_clear_install_error "$_label recovered" + return 0 + fi + tauri_stream_log stdout "ERROR_OUTPUT" "$_label failed (exit code $_rc)" step "error" "$_label failed (exit code $_rc)" "$C_ERR" >&2 return "$_rc" fi _log=$(mktemp) - "$@" >"$_log" 2>&1 && { rm -f "$_log"; return 0; } + tauri_stream_log stderr "OUTPUT_CLEAR" "$_label" + "$@" >"$_log" 2>&1 && { + rm -f "$_log" + tauri_clear_install_error "$_label recovered" + return 0 + } _rc=$? step "error" "$_label failed (exit code $_rc)" "$C_ERR" >&2 _redact_install_output "$_log" >&2 + tauri_stream_log stderr "ERROR_OUTPUT" "$_label failed (exit code $_rc)" rm -f "$_log" return $_rc } @@ -257,9 +287,55 @@ run_install_cmd_retry() { done } +# True when the runtime target is gfx906 (MI50/Radeon VII): the prebuilt AMD +# bitsandbytes wheel carries no gfx906 kernels, and force-reinstalling it would +# clobber a user's source-built bnb (the only 4-bit path on this arch) on every +# `studio update`. So skip the auto-install and leave whatever bnb is present. +# _gfx906_target is set during torch-index resolution; also honor an explicit +# UNSLOTH_ROCM_GFX_ARCH so a pinned-index install still skips. The override is +# normalized (gfx906:sramecc-:xnack- -> gfx906) so a copied HIP gcnArchName counts. +_is_gfx906_bnb_skip() { + [ "${_gfx906_target:-false}" = true ] && return 0 + _bnb_gfx_env=$(printf '%s' "${UNSLOTH_ROCM_GFX_ARCH:-}" | tr '[:upper:]' '[:lower:]' | tr -d '[:space:]') + _bnb_gfx_env=${_bnb_gfx_env%%:*} + [ "$_bnb_gfx_env" = "gfx906" ] && return 0 + # A pinned index (UNSLOTH_TORCH_INDEX_URL/_FAMILY) skips the reroute block that + # sets _gfx906_target, so a real gfx906 host with a pinned rocm6.3 index and no + # UNSLOTH_ROCM_GFX_ARCH would otherwise clobber a source-built bnb. Probe here + # in that gap; skip only when gfx906 is the SOLE distinct arch (mixed hosts + # opt in via the env var, mirroring the reroute block's de-dup rule). + if [ -z "$_bnb_gfx_env" ] && [ "${_torch_index_pinned:-false}" = true ]; then + _bnb_gfx_probe=$(_probe_amd_gfx_arch | awk 'NF && !seen[$0]++') + [ "$_bnb_gfx_probe" = "gfx906" ] && return 0 + fi + return 1 +} + +# `pip install unsloth` resolves its unconditional bitsandbytes dep to a generic +# CUDA wheel (no gfx906 kernels) once we skip the prebuilt one. Snapshot bnb before +# the unsloth install, then drop a freshly pulled wheel afterwards while leaving a +# pre-existing source build in place. +_gfx906_bnb_installed() { + "$_VENV_PY" -c "import importlib.util as u, sys; sys.exit(0 if u.find_spec('bitsandbytes') else 1)" >/dev/null 2>&1 +} +_gfx906_bnb_snapshot() { + _gfx906_bnb_absent_before=false + _is_gfx906_bnb_skip || return 0 + _gfx906_bnb_installed || _gfx906_bnb_absent_before=true +} +_gfx906_bnb_prune() { + _is_gfx906_bnb_skip || return 0 + [ "${_gfx906_bnb_absent_before:-false}" = true ] || return 0 + _gfx906_bnb_installed || return 0 + substep "gfx906: removing generic bitsandbytes pulled in as a dependency (no gfx906 kernels; build from source for 4-bit QLoRA)" "$C_WARN" + uv pip uninstall --python "$_VENV_PY" bitsandbytes >/dev/null 2>&1 \ + || "$_VENV_PY" -m pip uninstall -y bitsandbytes >/dev/null 2>&1 || true +} + # Install bitsandbytes on AMD ROCm hosts. bnb <= 0.49.2 NaNs at 4-bit decode -# shape on every AMD GPU; the fix (bnb #1887) is in the continuous-release_main -# wheel used below and first ships on PyPI in 0.50.0, hence the fallback floor. +# shape on every AMD GPU; the fix (bnb #1887) ships in continuous-release_main +# and, on PyPI, first in 0.50.0. Keep this floor in step with the amd extra in +# pyproject.toml and studio/install_python_stack.py. _BNB_ROCM_PYPI_FALLBACK="bitsandbytes>=0.50.0" # bitsandbytes ships no ROCm binary in its aarch64 wheel at any version: the PyPI # 0.50.0 and continuous-release_main aarch64 wheels both carry only @@ -289,8 +365,8 @@ _install_bnb_rocm() { _bnb_whl_url="" ;; esac - # uv rejects the pre-release wheel: its filename version (1.33.7rc0) does not - # match its metadata version (0.50.x.dev0). pip accepts it, so bootstrap pip. + # uv rejects the pre-release wheel: filename version (1.33.7rc0) does not + # match metadata (0.50.x.dev0). pip accepts it, so bootstrap pip and use it. if ! "$_venv_py" -m pip --version >/dev/null 2>&1; then if ! run_maybe_quiet "$_venv_py" -m ensurepip --upgrade; then run_maybe_quiet uv pip install --python "$_venv_py" pip || \ @@ -359,6 +435,34 @@ tauri_log() { fi } +tauri_stream_log() { + _tsl_stream="$1" + _tsl_tag="$2" + shift 2 + if [ "$TAURI_MODE" = true ]; then + if [ "$_tsl_stream" = stderr ]; then + printf '[TAURI:%s] %s\n' "$_tsl_tag" "$*" >&2 + else + printf '[TAURI:%s] %s\n' "$_tsl_tag" "$*" + fi + fi +} + +rollback_substep() { + if [ "$TAURI_MODE" = true ]; then + tauri_log "PROGRESS" "$1" + else + substep "$@" + fi +} + +tauri_clear_install_error() { + if [ "$TAURI_MODE" = true ]; then + tauri_log "ERROR_CLEAR" "$1" + printf '[TAURI:ERROR_CLEAR] %s\n' "$1" >&2 + fi +} + tauri_diag_marker() { _diag_gpu_branch="${1:-unknown}" _diag_torch_index_family="${2:-none}" @@ -519,10 +623,10 @@ _restore_studio_venv_replacement() { _VENV_ROLLBACK_ACTIVE=false return 0 } - substep "restoring previous environment after failed install..." "$C_WARN" + rollback_substep "restoring previous environment after failed install..." "$C_WARN" rm -rf "$_VENV_ROLLBACK_TARGET" if mv "$_VENV_ROLLBACK_DIR" "$_VENV_ROLLBACK_TARGET"; then - substep "restored previous environment" + rollback_substep "restored previous environment" _VENV_ROLLBACK_ACTIVE=false _VENV_ROLLBACK_DIR="" else @@ -707,8 +811,17 @@ _smart_apt_install() { return 0 fi - # In Tauri mode, report needed packages and exit — Rust handles elevation + # Optional callers never elevate, in any mode: nothing on the consumer path + # builds anything, so neither the terminal sudo prompt below nor the Tauri + # NEED_SUDO dialog (whose Cancel leaves the user not installed) may gate the + # run over unused tools. The caller falls through to prebuilt llama.cpp. + # Required packages such as curl still escalate. + if [ "${_SMART_APT_OPTIONAL:-false}" = true ]; then + return 2 + fi + if [ "$TAURI_MODE" = true ]; then + # Report needed packages and exit — Rust handles elevation. tauri_log "NEED_SUDO" "$_STILL_MISSING" exit 2 fi @@ -1905,67 +2018,142 @@ _maybe_reroute_strixhalo_to_2404() { _maybe_reroute_strixhalo_to_2404 || true # ── Check system dependencies ── -# cmake/git are only needed to *build* llama.cpp from source. Unsloth downloads a -# prebuilt by default, and setup.sh self-skips the source build when they're -# absent -- so macOS doesn't block on cmake (requiring it would force a manual -# Homebrew install). Linux keeps requiring them; its package manager has them. tauri_log "STEP" "Checking system dependencies" +# Without the Xcode CLT, macOS still ships /usr/bin/git as a stub that errors and pops +# a GUI dialog, so `command -v git` is not enough -- only running it tells the truth. +_has_working_git() { + command -v git >/dev/null 2>&1 || return 1 + git --version >/dev/null 2>&1 +} + +# macOS system-dependency check. A function so tests/sh can sed-extract it; the old +# inline form was untestable, which is why this gate shipped broken. +# +# The consumer install needs no developer toolchain: uv is a prebuilt binary, CPython +# is uv-managed, llama.cpp/whisper.cpp/Node are prebuilt downloads, and triton is +# skipped on macOS. Only `--local` needs git, for the unsloth-zoo git+https URL. +_check_macos_deps() { + _clt_missing=false + xcode-select -p >/dev/null 2>&1 || _clt_missing=true + + if [ "$STUDIO_LOCAL_INSTALL" = true ] && ! _has_working_git; then + echo "" + step "deps" "git is required for --local installs" "$C_ERR" + substep "--local installs unsloth-zoo from git+https://github.com/unslothai/unsloth-zoo," + substep "which needs a working git. Install the Xcode Command Line Tools:" + substep " xcode-select --install" + substep "Then re-run this script. A normal (non---local) install needs no compiler" + substep "and no git -- it uses prebuilt binaries and wheels only." + tauri_log "NEED_XCODE_CLT" "git" + return 1 + fi + + if [ "$_clt_missing" = true ]; then + # Not fatal, and no GUI dialog: firing xcode-select --install and exiting is + # what stranded clean Macs. + step "deps" "no Xcode Command Line Tools (not required)" "$C_WARN" + substep "Unsloth installs prebuilt binaries and wheels, so no compiler is needed." + substep "Install them only for a llama.cpp source build: xcode-select --install" + elif command -v cmake >/dev/null 2>&1; then + step "deps" "all system dependencies found" + else + # cmake is only for a source build, so its absence is not fatal. + step "deps" "using prebuilt llama.cpp (cmake not found)" "$C_WARN" + substep "Install cmake only if you want a source build: brew install cmake" + fi + return 0 +} + +# Linux/WSL system-dependency check. Same split as macOS, and a function for the same +# reason: tests/sh can extract it. +# +# Only a download transport is required. cmake, gcc and the libcurl headers exist +# solely for a llama.cpp source build the consumer path never does -- unslothai/ +# llama.cpp publishes linux-x64/arm64 prebuilts for cpu, cuda12, cuda13, rocm and +# vulkan. Requiring them turned every non-apt distro into a hard exit 1 over unused +# tooling. git follows macOS: --local only. +_check_linux_deps() { + _transport_missing=false + if ! command -v curl >/dev/null 2>&1 && ! command -v wget >/dev/null 2>&1; then + _transport_missing=true + fi + + # Wanted, never required: git fetches the triton_kernels git+https requirement (a + # training speedup), the rest serve the optional source build. Warn, never stop. + _optional_missing="" + command -v cmake >/dev/null 2>&1 || _optional_missing="$_optional_missing cmake" + _has_working_git || _optional_missing="$_optional_missing git" + command -v gcc >/dev/null 2>&1 || _optional_missing="$_optional_missing build-essential" + command -v curl-config >/dev/null 2>&1 || _optional_missing="$_optional_missing libcurl4-openssl-dev" + # Parameter expansion, not `sed`: sed may be absent on a minimal image, and a + # failed `$(... | sed ...)` yields "" -- "all found" on a machine that has none. + _optional_missing="${_optional_missing# }" + + if [ "$STUDIO_LOCAL_INSTALL" = true ] && ! _has_working_git; then + echo "" + step "deps" "git is required for --local installs" "$C_ERR" + substep "--local installs unsloth-zoo from git+https://github.com/unslothai/unsloth-zoo," + substep "which needs git. Install it with your package manager, then re-run." + substep "A normal (non---local) install needs no git and no compiler." + return 1 + fi + + # The one fatal case: nothing can be downloaded. apt is the only distro family we + # can drive unattended. + if [ "$_transport_missing" = true ]; then + if command -v apt-get >/dev/null 2>&1; then + echo "" + step "deps" "missing: curl" "$C_WARN" + substep "Needed to download uv, Python and the prebuilt inference engine." + _smart_apt_install curl + echo "" + else + echo "" + step "deps" "missing: curl (or wget)" "$C_ERR" + substep "Unsloth needs one of them to download uv, Python and the prebuilt" + substep "inference engine. Install one, then re-run setup:" + substep " Fedora/RHEL: sudo dnf install curl" + substep " Arch: sudo pacman -S --needed curl" + substep " openSUSE: sudo zypper install curl" + return 1 + fi + fi + + # Try apt for the optional set too; failing only costs the features warned about + # below. + if [ -n "$_optional_missing" ] && command -v apt-get >/dev/null 2>&1; then + step "deps" "installing optional build tools: $_optional_missing" "$C_DIM" + # Subshell because _smart_apt_install exits rather than returns, so `|| true` + # alone would not catch it. _SMART_APT_OPTIONAL suppresses every escalation + # path, so no install hinges on a prompt for tools nothing here needs. + ( _SMART_APT_OPTIONAL=true; _smart_apt_install $_optional_missing ) || true + _optional_missing="" + command -v cmake >/dev/null 2>&1 || _optional_missing="$_optional_missing cmake" + _has_working_git || _optional_missing="$_optional_missing git" + command -v gcc >/dev/null 2>&1 || _optional_missing="$_optional_missing build-essential" + command -v curl-config >/dev/null 2>&1 || _optional_missing="$_optional_missing libcurl4-openssl-dev" + _optional_missing="${_optional_missing# }" + fi + + if [ -n "$_optional_missing" ]; then + step "deps" "using prebuilt llama.cpp (missing: $_optional_missing)" "$C_WARN" + substep "Not required to run: Unsloth downloads a prebuilt inference engine." + case " $_optional_missing " in + *" git "*) substep "Without git the triton kernels training speedup is skipped." ;; + esac + else + step "deps" "all system dependencies found" + fi + return 0 +} + case "$OS" in macos) - # Xcode Command Line Tools provide the C/C++ compiler and git. - if ! xcode-select -p >/dev/null 2>&1; then - echo "" - echo "==> Xcode Command Line Tools are required." - echo " Installing (a system dialog will appear)..." - xcode-select --install /dev/null || true - echo " After the installation completes, please re-run this script." - exit 1 - fi - # cmake is only needed for a source build; the default prebuilt path - # doesn't use it, so its absence is not fatal -- no Homebrew prerequisite. - if command -v cmake >/dev/null 2>&1; then - step "deps" "all system dependencies found" - else - step "deps" "using prebuilt llama.cpp (cmake not found)" "$C_WARN" - substep "Install cmake only if you want a source build: brew install cmake" - fi + _check_macos_deps || exit 1 ;; linux|wsl) - MISSING="" - command -v cmake >/dev/null 2>&1 || MISSING="$MISSING cmake" - command -v git >/dev/null 2>&1 || MISSING="$MISSING git" - # curl or wget is needed for downloads; check both - if ! command -v curl >/dev/null 2>&1 && ! command -v wget >/dev/null 2>&1; then - MISSING="$MISSING curl" - fi - command -v gcc >/dev/null 2>&1 || MISSING="$MISSING build-essential" - # libcurl dev headers for llama.cpp HTTPS support - command -v curl-config >/dev/null 2>&1 || MISSING="$MISSING libcurl4-openssl-dev" - - MISSING=$(echo "$MISSING" | sed 's/^ *//') - if [ -n "$MISSING" ]; then - echo "" - step "deps" "missing: $MISSING" "$C_WARN" - substep "These are needed to build the GGUF inference engine." - if command -v apt-get >/dev/null 2>&1; then - _smart_apt_install $MISSING - else - echo " Automatic system package installation is supported on apt-based" - echo " Linux distributions (Ubuntu/Debian) only. Please install the" - echo " missing dependencies with your package manager, then re-run setup:" - echo " $MISSING" - echo "" - echo " Examples:" - echo " Fedora/RHEL: sudo dnf install cmake git gcc gcc-c++ make libcurl-devel" - echo " Arch: sudo pacman -S --needed cmake git base-devel curl" - echo " openSUSE: sudo zypper install cmake git gcc gcc-c++ make libcurl-devel" - exit 1 - fi - echo "" - else - step "deps" "all system dependencies found" - fi + _check_linux_deps || exit 1 ;; esac @@ -3317,10 +3505,20 @@ case "$_torch_index_leaf" in if (n > 0) print vals[idx] }') fi + # An explicit UNSLOTH_ROCM_GFX_ARCH=gfx906 pins the runtime target to the + # MI50 / Radeon VII path and must win over Strix probe-order detection on a + # mixed Strix + MI50 host, so the Strix reroute is suppressed when it is set. + # Normalize a copied HIP gcnArchName (gfx906:sramecc-:xnack- -> gfx906) and + # trim whitespace (mirrors the Python .strip()) so the feature-flag suffix or + # a stray newline does not defeat the exact gfx906 comparisons below. + _gfx906_env=$(printf '%s' "${UNSLOTH_ROCM_GFX_ARCH:-}" | tr '[:upper:]' '[:lower:]' | tr -d '[:space:]') + _gfx906_env=${_gfx906_env%%:*} _strix_gfx="" - case "$_runtime_gfx" in - gfx1151|gfx1150|gfx1152) _strix_gfx="$_runtime_gfx" ;; - esac + if [ "$_gfx906_env" != "gfx906" ]; then + case "$_runtime_gfx" in + gfx1151|gfx1150|gfx1152) _strix_gfx="$_runtime_gfx" ;; + esac + fi # Skip rocm7.13+ generic indexes: they already ship the fixes, so the # arch build (rocm7.13) would be a downgrade rather than a rescue. if [ -n "$_strix_gfx" ] && _rocm_leaf_below "$_torch_index_leaf" 7 13; then @@ -3348,6 +3546,57 @@ case "$_torch_index_leaf" in TORCHAUDIO_CONSTRAINT="torchaudio>=2.11.0,<2.12.0" _amd_gpu_radeon=false fi + # ── MI50 / Radeon VII (gfx906, Vega 20): legacy community-supported path ── + # Newer rocm wheel families bundle ROCm libraries whose Tensile kernels + # dropped gfx906 (rocBLAS "TensileLibrary.dat ... not read for gfx906", + # ROCm/TheRock#1844), so a rocm6.4+/7.x index installs a torch that fails + # at the first BLAS call. The rocm6.3 index is the last one whose wheels + # run on gfx906 (torch 2.7.0 verified on MI50 32GB; up to 2.9 in community + # use). Reroute any newer picked index; leave rocm6.0-6.3 alone. + # + # Target resolution: an explicit UNSLOTH_ROCM_GFX_ARCH wins (lets a host + # whose rocminfo/amd-smi emit no gfx token still opt in; _gfx906_env was + # lowercased above, before the Strix block it suppresses). Otherwise only + # treat gfx906 as the target when it is the SOLE distinct arch present: + # _gfx_all is de-duplicated by visible index, which loses per-device + # ordinals on a mixed host, so a non-gfx906 selection must never be + # downgraded to rocm6.3 -- such hosts set UNSLOTH_ROCM_GFX_ARCH to opt in. + _gfx906_target=false + if [ -n "$_gfx906_env" ]; then + [ "$_gfx906_env" = "gfx906" ] && _gfx906_target=true + elif [ -n "$_gfx_all" ]; then + _gfx906_uniq=$(printf '%s\n' "$_gfx_all" | awk 'NF && !seen[$0]++') + [ "$_gfx906_uniq" = "gfx906" ] && _gfx906_target=true + fi + # gfx906 always trains from the PyTorch rocm6.3 wheels, never the Radeon repo + # (repo.radeon.com wheels carry no gfx906 BLAS kernels). Clear the Radeon + # marketing-name flag as soon as gfx906 is the target -- even when the host + # already picks rocm6.0-6.3 and the reroute below is a no-op -- so a Radeon VII + # does not divert to the radeon branch on those versions. + if [ "$_gfx906_target" = true ]; then + _amd_gpu_radeon=false + fi + if [ "$_gfx906_target" = true ] && ! _rocm_leaf_below "$_torch_index_leaf" 6 4; then + echo "" >&2 + echo " [WARN] gfx906 (MI50 / Radeon VII / Vega 20) detected -- routing torch to the" >&2 + echo " [WARN] rocm6.3 index: it is the last wheel family that runs on gfx906 (newer" >&2 + echo " [WARN] rocm wheels ship without gfx906 BLAS kernels and fail at first use)." >&2 + echo " [WARN] gfx906 is a community-maintained legacy path: 16-bit LoRA and full" >&2 + echo " [WARN] finetuning work out of the box; bitsandbytes 4-bit QLoRA requires a" >&2 + echo " [WARN] source build of bitsandbytes for gfx906 (see docs.unsloth.ai/amd)." >&2 + echo "" >&2 + _amd_gfx906_base="${UNSLOTH_PYTORCH_MIRROR:-https://download.pytorch.org/whl}" + while [ "${_amd_gfx906_base%/}" != "$_amd_gfx906_base" ]; do + _amd_gfx906_base="${_amd_gfx906_base%/}" + done + TORCH_INDEX_URL="${_amd_gfx906_base}/rocm6.3" + # Reset to the default (<2.11) window: a rocm7.2 pick raised the floor + # to 2.11 above, which the rocm6.3 index (torch <= 2.9.x) cannot satisfy. + TORCH_CONSTRAINT="torch>=2.4,<2.11.0" + TORCHVISION_CONSTRAINT="torchvision>=0.19,<0.26.0" + TORCHAUDIO_CONSTRAINT="torchaudio>=2.4,<2.11.0" + # (_amd_gpu_radeon already cleared above for every gfx906 target.) + fi ;; esac fi # _torch_index_pinned guard (Radeon + Strix reroute) @@ -3574,6 +3823,7 @@ for _p in ('torch', 'torchvision', 'torchaudio'): if [ "$_MIGRATED" = true ]; then # Migrated env: force-reinstall unsloth+unsloth-zoo for a clean state, preserving # existing torch/CUDA unless the ROCm repair below fires. + _gfx906_bnb_snapshot substep "upgrading unsloth in migrated environment..." if [ "$SKIP_TORCH" = true ]; then # No-torch: install unsloth + unsloth-zoo with --no-deps (current @@ -3615,13 +3865,18 @@ if [ "$_MIGRATED" = true ]; then # existing ROCm installs gain the AMD bitsandbytes build without a # fresh reinstall. if [ "$SKIP_TORCH" = false ] && [ "$_torch_index_is_rocm_family" = true ]; then - _install_bnb_rocm "install bitsandbytes (AMD)" "$_VENV_PY" + if _is_gfx906_bnb_skip; then + substep "gfx906: skipping prebuilt bitsandbytes (no gfx906 kernels); build from source for 4-bit QLoRA -- https://docs.unsloth.ai/get-started/install-and-update/amd" "$C_WARN" + else + _install_bnb_rocm "install bitsandbytes (AMD)" "$_VENV_PY" + fi # Repair ROCm torch if overwritten during migrated install _has_hip=$("$_VENV_PY" -c "import torch; print(getattr(torch.version,'hip','') or '')" 2>/dev/null || true) if [ -z "$_has_hip" ]; then substep "repairing ROCm torch (overwritten by dependency resolution)..." _install_torch_default_index --force-reinstall fi + _gfx906_bnb_prune fi elif [ -n "$TORCH_INDEX_URL" ]; then # Fresh: Step 1 - install torch from explicit index (skip when --no-torch or Intel Mac) @@ -3812,8 +4067,13 @@ elif [ -n "$TORCH_INDEX_URL" ]; then # host stays in GGUF-only mode rather than pulling in bitsandbytes, # which is only useful once torch is present for training. if [ "$SKIP_TORCH" = false ] && [ "$_torch_index_is_rocm_family" = true ]; then - _install_bnb_rocm "install bitsandbytes (AMD)" "$_VENV_PY" + if _is_gfx906_bnb_skip; then + substep "gfx906: skipping prebuilt bitsandbytes (no gfx906 kernels); build from source for 4-bit QLoRA -- https://docs.unsloth.ai/get-started/install-and-update/amd" "$C_WARN" + else + _install_bnb_rocm "install bitsandbytes (AMD)" "$_VENV_PY" + fi fi + _gfx906_bnb_snapshot # Fresh: Step 2 - install unsloth, preserving the torch Step 1 installed tauri_log "STEP" "Installing Unsloth" substep "installing unsloth (this may take a few minutes)..." @@ -3864,6 +4124,7 @@ elif [ -n "$TORCH_INDEX_URL" ]; then substep "repairing ROCm torch (overwritten by dependency resolution)..." _install_torch_default_index --force-reinstall fi + _gfx906_bnb_prune fi else # Fallback: GPU detection failed to produce a URL -- let uv resolve torch @@ -3958,6 +4219,7 @@ if [ -n "$VENV_ABS_BIN" ]; then fi if ! command -v bash >/dev/null 2>&1; then + tauri_log "ERROR" "bash is required to run studio setup" step "setup" "bash is required to run studio setup" "$C_ERR" substep "Please install bash and re-run install.sh" exit 1 @@ -3996,6 +4258,7 @@ if [ "$STUDIO_LOCAL_INSTALL" = true ]; then STUDIO_LOCAL_REPO="$_REPO_ROOT" \ UNSLOTH_NO_TORCH="$SKIP_TORCH" \ UNSLOTH_LOCAL_LLAMA_CPP_DIR="$_WITH_LLAMA_CPP_DIR" \ + UNSLOTH_TAURI_MODE="$TAURI_MODE" \ bash "$SETUP_SH" =2026.7.6", - "wheel>=0.42.0", - "packaging", - "torch>=2.4.0,<2.12.0", - "torchvision", - "numpy", - "tqdm", - "psutil", - "tyro", - "protobuf", - "xformers>=0.0.27.post2 ; ('linux' in sys_platform or sys_platform == 'win32') and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", - "bitsandbytes>=0.45.5,!=0.46.0,!=0.48.0", - "triton>=3.0.0 ; ('linux' in sys_platform)", - "triton-windows ; (sys_platform == 'win32') and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", - "sentencepiece>=0.2.0", - "datasets>=3.4.1,!=4.0.*,!=4.1.0,<4.4.0", - "accelerate>=0.34.1", - "peft>=0.18.0,!=0.11.0", - "huggingface_hub>=0.34.0", - "hf_transfer", - "diffusers", - "transformers>=4.51.3,!=4.52.0,!=4.52.1,!=4.52.2,!=4.52.3,!=4.53.0,!=4.54.0,!=4.55.0,!=4.55.1,!=4.57.0,!=4.57.4,!=4.57.5,!=5.0.0,!=5.1.0,<=5.5.0", - "trl>=0.18.2,!=0.19.0,<=0.24.0", "typer>=0.12.0", + "rich", "pydantic", "pyyaml", "nest-asyncio", # Every CLI command imports studio.backend.*, which reaches structlog at # module level. The rest of the server stack lives in the studio extra. "structlog>=24.1.0", - # unsloth_cli/commands/start.py imports click, and unsloth_cli/__init__.py - # imports that, so every command needs it. typer supplied it until 0.27 - # dropped the dependency, which left this satisfied only by chance. + # unsloth_cli/__init__.py reaches click via commands/start.py, so every + # command needs it. typer supplied it until 0.27 dropped the dependency. "click>=8.0", ] @@ -70,9 +47,14 @@ version = {attr = "unsloth.models._utils.__version__"} [tool.setuptools] include-package-data = true +[tool.setuptools.cmdclass] +# Snapshots CHANGELOG.md into studio/ so every build path ships it. +build_py = "_changelog_build.build_py" + [tool.setuptools.package-data] unsloth_cli = ["codex_fallback_prompt.md", "pi_subagent.ts"] studio = [ + "CHANGELOG.md", "*.sh", "*.ps1", "*.bat", @@ -97,7 +79,7 @@ include = ["unsloth*", "unsloth_cli*", "studio", "studio.backend*"] exclude = ["images*", "tests*", "*.node_modules", "*.node_modules.*"] [project.optional-dependencies] -# Studio's server stack. Mirrors studio/backend/requirements/studio.txt; +# Studio's server stack, mirroring studio/backend/requirements/studio.txt. # test_studio_extra_matches_requirements.py catches drift. studio = [ "typer", @@ -128,11 +110,11 @@ triton = [ "triton>=3.0.0 ; ('linux' in sys_platform)", "triton-windows ; (sys_platform == 'win32') and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", ] -huggingface = [ + +huggingfacenotorch = [ "unsloth_zoo>=2026.7.6", "wheel>=0.42.0", "packaging", - "torchvision", "numpy", "tqdm", "psutil", @@ -148,10 +130,28 @@ huggingface = [ "transformers>=4.51.3,!=4.52.0,!=4.52.1,!=4.52.2,!=4.52.3,!=4.53.0,!=4.54.0,!=4.55.0,!=4.55.1,!=4.57.0,!=4.57.4,!=4.57.5,!=5.0.0,!=5.1.0,<=5.5.0", "trl>=0.18.2,!=0.19.0,<=0.24.0", "sentence-transformers", - "typer>=0.12.0", - "pydantic", - "pyyaml", - "nest-asyncio", +] +# torchcodec backend for Gemma audio / datasets>=4 (#7225). +# Pick the audio-torch* pin matching your torch minor (see TORCH_TORCHCODEC). +# torchcodec publishes no sdist and only manylinux_2_28_x86_64, macosx_*_arm64 +# and win_amd64 wheels, so Linux aarch64, Windows ARM64 and Intel Mac have +# nothing to resolve and pip fails the whole install rather than skipping audio. +# Gate on the platforms that have a wheel, matching +# PLATFORM_LACKS_TORCHCODEC_WHEEL in studio/install_python_stack.py. +audio-torch210 = [ + "torchcodec>=0.10.0,<0.11.0 ; python_version >= '3.10' and (((sys_platform == 'linux' or sys_platform == 'win32') and (platform_machine == 'x86_64' or platform_machine == 'AMD64')) or (sys_platform == 'darwin' and platform_machine == 'arm64'))", +] +audio-torch290 = [ + "torchcodec>=0.8.0,<0.10.0 ; python_version >= '3.10' and (((sys_platform == 'linux' or sys_platform == 'win32') and (platform_machine == 'x86_64' or platform_machine == 'AMD64')) or (sys_platform == 'darwin' and platform_machine == 'arm64'))", +] +audio-torch280 = [ + "torchcodec>=0.6.0,<0.8.0 ; python_version >= '3.9' and (((sys_platform == 'linux' or sys_platform == 'win32') and (platform_machine == 'x86_64' or platform_machine == 'AMD64')) or (sys_platform == 'darwin' and platform_machine == 'arm64'))", +] +huggingface = [ + "unsloth[huggingfacenotorch]", + "unsloth_zoo>=2026.7.6", + "torchvision", + "unsloth[triton]", ] windows = [ "unsloth[huggingface]", @@ -162,126 +162,235 @@ base = [ "unsloth[huggingface]", ] cu118only = [ - "xformers==0.0.22.post7 ; ('linux' in sys_platform or sys_platform == 'win32') and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", + "xformers @ https://download.pytorch.org/whl/cu118/xformers-0.0.22.post7%2Bcu118-cp39-cp39-manylinux2014_x86_64.whl ; python_version=='3.9' and ('linux' in sys_platform)", + "xformers @ https://download.pytorch.org/whl/cu118/xformers-0.0.22.post7%2Bcu118-cp310-cp310-manylinux2014_x86_64.whl ; python_version=='3.10' and ('linux' in sys_platform)", + "xformers @ https://download.pytorch.org/whl/cu118/xformers-0.0.22.post7%2Bcu118-cp311-cp311-manylinux2014_x86_64.whl ; python_version=='3.11' and ('linux' in sys_platform)", ] cu121only = [ - "xformers==0.0.22.post7 ; ('linux' in sys_platform or sys_platform == 'win32') and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", + "xformers @ https://download.pytorch.org/whl/cu121/xformers-0.0.22.post7-cp39-cp39-manylinux2014_x86_64.whl ; python_version=='3.9' and ('linux' in sys_platform)", + "xformers @ https://download.pytorch.org/whl/cu121/xformers-0.0.22.post7-cp310-cp310-manylinux2014_x86_64.whl ; python_version=='3.10' and ('linux' in sys_platform)", + "xformers @ https://download.pytorch.org/whl/cu121/xformers-0.0.22.post7-cp311-cp311-manylinux2014_x86_64.whl ; python_version=='3.11' and ('linux' in sys_platform)", ] cu118onlytorch211 = [ - "xformers==0.0.23 ; ('linux' in sys_platform or sys_platform == 'win32') and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", + "xformers @ https://download.pytorch.org/whl/cu118/xformers-0.0.23%2Bcu118-cp39-cp39-manylinux2014_x86_64.whl ; python_version=='3.9' and ('linux' in sys_platform)", + "xformers @ https://download.pytorch.org/whl/cu118/xformers-0.0.23%2Bcu118-cp310-cp310-manylinux2014_x86_64.whl ; python_version=='3.10' and ('linux' in sys_platform)", + "xformers @ https://download.pytorch.org/whl/cu118/xformers-0.0.23%2Bcu118-cp311-cp311-manylinux2014_x86_64.whl ; python_version=='3.11' and ('linux' in sys_platform)", ] cu121onlytorch211 = [ - "xformers==0.0.23 ; ('linux' in sys_platform or sys_platform == 'win32') and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", + "xformers @ https://download.pytorch.org/whl/cu121/xformers-0.0.23-cp39-cp39-manylinux2014_x86_64.whl ; python_version=='3.9' and ('linux' in sys_platform)", + "xformers @ https://download.pytorch.org/whl/cu121/xformers-0.0.23-cp310-cp310-manylinux2014_x86_64.whl ; python_version=='3.10' and ('linux' in sys_platform)", + "xformers @ https://download.pytorch.org/whl/cu121/xformers-0.0.23-cp311-cp311-manylinux2014_x86_64.whl ; python_version=='3.11' and ('linux' in sys_platform)", ] cu118onlytorch212 = [ - "xformers==0.0.23.post1 ; ('linux' in sys_platform or sys_platform == 'win32') and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", + "xformers @ https://download.pytorch.org/whl/cu118/xformers-0.0.23.post1%2Bcu118-cp39-cp39-manylinux2014_x86_64.whl ; python_version=='3.9' and ('linux' in sys_platform)", + "xformers @ https://download.pytorch.org/whl/cu118/xformers-0.0.23.post1%2Bcu118-cp310-cp310-manylinux2014_x86_64.whl ; python_version=='3.10' and ('linux' in sys_platform)", + "xformers @ https://download.pytorch.org/whl/cu118/xformers-0.0.23.post1%2Bcu118-cp311-cp311-manylinux2014_x86_64.whl ; python_version=='3.11' and ('linux' in sys_platform)", ] cu121onlytorch212 = [ - "xformers==0.0.23.post1 ; ('linux' in sys_platform or sys_platform == 'win32') and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", + "xformers @ https://download.pytorch.org/whl/cu121/xformers-0.0.23.post1-cp39-cp39-manylinux2014_x86_64.whl ; python_version=='3.9' and ('linux' in sys_platform)", + "xformers @ https://download.pytorch.org/whl/cu121/xformers-0.0.23.post1-cp310-cp310-manylinux2014_x86_64.whl ; python_version=='3.10' and ('linux' in sys_platform)", + "xformers @ https://download.pytorch.org/whl/cu121/xformers-0.0.23.post1-cp311-cp311-manylinux2014_x86_64.whl ; python_version=='3.11' and ('linux' in sys_platform)", ] cu118onlytorch220 = [ - "xformers==0.0.24 ; ('linux' in sys_platform or sys_platform == 'win32') and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", + "xformers @ https://download.pytorch.org/whl/cu118/xformers-0.0.24%2Bcu118-cp39-cp39-manylinux2014_x86_64.whl ; python_version=='3.9' and ('linux' in sys_platform)", + "xformers @ https://download.pytorch.org/whl/cu118/xformers-0.0.24%2Bcu118-cp310-cp310-manylinux2014_x86_64.whl ; python_version=='3.10' and ('linux' in sys_platform)", + "xformers @ https://download.pytorch.org/whl/cu118/xformers-0.0.24%2Bcu118-cp311-cp311-manylinux2014_x86_64.whl ; python_version=='3.11' and ('linux' in sys_platform)", ] cu121onlytorch220 = [ - "xformers==0.0.24 ; ('linux' in sys_platform or sys_platform == 'win32') and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", + "xformers @ https://download.pytorch.org/whl/cu121/xformers-0.0.24-cp39-cp39-manylinux2014_x86_64.whl ; python_version=='3.9' and ('linux' in sys_platform)", + "xformers @ https://download.pytorch.org/whl/cu121/xformers-0.0.24-cp310-cp310-manylinux2014_x86_64.whl ; python_version=='3.10' and ('linux' in sys_platform)", + "xformers @ https://download.pytorch.org/whl/cu121/xformers-0.0.24-cp311-cp311-manylinux2014_x86_64.whl ; python_version=='3.11' and ('linux' in sys_platform)", ] cu118onlytorch230 = [ - "xformers==0.0.27 ; ('linux' in sys_platform or sys_platform == 'win32') and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", + "xformers @ https://download.pytorch.org/whl/cu118/xformers-0.0.27%2Bcu118-cp39-cp39-manylinux2014_x86_64.whl ; python_version=='3.9' and ('linux' in sys_platform)", + "xformers @ https://download.pytorch.org/whl/cu118/xformers-0.0.27%2Bcu118-cp310-cp310-manylinux2014_x86_64.whl ; python_version=='3.10' and ('linux' in sys_platform)", + "xformers @ https://download.pytorch.org/whl/cu118/xformers-0.0.27%2Bcu118-cp311-cp311-manylinux2014_x86_64.whl ; python_version=='3.11' and ('linux' in sys_platform)", + "xformers @ https://download.pytorch.org/whl/cu118/xformers-0.0.27%2Bcu118-cp312-cp312-manylinux2014_x86_64.whl ; python_version=='3.12' and ('linux' in sys_platform)", ] cu121onlytorch230 = [ - "xformers==0.0.27 ; ('linux' in sys_platform or sys_platform == 'win32') and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", + "xformers @ https://download.pytorch.org/whl/cu121/xformers-0.0.27-cp39-cp39-manylinux2014_x86_64.whl ; python_version=='3.9' and ('linux' in sys_platform)", + "xformers @ https://download.pytorch.org/whl/cu121/xformers-0.0.27-cp310-cp310-manylinux2014_x86_64.whl ; python_version=='3.10' and ('linux' in sys_platform)", + "xformers @ https://download.pytorch.org/whl/cu121/xformers-0.0.27-cp311-cp311-manylinux2014_x86_64.whl ; python_version=='3.11' and ('linux' in sys_platform)", + "xformers @ https://download.pytorch.org/whl/cu121/xformers-0.0.27-cp312-cp312-manylinux2014_x86_64.whl ; python_version=='3.12' and ('linux' in sys_platform)", ] cu118onlytorch240 = [ - "xformers==0.0.27.post2 ; ('linux' in sys_platform or sys_platform == 'win32') and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", + "xformers @ https://download.pytorch.org/whl/cu118/xformers-0.0.27.post2%2Bcu118-cp39-cp39-manylinux2014_x86_64.whl ; python_version=='3.9' and ('linux' in sys_platform)", + "xformers @ https://download.pytorch.org/whl/cu118/xformers-0.0.27.post2%2Bcu118-cp310-cp310-manylinux2014_x86_64.whl ; python_version=='3.10' and ('linux' in sys_platform)", + "xformers @ https://download.pytorch.org/whl/cu118/xformers-0.0.27.post2%2Bcu118-cp311-cp311-manylinux2014_x86_64.whl ; python_version=='3.11' and ('linux' in sys_platform)", + "xformers @ https://download.pytorch.org/whl/cu118/xformers-0.0.27.post2%2Bcu118-cp312-cp312-manylinux2014_x86_64.whl ; python_version=='3.12' and ('linux' in sys_platform)", ] cu121onlytorch240 = [ - "xformers==0.0.27.post2 ; ('linux' in sys_platform or sys_platform == 'win32') and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", + "xformers @ https://download.pytorch.org/whl/cu121/xformers-0.0.28.post1-cp39-cp39-manylinux_2_28_x86_64.whl ; python_version=='3.9' and ('linux' in sys_platform)", + "xformers @ https://download.pytorch.org/whl/cu121/xformers-0.0.28.post1-cp310-cp310-manylinux_2_28_x86_64.whl ; python_version=='3.10' and ('linux' in sys_platform)", + "xformers @ https://download.pytorch.org/whl/cu121/xformers-0.0.28.post1-cp311-cp311-manylinux_2_28_x86_64.whl ; python_version=='3.11' and ('linux' in sys_platform)", + "xformers @ https://download.pytorch.org/whl/cu121/xformers-0.0.28.post1-cp312-cp312-manylinux_2_28_x86_64.whl ; python_version=='3.12' and ('linux' in sys_platform)", ] cu124onlytorch240 = [ - "xformers==0.0.28.post1 ; ('linux' in sys_platform or sys_platform == 'win32') and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", + "xformers @ https://download.pytorch.org/whl/cu124/xformers-0.0.28.post1-cp39-cp39-manylinux_2_28_x86_64.whl ; python_version=='3.9' and ('linux' in sys_platform)", + "xformers @ https://download.pytorch.org/whl/cu124/xformers-0.0.28.post1-cp310-cp310-manylinux_2_28_x86_64.whl ; python_version=='3.10' and ('linux' in sys_platform)", + "xformers @ https://download.pytorch.org/whl/cu124/xformers-0.0.28.post1-cp311-cp311-manylinux_2_28_x86_64.whl ; python_version=='3.11' and ('linux' in sys_platform)", + "xformers @ https://download.pytorch.org/whl/cu124/xformers-0.0.28.post1-cp312-cp312-manylinux_2_28_x86_64.whl ; python_version=='3.12' and ('linux' in sys_platform)", + "xformers @ https://download.pytorch.org/whl/cu124/xformers-0.0.28.post1-cp39-cp39-win_amd64.whl ; python_version=='3.9' and (sys_platform == 'win32')", + "xformers @ https://download.pytorch.org/whl/cu124/xformers-0.0.28.post1-cp310-cp310-win_amd64.whl ; python_version=='3.10' and (sys_platform == 'win32')", + "xformers @ https://download.pytorch.org/whl/cu124/xformers-0.0.28.post1-cp311-cp311-win_amd64.whl ; python_version=='3.11' and (sys_platform == 'win32')", + "xformers @ https://download.pytorch.org/whl/cu124/xformers-0.0.28.post1-cp312-cp312-win_amd64.whl ; python_version=='3.12' and (sys_platform == 'win32')", ] cu118onlytorch250 = [ - "xformers==0.0.28.post2 ; ('linux' in sys_platform or sys_platform == 'win32') and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", + "xformers @ https://download.pytorch.org/whl/cu118/xformers-0.0.28.post2-cp39-cp39-manylinux_2_28_x86_64.whl ; python_version=='3.9' and ('linux' in sys_platform)", + "xformers @ https://download.pytorch.org/whl/cu118/xformers-0.0.28.post2-cp310-cp310-manylinux_2_28_x86_64.whl ; python_version=='3.10' and ('linux' in sys_platform)", + "xformers @ https://download.pytorch.org/whl/cu118/xformers-0.0.28.post2-cp311-cp311-manylinux_2_28_x86_64.whl ; python_version=='3.11' and ('linux' in sys_platform)", + "xformers @ https://download.pytorch.org/whl/cu118/xformers-0.0.28.post2-cp312-cp312-manylinux_2_28_x86_64.whl ; python_version=='3.12' and ('linux' in sys_platform)", ] cu121onlytorch250 = [ - "xformers==0.0.28.post2 ; ('linux' in sys_platform or sys_platform == 'win32') and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", + "xformers @ https://download.pytorch.org/whl/cu121/xformers-0.0.28.post2-cp39-cp39-manylinux_2_28_x86_64.whl ; python_version=='3.9' and ('linux' in sys_platform)", + "xformers @ https://download.pytorch.org/whl/cu121/xformers-0.0.28.post2-cp310-cp310-manylinux_2_28_x86_64.whl ; python_version=='3.10' and ('linux' in sys_platform)", + "xformers @ https://download.pytorch.org/whl/cu121/xformers-0.0.28.post2-cp311-cp311-manylinux_2_28_x86_64.whl ; python_version=='3.11' and ('linux' in sys_platform)", + "xformers @ https://download.pytorch.org/whl/cu121/xformers-0.0.28.post2-cp312-cp312-manylinux_2_28_x86_64.whl ; python_version=='3.12' and ('linux' in sys_platform)", ] cu124onlytorch250 = [ - "xformers==0.0.28.post2 ; ('linux' in sys_platform or sys_platform == 'win32') and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", + "xformers @ https://download.pytorch.org/whl/cu124/xformers-0.0.28.post2-cp39-cp39-manylinux_2_28_x86_64.whl ; python_version=='3.9' and ('linux' in sys_platform)", + "xformers @ https://download.pytorch.org/whl/cu124/xformers-0.0.28.post2-cp310-cp310-manylinux_2_28_x86_64.whl ; python_version=='3.10' and ('linux' in sys_platform)", + "xformers @ https://download.pytorch.org/whl/cu124/xformers-0.0.28.post2-cp311-cp311-manylinux_2_28_x86_64.whl ; python_version=='3.11' and ('linux' in sys_platform)", + "xformers @ https://download.pytorch.org/whl/cu124/xformers-0.0.28.post2-cp312-cp312-manylinux_2_28_x86_64.whl ; python_version=='3.12' and ('linux' in sys_platform)", + "xformers @ https://download.pytorch.org/whl/cu124/xformers-0.0.28.post2-cp39-cp39-win_amd64.whl ; python_version=='3.9' and (sys_platform == 'win32')", + "xformers @ https://download.pytorch.org/whl/cu124/xformers-0.0.28.post2-cp310-cp310-win_amd64.whl ; python_version=='3.10' and (sys_platform == 'win32')", + "xformers @ https://download.pytorch.org/whl/cu124/xformers-0.0.28.post2-cp311-cp311-win_amd64.whl ; python_version=='3.11' and (sys_platform == 'win32')", + "xformers @ https://download.pytorch.org/whl/cu124/xformers-0.0.28.post2-cp312-cp312-win_amd64.whl ; python_version=='3.12' and (sys_platform == 'win32')", ] cu118onlytorch251 = [ - "xformers==0.0.29.post1 ; ('linux' in sys_platform or sys_platform == 'win32') and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", + "xformers @ https://download.pytorch.org/whl/cu118/xformers-0.0.29.post1-cp39-cp39-manylinux_2_28_x86_64.whl ; python_version=='3.9' and ('linux' in sys_platform)", + "xformers @ https://download.pytorch.org/whl/cu118/xformers-0.0.29.post1-cp310-cp310-manylinux_2_28_x86_64.whl ; python_version=='3.10' and ('linux' in sys_platform)", + "xformers @ https://download.pytorch.org/whl/cu118/xformers-0.0.29.post1-cp311-cp311-manylinux_2_28_x86_64.whl ; python_version=='3.11' and ('linux' in sys_platform)", + "xformers @ https://download.pytorch.org/whl/cu118/xformers-0.0.29.post1-cp312-cp312-manylinux_2_28_x86_64.whl ; python_version=='3.12' and ('linux' in sys_platform)", ] cu121onlytorch251 = [ - "xformers==0.0.29.post1 ; ('linux' in sys_platform or sys_platform == 'win32') and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", + "xformers @ https://download.pytorch.org/whl/cu121/xformers-0.0.29.post1-cp39-cp39-manylinux_2_28_x86_64.whl ; python_version=='3.9' and ('linux' in sys_platform)", + "xformers @ https://download.pytorch.org/whl/cu121/xformers-0.0.29.post1-cp310-cp310-manylinux_2_28_x86_64.whl ; python_version=='3.10' and ('linux' in sys_platform)", + "xformers @ https://download.pytorch.org/whl/cu121/xformers-0.0.29.post1-cp311-cp311-manylinux_2_28_x86_64.whl ; python_version=='3.11' and ('linux' in sys_platform)", + "xformers @ https://download.pytorch.org/whl/cu121/xformers-0.0.29.post1-cp312-cp312-manylinux_2_28_x86_64.whl ; python_version=='3.12' and ('linux' in sys_platform)", ] cu124onlytorch251 = [ - "xformers==0.0.29.post1 ; ('linux' in sys_platform or sys_platform == 'win32') and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", + "xformers @ https://download.pytorch.org/whl/cu124/xformers-0.0.29.post1-cp39-cp39-manylinux_2_28_x86_64.whl ; python_version=='3.9' and ('linux' in sys_platform)", + "xformers @ https://download.pytorch.org/whl/cu124/xformers-0.0.29.post1-cp310-cp310-manylinux_2_28_x86_64.whl ; python_version=='3.10' and ('linux' in sys_platform)", + "xformers @ https://download.pytorch.org/whl/cu124/xformers-0.0.29.post1-cp311-cp311-manylinux_2_28_x86_64.whl ; python_version=='3.11' and ('linux' in sys_platform)", + "xformers @ https://download.pytorch.org/whl/cu124/xformers-0.0.29.post1-cp312-cp312-manylinux_2_28_x86_64.whl ; python_version=='3.12' and ('linux' in sys_platform)", + "xformers @ https://download.pytorch.org/whl/cu124/xformers-0.0.29.post1-cp39-cp39-win_amd64.whl ; python_version=='3.9' and (sys_platform == 'win32')", + "xformers @ https://download.pytorch.org/whl/cu124/xformers-0.0.29.post1-cp310-cp310-win_amd64.whl ; python_version=='3.10' and (sys_platform == 'win32')", + "xformers @ https://download.pytorch.org/whl/cu124/xformers-0.0.29.post1-cp311-cp311-win_amd64.whl ; python_version=='3.11' and (sys_platform == 'win32')", + "xformers @ https://download.pytorch.org/whl/cu124/xformers-0.0.29.post1-cp312-cp312-win_amd64.whl ; python_version=='3.12' and (sys_platform == 'win32')", ] cu118onlytorch260 = [ - "xformers==0.0.29.post3 ; ('linux' in sys_platform or sys_platform == 'win32') and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", + "xformers @ https://download.pytorch.org/whl/cu118/xformers-0.0.29.post3-cp39-cp39-manylinux_2_28_x86_64.whl ; python_version=='3.9' and ('linux' in sys_platform)", + "xformers @ https://download.pytorch.org/whl/cu118/xformers-0.0.29.post3-cp310-cp310-manylinux_2_28_x86_64.whl ; python_version=='3.10' and ('linux' in sys_platform)", + "xformers @ https://download.pytorch.org/whl/cu118/xformers-0.0.29.post3-cp311-cp311-manylinux_2_28_x86_64.whl ; python_version=='3.11' and ('linux' in sys_platform)", + "xformers @ https://download.pytorch.org/whl/cu118/xformers-0.0.29.post3-cp312-cp312-manylinux_2_28_x86_64.whl ; python_version=='3.12' and ('linux' in sys_platform)", ] cu124onlytorch260 = [ - "xformers==0.0.29.post3 ; ('linux' in sys_platform or sys_platform == 'win32') and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", + "xformers @ https://download.pytorch.org/whl/cu124/xformers-0.0.29.post3-cp39-cp39-manylinux_2_28_x86_64.whl ; python_version=='3.9' and ('linux' in sys_platform)", + "xformers @ https://download.pytorch.org/whl/cu124/xformers-0.0.29.post3-cp310-cp310-manylinux_2_28_x86_64.whl ; python_version=='3.10' and ('linux' in sys_platform)", + "xformers @ https://download.pytorch.org/whl/cu124/xformers-0.0.29.post3-cp311-cp311-manylinux_2_28_x86_64.whl ; python_version=='3.11' and ('linux' in sys_platform)", + "xformers @ https://download.pytorch.org/whl/cu124/xformers-0.0.29.post3-cp312-cp312-manylinux_2_28_x86_64.whl ; python_version=='3.12' and ('linux' in sys_platform)", + "xformers @ https://download.pytorch.org/whl/cu124/xformers-0.0.29.post3-cp39-cp39-win_amd64.whl ; python_version=='3.9' and (sys_platform == 'win32')", + "xformers @ https://download.pytorch.org/whl/cu124/xformers-0.0.29.post3-cp310-cp310-win_amd64.whl ; python_version=='3.10' and (sys_platform == 'win32')", + "xformers @ https://download.pytorch.org/whl/cu124/xformers-0.0.29.post3-cp311-cp311-win_amd64.whl ; python_version=='3.11' and (sys_platform == 'win32')", + "xformers @ https://download.pytorch.org/whl/cu124/xformers-0.0.29.post3-cp312-cp312-win_amd64.whl ; python_version=='3.12' and (sys_platform == 'win32')", ] cu126onlytorch260 = [ - "xformers==0.0.29.post3 ; ('linux' in sys_platform or sys_platform == 'win32') and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", + "xformers @ https://download.pytorch.org/whl/cu126/xformers-0.0.29.post3-cp39-cp39-manylinux_2_28_x86_64.whl ; python_version=='3.9' and ('linux' in sys_platform)", + "xformers @ https://download.pytorch.org/whl/cu126/xformers-0.0.29.post3-cp310-cp310-manylinux_2_28_x86_64.whl ; python_version=='3.10' and ('linux' in sys_platform)", + "xformers @ https://download.pytorch.org/whl/cu126/xformers-0.0.29.post3-cp311-cp311-manylinux_2_28_x86_64.whl ; python_version=='3.11' and ('linux' in sys_platform)", + "xformers @ https://download.pytorch.org/whl/cu126/xformers-0.0.29.post3-cp312-cp312-manylinux_2_28_x86_64.whl ; python_version=='3.12' and ('linux' in sys_platform)", + "xformers @ https://download.pytorch.org/whl/cu126/xformers-0.0.29.post3-cp39-cp39-win_amd64.whl ; python_version=='3.9' and (sys_platform == 'win32')", + "xformers @ https://download.pytorch.org/whl/cu126/xformers-0.0.29.post3-cp310-cp310-win_amd64.whl ; python_version=='3.10' and (sys_platform == 'win32')", + "xformers @ https://download.pytorch.org/whl/cu126/xformers-0.0.29.post3-cp311-cp311-win_amd64.whl ; python_version=='3.11' and (sys_platform == 'win32')", + "xformers @ https://download.pytorch.org/whl/cu126/xformers-0.0.29.post3-cp312-cp312-win_amd64.whl ; python_version=='3.12' and (sys_platform == 'win32')", ] cu118onlytorch270 = [ - "xformers==0.0.30 ; ('linux' in sys_platform or sys_platform == 'win32') and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", + "xformers @ https://download.pytorch.org/whl/cu118/xformers-0.0.30-cp39-cp39-manylinux_2_28_x86_64.whl ; python_version=='3.9' and ('linux' in sys_platform)", + "xformers @ https://download.pytorch.org/whl/cu118/xformers-0.0.30-cp310-cp310-manylinux_2_28_x86_64.whl ; python_version=='3.10' and ('linux' in sys_platform)", + "xformers @ https://download.pytorch.org/whl/cu118/xformers-0.0.30-cp311-cp311-manylinux_2_28_x86_64.whl ; python_version=='3.11' and ('linux' in sys_platform)", + "xformers @ https://download.pytorch.org/whl/cu118/xformers-0.0.30-cp312-cp312-manylinux_2_28_x86_64.whl ; python_version=='3.12' and ('linux' in sys_platform)", ] cu126onlytorch270 = [ - "xformers==0.0.30 ; ('linux' in sys_platform or sys_platform == 'win32') and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", + "xformers @ https://download.pytorch.org/whl/cu126/xformers-0.0.30-cp39-cp39-manylinux_2_28_x86_64.whl ; python_version=='3.9' and ('linux' in sys_platform)", + "xformers @ https://download.pytorch.org/whl/cu126/xformers-0.0.30-cp310-cp310-manylinux_2_28_x86_64.whl ; python_version=='3.10' and ('linux' in sys_platform)", + "xformers @ https://download.pytorch.org/whl/cu126/xformers-0.0.30-cp311-cp311-manylinux_2_28_x86_64.whl ; python_version=='3.11' and ('linux' in sys_platform)", + "xformers @ https://download.pytorch.org/whl/cu126/xformers-0.0.30-cp312-cp312-manylinux_2_28_x86_64.whl ; python_version=='3.12' and ('linux' in sys_platform)", + "xformers @ https://download.pytorch.org/whl/cu126/xformers-0.0.30-cp39-cp39-win_amd64.whl ; python_version=='3.9' and (sys_platform == 'win32')", + "xformers @ https://download.pytorch.org/whl/cu126/xformers-0.0.30-cp310-cp310-win_amd64.whl ; python_version=='3.10' and (sys_platform == 'win32')", + "xformers @ https://download.pytorch.org/whl/cu126/xformers-0.0.30-cp311-cp311-win_amd64.whl ; python_version=='3.11' and (sys_platform == 'win32')", + "xformers @ https://download.pytorch.org/whl/cu126/xformers-0.0.30-cp312-cp312-win_amd64.whl ; python_version=='3.12' and (sys_platform == 'win32')", ] cu128onlytorch270 = [ - "xformers==0.0.30 ; ('linux' in sys_platform or sys_platform == 'win32') and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", + "xformers @ https://download.pytorch.org/whl/cu128/xformers-0.0.30-cp39-cp39-manylinux_2_28_x86_64.whl ; python_version=='3.9' and ('linux' in sys_platform)", + "xformers @ https://download.pytorch.org/whl/cu128/xformers-0.0.30-cp310-cp310-manylinux_2_28_x86_64.whl ; python_version=='3.10' and ('linux' in sys_platform)", + "xformers @ https://download.pytorch.org/whl/cu128/xformers-0.0.30-cp311-cp311-manylinux_2_28_x86_64.whl ; python_version=='3.11' and ('linux' in sys_platform)", + "xformers @ https://download.pytorch.org/whl/cu128/xformers-0.0.30-cp312-cp312-manylinux_2_28_x86_64.whl ; python_version=='3.12' and ('linux' in sys_platform)", + "xformers @ https://download.pytorch.org/whl/cu128/xformers-0.0.30-cp39-cp39-win_amd64.whl ; python_version=='3.9' and (sys_platform == 'win32')", + "xformers @ https://download.pytorch.org/whl/cu128/xformers-0.0.30-cp310-cp310-win_amd64.whl ; python_version=='3.10' and (sys_platform == 'win32')", + "xformers @ https://download.pytorch.org/whl/cu128/xformers-0.0.30-cp311-cp311-win_amd64.whl ; python_version=='3.11' and (sys_platform == 'win32')", + "xformers @ https://download.pytorch.org/whl/cu128/xformers-0.0.30-cp312-cp312-win_amd64.whl ; python_version=='3.12' and (sys_platform == 'win32')", ] cu118onlytorch271 = [ - "xformers==0.0.31.post1 ; ('linux' in sys_platform or sys_platform == 'win32') and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", + "xformers @ https://download.pytorch.org/whl/cu118/xformers-0.0.31.post1-cp39-abi3-manylinux_2_28_x86_64.whl ; ('linux' in sys_platform)", ] cu126onlytorch271 = [ - "xformers==0.0.31.post1 ; ('linux' in sys_platform or sys_platform == 'win32') and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", + "xformers @ https://download.pytorch.org/whl/cu126/xformers-0.0.31.post1-cp39-abi3-manylinux_2_28_x86_64.whl ; ('linux' in sys_platform)", + "xformers @ https://download.pytorch.org/whl/cu126/xformers-0.0.31.post1-cp39-abi3-win_amd64.whl ; (sys_platform == 'win32')", ] cu128onlytorch271 = [ - "xformers==0.0.31.post1 ; ('linux' in sys_platform or sys_platform == 'win32') and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", + "xformers @ https://download.pytorch.org/whl/cu128/xformers-0.0.31.post1-cp39-abi3-manylinux_2_28_x86_64.whl ; ('linux' in sys_platform)", + "xformers @ https://download.pytorch.org/whl/cu128/xformers-0.0.31.post1-cp39-abi3-win_amd64.whl ; (sys_platform == 'win32')", ] cu118onlytorch280 = [ - "xformers==0.0.32.post2 ; ('linux' in sys_platform or sys_platform == 'win32') and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", + "xformers @ https://download.pytorch.org/whl/cu126/xformers-0.0.32.post2-cp39-abi3-manylinux_2_28_x86_64.whl ; ('linux' in sys_platform)", + "xformers @ https://download.pytorch.org/whl/cu126/xformers-0.0.32.post2-cp39-abi3-win_amd64.whl ; (sys_platform == 'win32')", ] cu126onlytorch280 = [ - "xformers==0.0.32.post2 ; ('linux' in sys_platform or sys_platform == 'win32') and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", + "xformers @ https://download.pytorch.org/whl/cu128/xformers-0.0.32.post2-cp39-abi3-manylinux_2_28_x86_64.whl ; ('linux' in sys_platform)", + "xformers @ https://download.pytorch.org/whl/cu128/xformers-0.0.32.post2-cp39-abi3-win_amd64.whl ; (sys_platform == 'win32')", ] cu128onlytorch280 = [ - "xformers==0.0.32.post2 ; ('linux' in sys_platform or sys_platform == 'win32') and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", + "xformers @ https://download.pytorch.org/whl/cu129/xformers-0.0.32.post2-cp39-abi3-manylinux_2_28_x86_64.whl ; ('linux' in sys_platform)", + "xformers @ https://download.pytorch.org/whl/cu129/xformers-0.0.32.post2-cp39-abi3-win_amd64.whl ; (sys_platform == 'win32')", ] cu130onlytorch280 = [ ] cu126onlytorch290 = [ - "xformers==0.0.33.post1 ; ('linux' in sys_platform or sys_platform == 'win32') and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", + "xformers @ https://download.pytorch.org/whl/cu126/xformers-0.0.33.post1-cp39-abi3-manylinux_2_28_x86_64.whl ; ('linux' in sys_platform)", + "xformers @ https://download.pytorch.org/whl/cu126/xformers-0.0.33.post1-cp39-abi3-win_amd64.whl ; (sys_platform == 'win32')", ] cu128onlytorch290 = [ - "xformers==0.0.33.post1 ; ('linux' in sys_platform or sys_platform == 'win32') and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", + "xformers @ https://download.pytorch.org/whl/cu128/xformers-0.0.33.post1-cp39-abi3-manylinux_2_28_x86_64.whl ; ('linux' in sys_platform)", + "xformers @ https://download.pytorch.org/whl/cu128/xformers-0.0.33.post1-cp39-abi3-win_amd64.whl ; (sys_platform == 'win32')", ] cu130onlytorch290 = [ - "xformers==0.0.33.post1 ; ('linux' in sys_platform or sys_platform == 'win32') and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", + "xformers @ https://download.pytorch.org/whl/cu130/xformers-0.0.33.post1-cp39-abi3-manylinux_2_28_x86_64.whl ; ('linux' in sys_platform)", + "xformers @ https://download.pytorch.org/whl/cu130/xformers-0.0.33.post1-cp39-abi3-win_amd64.whl ; (sys_platform == 'win32')", ] cu126onlytorch291 = [ - "xformers==0.0.33.post2 ; ('linux' in sys_platform or sys_platform == 'win32') and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", + "xformers @ https://download.pytorch.org/whl/cu126/xformers-0.0.33.post2-cp39-abi3-manylinux_2_28_x86_64.whl ; ('linux' in sys_platform)", + "xformers @ https://download.pytorch.org/whl/cu126/xformers-0.0.33.post2-cp39-abi3-win_amd64.whl ; (sys_platform == 'win32')", ] cu128onlytorch291 = [ - "xformers==0.0.33.post2 ; ('linux' in sys_platform or sys_platform == 'win32') and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", + "xformers @ https://download.pytorch.org/whl/cu128/xformers-0.0.33.post2-cp39-abi3-manylinux_2_28_x86_64.whl ; ('linux' in sys_platform)", + "xformers @ https://download.pytorch.org/whl/cu128/xformers-0.0.33.post2-cp39-abi3-win_amd64.whl ; (sys_platform == 'win32')", ] cu130onlytorch291 = [ - "xformers==0.0.33.post2 ; ('linux' in sys_platform or sys_platform == 'win32') and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", + "xformers @ https://download.pytorch.org/whl/cu130/xformers-0.0.33.post2-cp39-abi3-manylinux_2_28_x86_64.whl ; ('linux' in sys_platform)", + "xformers @ https://download.pytorch.org/whl/cu130/xformers-0.0.33.post2-cp39-abi3-win_amd64.whl ; (sys_platform == 'win32')", ] cu126onlytorch2100 = [ - "xformers==0.0.34 ; ('linux' in sys_platform or sys_platform == 'win32') and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", + "xformers @ https://download.pytorch.org/whl/cu126/xformers-0.0.34-cp39-abi3-manylinux_2_28_x86_64.whl ; ('linux' in sys_platform)", + "xformers @ https://download.pytorch.org/whl/cu126/xformers-0.0.34-cp39-abi3-win_amd64.whl ; (sys_platform == 'win32')", ] cu128onlytorch2100 = [ - "xformers==0.0.34 ; ('linux' in sys_platform or sys_platform == 'win32') and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", + "xformers @ https://download.pytorch.org/whl/cu128/xformers-0.0.34-cp39-abi3-manylinux_2_28_x86_64.whl ; ('linux' in sys_platform)", + "xformers @ https://download.pytorch.org/whl/cu128/xformers-0.0.34-cp39-abi3-win_amd64.whl ; (sys_platform == 'win32')", ] cu130onlytorch2100 = [ - "xformers==0.0.34 ; ('linux' in sys_platform or sys_platform == 'win32') and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", + "xformers @ https://download.pytorch.org/whl/cu130/xformers-0.0.34-cp39-abi3-manylinux_2_28_x86_64.whl ; ('linux' in sys_platform)", + "xformers @ https://download.pytorch.org/whl/cu130/xformers-0.0.34-cp39-abi3-win_amd64.whl ; (sys_platform == 'win32')", ] cu118 = [ "unsloth[huggingface]", @@ -295,22 +404,22 @@ cu121 = [ ] cu118-torch211 = [ "unsloth[huggingface]", - "bitsandbytes>=0.45.5,!=0.46.0,!=0.48.0", + "bitsandbytes==0.45.5", "unsloth[cu118onlytorch211]", ] cu121-torch211 = [ "unsloth[huggingface]", - "bitsandbytes>=0.45.5,!=0.46.0,!=0.48.0", + "bitsandbytes==0.45.5", "unsloth[cu121onlytorch211]", ] cu118-torch212 = [ "unsloth[huggingface]", - "bitsandbytes>=0.45.5,!=0.46.0,!=0.48.0", + "bitsandbytes==0.45.5", "unsloth[cu118onlytorch212]", ] cu121-torch212 = [ "unsloth[huggingface]", - "bitsandbytes>=0.45.5,!=0.46.0,!=0.48.0", + "bitsandbytes==0.45.5", "unsloth[cu121onlytorch212]", ] cu118-torch220 = [ @@ -380,17 +489,17 @@ cu124-torch251 = [ ] cu118-torch260 = [ "unsloth[huggingface]", - "bitsandbytes>=0.45.1", + "bitsandbytes>=0.45.5,!=0.46.0,!=0.48.0", "unsloth[cu118onlytorch260]", ] cu124-torch260 = [ "unsloth[huggingface]", - "bitsandbytes>=0.45.1", + "bitsandbytes>=0.45.5,!=0.46.0,!=0.48.0", "unsloth[cu124onlytorch260]", ] cu126-torch260 = [ "unsloth[huggingface]", - "bitsandbytes>=0.45.1", + "bitsandbytes>=0.45.5,!=0.46.0,!=0.48.0", "unsloth[cu126onlytorch260]", ] cu118-torch270 = [ @@ -477,16 +586,19 @@ cu126-torch2100 = [ "unsloth[huggingface]", "bitsandbytes>=0.45.5,!=0.46.0,!=0.48.0", "unsloth[cu126onlytorch2100]", + "unsloth[audio-torch210]", ] cu128-torch2100 = [ "unsloth[huggingface]", "bitsandbytes>=0.45.5,!=0.46.0,!=0.48.0", "unsloth[cu128onlytorch2100]", + "unsloth[audio-torch210]", ] cu130-torch2100 = [ "unsloth[huggingface]", "bitsandbytes>=0.45.5,!=0.46.0,!=0.48.0", "unsloth[cu130onlytorch2100]", + "unsloth[audio-torch210]", ] kaggle = [ "unsloth[huggingface]", @@ -500,19 +612,16 @@ conda = [ ] colab-torch211 = [ "unsloth[huggingface]", - "bitsandbytes>=0.45.5,!=0.46.0,!=0.48.0", + "bitsandbytes==0.45.5", "unsloth[cu121onlytorch211]", ] -flashattention = [ - "packaging ; ('linux' in sys_platform)", - "ninja ; ('linux' in sys_platform)", - "flash-attn>=2.6.3 ; ('linux' in sys_platform)", -] colab-ampere-torch211 = [ "unsloth[huggingface]", - "bitsandbytes>=0.45.5,!=0.46.0,!=0.48.0", + "bitsandbytes==0.45.5", "unsloth[cu121onlytorch211]", - "unsloth[flashattention]", + "packaging", + "ninja", + "flash-attn>=2.6.3 ; ('linux' in sys_platform)", ] colab-torch220 = [ "unsloth[huggingface]", @@ -523,7 +632,9 @@ colab-ampere-torch220 = [ "unsloth[huggingface]", "bitsandbytes>=0.45.5,!=0.46.0,!=0.48.0", "unsloth[cu121onlytorch220]", - "unsloth[flashattention]", + "packaging", + "ninja", + "flash-attn>=2.6.3 ; ('linux' in sys_platform)", ] colab-new = [ "unsloth_zoo>=2026.7.6", @@ -542,10 +653,6 @@ colab-new = [ "bitsandbytes>=0.45.5,!=0.46.0,!=0.48.0", "unsloth[triton]", "sentence-transformers", - "typer>=0.12.0", - "pydantic", - "pyyaml", - "nest-asyncio", ] colab-no-deps = [ "accelerate>=0.34.1", @@ -558,6 +665,11 @@ colab-no-deps = [ colab = [ "unsloth[cu121]", ] +flashattention = [ + "packaging ; ('linux' in sys_platform)", + "ninja ; ('linux' in sys_platform)", + "flash-attn>=2.6.3 ; ('linux' in sys_platform)", +] colab-ampere = [ "unsloth[colab-ampere-torch220]", "unsloth[flashattention]", @@ -576,13 +688,13 @@ cu121-ampere = [ ] cu118-ampere-torch211 = [ "unsloth[huggingface]", - "bitsandbytes>=0.45.5,!=0.46.0,!=0.48.0", + "bitsandbytes==0.45.5", "unsloth[cu118onlytorch211]", "unsloth[flashattention]", ] cu121-ampere-torch211 = [ "unsloth[huggingface]", - "bitsandbytes>=0.45.5,!=0.46.0,!=0.48.0", + "bitsandbytes==0.45.5", "unsloth[cu121onlytorch211]", "unsloth[flashattention]", ] @@ -666,19 +778,19 @@ cu124-ampere-torch251 = [ ] cu118-ampere-torch260 = [ "unsloth[huggingface]", - "bitsandbytes>=0.45.1", + "bitsandbytes>=0.45.5,!=0.46.0,!=0.48.0", "unsloth[cu118onlytorch260]", "unsloth[flashattention]", ] cu124-ampere-torch260 = [ "unsloth[huggingface]", - "bitsandbytes>=0.45.1", + "bitsandbytes>=0.45.5,!=0.46.0,!=0.48.0", "unsloth[cu124onlytorch260]", "unsloth[flashattention]", ] cu126-ampere-torch260 = [ "unsloth[huggingface]", - "bitsandbytes>=0.45.1", + "bitsandbytes>=0.45.5,!=0.46.0,!=0.48.0", "unsloth[cu126onlytorch260]", "unsloth[flashattention]", ] @@ -740,6 +852,7 @@ cu130-ampere-torch280 = [ "unsloth[huggingface]", "bitsandbytes>=0.45.5,!=0.46.0,!=0.48.0", "unsloth[cu130onlytorch280]", + "unsloth[flashattention]", ] cu126-ampere-torch290 = [ "unsloth[huggingface]", @@ -775,16 +888,481 @@ cu126-ampere-torch2100 = [ "unsloth[huggingface]", "bitsandbytes>=0.45.5,!=0.46.0,!=0.48.0", "unsloth[cu126onlytorch2100]", + "unsloth[audio-torch210]", ] cu128-ampere-torch2100 = [ "unsloth[huggingface]", "bitsandbytes>=0.45.5,!=0.46.0,!=0.48.0", "unsloth[cu128onlytorch2100]", + "unsloth[audio-torch210]", ] cu130-ampere-torch2100 = [ "unsloth[huggingface]", "bitsandbytes>=0.45.5,!=0.46.0,!=0.48.0", "unsloth[cu130onlytorch2100]", + "unsloth[audio-torch210]", +] +flashattentiontorch260abiFALSEcu12x = [ + "flash-attn @ https://github.com/Dao-AILab/flash-attention/releases/download/v2.7.4.post1/flash_attn-2.7.4.post1+cu12torch2.6cxx11abiFALSE-cp39-cp39-linux_x86_64.whl ; ('linux' in sys_platform) and python_version == '3.9'", + "flash-attn @ https://github.com/Dao-AILab/flash-attention/releases/download/v2.7.4.post1/flash_attn-2.7.4.post1+cu12torch2.6cxx11abiFALSE-cp310-cp310-linux_x86_64.whl ; ('linux' in sys_platform) and python_version == '3.10'", + "flash-attn @ https://github.com/Dao-AILab/flash-attention/releases/download/v2.7.4.post1/flash_attn-2.7.4.post1+cu12torch2.6cxx11abiFALSE-cp311-cp311-linux_x86_64.whl ; ('linux' in sys_platform) and python_version == '3.11'", + "flash-attn @ https://github.com/Dao-AILab/flash-attention/releases/download/v2.7.4.post1/flash_attn-2.7.4.post1+cu12torch2.6cxx11abiFALSE-cp312-cp312-linux_x86_64.whl ; ('linux' in sys_platform) and python_version == '3.12'", + "flash-attn @ https://github.com/Dao-AILab/flash-attention/releases/download/v2.7.4.post1/flash_attn-2.7.4.post1+cu12torch2.6cxx11abiFALSE-cp313-cp313-linux_x86_64.whl ; ('linux' in sys_platform) and python_version == '3.13'", +] +flashattentiontorch260abiTRUEcu12x = [ + "flash-attn @ https://github.com/Dao-AILab/flash-attention/releases/download/v2.7.4.post1/flash_attn-2.7.4.post1+cu12torch2.6cxx11abiTRUE-cp39-cp39-linux_x86_64.whl ; ('linux' in sys_platform) and python_version == '3.9'", + "flash-attn @ https://github.com/Dao-AILab/flash-attention/releases/download/v2.7.4.post1/flash_attn-2.7.4.post1+cu12torch2.6cxx11abiTRUE-cp310-cp310-linux_x86_64.whl ; ('linux' in sys_platform) and python_version == '3.10'", + "flash-attn @ https://github.com/Dao-AILab/flash-attention/releases/download/v2.7.4.post1/flash_attn-2.7.4.post1+cu12torch2.6cxx11abiTRUE-cp311-cp311-linux_x86_64.whl ; ('linux' in sys_platform) and python_version == '3.11'", + "flash-attn @ https://github.com/Dao-AILab/flash-attention/releases/download/v2.7.4.post1/flash_attn-2.7.4.post1+cu12torch2.6cxx11abiTRUE-cp312-cp312-linux_x86_64.whl ; ('linux' in sys_platform) and python_version == '3.12'", + "flash-attn @ https://github.com/Dao-AILab/flash-attention/releases/download/v2.7.4.post1/flash_attn-2.7.4.post1+cu12torch2.6cxx11abiTRUE-cp313-cp313-linux_x86_64.whl ; ('linux' in sys_platform) and python_version == '3.13'", +] +flashattentiontorch250abiFALSEcu12x = [ + "flash-attn @ https://github.com/Dao-AILab/flash-attention/releases/download/v2.7.4.post1/flash_attn-2.7.4.post1+cu12torch2.5cxx11abiFALSE-cp39-cp39-linux_x86_64.whl ; ('linux' in sys_platform) and python_version == '3.9'", + "flash-attn @ https://github.com/Dao-AILab/flash-attention/releases/download/v2.7.4.post1/flash_attn-2.7.4.post1+cu12torch2.5cxx11abiFALSE-cp310-cp310-linux_x86_64.whl ; ('linux' in sys_platform) and python_version == '3.10'", + "flash-attn @ https://github.com/Dao-AILab/flash-attention/releases/download/v2.7.4.post1/flash_attn-2.7.4.post1+cu12torch2.5cxx11abiFALSE-cp311-cp311-linux_x86_64.whl ; ('linux' in sys_platform) and python_version == '3.11'", + "flash-attn @ https://github.com/Dao-AILab/flash-attention/releases/download/v2.7.4.post1/flash_attn-2.7.4.post1+cu12torch2.5cxx11abiFALSE-cp312-cp312-linux_x86_64.whl ; ('linux' in sys_platform) and python_version == '3.12'", + "flash-attn @ https://github.com/Dao-AILab/flash-attention/releases/download/v2.7.4.post1/flash_attn-2.7.4.post1+cu12torch2.5cxx11abiFALSE-cp313-cp313-linux_x86_64.whl ; ('linux' in sys_platform) and python_version == '3.13'", +] +flashattentiontorch250abiTRUEcu12x = [ + "flash-attn @ https://github.com/Dao-AILab/flash-attention/releases/download/v2.7.4.post1/flash_attn-2.7.4.post1+cu12torch2.5cxx11abiTRUE-cp39-cp39-linux_x86_64.whl ; ('linux' in sys_platform) and python_version == '3.9'", + "flash-attn @ https://github.com/Dao-AILab/flash-attention/releases/download/v2.7.4.post1/flash_attn-2.7.4.post1+cu12torch2.5cxx11abiTRUE-cp310-cp310-linux_x86_64.whl ; ('linux' in sys_platform) and python_version == '3.10'", + "flash-attn @ https://github.com/Dao-AILab/flash-attention/releases/download/v2.7.4.post1/flash_attn-2.7.4.post1+cu12torch2.5cxx11abiTRUE-cp311-cp311-linux_x86_64.whl ; ('linux' in sys_platform) and python_version == '3.11'", + "flash-attn @ https://github.com/Dao-AILab/flash-attention/releases/download/v2.7.4.post1/flash_attn-2.7.4.post1+cu12torch2.5cxx11abiTRUE-cp312-cp312-linux_x86_64.whl ; ('linux' in sys_platform) and python_version == '3.12'", + "flash-attn @ https://github.com/Dao-AILab/flash-attention/releases/download/v2.7.4.post1/flash_attn-2.7.4.post1+cu12torch2.5cxx11abiTRUE-cp313-cp313-linux_x86_64.whl ; ('linux' in sys_platform) and python_version == '3.13'", +] +flashattentiontorch240abiFALSEcu12x = [ + "flash-attn @ https://github.com/Dao-AILab/flash-attention/releases/download/v2.7.4.post1/flash_attn-2.7.4.post1+cu12torch2.4cxx11abiFALSE-cp39-cp39-linux_x86_64.whl ; ('linux' in sys_platform) and python_version == '3.9'", + "flash-attn @ https://github.com/Dao-AILab/flash-attention/releases/download/v2.7.4.post1/flash_attn-2.7.4.post1+cu12torch2.4cxx11abiFALSE-cp310-cp310-linux_x86_64.whl ; ('linux' in sys_platform) and python_version == '3.10'", + "flash-attn @ https://github.com/Dao-AILab/flash-attention/releases/download/v2.7.4.post1/flash_attn-2.7.4.post1+cu12torch2.4cxx11abiFALSE-cp311-cp311-linux_x86_64.whl ; ('linux' in sys_platform) and python_version == '3.11'", + "flash-attn @ https://github.com/Dao-AILab/flash-attention/releases/download/v2.7.4.post1/flash_attn-2.7.4.post1+cu12torch2.4cxx11abiFALSE-cp312-cp312-linux_x86_64.whl ; ('linux' in sys_platform) and python_version == '3.12'", +] +flashattentiontorch240abiTRUEcu12x = [ + "flash-attn @ https://github.com/Dao-AILab/flash-attention/releases/download/v2.7.4.post1/flash_attn-2.7.4.post1+cu12torch2.4cxx11abiTRUE-cp39-cp39-linux_x86_64.whl ; ('linux' in sys_platform) and python_version == '3.9'", + "flash-attn @ https://github.com/Dao-AILab/flash-attention/releases/download/v2.7.4.post1/flash_attn-2.7.4.post1+cu12torch2.4cxx11abiTRUE-cp310-cp310-linux_x86_64.whl ; ('linux' in sys_platform) and python_version == '3.10'", + "flash-attn @ https://github.com/Dao-AILab/flash-attention/releases/download/v2.7.4.post1/flash_attn-2.7.4.post1+cu12torch2.4cxx11abiTRUE-cp311-cp311-linux_x86_64.whl ; ('linux' in sys_platform) and python_version == '3.11'", + "flash-attn @ https://github.com/Dao-AILab/flash-attention/releases/download/v2.7.4.post1/flash_attn-2.7.4.post1+cu12torch2.4cxx11abiTRUE-cp312-cp312-linux_x86_64.whl ; ('linux' in sys_platform) and python_version == '3.12'", +] +intelgputorch260 = [ + "unsloth_zoo[intelgpu]", + "unsloth[huggingfacenotorch]", + + "pytorch_triton_xpu @ https://download.pytorch.org/whl/pytorch_triton_xpu-3.2.0-cp39-cp39-linux_x86_64.whl#sha256=147607f190a7d7aa24ba454def5977fbbfec792fdae18e4ed278cfec29b69271 ; ('linux' in sys_platform) and python_version == '3.9' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", + "pytorch_triton_xpu @ https://download.pytorch.org/whl/pytorch_triton_xpu-3.2.0-cp310-cp310-linux_x86_64.whl#sha256=23aa423fa1542afc34f67eb3ba8ef20060f6d1b3a4697eaeab22b11c92b30f2b ; ('linux' in sys_platform) and python_version == '3.10' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", + "pytorch_triton_xpu @ https://download.pytorch.org/whl/pytorch_triton_xpu-3.2.0-cp311-cp311-linux_x86_64.whl#sha256=bcfa995229bbfd9ffd8d6c8d9f6428d393e876fa6e23ee3c20e3c0d73ca75ca5 ; ('linux' in sys_platform) and python_version == '3.11' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", + "pytorch_triton_xpu @ https://download.pytorch.org/whl/pytorch_triton_xpu-3.2.0-cp312-cp312-linux_x86_64.whl#sha256=bd340903d03470708df3442438acb8b7e08087ab9e61fbe349b2872bf9257ab0 ; ('linux' in sys_platform) and python_version == '3.12' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", + "pytorch_triton_xpu @ https://download.pytorch.org/whl/pytorch_triton_xpu-3.2.0-cp313-cp313-linux_x86_64.whl#sha256=814dccc8a07159e6eca74bed70091bc8fea2d9dd87b0d91845f9f38cde62f01c ; ('linux' in sys_platform) and python_version == '3.13' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", + + "bitsandbytes @ https://github.com/bitsandbytes-foundation/bitsandbytes/releases/download/continuous-release_main/bitsandbytes-1.33.7.preview-py3-none-manylinux_2_24_x86_64.whl ; ('linux' in sys_platform) and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", + "bitsandbytes @ https://github.com/bitsandbytes-foundation/bitsandbytes/releases/download/continuous-release_main/bitsandbytes-1.33.7.preview-py3-none-win_amd64.whl ; (sys_platform == 'win32') and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", + + "torch @ https://download.pytorch.org/whl/xpu/torch-2.6.0%2Bxpu-cp39-cp39-linux_x86_64.whl#sha256=6a8adf6dc4c089406e8b3a7e58ab57a463bddf9b07130d2576e76eced43e92af ; ('linux' in sys_platform) and python_version == '3.9' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", + "torch @ https://download.pytorch.org/whl/xpu/torch-2.6.0%2Bxpu-cp310-cp310-linux_x86_64.whl#sha256=ff4561cbf07c83bbccaa0f6e9bb0e6dcf721bacd53c9c43c4eb0e7331b4792f9 ; ('linux' in sys_platform) and python_version == '3.10' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", + "torch @ https://download.pytorch.org/whl/xpu/torch-2.6.0%2Bxpu-cp311-cp311-linux_x86_64.whl#sha256=12005f66b810ddd3ab93f86c4522bcfdd412cbd27fc9d189b661ff7509bc5e8a ; ('linux' in sys_platform) and python_version == '3.11' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", + "torch @ https://download.pytorch.org/whl/xpu/torch-2.6.0%2Bxpu-cp312-cp312-linux_x86_64.whl#sha256=c4c5c67625cdacf35765c2b94e61fe166e3c3f4a14521b1212a59ad1b3eb0f2e ; ('linux' in sys_platform) and python_version == '3.12' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", + "torch @ https://download.pytorch.org/whl/xpu/torch-2.6.0%2Bxpu-cp313-cp313-linux_x86_64.whl#sha256=e6864f7a60a5ecc43d5d38f59a16e5dd132384f73dfd3a697f74944026038f7b ; ('linux' in sys_platform) and python_version == '3.13' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", +] +intel-gpu-torch260 = [ + "unsloth[intelgputorch260]" +] +intelgputorch270 = [ + "unsloth_zoo[intelgpu]", + "unsloth[huggingfacenotorch]", + + "pytorch_triton_xpu @ https://download.pytorch.org/whl/pytorch_triton_xpu-3.3.0-cp39-cp39-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl#sha256=749a7098492c6a27b356c97149a4a62973b953eae60bc1b6259260974f344913 ; ('linux' in sys_platform) and python_version == '3.9' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", + "pytorch_triton_xpu @ https://download.pytorch.org/whl/pytorch_triton_xpu-3.3.0-cp310-cp310-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl#sha256=44362e80abd752471a08341093321955b066daa2cfb4810e73b8e3b240850f93 ; ('linux' in sys_platform) and python_version == '3.10' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", + "pytorch_triton_xpu @ https://download.pytorch.org/whl/pytorch_triton_xpu-3.3.0-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl#sha256=faa6b8c945a837a080f641bc8ccc77a98fa66980dcd7e62e715fd853737343fd ; ('linux' in sys_platform) and python_version == '3.11' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", + "pytorch_triton_xpu @ https://download.pytorch.org/whl/pytorch_triton_xpu-3.3.0-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl#sha256=40f6fb65b345dc9a61813abe7ac9a585f2c9808f414d140cc2a5f11f53ee063c ; ('linux' in sys_platform) and python_version == '3.12' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", + "pytorch_triton_xpu @ https://download.pytorch.org/whl/pytorch_triton_xpu-3.3.0-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl#sha256=b22b4c02ec71b4bfc862ae3cdfd2871dc0b05d2b1802f5db2196e0f897d581e9 ; ('linux' in sys_platform) and python_version == '3.13' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", + "pytorch_triton_xpu @ https://download.pytorch.org/whl/pytorch_triton_xpu-3.3.0-cp39-cp39-win_amd64.whl#sha256=d4b738d7fa5100c1bd766f91614962828a4810eb57b4df92cd5214a83505a752 ; sys_platform == 'win32' and python_version == '3.9' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", + "pytorch_triton_xpu @ https://download.pytorch.org/whl/pytorch_triton_xpu-3.3.0-cp310-cp310-win_amd64.whl#sha256=143fe8a64d807bcdb7d81bbc062816add325570aa160448454ab6ded4a0a17a1 ; sys_platform == 'win32' and python_version == '3.10' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", + "pytorch_triton_xpu @ https://download.pytorch.org/whl/pytorch_triton_xpu-3.3.0-cp311-cp311-win_amd64.whl#sha256=a8025459ff325d6e3532eb5cf72519db1b178155e7d60aff6c56beb5968fc758 ; sys_platform == 'win32' and python_version == '3.11' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", + "pytorch_triton_xpu @ https://download.pytorch.org/whl/pytorch_triton_xpu-3.3.0-cp312-cp312-win_amd64.whl#sha256=0dd07e6d5b872e42e48f5ee140e609d4554ca3cc509d5bf509ac232267cf358e ; sys_platform == 'win32' and python_version == '3.12' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", + "pytorch_triton_xpu @ https://download.pytorch.org/whl/pytorch_triton_xpu-3.3.0-cp313-cp313-win_amd64.whl#sha256=a936a18182d8e065a9933afc9a3ebbffadd38604969f87c493831214539fc027 ; sys_platform == 'win32' and python_version == '3.13' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", + + "bitsandbytes @ https://github.com/bitsandbytes-foundation/bitsandbytes/releases/download/continuous-release_main/bitsandbytes-1.33.7.preview-py3-none-manylinux_2_24_x86_64.whl ; ('linux' in sys_platform) and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", + "bitsandbytes @ https://github.com/bitsandbytes-foundation/bitsandbytes/releases/download/continuous-release_main/bitsandbytes-1.33.7.preview-py3-none-win_amd64.whl ; (sys_platform == 'win32') and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", + + "torch @ https://download.pytorch.org/whl/xpu/torch-2.7.0%2Bxpu-cp39-cp39-linux_x86_64.whl#sha256=f8ee75e50fcbb37ed5b498299ca2264da99ab278a93fae2358e921e4a6e28273 ; ('linux' in sys_platform) and python_version == '3.9' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", + "torch @ https://download.pytorch.org/whl/xpu/torch-2.7.0%2Bxpu-cp310-cp310-linux_x86_64.whl#sha256=d6fdc342961d98fdcd9d03dfd491a3208bb5f7fbb435841f8f72ce9fdcd2d026 ; ('linux' in sys_platform) and python_version == '3.10' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", + "torch @ https://download.pytorch.org/whl/xpu/torch-2.7.0%2Bxpu-cp311-cp311-linux_x86_64.whl#sha256=74d07f9357df5cf2bf223ad3c84de16346bfaa0504f988fdd5590d3e177e5e86 ; ('linux' in sys_platform) and python_version == '3.11' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", + "torch @ https://download.pytorch.org/whl/xpu/torch-2.7.0%2Bxpu-cp312-cp312-linux_x86_64.whl#sha256=c806d44aa2ca5d225629f6fbc6c994d5deaac2d2cde449195bc8e3522ddd219a ; ('linux' in sys_platform) and python_version == '3.12' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", + "torch @ https://download.pytorch.org/whl/xpu/torch-2.7.0%2Bxpu-cp313-cp313-linux_x86_64.whl#sha256=25d8277b7f01d42e2e014ccbab57a2692b6ec4eff8dcf894eda1b297407cf97a ; ('linux' in sys_platform) and python_version == '3.13' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", + "torch @ https://download.pytorch.org/whl/xpu/torch-2.7.0%2Bxpu-cp39-cp39-win_amd64.whl#sha256=046e85125266ae69c1a0d083e6c092f947ab4b6b41532c16bafe40dbced845df ; sys_platform == 'win32' and python_version == '3.9' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", + "torch @ https://download.pytorch.org/whl/xpu/torch-2.7.0%2Bxpu-cp310-cp310-win_amd64.whl#sha256=9ebaeffb82b0b3e39b6030927d3ebe0eb62a0e9045a3b2d7b0a9e7b15222c0db ; sys_platform == 'win32' and python_version == '3.10' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", + "torch @ https://download.pytorch.org/whl/xpu/torch-2.7.0%2Bxpu-cp311-cp311-win_amd64.whl#sha256=356ba66cee127e7e2c942880bd50e03768306a4ea08d358a0f29c6eebfc4bc81 ; sys_platform == 'win32' and python_version == '3.11' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", + "torch @ https://download.pytorch.org/whl/xpu/torch-2.7.0%2Bxpu-cp312-cp312-win_amd64.whl#sha256=94739e665d9b4d5cd7af5f517cb6103f6f9fb421c095184609653a24524040f5 ; sys_platform == 'win32' and python_version == '3.12' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", + "torch @ https://download.pytorch.org/whl/xpu/torch-2.7.0%2Bxpu-cp313-cp313-win_amd64.whl#sha256=31df3cb674918e89bc8c532baa331dc84f4430e1f9c0ec379232db44cba78355 ; sys_platform == 'win32' and python_version == '3.13' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", +] +intel-gpu-torch270 = [ + "unsloth[intelgputorch270]" +] +intelgputorch280 = [ + "unsloth_zoo[intelgpu]", + "unsloth[huggingfacenotorch]", + + "pytorch_triton_xpu @ https://download.pytorch.org/whl/pytorch_triton_xpu-3.4.0-cp39-cp39-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl#sha256=ac4d8e33986b1c3c5e48151640539272b2187e83016985853111b46fb82c3c94 ; 'linux' in sys_platform and python_version == '3.9' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", + "pytorch_triton_xpu @ https://download.pytorch.org/whl/pytorch_triton_xpu-3.4.0-cp310-cp310-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl#sha256=999fef4c1f711092b9d3086525920545df490de476ecebe899ffc777019ae17f ; 'linux' in sys_platform and python_version == '3.10' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", + "pytorch_triton_xpu @ https://download.pytorch.org/whl/pytorch_triton_xpu-3.4.0-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl#sha256=57b09c8c492985ff6a27cd3a22b08e8f7b96b407bd8030967b6efbb9f63b80cf ; 'linux' in sys_platform and python_version == '3.11' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", + "pytorch_triton_xpu @ https://download.pytorch.org/whl/pytorch_triton_xpu-3.4.0-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl#sha256=df4bb3282bac9a3b90231700077110d8680b338416de03c2b7c6133c9b602649 ; 'linux' in sys_platform and python_version == '3.12' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", + "pytorch_triton_xpu @ https://download.pytorch.org/whl/pytorch_triton_xpu-3.4.0-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl#sha256=60da63c99ca827bdcb0df28e0298bf7d066dc607454c6d6176783cb4e79d838b ; 'linux' in sys_platform and python_version == '3.13' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", + "pytorch_triton_xpu @ https://download.pytorch.org/whl/pytorch_triton_xpu-3.4.0-cp39-cp39-win_amd64.whl#sha256=64aea8de349f3e2e0ebf4c24b011a8122531fdffda5776edaef45829cc241cf8 ; sys_platform == 'win32' and python_version == '3.9' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", + "pytorch_triton_xpu @ https://download.pytorch.org/whl/pytorch_triton_xpu-3.4.0-cp310-cp310-win_amd64.whl#sha256=ae573d255b257fdbed319a3440dc9d0a721e31160ab7f6eba1b2226e6a409a1d ; sys_platform == 'win32' and python_version == '3.10' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", + "pytorch_triton_xpu @ https://download.pytorch.org/whl/pytorch_triton_xpu-3.4.0-cp311-cp311-win_amd64.whl#sha256=8e0ea4558e5776d8ddab0264310be9b26aee5641bcac0da023537556d4317b86 ; sys_platform == 'win32' and python_version == '3.11' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", + "pytorch_triton_xpu @ https://download.pytorch.org/whl/pytorch_triton_xpu-3.4.0-cp312-cp312-win_amd64.whl#sha256=4090dde07a4fffc34aaf855701a9db28e9fccb57b368ade520f1a0f8e811c878 ; sys_platform == 'win32' and python_version == '3.12' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", + "pytorch_triton_xpu @ https://download.pytorch.org/whl/pytorch_triton_xpu-3.4.0-cp313-cp313-win_amd64.whl#sha256=a33d0888f3c8df028a2d028842715837d0049524d6c06b9bb11869890a13601a ; sys_platform == 'win32' and python_version == '3.13' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", + + "torch @ https://download.pytorch.org/whl/xpu/torch-2.8.0%2Bxpu-cp39-cp39-linux_x86_64.whl ; 'linux' in sys_platform and python_version == '3.9' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", + "torch @ https://download.pytorch.org/whl/xpu/torch-2.8.0%2Bxpu-cp310-cp310-linux_x86_64.whl ; 'linux' in sys_platform and python_version == '3.10' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", + "torch @ https://download.pytorch.org/whl/xpu/torch-2.8.0%2Bxpu-cp311-cp311-linux_x86_64.whl ; 'linux' in sys_platform and python_version == '3.11' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", + "torch @ https://download.pytorch.org/whl/xpu/torch-2.8.0%2Bxpu-cp312-cp312-linux_x86_64.whl ; 'linux' in sys_platform and python_version == '3.12' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", + "torch @ https://download.pytorch.org/whl/xpu/torch-2.8.0%2Bxpu-cp313-cp313-linux_x86_64.whl ; 'linux' in sys_platform and python_version == '3.13' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", + "torch @ https://download.pytorch.org/whl/xpu/torch-2.8.0%2Bxpu-cp39-cp39-win_amd64.whl#sha256=f2f401276892428e4875cf1d8717c5cbab704b16fc594ccf23795e7b16549a99 ; sys_platform == 'win32' and python_version == '3.9' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", + "torch @ https://download.pytorch.org/whl/xpu/torch-2.8.0%2Bxpu-cp310-cp310-win_amd64.whl#sha256=125c60cd59d51b39581a7e9afcd4679bc3a6b8c1f9440b1bb502a23fdd60571e ; sys_platform == 'win32' and python_version == '3.10' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", + "torch @ https://download.pytorch.org/whl/xpu/torch-2.8.0%2Bxpu-cp311-cp311-win_amd64.whl#sha256=47f1a57258cd460e80b38b2ed6744e31587ab77a96b4215bf59546cb4bab5cc0 ; sys_platform == 'win32' and python_version == '3.11' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", + "torch @ https://download.pytorch.org/whl/xpu/torch-2.8.0%2Bxpu-cp312-cp312-win_amd64.whl#sha256=0937d8943c145a83d9bafc6f80ef28971167817f9eda26066d33f72caf8a6646 ; sys_platform == 'win32' and python_version == '3.12' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", + "torch @ https://download.pytorch.org/whl/xpu/torch-2.8.0%2Bxpu-cp313-cp313-win_amd64.whl#sha256=e034aab1d71760dc80a731531be43673ffe15e99033b82d24e40d2e6d41bd8bf ; sys_platform == 'win32' and python_version == '3.13' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", + + "bitsandbytes @ https://github.com/bitsandbytes-foundation/bitsandbytes/releases/download/continuous-release_main/bitsandbytes-1.33.7.preview-py3-none-manylinux_2_24_x86_64.whl ; ('linux' in sys_platform) and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", + "bitsandbytes @ https://github.com/bitsandbytes-foundation/bitsandbytes/releases/download/continuous-release_main/bitsandbytes-1.33.7.preview-py3-none-win_amd64.whl ; (sys_platform == 'win32') and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", + + "torchvision @ https://download.pytorch.org/whl/xpu/torchvision-0.23.0%2Bxpu-cp39-cp39-manylinux_2_28_x86_64.whl#sha256=6e981c192045fc249c008441179ff237bb00174d818b875b0475730b63f0eaca ; 'linux' in sys_platform and python_version == '3.9' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", + "torchvision @ https://download.pytorch.org/whl/xpu/torchvision-0.23.0%2Bxpu-cp310-cp310-manylinux_2_28_x86_64.whl#sha256=e5ba4805969277175ebfd59cc717093528cc6e3ada89ac2725fc7a3c1fee6169 ; 'linux' in sys_platform and python_version == '3.10' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", + "torchvision @ https://download.pytorch.org/whl/xpu/torchvision-0.23.0%2Bxpu-cp311-cp311-manylinux_2_28_x86_64.whl#sha256=74c39c144104416bc4c5ad8c26ab0c169dc5cc6be58059e01bc3665dd0ef676f ; 'linux' in sys_platform and python_version == '3.11' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", + "torchvision @ https://download.pytorch.org/whl/xpu/torchvision-0.23.0%2Bxpu-cp312-cp312-manylinux_2_28_x86_64.whl#sha256=0acec355b80c3899841184084f365df336c508602812e34a44007b8b60d53af4 ; 'linux' in sys_platform and python_version == '3.12' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", + "torchvision @ https://download.pytorch.org/whl/xpu/torchvision-0.23.0%2Bxpu-cp313-cp313-manylinux_2_28_x86_64.whl#sha256=e2109ae773dad27b98ca17681044b4f876563c37f2382b75de3a371399edcff8 ; 'linux' in sys_platform and python_version == '3.13' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", + "torchvision @ https://download.pytorch.org/whl/xpu/torchvision-0.23.0%2Bxpu-cp39-cp39-win_amd64.whl#sha256=5f7904e7048d414379bc8c1167260f1e84204f105db2d0a2f9c89e87ce1cf205 ; sys_platform == 'win32' and python_version == '3.9' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", + "torchvision @ https://download.pytorch.org/whl/xpu/torchvision-0.23.0%2Bxpu-cp310-cp310-win_amd64.whl#sha256=005fca5e658ca8e37adb63c1a021c84f5e56dfa6cf0d601d89cfe40b9473f79f ; sys_platform == 'win32' and python_version == '3.10' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", + "torchvision @ https://download.pytorch.org/whl/xpu/torchvision-0.23.0%2Bxpu-cp311-cp311-win_amd64.whl#sha256=c6d030f5361461550c0ff1339b5bca8585fc1e84fda2e64b6184e65a581e4f98 ; sys_platform == 'win32' and python_version == '3.11' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", + "torchvision @ https://download.pytorch.org/whl/xpu/torchvision-0.23.0%2Bxpu-cp312-cp312-win_amd64.whl#sha256=91aafd61864cdce27461cbec13ddbf28c1bc6494265a1e4b80131c64a3b7d18f ; sys_platform == 'win32' and python_version == '3.12' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", + "torchvision @ https://download.pytorch.org/whl/xpu/torchvision-0.23.0%2Bxpu-cp313-cp313-win_amd64.whl#sha256=71dc4a6421742ed1e7f585b04a100ad53615c341fbccfbc255aefb38ea9091da ; sys_platform == 'win32' and python_version == '3.13' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", +] +intel-gpu-torch280 = [ + "unsloth[intelgputorch280]" +] +intelgputorch290 = [ + "unsloth_zoo[intelgpu]", + "unsloth[huggingfacenotorch]", + + "pytorch_triton_xpu @ https://download.pytorch.org/whl/pytorch_triton_xpu-3.5.0-cp310-cp310-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl#sha256=c169a1de14c19673b17c751290d467fa282fc90fa5da4314b2e5cdab1f553146 ; platform_system == 'Linux' and python_version == '3.10' and platform_machine == 'x86_64'", + "pytorch_triton_xpu @ https://download.pytorch.org/whl/pytorch_triton_xpu-3.5.0-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl#sha256=013d9dd5d6479bd22983161f462e61c8dbe1d82e6730624a7a8d5945507eaa61 ; platform_system == 'Linux' and python_version == '3.11' and platform_machine == 'x86_64'", + "pytorch_triton_xpu @ https://download.pytorch.org/whl/pytorch_triton_xpu-3.5.0-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl#sha256=afc8cabfbf7ed51fd278d1e0f88d6afc157b0201bad4b99d681e4d542f9e66d4 ; platform_system == 'Linux' and python_version == '3.12' and platform_machine == 'x86_64'", + "pytorch_triton_xpu @ https://download.pytorch.org/whl/pytorch_triton_xpu-3.5.0-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl#sha256=0d24c1716088f2764d0d24c64227732195b6a42706c3c5fc89eeb4904bfa0818 ; platform_system == 'Linux' and python_version == '3.13' and platform_machine == 'x86_64'", + "pytorch_triton_xpu @ https://download.pytorch.org/whl/pytorch_triton_xpu-3.5.0-cp310-cp310-win_amd64.whl#sha256=c83ab007311d9cfb6e809ee5a4587d99a9eef4be720b90da4f1aaa68b45139a0 ; sys_platform == 'win32' and python_version == '3.10' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", + "pytorch_triton_xpu @ https://download.pytorch.org/whl/pytorch_triton_xpu-3.5.0-cp311-cp311-win_amd64.whl#sha256=debf75348da8e8c7166b4d4a9b91d1508bb8d6581e339f79f7604b2e6746bacd ; sys_platform == 'win32' and python_version == '3.11' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", + "pytorch_triton_xpu @ https://download.pytorch.org/whl/pytorch_triton_xpu-3.5.0-cp312-cp312-win_amd64.whl#sha256=97337a47425f1963a723475bd61037460e84ba01db4f87a1d662c3718ff6c47e ; sys_platform == 'win32' and python_version == '3.12' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", + "pytorch_triton_xpu @ https://download.pytorch.org/whl/pytorch_triton_xpu-3.5.0-cp313-cp313-win_amd64.whl#sha256=2caf8138695f6abb023ecd02031a2611ba1bf8fff2f19802567cb2fadefe9e87 ; sys_platform == 'win32' and python_version == '3.13' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", + + "torch @ https://download.pytorch.org/whl/xpu/torch-2.9.0%2Bxpu-cp310-cp310-linux_x86_64.whl#sha256=5afbe860ce991825a36b75706a523601087e414b77598ef0d9d3d565741c277d ; platform_system == 'Linux' and python_version == '3.10' and platform_machine == 'x86_64'", + "torch @ https://download.pytorch.org/whl/xpu/torch-2.9.0%2Bxpu-cp311-cp311-linux_x86_64.whl#sha256=607fe419c32d6e8e0556f745742e7cff1d0babce51f54be890e0c1422359c442 ; platform_system == 'Linux' and python_version == '3.11' and platform_machine == 'x86_64'", + "torch @ https://download.pytorch.org/whl/xpu/torch-2.9.0%2Bxpu-cp312-cp312-linux_x86_64.whl#sha256=376bae584d89980b8e59934d248c38d5fa3b7d4687a4df1a19f4bc1d23dcc8c1 ; platform_system == 'Linux' and python_version == '3.12' and platform_machine == 'x86_64'", + "torch @ https://download.pytorch.org/whl/xpu/torch-2.9.0%2Bxpu-cp313-cp313-linux_x86_64.whl#sha256=98d6a06dd7fb185874367b18bd609f05f16fdce4142a5980ca94461949965cd2 ; platform_system == 'Linux' and python_version == '3.13' and platform_machine == 'x86_64'", + "torch @ https://download.pytorch.org/whl/xpu/torch-2.9.0%2Bxpu-cp310-cp310-win_amd64.whl#sha256=47cc68f631f65bd9c84924d052cd04dec7531023caa85e80345e9c94611c887d ; sys_platform == 'win32' and python_version == '3.10' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", + "torch @ https://download.pytorch.org/whl/xpu/torch-2.9.0%2Bxpu-cp311-cp311-win_amd64.whl#sha256=d56c44ab4818aba57e5c7b628f422d014e0d507427170a771c5be85e308b0bc6 ; sys_platform == 'win32' and python_version == '3.11' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", + "torch @ https://download.pytorch.org/whl/xpu/torch-2.9.0%2Bxpu-cp312-cp312-win_amd64.whl#sha256=18cad93aaff76a01ce73aef6935ece7cfc03344b905592ec731446c44d44592b ; sys_platform == 'win32' and python_version == '3.12' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", + "torch @ https://download.pytorch.org/whl/xpu/torch-2.9.0%2Bxpu-cp313-cp313-win_amd64.whl#sha256=579929cdc10a76800ead41289cac191ea36d1b16f5f501d3fc25607d4375cd83 ; sys_platform == 'win32' and python_version == '3.13' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", + + "bitsandbytes @ https://github.com/bitsandbytes-foundation/bitsandbytes/releases/download/continuous-release_main/bitsandbytes-1.33.7.preview-py3-none-manylinux_2_24_x86_64.whl ; ('linux' in sys_platform) and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", + "bitsandbytes @ https://github.com/bitsandbytes-foundation/bitsandbytes/releases/download/continuous-release_main/bitsandbytes-1.33.7.preview-py3-none-win_amd64.whl ; (sys_platform == 'win32') and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", + + "torchvision @ https://download.pytorch.org/whl/xpu/torchvision-0.24.0%2Bxpu-cp310-cp310-manylinux_2_28_x86_64.whl#sha256=cbfae2b79b7549fd368c2462fc8e94f8f26cc450782ee72138e908077c09a519 ; platform_system == 'Linux' and python_version == '3.10' and platform_machine == 'x86_64'", + "torchvision @ https://download.pytorch.org/whl/xpu/torchvision-0.24.0%2Bxpu-cp311-cp311-manylinux_2_28_x86_64.whl#sha256=044fa36ef4b6b43edcd490b75c853fa4b3eb033c2bded29f8fbcf27734713c67 ; platform_system == 'Linux' and python_version == '3.11' and platform_machine == 'x86_64'", + "torchvision @ https://download.pytorch.org/whl/xpu/torchvision-0.24.0%2Bxpu-cp312-cp312-manylinux_2_28_x86_64.whl#sha256=4b91e4bec1d740a6211f02578a79888550b73f3a4e1383035f8f6d72f587212c ; platform_system == 'Linux' and python_version == '3.12' and platform_machine == 'x86_64'", + "torchvision @ https://download.pytorch.org/whl/xpu/torchvision-0.24.0%2Bxpu-cp313-cp313-manylinux_2_28_x86_64.whl#sha256=88239e73ca37254bec84f29cd5887e10ff712de7edbbda3fbb3609cd6190d99e ; platform_system == 'Linux' and python_version == '3.13' and platform_machine == 'x86_64'", + "torchvision @ https://download.pytorch.org/whl/xpu/torchvision-0.24.0%2Bxpu-cp310-cp310-win_amd64.whl#sha256=19c7da8ca767d593e13a88a12bb08d06e34a673f6f26c2f9c191d60e81c02953 ; sys_platform == 'win32' and python_version == '3.10' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", + "torchvision @ https://download.pytorch.org/whl/xpu/torchvision-0.24.0%2Bxpu-cp311-cp311-win_amd64.whl#sha256=9bb0d1421c544ac8e2eca5b47daacaf54706dc9139c003aa5e77ee5f355c5931 ; sys_platform == 'win32' and python_version == '3.11' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", + "torchvision @ https://download.pytorch.org/whl/xpu/torchvision-0.24.0%2Bxpu-cp312-cp312-win_amd64.whl#sha256=6a5194bc736089606342d48a3f6822829b167617e9495d91d753dd1bd46fda18 ; sys_platform == 'win32' and python_version == '3.12' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", + "torchvision @ https://download.pytorch.org/whl/xpu/torchvision-0.24.0%2Bxpu-cp313-cp313-win_amd64.whl#sha256=da47a3ce2bb7f0301a31124668b5908f9b9e92d6241443de15a310ef9632fd83 ; sys_platform == 'win32' and python_version == '3.13' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", +] +intel-gpu-torch290 = [ + "unsloth[intelgputorch290]" +] +intelgputorch271 = [ + "unsloth_zoo[intelgpu]", + "unsloth[huggingfacenotorch]", + + "pytorch_triton_xpu @ https://download.pytorch.org/whl/pytorch_triton_xpu-3.3.1-cp310-cp310-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl#sha256=663ce21364096b268c6687f26f22862cb1001cae0c4ec9f98a0998415f99e2b0 ; platform_system == 'Linux' and python_version == '3.10' and platform_machine == 'x86_64'", + "pytorch_triton_xpu @ https://download.pytorch.org/whl/pytorch_triton_xpu-3.3.1-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl#sha256=dd92cc17000bad19f213b6a877d7f10cd71341b703cd188513ce9fff8d42e3dd ; platform_system == 'Linux' and python_version == '3.11' and platform_machine == 'x86_64'", + "pytorch_triton_xpu @ https://download.pytorch.org/whl/pytorch_triton_xpu-3.3.1-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl#sha256=aa5c3ec21a89e967d1dfe61e3d5b1c1ae9620c871ed804771d3378d6a44066f2 ; platform_system == 'Linux' and python_version == '3.12' and platform_machine == 'x86_64'", + "pytorch_triton_xpu @ https://download.pytorch.org/whl/pytorch_triton_xpu-3.3.1-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl#sha256=d1c6f522e11112a311b1a61ba7b40b43ad8305675fa29153017ccb1ad0b6816d ; platform_system == 'Linux' and python_version == '3.13' and platform_machine == 'x86_64'", + "pytorch_triton_xpu @ https://download.pytorch.org/whl/pytorch_triton_xpu-3.3.1-cp310-cp310-win_amd64.whl#sha256=a5c16dcf449a9cb62bc3788f7ec45782bb3ead6edc2637a12b60ef0f8f45dc55 ; sys_platform == 'win32' and python_version == '3.10' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", + "pytorch_triton_xpu @ https://download.pytorch.org/whl/pytorch_triton_xpu-3.3.1-cp311-cp311-win_amd64.whl#sha256=bc2d76ffa4ceed5b38ae34b52dbff643442e1a44d52ca72d7cb520ca1950e9ae ; sys_platform == 'win32' and python_version == '3.11' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", + "pytorch_triton_xpu @ https://download.pytorch.org/whl/pytorch_triton_xpu-3.3.1-cp312-cp312-win_amd64.whl#sha256=b09ca59ce52d6d27b1510df783cde222b703a71857a6fa953f1f155f9f50811a ; sys_platform == 'win32' and python_version == '3.12' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", + "pytorch_triton_xpu @ https://download.pytorch.org/whl/pytorch_triton_xpu-3.3.1-cp313-cp313-win_amd64.whl#sha256=1260c4a4bad426b6cd3c8f3e1a21835381c6f217bf434bcb55fedec08a206dea ; sys_platform == 'win32' and python_version == '3.13' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", + + "torch @ https://download.pytorch.org/whl/xpu/torch-2.7.1%2Bxpu-cp310-cp310-linux_x86_64.whl#sha256=231c3fbd88a75d94de5ccbbb7f4f9a96cb3c58b3d891c2a1b469d38df95f9be6 ; platform_system == 'Linux' and python_version == '3.10' and platform_machine == 'x86_64'", + "torch @ https://download.pytorch.org/whl/xpu/torch-2.7.1%2Bxpu-cp311-cp311-linux_x86_64.whl#sha256=78edcc27709dd819fc820f5eb9421bd10d3f3dcb14adb25ee60766c76f0e67f3 ; platform_system == 'Linux' and python_version == '3.11' and platform_machine == 'x86_64'", + "torch @ https://download.pytorch.org/whl/xpu/torch-2.7.1%2Bxpu-cp312-cp312-linux_x86_64.whl#sha256=b443df40bc9cb7d648a9f8f9ed1d5c3a1203e561ebd0a61dd55fb8a58833d5ec ; platform_system == 'Linux' and python_version == '3.12' and platform_machine == 'x86_64'", + "torch @ https://download.pytorch.org/whl/xpu/torch-2.7.1%2Bxpu-cp313-cp313-linux_x86_64.whl#sha256=412b58ffcceebea399c9a1bcdb22896aa10385c2650a8c4f8a677fb11c49b448 ; platform_system == 'Linux' and python_version == '3.13' and platform_machine == 'x86_64'", + "torch @ https://download.pytorch.org/whl/xpu/torch-2.7.1%2Bxpu-cp310-cp310-win_amd64.whl#sha256=2591228dc2cb73c78daf24277c4449ba9474f94cd31938147249269fe89d05d6 ; sys_platform == 'win32' and python_version == '3.10' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", + "torch @ https://download.pytorch.org/whl/xpu/torch-2.7.1%2Bxpu-cp311-cp311-win_amd64.whl#sha256=1aacb86e9a9684ffc8bde3db14b251d00df7019a9a434ec99a59076a2696325d ; sys_platform == 'win32' and python_version == '3.11' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", + "torch @ https://download.pytorch.org/whl/xpu/torch-2.7.1%2Bxpu-cp312-cp312-win_amd64.whl#sha256=9b65dc8562521b60d77aa653132bc03a19da0291318fcf919faa3f03080d8f7e ; sys_platform == 'win32' and python_version == '3.12' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", + "torch @ https://download.pytorch.org/whl/xpu/torch-2.7.1%2Bxpu-cp313-cp313-win_amd64.whl#sha256=cd3669fee311bc3ee5501d696bf989226a6f2bf957d120a04881a07af05526d6 ; sys_platform == 'win32' and python_version == '3.13' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", + + "bitsandbytes @ https://github.com/bitsandbytes-foundation/bitsandbytes/releases/download/continuous-release_main/bitsandbytes-1.33.7.preview-py3-none-manylinux_2_24_x86_64.whl ; ('linux' in sys_platform) and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", + "bitsandbytes @ https://github.com/bitsandbytes-foundation/bitsandbytes/releases/download/continuous-release_main/bitsandbytes-1.33.7.preview-py3-none-win_amd64.whl ; (sys_platform == 'win32') and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", + + "torchvision @ https://download.pytorch.org/whl/xpu/torchvision-0.22.1%2Bxpu-cp310-cp310-manylinux_2_28_x86_64.whl#sha256=f8cdf6889c02b3166679eef661b68757ea7e99c314432c3d41dac3d2ed4a59d4 ; platform_system == 'Linux' and python_version == '3.10' and platform_machine == 'x86_64'", + "torchvision @ https://download.pytorch.org/whl/xpu/torchvision-0.22.1%2Bxpu-cp311-cp311-manylinux_2_28_x86_64.whl#sha256=f7d15b65d52809745992e0001c25034f33ac01f2dff5248614e07b5d009a59b7 ; platform_system == 'Linux' and python_version == '3.11' and platform_machine == 'x86_64'", + "torchvision @ https://download.pytorch.org/whl/xpu/torchvision-0.22.1%2Bxpu-cp312-cp312-manylinux_2_28_x86_64.whl#sha256=1ff1f98d70846352c7f56833bedab1a055ead27b11c120b8c719063ee0383554 ; platform_system == 'Linux' and python_version == '3.12' and platform_machine == 'x86_64'", + "torchvision @ https://download.pytorch.org/whl/xpu/torchvision-0.22.1%2Bxpu-cp313-cp313-manylinux_2_28_x86_64.whl#sha256=f46945344ea911a70309231eaaf3b80c96f6646ce5515dc89aa94f94144e310e ; platform_system == 'Linux' and python_version == '3.13' and platform_machine == 'x86_64'", + "torchvision @ https://download.pytorch.org/whl/xpu/torchvision-0.22.1%2Bxpu-cp310-cp310-win_amd64.whl#sha256=ecae9a02de769e2070d37388116beb407c3f0d60b8e65c1da1423f4eafee361a ; sys_platform == 'win32' and python_version == '3.10' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", + "torchvision @ https://download.pytorch.org/whl/xpu/torchvision-0.22.1%2Bxpu-cp311-cp311-win_amd64.whl#sha256=2914e62782431bebd6ad9a3b98a2b7311e448e84a7534bb7f35874b9279a17de ; sys_platform == 'win32' and python_version == '3.11' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", + "torchvision @ https://download.pytorch.org/whl/xpu/torchvision-0.22.1%2Bxpu-cp312-cp312-win_amd64.whl#sha256=5b462c156f4e2097e1e53649d3f298ce352fa4c5d1e6addd360375b10ebd6c67 ; sys_platform == 'win32' and python_version == '3.12' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", + "torchvision @ https://download.pytorch.org/whl/xpu/torchvision-0.22.1%2Bxpu-cp313-cp313-win_amd64.whl#sha256=fa87b3677cd1af67ce423004283c1bde80e3571f391182a3e89b485e18e3c70f ; sys_platform == 'win32' and python_version == '3.13' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", +] +intel-gpu-torch271 = [ + "unsloth[intelgputorch271]" +] +intelgputorch291 = [ + "unsloth_zoo[intelgpu]", + "unsloth[huggingfacenotorch]", + + "pytorch_triton_xpu @ https://download.pytorch.org/whl/pytorch_triton_xpu-3.5.0-cp310-cp310-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl#sha256=c169a1de14c19673b17c751290d467fa282fc90fa5da4314b2e5cdab1f553146 ; platform_system == 'Linux' and python_version == '3.10' and platform_machine == 'x86_64'", + "pytorch_triton_xpu @ https://download.pytorch.org/whl/pytorch_triton_xpu-3.5.0-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl#sha256=013d9dd5d6479bd22983161f462e61c8dbe1d82e6730624a7a8d5945507eaa61 ; platform_system == 'Linux' and python_version == '3.11' and platform_machine == 'x86_64'", + "pytorch_triton_xpu @ https://download.pytorch.org/whl/pytorch_triton_xpu-3.5.0-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl#sha256=afc8cabfbf7ed51fd278d1e0f88d6afc157b0201bad4b99d681e4d542f9e66d4 ; platform_system == 'Linux' and python_version == '3.12' and platform_machine == 'x86_64'", + "pytorch_triton_xpu @ https://download.pytorch.org/whl/pytorch_triton_xpu-3.5.0-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl#sha256=0d24c1716088f2764d0d24c64227732195b6a42706c3c5fc89eeb4904bfa0818 ; platform_system == 'Linux' and python_version == '3.13' and platform_machine == 'x86_64'", + "pytorch_triton_xpu @ https://download.pytorch.org/whl/pytorch_triton_xpu-3.5.0-cp310-cp310-win_amd64.whl#sha256=c83ab007311d9cfb6e809ee5a4587d99a9eef4be720b90da4f1aaa68b45139a0 ; sys_platform == 'win32' and python_version == '3.10' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", + "pytorch_triton_xpu @ https://download.pytorch.org/whl/pytorch_triton_xpu-3.5.0-cp311-cp311-win_amd64.whl#sha256=debf75348da8e8c7166b4d4a9b91d1508bb8d6581e339f79f7604b2e6746bacd ; sys_platform == 'win32' and python_version == '3.11' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", + "pytorch_triton_xpu @ https://download.pytorch.org/whl/pytorch_triton_xpu-3.5.0-cp312-cp312-win_amd64.whl#sha256=97337a47425f1963a723475bd61037460e84ba01db4f87a1d662c3718ff6c47e ; sys_platform == 'win32' and python_version == '3.12' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", + "pytorch_triton_xpu @ https://download.pytorch.org/whl/pytorch_triton_xpu-3.5.0-cp313-cp313-win_amd64.whl#sha256=2caf8138695f6abb023ecd02031a2611ba1bf8fff2f19802567cb2fadefe9e87 ; sys_platform == 'win32' and python_version == '3.13' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", + + "torch @ https://download.pytorch.org/whl/xpu/torch-2.9.1%2Bxpu-cp310-cp310-linux_x86_64.whl#sha256=fb7895c744132d6a8e56ce8434ae1d8355c9bda4e9f58832744ff742d6268eaf ; platform_system == 'Linux' and python_version == '3.10' and platform_machine == 'x86_64'", + "torch @ https://download.pytorch.org/whl/xpu/torch-2.9.1%2Bxpu-cp311-cp311-linux_x86_64.whl#sha256=da2604a9114a28de71ce654819424d20a246adf644d191ae160837df9731b79e ; platform_system == 'Linux' and python_version == '3.11' and platform_machine == 'x86_64'", + "torch @ https://download.pytorch.org/whl/xpu/torch-2.9.1%2Bxpu-cp312-cp312-linux_x86_64.whl#sha256=d5968d78d81c1d01efc1b3bf83d7da3d83161dcc3a9fcf91f500591db1c6c75d ; platform_system == 'Linux' and python_version == '3.12' and platform_machine == 'x86_64'", + "torch @ https://download.pytorch.org/whl/xpu/torch-2.9.1%2Bxpu-cp313-cp313-linux_x86_64.whl#sha256=b56d6b0d65863f370527e971dbfa046a5dd2a1f61cc95071db26c764f36e4dce ; platform_system == 'Linux' and python_version == '3.13' and platform_machine == 'x86_64'", + "torch @ https://download.pytorch.org/whl/xpu/torch-2.9.1%2Bxpu-cp310-cp310-win_amd64.whl#sha256=2f318fb6a4bf1101cc17f35a5371f7c1768b41fceed03628397834e85b3edfdd ; sys_platform == 'win32' and python_version == '3.10' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", + "torch @ https://download.pytorch.org/whl/xpu/torch-2.9.1%2Bxpu-cp311-cp311-win_amd64.whl#sha256=c9cedc3fb099366b2e6c563df6578e323564b1b5d40ac27be73c674755343a1d ; sys_platform == 'win32' and python_version == '3.11' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", + "torch @ https://download.pytorch.org/whl/xpu/torch-2.9.1%2Bxpu-cp312-cp312-win_amd64.whl#sha256=bee9623254d0f95a1ca115dbd17e9a9d966fdb8ae123e2ada4a9eb2fb8d38db8 ; sys_platform == 'win32' and python_version == '3.12' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", + "torch @ https://download.pytorch.org/whl/xpu/torch-2.9.1%2Bxpu-cp313-cp313-win_amd64.whl#sha256=cd5c857da52a63c121561b30b0979e69ade70b575fd74e389787bc7c1ee2ac11 ; sys_platform == 'win32' and python_version == '3.13' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", + + "bitsandbytes @ https://github.com/bitsandbytes-foundation/bitsandbytes/releases/download/continuous-release_main/bitsandbytes-1.33.7.preview-py3-none-manylinux_2_24_x86_64.whl ; ('linux' in sys_platform) and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", + "bitsandbytes @ https://github.com/bitsandbytes-foundation/bitsandbytes/releases/download/continuous-release_main/bitsandbytes-1.33.7.preview-py3-none-win_amd64.whl ; (sys_platform == 'win32') and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", + + "torchvision @ https://download.pytorch.org/whl/xpu/torchvision-0.24.1%2Bxpu-cp310-cp310-manylinux_2_28_x86_64.whl#sha256=cc5272da2cb4554edf059eedd6d1f5ef2859033b0fb79d5dcb8e99a0697f3325 ; platform_system == 'Linux' and python_version == '3.10' and platform_machine == 'x86_64'", + "torchvision @ https://download.pytorch.org/whl/xpu/torchvision-0.24.1%2Bxpu-cp311-cp311-manylinux_2_28_x86_64.whl#sha256=3c80d6a068c32fc4ebddb27953e03a0141bd0f10ca8730417cbc0e0748158285 ; platform_system == 'Linux' and python_version == '3.11' and platform_machine == 'x86_64'", + "torchvision @ https://download.pytorch.org/whl/xpu/torchvision-0.24.1%2Bxpu-cp312-cp312-manylinux_2_28_x86_64.whl#sha256=8cf640a867cf270b3fda7a10002c29d3fc2ad6dfbd76404a8cdd820489adb04c ; platform_system == 'Linux' and python_version == '3.12' and platform_machine == 'x86_64'", + "torchvision @ https://download.pytorch.org/whl/xpu/torchvision-0.24.1%2Bxpu-cp313-cp313-manylinux_2_28_x86_64.whl#sha256=d9c59ee5ae3d0560f02401c8dfd8054d50813a8dbb5d33a8777de7d02f6fcb7b ; platform_system == 'Linux' and python_version == '3.13' and platform_machine == 'x86_64'", + "torchvision @ https://download.pytorch.org/whl/xpu/torchvision-0.24.1%2Bxpu-cp310-cp310-win_amd64.whl#sha256=843ea7fcd8f5a22ebbc20d2d61d9eec7593821a0372eb8cabb73953d12ef6acf ; sys_platform == 'win32' and python_version == '3.10' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", + "torchvision @ https://download.pytorch.org/whl/xpu/torchvision-0.24.1%2Bxpu-cp311-cp311-win_amd64.whl#sha256=e5ff8a31d3c700f8dbac59697c8e32298a43ec059609ebc6ea7bab3eff6384e1 ; sys_platform == 'win32' and python_version == '3.11' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", + "torchvision @ https://download.pytorch.org/whl/xpu/torchvision-0.24.1%2Bxpu-cp312-cp312-win_amd64.whl#sha256=8bae6d4c042f8d20818da4a5aa9109c6fbd6ec11bc422be152ce8adf9a7095bf ; sys_platform == 'win32' and python_version == '3.12' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", + "torchvision @ https://download.pytorch.org/whl/xpu/torchvision-0.24.1%2Bxpu-cp313-cp313-win_amd64.whl#sha256=47059e290fc2a41ba78666ffcde102c436abf7ff8a34d200268b48c4fa0f9c45 ; sys_platform == 'win32' and python_version == '3.13' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", +] +intel-gpu-torch291 = [ + "unsloth[intelgputorch291]" +] +intelgputorch210 = [ + "unsloth_zoo[intelgpu]", + "unsloth[huggingfacenotorch]", + + "triton-xpu @ https://download.pytorch.org/whl/triton_xpu-3.6.0-cp310-cp310-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl ; platform_system == 'Linux' and python_version == '3.10' and platform_machine == 'x86_64'", + "triton-xpu @ https://download.pytorch.org/whl/triton_xpu-3.6.0-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl ; platform_system == 'Linux' and python_version == '3.11' and platform_machine == 'x86_64'", + "triton-xpu @ https://download.pytorch.org/whl/triton_xpu-3.6.0-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl ; platform_system == 'Linux' and python_version == '3.12' and platform_machine == 'x86_64'", + "triton-xpu @ https://download.pytorch.org/whl/triton_xpu-3.6.0-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl ; platform_system == 'Linux' and python_version == '3.13' and platform_machine == 'x86_64'", + "triton-xpu @ https://download.pytorch.org/whl/triton_xpu-3.6.0-cp310-cp310-win_amd64.whl ; sys_platform == 'win32' and python_version == '3.10' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", + "triton-xpu @ https://download.pytorch.org/whl/triton_xpu-3.6.0-cp311-cp311-win_amd64.whl ; sys_platform == 'win32' and python_version == '3.11' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", + "triton-xpu @ https://download.pytorch.org/whl/triton_xpu-3.6.0-cp312-cp312-win_amd64.whl ; sys_platform == 'win32' and python_version == '3.12' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", + "triton-xpu @ https://download.pytorch.org/whl/triton_xpu-3.6.0-cp313-cp313-win_amd64.whl ; sys_platform == 'win32' and python_version == '3.13' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", + + "torch @ https://download.pytorch.org/whl/xpu/torch-2.10.0%2Bxpu-cp310-cp310-linux_x86_64.whl#sha256=abb1d1ec1ac672bac0ff35420c965f2df0c636ef9d94e2a830e34578489d0a57 ; platform_system == 'Linux' and python_version == '3.10' and platform_machine == 'x86_64'", + "torch @ https://download.pytorch.org/whl/xpu/torch-2.10.0%2Bxpu-cp311-cp311-linux_x86_64.whl#sha256=71ad2f82da0f41eaec159f39fc85854e27c2391efa91b373e550648a6f4aaad3 ; platform_system == 'Linux' and python_version == '3.11' and platform_machine == 'x86_64'", + "torch @ https://download.pytorch.org/whl/xpu/torch-2.10.0%2Bxpu-cp312-cp312-linux_x86_64.whl#sha256=b473571d478912f92881cc13f15fa18f8463fb0fb8a068c96ed47a7d45a4da0a ; platform_system == 'Linux' and python_version == '3.12' and platform_machine == 'x86_64'", + "torch @ https://download.pytorch.org/whl/xpu/torch-2.10.0%2Bxpu-cp313-cp313-linux_x86_64.whl#sha256=3bc64a746ff25a93de140902c60c9e819d7413f5cea1e88d80999c27a5901e9c ; platform_system == 'Linux' and python_version == '3.13' and platform_machine == 'x86_64'", + "torch @ https://download.pytorch.org/whl/xpu/torch-2.10.0%2Bxpu-cp310-cp310-win_amd64.whl#sha256=ce50691ab3fb6301d9b7bb8b3834cf5fa7152a2b5f91fd24c5efdc601a25b780 ; sys_platform == 'win32' and python_version == '3.10' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", + "torch @ https://download.pytorch.org/whl/xpu/torch-2.10.0%2Bxpu-cp311-cp311-win_amd64.whl#sha256=cb9d37f21cb9fb7df67d62863f021c3144e8d8832b9ea8e8523ac308bc620ea1 ; sys_platform == 'win32' and python_version == '3.11' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", + "torch @ https://download.pytorch.org/whl/xpu/torch-2.10.0%2Bxpu-cp312-cp312-win_amd64.whl#sha256=3ad605be4728b6d3a28a44d07dd794b1a9e45551b0057815bf25eb2a6d6a56a7 ; sys_platform == 'win32' and python_version == '3.12' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", + "torch @ https://download.pytorch.org/whl/xpu/torch-2.10.0%2Bxpu-cp313-cp313-win_amd64.whl#sha256=2b4b56dd6c792aef82006904fa888692e3782e4ae5da27526801bad4898f05a5 ; sys_platform == 'win32' and python_version == '3.13' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", + + "bitsandbytes @ https://github.com/bitsandbytes-foundation/bitsandbytes/releases/download/continuous-release_main/bitsandbytes-1.33.7.preview-py3-none-manylinux_2_24_x86_64.whl ; ('linux' in sys_platform) and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", + "bitsandbytes @ https://github.com/bitsandbytes-foundation/bitsandbytes/releases/download/continuous-release_main/bitsandbytes-1.33.7.preview-py3-none-win_amd64.whl ; (sys_platform == 'win32') and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", + + "torchvision @ https://download.pytorch.org/whl/xpu/torchvision-0.25.0%2Bxpu-cp310-cp310-manylinux_2_28_x86_64.whl#sha256=7e1e7b170fcf7161c8499b67156c5a05462243626dc0974010791a0bab4378d3 ; platform_system == 'Linux' and python_version == '3.10' and platform_machine == 'x86_64'", + "torchvision @ https://download.pytorch.org/whl/xpu/torchvision-0.25.0%2Bxpu-cp311-cp311-manylinux_2_28_x86_64.whl#sha256=bd6add201bd7628af70437292e1447abb368e0b5f4ff9abd334ae435efd44792 ; platform_system == 'Linux' and python_version == '3.11' and platform_machine == 'x86_64'", + "torchvision @ https://download.pytorch.org/whl/xpu/torchvision-0.25.0%2Bxpu-cp312-cp312-manylinux_2_28_x86_64.whl#sha256=6ad2543496bc29e59d3dd614a94d09aa9870318aedb66045344fffddfedd2cf8 ; platform_system == 'Linux' and python_version == '3.12' and platform_machine == 'x86_64'", + "torchvision @ https://download.pytorch.org/whl/xpu/torchvision-0.25.0%2Bxpu-cp313-cp313-manylinux_2_28_x86_64.whl#sha256=80269f37865fcd8b57f20e4786efae2200bfa2b2727926c3c7acc82f0e7d3548 ; platform_system == 'Linux' and python_version == '3.13' and platform_machine == 'x86_64'", + "torchvision @ https://download.pytorch.org/whl/xpu/torchvision-0.25.0%2Bxpu-cp310-cp310-win_amd64.whl#sha256=6b9485ba85dcba4d196d6134d9c3332fb228fb2556416bf0450a64e8a472fcba ; sys_platform == 'win32' and python_version == '3.10' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", + "torchvision @ https://download.pytorch.org/whl/xpu/torchvision-0.25.0%2Bxpu-cp311-cp311-win_amd64.whl#sha256=36cbaedf10f6412af5c89afd9aeea474e6a56a0050348ada8fabe1ecaf6b879e ; sys_platform == 'win32' and python_version == '3.11' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", + "torchvision @ https://download.pytorch.org/whl/xpu/torchvision-0.25.0%2Bxpu-cp312-cp312-win_amd64.whl#sha256=738357d97468d75fe3d510ac37e65130f2787f81d9bbc1518898f7396dc3403f ; sys_platform == 'win32' and python_version == '3.12' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", + "torchvision @ https://download.pytorch.org/whl/xpu/torchvision-0.25.0%2Bxpu-cp313-cp313-win_amd64.whl#sha256=1c4b44b36a557f7381e3076fb8843366742238648441d607c8d049c6da0f8886 ; sys_platform == 'win32' and python_version == '3.13' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", +] +intel-gpu-torch210 = [ + "unsloth[intelgputorch210]", + "unsloth[audio-torch210]", +] +intelgputorch2110 = [ + "unsloth_zoo[intelgpu]", + "unsloth[huggingfacenotorch]", + + "triton-xpu @ https://download.pytorch.org/whl/triton_xpu-3.7.0-cp310-cp310-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl#sha256=2a1841138750f708ec017becbf8d357526f3fa350deee6553be5735ad66160a3 ; platform_system == 'Linux' and python_version == '3.10' and platform_machine == 'x86_64'", + "triton-xpu @ https://download.pytorch.org/whl/triton_xpu-3.7.0-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl#sha256=e85378f1fc1ea002271de2a35475b75008fa554b86ef9d3bc55be9c513a63b51 ; platform_system == 'Linux' and python_version == '3.11' and platform_machine == 'x86_64'", + "triton-xpu @ https://download.pytorch.org/whl/triton_xpu-3.7.0-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl#sha256=a6663ebe43e3c0d560ff774708632d7a75208ee64a291c1724ed5c16a92d1c72 ; platform_system == 'Linux' and python_version == '3.12' and platform_machine == 'x86_64'", + "triton-xpu @ https://download.pytorch.org/whl/triton_xpu-3.7.0-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl#sha256=08c8d43b2831faf9d6799480df2b45dde58102257aebd810d07a2ce18cd4e5df ; platform_system == 'Linux' and python_version == '3.13' and platform_machine == 'x86_64'", + "triton-xpu @ https://download.pytorch.org/whl/triton_xpu-3.7.0-cp310-cp310-win_amd64.whl#sha256=90fb8f767950a4ffca627faa7f86d9c697237ea4352d7e23505c5c9ed8e72216 ; sys_platform == 'win32' and python_version == '3.10' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", + "triton-xpu @ https://download.pytorch.org/whl/triton_xpu-3.7.0-cp311-cp311-win_amd64.whl#sha256=aa7de82f4265089e74f25a2701b7532e5c47d74224d877b61da1d66156e3f0c1 ; sys_platform == 'win32' and python_version == '3.11' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", + "triton-xpu @ https://download.pytorch.org/whl/triton_xpu-3.7.0-cp312-cp312-win_amd64.whl#sha256=5ba3a31c6e1b259ad2d924e1b50f72a78c6ebd7eb4f364473bbf93e144734e80 ; sys_platform == 'win32' and python_version == '3.12' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", + "triton-xpu @ https://download.pytorch.org/whl/triton_xpu-3.7.0-cp313-cp313-win_amd64.whl#sha256=e8b4caba9b2399ea4c7f9a2777042564dea5d6f9e586a2dcb015a4ce20f000f7 ; sys_platform == 'win32' and python_version == '3.13' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", + + "torch @ https://download.pytorch.org/whl/xpu/torch-2.11.0%2Bxpu-cp310-cp310-linux_x86_64.whl ; platform_system == 'Linux' and python_version == '3.10' and platform_machine == 'x86_64'", + "torch @ https://download.pytorch.org/whl/xpu/torch-2.11.0%2Bxpu-cp311-cp311-linux_x86_64.whl ; platform_system == 'Linux' and python_version == '3.11' and platform_machine == 'x86_64'", + "torch @ https://download.pytorch.org/whl/xpu/torch-2.11.0%2Bxpu-cp312-cp312-linux_x86_64.whl ; platform_system == 'Linux' and python_version == '3.12' and platform_machine == 'x86_64'", + "torch @ https://download.pytorch.org/whl/xpu/torch-2.11.0%2Bxpu-cp313-cp313-linux_x86_64.whl ; platform_system == 'Linux' and python_version == '3.13' and platform_machine == 'x86_64'", + "torch @ https://download.pytorch.org/whl/xpu/torch-2.11.0%2Bxpu-cp310-cp310-win_amd64.whl ; sys_platform == 'win32' and python_version == '3.10' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", + "torch @ https://download.pytorch.org/whl/xpu/torch-2.11.0%2Bxpu-cp311-cp311-win_amd64.whl ; sys_platform == 'win32' and python_version == '3.11' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", + "torch @ https://download.pytorch.org/whl/xpu/torch-2.11.0%2Bxpu-cp312-cp312-win_amd64.whl ; sys_platform == 'win32' and python_version == '3.12' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", + "torch @ https://download.pytorch.org/whl/xpu/torch-2.11.0%2Bxpu-cp313-cp313-win_amd64.whl ; sys_platform == 'win32' and python_version == '3.13' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", + + "bitsandbytes @ https://github.com/bitsandbytes-foundation/bitsandbytes/releases/download/continuous-release_main/bitsandbytes-1.33.7.preview-py3-none-manylinux_2_24_x86_64.whl ; ('linux' in sys_platform) and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", + "bitsandbytes @ https://github.com/bitsandbytes-foundation/bitsandbytes/releases/download/continuous-release_main/bitsandbytes-1.33.7.preview-py3-none-win_amd64.whl ; (sys_platform == 'win32') and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", + + "torchvision @ https://download.pytorch.org/whl/xpu/torchvision-0.26.0%2Bxpu-cp310-cp310-manylinux_2_28_x86_64.whl#sha256=6e634354b752b7366e8ad16b84f3e7e5863776a7ab448bbabae4fd36668dee7a ; platform_system == 'Linux' and python_version == '3.10' and platform_machine == 'x86_64'", + "torchvision @ https://download.pytorch.org/whl/xpu/torchvision-0.26.0%2Bxpu-cp311-cp311-manylinux_2_28_x86_64.whl#sha256=293169899f562ce473a58836dd024f0b1e72a347400278287ab393d1b04991e4 ; platform_system == 'Linux' and python_version == '3.11' and platform_machine == 'x86_64'", + "torchvision @ https://download.pytorch.org/whl/xpu/torchvision-0.26.0%2Bxpu-cp312-cp312-manylinux_2_28_x86_64.whl#sha256=e204d14be6f0f84d5f0e6e9213556e80326c3ab682cac108bcbef340bf45297b ; platform_system == 'Linux' and python_version == '3.12' and platform_machine == 'x86_64'", + "torchvision @ https://download.pytorch.org/whl/xpu/torchvision-0.26.0%2Bxpu-cp313-cp313-manylinux_2_28_x86_64.whl#sha256=f134344006f0989a2d771554b7905fb05bd93d63b195e64626fde3495ec6f287 ; platform_system == 'Linux' and python_version == '3.13' and platform_machine == 'x86_64'", + "torchvision @ https://download.pytorch.org/whl/xpu/torchvision-0.26.0%2Bxpu-cp310-cp310-win_amd64.whl#sha256=7e52729cb9736c66dc79a7f42de6b31db93b9161d3357fd34cfa33f5fe32b8ea ; sys_platform == 'win32' and python_version == '3.10' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", + "torchvision @ https://download.pytorch.org/whl/xpu/torchvision-0.26.0%2Bxpu-cp311-cp311-win_amd64.whl#sha256=83a6130100c6b6750d8aa9fd29e5d0c53b1c85b1153b8ed4139aea54fc1892cc ; sys_platform == 'win32' and python_version == '3.11' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", + "torchvision @ https://download.pytorch.org/whl/xpu/torchvision-0.26.0%2Bxpu-cp312-cp312-win_amd64.whl#sha256=03788e0e5a5b85a2f09d11f0263d579fcb0cf5623d8810149be0e37836c2738c ; sys_platform == 'win32' and python_version == '3.12' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", + "torchvision @ https://download.pytorch.org/whl/xpu/torchvision-0.26.0%2Bxpu-cp313-cp313-win_amd64.whl#sha256=cb1da1d378ce440f7d1e0ed8cf21bd280d904ab25a55c9453f8377825818df74 ; sys_platform == 'win32' and python_version == '3.13' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", +] +intel-gpu-torch2110 = [ + "unsloth[intelgputorch2110]" +] +intelgputorch2120 = [ + "unsloth_zoo[intelgpu]", + "unsloth[huggingfacenotorch]", + + "triton-xpu @ https://download.pytorch.org/whl/triton_xpu-3.7.1-cp310-cp310-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl#sha256=81ff0eb0c4fc8e19d2510b28c3e1d9382a3c7d6fdaf6a9f9631a93a030d841cf ; platform_system == 'Linux' and python_version == '3.10' and platform_machine == 'x86_64'", + "triton-xpu @ https://download.pytorch.org/whl/triton_xpu-3.7.1-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl#sha256=55574a68d275b85cd4d5cbf185084bae019ebf09c3f43b0bd2831b14935ec8e7 ; platform_system == 'Linux' and python_version == '3.11' and platform_machine == 'x86_64'", + "triton-xpu @ https://download.pytorch.org/whl/triton_xpu-3.7.1-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl#sha256=a31c058c5c2e78ebe490a2e69f2f50caec6b1307ac096e944f116fdc06819d9a ; platform_system == 'Linux' and python_version == '3.12' and platform_machine == 'x86_64'", + "triton-xpu @ https://download.pytorch.org/whl/triton_xpu-3.7.1-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl#sha256=e701a31efa0334775f357c98716f3821775aa944219f7888e13c2dfe2daabe2a ; platform_system == 'Linux' and python_version == '3.13' and platform_machine == 'x86_64'", + "triton-xpu @ https://download.pytorch.org/whl/triton_xpu-3.7.1-cp310-cp310-win_amd64.whl#sha256=0d7730651c3e52fbf3a430cc201455f0c6600dc72e681aec495f131ea44f341a ; sys_platform == 'win32' and python_version == '3.10' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", + "triton-xpu @ https://download.pytorch.org/whl/triton_xpu-3.7.1-cp311-cp311-win_amd64.whl#sha256=8f4a63de73e3d632098f93c8f0bd77244958a47d7c5f728b8ff35f8a91fdb983 ; sys_platform == 'win32' and python_version == '3.11' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", + "triton-xpu @ https://download.pytorch.org/whl/triton_xpu-3.7.1-cp312-cp312-win_amd64.whl#sha256=6589ece3adc2b1ab88d90ff1267afc25df5c7b868f0b633e732cac70df36cbde ; sys_platform == 'win32' and python_version == '3.12' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", + "triton-xpu @ https://download.pytorch.org/whl/triton_xpu-3.7.1-cp313-cp313-win_amd64.whl#sha256=2fdf001a9b0575e8b1827127259bb9b13bf36e659882be74c2dfab46597d3e7a ; sys_platform == 'win32' and python_version == '3.13' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", + + "torch @ https://download.pytorch.org/whl/xpu/torch-2.12.0%2Bxpu-cp310-cp310-linux_x86_64.whl#sha256=e8923cd1fe560472904b1461b745d2f1826bb9c1bc0808225d5f28a450e4d553 ; platform_system == 'Linux' and python_version == '3.10' and platform_machine == 'x86_64'", + "torch @ https://download.pytorch.org/whl/xpu/torch-2.12.0%2Bxpu-cp311-cp311-linux_x86_64.whl#sha256=f7c082b2fc9b61def594d30ea57762dc4a8bc7111a9a9593953ed948de242e28 ; platform_system == 'Linux' and python_version == '3.11' and platform_machine == 'x86_64'", + "torch @ https://download.pytorch.org/whl/xpu/torch-2.12.0%2Bxpu-cp312-cp312-linux_x86_64.whl#sha256=f59decc04bec27862ed0197554a52370dbcba3e6892616d1fbce450e402bf2d5 ; platform_system == 'Linux' and python_version == '3.12' and platform_machine == 'x86_64'", + "torch @ https://download.pytorch.org/whl/xpu/torch-2.12.0%2Bxpu-cp313-cp313-linux_x86_64.whl#sha256=56f74e7c6c096e1a7ac215eb79ee590b764be3fbba8f4febc145bca47194a083 ; platform_system == 'Linux' and python_version == '3.13' and platform_machine == 'x86_64'", + "torch @ https://download.pytorch.org/whl/xpu/torch-2.12.0%2Bxpu-cp310-cp310-win_amd64.whl#sha256=b9779b71457b5a916ae052ed2467c10273cae4862d469b191359173b2038c53e ; sys_platform == 'win32' and python_version == '3.10' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", + "torch @ https://download.pytorch.org/whl/xpu/torch-2.12.0%2Bxpu-cp311-cp311-win_amd64.whl#sha256=7ef8e776c992e4e3ae007ebc108eb4f36b1d1dd9da97ecb308ab7fded89a2659 ; sys_platform == 'win32' and python_version == '3.11' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", + "torch @ https://download.pytorch.org/whl/xpu/torch-2.12.0%2Bxpu-cp312-cp312-win_amd64.whl#sha256=7f1d40febf2b8724adf4ff23866897d87478cc43de2a20f7776dc00be334c464 ; sys_platform == 'win32' and python_version == '3.12' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", + "torch @ https://download.pytorch.org/whl/xpu/torch-2.12.0%2Bxpu-cp313-cp313-win_amd64.whl#sha256=32770e2613df26e2c81ae64ea001b2ca12b8d152231285caff9b5f963a21ad75 ; sys_platform == 'win32' and python_version == '3.13' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", + + "bitsandbytes @ https://github.com/bitsandbytes-foundation/bitsandbytes/releases/download/continuous-release_main/bitsandbytes-1.33.7.preview-py3-none-manylinux_2_24_x86_64.whl ; ('linux' in sys_platform) and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", + "bitsandbytes @ https://github.com/bitsandbytes-foundation/bitsandbytes/releases/download/continuous-release_main/bitsandbytes-1.33.7.preview-py3-none-win_amd64.whl ; (sys_platform == 'win32') and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", + + "torchvision @ https://download.pytorch.org/whl/xpu/torchvision-0.27.0%2Bxpu-cp310-cp310-manylinux_2_28_x86_64.whl#sha256=0d517462caf6f5201c0d7c880f4ac431783c88fcc59b4587836da6c72a89509c ; platform_system == 'Linux' and python_version == '3.10' and platform_machine == 'x86_64'", + "torchvision @ https://download.pytorch.org/whl/xpu/torchvision-0.27.0%2Bxpu-cp311-cp311-manylinux_2_28_x86_64.whl#sha256=4b6feada86aa0bd606904b05898b33538106120d8ed706ba11d0011046534cb8 ; platform_system == 'Linux' and python_version == '3.11' and platform_machine == 'x86_64'", + "torchvision @ https://download.pytorch.org/whl/xpu/torchvision-0.27.0%2Bxpu-cp312-cp312-manylinux_2_28_x86_64.whl#sha256=e231819be0f87829c2344c909c1f0db9d6ae7d6faefe644a526a1a01d0c18d98 ; platform_system == 'Linux' and python_version == '3.12' and platform_machine == 'x86_64'", + "torchvision @ https://download.pytorch.org/whl/xpu/torchvision-0.27.0%2Bxpu-cp313-cp313-manylinux_2_28_x86_64.whl#sha256=8bc7d37515cea18af4c389d5fde58b1a9d76b015f2d87e4a7dc62ad50b1cc200 ; platform_system == 'Linux' and python_version == '3.13' and platform_machine == 'x86_64'", + "torchvision @ https://download.pytorch.org/whl/xpu/torchvision-0.27.0%2Bxpu-cp310-cp310-win_amd64.whl#sha256=65dbb041057dddfe369f29cfaab63f75563621779a23a7b1e2c0ff8a84d4376a ; sys_platform == 'win32' and python_version == '3.10' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", + "torchvision @ https://download.pytorch.org/whl/xpu/torchvision-0.27.0%2Bxpu-cp311-cp311-win_amd64.whl#sha256=df647445365924d69fe3bb2a15a7edfe5b63ef91e4ae69af11d93582985237a4 ; sys_platform == 'win32' and python_version == '3.11' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", + "torchvision @ https://download.pytorch.org/whl/xpu/torchvision-0.27.0%2Bxpu-cp312-cp312-win_amd64.whl#sha256=b0db3df0d0d154d18ba988ab420f1da2549f9372113ff54ff66e4ae3c7fe3bd0 ; sys_platform == 'win32' and python_version == '3.12' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", + "torchvision @ https://download.pytorch.org/whl/xpu/torchvision-0.27.0%2Bxpu-cp313-cp313-win_amd64.whl#sha256=c70850842068c43a0d50eaf139c25b6f6cc9b17a0dae70218c7e69edbee0bc80 ; sys_platform == 'win32' and python_version == '3.13' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", +] +intel-gpu-torch2120 = [ + "unsloth[intelgputorch2120]" +] +intel = [ + "unsloth[intelgputorch280]", +] +amd = [ + "unsloth[huggingfacenotorch]", + # 4-bit decode is unreliable on ROCm before 0.50.0, the first PyPI release + # carrying the full path: blocksize/warp decoupling (bnb #1887), fused SIMT + # GEMM on RDNA (#1979), RDNA3/4 workgroup fix (#2012). + "bitsandbytes>=0.50.0 ; ('linux' in sys_platform) and (platform_machine == 'AMD64' or platform_machine == 'x86_64' or platform_machine == 'aarch64')", + "bitsandbytes>=0.50.0 ; (sys_platform == 'win32') and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", +] +rocm702-torch280 = [ + "unsloth[amd]", + + "triton @ https://repo.radeon.com/rocm/manylinux/rocm-rel-7.0.2/triton-3.4.0%2Brocm7.0.2.gitf9e5bf54-cp311-cp311-linux_x86_64.whl ; platform_system == 'Linux' and python_version == '3.11' and platform_machine == 'x86_64'", + "triton @ https://repo.radeon.com/rocm/manylinux/rocm-rel-7.0.2/triton-3.4.0%2Brocm7.0.2.gitf9e5bf54-cp312-cp312-linux_x86_64.whl ; platform_system == 'Linux' and python_version == '3.12' and platform_machine == 'x86_64'", + "triton @ https://repo.radeon.com/rocm/manylinux/rocm-rel-7.0.2/triton-3.4.0%2Brocm7.0.2.gitf9e5bf54-cp313-cp313-linux_x86_64.whl ; platform_system == 'Linux' and python_version == '3.13' and platform_machine == 'x86_64'", + + "torch @ https://repo.radeon.com/rocm/manylinux/rocm-rel-7.0.2/torch-2.8.0%2Brocm7.0.2.lw.git245bf6ed-cp311-cp311-linux_x86_64.whl ; platform_system == 'Linux' and python_version == '3.11' and platform_machine == 'x86_64'", + "torch @ https://repo.radeon.com/rocm/manylinux/rocm-rel-7.0.2/torch-2.8.0%2Brocm7.0.2.lw.git245bf6ed-cp312-cp312-linux_x86_64.whl ; platform_system == 'Linux' and python_version == '3.12' and platform_machine == 'x86_64'", + "torch @ https://repo.radeon.com/rocm/manylinux/rocm-rel-7.0.2/torch-2.8.0%2Brocm7.0.2.lw.git245bf6ed-cp313-cp313-linux_x86_64.whl ; platform_system == 'Linux' and python_version == '3.13' and platform_machine == 'x86_64'", + + "torchvision @ https://repo.radeon.com/rocm/manylinux/rocm-rel-7.0.2/torchvision-0.23.0%2Brocm7.0.2.git824e8c87-cp311-cp311-linux_x86_64.whl ; platform_system == 'Linux' and python_version == '3.11' and platform_machine == 'x86_64'", + "torchvision @ https://repo.radeon.com/rocm/manylinux/rocm-rel-7.0.2/torchvision-0.23.0%2Brocm7.0.2.git824e8c87-cp312-cp312-linux_x86_64.whl ; platform_system == 'Linux' and python_version == '3.12' and platform_machine == 'x86_64'", + "torchvision @ https://repo.radeon.com/rocm/manylinux/rocm-rel-7.0.2/torchvision-0.23.0%2Brocm7.0.2.git824e8c87-cp313-cp313-linux_x86_64.whl ; platform_system == 'Linux' and python_version == '3.13' and platform_machine == 'x86_64'", +] +rocm72-torch291 = [ + "unsloth[amd]", + + "triton @ https://repo.radeon.com/rocm/manylinux/rocm-rel-7.2/triton-3.5.1%2Brocm7.2.0.gita272dfa8-cp310-cp310-linux_x86_64.whl ; platform_system == 'Linux' and python_version == '3.10' and platform_machine == 'x86_64'", + "triton @ https://repo.radeon.com/rocm/manylinux/rocm-rel-7.2/triton-3.5.1%2Brocm7.2.0.gita272dfa8-cp311-cp311-linux_x86_64.whl ; platform_system == 'Linux' and python_version == '3.11' and platform_machine == 'x86_64'", + "triton @ https://repo.radeon.com/rocm/manylinux/rocm-rel-7.2/triton-3.5.1%2Brocm7.2.0.gita272dfa8-cp312-cp312-linux_x86_64.whl ; platform_system == 'Linux' and python_version == '3.12' and platform_machine == 'x86_64'", + "triton @ https://repo.radeon.com/rocm/manylinux/rocm-rel-7.2/triton-3.5.1%2Brocm7.2.0.gita272dfa8-cp313-cp313-linux_x86_64.whl ; platform_system == 'Linux' and python_version == '3.13' and platform_machine == 'x86_64'", + + "torch @ https://repo.radeon.com/rocm/manylinux/rocm-rel-7.2/torch-2.9.1%2Brocm7.2.0.lw.git7e1940d4-cp310-cp310-linux_x86_64.whl ; platform_system == 'Linux' and python_version == '3.10' and platform_machine == 'x86_64'", + "torch @ https://repo.radeon.com/rocm/manylinux/rocm-rel-7.2/torch-2.9.1%2Brocm7.2.0.lw.git7e1940d4-cp311-cp311-linux_x86_64.whl ; platform_system == 'Linux' and python_version == '3.11' and platform_machine == 'x86_64'", + "torch @ https://repo.radeon.com/rocm/manylinux/rocm-rel-7.2/torch-2.9.1%2Brocm7.2.0.lw.git7e1940d4-cp312-cp312-linux_x86_64.whl ; platform_system == 'Linux' and python_version == '3.12' and platform_machine == 'x86_64'", + "torch @ https://repo.radeon.com/rocm/manylinux/rocm-rel-7.2/torch-2.9.1%2Brocm7.2.0.lw.git7e1940d4-cp313-cp313-linux_x86_64.whl ; platform_system == 'Linux' and python_version == '3.13' and platform_machine == 'x86_64'", + "torch @ https://repo.radeon.com/rocm/windows/rocm-rel-7.2/torch-2.9.1%2Brocmsdk20260116-cp312-cp312-win_amd64.whl ; sys_platform == 'win32' and python_version == '3.12'", + + "torchvision @ https://repo.radeon.com/rocm/manylinux/rocm-rel-7.2/torchvision-0.24.0%2Brocm7.2.0.gitb919bd0c-cp310-cp310-linux_x86_64.whl ; platform_system == 'Linux' and python_version == '3.10' and platform_machine == 'x86_64'", + "torchvision @ https://repo.radeon.com/rocm/manylinux/rocm-rel-7.2/torchvision-0.24.0%2Brocm7.2.0.gitb919bd0c-cp311-cp311-linux_x86_64.whl ; platform_system == 'Linux' and python_version == '3.11' and platform_machine == 'x86_64'", + "torchvision @ https://repo.radeon.com/rocm/manylinux/rocm-rel-7.2/torchvision-0.24.0%2Brocm7.2.0.gitb919bd0c-cp312-cp312-linux_x86_64.whl ; platform_system == 'Linux' and python_version == '3.12' and platform_machine == 'x86_64'", + "torchvision @ https://repo.radeon.com/rocm/manylinux/rocm-rel-7.2/torchvision-0.24.0%2Brocm7.2.0.gitb919bd0c-cp313-cp313-linux_x86_64.whl ; platform_system == 'Linux' and python_version == '3.13' and platform_machine == 'x86_64'", + "torchvision @ https://repo.radeon.com/rocm/windows/rocm-rel-7.2/torchvision-0.24.1%2Brocmsdk20260116-cp312-cp312-win_amd64.whl ; sys_platform == 'win32' and python_version == '3.12'", +] +rocm711-torch291 = [ + "unsloth[amd]", + + "triton @ https://repo.radeon.com/rocm/manylinux/rocm-rel-7.1.1/triton-3.5.1%2Brocm7.1.1.gita272dfa8-cp310-cp310-linux_x86_64.whl ; platform_system == 'Linux' and python_version == '3.10' and platform_machine == 'x86_64'", + "triton @ https://repo.radeon.com/rocm/manylinux/rocm-rel-7.1.1/triton-3.5.1%2Brocm7.1.1.gita272dfa8-cp311-cp311-linux_x86_64.whl ; platform_system == 'Linux' and python_version == '3.11' and platform_machine == 'x86_64'", + "triton @ https://repo.radeon.com/rocm/manylinux/rocm-rel-7.1.1/triton-3.5.1%2Brocm7.1.1.gita272dfa8-cp312-cp312-linux_x86_64.whl ; platform_system == 'Linux' and python_version == '3.12' and platform_machine == 'x86_64'", + "triton @ https://repo.radeon.com/rocm/manylinux/rocm-rel-7.1.1/triton-3.5.1%2Brocm7.1.1.gita272dfa8-cp313-cp313-linux_x86_64.whl ; platform_system == 'Linux' and python_version == '3.13' and platform_machine == 'x86_64'", + + "torch @ https://repo.radeon.com/rocm/manylinux/rocm-rel-7.1.1/torch-2.9.1%2Brocm7.1.1.lw.git351ff442-cp310-cp310-linux_x86_64.whl ; platform_system == 'Linux' and python_version == '3.10' and platform_machine == 'x86_64'", + "torch @ https://repo.radeon.com/rocm/manylinux/rocm-rel-7.1.1/torch-2.9.1%2Brocm7.1.1.lw.git351ff442-cp311-cp311-linux_x86_64.whl ; platform_system == 'Linux' and python_version == '3.11' and platform_machine == 'x86_64'", + "torch @ https://repo.radeon.com/rocm/manylinux/rocm-rel-7.1.1/torch-2.9.1%2Brocm7.1.1.lw.git351ff442-cp312-cp312-linux_x86_64.whl ; platform_system == 'Linux' and python_version == '3.12' and platform_machine == 'x86_64'", + "torch @ https://repo.radeon.com/rocm/manylinux/rocm-rel-7.1.1/torch-2.9.1%2Brocm7.1.1.lw.git351ff442-cp313-cp313-linux_x86_64.whl ; platform_system == 'Linux' and python_version == '3.13' and platform_machine == 'x86_64'", + + "torchvision @ https://repo.radeon.com/rocm/manylinux/rocm-rel-7.1.1/torchvision-0.24.0%2Brocm7.1.1.gitb919bd0c-cp310-cp310-linux_x86_64.whl ; platform_system == 'Linux' and python_version == '3.10' and platform_machine == 'x86_64'", + "torchvision @ https://repo.radeon.com/rocm/manylinux/rocm-rel-7.1.1/torchvision-0.24.0%2Brocm7.1.1.gitb919bd0c-cp311-cp311-linux_x86_64.whl ; platform_system == 'Linux' and python_version == '3.11' and platform_machine == 'x86_64'", + "torchvision @ https://repo.radeon.com/rocm/manylinux/rocm-rel-7.1.1/torchvision-0.24.0%2Brocm7.1.1.gitb919bd0c-cp312-cp312-linux_x86_64.whl ; platform_system == 'Linux' and python_version == '3.12' and platform_machine == 'x86_64'", + "torchvision @ https://repo.radeon.com/rocm/manylinux/rocm-rel-7.1.1/torchvision-0.24.0%2Brocm7.1.1.gitb919bd0c-cp313-cp313-linux_x86_64.whl ; platform_system == 'Linux' and python_version == '3.13' and platform_machine == 'x86_64'", +] +rocm72-torch2100 = [ + "unsloth[amd]", + + "triton @ https://repo.radeon.com/rocm/manylinux/rocm-rel-7.2/triton-3.6.0%2Brocm7.2.0.gitba5c1517-cp310-cp310-linux_x86_64.whl ; platform_system == 'Linux' and python_version == '3.10' and platform_machine == 'x86_64'", + "triton @ https://repo.radeon.com/rocm/manylinux/rocm-rel-7.2/triton-3.6.0%2Brocm7.2.0.gitba5c1517-cp311-cp311-linux_x86_64.whl ; platform_system == 'Linux' and python_version == '3.11' and platform_machine == 'x86_64'", + "triton @ https://repo.radeon.com/rocm/manylinux/rocm-rel-7.2/triton-3.6.0%2Brocm7.2.0.gitba5c1517-cp312-cp312-linux_x86_64.whl ; platform_system == 'Linux' and python_version == '3.12' and platform_machine == 'x86_64'", + "triton @ https://repo.radeon.com/rocm/manylinux/rocm-rel-7.2/triton-3.6.0%2Brocm7.2.0.gitba5c1517-cp313-cp313-linux_x86_64.whl ; platform_system == 'Linux' and python_version == '3.13' and platform_machine == 'x86_64'", + + "torch @ https://repo.radeon.com/rocm/manylinux/rocm-rel-7.2/torch-2.10.0%2Brocm7.2.0.lw.gitb6ee5fde-cp310-cp310-linux_x86_64.whl ; platform_system == 'Linux' and python_version == '3.10' and platform_machine == 'x86_64'", + "torch @ https://repo.radeon.com/rocm/manylinux/rocm-rel-7.2/torch-2.10.0%2Brocm7.2.0.lw.gitb6ee5fde-cp311-cp311-linux_x86_64.whl ; platform_system == 'Linux' and python_version == '3.11' and platform_machine == 'x86_64'", + "torch @ https://repo.radeon.com/rocm/manylinux/rocm-rel-7.2/torch-2.10.0%2Brocm7.2.0.lw.gitb6ee5fde-cp312-cp312-linux_x86_64.whl ; platform_system == 'Linux' and python_version == '3.12' and platform_machine == 'x86_64'", + "torch @ https://repo.radeon.com/rocm/manylinux/rocm-rel-7.2/torch-2.10.0%2Brocm7.2.0.lw.gitb6ee5fde-cp313-cp313-linux_x86_64.whl ; platform_system == 'Linux' and python_version == '3.13' and platform_machine == 'x86_64'", + + "torchvision @ https://repo.radeon.com/rocm/manylinux/rocm-rel-7.2/torchvision-0.25.0%2Brocm7.2.0.git82df5f59-cp310-cp310-linux_x86_64.whl ; platform_system == 'Linux' and python_version == '3.10' and platform_machine == 'x86_64'", + "torchvision @ https://repo.radeon.com/rocm/manylinux/rocm-rel-7.2/torchvision-0.25.0%2Brocm7.2.0.git82df5f59-cp311-cp311-linux_x86_64.whl ; platform_system == 'Linux' and python_version == '3.11' and platform_machine == 'x86_64'", + "torchvision @ https://repo.radeon.com/rocm/manylinux/rocm-rel-7.2/torchvision-0.25.0%2Brocm7.2.0.git82df5f59-cp312-cp312-linux_x86_64.whl ; platform_system == 'Linux' and python_version == '3.12' and platform_machine == 'x86_64'", + "torchvision @ https://repo.radeon.com/rocm/manylinux/rocm-rel-7.2/torchvision-0.25.0%2Brocm7.2.0.git82df5f59-cp313-cp313-linux_x86_64.whl ; platform_system == 'Linux' and python_version == '3.13' and platform_machine == 'x86_64'", + "unsloth[audio-torch210]", +] +rocm711-torch2100 = [ + "unsloth[amd]", + + "triton @ https://repo.radeon.com/rocm/manylinux/rocm-rel-7.1.1/triton-3.6.0%2Brocm7.1.1.gitba5c1517-cp310-cp310-linux_x86_64.whl ; platform_system == 'Linux' and python_version == '3.10' and platform_machine == 'x86_64'", + "triton @ https://repo.radeon.com/rocm/manylinux/rocm-rel-7.1.1/triton-3.6.0%2Brocm7.1.1.gitba5c1517-cp311-cp311-linux_x86_64.whl ; platform_system == 'Linux' and python_version == '3.11' and platform_machine == 'x86_64'", + "triton @ https://repo.radeon.com/rocm/manylinux/rocm-rel-7.1.1/triton-3.6.0%2Brocm7.1.1.gitba5c1517-cp312-cp312-linux_x86_64.whl ; platform_system == 'Linux' and python_version == '3.12' and platform_machine == 'x86_64'", + "triton @ https://repo.radeon.com/rocm/manylinux/rocm-rel-7.1.1/triton-3.6.0%2Brocm7.1.1.gitba5c1517-cp313-cp313-linux_x86_64.whl ; platform_system == 'Linux' and python_version == '3.13' and platform_machine == 'x86_64'", + + "torch @ https://repo.radeon.com/rocm/manylinux/rocm-rel-7.1.1/torch-2.10.0%2Brocm7.1.1.lw.gitd9556b05-cp310-cp310-linux_x86_64.whl ; platform_system == 'Linux' and python_version == '3.10' and platform_machine == 'x86_64'", + "torch @ https://repo.radeon.com/rocm/manylinux/rocm-rel-7.1.1/torch-2.10.0%2Brocm7.1.1.lw.gitd9556b05-cp311-cp311-linux_x86_64.whl ; platform_system == 'Linux' and python_version == '3.11' and platform_machine == 'x86_64'", + "torch @ https://repo.radeon.com/rocm/manylinux/rocm-rel-7.1.1/torch-2.10.0%2Brocm7.1.1.lw.gitd9556b05-cp312-cp312-linux_x86_64.whl ; platform_system == 'Linux' and python_version == '3.12' and platform_machine == 'x86_64'", + "torch @ https://repo.radeon.com/rocm/manylinux/rocm-rel-7.1.1/torch-2.10.0%2Brocm7.1.1.lw.gitd9556b05-cp313-cp313-linux_x86_64.whl ; platform_system == 'Linux' and python_version == '3.13' and platform_machine == 'x86_64'", + + "torchvision @ https://repo.radeon.com/rocm/manylinux/rocm-rel-7.1.1/torchvision-0.25.0%2Brocm7.1.1.git82df5f59-cp310-cp310-linux_x86_64.whl ; platform_system == 'Linux' and python_version == '3.10' and platform_machine == 'x86_64'", + "torchvision @ https://repo.radeon.com/rocm/manylinux/rocm-rel-7.1.1/torchvision-0.25.0%2Brocm7.1.1.git82df5f59-cp311-cp311-linux_x86_64.whl ; platform_system == 'Linux' and python_version == '3.11' and platform_machine == 'x86_64'", + "torchvision @ https://repo.radeon.com/rocm/manylinux/rocm-rel-7.1.1/torchvision-0.25.0%2Brocm7.1.1.git82df5f59-cp312-cp312-linux_x86_64.whl ; platform_system == 'Linux' and python_version == '3.12' and platform_machine == 'x86_64'", + "torchvision @ https://repo.radeon.com/rocm/manylinux/rocm-rel-7.1.1/torchvision-0.25.0%2Brocm7.1.1.git82df5f59-cp313-cp313-linux_x86_64.whl ; platform_system == 'Linux' and python_version == '3.13' and platform_machine == 'x86_64'", + "unsloth[audio-torch210]", ] [project.urls] @@ -826,5 +1404,5 @@ ignore = [ # Narrow the default test discovery so `pytest` from the repo root # does NOT pick up the GPU-heavy tests under tests/python, tests/qlora, # etc. The CI security job runs `pytest tests/security` explicitly. -pythonpath = ["."] testpaths = ["tests/security"] +pythonpath = ["."] diff --git a/scripts/profile_startup.py b/scripts/profile_startup.py new file mode 100644 index 0000000000..937d007ac1 --- /dev/null +++ b/scripts/profile_startup.py @@ -0,0 +1,377 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 + +"""Measure where Unsloth Studio's startup time goes, per platform. + +Nothing measured this before: the backend logs "lifespan startup completed in X ms" +but no test or CI job asserted a budget, and studio_test_kit discards the elapsed +time of its /healthz poll. A first local run (Linux, warm cache, fast server CPU) +found `import main` alone costs 6.6s before the server can bind, dominated by eager +module-level imports pulled in by the `routes` package: + + torch 1930 ms self + unsloth_zoo 914 ms self + routes 779 ms self + transformers 524 ms self + +Phases measured: + import `python -X importtime -c "import main"`, top cumulative + per-package self + spawn process start -> first byte on stdout + healthz process start -> /api/health (or /healthz) answers 200 + lifespan the backend's own "lifespan startup completed in X ms" log line + +Usage: + python scripts/profile_startup.py --repeats 3 --json out.json + python scripts/profile_startup.py --import-only # no server, no port needed + +Exit code is 0 unless --max-healthz-seconds is given and exceeded. +""" + +from __future__ import annotations + +import argparse +import json +import math +import os +import platform +import re +import shutil +import socket +import statistics +import subprocess +import sys +import threading +import time +import urllib.error +import urllib.request +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parents[1] +BACKEND = REPO_ROOT / "studio" / "backend" + +_IMPORTTIME_RE = re.compile(r"import time:\s+(\d+)\s+\|\s+(\d+)\s+\|(\s*)(\S.*)") + + +def _free_port() -> int: + with socket.socket() as s: + s.bind(("127.0.0.1", 0)) + return int(s.getsockname()[1]) + + +def profile_imports(python: str, top: int = 15) -> dict: + """Cumulative and self import cost for the backend's module graph. + + Run in a subprocess with -X importtime: the numbers are only meaningful for a + cold interpreter, and importing in-process would measure a warm sys.modules. + """ + proc = subprocess.run( + [python, "-X", "importtime", "-c", "import sys; sys.path.insert(0, '.'); import main"], + cwd = BACKEND, + capture_output = True, + text = True, + timeout = 900, + ) + rows = [] + for line in proc.stderr.splitlines(): + m = _IMPORTTIME_RE.match(line) + if m: + rows.append((int(m.group(1)), int(m.group(2)), m.group(4).strip())) + if not rows: + return {"ok": False, "error": (proc.stderr or proc.stdout)[-2000:]} + if proc.returncode != 0: + # Rows survive up to the failure, so any total from a partial graph is wrong. + return { + "ok": False, + "error": (proc.stderr or proc.stdout)[-2000:], + "partial_rows": len(rows), + } + + by_cum = sorted(rows, key = lambda r: -r[1]) + # Total comes from the `main` row, not by_cum[0]: -X importtime also prints the + # interpreter's own startup graph (`site`), which can outrank a trivial main. + main_row = next((r for r in reversed(rows) if r[2] == "main"), None) + if main_row is None: + return { + "ok": False, + "error": "no `import main` row in -X importtime output\n" + + (proc.stderr or proc.stdout)[-2000:], + } + self_by_pkg: dict[str, int] = {} + for self_us, _cum, name in rows: + pkg = name.split(".")[0] + self_by_pkg[pkg] = self_by_pkg.get(pkg, 0) + self_us + + return { + "ok": True, + "total_seconds": round(main_row[1] / 1e6, 3), + "top_cumulative": [ + {"module": n, "seconds": round(c / 1e6, 3)} for _s, c, n in by_cum[:top] + ], + "self_by_package_ms": { + k: round(v / 1000) for k, v in sorted(self_by_pkg.items(), key = lambda x: -x[1])[:top] + }, + } + + +def _terminate_tree(proc: subprocess.Popen) -> None: + """Stop the server AND its children, which on Windows are a separate process. + + CI profiles `Scripts/unsloth.exe`, a distlib launcher stub that CreateProcess's + the venv python and waits, so terminate() reaps the stub only: the real backend + keeps the inherited stdout handle, the reader thread never sees EOF, and + --repeats strands one server per iteration on the shared UNSLOTH_STUDIO_HOME. + taskkill /T walks the tree, as unsloth_cli/commands/start.py already does. + """ + if proc.poll() is not None: + return + if os.name == "nt": + try: + killed = subprocess.run( + ["taskkill", "/PID", str(proc.pid), "/T", "/F"], + capture_output = True, + timeout = 30, + check = False, + ) + if killed.returncode == 0: + return + except Exception: + # taskkill missing or timed out; fall through so the stub still dies. + pass + # check=False: a nonzero taskkill does not raise, so fall through as well. + proc.terminate() + + +def profile_launch( + bin_path: str, + port: int, + timeout_s: int = 300, +) -> dict: + """Spawn the backend the way the desktop app does and time it to first 200.""" + log_lines: list[str] = [] + first_byte: list[float] = [] + t0 = time.perf_counter() + proc = subprocess.Popen( + [bin_path, "studio", "--api-only", "-H", "127.0.0.1", "-p", str(port)], + cwd = REPO_ROOT, + stdout = subprocess.PIPE, + stderr = subprocess.STDOUT, + text = True, + bufsize = 1, + ) + + def _drain() -> None: + # Runs alongside the health polling: the first read timestamps the spawn + # phase, and an undrained pipe blocks the backend before it binds. + for line in proc.stdout: + if not first_byte: + first_byte.append(time.perf_counter() - t0) + log_lines.append(line.rstrip("\n")) + + reader = threading.Thread(target = _drain, daemon = True) + reader.start() + + t_healthz = None + deadline = t0 + timeout_s + try: + while time.perf_counter() < deadline: + if proc.poll() is not None: + break + if t_healthz is None: + for url in ( + f"http://127.0.0.1:{port}/api/health", + f"http://127.0.0.1:{port}/healthz", + ): + try: + with urllib.request.urlopen(url, timeout = 2) as r: + if r.status == 200: + t_healthz = time.perf_counter() - t0 + break + except (urllib.error.URLError, OSError, TimeoutError): + pass + if t_healthz is not None: + break + time.sleep(0.25) + finally: + _terminate_tree(proc) + try: + # Safe: the reader drains the pipe, so the child cannot block on write(). + proc.wait(timeout = 30) + except subprocess.TimeoutExpired: + proc.kill() + proc.wait() + reader.join(timeout = 10) + + t_first_byte = first_byte[0] if first_byte else None + lifespan_ms = None + for line in log_lines: + m = re.search(r"lifespan startup completed in ([\d.]+)ms", line) + if m: + lifespan_ms = float(m.group(1)) + return { + "spawn_seconds": round(t_first_byte, 3) if t_first_byte is not None else None, + "healthz_seconds": round(t_healthz, 3) if t_healthz is not None else None, + "lifespan_ms": lifespan_ms, + "reached_healthz": t_healthz is not None, + "log_tail": log_lines[-25:], + } + + +def python_version_of(python: str) -> str: + """Version of the interpreter that runs the imports, not the one running us. + + --python points at the installed Studio venv while this script runs under the + runner's system python, so platform.python_version() would label it wrong. + """ + if python == sys.executable: + return platform.python_version() + try: + proc = subprocess.run( + [python, "-c", "import platform; print(platform.python_version())"], + capture_output = True, + text = True, + timeout = 60, + ) + if proc.returncode == 0 and proc.stdout.strip(): + return proc.stdout.strip() + except (OSError, subprocess.SubprocessError): + pass + return "unknown" + + +def find_bin() -> str | None: + home = os.environ.get("UNSLOTH_STUDIO_HOME") or str(Path.home() / ".unsloth" / "studio") + names = ["unsloth.exe", "unsloth"] if platform.system() == "Windows" else ["unsloth"] + subdirs = ["unsloth_studio/Scripts", "unsloth_studio/bin", "bin", "Scripts"] + for sd in subdirs: + for n in names: + p = Path(home) / sd / n + if p.exists(): + return str(p) + return shutil.which("unsloth") + + +def main(argv: list[str]) -> int: + ap = argparse.ArgumentParser( + description = __doc__, formatter_class = argparse.RawDescriptionHelpFormatter + ) + ap.add_argument( + "--repeats", + type = int, + default = 1, + help = "launch repeats; the median is reported (imports are measured once)", + ) + ap.add_argument( + "--python", + default = sys.executable, + help = "interpreter used for the import profile (default: this one)", + ) + ap.add_argument("--bin", help = "path to the unsloth CLI (default: autodetect)") + ap.add_argument( + "--import-only", + action = "store_true", + help = "skip the server phases (no install needed beyond the deps)", + ) + ap.add_argument( + "--max-healthz-seconds", + type = float, + help = "fail if the median time to a healthy port exceeds this", + ) + ap.add_argument("--json", help = "write the full report here") + a = ap.parse_args(argv) + # range(0) launches nothing, leaving the budget check with nothing to fail on. + if a.repeats < 1: + ap.error("--repeats must be at least 1") + # Same reason: --import-only never launches anything. + if a.import_only and a.max_healthz_seconds is not None: + ap.error("--max-healthz-seconds cannot be combined with --import-only") + # nan and inf parse fine as floats but `med > budget` is then always False, + # so the gate would report success without ever bounding anything. + if a.max_healthz_seconds is not None and not math.isfinite(a.max_healthz_seconds): + ap.error("--max-healthz-seconds must be a finite number") + + report: dict = { + "platform": platform.system().lower(), + "machine": platform.machine(), + "python": python_version_of(a.python), + "cpu_count": os.cpu_count(), + } + + print("== import graph ==") + report["imports"] = profile_imports(a.python) + imp = report["imports"] + if imp.get("ok"): + print(f" import main: {imp['total_seconds']}s") + for row in imp["top_cumulative"][:8]: + print(f" {row['seconds']:7.3f}s {row['module']}") + print(" self time by package (ms):") + for k, v in list(imp["self_by_package_ms"].items())[:8]: + print(f" {v:8} ms {k}") + else: + print(f" FAILED: {imp.get('error', '')[:400]}") + + if not a.import_only: + bin_path = a.bin or find_bin() + if not bin_path: + print( + "== launch == skipped: no unsloth CLI found " + "(set UNSLOTH_STUDIO_HOME or pass --bin)" + ) + report["launch"] = {"skipped": "no unsloth CLI found"} + else: + print(f"== launch == {bin_path}") + runs = [] + for i in range(a.repeats): + r = profile_launch(bin_path, _free_port()) + runs.append(r) + print( + f" run {i + 1}: healthz={r['healthz_seconds']}s " + f"lifespan={r['lifespan_ms']}ms reached={r['reached_healthz']}" + ) + got = [r["healthz_seconds"] for r in runs if r["healthz_seconds"] is not None] + report["launch"] = { + "runs": runs, + "failed_runs": sum(1 for r in runs if not r["reached_healthz"]), + "healthz_median_seconds": round(statistics.median(got), 3) if got else None, + "healthz_max_seconds": round(max(got), 3) if got else None, + } + if got: + print( + f" median time to healthy port: {report['launch']['healthz_median_seconds']}s" + ) + + if a.json: + Path(a.json).write_text(json.dumps(report, indent = 2), encoding = "utf-8") + print(f"\nwrote {a.json}") + + if a.max_healthz_seconds is not None: + launch = report.get("launch") or {} + med = launch.get("healthz_median_seconds") + failed = launch.get("failed_runs") or 0 + if failed: + # Failed launches fail the budget; dropping them would keep only the fast ones. + print( + f"::error::startup regression: {failed} of {len(launch.get('runs') or [])} " + f"launches never became healthy within the timeout" + ) + return 1 + if med is None: + # Nothing measured: exiting 0 would pass a requested budget without a + # single health request, so fail closed. + print( + "::error::startup regression: no healthz measurement, so the " + f"{a.max_healthz_seconds}s budget was never checked " + f"({launch.get('skipped') or 'launch phase produced no runs'})" + ) + return 1 + elif med > a.max_healthz_seconds: + print( + f"::error::startup regression: {med}s median to a healthy port " + f"exceeds the {a.max_healthz_seconds}s budget" + ) + return 1 + return 0 + + +if __name__ == "__main__": + raise SystemExit(main(sys.argv[1:])) diff --git a/studio/backend/auth/authentication.py b/studio/backend/auth/authentication.py index dfb8fc513e..2e9520827e 100644 --- a/studio/backend/auth/authentication.py +++ b/studio/backend/auth/authentication.py @@ -11,11 +11,12 @@ import jwt from .storage import ( API_KEY_PREFIX, + credential_generation, get_jwt_secret, get_user_and_secret, load_jwt_secret, save_refresh_token, - validate_api_key, + validate_api_key_with_credential, verify_refresh_token, ) @@ -54,11 +55,14 @@ def create_access_token( expires_delta: Optional[timedelta] = None, *, desktop: bool = False, + secret: Optional[str] = None, ) -> str: """ Create a signed JWT for the given subject (e.g. username). - Valid across restarts: the signing secret is stored in SQLite. + Valid across restarts: the signing secret is stored in SQLite. Callers that + already verified a credential pass ``secret`` so a rotation landing mid-request + cannot sign the token with the credential that just replaced it. """ to_encode = {"sub": subject} if desktop: @@ -69,7 +73,7 @@ def create_access_token( to_encode.update({"exp": expire}) return jwt.encode( to_encode, - _get_secret_for_subject(subject), + secret if secret is not None else _get_secret_for_subject(subject), algorithm = ALGORITHM, ) @@ -96,15 +100,28 @@ def is_desktop_access_token(token: str) -> bool: return payload.get("sub") == subject and payload.get("desktop") is True -def create_refresh_token(subject: str, *, desktop: bool = False) -> str: +def create_refresh_token( + subject: str, + *, + desktop: bool = False, + secret: Optional[str] = None, +) -> str: """ Create a random refresh token, store its hash in SQLite, and return it. Refresh tokens are opaque (not JWTs); expire after REFRESH_TOKEN_EXPIRE_DAYS. + ``secret`` stamps the token with the credential version the caller verified, + so a rotation cannot leave a token minted from the replaced credential valid. """ token = secrets.token_urlsafe(48) expires_at = datetime.now(timezone.utc) + timedelta(days = REFRESH_TOKEN_EXPIRE_DAYS) - save_refresh_token(token, subject, expires_at.isoformat(), is_desktop = desktop) + save_refresh_token( + token, + subject, + expires_at.isoformat(), + is_desktop = desktop, + secret_gen = credential_generation(secret) if secret is not None else None, + ) return token @@ -137,7 +154,22 @@ def reload_secret() -> None: async def get_current_subject(credentials: HTTPAuthorizationCredentials = Depends(security)) -> str: """Validate JWT and require the password-change flow to be completed.""" - return await _get_current_subject( + subject, _generation = await _get_current_credential( + credentials, + allow_password_change = False, + ) + return subject + + +async def get_current_credential( + credentials: HTTPAuthorizationCredentials = Depends(security), +) -> Tuple[str, Optional[str]]: + """As get_current_subject, but also returns the credential generation. + + For routes that persist a new credential and must not do so on behalf of one + a concurrent reset has revoked. + """ + return await _get_current_credential( credentials, allow_password_change = False, ) @@ -158,27 +190,49 @@ async def get_current_subject_allow_password_change( credentials: HTTPAuthorizationCredentials = Depends(security), ) -> str: """Validate JWT but allow access to the password-change endpoint.""" - return await _get_current_subject( + subject, _generation = await _get_current_credential( credentials, allow_password_change = True, ) + return subject -async def _get_current_subject( +# The literal the examples ship with; pasted unedited more often than a revoked key. +API_KEY_PLACEHOLDER = f"{API_KEY_PREFIX}YOUR_KEY" + + +def _invalid_api_key_detail(token: str) -> str: + """Why the key failed. Only the example placeholder is called out; every real + key gets one indistinguishable message, so this leaks no key existence.""" + if token == API_KEY_PLACEHOLDER: + return ( + "This is the placeholder key from the example. Create an API key in " + f"Unsloth Studio under Settings > API and use it in place of {API_KEY_PLACEHOLDER}." + ) + return "Invalid or expired API key" + + +async def _get_current_credential( credentials: HTTPAuthorizationCredentials, *, allow_password_change: bool -) -> str: - """FastAPI dependency: validate the JWT and return the subject. Use on protected routes.""" +) -> Tuple[str, Optional[str]]: + """Validate the bearer and return ``(subject, credential generation)``. + + The generation is the credential version this request actually authenticated + against. Routes that persist new credentials must bind their write to it, or + a reset landing mid-request would bless what it just revoked. + """ token = credentials.credentials # --- API key path (sk-unsloth-...) --- if token.startswith(API_KEY_PREFIX): - username = validate_api_key(token) - if username is None: + verified = validate_api_key_with_credential(token) + if verified is None: raise HTTPException( status_code = status.HTTP_401_UNAUTHORIZED, - detail = "Invalid or expired API key", + detail = _invalid_api_key_detail(token), ) - return username + username, secret = verified + return username, credential_generation(secret) # --- JWT path --- subject = _decode_subject_without_verification(token) @@ -209,7 +263,7 @@ async def _get_current_subject( status_code = status.HTTP_403_FORBIDDEN, detail = "Password change required", ) - return subject + return subject, credential_generation(jwt_secret) except jwt.InvalidTokenError: raise HTTPException( status_code = status.HTTP_401_UNAUTHORIZED, diff --git a/studio/backend/auth/storage.py b/studio/backend/auth/storage.py index 5f80ad89a3..6cf4d44834 100644 --- a/studio/backend/auth/storage.py +++ b/studio/backend/auth/storage.py @@ -9,6 +9,7 @@ import ipaddress import os import secrets import sqlite3 +import tempfile import threading from datetime import datetime, timezone from typing import Optional, Tuple @@ -30,6 +31,97 @@ _BOOTSTRAP_PW_PATH = DB_PATH.parent / ".bootstrap_password" _bootstrap_password: Optional[str] = None +def _bootstrap_file_bytes(password: str) -> bytes: + """Exact on-disk form: the secret plus one LF. + + Bytes, not text: text mode writes CRLF on Windows, and `$(cat ...)` strips + the LF but leaves the CR attached to the credential. + """ + return (password + "\n").encode("utf-8") + + +def _persist_bootstrap_password(password: str) -> None: + """Atomically write the bootstrap password 0600, LF terminated on every OS. + + A partial write would destroy the only plaintext recovery credential. + """ + fd, tmp_name = tempfile.mkstemp( + prefix = f".{_BOOTSTRAP_PW_PATH.name}.", dir = _BOOTSTRAP_PW_PATH.parent + ) + try: + with os.fdopen(fd, "wb") as f: + f.write(_bootstrap_file_bytes(password)) + try: + os.chmod(tmp_name, 0o600) + except OSError: + pass + os.replace(tmp_name, _BOOTSTRAP_PW_PATH) + except BaseException: + try: + os.unlink(tmp_name) + except OSError: + pass + raise + + +def _normalise_bootstrap_file(raw: bytes, password: str) -> None: + """Append the LF a pre-newline release left off. + + Append-only, and only when the file is exactly the credential: + clear_bootstrap_password() may unlink or (when unlink fails, notably on + Windows while this descriptor is open) truncate through another descriptor + after we read, so a rewrite could restore revoked plaintext. An append + cannot: worst case is a lone "\\n" over a cleared file, which strips back to + no bootstrap password. Pre-newline releases wrote no terminator at all, so + that is the only shape in the wild; anything else reads fine, since every + reader strips, and is left alone. + """ + if raw != password.encode("utf-8"): + return + + # O_BINARY: without it Windows opens in text mode and turns the LF straight + # back into CRLF, the bug being fixed. + fd = os.open( + _BOOTSTRAP_PW_PATH, + os.O_WRONLY | os.O_APPEND | getattr(os, "O_BINARY", 0), + ) + try: + os.write(fd, b"\n") + try: + os.fchmod(fd, 0o600) + except (AttributeError, OSError): + # fchmod only reached Windows in 3.13. + pass + finally: + os.close(fd) + + +def _read_persisted_bootstrap_password() -> Optional[str]: + """Read the persisted password, normalising the file if it is malformed.""" + if not _BOOTSTRAP_PW_PATH.is_file(): + return None + + # No caller handles a raise, so an unreadable file has to mean "no bootstrap + # password", not a dead backend. We write UTF-8, so undecodable bytes are + # damage whose plaintext is worthless anyway. + try: + raw = _BOOTSTRAP_PW_PATH.read_bytes() + password = raw.decode("utf-8").strip() + except (OSError, UnicodeDecodeError): + return None + if not password: + return None + + # Older releases wrote no terminator; best-effort, a read-only auth dir must + # not fail startup. + if raw != _bootstrap_file_bytes(password): + try: + _normalise_bootstrap_file(raw, password) + except OSError: + pass + return password + + def generate_bootstrap_password() -> str: """Generate a 4-word diceware passphrase and persist it to disk. @@ -43,10 +135,10 @@ def generate_bootstrap_password() -> str: return _bootstrap_password # Persisted from a previous run? - if _BOOTSTRAP_PW_PATH.is_file(): - _bootstrap_password = _BOOTSTRAP_PW_PATH.read_text(encoding = "utf-8").strip() - if _bootstrap_password: - return _bootstrap_password + persisted = _read_persisted_bootstrap_password() + if persisted: + _bootstrap_password = persisted + return _bootstrap_password # First startup: generate a fresh passphrase. import diceware @@ -57,11 +149,7 @@ def generate_bootstrap_password() -> str: # Persist so the same passphrase survives restarts until password change. ensure_dir(_BOOTSTRAP_PW_PATH.parent) - _BOOTSTRAP_PW_PATH.write_text(_bootstrap_password, encoding = "utf-8") - try: - os.chmod(_BOOTSTRAP_PW_PATH, 0o600) - except OSError: - pass + _persist_bootstrap_password(_bootstrap_password) return _bootstrap_password @@ -72,13 +160,14 @@ def get_bootstrap_password() -> Optional[str]: def _load_bootstrap_password() -> Optional[str]: - """Load an existing bootstrap password without creating one.""" + """Load an existing bootstrap password without creating one. + + Upgrades take this path, not generate_bootstrap_password() + (ensure_default_admin short-circuits once the admin row exists), so it has + to normalise too. + """ global _bootstrap_password - _bootstrap_password = None - if _BOOTSTRAP_PW_PATH.is_file(): - bootstrap_password = _BOOTSTRAP_PW_PATH.read_text(encoding = "utf-8").strip() - if bootstrap_password: - _bootstrap_password = bootstrap_password + _bootstrap_password = _read_persisted_bootstrap_password() return _bootstrap_password @@ -97,7 +186,7 @@ def clear_bootstrap_password() -> None: # Removal failed (Windows AV, read-only auth dir). The hash is already # committed, so don't fail the change -- but truncate the file so its # stale plaintext can't be re-seeded by generate_bootstrap_password() - # if a later reset-password deletes auth.db and re-validates it. + # if auth.db is ever recreated. try: _BOOTSTRAP_PW_PATH.write_text("", encoding = "utf-8") cleared = True @@ -132,6 +221,31 @@ def _hash_token(token: str) -> str: return hashlib.sha256(token.encode("utf-8")).hexdigest() +class CredentialRotated(Exception): + """A password reset revoked the credential this request authenticated with.""" + + +def credential_generation(jwt_secret: str) -> str: + """Marker for the credential version a refresh token was issued under. + + Every password change rotates ``jwt_secret``, so a token stamped with the + previous one is rejected even if it was inserted after the revoking DELETE. + """ + return hashlib.sha256(jwt_secret.encode("utf-8")).hexdigest() + + +def _current_secret(conn: sqlite3.Connection, username: str) -> Optional[str]: + row = conn.execute( + "SELECT jwt_secret FROM auth_user WHERE username = ?", (username,) + ).fetchone() + return row["jwt_secret"] if row else None + + +def _current_generation(conn: sqlite3.Connection, username: str) -> Optional[str]: + secret = _current_secret(conn, username) + return credential_generation(secret) if secret is not None else None + + def get_connection() -> sqlite3.Connection: """Get a connection to the auth database, creating tables if needed.""" ensure_dir(DB_PATH.parent) @@ -175,7 +289,8 @@ def get_connection() -> sqlite3.Connection: token_hash TEXT NOT NULL, username TEXT NOT NULL, expires_at TEXT NOT NULL, - is_desktop INTEGER NOT NULL DEFAULT 0 + is_desktop INTEGER NOT NULL DEFAULT 0, + secret_gen TEXT ); """ ) @@ -214,6 +329,8 @@ def get_connection() -> sqlite3.Connection: refresh_columns = {row["name"] for row in conn.execute("PRAGMA table_info(refresh_tokens)")} if "is_desktop" not in refresh_columns: conn.execute("ALTER TABLE refresh_tokens ADD COLUMN is_desktop INTEGER NOT NULL DEFAULT 0") + if "secret_gen" not in refresh_columns: + conn.execute("ALTER TABLE refresh_tokens ADD COLUMN secret_gen TEXT") conn.commit() return conn @@ -587,12 +704,22 @@ def update_password( new_password: str, *, revoke_refresh_tokens: bool = False, -) -> bool: + expect_password_hash: Optional[str] = None, +) -> Optional[str]: """Update password, clear first-login requirement, rotate JWT secret. + Returns the new JWT secret, or None when nothing was updated. Callers that + mint tokens for the caller must sign with the returned secret: re-reading it + would pick up a reset that landed between this commit and the mint. + ``revoke_refresh_tokens`` deletes the user's refresh tokens in the SAME transaction: a separate delete could fail after the password commit and leave a pre-change token still able to mint access tokens. + + ``expect_password_hash`` makes the write conditional on the credential the + caller verified still being current, so a request that checked the old + password cannot overwrite a reset that landed while it was in flight. + Returns False when the credential moved underneath it. """ from .hashing import hash_password @@ -600,21 +727,32 @@ def update_password( jwt_secret = secrets.token_urlsafe(64) conn = get_connection() try: - cursor = conn.execute( - """ - UPDATE auth_user - SET password_salt = ?, password_hash = ?, jwt_secret = ?, must_change_password = 0 - WHERE username = ? - """, - (salt, pwd_hash, jwt_secret, username), - ) + if expect_password_hash is None: + cursor = conn.execute( + """ + UPDATE auth_user + SET password_salt = ?, password_hash = ?, jwt_secret = ?, must_change_password = 0 + WHERE username = ? + """, + (salt, pwd_hash, jwt_secret, username), + ) + else: + cursor = conn.execute( + """ + UPDATE auth_user + SET password_salt = ?, password_hash = ?, jwt_secret = ?, must_change_password = 0 + WHERE username = ? AND password_hash = ? + """, + (salt, pwd_hash, jwt_secret, username, expect_password_hash), + ) if revoke_refresh_tokens and cursor.rowcount > 0: conn.execute("DELETE FROM refresh_tokens WHERE username = ?", (username,)) conn.commit() if cursor.rowcount > 0: clear_bootstrap_password() clear_desktop_secret() - return cursor.rowcount > 0 + return jwt_secret + return None finally: conn.close() @@ -625,35 +763,49 @@ def save_refresh_token( expires_at: str, *, is_desktop: bool = False, + secret_gen: Optional[str] = None, ) -> None: """ Store a hashed refresh token with its associated username and expiry. + + ``secret_gen`` binds the token to a credential version; it defaults to the + current one, and callers that already verified a credential must pass the + version they verified rather than let this re-read a rotated one. """ token_hash = _hash_token(token) conn = get_connection() try: + if secret_gen is None: + secret_gen = _current_generation(conn, username) conn.execute( """ - INSERT INTO refresh_tokens (token_hash, username, expires_at, is_desktop) - VALUES (?, ?, ?, ?) + INSERT INTO refresh_tokens (token_hash, username, expires_at, is_desktop, secret_gen) + VALUES (?, ?, ?, ?, ?) """, - (token_hash, username, expires_at, int(is_desktop)), + (token_hash, username, expires_at, int(is_desktop), secret_gen), ) conn.commit() finally: conn.close() -def consume_refresh_token(token: str) -> Optional[Tuple[str, bool]]: +def consume_refresh_token(token: str) -> Optional[Tuple[str, bool, str]]: """Atomically validate-and-delete a refresh token for single-use rotation. DELETE RETURNING fuses validate and delete into one statement so two - concurrent refresh requests cannot both consume the same token. + concurrent refresh requests cannot both consume the same token. Returns + ``(username, is_desktop, jwt_secret)``; the caller must mint the replacement + tokens against that secret so a rotation landing mid-refresh cannot issue a + post-rotation session from a pre-rotation token. """ token_hash = _hash_token(token) now = datetime.now(timezone.utc).isoformat() conn = get_connection() try: + # One transaction with the delete: an unstamped legacy row has no + # generation to compare, so reading the credential after committing would + # hand a reset's new secret to a token issued before it. + conn.execute("BEGIN IMMEDIATE") conn.execute( "DELETE FROM refresh_tokens WHERE expires_at < ?", (now,), @@ -662,15 +814,21 @@ def consume_refresh_token(token: str) -> Optional[Tuple[str, bool]]: """ DELETE FROM refresh_tokens WHERE token_hash = ? AND expires_at >= ? - RETURNING username, is_desktop + RETURNING username, is_desktop, secret_gen """, (token_hash, now), ) row = cur.fetchone() - conn.commit() if row is None: + conn.commit() return None - return row["username"], bool(row["is_desktop"]) + secret = _current_secret(conn, row["username"]) + conn.commit() + if secret is None: + return None + if row["secret_gen"] is not None and row["secret_gen"] != credential_generation(secret): + return None + return row["username"], bool(row["is_desktop"]), secret finally: conn.close() @@ -694,7 +852,7 @@ def verify_refresh_token(token: str) -> Optional[Tuple[str, bool]]: cur = conn.execute( """ - SELECT id, username, expires_at, is_desktop FROM refresh_tokens + SELECT id, username, expires_at, is_desktop, secret_gen FROM refresh_tokens WHERE token_hash = ? """, (token_hash,), @@ -703,6 +861,13 @@ def verify_refresh_token(token: str) -> Optional[Tuple[str, bool]]: if row is None: return None + if row["secret_gen"] is not None and row["secret_gen"] != _current_generation( + conn, row["username"] + ): + conn.execute("DELETE FROM refresh_tokens WHERE id = ?", (row["id"],)) + conn.commit() + return None + # Check expiry expires_at = datetime.fromisoformat(row["expires_at"]) if datetime.now(timezone.utc) > expires_at: @@ -747,30 +912,41 @@ def create_desktop_secret() -> str: conn.close() -def validate_desktop_secret(raw_secret: str) -> Optional[str]: - """Return the real admin username when the desktop secret matches.""" +def validate_desktop_secret_with_credential(raw_secret: str) -> Optional[Tuple[str, str]]: + """Validate the desktop secret and return ``(username, jwt_secret)``. + + Both reads share one transaction so the returned secret is the credential + version the desktop secret was checked against; a reset landing mid-request + then invalidates the tokens minted from it rather than blessing them. + """ if not raw_secret.startswith(DESKTOP_SECRET_PREFIX): return None - if get_user_and_secret(DEFAULT_ADMIN_USERNAME) is None: - return None secret_hash = _pbkdf2_desktop_secret(raw_secret) conn = get_connection() try: - cur = conn.execute( + conn.execute("BEGIN") + row = conn.execute( "SELECT value FROM app_secrets WHERE key = ?", (_DESKTOP_SECRET_HASH_KEY,), - ) - row = cur.fetchone() - if row is None: + ).fetchone() + if row is None or not secrets.compare_digest(row["value"], secret_hash): return None - if not secrets.compare_digest(row["value"], secret_hash): + jwt_secret = _current_secret(conn, DEFAULT_ADMIN_USERNAME) + if jwt_secret is None: return None - return DEFAULT_ADMIN_USERNAME + return DEFAULT_ADMIN_USERNAME, jwt_secret finally: + conn.rollback() conn.close() +def validate_desktop_secret(raw_secret: str) -> Optional[str]: + """Return the real admin username when the desktop secret matches.""" + verified = validate_desktop_secret_with_credential(raw_secret) + return verified[0] if verified else None + + def clear_desktop_secret() -> None: """Remove backend-side desktop auth state.""" conn = get_connection() @@ -796,6 +972,7 @@ def create_api_key( name: str, expires_at: Optional[str] = None, internal: bool = False, + expect_gen: Optional[str] = None, ) -> Tuple[str, dict]: """Create a new API key for *username*. @@ -804,6 +981,10 @@ def create_api_key( Pass ``internal=True`` for keys minted by workflows (e.g. data-recipe runs) that should not appear in user-facing key listings. + + ``expect_gen`` ties the insert to the credential generation the request + authenticated under, so a session revoked by a concurrent password reset + cannot mint a key that outlives it. Raises ``CredentialRotated`` if it moved. """ raw_key = API_KEY_PREFIX + secrets.token_hex(16) key_hash = _pbkdf2_api_key(raw_key) @@ -812,6 +993,12 @@ def create_api_key( conn = get_connection() try: + if expect_gen is not None: + conn.execute("BEGIN IMMEDIATE") + if _current_generation(conn, username) != expect_gen: + raise CredentialRotated( + "The credential this request authenticated with was revoked." + ) conn.execute( """ INSERT INTO api_keys (username, key_prefix, key_hash, name, created_at, expires_at, is_internal) @@ -900,15 +1087,25 @@ def revoke_internal_api_key(key_id: int) -> bool: def validate_api_key(raw_key: str) -> Optional[str]: - """Validate *raw_key* and return the owning username, or ``None``. + """Validate *raw_key* and return the owning username, or ``None``.""" + verified = validate_api_key_with_credential(raw_key) + return verified[0] if verified else None - Also updates ``last_used_at`` on success. + +def validate_api_key_with_credential(raw_key: str) -> Optional[Tuple[str, str]]: + """Validate *raw_key* and return ``(username, jwt_secret)``, or ``None``. + + Also updates ``last_used_at`` on success. The key check and the credential + read share one write transaction, so the returned version is the one the key + was actually valid under: a reset committing right after cannot have its new + generation handed to a request the key it revoked authenticated. """ cache_id = _api_key_cache_id(raw_key) cached_hash = _api_key_hash_cache.get(cache_id) key_hash = cached_hash if cached_hash is not None else _pbkdf2_api_key(raw_key) conn = get_connection() try: + conn.execute("BEGIN IMMEDIATE") cur = conn.execute( "SELECT id, username, is_active, expires_at FROM api_keys WHERE key_hash = ?", (key_hash,), @@ -928,11 +1125,15 @@ def validate_api_key(raw_key: str) -> Optional[str]: expires = datetime.fromisoformat(row["expires_at"]) if datetime.now(timezone.utc) > expires: return None + secret = _current_secret(conn, row["username"]) + if secret is None: + return None conn.execute( "UPDATE api_keys SET last_used_at = ? WHERE id = ?", (datetime.now(timezone.utc).isoformat(), row["id"]), ) conn.commit() - return row["username"] + return row["username"], secret finally: + conn.rollback() conn.close() diff --git a/studio/backend/cloudflare_tunnel.py b/studio/backend/cloudflare_tunnel.py index 78fce0c70a..f7967e2faa 100644 --- a/studio/backend/cloudflare_tunnel.py +++ b/studio/backend/cloudflare_tunnel.py @@ -310,6 +310,7 @@ class CloudflareTunnel: stderr = subprocess.STDOUT, stdin = subprocess.DEVNULL, text = True, + encoding = "utf-8", errors = "replace", bufsize = 1, **_windows_hidden_kwargs(), diff --git a/studio/backend/core/data_recipe/local_callable_validators.py b/studio/backend/core/data_recipe/local_callable_validators.py index ffc81669ae..143895d781 100644 --- a/studio/backend/core/data_recipe/local_callable_validators.py +++ b/studio/backend/core/data_recipe/local_callable_validators.py @@ -257,6 +257,8 @@ def _run_oxc_batch( cwd = str(_OXC_TOOL_DIR), input = json.dumps(payload), text = True, + encoding = "utf-8", + errors = "replace", capture_output = True, check = False, env = env, diff --git a/studio/backend/core/inference/_vulkan_probe.py b/studio/backend/core/inference/_vulkan_probe.py index 706346daad..4bfefc21ce 100644 --- a/studio/backend/core/inference/_vulkan_probe.py +++ b/studio/backend/core/inference/_vulkan_probe.py @@ -6,12 +6,14 @@ Run in a short-lived subprocess (``python _vulkan_probe.py ``) so the Vulkan instance never lives in the long-running backend process. Loads the bundled ggml Vulkan backend from ```` and prints one -``\\t\\t\\t`` line per device to stdout. -Indices are ggml's own Vulkan device ordinals, which need not match nvidia-smi -order. ``is_igpu`` (from ggml's device type) is ``1`` for an integrated GPU -sharing system RAM. ``total_bytes`` is the device-local heap; the reader uses -it to reserve absolute headroom on a discrete card (parity with the CUDA/ROCm -fit) and ignores it for an iGPU, whose "VRAM" is shared system RAM. +``\\t\\t\\t\\t`` line per device to +stdout. Indices are ggml's own Vulkan device ordinals, which need not match +nvidia-smi order. ``is_igpu`` (from ggml's device type) is ``1`` for an +integrated GPU sharing system RAM. ``total_bytes`` is the device-local heap; +the reader uses it to reserve absolute headroom on a discrete card (parity +with the CUDA/ROCm fit) and ignores it for an iGPU, whose "VRAM" is shared +system RAM. ``name`` is ggml's device description (the marketing name, e.g. +"AMD Radeon RX 9070 XT"); empty when the registry lookup fails. Uses only the standard library so it stays runnable as a bare script. """ @@ -24,15 +26,30 @@ import sys _GGML_BACKEND_DEVICE_TYPE_IGPU = 2 -def _igpu_flags(base, lib, count: int) -> list[bool]: - """Per-device integrated-GPU flags via ggml's backend registry. +def _igpu_flags_and_names(base, lib, count: int) -> tuple[list[bool], list[str]]: + """Per-device integrated-GPU flags and descriptions via ggml's backend registry. The Vulkan reg enumerates devices in the same order as ``ggml_backend_vk_get_device_memory`` (each context uses ``ctx->device = - i``), so reg index == device ordinal. Returns all-False on any failure so - the reader never over-caps a discrete card. + i``), so reg index == device ordinal. Returns all-False / empty-name on any + failure so the reader never over-caps a discrete card and the memory + readings still get through. """ flags = [False] * count + names = [""] * count + + # The name lookup is bound OUTSIDE the type-detection try: a ggml-base + # without ggml_backend_dev_description (older/custom build) must degrade to + # unnamed devices, not abort before the iGPU flags are read (which would + # count an iGPU's shared RAM as VRAM). + describe = None + try: + base.ggml_backend_dev_description.restype = ctypes.c_char_p + base.ggml_backend_dev_description.argtypes = [ctypes.c_void_p] + describe = base.ggml_backend_dev_description + except Exception: + pass + try: lib.ggml_backend_vk_reg.restype = ctypes.c_void_p lib.ggml_backend_vk_reg.argtypes = [] @@ -45,17 +62,31 @@ def _igpu_flags(base, lib, count: int) -> list[bool]: reg = lib.ggml_backend_vk_reg() if not reg: - return flags + return flags, names dev_count = base.ggml_backend_reg_dev_count(reg) for i in range(min(count, dev_count)): dev = base.ggml_backend_reg_dev_get(reg, i) if dev: flags[i] = base.ggml_backend_dev_type(dev) == _GGML_BACKEND_DEVICE_TYPE_IGPU + if describe is not None: + try: + desc = describe(dev) + if desc: + # Tabs/newlines would corrupt the line protocol; + # spaces are safe. + names[i] = ( + desc.decode("utf-8", errors = "replace") + .replace("\t", " ") + .replace("\n", " ") + .strip() + ) + except Exception: + pass except Exception: - # Best-effort: any failure degrades to "discrete" so the memory - # readings still get through instead of crashing the probe. + # Best-effort: any failure degrades to "discrete"/"unnamed" so the + # memory readings still get through instead of crashing the probe. pass - return flags + return flags, names def main() -> int: @@ -63,6 +94,14 @@ def main() -> int: return 0 bindir = sys.argv[1] + # Device names can be non-ASCII (localized drivers); the platform-default + # stdout encoding (e.g. cp1252) would raise on them and lose the whole + # inventory. The reader decodes UTF-8 with the same error mode. + try: + sys.stdout.reconfigure(encoding = "utf-8", errors = "replace") + except Exception: + pass + # Hold add_dll_directory's handle for the rest of main() (the documented # idiom) so bindir stays on the search path while the sibling ggml DLLs # resolve below. @@ -96,12 +135,12 @@ def main() -> int: ] count = lib.ggml_backend_vk_get_device_count() - igpu = _igpu_flags(base, lib, count) + igpu, names = _igpu_flags_and_names(base, lib, count) rows = [] for i in range(count): free, total = ctypes.c_size_t(0), ctypes.c_size_t(0) lib.ggml_backend_vk_get_device_memory(i, ctypes.byref(free), ctypes.byref(total)) - rows.append("%d\t%d\t%d\t%d" % (i, free.value, int(igpu[i]), total.value)) + rows.append("%d\t%d\t%d\t%d\t%s" % (i, free.value, int(igpu[i]), total.value, names[i])) sys.stdout.write("\n".join(rows)) return 0 diff --git a/studio/backend/core/inference/anthropic_compat.py b/studio/backend/core/inference/anthropic_compat.py index 34445cc58e..a32e372d73 100644 --- a/studio/backend/core/inference/anthropic_compat.py +++ b/studio/backend/core/inference/anthropic_compat.py @@ -172,6 +172,136 @@ def anthropic_messages_to_openai( return result +_ANTHROPIC_SCHEMA_CLIENT_TOOL_PARAMETERS = { + "bash": { + "type": "object", + "properties": { + "command": {"type": "string"}, + "restart": {"type": "boolean"}, + }, + "anyOf": [ + {"required": ["command"]}, + {"properties": {"restart": {"const": True}}, "required": ["restart"]}, + ], + }, + "text_editor": { + "type": "object", + "properties": { + "command": { + "type": "string", + "enum": ["view", "str_replace", "create", "insert"], + }, + "path": {"type": "string"}, + "view_range": { + "type": "array", + "items": {"type": "integer"}, + "minItems": 2, + "maxItems": 2, + }, + "old_str": {"type": "string"}, + "new_str": {"type": "string"}, + "file_text": {"type": "string"}, + "insert_line": {"type": "integer"}, + "insert_text": {"type": "string"}, + }, + "required": ["command", "path"], + }, + "computer": { + "type": "object", + "properties": { + "action": {"type": "string"}, + "coordinate": { + "type": "array", + "items": {"type": "integer"}, + "minItems": 2, + "maxItems": 2, + }, + "text": {"type": "string"}, + "duration": {"type": "number"}, + "scroll_direction": {"type": "string"}, + "scroll_amount": {"type": "integer"}, + "start_coordinate": { + "type": "array", + "items": {"type": "integer"}, + "minItems": 2, + "maxItems": 2, + }, + "key": {"type": "string"}, + }, + "required": ["action"], + "additionalProperties": True, + }, + "memory": { + "type": "object", + "properties": { + "command": { + "type": "string", + "enum": ["view", "create", "str_replace", "insert", "delete", "rename"], + }, + "path": {"type": "string"}, + "view_range": { + "type": "array", + "items": {"type": "integer"}, + "minItems": 2, + "maxItems": 2, + }, + "file_text": {"type": "string"}, + "old_str": {"type": "string"}, + "new_str": {"type": "string"}, + "insert_line": {"type": "integer"}, + "insert_text": {"type": "string"}, + "old_path": {"type": "string"}, + "new_path": {"type": "string"}, + }, + "required": ["command"], + }, +} + +_ANTHROPIC_SCHEMA_CLIENT_TOOL_DESCRIPTIONS = { + "bash": "Run a command in the caller-owned persistent bash session, or restart it.", + "text_editor": "View, create, or edit files in the caller-owned filesystem.", + "computer": "Interact with the caller-owned computer using an action and its parameters.", + "memory": "Store and retrieve files in the caller-owned persistent memory directory.", +} + + +def anthropic_schema_client_tool_kind(tool) -> Optional[str]: + """Return the kind of a schema-less Anthropic client tool, if recognized.""" + td = tool if isinstance(tool, dict) else tool.model_dump() + if td.get("input_schema") is not None: + return None + type_ = td.get("type") + if not isinstance(type_, str): + return None + kind, separator, version = type_.rpartition("_") + if ( + separator + and kind in _ANTHROPIC_SCHEMA_CLIENT_TOOL_PARAMETERS + and len(version) == 8 + and version.isdigit() + ): + return kind + return None + + +def _anthropic_schema_client_tool_parameters(td: dict, kind: str) -> dict: + parameters = _ANTHROPIC_SCHEMA_CLIENT_TOOL_PARAMETERS[kind] + if kind != "text_editor": + return parameters + + version = td["type"].rpartition("_")[2] + commands = list(parameters["properties"]["command"]["enum"]) + if version < "20250429": + commands.append("undo_edit") + return { + **parameters, + "properties": { + **parameters["properties"], + "command": {**parameters["properties"]["command"], "enum": commands}, + }, + } + + def anthropic_tools_to_openai(tools: list) -> list[dict]: """Convert Anthropic client tools to OpenAI function-tool format.""" result = [] @@ -179,6 +309,9 @@ def anthropic_tools_to_openai(tools: list) -> list[dict]: td = t if isinstance(t, dict) else t.model_dump() name = td.get("name") input_schema = td.get("input_schema") + schema_client_kind = anthropic_schema_client_tool_kind(td) + if schema_client_kind is not None: + input_schema = _anthropic_schema_client_tool_parameters(td, schema_client_kind) if not name or input_schema is None: continue result.append( @@ -186,7 +319,8 @@ def anthropic_tools_to_openai(tools: list) -> list[dict]: "type": "function", "function": { "name": name, - "description": td.get("description", ""), + "description": td.get("description") + or _ANTHROPIC_SCHEMA_CLIENT_TOOL_DESCRIPTIONS.get(schema_client_kind, ""), "parameters": input_schema, }, } diff --git a/studio/backend/core/inference/api_monitor.py b/studio/backend/core/inference/api_monitor.py index f76a38576f..b637ba56d1 100644 --- a/studio/backend/core/inference/api_monitor.py +++ b/studio/backend/core/inference/api_monitor.py @@ -5,6 +5,7 @@ from __future__ import annotations +import os import threading import time import uuid @@ -18,6 +19,14 @@ _MAX_PROMPT_CHARS = 12000 _MAX_REPLY_CHARS = 12000 _PREVIEW_CHARS = 360 +# Opt-in startup kill switch for Studio's in-memory API monitor. +_DISABLE_ENV = "UNSLOTH_STUDIO_DISABLE_API_MONITOR" +_TRUE_VALUES = frozenset({"1", "true", "yes", "on"}) + + +def _api_monitor_disabled() -> bool: + return os.environ.get(_DISABLE_ENV, "").strip().lower() in _TRUE_VALUES + def _trim(text: Optional[str], limit: int) -> str: if not text: @@ -52,6 +61,13 @@ class ApiMonitorEntry: total_tokens: Optional[int] = None total_tokens_authoritative: bool = False error: Optional[str] = None + # "request" (HTTP call) or "lifecycle" (model load/unload: event/reason, not a prompt; shared). + kind: str = "request" + event: Optional[str] = None + reason: Optional[str] = None + shared: bool = False + # 0-100 for a running download row; None when not applicable. + progress: Optional[float] = None def snapshot(self, *, include_details: bool = True) -> dict[str, Any]: duration_ms = None @@ -85,6 +101,10 @@ class ApiMonitorEntry: "completion_tokens": self.completion_tokens, "total_tokens": self.total_tokens, "error": self.error, + "kind": self.kind, + "event": self.event, + "reason": self.reason, + "progress": self.progress, } if include_details: payload["prompt"] = self.prompt @@ -93,10 +113,16 @@ class ApiMonitorEntry: class ApiMonitor: - def __init__(self, max_entries: int = _MAX_ENTRIES): + def __init__( + self, + max_entries: int = _MAX_ENTRIES, + *, + enabled: bool = True, + ): self._entries: deque[ApiMonitorEntry] = deque() self._max_entries = max(0, max_entries) self._lock = threading.Lock() + self._enabled = enabled def start( self, @@ -108,6 +134,8 @@ class ApiMonitor: context_length: Optional[int] = None, subject: Optional[str] = None, ) -> str: + if not self._enabled: + return "" now = time.time() entry = ApiMonitorEntry( id = f"apireq_{uuid.uuid4().hex[:12]}", @@ -127,6 +155,75 @@ class ApiMonitor: self._trim_terminal_locked() return entry.id + def record_lifecycle( + self, + *, + event: str, + model: str, + reason: Optional[str] = None, + running: bool = False, + ) -> str: + """Record a model load/unload alongside the request traffic that caused it. + + ``running=True`` opens the row for the caller to close with :meth:`finish` / + :meth:`fail`; an unload is terminal on arrival. Rows are shared (visible to + every subject) and share the request retention budget. + """ + if not self._enabled: + return "" + now = time.time() + entry = ApiMonitorEntry( + id = f"apievt_{uuid.uuid4().hex[:12]}", + endpoint = f"model.{event}", + method = "", + model = model or "default", + prompt = "", + status = "running" if running else "completed", + started_at = now, + updated_at = now, + started_monotonic = time.monotonic(), + finished_at = None if running else now, + finished_monotonic = None if running else time.monotonic(), + kind = "lifecycle", + event = event, + reason = reason, + shared = True, + ) + with self._lock: + self._entries.appendleft(entry) + self._trim_terminal_locked() + return entry.id + + def relabel(self, entry_id: Optional[str], model: str) -> None: + """Rename an open lifecycle row once the load resolves its real id: up front + the caller only has the load path, which may be an HF snapshot dir.""" + if not entry_id or not model: + return + with self._lock: + entry = self._find_locked(entry_id) + if entry is not None: + entry.model = model + entry.updated_at = time.time() + + def set_progress(self, entry_id: Optional[str], progress: Optional[float]) -> None: + """Update an open download row's percentage (clamped to 0-100).""" + if not entry_id or progress is None: + return + with self._lock: + entry = self._find_locked(entry_id) + if entry is not None and entry.status == "running": + entry.progress = min(100.0, max(0.0, float(progress))) + entry.updated_at = time.time() + + def discard(self, entry_id: Optional[str]) -> None: + """Drop a row that turned out not to be an event (an already-satisfied load).""" + if not entry_id: + return + with self._lock: + entry = self._find_locked(entry_id) + if entry is not None: + self._entries.remove(entry) + def append_reply(self, entry_id: Optional[str], text: str) -> None: if not entry_id or not text: return @@ -212,6 +309,18 @@ class ApiMonitor: self._entries.appendleft(entry) self._trim_terminal_locked() + def fail_open(self, entry_id: Optional[str], error: str) -> None: + """Fail only a still-open row: unlike :meth:`fail`, a catch-all in a + ``finally`` cannot stamp an error onto a request that already succeeded.""" + if not entry_id: + return + with self._lock: + entry = self._find_locked(entry_id) + if entry is None or entry.finished_at is not None: + return + # Same lock as the check, so a finish() cannot land in between. + self._fail_locked(entry, error) + def fail(self, entry_id: Optional[str], error: str) -> None: if not entry_id: return @@ -224,15 +333,18 @@ class ApiMonitor: if error: entry.error = _trim(error, 1000) return - now = time.time() - entry.status = "error" - entry.error = _trim(error, 1000) - entry.updated_at = now - entry.finished_at = now - entry.finished_monotonic = time.monotonic() - self._entries.remove(entry) - self._entries.appendleft(entry) - self._trim_terminal_locked() + self._fail_locked(entry, error) + + def _fail_locked(self, entry: ApiMonitorEntry, error: str) -> None: + now = time.time() + entry.status = "error" + entry.error = _trim(error, 1000) + entry.updated_at = now + entry.finished_at = now + entry.finished_monotonic = time.monotonic() + self._entries.remove(entry) + self._entries.appendleft(entry) + self._trim_terminal_locked() def snapshot( self, @@ -244,7 +356,7 @@ class ApiMonitor: return [ entry.snapshot(include_details = include_details) for entry in self._entries - if subject is None or entry.subject == subject + if self._visible(entry, subject) ] def get( @@ -257,22 +369,29 @@ class ApiMonitor: entry = self._find_locked(entry_id) if entry is None: return None - if subject is not None and entry.subject != subject: + if not self._visible(entry, subject): return None return entry.snapshot(include_details = True) def active_count(self, *, subject: Optional[str] = None) -> int: + # Lifecycle rows show as "running" while loading but are not in-flight API requests. with self._lock: return sum( 1 for entry in self._entries - if entry.status == "running" and (subject is None or entry.subject == subject) + if entry.status == "running" + and entry.kind != "lifecycle" + and (subject is None or entry.subject == subject) ) def clear(self) -> None: with self._lock: self._entries.clear() + @staticmethod + def _visible(entry: ApiMonitorEntry, subject: Optional[str]) -> bool: + return subject is None or entry.subject == subject or entry.shared + def _find_locked(self, entry_id: str) -> Optional[ApiMonitorEntry]: for entry in self._entries: if entry.id == entry_id: @@ -292,4 +411,4 @@ class ApiMonitor: self._entries = kept -api_monitor = ApiMonitor() +api_monitor = ApiMonitor(enabled = not _api_monitor_disabled()) diff --git a/studio/backend/core/inference/chat_template_helpers.py b/studio/backend/core/inference/chat_template_helpers.py index 528c059fbc..3a8463855b 100644 --- a/studio/backend/core/inference/chat_template_helpers.py +++ b/studio/backend/core/inference/chat_template_helpers.py @@ -326,6 +326,58 @@ def _normalize_tool_call_arguments(messages: list) -> list: return out if mutated else messages +def _take_tool_result(pending: list, call_id) -> Optional[dict]: + if call_id: + for i, result in enumerate(pending): + if result.get("tool_call_id") == call_id: + return pending.pop(i) + for i, result in enumerate(pending): + if not result.get("tool_call_id"): + return pending.pop(i) + return None + + +def _split_parallel_tool_calls(messages: list) -> list: + """Llama 3.x templates render one call per message, so split parallel calls + into consecutive single-call messages, each followed by its own result.""" + if not any(isinstance(m, dict) and len(m.get("tool_calls") or ()) > 1 for m in messages): + return messages + + out: list = [] + i = 0 + total = len(messages) + while i < total: + msg = messages[i] + calls = msg.get("tool_calls") if isinstance(msg, dict) else None + if not calls or len(calls) <= 1: + out.append(msg) + i += 1 + continue + + # Tool results right after this message answer its calls. + j = i + 1 + pending: list = [] + while ( + j < total + and isinstance(messages[j], dict) + and messages[j].get("role") in ("tool", "ipython") + ): + pending.append(messages[j]) + j += 1 + + for idx, call in enumerate(calls): + piece = {**msg, "tool_calls": [call]} + if idx: + piece["content"] = "" + out.append(piece) + result = _take_tool_result(pending, call.get("id") if isinstance(call, dict) else None) + if result is not None: + out.append(result) + out.extend(pending) + i = j + return out + + def apply_chat_template_for_generation( tokenizer, messages: list, @@ -378,13 +430,21 @@ def apply_chat_template_for_generation( try: return _render(messages) except Exception: - # Strict tool templates reject the JSON-string ``arguments`` form via - # TypeError or a broad Jinja raise_exception, so retry with dicts coerced. - # Original messages render first, so working templates stay byte-identical. + # Retry with repairs applied cumulatively. Originals render first, so + # working templates stay byte-identical. + candidates: list = [] normalized = _normalize_tool_call_arguments(messages) - if normalized is messages: - raise - return _render(normalized) + if normalized is not messages: + candidates.append(normalized) + split = _split_parallel_tool_calls(normalized) + if split is not normalized: + candidates.append(split) + for candidate in candidates: + try: + return _render(candidate) + except Exception: + continue + raise def render_native_template( diff --git a/studio/backend/core/inference/inference.py b/studio/backend/core/inference/inference.py index 563a6732a1..e78bf1be8d 100644 --- a/studio/backend/core/inference/inference.py +++ b/studio/backend/core/inference/inference.py @@ -567,7 +567,7 @@ class InferenceBackend: _meta_path = Path(config.path) / "export_metadata.json" try: if _meta_path.exists(): - _meta = json.loads(_meta_path.read_text(encoding = "utf-8")) + _meta = json.loads(_meta_path.read_text(encoding = "utf-8-sig")) if _meta.get("base_model"): processor_source = _meta["base_model"] except Exception: @@ -2281,8 +2281,13 @@ class InferenceBackend: except Exception as e: logger.warning(f"Could not fully reset model state for {model_name}: {e}") - def reset_generation_state(self): - """Reset any cached generation state to prevent hanging after errors""" + def reset_generation_state(self, caller_cancel_event = None): + """Reset any cached generation state to prevent hanging after errors + + ``caller_cancel_event`` is accepted for signature parity with the + orchestrator, which uses it to drop a reset from a request that never + started. Nothing here cancels a live generation, so it is unused. + """ try: # Clear cached state for ALL loaded models for model_name in self.models.keys(): diff --git a/studio/backend/core/inference/llama_admission.py b/studio/backend/core/inference/llama_admission.py index b6a939c87b..7bf0dd7429 100644 --- a/studio/backend/core/inference/llama_admission.py +++ b/studio/backend/core/inference/llama_admission.py @@ -13,37 +13,159 @@ from __future__ import annotations import asyncio import os +import sys import threading from collections import deque from dataclasses import dataclass from typing import Deque, Optional -ADMISSION_CONTROL_ENV = "UNSLOTH_OPENAI_COMPAT_ADMISSION_CONTROL" -ADMISSION_QUEUE_TIMEOUT_ENV = "UNSLOTH_OPENAI_COMPAT_ADMISSION_QUEUE_TIMEOUT" -ADMISSION_KEEPALIVE_INTERVAL_ENV = "UNSLOTH_OPENAI_COMPAT_ADMISSION_KEEPALIVE_INTERVAL" -ADMISSION_MAX_QUEUE_ENV = "UNSLOTH_OPENAI_COMPAT_ADMISSION_MAX_QUEUE" +# dataclass(slots = True) halves per-instance overhead. Measured as perf-neutral +# here, not a speed win: it costs a little on construction and gains it back on +# access. It is 3.10+ and this package declares >=3.9, so gate it rather than +# dropping it outright. Empty on 3.9 means a plain dataclass. +_SLOTS = {"slots": True} if sys.version_info >= (3, 10) else {} + + +ADMISSION_CONTROL_ENV = "UNSLOTH_LLAMA_ADMISSION_CONTROL" +ADMISSION_QUEUE_TIMEOUT_ENV = "UNSLOTH_LLAMA_ADMISSION_QUEUE_TIMEOUT" +ADMISSION_KEEPALIVE_INTERVAL_ENV = "UNSLOTH_LLAMA_ADMISSION_KEEPALIVE_INTERVAL" +ADMISSION_MAX_QUEUE_ENV = "UNSLOTH_LLAMA_ADMISSION_MAX_QUEUE" +ADMISSION_QUEUE_PER_SLOT_ENV = "UNSLOTH_LLAMA_ADMISSION_QUEUE_PER_SLOT" + +# The UNSLOTH_OPENAI_COMPAT_* spellings predate this queue being shared with the +# Anthropic /v1/messages route (same llama-server slots). Still honored; the +# neutral name above wins when both are set. +_LEGACY_ENV = { + ADMISSION_CONTROL_ENV: "UNSLOTH_OPENAI_COMPAT_ADMISSION_CONTROL", + ADMISSION_QUEUE_TIMEOUT_ENV: "UNSLOTH_OPENAI_COMPAT_ADMISSION_QUEUE_TIMEOUT", + ADMISSION_KEEPALIVE_INTERVAL_ENV: "UNSLOTH_OPENAI_COMPAT_ADMISSION_KEEPALIVE_INTERVAL", + ADMISSION_MAX_QUEUE_ENV: "UNSLOTH_OPENAI_COMPAT_ADMISSION_MAX_QUEUE", +} DEFAULT_ADMISSION_ENABLED = True +# None: a queued request waits for its slot indefinitely rather than timing out. DEFAULT_ADMISSION_QUEUE_TIMEOUT_S = None DEFAULT_ADMISSION_KEEPALIVE_INTERVAL_S = 5.0 -DEFAULT_ADMISSION_MAX_QUEUE = 64 +# None: no absolute cap, the wait line is sized from the pool instead. +DEFAULT_ADMISSION_MAX_QUEUE = None +# Wait line = 16 x the serving slots, so it tracks --parallel (4 slots -> 64 +# waiters, 8 -> 128). Purely a memory guard; waiting itself is never timed out. +DEFAULT_ADMISSION_QUEUE_PER_SLOT = 16 +# Floor for the scaled line, so a 1-slot backend (plain `unsloth studio`, or any +# load downshifted to fit VRAM) keeps the depth it had before scaling existed +# rather than dropping to 16 and rejecting callers that used to queue. +DEFAULT_ADMISSION_MIN_QUEUE = 64 -@dataclass(frozen = True) +def _executor_workers() -> int: + """Threads asyncio's default executor runs to_thread work on. + + Mirrors ThreadPoolExecutor's own default sizing, which is what + ``run_in_executor(None, ...)`` builds. 3.13 sizes it from + ``process_cpu_count()``, which honours CPU affinity and cgroup quotas; + ``cpu_count()`` would budget from the whole host inside a one-core container. + """ + cpus = getattr(os, "process_cpu_count", os.cpu_count)() or 1 + return min(32, cpus + 4) + + +def _executor_reserve(workers: int) -> int: + """Threads kept clear of parked approvals, for generation steps, stream + teardown and unrelated to_thread work. Scaled rather than flat: a flat count + would leave a 5-worker executor (one usable CPU) no budget at all. + """ + return max(2, workers // 8) + + +def _max_parked(capacity: int) -> int: + """How many holders may sit on an approval prompt with their slot given back. + + A pending prompt parks an executor thread (the loop blocks inside + to_thread(next, gen)) whether or not it parked its slot, the pool already + permits `capacity` of those, and every park admits one more, so budget only + what the executor has left over. Zero on a backend whose --parallel alone + fills it: the prompt then holds its slot, as it did before parking existed. + """ + workers = _executor_workers() + spare = workers - _executor_reserve(workers) - max(0, capacity) + # A quarter of the executor, floored at two while `spare` allows: a quarter of + # five is one, and one park cannot cover the two simultaneous prompts #7455 + # exists for. + return max(0, min(max(2, workers // 4), spare)) + + +# Process-wide, not per queue: there is one executor, and base_url takes a fresh +# port on every load, so a per-queue budget would hand the same allowance to each +# backend and to every reload, blind to the approvals parked on the old queue. +_PARK_LOCK = threading.Lock() +_parked_total = 0 + + +def _claim_park(limit: int) -> bool: + global _parked_total + with _PARK_LOCK: + if _parked_total >= limit: + return False + _parked_total += 1 + return True + + +def _drop_park() -> None: + global _parked_total + with _PARK_LOCK: + _parked_total = max(0, _parked_total - 1) + + +def _live_capacity(current: "LlamaAdmissionQueue") -> int: + """Slots across every backend still serving requests. + + One queue's capacity is the wrong denominator for a budget sized against the + one executor: a reload drains the old queue alongside the new one, and + prompts on both park threads. Idle queues hold nothing and are about to be + evicted. + """ + with _QUEUES_LOCK: + queues = list(_QUEUES.values()) + # is_idle takes each queue's own lock, so never while holding _QUEUES_LOCK. + total = sum(queue._capacity for queue in queues if queue is current or not queue.is_idle()) + return total if any(queue is current for queue in queues) else total + current._capacity + + +@dataclass(frozen = True, **_SLOTS) class LlamaAdmissionConfig: enabled: bool = DEFAULT_ADMISSION_ENABLED queue_timeout_s: Optional[float] = DEFAULT_ADMISSION_QUEUE_TIMEOUT_S keepalive_interval_s: float = DEFAULT_ADMISSION_KEEPALIVE_INTERVAL_S max_queue: Optional[int] = DEFAULT_ADMISSION_MAX_QUEUE + queue_per_slot: Optional[int] = DEFAULT_ADMISSION_QUEUE_PER_SLOT + # Unconditional floor on the scaled line. The env path clears it when the + # operator sets QUEUE_PER_SLOT, so only the default multiplier is floored. + min_queue: Optional[int] = DEFAULT_ADMISSION_MIN_QUEUE + + def queue_limit(self, capacity: int) -> Optional[int]: + """How many callers may line up for a pool of ``capacity`` slots. + + An explicit ``max_queue`` wins; otherwise the line scales with the slots + so it follows ``--parallel``. The default multiplier is floored, so a + 1-slot backend does not end up shallower than it was before scaling. None + (or any non-positive setting) means an unbounded line. + """ + if self.max_queue is not None: + return self.max_queue if self.max_queue > 0 else None + if not self.queue_per_slot or self.queue_per_slot <= 0: + return None + scaled = self.queue_per_slot * max(1, capacity) + return max(self.min_queue, scaled) if self.min_queue else scaled -@dataclass(frozen = True) +@dataclass(frozen = True, **_SLOTS) class LlamaAdmissionSnapshot: key: str capacity: int active: int queued: int + free: int = 0 class LlamaAdmissionError(Exception): @@ -69,8 +191,17 @@ class LlamaAdmissionCancelled(LlamaAdmissionError): pass -def _bool_env(name: str, default: bool) -> bool: +def _raw_env(name: str) -> Optional[str]: + """Value for a canonical name, falling back to its legacy spelling.""" value = os.environ.get(name) + if value is None or not value.strip(): + legacy = _LEGACY_ENV.get(name) + value = os.environ.get(legacy) if legacy else None + return value + + +def _bool_env(name: str, default: bool) -> bool: + value = _raw_env(name) if value is None or not value.strip(): return default value = value.strip().lower() @@ -82,7 +213,7 @@ def _bool_env(name: str, default: bool) -> bool: def _optional_positive_float_env(name: str, default: Optional[float]) -> Optional[float]: - value = os.environ.get(name) + value = _raw_env(name) if value is None or not value.strip(): return default try: @@ -93,7 +224,7 @@ def _optional_positive_float_env(name: str, default: Optional[float]) -> Optiona def _positive_float_env(name: str, default: float) -> float: - value = os.environ.get(name) + value = _raw_env(name) if value is None or not value.strip(): return default try: @@ -103,19 +234,38 @@ def _positive_float_env(name: str, default: float) -> float: return parsed if parsed > 0 else default -def _optional_positive_int_env(name: str, default: Optional[int]) -> Optional[int]: - value = os.environ.get(name) - if value is None or not value.strip(): - return default +def _queue_limits_from_env() -> tuple[Optional[int], Optional[int], Optional[int]]: + """(max_queue, queue_per_slot, min_queue) from the environment. + + An absolute MAX_QUEUE wins outright; MAX_QUEUE=0 asks for an unbounded line. + Unset leaves the per-slot multiplier in charge (itself 0 for unbounded). The + floor applies only to the default multiplier: setting QUEUE_PER_SLOT means + the operator wants that exact depth, however shallow. + """ + # Explicit means it parsed, not just that something was set: a typo falls back + # to the default multiplier, so it has to keep the default's floor too. + raw_per_slot = _raw_env(ADMISSION_QUEUE_PER_SLOT_ENV) try: - parsed = int(value.strip()) + per_slot = int((raw_per_slot or "").strip()) except ValueError: - return default - return parsed if parsed > 0 else None + per_slot, min_queue = DEFAULT_ADMISSION_QUEUE_PER_SLOT, DEFAULT_ADMISSION_MIN_QUEUE + else: + per_slot, min_queue = (per_slot if per_slot > 0 else None), None + raw = _raw_env(ADMISSION_MAX_QUEUE_ENV) + if raw is None or not raw.strip(): + return None, per_slot, min_queue + try: + parsed = int(raw.strip()) + except ValueError: + return None, per_slot, min_queue + return (parsed, None, None) if parsed > 0 else (None, None, None) def llama_admission_config_from_env() -> LlamaAdmissionConfig: + max_queue, queue_per_slot, min_queue = _queue_limits_from_env() return LlamaAdmissionConfig( + queue_per_slot = queue_per_slot, + min_queue = min_queue, enabled = _bool_env(ADMISSION_CONTROL_ENV, DEFAULT_ADMISSION_ENABLED), queue_timeout_s = _optional_positive_float_env( ADMISSION_QUEUE_TIMEOUT_ENV, @@ -125,14 +275,11 @@ def llama_admission_config_from_env() -> LlamaAdmissionConfig: ADMISSION_KEEPALIVE_INTERVAL_ENV, DEFAULT_ADMISSION_KEEPALIVE_INTERVAL_S, ), - max_queue = _optional_positive_int_env( - ADMISSION_MAX_QUEUE_ENV, - DEFAULT_ADMISSION_MAX_QUEUE, - ), + max_queue = max_queue, ) -@dataclass +@dataclass(**_SLOTS) class _Waiter: loop: asyncio.AbstractEventLoop future: asyncio.Future @@ -141,20 +288,130 @@ class _Waiter: class LlamaAdmissionLease: - def __init__(self, queue: Optional["LlamaAdmissionQueue"]): + __slots__ = ("_queue", "_slot", "_released", "_release_lock", "_parked", "_budgeted") + + def __init__( + self, + queue: Optional["LlamaAdmissionQueue"], + slot: Optional[int] = None, + ): self._queue = queue + self._slot = slot self._released = False self._release_lock = threading.Lock() + self._parked = False + self._budgeted = False + + @property + def slot(self) -> Optional[int]: + """Pool slot this lease holds, or None when admission is disabled.""" + return self._slot + + def park(self) -> bool: + """Hand the slot back while this holder waits on something off the GPU. + + A run stopped on a tool approval prompt is not decoding, so holding its + slot would let unanswered prompts fill the pool while llama-server idles. + The lease itself stays valid: releasing it after a park is still correct. + + False when the park budget is spent and nothing was given back: the + caller keeps its slot across the prompt, as it did before parking + existed. Slower for whoever is behind it, but each freed slot admits + another run that can park too, on the executor the generators run on. + """ + queue = self._queue + with self._release_lock: + if queue is None or self._released or self._parked: + return False + # Under the lease lock so the decision and the handover cannot split. + # Nothing takes the queue lock then a lease lock, so this order is + # the only one in play. + if not queue.try_park(self._slot): + return False + self._parked = True + self._budgeted = True + self._slot = None + return True + + def _drop_budget(self) -> None: + """Give the executor budget back now the prompt wait is over. + + Separate from the queue's parked count, which lasts until the slot is + back: the executor thread is free the moment the answer arrives. Holding + the budget until the resume lands would refuse someone else's park for a + finished wait, and that someone holds the slot the resumer wants. + """ + with self._release_lock: + if not self._budgeted: + return + self._budgeted = False + _drop_park() + + def unpark(self) -> None: + """Drop the parked state without reclaiming a slot. + + For a holder that is tearing down: it will not decode again. Resuming + holders must use ``unpark_async``, which waits for a slot instead of + going back to llama-server past the admission limit. + """ + with self._release_lock: + if not self._parked: + return + self._parked = False + self._drop_budget() + if self._queue is not None: + self._queue.unpark() + + async def unpark_async( + self, + *, + cancel_event = None, + poll_s: float = 0.02, + ) -> None: + """Take a slot back, waiting until the pool has room. + + ``park`` gave the slot to a waiter, so by the time the user answers the + prompt someone else may be decoding in it. Resuming regardless put two + holders on a one-slot server. Gives up if the caller is cancelled, since + the holder is then leaving anyway and must not be stuck here. + """ + queue = self._queue + if queue is None or not self._parked: + return + # Before the wait, not after: the prompt is answered, so this holder is + # already off the executor and must not keep anyone else off it. + self._drop_budget() + slot = await queue.acquire_parked_slot(cancel_event = cancel_event, poll_s = poll_s) + stranded = None + with self._release_lock: + # release() may have run during the wait; it clears the flag and does + # the unpark itself, so only the caller that clears it here repeats one. + parked, self._parked = self._parked, False + if self._released: + # Released while waiting: this lease will never hand the slot + # back, so return it here rather than strand it for good. + stranded = slot + else: + self._slot = slot + if parked: + queue.unpark() + if stranded is not None: + queue.release(stranded) def release(self) -> None: queue = None + parked = False with self._release_lock: if self._released: return self._released = True queue = self._queue + parked, self._parked = self._parked, False + self._drop_budget() if queue is not None: - queue.release() + if parked: + queue.unpark() + queue.release(self._slot) async def __aenter__(self) -> "LlamaAdmissionLease": return self @@ -164,6 +421,8 @@ class LlamaAdmissionLease: class LlamaAdmissionReservation: + __slots__ = ("_queue", "_lease", "_waiter", "snapshot") + def __init__( self, *, @@ -195,6 +454,13 @@ class LlamaAdmissionReservation: return self._lease async def wait(self, timeout_s: float) -> Optional[LlamaAdmissionLease]: + """Wait up to ``timeout_s`` for a slot. + + A timeout leaves this reservation queued so the caller can poll again. + Any exit that abandons the wait for good must call ``cancel()``, or the + slot granted later is delivered to a future nobody reads and is never + released. + """ lease = self.lease_nowait() if lease is not None: return lease @@ -229,12 +495,74 @@ class LlamaAdmissionReservation: class LlamaAdmissionQueue: + """A fixed pool of generation slots for one llama-server, plus a FIFO wait line. + + The pool mirrors llama-server's own ``--parallel`` slots: ``capacity`` slot ids + are each either free or held by exactly one caller. A caller that finds every + slot busy waits in arrival order and is handed the next slot to free, so no + caller is starved. This bounds only the callers that reserve: chat completions + and messages do, while /v1/completions, Studio's own chat endpoint and RAG + captioning all reach llama-server directly, so it is not a global cap. + Waiting is unbounded in time by default (``queue_timeout_s`` + None); the wait line itself is bounded, and only how many may line up before + new arrivals are rejected. By default that is ``16 x slots`` floored at 64, + not unlimited: an unbounded line takes ``max_queue`` or ``queue_per_slot`` + set to 0. See ``LlamaAdmissionConfig.queue_limit``. + """ + + __slots__ = ( + "key", + "_lock", + "_capacity", + "_free", + "_in_use", + "_held", + "_waiters", + "_parked", + "_unpark_tickets", + "_unpark_seq", + ) + def __init__(self, key: str): self.key = key self._lock = threading.Lock() - self._active = 0 self._capacity = 1 + self._free: list[int] = [0] + # Held slots as a bitmask: one int instead of a set, so the pool costs the + # same whether it is idle or saturated. _held is its popcount, kept as a + # counter because int.bit_count() is 3.10+ and this package targets 3.9. + self._in_use = 0 + self._held = 0 self._waiters: Deque[_Waiter] = deque() + # Holders parked on a tool approval prompt. They hold no slot, so this only + # keeps the queue off the idle-eviction list while they are away. + self._parked = 0 + # FIFO tickets for holders resuming from a park (see acquire_parked_slot). A + # bare count deadlocked: every approved holder blocked every other one. + self._unpark_tickets: Deque[int] = deque() + self._unpark_seq = 0 + + def _resize_pool_locked(self, capacity: int) -> None: + # Slots past a shrunk capacity retire when their holder releases them. + if capacity == self._capacity: + return + self._capacity = capacity + self._free = [slot for slot in range(capacity) if not self._in_use >> slot & 1] + + def _can_admit_locked(self, reserved: int) -> bool: + # Slots still held above a shrunk capacity keep occupying the backend, so + # count every held slot against the ceiling, not just the ids below it. + # ``reserved`` holds slots back for approved holders waiting to resume; + # without it a stream of new arrivals took the next slot, forever. + return bool(self._free) and (self._held + reserved) < self._capacity + + def _take_slot_locked(self, reserved: int) -> Optional[int]: + if not self._can_admit_locked(reserved): + return None + slot = self._free.pop() + self._in_use |= 1 << slot + self._held += 1 + return slot def reserve(self, *, capacity: int, config: LlamaAdmissionConfig) -> LlamaAdmissionReservation: capacity = max(1, int(capacity or 1)) @@ -242,22 +570,25 @@ class LlamaAdmissionQueue: return LlamaAdmissionReservation( queue = None, lease = LlamaAdmissionLease(None), - snapshot = LlamaAdmissionSnapshot(self.key, capacity, 0, 0), + snapshot = LlamaAdmissionSnapshot(self.key, capacity, 0, 0, capacity), ) loop = asyncio.get_running_loop() with self._lock: - self._capacity = capacity - self._prune_waiters_locked() + self._resize_pool_locked(capacity) self._grant_waiters_locked() - if self._active < self._capacity and not self._waiters: - self._active += 1 - return LlamaAdmissionReservation( - queue = self, - lease = LlamaAdmissionLease(self), - snapshot = self._snapshot_locked(), - ) - if config.max_queue is not None and len(self._waiters) >= config.max_queue: + if not self._waiters: + slot = self._take_slot_locked(len(self._unpark_tickets)) + if slot is not None: + # No snapshot here: callers read it through snapshot_now(), + # which re-reads the queue, so building one per admitted + # request would be pure allocation on the hot path. + return LlamaAdmissionReservation( + queue = self, + lease = LlamaAdmissionLease(self, slot), + ) + limit = config.queue_limit(self._capacity) + if limit is not None and self._live_waiters_locked() >= limit: raise LlamaAdmissionQueueFull( "llama-server generation queue is full", snapshot = self._snapshot_locked(), @@ -270,15 +601,82 @@ class LlamaAdmissionQueue: return LlamaAdmissionReservation( queue = self, waiter = waiter, - snapshot = self._snapshot_locked(), ) - def release(self) -> None: + def _release_slot_locked(self, slot: Optional[int]) -> None: + # A slot id at or past a shrunk capacity retires instead of returning. + if slot is None or not self._in_use >> slot & 1: + return + self._in_use &= ~(1 << slot) + self._held -= 1 + if slot < self._capacity: + self._free.append(slot) + + def release(self, slot: Optional[int]) -> None: with self._lock: - if self._active > 0: - self._active -= 1 + self._release_slot_locked(slot) self._grant_waiters_locked() + def try_park(self, slot: Optional[int]) -> bool: + """Return a parked holder's slot to the pool. See ``LlamaAdmissionLease.park``. + + False leaves the slot with its holder, so a refused park costs nothing to + undo. The per-queue count is only what ``is_idle`` reads; the budget and + the capacity it is sized from are both process-wide. + """ + if not _claim_park(_max_parked(_live_capacity(self))): + return False + with self._lock: + self._parked += 1 + self._release_slot_locked(slot) + self._grant_waiters_locked() + return True + + def unpark(self) -> None: + with self._lock: + if self._parked > 0: + self._parked -= 1 + + async def acquire_parked_slot( + self, + *, + cancel_event = None, + poll_s: float = 0.02, + ) -> Optional[int]: + """Wait for a slot for a holder resuming from a park, None if cancelled. + + Ordered by ticket rather than counted, so approvals resume in the order + they came back: counting them made every approved holder block every + other one, and with nothing decoding that never resolved. + """ + with self._lock: + self._unpark_seq += 1 + ticket = self._unpark_seq + self._unpark_tickets.append(ticket) + try: + while True: + with self._lock: + ahead = 0 + for queued in self._unpark_tickets: + if queued == ticket: + break + ahead += 1 + # Only the approvals ahead of this one hold slots back from it. + slot = self._take_slot_locked(ahead) + if slot is not None: + return slot + if cancel_event is not None and cancel_event.is_set(): + return None + await asyncio.sleep(poll_s) + finally: + with self._lock: + try: + self._unpark_tickets.remove(ticket) + except ValueError: + pass + # This ticket was holding a slot back from the wait line. + self._grant_waiters_locked() + def cancel(self, waiter: _Waiter) -> None: lease_to_release = None with self._lock: @@ -291,7 +689,13 @@ class LlamaAdmissionQueue: lease_to_release = waiter.granted_lease waiter.granted_lease = None if not waiter.future.done(): - waiter.loop.call_soon_threadsafe(waiter.future.cancel) + try: + waiter.loop.call_soon_threadsafe(waiter.future.cancel) + except RuntimeError: + # Loop gone. Routes call cancel() from finally blocks, so + # raising here would both mask their exception and skip the + # release below, stranding the slot for the process lifetime. + pass if lease_to_release is not None: lease_to_release.release() @@ -303,20 +707,32 @@ class LlamaAdmissionQueue: def is_idle(self) -> bool: with self._lock: self._prune_waiters_locked() - return self._active == 0 and not self._waiters + # A parked holder owns no slot but is coming back to this queue, so + # evicting it here would resume it against a fresh 1-slot pool. + return self._in_use == 0 and not self._waiters and not self._parked def _grant_waiters_locked(self) -> None: - self._prune_waiters_locked() - while self._waiters and self._active < self._capacity: + # Dead waiters are skipped as they are popped, so no prune is needed here. + while self._waiters and self._can_admit_locked(len(self._unpark_tickets)): waiter = self._waiters.popleft() if waiter.cancelled or waiter.future.done(): continue - self._active += 1 - lease = LlamaAdmissionLease(self) + slot = self._take_slot_locked(len(self._unpark_tickets)) + lease = LlamaAdmissionLease(self, slot) waiter.granted_lease = lease - waiter.loop.call_soon_threadsafe(self._deliver_lease, waiter, lease) + try: + waiter.loop.call_soon_threadsafe(self._deliver_lease, waiter, lease) + except RuntimeError: + # Waiter's loop is gone. Reclaim the slot; leaving the bit set + # would strand it, since _free is rebuilt from the bitmask. + waiter.granted_lease = None + self._release_slot_locked(slot) def _deliver_lease(self, waiter: _Waiter, lease: LlamaAdmissionLease) -> None: + # Runs on the waiter's own loop thread, which is also the only thread that + # cancels that reservation, so waiter state is safe to touch unlocked here. + # release() may be called from any thread, but only reaches this via + # call_soon_threadsafe. Cancelling off-loop would need this under _lock. if waiter.cancelled or waiter.future.done(): waiter.granted_lease = None if not waiter.future.done(): @@ -331,16 +747,32 @@ class LlamaAdmissionQueue: lease.release() def _prune_waiters_locked(self) -> None: + # Rebuilding the deque on every reserve/release dominated the hot path, so + # only pay it when a waiter actually died out of band (an externally + # cancelled future); cancel() already drops its own waiter eagerly. + for waiter in self._waiters: + if waiter.cancelled or waiter.future.done(): + break + else: + return self._waiters = deque( waiter for waiter in self._waiters if not waiter.cancelled and not waiter.future.done() ) + def _live_waiters_locked(self) -> int: + self._prune_waiters_locked() + return len(self._waiters) + def _snapshot_locked(self) -> LlamaAdmissionSnapshot: return LlamaAdmissionSnapshot( key = self.key, capacity = self._capacity, - active = self._active, + active = self._held, queued = len(self._waiters), + # What another caller could actually take, so the admission log never + # shows free slots next to queued requests: after a shrink, ids below + # the new capacity can be free while holdovers still fill the ceiling. + free = min(len(self._free), max(0, self._capacity - self._held)), ) @@ -364,5 +796,10 @@ def get_llama_admission_queue(key: str) -> LlamaAdmissionQueue: def reset_llama_admission_queues() -> None: + global _parked_total with _QUEUES_LOCK: _QUEUES.clear() + # The budget outlives the queues it was claimed against, so dropping them + # without it leaks the count and shrinks the budget for good. + with _PARK_LOCK: + _parked_total = 0 diff --git a/studio/backend/core/inference/llama_cpp.py b/studio/backend/core/inference/llama_cpp.py index 0621a7f9c8..712caf43e5 100644 --- a/studio/backend/core/inference/llama_cpp.py +++ b/studio/backend/core/inference/llama_cpp.py @@ -43,6 +43,7 @@ import httpx from core.inference.llama_server_args import ( _LAYER_OFFLOAD_FLAGS, _effective_tensor_parallel, + _flag_name, _tensor_parallel_matches_loaded, extra_args_disable_mmproj, parse_cache_override, @@ -84,6 +85,7 @@ from core.tool_healing import ( strip_outside_think, ) from utils.native_path_leases import child_env_without_native_path_secret +from utils.child_stdio import utf8_child_env from utils.hf_xet_fallback import hf_hub_download_with_xet_fallback from utils.subprocess_compat import ( windows_hidden_subprocess_kwargs as _windows_hidden_subprocess_kwargs, @@ -91,13 +93,17 @@ from utils.subprocess_compat import ( from utils.process_lifetime import child_popen_kwargs as _child_popen_kwargs from core.inference.tool_call_parser import ( MAX_ACT_REPROMPTS as _MAX_REPROMPTS, + NUDGE_TOOL_CALLS_STATUS as _NUDGE_TOOL_CALLS_STATUS, REPROMPT_MAX_CHARS as _REPROMPT_MAX_CHARS, + is_reprompt_repeat as _is_reprompt_repeat, + is_reprompt_restatement as _is_reprompt_restatement, is_short_intent_without_action as _is_short_intent_without_action, reprompt_to_act_message as _reprompt_to_act_message, ) from core.inference.tool_loop_controller import ( ToolLoopController, append_deferred_nudges, + awaiting_approval_status, tool_event_provenance, ) from state.tool_approvals import ( @@ -307,6 +313,15 @@ def _native_linux_system_rocm_lib_dirs(binary_dir: str = "") -> "list[str]": os.path.join(d, "libhsa-runtime64.so.1") ): out.append(d) + # ROCm keeps LLVM's versioned runtime under /lib/llvm, so a + # lib64 host still finds it under lib. Probe both and keep them + # ahead of the bundle, else system libamd_comgr binds to the + # bundle's incompatible libLLVM.so.*. + for _sub in (lib_sub, "lib"): + llvm_lib = os.path.join(base, _sub, "llvm", "lib") + if llvm_lib not in seen and os.path.isdir(llvm_lib): + seen.add(llvm_lib) + out.append(llvm_lib) return out @@ -348,12 +363,32 @@ _DEFAULT_STREAM_STALL_TIMEOUT_S = 120.0 # 2 min # loop). Structured delta.tool_calls are grammar-bounded by llama-server; text # parsed from content is not, so one runaway turn could fan out unbounded. _MAX_TOOL_CALLS_PER_TURN = 8 -_FORCED_REPEAT_PLAN_SIGNAL = re.compile( - r"\b(?:i\s+will|i'll|let\s+me|going\s+to|need\s+to|call|use|run|search|fetch|render)\b", +# Obligation phrasing INTENT_SIGNAL leaves alone ("I need to call ..."), paired with +# an action verb. Sentence-anchored: mid-sentence the same words are prose that names +# a tool ("The API I should invoke is foo() because ..."), and suppressing that loses +# a real answer. "should"/"must" sit outside the need|have|ought group because they +# take a bare infinitive. "invoke"/"query" stay out of the verb list: they read as +# technical prose far more often than as a stall. +_FORCED_PLAN_INTENT = re.compile( + r"(?:^|[.!?]\s+)\s*" + r"(?:i\s+(?:(?:need|have|ought)\s+to|should|must)|need\s+to|going\s+to|must|should)" + r"\s+(?:\w+\s+){0,2}?(?:call|use|run|search|fetch|render)\b", + re.I | re.M, +) +# "the answer is not in the context" announces a *missing* answer, so the negated +# forms are excluded or the plan behind them would ship as the final response. +_FINAL_ANSWER_SIGNAL = re.compile( + r"\b(?:final\s+answer|answer\s*:|here\s+is|here's|in\s+summary|result\s*:" + r"|(?:the\s+)?answer\s+is(?!\s+(?:not|unavailable|unknown|unclear|missing)\b))\b", re.I, ) -_FINAL_ANSWER_SIGNAL = re.compile( - r"\b(?:final\s+answer|answer\s*:|here\s+is|here's|in\s+summary|result\s*:)\b", +# A plan that pivots ("I should call web_search, but Tokyo is the capital") has an +# answer attached, so the turn must survive. Leaking a plan sentence is cosmetic; +# dropping an answer is not, so the doubtful case keeps the output. The pivot has to +# carry text of its own: "I should call web_search, though." answers nothing. +_ANSWER_PIVOT = re.compile( + r"\b(?:but|however|although|though|that\s+said|in\s+the\s+meantime|meanwhile)\b" + r"[\W_]*(?:\w+[\W_]+){1,}\w", re.I, ) @@ -445,14 +480,28 @@ def _held_rehearsal_tail_len(text: str, active_tools: list[dict]) -> int: return len(tail) if tail and _is_rehearsal_prefix(tail, active_tools) else 0 -def _should_suppress_forced_no_tool_output(text: str) -> bool: - """Suppress only repeated forced-turn planning text, not final answers.""" +def _should_suppress_forced_no_tool_output(text: str, previous: str = "") -> bool: + """Suppress only repeated forced-turn planning text, not final answers. + + ``previous`` is the stall text that triggered the nudge, so a retry that + moved on can be told from one that just said the same thing again. + """ stripped = text.strip() if not stripped or len(stripped) >= _REPROMPT_MAX_CHARS: return False if _FINAL_ANSWER_SIGNAL.search(stripped): return False - return _FORCED_REPEAT_PLAN_SIGNAL.search(stripped) is not None + plan = _FORCED_PLAN_INTENT.search(stripped) + if plan is not None: + # Only the plan itself is safe to drop; anything the turn pivots to after it + # is the answer the user is waiting for. + return _ANSWER_PIVOT.search(stripped[plan.end() :]) is None + if not _is_short_intent_without_action(stripped): + return False + # INTENT_SIGNAL also fires on lead-ins to a real answer ("Now I have the results. + # The capital is Tokyo."), so a bare intent match is a stall only when the retry + # adds nothing. No ``previous`` keeps the standalone "is this a stall?" contract. + return not previous or _is_reprompt_restatement(stripped, previous) # ── Pre-compiled patterns for GGUF shard detection ─────────── @@ -569,7 +618,7 @@ def _load_swa_cache() -> dict: if _SWA_CACHE is not None: return _SWA_CACHE try: - with open(_swa_cache_path(), encoding = "utf-8") as f: + with open(_swa_cache_path(), encoding = "utf-8-sig") as f: _SWA_CACHE = json.load(f) if not isinstance(_SWA_CACHE, dict): _SWA_CACHE = {} @@ -620,7 +669,7 @@ def _fetch_swa_entry_from_hf(repo_id: str) -> Optional[object]: repo_type = "model", cache_dir = active_hf_hub_cache(), ) - with open(cfg_path, encoding = "utf-8") as f: + with open(cfg_path, encoding = "utf-8-sig") as f: cfg = json.load(f) except Exception: return None @@ -1508,6 +1557,21 @@ def _kv_bytes_per_elem(cache_type: Optional[str]) -> float: }.get((cache_type or "f16").strip().lower(), 2.0) +def _pad_kv_cells(cells: int) -> int: + return ((cells + 255) // 256) * 256 + + +def _kv_cache_cell_layout(n_ctx: int, n_parallel: int, kv_unified: bool) -> tuple[int, int, int]: + """Return llama.cpp's slot count, stream count, and cells per stream.""" + slots = max(1, n_parallel) + padded_ctx = _pad_kv_cells(n_ctx) + streams = 1 if kv_unified else slots + if padded_ctx <= 0: + return slots, streams, 0 + cells_per_stream = padded_ctx if kv_unified else _pad_kv_cells(padded_ctx // slots) + return slots, streams, cells_per_stream + + def _env_main_cache_type_for_budget(env: Optional[Mapping[str, str]] = None) -> Optional[str]: """Heavier of the inherited LLAMA_ARG_CACHE_TYPE_K/_V env types when it exceeds the f16 default, else None. Unsloth emits --cache-type only for the @@ -1540,6 +1604,39 @@ def _extra_args_main_cache_type_for_budget(extra_args: Optional[Iterable[str]]) return max(candidates, key = _kv_bytes_per_elem) +def _effective_main_cache_types( + args: Optional[Iterable[str]], env: Optional[Mapping[str, str]] = None +) -> tuple[str, str]: + """Effective main K/V cache types after environment and CLI precedence.""" + source_env = os.environ if env is None else env + env_k = (source_env.get("LLAMA_ARG_CACHE_TYPE_K") or "f16").strip().lower() + env_v = (source_env.get("LLAMA_ARG_CACHE_TYPE_V") or "f16").strip().lower() + arg_k, arg_v = parse_cache_override_per_axis(args) + return ( + (arg_k or env_k).strip().lower(), + (arg_v or env_v).strip().lower(), + ) + + +def _planned_main_cache_types( + cache_type_kv: Optional[str], + extra_args: Optional[Iterable[str]], + env: Optional[Mapping[str, str]] = None, +) -> tuple[str, str]: + """Main K/V types the loader's managed flags and user extras will produce.""" + args = list(extra_args or ()) + emitted_type = _extra_args_main_cache_type_for_budget(args) or cache_type_kv + if emitted_type: + args = [ + "--cache-type-k", + emitted_type, + "--cache-type-v", + emitted_type, + *args, + ] + return _effective_main_cache_types(args, env) + + def _auto_mode_drops_mtp( req_mode: Optional[str], size_b: Optional[float], @@ -1583,26 +1680,90 @@ def _extra_args_set_spec_type(extra_args: Optional[Iterable[str]]) -> bool: # set keeps detection and stripping from drifting. _GPU_OFFLOAD_OVERRIDE_FLAGS = _LAYER_OFFLOAD_FLAGS _THREAD_OVERRIDE_FLAGS = frozenset({"-t", "--threads"}) - - -def _extra_arg_flag_name(token: str) -> Optional[str]: - if not token.startswith("-") or token in {"-", "--"}: - return None - if len(token) >= 2 and (token[1].isdigit() or token[1] == "."): - return None - return token.split("=", 1)[0] +# common_params defaults in the bundled llama.cpp runtime. +_DEFAULT_LLAMA_N_BATCH = 2048 +_DEFAULT_LLAMA_N_UBATCH = 512 +_LLAMA_ARG_TRUE_VALUES = frozenset({"on", "enabled", "true", "1"}) +_LLAMA_ARG_FALSE_VALUES = frozenset({"off", "disabled", "false", "0"}) +_LLAMA_ARG_AUTO_VALUES = frozenset({"auto", "-1"}) +_LLAMA_ARG_TRUE_OR_AUTO_VALUES = _LLAMA_ARG_TRUE_VALUES | _LLAMA_ARG_AUTO_VALUES +_LLAMA_ARG_TRUE_FALSE_AUTO_VALUES = _LLAMA_ARG_TRUE_OR_AUTO_VALUES | _LLAMA_ARG_FALSE_VALUES def _extra_args_set_any_flag(extra_args: Optional[Iterable[str]], flags: Collection[str]) -> bool: if not extra_args: return False for raw in extra_args: - flag = _extra_arg_flag_name(str(raw)) + flag = _flag_name(str(raw)) if flag in flags: return True return False +def _swa_full_from_args_or_env( + extra_args: Optional[Iterable[str]], env: Optional[Mapping[str, str]] = None +) -> bool: + """Whether llama.cpp receives the enable-only full-size SWA option.""" + if _extra_args_set_any_flag(extra_args, {"--swa-full"}): + return True + value = (os.environ if env is None else env).get("LLAMA_ARG_SWA_FULL") + return value in _LLAMA_ARG_TRUE_VALUES + + +def _kv_unified_from_args( + extra_args: Optional[Iterable[str]], + default: bool = False, + env: Optional[Mapping[str, str]] = None, +) -> bool: + """Resolve llama.cpp's environment and last-wins unified KV flags.""" + enabled = False + value = (os.environ if env is None else env).get("LLAMA_ARG_KV_UNIFIED") + if value in _LLAMA_ARG_TRUE_VALUES: + enabled = True + elif value in _LLAMA_ARG_FALSE_VALUES: + enabled = False + if default: + # Studio's managed --kv-unified flag is appended after environment + # parsing and before user extras. + enabled = True + for raw in extra_args or (): + flag = _flag_name(str(raw)) + if flag in {"-kvu", "--kv-unified"}: + enabled = True + elif flag in {"-no-kvu", "--no-kv-unified"}: + enabled = False + return enabled + + +def _flash_attn_enabled_from_args( + args: Optional[Iterable[str]], + default: bool = True, + env: Optional[Mapping[str, str]] = None, +) -> bool: + """Resolve llama.cpp's environment and last-wins flash-attention settings.""" + enabled = default + # llama.cpp applies LLAMA_ARG_FLASH_ATTN before parsing argv (arg.cpp set_env), + # so the CLI still wins. --flash-attn has no args_neg, so no LLAMA_ARG_NO_ twin. + value = (os.environ if env is None else env).get("LLAMA_ARG_FLASH_ATTN") + if value in _LLAMA_ARG_FALSE_VALUES: + enabled = False + elif value in _LLAMA_ARG_TRUE_OR_AUTO_VALUES: + enabled = True + values = [str(arg) for arg in args] if args else [] + for i, raw in enumerate(values): + if _flag_name(raw) not in {"-fa", "--flash-attn"}: + continue + _, eq, inline = raw.partition("=") + value = inline if eq else "on" + if not eq and i + 1 < len(values) and values[i + 1] in _LLAMA_ARG_TRUE_FALSE_AUTO_VALUES: + value = values[i + 1] + if value in _LLAMA_ARG_FALSE_VALUES: + enabled = False + elif value in _LLAMA_ARG_TRUE_OR_AUTO_VALUES: + enabled = True + return enabled + + def _effective_spec_type( extra_args: Optional[Iterable[str]], env: Optional[Mapping[str, str]] = None ) -> Optional[str]: @@ -1614,7 +1775,8 @@ def _effective_spec_type( cli_present = False cli_value: Optional[str] = None for i, raw in enumerate(args): - flag, eq, inline = raw.partition("=") + flag = _flag_name(raw) + _, eq, inline = raw.partition("=") if flag == "--spec-default": cli_present = True cli_value = "default" @@ -1658,7 +1820,8 @@ def _extra_args_spec_draft_n_max(extra_args: Optional[Iterable[str]]) -> Optiona args = [str(a) for a in extra_args] found: Optional[int] = None for i, raw in enumerate(args): - flag, eq, inline = raw.partition("=") + flag = _flag_name(raw) + _, eq, inline = raw.partition("=") if flag not in ("--spec-draft-n-max", "--draft-max"): continue value = inline if eq else (args[i + 1] if i + 1 < len(args) else "") @@ -1688,7 +1851,8 @@ def _extra_args_mtp_draft_path( args = [str(a) for a in extra_args] if extra_args else [] found: Optional[str] = None for i, raw in enumerate(args): - flag, eq, inline = raw.partition("=") + flag = _flag_name(raw) + _, eq, inline = raw.partition("=") if flag not in flags: continue value = inline if eq else (args[i + 1] if i + 1 < len(args) else "") @@ -1712,7 +1876,8 @@ def _extra_args_draft_cache_types( k_type: Optional[str] = None v_type: Optional[str] = None for i, raw in enumerate(args): - flag, eq, inline = raw.partition("=") + flag = _flag_name(raw) + _, eq, inline = raw.partition("=") if flag not in k_flags and flag not in v_flags: continue value = inline if eq else (args[i + 1] if i + 1 < len(args) else "") @@ -1744,7 +1909,8 @@ def _extra_args_draft_offloaded_to_cpu( last_ngl: Optional[str] = None last_dev: Optional[str] = None for i, raw in enumerate(args): - flag, eq, inline = raw.partition("=") + flag = _flag_name(raw) + _, eq, inline = raw.partition("=") value = inline if eq else (args[i + 1] if i + 1 < len(args) else "") if flag in ngl_flags: last_ngl = value @@ -1766,31 +1932,61 @@ def _extra_args_draft_offloaded_to_cpu( def _extra_args_n_ubatch( - extra_args: Optional[Iterable[str]], env: Optional[Mapping[str, str]] = None + extra_args: Optional[Iterable[str]], + env: Optional[Mapping[str, str]] = None, + n_ctx: Optional[int] = None, ) -> Optional[int]: - """Physical micro-batch from extras (--ubatch-size/-ub) else the LLAMA_ARG_UBATCH - env, else None. It sizes the compute-graph buffer, so an override must reach - the VRAM reserve.""" + """Effective ubatch after llama.cpp normalizes it, or None at defaults.""" + values = { + "batch": _DEFAULT_LLAMA_N_BATCH, + "ubatch": _DEFAULT_LLAMA_N_UBATCH, + } + source_env = os.environ if env is None else env + overridden = False + for key, env_name in ( + ("batch", "LLAMA_ARG_BATCH"), + ("ubatch", "LLAMA_ARG_UBATCH"), + ): + raw = source_env.get(env_name) + if raw: + try: + values[key] = int(raw) + overridden = True + except (TypeError, ValueError): + pass + args = [str(a) for a in extra_args] if extra_args else [] - found: Optional[int] = None + flags = { + "-b": "batch", + "--batch-size": "batch", + "-ub": "ubatch", + "--ubatch-size": "ubatch", + } for i, raw in enumerate(args): - flag, eq, inline = raw.partition("=") - if flag not in ("--ubatch-size", "-ub"): + flag = _flag_name(raw) + _, eq, inline = raw.partition("=") + key = flags.get(flag) + if key is None: continue value = inline if eq else (args[i + 1] if i + 1 < len(args) else "") try: - found = int(value) + values[key] = int(value) + overridden = True except (TypeError, ValueError): continue - if found is not None: - return found - raw = (os.environ if env is None else env).get("LLAMA_ARG_UBATCH") - if raw: - try: - return int(raw) - except (TypeError, ValueError): - pass - return None + if not overridden: + return None + + # common_params stores signed values, then llama_context_params converts + # them to uint32_t. A zero ubatch means "use batch"; the context then caps + # ubatch at batch size. + batch = values["batch"] & 0xFFFFFFFF + raw_ubatch = values["ubatch"] + ubatch = batch if raw_ubatch == 0 else raw_ubatch & 0xFFFFFFFF + effective = min(batch, ubatch) + if n_ctx is not None and n_ctx > 0: + effective = min(effective, n_ctx) + return effective def _build_ngram_mod_flags( @@ -2034,6 +2230,8 @@ class LlamaCppBackend: self._effective_context_length: Optional[int] = None self._max_context_length: Optional[int] = None self._effective_parallel_slots: int = 1 + # --parallel the last load asked for, before any fit-time reduction. + self._requested_n_parallel: int = 1 self._chat_template: Optional[str] = None self._chat_template_override: Optional[str] = None self._supports_reasoning: bool = False @@ -2149,6 +2347,14 @@ class LlamaCppBackend: # save can tell whether the model files were swapped on disk since load. self._slot_loaded_identity: Optional[tuple] = None self._prompt_cache_disabled: bool = False + self._swa_full: bool = False + self._kv_cache_unified: bool = False + self._n_ubatch: int = self._DEFAULT_N_UBATCH + self._flash_attn_enabled: bool = True + self._effective_cache_types: tuple[str, str] = ("f16", "f16") + # Total KV allocation context across all slots. _effective_context_length + # becomes the per-slot request limit after /props reconciliation. + self._kv_cache_context_total: Optional[int] = None # True once a probe has completed; cleared on transient failure. self._is_audio: bool = False self._audio_type: Optional[str] = None @@ -2201,6 +2407,11 @@ class LlamaCppBackend: """True when the loaded GGUF is a block-diffusion model (DiffusionGemma).""" return self._is_diffusion + @property + def swa_full(self) -> bool: + """Whether the active llama-server received full-size SWA mode.""" + return self._swa_full + @property def hf_variant(self) -> Optional[str]: return self._hf_variant @@ -2257,6 +2468,17 @@ class LlamaCppBackend: slots = 1 return max(1, slots) + @property + def requested_parallel_slots(self) -> int: + """--parallel the last load asked for, before any fit-time reduction. + The reload dedupe compares requested-vs-requested (like requested_n_ctx); + the effective count would reload forever after a fitter reduction.""" + try: + slots = int(getattr(self, "_requested_n_parallel", 1)) + except (TypeError, ValueError): + slots = 1 + return max(1, slots) + @property def max_context_length(self) -> Optional[int]: """Return the largest context that fits on this hardware at load time. @@ -2282,6 +2504,8 @@ class LlamaCppBackend: def _reset_effective_parallel_slots(self) -> None: self._effective_parallel_slots = 1 + # Cleared with the effective count so a stale value can't skew the dedupe. + self._requested_n_parallel = 1 @staticmethod def _read_rss_bytes(pid: int) -> Optional[int]: @@ -2859,6 +3083,7 @@ class LlamaCppBackend: [bin_path, "--help"], capture_output = True, text = True, + encoding = "utf-8", errors = "replace", timeout = 10, check = False, @@ -3116,8 +3341,9 @@ class LlamaCppBackend: prefer_rocr masks at the ROCr/HSA layer instead (clearing HIP). A HIP mask filters only AFTER the HSA runtime enumerates every agent, and that enumeration segfaults at startup on a GPU the build has no kernels for - (e.g. a gfx1103 iGPU under a gfx110X prebuilt), before llama-server logs a - line. ROCR drops the device at the driver layer, consuming physical ids. + (e.g. a gfx1036 iGPU under a gfx103X prebuilt: that bundle maps only + gfx1030/1031/1032/1034), before llama-server logs a line. ROCR drops the + device at the driver layer, consuming physical ids. The CPU-only sentinel ("-1") has no portable ROCR spelling, so it keeps the HIP mask. Windows keeps the HIP mask too: ROCR_VISIBLE_DEVICES is a Linux ROCr variable (Windows HIP has no ROCr layer), so the ROCR pin @@ -3430,6 +3656,8 @@ class LlamaCppBackend: ], capture_output = True, text = True, + encoding = "utf-8", + errors = "replace", timeout = 10, env = child_env_without_native_path_secret(), **_windows_hidden_subprocess_kwargs(), @@ -3501,18 +3729,17 @@ class LlamaCppBackend: return [] @staticmethod - def _get_gpu_free_memory_vulkan(binary: Optional[str] = None) -> list[tuple[int, int, int]]: - """Query free (and total) VRAM per device via the bundled ggml Vulkan backend. + def _run_vulkan_probe(binary: Optional[str] = None) -> list[dict]: + """Run ``_vulkan_probe.py`` and parse its per-device lines. - Loads ``libggml-vulkan`` in a short-lived subprocess (no Vulkan instance - in this process) and returns (device_index, free_mib, total_mib) sorted - by index. The index is ggml's compact Vulkan ordinal -- the one the - registry names ``Vulkan`` and load_model pins with ``--device``, - NOT the raw ``GGML_VK_VISIBLE_DEVICES`` space. A user-set - ``GGML_VK_VISIBLE_DEVICES`` is honored by ggml (passed through), so the - list already reflects it. iGPUs leave a host-RAM margin (see - ``_apply_igpu_host_reserve_mib``) and report total 0; discrete cards pass - their real total through. [] when no Vulkan build or device is reachable. + Returns raw (uncapped) rows sorted by index: + ``{"index", "free_mib", "total_mib", "is_igpu", "name"}``. The index is + ggml's compact Vulkan ordinal -- the one the registry names + ``Vulkan`` and load_model pins with ``--device``, NOT the raw + ``GGML_VK_VISIBLE_DEVICES`` space. A user-set ``GGML_VK_VISIBLE_DEVICES`` + is honored by ggml (passed through), so the list already reflects it. + ``name`` is ggml's device description; "" from an older 4-column probe. + [] when no Vulkan build or device is reachable. """ binary = binary or LlamaCppBackend._find_llama_server_binary() if not binary: @@ -3537,12 +3764,15 @@ class LlamaCppBackend: ) probe_script = Path(__file__).with_name("_vulkan_probe.py") try: + # UTF-8 to match the probe's stdout reconfigure: device names can be + # non-ASCII, and the platform-default decode (cp1252) could throw. result = subprocess.run( [sys.executable, str(probe_script), str(binary_dir)], capture_output = True, - text = True, + encoding = "utf-8", + errors = "replace", timeout = 15, - env = env, + env = utf8_child_env(env), **_windows_hidden_subprocess_kwargs(), ) if result.returncode != 0: @@ -3554,21 +3784,56 @@ class LlamaCppBackend: logger.debug(f"vulkan GPU probe failed: {e}") return [] - gpus: list[tuple[int, int, int]] = [] + rows: list[dict] = [] for line in result.stdout.strip().splitlines(): parts = line.split("\t") - if len(parts) != 4: + # 4 columns from an older probe (no name); 5 with the name column. + if len(parts) not in (4, 5): continue try: - idx = int(parts[0]) - free_mib = int(parts[1]) // (1024 * 1024) - is_igpu = parts[2] == "1" - # iGPU "total" is shared RAM, not a VRAM budget -> keep 0 so the - # fit stays on free*frac (the host reserve below is its - # headroom); a discrete card passes its real total through. - total_mib = 0 if is_igpu else int(parts[3]) // (1024 * 1024) + rows.append( + { + "index": int(parts[0]), + "free_mib": int(parts[1]) // (1024 * 1024), + "is_igpu": parts[2] == "1", + "total_mib": int(parts[3]) // (1024 * 1024), + "name": parts[4].strip() if len(parts) == 5 else "", + } + ) except ValueError: continue + rows.sort(key = lambda r: r["index"]) + return rows + + @staticmethod + def vulkan_device_inventory(binary: Optional[str] = None) -> list[dict]: + """UI-facing Vulkan device list: the devices llama-server will actually + use, with real totals (an iGPU keeps its shared-RAM total here -- the + caller labels it, unlike the fit which zeroes it). Same rows as + ``_run_vulkan_probe``; names fall back to ``Vulkan``. + """ + rows = LlamaCppBackend._run_vulkan_probe(binary) + for row in rows: + if not row["name"]: + row["name"] = f"Vulkan{row['index']}" + return rows + + @staticmethod + def _get_gpu_free_memory_vulkan(binary: Optional[str] = None) -> list[tuple[int, int, int]]: + """Query free (and total) VRAM per device via the bundled ggml Vulkan backend. + + Fit-oriented view of ``_run_vulkan_probe``: returns (device_index, + free_mib, total_mib) sorted by index. iGPUs leave a host-RAM margin (see + ``_apply_igpu_host_reserve_mib``) and report total 0; discrete cards pass + their real total through. [] when no Vulkan build or device is reachable. + """ + gpus: list[tuple[int, int, int]] = [] + for row in LlamaCppBackend._run_vulkan_probe(binary): + idx, free_mib, is_igpu = row["index"], row["free_mib"], row["is_igpu"] + # iGPU "total" is shared RAM, not a VRAM budget -> keep 0 so the + # fit stays on free*frac (the host reserve below is its + # headroom); a discrete card passes its real total through. + total_mib = 0 if is_igpu else row["total_mib"] capped = _apply_igpu_host_reserve_mib(free_mib, is_igpu) if capped < free_mib: logger.info( @@ -3577,7 +3842,6 @@ class LlamaCppBackend: f"({free_mib}->{capped}MiB usable)" ) gpus.append((idx, capped, total_mib)) - gpus.sort(key = lambda g: g[0]) if gpus: logger.info( "Vulkan GPU memory detected: " @@ -4019,6 +4283,32 @@ class LlamaCppBackend: is non-None here.""" return self._embedding_length // self._n_heads if self._n_heads else 128 # type: ignore[operator] + def _max_kv_value_width( + self, + default_len: int, + swa_len: Optional[int] = None, + ) -> int: + """llama.cpp's hparams.n_embd_v_gqa_max() over every model layer.""" + n_layers = self._n_layers or 1 + n_kv = self._n_kv_heads or self._n_heads or 1 + if self._sliding_window_pattern is None: + max_len = max(default_len, swa_len or default_len) + return max( + self._kv_heads_for_layer(layer_idx, n_kv) * max_len for layer_idx in range(n_layers) + ) + return max( + self._kv_heads_for_layer(layer_idx, n_kv) + * ( + (swa_len or default_len) + if ( + layer_idx < len(self._sliding_window_pattern) + and self._sliding_window_pattern[layer_idx] + ) + else default_len + ) + for layer_idx in range(n_layers) + ) + def _estimate_kv_cache_bytes( self, n_ctx: int, @@ -4027,22 +4317,26 @@ class LlamaCppBackend: swa_full: bool = False, n_parallel: int = 1, kv_unified: bool = True, + n_ubatch: Optional[int] = None, ctx_checkpoints: int = 0, + flash_attn: bool = True, ) -> int: """Estimate KV cache VRAM for a given context length. 5-path architecture-aware estimation: 1. MLA -- compressed KV latent + RoPE, K-only (no separate V) 2. Hybrid -- only attention layers need KV (Mamba layers don't) - 3. SWA -- sliding-window layers cache min(ctx, window) tokens + 3. SWA -- sliding-window layers use compact or full cache cells 4. GQA -- standard full KV with explicit key/value dimensions 5. Legacy -- fallback using embed // n_heads Server-flag knobs (mirror llama-server's CLI): swa_full -- --swa-full: SWA layers cache full n_ctx (path 3->4). - n_parallel -- --parallel slots: non-SWA constant, SWA scale linearly. - kv_unified -- --kv-unified: memory no-op (API forward-compat). + n_parallel -- --parallel slots: controls per-slot stream padding. + kv_unified -- --kv-unified: one shared stream vs one per slot. + n_ubatch -- --ubatch-size: SWA cache's processing headroom. ctx_checkpoints -- --ctx-checkpoints: N SWA snapshots per slot. + flash_attn -- False pads variable-width V tensors to the model max. Returns 0 if metadata is insufficient. """ @@ -4057,9 +4351,17 @@ class LlamaCppBackend: n_kv = self._n_kv_heads or self._n_heads or 1 # type: ignore[assignment] # Bytes per element depends on KV cache quantization - bpe = _kv_bytes_per_elem(cache_type_kv) + bpe_k = _kv_bytes_per_elem(cache_type_kv) + # The automatic FA-off retry rewrites an invalid quantized V cache to + # f16. Pricing that viable retry here avoids under-reserving it. + bpe_v = bpe_k if flash_attn else max(bpe_k, _kv_bytes_per_elem("f16")) - slots = max(1, n_parallel) + slots, streams, cells_per_stream = _kv_cache_cell_layout(n_ctx, n_parallel, kv_unified) + total_cells = cells_per_stream * streams + ubatch = max( + 0, + int(self._DEFAULT_N_UBATCH if n_ubatch is None else n_ubatch), + ) # Path 1: MLA (DeepSeek-V2/V3, GLM-4.7, GLM-5, Kimi-K2.5) # One compressed KV latent per token/layer (shared across heads); V is @@ -4070,7 +4372,7 @@ class LlamaCppBackend: n_kv_mla = self._n_kv_heads or 1 rope_dim = self._key_length_mla or 64 key_len = self._kv_key_length or (self._kv_lora_rank + rope_dim) - return int(n_layers_kv * n_ctx * n_kv_mla * key_len * bpe) + return int(n_layers_kv * total_cells * n_kv_mla * key_len * bpe_k) key_len = self._kv_key_length val_len = self._kv_value_length @@ -4081,16 +4383,18 @@ class LlamaCppBackend: fai = self._full_attention_interval n_attn = -(-n_layers // fai) if fai > 0 else n_layers # ceiling division if key_len is not None and val_len is not None: - return int(n_attn * n_ctx * n_kv * (key_len + val_len) * bpe) + v_width = n_kv * val_len if flash_attn else self._max_kv_value_width(val_len) + return int(n_attn * total_cells * (n_kv * key_len * bpe_k + v_width * bpe_v)) head_dim = self._legacy_head_dim() - return int(n_attn * n_ctx * n_kv * 2 * head_dim * bpe) + return int(n_attn * total_cells * n_kv * 2 * head_dim * bpe_k) # Path 3: Sliding window (Gemma 2/3/3n/4, gpt-oss, Cohere2 ...). Pattern # from the resolver; if absent, falls through to the legacy 1/4-global # heuristic. --parallel N accounting (verified against llama-server): - # non-SWA cells = n_ctx split across slots (CONSTANT); SWA per-slot cells - # = 2*sliding_window (capped at n_ctx/per_slot_ctx) -> LINEAR in slots. - # --swa-full forces full n_ctx for SWA; --ctx-checkpoints N adds snapshots. + # non-SWA cells total n_ctx across streams. Compact SWA adds one processing + # micro-batch to the window allowance and pads to 256 cells; unified mode + # holds all slots in one stream, while non-unified mode has one stream per + # slot. --swa-full expands SWA to each stream's full context. if ( self._sliding_window is not None and self._sliding_window > 0 @@ -4098,15 +4402,19 @@ class LlamaCppBackend: and val_len is not None ): swa = self._sliding_window - per_slot_ctx = max(1, n_ctx // slots) - # --swa-full caches full per_slot_ctx (constant n_ctx total); else SWA - # caches 2*sliding_window per slot, clamped at per-slot ctx. - swa_cells_per_slot = per_slot_ctx if swa_full else min(n_ctx, 2 * swa, per_slot_ctx) + if swa_full: + swa_cells_total = total_cells + else: + swa_limit = swa * (slots if kv_unified else 1) + ubatch + swa_cells_per_stream = min(cells_per_stream, swa_limit) + swa_cells_per_stream = _pad_kv_cells(swa_cells_per_stream) + swa_cells_total = swa_cells_per_stream * streams key_len_swa = self._kv_key_length_swa or key_len val_len_swa = self._kv_value_length_swa or val_len + padded_v_width = None if flash_attn else self._max_kv_value_width(val_len, val_len_swa) if self._sliding_window_pattern is not None: - global_bytes = 0.0 # constant across slots - swa_bytes_per_slot = 0.0 # multiplied by slots + global_bytes = 0.0 + swa_bytes = 0.0 checkpoint_extra_per_slot = 0.0 # Only layers that allocate their own KV; trailing shared layers # reuse earlier caches. @@ -4116,41 +4424,48 @@ class LlamaCppBackend: layer_idx < len(self._sliding_window_pattern) and self._sliding_window_pattern[layer_idx] ) + layer_key_bytes = layer_n_kv * (key_len_swa if is_swa else key_len) * bpe_k + layer_value_bytes = ( + layer_n_kv * (val_len_swa if is_swa else val_len) + if padded_v_width is None + else padded_v_width + ) * bpe_v + layer_kv_bytes = layer_key_bytes + layer_value_bytes if is_swa: - swa_bytes_per_slot += ( - swa_cells_per_slot * layer_n_kv * (key_len_swa + val_len_swa) * bpe - ) + swa_bytes += swa_cells_total * layer_kv_bytes if ctx_checkpoints > 0 and not swa_full: - checkpoint_extra_per_slot += ( - ctx_checkpoints - * swa - * layer_n_kv - * (key_len_swa + val_len_swa) - * bpe - ) + checkpoint_extra_per_slot += ctx_checkpoints * swa * layer_kv_bytes else: - global_bytes += n_ctx * layer_n_kv * (key_len + val_len) * bpe - return int(global_bytes + slots * (swa_bytes_per_slot + checkpoint_extra_per_slot)) + global_bytes += total_cells * layer_kv_bytes + return int(global_bytes + swa_bytes + slots * checkpoint_extra_per_slot) n_global = max(1, n_layers_kv // 4) n_swa = n_layers_kv - n_global - kv_per_token = n_kv * (key_len + val_len) * bpe - kv_per_token_swa = n_kv * (key_len_swa + val_len_swa) * bpe - global_bytes = n_global * n_ctx * kv_per_token - swa_bytes_per_slot = n_swa * swa_cells_per_slot * kv_per_token_swa + global_v_width = n_kv * val_len if padded_v_width is None else padded_v_width + swa_v_width = n_kv * val_len_swa if padded_v_width is None else padded_v_width + kv_per_token = n_kv * key_len * bpe_k + global_v_width * bpe_v + kv_per_token_swa = n_kv * key_len_swa * bpe_k + swa_v_width * bpe_v + global_bytes = n_global * total_cells * kv_per_token + swa_bytes = n_swa * swa_cells_total * kv_per_token_swa checkpoint_extra_per_slot = ( ctx_checkpoints * n_swa * swa * kv_per_token_swa if ctx_checkpoints > 0 and not swa_full else 0.0 ) - return int(global_bytes + slots * (swa_bytes_per_slot + checkpoint_extra_per_slot)) + return int(global_bytes + swa_bytes + slots * checkpoint_extra_per_slot) # Path 4: Standard GQA with explicit key/value dimensions if key_len is not None and val_len is not None: - return int(n_layers_kv * n_ctx * n_kv * (key_len + val_len) * bpe) + padded_v_width = None if flash_attn else self._max_kv_value_width(val_len) + bytes_per_cell = 0.0 + for layer_idx in range(n_layers_kv): + layer_n_kv = self._kv_heads_for_layer(layer_idx, n_kv) + v_width = layer_n_kv * val_len if padded_v_width is None else padded_v_width + bytes_per_cell += layer_n_kv * key_len * bpe_k + v_width * bpe_v + return int(total_cells * bytes_per_cell) # Path 5: Legacy fallback (old GGUFs without explicit dimensions) head_dim = self._legacy_head_dim() - return int(2 * n_kv * head_dim * n_layers_kv * n_ctx * bpe) + return int(2 * n_kv * head_dim * n_layers_kv * total_cells * bpe_k) def _draft_backend_for(self, drafter_path: str) -> Optional["LlamaCppBackend"]: """Lightweight backend with a drafter GGUF's metadata, to size its own KV @@ -4198,6 +4513,10 @@ class LlamaCppBackend: draft_cache_type_k: Optional[str] = None, draft_cache_type_v: Optional[str] = None, n_parallel: int = 1, + swa_full: bool = False, + kv_unified: bool = True, + n_ubatch: Optional[int] = None, + flash_attn: bool = True, ) -> Optional[int]: """Draft KV cache bytes at n_ctx, sized from GGUF dims (K and V types are independent). Separate drafter (Gemma): its own KV via _estimate_kv_cache_bytes @@ -4211,12 +4530,23 @@ class LlamaCppBackend: db = self._draft_backend_for(drafter_path) if db is None or not db._can_estimate_kv(): return None + # Gemma 4 assistant layers share the target context's final global + # and SWA KV tensors, so only the drafter weights add memory. + if getattr(db, "_architecture", None) == "gemma4-assistant": + return 0 heavier = draft_cache_type_k if bpe_k >= bpe_v else draft_cache_type_v - # The drafter is served under the same --parallel slot count as the - # main model, so price its KV per slot too: a sliding-window drafter - # (Gemma) grows KV with slots and would otherwise be under-reserved. - kv = db._estimate_kv_cache_bytes(n_ctx, heavier, n_parallel = n_parallel) - return kv or None + # The drafter uses the main model's slot and stream layout, so its + # compact SWA and per-stream padding must follow the same settings. + kv = db._estimate_kv_cache_bytes( + n_ctx, + heavier, + n_parallel = n_parallel, + swa_full = swa_full, + kv_unified = kv_unified, + n_ubatch = n_ubatch, + flash_attn = flash_attn, + ) + return kv if kv > 0 else None nextn = self._nextn_predict_layers or 0 n_kv = self._n_kv_heads or self._n_heads k_len = self._kv_key_length @@ -4230,7 +4560,14 @@ class LlamaCppBackend: f16_bpe = _kv_bytes_per_elem("f16") bpe_k = max(bpe_k, f16_bpe) bpe_v = max(bpe_v, f16_bpe) - return int(nextn * n_kv * (k_len * bpe_k + v_len * bpe_v) * n_ctx) + _, streams, cells_per_stream = _kv_cache_cell_layout(n_ctx, n_parallel, kv_unified) + v_width = n_kv * v_len + if not flash_attn: + v_width = self._max_kv_value_width( + v_len, + self._kv_value_length_swa, + ) + return int(nextn * (n_kv * k_len * bpe_k + v_width * bpe_v) * cells_per_stream * streams) def _estimate_mtp_overhead_bytes( self, @@ -4243,6 +4580,10 @@ class LlamaCppBackend: draft_weights_bytes: int = 0, n_parallel: int = 1, mtp_keeps_target_ctx: bool = True, + swa_full: bool = False, + kv_unified: bool = True, + n_ubatch: Optional[int] = None, + flash_attn: bool = True, ) -> Optional[int]: """MTP draft reserve at ``n_ctx`` = draft KV (grows with ctx) + separate- drafter weights + (MTP + MLA only) a duplicated target KV context. The @@ -4258,6 +4599,10 @@ class LlamaCppBackend: draft_cache_type_k = draft_cache_type_k, draft_cache_type_v = draft_cache_type_v, n_parallel = n_parallel, + swa_full = swa_full, + kv_unified = kv_unified, + n_ubatch = n_ubatch, + flash_attn = flash_attn, ) weights = max(0, draft_weights_bytes) # MLA models (GLM-5.x, DeepSeek, Kimi-K2) under MTP keep a *second* full copy @@ -4273,7 +4618,15 @@ class LlamaCppBackend: # rather than duplicating the target, so they must not be charged for it. target_ctx_copy = 0 if mtp_keeps_target_ctx and self._kv_lora_rank is not None: - target_ctx_copy = self._estimate_kv_cache_bytes(n_ctx, "f16", n_parallel = n_parallel) + target_ctx_copy = self._estimate_kv_cache_bytes( + n_ctx, + "f16", + n_parallel = n_parallel, + swa_full = swa_full, + kv_unified = kv_unified, + n_ubatch = n_ubatch, + flash_attn = flash_attn, + ) if draft_kv is None: # KV unsized (exotic/remote drafter): still reserve known weights + any # MLA target copy so a large config can't launch over budget (the small @@ -4283,7 +4636,7 @@ class LlamaCppBackend: return total if total > 0 else None return draft_kv + weights + target_ctx_copy - _DEFAULT_N_UBATCH = 512 # llama.cpp --ubatch default; Unsloth does not override it + _DEFAULT_N_UBATCH = _DEFAULT_LLAMA_N_UBATCH _COMPUTE_BUFFER_SAFETY = 1.15 # upper-bound margin on the compute-buffer estimate # Soft VRAM the modeled terms omit; charged to the fit budget on tight tiers (#6682). _CUDA_CONTEXT_RESERVE_BYTES = 320 * 1024 * 1024 # CUDA ctx + cuBLAS workspace (~330 MiB) @@ -4341,7 +4694,10 @@ class LlamaCppBackend: n_embd = self._embedding_length or 0 if n_vocab <= 0 or n_embd <= 0: return 0 - ub = max(1, int(n_ubatch if n_ubatch else self._DEFAULT_N_UBATCH)) + ub = max( + 1, + int(self._DEFAULT_N_UBATCH if n_ubatch is None else n_ubatch), + ) par = max(1, int(n_parallel)) out_buffer = n_vocab * ub * 4 # f32 output/logits buffer act_scratch = 4 * n_embd * ub * 4 # a few resident hidden-width buffers @@ -4373,7 +4729,10 @@ class LlamaCppBackend: n_embd = self._embedding_length or 0 if n_embd <= 0 or n_ctx <= 0: return 0 - ub = max(1, int(n_ubatch if n_ubatch else self._DEFAULT_N_UBATCH)) + ub = max( + 1, + int(self._DEFAULT_N_UBATCH if n_ubatch is None else n_ubatch), + ) if getattr(self, "_architecture", None) == "deepseek4": # DSV4 indexer/CSA buffer (see constants): flat + linear, ub-scaled. Fires # for any KV type -- the indexer scratch is present even with an f16 cache. @@ -4421,6 +4780,9 @@ class LlamaCppBackend: per_device_overhead_bytes: int, min_gpus: int, n_ubatch: Optional[int] = None, + swa_full: bool = False, + kv_unified: bool = True, + flash_attn: bool = True, ) -> tuple[Optional[list[int]], bool, int]: """Largest serving-slot count in [1, n_parallel) whose fully-on-GPU footprint fits, so Unsloth keeps the model on GPU (-ngl -1) instead of --fit on, which offloads layers @@ -4439,7 +4801,15 @@ class LlamaCppBackend: total = ( base_footprint_bytes + cb - + self._estimate_kv_cache_bytes(effective_ctx, cache_type_kv, n_parallel = slots) + + self._estimate_kv_cache_bytes( + effective_ctx, + cache_type_kv, + n_parallel = slots, + swa_full = swa_full, + kv_unified = kv_unified, + n_ubatch = n_ubatch, + flash_attn = flash_attn, + ) ) gpu_indices, use_fit = self._select_gpus( total, @@ -4464,7 +4834,9 @@ class LlamaCppBackend: swa_full: bool = False, n_parallel: int = 1, kv_unified: bool = True, + n_ubatch: Optional[int] = None, ctx_checkpoints: int = 0, + flash_attn: bool = True, kv_on_gpu: bool = True, mtp_engaged: bool = False, mtp_overhead_fn: Optional[Callable[[int], int]] = None, @@ -4501,7 +4873,9 @@ class LlamaCppBackend: swa_full = swa_full, n_parallel = n_parallel, kv_unified = kv_unified, + n_ubatch = n_ubatch, ctx_checkpoints = ctx_checkpoints, + flash_attn = flash_attn, ) # byte-accurate mtp_overhead_fn supersedes the flat fraction (the fallback @@ -5148,7 +5522,9 @@ class LlamaCppBackend: stdout = subprocess.PIPE, stderr = subprocess.STDOUT, text = True, - env = env, + encoding = "utf-8", + errors = "replace", + env = utf8_child_env(env), **_windows_hidden_subprocess_kwargs(), **_child_popen_kwargs(), ) @@ -5164,6 +5540,12 @@ class LlamaCppBackend: self._is_audio = False # clear any prior TTS/audio model's routing flag self._model_identifier = model_identifier self._cache_type_kv = None + self._swa_full = False + self._kv_cache_unified = False + self._n_ubatch = self._DEFAULT_N_UBATCH + self._flash_attn_enabled = True + self._effective_cache_types = ("f16", "f16") + self._kv_cache_context_total = None self._gpu_offload_active = True # Diffusion doesn't use the llama.cpp GPU-memory knobs; reset them to # defaults (the picked device is still recorded below) so /load, /status @@ -5905,6 +6287,9 @@ class LlamaCppBackend: total_by_idx: Optional[dict[int, int]] = None, n_ubatch: Optional[int] = None, soft_overhead_bytes: int = 0, + swa_full: bool = False, + kv_unified: bool = True, + flash_attn: bool = True, ) -> tuple[int, int, list[int], Optional[list[int]]]: """Plan a ``--split-mode tensor`` load. Pure: no model or GPU needed. @@ -5992,6 +6377,17 @@ class LlamaCppBackend: def _mtp_at(ctx: int) -> int: return mtp_overhead_fn(ctx) if mtp_overhead_fn is not None else 0 + def _kv_at(ctx: int) -> int: + return self._estimate_kv_cache_bytes( + ctx, + cache_type_kv, + n_parallel = n_parallel, + swa_full = swa_full, + kv_unified = kv_unified, + n_ubatch = n_ubatch, + flash_attn = flash_attn, + ) + # Context-linear compute buffer, summed over the split. Tensor mode # replicates the compute graph on EVERY device (measured: the per-device # buffer grows a flat n_ubatch*2 bytes/token, ~1024 B/tok on Qwen3.5-9B at @@ -6017,31 +6413,21 @@ class LlamaCppBackend: # Weights + buffers exceed the pool -> floor; the load then # falls back to layer split. return ctx_floor - if mtp_overhead_fn is not None: - # kv(ctx)+mtp(ctx)+compute(ctx) is not single-linear, so binary search. - def _consumer(c: int) -> int: - return ( - self._estimate_kv_cache_bytes(c, cache_type_kv, n_parallel = n_parallel) - + _mtp_at(c) - + _cc_ctx(c) - ) - if _consumer(ctx) <= kv_budget_b: - return ctx - lo, hi, best = ctx_floor, ctx, ctx_floor - while lo <= hi: - mid = (lo + hi) // 2 - if _consumer(mid) <= kv_budget_b: - best = mid - lo = mid + 1 - else: - hi = mid - 1 - return best - kv_at = self._estimate_kv_cache_bytes(ctx, cache_type_kv, n_parallel = n_parallel) - total_at = kv_at + _cc_ctx(ctx) # both ~linear through the origin - if total_at <= kv_budget_b: + def _consumer(c: int) -> int: + return _kv_at(c) + _mtp_at(c) + _cc_ctx(c) + + if _consumer(ctx) <= kv_budget_b: return ctx - return max(ctx_floor, int(ctx * kv_budget_b / total_at)) + lo, hi, best = ctx_floor, ctx, ctx_floor + while lo <= hi: + mid = (lo + hi) // 2 + if _consumer(mid) <= kv_budget_b: + best = mid + lo = mid + 1 + else: + hi = mid - 1 + return best # KV size unknown -> can't prove a safe cap; floor. return min(4096, ctx) if ctx > 0 else 4096 @@ -6053,11 +6439,7 @@ class LlamaCppBackend: effective_ctx = min(_fit_ctx(target_ctx), max_available_ctx) min_usable_mib = min(usable_by_idx.values()) - kv_bytes = ( - self._estimate_kv_cache_bytes(effective_ctx, cache_type_kv, n_parallel = n_parallel) - if (self._can_estimate_kv() and effective_ctx > 0) - else 0 - ) + kv_bytes = _kv_at(effective_ctx) if (self._can_estimate_kv() and effective_ctx > 0) else 0 # The MTP reserve also has to fit the even split (mirror the pooled budget): # byte-accurate per-ctx (0 when no fn) plus the same flat cushion as above. mtp_bytes = (_mtp_at(effective_ctx) if effective_ctx > 0 else 0) + flat_mtp_bytes @@ -6182,21 +6564,6 @@ class LlamaCppBackend: cls._is_signal_crash(returncode) or cls._is_abort_exit(returncode) ) - @staticmethod - def _canonical_long_flag(name: str) -> str: - """Return ``name`` with llama.cpp's long-option underscore normalization. - - llama.cpp runs ``std::replace(arg.begin(), arg.end(), '_', '-')`` on any - argv token that starts with ``--`` before looking it up, so a legal - pass-through spelling like ``--cache_type_v`` parses as - ``--cache-type-v``. Mirror that here so managed-flag matching sees the - same canonical name. Short flags (``-ctv``) never carry underscores and - keep their exact spelling; pass only the flag name (no attached value). - """ - if name.startswith("--"): - return name.replace("_", "-") - return name - @staticmethod def _with_flash_attn_off(cmd: list[str]) -> Optional[list[str]]: """Return cmd with flash attention forced off, or None when its effective @@ -6209,23 +6576,25 @@ class LlamaCppBackend: def explicit(i): nxt = out[i + 1] if i + 1 < len(out) else None - return nxt if nxt in ("on", "auto", "off") else None + return nxt if nxt in _LLAMA_ARG_TRUE_FALSE_AUTO_VALUES else None effective = None for i, tok in enumerate(out): - if tok.startswith(("--flash-attn=", "-fa=")): + name = _flag_name(tok) + if name in ("--flash-attn", "-fa") and "=" in tok: effective = tok.partition("=")[2] - elif tok in ("--flash-attn", "-fa"): + elif name in ("--flash-attn", "-fa"): effective = explicit(i) or "on" - if effective not in ("on", "auto"): + if effective not in _LLAMA_ARG_TRUE_OR_AUTO_VALUES: return None for i, tok in enumerate(out): - if tok.startswith(("--flash-attn=", "-fa=")): + name = _flag_name(tok) + if name in ("--flash-attn", "-fa") and "=" in tok: flag, _, value = tok.partition("=") - if value in ("on", "auto"): + if value in _LLAMA_ARG_TRUE_OR_AUTO_VALUES: out[i] = f"{flag}=off" - elif tok in ("--flash-attn", "-fa"): - if explicit(i) in ("on", "auto"): + elif name in ("--flash-attn", "-fa"): + if explicit(i) in _LLAMA_ARG_TRUE_OR_AUTO_VALUES: out[i + 1] = "off" elif explicit(i) is None: # bare flag (reads as on) -> explicit off out[i] = f"{tok}=off" @@ -6257,7 +6626,7 @@ class LlamaCppBackend: # quantized V cache. Canonicalize the flag name the same way so the # reset recognizes the underscore aliases too; short flags (-ctv) # and the type value are left untouched. - name = LlamaCppBackend._canonical_long_flag(tok.partition("=")[0]) + name = _flag_name(tok) if name not in _v_cache_flags: continue if "=" in tok: @@ -6369,6 +6738,8 @@ class LlamaCppBackend: stdout = subprocess.PIPE, stderr = subprocess.STDOUT, text = True, + encoding = "utf-8", + errors = "replace", env = env, **_windows_hidden_subprocess_kwargs(), **_child_popen_kwargs(), @@ -6487,6 +6858,7 @@ class LlamaCppBackend: chat_template_override = chat_template_override, extra_args = extra_args, is_vision = is_vision, + n_parallel = n_parallel, preserve_multi_gpu_on_layer = preserve_multi_gpu_on_layer, ): logger.info( @@ -6514,6 +6886,25 @@ class LlamaCppBackend: binary = self._find_llama_server_binary() is_vulkan_backend = self._is_vulkan_backend(binary) + # Without --kv-unified an explicit --parallel N splits -c into windows of -c/N, so on a + # build lacking the flag the default of 4 would quarter every context window for a + # feature it cannot serve: fall back to one slot. Ahead of the KV estimates so the + # fit matches what launches. + if ( + n_parallel > 1 + and binary + and not self.probe_server_capabilities(binary).get("supports_kv_unified") + ): + logger.warning( + "llama-server at %s has no --kv-unified, so %d parallel slots would " + "split the context window %d ways. Using 1 slot instead; update " + "llama.cpp to run chats in parallel.", + binary, + n_parallel, + n_parallel, + ) + n_parallel = 1 + # ── Vulkan-ordinal preflight (BEFORE the Phase 1 kill) ──────── # An explicit Vulkan pin the ggml probe never enumerated cannot be honored. # Validate it ABOVE the kill so an invalid selection leaves the live model @@ -6635,12 +7026,23 @@ class LlamaCppBackend: # Block-diffusion GGUFs (DiffusionGemma) cannot run on llama-server; # serve them with the diffusion runner (same OpenAI-compat interface). if self._is_diffusion: - # Final defense: route and pre-teardown preflights reject before Phase 1. - if is_vulkan_backend and gpu_ids: - raise ValueError(_VULKAN_DIFFUSION_GPU_IDS_ERROR) # Not a tensor/layer GGUF: clear any preserved-fallback flag from a # prior load (this path skips the command builder that clears it). self._layer_preserves_tensor_intent = False + # On a Vulkan build gpu_ids are ggml Vulkan ordinals, but the diffusion + # runner selects its device by CUDA physical index (_diffusion_gpu_arg + # forwards gpu_ids[0] as a CUDA/DG_GPU token) with no mapping to them. + # The route rejects a CONFIRMED-diffusion pick up front; an uncached GGUF + # only classified as diffusion post-download still reaches here with a + # pin, so drop it and serve on the default device (like an unpinned load). + if gpu_ids and is_vulkan_backend: + logger.warning( + "Ignoring gpu_ids %s for diffusion GGUF on a Vulkan build: " + "the diffusion runner cannot map ggml Vulkan ordinals; " + "serving on the default device.", + gpu_ids, + ) + gpu_ids = None with self._lock: if self._cancel_event.is_set(): logger.info("Load cancelled before diffusion server start") @@ -6672,6 +7074,8 @@ class LlamaCppBackend: # same message remote validation already shows. raise LlamaServerNotFoundError(LLAMA_SERVER_NOT_FOUND_DETAIL) + server_caps = self.probe_server_capabilities(binary) + # Outside ``self._lock`` so /unload, /cancel, /status aren't # blocked. ``unload_model`` also records the kill, so the # frontend /unload+/load Apply path engages the wait here even @@ -6692,6 +7096,18 @@ class LlamaCppBackend: # state to publish. ctx_override = parse_ctx_override(extra_args) requested_ctx = resolve_requested_ctx(extra_args, n_ctx) + swa_full = _swa_full_from_args_or_env(extra_args) + _effective_ubatch = _extra_args_n_ubatch( + extra_args, + n_ctx = (requested_ctx if requested_ctx > 0 else self._context_length), + ) + planned_kv_unified = _kv_unified_from_args( + extra_args, + default = n_parallel > 1 and server_caps.get("supports_kv_unified", False), + ) + # A hard-crash recovery may relaunch this same plan with FA off. + # Size that larger cache up front so the recovery cannot OOM. + planned_flash_attn = False cache_override = parse_cache_override(extra_args) # Budget the heavier of asymmetric --cache-type-k/-v extras (they # win per axis at launch, appended last); resolve_cache_type_kv only @@ -7122,6 +7538,10 @@ class LlamaCppBackend: draft_cache_type_k = _mtp_draft_ck, draft_cache_type_v = _mtp_draft_cv, n_parallel = n_parallel, + swa_full = swa_full, + kv_unified = planned_kv_unified, + n_ubatch = _effective_ubatch, + flash_attn = planned_flash_attn, ) if ( self._estimate_mtp_overhead_bytes( @@ -7133,6 +7553,10 @@ class LlamaCppBackend: draft_weights_bytes = _mtp_draft_weights, n_parallel = n_parallel, mtp_keeps_target_ctx = _engaged_is_mtp, + swa_full = swa_full, + kv_unified = planned_kv_unified, + n_ubatch = _effective_ubatch, + flash_attn = planned_flash_attn, ) is not None ): @@ -7149,6 +7573,10 @@ class LlamaCppBackend: _w: int = _mtp_draft_weights, _np: int = n_parallel, _mtp: bool = _engaged_is_mtp, + _swa_full: bool = swa_full, + _kv_unified: bool = planned_kv_unified, + _n_ubatch: Optional[int] = _effective_ubatch, + _flash_attn: bool = planned_flash_attn, ) -> int: v = self._estimate_mtp_overhead_bytes( ctx, @@ -7159,15 +7587,26 @@ class LlamaCppBackend: draft_weights_bytes = _w, n_parallel = _np, mtp_keeps_target_ctx = _mtp, + swa_full = _swa_full, + kv_unified = _kv_unified, + n_ubatch = _n_ubatch, + flash_attn = _flash_attn, ) return v if v is not None else 0 def _mtp_bytes(ctx: int) -> int: return mtp_overhead_fn(ctx) if mtp_overhead_fn is not None else 0 - # Effective micro-batch (a user --ubatch override scales the - # compute buffer); None -> the 512 default in the estimate. - _effective_ubatch = _extra_args_n_ubatch(extra_args) + def _kv_bytes(ctx: int) -> int: + return self._estimate_kv_cache_bytes( + ctx, + cache_type_kv, + n_parallel = n_parallel, + swa_full = swa_full, + kv_unified = planned_kv_unified, + n_ubatch = _effective_ubatch, + flash_attn = planned_flash_attn, + ) def _cc_bytes(ctx: int, n_gpus: int = 1) -> int: # Context-linear compute-buffer growth (flash-attn KQ mask + @@ -7407,6 +7846,9 @@ class LlamaCppBackend: total_by_idx = total_by_idx, n_ubatch = _effective_ubatch, soft_overhead_bytes = _soft_overhead, + swa_full = swa_full, + kv_unified = planned_kv_unified, + flash_attn = planned_flash_attn, ) use_fit = False elif gpus and self._can_estimate_kv() and effective_ctx > 0: @@ -7439,16 +7881,18 @@ class LlamaCppBackend: pool_budget, _ms, cache_type_kv, + swa_full = swa_full, n_parallel = n_parallel, + kv_unified = planned_kv_unified, + n_ubatch = _effective_ubatch, + flash_attn = planned_flash_attn, mtp_engaged = _mtp_reserves_gpu, mtp_overhead_fn = mtp_overhead_fn, compute_ctx_bytes_fn = _cc_sub, budget_frac = 1.0, total_mib = None, ) - kv = self._estimate_kv_cache_bytes( - capped, cache_type_kv, n_parallel = n_parallel - ) + kv = _kv_bytes(capped) footprint_mib = ( _ms + kv + _mtp_bytes(capped) + _cc_sub(capped) ) / (1024 * 1024) @@ -7468,9 +7912,7 @@ class LlamaCppBackend: # on and let llama-server flex -ngl (CPU offload). requested_total = ( model_size_fit - + self._estimate_kv_cache_bytes( - effective_ctx, cache_type_kv, n_parallel = n_parallel - ) + + _kv_bytes(effective_ctx) + _mtp_bytes(effective_ctx) + _cc_bytes(effective_ctx) ) @@ -7522,16 +7964,18 @@ class LlamaCppBackend: pool_budget, _ms, cache_type_kv, + swa_full = swa_full, n_parallel = n_parallel, + kv_unified = planned_kv_unified, + n_ubatch = _effective_ubatch, + flash_attn = planned_flash_attn, mtp_engaged = _mtp_reserves_gpu, mtp_overhead_fn = mtp_overhead_fn, compute_ctx_bytes_fn = _cc_sub, budget_frac = 1.0, total_mib = None, ) - kv = self._estimate_kv_cache_bytes( - capped, cache_type_kv, n_parallel = n_parallel - ) + kv = _kv_bytes(capped) footprint_mib = ( _ms + kv + _mtp_bytes(capped) + _cc_sub(capped) ) / (1024 * 1024) @@ -7548,11 +7992,7 @@ class LlamaCppBackend: if effective_ctx > 0: for n_gpus in range(_auto_min_gpus, len(ranked) + 1): subset = ranked[:n_gpus] - kv = self._estimate_kv_cache_bytes( - effective_ctx, - cache_type_kv, - n_parallel = n_parallel, - ) + kv = _kv_bytes(effective_ctx) footprint_mib = ( _subset_model_size(n_gpus) + kv @@ -7609,7 +8049,11 @@ class LlamaCppBackend: _apple_fit_budget_mib, model_size_fit, cache_type_kv, + swa_full = swa_full, n_parallel = n_parallel, + kv_unified = planned_kv_unified, + n_ubatch = _effective_ubatch, + flash_attn = planned_flash_attn, mtp_engaged = _mtp_reserves_gpu, mtp_overhead_fn = mtp_overhead_fn, compute_ctx_bytes_fn = _cc_bytes, @@ -7617,12 +8061,7 @@ class LlamaCppBackend: total_mib = None, ) _cap_footprint_mib = ( - model_size_fit - + self._estimate_kv_cache_bytes( - cap, cache_type_kv, n_parallel = n_parallel - ) - + _mtp_bytes(cap) - + _cc_bytes(cap) + model_size_fit + _kv_bytes(cap) + _mtp_bytes(cap) + _cc_bytes(cap) ) / (1024 * 1024) # Fit returns the request unchanged when it fits OR weights # exceed budget; only the latter over-commits, so floor to 4096. @@ -7669,6 +8108,9 @@ class LlamaCppBackend: _pipeline_overhead_bytes + _cc_bytes(effective_ctx), _layer_min_gpus, _effective_ubatch, + swa_full = swa_full, + kv_unified = planned_kv_unified, + flash_attn = planned_flash_attn, ) if not _uf_slots: logger.info( @@ -7693,9 +8135,7 @@ class LlamaCppBackend: _mtp_note = "" if effective_ctx < original_ctx: - kv_est = self._estimate_kv_cache_bytes( - effective_ctx, cache_type_kv, n_parallel = n_parallel - ) + kv_est = _kv_bytes(effective_ctx) logger.info( f"Context auto-reduced: {original_ctx} -> {effective_ctx} " f"(model: {model_size / (1024**3):.1f} GB, " @@ -7704,9 +8144,7 @@ class LlamaCppBackend: + ")" ) - kv_cache_bytes = self._estimate_kv_cache_bytes( - effective_ctx, cache_type_kv, n_parallel = n_parallel - ) + kv_cache_bytes = _kv_bytes(effective_ctx) mmproj_note = ( f"mmproj: {mmproj_size / (1024**3):.1f} GB, " if mmproj_size else "" ) @@ -7871,7 +8309,6 @@ class LlamaCppBackend: cmd.extend(["-ngl", "-1", "--fit", "off"]) fully_gpu_offloaded = True - server_caps = self.probe_server_capabilities(binary) # Expose Prometheus /metrics for the engine-stats logger, only # when the binary advertises it (older/custom binaries may not). if server_caps.get("supports_metrics"): @@ -7943,6 +8380,11 @@ class LlamaCppBackend: "iq4_nl", "f32", } + # Normalize like the budget does (_planned_main_cache_types): a + # case-sensitive match drops "Q8_0", emitting no flag, so llama.cpp + # runs f16 while the estimate priced q8_0. Emit the normalized + # spelling; kv_cache_type_from_str is case-sensitive. + cache_type_kv = cache_type_kv.strip().lower() if cache_type_kv else cache_type_kv if ( cache_type_kv and cache_type_kv in _valid_cache_types @@ -8145,6 +8587,8 @@ class LlamaCppBackend: cmd.extend(str(a) for a in extra_args) logger.info(f"Appending user extra args to llama-server: {list(extra_args)}") + kv_cache_unified = _kv_unified_from_args(cmd) + logger.info(f"Starting llama-server: {' '.join(self._redacted_cmd_for_log(cmd))}") # Library paths so llama-server finds its shared libs and CUDA DLLs. @@ -8236,7 +8680,7 @@ class LlamaCppBackend: env["CUDA_DEVICE_ORDER"] = "PCI_BUS_ID" # Mask on AMD at the ROCr/HSA layer: HIP-only masking still # enumerates every agent first, which segfaults on a deselected - # unsupported GPU (e.g. gfx1103 iGPU under a gfx110X prebuilt). + # unsupported GPU (e.g. gfx1036 iGPU under a gfx103X prebuilt). self._emit_child_gpu_visibility( env, ",".join(str(i) for i in gpu_indices), prefer_rocr = True ) @@ -8312,6 +8756,8 @@ class LlamaCppBackend: stdout = subprocess.PIPE, stderr = subprocess.STDOUT, text = True, + encoding = "utf-8", + errors = "replace", env = env, **_windows_hidden_subprocess_kwargs(), **_child_popen_kwargs(), @@ -8659,6 +9105,21 @@ class LlamaCppBackend: self._healthy = True self._commit_effective_parallel_slots(n_parallel) + self._swa_full = swa_full + self._kv_cache_unified = kv_cache_unified + self._n_ubatch = max( + 0, + int(self._DEFAULT_N_UBATCH if _effective_ubatch is None else _effective_ubatch), + ) + self._flash_attn_enabled = ( + _flash_attn_enabled_from_args(_last_spawn_cmd, env = env) + and self._architecture != "grok" + ) + self._effective_cache_types = _effective_main_cache_types( + _last_spawn_cmd, + env, + ) + self._kv_cache_context_total = effective_ctx if effective_ctx > 0 else None # Server is up: adopt the real per-request context it allocated # -- the length --fit chose, or a --parallel slot split -- so the @@ -8666,6 +9127,11 @@ class LlamaCppBackend: # before the spawn above always failed; the seeded value was the # requested/native length.) self._reconcile_effective_ctx_with_server() + if self._kv_cache_context_total is not None: + self._n_ubatch = min( + self._n_ubatch, + self._kv_cache_context_total, + ) # Commit caller intent only after _healthy=True so a failed start # can't poison the next inheritance check. None keeps prior, [] @@ -8675,6 +9141,8 @@ class LlamaCppBackend: self._extra_args = list(extra_args) self._extra_args_source = (model_identifier, hf_variant) self._requested_n_ctx = int(n_ctx) + # Local n_parallel may have been reduced above; the snapshot has the ask. + self._requested_n_parallel = max(1, int(_pending_load_kwargs["n_parallel"])) # Commit the known-good snapshot + whether MTP+tensor is live, then # watch this load for a mid-generation crash. self._last_load_kwargs = _pending_load_kwargs @@ -9087,6 +9555,7 @@ class LlamaCppBackend: tensor_split: Optional[List[float]] = None, gpu_ids: Optional[List[int]] = None, mtp_draft_path: Optional[str] = None, + n_parallel: int = 1, preserve_multi_gpu_on_layer: bool = False, ) -> bool: """True iff the live server already satisfies these load kwargs. @@ -9122,7 +9591,6 @@ class LlamaCppBackend: if _norm(self._cache_type_kv) != _norm(cache_type_kv): return False - # Reconcile a user --split-mode in extras AND an inherited tensor # LLAMA_ARG_SPLIT_MODE env, but only against a server that actually # launched tensor: if load_model downgraded to layer split it scrubbed @@ -9146,9 +9614,16 @@ class LlamaCppBackend: # layer/MoE/split knobs), so a standing manual preference in the # request must not force a needless reload -- only the GPU pick matters. if not self._is_diffusion: + requested_extra_args = extra_args if extra_args is not None else self._extra_args + if self._swa_full != _swa_full_from_args_or_env(requested_extra_args): + return False # A GPU-memory-mode flip (Unsloth / manual) must always reload. if self._gpu_memory_mode != gpu_memory_mode: return False + # Requested-vs-requested (like n_ctx): comparing the effective count + # would reload forever whenever the fitter launched fewer slots. + if self._requested_n_parallel != max(1, int(n_parallel)): + return False # Manual: a layer-count change always reloads (covers Auto(-1) <-> a # pinned count); MoE/split only matter with an explicit offload. if gpu_memory_mode == "manual" and ( @@ -9273,7 +9748,8 @@ class LlamaCppBackend: last_draft: Optional[str] = None args = [str(arg) for arg in cmd] for index, raw in enumerate(args): - flag, equals, inline = raw.partition("=") + flag = _flag_name(raw) + _, equals, inline = raw.partition("=") if flag not in main_flags and flag not in draft_flags: continue value = inline if equals else (args[index + 1] if index + 1 < len(args) else "") @@ -9346,6 +9822,12 @@ class LlamaCppBackend: self._slot_save_binary = None self._slot_loaded_identity = None self._prompt_cache_disabled = False + self._swa_full = False + self._kv_cache_unified = False + self._n_ubatch = self._DEFAULT_N_UBATCH + self._flash_attn_enabled = True + self._effective_cache_types = ("f16", "f16") + self._kv_cache_context_total = None self._chat_template = None self._chat_template_override = None self._supports_reasoning = False @@ -9778,6 +10260,8 @@ class LlamaCppBackend: ["pgrep", "-a", "-f", "llama-server"], capture_output = True, text = True, + encoding = "utf-8", + errors = "replace", timeout = 5, env = child_env_without_native_path_secret(), ) @@ -9889,8 +10373,12 @@ class LlamaCppBackend: tuple(sidecars), self._requested_n_ctx, self._effective_context_length, - getattr(self, "_cache_type_kv", None), + self._effective_cache_types, self.effective_parallel_slots, + self._swa_full, + self._kv_cache_unified, + self._n_ubatch, + self._flash_attn_enabled, ) def _gguf_file_identity(self, path) -> Optional[tuple]: @@ -9921,7 +10409,8 @@ class LlamaCppBackend: args = [str(a).strip() for a in (self._extra_args or ())] files: list[str] = [] for i, arg in enumerate(args): - flag, sep, inline = arg.partition("=") + flag = _flag_name(arg) + _, sep, inline = arg.partition("=") if flag not in self._SIDECAR_WEIGHT_FLAGS: continue operand = inline if sep else (args[i + 1] if i + 1 < len(args) else "") @@ -9961,7 +10450,7 @@ class LlamaCppBackend: if os.environ.get("LLAMA_ARG_NO_CACHE_PROMPT") is not None: return True env = (os.environ.get("LLAMA_ARG_CACHE_PROMPT") or "").strip().lower() - return env in {"off", "disabled", "false", "0"} + return env in _LLAMA_ARG_FALSE_VALUES def save_slots_for_resume( self, should_abort: Optional[Callable[[], bool]] = None @@ -9973,6 +10462,17 @@ class LlamaCppBackend: or self._prompt_cache_off() ): return None + # Same predicate as the estimator's SWA path: a window alone is not enough. + # phi3 GGUFs carry attention.sliding_window but no key/value length, and + # llama.cpp forces them back to a non-SWA cache, so their slots do restore. + if ( + (self._sliding_window or 0) > 0 + and self._kv_key_length is not None + and self._kv_value_length is not None + and not self._swa_full + ): + logger.debug("Skipping slot save: compact SWA cache cannot be reused after restart") + return None save_dir = Path(self._slot_save_dir) gguf_stat = self._gguf_file_identity(self._gguf_path) if gguf_stat is None: @@ -9989,9 +10489,16 @@ class LlamaCppBackend: return None try: estimate = self._estimate_kv_cache_bytes( - self._effective_context_length or self._context_length or 0, - self._cache_type_kv, + self._kv_cache_context_total + or self._effective_context_length + or self._context_length + or 0, + max(self._effective_cache_types, key = _kv_bytes_per_elem), n_parallel = self.effective_parallel_slots, + swa_full = self._swa_full, + kv_unified = self._kv_cache_unified, + n_ubatch = self._n_ubatch, + flash_attn = self._flash_attn_enabled, ) # Skip before writing anything when the estimate alone blows the cap, # rather than fully writing a slot and discarding it afterwards. @@ -10347,6 +10854,8 @@ class LlamaCppBackend: actual_n_ctx = self._query_server_n_ctx() if not actual_n_ctx or actual_n_ctx <= 0: return + slots = 1 if self._kv_cache_unified else self.effective_parallel_slots + self._kv_cache_context_total = actual_n_ctx * slots if self._effective_context_length and actual_n_ctx < self._effective_context_length: logger.warning( "llama-server allocated a smaller per-request context than " @@ -11053,6 +11562,7 @@ class LlamaCppBackend: from core.inference.tools import ( build_rag_autoinject, execute_tool, + has_text_only_provisional_card, is_always_safe_tool, is_high_risk_tool_call, ) @@ -11250,6 +11760,10 @@ class LlamaCppBackend: # direct answer ("4", "Hello!") won't match. Pattern shared with the # safetensors loop (tool_call_parser.INTENT_SIGNAL). _reprompt_count = 0 + # Budgeted apart from _reprompt_count so a pre-tool nudge can't spend it. + _post_tool_reprompts = 0 + # Text that triggered the last nudge; if the retry restates it, stop. + _last_reprompt_text = "" # Gates ``max_tool_iterations`` on real tool turns (not the enlarged range) so reserved # re-prompt slots don't extend the budget. Mirrors the safetensors guard. _tool_iters_done = 0 @@ -11257,7 +11771,7 @@ class LlamaCppBackend: # Reserve extra iterations for re-prompts so they don't consume the # caller's tool-call budget; only when tool iterations are allowed. - _extra = _MAX_REPROMPTS if max_tool_iterations > 0 else 0 + _extra = _MAX_REPROMPTS + 1 if max_tool_iterations > 0 else 0 for iteration in range(max_tool_iterations + _extra): if cancel_event is not None and cancel_event.is_set(): return @@ -11323,6 +11837,7 @@ class LlamaCppBackend: # Time each reasoning pass so final answers can replace tool timing. _reasoning_started_at = None _reasoning_summary_emitted = False + _deferred_reasoning_summary = None cumulative_display = "" # Cumulative yielded text (with ) in_thinking = False has_content_tokens = False @@ -11479,6 +11994,9 @@ class LlamaCppBackend: permission_mode == "auto" and is_always_safe_tool(current_name) ) + # A text-preview card still streams while gated; + # hiding it blanks the chat. + and not has_text_only_provisional_card(current_name) ) # Keep small-argument tools on the normal path. _args_len = len( @@ -11571,7 +12089,11 @@ class LlamaCppBackend: and not _reasoning_summary_emitted ): _reasoning_summary_emitted = True - yield _reasoning_summary_event(_reasoning_started_at) + _summary = _reasoning_summary_event(_reasoning_started_at) + if _suppress_visible_output: + _deferred_reasoning_summary = _summary + else: + yield _summary has_content_tokens = True content_accum += token @@ -11580,20 +12102,27 @@ class LlamaCppBackend: # TEXT call to a provisional card. Gated on an enabled-name # sniff + size floor so prose/small calls spawn no pane; id # matches the first call so the final tool_start reconciles. - if ( - not has_structured_tc - and not _confirm_gated_iteration - and _text_args_call_start >= 0 - ): + if not has_structured_tc and _text_args_call_start >= 0: if not _text_args_id: _call_text = content_accum[_text_args_call_start:] _sniffed = _sniff_text_tool_name( _call_text, _enabled_tool_names ) - if _sniffed and ( - _sniffed == "render_html" - or len(_call_text) - >= _PROVISIONAL_ARGS_MIN_CHARS + # Structured-path rule: gated calls + # stream only from a text-preview card. + if ( + _sniffed + and not ( + _confirm_gated_iteration + and not has_text_only_provisional_card( + _sniffed + ) + ) + and ( + _sniffed == "render_html" + or len(_call_text) + >= _PROVISIONAL_ARGS_MIN_CHARS + ) ): _text_args_id = "call_0" _text_args_name = _sniffed @@ -11848,7 +12377,11 @@ class LlamaCppBackend: # route's extractor closes the streamed ). if _reasoning_started_at is not None and not _reasoning_summary_emitted: _reasoning_summary_emitted = True - yield _reasoning_summary_event(_reasoning_started_at) + _summary = _reasoning_summary_event(_reasoning_started_at) + if _suppress_visible_output: + _deferred_reasoning_summary = _summary + else: + yield _summary cumulative_display = _finalize_reasoning_only_cumulative( cumulative_display, reasoning_accum, @@ -11883,12 +12416,10 @@ class LlamaCppBackend: ) if not _safety_tc: # ── Re-prompt on plan-without-action ── - # If the model described its intent (forward-looking - # language) without calling a tool, nudge it to act. - # Fires at most once per request, only on short - # responses with intent signals -- "4" or "Hello!" - # won't trigger it. Use content if available, else - # fall back to reasoning text (reasoning-only stalls). + # Intent described without a tool call: nudge it to act. Up + # to _MAX_REPROMPTS times, only on short responses with intent + # signals -- "4" or "Hello!" won't trigger it. Uses content, + # else reasoning text (reasoning-only stalls). _stripped = content_accum.strip() if not _stripped: _stripped = reasoning_accum.strip() @@ -11898,18 +12429,33 @@ class LlamaCppBackend: r"(?i)\brender[_\s-]?html\b", _stripped, ) + # A post-tool stall still deserves a nudge, but each retry + # re-runs tools, so allow only one. RAG autoinject never lands + # in history, so _auto keeps a doc-grounded turn from reading + # as pre-tool (mirrors safetensors rag_autoinjected). + _already_acted = bool(_auto) or any( + record.executed for record in tool_controller.history + ) + if _already_acted: + _reprompt_used, _reprompt_cap = _post_tool_reprompts, 1 + else: + _reprompt_used, _reprompt_cap = _reprompt_count, _MAX_REPROMPTS # None keeps the default-on re-prompt; False disables it. if ( auto_heal_tool_calls and (nudge_tool_calls is None or nudge_tool_calls) and active_tools and not _render_html_already_done_intent - and _reprompt_count < _MAX_REPROMPTS + and _reprompt_used < _reprompt_cap + and not _is_reprompt_repeat(_stripped, _last_reprompt_text) and _is_short_intent_without_action(_stripped) ): _reprompt_count += 1 + if _already_acted: + _post_tool_reprompts += 1 + _last_reprompt_text = _stripped logger.info( - f"Re-prompt {_reprompt_count}/{_MAX_REPROMPTS}: " + f"Re-prompt {_reprompt_used + 1}/{_reprompt_cap}: " f"model responded without calling tools " f"({len(_stripped)} chars)" ) @@ -11939,12 +12485,18 @@ class LlamaCppBackend: _it_r = _iter_timings or {} _accumulated_predicted_ms += _it_r.get("predicted_ms", 0) _accumulated_predicted_n += _it_r.get("predicted_n", 0) + # Blank first (the route resets its text cursor only on an + # empty status), then the badge so the retry is not a hang. yield {"type": "status", "text": ""} + yield {"type": "status", "text": _NUDGE_TOOL_CALLS_STATUS} continue if _forced_tool_call_pending: _forced_tool_call_pending = False - if not _should_suppress_forced_no_tool_output(_stripped): + if not _should_suppress_forced_no_tool_output( + _stripped, + _last_reprompt_text, + ): if cumulative_display: forced_visible_text = _strip_tool_markup( cumulative_display, @@ -11962,6 +12514,8 @@ class LlamaCppBackend: "type": "content", "text": forced_visible_text, } + if _deferred_reasoning_summary is not None: + yield _deferred_reasoning_summary elif not _suppress_visible_output: # Turn ended as a plain answer (no [ARGS] followed): the held # rehearsal tail is real prose, release it. @@ -12182,18 +12736,31 @@ class LlamaCppBackend: start_event["awaiting_confirmation"] = needs_confirm try: - yield {"type": "status", "text": decision.status_text} + # Gated calls are not running yet; a "Running ..." badge + # counting up while it waits on a human reads as a hang. + yield { + "type": "status", + "text": ( + awaiting_approval_status(decision.tool_name) + if needs_confirm + else decision.status_text + ), + } yield start_event - if ( - decision_slot is not None - and wait_tool_decision( + _decision = ( + wait_tool_decision( decision_slot, approval_id, cancel_event = cancel_event, ) - == "deny" - ): + if decision_slot is not None + else None + ) + if _decision is not None and _decision != "deny": + # Approved: now it really is running. + yield {"type": "status", "text": decision.status_text} + if _decision == "deny": decision_slot = None resolved_provisional_tool_call_ids.add(decision.tool_call_id) yield { @@ -12259,6 +12826,10 @@ class LlamaCppBackend: _kb_search_count += 1 completion = tool_controller.record_result(decision, result) resolved_provisional_tool_call_ids.add(decision.tool_call_id) + # A real execution opens the post-tool phase; carrying the pre-tool + # stall text over would read the same sentence as a repeat and + # swallow the one post-tool nudge. + _last_reprompt_text = "" # A tool ran this turn, so it counts against the caller's budget. _turn_executed_real_tool = True yield completion.tool_end_event() @@ -12761,10 +13332,15 @@ class LlamaCppBackend: min_p: float = 0.0, max_new_tokens: int = 2048, repetition_penalty: float = 1.1, + cancel_event: Optional[threading.Event] = None, ) -> tuple: """ Generate TTS audio via llama-server /completion + codec decode. Returns (wav_bytes, sample_rate). + + ``cancel_event`` lets a Stop or a forced model swap end the request: the + decode is one blocking POST, so a watcher closes the client out from under + it rather than polling. Raises RuntimeError once cancelled. """ if audio_type not in self._TTS_PROMPTS: raise RuntimeError(f"GGUF TTS does not support '{audio_type}' codec.") @@ -12786,15 +13362,47 @@ class LlamaCppBackend: if need_ids: payload["n_probs"] = 1 + if cancel_event is not None and cancel_event.is_set(): + raise RuntimeError("Audio generation cancelled") + with httpx.Client( timeout = httpx.Timeout(300, connect = 10), headers = self._auth_headers, trust_env = False, ) as client: - resp = client.post(f"{self.base_url}/completion", json = payload) + finished = threading.Event() + watcher: Optional[threading.Thread] = None + if cancel_event is not None: + + def _close_when_cancelled() -> None: + while not finished.wait(0.05): + if cancel_event.is_set(): + # Closing mid-request makes the blocking post raise + # httpx.RequestError, the only way out of it. + with contextlib.suppress(Exception): + client.close() + return + + watcher = threading.Thread(target = _close_when_cancelled, daemon = True) + watcher.start() + try: + resp = client.post(f"{self.base_url}/completion", json = payload) + except httpx.RequestError: + if cancel_event is not None and cancel_event.is_set(): + raise RuntimeError("Audio generation cancelled") from None + raise + finally: + finished.set() + if watcher is not None: + watcher.join(timeout = 0.5) if resp.status_code != 200: raise RuntimeError(f"llama-server returned {resp.status_code}: {resp.text}") + # The codec decode below is GPU work with no interruption point, so check here: + # cancelling after this only wastes the decode it cannot stop. + if cancel_event is not None and cancel_event.is_set(): + raise RuntimeError("Audio generation cancelled") + data = resp.json() token_ids = ( [p["id"] for p in data.get("completion_probabilities", []) if "id" in p] diff --git a/studio/backend/core/inference/llama_keepwarm.py b/studio/backend/core/inference/llama_keepwarm.py index 3380ebf5f5..05b1271b27 100644 --- a/studio/backend/core/inference/llama_keepwarm.py +++ b/studio/backend/core/inference/llama_keepwarm.py @@ -345,6 +345,22 @@ def _loaded_identity(backend): return (backend.model_identifier, getattr(backend, "hf_variant", None), advertised) +def _note_idle_unload_event(freed) -> None: + """Monitor row for an idle auto-unload. Best-effort; uses the stash's + advertised repo id so the row never shows the on-disk load path.""" + try: + from core.inference.api_monitor import api_monitor + from core.inference.model_ids import public_model_id + + identifier, variant, advertised = (list(freed) + [None, None, None])[:3] + label = public_model_id(advertised or identifier) or "model" + if variant and ":" not in label: + label = f"{label}:{variant}" + api_monitor.record_lifecycle(event = "unload", model = label, reason = "idle") + except Exception as exc: + logger.debug("idle unload monitor event failed: %s", exc) + + async def idle_unload_loop(poll_seconds: float = 15.0) -> None: """Unload the loaded GGUF once idle past the configured TTL. Inert when off.""" from utils.openai_auto_switch_settings import ( @@ -407,6 +423,8 @@ async def idle_unload_loop(poll_seconds: float = 15.0) -> None: elif manifest: _delete_resume_files(manifest) logger.info("Idle auto-unload: freed GGUF after %ss idle", ttl) + # An idle unload stashes for reload and skips note_model_unloaded. + _note_idle_unload_event(freed) seen_model = None except Exception as exc: logger.debug("idle_unload_loop iteration failed: %s", exc) diff --git a/studio/backend/core/inference/llama_server_args.py b/studio/backend/core/inference/llama_server_args.py index 7b42d2f40d..7391e62516 100644 --- a/studio/backend/core/inference/llama_server_args.py +++ b/studio/backend/core/inference/llama_server_args.py @@ -16,11 +16,18 @@ from __future__ import annotations import os from typing import Iterable, Mapping, Optional +# Valid llama-server --parallel range, shared with LoadRequest.n_parallel. +# Mirrored by callers that cannot import this: run.py and unsloth_cli/commands/ +# studio.py (_PARALLEL_MIN/MAX), per-model-config.ts (N_PARALLEL_MIN/MAX); +# test_parallel_slots_per_load.py pins them together. +PARALLEL_MIN = 1 +PARALLEL_MAX = 64 + # Each group = every alias (short + long) of one hard-denied flag. # Extend the matching group when llama.cpp adds a new alias. _DENYLIST_GROUPS: tuple[frozenset[str], ...] = ( - # Parallel slots: owned by typer --parallel; a pass-through would desync - # app.state.llama_parallel_slots from llama-server. + # Parallel slots: owned by typer --parallel and LoadRequest.n_parallel; a + # pass-through would desync the slot bookkeeping from llama-server. frozenset({"-np", "--parallel", "--n-parallel"}), # Model identity: Unsloth resolves it from LoadRequest; a second -m would # load a different model than Unsloth thinks it loaded. @@ -80,9 +87,10 @@ _DENYLIST: frozenset[str] = frozenset().union(*_DENYLIST_GROUPS) def _flag_name(token: str) -> Optional[str]: """Flag name for ``token``, or None if it isn't a flag. - Peels `--key=value` to `--key`, treats `-1`/`-0.5` as values (shorts - always start with a letter), and normalises attached `-np8` / `-np-1` / - `-np8x` to `-np`. Mirrors the CLI's `_expand_attached_np_short`. + Peels `--key=value` to `--key`, normalises long-option underscores like + llama.cpp, treats `-1`/`-0.5` as values (shorts always start with a letter), + and normalises attached `-np8` / `-np-1` / `-np8x` to `-np`. Mirrors the + CLI's `_expand_attached_np_short`. """ token = token.strip() if not token.startswith("-") or token in {"-", "--"}: @@ -90,6 +98,8 @@ def _flag_name(token: str) -> Optional[str]: if len(token) >= 2 and (token[1].isdigit() or token[1] == "."): return None name = token.split("=", 1)[0] + if name.startswith("--"): + name = name.replace("_", "-") if len(name) > 3 and name.startswith("-np"): suffix = name[3:] if suffix[0].isdigit() or ( @@ -118,6 +128,7 @@ def validate_extra_args(args: Optional[Iterable[str]]) -> list[str]: parse_ctx_override(out) parse_cache_override(out) parse_split_mode_override(out) + parse_gpu_layers_override(out) return out @@ -193,9 +204,8 @@ _SPLIT_SHADOWING_FLAGS: frozenset[str] = _SPLIT_MODE_FLAGS | _TENSOR_SPLIT_FLAGS # inherited -ngl is respected (the offload_overridden path), so this group is # opt-in, not default. Layer flags are shared with llama_cpp's override # detection; the MoE flags are strip-only (manual's --n-cpu-moe slider owns them). -_LAYER_OFFLOAD_FLAGS: frozenset[str] = frozenset( - {"-ngl", "--gpu-layers", "--n-gpu-layers", "-fit", "--fit"} -) +_GPU_LAYER_FLAGS: frozenset[str] = frozenset({"-ngl", "--gpu-layers", "--n-gpu-layers"}) +_LAYER_OFFLOAD_FLAGS: frozenset[str] = _GPU_LAYER_FLAGS | frozenset({"-fit", "--fit"}) _MOE_OFFLOAD_FLAGS: frozenset[str] = frozenset({"-ncmoe", "--n-cpu-moe", "-cmoe", "--cpu-moe"}) _OFFLOAD_SHADOWING_FLAGS: frozenset[str] = _LAYER_OFFLOAD_FLAGS | _MOE_OFFLOAD_FLAGS @@ -306,6 +316,26 @@ def parse_cache_override(args: Optional[Iterable[str]]) -> Optional[str]: return _last_flag_value(args, _CACHE_FLAGS) +def parse_gpu_layers_override(args: Optional[Iterable[str]]) -> Optional[int]: + """Return the last user-supplied GPU layer count from extras. + + Manual GPU memory mode strips llama.cpp offload flags because the + first-class load fields own them. Callers use this parser first to preserve + an explicit ``-ngl`` / ``--gpu-layers`` / ``--n-gpu-layers`` value when + translating the extras into those fields. + """ + raw_value = _last_flag_value(args, _GPU_LAYER_FLAGS) + if raw_value is None: + return None + try: + value = int(raw_value) + except ValueError as exc: + raise ValueError("llama-server GPU layers flag requires an integer value") from exc + if value < -1: + raise ValueError("llama-server GPU layers flag requires an integer value of at least -1") + return value + + def parse_cache_override_per_axis( args: Optional[Iterable[str]], ) -> tuple[Optional[str], Optional[str]]: diff --git a/studio/backend/core/inference/local_model_resolver.py b/studio/backend/core/inference/local_model_resolver.py index e6014f442d..5d2a9e9c87 100644 --- a/studio/backend/core/inference/local_model_resolver.py +++ b/studio/backend/core/inference/local_model_resolver.py @@ -34,6 +34,15 @@ class _LocalGgufEntry: _CACHE_TTL_S = 5.0 _lock = threading.Lock() _scan: tuple[float, dict[str, _LocalGgufEntry]] = (0.0, {}) +# Not _lock: that is held for the whole scan, so the request path would wait on it. +_warm_lock = threading.Lock() +# Repos that finished downloading but are not in the published index yet: nothing +# else covers them until the next scan, and the request path must not call them absent. +_just_downloaded: set[str] = set() +_warming = False +_last_scan_s = 0.0 +# Rescan at most a tenth of the time: on the TTL alone a slow scan would run continuously. +_WARM_DUTY = 10.0 def _is_abs_path_id(value: str) -> bool: @@ -103,17 +112,26 @@ def _local_gguf_entry(loader_id: str, info) -> Optional[_LocalGgufEntry]: load_dir = _resolve_load_dir(p) variants, _ = list_local_gguf_variants(str(load_dir)) quants = tuple(v.quant for v in variants if getattr(v, "quant", None)) - return _LocalGgufEntry(loader_id, str(load_dir), quants) if quants else None + if not quants: + return None + # That call orders by descending size, so the head is the biggest quant (often + # F16). Downstream reads [0], and a bare id must mean whichever quant a plain + # load would take: answering with the largest can evict a model and then OOM. + from core.inference.openai_auto_download import preferred_quant + + best = preferred_quant(quants) + if best and quants[0] != best: + quants = (best, *(q for q in quants if q != best)) + return _LocalGgufEntry(loader_id, str(load_dir), quants) except Exception: return None -def info_has_local_gguf(info) -> bool: - """True when *info* (a LocalModelInfo) points to on-disk GGUF weights the - auto-switch path can load. Read from the files, not ``info.model_format``: the - HF-cache scanner leaves model_format unset for GGUF snapshots, so a - model_format filter would drop every cached GGUF. Lets /v1/models advertise - exactly what /v1 can serve.""" +def local_gguf_quants(info) -> Optional[tuple[str, ...]]: + """On-disk quant labels for *info*, or None when it is not a servable local + GGUF. Read from the files, not ``info.model_format``: the HF-cache scanner + leaves that unset for GGUF snapshots, so filtering on it drops every cached + GGUF. One scan tells /v1/models what it can serve and which quant to name.""" from pathlib import Path path = getattr(info, "path", None) @@ -123,8 +141,14 @@ def info_has_local_gguf(info) -> bool: if isinstance(path, str) and any( seg in (".studio_links", "ollama_links") for seg in Path(path).parts ): - return False - return _local_gguf_entry(getattr(info, "id", "") or "", info) is not None + return None + entry = _local_gguf_entry(getattr(info, "id", "") or "", info) + return entry.variants if entry is not None else None + + +def info_has_local_gguf(info) -> bool: + """True when *info* points to on-disk GGUF weights the auto-switch path can load.""" + return local_gguf_quants(info) is not None def _build_index() -> dict[str, _LocalGgufEntry]: @@ -287,6 +311,36 @@ def _sibling_revision_entries(raw_id: str, loader_id: str): yield sibling.name, entry +def note_downloaded(repo_id: Optional[str]) -> None: + """Record a repo as present ahead of the scan that will index it.""" + if not repo_id: + return + with _lock: + _just_downloaded.add(repo_id.strip().lower()) + + +def recently_downloaded(repo_id: str) -> bool: + """Whether *repo_id* finished downloading since the last completed scan.""" + if not isinstance(repo_id, str) or not repo_id.strip(): + return False + return repo_id.strip().lower() in _just_downloaded + + +def invalidate_index() -> None: + """Mark the cached scan stale so the next resolve sees a just-finished download + instead of waiting out the TTL. + + Keeps the entries: the request path reads this cache without scanning, so + emptying it would leave it with no evidence about any local model until the + rebuild lands, and a bare request for one would be answered by whatever is + resident. Only a completed download invalidates, and that only adds, so the + retained entries stay true. + """ + global _scan + with _lock: + _scan = (0.0, _scan[1]) + + def _index() -> dict[str, _LocalGgufEntry]: global _scan # Build under the lock so concurrent callers with an expired cache don't all @@ -301,23 +355,74 @@ def _index() -> dict[str, _LocalGgufEntry]: # an install with many local models can itself exceed the TTL, which would # store the cache already expired and make every request rebuild the index. _scan = (time.monotonic(), fresh) + # The scan supersedes the notes: whatever landed is in the index now. + _just_downloaded.clear() return fresh -def resolve_local_gguf(requested: str) -> Optional[tuple[str, Optional[str], str]]: +def index_is_built() -> bool: + """Whether a scan has ever completed, freshness aside. + + Lock-free on purpose: ``_lock`` is held for the whole scan, so taking it would + park the request path on the scan it is trying to stay off. Safe because + ``_scan`` is only ever rebound, never mutated. + """ + return bool(_scan[0]) + + +def warm_index_soon() -> None: + """(Re)build the index off the request path when it is missing or past its TTL. + + The only refresh for callers using ``allow_scan=False``. Covers a stale index, + not just an absent one: a model downloaded through the Hub UI or dropped into a + scan folder has no invalidation hook and would otherwise stay invisible to them + for the life of the process. Never blocks, and never touches ``_lock``. + """ + global _warming + if time.monotonic() - _scan[0] < max(_CACHE_TTL_S, _last_scan_s * _WARM_DUTY): + return + with _warm_lock: + if _warming: + return + _warming = True + + def _run() -> None: + global _warming, _last_scan_s + started = time.monotonic() + try: + _index() + except Exception: + pass + finally: + _last_scan_s = time.monotonic() - started + with _warm_lock: + _warming = False + + threading.Thread(target = _run, name = "local-model-index-warm", daemon = True).start() + + +def resolve_local_gguf( + requested: str, *, allow_scan: bool = True +) -> Optional[tuple[str, Optional[str], str]]: """Return ``(load_path, gguf_variant, loader_id)`` for a local match, else None. ``load_path`` is the concrete on-disk path to hand /load (so it never fetches a remote), ``loader_id`` is the advertised id used as the launch-override key. ``requested`` is ``repo`` or ``repo:VARIANT``. An exact id match wins first (so ids containing a colon still resolve); else the last ``:VARIANT`` is split - off and resolves only when that quant is on disk. + off and resolves only when that quant is on disk, unless it names no quant at + all (an Ollama-style ":latest"), which means the repo. + + ``allow_scan=False`` answers from the last built index and never rebuilds, for + the request path: the scan walks several model dirs and HF caches, takes seconds + on a large install, and holds a lock everyone queues behind. Stale is fine there, + since disk barely moves and a finished download calls :func:`invalidate_index`. """ if not isinstance(requested, str) or not requested.strip(): return None requested = requested.strip() try: - index = _index() + index = _index() if allow_scan else _scan[1] entry = index.get(requested.lower()) if entry is not None: variant = entry.variants[0] if entry.variants else None @@ -333,8 +438,44 @@ def resolve_local_gguf(requested: str) -> Optional[tuple[str, Optional[str], str for v in entry.variants: if v.lower() == wanted: return entry.load_path, v, entry.loader_id - return None + from core.inference.openai_auto_download import looks_like_quant + + if looks_like_quant(variant): + return None + # ":latest" or ":8b" names no file, so it means the repo; a real quant that + # is not on disk still misses, or a swap would serve the wrong weights. + return entry.load_path, (entry.variants[0] if entry.variants else None), entry.loader_id except Exception: # Best-effort: any resolver failure falls through to the loaded model, # so a malformed name can never turn a servable request into a 500. return None + + +MISS_MODEL_NOT_FOUND = "model_not_found" +MISS_VARIANT_NOT_FOUND = "variant_not_found" + + +def describe_local_miss(requested: str) -> tuple[str, tuple[str, ...]]: + """Why :func:`resolve_local_gguf` missed, so an error can say "wrong quant" + instead of "no such model". + + ``(MISS_VARIANT_NOT_FOUND, )`` when the repo is downloaded but the + requested ``:VARIANT`` is not, else ``(MISS_MODEL_NOT_FOUND, ())``. Fail-safe: a + scan failure reports the generic miss rather than raising into the handler. + """ + if not isinstance(requested, str) or not requested.strip(): + return MISS_MODEL_NOT_FOUND, () + base, sep, variant = requested.strip().rpartition(":") + from core.inference.openai_auto_download import looks_like_quant + + # Split like the resolver or the two disagree: a tag naming no quant means the + # repo there, so reporting a missing quant for it would name one nobody asked for. + if not sep or not looks_like_quant(variant): + return MISS_MODEL_NOT_FOUND, () + try: + entry = _index().get(base.strip().lower()) + except Exception: + return MISS_MODEL_NOT_FOUND, () + if entry is None or not entry.variants: + return MISS_MODEL_NOT_FOUND, () + return MISS_VARIANT_NOT_FOUND, entry.variants diff --git a/studio/backend/core/inference/mcp_client.py b/studio/backend/core/inference/mcp_client.py index 0256df944e..98112c6d5b 100644 --- a/studio/backend/core/inference/mcp_client.py +++ b/studio/backend/core/inference/mcp_client.py @@ -971,7 +971,12 @@ def _call_stdio_tool( raise RuntimeError("MCP server connection is not available") else: rem = _remaining() - coro = _race_tool_call(session.client.call_tool(name, args), rem, cancel_event) + # raise_on_error=False for the same reason as the one-shot path. + coro = _race_tool_call( + session.client.call_tool(name, args, raise_on_error = False), + rem, + cancel_event, + ) return session.run(coro, rem) except (_MCPCancelled, asyncio.TimeoutError): # _race_tool_call cancels the pending call but cancellation is diff --git a/studio/backend/core/inference/mlx_inference.py b/studio/backend/core/inference/mlx_inference.py index d19c67a01a..2b300a32b1 100644 --- a/studio/backend/core/inference/mlx_inference.py +++ b/studio/backend/core/inference/mlx_inference.py @@ -1189,7 +1189,8 @@ class MLXInferenceBackend: **gen_kwargs, ) - def reset_generation_state(self): + def reset_generation_state(self, caller_cancel_event = None): + # caller_cancel_event: signature parity with the orchestrator; unused here. import mlx.core as mx import gc diff --git a/studio/backend/core/inference/model_ids.py b/studio/backend/core/inference/model_ids.py index 548cc60f94..3886307ae2 100644 --- a/studio/backend/core/inference/model_ids.py +++ b/studio/backend/core/inference/model_ids.py @@ -39,10 +39,29 @@ def _looks_like_path(identifier: str) -> bool: return False +def hf_cache_repo_id(path: Optional[str]) -> Optional[str]: + """``.../models--org--name/snapshots/`` -> ``org/name``, else None. + + A model loaded from the HF cache is identified by its snapshot dir, whose + basename is a commit hash; recover the repo id so callers don't show that. + """ + if not path: + return None + parts = str(path).replace("\\", "/").split("/") + for index, part in enumerate(parts): + # Only inside the real cache layout: a "models--" name alone is not a repo id. + if part.startswith("models--") and parts[index + 1 : index + 2] == ["snapshots"]: + return part[len("models--") :].replace("--", "/") + return None + + def public_model_id(identifier: Optional[str]) -> Optional[str]: """Return a clean, path-free public id for *identifier*. - - Local GGUF path -> the file stem with ``.gguf`` stripped, e.g. + - HF cache path -> the repo id it came from, e.g. + ``~/.cache/huggingface/hub/models--unsloth--X-GGUF/snapshots/`` -> + ``unsloth/X-GGUF``. + - Other local GGUF path -> the file stem with ``.gguf`` stripped, e.g. ``/srv/models/Qwen3-30B-A3B-Q4_K_M.gguf`` -> ``Qwen3-30B-A3B-Q4_K_M``. - HF repo id (``org/model``) and already-clean names -> returned unchanged. - ``None`` / empty -> returned unchanged. @@ -51,6 +70,9 @@ def public_model_id(identifier: Optional[str]) -> Optional[str]: return identifier if not _looks_like_path(identifier): return identifier + repo_id = hf_cache_repo_id(identifier) + if repo_id: + return repo_id name = os.path.basename(identifier.replace("\\", "/").rstrip("/")) if name.lower().endswith(_GGUF_SUFFIX): name = name[: -len(_GGUF_SUFFIX)] diff --git a/studio/backend/core/inference/openai_auto_download.py b/studio/backend/core/inference/openai_auto_download.py new file mode 100644 index 0000000000..cad5e40d14 --- /dev/null +++ b/studio/backend/core/inference/openai_auto_download.py @@ -0,0 +1,812 @@ +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 + +"""Opt-in: fetch a GGUF a /v1 request names but this server doesn't have. + +Auto-switch only loads models already on disk. With +``openai_api_auto_download_model`` on, a miss that looks like a real Hub repo is +fetched in the background and the request is told to retry rather than held +open: a quant is routinely tens of GB, far longer than any client (or the +Cloudflare edge on ``--secure``) will wait, and the inference lifecycle gate must +not be held meanwhile. The resident model keeps serving, and the retry that lands +after the download goes through the ordinary auto-switch path. + +Admission is deliberately narrow, since a request only needs an API key: +- ``namespace/name`` only, and only when the Hub confirms GGUF weights. A + namespace is not evidence of intent (LiteLLM and OpenRouter address every + provider that way), so ``gpt-4`` and ``anthropic/claude-3.5-sonnet`` alike + fall through to the resident model as before. +- GGUF only, decided from the remote file list, not the repo name: GGUF runs + under llama.cpp, which never imports repo Python. +- ``auto_map`` is refused, so ``trust_remote_code`` is only ever granted + deliberately in the UI, never by an API call. +- One download at a time, so a key holder cannot fan out fetches. +""" + +from __future__ import annotations + +import asyncio +import shutil +import threading +import time +from dataclasses import dataclass +from typing import Optional + +from loggers import get_logger + +logger = get_logger(__name__) + +# Keep the Hub probe short so a slow Hub can't stall the request path. +_MODEL_INFO_TIMEOUT_S = 8.0 +# auth_check and hf_hub_download take no timeout of their own and run while the +# provisional slot is held, so an unresponsive Hub would pin the single flight. The +# code probe fetches up to three configs, so it gets more room than the auth call. +_CODE_PROBE_TIMEOUT_S = 20.0 +# Headroom left free after the download, so filling the disk can't wedge the box. +_DISK_RESERVE_BYTES = 5 * 1024**3 +_WATCH_POLL_S = 2.0 +# A stalled watcher must not pin the single-flight slot forever. +_MAX_WATCH_S = 24 * 60 * 60 +# Past the watch window the row is resolved, so poll only to see whether the +# worker is still alive and still owns the slot. +_TIMED_OUT_POLL_S = 60.0 +_RETRY_AFTER_S = 30 +# Long enough for a client honouring Retry-After to come back and be told, short +# enough that one that never returns cannot hold the slot. +_FAILED_HOLD_S = 3 * _RETRY_AFTER_S +_MAX_LISTED_VARIANTS = 8 + + +@dataclass(frozen = True) +class AutoDownloadRefusal: + """Why this request cannot be served yet; the route raises it in the + surface's own error envelope.""" + + status: int + code: str + message: str + retry_after: Optional[int] = None + + +@dataclass +class _Active: + repo_id: str + # None while the Hub probe is still deciding which quant to fetch. + variant: Optional[str] = None + expected_bytes: int = 0 + monitor_id: Optional[str] = None + started_at: float = 0.0 + # Set when the worker failed. Held until a retry surfaces it: Retry-After is far + # longer than the watcher poll, so the client would restart the same failing download. + error: Optional[str] = None + failed_at: float = 0.0 + + +_lock = threading.Lock() +_active: Optional[_Active] = None + +# Repos the Hub says are not servable, so a "vendor/model" miss doesn't re-probe every request. +_NOT_SERVABLE_TTL_S = 10 * 60 +_NOT_SERVABLE_MAX = 256 +_cache_lock = threading.Lock() +_not_servable: dict[str, float] = {} + + +def _public_label(repo_id: str, variant: Optional[str]) -> str: + return f"{repo_id}:{variant}" if variant else repo_id + + +def split_model_ref(requested: str) -> tuple[str, Optional[str]]: + """``org/repo:QUANT`` -> ``("org/repo", "QUANT")``; no suffix -> variant None. + + Splits on the last colon. A slash-bearing suffix is only a variant when a real + Hub repo precedes it: "build/llama-13b" is a subdirectory GGUF key the catalog + advertises, while "C:/models/x.gguf" leaves a drive letter that is no repo id. + """ + text = (requested or "").strip() + base, sep, suffix = text.rpartition(":") + if not sep or not base or not suffix: + return text, None + stripped = base.strip() + if "/" in suffix: + from hub.utils.paths import is_valid_repo_id + if "/" not in stripped or not is_valid_repo_id(stripped): + return text, None + return stripped, suffix.strip() + + +def is_downloadable_ref(requested: str) -> bool: + """Whether *requested* is shaped like a Hub repo we may fetch. + + Requires an explicit namespace: keeps ``gpt-4`` and other foreign ids falling + through, and stops ModelConfig.from_identifier's bare-name ``unsloth/`` + prefixing from turning an unrelated label into a real repo. + """ + from hub.utils.paths import is_valid_repo_id + + repo_id, variant = split_model_ref(requested) + if "/" not in repo_id or not is_valid_repo_id(repo_id): + return False + if variant is not None: + from hub.utils.paths import is_valid_gguf_variant + return is_valid_gguf_variant(variant) + return True + + +def looks_like_quant(variant: Optional[str]) -> bool: + """Whether a ``:suffix`` names a GGUF quant rather than a foreign tag. + + Neither a namespace nor a colon proves a request was meant for this server + (``vendor/model`` is LiteLLM/OpenRouter, ``name:latest`` is Ollama). A real + quant label does. + """ + import re + + from utils.models.model_config import _GGUF_KNOWN_QUANT_RE + + if not variant: + return False + # _extract_quant_label can append a bpw modifier (IQ4_XS-3.53bpw); still a quant. + label = re.sub(r"-[0-9]+(?:\.[0-9]+)?bpw$", "", variant.strip(), flags = re.IGNORECASE) + return _GGUF_KNOWN_QUANT_RE.fullmatch(label) is not None + + +def _hub_token(hf_token: Optional[str]): + """The caller's token, or an explicit False. None makes huggingface_hub fall + back to a cached login (here the server owner's); only False is anonymous.""" + return hf_token or False + + +def _servable_key(repo_id: str, hf_token: Optional[str]) -> str: + """Cache key, per credential. + + The Hub 404s a private repo the caller cannot see, so a tokenless verdict says + nothing about a caller who has one. Digested, so no token is held here. + """ + import hashlib + + seen_as = hashlib.sha256(hf_token.encode()).hexdigest()[:16] if hf_token else "anon" + return f"{repo_id.lower()}\n{seen_as}" + + +def _mark_not_servable(repo_id: str, hf_token: Optional[str]) -> None: + with _cache_lock: + if len(_not_servable) >= _NOT_SERVABLE_MAX: + _not_servable.clear() + _not_servable[_servable_key(repo_id, hf_token)] = time.monotonic() + _NOT_SERVABLE_TTL_S + + +def _is_not_servable(repo_id: str, hf_token: Optional[str]) -> bool: + key = _servable_key(repo_id, hf_token) + with _cache_lock: + expires = _not_servable.get(key) + if expires is None: + return False + if expires <= time.monotonic(): + del _not_servable[key] + return False + return True + + +def _gated_refusal(repo_id: str) -> AutoDownloadRefusal: + return AutoDownloadRefusal( + status = 403, + code = "model_access_denied", + message = ( + f"'{repo_id}' is gated on Hugging Face. Accept its licence, then retry with " + "your own token in the X-Unsloth-HF-Token header: automatic download never " + "uses this server's Hugging Face identity." + ), + ) + + +async def _bounded_probe(fn, *args, timeout: float, default): + """Run a blocking Hub probe off the loop, bounding only the wait. + + The thread is left to finish (a blocking socket read cannot be cancelled); the + caller takes *default*, chosen per call site so a timeout errs the safe way. + """ + try: + return await asyncio.wait_for(asyncio.to_thread(fn, *args), timeout) + except (TimeoutError, asyncio.TimeoutError): + logger.debug("hub probe %s timed out after %ss", getattr(fn, "__name__", fn), timeout) + return default + + +def _auth_denied(repo_id: str, hf_token: Optional[str]) -> bool: + """Whether this token lacks file access to a gated repo. False when the + check is inconclusive: the download's own auth is the real gate.""" + from hub.utils.hf_errors import hf_error_status + + try: + from huggingface_hub import auth_check + auth_check(repo_id, token = _hub_token(hf_token)) + except Exception as exc: + return hf_error_status(exc) in (401, 403) + return False + + +def _gguf_variants(siblings) -> dict[str, int]: + """Quant label -> bytes the download will actually fetch. + + Mirrors list_gguf_variants for the selectable labels: companions (mmproj/MTP) + and big-endian builds are not quants, and sharded quants sum across shards. + Bytes come from the download plan, which folds companions back into every + quant, so the disk reserve is measured against what the worker fetches. + """ + from hub.utils.gguf import extract_quant_label as canonical_quant_label + from hub.utils.gguf_plan import build_gguf_variant_plans + from utils.models.model_config import ( + _extract_quant_label, + _is_big_endian_gguf_path, + _is_mmproj, + _is_mtp_drafter, + ) + + siblings = list(siblings or []) + plans = build_gguf_variant_plans(siblings) + sizes: dict[str, int] = {} + for sibling in siblings: + name = getattr(sibling, "rfilename", "") or "" + if not name.lower().endswith(".gguf"): + continue + quant = _extract_quant_label(name) + if not looks_like_quant(quant): + # With no recognized quant token the extractors part ways: this one takes + # the last hyphenated segment ("7b" of llama-7b) while the plan and worker + # key the whole stem, so advertising ours dispatches an unresolvable variant. + quant = canonical_quant_label(name) or quant + if _is_mmproj(name) or _is_mtp_drafter(name) or _is_big_endian_gguf_path(name, quant): + continue + plan = plans.get(quant.lower()) + if plan is not None: + sizes[quant] = plan.download_size_bytes + else: + sizes[quant] = sizes.get(quant, 0) + int(getattr(sibling, "size", 0) or 0) + return sizes + + +def _remaining_bytes(repo_id: str, plan, expected_bytes: int) -> int: + """Bytes still to fetch: a resumed quant or a companion shared with another + quant is already on disk, and charging for it can 507 a download that fits.""" + try: + from hub.utils.download_registry import existing_blob_bytes + + hashes = frozenset( + file.sha256 for file in getattr(plan, "expected_files", ()) or () if file.sha256 + ) + if not hashes: + return expected_bytes + return max(0, expected_bytes - existing_blob_bytes("model", repo_id, hashes)) + except Exception: + return expected_bytes + + +def _enough_disk(need_bytes: int) -> tuple[bool, int]: + """(fits, free_bytes). Fail-open on an unreadable cache root: the download + worker runs its own preflight, this only adds the reserve margin.""" + try: + from hub.utils.hf_cache_state import hf_cache_root + + root = hf_cache_root(create = True) + if root is None: + return True, 0 + free = shutil.disk_usage(root).free + except Exception: + return True, 0 + return free >= need_bytes + _DISK_RESERVE_BYTES, free + + +def _gb(num_bytes: int) -> str: + return f"{num_bytes / 1024**3:.1f} GB" + + +async def _job_state(repo_id: str, variant: Optional[str]) -> tuple[str, Optional[str]]: + from hub.services.models import downloads + try: + status = await downloads.get_download_status_response(repo_id, variant or "") + return status.state, status.error + except Exception as exc: + # "unknown", not "idle": idle ends the watch, and a failed probe proves nothing. + logger.debug("auto-download: status probe failed for %r: %s", repo_id, exc) + return "unknown", None + + +async def _progress_percent( + repo_id: str, variant: Optional[str], expected_bytes: int, hf_token: Optional[str] +) -> Optional[float]: + """0-100, or None. The hub service reports a 0-1 fraction, so scale it.""" + from hub.services.models import downloads + try: + payload = await downloads.get_gguf_download_progress_response( + repo_id, variant or "", expected_bytes, hf_token + ) + fraction = payload.get("progress") + if not isinstance(fraction, (int, float)): + return None + return min(100.0, max(0.0, float(fraction) * 100.0)) + except Exception: + return None + + +def _release(active: Optional[_Active]) -> None: + """Free the single-flight slot, but only while *active* still owns it. + + Keying on ``repo_id`` alone let a stale operation clear a newer one: variant A + errors, an adopting request frees the slot, a retry starts B, then A's watcher + matches the repo and clears B, admitting a second download alongside it. + """ + global _active + if active is None: + return + with _lock: + if _active is active: + _active = None + + +async def _watch(active: _Active, hf_token: Optional[str]) -> None: + """Poll a dispatched job so the monitor row resolves and the resolver cache + is dropped the moment the weights land.""" + from core.inference import api_monitor as monitor_module + + api_monitor = monitor_module.api_monitor + deadline = time.monotonic() + _MAX_WATCH_S + timed_out = False + try: + while True: + await asyncio.sleep(_TIMED_OUT_POLL_S if timed_out else _WATCH_POLL_S) + state, error = await _job_state(active.repo_id, active.variant) + if state in ("running", "cancelling", "unknown"): + if timed_out: + # A running worker still owns the slot: releasing on the clock alone + # would admit a second multi-GB download beside it. "unknown" cannot + # confirm it is alive, so release then, or a broken probe wedges us. + if state == "unknown": + return + continue + if time.monotonic() >= deadline: + api_monitor.fail_open(active.monitor_id, "Download timed out") + timed_out = True + continue + # Only "running" has progress; the others are still in flight, so keep the slot. + if state == "running": + api_monitor.set_progress( + active.monitor_id, + await _progress_percent( + active.repo_id, active.variant, active.expected_bytes, hf_token + ), + ) + continue + if state == "cancelled": + api_monitor.finish(active.monitor_id, status = "cancelled") + return + if state == "complete": + # No invalidate here: finalize_worker_exit already dropped the cache and + # warmed it; a second would mark that fresh scan stale and push a + # synchronous rescan onto the client's retry. + api_monitor.finish(active.monitor_id, status = "completed") + elif state == "idle": + # The job vanished without a terminal state (worker killed). + api_monitor.fail_open(active.monitor_id, "Download did not complete") + else: + api_monitor.fail_open(active.monitor_id, error or f"Download {state}") + # Keep the slot so the next retry is told it failed instead of + # silently restarting the same download. + active.error = error or f"Download {state}" + active.failed_at = time.monotonic() + return + return + except asyncio.CancelledError: + raise + except Exception as exc: + logger.warning("auto-download: watcher failed for %r: %s", active.repo_id, exc) + api_monitor.fail_open(active.monitor_id, "Download tracking failed") + finally: + if not active.failed_at: + _release(active) + + +def _downloading_refusal(label: str, percent: Optional[float]) -> AutoDownloadRefusal: + progress = f" ({percent:.0f}% done)" if percent is not None else "" + return AutoDownloadRefusal( + status = 503, + code = "model_downloading", + message = (f"Downloading '{label}'{progress}. Retry shortly. Track it in Unsloth Studio."), + retry_after = _RETRY_AFTER_S, + ) + + +async def _is_downloadable_model(repo_id: str, hf_token: Optional[str]) -> bool: + """Whether the Hub has this repo with GGUF weights we could fetch. + + Only asked while another download holds the slot, to tell a second download + apart from an ordinary foreign label. Any failure answers False: refusing + would strand normal traffic for the length of the download. + """ + if _is_not_servable(repo_id, hf_token): + return False + + def _probe(): + from huggingface_hub import HfApi + return HfApi(token = _hub_token(hf_token)).model_info(repo_id, timeout = _MODEL_INFO_TIMEOUT_S) + + try: + info = await asyncio.to_thread(_probe) + except Exception: + return False + # The same filter admission uses, not a bare .gguf test: mmproj, MTP drafters and + # big-endian builds are companions, not quants. Answering otherwise would hold an + # ordinary foreign label at model_download_busy for an unrelated download. + servable = bool(_gguf_variants(getattr(info, "siblings", None))) + if not servable: + _mark_not_servable(repo_id, hf_token) + return servable + + +async def maybe_auto_download( + requested_model: str, + *, + hf_token: Optional[str] = None, + require_vision: bool = False, +) -> Optional[AutoDownloadRefusal]: + """Start (or report on) a background fetch of *requested_model*. + + Returns None when the request should carry on unchanged, or a refusal the + caller must raise. Only called after the local resolver has already missed. + + ``require_vision`` refuses a target with no mmproj companion rather than spend + gigabytes on weights that cannot answer the request; the local capability guard + only ever sees an already-downloaded model. + """ + global _active + + repo_id, wanted_variant = split_model_ref(requested_model) + if not is_downloadable_ref(requested_model): + return None + if _is_not_servable(repo_id, hf_token) and not looks_like_quant(wanted_variant): + return None + + # Settle the single-flight slot before the network, so retries during a download stay cheap. + busy: Optional[_Active] = None + with _lock: + current = _active + if current is not None and current.failed_at: + # A held failure only owns the slot until someone is told about it. + if current.repo_id != repo_id and time.monotonic() - current.failed_at > _FAILED_HOLD_S: + _active = current = None + if current is not None and current.repo_id == repo_id: + adopted = current + elif current is not None: + adopted = None + busy = current + else: + adopted = None + provisional = _Active(repo_id = repo_id, started_at = time.time()) + _active = provisional + + if busy is not None: + # Refusing before the probe blocks ordinary drop-in traffic: a namespaced label + # that is no downloadable GGUF repo (LiteLLM/OpenRouter style) would be told to + # wait out a multi-hour download. Only a downloadable label is a 2nd download. + if not await _is_downloadable_model(repo_id, hf_token): + return None + return AutoDownloadRefusal( + status = 503, + code = "model_download_busy", + message = ( + f"Already downloading '{_public_label(busy.repo_id, busy.variant)}'. " + f"Retry '{requested_model}' once it finishes." + ), + retry_after = _RETRY_AFTER_S, + ) + + if adopted is not None: + if adopted.variant is None: + # Still probing: no job yet, and a stale whole-repo error would free the probe's slot. + return _downloading_refusal(adopted.repo_id, None) + state, error = await _job_state(adopted.repo_id, adopted.variant) + if state in ("running", "cancelling", "unknown"): + return _downloading_refusal( + _public_label(adopted.repo_id, adopted.variant), + await _progress_percent( + adopted.repo_id, adopted.variant, adopted.expected_bytes, hf_token + ), + ) + if state == "error" or adopted.error: + error = error or adopted.error + # Surface once, then free the slot so a retry can start over. + _release(adopted) + return AutoDownloadRefusal( + status = 502, + code = "model_download_failed", + message = f"Downloading '{requested_model}' failed: {error or 'unknown error'}", + ) + # complete/idle/cancelled: the watcher is about to free the slot, so retry once more. + return _downloading_refusal( + _public_label(adopted.repo_id, adopted.variant), + 100.0 if state == "complete" else None, + ) + + try: + return await _admit_and_start( + repo_id, wanted_variant, requested_model, hf_token, provisional, require_vision + ) + except BaseException: + # Not `except Exception`: a cancel mid-probe would otherwise wedge the provisional slot. + _release(provisional) + raise + + +async def _admit_and_start( + repo_id: str, + wanted_variant: Optional[str], + requested_model: str, + hf_token: Optional[str], + active: _Active, + require_vision: bool = False, +) -> Optional[AutoDownloadRefusal]: + from hub.utils.hf_errors import hf_error_status + + def _probe(): + from huggingface_hub import HfApi + return HfApi(token = _hub_token(hf_token)).model_info( + repo_id, files_metadata = True, timeout = _MODEL_INFO_TIMEOUT_S + ) + + try: + info = await asyncio.to_thread(_probe) + except Exception as exc: + _release(active) + status = hf_error_status(exc) + if status == 401: + return AutoDownloadRefusal( + status = 401, + code = "model_access_denied", + message = ( + f"Hugging Face rejected the token sent for '{repo_id}'. Replace the " + "X-Unsloth-HF-Token header with a valid token; retrying will not help." + ), + ) + if status == 403: + return _gated_refusal(repo_id) + if status == 404: + _mark_not_servable(repo_id, hf_token) + # Unknown to the Hub reads as a foreign label; only an explicit quant makes it ours. + if not looks_like_quant(wanted_variant): + return None + # A private repo reads as absent without a token; don't confirm either way. + return AutoDownloadRefusal( + status = 404, + code = "model_not_found", + message = ( + f"'{repo_id}' was not found on Hugging Face, or is not accessible. " + "If it is private, send a token in the X-Unsloth-HF-Token header." + ), + ) + logger.warning("auto-download: Hub lookup failed for %r: %s", repo_id, exc) + return AutoDownloadRefusal( + status = 503, + code = "model_lookup_failed", + message = f"Could not reach Hugging Face to look up '{repo_id}'. Retry shortly.", + retry_after = _RETRY_AFTER_S, + ) + + # Inconclusive on timeout: the download's own auth is the real gate. + if getattr(info, "gated", False) and await _bounded_probe( + _auth_denied, repo_id, hf_token, timeout = _MODEL_INFO_TIMEOUT_S, default = False + ): + # Metadata for a gated repo is not file access; unchecked, the config read below lies. + _release(active) + return _gated_refusal(repo_id) + + variants = _gguf_variants(getattr(info, "siblings", None)) + if not variants: + _release(active) + _mark_not_servable(repo_id, hf_token) + if not looks_like_quant(wanted_variant): + return None + return AutoDownloadRefusal( + status = 400, + code = "model_not_supported", + message = ( + f"'{repo_id}' has no GGUF weights. Automatic download serves GGUF only; " + "load other formats from Unsloth Studio." + ), + ) + + # trust_remote_code gate: _config_has_auto_map is tri-state, so refuse on True and on None. + from utils.security.consent import _config_has_auto_map + + # _hub_token, not the raw token: None lets huggingface_hub fall back to a cached + # server login, so a caller-named repo would be probed with this server's identity. + # Defaults to None on timeout, which refuses: unchecked is not cleared. + has_auto_map = await _bounded_probe( + _config_has_auto_map, + repo_id, + _hub_token(hf_token), + timeout = _CODE_PROBE_TIMEOUT_S, + default = None, + ) + if has_auto_map is not False: + _release(active) + unknown = has_auto_map is None + return AutoDownloadRefusal( + status = 403, + code = "remote_code_consent_required", + message = ( + f"'{repo_id}' " + + ( + "could not be checked for custom code" + if unknown + else "ships custom code that runs on load" + ) + + ". Load it once in Unsloth Studio to review and approve it, then retry." + ), + ) + + variant = _match_variant(wanted_variant, variants) + if variant is None: + _release(active) + listed = sorted(variants) + shown = ", ".join(listed[:_MAX_LISTED_VARIANTS]) + extra = len(listed) - _MAX_LISTED_VARIANTS + return AutoDownloadRefusal( + status = 404, + code = "model_not_found", + message = ( + f"'{repo_id}' has no quant '{wanted_variant}'. Available quants: " + f"{shown}{f' and {extra} more' if extra > 0 else ''}." + ), + ) + + expected_bytes = variants[variant] + from hub.utils.gguf_plan import build_gguf_variant_plans + + plan = build_gguf_variant_plans(list(getattr(info, "siblings", None) or [])).get( + variant.lower() + ) + if require_vision and not (plan and plan.mmproj_filenames): + _release(active) + return AutoDownloadRefusal( + status = 400, + code = "invalid_value", + message = ( + f"'{_public_label(repo_id, variant)}' ships no mmproj companion, so it " + "cannot answer the image or audio input in this request. It was not " + "downloaded." + ), + ) + + need_bytes = _remaining_bytes(repo_id, plan, expected_bytes) + fits, free = _enough_disk(need_bytes) + if not fits: + _release(active) + return AutoDownloadRefusal( + status = 507, + code = "insufficient_disk_space", + message = ( + f"'{_public_label(repo_id, variant)}' needs {_gb(need_bytes)} plus " + f"{_gb(_DISK_RESERVE_BYTES)} headroom, but only {_gb(free)} is free." + ), + ) + + return await _dispatch(repo_id, variant, expected_bytes, requested_model, hf_token, active) + + +def preferred_quant(labels) -> Optional[str]: + """The quant a plain load would pick from *labels*, or None. + + The one ranking for "which quant did they mean": local resolution, remote + admission and /v1/models must agree, or a bare id means a different quant + depending on which of them answered it. + """ + from utils.models.model_config import _pick_best_gguf + + # _pick_best_gguf ranks filenames and matches upper-case tokens, so feed "