diff --git a/.github/scripts/run-studio-permission-browser.sh b/.github/scripts/run-studio-permission-browser.sh index e5a9a4c135..2007789035 100755 --- a/.github/scripts/run-studio-permission-browser.sh +++ b/.github/scripts/run-studio-permission-browser.sh @@ -17,8 +17,7 @@ if [ -n "${STUDIO_PERMISSION_FRONTEND:-}" ]; then fi mkdir -p "$artifact_dir" -# Wipe (not reset-password): the boot below must re-seed a fresh .bootstrap_password. -rm -rf "$studio_home/auth" +unsloth studio reset-password UNSLOTH_API_ONLY=1 unsloth studio -H 127.0.0.1 -p "$port" "$@" \ >"$server_log" 2>&1 & studio_pid=$! diff --git a/.github/workflows/consolidated-tests-ci.yml b/.github/workflows/consolidated-tests-ci.yml index afad1b6c46..d1bea819eb 100644 --- a/.github/workflows/consolidated-tests-ci.yml +++ b/.github/workflows/consolidated-tests-ci.yml @@ -7,7 +7,7 @@ # # Why a separate workflow: # - studio-backend-ci.yml's "Repo tests (CPU)" job already auto-discovers -# tests/ minus tests/qlora, tests/saving, tests/utils, tests/sh. The 17 +# tests/ minus tests/qlora, tests/saving, tests/utils, tests/sh. The 16 # Bucket-A tests below live inside those --ignore dirs (CPU-runnable but # historically excluded with their GPU siblings); pulling them out into # a sibling job keeps the existing 760-passed baseline stable while we @@ -274,7 +274,6 @@ jobs: tests/saving/test_export_dispatch.py \ tests/saving/test_imatrix_export.py \ tests/saving/test_gguf_single_pass_export.py \ - tests/saving/test_offline_gguf_vlm_tokenizer_7481.py \ tests/utils/test_attention_masks.py \ tests/utils/test_trunc_normal_patch.py \ tests/python/test_fast_language_model_text_only.py @@ -366,17 +365,17 @@ jobs: tests/saving/test_export_dispatch.py \ tests/saving/test_imatrix_export.py \ tests/saving/test_gguf_single_pass_export.py \ - tests/saving/test_offline_gguf_vlm_tokenizer_7481.py \ tests/utils/test_attention_masks.py \ tests/utils/test_trunc_normal_patch.py \ tests/python/test_fast_language_model_text_only.py \ tests/test_bad_mappings_redirect.py \ tests/test_prefetch_snapshot_scope.py \ tests/test_gemma_2b_mapper_key.py \ - tests/test_raw_text_json_loading.py - # test_run_attention_flash_varlen_receives_window_and_softcap was deselected - # until attention_dispatch.py predefined flash_attn_varlen_func as None; it - # monkeypatches that name, so it no longer needs flash_attn on this runner. + --deselect 'tests/utils/test_attention_masks.py::test_run_attention_flash_varlen_receives_window_and_softcap' + # The deselected test monkeypatches flash_attn_varlen_func, which is + # only bound on the module when `flash_attn` is importable. flash_attn + # requires CUDA + dev toolchain, which the CPU-only ubuntu-latest + # runner does not have. The other Bucket-A tests pass cleanly. - name: unsloth_zoo @ ${{ env.UNSLOTH_ZOO_REF }} — full pytest (CPU) # 106 of 111 test_* in unsloth_zoo are CPU-only. The two CUDA-skip @@ -2130,7 +2129,7 @@ jobs: pip show unsloth_zoo echo "::endgroup::" echo "Consolidated job done. Coverage:" - echo " - 17 unsloth Bucket-A tests under tests/saving/ + tests/utils/" + echo " - 16 unsloth Bucket-A tests under tests/saving/ + tests/utils/" echo " - unsloth_zoo @ ${UNSLOTH_ZOO_REF} pytest tests/ (5 GPU cases deselected)" echo " - unsloth_zoo.compiler.test_apply_fused_lm_head" diff --git a/.github/workflows/local-agent-guides-ci.yml b/.github/workflows/local-agent-guides-ci.yml index 0dc0cc66d7..c48328e90f 100644 --- a/.github/workflows/local-agent-guides-ci.yml +++ b/.github/workflows/local-agent-guides-ci.yml @@ -167,9 +167,7 @@ jobs: # ── boot the server under test (factored helper) ────────────────── - name: Serve unsloth run --disable-tools (gemma-4-E4B) run: | - # Wipe, not reset-password: since #7573 the reset rotates in place and - # prints the new passphrase, which would land unmasked in the job log. - rm -rf ~/.unsloth/studio/auth + unsloth studio reset-password bash .github/scripts/serve-unsloth-run.sh \ --gguf-file "$GITHUB_WORKSPACE/gguf-cache/${GGUF_FILE}" \ --port "$STUDIO_PORT" --log-dir logs \ @@ -373,7 +371,7 @@ jobs: - name: Serve unsloth run --disable-tools (gemma-4-E4B) run: | - rm -rf ~/.unsloth/studio/auth + unsloth studio reset-password bash .github/scripts/serve-unsloth-run.sh \ --gguf-file "$GITHUB_WORKSPACE/gguf-cache/${GGUF_FILE}" \ --port "$STUDIO_PORT" --log-dir logs \ @@ -556,7 +554,7 @@ jobs: - name: Serve unsloth run --disable-tools (gemma-4-E4B) run: | - rm -rf ~/.unsloth/studio/auth + unsloth studio reset-password bash .github/scripts/serve-unsloth-run.sh \ --gguf-file "$GITHUB_WORKSPACE/gguf-cache/${GGUF_FILE}" \ --port "$STUDIO_PORT" --log-dir logs \ @@ -720,7 +718,7 @@ jobs: - name: Serve unsloth run --disable-tools (gemma-3-270m) run: | - rm -rf ~/.unsloth/studio/auth + unsloth studio reset-password bash .github/scripts/serve-unsloth-run.sh \ --model "$GGUF_REPO" --gguf-variant "$GGUF_VARIANT" \ --port "$STUDIO_PORT" --log-dir logs \ diff --git a/.github/workflows/release-desktop.yml b/.github/workflows/release-desktop.yml index 0a8d71610d..081eda4e32 100644 --- a/.github/workflows/release-desktop.yml +++ b/.github/workflows/release-desktop.yml @@ -766,7 +766,6 @@ jobs: env: GH_REPO: ${{ github.repository }} APP_VERSION: ${{ needs.prepare-version.outputs.app_version }} - PYPI_VERSION: ${{ needs.prepare-version.outputs.pypi_version }} STUDIO_VERSION: ${{ needs.prepare-version.outputs.studio_version }} DESKTOP_RELEASE_TAG: ${{ needs.prepare-version.outputs.desktop_release_tag }} DESKTOP_PRERELEASE: ${{ needs.prepare-version.outputs.prerelease }} @@ -912,8 +911,6 @@ jobs: notes = pathlib.Path(os.environ['RUNNER_TEMP'], 'desktop-release-notes.md').read_text() metadata = { 'version': os.environ['APP_VERSION'], - # App version is SemVer; CHANGELOG.md is keyed by the backend release. - 'pypi_version': os.environ['PYPI_VERSION'], 'notes': notes, 'pub_date': datetime.datetime.now(datetime.timezone.utc).isoformat(timespec='milliseconds').replace('+00:00', 'Z'), 'platforms': { diff --git a/.github/workflows/startup-profile-ci.yml b/.github/workflows/startup-profile-ci.yml deleted file mode 100644 index fbde99836d..0000000000 --- a/.github/workflows/startup-profile-ci.yml +++ /dev/null @@ -1,156 +0,0 @@ -# SPDX-License-Identifier: AGPL-3.0-only -# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. - -# Measures where Studio's startup time goes, on each platform. -# -# Nothing recorded a number before: main.py logs "lifespan startup completed in X ms" -# and studio_test_kit polls /healthz, but both throw the elapsed time away. A first -# local run (Linux, warm cache, 18-core server) put `import main` at 5.7-6.6s BEFORE -# the server can bind, dominated by eager module-level imports pulled in by routes: -# torch ~1.9s self, unsloth_zoo ~0.8s, routes ~0.6s, transformers ~0.5s. -# -# Not a gate yet: --max-healthz-seconds exists, but a budget should come from -# observed numbers rather than a guess. - -name: Startup profile - -on: - pull_request: - paths: - # The measured import graph is the whole backend tree: main.py imports auth, - # core, hub, loggers, models, picker, routes and utils at module scope. - - 'studio/backend/**' - - '!studio/backend/tests/**' - # The launch phase spawns `unsloth studio --api-only`, so the CLI counts too. - - 'unsloth_cli/**' - - 'studio/src-tauri/src/preflight**' - # The profiler hardcodes the desktop argv that process.rs::backend_args builds, - # so a change there must schedule a run or the two silently diverge. - - 'studio/src-tauri/src/process.rs' - - 'scripts/profile_startup.py' - - '.github/workflows/startup-profile-ci.yml' - # The job profiles whatever `install.sh --local` built: the installers pick the - # venv's Python and the dependency specs, and pyproject's include list is what - # makes --local overlay studio.backend*. - - 'install.sh' - - 'install.ps1' - - 'pyproject.toml' - # --local also runs the checkout's setup scripts (install.sh picks - # $_REPO_ROOT/studio/setup.sh, the editable install resolves setup.ps1 to the - # repo), and both call install_python_stack.py, which picks the dependencies. - - 'studio/setup.sh' - - 'studio/setup.ps1' - - 'studio/install_python_stack.py' - workflow_dispatch: - inputs: - repeats: - description: 'launch repeats per OS (median reported)' - type: string - default: '3' - -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - -permissions: - contents: read - -jobs: - profile: - name: startup ${{ matrix.os }} - runs-on: ${{ matrix.os }} - timeout-minutes: 60 - continue-on-error: true - strategy: - fail-fast: false - matrix: - os: [ubuntu-latest, macos-14, windows-latest] - - env: - UNSLOTH_STUDIO_HOME: ${{ github.workspace }}/.studio-home - # A wildcard bind calls ifconfig.me on the startup path; loopback times our code. - UNSLOTH_STUDIO_DISABLE_PUBLIC_CHECK: '1' - - steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - with: - persist-credentials: false - - - name: Install Studio - shell: bash - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - set -o pipefail - mkdir -p logs - # --local is load-bearing: it overlays the checkout, so the profiled server - # is this diff. Without it install.sh resolves unsloth from PyPI. - if [ "${{ runner.os }}" = "Windows" ]; then - pwsh -NoProfile -File ./install.ps1 --local 2>&1 | tee logs/install.log - else - bash install.sh --local 2>&1 | tee logs/install.log - fi - - - name: Profile startup - shell: bash - run: | - BIN="$UNSLOTH_STUDIO_HOME/unsloth_studio/bin/unsloth" - [ -x "$BIN" ] || BIN="$UNSLOTH_STUDIO_HOME/unsloth_studio/Scripts/unsloth.exe" - [ -x "$BIN" ] || BIN="" - # Profile imports with the INSTALLED interpreter: that venv is what launches. - PY="$UNSLOTH_STUDIO_HOME/unsloth_studio/bin/python" - [ -x "$PY" ] || PY="$UNSLOTH_STUDIO_HOME/unsloth_studio/Scripts/python.exe" - [ -x "$PY" ] || PY="$(command -v python3 || command -v python)" - python3 scripts/profile_startup.py \ - --python "$PY" \ - ${BIN:+--bin "$BIN"} \ - --repeats "${{ inputs.repeats || '3' }}" \ - --json "startup-${{ matrix.os }}.json" 2>&1 | tee logs/profile.log - - - name: Summary - if: always() - shell: bash - run: | - f="startup-${{ matrix.os }}.json" - [ -f "$f" ] || { echo "no profile produced"; exit 0; } - python3 - "$f" >> "$GITHUB_STEP_SUMMARY" <<'PY' - import json, sys - d = json.load(open(sys.argv[1])) - print(f"### {d['platform']} / {d['machine']} (py {d['python']}, {d['cpu_count']} cpu)\n") - imp = d.get("imports", {}) - # Gate on ok: a failed `import main` still leaves rows, so a total can lie. - if imp.get("ok"): - print(f"**`import main`: {imp['total_seconds']}s**\n") - print("| package | self ms |") - print("|---|---:|") - for k, v in list(imp.get("self_by_package_ms", {}).items())[:8]: - print(f"| {k} | {v} |") - print() - else: - print("**`import main` failed - no valid import profile**\n") - print("```\n" + (imp.get("error") or "")[-1500:] + "\n```\n") - lau = d.get("launch") or {} - runs = len(lau.get("runs") or []) - failed = lau.get("failed_runs") or 0 - if lau.get("healthz_median_seconds") is not None: - # The aggregates cover only the runs that reached healthz, so flag the - # failures: bare numbers would read as a normal fast startup. - note = f" _({runs - failed} of {runs} launches; {failed} never became healthy)_" if failed else "" - print(f"**time to a healthy port: {lau['healthz_median_seconds']}s median, " - f"{lau['healthz_max_seconds']}s max**{note}\n") - elif lau.get("skipped"): - print(f"_launch phase skipped: {lau['skipped']}_\n") - elif runs: - print(f"**no launch measurement: all {runs} launches failed to become healthy**\n") - PY - - - name: Upload profile - if: always() - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 - with: - name: startup-profile-${{ matrix.os }} - path: | - startup-*.json - logs/ - retention-days: 14 - if-no-files-found: warn diff --git a/.github/workflows/studio-api-smoke.yml b/.github/workflows/studio-api-smoke.yml index 1cfa66fea4..cdf1f6bf12 100644 --- a/.github/workflows/studio-api-smoke.yml +++ b/.github/workflows/studio-api-smoke.yml @@ -113,8 +113,7 @@ jobs: - name: Reset auth + boot Unsloth (API-only) run: | - # Wipe (not reset-password): the boot below must re-seed a fresh .bootstrap_password. - rm -rf ~/.unsloth/studio/auth + unsloth studio reset-password mkdir -p logs UNSLOTH_API_ONLY=1 unsloth studio -H 127.0.0.1 -p "$STUDIO_PORT" \ > logs/studio.log 2>&1 & diff --git a/.github/workflows/studio-backend-ci.yml b/.github/workflows/studio-backend-ci.yml index dd5efbb299..ec437e0c32 100644 --- a/.github/workflows/studio-backend-ci.yml +++ b/.github/workflows/studio-backend-ci.yml @@ -223,16 +223,6 @@ jobs: tests/studio/test_is_mlx_dispatch_gate.py \ tests/studio/test_xpu_spoof_pipeline.py - - name: CLI tests (unsloth_cli) - # unsloth_cli/tests had no CI at all: `unsloth_cli/**` was only a paths - # trigger and a ruff target, so 673 tests covering the studio launcher, - # the pre-exposure gate and the auth secret writers ran nowhere, and - # four of them had been failing on main unnoticed. - # Own step, not folded into the tests/ discovery above: pyproject's - # testpaths is tests/, and this suite needs no PYTHONPATH or CUDA spoof - # (it self-bootstraps sys.path and imports neither unsloth nor torch). - run: python -m pytest unsloth_cli/tests -q --tb=short - - name: Shell installer tests # Auto-discovered rather than allowlisted. The old hardcoded list had # silently fallen seven files behind tests/run_all.sh, including diff --git a/.github/workflows/studio-frontend-ci.yml b/.github/workflows/studio-frontend-ci.yml index 773e555c8b..3a9e373915 100644 --- a/.github/workflows/studio-frontend-ci.yml +++ b/.github/workflows/studio-frontend-ci.yml @@ -133,9 +133,6 @@ jobs: - name: Typecheck run: npm run typecheck - - name: Unit tests - run: npm test - - name: Build run: npm run build diff --git a/.github/workflows/studio-inference-smoke.yml b/.github/workflows/studio-inference-smoke.yml index c37c9555bf..c2d52eac22 100644 --- a/.github/workflows/studio-inference-smoke.yml +++ b/.github/workflows/studio-inference-smoke.yml @@ -127,8 +127,7 @@ jobs: - name: Reset auth + boot Unsloth (API-only) run: | - # Wipe (not reset-password): the boot below must re-seed a fresh .bootstrap_password. - rm -rf ~/.unsloth/studio/auth + unsloth studio reset-password mkdir -p logs UNSLOTH_API_ONLY=1 unsloth studio -H 127.0.0.1 -p "$STUDIO_PORT" \ > logs/studio.log 2>&1 & @@ -401,7 +400,7 @@ jobs: # tool_policy=None so each request's `enable_tools` field is # honoured. run: | - rm -rf ~/.unsloth/studio/auth + unsloth studio reset-password mkdir -p logs UNSLOTH_API_ONLY=1 unsloth studio -H 127.0.0.1 -p "$STUDIO_PORT" \ > logs/studio.log 2>&1 & @@ -979,7 +978,7 @@ jobs: # response_format requests aren't routed through the agentic # tool loop. run: | - rm -rf ~/.unsloth/studio/auth + unsloth studio reset-password mkdir -p logs UNSLOTH_API_ONLY=1 unsloth studio -H 127.0.0.1 -p "$STUDIO_PORT" \ > logs/studio.log 2>&1 & diff --git a/.github/workflows/studio-mac-api-smoke.yml b/.github/workflows/studio-mac-api-smoke.yml index c2307f17a1..1968885a1d 100644 --- a/.github/workflows/studio-mac-api-smoke.yml +++ b/.github/workflows/studio-mac-api-smoke.yml @@ -101,8 +101,7 @@ jobs: - name: Reset auth + boot Unsloth (API-only) run: | - # Wipe (not reset-password): the boot below must re-seed a fresh .bootstrap_password. - rm -rf ~/.unsloth/studio/auth + unsloth studio reset-password mkdir -p logs UNSLOTH_API_ONLY=1 unsloth studio -H 127.0.0.1 -p "$STUDIO_PORT" \ > logs/studio.log 2>&1 & diff --git a/.github/workflows/studio-mac-inference-smoke.yml b/.github/workflows/studio-mac-inference-smoke.yml index 1dbf86ae98..ce15eed5c8 100644 --- a/.github/workflows/studio-mac-inference-smoke.yml +++ b/.github/workflows/studio-mac-inference-smoke.yml @@ -126,8 +126,7 @@ jobs: - name: Reset auth + boot Unsloth (API-only) run: | - # Wipe (not reset-password): the boot below must re-seed a fresh .bootstrap_password. - rm -rf ~/.unsloth/studio/auth + unsloth studio reset-password mkdir -p logs UNSLOTH_API_ONLY=1 unsloth studio -H 127.0.0.1 -p "$STUDIO_PORT" \ > logs/studio.log 2>&1 & @@ -387,7 +386,7 @@ jobs: # tool_policy=None so each request's `enable_tools` field is # honoured. run: | - rm -rf ~/.unsloth/studio/auth + unsloth studio reset-password mkdir -p logs UNSLOTH_API_ONLY=1 unsloth studio -H 127.0.0.1 -p "$STUDIO_PORT" \ > logs/studio.log 2>&1 & @@ -832,7 +831,7 @@ jobs: # response_format requests aren't routed through the agentic # tool loop. run: | - rm -rf ~/.unsloth/studio/auth + unsloth studio reset-password mkdir -p logs UNSLOTH_API_ONLY=1 unsloth studio -H 127.0.0.1 -p "$STUDIO_PORT" \ > logs/studio.log 2>&1 & diff --git a/.github/workflows/studio-mac-ui-smoke.yml b/.github/workflows/studio-mac-ui-smoke.yml index 3bed2fcdff..7375e9bcbf 100644 --- a/.github/workflows/studio-mac-ui-smoke.yml +++ b/.github/workflows/studio-mac-ui-smoke.yml @@ -146,8 +146,7 @@ jobs: - name: Reset auth + boot Unsloth run: | - # Wipe (not reset-password): the boot below must re-seed a fresh .bootstrap_password. - rm -rf ~/.unsloth/studio/auth + unsloth studio reset-password mkdir -p logs UNSLOTH_API_ONLY=1 unsloth studio -H 127.0.0.1 -p "$STUDIO_PORT" \ > logs/studio.log 2>&1 & @@ -191,7 +190,7 @@ jobs: # runner's kernel briefly runs out of socket buffers, and (3) a # goto 'interrupted by another navigation' when the SPA auth # guard redirects mid-navigation. The retry FULLY resets Unsloth - # (kill, wipe auth, reboot, wait /api/health, re-export + # (kill, reset-password, reboot, wait /api/health, re-export # bootstrap pw) before re-running the script. A real test failure # (assertion / timeout) does NOT match any pattern so it bypasses # retry and surfaces immediately. @@ -214,7 +213,7 @@ jobs: echo "::warning::Playwright flake on attempt ${attempt}; resetting Unsloth and retrying..." kill "${STUDIO_PID}" 2>/dev/null || true sleep 2 - rm -rf ~/.unsloth/studio/auth + unsloth studio reset-password UNSLOTH_API_ONLY=1 unsloth studio -H 127.0.0.1 -p "$STUDIO_PORT" \ > "logs/studio_retry_${attempt}.log" 2>&1 & STUDIO_PID=$! @@ -252,7 +251,7 @@ jobs: - name: Reset auth + boot Unsloth for extra UI tests (port 18897) run: | - rm -rf ~/.unsloth/studio/auth + unsloth studio reset-password mkdir -p logs UNSLOTH_API_ONLY=1 unsloth studio -H 127.0.0.1 -p 18897 \ > logs/studio_extra.log 2>&1 & @@ -309,7 +308,7 @@ jobs: echo "::warning::Playwright flake on attempt ${attempt}; resetting Unsloth and retrying..." kill "${STUDIO_EXTRA_PID}" 2>/dev/null || true sleep 2 - rm -rf ~/.unsloth/studio/auth + unsloth studio reset-password UNSLOTH_API_ONLY=1 unsloth studio -H 127.0.0.1 -p 18897 \ > "logs/studio_extra_retry_${attempt}.log" 2>&1 & STUDIO_EXTRA_PID=$! diff --git a/.github/workflows/studio-tauri-smoke.yml b/.github/workflows/studio-tauri-smoke.yml index c6dad07f37..8e26b9fd0c 100644 --- a/.github/workflows/studio-tauri-smoke.yml +++ b/.github/workflows/studio-tauri-smoke.yml @@ -91,16 +91,6 @@ jobs: npm run build test -f dist/index.html - # The crate carries ~100 unit tests (native_file_dialogs, preflight, - # install, desktop_auth, ...) that nothing ran until now: this workflow - # only ever built. Run them here, where the toolchain and the WebKit dev - # packages are already installed, so a broken assertion fails the PR - # instead of sitting unnoticed. `--no-fail-fast` reports every failing - # test in one run rather than stopping at the first. - - name: Rust unit tests (studio/src-tauri) - working-directory: studio/src-tauri - run: cargo test --no-fail-fast - - name: Tauri debug build (Linux, no bundle, no codesign) # `--debug` + `--no-bundle` keeps this lean: compiles the Rust crate, # confirms the frontend dist is wired into Tauri, but skips the AppImage diff --git a/.github/workflows/studio-ui-smoke.yml b/.github/workflows/studio-ui-smoke.yml index 3a0713f301..97eb07b2d8 100644 --- a/.github/workflows/studio-ui-smoke.yml +++ b/.github/workflows/studio-ui-smoke.yml @@ -115,8 +115,7 @@ jobs: - name: Reset auth + boot Unsloth run: | - # Wipe (not reset-password): the boot below must re-seed a fresh .bootstrap_password. - rm -rf ~/.unsloth/studio/auth + unsloth studio reset-password mkdir -p logs UNSLOTH_API_ONLY=1 unsloth studio -H 127.0.0.1 -p "$STUDIO_PORT" \ > logs/studio.log 2>&1 & @@ -194,7 +193,7 @@ jobs: # warm install we already did) so this adds little wall time. - name: Reset auth + boot Unsloth for extra UI tests (port 18894) run: | - rm -rf ~/.unsloth/studio/auth + unsloth studio reset-password mkdir -p logs UNSLOTH_API_ONLY=1 unsloth studio -H 127.0.0.1 -p 18894 \ > logs/studio_extra.log 2>&1 & @@ -254,7 +253,7 @@ jobs: # (RAG embedder + llama.cpp probe) stay hidden from the picker. - name: Reset auth + boot Unsloth for model-config tests (port 18898) run: | - rm -rf ~/.unsloth/studio/auth + unsloth studio reset-password mkdir -p logs UNSLOTH_API_ONLY=1 unsloth studio -H 127.0.0.1 -p 18898 \ > logs/studio_modelcfg.log 2>&1 & @@ -300,7 +299,7 @@ jobs: # earlier UI tests. No GGUF -- the bug surface is the composer. - name: Reset auth + boot Unsloth for IME / i18n tests (port 18896) run: | - rm -rf ~/.unsloth/studio/auth + unsloth studio reset-password mkdir -p logs UNSLOTH_API_ONLY=1 unsloth studio -H 127.0.0.1 -p 18896 \ > logs/studio_ime.log 2>&1 & diff --git a/.github/workflows/studio-update-smoke.yml b/.github/workflows/studio-update-smoke.yml index 047840e41c..625c2c7811 100644 --- a/.github/workflows/studio-update-smoke.yml +++ b/.github/workflows/studio-update-smoke.yml @@ -146,46 +146,6 @@ jobs: kill "$PID" 2>/dev/null || true echo "post-update Unsloth /api/health OK" - - name: A complete install reports itself complete - run: | - set -o pipefail - unsloth studio verify-install - unsloth studio desktop-capabilities --json | tee /tmp/caps.json - jq -e '.studio_install_ok == true' /tmp/caps.json - jq -e '.desktop_manageability_version >= 2' /tmp/caps.json - - - name: An incomplete install must not report itself ready - # An installer killed part-way leaves a working CLI but no studio.txt - # deps, which the old preflight called ManagedReady. The manifest is - # written last, so removing it reproduces that state. - run: | - set -o pipefail - # install.sh's default root, resolved explicitly: `python` on PATH - # here is setup-python's, not the managed venv. - MANIFEST="$HOME/.unsloth/studio/unsloth_studio/unsloth_install_manifest.json" - test -f "$MANIFEST" || { echo "::error::installer never wrote $MANIFEST"; exit 1; } - rm -f "$MANIFEST" - unsloth studio desktop-capabilities --json | tee /tmp/caps_bad.json - jq -e '.studio_install_ok == false' /tmp/caps_bad.json - if unsloth studio verify-install; then - echo "::error::verify-install passed on an install with no manifest" - exit 1 - fi - echo "incomplete install correctly reported not-ready" - - - name: Update repairs an incomplete install - # `--local` bypasses setup.sh's PyPI version compare, so this asserts - # the repair OUTCOME. The non-local fast path the desktop Repair button - # uses is covered by tests/studio/install/test_setup_fast_path_guard.py. - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - set -o pipefail - unsloth studio update --local 2>&1 | tee logs/update_repair.log - unsloth studio verify-install - unsloth studio desktop-capabilities --json | jq -e '.studio_install_ok == true' - echo "update repaired the incomplete install" - - name: Uninstall and verify clean # Round-trip the installer through scripts/uninstall.sh: confirms the # uninstaller actually finds and removes everything install.sh + diff --git a/.github/workflows/studio-windows-api-smoke.yml b/.github/workflows/studio-windows-api-smoke.yml index b328939846..6dbcceebbd 100644 --- a/.github/workflows/studio-windows-api-smoke.yml +++ b/.github/workflows/studio-windows-api-smoke.yml @@ -179,8 +179,7 @@ jobs: - name: Reset auth + boot Unsloth (API-only) run: | - # Wipe (not reset-password): the boot below must re-seed a fresh .bootstrap_password. - rm -rf ~/.unsloth/studio/auth + unsloth studio reset-password mkdir -p logs UNSLOTH_API_ONLY=1 unsloth studio -H 127.0.0.1 -p "$STUDIO_PORT" \ > logs/studio.log 2>&1 & diff --git a/.github/workflows/studio-windows-inference-smoke.yml b/.github/workflows/studio-windows-inference-smoke.yml index d821664327..3ebe442f52 100644 --- a/.github/workflows/studio-windows-inference-smoke.yml +++ b/.github/workflows/studio-windows-inference-smoke.yml @@ -229,8 +229,7 @@ jobs: - name: Reset auth + boot Unsloth (API-only) run: | - # Wipe (not reset-password): the boot below must re-seed a fresh .bootstrap_password. - rm -rf ~/.unsloth/studio/auth + unsloth studio reset-password mkdir -p logs UNSLOTH_API_ONLY=1 unsloth studio -H 127.0.0.1 -p "$STUDIO_PORT" \ > logs/studio.log 2>&1 & @@ -574,7 +573,7 @@ jobs: - name: Reset auth + boot Unsloth (API-only, default tool policy) run: | - rm -rf ~/.unsloth/studio/auth + unsloth studio reset-password mkdir -p logs UNSLOTH_API_ONLY=1 unsloth studio -H 127.0.0.1 -p "$STUDIO_PORT" \ > logs/studio.log 2>&1 & @@ -1075,7 +1074,7 @@ jobs: - name: Reset auth + boot Unsloth (API-only) run: | - rm -rf ~/.unsloth/studio/auth + unsloth studio reset-password mkdir -p logs UNSLOTH_API_ONLY=1 unsloth studio -H 127.0.0.1 -p "$STUDIO_PORT" \ > logs/studio.log 2>&1 & @@ -1547,7 +1546,7 @@ jobs: - name: Reset auth + boot Unsloth (API-only) run: | - rm -rf ~/.unsloth/studio/auth + unsloth studio reset-password mkdir -p logs UNSLOTH_API_ONLY=1 unsloth studio -H 127.0.0.1 -p "$STUDIO_PORT" \ > logs/studio.log 2>&1 & @@ -1889,11 +1888,8 @@ jobs: # (step/substep -> Write-StudioStdoutMirror / Get-StudioAnsi). $script:StudioVtOk = $false $script:UnslothVerbose = $false - # Get-HostMachineArch is reached only on the absent path, where - # Test-VCRedistInstalled consults it before trusting the System32 DLL, so - # part A passes without it and only the clean-box part fails. foreach ($fn in @('Get-StudioAnsi', 'Write-StudioStdoutMirror', 'step', 'substep', - 'Invoke-SetupCommand', 'Refresh-Environment', 'Get-HostMachineArch', + 'Invoke-SetupCommand', 'Refresh-Environment', 'Test-VCRedistInstalled', 'Ensure-VCRedist')) { $src = Get-FunctionSource -Path $setup -Name $fn if (-not $src) { throw "Function '$fn' not found in setup.ps1" } diff --git a/.github/workflows/studio-windows-ui-smoke.yml b/.github/workflows/studio-windows-ui-smoke.yml index d23cca323f..f401f7be44 100644 --- a/.github/workflows/studio-windows-ui-smoke.yml +++ b/.github/workflows/studio-windows-ui-smoke.yml @@ -297,8 +297,7 @@ jobs: - name: Reset auth + boot Unsloth run: | - # Wipe (not reset-password): the boot below must re-seed a fresh .bootstrap_password. - rm -rf ~/.unsloth/studio/auth + unsloth studio reset-password mkdir -p logs UNSLOTH_API_ONLY=1 unsloth studio -H 127.0.0.1 -p "$STUDIO_PORT" \ > logs/studio.log 2>&1 & @@ -353,7 +352,7 @@ jobs: - name: Reset auth + boot Unsloth for extra UI tests (port 18897) run: | - rm -rf ~/.unsloth/studio/auth + unsloth studio reset-password mkdir -p logs UNSLOTH_API_ONLY=1 unsloth studio -H 127.0.0.1 -p 18897 \ > logs/studio_extra.log 2>&1 & diff --git a/.github/workflows/studio-windows-update-smoke.yml b/.github/workflows/studio-windows-update-smoke.yml index 0dcc828e6b..42d74d47d2 100644 --- a/.github/workflows/studio-windows-update-smoke.yml +++ b/.github/workflows/studio-windows-update-smoke.yml @@ -198,31 +198,6 @@ jobs: fi echo "update path took the prebuilt fast path" - - name: Update must keep the --no-torch install GGUF-only - run: | - # `unsloth studio update` exports no UNSLOTH_NO_TORCH, so setup.ps1 has - # to recover the mode from the install manifest. Without that it reads - # the missing torch as a stale venv and tries to delete the venv it is - # running out of, and the shared dependency pass pulls torch back in. - # The skip line only prints when the dependency pass actually runs, so - # don't demand it if the fast path short-circuited that pass. - if grep -q "running ordered dependency installation" logs/update.log \ - && ! grep -q "skipping direct PyTorch and Triton installation (no-torch mode)" logs/update.log; then - echo "::error::studio update left no-torch mode; it would reinstall PyTorch." - grep -iE "no-torch|stale venv|PyTorch" logs/update.log | tail -40 - exit 1 - fi - PY="$HOME/.unsloth/studio/unsloth_studio/Scripts/python.exe" - if [ ! -f "$PY" ]; then - echo "::error::studio venv interpreter missing at $PY" - exit 1 - fi - if "$PY" -c "import torch" 2>/dev/null; then - echo "::error::torch was reinstalled into the --no-torch venv." - exit 1 - fi - echo "update preserved no-torch mode" - - name: Second update must also be a no-op env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/wheel-smoke.yml b/.github/workflows/wheel-smoke.yml index f7a7511616..cdad617027 100644 --- a/.github/workflows/wheel-smoke.yml +++ b/.github/workflows/wheel-smoke.yml @@ -127,31 +127,6 @@ jobs: cd /tmp /tmp/v/bin/python -c "from studio.backend.main import app; print('Unsloth backend OK:', app.title)" - - name: CLI without the Studio stack guides instead of tracebacking - # The smoke above installs studio.txt first, so it cannot catch a wheel - # that ships studio/ without declaring what it imports (#4701, #5260, - # #7147). Drop only structlog to reuse that venv without a re-download. - run: | - set -eu - /tmp/v/bin/pip uninstall -y structlog >/dev/null - cd /tmp - status=0 - for args in "export ./nope ./out" "list-checkpoints"; do - echo "--- unsloth $args" - out=$(/tmp/v/bin/unsloth $args 2>&1 || true) - printf '%s\n' "$out" - case "$out" in - *Traceback*) - echo "FAIL: raw traceback instead of guidance"; status=1 ;; - esac - case "$out" in - *'unsloth studio update'*) ;; - *) echo "FAIL: no remediation in the message"; status=1 ;; - esac - done - /tmp/v/bin/pip install -q structlog >/dev/null - exit "$status" - - name: Upload wheel on failure if: failure() uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 diff --git a/.gitignore b/.gitignore index fa6997cb06..39ca2226ca 100644 --- a/.gitignore +++ b/.gitignore @@ -208,9 +208,6 @@ tmp/ **/node_modules/ auth.db -# Packaging snapshot of the root CHANGELOG.md (written by build.sh) -studio/CHANGELOG.md - # Tauri local build/generated output studio/src-tauri/target/ studio/src-tauri/gen/ @@ -241,5 +238,4 @@ package-lock.json !studio/package-lock.json llama.cpp/ # Stray "~" dir some tools create from a literal ~ TMPDIR; never part of the repo. -~/ -/temp/ +/~/ diff --git a/CHANGELOG.md b/CHANGELOG.md deleted file mode 100644 index 241e013cea..0000000000 --- a/CHANGELOG.md +++ /dev/null @@ -1,88 +0,0 @@ -# Changelog - -Release notes for Unsloth and Unsloth Studio. - -Unsloth Studio reads this file to show release notes inside the "New Unsloth -version" update popup. Edit it here and the popup picks the change up on the -next update check, with no release or rebuild required. - -## Format - -Every release is a level-2 heading whose first token is the version, optionally -followed by a date: - -```md -## 2026.7.6 - 2026-07-22 -``` - -`## [2026.7.6] - 2026-07-22` and `## v2026.7.6` also work. Everything under a -heading, up to the next level-2 heading, is that release's notes and renders as -Markdown in the popup. - -Notes are matched to one exact version. When Studio offers an update to -`2026.7.6` it renders the `2026.7.6` section and nothing else. If that section -is missing, the popup links out to the online changelog rather than showing -notes from an unrelated release, so a new version needs its own section here -before its notes can appear. - -Keep the newest release at the top. Lead each bullet with the change itself: -the collapsed popup highlights the first sentence and dims the rest. -`## Unreleased` is ignored by the popup, so it is safe to stage notes there and -rename the heading at release time. - - - -## Unreleased - -## 2026.7.5 - -### What's Changed - -- AMD support is here. Train, run RL, chat with and deploy 500+ models on - Radeon, Instinct, Ryzen and data center GPUs across Windows, WSL and Linux, - up to 2x faster with 70% less VRAM and no accuracy loss. -- Intel XPU support lands in Studio, so Arc and Data Center GPUs run chat and - training alongside the NVIDIA, AMD and Apple paths. -- Local speech to text dictation runs fully offline, with slim Whisper bundles - and a picker for custom models. -- DoRA training is available in Studio, selectable next to LoRA and full - fine-tuning in the training tab. -- The update popup previews release notes inline, pulled from this file and - matched to the exact version being offered. - -### AMD, 23 July update - -Our AMD collaboration, custom Triton kernels and math algorithms bring local -training and inference to AMD hardware. The 23 July update builds on the -[AMD release](https://github.com/unslothai/unsloth/releases/tag/v0.1.501-beta): - -- RDNA2 and Gorgon Halo are supported, and the installer no longer fails to - detect GPUs on Strix Halo and other AMD cards. -- RDNA4 handling is better, and HIP and ROCm failures are caught and fixed - automatically instead of stopping the install. -- Unified memory safetensors loading is 2x faster, with much faster gradient - checkpointing on unified memory devices. -- Voice dictation through whisper.cpp has preliminary support. -- Rollback environments left by installs no longer eat 5GB of disk. They are - cleaned up automatically. - -Optimized ROCm builds cover GGUF and safetensors inference, and ROCm -compatibility is improved for MI300X and MI325X. Full guide: -[unsloth.ai/docs/basics/amd](https://unsloth.ai/docs/basics/amd). - -### Running larger models - -- Automatic GPU placement, or pick exactly which GPUs and layers to use. -- Move MoE expert layers into system memory so larger models fit. -- Split a model across several GPUs, or use tensor parallelism. -- Hardware settings are saved per model and quant. - -### Also in this release - -- Remote access with `unsloth studio --secure` over free HTTPS via Cloudflare. -- Web search reads PDF papers and manuals, and parallel tool calls, reasoning - output and tool retries are more reliable. -- The model download location is configurable, so weights can live on a second - drive instead of the default cache. -- Stalled Hugging Face XET downloads retry over standard HTTP, and existing - GGUF files are reused instead of downloaded again. diff --git a/MANIFEST.in b/MANIFEST.in deleted file mode 100644 index 7bce036343..0000000000 --- a/MANIFEST.in +++ /dev/null @@ -1,2 +0,0 @@ -include _changelog_build.py -include CHANGELOG.md diff --git a/_changelog_build.py b/_changelog_build.py deleted file mode 100644 index f5bcf2052c..0000000000 --- a/_changelog_build.py +++ /dev/null @@ -1,36 +0,0 @@ -# SPDX-License-Identifier: AGPL-3.0-only -# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. - -"""Snapshot CHANGELOG.md into the studio package at build time. - -CHANGELOG.md at the repo root stays the one file to edit. Copying it here, -rather than in build.sh, means every packaging path ships it, so release notes -still render when the popup cannot reach GitHub.""" - -from __future__ import annotations - -import shutil -from pathlib import Path - -from setuptools.command.build_py import build_py as _build_py - -ROOT = Path(__file__).resolve().parent -SOURCE = ROOT / "CHANGELOG.md" -SNAPSHOT = ROOT / "studio" / "CHANGELOG.md" - - -class build_py(_build_py): - def run(self) -> None: - # Beside the sources only if writable (PEP 517 may build an immutable - # checkout); into the staging directory always. - if SOURCE.is_file(): - try: - shutil.copyfile(SOURCE, SNAPSHOT) - except OSError: - pass - super().run() - if not SOURCE.is_file(): - return - staged = Path(self.build_lib) / "studio" / "CHANGELOG.md" - staged.parent.mkdir(parents = True, exist_ok = True) - shutil.copyfile(SOURCE, staged) diff --git a/build.sh b/build.sh index 5b09a7791b..2a836e19d9 100644 --- a/build.sh +++ b/build.sh @@ -103,13 +103,9 @@ else STUDIO_STAMPED_VERSION="$(python scripts/stamp_studio_release.py)" fi -# 4. Build wheel/sdist. _changelog_build.py snapshots CHANGELOG.md into the studio -# package so release notes render offline. +# 4. Build wheel/sdist python -m build -# Drop the snapshot so a source checkout never serves a stale copy. -rm -f studio/CHANGELOG.md - if [ "${1:-}" = "publish" ]; then python scripts/stamp_studio_release.py --verify-dist dist --expected "$STUDIO_STAMPED_VERSION" fi diff --git a/install.ps1 b/install.ps1 index 5b205df96d..a2aff0b69a 100644 --- a/install.ps1 +++ b/install.ps1 @@ -28,14 +28,6 @@ function Install-UnslothStudio { } } - function Clear-TauriInstallError { - param([string]$Message) - if ($TauriMode) { - Write-TauriLog "ERROR_CLEAR" $Message - [Console]::Error.WriteLine("[TAURI:ERROR_CLEAR] $Message") - } - } - function Format-TauriDiagBool { param([bool]$Value) if ($Value) { return "true" } @@ -57,26 +49,6 @@ function Install-UnslothStudio { } } - # Machine arch; Get-TauriDiagArch above reports the process. An emulated x64 shell on - # ARM64 reports AMD64, but PROCESSOR_ARCHITEW6432 is ARM64 in exactly that case. - function Get-HostMachineArch { - $osArch = "" - try { $osArch = [System.Runtime.InteropServices.RuntimeInformation]::OSArchitecture.ToString() } catch { $osArch = "" } - $signals = @([string]$env:PROCESSOR_ARCHITEW6432, [string]$env:PROCESSOR_ARCHITECTURE, $osArch) - foreach ($s in $signals) { - if ($s.ToLowerInvariant() -eq "arm64") { return "arm64" } - } - foreach ($s in $signals) { - if ([string]::IsNullOrWhiteSpace($s)) { continue } - switch ($s.ToLowerInvariant()) { - "amd64" { return "x86_64" } - "x64" { return "x86_64" } - "x86" { return "x86" } - } - } - return "unknown" - } - function Get-TauriTorchIndexFamily { param([string]$TorchIndexUrl) if ($SkipTorch) { return "none" } @@ -114,7 +86,7 @@ function Install-UnslothStudio { [int]$Code = 1 ) if ($Code -eq 0) { $Code = 1 } - Write-TauriLog "ERROR_DEFAULT" $Message + Write-TauriLog "ERROR" $Message if (Get-Command Restore-StudioVenvRollback -CommandType Function -ErrorAction SilentlyContinue) { Restore-StudioVenvRollback } @@ -513,8 +485,7 @@ function Install-UnslothStudio { # Full command output is shown only when --verbose / UNSLOTH_VERBOSE=1. function Invoke-InstallCommand { param( - [Parameter(Mandatory = $true)][ScriptBlock]$Command, - [string]$Label = "install command" + [Parameter(Mandatory = $true)][ScriptBlock]$Command ) # Installer-pinned index installs (torch) must beat an inherited uv mirror (#6898): # for --default-index, clear the uv index env vars (restore in finally) and set @@ -533,7 +504,6 @@ function Install-UnslothStudio { try { # Reset to avoid stale values from prior native commands. $global:LASTEXITCODE = 0 - Write-TauriLog "OUTPUT_CLEAR" $Label if ($script:UnslothVerbose) { # Merge stderr into stdout so progress/warning output stays visible # without flipping $? on successful native commands (PS 5.1 treats @@ -548,13 +518,7 @@ function Install-UnslothStudio { Write-Host (Redact-InstallOutput $output) -ForegroundColor Red } } - $exitCode = [int]$LASTEXITCODE - if ($exitCode -eq 0) { - Clear-TauriInstallError "$Label recovered" - } else { - Write-TauriLog "ERROR_OUTPUT" "$Label failed (exit code $exitCode)" - } - return $exitCode + return [int]$LASTEXITCODE } finally { $ErrorActionPreference = $prevEap if ($savedUvIndex) { @@ -585,7 +549,7 @@ function Install-UnslothStudio { } $attempt = 1 while ($true) { - $code = Invoke-InstallCommand -Command $Command -Label $Label + $code = Invoke-InstallCommand $Command if ($code -eq 0) { return 0 } if ($attempt -ge $maxAttempts) { return $code } substep ("retrying ""$Label"" after transient failure (attempt $($attempt + 1)/$maxAttempts, waiting ${delay}s)...") "Yellow" @@ -1144,27 +1108,10 @@ exit 0 return $false } - # The interpreter's own arch, asked of it: win-amd64|win-arm64|win32|"". - function Get-PythonPlatformTag { - param([string]$Exe) - try { - return (& $Exe -c "import sysconfig; print(sysconfig.get_platform())" 2>$null | Out-String).Trim().ToLowerInvariant() - } catch { return "" } - } - # Returns @{ Version = "3.13"; Path = "C:\...\python.exe" } or $null. # The resolved Path is passed to `uv venv --python` to prevent uv from # re-resolving the version string back to a conda interpreter. function Find-CompatiblePython { - # -X64Only: best installed x64 interpreter or $null, never ARM64. Last resort for - # Install-X64Python, where x64 of a lower-priority minor beats ARM64. - param([switch]$X64Only) - # Windows on ARM: prefer x64. pyarrow (via datasets) and hf-transfer ship no - # win_arm64 wheel, so a native ARM64 Python source-builds both and dies on CMake / - # Rust minutes in; x64 runs fine emulated. ARM64 is still returned when it is all - # there is, and the caller then bootstraps x64 or warns. - $preferX64 = $X64Only -or ((Get-HostMachineArch) -eq "arm64") - $candidates = @() # Try the Python Launcher first (most reliable on Windows) # py.exe resolves to the standard CPython install, not conda. # Prefer the requested $PythonVersion, then newest-first fallback. @@ -1182,8 +1129,7 @@ exit 0 # Resolve the actual executable path and verify it is not conda-based $resolvedExe = (& $pyLauncher.Source "-$minor" -c "import sys; print(sys.executable)" 2>$null | Out-String).Trim() if ($resolvedExe -and (Test-Path $resolvedExe) -and -not (Test-IsCondaPython $resolvedExe)) { - if (-not $preferX64) { return @{ Version = $ver; Path = $resolvedExe; Arch = "" } } - $candidates += @{ Version = $ver; Path = $resolvedExe } + return @{ Version = $ver; Path = $resolvedExe } } } } catch {} @@ -1204,53 +1150,11 @@ exit 0 try { $out = & $cmd.Source --version 2>&1 | Out-String if ($out -match "Python (3\.1[1-3])\.\d+") { - if (-not $preferX64) { return @{ Version = $Matches[1]; Path = $cmd.Source; Arch = "" } } - $candidates += @{ Version = $Matches[1]; Path = $cmd.Source } + return @{ Version = $Matches[1]; Path = $cmd.Source } } } catch {} } } - # `py -3.12` runs the launcher's preferred build, normally the native ARM64 one, so - # a same-minor x64 install that is neither preferred nor on PATH never becomes a - # candidate. `-3.12-64` cannot disambiguate (deprecated, it only means "not - # 32-bit"), so enumerate every registration with -0p and probe each path. - if ($preferX64) { - foreach ($pyLauncher in @(Get-Command py -All -CommandType Application -ErrorAction SilentlyContinue)) { - if ($pyLauncher.Source -match $script:CondaSkipPattern) { continue } - $listed = @() - try { $listed = @(& $pyLauncher.Source "-0p" 2>$null) } catch {} - foreach ($line in $listed) { - # " -V:3.12 * C:\...\python.exe": tag, optional default marker, path. - $m = [regex]::Match([string]$line, '(?i)^\s*-\S+\s+\*?\s*"?(?

\S.*?\.exe)"?\s*$') - if (-not $m.Success) { continue } - $exe = $m.Groups['p'].Value.Trim() - if ($candidates | Where-Object { $_.Path -eq $exe }) { continue } - if (-not (Test-Path -LiteralPath $exe)) { continue } - if (Test-IsCondaPython $exe) { continue } - try { - $out = & $exe --version 2>&1 | Out-String - if ($out -match "Python (3\.1[1-3])\.\d+") { - $candidates += @{ Version = $Matches[1]; Path = $exe } - } - } catch {} - } - } - } - # Prefer x64, but only within one minor: $minors is the caller's version preference, - # so ranking on arch alone would answer UNSLOTH_PYTHON=3.12 with an x64 3.13 and - # never bootstrap x64 3.12. Probing costs a subprocess, so non-ARM returned above. - foreach ($c in $candidates) { - $tag = Get-PythonPlatformTag $c.Path - $c.Arch = if ($tag -eq "win-amd64") { "x86_64" } elseif ($tag -eq "win-arm64") { "arm64" } else { "unknown" } - } - foreach ($minor in $minors) { - $sameMinor = @($candidates | Where-Object { $_.Version -eq $minor }) - if ($sameMinor.Count -eq 0) { continue } - $x64 = $sameMinor | Where-Object { $_.Arch -eq "x86_64" } | Select-Object -First 1 - if ($x64) { return $x64 } - if (-not $X64Only) { return $sameMinor[0] } - } - if (-not $X64Only -and $candidates.Count -gt 0) { return $candidates[0] } return $null } @@ -1261,11 +1165,8 @@ exit 0 # (no UAC), putting python.exe + the py launcher on PATH. Mirrors the uv -> # astral.sh fallback below. Returns @{ Version; Path } or $null. function Install-PythonFromPythonOrg { - # $Arch overrides the host arch, to pull x64 onto an ARM64 box. - param([string]$Arch = "") # python.org ships one installer per architecture. - $targetArch = if ($Arch) { $Arch } else { Get-TauriDiagArch } - $archSuffix = switch ($targetArch) { + $archSuffix = switch (Get-TauriDiagArch) { "x86_64" { "-amd64" } "arm64" { "-arm64" } "x86" { "" } @@ -1330,28 +1231,6 @@ exit 0 return (Find-CompatiblePython) } - # ── Windows on ARM: get an x64 CPython ── - # --architecture x64 forces winget off the ARM64 build; python.org takes the same override. - function Install-X64Python { - if ($script:WingetAvailable) { - $prevEAP = $ErrorActionPreference - $ErrorActionPreference = "Continue" - try { - winget install -e --id "Python.Python.$PythonVersion" --source winget --architecture x64 --accept-package-agreements --accept-source-agreements - } catch { } - $ErrorActionPreference = $prevEAP - Refresh-SessionPath - $found = Find-CompatiblePython - if ($found -and $found.Arch -eq "x86_64") { return $found } - substep "winget could not provide an x64 Python -- trying python.org..." "Yellow" - } - $found = Install-PythonFromPythonOrg -Arch "x86_64" - if ($found -and $found.Arch -eq "x86_64") { return $found } - # Nothing installable (offline / no winget): an x64 build of another supported minor - # still runs the wheels ARM64 cannot, so take it over the native interpreter. - return (Find-CompatiblePython -X64Only) - } - # ── Install Python if no compatible version (3.11-3.13) found ── # Find-CompatiblePython returns @{ Version = "3.13"; Path = "C:\...\python.exe" } or $null. Write-TauriLog "STEP" "Installing Python" @@ -1423,26 +1302,6 @@ exit 0 return (Exit-InstallFailure "Python installation failed") } } - # ── Windows on ARM: swap a native ARM64 interpreter for x64 ── - # pyarrow and hf-transfer publish no win_arm64 wheel, so an ARM64 Python source-builds - # both and fails deep into the run. Warn up front if x64 is unobtainable. - if ($DetectedPython -and (Get-HostMachineArch) -eq "arm64" -and $DetectedPython.Arch -ne "x86_64") { - substep "windows on arm: only a native ARM64 Python $($DetectedPython.Version) was found." "Yellow" - substep "pyarrow and hf-transfer publish no win_arm64 wheels, so installing x64 Python..." "Yellow" - $X64Python = Install-X64Python - if ($X64Python) { - $DetectedPython = $X64Python - step "python" "using x64 Python $($DetectedPython.Version) under emulation" - } else { - Write-Host "[WARN] Could not install an x64 Python on this ARM64 machine." -ForegroundColor Yellow - Write-Host " Continuing with ARM64 Python $($DetectedPython.Version), but the install is likely to fail:" -ForegroundColor Yellow - Write-Host " pyarrow (via datasets) and hf-transfer ship no win_arm64 wheels and will be" -ForegroundColor Yellow - Write-Host " built from source, which needs CMake plus the MSVC and Rust toolchains." -ForegroundColor Yellow - Write-Host " Fix: install x64 Python from https://www.python.org/downloads/windows/" -ForegroundColor Yellow - Write-Host " (choose 'Windows installer (64-bit)', not ARM64), then re-run this installer." -ForegroundColor Yellow - } - } - $DiagPythonVersion = $PythonVersion if ($DetectedPython) { $DiagPythonVersion = $DetectedPython.Version } $InitialGpuBranch = "unknown" @@ -1744,7 +1603,7 @@ exit 0 if (-not (Test-Path -LiteralPath $VenvPython)) { step "venv" "creating Python $($DetectedPython.Version) virtual environment" substep "$VenvDir" - $venvExit = Invoke-InstallCommand -Label "create virtual environment" { uv venv $VenvDir --python "$($DetectedPython.Path)" } + $venvExit = Invoke-InstallCommand { uv venv $VenvDir --python "$($DetectedPython.Path)" } if ($venvExit -ne 0) { Write-Host "[ERROR] Failed to create virtual environment (exit code $venvExit)" -ForegroundColor Red return (Exit-InstallFailure "Failed to create virtual environment (exit code $venvExit)" $venvExit) @@ -2516,7 +2375,7 @@ exit 0 } if ($StudioLocalInstall) { substep "overlaying local repo (editable)..." - $overlayExit = Invoke-InstallCommand -Label "overlay local repo" { uv pip install --python $VenvPython -e $RepoRoot --no-deps } + $overlayExit = Invoke-InstallCommand { uv pip install --python $VenvPython -e $RepoRoot --no-deps } if ($overlayExit -ne 0) { Write-Host "[ERROR] Failed to overlay local repo (exit code $overlayExit)" -ForegroundColor Red return (Exit-InstallFailure "Failed to overlay local repo (exit code $overlayExit)" $overlayExit) @@ -2563,13 +2422,6 @@ exit 0 } } else { Write-TauriLog "STEP" "Installing PyTorch" - # Windows on ARM lacks only torchaudio (whl/cpu win_arm64: torch 42, - # torchvision 60, torchaudio 0), so drop that pin instead of aborting. Ask the - # interpreter, not PROCESSOR_ARCHITECTURE; reached when no x64 Python exists. - $VenvPlatform = "" - try { - $VenvPlatform = (& $VenvPython -c "import sysconfig; print(sysconfig.get_platform())" 2>$null | Out-String).Trim().ToLowerInvariant() - } catch { $VenvPlatform = "" } substep "installing PyTorch ($(Remove-IndexUrlCredentials $TorchIndexUrl))..." # Bound the companions to the capped torch on EVERY index, cu # families included: torchaudio 2.11 dropped its exact torch pin from @@ -2577,13 +2429,7 @@ exit 0 # resolve a mismatched 2.11.0 build. Mirrors install.sh. $_pinVisionSpec = "torchvision>=0.19,<0.26.0" $_pinAudioSpec = "torchaudio>=2.4,<2.11.0" - $_torchSpecs = @("torch>=2.4,<2.11.0", $_pinVisionSpec, $_pinAudioSpec) - if ($VenvPlatform -eq "win-arm64") { - substep "windows on arm: skipping torchaudio (upstream publishes no" - substep "win_arm64 wheel); torch and torchvision install normally." - $_torchSpecs = @("torch>=2.4,<2.11.0", $_pinVisionSpec) - } - $torchInstallExit = Invoke-InstallCommandRetry -Label "install PyTorch" { uv pip install --python $VenvPython @_torchSpecs --default-index $TorchIndexUrl } + $torchInstallExit = Invoke-InstallCommandRetry -Label "install PyTorch" { uv pip install --python $VenvPython "torch>=2.4,<2.11.0" $_pinVisionSpec $_pinAudioSpec --default-index $TorchIndexUrl } if ($torchInstallExit -ne 0) { Write-Host "[ERROR] Failed to install PyTorch (exit code $torchInstallExit)" -ForegroundColor Red return (Exit-InstallFailure "Failed to install PyTorch (exit code $torchInstallExit)" $torchInstallExit) @@ -2618,7 +2464,7 @@ exit 0 if ($StudioLocalInstall) { substep "overlaying local repo (editable)..." - $overlayExit = Invoke-InstallCommand -Label "overlay local repo" { uv pip install --python $VenvPython -e $RepoRoot --no-deps } + $overlayExit = Invoke-InstallCommand { uv pip install --python $VenvPython -e $RepoRoot --no-deps } if ($overlayExit -ne 0) { Write-Host "[ERROR] Failed to overlay local repo (exit code $overlayExit)" -ForegroundColor Red return (Exit-InstallFailure "Failed to overlay local repo (exit code $overlayExit)" $overlayExit) @@ -2641,7 +2487,7 @@ exit 0 return (Exit-InstallFailure "Failed to install unsloth (exit code $baseInstallExit)" $baseInstallExit) } substep "overlaying local repo (editable)..." - $overlayExit = Invoke-InstallCommand -Label "overlay local repo" { uv pip install --python $VenvPython -e $RepoRoot --no-deps } + $overlayExit = Invoke-InstallCommand { uv pip install --python $VenvPython -e $RepoRoot --no-deps } if ($overlayExit -ne 0) { Write-Host "[ERROR] Failed to overlay local repo (exit code $overlayExit)" -ForegroundColor Red return (Exit-InstallFailure "Failed to overlay local repo (exit code $overlayExit)" $overlayExit) @@ -2689,7 +2535,7 @@ exit 0 $visionSpec = if ($PinnedRocmVisionSpec) { $PinnedRocmVisionSpec } elseif ($ROCmGfxArch -and $torchvisionFloorMap -and $torchvisionFloorMap.ContainsKey($ROCmGfxArch)) { $torchvisionFloorMap[$ROCmGfxArch] } else { "torchvision" } $audioSpec = if ($PinnedRocmAudioSpec) { $PinnedRocmAudioSpec } elseif ($ROCmGfxArch -and $torchaudioFloorMap -and $torchaudioFloorMap.ContainsKey($ROCmGfxArch)) { $torchaudioFloorMap[$ROCmGfxArch] } else { "torchaudio" } substep "PyTorch flavor mismatch (installed $installedTorchTag, need ROCm) -- reinstalling correct build..." "Yellow" - $torchFixExit = Invoke-InstallCommand -Label "reinstall PyTorch (ROCm)" { uv pip install --python $VenvPython --force-reinstall --default-index $ROCmIndexUrl $rocmSpec $visionSpec $audioSpec } + $torchFixExit = Invoke-InstallCommand { uv pip install --python $VenvPython --force-reinstall --default-index $ROCmIndexUrl $rocmSpec $visionSpec $audioSpec } if ($torchFixExit -ne 0) { Write-Host "[ERROR] Failed to reinstall PyTorch with the correct ROCm build (exit code $torchFixExit)" -ForegroundColor Red return (Exit-InstallFailure "Failed to reinstall PyTorch (ROCm) (exit code $torchFixExit)" $torchFixExit) @@ -2698,7 +2544,7 @@ exit 0 } elseif ($expectedTorchTag -ne 'rocm') { # CUDA: stale +cpu (or wrong cuXXX) against a CUDA index -> reinstall triplet. substep "PyTorch flavor mismatch (installed $installedTorchTag, need $expectedTorchTag) -- reinstalling correct build..." "Yellow" - $torchFixExit = Invoke-InstallCommand -Label "reinstall PyTorch ($expectedTorchTag)" { uv pip install --python $VenvPython "torch>=2.4,<2.11.0" "torchvision>=0.19,<0.26.0" "torchaudio>=2.4,<2.11.0" --default-index $TorchIndexUrl --reinstall-package torch --reinstall-package torchvision --reinstall-package torchaudio } + $torchFixExit = Invoke-InstallCommand { uv pip install --python $VenvPython "torch>=2.4,<2.11.0" "torchvision>=0.19,<0.26.0" "torchaudio>=2.4,<2.11.0" --default-index $TorchIndexUrl --reinstall-package torch --reinstall-package torchvision --reinstall-package torchaudio } if ($torchFixExit -ne 0) { Write-Host "[ERROR] Failed to reinstall PyTorch with the correct CUDA build (exit code $torchFixExit)" -ForegroundColor Red return (Exit-InstallFailure "Failed to reinstall PyTorch ($expectedTorchTag) (exit code $torchFixExit)" $torchFixExit) @@ -2799,9 +2645,6 @@ exit 0 # an inherited value would put llama.cpp in the wrong place. $previousUnslothStudioHome = $env:UNSLOTH_STUDIO_HOME $hadPreviousUnslothStudioHome = ($null -ne $previousUnslothStudioHome) - $previousTauriMode = $env:UNSLOTH_TAURI_MODE - $hadPreviousTauriMode = ($null -ne $previousTauriMode) - $env:UNSLOTH_TAURI_MODE = if ($TauriMode) { "1" } else { "0" } if ($StudioRedirectMode -eq 'env') { $env:UNSLOTH_STUDIO_HOME = $StudioHome } else { @@ -2831,22 +2674,14 @@ exit 0 } else { Remove-Item Env:UNSLOTH_STUDIO_HOME -ErrorAction SilentlyContinue } - if ($hadPreviousTauriMode) { - $env:UNSLOTH_TAURI_MODE = $previousTauriMode - } else { - Remove-Item Env:UNSLOTH_TAURI_MODE -ErrorAction SilentlyContinue - } Remove-Item Env:UNSLOTH_LOCAL_LLAMA_CPP_DIR -ErrorAction SilentlyContinue Remove-Item Env:UNSLOTH_INSTALL_ROLLBACK_MANAGED -ErrorAction SilentlyContinue Remove-Item Env:UNSLOTH_SETUP_PYTHON -ErrorAction SilentlyContinue } if ($setupExit -ne 0) { - if (-not $TauriMode) { - Write-Host "[ERROR] unsloth studio setup failed (exit code $setupExit)" -ForegroundColor Red - } + Write-Host "[ERROR] unsloth studio setup failed (exit code $setupExit)" -ForegroundColor Red return (Exit-InstallFailure "unsloth studio setup failed (exit code $setupExit)" $setupExit) } - Clear-TauriInstallError "studio setup completed" # ── Expose `unsloth` via a shim dir containing only unsloth.exe ── # We do NOT add the venv Scripts dir to PATH (it also holds python.exe diff --git a/install.sh b/install.sh index 166beeb52c..146a64e692 100755 --- a/install.sh +++ b/install.sh @@ -19,17 +19,6 @@ # SPDX-License-Identifier: AGPL-3.0-only # Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 set -e -# ── Why the installer lives in a function ── -# Under `curl ... | sh`, sh is the pipe READER. This file is ~150KB, so a top-level -# `exit` left most of it unread, the write end failed, and curl tacked -# "(56) Failure writing output to destination" onto our own error message. Wrapping -# the body forces sh to parse to the closing brace first, so the pipe always drains -# (install.ps1 has always had this shape). -# -# Body is deliberately NOT reindented: reflowing 4000+ lines would bury the change, -# and `exit` still exits the shell from inside a function. Do not add -# `exec < /dev/null`: for a piped shell that closes the script's own source. -_unsloth_main() { # ── Output style (aligned with studio/setup.sh) ── RULE="" @@ -218,37 +207,18 @@ run_install_cmd() { # command's exit code across the pipe without relying on pipefail # (this script runs under plain sh). _rcf=$(mktemp) - tauri_stream_log stdout "OUTPUT_CLEAR" "$_label" - { - if "$@" 2>&1; then - _cmd_rc=0 - else - _cmd_rc=$? - fi - printf '%s' "$_cmd_rc" > "$_rcf" - } | _redact_install_output + { "$@" 2>&1; printf '%s' "$?" > "$_rcf"; } | _redact_install_output _rc=$(cat "$_rcf" 2>/dev/null || echo 1) rm -f "$_rcf" - _rc=${_rc:-1} - if [ "$_rc" -eq 0 ] 2>/dev/null; then - tauri_clear_install_error "$_label recovered" - return 0 - fi - tauri_stream_log stdout "ERROR_OUTPUT" "$_label failed (exit code $_rc)" + [ "${_rc:-1}" -eq 0 ] 2>/dev/null && return 0 step "error" "$_label failed (exit code $_rc)" "$C_ERR" >&2 return "$_rc" fi _log=$(mktemp) - tauri_stream_log stderr "OUTPUT_CLEAR" "$_label" - "$@" >"$_log" 2>&1 && { - rm -f "$_log" - tauri_clear_install_error "$_label recovered" - return 0 - } + "$@" >"$_log" 2>&1 && { rm -f "$_log"; return 0; } _rc=$? step "error" "$_label failed (exit code $_rc)" "$C_ERR" >&2 _redact_install_output "$_log" >&2 - tauri_stream_log stderr "ERROR_OUTPUT" "$_label failed (exit code $_rc)" rm -f "$_log" return $_rc } @@ -287,55 +257,9 @@ run_install_cmd_retry() { done } -# True when the runtime target is gfx906 (MI50/Radeon VII): the prebuilt AMD -# bitsandbytes wheel carries no gfx906 kernels, and force-reinstalling it would -# clobber a user's source-built bnb (the only 4-bit path on this arch) on every -# `studio update`. So skip the auto-install and leave whatever bnb is present. -# _gfx906_target is set during torch-index resolution; also honor an explicit -# UNSLOTH_ROCM_GFX_ARCH so a pinned-index install still skips. The override is -# normalized (gfx906:sramecc-:xnack- -> gfx906) so a copied HIP gcnArchName counts. -_is_gfx906_bnb_skip() { - [ "${_gfx906_target:-false}" = true ] && return 0 - _bnb_gfx_env=$(printf '%s' "${UNSLOTH_ROCM_GFX_ARCH:-}" | tr '[:upper:]' '[:lower:]' | tr -d '[:space:]') - _bnb_gfx_env=${_bnb_gfx_env%%:*} - [ "$_bnb_gfx_env" = "gfx906" ] && return 0 - # A pinned index (UNSLOTH_TORCH_INDEX_URL/_FAMILY) skips the reroute block that - # sets _gfx906_target, so a real gfx906 host with a pinned rocm6.3 index and no - # UNSLOTH_ROCM_GFX_ARCH would otherwise clobber a source-built bnb. Probe here - # in that gap; skip only when gfx906 is the SOLE distinct arch (mixed hosts - # opt in via the env var, mirroring the reroute block's de-dup rule). - if [ -z "$_bnb_gfx_env" ] && [ "${_torch_index_pinned:-false}" = true ]; then - _bnb_gfx_probe=$(_probe_amd_gfx_arch | awk 'NF && !seen[$0]++') - [ "$_bnb_gfx_probe" = "gfx906" ] && return 0 - fi - return 1 -} - -# `pip install unsloth` resolves its unconditional bitsandbytes dep to a generic -# CUDA wheel (no gfx906 kernels) once we skip the prebuilt one. Snapshot bnb before -# the unsloth install, then drop a freshly pulled wheel afterwards while leaving a -# pre-existing source build in place. -_gfx906_bnb_installed() { - "$_VENV_PY" -c "import importlib.util as u, sys; sys.exit(0 if u.find_spec('bitsandbytes') else 1)" >/dev/null 2>&1 -} -_gfx906_bnb_snapshot() { - _gfx906_bnb_absent_before=false - _is_gfx906_bnb_skip || return 0 - _gfx906_bnb_installed || _gfx906_bnb_absent_before=true -} -_gfx906_bnb_prune() { - _is_gfx906_bnb_skip || return 0 - [ "${_gfx906_bnb_absent_before:-false}" = true ] || return 0 - _gfx906_bnb_installed || return 0 - substep "gfx906: removing generic bitsandbytes pulled in as a dependency (no gfx906 kernels; build from source for 4-bit QLoRA)" "$C_WARN" - uv pip uninstall --python "$_VENV_PY" bitsandbytes >/dev/null 2>&1 \ - || "$_VENV_PY" -m pip uninstall -y bitsandbytes >/dev/null 2>&1 || true -} - # Install bitsandbytes on AMD ROCm hosts. bnb <= 0.49.2 NaNs at 4-bit decode -# shape on every AMD GPU; the fix (bnb #1887) ships in continuous-release_main -# and, on PyPI, first in 0.50.0. Keep this floor in step with the amd extra in -# pyproject.toml and studio/install_python_stack.py. +# shape on every AMD GPU; the fix (bnb #1887) is in the continuous-release_main +# wheel used below and first ships on PyPI in 0.50.0, hence the fallback floor. _BNB_ROCM_PYPI_FALLBACK="bitsandbytes>=0.50.0" # bitsandbytes ships no ROCm binary in its aarch64 wheel at any version: the PyPI # 0.50.0 and continuous-release_main aarch64 wheels both carry only @@ -365,8 +289,8 @@ _install_bnb_rocm() { _bnb_whl_url="" ;; esac - # uv rejects the pre-release wheel: filename version (1.33.7rc0) does not - # match metadata (0.50.x.dev0). pip accepts it, so bootstrap pip and use it. + # uv rejects the pre-release wheel: its filename version (1.33.7rc0) does not + # match its metadata version (0.50.x.dev0). pip accepts it, so bootstrap pip. if ! "$_venv_py" -m pip --version >/dev/null 2>&1; then if ! run_maybe_quiet "$_venv_py" -m ensurepip --upgrade; then run_maybe_quiet uv pip install --python "$_venv_py" pip || \ @@ -435,34 +359,6 @@ tauri_log() { fi } -tauri_stream_log() { - _tsl_stream="$1" - _tsl_tag="$2" - shift 2 - if [ "$TAURI_MODE" = true ]; then - if [ "$_tsl_stream" = stderr ]; then - printf '[TAURI:%s] %s\n' "$_tsl_tag" "$*" >&2 - else - printf '[TAURI:%s] %s\n' "$_tsl_tag" "$*" - fi - fi -} - -rollback_substep() { - if [ "$TAURI_MODE" = true ]; then - tauri_log "PROGRESS" "$1" - else - substep "$@" - fi -} - -tauri_clear_install_error() { - if [ "$TAURI_MODE" = true ]; then - tauri_log "ERROR_CLEAR" "$1" - printf '[TAURI:ERROR_CLEAR] %s\n' "$1" >&2 - fi -} - tauri_diag_marker() { _diag_gpu_branch="${1:-unknown}" _diag_torch_index_family="${2:-none}" @@ -623,10 +519,10 @@ _restore_studio_venv_replacement() { _VENV_ROLLBACK_ACTIVE=false return 0 } - rollback_substep "restoring previous environment after failed install..." "$C_WARN" + substep "restoring previous environment after failed install..." "$C_WARN" rm -rf "$_VENV_ROLLBACK_TARGET" if mv "$_VENV_ROLLBACK_DIR" "$_VENV_ROLLBACK_TARGET"; then - rollback_substep "restored previous environment" + substep "restored previous environment" _VENV_ROLLBACK_ACTIVE=false _VENV_ROLLBACK_DIR="" else @@ -811,17 +707,8 @@ _smart_apt_install() { return 0 fi - # Optional callers never elevate, in any mode: nothing on the consumer path - # builds anything, so neither the terminal sudo prompt below nor the Tauri - # NEED_SUDO dialog (whose Cancel leaves the user not installed) may gate the - # run over unused tools. The caller falls through to prebuilt llama.cpp. - # Required packages such as curl still escalate. - if [ "${_SMART_APT_OPTIONAL:-false}" = true ]; then - return 2 - fi - + # In Tauri mode, report needed packages and exit — Rust handles elevation if [ "$TAURI_MODE" = true ]; then - # Report needed packages and exit — Rust handles elevation. tauri_log "NEED_SUDO" "$_STILL_MISSING" exit 2 fi @@ -2018,142 +1905,67 @@ _maybe_reroute_strixhalo_to_2404() { _maybe_reroute_strixhalo_to_2404 || true # ── Check system dependencies ── +# cmake/git are only needed to *build* llama.cpp from source. Unsloth downloads a +# prebuilt by default, and setup.sh self-skips the source build when they're +# absent -- so macOS doesn't block on cmake (requiring it would force a manual +# Homebrew install). Linux keeps requiring them; its package manager has them. tauri_log "STEP" "Checking system dependencies" -# Without the Xcode CLT, macOS still ships /usr/bin/git as a stub that errors and pops -# a GUI dialog, so `command -v git` is not enough -- only running it tells the truth. -_has_working_git() { - command -v git >/dev/null 2>&1 || return 1 - git --version >/dev/null 2>&1 -} - -# macOS system-dependency check. A function so tests/sh can sed-extract it; the old -# inline form was untestable, which is why this gate shipped broken. -# -# The consumer install needs no developer toolchain: uv is a prebuilt binary, CPython -# is uv-managed, llama.cpp/whisper.cpp/Node are prebuilt downloads, and triton is -# skipped on macOS. Only `--local` needs git, for the unsloth-zoo git+https URL. -_check_macos_deps() { - _clt_missing=false - xcode-select -p >/dev/null 2>&1 || _clt_missing=true - - if [ "$STUDIO_LOCAL_INSTALL" = true ] && ! _has_working_git; then - echo "" - step "deps" "git is required for --local installs" "$C_ERR" - substep "--local installs unsloth-zoo from git+https://github.com/unslothai/unsloth-zoo," - substep "which needs a working git. Install the Xcode Command Line Tools:" - substep " xcode-select --install" - substep "Then re-run this script. A normal (non---local) install needs no compiler" - substep "and no git -- it uses prebuilt binaries and wheels only." - tauri_log "NEED_XCODE_CLT" "git" - return 1 - fi - - if [ "$_clt_missing" = true ]; then - # Not fatal, and no GUI dialog: firing xcode-select --install and exiting is - # what stranded clean Macs. - step "deps" "no Xcode Command Line Tools (not required)" "$C_WARN" - substep "Unsloth installs prebuilt binaries and wheels, so no compiler is needed." - substep "Install them only for a llama.cpp source build: xcode-select --install" - elif command -v cmake >/dev/null 2>&1; then - step "deps" "all system dependencies found" - else - # cmake is only for a source build, so its absence is not fatal. - step "deps" "using prebuilt llama.cpp (cmake not found)" "$C_WARN" - substep "Install cmake only if you want a source build: brew install cmake" - fi - return 0 -} - -# Linux/WSL system-dependency check. Same split as macOS, and a function for the same -# reason: tests/sh can extract it. -# -# Only a download transport is required. cmake, gcc and the libcurl headers exist -# solely for a llama.cpp source build the consumer path never does -- unslothai/ -# llama.cpp publishes linux-x64/arm64 prebuilts for cpu, cuda12, cuda13, rocm and -# vulkan. Requiring them turned every non-apt distro into a hard exit 1 over unused -# tooling. git follows macOS: --local only. -_check_linux_deps() { - _transport_missing=false - if ! command -v curl >/dev/null 2>&1 && ! command -v wget >/dev/null 2>&1; then - _transport_missing=true - fi - - # Wanted, never required: git fetches the triton_kernels git+https requirement (a - # training speedup), the rest serve the optional source build. Warn, never stop. - _optional_missing="" - command -v cmake >/dev/null 2>&1 || _optional_missing="$_optional_missing cmake" - _has_working_git || _optional_missing="$_optional_missing git" - command -v gcc >/dev/null 2>&1 || _optional_missing="$_optional_missing build-essential" - command -v curl-config >/dev/null 2>&1 || _optional_missing="$_optional_missing libcurl4-openssl-dev" - # Parameter expansion, not `sed`: sed may be absent on a minimal image, and a - # failed `$(... | sed ...)` yields "" -- "all found" on a machine that has none. - _optional_missing="${_optional_missing# }" - - if [ "$STUDIO_LOCAL_INSTALL" = true ] && ! _has_working_git; then - echo "" - step "deps" "git is required for --local installs" "$C_ERR" - substep "--local installs unsloth-zoo from git+https://github.com/unslothai/unsloth-zoo," - substep "which needs git. Install it with your package manager, then re-run." - substep "A normal (non---local) install needs no git and no compiler." - return 1 - fi - - # The one fatal case: nothing can be downloaded. apt is the only distro family we - # can drive unattended. - if [ "$_transport_missing" = true ]; then - if command -v apt-get >/dev/null 2>&1; then - echo "" - step "deps" "missing: curl" "$C_WARN" - substep "Needed to download uv, Python and the prebuilt inference engine." - _smart_apt_install curl - echo "" - else - echo "" - step "deps" "missing: curl (or wget)" "$C_ERR" - substep "Unsloth needs one of them to download uv, Python and the prebuilt" - substep "inference engine. Install one, then re-run setup:" - substep " Fedora/RHEL: sudo dnf install curl" - substep " Arch: sudo pacman -S --needed curl" - substep " openSUSE: sudo zypper install curl" - return 1 - fi - fi - - # Try apt for the optional set too; failing only costs the features warned about - # below. - if [ -n "$_optional_missing" ] && command -v apt-get >/dev/null 2>&1; then - step "deps" "installing optional build tools: $_optional_missing" "$C_DIM" - # Subshell because _smart_apt_install exits rather than returns, so `|| true` - # alone would not catch it. _SMART_APT_OPTIONAL suppresses every escalation - # path, so no install hinges on a prompt for tools nothing here needs. - ( _SMART_APT_OPTIONAL=true; _smart_apt_install $_optional_missing ) || true - _optional_missing="" - command -v cmake >/dev/null 2>&1 || _optional_missing="$_optional_missing cmake" - _has_working_git || _optional_missing="$_optional_missing git" - command -v gcc >/dev/null 2>&1 || _optional_missing="$_optional_missing build-essential" - command -v curl-config >/dev/null 2>&1 || _optional_missing="$_optional_missing libcurl4-openssl-dev" - _optional_missing="${_optional_missing# }" - fi - - if [ -n "$_optional_missing" ]; then - step "deps" "using prebuilt llama.cpp (missing: $_optional_missing)" "$C_WARN" - substep "Not required to run: Unsloth downloads a prebuilt inference engine." - case " $_optional_missing " in - *" git "*) substep "Without git the triton kernels training speedup is skipped." ;; - esac - else - step "deps" "all system dependencies found" - fi - return 0 -} - case "$OS" in macos) - _check_macos_deps || exit 1 + # Xcode Command Line Tools provide the C/C++ compiler and git. + if ! xcode-select -p >/dev/null 2>&1; then + echo "" + echo "==> Xcode Command Line Tools are required." + echo " Installing (a system dialog will appear)..." + xcode-select --install /dev/null || true + echo " After the installation completes, please re-run this script." + exit 1 + fi + # cmake is only needed for a source build; the default prebuilt path + # doesn't use it, so its absence is not fatal -- no Homebrew prerequisite. + if command -v cmake >/dev/null 2>&1; then + step "deps" "all system dependencies found" + else + step "deps" "using prebuilt llama.cpp (cmake not found)" "$C_WARN" + substep "Install cmake only if you want a source build: brew install cmake" + fi ;; linux|wsl) - _check_linux_deps || exit 1 + MISSING="" + command -v cmake >/dev/null 2>&1 || MISSING="$MISSING cmake" + command -v git >/dev/null 2>&1 || MISSING="$MISSING git" + # curl or wget is needed for downloads; check both + if ! command -v curl >/dev/null 2>&1 && ! command -v wget >/dev/null 2>&1; then + MISSING="$MISSING curl" + fi + command -v gcc >/dev/null 2>&1 || MISSING="$MISSING build-essential" + # libcurl dev headers for llama.cpp HTTPS support + command -v curl-config >/dev/null 2>&1 || MISSING="$MISSING libcurl4-openssl-dev" + + MISSING=$(echo "$MISSING" | sed 's/^ *//') + if [ -n "$MISSING" ]; then + echo "" + step "deps" "missing: $MISSING" "$C_WARN" + substep "These are needed to build the GGUF inference engine." + if command -v apt-get >/dev/null 2>&1; then + _smart_apt_install $MISSING + else + echo " Automatic system package installation is supported on apt-based" + echo " Linux distributions (Ubuntu/Debian) only. Please install the" + echo " missing dependencies with your package manager, then re-run setup:" + echo " $MISSING" + echo "" + echo " Examples:" + echo " Fedora/RHEL: sudo dnf install cmake git gcc gcc-c++ make libcurl-devel" + echo " Arch: sudo pacman -S --needed cmake git base-devel curl" + echo " openSUSE: sudo zypper install cmake git gcc gcc-c++ make libcurl-devel" + exit 1 + fi + echo "" + else + step "deps" "all system dependencies found" + fi ;; esac @@ -3505,20 +3317,10 @@ case "$_torch_index_leaf" in if (n > 0) print vals[idx] }') fi - # An explicit UNSLOTH_ROCM_GFX_ARCH=gfx906 pins the runtime target to the - # MI50 / Radeon VII path and must win over Strix probe-order detection on a - # mixed Strix + MI50 host, so the Strix reroute is suppressed when it is set. - # Normalize a copied HIP gcnArchName (gfx906:sramecc-:xnack- -> gfx906) and - # trim whitespace (mirrors the Python .strip()) so the feature-flag suffix or - # a stray newline does not defeat the exact gfx906 comparisons below. - _gfx906_env=$(printf '%s' "${UNSLOTH_ROCM_GFX_ARCH:-}" | tr '[:upper:]' '[:lower:]' | tr -d '[:space:]') - _gfx906_env=${_gfx906_env%%:*} _strix_gfx="" - if [ "$_gfx906_env" != "gfx906" ]; then - case "$_runtime_gfx" in - gfx1151|gfx1150|gfx1152) _strix_gfx="$_runtime_gfx" ;; - esac - fi + case "$_runtime_gfx" in + gfx1151|gfx1150|gfx1152) _strix_gfx="$_runtime_gfx" ;; + esac # Skip rocm7.13+ generic indexes: they already ship the fixes, so the # arch build (rocm7.13) would be a downgrade rather than a rescue. if [ -n "$_strix_gfx" ] && _rocm_leaf_below "$_torch_index_leaf" 7 13; then @@ -3546,57 +3348,6 @@ case "$_torch_index_leaf" in TORCHAUDIO_CONSTRAINT="torchaudio>=2.11.0,<2.12.0" _amd_gpu_radeon=false fi - # ── MI50 / Radeon VII (gfx906, Vega 20): legacy community-supported path ── - # Newer rocm wheel families bundle ROCm libraries whose Tensile kernels - # dropped gfx906 (rocBLAS "TensileLibrary.dat ... not read for gfx906", - # ROCm/TheRock#1844), so a rocm6.4+/7.x index installs a torch that fails - # at the first BLAS call. The rocm6.3 index is the last one whose wheels - # run on gfx906 (torch 2.7.0 verified on MI50 32GB; up to 2.9 in community - # use). Reroute any newer picked index; leave rocm6.0-6.3 alone. - # - # Target resolution: an explicit UNSLOTH_ROCM_GFX_ARCH wins (lets a host - # whose rocminfo/amd-smi emit no gfx token still opt in; _gfx906_env was - # lowercased above, before the Strix block it suppresses). Otherwise only - # treat gfx906 as the target when it is the SOLE distinct arch present: - # _gfx_all is de-duplicated by visible index, which loses per-device - # ordinals on a mixed host, so a non-gfx906 selection must never be - # downgraded to rocm6.3 -- such hosts set UNSLOTH_ROCM_GFX_ARCH to opt in. - _gfx906_target=false - if [ -n "$_gfx906_env" ]; then - [ "$_gfx906_env" = "gfx906" ] && _gfx906_target=true - elif [ -n "$_gfx_all" ]; then - _gfx906_uniq=$(printf '%s\n' "$_gfx_all" | awk 'NF && !seen[$0]++') - [ "$_gfx906_uniq" = "gfx906" ] && _gfx906_target=true - fi - # gfx906 always trains from the PyTorch rocm6.3 wheels, never the Radeon repo - # (repo.radeon.com wheels carry no gfx906 BLAS kernels). Clear the Radeon - # marketing-name flag as soon as gfx906 is the target -- even when the host - # already picks rocm6.0-6.3 and the reroute below is a no-op -- so a Radeon VII - # does not divert to the radeon branch on those versions. - if [ "$_gfx906_target" = true ]; then - _amd_gpu_radeon=false - fi - if [ "$_gfx906_target" = true ] && ! _rocm_leaf_below "$_torch_index_leaf" 6 4; then - echo "" >&2 - echo " [WARN] gfx906 (MI50 / Radeon VII / Vega 20) detected -- routing torch to the" >&2 - echo " [WARN] rocm6.3 index: it is the last wheel family that runs on gfx906 (newer" >&2 - echo " [WARN] rocm wheels ship without gfx906 BLAS kernels and fail at first use)." >&2 - echo " [WARN] gfx906 is a community-maintained legacy path: 16-bit LoRA and full" >&2 - echo " [WARN] finetuning work out of the box; bitsandbytes 4-bit QLoRA requires a" >&2 - echo " [WARN] source build of bitsandbytes for gfx906 (see docs.unsloth.ai/amd)." >&2 - echo "" >&2 - _amd_gfx906_base="${UNSLOTH_PYTORCH_MIRROR:-https://download.pytorch.org/whl}" - while [ "${_amd_gfx906_base%/}" != "$_amd_gfx906_base" ]; do - _amd_gfx906_base="${_amd_gfx906_base%/}" - done - TORCH_INDEX_URL="${_amd_gfx906_base}/rocm6.3" - # Reset to the default (<2.11) window: a rocm7.2 pick raised the floor - # to 2.11 above, which the rocm6.3 index (torch <= 2.9.x) cannot satisfy. - TORCH_CONSTRAINT="torch>=2.4,<2.11.0" - TORCHVISION_CONSTRAINT="torchvision>=0.19,<0.26.0" - TORCHAUDIO_CONSTRAINT="torchaudio>=2.4,<2.11.0" - # (_amd_gpu_radeon already cleared above for every gfx906 target.) - fi ;; esac fi # _torch_index_pinned guard (Radeon + Strix reroute) @@ -3823,7 +3574,6 @@ for _p in ('torch', 'torchvision', 'torchaudio'): if [ "$_MIGRATED" = true ]; then # Migrated env: force-reinstall unsloth+unsloth-zoo for a clean state, preserving # existing torch/CUDA unless the ROCm repair below fires. - _gfx906_bnb_snapshot substep "upgrading unsloth in migrated environment..." if [ "$SKIP_TORCH" = true ]; then # No-torch: install unsloth + unsloth-zoo with --no-deps (current @@ -3865,18 +3615,13 @@ if [ "$_MIGRATED" = true ]; then # existing ROCm installs gain the AMD bitsandbytes build without a # fresh reinstall. if [ "$SKIP_TORCH" = false ] && [ "$_torch_index_is_rocm_family" = true ]; then - if _is_gfx906_bnb_skip; then - substep "gfx906: skipping prebuilt bitsandbytes (no gfx906 kernels); build from source for 4-bit QLoRA -- https://docs.unsloth.ai/get-started/install-and-update/amd" "$C_WARN" - else - _install_bnb_rocm "install bitsandbytes (AMD)" "$_VENV_PY" - fi + _install_bnb_rocm "install bitsandbytes (AMD)" "$_VENV_PY" # Repair ROCm torch if overwritten during migrated install _has_hip=$("$_VENV_PY" -c "import torch; print(getattr(torch.version,'hip','') or '')" 2>/dev/null || true) if [ -z "$_has_hip" ]; then substep "repairing ROCm torch (overwritten by dependency resolution)..." _install_torch_default_index --force-reinstall fi - _gfx906_bnb_prune fi elif [ -n "$TORCH_INDEX_URL" ]; then # Fresh: Step 1 - install torch from explicit index (skip when --no-torch or Intel Mac) @@ -4067,13 +3812,8 @@ elif [ -n "$TORCH_INDEX_URL" ]; then # host stays in GGUF-only mode rather than pulling in bitsandbytes, # which is only useful once torch is present for training. if [ "$SKIP_TORCH" = false ] && [ "$_torch_index_is_rocm_family" = true ]; then - if _is_gfx906_bnb_skip; then - substep "gfx906: skipping prebuilt bitsandbytes (no gfx906 kernels); build from source for 4-bit QLoRA -- https://docs.unsloth.ai/get-started/install-and-update/amd" "$C_WARN" - else - _install_bnb_rocm "install bitsandbytes (AMD)" "$_VENV_PY" - fi + _install_bnb_rocm "install bitsandbytes (AMD)" "$_VENV_PY" fi - _gfx906_bnb_snapshot # Fresh: Step 2 - install unsloth, preserving the torch Step 1 installed tauri_log "STEP" "Installing Unsloth" substep "installing unsloth (this may take a few minutes)..." @@ -4124,7 +3864,6 @@ elif [ -n "$TORCH_INDEX_URL" ]; then substep "repairing ROCm torch (overwritten by dependency resolution)..." _install_torch_default_index --force-reinstall fi - _gfx906_bnb_prune fi else # Fallback: GPU detection failed to produce a URL -- let uv resolve torch @@ -4219,7 +3958,6 @@ if [ -n "$VENV_ABS_BIN" ]; then fi if ! command -v bash >/dev/null 2>&1; then - tauri_log "ERROR" "bash is required to run studio setup" step "setup" "bash is required to run studio setup" "$C_ERR" substep "Please install bash and re-run install.sh" exit 1 @@ -4258,7 +3996,6 @@ if [ "$STUDIO_LOCAL_INSTALL" = true ]; then STUDIO_LOCAL_REPO="$_REPO_ROOT" \ UNSLOTH_NO_TORCH="$SKIP_TORCH" \ UNSLOTH_LOCAL_LLAMA_CPP_DIR="$_WITH_LLAMA_CPP_DIR" \ - UNSLOTH_TAURI_MODE="$TAURI_MODE" \ bash "$SETUP_SH" =2026.7.6", + "wheel>=0.42.0", + "packaging", + "torch>=2.4.0,<2.12.0", + "torchvision", + "numpy", + "tqdm", + "psutil", + "tyro", + "protobuf", + "xformers>=0.0.27.post2 ; ('linux' in sys_platform or sys_platform == 'win32') and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", + "bitsandbytes>=0.45.5,!=0.46.0,!=0.48.0", + "triton>=3.0.0 ; ('linux' in sys_platform)", + "triton-windows ; (sys_platform == 'win32') and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", + "sentencepiece>=0.2.0", + "datasets>=3.4.1,!=4.0.*,!=4.1.0,<4.4.0", + "accelerate>=0.34.1", + "peft>=0.18.0,!=0.11.0", + "huggingface_hub>=0.34.0", + "hf_transfer", + "diffusers", + "transformers>=4.51.3,!=4.52.0,!=4.52.1,!=4.52.2,!=4.52.3,!=4.53.0,!=4.54.0,!=4.55.0,!=4.55.1,!=4.57.0,!=4.57.4,!=4.57.5,!=5.0.0,!=5.1.0,<=5.5.0", + "trl>=0.18.2,!=0.19.0,<=0.24.0", "typer>=0.12.0", - "rich", "pydantic", "pyyaml", "nest-asyncio", # Every CLI command imports studio.backend.*, which reaches structlog at # module level. The rest of the server stack lives in the studio extra. "structlog>=24.1.0", - # unsloth_cli/__init__.py reaches click via commands/start.py, so every - # command needs it. typer supplied it until 0.27 dropped the dependency. + # unsloth_cli/commands/start.py imports click, and unsloth_cli/__init__.py + # imports that, so every command needs it. typer supplied it until 0.27 + # dropped the dependency, which left this satisfied only by chance. "click>=8.0", ] @@ -47,14 +70,9 @@ version = {attr = "unsloth.models._utils.__version__"} [tool.setuptools] include-package-data = true -[tool.setuptools.cmdclass] -# Snapshots CHANGELOG.md into studio/ so every build path ships it. -build_py = "_changelog_build.build_py" - [tool.setuptools.package-data] unsloth_cli = ["codex_fallback_prompt.md", "pi_subagent.ts"] studio = [ - "CHANGELOG.md", "*.sh", "*.ps1", "*.bat", @@ -79,7 +97,7 @@ include = ["unsloth*", "unsloth_cli*", "studio", "studio.backend*"] exclude = ["images*", "tests*", "*.node_modules", "*.node_modules.*"] [project.optional-dependencies] -# Studio's server stack, mirroring studio/backend/requirements/studio.txt. +# Studio's server stack. Mirrors studio/backend/requirements/studio.txt; # test_studio_extra_matches_requirements.py catches drift. studio = [ "typer", @@ -110,11 +128,11 @@ triton = [ "triton>=3.0.0 ; ('linux' in sys_platform)", "triton-windows ; (sys_platform == 'win32') and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", ] - -huggingfacenotorch = [ +huggingface = [ "unsloth_zoo>=2026.7.6", "wheel>=0.42.0", "packaging", + "torchvision", "numpy", "tqdm", "psutil", @@ -130,28 +148,10 @@ huggingfacenotorch = [ "transformers>=4.51.3,!=4.52.0,!=4.52.1,!=4.52.2,!=4.52.3,!=4.53.0,!=4.54.0,!=4.55.0,!=4.55.1,!=4.57.0,!=4.57.4,!=4.57.5,!=5.0.0,!=5.1.0,<=5.5.0", "trl>=0.18.2,!=0.19.0,<=0.24.0", "sentence-transformers", -] -# torchcodec backend for Gemma audio / datasets>=4 (#7225). -# Pick the audio-torch* pin matching your torch minor (see TORCH_TORCHCODEC). -# torchcodec publishes no sdist and only manylinux_2_28_x86_64, macosx_*_arm64 -# and win_amd64 wheels, so Linux aarch64, Windows ARM64 and Intel Mac have -# nothing to resolve and pip fails the whole install rather than skipping audio. -# Gate on the platforms that have a wheel, matching -# PLATFORM_LACKS_TORCHCODEC_WHEEL in studio/install_python_stack.py. -audio-torch210 = [ - "torchcodec>=0.10.0,<0.11.0 ; python_version >= '3.10' and (((sys_platform == 'linux' or sys_platform == 'win32') and (platform_machine == 'x86_64' or platform_machine == 'AMD64')) or (sys_platform == 'darwin' and platform_machine == 'arm64'))", -] -audio-torch290 = [ - "torchcodec>=0.8.0,<0.10.0 ; python_version >= '3.10' and (((sys_platform == 'linux' or sys_platform == 'win32') and (platform_machine == 'x86_64' or platform_machine == 'AMD64')) or (sys_platform == 'darwin' and platform_machine == 'arm64'))", -] -audio-torch280 = [ - "torchcodec>=0.6.0,<0.8.0 ; python_version >= '3.9' and (((sys_platform == 'linux' or sys_platform == 'win32') and (platform_machine == 'x86_64' or platform_machine == 'AMD64')) or (sys_platform == 'darwin' and platform_machine == 'arm64'))", -] -huggingface = [ - "unsloth[huggingfacenotorch]", - "unsloth_zoo>=2026.7.6", - "torchvision", - "unsloth[triton]", + "typer>=0.12.0", + "pydantic", + "pyyaml", + "nest-asyncio", ] windows = [ "unsloth[huggingface]", @@ -162,235 +162,126 @@ base = [ "unsloth[huggingface]", ] cu118only = [ - "xformers @ https://download.pytorch.org/whl/cu118/xformers-0.0.22.post7%2Bcu118-cp39-cp39-manylinux2014_x86_64.whl ; python_version=='3.9' and ('linux' in sys_platform)", - "xformers @ https://download.pytorch.org/whl/cu118/xformers-0.0.22.post7%2Bcu118-cp310-cp310-manylinux2014_x86_64.whl ; python_version=='3.10' and ('linux' in sys_platform)", - "xformers @ https://download.pytorch.org/whl/cu118/xformers-0.0.22.post7%2Bcu118-cp311-cp311-manylinux2014_x86_64.whl ; python_version=='3.11' and ('linux' in sys_platform)", + "xformers==0.0.22.post7 ; ('linux' in sys_platform or sys_platform == 'win32') and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", ] cu121only = [ - "xformers @ https://download.pytorch.org/whl/cu121/xformers-0.0.22.post7-cp39-cp39-manylinux2014_x86_64.whl ; python_version=='3.9' and ('linux' in sys_platform)", - "xformers @ https://download.pytorch.org/whl/cu121/xformers-0.0.22.post7-cp310-cp310-manylinux2014_x86_64.whl ; python_version=='3.10' and ('linux' in sys_platform)", - "xformers @ https://download.pytorch.org/whl/cu121/xformers-0.0.22.post7-cp311-cp311-manylinux2014_x86_64.whl ; python_version=='3.11' and ('linux' in sys_platform)", + "xformers==0.0.22.post7 ; ('linux' in sys_platform or sys_platform == 'win32') and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", ] cu118onlytorch211 = [ - "xformers @ https://download.pytorch.org/whl/cu118/xformers-0.0.23%2Bcu118-cp39-cp39-manylinux2014_x86_64.whl ; python_version=='3.9' and ('linux' in sys_platform)", - "xformers @ https://download.pytorch.org/whl/cu118/xformers-0.0.23%2Bcu118-cp310-cp310-manylinux2014_x86_64.whl ; python_version=='3.10' and ('linux' in sys_platform)", - "xformers @ https://download.pytorch.org/whl/cu118/xformers-0.0.23%2Bcu118-cp311-cp311-manylinux2014_x86_64.whl ; python_version=='3.11' and ('linux' in sys_platform)", + "xformers==0.0.23 ; ('linux' in sys_platform or sys_platform == 'win32') and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", ] cu121onlytorch211 = [ - "xformers @ https://download.pytorch.org/whl/cu121/xformers-0.0.23-cp39-cp39-manylinux2014_x86_64.whl ; python_version=='3.9' and ('linux' in sys_platform)", - "xformers @ https://download.pytorch.org/whl/cu121/xformers-0.0.23-cp310-cp310-manylinux2014_x86_64.whl ; python_version=='3.10' and ('linux' in sys_platform)", - "xformers @ https://download.pytorch.org/whl/cu121/xformers-0.0.23-cp311-cp311-manylinux2014_x86_64.whl ; python_version=='3.11' and ('linux' in sys_platform)", + "xformers==0.0.23 ; ('linux' in sys_platform or sys_platform == 'win32') and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", ] cu118onlytorch212 = [ - "xformers @ https://download.pytorch.org/whl/cu118/xformers-0.0.23.post1%2Bcu118-cp39-cp39-manylinux2014_x86_64.whl ; python_version=='3.9' and ('linux' in sys_platform)", - "xformers @ https://download.pytorch.org/whl/cu118/xformers-0.0.23.post1%2Bcu118-cp310-cp310-manylinux2014_x86_64.whl ; python_version=='3.10' and ('linux' in sys_platform)", - "xformers @ https://download.pytorch.org/whl/cu118/xformers-0.0.23.post1%2Bcu118-cp311-cp311-manylinux2014_x86_64.whl ; python_version=='3.11' and ('linux' in sys_platform)", + "xformers==0.0.23.post1 ; ('linux' in sys_platform or sys_platform == 'win32') and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", ] cu121onlytorch212 = [ - "xformers @ https://download.pytorch.org/whl/cu121/xformers-0.0.23.post1-cp39-cp39-manylinux2014_x86_64.whl ; python_version=='3.9' and ('linux' in sys_platform)", - "xformers @ https://download.pytorch.org/whl/cu121/xformers-0.0.23.post1-cp310-cp310-manylinux2014_x86_64.whl ; python_version=='3.10' and ('linux' in sys_platform)", - "xformers @ https://download.pytorch.org/whl/cu121/xformers-0.0.23.post1-cp311-cp311-manylinux2014_x86_64.whl ; python_version=='3.11' and ('linux' in sys_platform)", + "xformers==0.0.23.post1 ; ('linux' in sys_platform or sys_platform == 'win32') and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", ] cu118onlytorch220 = [ - "xformers @ https://download.pytorch.org/whl/cu118/xformers-0.0.24%2Bcu118-cp39-cp39-manylinux2014_x86_64.whl ; python_version=='3.9' and ('linux' in sys_platform)", - "xformers @ https://download.pytorch.org/whl/cu118/xformers-0.0.24%2Bcu118-cp310-cp310-manylinux2014_x86_64.whl ; python_version=='3.10' and ('linux' in sys_platform)", - "xformers @ https://download.pytorch.org/whl/cu118/xformers-0.0.24%2Bcu118-cp311-cp311-manylinux2014_x86_64.whl ; python_version=='3.11' and ('linux' in sys_platform)", + "xformers==0.0.24 ; ('linux' in sys_platform or sys_platform == 'win32') and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", ] cu121onlytorch220 = [ - "xformers @ https://download.pytorch.org/whl/cu121/xformers-0.0.24-cp39-cp39-manylinux2014_x86_64.whl ; python_version=='3.9' and ('linux' in sys_platform)", - "xformers @ https://download.pytorch.org/whl/cu121/xformers-0.0.24-cp310-cp310-manylinux2014_x86_64.whl ; python_version=='3.10' and ('linux' in sys_platform)", - "xformers @ https://download.pytorch.org/whl/cu121/xformers-0.0.24-cp311-cp311-manylinux2014_x86_64.whl ; python_version=='3.11' and ('linux' in sys_platform)", + "xformers==0.0.24 ; ('linux' in sys_platform or sys_platform == 'win32') and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", ] cu118onlytorch230 = [ - "xformers @ https://download.pytorch.org/whl/cu118/xformers-0.0.27%2Bcu118-cp39-cp39-manylinux2014_x86_64.whl ; python_version=='3.9' and ('linux' in sys_platform)", - "xformers @ https://download.pytorch.org/whl/cu118/xformers-0.0.27%2Bcu118-cp310-cp310-manylinux2014_x86_64.whl ; python_version=='3.10' and ('linux' in sys_platform)", - "xformers @ https://download.pytorch.org/whl/cu118/xformers-0.0.27%2Bcu118-cp311-cp311-manylinux2014_x86_64.whl ; python_version=='3.11' and ('linux' in sys_platform)", - "xformers @ https://download.pytorch.org/whl/cu118/xformers-0.0.27%2Bcu118-cp312-cp312-manylinux2014_x86_64.whl ; python_version=='3.12' and ('linux' in sys_platform)", + "xformers==0.0.27 ; ('linux' in sys_platform or sys_platform == 'win32') and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", ] cu121onlytorch230 = [ - "xformers @ https://download.pytorch.org/whl/cu121/xformers-0.0.27-cp39-cp39-manylinux2014_x86_64.whl ; python_version=='3.9' and ('linux' in sys_platform)", - "xformers @ https://download.pytorch.org/whl/cu121/xformers-0.0.27-cp310-cp310-manylinux2014_x86_64.whl ; python_version=='3.10' and ('linux' in sys_platform)", - "xformers @ https://download.pytorch.org/whl/cu121/xformers-0.0.27-cp311-cp311-manylinux2014_x86_64.whl ; python_version=='3.11' and ('linux' in sys_platform)", - "xformers @ https://download.pytorch.org/whl/cu121/xformers-0.0.27-cp312-cp312-manylinux2014_x86_64.whl ; python_version=='3.12' and ('linux' in sys_platform)", + "xformers==0.0.27 ; ('linux' in sys_platform or sys_platform == 'win32') and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", ] cu118onlytorch240 = [ - "xformers @ https://download.pytorch.org/whl/cu118/xformers-0.0.27.post2%2Bcu118-cp39-cp39-manylinux2014_x86_64.whl ; python_version=='3.9' and ('linux' in sys_platform)", - "xformers @ https://download.pytorch.org/whl/cu118/xformers-0.0.27.post2%2Bcu118-cp310-cp310-manylinux2014_x86_64.whl ; python_version=='3.10' and ('linux' in sys_platform)", - "xformers @ https://download.pytorch.org/whl/cu118/xformers-0.0.27.post2%2Bcu118-cp311-cp311-manylinux2014_x86_64.whl ; python_version=='3.11' and ('linux' in sys_platform)", - "xformers @ https://download.pytorch.org/whl/cu118/xformers-0.0.27.post2%2Bcu118-cp312-cp312-manylinux2014_x86_64.whl ; python_version=='3.12' and ('linux' in sys_platform)", + "xformers==0.0.27.post2 ; ('linux' in sys_platform or sys_platform == 'win32') and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", ] cu121onlytorch240 = [ - "xformers @ https://download.pytorch.org/whl/cu121/xformers-0.0.28.post1-cp39-cp39-manylinux_2_28_x86_64.whl ; python_version=='3.9' and ('linux' in sys_platform)", - "xformers @ https://download.pytorch.org/whl/cu121/xformers-0.0.28.post1-cp310-cp310-manylinux_2_28_x86_64.whl ; python_version=='3.10' and ('linux' in sys_platform)", - "xformers @ https://download.pytorch.org/whl/cu121/xformers-0.0.28.post1-cp311-cp311-manylinux_2_28_x86_64.whl ; python_version=='3.11' and ('linux' in sys_platform)", - "xformers @ https://download.pytorch.org/whl/cu121/xformers-0.0.28.post1-cp312-cp312-manylinux_2_28_x86_64.whl ; python_version=='3.12' and ('linux' in sys_platform)", + "xformers==0.0.27.post2 ; ('linux' in sys_platform or sys_platform == 'win32') and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", ] cu124onlytorch240 = [ - "xformers @ https://download.pytorch.org/whl/cu124/xformers-0.0.28.post1-cp39-cp39-manylinux_2_28_x86_64.whl ; python_version=='3.9' and ('linux' in sys_platform)", - "xformers @ https://download.pytorch.org/whl/cu124/xformers-0.0.28.post1-cp310-cp310-manylinux_2_28_x86_64.whl ; python_version=='3.10' and ('linux' in sys_platform)", - "xformers @ https://download.pytorch.org/whl/cu124/xformers-0.0.28.post1-cp311-cp311-manylinux_2_28_x86_64.whl ; python_version=='3.11' and ('linux' in sys_platform)", - "xformers @ https://download.pytorch.org/whl/cu124/xformers-0.0.28.post1-cp312-cp312-manylinux_2_28_x86_64.whl ; python_version=='3.12' and ('linux' in sys_platform)", - "xformers @ https://download.pytorch.org/whl/cu124/xformers-0.0.28.post1-cp39-cp39-win_amd64.whl ; python_version=='3.9' and (sys_platform == 'win32')", - "xformers @ https://download.pytorch.org/whl/cu124/xformers-0.0.28.post1-cp310-cp310-win_amd64.whl ; python_version=='3.10' and (sys_platform == 'win32')", - "xformers @ https://download.pytorch.org/whl/cu124/xformers-0.0.28.post1-cp311-cp311-win_amd64.whl ; python_version=='3.11' and (sys_platform == 'win32')", - "xformers @ https://download.pytorch.org/whl/cu124/xformers-0.0.28.post1-cp312-cp312-win_amd64.whl ; python_version=='3.12' and (sys_platform == 'win32')", + "xformers==0.0.28.post1 ; ('linux' in sys_platform or sys_platform == 'win32') and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", ] cu118onlytorch250 = [ - "xformers @ https://download.pytorch.org/whl/cu118/xformers-0.0.28.post2-cp39-cp39-manylinux_2_28_x86_64.whl ; python_version=='3.9' and ('linux' in sys_platform)", - "xformers @ https://download.pytorch.org/whl/cu118/xformers-0.0.28.post2-cp310-cp310-manylinux_2_28_x86_64.whl ; python_version=='3.10' and ('linux' in sys_platform)", - "xformers @ https://download.pytorch.org/whl/cu118/xformers-0.0.28.post2-cp311-cp311-manylinux_2_28_x86_64.whl ; python_version=='3.11' and ('linux' in sys_platform)", - "xformers @ https://download.pytorch.org/whl/cu118/xformers-0.0.28.post2-cp312-cp312-manylinux_2_28_x86_64.whl ; python_version=='3.12' and ('linux' in sys_platform)", + "xformers==0.0.28.post2 ; ('linux' in sys_platform or sys_platform == 'win32') and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", ] cu121onlytorch250 = [ - "xformers @ https://download.pytorch.org/whl/cu121/xformers-0.0.28.post2-cp39-cp39-manylinux_2_28_x86_64.whl ; python_version=='3.9' and ('linux' in sys_platform)", - "xformers @ https://download.pytorch.org/whl/cu121/xformers-0.0.28.post2-cp310-cp310-manylinux_2_28_x86_64.whl ; python_version=='3.10' and ('linux' in sys_platform)", - "xformers @ https://download.pytorch.org/whl/cu121/xformers-0.0.28.post2-cp311-cp311-manylinux_2_28_x86_64.whl ; python_version=='3.11' and ('linux' in sys_platform)", - "xformers @ https://download.pytorch.org/whl/cu121/xformers-0.0.28.post2-cp312-cp312-manylinux_2_28_x86_64.whl ; python_version=='3.12' and ('linux' in sys_platform)", + "xformers==0.0.28.post2 ; ('linux' in sys_platform or sys_platform == 'win32') and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", ] cu124onlytorch250 = [ - "xformers @ https://download.pytorch.org/whl/cu124/xformers-0.0.28.post2-cp39-cp39-manylinux_2_28_x86_64.whl ; python_version=='3.9' and ('linux' in sys_platform)", - "xformers @ https://download.pytorch.org/whl/cu124/xformers-0.0.28.post2-cp310-cp310-manylinux_2_28_x86_64.whl ; python_version=='3.10' and ('linux' in sys_platform)", - "xformers @ https://download.pytorch.org/whl/cu124/xformers-0.0.28.post2-cp311-cp311-manylinux_2_28_x86_64.whl ; python_version=='3.11' and ('linux' in sys_platform)", - "xformers @ https://download.pytorch.org/whl/cu124/xformers-0.0.28.post2-cp312-cp312-manylinux_2_28_x86_64.whl ; python_version=='3.12' and ('linux' in sys_platform)", - "xformers @ https://download.pytorch.org/whl/cu124/xformers-0.0.28.post2-cp39-cp39-win_amd64.whl ; python_version=='3.9' and (sys_platform == 'win32')", - "xformers @ https://download.pytorch.org/whl/cu124/xformers-0.0.28.post2-cp310-cp310-win_amd64.whl ; python_version=='3.10' and (sys_platform == 'win32')", - "xformers @ https://download.pytorch.org/whl/cu124/xformers-0.0.28.post2-cp311-cp311-win_amd64.whl ; python_version=='3.11' and (sys_platform == 'win32')", - "xformers @ https://download.pytorch.org/whl/cu124/xformers-0.0.28.post2-cp312-cp312-win_amd64.whl ; python_version=='3.12' and (sys_platform == 'win32')", + "xformers==0.0.28.post2 ; ('linux' in sys_platform or sys_platform == 'win32') and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", ] cu118onlytorch251 = [ - "xformers @ https://download.pytorch.org/whl/cu118/xformers-0.0.29.post1-cp39-cp39-manylinux_2_28_x86_64.whl ; python_version=='3.9' and ('linux' in sys_platform)", - "xformers @ https://download.pytorch.org/whl/cu118/xformers-0.0.29.post1-cp310-cp310-manylinux_2_28_x86_64.whl ; python_version=='3.10' and ('linux' in sys_platform)", - "xformers @ https://download.pytorch.org/whl/cu118/xformers-0.0.29.post1-cp311-cp311-manylinux_2_28_x86_64.whl ; python_version=='3.11' and ('linux' in sys_platform)", - "xformers @ https://download.pytorch.org/whl/cu118/xformers-0.0.29.post1-cp312-cp312-manylinux_2_28_x86_64.whl ; python_version=='3.12' and ('linux' in sys_platform)", + "xformers==0.0.29.post1 ; ('linux' in sys_platform or sys_platform == 'win32') and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", ] cu121onlytorch251 = [ - "xformers @ https://download.pytorch.org/whl/cu121/xformers-0.0.29.post1-cp39-cp39-manylinux_2_28_x86_64.whl ; python_version=='3.9' and ('linux' in sys_platform)", - "xformers @ https://download.pytorch.org/whl/cu121/xformers-0.0.29.post1-cp310-cp310-manylinux_2_28_x86_64.whl ; python_version=='3.10' and ('linux' in sys_platform)", - "xformers @ https://download.pytorch.org/whl/cu121/xformers-0.0.29.post1-cp311-cp311-manylinux_2_28_x86_64.whl ; python_version=='3.11' and ('linux' in sys_platform)", - "xformers @ https://download.pytorch.org/whl/cu121/xformers-0.0.29.post1-cp312-cp312-manylinux_2_28_x86_64.whl ; python_version=='3.12' and ('linux' in sys_platform)", + "xformers==0.0.29.post1 ; ('linux' in sys_platform or sys_platform == 'win32') and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", ] cu124onlytorch251 = [ - "xformers @ https://download.pytorch.org/whl/cu124/xformers-0.0.29.post1-cp39-cp39-manylinux_2_28_x86_64.whl ; python_version=='3.9' and ('linux' in sys_platform)", - "xformers @ https://download.pytorch.org/whl/cu124/xformers-0.0.29.post1-cp310-cp310-manylinux_2_28_x86_64.whl ; python_version=='3.10' and ('linux' in sys_platform)", - "xformers @ https://download.pytorch.org/whl/cu124/xformers-0.0.29.post1-cp311-cp311-manylinux_2_28_x86_64.whl ; python_version=='3.11' and ('linux' in sys_platform)", - "xformers @ https://download.pytorch.org/whl/cu124/xformers-0.0.29.post1-cp312-cp312-manylinux_2_28_x86_64.whl ; python_version=='3.12' and ('linux' in sys_platform)", - "xformers @ https://download.pytorch.org/whl/cu124/xformers-0.0.29.post1-cp39-cp39-win_amd64.whl ; python_version=='3.9' and (sys_platform == 'win32')", - "xformers @ https://download.pytorch.org/whl/cu124/xformers-0.0.29.post1-cp310-cp310-win_amd64.whl ; python_version=='3.10' and (sys_platform == 'win32')", - "xformers @ https://download.pytorch.org/whl/cu124/xformers-0.0.29.post1-cp311-cp311-win_amd64.whl ; python_version=='3.11' and (sys_platform == 'win32')", - "xformers @ https://download.pytorch.org/whl/cu124/xformers-0.0.29.post1-cp312-cp312-win_amd64.whl ; python_version=='3.12' and (sys_platform == 'win32')", + "xformers==0.0.29.post1 ; ('linux' in sys_platform or sys_platform == 'win32') and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", ] cu118onlytorch260 = [ - "xformers @ https://download.pytorch.org/whl/cu118/xformers-0.0.29.post3-cp39-cp39-manylinux_2_28_x86_64.whl ; python_version=='3.9' and ('linux' in sys_platform)", - "xformers @ https://download.pytorch.org/whl/cu118/xformers-0.0.29.post3-cp310-cp310-manylinux_2_28_x86_64.whl ; python_version=='3.10' and ('linux' in sys_platform)", - "xformers @ https://download.pytorch.org/whl/cu118/xformers-0.0.29.post3-cp311-cp311-manylinux_2_28_x86_64.whl ; python_version=='3.11' and ('linux' in sys_platform)", - "xformers @ https://download.pytorch.org/whl/cu118/xformers-0.0.29.post3-cp312-cp312-manylinux_2_28_x86_64.whl ; python_version=='3.12' and ('linux' in sys_platform)", + "xformers==0.0.29.post3 ; ('linux' in sys_platform or sys_platform == 'win32') and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", ] cu124onlytorch260 = [ - "xformers @ https://download.pytorch.org/whl/cu124/xformers-0.0.29.post3-cp39-cp39-manylinux_2_28_x86_64.whl ; python_version=='3.9' and ('linux' in sys_platform)", - "xformers @ https://download.pytorch.org/whl/cu124/xformers-0.0.29.post3-cp310-cp310-manylinux_2_28_x86_64.whl ; python_version=='3.10' and ('linux' in sys_platform)", - "xformers @ https://download.pytorch.org/whl/cu124/xformers-0.0.29.post3-cp311-cp311-manylinux_2_28_x86_64.whl ; python_version=='3.11' and ('linux' in sys_platform)", - "xformers @ https://download.pytorch.org/whl/cu124/xformers-0.0.29.post3-cp312-cp312-manylinux_2_28_x86_64.whl ; python_version=='3.12' and ('linux' in sys_platform)", - "xformers @ https://download.pytorch.org/whl/cu124/xformers-0.0.29.post3-cp39-cp39-win_amd64.whl ; python_version=='3.9' and (sys_platform == 'win32')", - "xformers @ https://download.pytorch.org/whl/cu124/xformers-0.0.29.post3-cp310-cp310-win_amd64.whl ; python_version=='3.10' and (sys_platform == 'win32')", - "xformers @ https://download.pytorch.org/whl/cu124/xformers-0.0.29.post3-cp311-cp311-win_amd64.whl ; python_version=='3.11' and (sys_platform == 'win32')", - "xformers @ https://download.pytorch.org/whl/cu124/xformers-0.0.29.post3-cp312-cp312-win_amd64.whl ; python_version=='3.12' and (sys_platform == 'win32')", + "xformers==0.0.29.post3 ; ('linux' in sys_platform or sys_platform == 'win32') and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", ] cu126onlytorch260 = [ - "xformers @ https://download.pytorch.org/whl/cu126/xformers-0.0.29.post3-cp39-cp39-manylinux_2_28_x86_64.whl ; python_version=='3.9' and ('linux' in sys_platform)", - "xformers @ https://download.pytorch.org/whl/cu126/xformers-0.0.29.post3-cp310-cp310-manylinux_2_28_x86_64.whl ; python_version=='3.10' and ('linux' in sys_platform)", - "xformers @ https://download.pytorch.org/whl/cu126/xformers-0.0.29.post3-cp311-cp311-manylinux_2_28_x86_64.whl ; python_version=='3.11' and ('linux' in sys_platform)", - "xformers @ https://download.pytorch.org/whl/cu126/xformers-0.0.29.post3-cp312-cp312-manylinux_2_28_x86_64.whl ; python_version=='3.12' and ('linux' in sys_platform)", - "xformers @ https://download.pytorch.org/whl/cu126/xformers-0.0.29.post3-cp39-cp39-win_amd64.whl ; python_version=='3.9' and (sys_platform == 'win32')", - "xformers @ https://download.pytorch.org/whl/cu126/xformers-0.0.29.post3-cp310-cp310-win_amd64.whl ; python_version=='3.10' and (sys_platform == 'win32')", - "xformers @ https://download.pytorch.org/whl/cu126/xformers-0.0.29.post3-cp311-cp311-win_amd64.whl ; python_version=='3.11' and (sys_platform == 'win32')", - "xformers @ https://download.pytorch.org/whl/cu126/xformers-0.0.29.post3-cp312-cp312-win_amd64.whl ; python_version=='3.12' and (sys_platform == 'win32')", + "xformers==0.0.29.post3 ; ('linux' in sys_platform or sys_platform == 'win32') and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", ] cu118onlytorch270 = [ - "xformers @ https://download.pytorch.org/whl/cu118/xformers-0.0.30-cp39-cp39-manylinux_2_28_x86_64.whl ; python_version=='3.9' and ('linux' in sys_platform)", - "xformers @ https://download.pytorch.org/whl/cu118/xformers-0.0.30-cp310-cp310-manylinux_2_28_x86_64.whl ; python_version=='3.10' and ('linux' in sys_platform)", - "xformers @ https://download.pytorch.org/whl/cu118/xformers-0.0.30-cp311-cp311-manylinux_2_28_x86_64.whl ; python_version=='3.11' and ('linux' in sys_platform)", - "xformers @ https://download.pytorch.org/whl/cu118/xformers-0.0.30-cp312-cp312-manylinux_2_28_x86_64.whl ; python_version=='3.12' and ('linux' in sys_platform)", + "xformers==0.0.30 ; ('linux' in sys_platform or sys_platform == 'win32') and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", ] cu126onlytorch270 = [ - "xformers @ https://download.pytorch.org/whl/cu126/xformers-0.0.30-cp39-cp39-manylinux_2_28_x86_64.whl ; python_version=='3.9' and ('linux' in sys_platform)", - "xformers @ https://download.pytorch.org/whl/cu126/xformers-0.0.30-cp310-cp310-manylinux_2_28_x86_64.whl ; python_version=='3.10' and ('linux' in sys_platform)", - "xformers @ https://download.pytorch.org/whl/cu126/xformers-0.0.30-cp311-cp311-manylinux_2_28_x86_64.whl ; python_version=='3.11' and ('linux' in sys_platform)", - "xformers @ https://download.pytorch.org/whl/cu126/xformers-0.0.30-cp312-cp312-manylinux_2_28_x86_64.whl ; python_version=='3.12' and ('linux' in sys_platform)", - "xformers @ https://download.pytorch.org/whl/cu126/xformers-0.0.30-cp39-cp39-win_amd64.whl ; python_version=='3.9' and (sys_platform == 'win32')", - "xformers @ https://download.pytorch.org/whl/cu126/xformers-0.0.30-cp310-cp310-win_amd64.whl ; python_version=='3.10' and (sys_platform == 'win32')", - "xformers @ https://download.pytorch.org/whl/cu126/xformers-0.0.30-cp311-cp311-win_amd64.whl ; python_version=='3.11' and (sys_platform == 'win32')", - "xformers @ https://download.pytorch.org/whl/cu126/xformers-0.0.30-cp312-cp312-win_amd64.whl ; python_version=='3.12' and (sys_platform == 'win32')", + "xformers==0.0.30 ; ('linux' in sys_platform or sys_platform == 'win32') and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", ] cu128onlytorch270 = [ - "xformers @ https://download.pytorch.org/whl/cu128/xformers-0.0.30-cp39-cp39-manylinux_2_28_x86_64.whl ; python_version=='3.9' and ('linux' in sys_platform)", - "xformers @ https://download.pytorch.org/whl/cu128/xformers-0.0.30-cp310-cp310-manylinux_2_28_x86_64.whl ; python_version=='3.10' and ('linux' in sys_platform)", - "xformers @ https://download.pytorch.org/whl/cu128/xformers-0.0.30-cp311-cp311-manylinux_2_28_x86_64.whl ; python_version=='3.11' and ('linux' in sys_platform)", - "xformers @ https://download.pytorch.org/whl/cu128/xformers-0.0.30-cp312-cp312-manylinux_2_28_x86_64.whl ; python_version=='3.12' and ('linux' in sys_platform)", - "xformers @ https://download.pytorch.org/whl/cu128/xformers-0.0.30-cp39-cp39-win_amd64.whl ; python_version=='3.9' and (sys_platform == 'win32')", - "xformers @ https://download.pytorch.org/whl/cu128/xformers-0.0.30-cp310-cp310-win_amd64.whl ; python_version=='3.10' and (sys_platform == 'win32')", - "xformers @ https://download.pytorch.org/whl/cu128/xformers-0.0.30-cp311-cp311-win_amd64.whl ; python_version=='3.11' and (sys_platform == 'win32')", - "xformers @ https://download.pytorch.org/whl/cu128/xformers-0.0.30-cp312-cp312-win_amd64.whl ; python_version=='3.12' and (sys_platform == 'win32')", + "xformers==0.0.30 ; ('linux' in sys_platform or sys_platform == 'win32') and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", ] cu118onlytorch271 = [ - "xformers @ https://download.pytorch.org/whl/cu118/xformers-0.0.31.post1-cp39-abi3-manylinux_2_28_x86_64.whl ; ('linux' in sys_platform)", + "xformers==0.0.31.post1 ; ('linux' in sys_platform or sys_platform == 'win32') and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", ] cu126onlytorch271 = [ - "xformers @ https://download.pytorch.org/whl/cu126/xformers-0.0.31.post1-cp39-abi3-manylinux_2_28_x86_64.whl ; ('linux' in sys_platform)", - "xformers @ https://download.pytorch.org/whl/cu126/xformers-0.0.31.post1-cp39-abi3-win_amd64.whl ; (sys_platform == 'win32')", + "xformers==0.0.31.post1 ; ('linux' in sys_platform or sys_platform == 'win32') and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", ] cu128onlytorch271 = [ - "xformers @ https://download.pytorch.org/whl/cu128/xformers-0.0.31.post1-cp39-abi3-manylinux_2_28_x86_64.whl ; ('linux' in sys_platform)", - "xformers @ https://download.pytorch.org/whl/cu128/xformers-0.0.31.post1-cp39-abi3-win_amd64.whl ; (sys_platform == 'win32')", + "xformers==0.0.31.post1 ; ('linux' in sys_platform or sys_platform == 'win32') and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", ] cu118onlytorch280 = [ - "xformers @ https://download.pytorch.org/whl/cu126/xformers-0.0.32.post2-cp39-abi3-manylinux_2_28_x86_64.whl ; ('linux' in sys_platform)", - "xformers @ https://download.pytorch.org/whl/cu126/xformers-0.0.32.post2-cp39-abi3-win_amd64.whl ; (sys_platform == 'win32')", + "xformers==0.0.32.post2 ; ('linux' in sys_platform or sys_platform == 'win32') and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", ] cu126onlytorch280 = [ - "xformers @ https://download.pytorch.org/whl/cu128/xformers-0.0.32.post2-cp39-abi3-manylinux_2_28_x86_64.whl ; ('linux' in sys_platform)", - "xformers @ https://download.pytorch.org/whl/cu128/xformers-0.0.32.post2-cp39-abi3-win_amd64.whl ; (sys_platform == 'win32')", + "xformers==0.0.32.post2 ; ('linux' in sys_platform or sys_platform == 'win32') and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", ] cu128onlytorch280 = [ - "xformers @ https://download.pytorch.org/whl/cu129/xformers-0.0.32.post2-cp39-abi3-manylinux_2_28_x86_64.whl ; ('linux' in sys_platform)", - "xformers @ https://download.pytorch.org/whl/cu129/xformers-0.0.32.post2-cp39-abi3-win_amd64.whl ; (sys_platform == 'win32')", + "xformers==0.0.32.post2 ; ('linux' in sys_platform or sys_platform == 'win32') and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", ] cu130onlytorch280 = [ ] cu126onlytorch290 = [ - "xformers @ https://download.pytorch.org/whl/cu126/xformers-0.0.33.post1-cp39-abi3-manylinux_2_28_x86_64.whl ; ('linux' in sys_platform)", - "xformers @ https://download.pytorch.org/whl/cu126/xformers-0.0.33.post1-cp39-abi3-win_amd64.whl ; (sys_platform == 'win32')", + "xformers==0.0.33.post1 ; ('linux' in sys_platform or sys_platform == 'win32') and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", ] cu128onlytorch290 = [ - "xformers @ https://download.pytorch.org/whl/cu128/xformers-0.0.33.post1-cp39-abi3-manylinux_2_28_x86_64.whl ; ('linux' in sys_platform)", - "xformers @ https://download.pytorch.org/whl/cu128/xformers-0.0.33.post1-cp39-abi3-win_amd64.whl ; (sys_platform == 'win32')", + "xformers==0.0.33.post1 ; ('linux' in sys_platform or sys_platform == 'win32') and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", ] cu130onlytorch290 = [ - "xformers @ https://download.pytorch.org/whl/cu130/xformers-0.0.33.post1-cp39-abi3-manylinux_2_28_x86_64.whl ; ('linux' in sys_platform)", - "xformers @ https://download.pytorch.org/whl/cu130/xformers-0.0.33.post1-cp39-abi3-win_amd64.whl ; (sys_platform == 'win32')", + "xformers==0.0.33.post1 ; ('linux' in sys_platform or sys_platform == 'win32') and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", ] cu126onlytorch291 = [ - "xformers @ https://download.pytorch.org/whl/cu126/xformers-0.0.33.post2-cp39-abi3-manylinux_2_28_x86_64.whl ; ('linux' in sys_platform)", - "xformers @ https://download.pytorch.org/whl/cu126/xformers-0.0.33.post2-cp39-abi3-win_amd64.whl ; (sys_platform == 'win32')", + "xformers==0.0.33.post2 ; ('linux' in sys_platform or sys_platform == 'win32') and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", ] cu128onlytorch291 = [ - "xformers @ https://download.pytorch.org/whl/cu128/xformers-0.0.33.post2-cp39-abi3-manylinux_2_28_x86_64.whl ; ('linux' in sys_platform)", - "xformers @ https://download.pytorch.org/whl/cu128/xformers-0.0.33.post2-cp39-abi3-win_amd64.whl ; (sys_platform == 'win32')", + "xformers==0.0.33.post2 ; ('linux' in sys_platform or sys_platform == 'win32') and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", ] cu130onlytorch291 = [ - "xformers @ https://download.pytorch.org/whl/cu130/xformers-0.0.33.post2-cp39-abi3-manylinux_2_28_x86_64.whl ; ('linux' in sys_platform)", - "xformers @ https://download.pytorch.org/whl/cu130/xformers-0.0.33.post2-cp39-abi3-win_amd64.whl ; (sys_platform == 'win32')", + "xformers==0.0.33.post2 ; ('linux' in sys_platform or sys_platform == 'win32') and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", ] cu126onlytorch2100 = [ - "xformers @ https://download.pytorch.org/whl/cu126/xformers-0.0.34-cp39-abi3-manylinux_2_28_x86_64.whl ; ('linux' in sys_platform)", - "xformers @ https://download.pytorch.org/whl/cu126/xformers-0.0.34-cp39-abi3-win_amd64.whl ; (sys_platform == 'win32')", + "xformers==0.0.34 ; ('linux' in sys_platform or sys_platform == 'win32') and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", ] cu128onlytorch2100 = [ - "xformers @ https://download.pytorch.org/whl/cu128/xformers-0.0.34-cp39-abi3-manylinux_2_28_x86_64.whl ; ('linux' in sys_platform)", - "xformers @ https://download.pytorch.org/whl/cu128/xformers-0.0.34-cp39-abi3-win_amd64.whl ; (sys_platform == 'win32')", + "xformers==0.0.34 ; ('linux' in sys_platform or sys_platform == 'win32') and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", ] cu130onlytorch2100 = [ - "xformers @ https://download.pytorch.org/whl/cu130/xformers-0.0.34-cp39-abi3-manylinux_2_28_x86_64.whl ; ('linux' in sys_platform)", - "xformers @ https://download.pytorch.org/whl/cu130/xformers-0.0.34-cp39-abi3-win_amd64.whl ; (sys_platform == 'win32')", + "xformers==0.0.34 ; ('linux' in sys_platform or sys_platform == 'win32') and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", ] cu118 = [ "unsloth[huggingface]", @@ -404,22 +295,22 @@ cu121 = [ ] cu118-torch211 = [ "unsloth[huggingface]", - "bitsandbytes==0.45.5", + "bitsandbytes>=0.45.5,!=0.46.0,!=0.48.0", "unsloth[cu118onlytorch211]", ] cu121-torch211 = [ "unsloth[huggingface]", - "bitsandbytes==0.45.5", + "bitsandbytes>=0.45.5,!=0.46.0,!=0.48.0", "unsloth[cu121onlytorch211]", ] cu118-torch212 = [ "unsloth[huggingface]", - "bitsandbytes==0.45.5", + "bitsandbytes>=0.45.5,!=0.46.0,!=0.48.0", "unsloth[cu118onlytorch212]", ] cu121-torch212 = [ "unsloth[huggingface]", - "bitsandbytes==0.45.5", + "bitsandbytes>=0.45.5,!=0.46.0,!=0.48.0", "unsloth[cu121onlytorch212]", ] cu118-torch220 = [ @@ -489,17 +380,17 @@ cu124-torch251 = [ ] cu118-torch260 = [ "unsloth[huggingface]", - "bitsandbytes>=0.45.5,!=0.46.0,!=0.48.0", + "bitsandbytes>=0.45.1", "unsloth[cu118onlytorch260]", ] cu124-torch260 = [ "unsloth[huggingface]", - "bitsandbytes>=0.45.5,!=0.46.0,!=0.48.0", + "bitsandbytes>=0.45.1", "unsloth[cu124onlytorch260]", ] cu126-torch260 = [ "unsloth[huggingface]", - "bitsandbytes>=0.45.5,!=0.46.0,!=0.48.0", + "bitsandbytes>=0.45.1", "unsloth[cu126onlytorch260]", ] cu118-torch270 = [ @@ -586,19 +477,16 @@ cu126-torch2100 = [ "unsloth[huggingface]", "bitsandbytes>=0.45.5,!=0.46.0,!=0.48.0", "unsloth[cu126onlytorch2100]", - "unsloth[audio-torch210]", ] cu128-torch2100 = [ "unsloth[huggingface]", "bitsandbytes>=0.45.5,!=0.46.0,!=0.48.0", "unsloth[cu128onlytorch2100]", - "unsloth[audio-torch210]", ] cu130-torch2100 = [ "unsloth[huggingface]", "bitsandbytes>=0.45.5,!=0.46.0,!=0.48.0", "unsloth[cu130onlytorch2100]", - "unsloth[audio-torch210]", ] kaggle = [ "unsloth[huggingface]", @@ -612,16 +500,19 @@ conda = [ ] colab-torch211 = [ "unsloth[huggingface]", - "bitsandbytes==0.45.5", + "bitsandbytes>=0.45.5,!=0.46.0,!=0.48.0", "unsloth[cu121onlytorch211]", ] +flashattention = [ + "packaging ; ('linux' in sys_platform)", + "ninja ; ('linux' in sys_platform)", + "flash-attn>=2.6.3 ; ('linux' in sys_platform)", +] colab-ampere-torch211 = [ "unsloth[huggingface]", - "bitsandbytes==0.45.5", + "bitsandbytes>=0.45.5,!=0.46.0,!=0.48.0", "unsloth[cu121onlytorch211]", - "packaging", - "ninja", - "flash-attn>=2.6.3 ; ('linux' in sys_platform)", + "unsloth[flashattention]", ] colab-torch220 = [ "unsloth[huggingface]", @@ -632,9 +523,7 @@ colab-ampere-torch220 = [ "unsloth[huggingface]", "bitsandbytes>=0.45.5,!=0.46.0,!=0.48.0", "unsloth[cu121onlytorch220]", - "packaging", - "ninja", - "flash-attn>=2.6.3 ; ('linux' in sys_platform)", + "unsloth[flashattention]", ] colab-new = [ "unsloth_zoo>=2026.7.6", @@ -653,6 +542,10 @@ colab-new = [ "bitsandbytes>=0.45.5,!=0.46.0,!=0.48.0", "unsloth[triton]", "sentence-transformers", + "typer>=0.12.0", + "pydantic", + "pyyaml", + "nest-asyncio", ] colab-no-deps = [ "accelerate>=0.34.1", @@ -665,11 +558,6 @@ colab-no-deps = [ colab = [ "unsloth[cu121]", ] -flashattention = [ - "packaging ; ('linux' in sys_platform)", - "ninja ; ('linux' in sys_platform)", - "flash-attn>=2.6.3 ; ('linux' in sys_platform)", -] colab-ampere = [ "unsloth[colab-ampere-torch220]", "unsloth[flashattention]", @@ -688,13 +576,13 @@ cu121-ampere = [ ] cu118-ampere-torch211 = [ "unsloth[huggingface]", - "bitsandbytes==0.45.5", + "bitsandbytes>=0.45.5,!=0.46.0,!=0.48.0", "unsloth[cu118onlytorch211]", "unsloth[flashattention]", ] cu121-ampere-torch211 = [ "unsloth[huggingface]", - "bitsandbytes==0.45.5", + "bitsandbytes>=0.45.5,!=0.46.0,!=0.48.0", "unsloth[cu121onlytorch211]", "unsloth[flashattention]", ] @@ -778,19 +666,19 @@ cu124-ampere-torch251 = [ ] cu118-ampere-torch260 = [ "unsloth[huggingface]", - "bitsandbytes>=0.45.5,!=0.46.0,!=0.48.0", + "bitsandbytes>=0.45.1", "unsloth[cu118onlytorch260]", "unsloth[flashattention]", ] cu124-ampere-torch260 = [ "unsloth[huggingface]", - "bitsandbytes>=0.45.5,!=0.46.0,!=0.48.0", + "bitsandbytes>=0.45.1", "unsloth[cu124onlytorch260]", "unsloth[flashattention]", ] cu126-ampere-torch260 = [ "unsloth[huggingface]", - "bitsandbytes>=0.45.5,!=0.46.0,!=0.48.0", + "bitsandbytes>=0.45.1", "unsloth[cu126onlytorch260]", "unsloth[flashattention]", ] @@ -852,7 +740,6 @@ cu130-ampere-torch280 = [ "unsloth[huggingface]", "bitsandbytes>=0.45.5,!=0.46.0,!=0.48.0", "unsloth[cu130onlytorch280]", - "unsloth[flashattention]", ] cu126-ampere-torch290 = [ "unsloth[huggingface]", @@ -888,481 +775,16 @@ cu126-ampere-torch2100 = [ "unsloth[huggingface]", "bitsandbytes>=0.45.5,!=0.46.0,!=0.48.0", "unsloth[cu126onlytorch2100]", - "unsloth[audio-torch210]", ] cu128-ampere-torch2100 = [ "unsloth[huggingface]", "bitsandbytes>=0.45.5,!=0.46.0,!=0.48.0", "unsloth[cu128onlytorch2100]", - "unsloth[audio-torch210]", ] cu130-ampere-torch2100 = [ "unsloth[huggingface]", "bitsandbytes>=0.45.5,!=0.46.0,!=0.48.0", "unsloth[cu130onlytorch2100]", - "unsloth[audio-torch210]", -] -flashattentiontorch260abiFALSEcu12x = [ - "flash-attn @ https://github.com/Dao-AILab/flash-attention/releases/download/v2.7.4.post1/flash_attn-2.7.4.post1+cu12torch2.6cxx11abiFALSE-cp39-cp39-linux_x86_64.whl ; ('linux' in sys_platform) and python_version == '3.9'", - "flash-attn @ https://github.com/Dao-AILab/flash-attention/releases/download/v2.7.4.post1/flash_attn-2.7.4.post1+cu12torch2.6cxx11abiFALSE-cp310-cp310-linux_x86_64.whl ; ('linux' in sys_platform) and python_version == '3.10'", - "flash-attn @ https://github.com/Dao-AILab/flash-attention/releases/download/v2.7.4.post1/flash_attn-2.7.4.post1+cu12torch2.6cxx11abiFALSE-cp311-cp311-linux_x86_64.whl ; ('linux' in sys_platform) and python_version == '3.11'", - "flash-attn @ https://github.com/Dao-AILab/flash-attention/releases/download/v2.7.4.post1/flash_attn-2.7.4.post1+cu12torch2.6cxx11abiFALSE-cp312-cp312-linux_x86_64.whl ; ('linux' in sys_platform) and python_version == '3.12'", - "flash-attn @ https://github.com/Dao-AILab/flash-attention/releases/download/v2.7.4.post1/flash_attn-2.7.4.post1+cu12torch2.6cxx11abiFALSE-cp313-cp313-linux_x86_64.whl ; ('linux' in sys_platform) and python_version == '3.13'", -] -flashattentiontorch260abiTRUEcu12x = [ - "flash-attn @ https://github.com/Dao-AILab/flash-attention/releases/download/v2.7.4.post1/flash_attn-2.7.4.post1+cu12torch2.6cxx11abiTRUE-cp39-cp39-linux_x86_64.whl ; ('linux' in sys_platform) and python_version == '3.9'", - "flash-attn @ https://github.com/Dao-AILab/flash-attention/releases/download/v2.7.4.post1/flash_attn-2.7.4.post1+cu12torch2.6cxx11abiTRUE-cp310-cp310-linux_x86_64.whl ; ('linux' in sys_platform) and python_version == '3.10'", - "flash-attn @ https://github.com/Dao-AILab/flash-attention/releases/download/v2.7.4.post1/flash_attn-2.7.4.post1+cu12torch2.6cxx11abiTRUE-cp311-cp311-linux_x86_64.whl ; ('linux' in sys_platform) and python_version == '3.11'", - "flash-attn @ https://github.com/Dao-AILab/flash-attention/releases/download/v2.7.4.post1/flash_attn-2.7.4.post1+cu12torch2.6cxx11abiTRUE-cp312-cp312-linux_x86_64.whl ; ('linux' in sys_platform) and python_version == '3.12'", - "flash-attn @ https://github.com/Dao-AILab/flash-attention/releases/download/v2.7.4.post1/flash_attn-2.7.4.post1+cu12torch2.6cxx11abiTRUE-cp313-cp313-linux_x86_64.whl ; ('linux' in sys_platform) and python_version == '3.13'", -] -flashattentiontorch250abiFALSEcu12x = [ - "flash-attn @ https://github.com/Dao-AILab/flash-attention/releases/download/v2.7.4.post1/flash_attn-2.7.4.post1+cu12torch2.5cxx11abiFALSE-cp39-cp39-linux_x86_64.whl ; ('linux' in sys_platform) and python_version == '3.9'", - "flash-attn @ https://github.com/Dao-AILab/flash-attention/releases/download/v2.7.4.post1/flash_attn-2.7.4.post1+cu12torch2.5cxx11abiFALSE-cp310-cp310-linux_x86_64.whl ; ('linux' in sys_platform) and python_version == '3.10'", - "flash-attn @ https://github.com/Dao-AILab/flash-attention/releases/download/v2.7.4.post1/flash_attn-2.7.4.post1+cu12torch2.5cxx11abiFALSE-cp311-cp311-linux_x86_64.whl ; ('linux' in sys_platform) and python_version == '3.11'", - "flash-attn @ https://github.com/Dao-AILab/flash-attention/releases/download/v2.7.4.post1/flash_attn-2.7.4.post1+cu12torch2.5cxx11abiFALSE-cp312-cp312-linux_x86_64.whl ; ('linux' in sys_platform) and python_version == '3.12'", - "flash-attn @ https://github.com/Dao-AILab/flash-attention/releases/download/v2.7.4.post1/flash_attn-2.7.4.post1+cu12torch2.5cxx11abiFALSE-cp313-cp313-linux_x86_64.whl ; ('linux' in sys_platform) and python_version == '3.13'", -] -flashattentiontorch250abiTRUEcu12x = [ - "flash-attn @ https://github.com/Dao-AILab/flash-attention/releases/download/v2.7.4.post1/flash_attn-2.7.4.post1+cu12torch2.5cxx11abiTRUE-cp39-cp39-linux_x86_64.whl ; ('linux' in sys_platform) and python_version == '3.9'", - "flash-attn @ https://github.com/Dao-AILab/flash-attention/releases/download/v2.7.4.post1/flash_attn-2.7.4.post1+cu12torch2.5cxx11abiTRUE-cp310-cp310-linux_x86_64.whl ; ('linux' in sys_platform) and python_version == '3.10'", - "flash-attn @ https://github.com/Dao-AILab/flash-attention/releases/download/v2.7.4.post1/flash_attn-2.7.4.post1+cu12torch2.5cxx11abiTRUE-cp311-cp311-linux_x86_64.whl ; ('linux' in sys_platform) and python_version == '3.11'", - "flash-attn @ https://github.com/Dao-AILab/flash-attention/releases/download/v2.7.4.post1/flash_attn-2.7.4.post1+cu12torch2.5cxx11abiTRUE-cp312-cp312-linux_x86_64.whl ; ('linux' in sys_platform) and python_version == '3.12'", - "flash-attn @ https://github.com/Dao-AILab/flash-attention/releases/download/v2.7.4.post1/flash_attn-2.7.4.post1+cu12torch2.5cxx11abiTRUE-cp313-cp313-linux_x86_64.whl ; ('linux' in sys_platform) and python_version == '3.13'", -] -flashattentiontorch240abiFALSEcu12x = [ - "flash-attn @ https://github.com/Dao-AILab/flash-attention/releases/download/v2.7.4.post1/flash_attn-2.7.4.post1+cu12torch2.4cxx11abiFALSE-cp39-cp39-linux_x86_64.whl ; ('linux' in sys_platform) and python_version == '3.9'", - "flash-attn @ https://github.com/Dao-AILab/flash-attention/releases/download/v2.7.4.post1/flash_attn-2.7.4.post1+cu12torch2.4cxx11abiFALSE-cp310-cp310-linux_x86_64.whl ; ('linux' in sys_platform) and python_version == '3.10'", - "flash-attn @ https://github.com/Dao-AILab/flash-attention/releases/download/v2.7.4.post1/flash_attn-2.7.4.post1+cu12torch2.4cxx11abiFALSE-cp311-cp311-linux_x86_64.whl ; ('linux' in sys_platform) and python_version == '3.11'", - "flash-attn @ https://github.com/Dao-AILab/flash-attention/releases/download/v2.7.4.post1/flash_attn-2.7.4.post1+cu12torch2.4cxx11abiFALSE-cp312-cp312-linux_x86_64.whl ; ('linux' in sys_platform) and python_version == '3.12'", -] -flashattentiontorch240abiTRUEcu12x = [ - "flash-attn @ https://github.com/Dao-AILab/flash-attention/releases/download/v2.7.4.post1/flash_attn-2.7.4.post1+cu12torch2.4cxx11abiTRUE-cp39-cp39-linux_x86_64.whl ; ('linux' in sys_platform) and python_version == '3.9'", - "flash-attn @ https://github.com/Dao-AILab/flash-attention/releases/download/v2.7.4.post1/flash_attn-2.7.4.post1+cu12torch2.4cxx11abiTRUE-cp310-cp310-linux_x86_64.whl ; ('linux' in sys_platform) and python_version == '3.10'", - "flash-attn @ https://github.com/Dao-AILab/flash-attention/releases/download/v2.7.4.post1/flash_attn-2.7.4.post1+cu12torch2.4cxx11abiTRUE-cp311-cp311-linux_x86_64.whl ; ('linux' in sys_platform) and python_version == '3.11'", - "flash-attn @ https://github.com/Dao-AILab/flash-attention/releases/download/v2.7.4.post1/flash_attn-2.7.4.post1+cu12torch2.4cxx11abiTRUE-cp312-cp312-linux_x86_64.whl ; ('linux' in sys_platform) and python_version == '3.12'", -] -intelgputorch260 = [ - "unsloth_zoo[intelgpu]", - "unsloth[huggingfacenotorch]", - - "pytorch_triton_xpu @ https://download.pytorch.org/whl/pytorch_triton_xpu-3.2.0-cp39-cp39-linux_x86_64.whl#sha256=147607f190a7d7aa24ba454def5977fbbfec792fdae18e4ed278cfec29b69271 ; ('linux' in sys_platform) and python_version == '3.9' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", - "pytorch_triton_xpu @ https://download.pytorch.org/whl/pytorch_triton_xpu-3.2.0-cp310-cp310-linux_x86_64.whl#sha256=23aa423fa1542afc34f67eb3ba8ef20060f6d1b3a4697eaeab22b11c92b30f2b ; ('linux' in sys_platform) and python_version == '3.10' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", - "pytorch_triton_xpu @ https://download.pytorch.org/whl/pytorch_triton_xpu-3.2.0-cp311-cp311-linux_x86_64.whl#sha256=bcfa995229bbfd9ffd8d6c8d9f6428d393e876fa6e23ee3c20e3c0d73ca75ca5 ; ('linux' in sys_platform) and python_version == '3.11' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", - "pytorch_triton_xpu @ https://download.pytorch.org/whl/pytorch_triton_xpu-3.2.0-cp312-cp312-linux_x86_64.whl#sha256=bd340903d03470708df3442438acb8b7e08087ab9e61fbe349b2872bf9257ab0 ; ('linux' in sys_platform) and python_version == '3.12' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", - "pytorch_triton_xpu @ https://download.pytorch.org/whl/pytorch_triton_xpu-3.2.0-cp313-cp313-linux_x86_64.whl#sha256=814dccc8a07159e6eca74bed70091bc8fea2d9dd87b0d91845f9f38cde62f01c ; ('linux' in sys_platform) and python_version == '3.13' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", - - "bitsandbytes @ https://github.com/bitsandbytes-foundation/bitsandbytes/releases/download/continuous-release_main/bitsandbytes-1.33.7.preview-py3-none-manylinux_2_24_x86_64.whl ; ('linux' in sys_platform) and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", - "bitsandbytes @ https://github.com/bitsandbytes-foundation/bitsandbytes/releases/download/continuous-release_main/bitsandbytes-1.33.7.preview-py3-none-win_amd64.whl ; (sys_platform == 'win32') and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", - - "torch @ https://download.pytorch.org/whl/xpu/torch-2.6.0%2Bxpu-cp39-cp39-linux_x86_64.whl#sha256=6a8adf6dc4c089406e8b3a7e58ab57a463bddf9b07130d2576e76eced43e92af ; ('linux' in sys_platform) and python_version == '3.9' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", - "torch @ https://download.pytorch.org/whl/xpu/torch-2.6.0%2Bxpu-cp310-cp310-linux_x86_64.whl#sha256=ff4561cbf07c83bbccaa0f6e9bb0e6dcf721bacd53c9c43c4eb0e7331b4792f9 ; ('linux' in sys_platform) and python_version == '3.10' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", - "torch @ https://download.pytorch.org/whl/xpu/torch-2.6.0%2Bxpu-cp311-cp311-linux_x86_64.whl#sha256=12005f66b810ddd3ab93f86c4522bcfdd412cbd27fc9d189b661ff7509bc5e8a ; ('linux' in sys_platform) and python_version == '3.11' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", - "torch @ https://download.pytorch.org/whl/xpu/torch-2.6.0%2Bxpu-cp312-cp312-linux_x86_64.whl#sha256=c4c5c67625cdacf35765c2b94e61fe166e3c3f4a14521b1212a59ad1b3eb0f2e ; ('linux' in sys_platform) and python_version == '3.12' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", - "torch @ https://download.pytorch.org/whl/xpu/torch-2.6.0%2Bxpu-cp313-cp313-linux_x86_64.whl#sha256=e6864f7a60a5ecc43d5d38f59a16e5dd132384f73dfd3a697f74944026038f7b ; ('linux' in sys_platform) and python_version == '3.13' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", -] -intel-gpu-torch260 = [ - "unsloth[intelgputorch260]" -] -intelgputorch270 = [ - "unsloth_zoo[intelgpu]", - "unsloth[huggingfacenotorch]", - - "pytorch_triton_xpu @ https://download.pytorch.org/whl/pytorch_triton_xpu-3.3.0-cp39-cp39-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl#sha256=749a7098492c6a27b356c97149a4a62973b953eae60bc1b6259260974f344913 ; ('linux' in sys_platform) and python_version == '3.9' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", - "pytorch_triton_xpu @ https://download.pytorch.org/whl/pytorch_triton_xpu-3.3.0-cp310-cp310-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl#sha256=44362e80abd752471a08341093321955b066daa2cfb4810e73b8e3b240850f93 ; ('linux' in sys_platform) and python_version == '3.10' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", - "pytorch_triton_xpu @ https://download.pytorch.org/whl/pytorch_triton_xpu-3.3.0-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl#sha256=faa6b8c945a837a080f641bc8ccc77a98fa66980dcd7e62e715fd853737343fd ; ('linux' in sys_platform) and python_version == '3.11' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", - "pytorch_triton_xpu @ https://download.pytorch.org/whl/pytorch_triton_xpu-3.3.0-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl#sha256=40f6fb65b345dc9a61813abe7ac9a585f2c9808f414d140cc2a5f11f53ee063c ; ('linux' in sys_platform) and python_version == '3.12' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", - "pytorch_triton_xpu @ https://download.pytorch.org/whl/pytorch_triton_xpu-3.3.0-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl#sha256=b22b4c02ec71b4bfc862ae3cdfd2871dc0b05d2b1802f5db2196e0f897d581e9 ; ('linux' in sys_platform) and python_version == '3.13' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", - "pytorch_triton_xpu @ https://download.pytorch.org/whl/pytorch_triton_xpu-3.3.0-cp39-cp39-win_amd64.whl#sha256=d4b738d7fa5100c1bd766f91614962828a4810eb57b4df92cd5214a83505a752 ; sys_platform == 'win32' and python_version == '3.9' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", - "pytorch_triton_xpu @ https://download.pytorch.org/whl/pytorch_triton_xpu-3.3.0-cp310-cp310-win_amd64.whl#sha256=143fe8a64d807bcdb7d81bbc062816add325570aa160448454ab6ded4a0a17a1 ; sys_platform == 'win32' and python_version == '3.10' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", - "pytorch_triton_xpu @ https://download.pytorch.org/whl/pytorch_triton_xpu-3.3.0-cp311-cp311-win_amd64.whl#sha256=a8025459ff325d6e3532eb5cf72519db1b178155e7d60aff6c56beb5968fc758 ; sys_platform == 'win32' and python_version == '3.11' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", - "pytorch_triton_xpu @ https://download.pytorch.org/whl/pytorch_triton_xpu-3.3.0-cp312-cp312-win_amd64.whl#sha256=0dd07e6d5b872e42e48f5ee140e609d4554ca3cc509d5bf509ac232267cf358e ; sys_platform == 'win32' and python_version == '3.12' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", - "pytorch_triton_xpu @ https://download.pytorch.org/whl/pytorch_triton_xpu-3.3.0-cp313-cp313-win_amd64.whl#sha256=a936a18182d8e065a9933afc9a3ebbffadd38604969f87c493831214539fc027 ; sys_platform == 'win32' and python_version == '3.13' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", - - "bitsandbytes @ https://github.com/bitsandbytes-foundation/bitsandbytes/releases/download/continuous-release_main/bitsandbytes-1.33.7.preview-py3-none-manylinux_2_24_x86_64.whl ; ('linux' in sys_platform) and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", - "bitsandbytes @ https://github.com/bitsandbytes-foundation/bitsandbytes/releases/download/continuous-release_main/bitsandbytes-1.33.7.preview-py3-none-win_amd64.whl ; (sys_platform == 'win32') and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", - - "torch @ https://download.pytorch.org/whl/xpu/torch-2.7.0%2Bxpu-cp39-cp39-linux_x86_64.whl#sha256=f8ee75e50fcbb37ed5b498299ca2264da99ab278a93fae2358e921e4a6e28273 ; ('linux' in sys_platform) and python_version == '3.9' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", - "torch @ https://download.pytorch.org/whl/xpu/torch-2.7.0%2Bxpu-cp310-cp310-linux_x86_64.whl#sha256=d6fdc342961d98fdcd9d03dfd491a3208bb5f7fbb435841f8f72ce9fdcd2d026 ; ('linux' in sys_platform) and python_version == '3.10' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", - "torch @ https://download.pytorch.org/whl/xpu/torch-2.7.0%2Bxpu-cp311-cp311-linux_x86_64.whl#sha256=74d07f9357df5cf2bf223ad3c84de16346bfaa0504f988fdd5590d3e177e5e86 ; ('linux' in sys_platform) and python_version == '3.11' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", - "torch @ https://download.pytorch.org/whl/xpu/torch-2.7.0%2Bxpu-cp312-cp312-linux_x86_64.whl#sha256=c806d44aa2ca5d225629f6fbc6c994d5deaac2d2cde449195bc8e3522ddd219a ; ('linux' in sys_platform) and python_version == '3.12' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", - "torch @ https://download.pytorch.org/whl/xpu/torch-2.7.0%2Bxpu-cp313-cp313-linux_x86_64.whl#sha256=25d8277b7f01d42e2e014ccbab57a2692b6ec4eff8dcf894eda1b297407cf97a ; ('linux' in sys_platform) and python_version == '3.13' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", - "torch @ https://download.pytorch.org/whl/xpu/torch-2.7.0%2Bxpu-cp39-cp39-win_amd64.whl#sha256=046e85125266ae69c1a0d083e6c092f947ab4b6b41532c16bafe40dbced845df ; sys_platform == 'win32' and python_version == '3.9' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", - "torch @ https://download.pytorch.org/whl/xpu/torch-2.7.0%2Bxpu-cp310-cp310-win_amd64.whl#sha256=9ebaeffb82b0b3e39b6030927d3ebe0eb62a0e9045a3b2d7b0a9e7b15222c0db ; sys_platform == 'win32' and python_version == '3.10' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", - "torch @ https://download.pytorch.org/whl/xpu/torch-2.7.0%2Bxpu-cp311-cp311-win_amd64.whl#sha256=356ba66cee127e7e2c942880bd50e03768306a4ea08d358a0f29c6eebfc4bc81 ; sys_platform == 'win32' and python_version == '3.11' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", - "torch @ https://download.pytorch.org/whl/xpu/torch-2.7.0%2Bxpu-cp312-cp312-win_amd64.whl#sha256=94739e665d9b4d5cd7af5f517cb6103f6f9fb421c095184609653a24524040f5 ; sys_platform == 'win32' and python_version == '3.12' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", - "torch @ https://download.pytorch.org/whl/xpu/torch-2.7.0%2Bxpu-cp313-cp313-win_amd64.whl#sha256=31df3cb674918e89bc8c532baa331dc84f4430e1f9c0ec379232db44cba78355 ; sys_platform == 'win32' and python_version == '3.13' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", -] -intel-gpu-torch270 = [ - "unsloth[intelgputorch270]" -] -intelgputorch280 = [ - "unsloth_zoo[intelgpu]", - "unsloth[huggingfacenotorch]", - - "pytorch_triton_xpu @ https://download.pytorch.org/whl/pytorch_triton_xpu-3.4.0-cp39-cp39-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl#sha256=ac4d8e33986b1c3c5e48151640539272b2187e83016985853111b46fb82c3c94 ; 'linux' in sys_platform and python_version == '3.9' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", - "pytorch_triton_xpu @ https://download.pytorch.org/whl/pytorch_triton_xpu-3.4.0-cp310-cp310-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl#sha256=999fef4c1f711092b9d3086525920545df490de476ecebe899ffc777019ae17f ; 'linux' in sys_platform and python_version == '3.10' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", - "pytorch_triton_xpu @ https://download.pytorch.org/whl/pytorch_triton_xpu-3.4.0-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl#sha256=57b09c8c492985ff6a27cd3a22b08e8f7b96b407bd8030967b6efbb9f63b80cf ; 'linux' in sys_platform and python_version == '3.11' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", - "pytorch_triton_xpu @ https://download.pytorch.org/whl/pytorch_triton_xpu-3.4.0-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl#sha256=df4bb3282bac9a3b90231700077110d8680b338416de03c2b7c6133c9b602649 ; 'linux' in sys_platform and python_version == '3.12' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", - "pytorch_triton_xpu @ https://download.pytorch.org/whl/pytorch_triton_xpu-3.4.0-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl#sha256=60da63c99ca827bdcb0df28e0298bf7d066dc607454c6d6176783cb4e79d838b ; 'linux' in sys_platform and python_version == '3.13' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", - "pytorch_triton_xpu @ https://download.pytorch.org/whl/pytorch_triton_xpu-3.4.0-cp39-cp39-win_amd64.whl#sha256=64aea8de349f3e2e0ebf4c24b011a8122531fdffda5776edaef45829cc241cf8 ; sys_platform == 'win32' and python_version == '3.9' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", - "pytorch_triton_xpu @ https://download.pytorch.org/whl/pytorch_triton_xpu-3.4.0-cp310-cp310-win_amd64.whl#sha256=ae573d255b257fdbed319a3440dc9d0a721e31160ab7f6eba1b2226e6a409a1d ; sys_platform == 'win32' and python_version == '3.10' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", - "pytorch_triton_xpu @ https://download.pytorch.org/whl/pytorch_triton_xpu-3.4.0-cp311-cp311-win_amd64.whl#sha256=8e0ea4558e5776d8ddab0264310be9b26aee5641bcac0da023537556d4317b86 ; sys_platform == 'win32' and python_version == '3.11' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", - "pytorch_triton_xpu @ https://download.pytorch.org/whl/pytorch_triton_xpu-3.4.0-cp312-cp312-win_amd64.whl#sha256=4090dde07a4fffc34aaf855701a9db28e9fccb57b368ade520f1a0f8e811c878 ; sys_platform == 'win32' and python_version == '3.12' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", - "pytorch_triton_xpu @ https://download.pytorch.org/whl/pytorch_triton_xpu-3.4.0-cp313-cp313-win_amd64.whl#sha256=a33d0888f3c8df028a2d028842715837d0049524d6c06b9bb11869890a13601a ; sys_platform == 'win32' and python_version == '3.13' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", - - "torch @ https://download.pytorch.org/whl/xpu/torch-2.8.0%2Bxpu-cp39-cp39-linux_x86_64.whl ; 'linux' in sys_platform and python_version == '3.9' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", - "torch @ https://download.pytorch.org/whl/xpu/torch-2.8.0%2Bxpu-cp310-cp310-linux_x86_64.whl ; 'linux' in sys_platform and python_version == '3.10' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", - "torch @ https://download.pytorch.org/whl/xpu/torch-2.8.0%2Bxpu-cp311-cp311-linux_x86_64.whl ; 'linux' in sys_platform and python_version == '3.11' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", - "torch @ https://download.pytorch.org/whl/xpu/torch-2.8.0%2Bxpu-cp312-cp312-linux_x86_64.whl ; 'linux' in sys_platform and python_version == '3.12' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", - "torch @ https://download.pytorch.org/whl/xpu/torch-2.8.0%2Bxpu-cp313-cp313-linux_x86_64.whl ; 'linux' in sys_platform and python_version == '3.13' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", - "torch @ https://download.pytorch.org/whl/xpu/torch-2.8.0%2Bxpu-cp39-cp39-win_amd64.whl#sha256=f2f401276892428e4875cf1d8717c5cbab704b16fc594ccf23795e7b16549a99 ; sys_platform == 'win32' and python_version == '3.9' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", - "torch @ https://download.pytorch.org/whl/xpu/torch-2.8.0%2Bxpu-cp310-cp310-win_amd64.whl#sha256=125c60cd59d51b39581a7e9afcd4679bc3a6b8c1f9440b1bb502a23fdd60571e ; sys_platform == 'win32' and python_version == '3.10' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", - "torch @ https://download.pytorch.org/whl/xpu/torch-2.8.0%2Bxpu-cp311-cp311-win_amd64.whl#sha256=47f1a57258cd460e80b38b2ed6744e31587ab77a96b4215bf59546cb4bab5cc0 ; sys_platform == 'win32' and python_version == '3.11' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", - "torch @ https://download.pytorch.org/whl/xpu/torch-2.8.0%2Bxpu-cp312-cp312-win_amd64.whl#sha256=0937d8943c145a83d9bafc6f80ef28971167817f9eda26066d33f72caf8a6646 ; sys_platform == 'win32' and python_version == '3.12' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", - "torch @ https://download.pytorch.org/whl/xpu/torch-2.8.0%2Bxpu-cp313-cp313-win_amd64.whl#sha256=e034aab1d71760dc80a731531be43673ffe15e99033b82d24e40d2e6d41bd8bf ; sys_platform == 'win32' and python_version == '3.13' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", - - "bitsandbytes @ https://github.com/bitsandbytes-foundation/bitsandbytes/releases/download/continuous-release_main/bitsandbytes-1.33.7.preview-py3-none-manylinux_2_24_x86_64.whl ; ('linux' in sys_platform) and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", - "bitsandbytes @ https://github.com/bitsandbytes-foundation/bitsandbytes/releases/download/continuous-release_main/bitsandbytes-1.33.7.preview-py3-none-win_amd64.whl ; (sys_platform == 'win32') and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", - - "torchvision @ https://download.pytorch.org/whl/xpu/torchvision-0.23.0%2Bxpu-cp39-cp39-manylinux_2_28_x86_64.whl#sha256=6e981c192045fc249c008441179ff237bb00174d818b875b0475730b63f0eaca ; 'linux' in sys_platform and python_version == '3.9' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", - "torchvision @ https://download.pytorch.org/whl/xpu/torchvision-0.23.0%2Bxpu-cp310-cp310-manylinux_2_28_x86_64.whl#sha256=e5ba4805969277175ebfd59cc717093528cc6e3ada89ac2725fc7a3c1fee6169 ; 'linux' in sys_platform and python_version == '3.10' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", - "torchvision @ https://download.pytorch.org/whl/xpu/torchvision-0.23.0%2Bxpu-cp311-cp311-manylinux_2_28_x86_64.whl#sha256=74c39c144104416bc4c5ad8c26ab0c169dc5cc6be58059e01bc3665dd0ef676f ; 'linux' in sys_platform and python_version == '3.11' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", - "torchvision @ https://download.pytorch.org/whl/xpu/torchvision-0.23.0%2Bxpu-cp312-cp312-manylinux_2_28_x86_64.whl#sha256=0acec355b80c3899841184084f365df336c508602812e34a44007b8b60d53af4 ; 'linux' in sys_platform and python_version == '3.12' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", - "torchvision @ https://download.pytorch.org/whl/xpu/torchvision-0.23.0%2Bxpu-cp313-cp313-manylinux_2_28_x86_64.whl#sha256=e2109ae773dad27b98ca17681044b4f876563c37f2382b75de3a371399edcff8 ; 'linux' in sys_platform and python_version == '3.13' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", - "torchvision @ https://download.pytorch.org/whl/xpu/torchvision-0.23.0%2Bxpu-cp39-cp39-win_amd64.whl#sha256=5f7904e7048d414379bc8c1167260f1e84204f105db2d0a2f9c89e87ce1cf205 ; sys_platform == 'win32' and python_version == '3.9' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", - "torchvision @ https://download.pytorch.org/whl/xpu/torchvision-0.23.0%2Bxpu-cp310-cp310-win_amd64.whl#sha256=005fca5e658ca8e37adb63c1a021c84f5e56dfa6cf0d601d89cfe40b9473f79f ; sys_platform == 'win32' and python_version == '3.10' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", - "torchvision @ https://download.pytorch.org/whl/xpu/torchvision-0.23.0%2Bxpu-cp311-cp311-win_amd64.whl#sha256=c6d030f5361461550c0ff1339b5bca8585fc1e84fda2e64b6184e65a581e4f98 ; sys_platform == 'win32' and python_version == '3.11' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", - "torchvision @ https://download.pytorch.org/whl/xpu/torchvision-0.23.0%2Bxpu-cp312-cp312-win_amd64.whl#sha256=91aafd61864cdce27461cbec13ddbf28c1bc6494265a1e4b80131c64a3b7d18f ; sys_platform == 'win32' and python_version == '3.12' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", - "torchvision @ https://download.pytorch.org/whl/xpu/torchvision-0.23.0%2Bxpu-cp313-cp313-win_amd64.whl#sha256=71dc4a6421742ed1e7f585b04a100ad53615c341fbccfbc255aefb38ea9091da ; sys_platform == 'win32' and python_version == '3.13' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", -] -intel-gpu-torch280 = [ - "unsloth[intelgputorch280]" -] -intelgputorch290 = [ - "unsloth_zoo[intelgpu]", - "unsloth[huggingfacenotorch]", - - "pytorch_triton_xpu @ https://download.pytorch.org/whl/pytorch_triton_xpu-3.5.0-cp310-cp310-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl#sha256=c169a1de14c19673b17c751290d467fa282fc90fa5da4314b2e5cdab1f553146 ; platform_system == 'Linux' and python_version == '3.10' and platform_machine == 'x86_64'", - "pytorch_triton_xpu @ https://download.pytorch.org/whl/pytorch_triton_xpu-3.5.0-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl#sha256=013d9dd5d6479bd22983161f462e61c8dbe1d82e6730624a7a8d5945507eaa61 ; platform_system == 'Linux' and python_version == '3.11' and platform_machine == 'x86_64'", - "pytorch_triton_xpu @ https://download.pytorch.org/whl/pytorch_triton_xpu-3.5.0-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl#sha256=afc8cabfbf7ed51fd278d1e0f88d6afc157b0201bad4b99d681e4d542f9e66d4 ; platform_system == 'Linux' and python_version == '3.12' and platform_machine == 'x86_64'", - "pytorch_triton_xpu @ https://download.pytorch.org/whl/pytorch_triton_xpu-3.5.0-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl#sha256=0d24c1716088f2764d0d24c64227732195b6a42706c3c5fc89eeb4904bfa0818 ; platform_system == 'Linux' and python_version == '3.13' and platform_machine == 'x86_64'", - "pytorch_triton_xpu @ https://download.pytorch.org/whl/pytorch_triton_xpu-3.5.0-cp310-cp310-win_amd64.whl#sha256=c83ab007311d9cfb6e809ee5a4587d99a9eef4be720b90da4f1aaa68b45139a0 ; sys_platform == 'win32' and python_version == '3.10' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", - "pytorch_triton_xpu @ https://download.pytorch.org/whl/pytorch_triton_xpu-3.5.0-cp311-cp311-win_amd64.whl#sha256=debf75348da8e8c7166b4d4a9b91d1508bb8d6581e339f79f7604b2e6746bacd ; sys_platform == 'win32' and python_version == '3.11' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", - "pytorch_triton_xpu @ https://download.pytorch.org/whl/pytorch_triton_xpu-3.5.0-cp312-cp312-win_amd64.whl#sha256=97337a47425f1963a723475bd61037460e84ba01db4f87a1d662c3718ff6c47e ; sys_platform == 'win32' and python_version == '3.12' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", - "pytorch_triton_xpu @ https://download.pytorch.org/whl/pytorch_triton_xpu-3.5.0-cp313-cp313-win_amd64.whl#sha256=2caf8138695f6abb023ecd02031a2611ba1bf8fff2f19802567cb2fadefe9e87 ; sys_platform == 'win32' and python_version == '3.13' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", - - "torch @ https://download.pytorch.org/whl/xpu/torch-2.9.0%2Bxpu-cp310-cp310-linux_x86_64.whl#sha256=5afbe860ce991825a36b75706a523601087e414b77598ef0d9d3d565741c277d ; platform_system == 'Linux' and python_version == '3.10' and platform_machine == 'x86_64'", - "torch @ https://download.pytorch.org/whl/xpu/torch-2.9.0%2Bxpu-cp311-cp311-linux_x86_64.whl#sha256=607fe419c32d6e8e0556f745742e7cff1d0babce51f54be890e0c1422359c442 ; platform_system == 'Linux' and python_version == '3.11' and platform_machine == 'x86_64'", - "torch @ https://download.pytorch.org/whl/xpu/torch-2.9.0%2Bxpu-cp312-cp312-linux_x86_64.whl#sha256=376bae584d89980b8e59934d248c38d5fa3b7d4687a4df1a19f4bc1d23dcc8c1 ; platform_system == 'Linux' and python_version == '3.12' and platform_machine == 'x86_64'", - "torch @ https://download.pytorch.org/whl/xpu/torch-2.9.0%2Bxpu-cp313-cp313-linux_x86_64.whl#sha256=98d6a06dd7fb185874367b18bd609f05f16fdce4142a5980ca94461949965cd2 ; platform_system == 'Linux' and python_version == '3.13' and platform_machine == 'x86_64'", - "torch @ https://download.pytorch.org/whl/xpu/torch-2.9.0%2Bxpu-cp310-cp310-win_amd64.whl#sha256=47cc68f631f65bd9c84924d052cd04dec7531023caa85e80345e9c94611c887d ; sys_platform == 'win32' and python_version == '3.10' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", - "torch @ https://download.pytorch.org/whl/xpu/torch-2.9.0%2Bxpu-cp311-cp311-win_amd64.whl#sha256=d56c44ab4818aba57e5c7b628f422d014e0d507427170a771c5be85e308b0bc6 ; sys_platform == 'win32' and python_version == '3.11' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", - "torch @ https://download.pytorch.org/whl/xpu/torch-2.9.0%2Bxpu-cp312-cp312-win_amd64.whl#sha256=18cad93aaff76a01ce73aef6935ece7cfc03344b905592ec731446c44d44592b ; sys_platform == 'win32' and python_version == '3.12' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", - "torch @ https://download.pytorch.org/whl/xpu/torch-2.9.0%2Bxpu-cp313-cp313-win_amd64.whl#sha256=579929cdc10a76800ead41289cac191ea36d1b16f5f501d3fc25607d4375cd83 ; sys_platform == 'win32' and python_version == '3.13' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", - - "bitsandbytes @ https://github.com/bitsandbytes-foundation/bitsandbytes/releases/download/continuous-release_main/bitsandbytes-1.33.7.preview-py3-none-manylinux_2_24_x86_64.whl ; ('linux' in sys_platform) and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", - "bitsandbytes @ https://github.com/bitsandbytes-foundation/bitsandbytes/releases/download/continuous-release_main/bitsandbytes-1.33.7.preview-py3-none-win_amd64.whl ; (sys_platform == 'win32') and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", - - "torchvision @ https://download.pytorch.org/whl/xpu/torchvision-0.24.0%2Bxpu-cp310-cp310-manylinux_2_28_x86_64.whl#sha256=cbfae2b79b7549fd368c2462fc8e94f8f26cc450782ee72138e908077c09a519 ; platform_system == 'Linux' and python_version == '3.10' and platform_machine == 'x86_64'", - "torchvision @ https://download.pytorch.org/whl/xpu/torchvision-0.24.0%2Bxpu-cp311-cp311-manylinux_2_28_x86_64.whl#sha256=044fa36ef4b6b43edcd490b75c853fa4b3eb033c2bded29f8fbcf27734713c67 ; platform_system == 'Linux' and python_version == '3.11' and platform_machine == 'x86_64'", - "torchvision @ https://download.pytorch.org/whl/xpu/torchvision-0.24.0%2Bxpu-cp312-cp312-manylinux_2_28_x86_64.whl#sha256=4b91e4bec1d740a6211f02578a79888550b73f3a4e1383035f8f6d72f587212c ; platform_system == 'Linux' and python_version == '3.12' and platform_machine == 'x86_64'", - "torchvision @ https://download.pytorch.org/whl/xpu/torchvision-0.24.0%2Bxpu-cp313-cp313-manylinux_2_28_x86_64.whl#sha256=88239e73ca37254bec84f29cd5887e10ff712de7edbbda3fbb3609cd6190d99e ; platform_system == 'Linux' and python_version == '3.13' and platform_machine == 'x86_64'", - "torchvision @ https://download.pytorch.org/whl/xpu/torchvision-0.24.0%2Bxpu-cp310-cp310-win_amd64.whl#sha256=19c7da8ca767d593e13a88a12bb08d06e34a673f6f26c2f9c191d60e81c02953 ; sys_platform == 'win32' and python_version == '3.10' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", - "torchvision @ https://download.pytorch.org/whl/xpu/torchvision-0.24.0%2Bxpu-cp311-cp311-win_amd64.whl#sha256=9bb0d1421c544ac8e2eca5b47daacaf54706dc9139c003aa5e77ee5f355c5931 ; sys_platform == 'win32' and python_version == '3.11' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", - "torchvision @ https://download.pytorch.org/whl/xpu/torchvision-0.24.0%2Bxpu-cp312-cp312-win_amd64.whl#sha256=6a5194bc736089606342d48a3f6822829b167617e9495d91d753dd1bd46fda18 ; sys_platform == 'win32' and python_version == '3.12' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", - "torchvision @ https://download.pytorch.org/whl/xpu/torchvision-0.24.0%2Bxpu-cp313-cp313-win_amd64.whl#sha256=da47a3ce2bb7f0301a31124668b5908f9b9e92d6241443de15a310ef9632fd83 ; sys_platform == 'win32' and python_version == '3.13' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", -] -intel-gpu-torch290 = [ - "unsloth[intelgputorch290]" -] -intelgputorch271 = [ - "unsloth_zoo[intelgpu]", - "unsloth[huggingfacenotorch]", - - "pytorch_triton_xpu @ https://download.pytorch.org/whl/pytorch_triton_xpu-3.3.1-cp310-cp310-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl#sha256=663ce21364096b268c6687f26f22862cb1001cae0c4ec9f98a0998415f99e2b0 ; platform_system == 'Linux' and python_version == '3.10' and platform_machine == 'x86_64'", - "pytorch_triton_xpu @ https://download.pytorch.org/whl/pytorch_triton_xpu-3.3.1-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl#sha256=dd92cc17000bad19f213b6a877d7f10cd71341b703cd188513ce9fff8d42e3dd ; platform_system == 'Linux' and python_version == '3.11' and platform_machine == 'x86_64'", - "pytorch_triton_xpu @ https://download.pytorch.org/whl/pytorch_triton_xpu-3.3.1-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl#sha256=aa5c3ec21a89e967d1dfe61e3d5b1c1ae9620c871ed804771d3378d6a44066f2 ; platform_system == 'Linux' and python_version == '3.12' and platform_machine == 'x86_64'", - "pytorch_triton_xpu @ https://download.pytorch.org/whl/pytorch_triton_xpu-3.3.1-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl#sha256=d1c6f522e11112a311b1a61ba7b40b43ad8305675fa29153017ccb1ad0b6816d ; platform_system == 'Linux' and python_version == '3.13' and platform_machine == 'x86_64'", - "pytorch_triton_xpu @ https://download.pytorch.org/whl/pytorch_triton_xpu-3.3.1-cp310-cp310-win_amd64.whl#sha256=a5c16dcf449a9cb62bc3788f7ec45782bb3ead6edc2637a12b60ef0f8f45dc55 ; sys_platform == 'win32' and python_version == '3.10' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", - "pytorch_triton_xpu @ https://download.pytorch.org/whl/pytorch_triton_xpu-3.3.1-cp311-cp311-win_amd64.whl#sha256=bc2d76ffa4ceed5b38ae34b52dbff643442e1a44d52ca72d7cb520ca1950e9ae ; sys_platform == 'win32' and python_version == '3.11' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", - "pytorch_triton_xpu @ https://download.pytorch.org/whl/pytorch_triton_xpu-3.3.1-cp312-cp312-win_amd64.whl#sha256=b09ca59ce52d6d27b1510df783cde222b703a71857a6fa953f1f155f9f50811a ; sys_platform == 'win32' and python_version == '3.12' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", - "pytorch_triton_xpu @ https://download.pytorch.org/whl/pytorch_triton_xpu-3.3.1-cp313-cp313-win_amd64.whl#sha256=1260c4a4bad426b6cd3c8f3e1a21835381c6f217bf434bcb55fedec08a206dea ; sys_platform == 'win32' and python_version == '3.13' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", - - "torch @ https://download.pytorch.org/whl/xpu/torch-2.7.1%2Bxpu-cp310-cp310-linux_x86_64.whl#sha256=231c3fbd88a75d94de5ccbbb7f4f9a96cb3c58b3d891c2a1b469d38df95f9be6 ; platform_system == 'Linux' and python_version == '3.10' and platform_machine == 'x86_64'", - "torch @ https://download.pytorch.org/whl/xpu/torch-2.7.1%2Bxpu-cp311-cp311-linux_x86_64.whl#sha256=78edcc27709dd819fc820f5eb9421bd10d3f3dcb14adb25ee60766c76f0e67f3 ; platform_system == 'Linux' and python_version == '3.11' and platform_machine == 'x86_64'", - "torch @ https://download.pytorch.org/whl/xpu/torch-2.7.1%2Bxpu-cp312-cp312-linux_x86_64.whl#sha256=b443df40bc9cb7d648a9f8f9ed1d5c3a1203e561ebd0a61dd55fb8a58833d5ec ; platform_system == 'Linux' and python_version == '3.12' and platform_machine == 'x86_64'", - "torch @ https://download.pytorch.org/whl/xpu/torch-2.7.1%2Bxpu-cp313-cp313-linux_x86_64.whl#sha256=412b58ffcceebea399c9a1bcdb22896aa10385c2650a8c4f8a677fb11c49b448 ; platform_system == 'Linux' and python_version == '3.13' and platform_machine == 'x86_64'", - "torch @ https://download.pytorch.org/whl/xpu/torch-2.7.1%2Bxpu-cp310-cp310-win_amd64.whl#sha256=2591228dc2cb73c78daf24277c4449ba9474f94cd31938147249269fe89d05d6 ; sys_platform == 'win32' and python_version == '3.10' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", - "torch @ https://download.pytorch.org/whl/xpu/torch-2.7.1%2Bxpu-cp311-cp311-win_amd64.whl#sha256=1aacb86e9a9684ffc8bde3db14b251d00df7019a9a434ec99a59076a2696325d ; sys_platform == 'win32' and python_version == '3.11' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", - "torch @ https://download.pytorch.org/whl/xpu/torch-2.7.1%2Bxpu-cp312-cp312-win_amd64.whl#sha256=9b65dc8562521b60d77aa653132bc03a19da0291318fcf919faa3f03080d8f7e ; sys_platform == 'win32' and python_version == '3.12' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", - "torch @ https://download.pytorch.org/whl/xpu/torch-2.7.1%2Bxpu-cp313-cp313-win_amd64.whl#sha256=cd3669fee311bc3ee5501d696bf989226a6f2bf957d120a04881a07af05526d6 ; sys_platform == 'win32' and python_version == '3.13' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", - - "bitsandbytes @ https://github.com/bitsandbytes-foundation/bitsandbytes/releases/download/continuous-release_main/bitsandbytes-1.33.7.preview-py3-none-manylinux_2_24_x86_64.whl ; ('linux' in sys_platform) and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", - "bitsandbytes @ https://github.com/bitsandbytes-foundation/bitsandbytes/releases/download/continuous-release_main/bitsandbytes-1.33.7.preview-py3-none-win_amd64.whl ; (sys_platform == 'win32') and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", - - "torchvision @ https://download.pytorch.org/whl/xpu/torchvision-0.22.1%2Bxpu-cp310-cp310-manylinux_2_28_x86_64.whl#sha256=f8cdf6889c02b3166679eef661b68757ea7e99c314432c3d41dac3d2ed4a59d4 ; platform_system == 'Linux' and python_version == '3.10' and platform_machine == 'x86_64'", - "torchvision @ https://download.pytorch.org/whl/xpu/torchvision-0.22.1%2Bxpu-cp311-cp311-manylinux_2_28_x86_64.whl#sha256=f7d15b65d52809745992e0001c25034f33ac01f2dff5248614e07b5d009a59b7 ; platform_system == 'Linux' and python_version == '3.11' and platform_machine == 'x86_64'", - "torchvision @ https://download.pytorch.org/whl/xpu/torchvision-0.22.1%2Bxpu-cp312-cp312-manylinux_2_28_x86_64.whl#sha256=1ff1f98d70846352c7f56833bedab1a055ead27b11c120b8c719063ee0383554 ; platform_system == 'Linux' and python_version == '3.12' and platform_machine == 'x86_64'", - "torchvision @ https://download.pytorch.org/whl/xpu/torchvision-0.22.1%2Bxpu-cp313-cp313-manylinux_2_28_x86_64.whl#sha256=f46945344ea911a70309231eaaf3b80c96f6646ce5515dc89aa94f94144e310e ; platform_system == 'Linux' and python_version == '3.13' and platform_machine == 'x86_64'", - "torchvision @ https://download.pytorch.org/whl/xpu/torchvision-0.22.1%2Bxpu-cp310-cp310-win_amd64.whl#sha256=ecae9a02de769e2070d37388116beb407c3f0d60b8e65c1da1423f4eafee361a ; sys_platform == 'win32' and python_version == '3.10' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", - "torchvision @ https://download.pytorch.org/whl/xpu/torchvision-0.22.1%2Bxpu-cp311-cp311-win_amd64.whl#sha256=2914e62782431bebd6ad9a3b98a2b7311e448e84a7534bb7f35874b9279a17de ; sys_platform == 'win32' and python_version == '3.11' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", - "torchvision @ https://download.pytorch.org/whl/xpu/torchvision-0.22.1%2Bxpu-cp312-cp312-win_amd64.whl#sha256=5b462c156f4e2097e1e53649d3f298ce352fa4c5d1e6addd360375b10ebd6c67 ; sys_platform == 'win32' and python_version == '3.12' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", - "torchvision @ https://download.pytorch.org/whl/xpu/torchvision-0.22.1%2Bxpu-cp313-cp313-win_amd64.whl#sha256=fa87b3677cd1af67ce423004283c1bde80e3571f391182a3e89b485e18e3c70f ; sys_platform == 'win32' and python_version == '3.13' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", -] -intel-gpu-torch271 = [ - "unsloth[intelgputorch271]" -] -intelgputorch291 = [ - "unsloth_zoo[intelgpu]", - "unsloth[huggingfacenotorch]", - - "pytorch_triton_xpu @ https://download.pytorch.org/whl/pytorch_triton_xpu-3.5.0-cp310-cp310-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl#sha256=c169a1de14c19673b17c751290d467fa282fc90fa5da4314b2e5cdab1f553146 ; platform_system == 'Linux' and python_version == '3.10' and platform_machine == 'x86_64'", - "pytorch_triton_xpu @ https://download.pytorch.org/whl/pytorch_triton_xpu-3.5.0-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl#sha256=013d9dd5d6479bd22983161f462e61c8dbe1d82e6730624a7a8d5945507eaa61 ; platform_system == 'Linux' and python_version == '3.11' and platform_machine == 'x86_64'", - "pytorch_triton_xpu @ https://download.pytorch.org/whl/pytorch_triton_xpu-3.5.0-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl#sha256=afc8cabfbf7ed51fd278d1e0f88d6afc157b0201bad4b99d681e4d542f9e66d4 ; platform_system == 'Linux' and python_version == '3.12' and platform_machine == 'x86_64'", - "pytorch_triton_xpu @ https://download.pytorch.org/whl/pytorch_triton_xpu-3.5.0-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl#sha256=0d24c1716088f2764d0d24c64227732195b6a42706c3c5fc89eeb4904bfa0818 ; platform_system == 'Linux' and python_version == '3.13' and platform_machine == 'x86_64'", - "pytorch_triton_xpu @ https://download.pytorch.org/whl/pytorch_triton_xpu-3.5.0-cp310-cp310-win_amd64.whl#sha256=c83ab007311d9cfb6e809ee5a4587d99a9eef4be720b90da4f1aaa68b45139a0 ; sys_platform == 'win32' and python_version == '3.10' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", - "pytorch_triton_xpu @ https://download.pytorch.org/whl/pytorch_triton_xpu-3.5.0-cp311-cp311-win_amd64.whl#sha256=debf75348da8e8c7166b4d4a9b91d1508bb8d6581e339f79f7604b2e6746bacd ; sys_platform == 'win32' and python_version == '3.11' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", - "pytorch_triton_xpu @ https://download.pytorch.org/whl/pytorch_triton_xpu-3.5.0-cp312-cp312-win_amd64.whl#sha256=97337a47425f1963a723475bd61037460e84ba01db4f87a1d662c3718ff6c47e ; sys_platform == 'win32' and python_version == '3.12' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", - "pytorch_triton_xpu @ https://download.pytorch.org/whl/pytorch_triton_xpu-3.5.0-cp313-cp313-win_amd64.whl#sha256=2caf8138695f6abb023ecd02031a2611ba1bf8fff2f19802567cb2fadefe9e87 ; sys_platform == 'win32' and python_version == '3.13' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", - - "torch @ https://download.pytorch.org/whl/xpu/torch-2.9.1%2Bxpu-cp310-cp310-linux_x86_64.whl#sha256=fb7895c744132d6a8e56ce8434ae1d8355c9bda4e9f58832744ff742d6268eaf ; platform_system == 'Linux' and python_version == '3.10' and platform_machine == 'x86_64'", - "torch @ https://download.pytorch.org/whl/xpu/torch-2.9.1%2Bxpu-cp311-cp311-linux_x86_64.whl#sha256=da2604a9114a28de71ce654819424d20a246adf644d191ae160837df9731b79e ; platform_system == 'Linux' and python_version == '3.11' and platform_machine == 'x86_64'", - "torch @ https://download.pytorch.org/whl/xpu/torch-2.9.1%2Bxpu-cp312-cp312-linux_x86_64.whl#sha256=d5968d78d81c1d01efc1b3bf83d7da3d83161dcc3a9fcf91f500591db1c6c75d ; platform_system == 'Linux' and python_version == '3.12' and platform_machine == 'x86_64'", - "torch @ https://download.pytorch.org/whl/xpu/torch-2.9.1%2Bxpu-cp313-cp313-linux_x86_64.whl#sha256=b56d6b0d65863f370527e971dbfa046a5dd2a1f61cc95071db26c764f36e4dce ; platform_system == 'Linux' and python_version == '3.13' and platform_machine == 'x86_64'", - "torch @ https://download.pytorch.org/whl/xpu/torch-2.9.1%2Bxpu-cp310-cp310-win_amd64.whl#sha256=2f318fb6a4bf1101cc17f35a5371f7c1768b41fceed03628397834e85b3edfdd ; sys_platform == 'win32' and python_version == '3.10' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", - "torch @ https://download.pytorch.org/whl/xpu/torch-2.9.1%2Bxpu-cp311-cp311-win_amd64.whl#sha256=c9cedc3fb099366b2e6c563df6578e323564b1b5d40ac27be73c674755343a1d ; sys_platform == 'win32' and python_version == '3.11' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", - "torch @ https://download.pytorch.org/whl/xpu/torch-2.9.1%2Bxpu-cp312-cp312-win_amd64.whl#sha256=bee9623254d0f95a1ca115dbd17e9a9d966fdb8ae123e2ada4a9eb2fb8d38db8 ; sys_platform == 'win32' and python_version == '3.12' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", - "torch @ https://download.pytorch.org/whl/xpu/torch-2.9.1%2Bxpu-cp313-cp313-win_amd64.whl#sha256=cd5c857da52a63c121561b30b0979e69ade70b575fd74e389787bc7c1ee2ac11 ; sys_platform == 'win32' and python_version == '3.13' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", - - "bitsandbytes @ https://github.com/bitsandbytes-foundation/bitsandbytes/releases/download/continuous-release_main/bitsandbytes-1.33.7.preview-py3-none-manylinux_2_24_x86_64.whl ; ('linux' in sys_platform) and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", - "bitsandbytes @ https://github.com/bitsandbytes-foundation/bitsandbytes/releases/download/continuous-release_main/bitsandbytes-1.33.7.preview-py3-none-win_amd64.whl ; (sys_platform == 'win32') and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", - - "torchvision @ https://download.pytorch.org/whl/xpu/torchvision-0.24.1%2Bxpu-cp310-cp310-manylinux_2_28_x86_64.whl#sha256=cc5272da2cb4554edf059eedd6d1f5ef2859033b0fb79d5dcb8e99a0697f3325 ; platform_system == 'Linux' and python_version == '3.10' and platform_machine == 'x86_64'", - "torchvision @ https://download.pytorch.org/whl/xpu/torchvision-0.24.1%2Bxpu-cp311-cp311-manylinux_2_28_x86_64.whl#sha256=3c80d6a068c32fc4ebddb27953e03a0141bd0f10ca8730417cbc0e0748158285 ; platform_system == 'Linux' and python_version == '3.11' and platform_machine == 'x86_64'", - "torchvision @ https://download.pytorch.org/whl/xpu/torchvision-0.24.1%2Bxpu-cp312-cp312-manylinux_2_28_x86_64.whl#sha256=8cf640a867cf270b3fda7a10002c29d3fc2ad6dfbd76404a8cdd820489adb04c ; platform_system == 'Linux' and python_version == '3.12' and platform_machine == 'x86_64'", - "torchvision @ https://download.pytorch.org/whl/xpu/torchvision-0.24.1%2Bxpu-cp313-cp313-manylinux_2_28_x86_64.whl#sha256=d9c59ee5ae3d0560f02401c8dfd8054d50813a8dbb5d33a8777de7d02f6fcb7b ; platform_system == 'Linux' and python_version == '3.13' and platform_machine == 'x86_64'", - "torchvision @ https://download.pytorch.org/whl/xpu/torchvision-0.24.1%2Bxpu-cp310-cp310-win_amd64.whl#sha256=843ea7fcd8f5a22ebbc20d2d61d9eec7593821a0372eb8cabb73953d12ef6acf ; sys_platform == 'win32' and python_version == '3.10' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", - "torchvision @ https://download.pytorch.org/whl/xpu/torchvision-0.24.1%2Bxpu-cp311-cp311-win_amd64.whl#sha256=e5ff8a31d3c700f8dbac59697c8e32298a43ec059609ebc6ea7bab3eff6384e1 ; sys_platform == 'win32' and python_version == '3.11' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", - "torchvision @ https://download.pytorch.org/whl/xpu/torchvision-0.24.1%2Bxpu-cp312-cp312-win_amd64.whl#sha256=8bae6d4c042f8d20818da4a5aa9109c6fbd6ec11bc422be152ce8adf9a7095bf ; sys_platform == 'win32' and python_version == '3.12' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", - "torchvision @ https://download.pytorch.org/whl/xpu/torchvision-0.24.1%2Bxpu-cp313-cp313-win_amd64.whl#sha256=47059e290fc2a41ba78666ffcde102c436abf7ff8a34d200268b48c4fa0f9c45 ; sys_platform == 'win32' and python_version == '3.13' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", -] -intel-gpu-torch291 = [ - "unsloth[intelgputorch291]" -] -intelgputorch210 = [ - "unsloth_zoo[intelgpu]", - "unsloth[huggingfacenotorch]", - - "triton-xpu @ https://download.pytorch.org/whl/triton_xpu-3.6.0-cp310-cp310-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl ; platform_system == 'Linux' and python_version == '3.10' and platform_machine == 'x86_64'", - "triton-xpu @ https://download.pytorch.org/whl/triton_xpu-3.6.0-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl ; platform_system == 'Linux' and python_version == '3.11' and platform_machine == 'x86_64'", - "triton-xpu @ https://download.pytorch.org/whl/triton_xpu-3.6.0-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl ; platform_system == 'Linux' and python_version == '3.12' and platform_machine == 'x86_64'", - "triton-xpu @ https://download.pytorch.org/whl/triton_xpu-3.6.0-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl ; platform_system == 'Linux' and python_version == '3.13' and platform_machine == 'x86_64'", - "triton-xpu @ https://download.pytorch.org/whl/triton_xpu-3.6.0-cp310-cp310-win_amd64.whl ; sys_platform == 'win32' and python_version == '3.10' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", - "triton-xpu @ https://download.pytorch.org/whl/triton_xpu-3.6.0-cp311-cp311-win_amd64.whl ; sys_platform == 'win32' and python_version == '3.11' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", - "triton-xpu @ https://download.pytorch.org/whl/triton_xpu-3.6.0-cp312-cp312-win_amd64.whl ; sys_platform == 'win32' and python_version == '3.12' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", - "triton-xpu @ https://download.pytorch.org/whl/triton_xpu-3.6.0-cp313-cp313-win_amd64.whl ; sys_platform == 'win32' and python_version == '3.13' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", - - "torch @ https://download.pytorch.org/whl/xpu/torch-2.10.0%2Bxpu-cp310-cp310-linux_x86_64.whl#sha256=abb1d1ec1ac672bac0ff35420c965f2df0c636ef9d94e2a830e34578489d0a57 ; platform_system == 'Linux' and python_version == '3.10' and platform_machine == 'x86_64'", - "torch @ https://download.pytorch.org/whl/xpu/torch-2.10.0%2Bxpu-cp311-cp311-linux_x86_64.whl#sha256=71ad2f82da0f41eaec159f39fc85854e27c2391efa91b373e550648a6f4aaad3 ; platform_system == 'Linux' and python_version == '3.11' and platform_machine == 'x86_64'", - "torch @ https://download.pytorch.org/whl/xpu/torch-2.10.0%2Bxpu-cp312-cp312-linux_x86_64.whl#sha256=b473571d478912f92881cc13f15fa18f8463fb0fb8a068c96ed47a7d45a4da0a ; platform_system == 'Linux' and python_version == '3.12' and platform_machine == 'x86_64'", - "torch @ https://download.pytorch.org/whl/xpu/torch-2.10.0%2Bxpu-cp313-cp313-linux_x86_64.whl#sha256=3bc64a746ff25a93de140902c60c9e819d7413f5cea1e88d80999c27a5901e9c ; platform_system == 'Linux' and python_version == '3.13' and platform_machine == 'x86_64'", - "torch @ https://download.pytorch.org/whl/xpu/torch-2.10.0%2Bxpu-cp310-cp310-win_amd64.whl#sha256=ce50691ab3fb6301d9b7bb8b3834cf5fa7152a2b5f91fd24c5efdc601a25b780 ; sys_platform == 'win32' and python_version == '3.10' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", - "torch @ https://download.pytorch.org/whl/xpu/torch-2.10.0%2Bxpu-cp311-cp311-win_amd64.whl#sha256=cb9d37f21cb9fb7df67d62863f021c3144e8d8832b9ea8e8523ac308bc620ea1 ; sys_platform == 'win32' and python_version == '3.11' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", - "torch @ https://download.pytorch.org/whl/xpu/torch-2.10.0%2Bxpu-cp312-cp312-win_amd64.whl#sha256=3ad605be4728b6d3a28a44d07dd794b1a9e45551b0057815bf25eb2a6d6a56a7 ; sys_platform == 'win32' and python_version == '3.12' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", - "torch @ https://download.pytorch.org/whl/xpu/torch-2.10.0%2Bxpu-cp313-cp313-win_amd64.whl#sha256=2b4b56dd6c792aef82006904fa888692e3782e4ae5da27526801bad4898f05a5 ; sys_platform == 'win32' and python_version == '3.13' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", - - "bitsandbytes @ https://github.com/bitsandbytes-foundation/bitsandbytes/releases/download/continuous-release_main/bitsandbytes-1.33.7.preview-py3-none-manylinux_2_24_x86_64.whl ; ('linux' in sys_platform) and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", - "bitsandbytes @ https://github.com/bitsandbytes-foundation/bitsandbytes/releases/download/continuous-release_main/bitsandbytes-1.33.7.preview-py3-none-win_amd64.whl ; (sys_platform == 'win32') and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", - - "torchvision @ https://download.pytorch.org/whl/xpu/torchvision-0.25.0%2Bxpu-cp310-cp310-manylinux_2_28_x86_64.whl#sha256=7e1e7b170fcf7161c8499b67156c5a05462243626dc0974010791a0bab4378d3 ; platform_system == 'Linux' and python_version == '3.10' and platform_machine == 'x86_64'", - "torchvision @ https://download.pytorch.org/whl/xpu/torchvision-0.25.0%2Bxpu-cp311-cp311-manylinux_2_28_x86_64.whl#sha256=bd6add201bd7628af70437292e1447abb368e0b5f4ff9abd334ae435efd44792 ; platform_system == 'Linux' and python_version == '3.11' and platform_machine == 'x86_64'", - "torchvision @ https://download.pytorch.org/whl/xpu/torchvision-0.25.0%2Bxpu-cp312-cp312-manylinux_2_28_x86_64.whl#sha256=6ad2543496bc29e59d3dd614a94d09aa9870318aedb66045344fffddfedd2cf8 ; platform_system == 'Linux' and python_version == '3.12' and platform_machine == 'x86_64'", - "torchvision @ https://download.pytorch.org/whl/xpu/torchvision-0.25.0%2Bxpu-cp313-cp313-manylinux_2_28_x86_64.whl#sha256=80269f37865fcd8b57f20e4786efae2200bfa2b2727926c3c7acc82f0e7d3548 ; platform_system == 'Linux' and python_version == '3.13' and platform_machine == 'x86_64'", - "torchvision @ https://download.pytorch.org/whl/xpu/torchvision-0.25.0%2Bxpu-cp310-cp310-win_amd64.whl#sha256=6b9485ba85dcba4d196d6134d9c3332fb228fb2556416bf0450a64e8a472fcba ; sys_platform == 'win32' and python_version == '3.10' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", - "torchvision @ https://download.pytorch.org/whl/xpu/torchvision-0.25.0%2Bxpu-cp311-cp311-win_amd64.whl#sha256=36cbaedf10f6412af5c89afd9aeea474e6a56a0050348ada8fabe1ecaf6b879e ; sys_platform == 'win32' and python_version == '3.11' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", - "torchvision @ https://download.pytorch.org/whl/xpu/torchvision-0.25.0%2Bxpu-cp312-cp312-win_amd64.whl#sha256=738357d97468d75fe3d510ac37e65130f2787f81d9bbc1518898f7396dc3403f ; sys_platform == 'win32' and python_version == '3.12' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", - "torchvision @ https://download.pytorch.org/whl/xpu/torchvision-0.25.0%2Bxpu-cp313-cp313-win_amd64.whl#sha256=1c4b44b36a557f7381e3076fb8843366742238648441d607c8d049c6da0f8886 ; sys_platform == 'win32' and python_version == '3.13' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", -] -intel-gpu-torch210 = [ - "unsloth[intelgputorch210]", - "unsloth[audio-torch210]", -] -intelgputorch2110 = [ - "unsloth_zoo[intelgpu]", - "unsloth[huggingfacenotorch]", - - "triton-xpu @ https://download.pytorch.org/whl/triton_xpu-3.7.0-cp310-cp310-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl#sha256=2a1841138750f708ec017becbf8d357526f3fa350deee6553be5735ad66160a3 ; platform_system == 'Linux' and python_version == '3.10' and platform_machine == 'x86_64'", - "triton-xpu @ https://download.pytorch.org/whl/triton_xpu-3.7.0-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl#sha256=e85378f1fc1ea002271de2a35475b75008fa554b86ef9d3bc55be9c513a63b51 ; platform_system == 'Linux' and python_version == '3.11' and platform_machine == 'x86_64'", - "triton-xpu @ https://download.pytorch.org/whl/triton_xpu-3.7.0-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl#sha256=a6663ebe43e3c0d560ff774708632d7a75208ee64a291c1724ed5c16a92d1c72 ; platform_system == 'Linux' and python_version == '3.12' and platform_machine == 'x86_64'", - "triton-xpu @ https://download.pytorch.org/whl/triton_xpu-3.7.0-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl#sha256=08c8d43b2831faf9d6799480df2b45dde58102257aebd810d07a2ce18cd4e5df ; platform_system == 'Linux' and python_version == '3.13' and platform_machine == 'x86_64'", - "triton-xpu @ https://download.pytorch.org/whl/triton_xpu-3.7.0-cp310-cp310-win_amd64.whl#sha256=90fb8f767950a4ffca627faa7f86d9c697237ea4352d7e23505c5c9ed8e72216 ; sys_platform == 'win32' and python_version == '3.10' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", - "triton-xpu @ https://download.pytorch.org/whl/triton_xpu-3.7.0-cp311-cp311-win_amd64.whl#sha256=aa7de82f4265089e74f25a2701b7532e5c47d74224d877b61da1d66156e3f0c1 ; sys_platform == 'win32' and python_version == '3.11' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", - "triton-xpu @ https://download.pytorch.org/whl/triton_xpu-3.7.0-cp312-cp312-win_amd64.whl#sha256=5ba3a31c6e1b259ad2d924e1b50f72a78c6ebd7eb4f364473bbf93e144734e80 ; sys_platform == 'win32' and python_version == '3.12' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", - "triton-xpu @ https://download.pytorch.org/whl/triton_xpu-3.7.0-cp313-cp313-win_amd64.whl#sha256=e8b4caba9b2399ea4c7f9a2777042564dea5d6f9e586a2dcb015a4ce20f000f7 ; sys_platform == 'win32' and python_version == '3.13' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", - - "torch @ https://download.pytorch.org/whl/xpu/torch-2.11.0%2Bxpu-cp310-cp310-linux_x86_64.whl ; platform_system == 'Linux' and python_version == '3.10' and platform_machine == 'x86_64'", - "torch @ https://download.pytorch.org/whl/xpu/torch-2.11.0%2Bxpu-cp311-cp311-linux_x86_64.whl ; platform_system == 'Linux' and python_version == '3.11' and platform_machine == 'x86_64'", - "torch @ https://download.pytorch.org/whl/xpu/torch-2.11.0%2Bxpu-cp312-cp312-linux_x86_64.whl ; platform_system == 'Linux' and python_version == '3.12' and platform_machine == 'x86_64'", - "torch @ https://download.pytorch.org/whl/xpu/torch-2.11.0%2Bxpu-cp313-cp313-linux_x86_64.whl ; platform_system == 'Linux' and python_version == '3.13' and platform_machine == 'x86_64'", - "torch @ https://download.pytorch.org/whl/xpu/torch-2.11.0%2Bxpu-cp310-cp310-win_amd64.whl ; sys_platform == 'win32' and python_version == '3.10' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", - "torch @ https://download.pytorch.org/whl/xpu/torch-2.11.0%2Bxpu-cp311-cp311-win_amd64.whl ; sys_platform == 'win32' and python_version == '3.11' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", - "torch @ https://download.pytorch.org/whl/xpu/torch-2.11.0%2Bxpu-cp312-cp312-win_amd64.whl ; sys_platform == 'win32' and python_version == '3.12' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", - "torch @ https://download.pytorch.org/whl/xpu/torch-2.11.0%2Bxpu-cp313-cp313-win_amd64.whl ; sys_platform == 'win32' and python_version == '3.13' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", - - "bitsandbytes @ https://github.com/bitsandbytes-foundation/bitsandbytes/releases/download/continuous-release_main/bitsandbytes-1.33.7.preview-py3-none-manylinux_2_24_x86_64.whl ; ('linux' in sys_platform) and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", - "bitsandbytes @ https://github.com/bitsandbytes-foundation/bitsandbytes/releases/download/continuous-release_main/bitsandbytes-1.33.7.preview-py3-none-win_amd64.whl ; (sys_platform == 'win32') and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", - - "torchvision @ https://download.pytorch.org/whl/xpu/torchvision-0.26.0%2Bxpu-cp310-cp310-manylinux_2_28_x86_64.whl#sha256=6e634354b752b7366e8ad16b84f3e7e5863776a7ab448bbabae4fd36668dee7a ; platform_system == 'Linux' and python_version == '3.10' and platform_machine == 'x86_64'", - "torchvision @ https://download.pytorch.org/whl/xpu/torchvision-0.26.0%2Bxpu-cp311-cp311-manylinux_2_28_x86_64.whl#sha256=293169899f562ce473a58836dd024f0b1e72a347400278287ab393d1b04991e4 ; platform_system == 'Linux' and python_version == '3.11' and platform_machine == 'x86_64'", - "torchvision @ https://download.pytorch.org/whl/xpu/torchvision-0.26.0%2Bxpu-cp312-cp312-manylinux_2_28_x86_64.whl#sha256=e204d14be6f0f84d5f0e6e9213556e80326c3ab682cac108bcbef340bf45297b ; platform_system == 'Linux' and python_version == '3.12' and platform_machine == 'x86_64'", - "torchvision @ https://download.pytorch.org/whl/xpu/torchvision-0.26.0%2Bxpu-cp313-cp313-manylinux_2_28_x86_64.whl#sha256=f134344006f0989a2d771554b7905fb05bd93d63b195e64626fde3495ec6f287 ; platform_system == 'Linux' and python_version == '3.13' and platform_machine == 'x86_64'", - "torchvision @ https://download.pytorch.org/whl/xpu/torchvision-0.26.0%2Bxpu-cp310-cp310-win_amd64.whl#sha256=7e52729cb9736c66dc79a7f42de6b31db93b9161d3357fd34cfa33f5fe32b8ea ; sys_platform == 'win32' and python_version == '3.10' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", - "torchvision @ https://download.pytorch.org/whl/xpu/torchvision-0.26.0%2Bxpu-cp311-cp311-win_amd64.whl#sha256=83a6130100c6b6750d8aa9fd29e5d0c53b1c85b1153b8ed4139aea54fc1892cc ; sys_platform == 'win32' and python_version == '3.11' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", - "torchvision @ https://download.pytorch.org/whl/xpu/torchvision-0.26.0%2Bxpu-cp312-cp312-win_amd64.whl#sha256=03788e0e5a5b85a2f09d11f0263d579fcb0cf5623d8810149be0e37836c2738c ; sys_platform == 'win32' and python_version == '3.12' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", - "torchvision @ https://download.pytorch.org/whl/xpu/torchvision-0.26.0%2Bxpu-cp313-cp313-win_amd64.whl#sha256=cb1da1d378ce440f7d1e0ed8cf21bd280d904ab25a55c9453f8377825818df74 ; sys_platform == 'win32' and python_version == '3.13' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", -] -intel-gpu-torch2110 = [ - "unsloth[intelgputorch2110]" -] -intelgputorch2120 = [ - "unsloth_zoo[intelgpu]", - "unsloth[huggingfacenotorch]", - - "triton-xpu @ https://download.pytorch.org/whl/triton_xpu-3.7.1-cp310-cp310-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl#sha256=81ff0eb0c4fc8e19d2510b28c3e1d9382a3c7d6fdaf6a9f9631a93a030d841cf ; platform_system == 'Linux' and python_version == '3.10' and platform_machine == 'x86_64'", - "triton-xpu @ https://download.pytorch.org/whl/triton_xpu-3.7.1-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl#sha256=55574a68d275b85cd4d5cbf185084bae019ebf09c3f43b0bd2831b14935ec8e7 ; platform_system == 'Linux' and python_version == '3.11' and platform_machine == 'x86_64'", - "triton-xpu @ https://download.pytorch.org/whl/triton_xpu-3.7.1-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl#sha256=a31c058c5c2e78ebe490a2e69f2f50caec6b1307ac096e944f116fdc06819d9a ; platform_system == 'Linux' and python_version == '3.12' and platform_machine == 'x86_64'", - "triton-xpu @ https://download.pytorch.org/whl/triton_xpu-3.7.1-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl#sha256=e701a31efa0334775f357c98716f3821775aa944219f7888e13c2dfe2daabe2a ; platform_system == 'Linux' and python_version == '3.13' and platform_machine == 'x86_64'", - "triton-xpu @ https://download.pytorch.org/whl/triton_xpu-3.7.1-cp310-cp310-win_amd64.whl#sha256=0d7730651c3e52fbf3a430cc201455f0c6600dc72e681aec495f131ea44f341a ; sys_platform == 'win32' and python_version == '3.10' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", - "triton-xpu @ https://download.pytorch.org/whl/triton_xpu-3.7.1-cp311-cp311-win_amd64.whl#sha256=8f4a63de73e3d632098f93c8f0bd77244958a47d7c5f728b8ff35f8a91fdb983 ; sys_platform == 'win32' and python_version == '3.11' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", - "triton-xpu @ https://download.pytorch.org/whl/triton_xpu-3.7.1-cp312-cp312-win_amd64.whl#sha256=6589ece3adc2b1ab88d90ff1267afc25df5c7b868f0b633e732cac70df36cbde ; sys_platform == 'win32' and python_version == '3.12' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", - "triton-xpu @ https://download.pytorch.org/whl/triton_xpu-3.7.1-cp313-cp313-win_amd64.whl#sha256=2fdf001a9b0575e8b1827127259bb9b13bf36e659882be74c2dfab46597d3e7a ; sys_platform == 'win32' and python_version == '3.13' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", - - "torch @ https://download.pytorch.org/whl/xpu/torch-2.12.0%2Bxpu-cp310-cp310-linux_x86_64.whl#sha256=e8923cd1fe560472904b1461b745d2f1826bb9c1bc0808225d5f28a450e4d553 ; platform_system == 'Linux' and python_version == '3.10' and platform_machine == 'x86_64'", - "torch @ https://download.pytorch.org/whl/xpu/torch-2.12.0%2Bxpu-cp311-cp311-linux_x86_64.whl#sha256=f7c082b2fc9b61def594d30ea57762dc4a8bc7111a9a9593953ed948de242e28 ; platform_system == 'Linux' and python_version == '3.11' and platform_machine == 'x86_64'", - "torch @ https://download.pytorch.org/whl/xpu/torch-2.12.0%2Bxpu-cp312-cp312-linux_x86_64.whl#sha256=f59decc04bec27862ed0197554a52370dbcba3e6892616d1fbce450e402bf2d5 ; platform_system == 'Linux' and python_version == '3.12' and platform_machine == 'x86_64'", - "torch @ https://download.pytorch.org/whl/xpu/torch-2.12.0%2Bxpu-cp313-cp313-linux_x86_64.whl#sha256=56f74e7c6c096e1a7ac215eb79ee590b764be3fbba8f4febc145bca47194a083 ; platform_system == 'Linux' and python_version == '3.13' and platform_machine == 'x86_64'", - "torch @ https://download.pytorch.org/whl/xpu/torch-2.12.0%2Bxpu-cp310-cp310-win_amd64.whl#sha256=b9779b71457b5a916ae052ed2467c10273cae4862d469b191359173b2038c53e ; sys_platform == 'win32' and python_version == '3.10' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", - "torch @ https://download.pytorch.org/whl/xpu/torch-2.12.0%2Bxpu-cp311-cp311-win_amd64.whl#sha256=7ef8e776c992e4e3ae007ebc108eb4f36b1d1dd9da97ecb308ab7fded89a2659 ; sys_platform == 'win32' and python_version == '3.11' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", - "torch @ https://download.pytorch.org/whl/xpu/torch-2.12.0%2Bxpu-cp312-cp312-win_amd64.whl#sha256=7f1d40febf2b8724adf4ff23866897d87478cc43de2a20f7776dc00be334c464 ; sys_platform == 'win32' and python_version == '3.12' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", - "torch @ https://download.pytorch.org/whl/xpu/torch-2.12.0%2Bxpu-cp313-cp313-win_amd64.whl#sha256=32770e2613df26e2c81ae64ea001b2ca12b8d152231285caff9b5f963a21ad75 ; sys_platform == 'win32' and python_version == '3.13' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", - - "bitsandbytes @ https://github.com/bitsandbytes-foundation/bitsandbytes/releases/download/continuous-release_main/bitsandbytes-1.33.7.preview-py3-none-manylinux_2_24_x86_64.whl ; ('linux' in sys_platform) and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", - "bitsandbytes @ https://github.com/bitsandbytes-foundation/bitsandbytes/releases/download/continuous-release_main/bitsandbytes-1.33.7.preview-py3-none-win_amd64.whl ; (sys_platform == 'win32') and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", - - "torchvision @ https://download.pytorch.org/whl/xpu/torchvision-0.27.0%2Bxpu-cp310-cp310-manylinux_2_28_x86_64.whl#sha256=0d517462caf6f5201c0d7c880f4ac431783c88fcc59b4587836da6c72a89509c ; platform_system == 'Linux' and python_version == '3.10' and platform_machine == 'x86_64'", - "torchvision @ https://download.pytorch.org/whl/xpu/torchvision-0.27.0%2Bxpu-cp311-cp311-manylinux_2_28_x86_64.whl#sha256=4b6feada86aa0bd606904b05898b33538106120d8ed706ba11d0011046534cb8 ; platform_system == 'Linux' and python_version == '3.11' and platform_machine == 'x86_64'", - "torchvision @ https://download.pytorch.org/whl/xpu/torchvision-0.27.0%2Bxpu-cp312-cp312-manylinux_2_28_x86_64.whl#sha256=e231819be0f87829c2344c909c1f0db9d6ae7d6faefe644a526a1a01d0c18d98 ; platform_system == 'Linux' and python_version == '3.12' and platform_machine == 'x86_64'", - "torchvision @ https://download.pytorch.org/whl/xpu/torchvision-0.27.0%2Bxpu-cp313-cp313-manylinux_2_28_x86_64.whl#sha256=8bc7d37515cea18af4c389d5fde58b1a9d76b015f2d87e4a7dc62ad50b1cc200 ; platform_system == 'Linux' and python_version == '3.13' and platform_machine == 'x86_64'", - "torchvision @ https://download.pytorch.org/whl/xpu/torchvision-0.27.0%2Bxpu-cp310-cp310-win_amd64.whl#sha256=65dbb041057dddfe369f29cfaab63f75563621779a23a7b1e2c0ff8a84d4376a ; sys_platform == 'win32' and python_version == '3.10' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", - "torchvision @ https://download.pytorch.org/whl/xpu/torchvision-0.27.0%2Bxpu-cp311-cp311-win_amd64.whl#sha256=df647445365924d69fe3bb2a15a7edfe5b63ef91e4ae69af11d93582985237a4 ; sys_platform == 'win32' and python_version == '3.11' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", - "torchvision @ https://download.pytorch.org/whl/xpu/torchvision-0.27.0%2Bxpu-cp312-cp312-win_amd64.whl#sha256=b0db3df0d0d154d18ba988ab420f1da2549f9372113ff54ff66e4ae3c7fe3bd0 ; sys_platform == 'win32' and python_version == '3.12' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", - "torchvision @ https://download.pytorch.org/whl/xpu/torchvision-0.27.0%2Bxpu-cp313-cp313-win_amd64.whl#sha256=c70850842068c43a0d50eaf139c25b6f6cc9b17a0dae70218c7e69edbee0bc80 ; sys_platform == 'win32' and python_version == '3.13' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", -] -intel-gpu-torch2120 = [ - "unsloth[intelgputorch2120]" -] -intel = [ - "unsloth[intelgputorch280]", -] -amd = [ - "unsloth[huggingfacenotorch]", - # 4-bit decode is unreliable on ROCm before 0.50.0, the first PyPI release - # carrying the full path: blocksize/warp decoupling (bnb #1887), fused SIMT - # GEMM on RDNA (#1979), RDNA3/4 workgroup fix (#2012). - "bitsandbytes>=0.50.0 ; ('linux' in sys_platform) and (platform_machine == 'AMD64' or platform_machine == 'x86_64' or platform_machine == 'aarch64')", - "bitsandbytes>=0.50.0 ; (sys_platform == 'win32') and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", -] -rocm702-torch280 = [ - "unsloth[amd]", - - "triton @ https://repo.radeon.com/rocm/manylinux/rocm-rel-7.0.2/triton-3.4.0%2Brocm7.0.2.gitf9e5bf54-cp311-cp311-linux_x86_64.whl ; platform_system == 'Linux' and python_version == '3.11' and platform_machine == 'x86_64'", - "triton @ https://repo.radeon.com/rocm/manylinux/rocm-rel-7.0.2/triton-3.4.0%2Brocm7.0.2.gitf9e5bf54-cp312-cp312-linux_x86_64.whl ; platform_system == 'Linux' and python_version == '3.12' and platform_machine == 'x86_64'", - "triton @ https://repo.radeon.com/rocm/manylinux/rocm-rel-7.0.2/triton-3.4.0%2Brocm7.0.2.gitf9e5bf54-cp313-cp313-linux_x86_64.whl ; platform_system == 'Linux' and python_version == '3.13' and platform_machine == 'x86_64'", - - "torch @ https://repo.radeon.com/rocm/manylinux/rocm-rel-7.0.2/torch-2.8.0%2Brocm7.0.2.lw.git245bf6ed-cp311-cp311-linux_x86_64.whl ; platform_system == 'Linux' and python_version == '3.11' and platform_machine == 'x86_64'", - "torch @ https://repo.radeon.com/rocm/manylinux/rocm-rel-7.0.2/torch-2.8.0%2Brocm7.0.2.lw.git245bf6ed-cp312-cp312-linux_x86_64.whl ; platform_system == 'Linux' and python_version == '3.12' and platform_machine == 'x86_64'", - "torch @ https://repo.radeon.com/rocm/manylinux/rocm-rel-7.0.2/torch-2.8.0%2Brocm7.0.2.lw.git245bf6ed-cp313-cp313-linux_x86_64.whl ; platform_system == 'Linux' and python_version == '3.13' and platform_machine == 'x86_64'", - - "torchvision @ https://repo.radeon.com/rocm/manylinux/rocm-rel-7.0.2/torchvision-0.23.0%2Brocm7.0.2.git824e8c87-cp311-cp311-linux_x86_64.whl ; platform_system == 'Linux' and python_version == '3.11' and platform_machine == 'x86_64'", - "torchvision @ https://repo.radeon.com/rocm/manylinux/rocm-rel-7.0.2/torchvision-0.23.0%2Brocm7.0.2.git824e8c87-cp312-cp312-linux_x86_64.whl ; platform_system == 'Linux' and python_version == '3.12' and platform_machine == 'x86_64'", - "torchvision @ https://repo.radeon.com/rocm/manylinux/rocm-rel-7.0.2/torchvision-0.23.0%2Brocm7.0.2.git824e8c87-cp313-cp313-linux_x86_64.whl ; platform_system == 'Linux' and python_version == '3.13' and platform_machine == 'x86_64'", -] -rocm72-torch291 = [ - "unsloth[amd]", - - "triton @ https://repo.radeon.com/rocm/manylinux/rocm-rel-7.2/triton-3.5.1%2Brocm7.2.0.gita272dfa8-cp310-cp310-linux_x86_64.whl ; platform_system == 'Linux' and python_version == '3.10' and platform_machine == 'x86_64'", - "triton @ https://repo.radeon.com/rocm/manylinux/rocm-rel-7.2/triton-3.5.1%2Brocm7.2.0.gita272dfa8-cp311-cp311-linux_x86_64.whl ; platform_system == 'Linux' and python_version == '3.11' and platform_machine == 'x86_64'", - "triton @ https://repo.radeon.com/rocm/manylinux/rocm-rel-7.2/triton-3.5.1%2Brocm7.2.0.gita272dfa8-cp312-cp312-linux_x86_64.whl ; platform_system == 'Linux' and python_version == '3.12' and platform_machine == 'x86_64'", - "triton @ https://repo.radeon.com/rocm/manylinux/rocm-rel-7.2/triton-3.5.1%2Brocm7.2.0.gita272dfa8-cp313-cp313-linux_x86_64.whl ; platform_system == 'Linux' and python_version == '3.13' and platform_machine == 'x86_64'", - - "torch @ https://repo.radeon.com/rocm/manylinux/rocm-rel-7.2/torch-2.9.1%2Brocm7.2.0.lw.git7e1940d4-cp310-cp310-linux_x86_64.whl ; platform_system == 'Linux' and python_version == '3.10' and platform_machine == 'x86_64'", - "torch @ https://repo.radeon.com/rocm/manylinux/rocm-rel-7.2/torch-2.9.1%2Brocm7.2.0.lw.git7e1940d4-cp311-cp311-linux_x86_64.whl ; platform_system == 'Linux' and python_version == '3.11' and platform_machine == 'x86_64'", - "torch @ https://repo.radeon.com/rocm/manylinux/rocm-rel-7.2/torch-2.9.1%2Brocm7.2.0.lw.git7e1940d4-cp312-cp312-linux_x86_64.whl ; platform_system == 'Linux' and python_version == '3.12' and platform_machine == 'x86_64'", - "torch @ https://repo.radeon.com/rocm/manylinux/rocm-rel-7.2/torch-2.9.1%2Brocm7.2.0.lw.git7e1940d4-cp313-cp313-linux_x86_64.whl ; platform_system == 'Linux' and python_version == '3.13' and platform_machine == 'x86_64'", - "torch @ https://repo.radeon.com/rocm/windows/rocm-rel-7.2/torch-2.9.1%2Brocmsdk20260116-cp312-cp312-win_amd64.whl ; sys_platform == 'win32' and python_version == '3.12'", - - "torchvision @ https://repo.radeon.com/rocm/manylinux/rocm-rel-7.2/torchvision-0.24.0%2Brocm7.2.0.gitb919bd0c-cp310-cp310-linux_x86_64.whl ; platform_system == 'Linux' and python_version == '3.10' and platform_machine == 'x86_64'", - "torchvision @ https://repo.radeon.com/rocm/manylinux/rocm-rel-7.2/torchvision-0.24.0%2Brocm7.2.0.gitb919bd0c-cp311-cp311-linux_x86_64.whl ; platform_system == 'Linux' and python_version == '3.11' and platform_machine == 'x86_64'", - "torchvision @ https://repo.radeon.com/rocm/manylinux/rocm-rel-7.2/torchvision-0.24.0%2Brocm7.2.0.gitb919bd0c-cp312-cp312-linux_x86_64.whl ; platform_system == 'Linux' and python_version == '3.12' and platform_machine == 'x86_64'", - "torchvision @ https://repo.radeon.com/rocm/manylinux/rocm-rel-7.2/torchvision-0.24.0%2Brocm7.2.0.gitb919bd0c-cp313-cp313-linux_x86_64.whl ; platform_system == 'Linux' and python_version == '3.13' and platform_machine == 'x86_64'", - "torchvision @ https://repo.radeon.com/rocm/windows/rocm-rel-7.2/torchvision-0.24.1%2Brocmsdk20260116-cp312-cp312-win_amd64.whl ; sys_platform == 'win32' and python_version == '3.12'", -] -rocm711-torch291 = [ - "unsloth[amd]", - - "triton @ https://repo.radeon.com/rocm/manylinux/rocm-rel-7.1.1/triton-3.5.1%2Brocm7.1.1.gita272dfa8-cp310-cp310-linux_x86_64.whl ; platform_system == 'Linux' and python_version == '3.10' and platform_machine == 'x86_64'", - "triton @ https://repo.radeon.com/rocm/manylinux/rocm-rel-7.1.1/triton-3.5.1%2Brocm7.1.1.gita272dfa8-cp311-cp311-linux_x86_64.whl ; platform_system == 'Linux' and python_version == '3.11' and platform_machine == 'x86_64'", - "triton @ https://repo.radeon.com/rocm/manylinux/rocm-rel-7.1.1/triton-3.5.1%2Brocm7.1.1.gita272dfa8-cp312-cp312-linux_x86_64.whl ; platform_system == 'Linux' and python_version == '3.12' and platform_machine == 'x86_64'", - "triton @ https://repo.radeon.com/rocm/manylinux/rocm-rel-7.1.1/triton-3.5.1%2Brocm7.1.1.gita272dfa8-cp313-cp313-linux_x86_64.whl ; platform_system == 'Linux' and python_version == '3.13' and platform_machine == 'x86_64'", - - "torch @ https://repo.radeon.com/rocm/manylinux/rocm-rel-7.1.1/torch-2.9.1%2Brocm7.1.1.lw.git351ff442-cp310-cp310-linux_x86_64.whl ; platform_system == 'Linux' and python_version == '3.10' and platform_machine == 'x86_64'", - "torch @ https://repo.radeon.com/rocm/manylinux/rocm-rel-7.1.1/torch-2.9.1%2Brocm7.1.1.lw.git351ff442-cp311-cp311-linux_x86_64.whl ; platform_system == 'Linux' and python_version == '3.11' and platform_machine == 'x86_64'", - "torch @ https://repo.radeon.com/rocm/manylinux/rocm-rel-7.1.1/torch-2.9.1%2Brocm7.1.1.lw.git351ff442-cp312-cp312-linux_x86_64.whl ; platform_system == 'Linux' and python_version == '3.12' and platform_machine == 'x86_64'", - "torch @ https://repo.radeon.com/rocm/manylinux/rocm-rel-7.1.1/torch-2.9.1%2Brocm7.1.1.lw.git351ff442-cp313-cp313-linux_x86_64.whl ; platform_system == 'Linux' and python_version == '3.13' and platform_machine == 'x86_64'", - - "torchvision @ https://repo.radeon.com/rocm/manylinux/rocm-rel-7.1.1/torchvision-0.24.0%2Brocm7.1.1.gitb919bd0c-cp310-cp310-linux_x86_64.whl ; platform_system == 'Linux' and python_version == '3.10' and platform_machine == 'x86_64'", - "torchvision @ https://repo.radeon.com/rocm/manylinux/rocm-rel-7.1.1/torchvision-0.24.0%2Brocm7.1.1.gitb919bd0c-cp311-cp311-linux_x86_64.whl ; platform_system == 'Linux' and python_version == '3.11' and platform_machine == 'x86_64'", - "torchvision @ https://repo.radeon.com/rocm/manylinux/rocm-rel-7.1.1/torchvision-0.24.0%2Brocm7.1.1.gitb919bd0c-cp312-cp312-linux_x86_64.whl ; platform_system == 'Linux' and python_version == '3.12' and platform_machine == 'x86_64'", - "torchvision @ https://repo.radeon.com/rocm/manylinux/rocm-rel-7.1.1/torchvision-0.24.0%2Brocm7.1.1.gitb919bd0c-cp313-cp313-linux_x86_64.whl ; platform_system == 'Linux' and python_version == '3.13' and platform_machine == 'x86_64'", -] -rocm72-torch2100 = [ - "unsloth[amd]", - - "triton @ https://repo.radeon.com/rocm/manylinux/rocm-rel-7.2/triton-3.6.0%2Brocm7.2.0.gitba5c1517-cp310-cp310-linux_x86_64.whl ; platform_system == 'Linux' and python_version == '3.10' and platform_machine == 'x86_64'", - "triton @ https://repo.radeon.com/rocm/manylinux/rocm-rel-7.2/triton-3.6.0%2Brocm7.2.0.gitba5c1517-cp311-cp311-linux_x86_64.whl ; platform_system == 'Linux' and python_version == '3.11' and platform_machine == 'x86_64'", - "triton @ https://repo.radeon.com/rocm/manylinux/rocm-rel-7.2/triton-3.6.0%2Brocm7.2.0.gitba5c1517-cp312-cp312-linux_x86_64.whl ; platform_system == 'Linux' and python_version == '3.12' and platform_machine == 'x86_64'", - "triton @ https://repo.radeon.com/rocm/manylinux/rocm-rel-7.2/triton-3.6.0%2Brocm7.2.0.gitba5c1517-cp313-cp313-linux_x86_64.whl ; platform_system == 'Linux' and python_version == '3.13' and platform_machine == 'x86_64'", - - "torch @ https://repo.radeon.com/rocm/manylinux/rocm-rel-7.2/torch-2.10.0%2Brocm7.2.0.lw.gitb6ee5fde-cp310-cp310-linux_x86_64.whl ; platform_system == 'Linux' and python_version == '3.10' and platform_machine == 'x86_64'", - "torch @ https://repo.radeon.com/rocm/manylinux/rocm-rel-7.2/torch-2.10.0%2Brocm7.2.0.lw.gitb6ee5fde-cp311-cp311-linux_x86_64.whl ; platform_system == 'Linux' and python_version == '3.11' and platform_machine == 'x86_64'", - "torch @ https://repo.radeon.com/rocm/manylinux/rocm-rel-7.2/torch-2.10.0%2Brocm7.2.0.lw.gitb6ee5fde-cp312-cp312-linux_x86_64.whl ; platform_system == 'Linux' and python_version == '3.12' and platform_machine == 'x86_64'", - "torch @ https://repo.radeon.com/rocm/manylinux/rocm-rel-7.2/torch-2.10.0%2Brocm7.2.0.lw.gitb6ee5fde-cp313-cp313-linux_x86_64.whl ; platform_system == 'Linux' and python_version == '3.13' and platform_machine == 'x86_64'", - - "torchvision @ https://repo.radeon.com/rocm/manylinux/rocm-rel-7.2/torchvision-0.25.0%2Brocm7.2.0.git82df5f59-cp310-cp310-linux_x86_64.whl ; platform_system == 'Linux' and python_version == '3.10' and platform_machine == 'x86_64'", - "torchvision @ https://repo.radeon.com/rocm/manylinux/rocm-rel-7.2/torchvision-0.25.0%2Brocm7.2.0.git82df5f59-cp311-cp311-linux_x86_64.whl ; platform_system == 'Linux' and python_version == '3.11' and platform_machine == 'x86_64'", - "torchvision @ https://repo.radeon.com/rocm/manylinux/rocm-rel-7.2/torchvision-0.25.0%2Brocm7.2.0.git82df5f59-cp312-cp312-linux_x86_64.whl ; platform_system == 'Linux' and python_version == '3.12' and platform_machine == 'x86_64'", - "torchvision @ https://repo.radeon.com/rocm/manylinux/rocm-rel-7.2/torchvision-0.25.0%2Brocm7.2.0.git82df5f59-cp313-cp313-linux_x86_64.whl ; platform_system == 'Linux' and python_version == '3.13' and platform_machine == 'x86_64'", - "unsloth[audio-torch210]", -] -rocm711-torch2100 = [ - "unsloth[amd]", - - "triton @ https://repo.radeon.com/rocm/manylinux/rocm-rel-7.1.1/triton-3.6.0%2Brocm7.1.1.gitba5c1517-cp310-cp310-linux_x86_64.whl ; platform_system == 'Linux' and python_version == '3.10' and platform_machine == 'x86_64'", - "triton @ https://repo.radeon.com/rocm/manylinux/rocm-rel-7.1.1/triton-3.6.0%2Brocm7.1.1.gitba5c1517-cp311-cp311-linux_x86_64.whl ; platform_system == 'Linux' and python_version == '3.11' and platform_machine == 'x86_64'", - "triton @ https://repo.radeon.com/rocm/manylinux/rocm-rel-7.1.1/triton-3.6.0%2Brocm7.1.1.gitba5c1517-cp312-cp312-linux_x86_64.whl ; platform_system == 'Linux' and python_version == '3.12' and platform_machine == 'x86_64'", - "triton @ https://repo.radeon.com/rocm/manylinux/rocm-rel-7.1.1/triton-3.6.0%2Brocm7.1.1.gitba5c1517-cp313-cp313-linux_x86_64.whl ; platform_system == 'Linux' and python_version == '3.13' and platform_machine == 'x86_64'", - - "torch @ https://repo.radeon.com/rocm/manylinux/rocm-rel-7.1.1/torch-2.10.0%2Brocm7.1.1.lw.gitd9556b05-cp310-cp310-linux_x86_64.whl ; platform_system == 'Linux' and python_version == '3.10' and platform_machine == 'x86_64'", - "torch @ https://repo.radeon.com/rocm/manylinux/rocm-rel-7.1.1/torch-2.10.0%2Brocm7.1.1.lw.gitd9556b05-cp311-cp311-linux_x86_64.whl ; platform_system == 'Linux' and python_version == '3.11' and platform_machine == 'x86_64'", - "torch @ https://repo.radeon.com/rocm/manylinux/rocm-rel-7.1.1/torch-2.10.0%2Brocm7.1.1.lw.gitd9556b05-cp312-cp312-linux_x86_64.whl ; platform_system == 'Linux' and python_version == '3.12' and platform_machine == 'x86_64'", - "torch @ https://repo.radeon.com/rocm/manylinux/rocm-rel-7.1.1/torch-2.10.0%2Brocm7.1.1.lw.gitd9556b05-cp313-cp313-linux_x86_64.whl ; platform_system == 'Linux' and python_version == '3.13' and platform_machine == 'x86_64'", - - "torchvision @ https://repo.radeon.com/rocm/manylinux/rocm-rel-7.1.1/torchvision-0.25.0%2Brocm7.1.1.git82df5f59-cp310-cp310-linux_x86_64.whl ; platform_system == 'Linux' and python_version == '3.10' and platform_machine == 'x86_64'", - "torchvision @ https://repo.radeon.com/rocm/manylinux/rocm-rel-7.1.1/torchvision-0.25.0%2Brocm7.1.1.git82df5f59-cp311-cp311-linux_x86_64.whl ; platform_system == 'Linux' and python_version == '3.11' and platform_machine == 'x86_64'", - "torchvision @ https://repo.radeon.com/rocm/manylinux/rocm-rel-7.1.1/torchvision-0.25.0%2Brocm7.1.1.git82df5f59-cp312-cp312-linux_x86_64.whl ; platform_system == 'Linux' and python_version == '3.12' and platform_machine == 'x86_64'", - "torchvision @ https://repo.radeon.com/rocm/manylinux/rocm-rel-7.1.1/torchvision-0.25.0%2Brocm7.1.1.git82df5f59-cp313-cp313-linux_x86_64.whl ; platform_system == 'Linux' and python_version == '3.13' and platform_machine == 'x86_64'", - "unsloth[audio-torch210]", ] [project.urls] @@ -1404,5 +826,5 @@ ignore = [ # Narrow the default test discovery so `pytest` from the repo root # does NOT pick up the GPU-heavy tests under tests/python, tests/qlora, # etc. The CI security job runs `pytest tests/security` explicitly. -testpaths = ["tests/security"] pythonpath = ["."] +testpaths = ["tests/security"] diff --git a/scripts/profile_startup.py b/scripts/profile_startup.py deleted file mode 100644 index 937d007ac1..0000000000 --- a/scripts/profile_startup.py +++ /dev/null @@ -1,377 +0,0 @@ -#!/usr/bin/env python3 -# SPDX-License-Identifier: AGPL-3.0-only -# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 - -"""Measure where Unsloth Studio's startup time goes, per platform. - -Nothing measured this before: the backend logs "lifespan startup completed in X ms" -but no test or CI job asserted a budget, and studio_test_kit discards the elapsed -time of its /healthz poll. A first local run (Linux, warm cache, fast server CPU) -found `import main` alone costs 6.6s before the server can bind, dominated by eager -module-level imports pulled in by the `routes` package: - - torch 1930 ms self - unsloth_zoo 914 ms self - routes 779 ms self - transformers 524 ms self - -Phases measured: - import `python -X importtime -c "import main"`, top cumulative + per-package self - spawn process start -> first byte on stdout - healthz process start -> /api/health (or /healthz) answers 200 - lifespan the backend's own "lifespan startup completed in X ms" log line - -Usage: - python scripts/profile_startup.py --repeats 3 --json out.json - python scripts/profile_startup.py --import-only # no server, no port needed - -Exit code is 0 unless --max-healthz-seconds is given and exceeded. -""" - -from __future__ import annotations - -import argparse -import json -import math -import os -import platform -import re -import shutil -import socket -import statistics -import subprocess -import sys -import threading -import time -import urllib.error -import urllib.request -from pathlib import Path - -REPO_ROOT = Path(__file__).resolve().parents[1] -BACKEND = REPO_ROOT / "studio" / "backend" - -_IMPORTTIME_RE = re.compile(r"import time:\s+(\d+)\s+\|\s+(\d+)\s+\|(\s*)(\S.*)") - - -def _free_port() -> int: - with socket.socket() as s: - s.bind(("127.0.0.1", 0)) - return int(s.getsockname()[1]) - - -def profile_imports(python: str, top: int = 15) -> dict: - """Cumulative and self import cost for the backend's module graph. - - Run in a subprocess with -X importtime: the numbers are only meaningful for a - cold interpreter, and importing in-process would measure a warm sys.modules. - """ - proc = subprocess.run( - [python, "-X", "importtime", "-c", "import sys; sys.path.insert(0, '.'); import main"], - cwd = BACKEND, - capture_output = True, - text = True, - timeout = 900, - ) - rows = [] - for line in proc.stderr.splitlines(): - m = _IMPORTTIME_RE.match(line) - if m: - rows.append((int(m.group(1)), int(m.group(2)), m.group(4).strip())) - if not rows: - return {"ok": False, "error": (proc.stderr or proc.stdout)[-2000:]} - if proc.returncode != 0: - # Rows survive up to the failure, so any total from a partial graph is wrong. - return { - "ok": False, - "error": (proc.stderr or proc.stdout)[-2000:], - "partial_rows": len(rows), - } - - by_cum = sorted(rows, key = lambda r: -r[1]) - # Total comes from the `main` row, not by_cum[0]: -X importtime also prints the - # interpreter's own startup graph (`site`), which can outrank a trivial main. - main_row = next((r for r in reversed(rows) if r[2] == "main"), None) - if main_row is None: - return { - "ok": False, - "error": "no `import main` row in -X importtime output\n" - + (proc.stderr or proc.stdout)[-2000:], - } - self_by_pkg: dict[str, int] = {} - for self_us, _cum, name in rows: - pkg = name.split(".")[0] - self_by_pkg[pkg] = self_by_pkg.get(pkg, 0) + self_us - - return { - "ok": True, - "total_seconds": round(main_row[1] / 1e6, 3), - "top_cumulative": [ - {"module": n, "seconds": round(c / 1e6, 3)} for _s, c, n in by_cum[:top] - ], - "self_by_package_ms": { - k: round(v / 1000) for k, v in sorted(self_by_pkg.items(), key = lambda x: -x[1])[:top] - }, - } - - -def _terminate_tree(proc: subprocess.Popen) -> None: - """Stop the server AND its children, which on Windows are a separate process. - - CI profiles `Scripts/unsloth.exe`, a distlib launcher stub that CreateProcess's - the venv python and waits, so terminate() reaps the stub only: the real backend - keeps the inherited stdout handle, the reader thread never sees EOF, and - --repeats strands one server per iteration on the shared UNSLOTH_STUDIO_HOME. - taskkill /T walks the tree, as unsloth_cli/commands/start.py already does. - """ - if proc.poll() is not None: - return - if os.name == "nt": - try: - killed = subprocess.run( - ["taskkill", "/PID", str(proc.pid), "/T", "/F"], - capture_output = True, - timeout = 30, - check = False, - ) - if killed.returncode == 0: - return - except Exception: - # taskkill missing or timed out; fall through so the stub still dies. - pass - # check=False: a nonzero taskkill does not raise, so fall through as well. - proc.terminate() - - -def profile_launch( - bin_path: str, - port: int, - timeout_s: int = 300, -) -> dict: - """Spawn the backend the way the desktop app does and time it to first 200.""" - log_lines: list[str] = [] - first_byte: list[float] = [] - t0 = time.perf_counter() - proc = subprocess.Popen( - [bin_path, "studio", "--api-only", "-H", "127.0.0.1", "-p", str(port)], - cwd = REPO_ROOT, - stdout = subprocess.PIPE, - stderr = subprocess.STDOUT, - text = True, - bufsize = 1, - ) - - def _drain() -> None: - # Runs alongside the health polling: the first read timestamps the spawn - # phase, and an undrained pipe blocks the backend before it binds. - for line in proc.stdout: - if not first_byte: - first_byte.append(time.perf_counter() - t0) - log_lines.append(line.rstrip("\n")) - - reader = threading.Thread(target = _drain, daemon = True) - reader.start() - - t_healthz = None - deadline = t0 + timeout_s - try: - while time.perf_counter() < deadline: - if proc.poll() is not None: - break - if t_healthz is None: - for url in ( - f"http://127.0.0.1:{port}/api/health", - f"http://127.0.0.1:{port}/healthz", - ): - try: - with urllib.request.urlopen(url, timeout = 2) as r: - if r.status == 200: - t_healthz = time.perf_counter() - t0 - break - except (urllib.error.URLError, OSError, TimeoutError): - pass - if t_healthz is not None: - break - time.sleep(0.25) - finally: - _terminate_tree(proc) - try: - # Safe: the reader drains the pipe, so the child cannot block on write(). - proc.wait(timeout = 30) - except subprocess.TimeoutExpired: - proc.kill() - proc.wait() - reader.join(timeout = 10) - - t_first_byte = first_byte[0] if first_byte else None - lifespan_ms = None - for line in log_lines: - m = re.search(r"lifespan startup completed in ([\d.]+)ms", line) - if m: - lifespan_ms = float(m.group(1)) - return { - "spawn_seconds": round(t_first_byte, 3) if t_first_byte is not None else None, - "healthz_seconds": round(t_healthz, 3) if t_healthz is not None else None, - "lifespan_ms": lifespan_ms, - "reached_healthz": t_healthz is not None, - "log_tail": log_lines[-25:], - } - - -def python_version_of(python: str) -> str: - """Version of the interpreter that runs the imports, not the one running us. - - --python points at the installed Studio venv while this script runs under the - runner's system python, so platform.python_version() would label it wrong. - """ - if python == sys.executable: - return platform.python_version() - try: - proc = subprocess.run( - [python, "-c", "import platform; print(platform.python_version())"], - capture_output = True, - text = True, - timeout = 60, - ) - if proc.returncode == 0 and proc.stdout.strip(): - return proc.stdout.strip() - except (OSError, subprocess.SubprocessError): - pass - return "unknown" - - -def find_bin() -> str | None: - home = os.environ.get("UNSLOTH_STUDIO_HOME") or str(Path.home() / ".unsloth" / "studio") - names = ["unsloth.exe", "unsloth"] if platform.system() == "Windows" else ["unsloth"] - subdirs = ["unsloth_studio/Scripts", "unsloth_studio/bin", "bin", "Scripts"] - for sd in subdirs: - for n in names: - p = Path(home) / sd / n - if p.exists(): - return str(p) - return shutil.which("unsloth") - - -def main(argv: list[str]) -> int: - ap = argparse.ArgumentParser( - description = __doc__, formatter_class = argparse.RawDescriptionHelpFormatter - ) - ap.add_argument( - "--repeats", - type = int, - default = 1, - help = "launch repeats; the median is reported (imports are measured once)", - ) - ap.add_argument( - "--python", - default = sys.executable, - help = "interpreter used for the import profile (default: this one)", - ) - ap.add_argument("--bin", help = "path to the unsloth CLI (default: autodetect)") - ap.add_argument( - "--import-only", - action = "store_true", - help = "skip the server phases (no install needed beyond the deps)", - ) - ap.add_argument( - "--max-healthz-seconds", - type = float, - help = "fail if the median time to a healthy port exceeds this", - ) - ap.add_argument("--json", help = "write the full report here") - a = ap.parse_args(argv) - # range(0) launches nothing, leaving the budget check with nothing to fail on. - if a.repeats < 1: - ap.error("--repeats must be at least 1") - # Same reason: --import-only never launches anything. - if a.import_only and a.max_healthz_seconds is not None: - ap.error("--max-healthz-seconds cannot be combined with --import-only") - # nan and inf parse fine as floats but `med > budget` is then always False, - # so the gate would report success without ever bounding anything. - if a.max_healthz_seconds is not None and not math.isfinite(a.max_healthz_seconds): - ap.error("--max-healthz-seconds must be a finite number") - - report: dict = { - "platform": platform.system().lower(), - "machine": platform.machine(), - "python": python_version_of(a.python), - "cpu_count": os.cpu_count(), - } - - print("== import graph ==") - report["imports"] = profile_imports(a.python) - imp = report["imports"] - if imp.get("ok"): - print(f" import main: {imp['total_seconds']}s") - for row in imp["top_cumulative"][:8]: - print(f" {row['seconds']:7.3f}s {row['module']}") - print(" self time by package (ms):") - for k, v in list(imp["self_by_package_ms"].items())[:8]: - print(f" {v:8} ms {k}") - else: - print(f" FAILED: {imp.get('error', '')[:400]}") - - if not a.import_only: - bin_path = a.bin or find_bin() - if not bin_path: - print( - "== launch == skipped: no unsloth CLI found " - "(set UNSLOTH_STUDIO_HOME or pass --bin)" - ) - report["launch"] = {"skipped": "no unsloth CLI found"} - else: - print(f"== launch == {bin_path}") - runs = [] - for i in range(a.repeats): - r = profile_launch(bin_path, _free_port()) - runs.append(r) - print( - f" run {i + 1}: healthz={r['healthz_seconds']}s " - f"lifespan={r['lifespan_ms']}ms reached={r['reached_healthz']}" - ) - got = [r["healthz_seconds"] for r in runs if r["healthz_seconds"] is not None] - report["launch"] = { - "runs": runs, - "failed_runs": sum(1 for r in runs if not r["reached_healthz"]), - "healthz_median_seconds": round(statistics.median(got), 3) if got else None, - "healthz_max_seconds": round(max(got), 3) if got else None, - } - if got: - print( - f" median time to healthy port: {report['launch']['healthz_median_seconds']}s" - ) - - if a.json: - Path(a.json).write_text(json.dumps(report, indent = 2), encoding = "utf-8") - print(f"\nwrote {a.json}") - - if a.max_healthz_seconds is not None: - launch = report.get("launch") or {} - med = launch.get("healthz_median_seconds") - failed = launch.get("failed_runs") or 0 - if failed: - # Failed launches fail the budget; dropping them would keep only the fast ones. - print( - f"::error::startup regression: {failed} of {len(launch.get('runs') or [])} " - f"launches never became healthy within the timeout" - ) - return 1 - if med is None: - # Nothing measured: exiting 0 would pass a requested budget without a - # single health request, so fail closed. - print( - "::error::startup regression: no healthz measurement, so the " - f"{a.max_healthz_seconds}s budget was never checked " - f"({launch.get('skipped') or 'launch phase produced no runs'})" - ) - return 1 - elif med > a.max_healthz_seconds: - print( - f"::error::startup regression: {med}s median to a healthy port " - f"exceeds the {a.max_healthz_seconds}s budget" - ) - return 1 - return 0 - - -if __name__ == "__main__": - raise SystemExit(main(sys.argv[1:])) diff --git a/studio/backend/auth/authentication.py b/studio/backend/auth/authentication.py index 2e9520827e..dfb8fc513e 100644 --- a/studio/backend/auth/authentication.py +++ b/studio/backend/auth/authentication.py @@ -11,12 +11,11 @@ import jwt from .storage import ( API_KEY_PREFIX, - credential_generation, get_jwt_secret, get_user_and_secret, load_jwt_secret, save_refresh_token, - validate_api_key_with_credential, + validate_api_key, verify_refresh_token, ) @@ -55,14 +54,11 @@ def create_access_token( expires_delta: Optional[timedelta] = None, *, desktop: bool = False, - secret: Optional[str] = None, ) -> str: """ Create a signed JWT for the given subject (e.g. username). - Valid across restarts: the signing secret is stored in SQLite. Callers that - already verified a credential pass ``secret`` so a rotation landing mid-request - cannot sign the token with the credential that just replaced it. + Valid across restarts: the signing secret is stored in SQLite. """ to_encode = {"sub": subject} if desktop: @@ -73,7 +69,7 @@ def create_access_token( to_encode.update({"exp": expire}) return jwt.encode( to_encode, - secret if secret is not None else _get_secret_for_subject(subject), + _get_secret_for_subject(subject), algorithm = ALGORITHM, ) @@ -100,28 +96,15 @@ def is_desktop_access_token(token: str) -> bool: return payload.get("sub") == subject and payload.get("desktop") is True -def create_refresh_token( - subject: str, - *, - desktop: bool = False, - secret: Optional[str] = None, -) -> str: +def create_refresh_token(subject: str, *, desktop: bool = False) -> str: """ Create a random refresh token, store its hash in SQLite, and return it. Refresh tokens are opaque (not JWTs); expire after REFRESH_TOKEN_EXPIRE_DAYS. - ``secret`` stamps the token with the credential version the caller verified, - so a rotation cannot leave a token minted from the replaced credential valid. """ token = secrets.token_urlsafe(48) expires_at = datetime.now(timezone.utc) + timedelta(days = REFRESH_TOKEN_EXPIRE_DAYS) - save_refresh_token( - token, - subject, - expires_at.isoformat(), - is_desktop = desktop, - secret_gen = credential_generation(secret) if secret is not None else None, - ) + save_refresh_token(token, subject, expires_at.isoformat(), is_desktop = desktop) return token @@ -154,22 +137,7 @@ def reload_secret() -> None: async def get_current_subject(credentials: HTTPAuthorizationCredentials = Depends(security)) -> str: """Validate JWT and require the password-change flow to be completed.""" - subject, _generation = await _get_current_credential( - credentials, - allow_password_change = False, - ) - return subject - - -async def get_current_credential( - credentials: HTTPAuthorizationCredentials = Depends(security), -) -> Tuple[str, Optional[str]]: - """As get_current_subject, but also returns the credential generation. - - For routes that persist a new credential and must not do so on behalf of one - a concurrent reset has revoked. - """ - return await _get_current_credential( + return await _get_current_subject( credentials, allow_password_change = False, ) @@ -190,49 +158,27 @@ async def get_current_subject_allow_password_change( credentials: HTTPAuthorizationCredentials = Depends(security), ) -> str: """Validate JWT but allow access to the password-change endpoint.""" - subject, _generation = await _get_current_credential( + return await _get_current_subject( credentials, allow_password_change = True, ) - return subject -# The literal the examples ship with; pasted unedited more often than a revoked key. -API_KEY_PLACEHOLDER = f"{API_KEY_PREFIX}YOUR_KEY" - - -def _invalid_api_key_detail(token: str) -> str: - """Why the key failed. Only the example placeholder is called out; every real - key gets one indistinguishable message, so this leaks no key existence.""" - if token == API_KEY_PLACEHOLDER: - return ( - "This is the placeholder key from the example. Create an API key in " - f"Unsloth Studio under Settings > API and use it in place of {API_KEY_PLACEHOLDER}." - ) - return "Invalid or expired API key" - - -async def _get_current_credential( +async def _get_current_subject( credentials: HTTPAuthorizationCredentials, *, allow_password_change: bool -) -> Tuple[str, Optional[str]]: - """Validate the bearer and return ``(subject, credential generation)``. - - The generation is the credential version this request actually authenticated - against. Routes that persist new credentials must bind their write to it, or - a reset landing mid-request would bless what it just revoked. - """ +) -> str: + """FastAPI dependency: validate the JWT and return the subject. Use on protected routes.""" token = credentials.credentials # --- API key path (sk-unsloth-...) --- if token.startswith(API_KEY_PREFIX): - verified = validate_api_key_with_credential(token) - if verified is None: + username = validate_api_key(token) + if username is None: raise HTTPException( status_code = status.HTTP_401_UNAUTHORIZED, - detail = _invalid_api_key_detail(token), + detail = "Invalid or expired API key", ) - username, secret = verified - return username, credential_generation(secret) + return username # --- JWT path --- subject = _decode_subject_without_verification(token) @@ -263,7 +209,7 @@ async def _get_current_credential( status_code = status.HTTP_403_FORBIDDEN, detail = "Password change required", ) - return subject, credential_generation(jwt_secret) + return subject except jwt.InvalidTokenError: raise HTTPException( status_code = status.HTTP_401_UNAUTHORIZED, diff --git a/studio/backend/auth/storage.py b/studio/backend/auth/storage.py index 6cf4d44834..5f80ad89a3 100644 --- a/studio/backend/auth/storage.py +++ b/studio/backend/auth/storage.py @@ -9,7 +9,6 @@ import ipaddress import os import secrets import sqlite3 -import tempfile import threading from datetime import datetime, timezone from typing import Optional, Tuple @@ -31,97 +30,6 @@ _BOOTSTRAP_PW_PATH = DB_PATH.parent / ".bootstrap_password" _bootstrap_password: Optional[str] = None -def _bootstrap_file_bytes(password: str) -> bytes: - """Exact on-disk form: the secret plus one LF. - - Bytes, not text: text mode writes CRLF on Windows, and `$(cat ...)` strips - the LF but leaves the CR attached to the credential. - """ - return (password + "\n").encode("utf-8") - - -def _persist_bootstrap_password(password: str) -> None: - """Atomically write the bootstrap password 0600, LF terminated on every OS. - - A partial write would destroy the only plaintext recovery credential. - """ - fd, tmp_name = tempfile.mkstemp( - prefix = f".{_BOOTSTRAP_PW_PATH.name}.", dir = _BOOTSTRAP_PW_PATH.parent - ) - try: - with os.fdopen(fd, "wb") as f: - f.write(_bootstrap_file_bytes(password)) - try: - os.chmod(tmp_name, 0o600) - except OSError: - pass - os.replace(tmp_name, _BOOTSTRAP_PW_PATH) - except BaseException: - try: - os.unlink(tmp_name) - except OSError: - pass - raise - - -def _normalise_bootstrap_file(raw: bytes, password: str) -> None: - """Append the LF a pre-newline release left off. - - Append-only, and only when the file is exactly the credential: - clear_bootstrap_password() may unlink or (when unlink fails, notably on - Windows while this descriptor is open) truncate through another descriptor - after we read, so a rewrite could restore revoked plaintext. An append - cannot: worst case is a lone "\\n" over a cleared file, which strips back to - no bootstrap password. Pre-newline releases wrote no terminator at all, so - that is the only shape in the wild; anything else reads fine, since every - reader strips, and is left alone. - """ - if raw != password.encode("utf-8"): - return - - # O_BINARY: without it Windows opens in text mode and turns the LF straight - # back into CRLF, the bug being fixed. - fd = os.open( - _BOOTSTRAP_PW_PATH, - os.O_WRONLY | os.O_APPEND | getattr(os, "O_BINARY", 0), - ) - try: - os.write(fd, b"\n") - try: - os.fchmod(fd, 0o600) - except (AttributeError, OSError): - # fchmod only reached Windows in 3.13. - pass - finally: - os.close(fd) - - -def _read_persisted_bootstrap_password() -> Optional[str]: - """Read the persisted password, normalising the file if it is malformed.""" - if not _BOOTSTRAP_PW_PATH.is_file(): - return None - - # No caller handles a raise, so an unreadable file has to mean "no bootstrap - # password", not a dead backend. We write UTF-8, so undecodable bytes are - # damage whose plaintext is worthless anyway. - try: - raw = _BOOTSTRAP_PW_PATH.read_bytes() - password = raw.decode("utf-8").strip() - except (OSError, UnicodeDecodeError): - return None - if not password: - return None - - # Older releases wrote no terminator; best-effort, a read-only auth dir must - # not fail startup. - if raw != _bootstrap_file_bytes(password): - try: - _normalise_bootstrap_file(raw, password) - except OSError: - pass - return password - - def generate_bootstrap_password() -> str: """Generate a 4-word diceware passphrase and persist it to disk. @@ -135,10 +43,10 @@ def generate_bootstrap_password() -> str: return _bootstrap_password # Persisted from a previous run? - persisted = _read_persisted_bootstrap_password() - if persisted: - _bootstrap_password = persisted - return _bootstrap_password + if _BOOTSTRAP_PW_PATH.is_file(): + _bootstrap_password = _BOOTSTRAP_PW_PATH.read_text(encoding = "utf-8").strip() + if _bootstrap_password: + return _bootstrap_password # First startup: generate a fresh passphrase. import diceware @@ -149,7 +57,11 @@ def generate_bootstrap_password() -> str: # Persist so the same passphrase survives restarts until password change. ensure_dir(_BOOTSTRAP_PW_PATH.parent) - _persist_bootstrap_password(_bootstrap_password) + _BOOTSTRAP_PW_PATH.write_text(_bootstrap_password, encoding = "utf-8") + try: + os.chmod(_BOOTSTRAP_PW_PATH, 0o600) + except OSError: + pass return _bootstrap_password @@ -160,14 +72,13 @@ def get_bootstrap_password() -> Optional[str]: def _load_bootstrap_password() -> Optional[str]: - """Load an existing bootstrap password without creating one. - - Upgrades take this path, not generate_bootstrap_password() - (ensure_default_admin short-circuits once the admin row exists), so it has - to normalise too. - """ + """Load an existing bootstrap password without creating one.""" global _bootstrap_password - _bootstrap_password = _read_persisted_bootstrap_password() + _bootstrap_password = None + if _BOOTSTRAP_PW_PATH.is_file(): + bootstrap_password = _BOOTSTRAP_PW_PATH.read_text(encoding = "utf-8").strip() + if bootstrap_password: + _bootstrap_password = bootstrap_password return _bootstrap_password @@ -186,7 +97,7 @@ def clear_bootstrap_password() -> None: # Removal failed (Windows AV, read-only auth dir). The hash is already # committed, so don't fail the change -- but truncate the file so its # stale plaintext can't be re-seeded by generate_bootstrap_password() - # if auth.db is ever recreated. + # if a later reset-password deletes auth.db and re-validates it. try: _BOOTSTRAP_PW_PATH.write_text("", encoding = "utf-8") cleared = True @@ -221,31 +132,6 @@ def _hash_token(token: str) -> str: return hashlib.sha256(token.encode("utf-8")).hexdigest() -class CredentialRotated(Exception): - """A password reset revoked the credential this request authenticated with.""" - - -def credential_generation(jwt_secret: str) -> str: - """Marker for the credential version a refresh token was issued under. - - Every password change rotates ``jwt_secret``, so a token stamped with the - previous one is rejected even if it was inserted after the revoking DELETE. - """ - return hashlib.sha256(jwt_secret.encode("utf-8")).hexdigest() - - -def _current_secret(conn: sqlite3.Connection, username: str) -> Optional[str]: - row = conn.execute( - "SELECT jwt_secret FROM auth_user WHERE username = ?", (username,) - ).fetchone() - return row["jwt_secret"] if row else None - - -def _current_generation(conn: sqlite3.Connection, username: str) -> Optional[str]: - secret = _current_secret(conn, username) - return credential_generation(secret) if secret is not None else None - - def get_connection() -> sqlite3.Connection: """Get a connection to the auth database, creating tables if needed.""" ensure_dir(DB_PATH.parent) @@ -289,8 +175,7 @@ def get_connection() -> sqlite3.Connection: token_hash TEXT NOT NULL, username TEXT NOT NULL, expires_at TEXT NOT NULL, - is_desktop INTEGER NOT NULL DEFAULT 0, - secret_gen TEXT + is_desktop INTEGER NOT NULL DEFAULT 0 ); """ ) @@ -329,8 +214,6 @@ def get_connection() -> sqlite3.Connection: refresh_columns = {row["name"] for row in conn.execute("PRAGMA table_info(refresh_tokens)")} if "is_desktop" not in refresh_columns: conn.execute("ALTER TABLE refresh_tokens ADD COLUMN is_desktop INTEGER NOT NULL DEFAULT 0") - if "secret_gen" not in refresh_columns: - conn.execute("ALTER TABLE refresh_tokens ADD COLUMN secret_gen TEXT") conn.commit() return conn @@ -704,22 +587,12 @@ def update_password( new_password: str, *, revoke_refresh_tokens: bool = False, - expect_password_hash: Optional[str] = None, -) -> Optional[str]: +) -> bool: """Update password, clear first-login requirement, rotate JWT secret. - Returns the new JWT secret, or None when nothing was updated. Callers that - mint tokens for the caller must sign with the returned secret: re-reading it - would pick up a reset that landed between this commit and the mint. - ``revoke_refresh_tokens`` deletes the user's refresh tokens in the SAME transaction: a separate delete could fail after the password commit and leave a pre-change token still able to mint access tokens. - - ``expect_password_hash`` makes the write conditional on the credential the - caller verified still being current, so a request that checked the old - password cannot overwrite a reset that landed while it was in flight. - Returns False when the credential moved underneath it. """ from .hashing import hash_password @@ -727,32 +600,21 @@ def update_password( jwt_secret = secrets.token_urlsafe(64) conn = get_connection() try: - if expect_password_hash is None: - cursor = conn.execute( - """ - UPDATE auth_user - SET password_salt = ?, password_hash = ?, jwt_secret = ?, must_change_password = 0 - WHERE username = ? - """, - (salt, pwd_hash, jwt_secret, username), - ) - else: - cursor = conn.execute( - """ - UPDATE auth_user - SET password_salt = ?, password_hash = ?, jwt_secret = ?, must_change_password = 0 - WHERE username = ? AND password_hash = ? - """, - (salt, pwd_hash, jwt_secret, username, expect_password_hash), - ) + cursor = conn.execute( + """ + UPDATE auth_user + SET password_salt = ?, password_hash = ?, jwt_secret = ?, must_change_password = 0 + WHERE username = ? + """, + (salt, pwd_hash, jwt_secret, username), + ) if revoke_refresh_tokens and cursor.rowcount > 0: conn.execute("DELETE FROM refresh_tokens WHERE username = ?", (username,)) conn.commit() if cursor.rowcount > 0: clear_bootstrap_password() clear_desktop_secret() - return jwt_secret - return None + return cursor.rowcount > 0 finally: conn.close() @@ -763,49 +625,35 @@ def save_refresh_token( expires_at: str, *, is_desktop: bool = False, - secret_gen: Optional[str] = None, ) -> None: """ Store a hashed refresh token with its associated username and expiry. - - ``secret_gen`` binds the token to a credential version; it defaults to the - current one, and callers that already verified a credential must pass the - version they verified rather than let this re-read a rotated one. """ token_hash = _hash_token(token) conn = get_connection() try: - if secret_gen is None: - secret_gen = _current_generation(conn, username) conn.execute( """ - INSERT INTO refresh_tokens (token_hash, username, expires_at, is_desktop, secret_gen) - VALUES (?, ?, ?, ?, ?) + INSERT INTO refresh_tokens (token_hash, username, expires_at, is_desktop) + VALUES (?, ?, ?, ?) """, - (token_hash, username, expires_at, int(is_desktop), secret_gen), + (token_hash, username, expires_at, int(is_desktop)), ) conn.commit() finally: conn.close() -def consume_refresh_token(token: str) -> Optional[Tuple[str, bool, str]]: +def consume_refresh_token(token: str) -> Optional[Tuple[str, bool]]: """Atomically validate-and-delete a refresh token for single-use rotation. DELETE RETURNING fuses validate and delete into one statement so two - concurrent refresh requests cannot both consume the same token. Returns - ``(username, is_desktop, jwt_secret)``; the caller must mint the replacement - tokens against that secret so a rotation landing mid-refresh cannot issue a - post-rotation session from a pre-rotation token. + concurrent refresh requests cannot both consume the same token. """ token_hash = _hash_token(token) now = datetime.now(timezone.utc).isoformat() conn = get_connection() try: - # One transaction with the delete: an unstamped legacy row has no - # generation to compare, so reading the credential after committing would - # hand a reset's new secret to a token issued before it. - conn.execute("BEGIN IMMEDIATE") conn.execute( "DELETE FROM refresh_tokens WHERE expires_at < ?", (now,), @@ -814,21 +662,15 @@ def consume_refresh_token(token: str) -> Optional[Tuple[str, bool, str]]: """ DELETE FROM refresh_tokens WHERE token_hash = ? AND expires_at >= ? - RETURNING username, is_desktop, secret_gen + RETURNING username, is_desktop """, (token_hash, now), ) row = cur.fetchone() - if row is None: - conn.commit() - return None - secret = _current_secret(conn, row["username"]) conn.commit() - if secret is None: + if row is None: return None - if row["secret_gen"] is not None and row["secret_gen"] != credential_generation(secret): - return None - return row["username"], bool(row["is_desktop"]), secret + return row["username"], bool(row["is_desktop"]) finally: conn.close() @@ -852,7 +694,7 @@ def verify_refresh_token(token: str) -> Optional[Tuple[str, bool]]: cur = conn.execute( """ - SELECT id, username, expires_at, is_desktop, secret_gen FROM refresh_tokens + SELECT id, username, expires_at, is_desktop FROM refresh_tokens WHERE token_hash = ? """, (token_hash,), @@ -861,13 +703,6 @@ def verify_refresh_token(token: str) -> Optional[Tuple[str, bool]]: if row is None: return None - if row["secret_gen"] is not None and row["secret_gen"] != _current_generation( - conn, row["username"] - ): - conn.execute("DELETE FROM refresh_tokens WHERE id = ?", (row["id"],)) - conn.commit() - return None - # Check expiry expires_at = datetime.fromisoformat(row["expires_at"]) if datetime.now(timezone.utc) > expires_at: @@ -912,41 +747,30 @@ def create_desktop_secret() -> str: conn.close() -def validate_desktop_secret_with_credential(raw_secret: str) -> Optional[Tuple[str, str]]: - """Validate the desktop secret and return ``(username, jwt_secret)``. - - Both reads share one transaction so the returned secret is the credential - version the desktop secret was checked against; a reset landing mid-request - then invalidates the tokens minted from it rather than blessing them. - """ +def validate_desktop_secret(raw_secret: str) -> Optional[str]: + """Return the real admin username when the desktop secret matches.""" if not raw_secret.startswith(DESKTOP_SECRET_PREFIX): return None + if get_user_and_secret(DEFAULT_ADMIN_USERNAME) is None: + return None secret_hash = _pbkdf2_desktop_secret(raw_secret) conn = get_connection() try: - conn.execute("BEGIN") - row = conn.execute( + cur = conn.execute( "SELECT value FROM app_secrets WHERE key = ?", (_DESKTOP_SECRET_HASH_KEY,), - ).fetchone() - if row is None or not secrets.compare_digest(row["value"], secret_hash): + ) + row = cur.fetchone() + if row is None: return None - jwt_secret = _current_secret(conn, DEFAULT_ADMIN_USERNAME) - if jwt_secret is None: + if not secrets.compare_digest(row["value"], secret_hash): return None - return DEFAULT_ADMIN_USERNAME, jwt_secret + return DEFAULT_ADMIN_USERNAME finally: - conn.rollback() conn.close() -def validate_desktop_secret(raw_secret: str) -> Optional[str]: - """Return the real admin username when the desktop secret matches.""" - verified = validate_desktop_secret_with_credential(raw_secret) - return verified[0] if verified else None - - def clear_desktop_secret() -> None: """Remove backend-side desktop auth state.""" conn = get_connection() @@ -972,7 +796,6 @@ def create_api_key( name: str, expires_at: Optional[str] = None, internal: bool = False, - expect_gen: Optional[str] = None, ) -> Tuple[str, dict]: """Create a new API key for *username*. @@ -981,10 +804,6 @@ def create_api_key( Pass ``internal=True`` for keys minted by workflows (e.g. data-recipe runs) that should not appear in user-facing key listings. - - ``expect_gen`` ties the insert to the credential generation the request - authenticated under, so a session revoked by a concurrent password reset - cannot mint a key that outlives it. Raises ``CredentialRotated`` if it moved. """ raw_key = API_KEY_PREFIX + secrets.token_hex(16) key_hash = _pbkdf2_api_key(raw_key) @@ -993,12 +812,6 @@ def create_api_key( conn = get_connection() try: - if expect_gen is not None: - conn.execute("BEGIN IMMEDIATE") - if _current_generation(conn, username) != expect_gen: - raise CredentialRotated( - "The credential this request authenticated with was revoked." - ) conn.execute( """ INSERT INTO api_keys (username, key_prefix, key_hash, name, created_at, expires_at, is_internal) @@ -1087,25 +900,15 @@ def revoke_internal_api_key(key_id: int) -> bool: def validate_api_key(raw_key: str) -> Optional[str]: - """Validate *raw_key* and return the owning username, or ``None``.""" - verified = validate_api_key_with_credential(raw_key) - return verified[0] if verified else None + """Validate *raw_key* and return the owning username, or ``None``. - -def validate_api_key_with_credential(raw_key: str) -> Optional[Tuple[str, str]]: - """Validate *raw_key* and return ``(username, jwt_secret)``, or ``None``. - - Also updates ``last_used_at`` on success. The key check and the credential - read share one write transaction, so the returned version is the one the key - was actually valid under: a reset committing right after cannot have its new - generation handed to a request the key it revoked authenticated. + Also updates ``last_used_at`` on success. """ cache_id = _api_key_cache_id(raw_key) cached_hash = _api_key_hash_cache.get(cache_id) key_hash = cached_hash if cached_hash is not None else _pbkdf2_api_key(raw_key) conn = get_connection() try: - conn.execute("BEGIN IMMEDIATE") cur = conn.execute( "SELECT id, username, is_active, expires_at FROM api_keys WHERE key_hash = ?", (key_hash,), @@ -1125,15 +928,11 @@ def validate_api_key_with_credential(raw_key: str) -> Optional[Tuple[str, str]]: expires = datetime.fromisoformat(row["expires_at"]) if datetime.now(timezone.utc) > expires: return None - secret = _current_secret(conn, row["username"]) - if secret is None: - return None conn.execute( "UPDATE api_keys SET last_used_at = ? WHERE id = ?", (datetime.now(timezone.utc).isoformat(), row["id"]), ) conn.commit() - return row["username"], secret + return row["username"] finally: - conn.rollback() conn.close() diff --git a/studio/backend/cloudflare_tunnel.py b/studio/backend/cloudflare_tunnel.py index f7967e2faa..78fce0c70a 100644 --- a/studio/backend/cloudflare_tunnel.py +++ b/studio/backend/cloudflare_tunnel.py @@ -310,7 +310,6 @@ class CloudflareTunnel: stderr = subprocess.STDOUT, stdin = subprocess.DEVNULL, text = True, - encoding = "utf-8", errors = "replace", bufsize = 1, **_windows_hidden_kwargs(), diff --git a/studio/backend/core/data_recipe/local_callable_validators.py b/studio/backend/core/data_recipe/local_callable_validators.py index 143895d781..ffc81669ae 100644 --- a/studio/backend/core/data_recipe/local_callable_validators.py +++ b/studio/backend/core/data_recipe/local_callable_validators.py @@ -257,8 +257,6 @@ def _run_oxc_batch( cwd = str(_OXC_TOOL_DIR), input = json.dumps(payload), text = True, - encoding = "utf-8", - errors = "replace", capture_output = True, check = False, env = env, diff --git a/studio/backend/core/inference/_vulkan_probe.py b/studio/backend/core/inference/_vulkan_probe.py index 4bfefc21ce..706346daad 100644 --- a/studio/backend/core/inference/_vulkan_probe.py +++ b/studio/backend/core/inference/_vulkan_probe.py @@ -6,14 +6,12 @@ Run in a short-lived subprocess (``python _vulkan_probe.py ``) so the Vulkan instance never lives in the long-running backend process. Loads the bundled ggml Vulkan backend from ```` and prints one -``\\t\\t\\t\\t`` line per device to -stdout. Indices are ggml's own Vulkan device ordinals, which need not match -nvidia-smi order. ``is_igpu`` (from ggml's device type) is ``1`` for an -integrated GPU sharing system RAM. ``total_bytes`` is the device-local heap; -the reader uses it to reserve absolute headroom on a discrete card (parity -with the CUDA/ROCm fit) and ignores it for an iGPU, whose "VRAM" is shared -system RAM. ``name`` is ggml's device description (the marketing name, e.g. -"AMD Radeon RX 9070 XT"); empty when the registry lookup fails. +``\\t\\t\\t`` line per device to stdout. +Indices are ggml's own Vulkan device ordinals, which need not match nvidia-smi +order. ``is_igpu`` (from ggml's device type) is ``1`` for an integrated GPU +sharing system RAM. ``total_bytes`` is the device-local heap; the reader uses +it to reserve absolute headroom on a discrete card (parity with the CUDA/ROCm +fit) and ignores it for an iGPU, whose "VRAM" is shared system RAM. Uses only the standard library so it stays runnable as a bare script. """ @@ -26,30 +24,15 @@ import sys _GGML_BACKEND_DEVICE_TYPE_IGPU = 2 -def _igpu_flags_and_names(base, lib, count: int) -> tuple[list[bool], list[str]]: - """Per-device integrated-GPU flags and descriptions via ggml's backend registry. +def _igpu_flags(base, lib, count: int) -> list[bool]: + """Per-device integrated-GPU flags via ggml's backend registry. The Vulkan reg enumerates devices in the same order as ``ggml_backend_vk_get_device_memory`` (each context uses ``ctx->device = - i``), so reg index == device ordinal. Returns all-False / empty-name on any - failure so the reader never over-caps a discrete card and the memory - readings still get through. + i``), so reg index == device ordinal. Returns all-False on any failure so + the reader never over-caps a discrete card. """ flags = [False] * count - names = [""] * count - - # The name lookup is bound OUTSIDE the type-detection try: a ggml-base - # without ggml_backend_dev_description (older/custom build) must degrade to - # unnamed devices, not abort before the iGPU flags are read (which would - # count an iGPU's shared RAM as VRAM). - describe = None - try: - base.ggml_backend_dev_description.restype = ctypes.c_char_p - base.ggml_backend_dev_description.argtypes = [ctypes.c_void_p] - describe = base.ggml_backend_dev_description - except Exception: - pass - try: lib.ggml_backend_vk_reg.restype = ctypes.c_void_p lib.ggml_backend_vk_reg.argtypes = [] @@ -62,31 +45,17 @@ def _igpu_flags_and_names(base, lib, count: int) -> tuple[list[bool], list[str]] reg = lib.ggml_backend_vk_reg() if not reg: - return flags, names + return flags dev_count = base.ggml_backend_reg_dev_count(reg) for i in range(min(count, dev_count)): dev = base.ggml_backend_reg_dev_get(reg, i) if dev: flags[i] = base.ggml_backend_dev_type(dev) == _GGML_BACKEND_DEVICE_TYPE_IGPU - if describe is not None: - try: - desc = describe(dev) - if desc: - # Tabs/newlines would corrupt the line protocol; - # spaces are safe. - names[i] = ( - desc.decode("utf-8", errors = "replace") - .replace("\t", " ") - .replace("\n", " ") - .strip() - ) - except Exception: - pass except Exception: - # Best-effort: any failure degrades to "discrete"/"unnamed" so the - # memory readings still get through instead of crashing the probe. + # Best-effort: any failure degrades to "discrete" so the memory + # readings still get through instead of crashing the probe. pass - return flags, names + return flags def main() -> int: @@ -94,14 +63,6 @@ def main() -> int: return 0 bindir = sys.argv[1] - # Device names can be non-ASCII (localized drivers); the platform-default - # stdout encoding (e.g. cp1252) would raise on them and lose the whole - # inventory. The reader decodes UTF-8 with the same error mode. - try: - sys.stdout.reconfigure(encoding = "utf-8", errors = "replace") - except Exception: - pass - # Hold add_dll_directory's handle for the rest of main() (the documented # idiom) so bindir stays on the search path while the sibling ggml DLLs # resolve below. @@ -135,12 +96,12 @@ def main() -> int: ] count = lib.ggml_backend_vk_get_device_count() - igpu, names = _igpu_flags_and_names(base, lib, count) + igpu = _igpu_flags(base, lib, count) rows = [] for i in range(count): free, total = ctypes.c_size_t(0), ctypes.c_size_t(0) lib.ggml_backend_vk_get_device_memory(i, ctypes.byref(free), ctypes.byref(total)) - rows.append("%d\t%d\t%d\t%d\t%s" % (i, free.value, int(igpu[i]), total.value, names[i])) + rows.append("%d\t%d\t%d\t%d" % (i, free.value, int(igpu[i]), total.value)) sys.stdout.write("\n".join(rows)) return 0 diff --git a/studio/backend/core/inference/anthropic_compat.py b/studio/backend/core/inference/anthropic_compat.py index a32e372d73..34445cc58e 100644 --- a/studio/backend/core/inference/anthropic_compat.py +++ b/studio/backend/core/inference/anthropic_compat.py @@ -172,136 +172,6 @@ def anthropic_messages_to_openai( return result -_ANTHROPIC_SCHEMA_CLIENT_TOOL_PARAMETERS = { - "bash": { - "type": "object", - "properties": { - "command": {"type": "string"}, - "restart": {"type": "boolean"}, - }, - "anyOf": [ - {"required": ["command"]}, - {"properties": {"restart": {"const": True}}, "required": ["restart"]}, - ], - }, - "text_editor": { - "type": "object", - "properties": { - "command": { - "type": "string", - "enum": ["view", "str_replace", "create", "insert"], - }, - "path": {"type": "string"}, - "view_range": { - "type": "array", - "items": {"type": "integer"}, - "minItems": 2, - "maxItems": 2, - }, - "old_str": {"type": "string"}, - "new_str": {"type": "string"}, - "file_text": {"type": "string"}, - "insert_line": {"type": "integer"}, - "insert_text": {"type": "string"}, - }, - "required": ["command", "path"], - }, - "computer": { - "type": "object", - "properties": { - "action": {"type": "string"}, - "coordinate": { - "type": "array", - "items": {"type": "integer"}, - "minItems": 2, - "maxItems": 2, - }, - "text": {"type": "string"}, - "duration": {"type": "number"}, - "scroll_direction": {"type": "string"}, - "scroll_amount": {"type": "integer"}, - "start_coordinate": { - "type": "array", - "items": {"type": "integer"}, - "minItems": 2, - "maxItems": 2, - }, - "key": {"type": "string"}, - }, - "required": ["action"], - "additionalProperties": True, - }, - "memory": { - "type": "object", - "properties": { - "command": { - "type": "string", - "enum": ["view", "create", "str_replace", "insert", "delete", "rename"], - }, - "path": {"type": "string"}, - "view_range": { - "type": "array", - "items": {"type": "integer"}, - "minItems": 2, - "maxItems": 2, - }, - "file_text": {"type": "string"}, - "old_str": {"type": "string"}, - "new_str": {"type": "string"}, - "insert_line": {"type": "integer"}, - "insert_text": {"type": "string"}, - "old_path": {"type": "string"}, - "new_path": {"type": "string"}, - }, - "required": ["command"], - }, -} - -_ANTHROPIC_SCHEMA_CLIENT_TOOL_DESCRIPTIONS = { - "bash": "Run a command in the caller-owned persistent bash session, or restart it.", - "text_editor": "View, create, or edit files in the caller-owned filesystem.", - "computer": "Interact with the caller-owned computer using an action and its parameters.", - "memory": "Store and retrieve files in the caller-owned persistent memory directory.", -} - - -def anthropic_schema_client_tool_kind(tool) -> Optional[str]: - """Return the kind of a schema-less Anthropic client tool, if recognized.""" - td = tool if isinstance(tool, dict) else tool.model_dump() - if td.get("input_schema") is not None: - return None - type_ = td.get("type") - if not isinstance(type_, str): - return None - kind, separator, version = type_.rpartition("_") - if ( - separator - and kind in _ANTHROPIC_SCHEMA_CLIENT_TOOL_PARAMETERS - and len(version) == 8 - and version.isdigit() - ): - return kind - return None - - -def _anthropic_schema_client_tool_parameters(td: dict, kind: str) -> dict: - parameters = _ANTHROPIC_SCHEMA_CLIENT_TOOL_PARAMETERS[kind] - if kind != "text_editor": - return parameters - - version = td["type"].rpartition("_")[2] - commands = list(parameters["properties"]["command"]["enum"]) - if version < "20250429": - commands.append("undo_edit") - return { - **parameters, - "properties": { - **parameters["properties"], - "command": {**parameters["properties"]["command"], "enum": commands}, - }, - } - - def anthropic_tools_to_openai(tools: list) -> list[dict]: """Convert Anthropic client tools to OpenAI function-tool format.""" result = [] @@ -309,9 +179,6 @@ def anthropic_tools_to_openai(tools: list) -> list[dict]: td = t if isinstance(t, dict) else t.model_dump() name = td.get("name") input_schema = td.get("input_schema") - schema_client_kind = anthropic_schema_client_tool_kind(td) - if schema_client_kind is not None: - input_schema = _anthropic_schema_client_tool_parameters(td, schema_client_kind) if not name or input_schema is None: continue result.append( @@ -319,8 +186,7 @@ def anthropic_tools_to_openai(tools: list) -> list[dict]: "type": "function", "function": { "name": name, - "description": td.get("description") - or _ANTHROPIC_SCHEMA_CLIENT_TOOL_DESCRIPTIONS.get(schema_client_kind, ""), + "description": td.get("description", ""), "parameters": input_schema, }, } diff --git a/studio/backend/core/inference/api_monitor.py b/studio/backend/core/inference/api_monitor.py index b637ba56d1..f76a38576f 100644 --- a/studio/backend/core/inference/api_monitor.py +++ b/studio/backend/core/inference/api_monitor.py @@ -5,7 +5,6 @@ from __future__ import annotations -import os import threading import time import uuid @@ -19,14 +18,6 @@ _MAX_PROMPT_CHARS = 12000 _MAX_REPLY_CHARS = 12000 _PREVIEW_CHARS = 360 -# Opt-in startup kill switch for Studio's in-memory API monitor. -_DISABLE_ENV = "UNSLOTH_STUDIO_DISABLE_API_MONITOR" -_TRUE_VALUES = frozenset({"1", "true", "yes", "on"}) - - -def _api_monitor_disabled() -> bool: - return os.environ.get(_DISABLE_ENV, "").strip().lower() in _TRUE_VALUES - def _trim(text: Optional[str], limit: int) -> str: if not text: @@ -61,13 +52,6 @@ class ApiMonitorEntry: total_tokens: Optional[int] = None total_tokens_authoritative: bool = False error: Optional[str] = None - # "request" (HTTP call) or "lifecycle" (model load/unload: event/reason, not a prompt; shared). - kind: str = "request" - event: Optional[str] = None - reason: Optional[str] = None - shared: bool = False - # 0-100 for a running download row; None when not applicable. - progress: Optional[float] = None def snapshot(self, *, include_details: bool = True) -> dict[str, Any]: duration_ms = None @@ -101,10 +85,6 @@ class ApiMonitorEntry: "completion_tokens": self.completion_tokens, "total_tokens": self.total_tokens, "error": self.error, - "kind": self.kind, - "event": self.event, - "reason": self.reason, - "progress": self.progress, } if include_details: payload["prompt"] = self.prompt @@ -113,16 +93,10 @@ class ApiMonitorEntry: class ApiMonitor: - def __init__( - self, - max_entries: int = _MAX_ENTRIES, - *, - enabled: bool = True, - ): + def __init__(self, max_entries: int = _MAX_ENTRIES): self._entries: deque[ApiMonitorEntry] = deque() self._max_entries = max(0, max_entries) self._lock = threading.Lock() - self._enabled = enabled def start( self, @@ -134,8 +108,6 @@ class ApiMonitor: context_length: Optional[int] = None, subject: Optional[str] = None, ) -> str: - if not self._enabled: - return "" now = time.time() entry = ApiMonitorEntry( id = f"apireq_{uuid.uuid4().hex[:12]}", @@ -155,75 +127,6 @@ class ApiMonitor: self._trim_terminal_locked() return entry.id - def record_lifecycle( - self, - *, - event: str, - model: str, - reason: Optional[str] = None, - running: bool = False, - ) -> str: - """Record a model load/unload alongside the request traffic that caused it. - - ``running=True`` opens the row for the caller to close with :meth:`finish` / - :meth:`fail`; an unload is terminal on arrival. Rows are shared (visible to - every subject) and share the request retention budget. - """ - if not self._enabled: - return "" - now = time.time() - entry = ApiMonitorEntry( - id = f"apievt_{uuid.uuid4().hex[:12]}", - endpoint = f"model.{event}", - method = "", - model = model or "default", - prompt = "", - status = "running" if running else "completed", - started_at = now, - updated_at = now, - started_monotonic = time.monotonic(), - finished_at = None if running else now, - finished_monotonic = None if running else time.monotonic(), - kind = "lifecycle", - event = event, - reason = reason, - shared = True, - ) - with self._lock: - self._entries.appendleft(entry) - self._trim_terminal_locked() - return entry.id - - def relabel(self, entry_id: Optional[str], model: str) -> None: - """Rename an open lifecycle row once the load resolves its real id: up front - the caller only has the load path, which may be an HF snapshot dir.""" - if not entry_id or not model: - return - with self._lock: - entry = self._find_locked(entry_id) - if entry is not None: - entry.model = model - entry.updated_at = time.time() - - def set_progress(self, entry_id: Optional[str], progress: Optional[float]) -> None: - """Update an open download row's percentage (clamped to 0-100).""" - if not entry_id or progress is None: - return - with self._lock: - entry = self._find_locked(entry_id) - if entry is not None and entry.status == "running": - entry.progress = min(100.0, max(0.0, float(progress))) - entry.updated_at = time.time() - - def discard(self, entry_id: Optional[str]) -> None: - """Drop a row that turned out not to be an event (an already-satisfied load).""" - if not entry_id: - return - with self._lock: - entry = self._find_locked(entry_id) - if entry is not None: - self._entries.remove(entry) - def append_reply(self, entry_id: Optional[str], text: str) -> None: if not entry_id or not text: return @@ -309,18 +212,6 @@ class ApiMonitor: self._entries.appendleft(entry) self._trim_terminal_locked() - def fail_open(self, entry_id: Optional[str], error: str) -> None: - """Fail only a still-open row: unlike :meth:`fail`, a catch-all in a - ``finally`` cannot stamp an error onto a request that already succeeded.""" - if not entry_id: - return - with self._lock: - entry = self._find_locked(entry_id) - if entry is None or entry.finished_at is not None: - return - # Same lock as the check, so a finish() cannot land in between. - self._fail_locked(entry, error) - def fail(self, entry_id: Optional[str], error: str) -> None: if not entry_id: return @@ -333,18 +224,15 @@ class ApiMonitor: if error: entry.error = _trim(error, 1000) return - self._fail_locked(entry, error) - - def _fail_locked(self, entry: ApiMonitorEntry, error: str) -> None: - now = time.time() - entry.status = "error" - entry.error = _trim(error, 1000) - entry.updated_at = now - entry.finished_at = now - entry.finished_monotonic = time.monotonic() - self._entries.remove(entry) - self._entries.appendleft(entry) - self._trim_terminal_locked() + now = time.time() + entry.status = "error" + entry.error = _trim(error, 1000) + entry.updated_at = now + entry.finished_at = now + entry.finished_monotonic = time.monotonic() + self._entries.remove(entry) + self._entries.appendleft(entry) + self._trim_terminal_locked() def snapshot( self, @@ -356,7 +244,7 @@ class ApiMonitor: return [ entry.snapshot(include_details = include_details) for entry in self._entries - if self._visible(entry, subject) + if subject is None or entry.subject == subject ] def get( @@ -369,29 +257,22 @@ class ApiMonitor: entry = self._find_locked(entry_id) if entry is None: return None - if not self._visible(entry, subject): + if subject is not None and entry.subject != subject: return None return entry.snapshot(include_details = True) def active_count(self, *, subject: Optional[str] = None) -> int: - # Lifecycle rows show as "running" while loading but are not in-flight API requests. with self._lock: return sum( 1 for entry in self._entries - if entry.status == "running" - and entry.kind != "lifecycle" - and (subject is None or entry.subject == subject) + if entry.status == "running" and (subject is None or entry.subject == subject) ) def clear(self) -> None: with self._lock: self._entries.clear() - @staticmethod - def _visible(entry: ApiMonitorEntry, subject: Optional[str]) -> bool: - return subject is None or entry.subject == subject or entry.shared - def _find_locked(self, entry_id: str) -> Optional[ApiMonitorEntry]: for entry in self._entries: if entry.id == entry_id: @@ -411,4 +292,4 @@ class ApiMonitor: self._entries = kept -api_monitor = ApiMonitor(enabled = not _api_monitor_disabled()) +api_monitor = ApiMonitor() diff --git a/studio/backend/core/inference/chat_template_helpers.py b/studio/backend/core/inference/chat_template_helpers.py index 3a8463855b..528c059fbc 100644 --- a/studio/backend/core/inference/chat_template_helpers.py +++ b/studio/backend/core/inference/chat_template_helpers.py @@ -326,58 +326,6 @@ def _normalize_tool_call_arguments(messages: list) -> list: return out if mutated else messages -def _take_tool_result(pending: list, call_id) -> Optional[dict]: - if call_id: - for i, result in enumerate(pending): - if result.get("tool_call_id") == call_id: - return pending.pop(i) - for i, result in enumerate(pending): - if not result.get("tool_call_id"): - return pending.pop(i) - return None - - -def _split_parallel_tool_calls(messages: list) -> list: - """Llama 3.x templates render one call per message, so split parallel calls - into consecutive single-call messages, each followed by its own result.""" - if not any(isinstance(m, dict) and len(m.get("tool_calls") or ()) > 1 for m in messages): - return messages - - out: list = [] - i = 0 - total = len(messages) - while i < total: - msg = messages[i] - calls = msg.get("tool_calls") if isinstance(msg, dict) else None - if not calls or len(calls) <= 1: - out.append(msg) - i += 1 - continue - - # Tool results right after this message answer its calls. - j = i + 1 - pending: list = [] - while ( - j < total - and isinstance(messages[j], dict) - and messages[j].get("role") in ("tool", "ipython") - ): - pending.append(messages[j]) - j += 1 - - for idx, call in enumerate(calls): - piece = {**msg, "tool_calls": [call]} - if idx: - piece["content"] = "" - out.append(piece) - result = _take_tool_result(pending, call.get("id") if isinstance(call, dict) else None) - if result is not None: - out.append(result) - out.extend(pending) - i = j - return out - - def apply_chat_template_for_generation( tokenizer, messages: list, @@ -430,21 +378,13 @@ def apply_chat_template_for_generation( try: return _render(messages) except Exception: - # Retry with repairs applied cumulatively. Originals render first, so - # working templates stay byte-identical. - candidates: list = [] + # Strict tool templates reject the JSON-string ``arguments`` form via + # TypeError or a broad Jinja raise_exception, so retry with dicts coerced. + # Original messages render first, so working templates stay byte-identical. normalized = _normalize_tool_call_arguments(messages) - if normalized is not messages: - candidates.append(normalized) - split = _split_parallel_tool_calls(normalized) - if split is not normalized: - candidates.append(split) - for candidate in candidates: - try: - return _render(candidate) - except Exception: - continue - raise + if normalized is messages: + raise + return _render(normalized) def render_native_template( diff --git a/studio/backend/core/inference/inference.py b/studio/backend/core/inference/inference.py index e78bf1be8d..563a6732a1 100644 --- a/studio/backend/core/inference/inference.py +++ b/studio/backend/core/inference/inference.py @@ -567,7 +567,7 @@ class InferenceBackend: _meta_path = Path(config.path) / "export_metadata.json" try: if _meta_path.exists(): - _meta = json.loads(_meta_path.read_text(encoding = "utf-8-sig")) + _meta = json.loads(_meta_path.read_text(encoding = "utf-8")) if _meta.get("base_model"): processor_source = _meta["base_model"] except Exception: @@ -2281,13 +2281,8 @@ class InferenceBackend: except Exception as e: logger.warning(f"Could not fully reset model state for {model_name}: {e}") - def reset_generation_state(self, caller_cancel_event = None): - """Reset any cached generation state to prevent hanging after errors - - ``caller_cancel_event`` is accepted for signature parity with the - orchestrator, which uses it to drop a reset from a request that never - started. Nothing here cancels a live generation, so it is unused. - """ + def reset_generation_state(self): + """Reset any cached generation state to prevent hanging after errors""" try: # Clear cached state for ALL loaded models for model_name in self.models.keys(): diff --git a/studio/backend/core/inference/llama_admission.py b/studio/backend/core/inference/llama_admission.py index 7bf0dd7429..b6a939c87b 100644 --- a/studio/backend/core/inference/llama_admission.py +++ b/studio/backend/core/inference/llama_admission.py @@ -13,159 +13,37 @@ from __future__ import annotations import asyncio import os -import sys import threading from collections import deque from dataclasses import dataclass from typing import Deque, Optional -# dataclass(slots = True) halves per-instance overhead. Measured as perf-neutral -# here, not a speed win: it costs a little on construction and gains it back on -# access. It is 3.10+ and this package declares >=3.9, so gate it rather than -# dropping it outright. Empty on 3.9 means a plain dataclass. -_SLOTS = {"slots": True} if sys.version_info >= (3, 10) else {} - - -ADMISSION_CONTROL_ENV = "UNSLOTH_LLAMA_ADMISSION_CONTROL" -ADMISSION_QUEUE_TIMEOUT_ENV = "UNSLOTH_LLAMA_ADMISSION_QUEUE_TIMEOUT" -ADMISSION_KEEPALIVE_INTERVAL_ENV = "UNSLOTH_LLAMA_ADMISSION_KEEPALIVE_INTERVAL" -ADMISSION_MAX_QUEUE_ENV = "UNSLOTH_LLAMA_ADMISSION_MAX_QUEUE" -ADMISSION_QUEUE_PER_SLOT_ENV = "UNSLOTH_LLAMA_ADMISSION_QUEUE_PER_SLOT" - -# The UNSLOTH_OPENAI_COMPAT_* spellings predate this queue being shared with the -# Anthropic /v1/messages route (same llama-server slots). Still honored; the -# neutral name above wins when both are set. -_LEGACY_ENV = { - ADMISSION_CONTROL_ENV: "UNSLOTH_OPENAI_COMPAT_ADMISSION_CONTROL", - ADMISSION_QUEUE_TIMEOUT_ENV: "UNSLOTH_OPENAI_COMPAT_ADMISSION_QUEUE_TIMEOUT", - ADMISSION_KEEPALIVE_INTERVAL_ENV: "UNSLOTH_OPENAI_COMPAT_ADMISSION_KEEPALIVE_INTERVAL", - ADMISSION_MAX_QUEUE_ENV: "UNSLOTH_OPENAI_COMPAT_ADMISSION_MAX_QUEUE", -} +ADMISSION_CONTROL_ENV = "UNSLOTH_OPENAI_COMPAT_ADMISSION_CONTROL" +ADMISSION_QUEUE_TIMEOUT_ENV = "UNSLOTH_OPENAI_COMPAT_ADMISSION_QUEUE_TIMEOUT" +ADMISSION_KEEPALIVE_INTERVAL_ENV = "UNSLOTH_OPENAI_COMPAT_ADMISSION_KEEPALIVE_INTERVAL" +ADMISSION_MAX_QUEUE_ENV = "UNSLOTH_OPENAI_COMPAT_ADMISSION_MAX_QUEUE" DEFAULT_ADMISSION_ENABLED = True -# None: a queued request waits for its slot indefinitely rather than timing out. DEFAULT_ADMISSION_QUEUE_TIMEOUT_S = None DEFAULT_ADMISSION_KEEPALIVE_INTERVAL_S = 5.0 -# None: no absolute cap, the wait line is sized from the pool instead. -DEFAULT_ADMISSION_MAX_QUEUE = None -# Wait line = 16 x the serving slots, so it tracks --parallel (4 slots -> 64 -# waiters, 8 -> 128). Purely a memory guard; waiting itself is never timed out. -DEFAULT_ADMISSION_QUEUE_PER_SLOT = 16 -# Floor for the scaled line, so a 1-slot backend (plain `unsloth studio`, or any -# load downshifted to fit VRAM) keeps the depth it had before scaling existed -# rather than dropping to 16 and rejecting callers that used to queue. -DEFAULT_ADMISSION_MIN_QUEUE = 64 +DEFAULT_ADMISSION_MAX_QUEUE = 64 -def _executor_workers() -> int: - """Threads asyncio's default executor runs to_thread work on. - - Mirrors ThreadPoolExecutor's own default sizing, which is what - ``run_in_executor(None, ...)`` builds. 3.13 sizes it from - ``process_cpu_count()``, which honours CPU affinity and cgroup quotas; - ``cpu_count()`` would budget from the whole host inside a one-core container. - """ - cpus = getattr(os, "process_cpu_count", os.cpu_count)() or 1 - return min(32, cpus + 4) - - -def _executor_reserve(workers: int) -> int: - """Threads kept clear of parked approvals, for generation steps, stream - teardown and unrelated to_thread work. Scaled rather than flat: a flat count - would leave a 5-worker executor (one usable CPU) no budget at all. - """ - return max(2, workers // 8) - - -def _max_parked(capacity: int) -> int: - """How many holders may sit on an approval prompt with their slot given back. - - A pending prompt parks an executor thread (the loop blocks inside - to_thread(next, gen)) whether or not it parked its slot, the pool already - permits `capacity` of those, and every park admits one more, so budget only - what the executor has left over. Zero on a backend whose --parallel alone - fills it: the prompt then holds its slot, as it did before parking existed. - """ - workers = _executor_workers() - spare = workers - _executor_reserve(workers) - max(0, capacity) - # A quarter of the executor, floored at two while `spare` allows: a quarter of - # five is one, and one park cannot cover the two simultaneous prompts #7455 - # exists for. - return max(0, min(max(2, workers // 4), spare)) - - -# Process-wide, not per queue: there is one executor, and base_url takes a fresh -# port on every load, so a per-queue budget would hand the same allowance to each -# backend and to every reload, blind to the approvals parked on the old queue. -_PARK_LOCK = threading.Lock() -_parked_total = 0 - - -def _claim_park(limit: int) -> bool: - global _parked_total - with _PARK_LOCK: - if _parked_total >= limit: - return False - _parked_total += 1 - return True - - -def _drop_park() -> None: - global _parked_total - with _PARK_LOCK: - _parked_total = max(0, _parked_total - 1) - - -def _live_capacity(current: "LlamaAdmissionQueue") -> int: - """Slots across every backend still serving requests. - - One queue's capacity is the wrong denominator for a budget sized against the - one executor: a reload drains the old queue alongside the new one, and - prompts on both park threads. Idle queues hold nothing and are about to be - evicted. - """ - with _QUEUES_LOCK: - queues = list(_QUEUES.values()) - # is_idle takes each queue's own lock, so never while holding _QUEUES_LOCK. - total = sum(queue._capacity for queue in queues if queue is current or not queue.is_idle()) - return total if any(queue is current for queue in queues) else total + current._capacity - - -@dataclass(frozen = True, **_SLOTS) +@dataclass(frozen = True) class LlamaAdmissionConfig: enabled: bool = DEFAULT_ADMISSION_ENABLED queue_timeout_s: Optional[float] = DEFAULT_ADMISSION_QUEUE_TIMEOUT_S keepalive_interval_s: float = DEFAULT_ADMISSION_KEEPALIVE_INTERVAL_S max_queue: Optional[int] = DEFAULT_ADMISSION_MAX_QUEUE - queue_per_slot: Optional[int] = DEFAULT_ADMISSION_QUEUE_PER_SLOT - # Unconditional floor on the scaled line. The env path clears it when the - # operator sets QUEUE_PER_SLOT, so only the default multiplier is floored. - min_queue: Optional[int] = DEFAULT_ADMISSION_MIN_QUEUE - - def queue_limit(self, capacity: int) -> Optional[int]: - """How many callers may line up for a pool of ``capacity`` slots. - - An explicit ``max_queue`` wins; otherwise the line scales with the slots - so it follows ``--parallel``. The default multiplier is floored, so a - 1-slot backend does not end up shallower than it was before scaling. None - (or any non-positive setting) means an unbounded line. - """ - if self.max_queue is not None: - return self.max_queue if self.max_queue > 0 else None - if not self.queue_per_slot or self.queue_per_slot <= 0: - return None - scaled = self.queue_per_slot * max(1, capacity) - return max(self.min_queue, scaled) if self.min_queue else scaled -@dataclass(frozen = True, **_SLOTS) +@dataclass(frozen = True) class LlamaAdmissionSnapshot: key: str capacity: int active: int queued: int - free: int = 0 class LlamaAdmissionError(Exception): @@ -191,17 +69,8 @@ class LlamaAdmissionCancelled(LlamaAdmissionError): pass -def _raw_env(name: str) -> Optional[str]: - """Value for a canonical name, falling back to its legacy spelling.""" - value = os.environ.get(name) - if value is None or not value.strip(): - legacy = _LEGACY_ENV.get(name) - value = os.environ.get(legacy) if legacy else None - return value - - def _bool_env(name: str, default: bool) -> bool: - value = _raw_env(name) + value = os.environ.get(name) if value is None or not value.strip(): return default value = value.strip().lower() @@ -213,7 +82,7 @@ def _bool_env(name: str, default: bool) -> bool: def _optional_positive_float_env(name: str, default: Optional[float]) -> Optional[float]: - value = _raw_env(name) + value = os.environ.get(name) if value is None or not value.strip(): return default try: @@ -224,7 +93,7 @@ def _optional_positive_float_env(name: str, default: Optional[float]) -> Optiona def _positive_float_env(name: str, default: float) -> float: - value = _raw_env(name) + value = os.environ.get(name) if value is None or not value.strip(): return default try: @@ -234,38 +103,19 @@ def _positive_float_env(name: str, default: float) -> float: return parsed if parsed > 0 else default -def _queue_limits_from_env() -> tuple[Optional[int], Optional[int], Optional[int]]: - """(max_queue, queue_per_slot, min_queue) from the environment. - - An absolute MAX_QUEUE wins outright; MAX_QUEUE=0 asks for an unbounded line. - Unset leaves the per-slot multiplier in charge (itself 0 for unbounded). The - floor applies only to the default multiplier: setting QUEUE_PER_SLOT means - the operator wants that exact depth, however shallow. - """ - # Explicit means it parsed, not just that something was set: a typo falls back - # to the default multiplier, so it has to keep the default's floor too. - raw_per_slot = _raw_env(ADMISSION_QUEUE_PER_SLOT_ENV) +def _optional_positive_int_env(name: str, default: Optional[int]) -> Optional[int]: + value = os.environ.get(name) + if value is None or not value.strip(): + return default try: - per_slot = int((raw_per_slot or "").strip()) + parsed = int(value.strip()) except ValueError: - per_slot, min_queue = DEFAULT_ADMISSION_QUEUE_PER_SLOT, DEFAULT_ADMISSION_MIN_QUEUE - else: - per_slot, min_queue = (per_slot if per_slot > 0 else None), None - raw = _raw_env(ADMISSION_MAX_QUEUE_ENV) - if raw is None or not raw.strip(): - return None, per_slot, min_queue - try: - parsed = int(raw.strip()) - except ValueError: - return None, per_slot, min_queue - return (parsed, None, None) if parsed > 0 else (None, None, None) + return default + return parsed if parsed > 0 else None def llama_admission_config_from_env() -> LlamaAdmissionConfig: - max_queue, queue_per_slot, min_queue = _queue_limits_from_env() return LlamaAdmissionConfig( - queue_per_slot = queue_per_slot, - min_queue = min_queue, enabled = _bool_env(ADMISSION_CONTROL_ENV, DEFAULT_ADMISSION_ENABLED), queue_timeout_s = _optional_positive_float_env( ADMISSION_QUEUE_TIMEOUT_ENV, @@ -275,11 +125,14 @@ def llama_admission_config_from_env() -> LlamaAdmissionConfig: ADMISSION_KEEPALIVE_INTERVAL_ENV, DEFAULT_ADMISSION_KEEPALIVE_INTERVAL_S, ), - max_queue = max_queue, + max_queue = _optional_positive_int_env( + ADMISSION_MAX_QUEUE_ENV, + DEFAULT_ADMISSION_MAX_QUEUE, + ), ) -@dataclass(**_SLOTS) +@dataclass class _Waiter: loop: asyncio.AbstractEventLoop future: asyncio.Future @@ -288,130 +141,20 @@ class _Waiter: class LlamaAdmissionLease: - __slots__ = ("_queue", "_slot", "_released", "_release_lock", "_parked", "_budgeted") - - def __init__( - self, - queue: Optional["LlamaAdmissionQueue"], - slot: Optional[int] = None, - ): + def __init__(self, queue: Optional["LlamaAdmissionQueue"]): self._queue = queue - self._slot = slot self._released = False self._release_lock = threading.Lock() - self._parked = False - self._budgeted = False - - @property - def slot(self) -> Optional[int]: - """Pool slot this lease holds, or None when admission is disabled.""" - return self._slot - - def park(self) -> bool: - """Hand the slot back while this holder waits on something off the GPU. - - A run stopped on a tool approval prompt is not decoding, so holding its - slot would let unanswered prompts fill the pool while llama-server idles. - The lease itself stays valid: releasing it after a park is still correct. - - False when the park budget is spent and nothing was given back: the - caller keeps its slot across the prompt, as it did before parking - existed. Slower for whoever is behind it, but each freed slot admits - another run that can park too, on the executor the generators run on. - """ - queue = self._queue - with self._release_lock: - if queue is None or self._released or self._parked: - return False - # Under the lease lock so the decision and the handover cannot split. - # Nothing takes the queue lock then a lease lock, so this order is - # the only one in play. - if not queue.try_park(self._slot): - return False - self._parked = True - self._budgeted = True - self._slot = None - return True - - def _drop_budget(self) -> None: - """Give the executor budget back now the prompt wait is over. - - Separate from the queue's parked count, which lasts until the slot is - back: the executor thread is free the moment the answer arrives. Holding - the budget until the resume lands would refuse someone else's park for a - finished wait, and that someone holds the slot the resumer wants. - """ - with self._release_lock: - if not self._budgeted: - return - self._budgeted = False - _drop_park() - - def unpark(self) -> None: - """Drop the parked state without reclaiming a slot. - - For a holder that is tearing down: it will not decode again. Resuming - holders must use ``unpark_async``, which waits for a slot instead of - going back to llama-server past the admission limit. - """ - with self._release_lock: - if not self._parked: - return - self._parked = False - self._drop_budget() - if self._queue is not None: - self._queue.unpark() - - async def unpark_async( - self, - *, - cancel_event = None, - poll_s: float = 0.02, - ) -> None: - """Take a slot back, waiting until the pool has room. - - ``park`` gave the slot to a waiter, so by the time the user answers the - prompt someone else may be decoding in it. Resuming regardless put two - holders on a one-slot server. Gives up if the caller is cancelled, since - the holder is then leaving anyway and must not be stuck here. - """ - queue = self._queue - if queue is None or not self._parked: - return - # Before the wait, not after: the prompt is answered, so this holder is - # already off the executor and must not keep anyone else off it. - self._drop_budget() - slot = await queue.acquire_parked_slot(cancel_event = cancel_event, poll_s = poll_s) - stranded = None - with self._release_lock: - # release() may have run during the wait; it clears the flag and does - # the unpark itself, so only the caller that clears it here repeats one. - parked, self._parked = self._parked, False - if self._released: - # Released while waiting: this lease will never hand the slot - # back, so return it here rather than strand it for good. - stranded = slot - else: - self._slot = slot - if parked: - queue.unpark() - if stranded is not None: - queue.release(stranded) def release(self) -> None: queue = None - parked = False with self._release_lock: if self._released: return self._released = True queue = self._queue - parked, self._parked = self._parked, False - self._drop_budget() if queue is not None: - if parked: - queue.unpark() - queue.release(self._slot) + queue.release() async def __aenter__(self) -> "LlamaAdmissionLease": return self @@ -421,8 +164,6 @@ class LlamaAdmissionLease: class LlamaAdmissionReservation: - __slots__ = ("_queue", "_lease", "_waiter", "snapshot") - def __init__( self, *, @@ -454,13 +195,6 @@ class LlamaAdmissionReservation: return self._lease async def wait(self, timeout_s: float) -> Optional[LlamaAdmissionLease]: - """Wait up to ``timeout_s`` for a slot. - - A timeout leaves this reservation queued so the caller can poll again. - Any exit that abandons the wait for good must call ``cancel()``, or the - slot granted later is delivered to a future nobody reads and is never - released. - """ lease = self.lease_nowait() if lease is not None: return lease @@ -495,74 +229,12 @@ class LlamaAdmissionReservation: class LlamaAdmissionQueue: - """A fixed pool of generation slots for one llama-server, plus a FIFO wait line. - - The pool mirrors llama-server's own ``--parallel`` slots: ``capacity`` slot ids - are each either free or held by exactly one caller. A caller that finds every - slot busy waits in arrival order and is handed the next slot to free, so no - caller is starved. This bounds only the callers that reserve: chat completions - and messages do, while /v1/completions, Studio's own chat endpoint and RAG - captioning all reach llama-server directly, so it is not a global cap. - Waiting is unbounded in time by default (``queue_timeout_s`` - None); the wait line itself is bounded, and only how many may line up before - new arrivals are rejected. By default that is ``16 x slots`` floored at 64, - not unlimited: an unbounded line takes ``max_queue`` or ``queue_per_slot`` - set to 0. See ``LlamaAdmissionConfig.queue_limit``. - """ - - __slots__ = ( - "key", - "_lock", - "_capacity", - "_free", - "_in_use", - "_held", - "_waiters", - "_parked", - "_unpark_tickets", - "_unpark_seq", - ) - def __init__(self, key: str): self.key = key self._lock = threading.Lock() + self._active = 0 self._capacity = 1 - self._free: list[int] = [0] - # Held slots as a bitmask: one int instead of a set, so the pool costs the - # same whether it is idle or saturated. _held is its popcount, kept as a - # counter because int.bit_count() is 3.10+ and this package targets 3.9. - self._in_use = 0 - self._held = 0 self._waiters: Deque[_Waiter] = deque() - # Holders parked on a tool approval prompt. They hold no slot, so this only - # keeps the queue off the idle-eviction list while they are away. - self._parked = 0 - # FIFO tickets for holders resuming from a park (see acquire_parked_slot). A - # bare count deadlocked: every approved holder blocked every other one. - self._unpark_tickets: Deque[int] = deque() - self._unpark_seq = 0 - - def _resize_pool_locked(self, capacity: int) -> None: - # Slots past a shrunk capacity retire when their holder releases them. - if capacity == self._capacity: - return - self._capacity = capacity - self._free = [slot for slot in range(capacity) if not self._in_use >> slot & 1] - - def _can_admit_locked(self, reserved: int) -> bool: - # Slots still held above a shrunk capacity keep occupying the backend, so - # count every held slot against the ceiling, not just the ids below it. - # ``reserved`` holds slots back for approved holders waiting to resume; - # without it a stream of new arrivals took the next slot, forever. - return bool(self._free) and (self._held + reserved) < self._capacity - - def _take_slot_locked(self, reserved: int) -> Optional[int]: - if not self._can_admit_locked(reserved): - return None - slot = self._free.pop() - self._in_use |= 1 << slot - self._held += 1 - return slot def reserve(self, *, capacity: int, config: LlamaAdmissionConfig) -> LlamaAdmissionReservation: capacity = max(1, int(capacity or 1)) @@ -570,25 +242,22 @@ class LlamaAdmissionQueue: return LlamaAdmissionReservation( queue = None, lease = LlamaAdmissionLease(None), - snapshot = LlamaAdmissionSnapshot(self.key, capacity, 0, 0, capacity), + snapshot = LlamaAdmissionSnapshot(self.key, capacity, 0, 0), ) loop = asyncio.get_running_loop() with self._lock: - self._resize_pool_locked(capacity) + self._capacity = capacity + self._prune_waiters_locked() self._grant_waiters_locked() - if not self._waiters: - slot = self._take_slot_locked(len(self._unpark_tickets)) - if slot is not None: - # No snapshot here: callers read it through snapshot_now(), - # which re-reads the queue, so building one per admitted - # request would be pure allocation on the hot path. - return LlamaAdmissionReservation( - queue = self, - lease = LlamaAdmissionLease(self, slot), - ) - limit = config.queue_limit(self._capacity) - if limit is not None and self._live_waiters_locked() >= limit: + if self._active < self._capacity and not self._waiters: + self._active += 1 + return LlamaAdmissionReservation( + queue = self, + lease = LlamaAdmissionLease(self), + snapshot = self._snapshot_locked(), + ) + if config.max_queue is not None and len(self._waiters) >= config.max_queue: raise LlamaAdmissionQueueFull( "llama-server generation queue is full", snapshot = self._snapshot_locked(), @@ -601,82 +270,15 @@ class LlamaAdmissionQueue: return LlamaAdmissionReservation( queue = self, waiter = waiter, + snapshot = self._snapshot_locked(), ) - def _release_slot_locked(self, slot: Optional[int]) -> None: - # A slot id at or past a shrunk capacity retires instead of returning. - if slot is None or not self._in_use >> slot & 1: - return - self._in_use &= ~(1 << slot) - self._held -= 1 - if slot < self._capacity: - self._free.append(slot) - - def release(self, slot: Optional[int]) -> None: + def release(self) -> None: with self._lock: - self._release_slot_locked(slot) + if self._active > 0: + self._active -= 1 self._grant_waiters_locked() - def try_park(self, slot: Optional[int]) -> bool: - """Return a parked holder's slot to the pool. See ``LlamaAdmissionLease.park``. - - False leaves the slot with its holder, so a refused park costs nothing to - undo. The per-queue count is only what ``is_idle`` reads; the budget and - the capacity it is sized from are both process-wide. - """ - if not _claim_park(_max_parked(_live_capacity(self))): - return False - with self._lock: - self._parked += 1 - self._release_slot_locked(slot) - self._grant_waiters_locked() - return True - - def unpark(self) -> None: - with self._lock: - if self._parked > 0: - self._parked -= 1 - - async def acquire_parked_slot( - self, - *, - cancel_event = None, - poll_s: float = 0.02, - ) -> Optional[int]: - """Wait for a slot for a holder resuming from a park, None if cancelled. - - Ordered by ticket rather than counted, so approvals resume in the order - they came back: counting them made every approved holder block every - other one, and with nothing decoding that never resolved. - """ - with self._lock: - self._unpark_seq += 1 - ticket = self._unpark_seq - self._unpark_tickets.append(ticket) - try: - while True: - with self._lock: - ahead = 0 - for queued in self._unpark_tickets: - if queued == ticket: - break - ahead += 1 - # Only the approvals ahead of this one hold slots back from it. - slot = self._take_slot_locked(ahead) - if slot is not None: - return slot - if cancel_event is not None and cancel_event.is_set(): - return None - await asyncio.sleep(poll_s) - finally: - with self._lock: - try: - self._unpark_tickets.remove(ticket) - except ValueError: - pass - # This ticket was holding a slot back from the wait line. - self._grant_waiters_locked() - def cancel(self, waiter: _Waiter) -> None: lease_to_release = None with self._lock: @@ -689,13 +291,7 @@ class LlamaAdmissionQueue: lease_to_release = waiter.granted_lease waiter.granted_lease = None if not waiter.future.done(): - try: - waiter.loop.call_soon_threadsafe(waiter.future.cancel) - except RuntimeError: - # Loop gone. Routes call cancel() from finally blocks, so - # raising here would both mask their exception and skip the - # release below, stranding the slot for the process lifetime. - pass + waiter.loop.call_soon_threadsafe(waiter.future.cancel) if lease_to_release is not None: lease_to_release.release() @@ -707,32 +303,20 @@ class LlamaAdmissionQueue: def is_idle(self) -> bool: with self._lock: self._prune_waiters_locked() - # A parked holder owns no slot but is coming back to this queue, so - # evicting it here would resume it against a fresh 1-slot pool. - return self._in_use == 0 and not self._waiters and not self._parked + return self._active == 0 and not self._waiters def _grant_waiters_locked(self) -> None: - # Dead waiters are skipped as they are popped, so no prune is needed here. - while self._waiters and self._can_admit_locked(len(self._unpark_tickets)): + self._prune_waiters_locked() + while self._waiters and self._active < self._capacity: waiter = self._waiters.popleft() if waiter.cancelled or waiter.future.done(): continue - slot = self._take_slot_locked(len(self._unpark_tickets)) - lease = LlamaAdmissionLease(self, slot) + self._active += 1 + lease = LlamaAdmissionLease(self) waiter.granted_lease = lease - try: - waiter.loop.call_soon_threadsafe(self._deliver_lease, waiter, lease) - except RuntimeError: - # Waiter's loop is gone. Reclaim the slot; leaving the bit set - # would strand it, since _free is rebuilt from the bitmask. - waiter.granted_lease = None - self._release_slot_locked(slot) + waiter.loop.call_soon_threadsafe(self._deliver_lease, waiter, lease) def _deliver_lease(self, waiter: _Waiter, lease: LlamaAdmissionLease) -> None: - # Runs on the waiter's own loop thread, which is also the only thread that - # cancels that reservation, so waiter state is safe to touch unlocked here. - # release() may be called from any thread, but only reaches this via - # call_soon_threadsafe. Cancelling off-loop would need this under _lock. if waiter.cancelled or waiter.future.done(): waiter.granted_lease = None if not waiter.future.done(): @@ -747,32 +331,16 @@ class LlamaAdmissionQueue: lease.release() def _prune_waiters_locked(self) -> None: - # Rebuilding the deque on every reserve/release dominated the hot path, so - # only pay it when a waiter actually died out of band (an externally - # cancelled future); cancel() already drops its own waiter eagerly. - for waiter in self._waiters: - if waiter.cancelled or waiter.future.done(): - break - else: - return self._waiters = deque( waiter for waiter in self._waiters if not waiter.cancelled and not waiter.future.done() ) - def _live_waiters_locked(self) -> int: - self._prune_waiters_locked() - return len(self._waiters) - def _snapshot_locked(self) -> LlamaAdmissionSnapshot: return LlamaAdmissionSnapshot( key = self.key, capacity = self._capacity, - active = self._held, + active = self._active, queued = len(self._waiters), - # What another caller could actually take, so the admission log never - # shows free slots next to queued requests: after a shrink, ids below - # the new capacity can be free while holdovers still fill the ceiling. - free = min(len(self._free), max(0, self._capacity - self._held)), ) @@ -796,10 +364,5 @@ def get_llama_admission_queue(key: str) -> LlamaAdmissionQueue: def reset_llama_admission_queues() -> None: - global _parked_total with _QUEUES_LOCK: _QUEUES.clear() - # The budget outlives the queues it was claimed against, so dropping them - # without it leaks the count and shrinks the budget for good. - with _PARK_LOCK: - _parked_total = 0 diff --git a/studio/backend/core/inference/llama_cpp.py b/studio/backend/core/inference/llama_cpp.py index 712caf43e5..0621a7f9c8 100644 --- a/studio/backend/core/inference/llama_cpp.py +++ b/studio/backend/core/inference/llama_cpp.py @@ -43,7 +43,6 @@ import httpx from core.inference.llama_server_args import ( _LAYER_OFFLOAD_FLAGS, _effective_tensor_parallel, - _flag_name, _tensor_parallel_matches_loaded, extra_args_disable_mmproj, parse_cache_override, @@ -85,7 +84,6 @@ from core.tool_healing import ( strip_outside_think, ) from utils.native_path_leases import child_env_without_native_path_secret -from utils.child_stdio import utf8_child_env from utils.hf_xet_fallback import hf_hub_download_with_xet_fallback from utils.subprocess_compat import ( windows_hidden_subprocess_kwargs as _windows_hidden_subprocess_kwargs, @@ -93,17 +91,13 @@ from utils.subprocess_compat import ( from utils.process_lifetime import child_popen_kwargs as _child_popen_kwargs from core.inference.tool_call_parser import ( MAX_ACT_REPROMPTS as _MAX_REPROMPTS, - NUDGE_TOOL_CALLS_STATUS as _NUDGE_TOOL_CALLS_STATUS, REPROMPT_MAX_CHARS as _REPROMPT_MAX_CHARS, - is_reprompt_repeat as _is_reprompt_repeat, - is_reprompt_restatement as _is_reprompt_restatement, is_short_intent_without_action as _is_short_intent_without_action, reprompt_to_act_message as _reprompt_to_act_message, ) from core.inference.tool_loop_controller import ( ToolLoopController, append_deferred_nudges, - awaiting_approval_status, tool_event_provenance, ) from state.tool_approvals import ( @@ -313,15 +307,6 @@ def _native_linux_system_rocm_lib_dirs(binary_dir: str = "") -> "list[str]": os.path.join(d, "libhsa-runtime64.so.1") ): out.append(d) - # ROCm keeps LLVM's versioned runtime under /lib/llvm, so a - # lib64 host still finds it under lib. Probe both and keep them - # ahead of the bundle, else system libamd_comgr binds to the - # bundle's incompatible libLLVM.so.*. - for _sub in (lib_sub, "lib"): - llvm_lib = os.path.join(base, _sub, "llvm", "lib") - if llvm_lib not in seen and os.path.isdir(llvm_lib): - seen.add(llvm_lib) - out.append(llvm_lib) return out @@ -363,32 +348,12 @@ _DEFAULT_STREAM_STALL_TIMEOUT_S = 120.0 # 2 min # loop). Structured delta.tool_calls are grammar-bounded by llama-server; text # parsed from content is not, so one runaway turn could fan out unbounded. _MAX_TOOL_CALLS_PER_TURN = 8 -# Obligation phrasing INTENT_SIGNAL leaves alone ("I need to call ..."), paired with -# an action verb. Sentence-anchored: mid-sentence the same words are prose that names -# a tool ("The API I should invoke is foo() because ..."), and suppressing that loses -# a real answer. "should"/"must" sit outside the need|have|ought group because they -# take a bare infinitive. "invoke"/"query" stay out of the verb list: they read as -# technical prose far more often than as a stall. -_FORCED_PLAN_INTENT = re.compile( - r"(?:^|[.!?]\s+)\s*" - r"(?:i\s+(?:(?:need|have|ought)\s+to|should|must)|need\s+to|going\s+to|must|should)" - r"\s+(?:\w+\s+){0,2}?(?:call|use|run|search|fetch|render)\b", - re.I | re.M, -) -# "the answer is not in the context" announces a *missing* answer, so the negated -# forms are excluded or the plan behind them would ship as the final response. -_FINAL_ANSWER_SIGNAL = re.compile( - r"\b(?:final\s+answer|answer\s*:|here\s+is|here's|in\s+summary|result\s*:" - r"|(?:the\s+)?answer\s+is(?!\s+(?:not|unavailable|unknown|unclear|missing)\b))\b", +_FORCED_REPEAT_PLAN_SIGNAL = re.compile( + r"\b(?:i\s+will|i'll|let\s+me|going\s+to|need\s+to|call|use|run|search|fetch|render)\b", re.I, ) -# A plan that pivots ("I should call web_search, but Tokyo is the capital") has an -# answer attached, so the turn must survive. Leaking a plan sentence is cosmetic; -# dropping an answer is not, so the doubtful case keeps the output. The pivot has to -# carry text of its own: "I should call web_search, though." answers nothing. -_ANSWER_PIVOT = re.compile( - r"\b(?:but|however|although|though|that\s+said|in\s+the\s+meantime|meanwhile)\b" - r"[\W_]*(?:\w+[\W_]+){1,}\w", +_FINAL_ANSWER_SIGNAL = re.compile( + r"\b(?:final\s+answer|answer\s*:|here\s+is|here's|in\s+summary|result\s*:)\b", re.I, ) @@ -480,28 +445,14 @@ def _held_rehearsal_tail_len(text: str, active_tools: list[dict]) -> int: return len(tail) if tail and _is_rehearsal_prefix(tail, active_tools) else 0 -def _should_suppress_forced_no_tool_output(text: str, previous: str = "") -> bool: - """Suppress only repeated forced-turn planning text, not final answers. - - ``previous`` is the stall text that triggered the nudge, so a retry that - moved on can be told from one that just said the same thing again. - """ +def _should_suppress_forced_no_tool_output(text: str) -> bool: + """Suppress only repeated forced-turn planning text, not final answers.""" stripped = text.strip() if not stripped or len(stripped) >= _REPROMPT_MAX_CHARS: return False if _FINAL_ANSWER_SIGNAL.search(stripped): return False - plan = _FORCED_PLAN_INTENT.search(stripped) - if plan is not None: - # Only the plan itself is safe to drop; anything the turn pivots to after it - # is the answer the user is waiting for. - return _ANSWER_PIVOT.search(stripped[plan.end() :]) is None - if not _is_short_intent_without_action(stripped): - return False - # INTENT_SIGNAL also fires on lead-ins to a real answer ("Now I have the results. - # The capital is Tokyo."), so a bare intent match is a stall only when the retry - # adds nothing. No ``previous`` keeps the standalone "is this a stall?" contract. - return not previous or _is_reprompt_restatement(stripped, previous) + return _FORCED_REPEAT_PLAN_SIGNAL.search(stripped) is not None # ── Pre-compiled patterns for GGUF shard detection ─────────── @@ -618,7 +569,7 @@ def _load_swa_cache() -> dict: if _SWA_CACHE is not None: return _SWA_CACHE try: - with open(_swa_cache_path(), encoding = "utf-8-sig") as f: + with open(_swa_cache_path(), encoding = "utf-8") as f: _SWA_CACHE = json.load(f) if not isinstance(_SWA_CACHE, dict): _SWA_CACHE = {} @@ -669,7 +620,7 @@ def _fetch_swa_entry_from_hf(repo_id: str) -> Optional[object]: repo_type = "model", cache_dir = active_hf_hub_cache(), ) - with open(cfg_path, encoding = "utf-8-sig") as f: + with open(cfg_path, encoding = "utf-8") as f: cfg = json.load(f) except Exception: return None @@ -1557,21 +1508,6 @@ def _kv_bytes_per_elem(cache_type: Optional[str]) -> float: }.get((cache_type or "f16").strip().lower(), 2.0) -def _pad_kv_cells(cells: int) -> int: - return ((cells + 255) // 256) * 256 - - -def _kv_cache_cell_layout(n_ctx: int, n_parallel: int, kv_unified: bool) -> tuple[int, int, int]: - """Return llama.cpp's slot count, stream count, and cells per stream.""" - slots = max(1, n_parallel) - padded_ctx = _pad_kv_cells(n_ctx) - streams = 1 if kv_unified else slots - if padded_ctx <= 0: - return slots, streams, 0 - cells_per_stream = padded_ctx if kv_unified else _pad_kv_cells(padded_ctx // slots) - return slots, streams, cells_per_stream - - def _env_main_cache_type_for_budget(env: Optional[Mapping[str, str]] = None) -> Optional[str]: """Heavier of the inherited LLAMA_ARG_CACHE_TYPE_K/_V env types when it exceeds the f16 default, else None. Unsloth emits --cache-type only for the @@ -1604,39 +1540,6 @@ def _extra_args_main_cache_type_for_budget(extra_args: Optional[Iterable[str]]) return max(candidates, key = _kv_bytes_per_elem) -def _effective_main_cache_types( - args: Optional[Iterable[str]], env: Optional[Mapping[str, str]] = None -) -> tuple[str, str]: - """Effective main K/V cache types after environment and CLI precedence.""" - source_env = os.environ if env is None else env - env_k = (source_env.get("LLAMA_ARG_CACHE_TYPE_K") or "f16").strip().lower() - env_v = (source_env.get("LLAMA_ARG_CACHE_TYPE_V") or "f16").strip().lower() - arg_k, arg_v = parse_cache_override_per_axis(args) - return ( - (arg_k or env_k).strip().lower(), - (arg_v or env_v).strip().lower(), - ) - - -def _planned_main_cache_types( - cache_type_kv: Optional[str], - extra_args: Optional[Iterable[str]], - env: Optional[Mapping[str, str]] = None, -) -> tuple[str, str]: - """Main K/V types the loader's managed flags and user extras will produce.""" - args = list(extra_args or ()) - emitted_type = _extra_args_main_cache_type_for_budget(args) or cache_type_kv - if emitted_type: - args = [ - "--cache-type-k", - emitted_type, - "--cache-type-v", - emitted_type, - *args, - ] - return _effective_main_cache_types(args, env) - - def _auto_mode_drops_mtp( req_mode: Optional[str], size_b: Optional[float], @@ -1680,90 +1583,26 @@ def _extra_args_set_spec_type(extra_args: Optional[Iterable[str]]) -> bool: # set keeps detection and stripping from drifting. _GPU_OFFLOAD_OVERRIDE_FLAGS = _LAYER_OFFLOAD_FLAGS _THREAD_OVERRIDE_FLAGS = frozenset({"-t", "--threads"}) -# common_params defaults in the bundled llama.cpp runtime. -_DEFAULT_LLAMA_N_BATCH = 2048 -_DEFAULT_LLAMA_N_UBATCH = 512 -_LLAMA_ARG_TRUE_VALUES = frozenset({"on", "enabled", "true", "1"}) -_LLAMA_ARG_FALSE_VALUES = frozenset({"off", "disabled", "false", "0"}) -_LLAMA_ARG_AUTO_VALUES = frozenset({"auto", "-1"}) -_LLAMA_ARG_TRUE_OR_AUTO_VALUES = _LLAMA_ARG_TRUE_VALUES | _LLAMA_ARG_AUTO_VALUES -_LLAMA_ARG_TRUE_FALSE_AUTO_VALUES = _LLAMA_ARG_TRUE_OR_AUTO_VALUES | _LLAMA_ARG_FALSE_VALUES + + +def _extra_arg_flag_name(token: str) -> Optional[str]: + if not token.startswith("-") or token in {"-", "--"}: + return None + if len(token) >= 2 and (token[1].isdigit() or token[1] == "."): + return None + return token.split("=", 1)[0] def _extra_args_set_any_flag(extra_args: Optional[Iterable[str]], flags: Collection[str]) -> bool: if not extra_args: return False for raw in extra_args: - flag = _flag_name(str(raw)) + flag = _extra_arg_flag_name(str(raw)) if flag in flags: return True return False -def _swa_full_from_args_or_env( - extra_args: Optional[Iterable[str]], env: Optional[Mapping[str, str]] = None -) -> bool: - """Whether llama.cpp receives the enable-only full-size SWA option.""" - if _extra_args_set_any_flag(extra_args, {"--swa-full"}): - return True - value = (os.environ if env is None else env).get("LLAMA_ARG_SWA_FULL") - return value in _LLAMA_ARG_TRUE_VALUES - - -def _kv_unified_from_args( - extra_args: Optional[Iterable[str]], - default: bool = False, - env: Optional[Mapping[str, str]] = None, -) -> bool: - """Resolve llama.cpp's environment and last-wins unified KV flags.""" - enabled = False - value = (os.environ if env is None else env).get("LLAMA_ARG_KV_UNIFIED") - if value in _LLAMA_ARG_TRUE_VALUES: - enabled = True - elif value in _LLAMA_ARG_FALSE_VALUES: - enabled = False - if default: - # Studio's managed --kv-unified flag is appended after environment - # parsing and before user extras. - enabled = True - for raw in extra_args or (): - flag = _flag_name(str(raw)) - if flag in {"-kvu", "--kv-unified"}: - enabled = True - elif flag in {"-no-kvu", "--no-kv-unified"}: - enabled = False - return enabled - - -def _flash_attn_enabled_from_args( - args: Optional[Iterable[str]], - default: bool = True, - env: Optional[Mapping[str, str]] = None, -) -> bool: - """Resolve llama.cpp's environment and last-wins flash-attention settings.""" - enabled = default - # llama.cpp applies LLAMA_ARG_FLASH_ATTN before parsing argv (arg.cpp set_env), - # so the CLI still wins. --flash-attn has no args_neg, so no LLAMA_ARG_NO_ twin. - value = (os.environ if env is None else env).get("LLAMA_ARG_FLASH_ATTN") - if value in _LLAMA_ARG_FALSE_VALUES: - enabled = False - elif value in _LLAMA_ARG_TRUE_OR_AUTO_VALUES: - enabled = True - values = [str(arg) for arg in args] if args else [] - for i, raw in enumerate(values): - if _flag_name(raw) not in {"-fa", "--flash-attn"}: - continue - _, eq, inline = raw.partition("=") - value = inline if eq else "on" - if not eq and i + 1 < len(values) and values[i + 1] in _LLAMA_ARG_TRUE_FALSE_AUTO_VALUES: - value = values[i + 1] - if value in _LLAMA_ARG_FALSE_VALUES: - enabled = False - elif value in _LLAMA_ARG_TRUE_OR_AUTO_VALUES: - enabled = True - return enabled - - def _effective_spec_type( extra_args: Optional[Iterable[str]], env: Optional[Mapping[str, str]] = None ) -> Optional[str]: @@ -1775,8 +1614,7 @@ def _effective_spec_type( cli_present = False cli_value: Optional[str] = None for i, raw in enumerate(args): - flag = _flag_name(raw) - _, eq, inline = raw.partition("=") + flag, eq, inline = raw.partition("=") if flag == "--spec-default": cli_present = True cli_value = "default" @@ -1820,8 +1658,7 @@ def _extra_args_spec_draft_n_max(extra_args: Optional[Iterable[str]]) -> Optiona args = [str(a) for a in extra_args] found: Optional[int] = None for i, raw in enumerate(args): - flag = _flag_name(raw) - _, eq, inline = raw.partition("=") + flag, eq, inline = raw.partition("=") if flag not in ("--spec-draft-n-max", "--draft-max"): continue value = inline if eq else (args[i + 1] if i + 1 < len(args) else "") @@ -1851,8 +1688,7 @@ def _extra_args_mtp_draft_path( args = [str(a) for a in extra_args] if extra_args else [] found: Optional[str] = None for i, raw in enumerate(args): - flag = _flag_name(raw) - _, eq, inline = raw.partition("=") + flag, eq, inline = raw.partition("=") if flag not in flags: continue value = inline if eq else (args[i + 1] if i + 1 < len(args) else "") @@ -1876,8 +1712,7 @@ def _extra_args_draft_cache_types( k_type: Optional[str] = None v_type: Optional[str] = None for i, raw in enumerate(args): - flag = _flag_name(raw) - _, eq, inline = raw.partition("=") + flag, eq, inline = raw.partition("=") if flag not in k_flags and flag not in v_flags: continue value = inline if eq else (args[i + 1] if i + 1 < len(args) else "") @@ -1909,8 +1744,7 @@ def _extra_args_draft_offloaded_to_cpu( last_ngl: Optional[str] = None last_dev: Optional[str] = None for i, raw in enumerate(args): - flag = _flag_name(raw) - _, eq, inline = raw.partition("=") + flag, eq, inline = raw.partition("=") value = inline if eq else (args[i + 1] if i + 1 < len(args) else "") if flag in ngl_flags: last_ngl = value @@ -1932,61 +1766,31 @@ def _extra_args_draft_offloaded_to_cpu( def _extra_args_n_ubatch( - extra_args: Optional[Iterable[str]], - env: Optional[Mapping[str, str]] = None, - n_ctx: Optional[int] = None, + extra_args: Optional[Iterable[str]], env: Optional[Mapping[str, str]] = None ) -> Optional[int]: - """Effective ubatch after llama.cpp normalizes it, or None at defaults.""" - values = { - "batch": _DEFAULT_LLAMA_N_BATCH, - "ubatch": _DEFAULT_LLAMA_N_UBATCH, - } - source_env = os.environ if env is None else env - overridden = False - for key, env_name in ( - ("batch", "LLAMA_ARG_BATCH"), - ("ubatch", "LLAMA_ARG_UBATCH"), - ): - raw = source_env.get(env_name) - if raw: - try: - values[key] = int(raw) - overridden = True - except (TypeError, ValueError): - pass - + """Physical micro-batch from extras (--ubatch-size/-ub) else the LLAMA_ARG_UBATCH + env, else None. It sizes the compute-graph buffer, so an override must reach + the VRAM reserve.""" args = [str(a) for a in extra_args] if extra_args else [] - flags = { - "-b": "batch", - "--batch-size": "batch", - "-ub": "ubatch", - "--ubatch-size": "ubatch", - } + found: Optional[int] = None for i, raw in enumerate(args): - flag = _flag_name(raw) - _, eq, inline = raw.partition("=") - key = flags.get(flag) - if key is None: + flag, eq, inline = raw.partition("=") + if flag not in ("--ubatch-size", "-ub"): continue value = inline if eq else (args[i + 1] if i + 1 < len(args) else "") try: - values[key] = int(value) - overridden = True + found = int(value) except (TypeError, ValueError): continue - if not overridden: - return None - - # common_params stores signed values, then llama_context_params converts - # them to uint32_t. A zero ubatch means "use batch"; the context then caps - # ubatch at batch size. - batch = values["batch"] & 0xFFFFFFFF - raw_ubatch = values["ubatch"] - ubatch = batch if raw_ubatch == 0 else raw_ubatch & 0xFFFFFFFF - effective = min(batch, ubatch) - if n_ctx is not None and n_ctx > 0: - effective = min(effective, n_ctx) - return effective + if found is not None: + return found + raw = (os.environ if env is None else env).get("LLAMA_ARG_UBATCH") + if raw: + try: + return int(raw) + except (TypeError, ValueError): + pass + return None def _build_ngram_mod_flags( @@ -2230,8 +2034,6 @@ class LlamaCppBackend: self._effective_context_length: Optional[int] = None self._max_context_length: Optional[int] = None self._effective_parallel_slots: int = 1 - # --parallel the last load asked for, before any fit-time reduction. - self._requested_n_parallel: int = 1 self._chat_template: Optional[str] = None self._chat_template_override: Optional[str] = None self._supports_reasoning: bool = False @@ -2347,14 +2149,6 @@ class LlamaCppBackend: # save can tell whether the model files were swapped on disk since load. self._slot_loaded_identity: Optional[tuple] = None self._prompt_cache_disabled: bool = False - self._swa_full: bool = False - self._kv_cache_unified: bool = False - self._n_ubatch: int = self._DEFAULT_N_UBATCH - self._flash_attn_enabled: bool = True - self._effective_cache_types: tuple[str, str] = ("f16", "f16") - # Total KV allocation context across all slots. _effective_context_length - # becomes the per-slot request limit after /props reconciliation. - self._kv_cache_context_total: Optional[int] = None # True once a probe has completed; cleared on transient failure. self._is_audio: bool = False self._audio_type: Optional[str] = None @@ -2407,11 +2201,6 @@ class LlamaCppBackend: """True when the loaded GGUF is a block-diffusion model (DiffusionGemma).""" return self._is_diffusion - @property - def swa_full(self) -> bool: - """Whether the active llama-server received full-size SWA mode.""" - return self._swa_full - @property def hf_variant(self) -> Optional[str]: return self._hf_variant @@ -2468,17 +2257,6 @@ class LlamaCppBackend: slots = 1 return max(1, slots) - @property - def requested_parallel_slots(self) -> int: - """--parallel the last load asked for, before any fit-time reduction. - The reload dedupe compares requested-vs-requested (like requested_n_ctx); - the effective count would reload forever after a fitter reduction.""" - try: - slots = int(getattr(self, "_requested_n_parallel", 1)) - except (TypeError, ValueError): - slots = 1 - return max(1, slots) - @property def max_context_length(self) -> Optional[int]: """Return the largest context that fits on this hardware at load time. @@ -2504,8 +2282,6 @@ class LlamaCppBackend: def _reset_effective_parallel_slots(self) -> None: self._effective_parallel_slots = 1 - # Cleared with the effective count so a stale value can't skew the dedupe. - self._requested_n_parallel = 1 @staticmethod def _read_rss_bytes(pid: int) -> Optional[int]: @@ -3083,7 +2859,6 @@ class LlamaCppBackend: [bin_path, "--help"], capture_output = True, text = True, - encoding = "utf-8", errors = "replace", timeout = 10, check = False, @@ -3341,9 +3116,8 @@ class LlamaCppBackend: prefer_rocr masks at the ROCr/HSA layer instead (clearing HIP). A HIP mask filters only AFTER the HSA runtime enumerates every agent, and that enumeration segfaults at startup on a GPU the build has no kernels for - (e.g. a gfx1036 iGPU under a gfx103X prebuilt: that bundle maps only - gfx1030/1031/1032/1034), before llama-server logs a line. ROCR drops the - device at the driver layer, consuming physical ids. + (e.g. a gfx1103 iGPU under a gfx110X prebuilt), before llama-server logs a + line. ROCR drops the device at the driver layer, consuming physical ids. The CPU-only sentinel ("-1") has no portable ROCR spelling, so it keeps the HIP mask. Windows keeps the HIP mask too: ROCR_VISIBLE_DEVICES is a Linux ROCr variable (Windows HIP has no ROCr layer), so the ROCR pin @@ -3656,8 +3430,6 @@ class LlamaCppBackend: ], capture_output = True, text = True, - encoding = "utf-8", - errors = "replace", timeout = 10, env = child_env_without_native_path_secret(), **_windows_hidden_subprocess_kwargs(), @@ -3729,17 +3501,18 @@ class LlamaCppBackend: return [] @staticmethod - def _run_vulkan_probe(binary: Optional[str] = None) -> list[dict]: - """Run ``_vulkan_probe.py`` and parse its per-device lines. + def _get_gpu_free_memory_vulkan(binary: Optional[str] = None) -> list[tuple[int, int, int]]: + """Query free (and total) VRAM per device via the bundled ggml Vulkan backend. - Returns raw (uncapped) rows sorted by index: - ``{"index", "free_mib", "total_mib", "is_igpu", "name"}``. The index is - ggml's compact Vulkan ordinal -- the one the registry names - ``Vulkan`` and load_model pins with ``--device``, NOT the raw - ``GGML_VK_VISIBLE_DEVICES`` space. A user-set ``GGML_VK_VISIBLE_DEVICES`` - is honored by ggml (passed through), so the list already reflects it. - ``name`` is ggml's device description; "" from an older 4-column probe. - [] when no Vulkan build or device is reachable. + Loads ``libggml-vulkan`` in a short-lived subprocess (no Vulkan instance + in this process) and returns (device_index, free_mib, total_mib) sorted + by index. The index is ggml's compact Vulkan ordinal -- the one the + registry names ``Vulkan`` and load_model pins with ``--device``, + NOT the raw ``GGML_VK_VISIBLE_DEVICES`` space. A user-set + ``GGML_VK_VISIBLE_DEVICES`` is honored by ggml (passed through), so the + list already reflects it. iGPUs leave a host-RAM margin (see + ``_apply_igpu_host_reserve_mib``) and report total 0; discrete cards pass + their real total through. [] when no Vulkan build or device is reachable. """ binary = binary or LlamaCppBackend._find_llama_server_binary() if not binary: @@ -3764,15 +3537,12 @@ class LlamaCppBackend: ) probe_script = Path(__file__).with_name("_vulkan_probe.py") try: - # UTF-8 to match the probe's stdout reconfigure: device names can be - # non-ASCII, and the platform-default decode (cp1252) could throw. result = subprocess.run( [sys.executable, str(probe_script), str(binary_dir)], capture_output = True, - encoding = "utf-8", - errors = "replace", + text = True, timeout = 15, - env = utf8_child_env(env), + env = env, **_windows_hidden_subprocess_kwargs(), ) if result.returncode != 0: @@ -3784,56 +3554,21 @@ class LlamaCppBackend: logger.debug(f"vulkan GPU probe failed: {e}") return [] - rows: list[dict] = [] + gpus: list[tuple[int, int, int]] = [] for line in result.stdout.strip().splitlines(): parts = line.split("\t") - # 4 columns from an older probe (no name); 5 with the name column. - if len(parts) not in (4, 5): + if len(parts) != 4: continue try: - rows.append( - { - "index": int(parts[0]), - "free_mib": int(parts[1]) // (1024 * 1024), - "is_igpu": parts[2] == "1", - "total_mib": int(parts[3]) // (1024 * 1024), - "name": parts[4].strip() if len(parts) == 5 else "", - } - ) + idx = int(parts[0]) + free_mib = int(parts[1]) // (1024 * 1024) + is_igpu = parts[2] == "1" + # iGPU "total" is shared RAM, not a VRAM budget -> keep 0 so the + # fit stays on free*frac (the host reserve below is its + # headroom); a discrete card passes its real total through. + total_mib = 0 if is_igpu else int(parts[3]) // (1024 * 1024) except ValueError: continue - rows.sort(key = lambda r: r["index"]) - return rows - - @staticmethod - def vulkan_device_inventory(binary: Optional[str] = None) -> list[dict]: - """UI-facing Vulkan device list: the devices llama-server will actually - use, with real totals (an iGPU keeps its shared-RAM total here -- the - caller labels it, unlike the fit which zeroes it). Same rows as - ``_run_vulkan_probe``; names fall back to ``Vulkan``. - """ - rows = LlamaCppBackend._run_vulkan_probe(binary) - for row in rows: - if not row["name"]: - row["name"] = f"Vulkan{row['index']}" - return rows - - @staticmethod - def _get_gpu_free_memory_vulkan(binary: Optional[str] = None) -> list[tuple[int, int, int]]: - """Query free (and total) VRAM per device via the bundled ggml Vulkan backend. - - Fit-oriented view of ``_run_vulkan_probe``: returns (device_index, - free_mib, total_mib) sorted by index. iGPUs leave a host-RAM margin (see - ``_apply_igpu_host_reserve_mib``) and report total 0; discrete cards pass - their real total through. [] when no Vulkan build or device is reachable. - """ - gpus: list[tuple[int, int, int]] = [] - for row in LlamaCppBackend._run_vulkan_probe(binary): - idx, free_mib, is_igpu = row["index"], row["free_mib"], row["is_igpu"] - # iGPU "total" is shared RAM, not a VRAM budget -> keep 0 so the - # fit stays on free*frac (the host reserve below is its - # headroom); a discrete card passes its real total through. - total_mib = 0 if is_igpu else row["total_mib"] capped = _apply_igpu_host_reserve_mib(free_mib, is_igpu) if capped < free_mib: logger.info( @@ -3842,6 +3577,7 @@ class LlamaCppBackend: f"({free_mib}->{capped}MiB usable)" ) gpus.append((idx, capped, total_mib)) + gpus.sort(key = lambda g: g[0]) if gpus: logger.info( "Vulkan GPU memory detected: " @@ -4283,32 +4019,6 @@ class LlamaCppBackend: is non-None here.""" return self._embedding_length // self._n_heads if self._n_heads else 128 # type: ignore[operator] - def _max_kv_value_width( - self, - default_len: int, - swa_len: Optional[int] = None, - ) -> int: - """llama.cpp's hparams.n_embd_v_gqa_max() over every model layer.""" - n_layers = self._n_layers or 1 - n_kv = self._n_kv_heads or self._n_heads or 1 - if self._sliding_window_pattern is None: - max_len = max(default_len, swa_len or default_len) - return max( - self._kv_heads_for_layer(layer_idx, n_kv) * max_len for layer_idx in range(n_layers) - ) - return max( - self._kv_heads_for_layer(layer_idx, n_kv) - * ( - (swa_len or default_len) - if ( - layer_idx < len(self._sliding_window_pattern) - and self._sliding_window_pattern[layer_idx] - ) - else default_len - ) - for layer_idx in range(n_layers) - ) - def _estimate_kv_cache_bytes( self, n_ctx: int, @@ -4317,26 +4027,22 @@ class LlamaCppBackend: swa_full: bool = False, n_parallel: int = 1, kv_unified: bool = True, - n_ubatch: Optional[int] = None, ctx_checkpoints: int = 0, - flash_attn: bool = True, ) -> int: """Estimate KV cache VRAM for a given context length. 5-path architecture-aware estimation: 1. MLA -- compressed KV latent + RoPE, K-only (no separate V) 2. Hybrid -- only attention layers need KV (Mamba layers don't) - 3. SWA -- sliding-window layers use compact or full cache cells + 3. SWA -- sliding-window layers cache min(ctx, window) tokens 4. GQA -- standard full KV with explicit key/value dimensions 5. Legacy -- fallback using embed // n_heads Server-flag knobs (mirror llama-server's CLI): swa_full -- --swa-full: SWA layers cache full n_ctx (path 3->4). - n_parallel -- --parallel slots: controls per-slot stream padding. - kv_unified -- --kv-unified: one shared stream vs one per slot. - n_ubatch -- --ubatch-size: SWA cache's processing headroom. + n_parallel -- --parallel slots: non-SWA constant, SWA scale linearly. + kv_unified -- --kv-unified: memory no-op (API forward-compat). ctx_checkpoints -- --ctx-checkpoints: N SWA snapshots per slot. - flash_attn -- False pads variable-width V tensors to the model max. Returns 0 if metadata is insufficient. """ @@ -4351,17 +4057,9 @@ class LlamaCppBackend: n_kv = self._n_kv_heads or self._n_heads or 1 # type: ignore[assignment] # Bytes per element depends on KV cache quantization - bpe_k = _kv_bytes_per_elem(cache_type_kv) - # The automatic FA-off retry rewrites an invalid quantized V cache to - # f16. Pricing that viable retry here avoids under-reserving it. - bpe_v = bpe_k if flash_attn else max(bpe_k, _kv_bytes_per_elem("f16")) + bpe = _kv_bytes_per_elem(cache_type_kv) - slots, streams, cells_per_stream = _kv_cache_cell_layout(n_ctx, n_parallel, kv_unified) - total_cells = cells_per_stream * streams - ubatch = max( - 0, - int(self._DEFAULT_N_UBATCH if n_ubatch is None else n_ubatch), - ) + slots = max(1, n_parallel) # Path 1: MLA (DeepSeek-V2/V3, GLM-4.7, GLM-5, Kimi-K2.5) # One compressed KV latent per token/layer (shared across heads); V is @@ -4372,7 +4070,7 @@ class LlamaCppBackend: n_kv_mla = self._n_kv_heads or 1 rope_dim = self._key_length_mla or 64 key_len = self._kv_key_length or (self._kv_lora_rank + rope_dim) - return int(n_layers_kv * total_cells * n_kv_mla * key_len * bpe_k) + return int(n_layers_kv * n_ctx * n_kv_mla * key_len * bpe) key_len = self._kv_key_length val_len = self._kv_value_length @@ -4383,18 +4081,16 @@ class LlamaCppBackend: fai = self._full_attention_interval n_attn = -(-n_layers // fai) if fai > 0 else n_layers # ceiling division if key_len is not None and val_len is not None: - v_width = n_kv * val_len if flash_attn else self._max_kv_value_width(val_len) - return int(n_attn * total_cells * (n_kv * key_len * bpe_k + v_width * bpe_v)) + return int(n_attn * n_ctx * n_kv * (key_len + val_len) * bpe) head_dim = self._legacy_head_dim() - return int(n_attn * total_cells * n_kv * 2 * head_dim * bpe_k) + return int(n_attn * n_ctx * n_kv * 2 * head_dim * bpe) # Path 3: Sliding window (Gemma 2/3/3n/4, gpt-oss, Cohere2 ...). Pattern # from the resolver; if absent, falls through to the legacy 1/4-global # heuristic. --parallel N accounting (verified against llama-server): - # non-SWA cells total n_ctx across streams. Compact SWA adds one processing - # micro-batch to the window allowance and pads to 256 cells; unified mode - # holds all slots in one stream, while non-unified mode has one stream per - # slot. --swa-full expands SWA to each stream's full context. + # non-SWA cells = n_ctx split across slots (CONSTANT); SWA per-slot cells + # = 2*sliding_window (capped at n_ctx/per_slot_ctx) -> LINEAR in slots. + # --swa-full forces full n_ctx for SWA; --ctx-checkpoints N adds snapshots. if ( self._sliding_window is not None and self._sliding_window > 0 @@ -4402,19 +4098,15 @@ class LlamaCppBackend: and val_len is not None ): swa = self._sliding_window - if swa_full: - swa_cells_total = total_cells - else: - swa_limit = swa * (slots if kv_unified else 1) + ubatch - swa_cells_per_stream = min(cells_per_stream, swa_limit) - swa_cells_per_stream = _pad_kv_cells(swa_cells_per_stream) - swa_cells_total = swa_cells_per_stream * streams + per_slot_ctx = max(1, n_ctx // slots) + # --swa-full caches full per_slot_ctx (constant n_ctx total); else SWA + # caches 2*sliding_window per slot, clamped at per-slot ctx. + swa_cells_per_slot = per_slot_ctx if swa_full else min(n_ctx, 2 * swa, per_slot_ctx) key_len_swa = self._kv_key_length_swa or key_len val_len_swa = self._kv_value_length_swa or val_len - padded_v_width = None if flash_attn else self._max_kv_value_width(val_len, val_len_swa) if self._sliding_window_pattern is not None: - global_bytes = 0.0 - swa_bytes = 0.0 + global_bytes = 0.0 # constant across slots + swa_bytes_per_slot = 0.0 # multiplied by slots checkpoint_extra_per_slot = 0.0 # Only layers that allocate their own KV; trailing shared layers # reuse earlier caches. @@ -4424,48 +4116,41 @@ class LlamaCppBackend: layer_idx < len(self._sliding_window_pattern) and self._sliding_window_pattern[layer_idx] ) - layer_key_bytes = layer_n_kv * (key_len_swa if is_swa else key_len) * bpe_k - layer_value_bytes = ( - layer_n_kv * (val_len_swa if is_swa else val_len) - if padded_v_width is None - else padded_v_width - ) * bpe_v - layer_kv_bytes = layer_key_bytes + layer_value_bytes if is_swa: - swa_bytes += swa_cells_total * layer_kv_bytes + swa_bytes_per_slot += ( + swa_cells_per_slot * layer_n_kv * (key_len_swa + val_len_swa) * bpe + ) if ctx_checkpoints > 0 and not swa_full: - checkpoint_extra_per_slot += ctx_checkpoints * swa * layer_kv_bytes + checkpoint_extra_per_slot += ( + ctx_checkpoints + * swa + * layer_n_kv + * (key_len_swa + val_len_swa) + * bpe + ) else: - global_bytes += total_cells * layer_kv_bytes - return int(global_bytes + swa_bytes + slots * checkpoint_extra_per_slot) + global_bytes += n_ctx * layer_n_kv * (key_len + val_len) * bpe + return int(global_bytes + slots * (swa_bytes_per_slot + checkpoint_extra_per_slot)) n_global = max(1, n_layers_kv // 4) n_swa = n_layers_kv - n_global - global_v_width = n_kv * val_len if padded_v_width is None else padded_v_width - swa_v_width = n_kv * val_len_swa if padded_v_width is None else padded_v_width - kv_per_token = n_kv * key_len * bpe_k + global_v_width * bpe_v - kv_per_token_swa = n_kv * key_len_swa * bpe_k + swa_v_width * bpe_v - global_bytes = n_global * total_cells * kv_per_token - swa_bytes = n_swa * swa_cells_total * kv_per_token_swa + kv_per_token = n_kv * (key_len + val_len) * bpe + kv_per_token_swa = n_kv * (key_len_swa + val_len_swa) * bpe + global_bytes = n_global * n_ctx * kv_per_token + swa_bytes_per_slot = n_swa * swa_cells_per_slot * kv_per_token_swa checkpoint_extra_per_slot = ( ctx_checkpoints * n_swa * swa * kv_per_token_swa if ctx_checkpoints > 0 and not swa_full else 0.0 ) - return int(global_bytes + swa_bytes + slots * checkpoint_extra_per_slot) + return int(global_bytes + slots * (swa_bytes_per_slot + checkpoint_extra_per_slot)) # Path 4: Standard GQA with explicit key/value dimensions if key_len is not None and val_len is not None: - padded_v_width = None if flash_attn else self._max_kv_value_width(val_len) - bytes_per_cell = 0.0 - for layer_idx in range(n_layers_kv): - layer_n_kv = self._kv_heads_for_layer(layer_idx, n_kv) - v_width = layer_n_kv * val_len if padded_v_width is None else padded_v_width - bytes_per_cell += layer_n_kv * key_len * bpe_k + v_width * bpe_v - return int(total_cells * bytes_per_cell) + return int(n_layers_kv * n_ctx * n_kv * (key_len + val_len) * bpe) # Path 5: Legacy fallback (old GGUFs without explicit dimensions) head_dim = self._legacy_head_dim() - return int(2 * n_kv * head_dim * n_layers_kv * total_cells * bpe_k) + return int(2 * n_kv * head_dim * n_layers_kv * n_ctx * bpe) def _draft_backend_for(self, drafter_path: str) -> Optional["LlamaCppBackend"]: """Lightweight backend with a drafter GGUF's metadata, to size its own KV @@ -4513,10 +4198,6 @@ class LlamaCppBackend: draft_cache_type_k: Optional[str] = None, draft_cache_type_v: Optional[str] = None, n_parallel: int = 1, - swa_full: bool = False, - kv_unified: bool = True, - n_ubatch: Optional[int] = None, - flash_attn: bool = True, ) -> Optional[int]: """Draft KV cache bytes at n_ctx, sized from GGUF dims (K and V types are independent). Separate drafter (Gemma): its own KV via _estimate_kv_cache_bytes @@ -4530,23 +4211,12 @@ class LlamaCppBackend: db = self._draft_backend_for(drafter_path) if db is None or not db._can_estimate_kv(): return None - # Gemma 4 assistant layers share the target context's final global - # and SWA KV tensors, so only the drafter weights add memory. - if getattr(db, "_architecture", None) == "gemma4-assistant": - return 0 heavier = draft_cache_type_k if bpe_k >= bpe_v else draft_cache_type_v - # The drafter uses the main model's slot and stream layout, so its - # compact SWA and per-stream padding must follow the same settings. - kv = db._estimate_kv_cache_bytes( - n_ctx, - heavier, - n_parallel = n_parallel, - swa_full = swa_full, - kv_unified = kv_unified, - n_ubatch = n_ubatch, - flash_attn = flash_attn, - ) - return kv if kv > 0 else None + # The drafter is served under the same --parallel slot count as the + # main model, so price its KV per slot too: a sliding-window drafter + # (Gemma) grows KV with slots and would otherwise be under-reserved. + kv = db._estimate_kv_cache_bytes(n_ctx, heavier, n_parallel = n_parallel) + return kv or None nextn = self._nextn_predict_layers or 0 n_kv = self._n_kv_heads or self._n_heads k_len = self._kv_key_length @@ -4560,14 +4230,7 @@ class LlamaCppBackend: f16_bpe = _kv_bytes_per_elem("f16") bpe_k = max(bpe_k, f16_bpe) bpe_v = max(bpe_v, f16_bpe) - _, streams, cells_per_stream = _kv_cache_cell_layout(n_ctx, n_parallel, kv_unified) - v_width = n_kv * v_len - if not flash_attn: - v_width = self._max_kv_value_width( - v_len, - self._kv_value_length_swa, - ) - return int(nextn * (n_kv * k_len * bpe_k + v_width * bpe_v) * cells_per_stream * streams) + return int(nextn * n_kv * (k_len * bpe_k + v_len * bpe_v) * n_ctx) def _estimate_mtp_overhead_bytes( self, @@ -4580,10 +4243,6 @@ class LlamaCppBackend: draft_weights_bytes: int = 0, n_parallel: int = 1, mtp_keeps_target_ctx: bool = True, - swa_full: bool = False, - kv_unified: bool = True, - n_ubatch: Optional[int] = None, - flash_attn: bool = True, ) -> Optional[int]: """MTP draft reserve at ``n_ctx`` = draft KV (grows with ctx) + separate- drafter weights + (MTP + MLA only) a duplicated target KV context. The @@ -4599,10 +4258,6 @@ class LlamaCppBackend: draft_cache_type_k = draft_cache_type_k, draft_cache_type_v = draft_cache_type_v, n_parallel = n_parallel, - swa_full = swa_full, - kv_unified = kv_unified, - n_ubatch = n_ubatch, - flash_attn = flash_attn, ) weights = max(0, draft_weights_bytes) # MLA models (GLM-5.x, DeepSeek, Kimi-K2) under MTP keep a *second* full copy @@ -4618,15 +4273,7 @@ class LlamaCppBackend: # rather than duplicating the target, so they must not be charged for it. target_ctx_copy = 0 if mtp_keeps_target_ctx and self._kv_lora_rank is not None: - target_ctx_copy = self._estimate_kv_cache_bytes( - n_ctx, - "f16", - n_parallel = n_parallel, - swa_full = swa_full, - kv_unified = kv_unified, - n_ubatch = n_ubatch, - flash_attn = flash_attn, - ) + target_ctx_copy = self._estimate_kv_cache_bytes(n_ctx, "f16", n_parallel = n_parallel) if draft_kv is None: # KV unsized (exotic/remote drafter): still reserve known weights + any # MLA target copy so a large config can't launch over budget (the small @@ -4636,7 +4283,7 @@ class LlamaCppBackend: return total if total > 0 else None return draft_kv + weights + target_ctx_copy - _DEFAULT_N_UBATCH = _DEFAULT_LLAMA_N_UBATCH + _DEFAULT_N_UBATCH = 512 # llama.cpp --ubatch default; Unsloth does not override it _COMPUTE_BUFFER_SAFETY = 1.15 # upper-bound margin on the compute-buffer estimate # Soft VRAM the modeled terms omit; charged to the fit budget on tight tiers (#6682). _CUDA_CONTEXT_RESERVE_BYTES = 320 * 1024 * 1024 # CUDA ctx + cuBLAS workspace (~330 MiB) @@ -4694,10 +4341,7 @@ class LlamaCppBackend: n_embd = self._embedding_length or 0 if n_vocab <= 0 or n_embd <= 0: return 0 - ub = max( - 1, - int(self._DEFAULT_N_UBATCH if n_ubatch is None else n_ubatch), - ) + ub = max(1, int(n_ubatch if n_ubatch else self._DEFAULT_N_UBATCH)) par = max(1, int(n_parallel)) out_buffer = n_vocab * ub * 4 # f32 output/logits buffer act_scratch = 4 * n_embd * ub * 4 # a few resident hidden-width buffers @@ -4729,10 +4373,7 @@ class LlamaCppBackend: n_embd = self._embedding_length or 0 if n_embd <= 0 or n_ctx <= 0: return 0 - ub = max( - 1, - int(self._DEFAULT_N_UBATCH if n_ubatch is None else n_ubatch), - ) + ub = max(1, int(n_ubatch if n_ubatch else self._DEFAULT_N_UBATCH)) if getattr(self, "_architecture", None) == "deepseek4": # DSV4 indexer/CSA buffer (see constants): flat + linear, ub-scaled. Fires # for any KV type -- the indexer scratch is present even with an f16 cache. @@ -4780,9 +4421,6 @@ class LlamaCppBackend: per_device_overhead_bytes: int, min_gpus: int, n_ubatch: Optional[int] = None, - swa_full: bool = False, - kv_unified: bool = True, - flash_attn: bool = True, ) -> tuple[Optional[list[int]], bool, int]: """Largest serving-slot count in [1, n_parallel) whose fully-on-GPU footprint fits, so Unsloth keeps the model on GPU (-ngl -1) instead of --fit on, which offloads layers @@ -4801,15 +4439,7 @@ class LlamaCppBackend: total = ( base_footprint_bytes + cb - + self._estimate_kv_cache_bytes( - effective_ctx, - cache_type_kv, - n_parallel = slots, - swa_full = swa_full, - kv_unified = kv_unified, - n_ubatch = n_ubatch, - flash_attn = flash_attn, - ) + + self._estimate_kv_cache_bytes(effective_ctx, cache_type_kv, n_parallel = slots) ) gpu_indices, use_fit = self._select_gpus( total, @@ -4834,9 +4464,7 @@ class LlamaCppBackend: swa_full: bool = False, n_parallel: int = 1, kv_unified: bool = True, - n_ubatch: Optional[int] = None, ctx_checkpoints: int = 0, - flash_attn: bool = True, kv_on_gpu: bool = True, mtp_engaged: bool = False, mtp_overhead_fn: Optional[Callable[[int], int]] = None, @@ -4873,9 +4501,7 @@ class LlamaCppBackend: swa_full = swa_full, n_parallel = n_parallel, kv_unified = kv_unified, - n_ubatch = n_ubatch, ctx_checkpoints = ctx_checkpoints, - flash_attn = flash_attn, ) # byte-accurate mtp_overhead_fn supersedes the flat fraction (the fallback @@ -5522,9 +5148,7 @@ class LlamaCppBackend: stdout = subprocess.PIPE, stderr = subprocess.STDOUT, text = True, - encoding = "utf-8", - errors = "replace", - env = utf8_child_env(env), + env = env, **_windows_hidden_subprocess_kwargs(), **_child_popen_kwargs(), ) @@ -5540,12 +5164,6 @@ class LlamaCppBackend: self._is_audio = False # clear any prior TTS/audio model's routing flag self._model_identifier = model_identifier self._cache_type_kv = None - self._swa_full = False - self._kv_cache_unified = False - self._n_ubatch = self._DEFAULT_N_UBATCH - self._flash_attn_enabled = True - self._effective_cache_types = ("f16", "f16") - self._kv_cache_context_total = None self._gpu_offload_active = True # Diffusion doesn't use the llama.cpp GPU-memory knobs; reset them to # defaults (the picked device is still recorded below) so /load, /status @@ -6287,9 +5905,6 @@ class LlamaCppBackend: total_by_idx: Optional[dict[int, int]] = None, n_ubatch: Optional[int] = None, soft_overhead_bytes: int = 0, - swa_full: bool = False, - kv_unified: bool = True, - flash_attn: bool = True, ) -> tuple[int, int, list[int], Optional[list[int]]]: """Plan a ``--split-mode tensor`` load. Pure: no model or GPU needed. @@ -6377,17 +5992,6 @@ class LlamaCppBackend: def _mtp_at(ctx: int) -> int: return mtp_overhead_fn(ctx) if mtp_overhead_fn is not None else 0 - def _kv_at(ctx: int) -> int: - return self._estimate_kv_cache_bytes( - ctx, - cache_type_kv, - n_parallel = n_parallel, - swa_full = swa_full, - kv_unified = kv_unified, - n_ubatch = n_ubatch, - flash_attn = flash_attn, - ) - # Context-linear compute buffer, summed over the split. Tensor mode # replicates the compute graph on EVERY device (measured: the per-device # buffer grows a flat n_ubatch*2 bytes/token, ~1024 B/tok on Qwen3.5-9B at @@ -6413,21 +6017,31 @@ class LlamaCppBackend: # Weights + buffers exceed the pool -> floor; the load then # falls back to layer split. return ctx_floor + if mtp_overhead_fn is not None: + # kv(ctx)+mtp(ctx)+compute(ctx) is not single-linear, so binary search. + def _consumer(c: int) -> int: + return ( + self._estimate_kv_cache_bytes(c, cache_type_kv, n_parallel = n_parallel) + + _mtp_at(c) + + _cc_ctx(c) + ) - def _consumer(c: int) -> int: - return _kv_at(c) + _mtp_at(c) + _cc_ctx(c) - - if _consumer(ctx) <= kv_budget_b: + if _consumer(ctx) <= kv_budget_b: + return ctx + lo, hi, best = ctx_floor, ctx, ctx_floor + while lo <= hi: + mid = (lo + hi) // 2 + if _consumer(mid) <= kv_budget_b: + best = mid + lo = mid + 1 + else: + hi = mid - 1 + return best + kv_at = self._estimate_kv_cache_bytes(ctx, cache_type_kv, n_parallel = n_parallel) + total_at = kv_at + _cc_ctx(ctx) # both ~linear through the origin + if total_at <= kv_budget_b: return ctx - lo, hi, best = ctx_floor, ctx, ctx_floor - while lo <= hi: - mid = (lo + hi) // 2 - if _consumer(mid) <= kv_budget_b: - best = mid - lo = mid + 1 - else: - hi = mid - 1 - return best + return max(ctx_floor, int(ctx * kv_budget_b / total_at)) # KV size unknown -> can't prove a safe cap; floor. return min(4096, ctx) if ctx > 0 else 4096 @@ -6439,7 +6053,11 @@ class LlamaCppBackend: effective_ctx = min(_fit_ctx(target_ctx), max_available_ctx) min_usable_mib = min(usable_by_idx.values()) - kv_bytes = _kv_at(effective_ctx) if (self._can_estimate_kv() and effective_ctx > 0) else 0 + kv_bytes = ( + self._estimate_kv_cache_bytes(effective_ctx, cache_type_kv, n_parallel = n_parallel) + if (self._can_estimate_kv() and effective_ctx > 0) + else 0 + ) # The MTP reserve also has to fit the even split (mirror the pooled budget): # byte-accurate per-ctx (0 when no fn) plus the same flat cushion as above. mtp_bytes = (_mtp_at(effective_ctx) if effective_ctx > 0 else 0) + flat_mtp_bytes @@ -6564,6 +6182,21 @@ class LlamaCppBackend: cls._is_signal_crash(returncode) or cls._is_abort_exit(returncode) ) + @staticmethod + def _canonical_long_flag(name: str) -> str: + """Return ``name`` with llama.cpp's long-option underscore normalization. + + llama.cpp runs ``std::replace(arg.begin(), arg.end(), '_', '-')`` on any + argv token that starts with ``--`` before looking it up, so a legal + pass-through spelling like ``--cache_type_v`` parses as + ``--cache-type-v``. Mirror that here so managed-flag matching sees the + same canonical name. Short flags (``-ctv``) never carry underscores and + keep their exact spelling; pass only the flag name (no attached value). + """ + if name.startswith("--"): + return name.replace("_", "-") + return name + @staticmethod def _with_flash_attn_off(cmd: list[str]) -> Optional[list[str]]: """Return cmd with flash attention forced off, or None when its effective @@ -6576,25 +6209,23 @@ class LlamaCppBackend: def explicit(i): nxt = out[i + 1] if i + 1 < len(out) else None - return nxt if nxt in _LLAMA_ARG_TRUE_FALSE_AUTO_VALUES else None + return nxt if nxt in ("on", "auto", "off") else None effective = None for i, tok in enumerate(out): - name = _flag_name(tok) - if name in ("--flash-attn", "-fa") and "=" in tok: + if tok.startswith(("--flash-attn=", "-fa=")): effective = tok.partition("=")[2] - elif name in ("--flash-attn", "-fa"): + elif tok in ("--flash-attn", "-fa"): effective = explicit(i) or "on" - if effective not in _LLAMA_ARG_TRUE_OR_AUTO_VALUES: + if effective not in ("on", "auto"): return None for i, tok in enumerate(out): - name = _flag_name(tok) - if name in ("--flash-attn", "-fa") and "=" in tok: + if tok.startswith(("--flash-attn=", "-fa=")): flag, _, value = tok.partition("=") - if value in _LLAMA_ARG_TRUE_OR_AUTO_VALUES: + if value in ("on", "auto"): out[i] = f"{flag}=off" - elif name in ("--flash-attn", "-fa"): - if explicit(i) in _LLAMA_ARG_TRUE_OR_AUTO_VALUES: + elif tok in ("--flash-attn", "-fa"): + if explicit(i) in ("on", "auto"): out[i + 1] = "off" elif explicit(i) is None: # bare flag (reads as on) -> explicit off out[i] = f"{tok}=off" @@ -6626,7 +6257,7 @@ class LlamaCppBackend: # quantized V cache. Canonicalize the flag name the same way so the # reset recognizes the underscore aliases too; short flags (-ctv) # and the type value are left untouched. - name = _flag_name(tok) + name = LlamaCppBackend._canonical_long_flag(tok.partition("=")[0]) if name not in _v_cache_flags: continue if "=" in tok: @@ -6738,8 +6369,6 @@ class LlamaCppBackend: stdout = subprocess.PIPE, stderr = subprocess.STDOUT, text = True, - encoding = "utf-8", - errors = "replace", env = env, **_windows_hidden_subprocess_kwargs(), **_child_popen_kwargs(), @@ -6858,7 +6487,6 @@ class LlamaCppBackend: chat_template_override = chat_template_override, extra_args = extra_args, is_vision = is_vision, - n_parallel = n_parallel, preserve_multi_gpu_on_layer = preserve_multi_gpu_on_layer, ): logger.info( @@ -6886,25 +6514,6 @@ class LlamaCppBackend: binary = self._find_llama_server_binary() is_vulkan_backend = self._is_vulkan_backend(binary) - # Without --kv-unified an explicit --parallel N splits -c into windows of -c/N, so on a - # build lacking the flag the default of 4 would quarter every context window for a - # feature it cannot serve: fall back to one slot. Ahead of the KV estimates so the - # fit matches what launches. - if ( - n_parallel > 1 - and binary - and not self.probe_server_capabilities(binary).get("supports_kv_unified") - ): - logger.warning( - "llama-server at %s has no --kv-unified, so %d parallel slots would " - "split the context window %d ways. Using 1 slot instead; update " - "llama.cpp to run chats in parallel.", - binary, - n_parallel, - n_parallel, - ) - n_parallel = 1 - # ── Vulkan-ordinal preflight (BEFORE the Phase 1 kill) ──────── # An explicit Vulkan pin the ggml probe never enumerated cannot be honored. # Validate it ABOVE the kill so an invalid selection leaves the live model @@ -7026,23 +6635,12 @@ class LlamaCppBackend: # Block-diffusion GGUFs (DiffusionGemma) cannot run on llama-server; # serve them with the diffusion runner (same OpenAI-compat interface). if self._is_diffusion: + # Final defense: route and pre-teardown preflights reject before Phase 1. + if is_vulkan_backend and gpu_ids: + raise ValueError(_VULKAN_DIFFUSION_GPU_IDS_ERROR) # Not a tensor/layer GGUF: clear any preserved-fallback flag from a # prior load (this path skips the command builder that clears it). self._layer_preserves_tensor_intent = False - # On a Vulkan build gpu_ids are ggml Vulkan ordinals, but the diffusion - # runner selects its device by CUDA physical index (_diffusion_gpu_arg - # forwards gpu_ids[0] as a CUDA/DG_GPU token) with no mapping to them. - # The route rejects a CONFIRMED-diffusion pick up front; an uncached GGUF - # only classified as diffusion post-download still reaches here with a - # pin, so drop it and serve on the default device (like an unpinned load). - if gpu_ids and is_vulkan_backend: - logger.warning( - "Ignoring gpu_ids %s for diffusion GGUF on a Vulkan build: " - "the diffusion runner cannot map ggml Vulkan ordinals; " - "serving on the default device.", - gpu_ids, - ) - gpu_ids = None with self._lock: if self._cancel_event.is_set(): logger.info("Load cancelled before diffusion server start") @@ -7074,8 +6672,6 @@ class LlamaCppBackend: # same message remote validation already shows. raise LlamaServerNotFoundError(LLAMA_SERVER_NOT_FOUND_DETAIL) - server_caps = self.probe_server_capabilities(binary) - # Outside ``self._lock`` so /unload, /cancel, /status aren't # blocked. ``unload_model`` also records the kill, so the # frontend /unload+/load Apply path engages the wait here even @@ -7096,18 +6692,6 @@ class LlamaCppBackend: # state to publish. ctx_override = parse_ctx_override(extra_args) requested_ctx = resolve_requested_ctx(extra_args, n_ctx) - swa_full = _swa_full_from_args_or_env(extra_args) - _effective_ubatch = _extra_args_n_ubatch( - extra_args, - n_ctx = (requested_ctx if requested_ctx > 0 else self._context_length), - ) - planned_kv_unified = _kv_unified_from_args( - extra_args, - default = n_parallel > 1 and server_caps.get("supports_kv_unified", False), - ) - # A hard-crash recovery may relaunch this same plan with FA off. - # Size that larger cache up front so the recovery cannot OOM. - planned_flash_attn = False cache_override = parse_cache_override(extra_args) # Budget the heavier of asymmetric --cache-type-k/-v extras (they # win per axis at launch, appended last); resolve_cache_type_kv only @@ -7538,10 +7122,6 @@ class LlamaCppBackend: draft_cache_type_k = _mtp_draft_ck, draft_cache_type_v = _mtp_draft_cv, n_parallel = n_parallel, - swa_full = swa_full, - kv_unified = planned_kv_unified, - n_ubatch = _effective_ubatch, - flash_attn = planned_flash_attn, ) if ( self._estimate_mtp_overhead_bytes( @@ -7553,10 +7133,6 @@ class LlamaCppBackend: draft_weights_bytes = _mtp_draft_weights, n_parallel = n_parallel, mtp_keeps_target_ctx = _engaged_is_mtp, - swa_full = swa_full, - kv_unified = planned_kv_unified, - n_ubatch = _effective_ubatch, - flash_attn = planned_flash_attn, ) is not None ): @@ -7573,10 +7149,6 @@ class LlamaCppBackend: _w: int = _mtp_draft_weights, _np: int = n_parallel, _mtp: bool = _engaged_is_mtp, - _swa_full: bool = swa_full, - _kv_unified: bool = planned_kv_unified, - _n_ubatch: Optional[int] = _effective_ubatch, - _flash_attn: bool = planned_flash_attn, ) -> int: v = self._estimate_mtp_overhead_bytes( ctx, @@ -7587,26 +7159,15 @@ class LlamaCppBackend: draft_weights_bytes = _w, n_parallel = _np, mtp_keeps_target_ctx = _mtp, - swa_full = _swa_full, - kv_unified = _kv_unified, - n_ubatch = _n_ubatch, - flash_attn = _flash_attn, ) return v if v is not None else 0 def _mtp_bytes(ctx: int) -> int: return mtp_overhead_fn(ctx) if mtp_overhead_fn is not None else 0 - def _kv_bytes(ctx: int) -> int: - return self._estimate_kv_cache_bytes( - ctx, - cache_type_kv, - n_parallel = n_parallel, - swa_full = swa_full, - kv_unified = planned_kv_unified, - n_ubatch = _effective_ubatch, - flash_attn = planned_flash_attn, - ) + # Effective micro-batch (a user --ubatch override scales the + # compute buffer); None -> the 512 default in the estimate. + _effective_ubatch = _extra_args_n_ubatch(extra_args) def _cc_bytes(ctx: int, n_gpus: int = 1) -> int: # Context-linear compute-buffer growth (flash-attn KQ mask + @@ -7846,9 +7407,6 @@ class LlamaCppBackend: total_by_idx = total_by_idx, n_ubatch = _effective_ubatch, soft_overhead_bytes = _soft_overhead, - swa_full = swa_full, - kv_unified = planned_kv_unified, - flash_attn = planned_flash_attn, ) use_fit = False elif gpus and self._can_estimate_kv() and effective_ctx > 0: @@ -7881,18 +7439,16 @@ class LlamaCppBackend: pool_budget, _ms, cache_type_kv, - swa_full = swa_full, n_parallel = n_parallel, - kv_unified = planned_kv_unified, - n_ubatch = _effective_ubatch, - flash_attn = planned_flash_attn, mtp_engaged = _mtp_reserves_gpu, mtp_overhead_fn = mtp_overhead_fn, compute_ctx_bytes_fn = _cc_sub, budget_frac = 1.0, total_mib = None, ) - kv = _kv_bytes(capped) + kv = self._estimate_kv_cache_bytes( + capped, cache_type_kv, n_parallel = n_parallel + ) footprint_mib = ( _ms + kv + _mtp_bytes(capped) + _cc_sub(capped) ) / (1024 * 1024) @@ -7912,7 +7468,9 @@ class LlamaCppBackend: # on and let llama-server flex -ngl (CPU offload). requested_total = ( model_size_fit - + _kv_bytes(effective_ctx) + + self._estimate_kv_cache_bytes( + effective_ctx, cache_type_kv, n_parallel = n_parallel + ) + _mtp_bytes(effective_ctx) + _cc_bytes(effective_ctx) ) @@ -7964,18 +7522,16 @@ class LlamaCppBackend: pool_budget, _ms, cache_type_kv, - swa_full = swa_full, n_parallel = n_parallel, - kv_unified = planned_kv_unified, - n_ubatch = _effective_ubatch, - flash_attn = planned_flash_attn, mtp_engaged = _mtp_reserves_gpu, mtp_overhead_fn = mtp_overhead_fn, compute_ctx_bytes_fn = _cc_sub, budget_frac = 1.0, total_mib = None, ) - kv = _kv_bytes(capped) + kv = self._estimate_kv_cache_bytes( + capped, cache_type_kv, n_parallel = n_parallel + ) footprint_mib = ( _ms + kv + _mtp_bytes(capped) + _cc_sub(capped) ) / (1024 * 1024) @@ -7992,7 +7548,11 @@ class LlamaCppBackend: if effective_ctx > 0: for n_gpus in range(_auto_min_gpus, len(ranked) + 1): subset = ranked[:n_gpus] - kv = _kv_bytes(effective_ctx) + kv = self._estimate_kv_cache_bytes( + effective_ctx, + cache_type_kv, + n_parallel = n_parallel, + ) footprint_mib = ( _subset_model_size(n_gpus) + kv @@ -8049,11 +7609,7 @@ class LlamaCppBackend: _apple_fit_budget_mib, model_size_fit, cache_type_kv, - swa_full = swa_full, n_parallel = n_parallel, - kv_unified = planned_kv_unified, - n_ubatch = _effective_ubatch, - flash_attn = planned_flash_attn, mtp_engaged = _mtp_reserves_gpu, mtp_overhead_fn = mtp_overhead_fn, compute_ctx_bytes_fn = _cc_bytes, @@ -8061,7 +7617,12 @@ class LlamaCppBackend: total_mib = None, ) _cap_footprint_mib = ( - model_size_fit + _kv_bytes(cap) + _mtp_bytes(cap) + _cc_bytes(cap) + model_size_fit + + self._estimate_kv_cache_bytes( + cap, cache_type_kv, n_parallel = n_parallel + ) + + _mtp_bytes(cap) + + _cc_bytes(cap) ) / (1024 * 1024) # Fit returns the request unchanged when it fits OR weights # exceed budget; only the latter over-commits, so floor to 4096. @@ -8108,9 +7669,6 @@ class LlamaCppBackend: _pipeline_overhead_bytes + _cc_bytes(effective_ctx), _layer_min_gpus, _effective_ubatch, - swa_full = swa_full, - kv_unified = planned_kv_unified, - flash_attn = planned_flash_attn, ) if not _uf_slots: logger.info( @@ -8135,7 +7693,9 @@ class LlamaCppBackend: _mtp_note = "" if effective_ctx < original_ctx: - kv_est = _kv_bytes(effective_ctx) + kv_est = self._estimate_kv_cache_bytes( + effective_ctx, cache_type_kv, n_parallel = n_parallel + ) logger.info( f"Context auto-reduced: {original_ctx} -> {effective_ctx} " f"(model: {model_size / (1024**3):.1f} GB, " @@ -8144,7 +7704,9 @@ class LlamaCppBackend: + ")" ) - kv_cache_bytes = _kv_bytes(effective_ctx) + kv_cache_bytes = self._estimate_kv_cache_bytes( + effective_ctx, cache_type_kv, n_parallel = n_parallel + ) mmproj_note = ( f"mmproj: {mmproj_size / (1024**3):.1f} GB, " if mmproj_size else "" ) @@ -8309,6 +7871,7 @@ class LlamaCppBackend: cmd.extend(["-ngl", "-1", "--fit", "off"]) fully_gpu_offloaded = True + server_caps = self.probe_server_capabilities(binary) # Expose Prometheus /metrics for the engine-stats logger, only # when the binary advertises it (older/custom binaries may not). if server_caps.get("supports_metrics"): @@ -8380,11 +7943,6 @@ class LlamaCppBackend: "iq4_nl", "f32", } - # Normalize like the budget does (_planned_main_cache_types): a - # case-sensitive match drops "Q8_0", emitting no flag, so llama.cpp - # runs f16 while the estimate priced q8_0. Emit the normalized - # spelling; kv_cache_type_from_str is case-sensitive. - cache_type_kv = cache_type_kv.strip().lower() if cache_type_kv else cache_type_kv if ( cache_type_kv and cache_type_kv in _valid_cache_types @@ -8587,8 +8145,6 @@ class LlamaCppBackend: cmd.extend(str(a) for a in extra_args) logger.info(f"Appending user extra args to llama-server: {list(extra_args)}") - kv_cache_unified = _kv_unified_from_args(cmd) - logger.info(f"Starting llama-server: {' '.join(self._redacted_cmd_for_log(cmd))}") # Library paths so llama-server finds its shared libs and CUDA DLLs. @@ -8680,7 +8236,7 @@ class LlamaCppBackend: env["CUDA_DEVICE_ORDER"] = "PCI_BUS_ID" # Mask on AMD at the ROCr/HSA layer: HIP-only masking still # enumerates every agent first, which segfaults on a deselected - # unsupported GPU (e.g. gfx1036 iGPU under a gfx103X prebuilt). + # unsupported GPU (e.g. gfx1103 iGPU under a gfx110X prebuilt). self._emit_child_gpu_visibility( env, ",".join(str(i) for i in gpu_indices), prefer_rocr = True ) @@ -8756,8 +8312,6 @@ class LlamaCppBackend: stdout = subprocess.PIPE, stderr = subprocess.STDOUT, text = True, - encoding = "utf-8", - errors = "replace", env = env, **_windows_hidden_subprocess_kwargs(), **_child_popen_kwargs(), @@ -9105,21 +8659,6 @@ class LlamaCppBackend: self._healthy = True self._commit_effective_parallel_slots(n_parallel) - self._swa_full = swa_full - self._kv_cache_unified = kv_cache_unified - self._n_ubatch = max( - 0, - int(self._DEFAULT_N_UBATCH if _effective_ubatch is None else _effective_ubatch), - ) - self._flash_attn_enabled = ( - _flash_attn_enabled_from_args(_last_spawn_cmd, env = env) - and self._architecture != "grok" - ) - self._effective_cache_types = _effective_main_cache_types( - _last_spawn_cmd, - env, - ) - self._kv_cache_context_total = effective_ctx if effective_ctx > 0 else None # Server is up: adopt the real per-request context it allocated # -- the length --fit chose, or a --parallel slot split -- so the @@ -9127,11 +8666,6 @@ class LlamaCppBackend: # before the spawn above always failed; the seeded value was the # requested/native length.) self._reconcile_effective_ctx_with_server() - if self._kv_cache_context_total is not None: - self._n_ubatch = min( - self._n_ubatch, - self._kv_cache_context_total, - ) # Commit caller intent only after _healthy=True so a failed start # can't poison the next inheritance check. None keeps prior, [] @@ -9141,8 +8675,6 @@ class LlamaCppBackend: self._extra_args = list(extra_args) self._extra_args_source = (model_identifier, hf_variant) self._requested_n_ctx = int(n_ctx) - # Local n_parallel may have been reduced above; the snapshot has the ask. - self._requested_n_parallel = max(1, int(_pending_load_kwargs["n_parallel"])) # Commit the known-good snapshot + whether MTP+tensor is live, then # watch this load for a mid-generation crash. self._last_load_kwargs = _pending_load_kwargs @@ -9555,7 +9087,6 @@ class LlamaCppBackend: tensor_split: Optional[List[float]] = None, gpu_ids: Optional[List[int]] = None, mtp_draft_path: Optional[str] = None, - n_parallel: int = 1, preserve_multi_gpu_on_layer: bool = False, ) -> bool: """True iff the live server already satisfies these load kwargs. @@ -9591,6 +9122,7 @@ class LlamaCppBackend: if _norm(self._cache_type_kv) != _norm(cache_type_kv): return False + # Reconcile a user --split-mode in extras AND an inherited tensor # LLAMA_ARG_SPLIT_MODE env, but only against a server that actually # launched tensor: if load_model downgraded to layer split it scrubbed @@ -9614,16 +9146,9 @@ class LlamaCppBackend: # layer/MoE/split knobs), so a standing manual preference in the # request must not force a needless reload -- only the GPU pick matters. if not self._is_diffusion: - requested_extra_args = extra_args if extra_args is not None else self._extra_args - if self._swa_full != _swa_full_from_args_or_env(requested_extra_args): - return False # A GPU-memory-mode flip (Unsloth / manual) must always reload. if self._gpu_memory_mode != gpu_memory_mode: return False - # Requested-vs-requested (like n_ctx): comparing the effective count - # would reload forever whenever the fitter launched fewer slots. - if self._requested_n_parallel != max(1, int(n_parallel)): - return False # Manual: a layer-count change always reloads (covers Auto(-1) <-> a # pinned count); MoE/split only matter with an explicit offload. if gpu_memory_mode == "manual" and ( @@ -9748,8 +9273,7 @@ class LlamaCppBackend: last_draft: Optional[str] = None args = [str(arg) for arg in cmd] for index, raw in enumerate(args): - flag = _flag_name(raw) - _, equals, inline = raw.partition("=") + flag, equals, inline = raw.partition("=") if flag not in main_flags and flag not in draft_flags: continue value = inline if equals else (args[index + 1] if index + 1 < len(args) else "") @@ -9822,12 +9346,6 @@ class LlamaCppBackend: self._slot_save_binary = None self._slot_loaded_identity = None self._prompt_cache_disabled = False - self._swa_full = False - self._kv_cache_unified = False - self._n_ubatch = self._DEFAULT_N_UBATCH - self._flash_attn_enabled = True - self._effective_cache_types = ("f16", "f16") - self._kv_cache_context_total = None self._chat_template = None self._chat_template_override = None self._supports_reasoning = False @@ -10260,8 +9778,6 @@ class LlamaCppBackend: ["pgrep", "-a", "-f", "llama-server"], capture_output = True, text = True, - encoding = "utf-8", - errors = "replace", timeout = 5, env = child_env_without_native_path_secret(), ) @@ -10373,12 +9889,8 @@ class LlamaCppBackend: tuple(sidecars), self._requested_n_ctx, self._effective_context_length, - self._effective_cache_types, + getattr(self, "_cache_type_kv", None), self.effective_parallel_slots, - self._swa_full, - self._kv_cache_unified, - self._n_ubatch, - self._flash_attn_enabled, ) def _gguf_file_identity(self, path) -> Optional[tuple]: @@ -10409,8 +9921,7 @@ class LlamaCppBackend: args = [str(a).strip() for a in (self._extra_args or ())] files: list[str] = [] for i, arg in enumerate(args): - flag = _flag_name(arg) - _, sep, inline = arg.partition("=") + flag, sep, inline = arg.partition("=") if flag not in self._SIDECAR_WEIGHT_FLAGS: continue operand = inline if sep else (args[i + 1] if i + 1 < len(args) else "") @@ -10450,7 +9961,7 @@ class LlamaCppBackend: if os.environ.get("LLAMA_ARG_NO_CACHE_PROMPT") is not None: return True env = (os.environ.get("LLAMA_ARG_CACHE_PROMPT") or "").strip().lower() - return env in _LLAMA_ARG_FALSE_VALUES + return env in {"off", "disabled", "false", "0"} def save_slots_for_resume( self, should_abort: Optional[Callable[[], bool]] = None @@ -10462,17 +9973,6 @@ class LlamaCppBackend: or self._prompt_cache_off() ): return None - # Same predicate as the estimator's SWA path: a window alone is not enough. - # phi3 GGUFs carry attention.sliding_window but no key/value length, and - # llama.cpp forces them back to a non-SWA cache, so their slots do restore. - if ( - (self._sliding_window or 0) > 0 - and self._kv_key_length is not None - and self._kv_value_length is not None - and not self._swa_full - ): - logger.debug("Skipping slot save: compact SWA cache cannot be reused after restart") - return None save_dir = Path(self._slot_save_dir) gguf_stat = self._gguf_file_identity(self._gguf_path) if gguf_stat is None: @@ -10489,16 +9989,9 @@ class LlamaCppBackend: return None try: estimate = self._estimate_kv_cache_bytes( - self._kv_cache_context_total - or self._effective_context_length - or self._context_length - or 0, - max(self._effective_cache_types, key = _kv_bytes_per_elem), + self._effective_context_length or self._context_length or 0, + self._cache_type_kv, n_parallel = self.effective_parallel_slots, - swa_full = self._swa_full, - kv_unified = self._kv_cache_unified, - n_ubatch = self._n_ubatch, - flash_attn = self._flash_attn_enabled, ) # Skip before writing anything when the estimate alone blows the cap, # rather than fully writing a slot and discarding it afterwards. @@ -10854,8 +10347,6 @@ class LlamaCppBackend: actual_n_ctx = self._query_server_n_ctx() if not actual_n_ctx or actual_n_ctx <= 0: return - slots = 1 if self._kv_cache_unified else self.effective_parallel_slots - self._kv_cache_context_total = actual_n_ctx * slots if self._effective_context_length and actual_n_ctx < self._effective_context_length: logger.warning( "llama-server allocated a smaller per-request context than " @@ -11562,7 +11053,6 @@ class LlamaCppBackend: from core.inference.tools import ( build_rag_autoinject, execute_tool, - has_text_only_provisional_card, is_always_safe_tool, is_high_risk_tool_call, ) @@ -11760,10 +11250,6 @@ class LlamaCppBackend: # direct answer ("4", "Hello!") won't match. Pattern shared with the # safetensors loop (tool_call_parser.INTENT_SIGNAL). _reprompt_count = 0 - # Budgeted apart from _reprompt_count so a pre-tool nudge can't spend it. - _post_tool_reprompts = 0 - # Text that triggered the last nudge; if the retry restates it, stop. - _last_reprompt_text = "" # Gates ``max_tool_iterations`` on real tool turns (not the enlarged range) so reserved # re-prompt slots don't extend the budget. Mirrors the safetensors guard. _tool_iters_done = 0 @@ -11771,7 +11257,7 @@ class LlamaCppBackend: # Reserve extra iterations for re-prompts so they don't consume the # caller's tool-call budget; only when tool iterations are allowed. - _extra = _MAX_REPROMPTS + 1 if max_tool_iterations > 0 else 0 + _extra = _MAX_REPROMPTS if max_tool_iterations > 0 else 0 for iteration in range(max_tool_iterations + _extra): if cancel_event is not None and cancel_event.is_set(): return @@ -11837,7 +11323,6 @@ class LlamaCppBackend: # Time each reasoning pass so final answers can replace tool timing. _reasoning_started_at = None _reasoning_summary_emitted = False - _deferred_reasoning_summary = None cumulative_display = "" # Cumulative yielded text (with ) in_thinking = False has_content_tokens = False @@ -11994,9 +11479,6 @@ class LlamaCppBackend: permission_mode == "auto" and is_always_safe_tool(current_name) ) - # A text-preview card still streams while gated; - # hiding it blanks the chat. - and not has_text_only_provisional_card(current_name) ) # Keep small-argument tools on the normal path. _args_len = len( @@ -12089,11 +11571,7 @@ class LlamaCppBackend: and not _reasoning_summary_emitted ): _reasoning_summary_emitted = True - _summary = _reasoning_summary_event(_reasoning_started_at) - if _suppress_visible_output: - _deferred_reasoning_summary = _summary - else: - yield _summary + yield _reasoning_summary_event(_reasoning_started_at) has_content_tokens = True content_accum += token @@ -12102,27 +11580,20 @@ class LlamaCppBackend: # TEXT call to a provisional card. Gated on an enabled-name # sniff + size floor so prose/small calls spawn no pane; id # matches the first call so the final tool_start reconciles. - if not has_structured_tc and _text_args_call_start >= 0: + if ( + not has_structured_tc + and not _confirm_gated_iteration + and _text_args_call_start >= 0 + ): if not _text_args_id: _call_text = content_accum[_text_args_call_start:] _sniffed = _sniff_text_tool_name( _call_text, _enabled_tool_names ) - # Structured-path rule: gated calls - # stream only from a text-preview card. - if ( - _sniffed - and not ( - _confirm_gated_iteration - and not has_text_only_provisional_card( - _sniffed - ) - ) - and ( - _sniffed == "render_html" - or len(_call_text) - >= _PROVISIONAL_ARGS_MIN_CHARS - ) + if _sniffed and ( + _sniffed == "render_html" + or len(_call_text) + >= _PROVISIONAL_ARGS_MIN_CHARS ): _text_args_id = "call_0" _text_args_name = _sniffed @@ -12377,11 +11848,7 @@ class LlamaCppBackend: # route's extractor closes the streamed ). if _reasoning_started_at is not None and not _reasoning_summary_emitted: _reasoning_summary_emitted = True - _summary = _reasoning_summary_event(_reasoning_started_at) - if _suppress_visible_output: - _deferred_reasoning_summary = _summary - else: - yield _summary + yield _reasoning_summary_event(_reasoning_started_at) cumulative_display = _finalize_reasoning_only_cumulative( cumulative_display, reasoning_accum, @@ -12416,10 +11883,12 @@ class LlamaCppBackend: ) if not _safety_tc: # ── Re-prompt on plan-without-action ── - # Intent described without a tool call: nudge it to act. Up - # to _MAX_REPROMPTS times, only on short responses with intent - # signals -- "4" or "Hello!" won't trigger it. Uses content, - # else reasoning text (reasoning-only stalls). + # If the model described its intent (forward-looking + # language) without calling a tool, nudge it to act. + # Fires at most once per request, only on short + # responses with intent signals -- "4" or "Hello!" + # won't trigger it. Use content if available, else + # fall back to reasoning text (reasoning-only stalls). _stripped = content_accum.strip() if not _stripped: _stripped = reasoning_accum.strip() @@ -12429,33 +11898,18 @@ class LlamaCppBackend: r"(?i)\brender[_\s-]?html\b", _stripped, ) - # A post-tool stall still deserves a nudge, but each retry - # re-runs tools, so allow only one. RAG autoinject never lands - # in history, so _auto keeps a doc-grounded turn from reading - # as pre-tool (mirrors safetensors rag_autoinjected). - _already_acted = bool(_auto) or any( - record.executed for record in tool_controller.history - ) - if _already_acted: - _reprompt_used, _reprompt_cap = _post_tool_reprompts, 1 - else: - _reprompt_used, _reprompt_cap = _reprompt_count, _MAX_REPROMPTS # None keeps the default-on re-prompt; False disables it. if ( auto_heal_tool_calls and (nudge_tool_calls is None or nudge_tool_calls) and active_tools and not _render_html_already_done_intent - and _reprompt_used < _reprompt_cap - and not _is_reprompt_repeat(_stripped, _last_reprompt_text) + and _reprompt_count < _MAX_REPROMPTS and _is_short_intent_without_action(_stripped) ): _reprompt_count += 1 - if _already_acted: - _post_tool_reprompts += 1 - _last_reprompt_text = _stripped logger.info( - f"Re-prompt {_reprompt_used + 1}/{_reprompt_cap}: " + f"Re-prompt {_reprompt_count}/{_MAX_REPROMPTS}: " f"model responded without calling tools " f"({len(_stripped)} chars)" ) @@ -12485,18 +11939,12 @@ class LlamaCppBackend: _it_r = _iter_timings or {} _accumulated_predicted_ms += _it_r.get("predicted_ms", 0) _accumulated_predicted_n += _it_r.get("predicted_n", 0) - # Blank first (the route resets its text cursor only on an - # empty status), then the badge so the retry is not a hang. yield {"type": "status", "text": ""} - yield {"type": "status", "text": _NUDGE_TOOL_CALLS_STATUS} continue if _forced_tool_call_pending: _forced_tool_call_pending = False - if not _should_suppress_forced_no_tool_output( - _stripped, - _last_reprompt_text, - ): + if not _should_suppress_forced_no_tool_output(_stripped): if cumulative_display: forced_visible_text = _strip_tool_markup( cumulative_display, @@ -12514,8 +11962,6 @@ class LlamaCppBackend: "type": "content", "text": forced_visible_text, } - if _deferred_reasoning_summary is not None: - yield _deferred_reasoning_summary elif not _suppress_visible_output: # Turn ended as a plain answer (no [ARGS] followed): the held # rehearsal tail is real prose, release it. @@ -12736,31 +12182,18 @@ class LlamaCppBackend: start_event["awaiting_confirmation"] = needs_confirm try: - # Gated calls are not running yet; a "Running ..." badge - # counting up while it waits on a human reads as a hang. - yield { - "type": "status", - "text": ( - awaiting_approval_status(decision.tool_name) - if needs_confirm - else decision.status_text - ), - } + yield {"type": "status", "text": decision.status_text} yield start_event - _decision = ( - wait_tool_decision( + if ( + decision_slot is not None + and wait_tool_decision( decision_slot, approval_id, cancel_event = cancel_event, ) - if decision_slot is not None - else None - ) - if _decision is not None and _decision != "deny": - # Approved: now it really is running. - yield {"type": "status", "text": decision.status_text} - if _decision == "deny": + == "deny" + ): decision_slot = None resolved_provisional_tool_call_ids.add(decision.tool_call_id) yield { @@ -12826,10 +12259,6 @@ class LlamaCppBackend: _kb_search_count += 1 completion = tool_controller.record_result(decision, result) resolved_provisional_tool_call_ids.add(decision.tool_call_id) - # A real execution opens the post-tool phase; carrying the pre-tool - # stall text over would read the same sentence as a repeat and - # swallow the one post-tool nudge. - _last_reprompt_text = "" # A tool ran this turn, so it counts against the caller's budget. _turn_executed_real_tool = True yield completion.tool_end_event() @@ -13332,15 +12761,10 @@ class LlamaCppBackend: min_p: float = 0.0, max_new_tokens: int = 2048, repetition_penalty: float = 1.1, - cancel_event: Optional[threading.Event] = None, ) -> tuple: """ Generate TTS audio via llama-server /completion + codec decode. Returns (wav_bytes, sample_rate). - - ``cancel_event`` lets a Stop or a forced model swap end the request: the - decode is one blocking POST, so a watcher closes the client out from under - it rather than polling. Raises RuntimeError once cancelled. """ if audio_type not in self._TTS_PROMPTS: raise RuntimeError(f"GGUF TTS does not support '{audio_type}' codec.") @@ -13362,47 +12786,15 @@ class LlamaCppBackend: if need_ids: payload["n_probs"] = 1 - if cancel_event is not None and cancel_event.is_set(): - raise RuntimeError("Audio generation cancelled") - with httpx.Client( timeout = httpx.Timeout(300, connect = 10), headers = self._auth_headers, trust_env = False, ) as client: - finished = threading.Event() - watcher: Optional[threading.Thread] = None - if cancel_event is not None: - - def _close_when_cancelled() -> None: - while not finished.wait(0.05): - if cancel_event.is_set(): - # Closing mid-request makes the blocking post raise - # httpx.RequestError, the only way out of it. - with contextlib.suppress(Exception): - client.close() - return - - watcher = threading.Thread(target = _close_when_cancelled, daemon = True) - watcher.start() - try: - resp = client.post(f"{self.base_url}/completion", json = payload) - except httpx.RequestError: - if cancel_event is not None and cancel_event.is_set(): - raise RuntimeError("Audio generation cancelled") from None - raise - finally: - finished.set() - if watcher is not None: - watcher.join(timeout = 0.5) + resp = client.post(f"{self.base_url}/completion", json = payload) if resp.status_code != 200: raise RuntimeError(f"llama-server returned {resp.status_code}: {resp.text}") - # The codec decode below is GPU work with no interruption point, so check here: - # cancelling after this only wastes the decode it cannot stop. - if cancel_event is not None and cancel_event.is_set(): - raise RuntimeError("Audio generation cancelled") - data = resp.json() token_ids = ( [p["id"] for p in data.get("completion_probabilities", []) if "id" in p] diff --git a/studio/backend/core/inference/llama_keepwarm.py b/studio/backend/core/inference/llama_keepwarm.py index 05b1271b27..3380ebf5f5 100644 --- a/studio/backend/core/inference/llama_keepwarm.py +++ b/studio/backend/core/inference/llama_keepwarm.py @@ -345,22 +345,6 @@ def _loaded_identity(backend): return (backend.model_identifier, getattr(backend, "hf_variant", None), advertised) -def _note_idle_unload_event(freed) -> None: - """Monitor row for an idle auto-unload. Best-effort; uses the stash's - advertised repo id so the row never shows the on-disk load path.""" - try: - from core.inference.api_monitor import api_monitor - from core.inference.model_ids import public_model_id - - identifier, variant, advertised = (list(freed) + [None, None, None])[:3] - label = public_model_id(advertised or identifier) or "model" - if variant and ":" not in label: - label = f"{label}:{variant}" - api_monitor.record_lifecycle(event = "unload", model = label, reason = "idle") - except Exception as exc: - logger.debug("idle unload monitor event failed: %s", exc) - - async def idle_unload_loop(poll_seconds: float = 15.0) -> None: """Unload the loaded GGUF once idle past the configured TTL. Inert when off.""" from utils.openai_auto_switch_settings import ( @@ -423,8 +407,6 @@ async def idle_unload_loop(poll_seconds: float = 15.0) -> None: elif manifest: _delete_resume_files(manifest) logger.info("Idle auto-unload: freed GGUF after %ss idle", ttl) - # An idle unload stashes for reload and skips note_model_unloaded. - _note_idle_unload_event(freed) seen_model = None except Exception as exc: logger.debug("idle_unload_loop iteration failed: %s", exc) diff --git a/studio/backend/core/inference/llama_server_args.py b/studio/backend/core/inference/llama_server_args.py index 7391e62516..7b42d2f40d 100644 --- a/studio/backend/core/inference/llama_server_args.py +++ b/studio/backend/core/inference/llama_server_args.py @@ -16,18 +16,11 @@ from __future__ import annotations import os from typing import Iterable, Mapping, Optional -# Valid llama-server --parallel range, shared with LoadRequest.n_parallel. -# Mirrored by callers that cannot import this: run.py and unsloth_cli/commands/ -# studio.py (_PARALLEL_MIN/MAX), per-model-config.ts (N_PARALLEL_MIN/MAX); -# test_parallel_slots_per_load.py pins them together. -PARALLEL_MIN = 1 -PARALLEL_MAX = 64 - # Each group = every alias (short + long) of one hard-denied flag. # Extend the matching group when llama.cpp adds a new alias. _DENYLIST_GROUPS: tuple[frozenset[str], ...] = ( - # Parallel slots: owned by typer --parallel and LoadRequest.n_parallel; a - # pass-through would desync the slot bookkeeping from llama-server. + # Parallel slots: owned by typer --parallel; a pass-through would desync + # app.state.llama_parallel_slots from llama-server. frozenset({"-np", "--parallel", "--n-parallel"}), # Model identity: Unsloth resolves it from LoadRequest; a second -m would # load a different model than Unsloth thinks it loaded. @@ -87,10 +80,9 @@ _DENYLIST: frozenset[str] = frozenset().union(*_DENYLIST_GROUPS) def _flag_name(token: str) -> Optional[str]: """Flag name for ``token``, or None if it isn't a flag. - Peels `--key=value` to `--key`, normalises long-option underscores like - llama.cpp, treats `-1`/`-0.5` as values (shorts always start with a letter), - and normalises attached `-np8` / `-np-1` / `-np8x` to `-np`. Mirrors the - CLI's `_expand_attached_np_short`. + Peels `--key=value` to `--key`, treats `-1`/`-0.5` as values (shorts + always start with a letter), and normalises attached `-np8` / `-np-1` / + `-np8x` to `-np`. Mirrors the CLI's `_expand_attached_np_short`. """ token = token.strip() if not token.startswith("-") or token in {"-", "--"}: @@ -98,8 +90,6 @@ def _flag_name(token: str) -> Optional[str]: if len(token) >= 2 and (token[1].isdigit() or token[1] == "."): return None name = token.split("=", 1)[0] - if name.startswith("--"): - name = name.replace("_", "-") if len(name) > 3 and name.startswith("-np"): suffix = name[3:] if suffix[0].isdigit() or ( @@ -128,7 +118,6 @@ def validate_extra_args(args: Optional[Iterable[str]]) -> list[str]: parse_ctx_override(out) parse_cache_override(out) parse_split_mode_override(out) - parse_gpu_layers_override(out) return out @@ -204,8 +193,9 @@ _SPLIT_SHADOWING_FLAGS: frozenset[str] = _SPLIT_MODE_FLAGS | _TENSOR_SPLIT_FLAGS # inherited -ngl is respected (the offload_overridden path), so this group is # opt-in, not default. Layer flags are shared with llama_cpp's override # detection; the MoE flags are strip-only (manual's --n-cpu-moe slider owns them). -_GPU_LAYER_FLAGS: frozenset[str] = frozenset({"-ngl", "--gpu-layers", "--n-gpu-layers"}) -_LAYER_OFFLOAD_FLAGS: frozenset[str] = _GPU_LAYER_FLAGS | frozenset({"-fit", "--fit"}) +_LAYER_OFFLOAD_FLAGS: frozenset[str] = frozenset( + {"-ngl", "--gpu-layers", "--n-gpu-layers", "-fit", "--fit"} +) _MOE_OFFLOAD_FLAGS: frozenset[str] = frozenset({"-ncmoe", "--n-cpu-moe", "-cmoe", "--cpu-moe"}) _OFFLOAD_SHADOWING_FLAGS: frozenset[str] = _LAYER_OFFLOAD_FLAGS | _MOE_OFFLOAD_FLAGS @@ -316,26 +306,6 @@ def parse_cache_override(args: Optional[Iterable[str]]) -> Optional[str]: return _last_flag_value(args, _CACHE_FLAGS) -def parse_gpu_layers_override(args: Optional[Iterable[str]]) -> Optional[int]: - """Return the last user-supplied GPU layer count from extras. - - Manual GPU memory mode strips llama.cpp offload flags because the - first-class load fields own them. Callers use this parser first to preserve - an explicit ``-ngl`` / ``--gpu-layers`` / ``--n-gpu-layers`` value when - translating the extras into those fields. - """ - raw_value = _last_flag_value(args, _GPU_LAYER_FLAGS) - if raw_value is None: - return None - try: - value = int(raw_value) - except ValueError as exc: - raise ValueError("llama-server GPU layers flag requires an integer value") from exc - if value < -1: - raise ValueError("llama-server GPU layers flag requires an integer value of at least -1") - return value - - def parse_cache_override_per_axis( args: Optional[Iterable[str]], ) -> tuple[Optional[str], Optional[str]]: diff --git a/studio/backend/core/inference/local_model_resolver.py b/studio/backend/core/inference/local_model_resolver.py index 5d2a9e9c87..e6014f442d 100644 --- a/studio/backend/core/inference/local_model_resolver.py +++ b/studio/backend/core/inference/local_model_resolver.py @@ -34,15 +34,6 @@ class _LocalGgufEntry: _CACHE_TTL_S = 5.0 _lock = threading.Lock() _scan: tuple[float, dict[str, _LocalGgufEntry]] = (0.0, {}) -# Not _lock: that is held for the whole scan, so the request path would wait on it. -_warm_lock = threading.Lock() -# Repos that finished downloading but are not in the published index yet: nothing -# else covers them until the next scan, and the request path must not call them absent. -_just_downloaded: set[str] = set() -_warming = False -_last_scan_s = 0.0 -# Rescan at most a tenth of the time: on the TTL alone a slow scan would run continuously. -_WARM_DUTY = 10.0 def _is_abs_path_id(value: str) -> bool: @@ -112,26 +103,17 @@ def _local_gguf_entry(loader_id: str, info) -> Optional[_LocalGgufEntry]: load_dir = _resolve_load_dir(p) variants, _ = list_local_gguf_variants(str(load_dir)) quants = tuple(v.quant for v in variants if getattr(v, "quant", None)) - if not quants: - return None - # That call orders by descending size, so the head is the biggest quant (often - # F16). Downstream reads [0], and a bare id must mean whichever quant a plain - # load would take: answering with the largest can evict a model and then OOM. - from core.inference.openai_auto_download import preferred_quant - - best = preferred_quant(quants) - if best and quants[0] != best: - quants = (best, *(q for q in quants if q != best)) - return _LocalGgufEntry(loader_id, str(load_dir), quants) + return _LocalGgufEntry(loader_id, str(load_dir), quants) if quants else None except Exception: return None -def local_gguf_quants(info) -> Optional[tuple[str, ...]]: - """On-disk quant labels for *info*, or None when it is not a servable local - GGUF. Read from the files, not ``info.model_format``: the HF-cache scanner - leaves that unset for GGUF snapshots, so filtering on it drops every cached - GGUF. One scan tells /v1/models what it can serve and which quant to name.""" +def info_has_local_gguf(info) -> bool: + """True when *info* (a LocalModelInfo) points to on-disk GGUF weights the + auto-switch path can load. Read from the files, not ``info.model_format``: the + HF-cache scanner leaves model_format unset for GGUF snapshots, so a + model_format filter would drop every cached GGUF. Lets /v1/models advertise + exactly what /v1 can serve.""" from pathlib import Path path = getattr(info, "path", None) @@ -141,14 +123,8 @@ def local_gguf_quants(info) -> Optional[tuple[str, ...]]: if isinstance(path, str) and any( seg in (".studio_links", "ollama_links") for seg in Path(path).parts ): - return None - entry = _local_gguf_entry(getattr(info, "id", "") or "", info) - return entry.variants if entry is not None else None - - -def info_has_local_gguf(info) -> bool: - """True when *info* points to on-disk GGUF weights the auto-switch path can load.""" - return local_gguf_quants(info) is not None + return False + return _local_gguf_entry(getattr(info, "id", "") or "", info) is not None def _build_index() -> dict[str, _LocalGgufEntry]: @@ -311,36 +287,6 @@ def _sibling_revision_entries(raw_id: str, loader_id: str): yield sibling.name, entry -def note_downloaded(repo_id: Optional[str]) -> None: - """Record a repo as present ahead of the scan that will index it.""" - if not repo_id: - return - with _lock: - _just_downloaded.add(repo_id.strip().lower()) - - -def recently_downloaded(repo_id: str) -> bool: - """Whether *repo_id* finished downloading since the last completed scan.""" - if not isinstance(repo_id, str) or not repo_id.strip(): - return False - return repo_id.strip().lower() in _just_downloaded - - -def invalidate_index() -> None: - """Mark the cached scan stale so the next resolve sees a just-finished download - instead of waiting out the TTL. - - Keeps the entries: the request path reads this cache without scanning, so - emptying it would leave it with no evidence about any local model until the - rebuild lands, and a bare request for one would be answered by whatever is - resident. Only a completed download invalidates, and that only adds, so the - retained entries stay true. - """ - global _scan - with _lock: - _scan = (0.0, _scan[1]) - - def _index() -> dict[str, _LocalGgufEntry]: global _scan # Build under the lock so concurrent callers with an expired cache don't all @@ -355,74 +301,23 @@ def _index() -> dict[str, _LocalGgufEntry]: # an install with many local models can itself exceed the TTL, which would # store the cache already expired and make every request rebuild the index. _scan = (time.monotonic(), fresh) - # The scan supersedes the notes: whatever landed is in the index now. - _just_downloaded.clear() return fresh -def index_is_built() -> bool: - """Whether a scan has ever completed, freshness aside. - - Lock-free on purpose: ``_lock`` is held for the whole scan, so taking it would - park the request path on the scan it is trying to stay off. Safe because - ``_scan`` is only ever rebound, never mutated. - """ - return bool(_scan[0]) - - -def warm_index_soon() -> None: - """(Re)build the index off the request path when it is missing or past its TTL. - - The only refresh for callers using ``allow_scan=False``. Covers a stale index, - not just an absent one: a model downloaded through the Hub UI or dropped into a - scan folder has no invalidation hook and would otherwise stay invisible to them - for the life of the process. Never blocks, and never touches ``_lock``. - """ - global _warming - if time.monotonic() - _scan[0] < max(_CACHE_TTL_S, _last_scan_s * _WARM_DUTY): - return - with _warm_lock: - if _warming: - return - _warming = True - - def _run() -> None: - global _warming, _last_scan_s - started = time.monotonic() - try: - _index() - except Exception: - pass - finally: - _last_scan_s = time.monotonic() - started - with _warm_lock: - _warming = False - - threading.Thread(target = _run, name = "local-model-index-warm", daemon = True).start() - - -def resolve_local_gguf( - requested: str, *, allow_scan: bool = True -) -> Optional[tuple[str, Optional[str], str]]: +def resolve_local_gguf(requested: str) -> Optional[tuple[str, Optional[str], str]]: """Return ``(load_path, gguf_variant, loader_id)`` for a local match, else None. ``load_path`` is the concrete on-disk path to hand /load (so it never fetches a remote), ``loader_id`` is the advertised id used as the launch-override key. ``requested`` is ``repo`` or ``repo:VARIANT``. An exact id match wins first (so ids containing a colon still resolve); else the last ``:VARIANT`` is split - off and resolves only when that quant is on disk, unless it names no quant at - all (an Ollama-style ":latest"), which means the repo. - - ``allow_scan=False`` answers from the last built index and never rebuilds, for - the request path: the scan walks several model dirs and HF caches, takes seconds - on a large install, and holds a lock everyone queues behind. Stale is fine there, - since disk barely moves and a finished download calls :func:`invalidate_index`. + off and resolves only when that quant is on disk. """ if not isinstance(requested, str) or not requested.strip(): return None requested = requested.strip() try: - index = _index() if allow_scan else _scan[1] + index = _index() entry = index.get(requested.lower()) if entry is not None: variant = entry.variants[0] if entry.variants else None @@ -438,44 +333,8 @@ def resolve_local_gguf( for v in entry.variants: if v.lower() == wanted: return entry.load_path, v, entry.loader_id - from core.inference.openai_auto_download import looks_like_quant - - if looks_like_quant(variant): - return None - # ":latest" or ":8b" names no file, so it means the repo; a real quant that - # is not on disk still misses, or a swap would serve the wrong weights. - return entry.load_path, (entry.variants[0] if entry.variants else None), entry.loader_id + return None except Exception: # Best-effort: any resolver failure falls through to the loaded model, # so a malformed name can never turn a servable request into a 500. return None - - -MISS_MODEL_NOT_FOUND = "model_not_found" -MISS_VARIANT_NOT_FOUND = "variant_not_found" - - -def describe_local_miss(requested: str) -> tuple[str, tuple[str, ...]]: - """Why :func:`resolve_local_gguf` missed, so an error can say "wrong quant" - instead of "no such model". - - ``(MISS_VARIANT_NOT_FOUND, )`` when the repo is downloaded but the - requested ``:VARIANT`` is not, else ``(MISS_MODEL_NOT_FOUND, ())``. Fail-safe: a - scan failure reports the generic miss rather than raising into the handler. - """ - if not isinstance(requested, str) or not requested.strip(): - return MISS_MODEL_NOT_FOUND, () - base, sep, variant = requested.strip().rpartition(":") - from core.inference.openai_auto_download import looks_like_quant - - # Split like the resolver or the two disagree: a tag naming no quant means the - # repo there, so reporting a missing quant for it would name one nobody asked for. - if not sep or not looks_like_quant(variant): - return MISS_MODEL_NOT_FOUND, () - try: - entry = _index().get(base.strip().lower()) - except Exception: - return MISS_MODEL_NOT_FOUND, () - if entry is None or not entry.variants: - return MISS_MODEL_NOT_FOUND, () - return MISS_VARIANT_NOT_FOUND, entry.variants diff --git a/studio/backend/core/inference/mcp_client.py b/studio/backend/core/inference/mcp_client.py index 98112c6d5b..0256df944e 100644 --- a/studio/backend/core/inference/mcp_client.py +++ b/studio/backend/core/inference/mcp_client.py @@ -971,12 +971,7 @@ def _call_stdio_tool( raise RuntimeError("MCP server connection is not available") else: rem = _remaining() - # raise_on_error=False for the same reason as the one-shot path. - coro = _race_tool_call( - session.client.call_tool(name, args, raise_on_error = False), - rem, - cancel_event, - ) + coro = _race_tool_call(session.client.call_tool(name, args), rem, cancel_event) return session.run(coro, rem) except (_MCPCancelled, asyncio.TimeoutError): # _race_tool_call cancels the pending call but cancellation is diff --git a/studio/backend/core/inference/mlx_inference.py b/studio/backend/core/inference/mlx_inference.py index 2b300a32b1..d19c67a01a 100644 --- a/studio/backend/core/inference/mlx_inference.py +++ b/studio/backend/core/inference/mlx_inference.py @@ -1189,8 +1189,7 @@ class MLXInferenceBackend: **gen_kwargs, ) - def reset_generation_state(self, caller_cancel_event = None): - # caller_cancel_event: signature parity with the orchestrator; unused here. + def reset_generation_state(self): import mlx.core as mx import gc diff --git a/studio/backend/core/inference/model_ids.py b/studio/backend/core/inference/model_ids.py index 3886307ae2..548cc60f94 100644 --- a/studio/backend/core/inference/model_ids.py +++ b/studio/backend/core/inference/model_ids.py @@ -39,29 +39,10 @@ def _looks_like_path(identifier: str) -> bool: return False -def hf_cache_repo_id(path: Optional[str]) -> Optional[str]: - """``.../models--org--name/snapshots/`` -> ``org/name``, else None. - - A model loaded from the HF cache is identified by its snapshot dir, whose - basename is a commit hash; recover the repo id so callers don't show that. - """ - if not path: - return None - parts = str(path).replace("\\", "/").split("/") - for index, part in enumerate(parts): - # Only inside the real cache layout: a "models--" name alone is not a repo id. - if part.startswith("models--") and parts[index + 1 : index + 2] == ["snapshots"]: - return part[len("models--") :].replace("--", "/") - return None - - def public_model_id(identifier: Optional[str]) -> Optional[str]: """Return a clean, path-free public id for *identifier*. - - HF cache path -> the repo id it came from, e.g. - ``~/.cache/huggingface/hub/models--unsloth--X-GGUF/snapshots/`` -> - ``unsloth/X-GGUF``. - - Other local GGUF path -> the file stem with ``.gguf`` stripped, e.g. + - Local GGUF path -> the file stem with ``.gguf`` stripped, e.g. ``/srv/models/Qwen3-30B-A3B-Q4_K_M.gguf`` -> ``Qwen3-30B-A3B-Q4_K_M``. - HF repo id (``org/model``) and already-clean names -> returned unchanged. - ``None`` / empty -> returned unchanged. @@ -70,9 +51,6 @@ def public_model_id(identifier: Optional[str]) -> Optional[str]: return identifier if not _looks_like_path(identifier): return identifier - repo_id = hf_cache_repo_id(identifier) - if repo_id: - return repo_id name = os.path.basename(identifier.replace("\\", "/").rstrip("/")) if name.lower().endswith(_GGUF_SUFFIX): name = name[: -len(_GGUF_SUFFIX)] diff --git a/studio/backend/core/inference/openai_auto_download.py b/studio/backend/core/inference/openai_auto_download.py deleted file mode 100644 index cad5e40d14..0000000000 --- a/studio/backend/core/inference/openai_auto_download.py +++ /dev/null @@ -1,812 +0,0 @@ -# SPDX-License-Identifier: AGPL-3.0-only -# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 - -"""Opt-in: fetch a GGUF a /v1 request names but this server doesn't have. - -Auto-switch only loads models already on disk. With -``openai_api_auto_download_model`` on, a miss that looks like a real Hub repo is -fetched in the background and the request is told to retry rather than held -open: a quant is routinely tens of GB, far longer than any client (or the -Cloudflare edge on ``--secure``) will wait, and the inference lifecycle gate must -not be held meanwhile. The resident model keeps serving, and the retry that lands -after the download goes through the ordinary auto-switch path. - -Admission is deliberately narrow, since a request only needs an API key: -- ``namespace/name`` only, and only when the Hub confirms GGUF weights. A - namespace is not evidence of intent (LiteLLM and OpenRouter address every - provider that way), so ``gpt-4`` and ``anthropic/claude-3.5-sonnet`` alike - fall through to the resident model as before. -- GGUF only, decided from the remote file list, not the repo name: GGUF runs - under llama.cpp, which never imports repo Python. -- ``auto_map`` is refused, so ``trust_remote_code`` is only ever granted - deliberately in the UI, never by an API call. -- One download at a time, so a key holder cannot fan out fetches. -""" - -from __future__ import annotations - -import asyncio -import shutil -import threading -import time -from dataclasses import dataclass -from typing import Optional - -from loggers import get_logger - -logger = get_logger(__name__) - -# Keep the Hub probe short so a slow Hub can't stall the request path. -_MODEL_INFO_TIMEOUT_S = 8.0 -# auth_check and hf_hub_download take no timeout of their own and run while the -# provisional slot is held, so an unresponsive Hub would pin the single flight. The -# code probe fetches up to three configs, so it gets more room than the auth call. -_CODE_PROBE_TIMEOUT_S = 20.0 -# Headroom left free after the download, so filling the disk can't wedge the box. -_DISK_RESERVE_BYTES = 5 * 1024**3 -_WATCH_POLL_S = 2.0 -# A stalled watcher must not pin the single-flight slot forever. -_MAX_WATCH_S = 24 * 60 * 60 -# Past the watch window the row is resolved, so poll only to see whether the -# worker is still alive and still owns the slot. -_TIMED_OUT_POLL_S = 60.0 -_RETRY_AFTER_S = 30 -# Long enough for a client honouring Retry-After to come back and be told, short -# enough that one that never returns cannot hold the slot. -_FAILED_HOLD_S = 3 * _RETRY_AFTER_S -_MAX_LISTED_VARIANTS = 8 - - -@dataclass(frozen = True) -class AutoDownloadRefusal: - """Why this request cannot be served yet; the route raises it in the - surface's own error envelope.""" - - status: int - code: str - message: str - retry_after: Optional[int] = None - - -@dataclass -class _Active: - repo_id: str - # None while the Hub probe is still deciding which quant to fetch. - variant: Optional[str] = None - expected_bytes: int = 0 - monitor_id: Optional[str] = None - started_at: float = 0.0 - # Set when the worker failed. Held until a retry surfaces it: Retry-After is far - # longer than the watcher poll, so the client would restart the same failing download. - error: Optional[str] = None - failed_at: float = 0.0 - - -_lock = threading.Lock() -_active: Optional[_Active] = None - -# Repos the Hub says are not servable, so a "vendor/model" miss doesn't re-probe every request. -_NOT_SERVABLE_TTL_S = 10 * 60 -_NOT_SERVABLE_MAX = 256 -_cache_lock = threading.Lock() -_not_servable: dict[str, float] = {} - - -def _public_label(repo_id: str, variant: Optional[str]) -> str: - return f"{repo_id}:{variant}" if variant else repo_id - - -def split_model_ref(requested: str) -> tuple[str, Optional[str]]: - """``org/repo:QUANT`` -> ``("org/repo", "QUANT")``; no suffix -> variant None. - - Splits on the last colon. A slash-bearing suffix is only a variant when a real - Hub repo precedes it: "build/llama-13b" is a subdirectory GGUF key the catalog - advertises, while "C:/models/x.gguf" leaves a drive letter that is no repo id. - """ - text = (requested or "").strip() - base, sep, suffix = text.rpartition(":") - if not sep or not base or not suffix: - return text, None - stripped = base.strip() - if "/" in suffix: - from hub.utils.paths import is_valid_repo_id - if "/" not in stripped or not is_valid_repo_id(stripped): - return text, None - return stripped, suffix.strip() - - -def is_downloadable_ref(requested: str) -> bool: - """Whether *requested* is shaped like a Hub repo we may fetch. - - Requires an explicit namespace: keeps ``gpt-4`` and other foreign ids falling - through, and stops ModelConfig.from_identifier's bare-name ``unsloth/`` - prefixing from turning an unrelated label into a real repo. - """ - from hub.utils.paths import is_valid_repo_id - - repo_id, variant = split_model_ref(requested) - if "/" not in repo_id or not is_valid_repo_id(repo_id): - return False - if variant is not None: - from hub.utils.paths import is_valid_gguf_variant - return is_valid_gguf_variant(variant) - return True - - -def looks_like_quant(variant: Optional[str]) -> bool: - """Whether a ``:suffix`` names a GGUF quant rather than a foreign tag. - - Neither a namespace nor a colon proves a request was meant for this server - (``vendor/model`` is LiteLLM/OpenRouter, ``name:latest`` is Ollama). A real - quant label does. - """ - import re - - from utils.models.model_config import _GGUF_KNOWN_QUANT_RE - - if not variant: - return False - # _extract_quant_label can append a bpw modifier (IQ4_XS-3.53bpw); still a quant. - label = re.sub(r"-[0-9]+(?:\.[0-9]+)?bpw$", "", variant.strip(), flags = re.IGNORECASE) - return _GGUF_KNOWN_QUANT_RE.fullmatch(label) is not None - - -def _hub_token(hf_token: Optional[str]): - """The caller's token, or an explicit False. None makes huggingface_hub fall - back to a cached login (here the server owner's); only False is anonymous.""" - return hf_token or False - - -def _servable_key(repo_id: str, hf_token: Optional[str]) -> str: - """Cache key, per credential. - - The Hub 404s a private repo the caller cannot see, so a tokenless verdict says - nothing about a caller who has one. Digested, so no token is held here. - """ - import hashlib - - seen_as = hashlib.sha256(hf_token.encode()).hexdigest()[:16] if hf_token else "anon" - return f"{repo_id.lower()}\n{seen_as}" - - -def _mark_not_servable(repo_id: str, hf_token: Optional[str]) -> None: - with _cache_lock: - if len(_not_servable) >= _NOT_SERVABLE_MAX: - _not_servable.clear() - _not_servable[_servable_key(repo_id, hf_token)] = time.monotonic() + _NOT_SERVABLE_TTL_S - - -def _is_not_servable(repo_id: str, hf_token: Optional[str]) -> bool: - key = _servable_key(repo_id, hf_token) - with _cache_lock: - expires = _not_servable.get(key) - if expires is None: - return False - if expires <= time.monotonic(): - del _not_servable[key] - return False - return True - - -def _gated_refusal(repo_id: str) -> AutoDownloadRefusal: - return AutoDownloadRefusal( - status = 403, - code = "model_access_denied", - message = ( - f"'{repo_id}' is gated on Hugging Face. Accept its licence, then retry with " - "your own token in the X-Unsloth-HF-Token header: automatic download never " - "uses this server's Hugging Face identity." - ), - ) - - -async def _bounded_probe(fn, *args, timeout: float, default): - """Run a blocking Hub probe off the loop, bounding only the wait. - - The thread is left to finish (a blocking socket read cannot be cancelled); the - caller takes *default*, chosen per call site so a timeout errs the safe way. - """ - try: - return await asyncio.wait_for(asyncio.to_thread(fn, *args), timeout) - except (TimeoutError, asyncio.TimeoutError): - logger.debug("hub probe %s timed out after %ss", getattr(fn, "__name__", fn), timeout) - return default - - -def _auth_denied(repo_id: str, hf_token: Optional[str]) -> bool: - """Whether this token lacks file access to a gated repo. False when the - check is inconclusive: the download's own auth is the real gate.""" - from hub.utils.hf_errors import hf_error_status - - try: - from huggingface_hub import auth_check - auth_check(repo_id, token = _hub_token(hf_token)) - except Exception as exc: - return hf_error_status(exc) in (401, 403) - return False - - -def _gguf_variants(siblings) -> dict[str, int]: - """Quant label -> bytes the download will actually fetch. - - Mirrors list_gguf_variants for the selectable labels: companions (mmproj/MTP) - and big-endian builds are not quants, and sharded quants sum across shards. - Bytes come from the download plan, which folds companions back into every - quant, so the disk reserve is measured against what the worker fetches. - """ - from hub.utils.gguf import extract_quant_label as canonical_quant_label - from hub.utils.gguf_plan import build_gguf_variant_plans - from utils.models.model_config import ( - _extract_quant_label, - _is_big_endian_gguf_path, - _is_mmproj, - _is_mtp_drafter, - ) - - siblings = list(siblings or []) - plans = build_gguf_variant_plans(siblings) - sizes: dict[str, int] = {} - for sibling in siblings: - name = getattr(sibling, "rfilename", "") or "" - if not name.lower().endswith(".gguf"): - continue - quant = _extract_quant_label(name) - if not looks_like_quant(quant): - # With no recognized quant token the extractors part ways: this one takes - # the last hyphenated segment ("7b" of llama-7b) while the plan and worker - # key the whole stem, so advertising ours dispatches an unresolvable variant. - quant = canonical_quant_label(name) or quant - if _is_mmproj(name) or _is_mtp_drafter(name) or _is_big_endian_gguf_path(name, quant): - continue - plan = plans.get(quant.lower()) - if plan is not None: - sizes[quant] = plan.download_size_bytes - else: - sizes[quant] = sizes.get(quant, 0) + int(getattr(sibling, "size", 0) or 0) - return sizes - - -def _remaining_bytes(repo_id: str, plan, expected_bytes: int) -> int: - """Bytes still to fetch: a resumed quant or a companion shared with another - quant is already on disk, and charging for it can 507 a download that fits.""" - try: - from hub.utils.download_registry import existing_blob_bytes - - hashes = frozenset( - file.sha256 for file in getattr(plan, "expected_files", ()) or () if file.sha256 - ) - if not hashes: - return expected_bytes - return max(0, expected_bytes - existing_blob_bytes("model", repo_id, hashes)) - except Exception: - return expected_bytes - - -def _enough_disk(need_bytes: int) -> tuple[bool, int]: - """(fits, free_bytes). Fail-open on an unreadable cache root: the download - worker runs its own preflight, this only adds the reserve margin.""" - try: - from hub.utils.hf_cache_state import hf_cache_root - - root = hf_cache_root(create = True) - if root is None: - return True, 0 - free = shutil.disk_usage(root).free - except Exception: - return True, 0 - return free >= need_bytes + _DISK_RESERVE_BYTES, free - - -def _gb(num_bytes: int) -> str: - return f"{num_bytes / 1024**3:.1f} GB" - - -async def _job_state(repo_id: str, variant: Optional[str]) -> tuple[str, Optional[str]]: - from hub.services.models import downloads - try: - status = await downloads.get_download_status_response(repo_id, variant or "") - return status.state, status.error - except Exception as exc: - # "unknown", not "idle": idle ends the watch, and a failed probe proves nothing. - logger.debug("auto-download: status probe failed for %r: %s", repo_id, exc) - return "unknown", None - - -async def _progress_percent( - repo_id: str, variant: Optional[str], expected_bytes: int, hf_token: Optional[str] -) -> Optional[float]: - """0-100, or None. The hub service reports a 0-1 fraction, so scale it.""" - from hub.services.models import downloads - try: - payload = await downloads.get_gguf_download_progress_response( - repo_id, variant or "", expected_bytes, hf_token - ) - fraction = payload.get("progress") - if not isinstance(fraction, (int, float)): - return None - return min(100.0, max(0.0, float(fraction) * 100.0)) - except Exception: - return None - - -def _release(active: Optional[_Active]) -> None: - """Free the single-flight slot, but only while *active* still owns it. - - Keying on ``repo_id`` alone let a stale operation clear a newer one: variant A - errors, an adopting request frees the slot, a retry starts B, then A's watcher - matches the repo and clears B, admitting a second download alongside it. - """ - global _active - if active is None: - return - with _lock: - if _active is active: - _active = None - - -async def _watch(active: _Active, hf_token: Optional[str]) -> None: - """Poll a dispatched job so the monitor row resolves and the resolver cache - is dropped the moment the weights land.""" - from core.inference import api_monitor as monitor_module - - api_monitor = monitor_module.api_monitor - deadline = time.monotonic() + _MAX_WATCH_S - timed_out = False - try: - while True: - await asyncio.sleep(_TIMED_OUT_POLL_S if timed_out else _WATCH_POLL_S) - state, error = await _job_state(active.repo_id, active.variant) - if state in ("running", "cancelling", "unknown"): - if timed_out: - # A running worker still owns the slot: releasing on the clock alone - # would admit a second multi-GB download beside it. "unknown" cannot - # confirm it is alive, so release then, or a broken probe wedges us. - if state == "unknown": - return - continue - if time.monotonic() >= deadline: - api_monitor.fail_open(active.monitor_id, "Download timed out") - timed_out = True - continue - # Only "running" has progress; the others are still in flight, so keep the slot. - if state == "running": - api_monitor.set_progress( - active.monitor_id, - await _progress_percent( - active.repo_id, active.variant, active.expected_bytes, hf_token - ), - ) - continue - if state == "cancelled": - api_monitor.finish(active.monitor_id, status = "cancelled") - return - if state == "complete": - # No invalidate here: finalize_worker_exit already dropped the cache and - # warmed it; a second would mark that fresh scan stale and push a - # synchronous rescan onto the client's retry. - api_monitor.finish(active.monitor_id, status = "completed") - elif state == "idle": - # The job vanished without a terminal state (worker killed). - api_monitor.fail_open(active.monitor_id, "Download did not complete") - else: - api_monitor.fail_open(active.monitor_id, error or f"Download {state}") - # Keep the slot so the next retry is told it failed instead of - # silently restarting the same download. - active.error = error or f"Download {state}" - active.failed_at = time.monotonic() - return - return - except asyncio.CancelledError: - raise - except Exception as exc: - logger.warning("auto-download: watcher failed for %r: %s", active.repo_id, exc) - api_monitor.fail_open(active.monitor_id, "Download tracking failed") - finally: - if not active.failed_at: - _release(active) - - -def _downloading_refusal(label: str, percent: Optional[float]) -> AutoDownloadRefusal: - progress = f" ({percent:.0f}% done)" if percent is not None else "" - return AutoDownloadRefusal( - status = 503, - code = "model_downloading", - message = (f"Downloading '{label}'{progress}. Retry shortly. Track it in Unsloth Studio."), - retry_after = _RETRY_AFTER_S, - ) - - -async def _is_downloadable_model(repo_id: str, hf_token: Optional[str]) -> bool: - """Whether the Hub has this repo with GGUF weights we could fetch. - - Only asked while another download holds the slot, to tell a second download - apart from an ordinary foreign label. Any failure answers False: refusing - would strand normal traffic for the length of the download. - """ - if _is_not_servable(repo_id, hf_token): - return False - - def _probe(): - from huggingface_hub import HfApi - return HfApi(token = _hub_token(hf_token)).model_info(repo_id, timeout = _MODEL_INFO_TIMEOUT_S) - - try: - info = await asyncio.to_thread(_probe) - except Exception: - return False - # The same filter admission uses, not a bare .gguf test: mmproj, MTP drafters and - # big-endian builds are companions, not quants. Answering otherwise would hold an - # ordinary foreign label at model_download_busy for an unrelated download. - servable = bool(_gguf_variants(getattr(info, "siblings", None))) - if not servable: - _mark_not_servable(repo_id, hf_token) - return servable - - -async def maybe_auto_download( - requested_model: str, - *, - hf_token: Optional[str] = None, - require_vision: bool = False, -) -> Optional[AutoDownloadRefusal]: - """Start (or report on) a background fetch of *requested_model*. - - Returns None when the request should carry on unchanged, or a refusal the - caller must raise. Only called after the local resolver has already missed. - - ``require_vision`` refuses a target with no mmproj companion rather than spend - gigabytes on weights that cannot answer the request; the local capability guard - only ever sees an already-downloaded model. - """ - global _active - - repo_id, wanted_variant = split_model_ref(requested_model) - if not is_downloadable_ref(requested_model): - return None - if _is_not_servable(repo_id, hf_token) and not looks_like_quant(wanted_variant): - return None - - # Settle the single-flight slot before the network, so retries during a download stay cheap. - busy: Optional[_Active] = None - with _lock: - current = _active - if current is not None and current.failed_at: - # A held failure only owns the slot until someone is told about it. - if current.repo_id != repo_id and time.monotonic() - current.failed_at > _FAILED_HOLD_S: - _active = current = None - if current is not None and current.repo_id == repo_id: - adopted = current - elif current is not None: - adopted = None - busy = current - else: - adopted = None - provisional = _Active(repo_id = repo_id, started_at = time.time()) - _active = provisional - - if busy is not None: - # Refusing before the probe blocks ordinary drop-in traffic: a namespaced label - # that is no downloadable GGUF repo (LiteLLM/OpenRouter style) would be told to - # wait out a multi-hour download. Only a downloadable label is a 2nd download. - if not await _is_downloadable_model(repo_id, hf_token): - return None - return AutoDownloadRefusal( - status = 503, - code = "model_download_busy", - message = ( - f"Already downloading '{_public_label(busy.repo_id, busy.variant)}'. " - f"Retry '{requested_model}' once it finishes." - ), - retry_after = _RETRY_AFTER_S, - ) - - if adopted is not None: - if adopted.variant is None: - # Still probing: no job yet, and a stale whole-repo error would free the probe's slot. - return _downloading_refusal(adopted.repo_id, None) - state, error = await _job_state(adopted.repo_id, adopted.variant) - if state in ("running", "cancelling", "unknown"): - return _downloading_refusal( - _public_label(adopted.repo_id, adopted.variant), - await _progress_percent( - adopted.repo_id, adopted.variant, adopted.expected_bytes, hf_token - ), - ) - if state == "error" or adopted.error: - error = error or adopted.error - # Surface once, then free the slot so a retry can start over. - _release(adopted) - return AutoDownloadRefusal( - status = 502, - code = "model_download_failed", - message = f"Downloading '{requested_model}' failed: {error or 'unknown error'}", - ) - # complete/idle/cancelled: the watcher is about to free the slot, so retry once more. - return _downloading_refusal( - _public_label(adopted.repo_id, adopted.variant), - 100.0 if state == "complete" else None, - ) - - try: - return await _admit_and_start( - repo_id, wanted_variant, requested_model, hf_token, provisional, require_vision - ) - except BaseException: - # Not `except Exception`: a cancel mid-probe would otherwise wedge the provisional slot. - _release(provisional) - raise - - -async def _admit_and_start( - repo_id: str, - wanted_variant: Optional[str], - requested_model: str, - hf_token: Optional[str], - active: _Active, - require_vision: bool = False, -) -> Optional[AutoDownloadRefusal]: - from hub.utils.hf_errors import hf_error_status - - def _probe(): - from huggingface_hub import HfApi - return HfApi(token = _hub_token(hf_token)).model_info( - repo_id, files_metadata = True, timeout = _MODEL_INFO_TIMEOUT_S - ) - - try: - info = await asyncio.to_thread(_probe) - except Exception as exc: - _release(active) - status = hf_error_status(exc) - if status == 401: - return AutoDownloadRefusal( - status = 401, - code = "model_access_denied", - message = ( - f"Hugging Face rejected the token sent for '{repo_id}'. Replace the " - "X-Unsloth-HF-Token header with a valid token; retrying will not help." - ), - ) - if status == 403: - return _gated_refusal(repo_id) - if status == 404: - _mark_not_servable(repo_id, hf_token) - # Unknown to the Hub reads as a foreign label; only an explicit quant makes it ours. - if not looks_like_quant(wanted_variant): - return None - # A private repo reads as absent without a token; don't confirm either way. - return AutoDownloadRefusal( - status = 404, - code = "model_not_found", - message = ( - f"'{repo_id}' was not found on Hugging Face, or is not accessible. " - "If it is private, send a token in the X-Unsloth-HF-Token header." - ), - ) - logger.warning("auto-download: Hub lookup failed for %r: %s", repo_id, exc) - return AutoDownloadRefusal( - status = 503, - code = "model_lookup_failed", - message = f"Could not reach Hugging Face to look up '{repo_id}'. Retry shortly.", - retry_after = _RETRY_AFTER_S, - ) - - # Inconclusive on timeout: the download's own auth is the real gate. - if getattr(info, "gated", False) and await _bounded_probe( - _auth_denied, repo_id, hf_token, timeout = _MODEL_INFO_TIMEOUT_S, default = False - ): - # Metadata for a gated repo is not file access; unchecked, the config read below lies. - _release(active) - return _gated_refusal(repo_id) - - variants = _gguf_variants(getattr(info, "siblings", None)) - if not variants: - _release(active) - _mark_not_servable(repo_id, hf_token) - if not looks_like_quant(wanted_variant): - return None - return AutoDownloadRefusal( - status = 400, - code = "model_not_supported", - message = ( - f"'{repo_id}' has no GGUF weights. Automatic download serves GGUF only; " - "load other formats from Unsloth Studio." - ), - ) - - # trust_remote_code gate: _config_has_auto_map is tri-state, so refuse on True and on None. - from utils.security.consent import _config_has_auto_map - - # _hub_token, not the raw token: None lets huggingface_hub fall back to a cached - # server login, so a caller-named repo would be probed with this server's identity. - # Defaults to None on timeout, which refuses: unchecked is not cleared. - has_auto_map = await _bounded_probe( - _config_has_auto_map, - repo_id, - _hub_token(hf_token), - timeout = _CODE_PROBE_TIMEOUT_S, - default = None, - ) - if has_auto_map is not False: - _release(active) - unknown = has_auto_map is None - return AutoDownloadRefusal( - status = 403, - code = "remote_code_consent_required", - message = ( - f"'{repo_id}' " - + ( - "could not be checked for custom code" - if unknown - else "ships custom code that runs on load" - ) - + ". Load it once in Unsloth Studio to review and approve it, then retry." - ), - ) - - variant = _match_variant(wanted_variant, variants) - if variant is None: - _release(active) - listed = sorted(variants) - shown = ", ".join(listed[:_MAX_LISTED_VARIANTS]) - extra = len(listed) - _MAX_LISTED_VARIANTS - return AutoDownloadRefusal( - status = 404, - code = "model_not_found", - message = ( - f"'{repo_id}' has no quant '{wanted_variant}'. Available quants: " - f"{shown}{f' and {extra} more' if extra > 0 else ''}." - ), - ) - - expected_bytes = variants[variant] - from hub.utils.gguf_plan import build_gguf_variant_plans - - plan = build_gguf_variant_plans(list(getattr(info, "siblings", None) or [])).get( - variant.lower() - ) - if require_vision and not (plan and plan.mmproj_filenames): - _release(active) - return AutoDownloadRefusal( - status = 400, - code = "invalid_value", - message = ( - f"'{_public_label(repo_id, variant)}' ships no mmproj companion, so it " - "cannot answer the image or audio input in this request. It was not " - "downloaded." - ), - ) - - need_bytes = _remaining_bytes(repo_id, plan, expected_bytes) - fits, free = _enough_disk(need_bytes) - if not fits: - _release(active) - return AutoDownloadRefusal( - status = 507, - code = "insufficient_disk_space", - message = ( - f"'{_public_label(repo_id, variant)}' needs {_gb(need_bytes)} plus " - f"{_gb(_DISK_RESERVE_BYTES)} headroom, but only {_gb(free)} is free." - ), - ) - - return await _dispatch(repo_id, variant, expected_bytes, requested_model, hf_token, active) - - -def preferred_quant(labels) -> Optional[str]: - """The quant a plain load would pick from *labels*, or None. - - The one ranking for "which quant did they mean": local resolution, remote - admission and /v1/models must agree, or a bare id means a different quant - depending on which of them answered it. - """ - from utils.models.model_config import _pick_best_gguf - - # _pick_best_gguf ranks filenames and matches upper-case tokens, so feed "