diff --git a/.github/scripts/run-studio-permission-browser.sh b/.github/scripts/run-studio-permission-browser.sh index 2007789035..e5a9a4c135 100755 --- a/.github/scripts/run-studio-permission-browser.sh +++ b/.github/scripts/run-studio-permission-browser.sh @@ -17,7 +17,8 @@ if [ -n "${STUDIO_PERMISSION_FRONTEND:-}" ]; then fi mkdir -p "$artifact_dir" -unsloth studio reset-password +# Wipe (not reset-password): the boot below must re-seed a fresh .bootstrap_password. +rm -rf "$studio_home/auth" UNSLOTH_API_ONLY=1 unsloth studio -H 127.0.0.1 -p "$port" "$@" \ >"$server_log" 2>&1 & studio_pid=$! diff --git a/.github/workflows/local-agent-guides-ci.yml b/.github/workflows/local-agent-guides-ci.yml index c48328e90f..0dc0cc66d7 100644 --- a/.github/workflows/local-agent-guides-ci.yml +++ b/.github/workflows/local-agent-guides-ci.yml @@ -167,7 +167,9 @@ jobs: # ── boot the server under test (factored helper) ────────────────── - name: Serve unsloth run --disable-tools (gemma-4-E4B) run: | - unsloth studio reset-password + # Wipe, not reset-password: since #7573 the reset rotates in place and + # prints the new passphrase, which would land unmasked in the job log. + rm -rf ~/.unsloth/studio/auth bash .github/scripts/serve-unsloth-run.sh \ --gguf-file "$GITHUB_WORKSPACE/gguf-cache/${GGUF_FILE}" \ --port "$STUDIO_PORT" --log-dir logs \ @@ -371,7 +373,7 @@ jobs: - name: Serve unsloth run --disable-tools (gemma-4-E4B) run: | - unsloth studio reset-password + rm -rf ~/.unsloth/studio/auth bash .github/scripts/serve-unsloth-run.sh \ --gguf-file "$GITHUB_WORKSPACE/gguf-cache/${GGUF_FILE}" \ --port "$STUDIO_PORT" --log-dir logs \ @@ -554,7 +556,7 @@ jobs: - name: Serve unsloth run --disable-tools (gemma-4-E4B) run: | - unsloth studio reset-password + rm -rf ~/.unsloth/studio/auth bash .github/scripts/serve-unsloth-run.sh \ --gguf-file "$GITHUB_WORKSPACE/gguf-cache/${GGUF_FILE}" \ --port "$STUDIO_PORT" --log-dir logs \ @@ -718,7 +720,7 @@ jobs: - name: Serve unsloth run --disable-tools (gemma-3-270m) run: | - unsloth studio reset-password + rm -rf ~/.unsloth/studio/auth bash .github/scripts/serve-unsloth-run.sh \ --model "$GGUF_REPO" --gguf-variant "$GGUF_VARIANT" \ --port "$STUDIO_PORT" --log-dir logs \ diff --git a/.github/workflows/release-desktop.yml b/.github/workflows/release-desktop.yml index 081eda4e32..0a8d71610d 100644 --- a/.github/workflows/release-desktop.yml +++ b/.github/workflows/release-desktop.yml @@ -766,6 +766,7 @@ jobs: env: GH_REPO: ${{ github.repository }} APP_VERSION: ${{ needs.prepare-version.outputs.app_version }} + PYPI_VERSION: ${{ needs.prepare-version.outputs.pypi_version }} STUDIO_VERSION: ${{ needs.prepare-version.outputs.studio_version }} DESKTOP_RELEASE_TAG: ${{ needs.prepare-version.outputs.desktop_release_tag }} DESKTOP_PRERELEASE: ${{ needs.prepare-version.outputs.prerelease }} @@ -911,6 +912,8 @@ jobs: notes = pathlib.Path(os.environ['RUNNER_TEMP'], 'desktop-release-notes.md').read_text() metadata = { 'version': os.environ['APP_VERSION'], + # App version is SemVer; CHANGELOG.md is keyed by the backend release. + 'pypi_version': os.environ['PYPI_VERSION'], 'notes': notes, 'pub_date': datetime.datetime.now(datetime.timezone.utc).isoformat(timespec='milliseconds').replace('+00:00', 'Z'), 'platforms': { diff --git a/.github/workflows/startup-profile-ci.yml b/.github/workflows/startup-profile-ci.yml new file mode 100644 index 0000000000..fbde99836d --- /dev/null +++ b/.github/workflows/startup-profile-ci.yml @@ -0,0 +1,156 @@ +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. + +# Measures where Studio's startup time goes, on each platform. +# +# Nothing recorded a number before: main.py logs "lifespan startup completed in X ms" +# and studio_test_kit polls /healthz, but both throw the elapsed time away. A first +# local run (Linux, warm cache, 18-core server) put `import main` at 5.7-6.6s BEFORE +# the server can bind, dominated by eager module-level imports pulled in by routes: +# torch ~1.9s self, unsloth_zoo ~0.8s, routes ~0.6s, transformers ~0.5s. +# +# Not a gate yet: --max-healthz-seconds exists, but a budget should come from +# observed numbers rather than a guess. + +name: Startup profile + +on: + pull_request: + paths: + # The measured import graph is the whole backend tree: main.py imports auth, + # core, hub, loggers, models, picker, routes and utils at module scope. + - 'studio/backend/**' + - '!studio/backend/tests/**' + # The launch phase spawns `unsloth studio --api-only`, so the CLI counts too. + - 'unsloth_cli/**' + - 'studio/src-tauri/src/preflight**' + # The profiler hardcodes the desktop argv that process.rs::backend_args builds, + # so a change there must schedule a run or the two silently diverge. + - 'studio/src-tauri/src/process.rs' + - 'scripts/profile_startup.py' + - '.github/workflows/startup-profile-ci.yml' + # The job profiles whatever `install.sh --local` built: the installers pick the + # venv's Python and the dependency specs, and pyproject's include list is what + # makes --local overlay studio.backend*. + - 'install.sh' + - 'install.ps1' + - 'pyproject.toml' + # --local also runs the checkout's setup scripts (install.sh picks + # $_REPO_ROOT/studio/setup.sh, the editable install resolves setup.ps1 to the + # repo), and both call install_python_stack.py, which picks the dependencies. + - 'studio/setup.sh' + - 'studio/setup.ps1' + - 'studio/install_python_stack.py' + workflow_dispatch: + inputs: + repeats: + description: 'launch repeats per OS (median reported)' + type: string + default: '3' + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +permissions: + contents: read + +jobs: + profile: + name: startup ${{ matrix.os }} + runs-on: ${{ matrix.os }} + timeout-minutes: 60 + continue-on-error: true + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, macos-14, windows-latest] + + env: + UNSLOTH_STUDIO_HOME: ${{ github.workspace }}/.studio-home + # A wildcard bind calls ifconfig.me on the startup path; loopback times our code. + UNSLOTH_STUDIO_DISABLE_PUBLIC_CHECK: '1' + + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + + - name: Install Studio + shell: bash + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + set -o pipefail + mkdir -p logs + # --local is load-bearing: it overlays the checkout, so the profiled server + # is this diff. Without it install.sh resolves unsloth from PyPI. + if [ "${{ runner.os }}" = "Windows" ]; then + pwsh -NoProfile -File ./install.ps1 --local 2>&1 | tee logs/install.log + else + bash install.sh --local 2>&1 | tee logs/install.log + fi + + - name: Profile startup + shell: bash + run: | + BIN="$UNSLOTH_STUDIO_HOME/unsloth_studio/bin/unsloth" + [ -x "$BIN" ] || BIN="$UNSLOTH_STUDIO_HOME/unsloth_studio/Scripts/unsloth.exe" + [ -x "$BIN" ] || BIN="" + # Profile imports with the INSTALLED interpreter: that venv is what launches. + PY="$UNSLOTH_STUDIO_HOME/unsloth_studio/bin/python" + [ -x "$PY" ] || PY="$UNSLOTH_STUDIO_HOME/unsloth_studio/Scripts/python.exe" + [ -x "$PY" ] || PY="$(command -v python3 || command -v python)" + python3 scripts/profile_startup.py \ + --python "$PY" \ + ${BIN:+--bin "$BIN"} \ + --repeats "${{ inputs.repeats || '3' }}" \ + --json "startup-${{ matrix.os }}.json" 2>&1 | tee logs/profile.log + + - name: Summary + if: always() + shell: bash + run: | + f="startup-${{ matrix.os }}.json" + [ -f "$f" ] || { echo "no profile produced"; exit 0; } + python3 - "$f" >> "$GITHUB_STEP_SUMMARY" <<'PY' + import json, sys + d = json.load(open(sys.argv[1])) + print(f"### {d['platform']} / {d['machine']} (py {d['python']}, {d['cpu_count']} cpu)\n") + imp = d.get("imports", {}) + # Gate on ok: a failed `import main` still leaves rows, so a total can lie. + if imp.get("ok"): + print(f"**`import main`: {imp['total_seconds']}s**\n") + print("| package | self ms |") + print("|---|---:|") + for k, v in list(imp.get("self_by_package_ms", {}).items())[:8]: + print(f"| {k} | {v} |") + print() + else: + print("**`import main` failed - no valid import profile**\n") + print("```\n" + (imp.get("error") or "")[-1500:] + "\n```\n") + lau = d.get("launch") or {} + runs = len(lau.get("runs") or []) + failed = lau.get("failed_runs") or 0 + if lau.get("healthz_median_seconds") is not None: + # The aggregates cover only the runs that reached healthz, so flag the + # failures: bare numbers would read as a normal fast startup. + note = f" _({runs - failed} of {runs} launches; {failed} never became healthy)_" if failed else "" + print(f"**time to a healthy port: {lau['healthz_median_seconds']}s median, " + f"{lau['healthz_max_seconds']}s max**{note}\n") + elif lau.get("skipped"): + print(f"_launch phase skipped: {lau['skipped']}_\n") + elif runs: + print(f"**no launch measurement: all {runs} launches failed to become healthy**\n") + PY + + - name: Upload profile + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: startup-profile-${{ matrix.os }} + path: | + startup-*.json + logs/ + retention-days: 14 + if-no-files-found: warn diff --git a/.github/workflows/studio-api-smoke.yml b/.github/workflows/studio-api-smoke.yml index cdf1f6bf12..1cfa66fea4 100644 --- a/.github/workflows/studio-api-smoke.yml +++ b/.github/workflows/studio-api-smoke.yml @@ -113,7 +113,8 @@ jobs: - name: Reset auth + boot Unsloth (API-only) run: | - unsloth studio reset-password + # Wipe (not reset-password): the boot below must re-seed a fresh .bootstrap_password. + rm -rf ~/.unsloth/studio/auth mkdir -p logs UNSLOTH_API_ONLY=1 unsloth studio -H 127.0.0.1 -p "$STUDIO_PORT" \ > logs/studio.log 2>&1 & diff --git a/.github/workflows/studio-backend-ci.yml b/.github/workflows/studio-backend-ci.yml index ec437e0c32..dd5efbb299 100644 --- a/.github/workflows/studio-backend-ci.yml +++ b/.github/workflows/studio-backend-ci.yml @@ -223,6 +223,16 @@ jobs: tests/studio/test_is_mlx_dispatch_gate.py \ tests/studio/test_xpu_spoof_pipeline.py + - name: CLI tests (unsloth_cli) + # unsloth_cli/tests had no CI at all: `unsloth_cli/**` was only a paths + # trigger and a ruff target, so 673 tests covering the studio launcher, + # the pre-exposure gate and the auth secret writers ran nowhere, and + # four of them had been failing on main unnoticed. + # Own step, not folded into the tests/ discovery above: pyproject's + # testpaths is tests/, and this suite needs no PYTHONPATH or CUDA spoof + # (it self-bootstraps sys.path and imports neither unsloth nor torch). + run: python -m pytest unsloth_cli/tests -q --tb=short + - name: Shell installer tests # Auto-discovered rather than allowlisted. The old hardcoded list had # silently fallen seven files behind tests/run_all.sh, including diff --git a/.github/workflows/studio-inference-smoke.yml b/.github/workflows/studio-inference-smoke.yml index c2d52eac22..c37c9555bf 100644 --- a/.github/workflows/studio-inference-smoke.yml +++ b/.github/workflows/studio-inference-smoke.yml @@ -127,7 +127,8 @@ jobs: - name: Reset auth + boot Unsloth (API-only) run: | - unsloth studio reset-password + # Wipe (not reset-password): the boot below must re-seed a fresh .bootstrap_password. + rm -rf ~/.unsloth/studio/auth mkdir -p logs UNSLOTH_API_ONLY=1 unsloth studio -H 127.0.0.1 -p "$STUDIO_PORT" \ > logs/studio.log 2>&1 & @@ -400,7 +401,7 @@ jobs: # tool_policy=None so each request's `enable_tools` field is # honoured. run: | - unsloth studio reset-password + rm -rf ~/.unsloth/studio/auth mkdir -p logs UNSLOTH_API_ONLY=1 unsloth studio -H 127.0.0.1 -p "$STUDIO_PORT" \ > logs/studio.log 2>&1 & @@ -978,7 +979,7 @@ jobs: # response_format requests aren't routed through the agentic # tool loop. run: | - unsloth studio reset-password + rm -rf ~/.unsloth/studio/auth mkdir -p logs UNSLOTH_API_ONLY=1 unsloth studio -H 127.0.0.1 -p "$STUDIO_PORT" \ > logs/studio.log 2>&1 & diff --git a/.github/workflows/studio-mac-api-smoke.yml b/.github/workflows/studio-mac-api-smoke.yml index 1968885a1d..c2307f17a1 100644 --- a/.github/workflows/studio-mac-api-smoke.yml +++ b/.github/workflows/studio-mac-api-smoke.yml @@ -101,7 +101,8 @@ jobs: - name: Reset auth + boot Unsloth (API-only) run: | - unsloth studio reset-password + # Wipe (not reset-password): the boot below must re-seed a fresh .bootstrap_password. + rm -rf ~/.unsloth/studio/auth mkdir -p logs UNSLOTH_API_ONLY=1 unsloth studio -H 127.0.0.1 -p "$STUDIO_PORT" \ > logs/studio.log 2>&1 & diff --git a/.github/workflows/studio-mac-inference-smoke.yml b/.github/workflows/studio-mac-inference-smoke.yml index ce15eed5c8..1dbf86ae98 100644 --- a/.github/workflows/studio-mac-inference-smoke.yml +++ b/.github/workflows/studio-mac-inference-smoke.yml @@ -126,7 +126,8 @@ jobs: - name: Reset auth + boot Unsloth (API-only) run: | - unsloth studio reset-password + # Wipe (not reset-password): the boot below must re-seed a fresh .bootstrap_password. + rm -rf ~/.unsloth/studio/auth mkdir -p logs UNSLOTH_API_ONLY=1 unsloth studio -H 127.0.0.1 -p "$STUDIO_PORT" \ > logs/studio.log 2>&1 & @@ -386,7 +387,7 @@ jobs: # tool_policy=None so each request's `enable_tools` field is # honoured. run: | - unsloth studio reset-password + rm -rf ~/.unsloth/studio/auth mkdir -p logs UNSLOTH_API_ONLY=1 unsloth studio -H 127.0.0.1 -p "$STUDIO_PORT" \ > logs/studio.log 2>&1 & @@ -831,7 +832,7 @@ jobs: # response_format requests aren't routed through the agentic # tool loop. run: | - unsloth studio reset-password + rm -rf ~/.unsloth/studio/auth mkdir -p logs UNSLOTH_API_ONLY=1 unsloth studio -H 127.0.0.1 -p "$STUDIO_PORT" \ > logs/studio.log 2>&1 & diff --git a/.github/workflows/studio-mac-ui-smoke.yml b/.github/workflows/studio-mac-ui-smoke.yml index 7375e9bcbf..3bed2fcdff 100644 --- a/.github/workflows/studio-mac-ui-smoke.yml +++ b/.github/workflows/studio-mac-ui-smoke.yml @@ -146,7 +146,8 @@ jobs: - name: Reset auth + boot Unsloth run: | - unsloth studio reset-password + # Wipe (not reset-password): the boot below must re-seed a fresh .bootstrap_password. + rm -rf ~/.unsloth/studio/auth mkdir -p logs UNSLOTH_API_ONLY=1 unsloth studio -H 127.0.0.1 -p "$STUDIO_PORT" \ > logs/studio.log 2>&1 & @@ -190,7 +191,7 @@ jobs: # runner's kernel briefly runs out of socket buffers, and (3) a # goto 'interrupted by another navigation' when the SPA auth # guard redirects mid-navigation. The retry FULLY resets Unsloth - # (kill, reset-password, reboot, wait /api/health, re-export + # (kill, wipe auth, reboot, wait /api/health, re-export # bootstrap pw) before re-running the script. A real test failure # (assertion / timeout) does NOT match any pattern so it bypasses # retry and surfaces immediately. @@ -213,7 +214,7 @@ jobs: echo "::warning::Playwright flake on attempt ${attempt}; resetting Unsloth and retrying..." kill "${STUDIO_PID}" 2>/dev/null || true sleep 2 - unsloth studio reset-password + rm -rf ~/.unsloth/studio/auth UNSLOTH_API_ONLY=1 unsloth studio -H 127.0.0.1 -p "$STUDIO_PORT" \ > "logs/studio_retry_${attempt}.log" 2>&1 & STUDIO_PID=$! @@ -251,7 +252,7 @@ jobs: - name: Reset auth + boot Unsloth for extra UI tests (port 18897) run: | - unsloth studio reset-password + rm -rf ~/.unsloth/studio/auth mkdir -p logs UNSLOTH_API_ONLY=1 unsloth studio -H 127.0.0.1 -p 18897 \ > logs/studio_extra.log 2>&1 & @@ -308,7 +309,7 @@ jobs: echo "::warning::Playwright flake on attempt ${attempt}; resetting Unsloth and retrying..." kill "${STUDIO_EXTRA_PID}" 2>/dev/null || true sleep 2 - unsloth studio reset-password + rm -rf ~/.unsloth/studio/auth UNSLOTH_API_ONLY=1 unsloth studio -H 127.0.0.1 -p 18897 \ > "logs/studio_extra_retry_${attempt}.log" 2>&1 & STUDIO_EXTRA_PID=$! diff --git a/.github/workflows/studio-ui-smoke.yml b/.github/workflows/studio-ui-smoke.yml index 97eb07b2d8..3a0713f301 100644 --- a/.github/workflows/studio-ui-smoke.yml +++ b/.github/workflows/studio-ui-smoke.yml @@ -115,7 +115,8 @@ jobs: - name: Reset auth + boot Unsloth run: | - unsloth studio reset-password + # Wipe (not reset-password): the boot below must re-seed a fresh .bootstrap_password. + rm -rf ~/.unsloth/studio/auth mkdir -p logs UNSLOTH_API_ONLY=1 unsloth studio -H 127.0.0.1 -p "$STUDIO_PORT" \ > logs/studio.log 2>&1 & @@ -193,7 +194,7 @@ jobs: # warm install we already did) so this adds little wall time. - name: Reset auth + boot Unsloth for extra UI tests (port 18894) run: | - unsloth studio reset-password + rm -rf ~/.unsloth/studio/auth mkdir -p logs UNSLOTH_API_ONLY=1 unsloth studio -H 127.0.0.1 -p 18894 \ > logs/studio_extra.log 2>&1 & @@ -253,7 +254,7 @@ jobs: # (RAG embedder + llama.cpp probe) stay hidden from the picker. - name: Reset auth + boot Unsloth for model-config tests (port 18898) run: | - unsloth studio reset-password + rm -rf ~/.unsloth/studio/auth mkdir -p logs UNSLOTH_API_ONLY=1 unsloth studio -H 127.0.0.1 -p 18898 \ > logs/studio_modelcfg.log 2>&1 & @@ -299,7 +300,7 @@ jobs: # earlier UI tests. No GGUF -- the bug surface is the composer. - name: Reset auth + boot Unsloth for IME / i18n tests (port 18896) run: | - unsloth studio reset-password + rm -rf ~/.unsloth/studio/auth mkdir -p logs UNSLOTH_API_ONLY=1 unsloth studio -H 127.0.0.1 -p 18896 \ > logs/studio_ime.log 2>&1 & diff --git a/.github/workflows/studio-windows-api-smoke.yml b/.github/workflows/studio-windows-api-smoke.yml index 6dbcceebbd..b328939846 100644 --- a/.github/workflows/studio-windows-api-smoke.yml +++ b/.github/workflows/studio-windows-api-smoke.yml @@ -179,7 +179,8 @@ jobs: - name: Reset auth + boot Unsloth (API-only) run: | - unsloth studio reset-password + # Wipe (not reset-password): the boot below must re-seed a fresh .bootstrap_password. + rm -rf ~/.unsloth/studio/auth mkdir -p logs UNSLOTH_API_ONLY=1 unsloth studio -H 127.0.0.1 -p "$STUDIO_PORT" \ > logs/studio.log 2>&1 & diff --git a/.github/workflows/studio-windows-inference-smoke.yml b/.github/workflows/studio-windows-inference-smoke.yml index 3ebe442f52..d821664327 100644 --- a/.github/workflows/studio-windows-inference-smoke.yml +++ b/.github/workflows/studio-windows-inference-smoke.yml @@ -229,7 +229,8 @@ jobs: - name: Reset auth + boot Unsloth (API-only) run: | - unsloth studio reset-password + # Wipe (not reset-password): the boot below must re-seed a fresh .bootstrap_password. + rm -rf ~/.unsloth/studio/auth mkdir -p logs UNSLOTH_API_ONLY=1 unsloth studio -H 127.0.0.1 -p "$STUDIO_PORT" \ > logs/studio.log 2>&1 & @@ -573,7 +574,7 @@ jobs: - name: Reset auth + boot Unsloth (API-only, default tool policy) run: | - unsloth studio reset-password + rm -rf ~/.unsloth/studio/auth mkdir -p logs UNSLOTH_API_ONLY=1 unsloth studio -H 127.0.0.1 -p "$STUDIO_PORT" \ > logs/studio.log 2>&1 & @@ -1074,7 +1075,7 @@ jobs: - name: Reset auth + boot Unsloth (API-only) run: | - unsloth studio reset-password + rm -rf ~/.unsloth/studio/auth mkdir -p logs UNSLOTH_API_ONLY=1 unsloth studio -H 127.0.0.1 -p "$STUDIO_PORT" \ > logs/studio.log 2>&1 & @@ -1546,7 +1547,7 @@ jobs: - name: Reset auth + boot Unsloth (API-only) run: | - unsloth studio reset-password + rm -rf ~/.unsloth/studio/auth mkdir -p logs UNSLOTH_API_ONLY=1 unsloth studio -H 127.0.0.1 -p "$STUDIO_PORT" \ > logs/studio.log 2>&1 & @@ -1888,8 +1889,11 @@ jobs: # (step/substep -> Write-StudioStdoutMirror / Get-StudioAnsi). $script:StudioVtOk = $false $script:UnslothVerbose = $false + # Get-HostMachineArch is reached only on the absent path, where + # Test-VCRedistInstalled consults it before trusting the System32 DLL, so + # part A passes without it and only the clean-box part fails. foreach ($fn in @('Get-StudioAnsi', 'Write-StudioStdoutMirror', 'step', 'substep', - 'Invoke-SetupCommand', 'Refresh-Environment', + 'Invoke-SetupCommand', 'Refresh-Environment', 'Get-HostMachineArch', 'Test-VCRedistInstalled', 'Ensure-VCRedist')) { $src = Get-FunctionSource -Path $setup -Name $fn if (-not $src) { throw "Function '$fn' not found in setup.ps1" } diff --git a/.github/workflows/studio-windows-ui-smoke.yml b/.github/workflows/studio-windows-ui-smoke.yml index f401f7be44..d23cca323f 100644 --- a/.github/workflows/studio-windows-ui-smoke.yml +++ b/.github/workflows/studio-windows-ui-smoke.yml @@ -297,7 +297,8 @@ jobs: - name: Reset auth + boot Unsloth run: | - unsloth studio reset-password + # Wipe (not reset-password): the boot below must re-seed a fresh .bootstrap_password. + rm -rf ~/.unsloth/studio/auth mkdir -p logs UNSLOTH_API_ONLY=1 unsloth studio -H 127.0.0.1 -p "$STUDIO_PORT" \ > logs/studio.log 2>&1 & @@ -352,7 +353,7 @@ jobs: - name: Reset auth + boot Unsloth for extra UI tests (port 18897) run: | - unsloth studio reset-password + rm -rf ~/.unsloth/studio/auth mkdir -p logs UNSLOTH_API_ONLY=1 unsloth studio -H 127.0.0.1 -p 18897 \ > logs/studio_extra.log 2>&1 & diff --git a/.gitignore b/.gitignore index fafd17aa95..fa6997cb06 100644 --- a/.gitignore +++ b/.gitignore @@ -208,6 +208,9 @@ tmp/ **/node_modules/ auth.db +# Packaging snapshot of the root CHANGELOG.md (written by build.sh) +studio/CHANGELOG.md + # Tauri local build/generated output studio/src-tauri/target/ studio/src-tauri/gen/ diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000000..241e013cea --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,88 @@ +# Changelog + +Release notes for Unsloth and Unsloth Studio. + +Unsloth Studio reads this file to show release notes inside the "New Unsloth +version" update popup. Edit it here and the popup picks the change up on the +next update check, with no release or rebuild required. + +## Format + +Every release is a level-2 heading whose first token is the version, optionally +followed by a date: + +```md +## 2026.7.6 - 2026-07-22 +``` + +`## [2026.7.6] - 2026-07-22` and `## v2026.7.6` also work. Everything under a +heading, up to the next level-2 heading, is that release's notes and renders as +Markdown in the popup. + +Notes are matched to one exact version. When Studio offers an update to +`2026.7.6` it renders the `2026.7.6` section and nothing else. If that section +is missing, the popup links out to the online changelog rather than showing +notes from an unrelated release, so a new version needs its own section here +before its notes can appear. + +Keep the newest release at the top. Lead each bullet with the change itself: +the collapsed popup highlights the first sentence and dims the rest. +`## Unreleased` is ignored by the popup, so it is safe to stage notes there and +rename the heading at release time. + + + +## Unreleased + +## 2026.7.5 + +### What's Changed + +- AMD support is here. Train, run RL, chat with and deploy 500+ models on + Radeon, Instinct, Ryzen and data center GPUs across Windows, WSL and Linux, + up to 2x faster with 70% less VRAM and no accuracy loss. +- Intel XPU support lands in Studio, so Arc and Data Center GPUs run chat and + training alongside the NVIDIA, AMD and Apple paths. +- Local speech to text dictation runs fully offline, with slim Whisper bundles + and a picker for custom models. +- DoRA training is available in Studio, selectable next to LoRA and full + fine-tuning in the training tab. +- The update popup previews release notes inline, pulled from this file and + matched to the exact version being offered. + +### AMD, 23 July update + +Our AMD collaboration, custom Triton kernels and math algorithms bring local +training and inference to AMD hardware. The 23 July update builds on the +[AMD release](https://github.com/unslothai/unsloth/releases/tag/v0.1.501-beta): + +- RDNA2 and Gorgon Halo are supported, and the installer no longer fails to + detect GPUs on Strix Halo and other AMD cards. +- RDNA4 handling is better, and HIP and ROCm failures are caught and fixed + automatically instead of stopping the install. +- Unified memory safetensors loading is 2x faster, with much faster gradient + checkpointing on unified memory devices. +- Voice dictation through whisper.cpp has preliminary support. +- Rollback environments left by installs no longer eat 5GB of disk. They are + cleaned up automatically. + +Optimized ROCm builds cover GGUF and safetensors inference, and ROCm +compatibility is improved for MI300X and MI325X. Full guide: +[unsloth.ai/docs/basics/amd](https://unsloth.ai/docs/basics/amd). + +### Running larger models + +- Automatic GPU placement, or pick exactly which GPUs and layers to use. +- Move MoE expert layers into system memory so larger models fit. +- Split a model across several GPUs, or use tensor parallelism. +- Hardware settings are saved per model and quant. + +### Also in this release + +- Remote access with `unsloth studio --secure` over free HTTPS via Cloudflare. +- Web search reads PDF papers and manuals, and parallel tool calls, reasoning + output and tool retries are more reliable. +- The model download location is configurable, so weights can live on a second + drive instead of the default cache. +- Stalled Hugging Face XET downloads retry over standard HTTP, and existing + GGUF files are reused instead of downloaded again. diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 0000000000..7bce036343 --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1,2 @@ +include _changelog_build.py +include CHANGELOG.md diff --git a/_changelog_build.py b/_changelog_build.py new file mode 100644 index 0000000000..f5bcf2052c --- /dev/null +++ b/_changelog_build.py @@ -0,0 +1,36 @@ +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. + +"""Snapshot CHANGELOG.md into the studio package at build time. + +CHANGELOG.md at the repo root stays the one file to edit. Copying it here, +rather than in build.sh, means every packaging path ships it, so release notes +still render when the popup cannot reach GitHub.""" + +from __future__ import annotations + +import shutil +from pathlib import Path + +from setuptools.command.build_py import build_py as _build_py + +ROOT = Path(__file__).resolve().parent +SOURCE = ROOT / "CHANGELOG.md" +SNAPSHOT = ROOT / "studio" / "CHANGELOG.md" + + +class build_py(_build_py): + def run(self) -> None: + # Beside the sources only if writable (PEP 517 may build an immutable + # checkout); into the staging directory always. + if SOURCE.is_file(): + try: + shutil.copyfile(SOURCE, SNAPSHOT) + except OSError: + pass + super().run() + if not SOURCE.is_file(): + return + staged = Path(self.build_lib) / "studio" / "CHANGELOG.md" + staged.parent.mkdir(parents = True, exist_ok = True) + shutil.copyfile(SOURCE, staged) diff --git a/build.sh b/build.sh index 2a836e19d9..5b09a7791b 100644 --- a/build.sh +++ b/build.sh @@ -103,9 +103,13 @@ else STUDIO_STAMPED_VERSION="$(python scripts/stamp_studio_release.py)" fi -# 4. Build wheel/sdist +# 4. Build wheel/sdist. _changelog_build.py snapshots CHANGELOG.md into the studio +# package so release notes render offline. python -m build +# Drop the snapshot so a source checkout never serves a stale copy. +rm -f studio/CHANGELOG.md + if [ "${1:-}" = "publish" ]; then python scripts/stamp_studio_release.py --verify-dist dist --expected "$STUDIO_STAMPED_VERSION" fi diff --git a/install.ps1 b/install.ps1 index 0b06cb3ea1..5b205df96d 100644 --- a/install.ps1 +++ b/install.ps1 @@ -57,6 +57,26 @@ function Install-UnslothStudio { } } + # Machine arch; Get-TauriDiagArch above reports the process. An emulated x64 shell on + # ARM64 reports AMD64, but PROCESSOR_ARCHITEW6432 is ARM64 in exactly that case. + function Get-HostMachineArch { + $osArch = "" + try { $osArch = [System.Runtime.InteropServices.RuntimeInformation]::OSArchitecture.ToString() } catch { $osArch = "" } + $signals = @([string]$env:PROCESSOR_ARCHITEW6432, [string]$env:PROCESSOR_ARCHITECTURE, $osArch) + foreach ($s in $signals) { + if ($s.ToLowerInvariant() -eq "arm64") { return "arm64" } + } + foreach ($s in $signals) { + if ([string]::IsNullOrWhiteSpace($s)) { continue } + switch ($s.ToLowerInvariant()) { + "amd64" { return "x86_64" } + "x64" { return "x86_64" } + "x86" { return "x86" } + } + } + return "unknown" + } + function Get-TauriTorchIndexFamily { param([string]$TorchIndexUrl) if ($SkipTorch) { return "none" } @@ -1124,10 +1144,27 @@ exit 0 return $false } + # The interpreter's own arch, asked of it: win-amd64|win-arm64|win32|"". + function Get-PythonPlatformTag { + param([string]$Exe) + try { + return (& $Exe -c "import sysconfig; print(sysconfig.get_platform())" 2>$null | Out-String).Trim().ToLowerInvariant() + } catch { return "" } + } + # Returns @{ Version = "3.13"; Path = "C:\...\python.exe" } or $null. # The resolved Path is passed to `uv venv --python` to prevent uv from # re-resolving the version string back to a conda interpreter. function Find-CompatiblePython { + # -X64Only: best installed x64 interpreter or $null, never ARM64. Last resort for + # Install-X64Python, where x64 of a lower-priority minor beats ARM64. + param([switch]$X64Only) + # Windows on ARM: prefer x64. pyarrow (via datasets) and hf-transfer ship no + # win_arm64 wheel, so a native ARM64 Python source-builds both and dies on CMake / + # Rust minutes in; x64 runs fine emulated. ARM64 is still returned when it is all + # there is, and the caller then bootstraps x64 or warns. + $preferX64 = $X64Only -or ((Get-HostMachineArch) -eq "arm64") + $candidates = @() # Try the Python Launcher first (most reliable on Windows) # py.exe resolves to the standard CPython install, not conda. # Prefer the requested $PythonVersion, then newest-first fallback. @@ -1145,7 +1182,8 @@ exit 0 # Resolve the actual executable path and verify it is not conda-based $resolvedExe = (& $pyLauncher.Source "-$minor" -c "import sys; print(sys.executable)" 2>$null | Out-String).Trim() if ($resolvedExe -and (Test-Path $resolvedExe) -and -not (Test-IsCondaPython $resolvedExe)) { - return @{ Version = $ver; Path = $resolvedExe } + if (-not $preferX64) { return @{ Version = $ver; Path = $resolvedExe; Arch = "" } } + $candidates += @{ Version = $ver; Path = $resolvedExe } } } } catch {} @@ -1166,11 +1204,53 @@ exit 0 try { $out = & $cmd.Source --version 2>&1 | Out-String if ($out -match "Python (3\.1[1-3])\.\d+") { - return @{ Version = $Matches[1]; Path = $cmd.Source } + if (-not $preferX64) { return @{ Version = $Matches[1]; Path = $cmd.Source; Arch = "" } } + $candidates += @{ Version = $Matches[1]; Path = $cmd.Source } } } catch {} } } + # `py -3.12` runs the launcher's preferred build, normally the native ARM64 one, so + # a same-minor x64 install that is neither preferred nor on PATH never becomes a + # candidate. `-3.12-64` cannot disambiguate (deprecated, it only means "not + # 32-bit"), so enumerate every registration with -0p and probe each path. + if ($preferX64) { + foreach ($pyLauncher in @(Get-Command py -All -CommandType Application -ErrorAction SilentlyContinue)) { + if ($pyLauncher.Source -match $script:CondaSkipPattern) { continue } + $listed = @() + try { $listed = @(& $pyLauncher.Source "-0p" 2>$null) } catch {} + foreach ($line in $listed) { + # " -V:3.12 * C:\...\python.exe": tag, optional default marker, path. + $m = [regex]::Match([string]$line, '(?i)^\s*-\S+\s+\*?\s*"?(?

\S.*?\.exe)"?\s*$') + if (-not $m.Success) { continue } + $exe = $m.Groups['p'].Value.Trim() + if ($candidates | Where-Object { $_.Path -eq $exe }) { continue } + if (-not (Test-Path -LiteralPath $exe)) { continue } + if (Test-IsCondaPython $exe) { continue } + try { + $out = & $exe --version 2>&1 | Out-String + if ($out -match "Python (3\.1[1-3])\.\d+") { + $candidates += @{ Version = $Matches[1]; Path = $exe } + } + } catch {} + } + } + } + # Prefer x64, but only within one minor: $minors is the caller's version preference, + # so ranking on arch alone would answer UNSLOTH_PYTHON=3.12 with an x64 3.13 and + # never bootstrap x64 3.12. Probing costs a subprocess, so non-ARM returned above. + foreach ($c in $candidates) { + $tag = Get-PythonPlatformTag $c.Path + $c.Arch = if ($tag -eq "win-amd64") { "x86_64" } elseif ($tag -eq "win-arm64") { "arm64" } else { "unknown" } + } + foreach ($minor in $minors) { + $sameMinor = @($candidates | Where-Object { $_.Version -eq $minor }) + if ($sameMinor.Count -eq 0) { continue } + $x64 = $sameMinor | Where-Object { $_.Arch -eq "x86_64" } | Select-Object -First 1 + if ($x64) { return $x64 } + if (-not $X64Only) { return $sameMinor[0] } + } + if (-not $X64Only -and $candidates.Count -gt 0) { return $candidates[0] } return $null } @@ -1181,8 +1261,11 @@ exit 0 # (no UAC), putting python.exe + the py launcher on PATH. Mirrors the uv -> # astral.sh fallback below. Returns @{ Version; Path } or $null. function Install-PythonFromPythonOrg { + # $Arch overrides the host arch, to pull x64 onto an ARM64 box. + param([string]$Arch = "") # python.org ships one installer per architecture. - $archSuffix = switch (Get-TauriDiagArch) { + $targetArch = if ($Arch) { $Arch } else { Get-TauriDiagArch } + $archSuffix = switch ($targetArch) { "x86_64" { "-amd64" } "arm64" { "-arm64" } "x86" { "" } @@ -1247,6 +1330,28 @@ exit 0 return (Find-CompatiblePython) } + # ── Windows on ARM: get an x64 CPython ── + # --architecture x64 forces winget off the ARM64 build; python.org takes the same override. + function Install-X64Python { + if ($script:WingetAvailable) { + $prevEAP = $ErrorActionPreference + $ErrorActionPreference = "Continue" + try { + winget install -e --id "Python.Python.$PythonVersion" --source winget --architecture x64 --accept-package-agreements --accept-source-agreements + } catch { } + $ErrorActionPreference = $prevEAP + Refresh-SessionPath + $found = Find-CompatiblePython + if ($found -and $found.Arch -eq "x86_64") { return $found } + substep "winget could not provide an x64 Python -- trying python.org..." "Yellow" + } + $found = Install-PythonFromPythonOrg -Arch "x86_64" + if ($found -and $found.Arch -eq "x86_64") { return $found } + # Nothing installable (offline / no winget): an x64 build of another supported minor + # still runs the wheels ARM64 cannot, so take it over the native interpreter. + return (Find-CompatiblePython -X64Only) + } + # ── Install Python if no compatible version (3.11-3.13) found ── # Find-CompatiblePython returns @{ Version = "3.13"; Path = "C:\...\python.exe" } or $null. Write-TauriLog "STEP" "Installing Python" @@ -1318,6 +1423,26 @@ exit 0 return (Exit-InstallFailure "Python installation failed") } } + # ── Windows on ARM: swap a native ARM64 interpreter for x64 ── + # pyarrow and hf-transfer publish no win_arm64 wheel, so an ARM64 Python source-builds + # both and fails deep into the run. Warn up front if x64 is unobtainable. + if ($DetectedPython -and (Get-HostMachineArch) -eq "arm64" -and $DetectedPython.Arch -ne "x86_64") { + substep "windows on arm: only a native ARM64 Python $($DetectedPython.Version) was found." "Yellow" + substep "pyarrow and hf-transfer publish no win_arm64 wheels, so installing x64 Python..." "Yellow" + $X64Python = Install-X64Python + if ($X64Python) { + $DetectedPython = $X64Python + step "python" "using x64 Python $($DetectedPython.Version) under emulation" + } else { + Write-Host "[WARN] Could not install an x64 Python on this ARM64 machine." -ForegroundColor Yellow + Write-Host " Continuing with ARM64 Python $($DetectedPython.Version), but the install is likely to fail:" -ForegroundColor Yellow + Write-Host " pyarrow (via datasets) and hf-transfer ship no win_arm64 wheels and will be" -ForegroundColor Yellow + Write-Host " built from source, which needs CMake plus the MSVC and Rust toolchains." -ForegroundColor Yellow + Write-Host " Fix: install x64 Python from https://www.python.org/downloads/windows/" -ForegroundColor Yellow + Write-Host " (choose 'Windows installer (64-bit)', not ARM64), then re-run this installer." -ForegroundColor Yellow + } + } + $DiagPythonVersion = $PythonVersion if ($DetectedPython) { $DiagPythonVersion = $DetectedPython.Version } $InitialGpuBranch = "unknown" @@ -2438,6 +2563,13 @@ exit 0 } } else { Write-TauriLog "STEP" "Installing PyTorch" + # Windows on ARM lacks only torchaudio (whl/cpu win_arm64: torch 42, + # torchvision 60, torchaudio 0), so drop that pin instead of aborting. Ask the + # interpreter, not PROCESSOR_ARCHITECTURE; reached when no x64 Python exists. + $VenvPlatform = "" + try { + $VenvPlatform = (& $VenvPython -c "import sysconfig; print(sysconfig.get_platform())" 2>$null | Out-String).Trim().ToLowerInvariant() + } catch { $VenvPlatform = "" } substep "installing PyTorch ($(Remove-IndexUrlCredentials $TorchIndexUrl))..." # Bound the companions to the capped torch on EVERY index, cu # families included: torchaudio 2.11 dropped its exact torch pin from @@ -2445,7 +2577,13 @@ exit 0 # resolve a mismatched 2.11.0 build. Mirrors install.sh. $_pinVisionSpec = "torchvision>=0.19,<0.26.0" $_pinAudioSpec = "torchaudio>=2.4,<2.11.0" - $torchInstallExit = Invoke-InstallCommandRetry -Label "install PyTorch" { uv pip install --python $VenvPython "torch>=2.4,<2.11.0" $_pinVisionSpec $_pinAudioSpec --default-index $TorchIndexUrl } + $_torchSpecs = @("torch>=2.4,<2.11.0", $_pinVisionSpec, $_pinAudioSpec) + if ($VenvPlatform -eq "win-arm64") { + substep "windows on arm: skipping torchaudio (upstream publishes no" + substep "win_arm64 wheel); torch and torchvision install normally." + $_torchSpecs = @("torch>=2.4,<2.11.0", $_pinVisionSpec) + } + $torchInstallExit = Invoke-InstallCommandRetry -Label "install PyTorch" { uv pip install --python $VenvPython @_torchSpecs --default-index $TorchIndexUrl } if ($torchInstallExit -ne 0) { Write-Host "[ERROR] Failed to install PyTorch (exit code $torchInstallExit)" -ForegroundColor Red return (Exit-InstallFailure "Failed to install PyTorch (exit code $torchInstallExit)" $torchInstallExit) diff --git a/install.sh b/install.sh index fc9aa0a431..166beeb52c 100755 --- a/install.sh +++ b/install.sh @@ -19,6 +19,17 @@ # SPDX-License-Identifier: AGPL-3.0-only # Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 set -e +# ── Why the installer lives in a function ── +# Under `curl ... | sh`, sh is the pipe READER. This file is ~150KB, so a top-level +# `exit` left most of it unread, the write end failed, and curl tacked +# "(56) Failure writing output to destination" onto our own error message. Wrapping +# the body forces sh to parse to the closing brace first, so the pipe always drains +# (install.ps1 has always had this shape). +# +# Body is deliberately NOT reindented: reflowing 4000+ lines would bury the change, +# and `exit` still exits the shell from inside a function. Do not add +# `exec < /dev/null`: for a piped shell that closes the script's own source. +_unsloth_main() { # ── Output style (aligned with studio/setup.sh) ── RULE="" @@ -4447,3 +4458,8 @@ else substep "(add -H 0.0.0.0 --cloudflare for a public Cloudflare HTTPS link, or --secure to keep the raw port private; anyone with the API key can run code)" echo "" fi + +} + +# Every byte above is parsed before this line runs, which is the point. +_unsloth_main "$@" diff --git a/pyproject.toml b/pyproject.toml index 7359a51fa6..8895bf0686 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -47,9 +47,14 @@ version = {attr = "unsloth.models._utils.__version__"} [tool.setuptools] include-package-data = true +[tool.setuptools.cmdclass] +# Snapshots CHANGELOG.md into studio/ so every build path ships it. +build_py = "_changelog_build.build_py" + [tool.setuptools.package-data] unsloth_cli = ["codex_fallback_prompt.md", "pi_subagent.ts"] studio = [ + "CHANGELOG.md", "*.sh", "*.ps1", "*.bat", @@ -128,14 +133,19 @@ huggingfacenotorch = [ ] # torchcodec backend for Gemma audio / datasets>=4 (#7225). # Pick the audio-torch* pin matching your torch minor (see TORCH_TORCHCODEC). +# torchcodec publishes no sdist and only manylinux_2_28_x86_64, macosx_*_arm64 +# and win_amd64 wheels, so Linux aarch64, Windows ARM64 and Intel Mac have +# nothing to resolve and pip fails the whole install rather than skipping audio. +# Gate on the platforms that have a wheel, matching +# PLATFORM_LACKS_TORCHCODEC_WHEEL in studio/install_python_stack.py. audio-torch210 = [ - "torchcodec>=0.10.0,<0.11.0 ; python_version >= '3.10'", + "torchcodec>=0.10.0,<0.11.0 ; python_version >= '3.10' and (((sys_platform == 'linux' or sys_platform == 'win32') and (platform_machine == 'x86_64' or platform_machine == 'AMD64')) or (sys_platform == 'darwin' and platform_machine == 'arm64'))", ] audio-torch290 = [ - "torchcodec>=0.8.0,<0.10.0 ; python_version >= '3.10'", + "torchcodec>=0.8.0,<0.10.0 ; python_version >= '3.10' and (((sys_platform == 'linux' or sys_platform == 'win32') and (platform_machine == 'x86_64' or platform_machine == 'AMD64')) or (sys_platform == 'darwin' and platform_machine == 'arm64'))", ] audio-torch280 = [ - "torchcodec>=0.6.0,<0.8.0 ; python_version >= '3.9'", + "torchcodec>=0.6.0,<0.8.0 ; python_version >= '3.9' and (((sys_platform == 'linux' or sys_platform == 'win32') and (platform_machine == 'x86_64' or platform_machine == 'AMD64')) or (sys_platform == 'darwin' and platform_machine == 'arm64'))", ] huggingface = [ "unsloth[huggingfacenotorch]", diff --git a/scripts/profile_startup.py b/scripts/profile_startup.py new file mode 100644 index 0000000000..937d007ac1 --- /dev/null +++ b/scripts/profile_startup.py @@ -0,0 +1,377 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 + +"""Measure where Unsloth Studio's startup time goes, per platform. + +Nothing measured this before: the backend logs "lifespan startup completed in X ms" +but no test or CI job asserted a budget, and studio_test_kit discards the elapsed +time of its /healthz poll. A first local run (Linux, warm cache, fast server CPU) +found `import main` alone costs 6.6s before the server can bind, dominated by eager +module-level imports pulled in by the `routes` package: + + torch 1930 ms self + unsloth_zoo 914 ms self + routes 779 ms self + transformers 524 ms self + +Phases measured: + import `python -X importtime -c "import main"`, top cumulative + per-package self + spawn process start -> first byte on stdout + healthz process start -> /api/health (or /healthz) answers 200 + lifespan the backend's own "lifespan startup completed in X ms" log line + +Usage: + python scripts/profile_startup.py --repeats 3 --json out.json + python scripts/profile_startup.py --import-only # no server, no port needed + +Exit code is 0 unless --max-healthz-seconds is given and exceeded. +""" + +from __future__ import annotations + +import argparse +import json +import math +import os +import platform +import re +import shutil +import socket +import statistics +import subprocess +import sys +import threading +import time +import urllib.error +import urllib.request +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parents[1] +BACKEND = REPO_ROOT / "studio" / "backend" + +_IMPORTTIME_RE = re.compile(r"import time:\s+(\d+)\s+\|\s+(\d+)\s+\|(\s*)(\S.*)") + + +def _free_port() -> int: + with socket.socket() as s: + s.bind(("127.0.0.1", 0)) + return int(s.getsockname()[1]) + + +def profile_imports(python: str, top: int = 15) -> dict: + """Cumulative and self import cost for the backend's module graph. + + Run in a subprocess with -X importtime: the numbers are only meaningful for a + cold interpreter, and importing in-process would measure a warm sys.modules. + """ + proc = subprocess.run( + [python, "-X", "importtime", "-c", "import sys; sys.path.insert(0, '.'); import main"], + cwd = BACKEND, + capture_output = True, + text = True, + timeout = 900, + ) + rows = [] + for line in proc.stderr.splitlines(): + m = _IMPORTTIME_RE.match(line) + if m: + rows.append((int(m.group(1)), int(m.group(2)), m.group(4).strip())) + if not rows: + return {"ok": False, "error": (proc.stderr or proc.stdout)[-2000:]} + if proc.returncode != 0: + # Rows survive up to the failure, so any total from a partial graph is wrong. + return { + "ok": False, + "error": (proc.stderr or proc.stdout)[-2000:], + "partial_rows": len(rows), + } + + by_cum = sorted(rows, key = lambda r: -r[1]) + # Total comes from the `main` row, not by_cum[0]: -X importtime also prints the + # interpreter's own startup graph (`site`), which can outrank a trivial main. + main_row = next((r for r in reversed(rows) if r[2] == "main"), None) + if main_row is None: + return { + "ok": False, + "error": "no `import main` row in -X importtime output\n" + + (proc.stderr or proc.stdout)[-2000:], + } + self_by_pkg: dict[str, int] = {} + for self_us, _cum, name in rows: + pkg = name.split(".")[0] + self_by_pkg[pkg] = self_by_pkg.get(pkg, 0) + self_us + + return { + "ok": True, + "total_seconds": round(main_row[1] / 1e6, 3), + "top_cumulative": [ + {"module": n, "seconds": round(c / 1e6, 3)} for _s, c, n in by_cum[:top] + ], + "self_by_package_ms": { + k: round(v / 1000) for k, v in sorted(self_by_pkg.items(), key = lambda x: -x[1])[:top] + }, + } + + +def _terminate_tree(proc: subprocess.Popen) -> None: + """Stop the server AND its children, which on Windows are a separate process. + + CI profiles `Scripts/unsloth.exe`, a distlib launcher stub that CreateProcess's + the venv python and waits, so terminate() reaps the stub only: the real backend + keeps the inherited stdout handle, the reader thread never sees EOF, and + --repeats strands one server per iteration on the shared UNSLOTH_STUDIO_HOME. + taskkill /T walks the tree, as unsloth_cli/commands/start.py already does. + """ + if proc.poll() is not None: + return + if os.name == "nt": + try: + killed = subprocess.run( + ["taskkill", "/PID", str(proc.pid), "/T", "/F"], + capture_output = True, + timeout = 30, + check = False, + ) + if killed.returncode == 0: + return + except Exception: + # taskkill missing or timed out; fall through so the stub still dies. + pass + # check=False: a nonzero taskkill does not raise, so fall through as well. + proc.terminate() + + +def profile_launch( + bin_path: str, + port: int, + timeout_s: int = 300, +) -> dict: + """Spawn the backend the way the desktop app does and time it to first 200.""" + log_lines: list[str] = [] + first_byte: list[float] = [] + t0 = time.perf_counter() + proc = subprocess.Popen( + [bin_path, "studio", "--api-only", "-H", "127.0.0.1", "-p", str(port)], + cwd = REPO_ROOT, + stdout = subprocess.PIPE, + stderr = subprocess.STDOUT, + text = True, + bufsize = 1, + ) + + def _drain() -> None: + # Runs alongside the health polling: the first read timestamps the spawn + # phase, and an undrained pipe blocks the backend before it binds. + for line in proc.stdout: + if not first_byte: + first_byte.append(time.perf_counter() - t0) + log_lines.append(line.rstrip("\n")) + + reader = threading.Thread(target = _drain, daemon = True) + reader.start() + + t_healthz = None + deadline = t0 + timeout_s + try: + while time.perf_counter() < deadline: + if proc.poll() is not None: + break + if t_healthz is None: + for url in ( + f"http://127.0.0.1:{port}/api/health", + f"http://127.0.0.1:{port}/healthz", + ): + try: + with urllib.request.urlopen(url, timeout = 2) as r: + if r.status == 200: + t_healthz = time.perf_counter() - t0 + break + except (urllib.error.URLError, OSError, TimeoutError): + pass + if t_healthz is not None: + break + time.sleep(0.25) + finally: + _terminate_tree(proc) + try: + # Safe: the reader drains the pipe, so the child cannot block on write(). + proc.wait(timeout = 30) + except subprocess.TimeoutExpired: + proc.kill() + proc.wait() + reader.join(timeout = 10) + + t_first_byte = first_byte[0] if first_byte else None + lifespan_ms = None + for line in log_lines: + m = re.search(r"lifespan startup completed in ([\d.]+)ms", line) + if m: + lifespan_ms = float(m.group(1)) + return { + "spawn_seconds": round(t_first_byte, 3) if t_first_byte is not None else None, + "healthz_seconds": round(t_healthz, 3) if t_healthz is not None else None, + "lifespan_ms": lifespan_ms, + "reached_healthz": t_healthz is not None, + "log_tail": log_lines[-25:], + } + + +def python_version_of(python: str) -> str: + """Version of the interpreter that runs the imports, not the one running us. + + --python points at the installed Studio venv while this script runs under the + runner's system python, so platform.python_version() would label it wrong. + """ + if python == sys.executable: + return platform.python_version() + try: + proc = subprocess.run( + [python, "-c", "import platform; print(platform.python_version())"], + capture_output = True, + text = True, + timeout = 60, + ) + if proc.returncode == 0 and proc.stdout.strip(): + return proc.stdout.strip() + except (OSError, subprocess.SubprocessError): + pass + return "unknown" + + +def find_bin() -> str | None: + home = os.environ.get("UNSLOTH_STUDIO_HOME") or str(Path.home() / ".unsloth" / "studio") + names = ["unsloth.exe", "unsloth"] if platform.system() == "Windows" else ["unsloth"] + subdirs = ["unsloth_studio/Scripts", "unsloth_studio/bin", "bin", "Scripts"] + for sd in subdirs: + for n in names: + p = Path(home) / sd / n + if p.exists(): + return str(p) + return shutil.which("unsloth") + + +def main(argv: list[str]) -> int: + ap = argparse.ArgumentParser( + description = __doc__, formatter_class = argparse.RawDescriptionHelpFormatter + ) + ap.add_argument( + "--repeats", + type = int, + default = 1, + help = "launch repeats; the median is reported (imports are measured once)", + ) + ap.add_argument( + "--python", + default = sys.executable, + help = "interpreter used for the import profile (default: this one)", + ) + ap.add_argument("--bin", help = "path to the unsloth CLI (default: autodetect)") + ap.add_argument( + "--import-only", + action = "store_true", + help = "skip the server phases (no install needed beyond the deps)", + ) + ap.add_argument( + "--max-healthz-seconds", + type = float, + help = "fail if the median time to a healthy port exceeds this", + ) + ap.add_argument("--json", help = "write the full report here") + a = ap.parse_args(argv) + # range(0) launches nothing, leaving the budget check with nothing to fail on. + if a.repeats < 1: + ap.error("--repeats must be at least 1") + # Same reason: --import-only never launches anything. + if a.import_only and a.max_healthz_seconds is not None: + ap.error("--max-healthz-seconds cannot be combined with --import-only") + # nan and inf parse fine as floats but `med > budget` is then always False, + # so the gate would report success without ever bounding anything. + if a.max_healthz_seconds is not None and not math.isfinite(a.max_healthz_seconds): + ap.error("--max-healthz-seconds must be a finite number") + + report: dict = { + "platform": platform.system().lower(), + "machine": platform.machine(), + "python": python_version_of(a.python), + "cpu_count": os.cpu_count(), + } + + print("== import graph ==") + report["imports"] = profile_imports(a.python) + imp = report["imports"] + if imp.get("ok"): + print(f" import main: {imp['total_seconds']}s") + for row in imp["top_cumulative"][:8]: + print(f" {row['seconds']:7.3f}s {row['module']}") + print(" self time by package (ms):") + for k, v in list(imp["self_by_package_ms"].items())[:8]: + print(f" {v:8} ms {k}") + else: + print(f" FAILED: {imp.get('error', '')[:400]}") + + if not a.import_only: + bin_path = a.bin or find_bin() + if not bin_path: + print( + "== launch == skipped: no unsloth CLI found " + "(set UNSLOTH_STUDIO_HOME or pass --bin)" + ) + report["launch"] = {"skipped": "no unsloth CLI found"} + else: + print(f"== launch == {bin_path}") + runs = [] + for i in range(a.repeats): + r = profile_launch(bin_path, _free_port()) + runs.append(r) + print( + f" run {i + 1}: healthz={r['healthz_seconds']}s " + f"lifespan={r['lifespan_ms']}ms reached={r['reached_healthz']}" + ) + got = [r["healthz_seconds"] for r in runs if r["healthz_seconds"] is not None] + report["launch"] = { + "runs": runs, + "failed_runs": sum(1 for r in runs if not r["reached_healthz"]), + "healthz_median_seconds": round(statistics.median(got), 3) if got else None, + "healthz_max_seconds": round(max(got), 3) if got else None, + } + if got: + print( + f" median time to healthy port: {report['launch']['healthz_median_seconds']}s" + ) + + if a.json: + Path(a.json).write_text(json.dumps(report, indent = 2), encoding = "utf-8") + print(f"\nwrote {a.json}") + + if a.max_healthz_seconds is not None: + launch = report.get("launch") or {} + med = launch.get("healthz_median_seconds") + failed = launch.get("failed_runs") or 0 + if failed: + # Failed launches fail the budget; dropping them would keep only the fast ones. + print( + f"::error::startup regression: {failed} of {len(launch.get('runs') or [])} " + f"launches never became healthy within the timeout" + ) + return 1 + if med is None: + # Nothing measured: exiting 0 would pass a requested budget without a + # single health request, so fail closed. + print( + "::error::startup regression: no healthz measurement, so the " + f"{a.max_healthz_seconds}s budget was never checked " + f"({launch.get('skipped') or 'launch phase produced no runs'})" + ) + return 1 + elif med > a.max_healthz_seconds: + print( + f"::error::startup regression: {med}s median to a healthy port " + f"exceeds the {a.max_healthz_seconds}s budget" + ) + return 1 + return 0 + + +if __name__ == "__main__": + raise SystemExit(main(sys.argv[1:])) diff --git a/studio/backend/auth/authentication.py b/studio/backend/auth/authentication.py index 94df994928..2e9520827e 100644 --- a/studio/backend/auth/authentication.py +++ b/studio/backend/auth/authentication.py @@ -11,11 +11,12 @@ import jwt from .storage import ( API_KEY_PREFIX, + credential_generation, get_jwt_secret, get_user_and_secret, load_jwt_secret, save_refresh_token, - validate_api_key, + validate_api_key_with_credential, verify_refresh_token, ) @@ -54,11 +55,14 @@ def create_access_token( expires_delta: Optional[timedelta] = None, *, desktop: bool = False, + secret: Optional[str] = None, ) -> str: """ Create a signed JWT for the given subject (e.g. username). - Valid across restarts: the signing secret is stored in SQLite. + Valid across restarts: the signing secret is stored in SQLite. Callers that + already verified a credential pass ``secret`` so a rotation landing mid-request + cannot sign the token with the credential that just replaced it. """ to_encode = {"sub": subject} if desktop: @@ -69,7 +73,7 @@ def create_access_token( to_encode.update({"exp": expire}) return jwt.encode( to_encode, - _get_secret_for_subject(subject), + secret if secret is not None else _get_secret_for_subject(subject), algorithm = ALGORITHM, ) @@ -96,15 +100,28 @@ def is_desktop_access_token(token: str) -> bool: return payload.get("sub") == subject and payload.get("desktop") is True -def create_refresh_token(subject: str, *, desktop: bool = False) -> str: +def create_refresh_token( + subject: str, + *, + desktop: bool = False, + secret: Optional[str] = None, +) -> str: """ Create a random refresh token, store its hash in SQLite, and return it. Refresh tokens are opaque (not JWTs); expire after REFRESH_TOKEN_EXPIRE_DAYS. + ``secret`` stamps the token with the credential version the caller verified, + so a rotation cannot leave a token minted from the replaced credential valid. """ token = secrets.token_urlsafe(48) expires_at = datetime.now(timezone.utc) + timedelta(days = REFRESH_TOKEN_EXPIRE_DAYS) - save_refresh_token(token, subject, expires_at.isoformat(), is_desktop = desktop) + save_refresh_token( + token, + subject, + expires_at.isoformat(), + is_desktop = desktop, + secret_gen = credential_generation(secret) if secret is not None else None, + ) return token @@ -137,7 +154,22 @@ def reload_secret() -> None: async def get_current_subject(credentials: HTTPAuthorizationCredentials = Depends(security)) -> str: """Validate JWT and require the password-change flow to be completed.""" - return await _get_current_subject( + subject, _generation = await _get_current_credential( + credentials, + allow_password_change = False, + ) + return subject + + +async def get_current_credential( + credentials: HTTPAuthorizationCredentials = Depends(security), +) -> Tuple[str, Optional[str]]: + """As get_current_subject, but also returns the credential generation. + + For routes that persist a new credential and must not do so on behalf of one + a concurrent reset has revoked. + """ + return await _get_current_credential( credentials, allow_password_change = False, ) @@ -158,10 +190,11 @@ async def get_current_subject_allow_password_change( credentials: HTTPAuthorizationCredentials = Depends(security), ) -> str: """Validate JWT but allow access to the password-change endpoint.""" - return await _get_current_subject( + subject, _generation = await _get_current_credential( credentials, allow_password_change = True, ) + return subject # The literal the examples ship with; pasted unedited more often than a revoked key. @@ -179,21 +212,27 @@ def _invalid_api_key_detail(token: str) -> str: return "Invalid or expired API key" -async def _get_current_subject( +async def _get_current_credential( credentials: HTTPAuthorizationCredentials, *, allow_password_change: bool -) -> str: - """FastAPI dependency: validate the JWT and return the subject. Use on protected routes.""" +) -> Tuple[str, Optional[str]]: + """Validate the bearer and return ``(subject, credential generation)``. + + The generation is the credential version this request actually authenticated + against. Routes that persist new credentials must bind their write to it, or + a reset landing mid-request would bless what it just revoked. + """ token = credentials.credentials # --- API key path (sk-unsloth-...) --- if token.startswith(API_KEY_PREFIX): - username = validate_api_key(token) - if username is None: + verified = validate_api_key_with_credential(token) + if verified is None: raise HTTPException( status_code = status.HTTP_401_UNAUTHORIZED, detail = _invalid_api_key_detail(token), ) - return username + username, secret = verified + return username, credential_generation(secret) # --- JWT path --- subject = _decode_subject_without_verification(token) @@ -224,7 +263,7 @@ async def _get_current_subject( status_code = status.HTTP_403_FORBIDDEN, detail = "Password change required", ) - return subject + return subject, credential_generation(jwt_secret) except jwt.InvalidTokenError: raise HTTPException( status_code = status.HTTP_401_UNAUTHORIZED, diff --git a/studio/backend/auth/storage.py b/studio/backend/auth/storage.py index 5f80ad89a3..6cf4d44834 100644 --- a/studio/backend/auth/storage.py +++ b/studio/backend/auth/storage.py @@ -9,6 +9,7 @@ import ipaddress import os import secrets import sqlite3 +import tempfile import threading from datetime import datetime, timezone from typing import Optional, Tuple @@ -30,6 +31,97 @@ _BOOTSTRAP_PW_PATH = DB_PATH.parent / ".bootstrap_password" _bootstrap_password: Optional[str] = None +def _bootstrap_file_bytes(password: str) -> bytes: + """Exact on-disk form: the secret plus one LF. + + Bytes, not text: text mode writes CRLF on Windows, and `$(cat ...)` strips + the LF but leaves the CR attached to the credential. + """ + return (password + "\n").encode("utf-8") + + +def _persist_bootstrap_password(password: str) -> None: + """Atomically write the bootstrap password 0600, LF terminated on every OS. + + A partial write would destroy the only plaintext recovery credential. + """ + fd, tmp_name = tempfile.mkstemp( + prefix = f".{_BOOTSTRAP_PW_PATH.name}.", dir = _BOOTSTRAP_PW_PATH.parent + ) + try: + with os.fdopen(fd, "wb") as f: + f.write(_bootstrap_file_bytes(password)) + try: + os.chmod(tmp_name, 0o600) + except OSError: + pass + os.replace(tmp_name, _BOOTSTRAP_PW_PATH) + except BaseException: + try: + os.unlink(tmp_name) + except OSError: + pass + raise + + +def _normalise_bootstrap_file(raw: bytes, password: str) -> None: + """Append the LF a pre-newline release left off. + + Append-only, and only when the file is exactly the credential: + clear_bootstrap_password() may unlink or (when unlink fails, notably on + Windows while this descriptor is open) truncate through another descriptor + after we read, so a rewrite could restore revoked plaintext. An append + cannot: worst case is a lone "\\n" over a cleared file, which strips back to + no bootstrap password. Pre-newline releases wrote no terminator at all, so + that is the only shape in the wild; anything else reads fine, since every + reader strips, and is left alone. + """ + if raw != password.encode("utf-8"): + return + + # O_BINARY: without it Windows opens in text mode and turns the LF straight + # back into CRLF, the bug being fixed. + fd = os.open( + _BOOTSTRAP_PW_PATH, + os.O_WRONLY | os.O_APPEND | getattr(os, "O_BINARY", 0), + ) + try: + os.write(fd, b"\n") + try: + os.fchmod(fd, 0o600) + except (AttributeError, OSError): + # fchmod only reached Windows in 3.13. + pass + finally: + os.close(fd) + + +def _read_persisted_bootstrap_password() -> Optional[str]: + """Read the persisted password, normalising the file if it is malformed.""" + if not _BOOTSTRAP_PW_PATH.is_file(): + return None + + # No caller handles a raise, so an unreadable file has to mean "no bootstrap + # password", not a dead backend. We write UTF-8, so undecodable bytes are + # damage whose plaintext is worthless anyway. + try: + raw = _BOOTSTRAP_PW_PATH.read_bytes() + password = raw.decode("utf-8").strip() + except (OSError, UnicodeDecodeError): + return None + if not password: + return None + + # Older releases wrote no terminator; best-effort, a read-only auth dir must + # not fail startup. + if raw != _bootstrap_file_bytes(password): + try: + _normalise_bootstrap_file(raw, password) + except OSError: + pass + return password + + def generate_bootstrap_password() -> str: """Generate a 4-word diceware passphrase and persist it to disk. @@ -43,10 +135,10 @@ def generate_bootstrap_password() -> str: return _bootstrap_password # Persisted from a previous run? - if _BOOTSTRAP_PW_PATH.is_file(): - _bootstrap_password = _BOOTSTRAP_PW_PATH.read_text(encoding = "utf-8").strip() - if _bootstrap_password: - return _bootstrap_password + persisted = _read_persisted_bootstrap_password() + if persisted: + _bootstrap_password = persisted + return _bootstrap_password # First startup: generate a fresh passphrase. import diceware @@ -57,11 +149,7 @@ def generate_bootstrap_password() -> str: # Persist so the same passphrase survives restarts until password change. ensure_dir(_BOOTSTRAP_PW_PATH.parent) - _BOOTSTRAP_PW_PATH.write_text(_bootstrap_password, encoding = "utf-8") - try: - os.chmod(_BOOTSTRAP_PW_PATH, 0o600) - except OSError: - pass + _persist_bootstrap_password(_bootstrap_password) return _bootstrap_password @@ -72,13 +160,14 @@ def get_bootstrap_password() -> Optional[str]: def _load_bootstrap_password() -> Optional[str]: - """Load an existing bootstrap password without creating one.""" + """Load an existing bootstrap password without creating one. + + Upgrades take this path, not generate_bootstrap_password() + (ensure_default_admin short-circuits once the admin row exists), so it has + to normalise too. + """ global _bootstrap_password - _bootstrap_password = None - if _BOOTSTRAP_PW_PATH.is_file(): - bootstrap_password = _BOOTSTRAP_PW_PATH.read_text(encoding = "utf-8").strip() - if bootstrap_password: - _bootstrap_password = bootstrap_password + _bootstrap_password = _read_persisted_bootstrap_password() return _bootstrap_password @@ -97,7 +186,7 @@ def clear_bootstrap_password() -> None: # Removal failed (Windows AV, read-only auth dir). The hash is already # committed, so don't fail the change -- but truncate the file so its # stale plaintext can't be re-seeded by generate_bootstrap_password() - # if a later reset-password deletes auth.db and re-validates it. + # if auth.db is ever recreated. try: _BOOTSTRAP_PW_PATH.write_text("", encoding = "utf-8") cleared = True @@ -132,6 +221,31 @@ def _hash_token(token: str) -> str: return hashlib.sha256(token.encode("utf-8")).hexdigest() +class CredentialRotated(Exception): + """A password reset revoked the credential this request authenticated with.""" + + +def credential_generation(jwt_secret: str) -> str: + """Marker for the credential version a refresh token was issued under. + + Every password change rotates ``jwt_secret``, so a token stamped with the + previous one is rejected even if it was inserted after the revoking DELETE. + """ + return hashlib.sha256(jwt_secret.encode("utf-8")).hexdigest() + + +def _current_secret(conn: sqlite3.Connection, username: str) -> Optional[str]: + row = conn.execute( + "SELECT jwt_secret FROM auth_user WHERE username = ?", (username,) + ).fetchone() + return row["jwt_secret"] if row else None + + +def _current_generation(conn: sqlite3.Connection, username: str) -> Optional[str]: + secret = _current_secret(conn, username) + return credential_generation(secret) if secret is not None else None + + def get_connection() -> sqlite3.Connection: """Get a connection to the auth database, creating tables if needed.""" ensure_dir(DB_PATH.parent) @@ -175,7 +289,8 @@ def get_connection() -> sqlite3.Connection: token_hash TEXT NOT NULL, username TEXT NOT NULL, expires_at TEXT NOT NULL, - is_desktop INTEGER NOT NULL DEFAULT 0 + is_desktop INTEGER NOT NULL DEFAULT 0, + secret_gen TEXT ); """ ) @@ -214,6 +329,8 @@ def get_connection() -> sqlite3.Connection: refresh_columns = {row["name"] for row in conn.execute("PRAGMA table_info(refresh_tokens)")} if "is_desktop" not in refresh_columns: conn.execute("ALTER TABLE refresh_tokens ADD COLUMN is_desktop INTEGER NOT NULL DEFAULT 0") + if "secret_gen" not in refresh_columns: + conn.execute("ALTER TABLE refresh_tokens ADD COLUMN secret_gen TEXT") conn.commit() return conn @@ -587,12 +704,22 @@ def update_password( new_password: str, *, revoke_refresh_tokens: bool = False, -) -> bool: + expect_password_hash: Optional[str] = None, +) -> Optional[str]: """Update password, clear first-login requirement, rotate JWT secret. + Returns the new JWT secret, or None when nothing was updated. Callers that + mint tokens for the caller must sign with the returned secret: re-reading it + would pick up a reset that landed between this commit and the mint. + ``revoke_refresh_tokens`` deletes the user's refresh tokens in the SAME transaction: a separate delete could fail after the password commit and leave a pre-change token still able to mint access tokens. + + ``expect_password_hash`` makes the write conditional on the credential the + caller verified still being current, so a request that checked the old + password cannot overwrite a reset that landed while it was in flight. + Returns False when the credential moved underneath it. """ from .hashing import hash_password @@ -600,21 +727,32 @@ def update_password( jwt_secret = secrets.token_urlsafe(64) conn = get_connection() try: - cursor = conn.execute( - """ - UPDATE auth_user - SET password_salt = ?, password_hash = ?, jwt_secret = ?, must_change_password = 0 - WHERE username = ? - """, - (salt, pwd_hash, jwt_secret, username), - ) + if expect_password_hash is None: + cursor = conn.execute( + """ + UPDATE auth_user + SET password_salt = ?, password_hash = ?, jwt_secret = ?, must_change_password = 0 + WHERE username = ? + """, + (salt, pwd_hash, jwt_secret, username), + ) + else: + cursor = conn.execute( + """ + UPDATE auth_user + SET password_salt = ?, password_hash = ?, jwt_secret = ?, must_change_password = 0 + WHERE username = ? AND password_hash = ? + """, + (salt, pwd_hash, jwt_secret, username, expect_password_hash), + ) if revoke_refresh_tokens and cursor.rowcount > 0: conn.execute("DELETE FROM refresh_tokens WHERE username = ?", (username,)) conn.commit() if cursor.rowcount > 0: clear_bootstrap_password() clear_desktop_secret() - return cursor.rowcount > 0 + return jwt_secret + return None finally: conn.close() @@ -625,35 +763,49 @@ def save_refresh_token( expires_at: str, *, is_desktop: bool = False, + secret_gen: Optional[str] = None, ) -> None: """ Store a hashed refresh token with its associated username and expiry. + + ``secret_gen`` binds the token to a credential version; it defaults to the + current one, and callers that already verified a credential must pass the + version they verified rather than let this re-read a rotated one. """ token_hash = _hash_token(token) conn = get_connection() try: + if secret_gen is None: + secret_gen = _current_generation(conn, username) conn.execute( """ - INSERT INTO refresh_tokens (token_hash, username, expires_at, is_desktop) - VALUES (?, ?, ?, ?) + INSERT INTO refresh_tokens (token_hash, username, expires_at, is_desktop, secret_gen) + VALUES (?, ?, ?, ?, ?) """, - (token_hash, username, expires_at, int(is_desktop)), + (token_hash, username, expires_at, int(is_desktop), secret_gen), ) conn.commit() finally: conn.close() -def consume_refresh_token(token: str) -> Optional[Tuple[str, bool]]: +def consume_refresh_token(token: str) -> Optional[Tuple[str, bool, str]]: """Atomically validate-and-delete a refresh token for single-use rotation. DELETE RETURNING fuses validate and delete into one statement so two - concurrent refresh requests cannot both consume the same token. + concurrent refresh requests cannot both consume the same token. Returns + ``(username, is_desktop, jwt_secret)``; the caller must mint the replacement + tokens against that secret so a rotation landing mid-refresh cannot issue a + post-rotation session from a pre-rotation token. """ token_hash = _hash_token(token) now = datetime.now(timezone.utc).isoformat() conn = get_connection() try: + # One transaction with the delete: an unstamped legacy row has no + # generation to compare, so reading the credential after committing would + # hand a reset's new secret to a token issued before it. + conn.execute("BEGIN IMMEDIATE") conn.execute( "DELETE FROM refresh_tokens WHERE expires_at < ?", (now,), @@ -662,15 +814,21 @@ def consume_refresh_token(token: str) -> Optional[Tuple[str, bool]]: """ DELETE FROM refresh_tokens WHERE token_hash = ? AND expires_at >= ? - RETURNING username, is_desktop + RETURNING username, is_desktop, secret_gen """, (token_hash, now), ) row = cur.fetchone() - conn.commit() if row is None: + conn.commit() return None - return row["username"], bool(row["is_desktop"]) + secret = _current_secret(conn, row["username"]) + conn.commit() + if secret is None: + return None + if row["secret_gen"] is not None and row["secret_gen"] != credential_generation(secret): + return None + return row["username"], bool(row["is_desktop"]), secret finally: conn.close() @@ -694,7 +852,7 @@ def verify_refresh_token(token: str) -> Optional[Tuple[str, bool]]: cur = conn.execute( """ - SELECT id, username, expires_at, is_desktop FROM refresh_tokens + SELECT id, username, expires_at, is_desktop, secret_gen FROM refresh_tokens WHERE token_hash = ? """, (token_hash,), @@ -703,6 +861,13 @@ def verify_refresh_token(token: str) -> Optional[Tuple[str, bool]]: if row is None: return None + if row["secret_gen"] is not None and row["secret_gen"] != _current_generation( + conn, row["username"] + ): + conn.execute("DELETE FROM refresh_tokens WHERE id = ?", (row["id"],)) + conn.commit() + return None + # Check expiry expires_at = datetime.fromisoformat(row["expires_at"]) if datetime.now(timezone.utc) > expires_at: @@ -747,30 +912,41 @@ def create_desktop_secret() -> str: conn.close() -def validate_desktop_secret(raw_secret: str) -> Optional[str]: - """Return the real admin username when the desktop secret matches.""" +def validate_desktop_secret_with_credential(raw_secret: str) -> Optional[Tuple[str, str]]: + """Validate the desktop secret and return ``(username, jwt_secret)``. + + Both reads share one transaction so the returned secret is the credential + version the desktop secret was checked against; a reset landing mid-request + then invalidates the tokens minted from it rather than blessing them. + """ if not raw_secret.startswith(DESKTOP_SECRET_PREFIX): return None - if get_user_and_secret(DEFAULT_ADMIN_USERNAME) is None: - return None secret_hash = _pbkdf2_desktop_secret(raw_secret) conn = get_connection() try: - cur = conn.execute( + conn.execute("BEGIN") + row = conn.execute( "SELECT value FROM app_secrets WHERE key = ?", (_DESKTOP_SECRET_HASH_KEY,), - ) - row = cur.fetchone() - if row is None: + ).fetchone() + if row is None or not secrets.compare_digest(row["value"], secret_hash): return None - if not secrets.compare_digest(row["value"], secret_hash): + jwt_secret = _current_secret(conn, DEFAULT_ADMIN_USERNAME) + if jwt_secret is None: return None - return DEFAULT_ADMIN_USERNAME + return DEFAULT_ADMIN_USERNAME, jwt_secret finally: + conn.rollback() conn.close() +def validate_desktop_secret(raw_secret: str) -> Optional[str]: + """Return the real admin username when the desktop secret matches.""" + verified = validate_desktop_secret_with_credential(raw_secret) + return verified[0] if verified else None + + def clear_desktop_secret() -> None: """Remove backend-side desktop auth state.""" conn = get_connection() @@ -796,6 +972,7 @@ def create_api_key( name: str, expires_at: Optional[str] = None, internal: bool = False, + expect_gen: Optional[str] = None, ) -> Tuple[str, dict]: """Create a new API key for *username*. @@ -804,6 +981,10 @@ def create_api_key( Pass ``internal=True`` for keys minted by workflows (e.g. data-recipe runs) that should not appear in user-facing key listings. + + ``expect_gen`` ties the insert to the credential generation the request + authenticated under, so a session revoked by a concurrent password reset + cannot mint a key that outlives it. Raises ``CredentialRotated`` if it moved. """ raw_key = API_KEY_PREFIX + secrets.token_hex(16) key_hash = _pbkdf2_api_key(raw_key) @@ -812,6 +993,12 @@ def create_api_key( conn = get_connection() try: + if expect_gen is not None: + conn.execute("BEGIN IMMEDIATE") + if _current_generation(conn, username) != expect_gen: + raise CredentialRotated( + "The credential this request authenticated with was revoked." + ) conn.execute( """ INSERT INTO api_keys (username, key_prefix, key_hash, name, created_at, expires_at, is_internal) @@ -900,15 +1087,25 @@ def revoke_internal_api_key(key_id: int) -> bool: def validate_api_key(raw_key: str) -> Optional[str]: - """Validate *raw_key* and return the owning username, or ``None``. + """Validate *raw_key* and return the owning username, or ``None``.""" + verified = validate_api_key_with_credential(raw_key) + return verified[0] if verified else None - Also updates ``last_used_at`` on success. + +def validate_api_key_with_credential(raw_key: str) -> Optional[Tuple[str, str]]: + """Validate *raw_key* and return ``(username, jwt_secret)``, or ``None``. + + Also updates ``last_used_at`` on success. The key check and the credential + read share one write transaction, so the returned version is the one the key + was actually valid under: a reset committing right after cannot have its new + generation handed to a request the key it revoked authenticated. """ cache_id = _api_key_cache_id(raw_key) cached_hash = _api_key_hash_cache.get(cache_id) key_hash = cached_hash if cached_hash is not None else _pbkdf2_api_key(raw_key) conn = get_connection() try: + conn.execute("BEGIN IMMEDIATE") cur = conn.execute( "SELECT id, username, is_active, expires_at FROM api_keys WHERE key_hash = ?", (key_hash,), @@ -928,11 +1125,15 @@ def validate_api_key(raw_key: str) -> Optional[str]: expires = datetime.fromisoformat(row["expires_at"]) if datetime.now(timezone.utc) > expires: return None + secret = _current_secret(conn, row["username"]) + if secret is None: + return None conn.execute( "UPDATE api_keys SET last_used_at = ? WHERE id = ?", (datetime.now(timezone.utc).isoformat(), row["id"]), ) conn.commit() - return row["username"] + return row["username"], secret finally: + conn.rollback() conn.close() diff --git a/studio/backend/cloudflare_tunnel.py b/studio/backend/cloudflare_tunnel.py index 78fce0c70a..f7967e2faa 100644 --- a/studio/backend/cloudflare_tunnel.py +++ b/studio/backend/cloudflare_tunnel.py @@ -310,6 +310,7 @@ class CloudflareTunnel: stderr = subprocess.STDOUT, stdin = subprocess.DEVNULL, text = True, + encoding = "utf-8", errors = "replace", bufsize = 1, **_windows_hidden_kwargs(), diff --git a/studio/backend/core/data_recipe/local_callable_validators.py b/studio/backend/core/data_recipe/local_callable_validators.py index ffc81669ae..143895d781 100644 --- a/studio/backend/core/data_recipe/local_callable_validators.py +++ b/studio/backend/core/data_recipe/local_callable_validators.py @@ -257,6 +257,8 @@ def _run_oxc_batch( cwd = str(_OXC_TOOL_DIR), input = json.dumps(payload), text = True, + encoding = "utf-8", + errors = "replace", capture_output = True, check = False, env = env, diff --git a/studio/backend/core/inference/inference.py b/studio/backend/core/inference/inference.py index 0af37e627f..e78bf1be8d 100644 --- a/studio/backend/core/inference/inference.py +++ b/studio/backend/core/inference/inference.py @@ -567,7 +567,7 @@ class InferenceBackend: _meta_path = Path(config.path) / "export_metadata.json" try: if _meta_path.exists(): - _meta = json.loads(_meta_path.read_text(encoding = "utf-8")) + _meta = json.loads(_meta_path.read_text(encoding = "utf-8-sig")) if _meta.get("base_model"): processor_source = _meta["base_model"] except Exception: diff --git a/studio/backend/core/inference/llama_cpp.py b/studio/backend/core/inference/llama_cpp.py index 5f1db5eeee..43ee67e093 100644 --- a/studio/backend/core/inference/llama_cpp.py +++ b/studio/backend/core/inference/llama_cpp.py @@ -85,6 +85,7 @@ from core.tool_healing import ( strip_outside_think, ) from utils.native_path_leases import child_env_without_native_path_secret +from utils.child_stdio import utf8_child_env from utils.hf_xet_fallback import hf_hub_download_with_xet_fallback from utils.subprocess_compat import ( windows_hidden_subprocess_kwargs as _windows_hidden_subprocess_kwargs, @@ -581,7 +582,7 @@ def _load_swa_cache() -> dict: if _SWA_CACHE is not None: return _SWA_CACHE try: - with open(_swa_cache_path(), encoding = "utf-8") as f: + with open(_swa_cache_path(), encoding = "utf-8-sig") as f: _SWA_CACHE = json.load(f) if not isinstance(_SWA_CACHE, dict): _SWA_CACHE = {} @@ -632,7 +633,7 @@ def _fetch_swa_entry_from_hf(repo_id: str) -> Optional[object]: repo_type = "model", cache_dir = active_hf_hub_cache(), ) - with open(cfg_path, encoding = "utf-8") as f: + with open(cfg_path, encoding = "utf-8-sig") as f: cfg = json.load(f) except Exception: return None @@ -3087,6 +3088,7 @@ class LlamaCppBackend: [bin_path, "--help"], capture_output = True, text = True, + encoding = "utf-8", errors = "replace", timeout = 10, check = False, @@ -3692,6 +3694,8 @@ class LlamaCppBackend: ], capture_output = True, text = True, + encoding = "utf-8", + errors = "replace", timeout = 10, env = child_env_without_native_path_secret(), **_windows_hidden_subprocess_kwargs(), @@ -3806,7 +3810,7 @@ class LlamaCppBackend: encoding = "utf-8", errors = "replace", timeout = 15, - env = env, + env = utf8_child_env(env), **_windows_hidden_subprocess_kwargs(), ) if result.returncode != 0: @@ -5556,7 +5560,9 @@ class LlamaCppBackend: stdout = subprocess.PIPE, stderr = subprocess.STDOUT, text = True, - env = env, + encoding = "utf-8", + errors = "replace", + env = utf8_child_env(env), **_windows_hidden_subprocess_kwargs(), **_child_popen_kwargs(), ) @@ -6770,6 +6776,8 @@ class LlamaCppBackend: stdout = subprocess.PIPE, stderr = subprocess.STDOUT, text = True, + encoding = "utf-8", + errors = "replace", env = env, **_windows_hidden_subprocess_kwargs(), **_child_popen_kwargs(), @@ -8841,6 +8849,8 @@ class LlamaCppBackend: stdout = subprocess.PIPE, stderr = subprocess.STDOUT, text = True, + encoding = "utf-8", + errors = "replace", env = env, **_windows_hidden_subprocess_kwargs(), **_child_popen_kwargs(), @@ -10357,6 +10367,8 @@ class LlamaCppBackend: ["pgrep", "-a", "-f", "llama-server"], capture_output = True, text = True, + encoding = "utf-8", + errors = "replace", timeout = 5, env = child_env_without_native_path_secret(), ) diff --git a/studio/backend/core/inference/worker.py b/studio/backend/core/inference/worker.py index 3f32b3bd57..f208183300 100644 --- a/studio/backend/core/inference/worker.py +++ b/studio/backend/core/inference/worker.py @@ -151,7 +151,7 @@ def _resolve_lora_4bit(mc, load_in_4bit: bool) -> bool: import json try: - with open(adapter_cfg_path, encoding = "utf-8") as f: + with open(adapter_cfg_path, encoding = "utf-8-sig") as f: adapter_cfg = json.load(f) training_method = adapter_cfg.get("unsloth_training_method") if training_method == "lora" and load_in_4bit: @@ -963,7 +963,7 @@ def run_inference_process( if _local_adapter_cfg.is_file(): try: _lora_base = ( - _json.loads(_local_adapter_cfg.read_text(encoding = "utf-8")).get( + _json.loads(_local_adapter_cfg.read_text(encoding = "utf-8-sig")).get( "base_model_name_or_path" ) or None diff --git a/studio/backend/core/rag/embed_llama_server.py b/studio/backend/core/rag/embed_llama_server.py index facd989b27..b3ac62e520 100644 --- a/studio/backend/core/rag/embed_llama_server.py +++ b/studio/backend/core/rag/embed_llama_server.py @@ -103,6 +103,8 @@ class LlamaServerBackend: [binary, "--help"], capture_output = True, text = True, + encoding = "utf-8", + errors = "replace", timeout = 30, **windows_hidden_subprocess_kwargs(), ) @@ -331,6 +333,8 @@ class LlamaServerBackend: stdout = subprocess.PIPE, stderr = subprocess.STDOUT, text = True, + encoding = "utf-8", + errors = "replace", env = env, **windows_hidden_subprocess_kwargs(), **child_popen_kwargs(), diff --git a/studio/backend/core/rag/embeddings.py b/studio/backend/core/rag/embeddings.py index c86c0d3c51..95b8a866b2 100644 --- a/studio/backend/core/rag/embeddings.py +++ b/studio/backend/core/rag/embeddings.py @@ -100,7 +100,7 @@ def _st_module_subdirs(name: str, token: str | None) -> tuple[str, ...]: path = Path(normalize_path(name)).expanduser() / "modules.json" if not path.is_file(): return () - data = json.loads(path.read_text(encoding = "utf-8")) + data = json.loads(path.read_text(encoding = "utf-8-sig")) else: from huggingface_hub import hf_hub_download from huggingface_hub.utils import EntryNotFoundError @@ -115,7 +115,7 @@ def _st_module_subdirs(name: str, token: str | None) -> tuple[str, ...]: ) except EntryNotFoundError: return () - data = json.loads(open(local, encoding = "utf-8").read()) + data = json.loads(open(local, encoding = "utf-8-sig").read()) subdirs = [] for module in data or (): sub = str((module or {}).get("path", "")).strip().strip("/") diff --git a/studio/backend/core/training/worker.py b/studio/backend/core/training/worker.py index baf6329dae..b5fb5d224e 100644 --- a/studio/backend/core/training/worker.py +++ b/studio/backend/core/training/worker.py @@ -43,6 +43,7 @@ if sys.platform.startswith("linux") and "HSA_ENABLE_DXG_DETECTION" not in os.env pass logger = get_logger(__name__) +from utils.child_stdio import utf8_child_env from utils.hardware import apply_gpu_ids from utils.training_runs import build_default_output_dir_name from utils.wheel_utils import ( @@ -385,6 +386,10 @@ def _install_package_wheel_first( "stdout": _sp.PIPE, "stderr": _sp.STDOUT, "text": True, + "encoding": "utf-8", + "errors": "replace", + # Make the Python child emit the UTF-8 we decode above. + "env": utf8_child_env(), } if is_hip: _run_kwargs["timeout"] = 1800 @@ -606,6 +611,9 @@ def _ensure_flash_linear_attention_unconditional(event_queue: Any) -> bool: stdout = _sp.PIPE, stderr = _sp.STDOUT, text = True, + encoding = "utf-8", + errors = "replace", + env = utf8_child_env(), timeout = _TILELANG_INSTALL_TIMEOUT_S, ) except _sp.TimeoutExpired: @@ -849,6 +857,9 @@ def _run_pip(cmd: list[str], event_queue: Any, label: str) -> bool: stdout = _sp.PIPE, stderr = _sp.STDOUT, text = True, + encoding = "utf-8", + errors = "replace", + env = utf8_child_env(), timeout = _TILELANG_INSTALL_TIMEOUT_S, ) except _sp.TimeoutExpired: diff --git a/studio/backend/hub/services/models/ollama.py b/studio/backend/hub/services/models/ollama.py index 56275c22a9..da30f7e98c 100644 --- a/studio/backend/hub/services/models/ollama.py +++ b/studio/backend/hub/services/models/ollama.py @@ -215,7 +215,7 @@ def _ollama_model_info_from_manifest( return None try: - manifest = json.loads(tag_file.read_text(encoding = "utf-8")) + manifest = json.loads(tag_file.read_text(encoding = "utf-8-sig")) except (json.JSONDecodeError, OSError, UnicodeDecodeError) as e: logger.debug("Skipping unreadable/invalid Ollama manifest %s: %s", tag_file, e) return None @@ -228,7 +228,7 @@ def _ollama_model_info_from_manifest( config_blob = _ollama_blob_path(blobs_dir, config_digest) if config_blob is not None and _safe_is_file(config_blob): try: - cfg = json.loads(config_blob.read_text(encoding = "utf-8")) + cfg = json.loads(config_blob.read_text(encoding = "utf-8-sig")) model_type = cfg.get("model_type", "") file_type = cfg.get("file_type", "") except (json.JSONDecodeError, OSError, UnicodeDecodeError) as e: diff --git a/studio/backend/hub/utils/download_registry.py b/studio/backend/hub/utils/download_registry.py index 39c27208b1..760ef6b01c 100644 --- a/studio/backend/hub/utils/download_registry.py +++ b/studio/backend/hub/utils/download_registry.py @@ -464,6 +464,8 @@ def _read_marker_value(marker: Path) -> Optional[str]: return None value = marker.read_text(encoding = "utf-8").strip() except (OSError, UnicodeDecodeError): + # UnicodeDecodeError is a ValueError, so it would escape and abort + # prepare_cache_for_transport. An unknown value just purges and restarts. return None return value if value in VALID_TRANSPORTS else None diff --git a/studio/backend/loggers/config.py b/studio/backend/loggers/config.py index 688d3c7ebe..57cf7cecd6 100644 --- a/studio/backend/loggers/config.py +++ b/studio/backend/loggers/config.py @@ -42,8 +42,12 @@ class LogConfig: log_level_name = os.getenv("LOG_LEVEL", "INFO").upper() log_level = getattr(logging, log_level_name, logging.INFO) - if sys.platform == "win32": - for stream in (sys.stdout, sys.stderr): + # Non-ASCII on a non-UTF-8 stream raises UnicodeEncodeError (Windows, + # LANG=C), so key off the stream, not the platform. + for stream in (sys.stdout, sys.stderr): + if getattr(stream, "encoding", "") and not str(stream.encoding).lower().replace( + "-", "" + ).startswith("utf8"): if hasattr(stream, "reconfigure"): try: stream.reconfigure(encoding = "utf-8", errors = "replace") diff --git a/studio/backend/main.py b/studio/backend/main.py index cadcf33bca..4c5cf5b4a5 100644 --- a/studio/backend/main.py +++ b/studio/backend/main.py @@ -347,6 +347,7 @@ from utils.update_status import ( get_studio_install_source_status, get_studio_update_status, ) +from utils.changelog import get_release_notes, is_supported_version_query from utils.studio_version import get_studio_version from utils.api_errors import install_api_error_handlers @@ -1154,6 +1155,18 @@ def studio_update_status(_current_subject: str = Depends(get_current_subject)): return get_studio_update_status(UNSLOTH_VERSION) +@app.get("/api/studio/release-notes") +def studio_release_notes( + version: str = Query(..., max_length = 64), + refresh: bool = Query(False), + _current_subject: str = Depends(get_current_subject), +): + """Return CHANGELOG.md notes for exactly `version` (never a nearby one).""" + if not is_supported_version_query(version): + raise HTTPException(status_code = 422, detail = "Invalid version.") + return get_release_notes(version, refresh = refresh) + + @app.get( "/api/studio/download-transport-capabilities", response_model = TransportCapabilities, diff --git a/studio/backend/plugins/data-designer-github-repo-seed/src/data_designer_github_repo_seed/scraper_impl/state_store.py b/studio/backend/plugins/data-designer-github-repo-seed/src/data_designer_github_repo_seed/scraper_impl/state_store.py index b4c226136b..b059fad7ff 100644 --- a/studio/backend/plugins/data-designer-github-repo-seed/src/data_designer_github_repo_seed/scraper_impl/state_store.py +++ b/studio/backend/plugins/data-designer-github-repo-seed/src/data_designer_github_repo_seed/scraper_impl/state_store.py @@ -6,10 +6,93 @@ from __future__ import annotations import json +import locale import os import threading from pathlib import Path -from typing import Any, Dict +from typing import Any, Dict, NamedTuple + + +def _locale_encoding() -> str: + """The codepage a pre-UTF-8 release here would have written, or "". + + Empty on a UTF-8 host, where there is no codepage to attribute the file to. + """ + try: + preferred = locale.getencoding() + except AttributeError: # Python < 3.11 + preferred = locale.getpreferredencoding(False) + if preferred.lower().replace("-", "").replace("_", "") == "utf8": + return "" + return preferred + + +# Trail bytes can land on JSON punctuation, so a single-byte fallback misreads these. +_DOUBLE_BYTE_ENCODINGS = ("cp932", "cp936", "cp949", "cp950") + + +def _parse(raw: bytes, encoding: str) -> Any: + """Parse one JSON document under *encoding*, or None if it does not. + + RecursionError is a RuntimeError, so nesting json.loads will not descend is + the one parse failure the other three miss. Both callers run this outside + any further handler, so it has to answer None here or a single damaged + record aborts the scraper at startup instead of being skipped. + """ + try: + return json.loads(raw.decode(encoding)) + except (UnicodeDecodeError, LookupError, ValueError, RecursionError): + return None + + +class _Reading(NamedTuple): + as_utf8: Any + as_legacy: Any + + +def _read_line(raw: bytes, codepage: str) -> _Reading: + """Read one line as UTF-8 and as a codepage, for dedup keys only. + + Requiring valid JSON, not merely a successful decode, is what separates a + genuine legacy record from a half-written UTF-8 one: a torn multibyte + character decodes under cp1252 but leaves the JSON unterminated. Some byte + strings parse both ways, e.g. cp1251 ``Р°`` is ``D0 B0``, which is also + UTF-8 ``а``. + + The codepage reading is never authoritative, because the file's own encoding + cannot be recovered from its bytes. Reading a cp1251 shard on a cp1252 + machine turns ``Привет`` into ``Ïðèâåò`` and every byte of it decodes + cleanly, so a successful decode proves nothing about who wrote it. It is + used only to recover the dedup keys, which are ASCII ids and come back the + same under any of these, so the first reading that parses will do. + + That is also why several are tried. latin-1 alone mangles the double-byte + codepages: cp932 ``表`` is ``95 5C``, and latin-1 turns the trail byte into + a JSON backslash, so the record fails to parse and its id is forgotten. + """ + as_utf8 = _parse(raw, "utf-8") + # A record that reads as UTF-8 needs no second reading: re-parsing cost 2.8x on a + # 76 MB shard, and these reach gigabytes. Only a dict, since key lookup falls + # through to the codepage when UTF-8 yields none. + if isinstance(as_utf8, dict): + return _Reading(as_utf8, None) + for encoding in (codepage, "latin-1", *_DOUBLE_BYTE_ENCODINGS): + if not encoding: + continue + as_legacy = _parse(raw, encoding) + if as_legacy is not None: + return _Reading(as_utf8, as_legacy) + return _Reading(as_utf8, None) + + +class _Scan(NamedTuple): + """What a pass over an existing shard established about it.""" + + legacy: bool # enough evidence to trust the codepage reading's keys + readable: bool + saw_non_ascii: bool # some line's meaning depends on the encoding + utf8_keys: set # keys from lines UTF-8 could read + legacy_keys: set # keys only the codepage reading yields class StateStore: @@ -18,12 +101,19 @@ class StateStore: self.path.parent.mkdir(parents = True, exist_ok = True) self._lock = threading.Lock() self._data: Dict[str, Any] = {} + # Read whole, and UTF-8 only unlike the shards below: a checkpoint holds + # nothing but base64 cursors and booleans, so a codepage retry could only ever + # add non-ASCII. That would resume on a mojibaked cursor, which GitHub rejects + # with INVALID_CURSOR_ARGUMENTS, and the empty page it returns marks the stream + # done and skips the rest for good. Dropping a damaged checkpoint re-scrapes + # from the first page, which the writers dedup. if self.path.exists(): try: - with self.path.open(encoding = "utf-8") as f: - self._data = json.load(f) - except Exception: - self._data = {} + raw = self.path.read_bytes() + except OSError: + raw = b"" + data = _parse(raw, "utf-8") + self._data = data if isinstance(data, dict) else {} def get( self, @@ -63,24 +153,83 @@ class JsonlWriter: self.path = Path(path) self.path.parent.mkdir(parents = True, exist_ok = True) self._lock = threading.Lock() - self._fh = self.path.open("a", buffering = 1, encoding = "utf-8") self._count_seen_keys: set[str] = set() - # Preload seen keys for dedup across resumes + self._codepage = _locale_encoding() + self._ensure_ascii = False + encoding = "utf-8" if self.path.exists() and self.path.stat().st_size > 0: - try: - # No guess is safe for a file an older build wrote in the - # operator's locale, so read past whatever will not decode. - with self.path.open(encoding = "utf-8", errors = "replace") as f: - for line in f: - try: - obj = json.loads(line) - k = self._key(obj) - if k is not None: - self._count_seen_keys.add(k) - except Exception: - pass - except Exception: - pass + scan = self._scan_existing() + self._count_seen_keys = scan.utf8_keys + if scan.legacy: + self._count_seen_keys |= scan.legacy_keys + if scan.saw_non_ascii or not scan.readable: + # Never convert: the writing encoding is unrecoverable and guessing + # mojibakes the records. Pure ASCII appends store identically under + # every codepage, and json.loads turns the \uXXXX escapes back. + encoding = "ascii" + self._ensure_ascii = True + self._fh = self.path.open("a", buffering = 1, encoding = encoding, errors = "strict") + + def _scan_existing(self) -> _Scan: + """Read the shard once to recover dedup keys and judge its encoding. + + Line by line: these shards reach gigabytes on a large scrape, so neither + the bytes nor the decoded text are held whole. + + The verdict weighs the whole file. Each line with non-ASCII bytes votes: + one that parses only under the codepage is evidence of a legacy shard, + one that parses as UTF-8 is evidence against, since arbitrary codepage + text almost never forms valid multibyte UTF-8. A single corrupt byte in + a healthy shard therefore cannot outvote the records around it, and a + genuinely legacy shard has a legacy vote on every line that carries an + umlaut. + + More than one such line is required, because a single one is genuinely + undecidable: a legacy record holding one accented character and an ASCII + record holding one stray byte are the same shape. Reading it as damage + risks a duplicate; reading it as legacy marks an unreadable record seen + and blocks the retry that would replace it, losing it for good. Only one + of those is recoverable. + + The verdict only picks which reading supplies the dedup keys. The file + itself is never rewritten either way, so a wrong answer costs at most a + duplicate, never a corrupted record. + """ + legacy_votes = 0 + utf8_votes = 0 + saw_non_ascii = False + utf8_keys: set[str] = set() + legacy_keys: set[str] = set() + try: + with self.path.open("rb") as handle: + for raw in handle: + line = raw.strip() + reading = _read_line(line, self._codepage) + # ASCII reads the same everywhere: no vote, no constraint. + if not line.isascii(): + saw_non_ascii = True + if reading.as_utf8 is None and reading.as_legacy is not None: + legacy_votes += 1 + elif reading.as_utf8 is not None: + utf8_votes += 1 + # Kept apart so a damaged line does not block its own retry. + if isinstance(reading.as_utf8, dict): + key = self._key(reading.as_utf8) + if key is not None: + utf8_keys.add(key) + elif isinstance(reading.as_legacy, dict): + key = self._key(reading.as_legacy) + if key is not None: + legacy_keys.add(key) + except OSError: + return _Scan(False, False, False, utf8_keys, legacy_keys) + return _Scan( + legacy_votes > 1 and legacy_votes > utf8_votes, + True, + saw_non_ascii, + utf8_keys, + legacy_keys, + ) def _key(self, obj: dict) -> str | None: for k in ("id", "node_id", "number", "sha", "url"): @@ -99,7 +248,7 @@ class JsonlWriter: return False if k is not None: self._count_seen_keys.add(k) - self._fh.write(json.dumps(obj, default = str, ensure_ascii = False)) + self._fh.write(json.dumps(obj, default = str, ensure_ascii = self._ensure_ascii)) self._fh.write("\n") self._fh.flush() return True diff --git a/studio/backend/plugins/data-designer-unstructured-seed/src/data_designer_unstructured_seed/impl.py b/studio/backend/plugins/data-designer-unstructured-seed/src/data_designer_unstructured_seed/impl.py index ce0c88e5bf..825b050e07 100644 --- a/studio/backend/plugins/data-designer-unstructured-seed/src/data_designer_unstructured_seed/impl.py +++ b/studio/backend/plugins/data-designer-unstructured-seed/src/data_designer_unstructured_seed/impl.py @@ -30,6 +30,8 @@ class UnstructuredSeedReader(SeedReader[UnstructuredSeedSource]): meta = json_mod.loads(meta_path.read_text(encoding = "utf-8")) orig_name = meta.get("original_filename", path_obj.name) except (json_mod.JSONDecodeError, OSError, UnicodeDecodeError): + # Undecodable metadata is as malformed as invalid JSON, so + # fall back to the file's own name rather than abort the seed. pass file_entries.append((path_obj, orig_name)) diff --git a/studio/backend/routes/auth.py b/studio/backend/routes/auth.py index 1acc48e3a3..fe2f09fcd9 100644 --- a/studio/backend/routes/auth.py +++ b/studio/backend/routes/auth.py @@ -31,6 +31,7 @@ from auth import storage, hashing from auth.authentication import ( create_access_token, create_refresh_token, + get_current_credential, get_current_subject, get_current_subject_allow_password_change, refresh_access_token, @@ -399,7 +400,7 @@ async def login(payload: AuthLoginRequest, request: Request) -> Token: detail = f"Incorrect password. To reset it, run this in your terminal: {_reset_password_command()}", ) - salt, pwd_hash, _jwt_secret, must_change_password = record + salt, pwd_hash, jwt_secret, must_change_password = record if not hashing.verify_password(payload.password, salt, pwd_hash): _record_login_failure(key) raise HTTPException( @@ -409,8 +410,10 @@ async def login(payload: AuthLoginRequest, request: Request) -> Token: _clear_login_bucket(key) _clear_login_bucket(unknown_key) - access_token = create_access_token(subject = payload.username) - refresh_token = create_refresh_token(subject = payload.username) + # Issue against the credential version just verified, not whatever is in the DB + # now: a concurrent reset-password must not hand this login a post-reset session. + access_token = create_access_token(subject = payload.username, secret = jwt_secret) + refresh_token = create_refresh_token(subject = payload.username, secret = jwt_secret) return Token( access_token = access_token, refresh_token = refresh_token, @@ -438,16 +441,17 @@ async def logout( @router.post("/desktop-login", response_model = Token) async def desktop_login(payload: DesktopLoginRequest) -> Token: """Exchange a local desktop secret for normal admin-subject tokens.""" - username = storage.validate_desktop_secret(payload.secret) - if username is None: + verified = storage.validate_desktop_secret_with_credential(payload.secret) + if verified is None: raise HTTPException( status_code = status.HTTP_401_UNAUTHORIZED, detail = "Desktop authentication failed", ) + username, jwt_secret = verified return Token( - access_token = create_access_token(subject = username, desktop = True), - refresh_token = create_refresh_token(subject = username, desktop = True), + access_token = create_access_token(subject = username, desktop = True, secret = jwt_secret), + refresh_token = create_refresh_token(subject = username, desktop = True, secret = jwt_secret), token_type = "bearer", must_change_password = False, ) @@ -462,9 +466,11 @@ async def refresh(payload: RefreshTokenRequest) -> Token: status_code = status.HTTP_401_UNAUTHORIZED, detail = "Invalid or expired refresh token", ) - username, is_desktop = consumed - new_access_token = create_access_token(subject = username, desktop = is_desktop) - new_refresh_token = create_refresh_token(subject = username, desktop = is_desktop) + username, is_desktop, jwt_secret = consumed + new_access_token = create_access_token(subject = username, desktop = is_desktop, secret = jwt_secret) + new_refresh_token = create_refresh_token( + subject = username, desktop = is_desktop, secret = jwt_secret + ) return Token( access_token = new_access_token, @@ -507,13 +513,25 @@ async def change_password( # Single transaction: a separate refresh-token purge could fail after the # password commit, leaving pre-change tokens able to mint access tokens. - storage.update_password(current_subject, payload.new_password, revoke_refresh_tokens = True) + # Conditional on the hash just verified: a reset-password that landed while + # this request was in flight must not be overwritten by it. + new_secret = storage.update_password( + current_subject, + payload.new_password, + revoke_refresh_tokens = True, + expect_password_hash = pwd_hash, + ) + if new_secret is None: + raise HTTPException( + status_code = status.HTTP_409_CONFLICT, + detail = "The password changed while this request was in flight. Sign in again.", + ) try: request.app.state.bootstrap_password = None except AttributeError: pass - access_token = create_access_token(subject = current_subject) - refresh_token = create_refresh_token(subject = current_subject) + access_token = create_access_token(subject = current_subject, secret = new_secret) + refresh_token = create_refresh_token(subject = current_subject, secret = new_secret) return Token( access_token = access_token, refresh_token = refresh_token, @@ -541,20 +559,28 @@ def _row_to_api_key_response(row: dict) -> ApiKeyResponse: @router.post("/api-keys", response_model = CreateApiKeyResponse) async def create_api_key( - payload: CreateApiKeyRequest, current_subject: str = Depends(get_current_subject) + payload: CreateApiKeyRequest, credential: tuple = Depends(get_current_credential) ) -> CreateApiKeyResponse: """Create a new API key. The raw key is returned once and cannot be retrieved later.""" + current_subject, generation = credential expires_at = None if payload.expires_in_days is not None: expires_at = ( datetime.now(timezone.utc) + timedelta(days = payload.expires_in_days) ).isoformat() - raw_key, row = storage.create_api_key( - username = current_subject, - name = payload.name, - expires_at = expires_at, - ) + try: + raw_key, row = storage.create_api_key( + username = current_subject, + name = payload.name, + expires_at = expires_at, + expect_gen = generation, + ) + except storage.CredentialRotated: + raise HTTPException( + status_code = status.HTTP_401_UNAUTHORIZED, + detail = "Invalid or expired token", + ) return CreateApiKeyResponse( key = raw_key, api_key = _row_to_api_key_response(row), diff --git a/studio/backend/routes/data_recipe/jobs.py b/studio/backend/routes/data_recipe/jobs.py index e870e8855e..7fdf0abada 100644 --- a/studio/backend/routes/data_recipe/jobs.py +++ b/studio/backend/routes/data_recipe/jobs.py @@ -10,7 +10,10 @@ from datetime import datetime, timedelta, timezone from typing import Any, Optional from urllib.parse import urlparse -from fastapi import APIRouter, HTTPException, Query, Request +from fastapi import APIRouter, Depends, HTTPException, Query, Request + +from auth.authentication import get_current_credential +from auth.storage import CredentialRotated from fastapi.responses import JSONResponse, StreamingResponse from pydantic import ValidationError @@ -257,7 +260,11 @@ def _inject_local_structured_response_format( model_configs.extend(new_configs) -def _inject_local_providers(recipe: dict[str, Any], request: Request) -> Optional[int]: +def _inject_local_providers( + recipe: dict[str, Any], + request: Request, + expect_gen: Optional[str] = None, +) -> Optional[int]: """Mutate recipe in-place: point is_local providers at this server and mint a short-lived internal sk-unsloth-* key for workflow auth. @@ -313,6 +320,7 @@ def _inject_local_providers(recipe: dict[str, Any], request: Request) -> Optiona name = "data-recipe workflow", expires_at = expires_at, internal = True, + expect_gen = expect_gen, ) internal_key_id = int(row["id"]) @@ -375,7 +383,11 @@ def _normalize_run_name(value: Any) -> str | None: @router.post("/jobs", response_class = JSONResponse, response_model = JobCreateResponse) -def create_job(payload: RecipePayload, request: Request): +def create_job( + payload: RecipePayload, + request: Request, + credential: tuple = Depends(get_current_credential), +): recipe = payload.recipe if not recipe.get("columns"): raise HTTPException(status_code = 400, detail = "Recipe must include columns.") @@ -406,7 +418,11 @@ def create_job(payload: RecipePayload, request: Request): ) from exc try: - internal_api_key_id = _inject_local_providers(recipe, request) + internal_api_key_id = _inject_local_providers(recipe, request, credential[1]) + except CredentialRotated as exc: + # A reset-password landed after this request authenticated; the workflow key + # is refused, so answer like any other revoked credential rather than 500. + raise HTTPException(status_code = 401, detail = "Invalid or expired token") from exc except ValueError as exc: raise log_and_http_error( exc, diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.py index 7ab7041921..6dbc63d09d 100644 --- a/studio/backend/routes/inference.py +++ b/studio/backend/routes/inference.py @@ -4451,7 +4451,7 @@ def _effective_load_in_4bit(config: ModelConfig, requested: bool) -> bool: if not adapter_cfg_path.exists(): return load_in_4bit try: - with open(adapter_cfg_path, encoding = "utf-8") as f: + with open(adapter_cfg_path, encoding = "utf-8-sig") as f: adapter_cfg = json.load(f) if not isinstance(adapter_cfg, dict): # malformed -> keep requested return load_in_4bit diff --git a/studio/backend/routes/models.py b/studio/backend/routes/models.py index 96c5b96d73..6e587c18e8 100644 --- a/studio/backend/routes/models.py +++ b/studio/backend/routes/models.py @@ -722,7 +722,7 @@ def _scan_ollama_dir(ollama_dir: Path, limit: Optional[int] = None) -> List[Loca stem_hash = hashlib.sha256(manifest_key.encode()).hexdigest()[:10] try: - manifest = json.loads(tag_file.read_text(encoding = "utf-8")) + manifest = json.loads(tag_file.read_text(encoding = "utf-8-sig")) except (json.JSONDecodeError, OSError, UnicodeDecodeError) as e: logger.debug( "Skipping unreadable/invalid Ollama manifest %s: %s", @@ -738,7 +738,7 @@ def _scan_ollama_dir(ollama_dir: Path, limit: Optional[int] = None) -> List[Loca config_blob = blobs_dir / config_digest.replace(":", "-") if config_blob.is_file(): try: - cfg = json.loads(config_blob.read_text(encoding = "utf-8")) + cfg = json.loads(config_blob.read_text(encoding = "utf-8-sig")) model_type = cfg.get("model_type", "") file_type = cfg.get("file_type", "") except (json.JSONDecodeError, OSError, UnicodeDecodeError) as e: @@ -1042,7 +1042,7 @@ def _dir_has_downloaded_model(directory: Path, max_entries: int = 4000) -> bool: if not m.is_file(): continue try: - manifest = json.loads(m.read_text(encoding = "utf-8")) + manifest = json.loads(m.read_text(encoding = "utf-8-sig")) except (json.JSONDecodeError, OSError, ValueError): continue for layer in manifest.get("layers") or []: @@ -3360,6 +3360,8 @@ def _wsl_reveal_in_explorer(path: Path) -> bool: ["wslpath", "-w", str(path)], capture_output = True, text = True, + encoding = "utf-8", + errors = "replace", check = True, timeout = 10, ).stdout.strip() diff --git a/studio/backend/run.py b/studio/backend/run.py index 08d1c5299e..2d9e714d90 100644 --- a/studio/backend/run.py +++ b/studio/backend/run.py @@ -10,7 +10,7 @@ import os import sys import time from pathlib import Path -from typing import Optional, Tuple +from typing import NoReturn, Optional, Sequence, Tuple def _fix_torch_cuda_ld_path(): @@ -689,6 +689,33 @@ def _get_pid_on_port(port: int) -> "tuple[int, str] | None": return None +def _bind_addresses(host: str, port: int) -> "set[str]": + """Every address *host* resolves to. `localhost` is both 127.0.0.1 and ::1, and + recording only the first lets a later launch on the other one miss us.""" + import socket + + try: + infos = socket.getaddrinfo(host, port, socket.AF_UNSPEC, socket.SOCK_STREAM) + except OSError: + return {host} + return {info[4][0] for info in infos} or {host} + + +def _addresses_collide(recorded: "str | None", host: str, port: int) -> bool: + """Would a server bound to *recorded* block a bind to *host*? + + *recorded* may list several addresses. Unknown or wildcard on either side + collides: refusing with a clear message beats silently starting a duplicate. + """ + wildcards = ("0.0.0.0", "::", "") + if not recorded or host in wildcards: + return True + listed = {a.strip() for a in recorded.split(",") if a.strip()} + if not listed or listed & set(wildcards): + return True + return bool(listed & _bind_addresses(host, port)) + + def _is_port_free(host: str, port: int) -> bool: """Check if a port is available for binding. @@ -733,18 +760,213 @@ def _find_free_port( host: str, start: int, max_attempts: int = 20, + avoid_own_studio: bool = False, ) -> int: - """Find a free port from `start`, trying up to max_attempts ports.""" + """Find a free port from `start`, trying up to max_attempts ports. + + ``avoid_own_studio`` aborts rather than skipping past one of our own servers + in the fallback range, which would start a duplicate on a later port. + """ for offset in range(max_attempts): candidate = start + offset if _is_port_free(host, candidate): return candidate + if avoid_own_studio: + own = _own_studio_on_port(candidate, host) + if own is not None: + _abort_already_running(own, candidate) raise RuntimeError(f"Could not find a free port in range {start}-{start + max_attempts - 1}") from utils.paths.storage_roots import studio_root as _studio_root +# Legacy single-instance file; still read so `stop` finds an older build's server. _PID_FILE = _studio_root() / "studio.pid" +PID_FILE_GLOB = "studio-*.pid" + + +def _pid_file_for_port(port: int) -> Path: + # PID in the name: 127.0.0.1 and ::1 can share a port, and one file per port + # would let the second bind overwrite the first. + return _studio_root() / f"studio-{port}-{os.getpid()}.pid" + + +def _pid_alive(pid: int) -> bool: + try: + import psutil + return psutil.pid_exists(pid) + except ImportError: + pass + if sys.platform == "win32": + # os.kill(pid, 0) raises OSError for every pid on Windows, so tasklist is + # the only usable probe here. + import subprocess + try: + out = subprocess.run( + ["tasklist", "/FI", f"PID eq {int(pid)}", "/NH", "/FO", "CSV"], + capture_output = True, + text = True, + timeout = 10, + ).stdout + except Exception: + # Unconfirmed means keep, matching the CLI's _pid_alive. Pruning a + # live server's record is what lets the next launch fall back past it + # and strand it, which is the bug this file exists to fix. A stale + # record instead costs one clear "already running" message. + return True + return f'"{int(pid)}"' in out + try: + os.kill(pid, 0) + except ProcessLookupError: + return False + except OSError: + return True + return True + + +def _process_create_time(pid: int) -> "float | None": + try: + import psutil + return psutil.Process(pid).create_time() + except Exception: + return None + + +def _read_pid_record(path: Path) -> "tuple[int, float | None, str | None] | None": + """Parse ``pid`` / optional ``create_time`` / optional bind address.""" + try: + lines = path.read_text(encoding = "utf-8").splitlines() + except (OSError, UnicodeDecodeError): + return None + if not lines or not lines[0].strip().isdigit(): + return None + try: + # isdigit() is not enough: a superscript two passes it but int() rejects it. + pid = int(lines[0].strip()) + except ValueError: + return None + # kill(0) signals our whole process group; kill(1) is init. Never either. + if pid < 2: + return None + created = None + if len(lines) > 1: + try: + created = float(lines[1].strip()) + except ValueError: + created = None + address = lines[2].strip() if len(lines) > 2 and lines[2].strip() else None + return pid, created, address + + +def _pid_is_studio_backend(pid: int, created_times: "Sequence[float | None]" = ()) -> bool: + """False only when a recorded start time proves this PID is a different process. + + Any recorded time matching is enough -- a stale record must not veto a live + server that reused the PID. Untimed records cannot be checked at all, so they + are trusted: a legacy `python run.py` has no telltale argv, and guessing from + the command line rejected real servers. + """ + known = [c for c in created_times if c is not None] + if not known: + return True + actual = _process_create_time(pid) + if actual is None: + return True + return any(abs(actual - c) < 1.0 for c in known) + + +def _own_studio_on_port(port: int, host: str) -> "int | None": + """PID of one of our own servers already bound to *port* for *host*. + + Reads our own records rather than enumerating listeners: psutil is optional, + and without it a listener scan finds nothing and we silently start a duplicate. + """ + try: + paths = list(_studio_root().glob(f"studio-{port}-*.pid")) + except OSError: + return None + for path in paths: + record = _read_pid_record(path) + if record is None: + continue + pid, created, address = record + if not _pid_alive(pid): + # Pruning is a courtesy; an undeletable record must not abort startup. + try: + path.unlink(missing_ok = True) + except OSError: + pass + continue + if not _addresses_collide(address, host, port): + continue + if _pid_is_studio_backend(pid, [created]): + return pid + return _legacy_studio_on_port(port) + + +def _legacy_studio_on_port(port: int) -> "int | None": + """A pre-upgrade server recorded only its PID, so match it to the listener. + + Falling back past one leaves it running while `_write_pid_file` overwrites the + only record of it. When the listener is unknowable, assume it is ours. + """ + record = _read_pid_record(_PID_FILE) + if record is None: + return None + pid, created, _address = record + if not _pid_alive(pid): + return None + # A current build writes a per-port file too, so its port is already known -- + # and this port's records were just checked. Only count a record that still + # matches the live process: a stale one may just share a reused PID. + for other in _per_port_records(): + if other and other[0] == pid and _pid_is_studio_backend(pid, [other[1]]): + return None + blocker = _get_pid_on_port(port) + if blocker is not None and blocker[0] != pid: + return None + if not _pid_is_studio_backend(pid, [created]): + return None + return pid + + +def _per_port_records() -> "list[tuple[int, float | None, str | None] | None]": + try: + return [_read_pid_record(p) for p in _studio_root().glob(PID_FILE_GLOB)] + except OSError: + return [] + + +def _resolve_port( + host: str, + port: int, + avoid_own_studio: bool = True, +) -> int: + """The requested port, or the next free one. + + With ``avoid_own_studio`` this aborts rather than falling back past one of our + own servers, on *port* itself or anywhere in the fallback range: skipping one + is what strands it. Callers that read the bound port back pass False and keep + the plain fallback. + """ + if _is_port_free(host, port): + return port + if avoid_own_studio: + own = _own_studio_on_port(port, host) + if own is not None: + _abort_already_running(own, port) + return _find_free_port(host, port + 1, avoid_own_studio = avoid_own_studio) + + +def _abort_already_running(pid: int, port: int) -> "NoReturn": + print( + f"Error: Unsloth Studio is already running on port {port} (PID {pid}). Run " + "`unsloth studio stop` first, or start this one on a different --port.", + file = sys.stderr, + flush = True, + ) + sys.exit(1) + # Direct backend launches bypass the CLI's env re-export; do it here for # real custom roots so unsloth-zoo's import-time LLAMA_CPP_DEFAULT_DIR @@ -770,23 +992,101 @@ if _STUDIO_ROOT_RESOLVED != _LEGACY_STUDIO_ROOT: os.environ.setdefault("UNSLOTH_IS_PRESENT", "1") -def _write_pid_file(): - """Write the current process PID to the studio PID file.""" +_OWN_PID_FILE: "Path | None" = None + + +def _write_pid_file(port: int, host: str = ""): + """Record this PID under its own port so `stop` can find every server.""" + global _OWN_PID_FILE + path = _pid_file_for_port(port) try: - _PID_FILE.parent.mkdir(parents = True, exist_ok = True) - _PID_FILE.write_text(str(os.getpid()), encoding = "utf-8") + path.parent.mkdir(parents = True, exist_ok = True) + except OSError: + pass + try: + # Start time pins the record to this process; the bind address tells a + # later launch whether this server would actually block it. + created = _process_create_time(os.getpid()) + address = ",".join(sorted(_bind_addresses(host, port))) if host else "" + body = f"{os.getpid()}\n{'' if created is None else repr(created)}\n{address}" + # Write-then-rename: `stop` reads these concurrently, and a reader that + # catches the truncate window sees a corrupt record and deletes it. + tmp = path.with_name(path.name + ".tmp") + try: + tmp.write_text(body, encoding = "utf-8") + os.replace(tmp, path) + finally: + # A failed replace would otherwise leave the scratch file behind. It + # does not end in .pid, so no glob picks it up either way. + tmp.unlink(missing_ok = True) + except OSError: + pass + else: + _OWN_PID_FILE = path + # An older CLI's `stop` only reads this one, and expects a bare PID. Written + # independently of the per-port record: if that one failed, this is the only + # thing keeping the server stoppable at all. + try: + # Never take it from a server that is still running. A pre-upgrade server + # is recorded here and nowhere else, so overwriting its entry is exactly + # what strands it -- the orphan this file exists to prevent. + prior = _read_pid_record(_PID_FILE) if _PID_FILE.is_file() else None + if prior is None or prior[0] == os.getpid() or not _pid_alive(prior[0]): + _PID_FILE.write_text(str(os.getpid()), encoding = "utf-8") except OSError: pass -def _remove_pid_file(): - """Remove the PID file if it belongs to this process.""" +def _legacy_heir() -> "int | None": + """Another live server's PID, to hand the legacy studio.pid over to. + + Only one server owns studio.pid at a time, so its exit would otherwise drop + the single record an older CLI can read, stranding any sibling that is still + serving. + """ try: - if _PID_FILE.is_file(): - stored = _PID_FILE.read_text(encoding = "utf-8").strip() - if stored == str(os.getpid()): + paths = sorted(_studio_root().glob(PID_FILE_GLOB)) + except OSError: + return None + for path in paths: + if _OWN_PID_FILE is not None and path == _OWN_PID_FILE: + continue + record = _read_pid_record(path) + if record is None or record[0] == os.getpid(): + continue + if _pid_alive(record[0]) and _pid_is_studio_backend(record[0], [record[1]]): + return record[0] + return None + + +def _remove_pid_file(): + """Remove the PID files that belong to this process. + + _PID_FILE is checked even when the per-port record was never written, since + _write_pid_file writes the two independently. + """ + # Nothing here may raise: _graceful_shutdown calls this at the end, and an + # unreadable or undeletable record must not abandon the rest of the exit + # path. _read_pid_record already swallows OSError/UnicodeDecodeError. + if _OWN_PID_FILE is not None: + try: + record = _read_pid_record(_OWN_PID_FILE) if _OWN_PID_FILE.is_file() else None + if record is not None and record[0] == os.getpid(): + _OWN_PID_FILE.unlink(missing_ok = True) + except OSError: + pass + try: + record = _read_pid_record(_PID_FILE) if _PID_FILE.is_file() else None + if record is not None and record[0] == os.getpid(): + # Hand the pointer to a live sibling rather than deleting it. An + # older CLI reads only this file, so dropping it while another + # server is still up leaves that server unstoppable. + heir = _legacy_heir() + if heir is None: _PID_FILE.unlink(missing_ok = True) - except (OSError, UnicodeDecodeError): + else: + _PID_FILE.write_text(str(heir), encoding = "utf-8") + except OSError: pass @@ -796,7 +1096,6 @@ def _graceful_shutdown(server = None): Called from signal handlers to clean up children before exit. Critical on Windows where atexit handlers are unreliable after Ctrl+C. """ - _remove_pid_file() logger.info("Graceful shutdown initiated -- cleaning up subprocesses...") # 1. Shut down uvicorn (releases the listening socket). @@ -849,6 +1148,9 @@ def _graceful_shutdown(server = None): except Exception as e: logger.warning("Error in process-lifetime sweep: %s", e) + # Last: while cleanup runs the server is still alive, and dropping the record + # early leaves a retried `stop` or a new launch unable to find it. + _remove_pid_file() logger.info("All subprocesses cleaned up") @@ -1326,7 +1628,8 @@ def _apply_supplied_password(password_value: "Optional[str]") -> None: if not _auth_storage.requires_password_change(_admin): print( "Error: an Unsloth admin password is already set; --password only sets " - "the initial password. Run `unsloth studio reset-password` first.", + "the initial password. Change it in the UI, or run `unsloth studio " + "reset-password` for a new one.", file = sys.stderr, flush = True, ) @@ -1397,6 +1700,7 @@ def run_server( enable_tools: "Optional[bool]" = None, password: "Optional[str]" = None, emit_tauri_port: bool = True, + abort_if_own_studio: "Optional[bool]" = None, ): """ Start the FastAPI server. @@ -1530,10 +1834,16 @@ def run_server( ) # Auto-find a free port if the requested one is in use. - if not _is_port_free(host, port): - original_port = port - blocker = _get_pid_on_port(port) - port = _find_free_port(host, port + 1) + original_port = port + # Refusing rather than falling back is for callers that cannot follow us to + # the new port. `studio run` reads app.state.server_port back and the desktop + # app reads TAURI_PORT, so both should keep the plain fallback; only the + # bare launch, which has nothing but the banner, benefits from the refusal. + if abort_if_own_studio is None: + abort_if_own_studio = not api_only + port = _resolve_port(host, port, avoid_own_studio = abort_if_own_studio) + if port != original_port: + blocker = _get_pid_on_port(original_port) if not silent: print("") print("=" * 50) @@ -1731,7 +2041,7 @@ def run_server( (time.perf_counter() - boot_started) * 1000, ) - _write_pid_file() + _write_pid_file(port, host) import atexit atexit.register(_remove_pid_file) diff --git a/studio/backend/tests/test_change_password_policy.py b/studio/backend/tests/test_change_password_policy.py index c73e9ed839..fc095760d0 100644 --- a/studio/backend/tests/test_change_password_policy.py +++ b/studio/backend/tests/test_change_password_policy.py @@ -67,9 +67,11 @@ def test_rejects_password_containing_spaces(_user): def test_allows_password_without_spaces(_user, monkeypatch): - monkeypatch.setattr(auth_routes.storage, "update_password", lambda *args, **kwargs: True) - monkeypatch.setattr(auth_routes, "create_access_token", lambda subject: "at") - monkeypatch.setattr(auth_routes, "create_refresh_token", lambda subject: "rt") + monkeypatch.setattr( + auth_routes.storage, "update_password", lambda *args, **kwargs: "rotated-secret" + ) + monkeypatch.setattr(auth_routes, "create_access_token", lambda subject, **kwargs: "at") + monkeypatch.setattr(auth_routes, "create_refresh_token", lambda subject, **kwargs: "rt") token = _change("correct-horse-battery") assert token.access_token == "at" assert token.must_change_password is False diff --git a/studio/backend/tests/test_chat_text_encoding.py b/studio/backend/tests/test_chat_text_encoding.py new file mode 100644 index 0000000000..64860dab1a --- /dev/null +++ b/studio/backend/tests/test_chat_text_encoding.py @@ -0,0 +1,195 @@ +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 + +"""Model text stays intact when it carries non-ASCII. + +``open()`` and ``Path.read_text()`` fall back to ``locale.getencoding()`` when +no ``encoding`` is passed. On Windows that is the ANSI codepage, not UTF-8, so +a chat template or model config holding ``ä ö ü → 世`` mojibakes or raises +``UnicodeDecodeError``. These files are UTF-8, so the reads must say so. + +Each fixture writes raw UTF-8 (``ensure_ascii = False``), matching what +Hugging Face actually ships, rather than ASCII ``\\uXXXX`` escapes. +""" + +from __future__ import annotations + +import json +import subprocess +import sys +import textwrap +from pathlib import Path + + +BACKEND_ROOT = Path(__file__).resolve().parent.parent + + +def test_config_json_round_trips_non_ascii(tmp_path: Path) -> None: + from utils import transformers_version + + name = "Modell für Grüße 世界" + (tmp_path / "config.json").write_text( + json.dumps({"model_type": "llama", "_name_or_path": name}, ensure_ascii = False), + encoding = "utf-8", + ) + transformers_version._config_json_cache.clear() + + cfg = transformers_version._load_config_json(str(tmp_path)) + + assert cfg is not None + assert cfg["_name_or_path"] == name + + +def test_tokenizer_config_round_trips_non_ascii_chat_template(tmp_path: Path) -> None: + """Chat templates commonly hold ``→`` and smart quotes, which cp1252 mangles.""" + from utils import transformers_version + + template = "{{ '→ Grüße 世界' }}" + (tmp_path / "tokenizer_config.json").write_text( + json.dumps( + {"tokenizer_class": "TokenizersBackend", "chat_template": template}, + ensure_ascii = False, + ), + encoding = "utf-8", + ) + transformers_version._tokenizer_class_cache.clear() + + assert transformers_version._check_tokenizer_config_needs_v5(str(tmp_path)) is True + + +def test_config_json_survives_a_utf8_bom(tmp_path: Path) -> None: + """Notepad wrote "UTF-8 with BOM" by default for years, so hand-edited + configs on Windows carry one. Plain utf-8 keeps the BOM and json.load then + fails on it; utf-8-sig strips it and is identical otherwise.""" + from utils import transformers_version + + name = "Grüße 世界" + (tmp_path / "config.json").write_text( + json.dumps({"model_type": "llama", "_name_or_path": name}, ensure_ascii = False), + encoding = "utf-8-sig", + ) + transformers_version._config_json_cache.clear() + + cfg = transformers_version._load_config_json(str(tmp_path)) + + assert cfg is not None + assert cfg["_name_or_path"] == name + + +def test_remote_code_scan_reads_non_ascii_sources(tmp_path: Path) -> None: + """A German Windows profile also puts umlauts in the model sources scanned.""" + from utils.security import remote_code_scan + + source = "# Grüße über Öl\nVALUE = '世界'\n" + # newline = "" pins the bytes on disk, so Windows line end translation cannot make the + # read back differ by \r. open() because Path.write_text() only grew newline in 3.10. + with open( + tmp_path / "modeling_custom.py", + "w", + encoding = "utf-8", + newline = "", + ) as handle: + handle.write(source) + + files = remote_code_scan.repo_remote_code_files(str(tmp_path)) + + assert files["modeling_custom.py"] == source + + +def test_model_config_reads_do_not_rely_on_the_locale_encoding(tmp_path: Path) -> None: + """The reads above pass anywhere the locale is already UTF-8, which hides + the Windows bug on Linux and macOS. ``-X warn_default_encoding`` makes + CPython flag any text I/O that falls back to the locale, so this fails on + every platform if an ``encoding`` argument goes missing again.""" + # The readers swallow exceptions, so record the warnings instead of raising. + script = textwrap.dedent( + f""" + import sys, warnings + sys.path.insert(0, {str(BACKEND_ROOT)!r}) + from utils import transformers_version + + target = {str(tmp_path)!r} + with warnings.catch_warnings(record = True) as caught: + warnings.simplefilter("always") + transformers_version._config_json_cache.clear() + transformers_version._tokenizer_class_cache.clear() + assert transformers_version._load_config_json(target) is not None + assert transformers_version._check_tokenizer_config_needs_v5(target) is True + + missing = [str(w.message) for w in caught if w.category is EncodingWarning] + if missing: + sys.exit("text I/O fell back to the locale encoding: " + "; ".join(missing)) + """ + ) + for name, payload in ( + ("config.json", {"model_type": "llama", "_name_or_path": "Grüße"}), + ("tokenizer_config.json", {"tokenizer_class": "TokenizersBackend"}), + ): + (tmp_path / name).write_text(json.dumps(payload, ensure_ascii = False), encoding = "utf-8") + + result = subprocess.run( + [sys.executable, "-X", "warn_default_encoding", "-c", script], + capture_output = True, + text = True, + encoding = "utf-8", + errors = "replace", + timeout = 120, + ) + + assert result.returncode == 0, result.stderr + + +def test_utf8_child_env_round_trips_non_ascii(tmp_path: Path) -> None: + """A Python child encodes stdout with its locale unless told otherwise, so + reading its pipe as utf-8 needs the child told to emit utf-8.""" + from utils.child_stdio import utf8_child_env + + payload = "Grüße über Öl → 世界" + child = tmp_path / "child.py" + child.write_text("import sys\nsys.stdout.write(" + repr(payload) + ")\n", encoding = "utf-8") + + env = utf8_child_env() + assert env["PYTHONIOENCODING"] == "utf-8" + + proc = subprocess.run( + [sys.executable, str(child)], + capture_output = True, + text = True, + encoding = "utf-8", + errors = "replace", + env = env, + timeout = 120, + ) + + assert proc.returncode == 0, proc.stderr + assert proc.stdout == payload + + +def test_python_children_are_told_to_emit_utf8() -> None: + """Any child we decode as utf-8 must also be told to write utf-8, or a + cp1252 console silently mangles what it prints.""" + import ast + + offenders: list[str] = [] + for path in sorted(BACKEND_ROOT.rglob("*.py")): + parts = path.relative_to(BACKEND_ROOT).parts + if any(p in ("tests", "node_modules", "plugins", "__pycache__") for p in parts): + continue + source = path.read_text(encoding = "utf-8") + for node in ast.walk(ast.parse(source, filename = str(path))): + if not isinstance(node, ast.Call): + continue + func = node.func + if not (isinstance(func, ast.Attribute) and func.attr in ("run", "Popen")): + continue + segment = ast.get_source_segment(source, node) or "" + if "sys.executable" not in segment or 'encoding = "utf-8"' not in segment: + continue + if "utf8_child_env" in segment or "PYTHONIOENCODING" in segment: + continue + offenders.append(f"{path.name}:{node.lineno}") + + assert not offenders, ( + "these spawn a Python child and decode it as utf-8 without setting the " + "child's own stdio encoding; wrap env in utf8_child_env():\n " + "\n ".join(offenders) + ) diff --git a/studio/backend/tests/test_credential_rotation_race.py b/studio/backend/tests/test_credential_rotation_race.py new file mode 100644 index 0000000000..9b0f95aa02 --- /dev/null +++ b/studio/backend/tests/test_credential_rotation_race.py @@ -0,0 +1,255 @@ +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 + +"""A password rotation must not leave a session minted from the replaced credential. + +`unsloth studio reset-password` rotates in place against a live server, so a login +can verify the old password, have the rotation land, and only then mint its tokens. +Issuance is bound to the credential version that was verified, so such a login gets +tokens that are already dead rather than a session that outlives the reset. +""" + +import secrets +from datetime import datetime, timedelta, timezone + +import jwt +import pytest + +from auth import hashing, storage +from auth.authentication import ALGORITHM, create_access_token, create_refresh_token + + +@pytest.fixture(autouse = True) +def isolated_auth_db(tmp_path, monkeypatch): + monkeypatch.setattr(storage, "DB_PATH", tmp_path / "auth.db") + monkeypatch.setattr(storage, "_BOOTSTRAP_PW_PATH", tmp_path / ".bootstrap_password") + monkeypatch.setattr(storage, "_bootstrap_password", None) + monkeypatch.setattr(storage, "_api_key_pbkdf2_salt_cache", None) + yield + + +@pytest.fixture +def admin(): + storage.create_initial_user( + username = storage.DEFAULT_ADMIN_USERNAME, + password = "old-password-123", + jwt_secret = secrets.token_urlsafe(64), + ) + return storage.DEFAULT_ADMIN_USERNAME + + +def _verified_secret(username): + return storage.get_user_and_secret(username)[2] + + +def test_access_token_from_the_replaced_credential_is_rejected(admin): + secret = _verified_secret(admin) + + storage.update_password(admin, "new-password-456", revoke_refresh_tokens = True) + token = create_access_token(subject = admin, secret = secret) + + with pytest.raises(jwt.InvalidTokenError): + jwt.decode(token, storage.get_jwt_secret(admin), algorithms = [ALGORITHM]) + + +def test_refresh_token_from_the_replaced_credential_is_rejected(admin): + secret = _verified_secret(admin) + + # Inserted AFTER the rotation's DELETE, so revocation alone cannot catch it. + storage.update_password(admin, "new-password-456", revoke_refresh_tokens = True) + token = create_refresh_token(subject = admin, secret = secret) + + assert storage.verify_refresh_token(token) is None + assert storage.consume_refresh_token(token) is None + + +def test_a_rejected_refresh_token_is_dropped(admin): + secret = _verified_secret(admin) + storage.update_password(admin, "new-password-456", revoke_refresh_tokens = True) + token = create_refresh_token(subject = admin, secret = secret) + + storage.verify_refresh_token(token) + + conn = storage.get_connection() + try: + assert conn.execute("SELECT COUNT(*) AS c FROM refresh_tokens").fetchone()["c"] == 0 + finally: + conn.close() + + +def test_tokens_from_the_current_credential_still_work(admin): + secret = _verified_secret(admin) + + access = create_access_token(subject = admin, secret = secret) + refresh = create_refresh_token(subject = admin, secret = secret) + + jwt.decode(access, storage.get_jwt_secret(admin), algorithms = [ALGORITHM]) + assert storage.verify_refresh_token(refresh) == (admin, False) + + +def test_refresh_cannot_outlive_a_rotation_it_raced(admin): + # /refresh consumes, then mints. A rotation landing in between must not let + # the replacement pair be signed with the credential that just replaced it. + secret = _verified_secret(admin) + token = create_refresh_token(subject = admin, secret = secret) + consumed = storage.consume_refresh_token(token) + assert consumed is not None + _username, _is_desktop, consumed_secret = consumed + + storage.update_password(admin, "new-password-456", revoke_refresh_tokens = True) + access = create_access_token(subject = admin, secret = consumed_secret) + refresh = create_refresh_token(subject = admin, secret = consumed_secret) + + with pytest.raises(jwt.InvalidTokenError): + jwt.decode(access, storage.get_jwt_secret(admin), algorithms = [ALGORITHM]) + assert storage.verify_refresh_token(refresh) is None + + +def test_desktop_login_cannot_outlive_a_rotation_it_raced(admin): + # The reset deletes the desktop secret, so a desktop-login that validated it + # just beforehand must not mint a session that survives. + raw = storage.create_desktop_secret() + verified = storage.validate_desktop_secret_with_credential(raw) + assert verified is not None + _username, verified_secret = verified + + storage.update_password(admin, "new-password-456", revoke_refresh_tokens = True) + access = create_access_token(subject = admin, desktop = True, secret = verified_secret) + refresh = create_refresh_token(subject = admin, desktop = True, secret = verified_secret) + + with pytest.raises(jwt.InvalidTokenError): + jwt.decode(access, storage.get_jwt_secret(admin), algorithms = [ALGORITHM]) + assert storage.verify_refresh_token(refresh) is None + + +def test_change_password_cannot_overwrite_a_rotation_it_raced(admin): + # A change-password that verified the old hash must not clobber a reset that + # committed while it was in flight. + _salt, verified_hash, _secret, _must_change = storage.get_user_and_secret(admin) + + storage.update_password(admin, "reset-by-the-cli-789", revoke_refresh_tokens = True) + + assert not storage.update_password( + admin, + "attacker-chosen-000", + revoke_refresh_tokens = True, + expect_password_hash = verified_hash, + ) + salt, pwd_hash, _s, _m = storage.get_user_and_secret(admin) + assert hashing.verify_password("reset-by-the-cli-789", salt, pwd_hash) + + +def test_api_key_creation_from_a_revoked_credential_is_refused(admin): + generation = storage.credential_generation(_verified_secret(admin)) + + storage.update_password(admin, "new-password-456", revoke_refresh_tokens = True) + + with pytest.raises(storage.CredentialRotated): + storage.create_api_key(username = admin, name = "k", expect_gen = generation) + conn = storage.get_connection() + try: + assert conn.execute("SELECT COUNT(*) AS c FROM api_keys").fetchone()["c"] == 0 + finally: + conn.close() + + +def test_api_key_creation_under_the_current_credential_still_works(admin): + generation = storage.credential_generation(_verified_secret(admin)) + + raw_key, _row = storage.create_api_key(username = admin, name = "k", expect_gen = generation) + + assert storage.validate_api_key(raw_key) == admin + + +def test_change_password_tokens_are_bound_to_its_own_write(admin): + # The tokens returned to a successful change-password must be signed with the + # secret that write produced, not whatever a later reset put in the DB. + _salt, verified_hash, _secret, _must = storage.get_user_and_secret(admin) + new_secret = storage.update_password( + admin, + "chosen-by-the-user", + revoke_refresh_tokens = True, + expect_password_hash = verified_hash, + ) + assert new_secret is not None + + storage.update_password(admin, "reset-by-the-cli-789", revoke_refresh_tokens = True) + access = create_access_token(subject = admin, secret = new_secret) + refresh = create_refresh_token(subject = admin, secret = new_secret) + + with pytest.raises(jwt.InvalidTokenError): + jwt.decode(access, storage.get_jwt_secret(admin), algorithms = [ALGORITHM]) + assert storage.verify_refresh_token(refresh) is None + + +def test_internal_api_key_minting_honours_the_request_generation(admin): + generation = storage.credential_generation(_verified_secret(admin)) + storage.update_password(admin, "new-password-456", revoke_refresh_tokens = True) + + with pytest.raises(storage.CredentialRotated): + storage.create_api_key( + username = admin, + name = "data-recipe workflow", + internal = True, + expect_gen = generation, + ) + + +def test_api_key_auth_reports_the_version_the_key_was_valid_under(admin): + # The generation must come from the same transaction as the key check, or a + # revoked key could hand a route the post-reset generation and mint again. + raw, _row = storage.create_api_key(username = admin, name = "agent") + verified = storage.validate_api_key_with_credential(raw) + assert verified is not None + _user, secret = verified + generation = storage.credential_generation(secret) + + storage.update_password(admin, "new-password-456", revoke_refresh_tokens = True) + conn = storage.get_connection() + try: + conn.execute("DELETE FROM api_keys") + conn.commit() + finally: + conn.close() + + assert storage.validate_api_key(raw) is None + with pytest.raises(storage.CredentialRotated): + storage.create_api_key(username = admin, name = "after", expect_gen = generation) + + +def test_consuming_a_legacy_token_reports_the_pre_reset_credential(admin): + # An unstamped row has no generation to compare, so consume must read the + # credential inside the delete transaction rather than after committing it. + token = secrets.token_urlsafe(48) + expires_at = (datetime.now(timezone.utc) + timedelta(days = 7)).isoformat() + storage.save_refresh_token(token, admin, expires_at, secret_gen = None) + conn = storage.get_connection() + try: + conn.execute("UPDATE refresh_tokens SET secret_gen = NULL") + conn.commit() + finally: + conn.close() + + consumed = storage.consume_refresh_token(token) + assert consumed is not None + _username, _is_desktop, consumed_secret = consumed + + storage.update_password(admin, "new-password-456", revoke_refresh_tokens = True) + access = create_access_token(subject = admin, secret = consumed_secret) + with pytest.raises(jwt.InvalidTokenError): + jwt.decode(access, storage.get_jwt_secret(admin), algorithms = [ALGORITHM]) + + +def test_unstamped_legacy_tokens_still_verify(admin): + # Rows written before the secret_gen column existed must not log users out. + token = secrets.token_urlsafe(48) + expires_at = (datetime.now(timezone.utc) + timedelta(days = 7)).isoformat() + storage.save_refresh_token(token, admin, expires_at, secret_gen = None) + conn = storage.get_connection() + try: + conn.execute("UPDATE refresh_tokens SET secret_gen = NULL") + conn.commit() + finally: + conn.close() + + assert storage.verify_refresh_token(token) == (admin, False) diff --git a/studio/backend/tests/test_desktop_auth.py b/studio/backend/tests/test_desktop_auth.py index bc995b6a59..039bb5e3e6 100644 --- a/studio/backend/tests/test_desktop_auth.py +++ b/studio/backend/tests/test_desktop_auth.py @@ -134,6 +134,218 @@ def test_ensure_default_admin_loads_existing_bootstrap_after_restart(monkeypatch assert storage.get_bootstrap_password() == bootstrap_pw +def test_bootstrap_password_file_ends_with_a_newline(): + # Otherwise `cat` welds the passphrase onto the shell prompt. + storage.ensure_default_admin() + + # Bytes: read_text would decode CRLF back to "\n" and hide a CR. + raw = storage._BOOTSTRAP_PW_PATH.read_bytes() + + assert raw == storage.get_bootstrap_password().encode("utf-8") + b"\n" + + +def test_bootstrap_password_round_trips_across_a_restart_with_the_newline(): + storage.ensure_default_admin() + original = storage.get_bootstrap_password() + + storage._bootstrap_password = None + + assert storage.generate_bootstrap_password() == original + + +def test_upgrade_normalises_the_bootstrap_file(): + # Upgrade path: the admin row exists, so generate_bootstrap_password() never runs. + seed_user() + storage._BOOTSTRAP_PW_PATH.write_bytes(b"legacy-bootstrap-secret") + + storage.ensure_default_admin() + + assert storage._BOOTSTRAP_PW_PATH.read_bytes() == b"legacy-bootstrap-secret\n" + assert storage.get_bootstrap_password() == "legacy-bootstrap-secret" + + +@pytest.mark.parametrize( + "other", + [ + b"legacy-bootstrap-secret\r\n", # only an unreleased build wrote this + b"legacy-bootstrap-secret\r", + b"legacy-bootstrap-secret ", + ], +) +def test_only_an_exactly_unterminated_bootstrap_file_is_touched(other): + # Appending is safe only because it is restricted to the one released shape. + seed_user() + storage._BOOTSTRAP_PW_PATH.write_bytes(other) + + storage.ensure_default_admin() + + assert storage.get_bootstrap_password() == "legacy-bootstrap-secret" + assert storage._BOOTSTRAP_PW_PATH.read_bytes() == other + + +def test_upgrade_normalises_when_the_admin_row_is_missing(): + storage._BOOTSTRAP_PW_PATH.write_bytes(b"legacy-bootstrap-secret") + + assert storage.generate_bootstrap_password() == "legacy-bootstrap-secret" + assert storage._BOOTSTRAP_PW_PATH.read_bytes() == b"legacy-bootstrap-secret\n" + + +def test_a_well_formed_bootstrap_file_is_not_rewritten(): + seed_user() + storage._BOOTSTRAP_PW_PATH.write_bytes(b"legacy-bootstrap-secret\n") + mtime = storage._BOOTSTRAP_PW_PATH.stat().st_mtime_ns + + storage.ensure_default_admin() + + assert storage._BOOTSTRAP_PW_PATH.stat().st_mtime_ns == mtime + + +def test_migration_failure_does_not_break_startup(monkeypatch): + seed_user() + storage._BOOTSTRAP_PW_PATH.write_bytes(b"legacy-bootstrap-secret") + + real_open = storage.os.open + + def refuse(path, flags, *args, **kwargs): + if str(path) == str(storage._BOOTSTRAP_PW_PATH): + raise PermissionError("read-only auth dir") + return real_open(path, flags, *args, **kwargs) + + monkeypatch.setattr(storage.os, "open", refuse) + + storage.ensure_default_admin() + + assert storage.get_bootstrap_password() == "legacy-bootstrap-secret" + assert storage._BOOTSTRAP_PW_PATH.read_bytes() == b"legacy-bootstrap-secret" + + +def test_normalising_never_recreates_a_cleared_bootstrap_file(monkeypatch): + # A rename would resurrect revoked plaintext if the password changed after the read. + seed_user() + storage._BOOTSTRAP_PW_PATH.write_bytes(b"legacy-bootstrap-secret") + + real_open = storage.os.open + + def clear_then_open(path, flags, *args, **kwargs): + if str(path) == str(storage._BOOTSTRAP_PW_PATH): + storage._BOOTSTRAP_PW_PATH.unlink(missing_ok = True) + return real_open(path, flags, *args, **kwargs) + + monkeypatch.setattr(storage.os, "open", clear_then_open) + + assert storage._read_persisted_bootstrap_password() == "legacy-bootstrap-secret" + assert not storage._BOOTSTRAP_PW_PATH.exists() + + +def test_normalising_does_not_overwrite_a_rotated_bootstrap_file(monkeypatch): + seed_user() + storage._BOOTSTRAP_PW_PATH.write_bytes(b"legacy-bootstrap-secret") + + real_open = storage.os.open + + def rotate_then_open(path, flags, *args, **kwargs): + if str(path) == str(storage._BOOTSTRAP_PW_PATH): + storage._BOOTSTRAP_PW_PATH.write_bytes(b"brand-new-secret\n") + return real_open(path, flags, *args, **kwargs) + + monkeypatch.setattr(storage.os, "open", rotate_then_open) + + storage._read_persisted_bootstrap_password() + + # The append may add a second newline; the rotated credential must survive. + raw = storage._BOOTSTRAP_PW_PATH.read_bytes() + assert raw.strip() == b"brand-new-secret" + storage._bootstrap_password = None + assert storage._load_bootstrap_password() == "brand-new-secret" + + +def test_leading_whitespace_bootstrap_file_is_left_alone(monkeypatch): + # An in-place rewrite is not atomic, so only the exact unterminated shape is touched. + seed_user() + storage._BOOTSTRAP_PW_PATH.write_bytes(b" legacy-bootstrap-secret ") + + storage.ensure_default_admin() + + assert storage.get_bootstrap_password() == "legacy-bootstrap-secret" + assert storage._BOOTSTRAP_PW_PATH.read_bytes() == b" legacy-bootstrap-secret " + + +def test_normalising_opens_the_file_in_binary_mode(monkeypatch): + # Without O_BINARY, Windows text mode turns the written LF back into CRLF. + seed_user() + storage._BOOTSTRAP_PW_PATH.write_bytes(b"legacy-bootstrap-secret") + monkeypatch.setattr(storage.os, "O_BINARY", 0x8000, raising = False) + seen = [] + real_open = storage.os.open + + def spy(path, flags, *args, **kwargs): + if str(path) == str(storage._BOOTSTRAP_PW_PATH): + seen.append(flags) + return real_open(path, flags & ~0x8000, *args, **kwargs) + + monkeypatch.setattr(storage.os, "open", spy) + + storage.ensure_default_admin() + + assert seen and all(f & 0x8000 for f in seen), seen + + +def test_clearing_by_truncation_mid_normalisation_is_not_undone(monkeypatch): + # clear_bootstrap_password() truncates through its own descriptor when the unlink + # fails (Windows, while ours is open); the append must not restore the plaintext. + seed_user() + storage._BOOTSTRAP_PW_PATH.write_bytes(b"legacy-bootstrap-secret") + + real_open = storage.os.open + + def truncate_then_open(path, flags, *args, **kwargs): + fd = real_open(path, flags, *args, **kwargs) + if str(path) == str(storage._BOOTSTRAP_PW_PATH): + storage._BOOTSTRAP_PW_PATH.write_text("", encoding = "utf-8") + return fd + + monkeypatch.setattr(storage.os, "open", truncate_then_open) + + storage._read_persisted_bootstrap_password() + + # A lone newline over a cleared file still reads back as no password. + assert storage._BOOTSTRAP_PW_PATH.read_bytes().strip() == b"" + storage._bootstrap_password = None + assert storage._load_bootstrap_password() is None + + +def test_normalising_works_without_fchmod(monkeypatch): + # os.fchmod only reached Windows in 3.13; its absence must not raise. + seed_user() + storage._BOOTSTRAP_PW_PATH.write_bytes(b"legacy-bootstrap-secret") + monkeypatch.delattr(storage.os, "fchmod", raising = False) + + storage.ensure_default_admin() + + assert storage._BOOTSTRAP_PW_PATH.read_bytes() == b"legacy-bootstrap-secret\n" + assert storage.get_bootstrap_password() == "legacy-bootstrap-secret" + + +def test_persisting_the_bootstrap_password_is_atomic(monkeypatch, tmp_path): + # A partial write would destroy the only plaintext recovery credential. + storage._persist_bootstrap_password("original-secret") + + def boom(src, dst): + raise OSError("crash before replace") + + monkeypatch.setattr(storage.os, "replace", boom) + with pytest.raises(OSError): + storage._persist_bootstrap_password("new-secret") + + assert storage._BOOTSTRAP_PW_PATH.read_bytes() == b"original-secret\n" + leftovers = [ + p.name + for p in storage._BOOTSTRAP_PW_PATH.parent.iterdir() + if "bootstrap_password." in p.name + ] + assert leftovers == [] + + def test_ensure_default_admin_does_not_generate_for_empty_existing_bootstrap(): seed_user() storage._BOOTSTRAP_PW_PATH.write_text(" \n", encoding = "utf-8") @@ -233,7 +445,7 @@ def test_consume_refresh_token_second_call_returns_none(): storage.save_refresh_token(raw, storage.DEFAULT_ADMIN_USERNAME, expires) first = storage.consume_refresh_token(raw) - assert first == (storage.DEFAULT_ADMIN_USERNAME, False) + assert first[:2] == (storage.DEFAULT_ADMIN_USERNAME, False) second = storage.consume_refresh_token(raw) assert second is None @@ -262,7 +474,7 @@ def test_consume_refresh_token_concurrent_only_one_succeeds(tmp_path, monkeypatc successes = [r for r in results if r is not None] assert len(successes) == 1, f"expected exactly one consumer to win, got {len(successes)}" - assert successes[0] == (storage.DEFAULT_ADMIN_USERNAME, False) + assert successes[0][:2] == (storage.DEFAULT_ADMIN_USERNAME, False) def test_consume_refresh_token_expired_returns_none(): @@ -336,6 +548,28 @@ def test_local_recipe_token_authenticates_as_admin_for_web_user(loaded_local_mod assert asyncio.run(get_current_subject(credentials)) == storage.DEFAULT_ADMIN_USERNAME +def test_rotated_credential_job_start_is_401_not_500(loaded_local_model): + # A reset-password landing mid-request makes the workflow-key mint refuse. + # That must reach the client as a revoked credential, not an unhandled error. + from fastapi import HTTPException + + seed_user() + jobs_route = data_recipe_jobs_module() + stale_gen = storage.credential_generation(secrets.token_urlsafe(64)) + + with pytest.raises(storage.CredentialRotated): + jobs_route._inject_local_providers(local_recipe(), local_recipe_request("t"), stale_gen) + + def _boom(*_a, **_k): + raise storage.CredentialRotated("revoked") + + jobs_route._inject_local_providers = _boom + payload = SimpleNamespace(recipe = local_recipe(), run = {}) + with pytest.raises(HTTPException) as excinfo: + jobs_route.create_job(payload, local_recipe_request("t"), ("unsloth", stale_gen)) + assert excinfo.value.status_code == 401 + + def test_desktop_login_rejects_invalid_secret(): seed_user(must_change_password = False) client = auth_client() @@ -358,7 +592,7 @@ def test_write_desktop_secret_file_is_0600_on_unix(tmp_path): studio_cli._write_auth_secret(path, "desktop-secret") - assert path.read_text() == "desktop-secret" + assert path.read_bytes() == b"desktop-secret\n" if platform.system() != "Windows": assert oct(path.stat().st_mode & 0o777) == "0o600" @@ -368,18 +602,31 @@ def test_reset_password_removes_desktop_secret_files(tmp_path, monkeypatch): from unsloth_cli.commands import studio as studio_cli auth_dir = tmp_path / "auth" - auth_dir.mkdir() - (auth_dir / "auth.db").write_text("db") - (auth_dir / ".bootstrap_password").write_text("boot") - (auth_dir / ".desktop_secret").write_text("new") monkeypatch.setattr(studio_cli, "STUDIO_HOME", tmp_path) + secret = studio_cli._create_desktop_secret_in_cli() + studio_cli._write_auth_secret(auth_dir / studio_cli.DESKTOP_SECRET_FILE, secret) + (auth_dir / studio_cli.BOOTSTRAP_PASSWORD_FILE).write_text("boot") result = CliRunner().invoke(studio_cli.studio_app, ["reset-password"]) - assert result.exit_code == 0 - assert not (auth_dir / "auth.db").exists() - assert not (auth_dir / ".bootstrap_password").exists() - assert not (auth_dir / ".desktop_secret").exists() + assert result.exit_code == 0, result.output + # The DB survives on purpose: a running server keeps serving from its admin row. + assert (auth_dir / "auth.db").exists() + assert not (auth_dir / studio_cli.BOOTSTRAP_PASSWORD_FILE).exists() + assert not (auth_dir / studio_cli.DESKTOP_SECRET_FILE).exists() + + conn = studio_cli._connect_auth_db() + try: + surviving = conn.execute( + "SELECT COUNT(*) FROM app_secrets WHERE key IN (?, ?)", + ( + studio_cli.DESKTOP_SECRET_HASH_KEY, + studio_cli.DESKTOP_SECRET_CREATED_AT_KEY, + ), + ).fetchone()[0] + finally: + conn.close() + assert surviving == 0 def test_reset_password_removes_desktop_secret_files_without_db(tmp_path, monkeypatch): @@ -525,7 +772,8 @@ if result.exit_code != 0: capture_output = True, ) assert result.returncode == 0, result.stderr + result.stdout - secret = (auth_dir / ".desktop_secret").read_text() + # Strip like the src-tauri readers do. + secret = (auth_dir / ".desktop_secret").read_text().strip() assert secret.startswith("desktop-") conn = sqlite3.connect(auth_dir / "auth.db") @@ -633,7 +881,7 @@ def test_update_password_clears_desktop_secret(): assert storage.validate_desktop_secret(raw) == storage.DEFAULT_ADMIN_USERNAME changed = storage.update_password(storage.DEFAULT_ADMIN_USERNAME, "new-admin-password") - assert changed is True + assert changed assert storage.validate_desktop_secret(raw) is None @@ -642,7 +890,7 @@ def test_update_password_on_unknown_user_leaves_desktop_secret_intact(): raw = storage.create_desktop_secret() changed = storage.update_password("not-a-user", "irrelevant") - assert changed is False + assert not changed assert storage.validate_desktop_secret(raw) == storage.DEFAULT_ADMIN_USERNAME diff --git a/studio/backend/tests/test_password_prompt_backstop.py b/studio/backend/tests/test_password_prompt_backstop.py index 3c2c1956f9..6c22532532 100644 --- a/studio/backend/tests/test_password_prompt_backstop.py +++ b/studio/backend/tests/test_password_prompt_backstop.py @@ -247,8 +247,8 @@ def test_lifespan_honors_bootstrap_suppression_in_source(): def test_clear_bootstrap_password_truncates_when_unlink_fails(monkeypatch, tmp_path): # If the file cannot be unlinked (Windows AV / read-only auth dir), clear must # truncate it so its stale plaintext cannot be re-seeded by - # generate_bootstrap_password() after a later reset-password deletes auth.db, - # which would re-validate the revoked bootstrap password. + # generate_bootstrap_password() if auth.db is ever recreated, which would + # re-validate the revoked bootstrap password. import pathlib pw_path = tmp_path / ".bootstrap_password" diff --git a/studio/backend/tests/test_rocm_multi_gpu_vram_system_wide.py b/studio/backend/tests/test_rocm_multi_gpu_vram_system_wide.py index bdafdeae9b..db89b02003 100644 --- a/studio/backend/tests/test_rocm_multi_gpu_vram_system_wide.py +++ b/studio/backend/tests/test_rocm_multi_gpu_vram_system_wide.py @@ -45,8 +45,20 @@ def _build_structlog_stub(): _maybe_stub("loggers", _build_loggers_stub) _maybe_stub("structlog", _build_structlog_stub) +import pytest + import utils.hardware.hardware as hw # noqa: E402 +# The DRM/KFD readers below are Linux-only in production: _rocm_linux_amdgpu_cards and +# _rocm_linux_sysfs_vram_by_pci_gb return early unless platform.system() is "Linux", and +# _rocm_kfd_gpu_pci_ids only ever globs /sys/class/kfd. Their fake sysfs tree needs PCI +# addresses like "0000:00:02.0" as directory names and POSIX separators in the paths the +# readers match; Windows permits neither, so the tree cannot be represented there. +linux_only = pytest.mark.skipif( + not sys.platform.startswith("linux"), + reason = "covers Linux-only DRM/KFD sysfs parsing driven by a fake /sys tree", +) + def _device( index, @@ -99,6 +111,7 @@ def _fake_drm(tmp_path, monkeypatch, cards): return card_paths +@linux_only def test_linux_vram_keyed_by_pci_excludes_foreign_adapters(monkeypatch, tmp_path): # Foreign (non-amdgpu) adapters contribute no entry, so they cannot shift ordinals. monkeypatch.setattr(hw.platform, "system", lambda: "Linux") @@ -117,6 +130,7 @@ def test_linux_vram_keyed_by_pci_excludes_foreign_adapters(monkeypatch, tmp_path } +@linux_only def test_linux_vram_omits_bad_cards_without_shifting(monkeypatch, tmp_path): # A zero-total card has no entry; identity keying means its absence renumbers nothing. monkeypatch.setattr(hw.platform, "system", lambda: "Linux") @@ -131,6 +145,7 @@ def test_linux_vram_omits_bad_cards_without_shifting(monkeypatch, tmp_path): assert hw._rocm_linux_sysfs_vram_by_pci_gb() == {"0000:41:00.0": (2.0, 16.0)} +@linux_only def test_linux_vram_omits_amd_card_without_vram_files(monkeypatch, tmp_path): # An APU with no mem_info_vram_* files has no entry; the discrete card keeps its address. monkeypatch.setattr(hw.platform, "system", lambda: "Linux") @@ -174,6 +189,7 @@ def _fake_kfd(tmp_path, monkeypatch, nodes): return node_paths +@linux_only def test_kfd_lists_gpu_nodes_in_device_order(monkeypatch, tmp_path): # The CPU node (simd_count 0) takes no ordinal; GPU nodes in node-id order are HIP's order. monkeypatch.setattr(hw.platform, "system", lambda: "Linux") @@ -189,12 +205,14 @@ def test_kfd_lists_gpu_nodes_in_device_order(monkeypatch, tmp_path): assert hw._rocm_kfd_gpu_pci_ids() == ["0000:03:00.0", "0000:41:00.0"] +@linux_only def test_kfd_decodes_domain_device_and_function(monkeypatch, tmp_path): monkeypatch.setattr(hw.platform, "system", lambda: "Linux") _fake_kfd(tmp_path, monkeypatch, [(1, 64, (0xC1 << 8) | (0x1F << 3) | 5, 0x1234, _AMD)]) assert hw._rocm_kfd_gpu_pci_ids() == ["1234:c1:1f.5"] +@linux_only def test_kfd_skips_non_amd_gpu_nodes(monkeypatch, tmp_path): # An NVIDIA KFD node is not a HIP device: it must take no ordinal, else it # shifts every AMD GPU and ROCm device 1 resolves to AMD GPU 0. @@ -212,6 +230,7 @@ def test_kfd_skips_non_amd_gpu_nodes(monkeypatch, tmp_path): assert hw._rocm_kfd_gpu_pci_ids() == ["0000:03:00.0", "0000:41:00.0"] +@linux_only def test_kfd_fails_closed_when_a_gpu_has_no_location(monkeypatch, tmp_path): # Dropping an unplaceable AMD GPU shifts later ordinals; fail closed for the whole map. monkeypatch.setattr(hw.platform, "system", lambda: "Linux") @@ -226,6 +245,7 @@ def test_kfd_fails_closed_when_a_gpu_has_no_location(monkeypatch, tmp_path): assert hw._rocm_kfd_gpu_pci_ids() == [] +@linux_only def test_kfd_fails_closed_when_a_node_is_unreadable(monkeypatch, tmp_path): # An unreadable node could be a GPU; assuming otherwise would shift ordinals. monkeypatch.setattr(hw.platform, "system", lambda: "Linux") @@ -241,6 +261,23 @@ def test_kfd_fails_closed_when_a_node_is_unreadable(monkeypatch, tmp_path): assert hw._rocm_kfd_gpu_pci_ids() == [] +@linux_only +def test_kfd_fails_closed_when_a_node_does_not_decode(monkeypatch, tmp_path): + # UnicodeDecodeError is a ValueError, so it slips past `except OSError` and + # would shift every later HIP ordinal. + monkeypatch.setattr(hw.platform, "system", lambda: "Linux") + paths = _fake_kfd( + tmp_path, + monkeypatch, + [ + (1, 304, (0x03 << 8) | 0, 0, _AMD), + (2, 304, (0x41 << 8) | 0, 0, _AMD), + ], + ) + (Path(paths[0]) / "properties").write_bytes(b"simd_count 304\nvendor_id \x80\xff\n") + assert hw._rocm_kfd_gpu_pci_ids() == [] + + def test_kfd_absent_yields_no_device_order(monkeypatch): monkeypatch.setattr(hw.glob, "glob", lambda pattern: []) assert hw._rocm_kfd_gpu_pci_ids() == [] @@ -422,6 +459,10 @@ def test_visible_utilization_rocm_fallback_overlays(monkeypatch): ): monkeypatch.delenv(_var, raising = False) monkeypatch.setattr(hw, "IS_ROCM", True) + # No AMD adapter data on this host. On Windows this branch runs ahead of the torch + # fallback under test, and probing it imports torch, which the CI runner does not + # install. Off Windows the real function is never reached, so this changes nothing. + monkeypatch.setattr(hw, "_rocm_windows_per_device_vram", lambda ids: []) monkeypatch.setattr(hw, "get_device", lambda: hw.DeviceType.CUDA) monkeypatch.setattr(hw, "_smi_query", lambda *a, **k: None) # amd-smi unavailable monkeypatch.setattr( @@ -450,6 +491,10 @@ def test_visible_utilization_rocm_fallback_overlays(monkeypatch): def test_visible_utilization_relative_index_skips_overlay(monkeypatch): # UUID/MIG mask gives relative indices; the overlay matches physical index, so it must not run. monkeypatch.setattr(hw, "IS_ROCM", True) + # No AMD adapter data on this host. On Windows this branch runs ahead of the torch + # fallback under test, and probing it imports torch, which the CI runner does not + # install. Off Windows the real function is never reached, so this changes nothing. + monkeypatch.setattr(hw, "_rocm_windows_per_device_vram", lambda ids: []) monkeypatch.setattr(hw, "get_device", lambda: hw.DeviceType.CUDA) monkeypatch.setattr(hw, "_smi_query", lambda *a, **k: None) monkeypatch.setattr( diff --git a/studio/backend/tests/test_studio_pid_files.py b/studio/backend/tests/test_studio_pid_files.py new file mode 100644 index 0000000000..df2c8e87f8 --- /dev/null +++ b/studio/backend/tests/test_studio_pid_files.py @@ -0,0 +1,568 @@ +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 + +"""Per-port PID files, so `unsloth studio stop` can find every server. + +Imports run.py directly, so run under the Unsloth venv. +""" + +from __future__ import annotations + +import os +import sys +from pathlib import Path +from types import SimpleNamespace + +import pytest + +_BACKEND = Path(__file__).resolve().parents[1] +if str(_BACKEND) not in sys.path: + sys.path.insert(0, str(_BACKEND)) + +import run # noqa: E402 + +# Captured before the autouse fixture stubs them, for the tests that exercise them. +_REAL_IS_STUDIO_BACKEND = run._pid_is_studio_backend +_REAL_PID_ALIVE = run._pid_alive + + +@pytest.fixture(autouse = True) +def isolated_root(tmp_path, monkeypatch): + monkeypatch.setattr(run, "_studio_root", lambda: tmp_path) + monkeypatch.setattr(run, "_PID_FILE", tmp_path / "studio.pid") + monkeypatch.setattr(run, "_OWN_PID_FILE", None) + monkeypatch.setattr(run, "_pid_alive", lambda pid: True) + monkeypatch.setattr(run, "_pid_is_studio_backend", lambda pid, created_times = (): True) + yield + + +def _files(tmp_path): + return sorted(p.name for p in tmp_path.glob("studio-*.pid")) + + +def _pid_of(path): + return path.read_text(encoding = "utf-8").splitlines()[0] + + +def test_write_pid_file_records_port_and_pid(tmp_path): + run._write_pid_file(8901) + + assert _files(tmp_path) == [f"studio-8901-{os.getpid()}.pid"] + assert _pid_of(tmp_path / f"studio-8901-{os.getpid()}.pid") == str(os.getpid()) + + +def test_write_pid_file_records_the_start_time(tmp_path): + # Pins the record to this process, so a reused PID isn't mistaken for it. + run._write_pid_file(8901) + + record = run._read_pid_record(tmp_path / f"studio-8901-{os.getpid()}.pid") + + assert record[0] == os.getpid() + assert record[1] == pytest.approx(run._process_create_time(os.getpid())) + + +def test_write_pid_file_keeps_the_legacy_file_a_bare_pid(tmp_path): + # An older CLI's `stop` reads studio.pid and expects only digits. + run._write_pid_file(8901) + + assert (tmp_path / "studio.pid").read_text(encoding = "utf-8") == str(os.getpid()) + + +def test_second_port_does_not_clobber_the_first(tmp_path): + (tmp_path / "studio-8901-8550.pid").write_text("8550", encoding = "utf-8") + + run._write_pid_file(8902) + + assert _pid_of(tmp_path / "studio-8901-8550.pid") == "8550" + assert (tmp_path / f"studio-8902-{os.getpid()}.pid").exists() + + +def test_same_port_on_two_binds_does_not_clobber(tmp_path): + # 127.0.0.1:8888 and ::1:8888 can both listen; one file per port would lose one. + (tmp_path / "studio-8888-8550.pid").write_text("8550", encoding = "utf-8") + + run._write_pid_file(8888) + + assert len(_files(tmp_path)) == 2 + + +def test_remove_pid_file_only_removes_our_own(tmp_path, monkeypatch): + run._write_pid_file(8901) + (tmp_path / "studio-8902-8600.pid").write_text("8600", encoding = "utf-8") + # Nothing to hand the legacy pointer to, so it goes away with us. + monkeypatch.setattr(run, "_pid_alive", lambda pid: pid == os.getpid()) + + run._remove_pid_file() + + assert _files(tmp_path) == ["studio-8902-8600.pid"] + assert not (tmp_path / "studio.pid").exists() + + +def test_the_legacy_pointer_moves_to_a_live_sibling(tmp_path): + # Only one server owns studio.pid. Deleting it on our way out would leave an + # older CLI, which reads nothing else, unable to stop the sibling still up. + run._write_pid_file(8901) + (tmp_path / "studio-8902-8600.pid").write_text("8600", encoding = "utf-8") + + run._remove_pid_file() + + assert (tmp_path / "studio.pid").read_text(encoding = "utf-8").strip() == "8600" + + +def test_the_legacy_pointer_is_not_handed_to_a_dead_sibling(tmp_path, monkeypatch): + run._write_pid_file(8901) + (tmp_path / "studio-8902-8600.pid").write_text("8600", encoding = "utf-8") + monkeypatch.setattr(run, "_pid_is_studio_backend", lambda pid, created_times = (): False) + + run._remove_pid_file() + + assert not (tmp_path / "studio.pid").exists() + + +def test_remove_pid_file_leaves_a_reused_entry_alone(tmp_path): + run._write_pid_file(8901) + own = tmp_path / f"studio-8901-{os.getpid()}.pid" + own.write_text("999999", encoding = "utf-8") + + run._remove_pid_file() + + assert own.read_text(encoding = "utf-8") == "999999" + + +def test_windows_liveness_does_not_call_every_pid_alive(monkeypatch): + # os.kill(pid, 0) raises OSError for every pid on Windows, so without the + # tasklist fallback a stale record would block its port forever. + import subprocess + + monkeypatch.setattr(run, "_pid_alive", _REAL_PID_ALIVE) + monkeypatch.setitem(sys.modules, "psutil", None) + monkeypatch.setattr(sys, "platform", "win32") + monkeypatch.setattr( + subprocess, "run", lambda *a, **k: SimpleNamespace(stdout = '"python.exe","8550",...') + ) + + assert run._pid_alive(8550) is True + assert run._pid_alive(9999) is False + + +def test_windows_liveness_keeps_the_record_when_tasklist_fails(monkeypatch): + # Unconfirmed must mean keep, matching the CLI's _pid_alive. Pruning a live + # server's record lets the next launch fall back past it and strand it, which + # is the bug this file exists to fix; a stale record costs one clear abort. + import subprocess + + def _boom(*a, **k): + raise OSError("tasklist missing") + + monkeypatch.setattr(run, "_pid_alive", _REAL_PID_ALIVE) + monkeypatch.setitem(sys.modules, "psutil", None) + monkeypatch.setattr(sys, "platform", "win32") + monkeypatch.setattr(subprocess, "run", _boom) + + assert run._pid_alive(8550) is True + + +def test_read_pid_record_parses_pid_time_and_address(tmp_path): + (tmp_path / "r.pid").write_text("8550\n111.5\n127.0.0.1", encoding = "utf-8") + + assert run._read_pid_record(tmp_path / "r.pid") == (8550, 111.5, "127.0.0.1") + + +def test_read_pid_record_tolerates_a_bare_pid(tmp_path): + (tmp_path / "r.pid").write_text("8550", encoding = "utf-8") + + assert run._read_pid_record(tmp_path / "r.pid") == (8550, None, None) + + +def test_read_pid_record_rejects_pid_zero_and_init(tmp_path): + # kill(0) signals our whole process group. + (tmp_path / "zero.pid").write_text("0", encoding = "utf-8") + (tmp_path / "init.pid").write_text("1", encoding = "utf-8") + + assert run._read_pid_record(tmp_path / "zero.pid") is None + assert run._read_pid_record(tmp_path / "init.pid") is None + + +def test_read_pid_record_rejects_a_corrupt_file(tmp_path): + (tmp_path / "r.pid").write_text("not-a-pid", encoding = "utf-8") + + assert run._read_pid_record(tmp_path / "r.pid") is None + + +def test_graceful_shutdown_drops_the_record_last(monkeypatch): + # Cleanup can take seconds while the server is still alive. Dropping the record + # first leaves a retried `stop` or a new launch unable to find it. + order = [] + monkeypatch.setattr(run, "_remove_pid_file", lambda: order.append("remove_record")) + + class _Server: + def __setattr__(self, name, value): + order.append("release_socket") + + run._graceful_shutdown(_Server()) + + assert order == ["release_socket", "remove_record"] + + +def test_own_studio_on_port_is_found_without_psutil(tmp_path, monkeypatch): + # psutil is optional; a listener scan finds nothing without it, so detection + # must come from our own records or we silently start a duplicate. + monkeypatch.setitem(sys.modules, "psutil", None) + (tmp_path / "studio-8901-8550.pid").write_text("8550\n\n127.0.0.1", encoding = "utf-8") + + assert run._own_studio_on_port(8901, "127.0.0.1") == 8550 + + +def test_no_record_for_the_port_means_no_own_studio(tmp_path): + # jupyter-lab on 8888 must keep the fallback, not abort the launch. + (tmp_path / "studio-8901-8550.pid").write_text("8550", encoding = "utf-8") + + assert run._own_studio_on_port(8888, "127.0.0.1") is None + + +def test_own_studio_on_port_prunes_a_dead_record(tmp_path, monkeypatch): + monkeypatch.setattr(run, "_pid_alive", lambda pid: False) + (tmp_path / "studio-8901-8550.pid").write_text("8550", encoding = "utf-8") + + assert run._own_studio_on_port(8901, "127.0.0.1") is None + assert not (tmp_path / "studio-8901-8550.pid").exists() + + +def test_a_reused_pid_is_not_treated_as_our_studio(tmp_path, monkeypatch): + # Stale record + the OS handing that PID to something else must not abort. + monkeypatch.setattr(run, "_pid_is_studio_backend", lambda pid, created_times = (): False) + (tmp_path / "studio-8901-8550.pid").write_text("8550", encoding = "utf-8") + + assert run._own_studio_on_port(8901, "127.0.0.1") is None + + +def test_an_unverifiable_record_still_blocks_a_duplicate(tmp_path, monkeypatch): + # Can't tell: refusing with a clear message beats a silent second instance. + monkeypatch.setattr(run, "_pid_is_studio_backend", lambda pid, created_times = (): True) + (tmp_path / "studio-8901-8550.pid").write_text("8550", encoding = "utf-8") + + assert run._own_studio_on_port(8901, "127.0.0.1") == 8550 + + +def test_start_time_mismatch_rejects_a_reused_pid(monkeypatch): + monkeypatch.setattr(run, "_pid_is_studio_backend", _REAL_IS_STUDIO_BACKEND) + monkeypatch.setattr(run, "_process_create_time", lambda pid: 999.0) + + assert run._pid_is_studio_backend(8550, [111.5]) is False + assert run._pid_is_studio_backend(8550, [999.0]) is True + + +def test_a_stale_record_does_not_veto_a_live_server_sharing_the_pid(monkeypatch): + # Crash leaves studio-8888-1234.pid, the OS reuses 1234 for a new server on + # another port. Keeping only the first timestamp would reject the live one. + monkeypatch.setattr(run, "_pid_is_studio_backend", _REAL_IS_STUDIO_BACKEND) + monkeypatch.setattr(run, "_process_create_time", lambda pid: 999.0) + + assert run._pid_is_studio_backend(1234, [111.5, 999.0]) is True + assert run._pid_is_studio_backend(1234, [111.5, 222.5]) is False + + +def test_a_stale_record_on_another_port_does_not_hide_a_live_server(tmp_path, monkeypatch): + # 1234 was reused: the stale 8888 record must not stop us seeing 9000. + monkeypatch.setattr(run, "_pid_is_studio_backend", _REAL_IS_STUDIO_BACKEND) + monkeypatch.setattr(run, "_process_create_time", lambda pid: 999.0) + (tmp_path / "studio-8888-1234.pid").write_text("1234\n111.5\n", encoding = "utf-8") + (tmp_path / "studio-9000-1234.pid").write_text("1234\n999.0\n", encoding = "utf-8") + + assert run._own_studio_on_port(8888, "127.0.0.1") is None + assert run._own_studio_on_port(9000, "127.0.0.1") == 1234 + + +def test_a_start_time_is_the_only_thing_that_disproves_a_record(monkeypatch): + monkeypatch.setattr(run, "_pid_is_studio_backend", _REAL_IS_STUDIO_BACKEND) + monkeypatch.setattr(run, "_process_create_time", lambda pid: 999.0) + + assert run._pid_is_studio_backend(8550, [999.0]) is True + assert run._pid_is_studio_backend(8550, [111.5]) is False + + +def test_a_bare_run_py_command_line_is_not_rejected(monkeypatch): + # `cd studio/backend && python run.py --port 8901` has no "studio" or "unsloth" + # in argv. Guessing from the command line called that "not ours". + monkeypatch.setattr(run, "_pid_is_studio_backend", _REAL_IS_STUDIO_BACKEND) + + class _FakeProcess: + def __init__(self, pid): + self.pid = pid + + def cmdline(self): + return ["python", "run.py", "--port", "8901"] + + def create_time(self): + return 111.5 + + monkeypatch.setitem(sys.modules, "psutil", SimpleNamespace(Process = _FakeProcess)) + + assert run._pid_is_studio_backend(8550) is True + + +def test_an_untimed_legacy_record_is_trusted(monkeypatch): + # `python run.py --port 8901` has no telltale argv, so guessing from the + # command line rejected real servers. Only a start time can disprove one. + monkeypatch.setattr(run, "_pid_is_studio_backend", _REAL_IS_STUDIO_BACKEND) + monkeypatch.setattr(run, "_process_create_time", lambda pid: 999.0) + + assert run._pid_is_studio_backend(8550) is True + assert run._pid_is_studio_backend(8550, [None]) is True + + +def test_the_untimed_legacy_record_does_not_cancel_a_timed_one(monkeypatch): + # Mirrors _pid_is_studio_server in the CLI. An untimed record carries no + # information, so it must not overrule a start time that says "not ours" -- + # every current server writes one of each, which made the check inert. + monkeypatch.setattr(run, "_pid_is_studio_backend", _REAL_IS_STUDIO_BACKEND) + monkeypatch.setattr(run, "_process_create_time", lambda pid: 999.0) + + assert run._pid_is_studio_backend(8550, [111.5, None]) is False + assert run._pid_is_studio_backend(8550, [111.5, 999.0]) is True + + +def test_a_legacy_server_on_the_port_is_recognised(tmp_path, monkeypatch): + # Pre-upgrade servers wrote only studio.pid. Falling back past one strands it + # and then overwrites its record. + monkeypatch.setattr(run, "_get_pid_on_port", lambda p: (8550, "python")) + (tmp_path / "studio.pid").write_text("8550", encoding = "utf-8") + + assert run._own_studio_on_port(8901, "127.0.0.1") == 8550 + + +def test_a_legacy_record_for_a_different_listener_falls_back(tmp_path, monkeypatch): + # jupyter holds the port; the legacy server is elsewhere. Keep falling back. + monkeypatch.setattr(run, "_get_pid_on_port", lambda p: (117, "jupyter-lab")) + (tmp_path / "studio.pid").write_text("8550", encoding = "utf-8") + + assert run._own_studio_on_port(8901, "127.0.0.1") is None + + +def test_an_unknowable_listener_treats_the_legacy_record_as_ours(tmp_path, monkeypatch): + # No psutil: _get_pid_on_port can't say. Refusing beats a silent duplicate. + monkeypatch.setattr(run, "_get_pid_on_port", lambda p: None) + (tmp_path / "studio.pid").write_text("8550", encoding = "utf-8") + + assert run._own_studio_on_port(8901, "127.0.0.1") == 8550 + + +def test_a_dead_legacy_record_falls_back(tmp_path, monkeypatch): + monkeypatch.setattr(run, "_pid_alive", lambda pid: False) + monkeypatch.setattr(run, "_get_pid_on_port", lambda p: None) + (tmp_path / "studio.pid").write_text("8550", encoding = "utf-8") + + assert run._own_studio_on_port(8901, "127.0.0.1") is None + + +def test_a_stale_per_port_record_does_not_mask_a_legacy_server(tmp_path, monkeypatch): + # Crashed current build left studio-8901-8550.pid; 8550 was then reused by a + # pre-upgrade server recorded only in studio.pid. The stale record must not + # count as "port already known" and send us falling back past the live one. + monkeypatch.setattr(run, "_pid_is_studio_backend", _REAL_IS_STUDIO_BACKEND) + monkeypatch.setattr(run, "_process_create_time", lambda pid: 999.0) + monkeypatch.setattr(run, "_get_pid_on_port", lambda p: (8550, "python")) + (tmp_path / "studio-8901-8550.pid").write_text("8550\n111.5\n127.0.0.1", encoding = "utf-8") + (tmp_path / "studio.pid").write_text("8550", encoding = "utf-8") + + assert run._own_studio_on_port(8901, "127.0.0.1") == 8550 + + +def test_a_current_server_elsewhere_does_not_block_a_foreign_port(tmp_path, monkeypatch): + # Current builds write studio.pid too. Without psutil the legacy check can't + # see the listener, so it must not claim our 8901 server holds jupyter's 8888. + monkeypatch.setattr(run, "_get_pid_on_port", lambda p: None) + (tmp_path / "studio-8901-5000.pid").write_text("5000\n\n127.0.0.1", encoding = "utf-8") + (tmp_path / "studio.pid").write_text("5000", encoding = "utf-8") + + assert run._own_studio_on_port(8888, "127.0.0.1") is None + + +def test_a_per_port_record_is_preferred_over_the_legacy_one(tmp_path, monkeypatch): + monkeypatch.setattr(run, "_get_pid_on_port", lambda p: (8550, "python")) + (tmp_path / "studio-8901-8600.pid").write_text("8600\n\n127.0.0.1", encoding = "utf-8") + (tmp_path / "studio.pid").write_text("8550", encoding = "utf-8") + + assert run._own_studio_on_port(8901, "127.0.0.1") == 8600 + + +def test_our_studio_on_another_bind_address_does_not_abort(tmp_path): + # Our server holds ::1:8889; binding 127.0.0.1:8889 is not a conflict with us, + # so fall through to the next port instead of refusing. + (tmp_path / "studio-8889-8550.pid").write_text("8550\n\n::1", encoding = "utf-8") + + assert run._own_studio_on_port(8889, "127.0.0.1") is None + assert run._own_studio_on_port(8889, "::1") == 8550 + + +def test_address_matching(tmp_path): + assert run._addresses_collide("0.0.0.0", "127.0.0.1", 8889) is True + assert run._addresses_collide("127.0.0.1", "0.0.0.0", 8889) is True + assert run._addresses_collide("127.0.0.1", "127.0.0.1", 8889) is True + assert run._addresses_collide("::1", "127.0.0.1", 8889) is False + # An unrecorded address is unknown, so assume a conflict. + assert run._addresses_collide(None, "127.0.0.1", 8889) is True + + +def test_a_hostname_resolves_the_same_way_the_bind_does(tmp_path): + # `localhost` and the address _is_port_free actually binds must agree, or a + # recorded server is missed and a duplicate starts. + recorded = ",".join(sorted(run._bind_addresses("localhost", 8889))) + + assert run._addresses_collide(recorded, "localhost", 8889) is True + + +def test_a_hostname_records_every_address_it_resolves_to(tmp_path): + # `localhost` binds 127.0.0.1 AND ::1. Recording only the first lets a later + # launch on the other literal miss us and start a duplicate. + addrs = run._bind_addresses("localhost", 8889) + recorded = ",".join(sorted(addrs)) + + for literal in addrs: + assert run._addresses_collide(recorded, literal, 8889) is True + + +def test_a_multi_address_record_matches_either_literal(tmp_path): + recorded = "127.0.0.1,::1" + + assert run._addresses_collide(recorded, "127.0.0.1", 8889) is True + assert run._addresses_collide(recorded, "::1", 8889) is True + assert run._addresses_collide("127.0.0.1", "::1", 8889) is False + + +def test_fallback_aborts_on_our_own_server_further_up_the_range(tmp_path, monkeypatch): + # jupyter holds 8888, our server holds 8889: skipping to 8890 is the duplicate. + (tmp_path / "studio-8889-8550.pid").write_text("8550\n\n127.0.0.1", encoding = "utf-8") + monkeypatch.setattr(run, "_is_port_free", lambda host, p: p >= 8890) + + with pytest.raises(SystemExit) as excinfo: + run._find_free_port("127.0.0.1", 8889, avoid_own_studio = True) + + assert excinfo.value.code == 1 + + +def test_fallback_still_skips_foreign_processes(tmp_path, monkeypatch): + # No record for 8889, so the blocker is not ours: keep falling back. + monkeypatch.setattr(run, "_is_port_free", lambda host, p: p >= 8890) + + assert run._find_free_port("127.0.0.1", 8889, avoid_own_studio = True) == 8890 + + +def test_the_requested_port_is_kept_when_it_is_free(monkeypatch): + monkeypatch.setattr(run, "_is_port_free", lambda host, p: True) + + assert run._resolve_port("127.0.0.1", 8888) == 8888 + + +def test_our_own_server_on_the_requested_port_aborts_rather_than_falling_back( + tmp_path, monkeypatch +): + # The reported bug: 8888 is ours, so falling back to 8889 is the duplicate + # that leaves 8888 serving with nothing recording it. + monkeypatch.setattr(run, "_is_port_free", lambda host, p: p != 8888) + (tmp_path / "studio-8888-8550.pid").write_text("8550\n\n127.0.0.1", encoding = "utf-8") + + with pytest.raises(SystemExit) as excinfo: + run._resolve_port("127.0.0.1", 8888) + + assert excinfo.value.code == 1 + + +def test_a_foreign_process_on_the_requested_port_still_falls_back(monkeypatch): + # jupyter-lab on 8888 must not stop Unsloth starting on 8889. + monkeypatch.setattr(run, "_is_port_free", lambda host, p: p != 8888) + + assert run._resolve_port("127.0.0.1", 8888) == 8889 + + +def test_a_caller_that_reads_the_port_back_keeps_the_plain_fallback(tmp_path, monkeypatch): + # api-only callers (the desktop app via TAURI_PORT, `studio run` via + # app.state.server_port) follow us to the new port, so aborting there only + # turns a working launch into a crash the desktop app reports as "stopped + # unexpectedly". Both servers are still recorded, so `stop` finds them. + monkeypatch.setattr(run, "_is_port_free", lambda host, p: p != 8888) + (tmp_path / "studio-8888-8550.pid").write_text("8550\n\n127.0.0.1", encoding = "utf-8") + + assert run._resolve_port("127.0.0.1", 8888, avoid_own_studio = False) == 8889 + + +def test_the_recorded_address_is_every_address_the_bind_resolves_to(tmp_path): + # The only test that runs the writer with a real host. Recording `host` + # verbatim, or dropping the line, passes every other test here and silently + # stops matching a launch that spells the same interface differently. + run._write_pid_file(8901, "localhost") + + record = run._read_pid_record(tmp_path / f"studio-8901-{os.getpid()}.pid") + + assert record[2] is not None, "no bind address recorded" + assert set(record[2].split(",")) == run._bind_addresses("localhost", 8901) + + +def test_a_server_started_on_a_hostname_is_found_again_by_ip(tmp_path): + run._write_pid_file(8901, "localhost") + + for literal in run._bind_addresses("localhost", 8901): + assert run._own_studio_on_port(8901, literal) == os.getpid() + + +def test_bind_addresses_keeps_every_family_a_hostname_resolves_to(monkeypatch): + # Independent oracle: the sibling test derives its expectation from this + # function's own output, so dropping a family would pass it. + import socket + monkeypatch.setattr( + socket, + "getaddrinfo", + lambda *a, **k: [ + (socket.AF_INET, socket.SOCK_STREAM, 6, "", ("127.0.0.1", 8889)), + (socket.AF_INET6, socket.SOCK_STREAM, 6, "", ("::1", 8889, 0, 0)), + ], + ) + + assert run._bind_addresses("localhost", 8889) == {"127.0.0.1", "::1"} + + +def test_the_legacy_file_is_written_even_when_the_per_port_record_fails(tmp_path, monkeypatch): + # A studio root that cannot take a new entry used to leave the server + # recorded nowhere at all, so the CLI could not stop it. studio.pid is an + # overwrite of an existing path, so it can still succeed and must be tried. + blocked = tmp_path / "not-a-directory" + blocked.write_text("", encoding = "utf-8") + monkeypatch.setattr( + run, "_pid_file_for_port", lambda port: blocked / f"studio-{port}-{os.getpid()}.pid" + ) + + run._write_pid_file(8901, "127.0.0.1") + + assert (tmp_path / "studio.pid").read_text(encoding = "utf-8") == str(os.getpid()) + assert run._OWN_PID_FILE is None + + +def test_a_record_whose_pid_is_not_ascii_digits_is_discarded(tmp_path): + # A superscript two passes isdigit() but int() rejects it, so that gate alone + # let a ValueError escape into every caller of _read_pid_record. + (tmp_path / "r.pid").write_text("²", encoding = "utf-8") + + assert run._read_pid_record(tmp_path / "r.pid") is None + + +def test_the_legacy_file_is_not_taken_from_a_live_server(tmp_path): + # A pre-upgrade server is recorded in studio.pid and nowhere else, so a + # second launch overwriting it is exactly what strands it. That is the + # orphan this file exists to prevent, reached from the other direction. + (tmp_path / "studio.pid").write_text("8550", encoding = "utf-8") + + run._write_pid_file(8902, "127.0.0.1") + + assert (tmp_path / "studio.pid").read_text(encoding = "utf-8") == "8550" + assert (tmp_path / f"studio-8902-{os.getpid()}.pid").exists() + + +def test_the_legacy_file_is_taken_over_from_a_dead_server(tmp_path, monkeypatch): + # A stale record must not keep the pointer forever, or an older CLI could + # never stop anything again. + monkeypatch.setattr(run, "_pid_alive", lambda pid: False) + (tmp_path / "studio.pid").write_text("8550", encoding = "utf-8") + + run._write_pid_file(8902, "127.0.0.1") + + assert (tmp_path / "studio.pid").read_text(encoding = "utf-8") == str(os.getpid()) diff --git a/studio/backend/tests/test_text_io_encoding.py b/studio/backend/tests/test_text_io_encoding.py new file mode 100644 index 0000000000..7eae3c7fef --- /dev/null +++ b/studio/backend/tests/test_text_io_encoding.py @@ -0,0 +1,809 @@ +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 + +"""Text I/O must name its encoding, or Windows silently uses the ANSI codepage. + +``open()``, ``Path.read_text()`` and ``subprocess(text = True)`` fall back to +``locale.getencoding()`` when no ``encoding`` is passed. On Windows that is +cp1252 (or cp932, cp1251, ... by system locale), not UTF-8, so a chat template, +model config or path containing ``ä ö ü → 世`` mojibakes or raises +``UnicodeDecodeError`` mid-load. Studio's files are UTF-8, so say so. +""" + +from __future__ import annotations + +import ast +import importlib.util +import json +import os +from pathlib import Path +from types import SimpleNamespace + +import pytest + + +BACKEND_ROOT = Path(__file__).resolve().parent.parent + +# Not runtime source. Shipped plugins under plugins/*/src are, so only builds are skipped. +_SKIPPED_DIRS = ("node_modules", "build", "tests", "__pycache__") + +# Path.open()'s signature is what tells it apart from other libraries' open(), +# e.g. fitz.open(stream=...) and av.open(..., metadata_errors=...). +_FILE_MODE_CHARS = set("rwxabt+") +_PATH_OPEN_ARGS = ("mode", "buffering", "encoding", "errors", "newline") +_PATH_OPEN_KWARGS = set(_PATH_OPEN_ARGS) +_PATH_OPEN_ENCODING_ARG = _PATH_OPEN_ARGS.index("encoding") + +_SUBPROCESS_CALLS = {"run", "Popen", "check_output", "check_call", "call"} + +# open(file, mode, buffering, encoding, ...), and os.fdopen forwards the same +# signature with a descriptor in place of the path. +_OPEN_ENCODING_ARG = 3 + + +def _studio_sources() -> list[Path]: + return [ + path + for path in sorted(BACKEND_ROOT.rglob("*.py")) + if not any(part in _SKIPPED_DIRS for part in path.relative_to(BACKEND_ROOT).parts) + ] + + +def _has_keyword(node: ast.Call, name: str) -> bool: + return any(keyword.arg == name for keyword in node.keywords) + + +def _mode_is_binary(node: ast.Call) -> bool: + mode: str | None = None + if len(node.args) >= 2 and isinstance(node.args[1], ast.Constant): + value = node.args[1].value + mode = value if isinstance(value, str) else None + for keyword in node.keywords: + if keyword.arg == "mode" and isinstance(keyword.value, ast.Constant): + value = keyword.value.value + if isinstance(value, str): + mode = value + return bool(mode and "b" in mode) + + +def _open_has_encoding(node: ast.Call) -> bool: + """open()/os.fdopen() also take encoding positionally: open(p, "w", 1, "utf-8").""" + return _has_keyword(node, "encoding") or len(node.args) > _OPEN_ENCODING_ARG + + +def _path_open_mode(node: ast.Call) -> str | None: + if node.args and isinstance(node.args[0], ast.Constant): + value = node.args[0].value + if isinstance(value, str): + return value + for keyword in node.keywords: + if keyword.arg == "mode" and isinstance(keyword.value, ast.Constant): + value = keyword.value.value + if isinstance(value, str): + return value + return None + + +def _is_path_open(node: ast.Call) -> bool: + """True only for calls matching ``Path.open``'s signature.""" + if len(node.args) > len(_PATH_OPEN_ARGS): + return False + if any(k.arg not in _PATH_OPEN_KWARGS for k in node.keywords): + return False + mode = _path_open_mode(node) + if mode is not None: + return bool(mode) and set(mode) <= _FILE_MODE_CHARS + return not node.args + + +def _path_open_has_encoding(node: ast.Call) -> bool: + """Path.open() also takes encoding positionally: open("w", 1, "utf-8").""" + return _has_keyword(node, "encoding") or len(node.args) > _PATH_OPEN_ENCODING_ARG + + +def _call_name(node: ast.Call) -> str | None: + func = node.func + if isinstance(func, ast.Name): + return func.id + if isinstance(func, ast.Attribute): + return func.attr + return None + + +def _subprocess_names(tree: ast.AST) -> set[str]: + """Names subprocess is reachable under here, e.g. `import subprocess as _sp`.""" + names = set() + for node in ast.walk(tree): + if isinstance(node, ast.Import): + for alias in node.names: + if alias.name == "subprocess": + names.add(alias.asname or alias.name) + return names + + +def _subprocess_aliases(tree: ast.AST, names: set[str]) -> set[str]: + """Plain names bound to a subprocess callable, called without the module. + + ``install_wheel(run = subprocess.run)`` calls its injected ``run`` as a bare + name, so matching only the attribute form leaves those installer calls + unguarded. Imports, assignments and parameter defaults all bind one. + """ + + def _is_bound(value: ast.expr | None) -> bool: + return ( + isinstance(value, ast.Attribute) + and value.attr in _SUBPROCESS_CALLS + and isinstance(value.value, ast.Name) + and value.value.id in names + ) + + aliases: set[str] = set() + for node in ast.walk(tree): + if isinstance(node, ast.ImportFrom) and node.module == "subprocess": + aliases.update(a.asname or a.name for a in node.names if a.name in _SUBPROCESS_CALLS) + elif isinstance(node, ast.Assign) and _is_bound(node.value): + aliases.update(t.id for t in node.targets if isinstance(t, ast.Name)) + elif isinstance(node, ast.AnnAssign) and _is_bound(node.value): + if isinstance(node.target, ast.Name): + aliases.add(node.target.id) + elif isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)): + args = node.args + positional = args.posonlyargs + args.args + # Defaults cover the tail of the positional parameters; kw_defaults + # is aligned with kwonlyargs already, holding None where absent. + padded = [None] * (len(positional) - len(args.defaults)) + list(args.defaults) + pairs = list(zip(positional, padded)) + list(zip(args.kwonlyargs, args.kw_defaults)) + aliases.update(arg.arg for arg, default in pairs if _is_bound(default)) + return aliases + + +def _is_subprocess_call(node: ast.Call, names: set[str], aliases: set[str]) -> bool: + func = node.func + if isinstance(func, ast.Name): + return func.id in aliases + if not isinstance(func, ast.Attribute) or func.attr not in _SUBPROCESS_CALLS: + return False + value = func.value + return isinstance(value, ast.Name) and value.id in names + + +def _text_mode_subprocess(node: ast.Call) -> bool: + for keyword in node.keywords: + if keyword.arg not in ("text", "universal_newlines"): + continue + if isinstance(keyword.value, ast.Constant) and keyword.value.value is True: + return True + return False + + +def _text_mode_dict(node: ast.Dict) -> bool: + """A ``{"text": True, ...}`` literal with no "encoding" key.""" + keys = [k.value for k in node.keys if isinstance(k, ast.Constant)] + if "encoding" in keys: + return False + for key, value in zip(node.keys, node.values): + if not isinstance(key, ast.Constant) or key.value not in ( + "text", + "universal_newlines", + ): + continue + if isinstance(value, ast.Constant) and value.value is True: + return True + return False + + +def _splatted_names(tree: ast.AST) -> set[str]: + """Names handed to a call as ``**name``.""" + names = set() + for node in ast.walk(tree): + if isinstance(node, ast.Call): + for keyword in node.keywords: + if keyword.arg is None and isinstance(keyword.value, ast.Name): + names.add(keyword.value.id) + return names + + +def _encoding_assigned_later(tree: ast.AST, name: str) -> bool: + """``name["encoding"] = ...`` somewhere, so the literal need not carry it.""" + for node in ast.walk(tree): + if not isinstance(node, ast.Subscript) or not isinstance(node.ctx, ast.Store): + continue + target, key = node.value, node.slice + if isinstance(target, ast.Name) and target.id == name: + if isinstance(key, ast.Constant) and key.value == "encoding": + return True + return False + + +def _splatted_kwargs_offenders(tree: ast.AST) -> list[ast.Dict]: + """Text-mode kwargs built in a dict and splatted into a call. + + Kwargs are collected in a dict and splatted (``run(cmd, **run_kwargs)``) + where a branch has to add a timeout or an env, and the call is often through + a helper, so neither the callee nor the keywords are visible at the call + site. Only dicts that reach a call this way are judged: an unrelated payload + that happens to carry ``"text": True`` is not subprocess configuration. + """ + found = [] + # ``run(cmd, **{...})``: the literal is at the call already. + for node in ast.walk(tree): + if not isinstance(node, ast.Call): + continue + for keyword in node.keywords: + if keyword.arg is None and isinstance(keyword.value, ast.Dict): + if _text_mode_dict(keyword.value): + found.append(keyword.value) + splatted = _splatted_names(tree) + if not splatted: + return found + for node in ast.walk(tree): + targets = [] + if isinstance(node, ast.Assign): + targets = [t for t in node.targets if isinstance(t, ast.Name)] + elif isinstance(node, ast.AnnAssign) and isinstance(node.target, ast.Name): + targets = [node.target] + if not targets or not isinstance(node.value, ast.Dict): + continue + if not _text_mode_dict(node.value): + continue + for target in targets: + if target.id in splatted and not _encoding_assigned_later(tree, target.id): + found.append(node.value) + break + return found + + +def _offenders(path: Path) -> list[str]: + source = path.read_text(encoding = "utf-8") + tree = ast.parse(source, filename = str(path)) + subprocess_names = _subprocess_names(tree) + subprocess_aliases = _subprocess_aliases(tree, subprocess_names) + found: list[str] = [] + for node in _splatted_kwargs_offenders(tree): + found.append( + f"{path.name}:{node.lineno}: subprocess kwargs with text = True and no encoding" + ) + for node in ast.walk(tree): + if not isinstance(node, ast.Call): + continue + name = _call_name(node) + + if _is_subprocess_call(node, subprocess_names, subprocess_aliases): + if _text_mode_subprocess(node) and not _has_keyword(node, "encoding"): + found.append(f"{path.name}:{node.lineno}: subprocess(text = True) without encoding") + continue + + if name == "open" and isinstance(node.func, ast.Name): + if _mode_is_binary(node) or _open_has_encoding(node): + continue + found.append(f"{path.name}:{node.lineno}: open() without encoding") + continue + + # os.fdopen(fd, "w") is open() on a descriptor, so text mode takes the + # same locale default. Its mode defaults to "r", i.e. text, like open's. + if name == "fdopen": + if _mode_is_binary(node) or _open_has_encoding(node): + continue + found.append(f"{path.name}:{node.lineno}: os.fdopen() without encoding") + continue + + if name == "open" and isinstance(node.func, ast.Attribute): + if not _is_path_open(node) or _path_open_has_encoding(node): + continue + if _path_open_mode(node) and "b" in _path_open_mode(node): + continue + found.append(f"{path.name}:{node.lineno}: Path.open() without encoding") + continue + + if name in ("read_text", "write_text") and isinstance(node.func, ast.Attribute): + if _has_keyword(node, "encoding"): + continue + # importlib.metadata Distribution.read_text() takes no encoding kwarg. + if isinstance(node.func.value, ast.Name) and node.func.value.id == "dist": + continue + found.append(f"{path.name}:{node.lineno}: {name}() without encoding") + return found + + +@pytest.mark.parametrize("path", _studio_sources(), ids = lambda p: str(p.name)) +def test_text_io_names_its_encoding(path: Path) -> None: + offenders = _offenders(path) + assert not offenders, ( + "Text I/O without an explicit encoding falls back to the Windows ANSI " + 'codepage and corrupts non-ASCII (ä ö ü → 世). Pass encoding = "utf-8":\n ' + + "\n ".join(offenders) + ) + + +_STATE_STORE = ( + BACKEND_ROOT + / "plugins/data-designer-github-repo-seed/src" + / "data_designer_github_repo_seed/scraper_impl/state_store.py" +) + + +def _load_state_store(codepage: str): + """Load state_store with the writing machine's codepage pinned.""" + spec = importlib.util.spec_from_file_location(f"state_store_{codepage}", _STATE_STORE) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + module.locale = SimpleNamespace( + getencoding = lambda: codepage, + getpreferredencoding = lambda _ = True: codepage, + ) + return module + + +@pytest.mark.parametrize( + ("codepage", "name"), [("cp1252", "Jürgen"), ("cp1251", "Юрий"), ("cp932", "田中")] +) +def test_resuming_a_legacy_jsonl_keeps_one_encoding( + tmp_path: Path, codepage: str, name: str +) -> None: + """A scrape written before UTF-8 was explicit must resume, not duplicate.""" + path = tmp_path / "out.jsonl" + records = [{"id": 1, "author": name}, {"id": 2, "author": name}] + body = "".join(json.dumps(r, ensure_ascii = False) + "\n" for r in records) + path.write_bytes(body.encode(codepage)) + before = path.read_bytes() + + writer = _load_state_store(codepage).JsonlWriter(path) + try: + # Seen keys survive the resume, so a repeat is refused, not appended. + assert writer.has("id:1") and writer.has("id:2") + assert writer.write(records[0]) is False + assert writer.write({"id": 3, "author": name}) is True + finally: + writer.close() + + # Never converted, so it still reads in its own codepage; the append is ASCII. + blob = path.read_bytes() + assert blob.startswith(before) + assert blob[len(before) :].isascii() + lines = [json.loads(x) for x in blob.decode(codepage).splitlines() if x.strip()] + assert len(lines) == 3 + assert [line["author"] for line in lines] == [name] * 3 + + +def test_a_coincidentally_utf8_legacy_line_is_left_alone(tmp_path: Path) -> None: + """cp1251 `Р°` is D0 B0, which is also UTF-8 `а`, and nothing can tell them apart.""" + path = tmp_path / "out.jsonl" + ambiguous = "Р°" + assert ambiguous.encode("cp1251").decode("utf-8") == "а" # the trap + authors = ["Привет", "Здравствуйте", "Москва", ambiguous] + path.write_bytes( + b"".join( + json.dumps({"id": i, "author": a}, ensure_ascii = False).encode("cp1251") + b"\n" + for i, a in enumerate(authors) + ) + ) + before = path.read_bytes() + + _load_state_store("cp1251").JsonlWriter(path).close() + + # Untouched, so the ambiguity never had to be resolved. + assert path.read_bytes() == before + rows = [json.loads(x) for x in path.read_text(encoding = "cp1251").splitlines() if x.strip()] + assert [row["author"] for row in rows] == authors + + +@pytest.mark.parametrize( + ("codepage", "word"), [("cp1251", "Привет"), ("cp932", "こんにちは"), ("cp1252", "Jürgen")] +) +def test_a_moved_shard_is_not_rewritten_by_guesswork( + tmp_path: Path, codepage: str, word: str +) -> None: + """Off the writing machine there is no codepage to attribute the file to.""" + path = tmp_path / "out.jsonl" + # Two records: a lone non-UTF-8 line would count as damage, not legacy. + path.write_bytes( + b"".join( + json.dumps({"id": i, "author": word}, ensure_ascii = False).encode(codepage) + b"\n" + for i in (1, 4) + ) + ) + before = path.read_bytes() + + # A UTF-8 host: latin-1 would read cp1251 `Привет` back as `Ïðèâåò`. + writer = _load_state_store("utf-8").JsonlWriter(path) + try: + assert writer.has("id:1") # ASCII keys still recover + assert writer.write({"id": 2, "author": "Grüße"}) is True + finally: + writer.close() + + blob = path.read_bytes() + assert blob.startswith(before) # never rewritten + assert blob[len(before) :].isascii() # appended as \uXXXX, so no second encoding + rows = [json.loads(x) for x in blob.decode(codepage).splitlines() if x.strip()] + assert [row["author"] for row in rows] == [word, word, "Grüße"] + + +def test_an_all_ambiguous_shard_still_gets_ascii_appends(tmp_path: Path) -> None: + """Every line valid under both readings still means the append must not pick one.""" + path = tmp_path / "out.jsonl" + ambiguous = "Р°" # cp1251 D0 B0, also valid UTF-8 for "а" + path.write_bytes( + b"".join( + json.dumps({"id": i, "a": ambiguous}, ensure_ascii = False).encode("cp1251") + b"\n" + for i in range(3) + ) + ) + before = path.read_bytes() + + writer = _load_state_store("cp1251").JsonlWriter(path) + try: + assert writer.write({"id": 9, "a": "世界"}) is True + finally: + writer.close() + + blob = path.read_bytes() + assert blob.startswith(before) + # ASCII, so the appended record survives whichever reading is chosen. + assert blob[len(before) :].isascii() + for codec in ("cp1251", "utf-8"): + rows = [json.loads(x) for x in blob.decode(codec).splitlines() if x.strip()] + assert rows[-1]["a"] == "世界" + + +def test_a_damaged_line_in_an_ascii_shard_does_not_block_its_retry(tmp_path: Path) -> None: + """With no non-ASCII records to outvote it, one damaged line is still damage.""" + path = tmp_path / "out.jsonl" + path.write_bytes( + b'{"id": 1, "author": "alice"}\n' + + b'{"id": 99, "author": "bad \x96 byte"}\n' + + b'{"id": 2, "author": "bob"}\n' + ) + + writer = _load_state_store("cp1252").JsonlWriter(path) + try: + assert writer.has("id:1") and writer.has("id:2") + assert not writer.has("id:99") + assert writer.write({"id": 99, "author": "good byte"}) is True + finally: + writer.close() + + +def test_a_damaged_line_does_not_block_its_own_retry(tmp_path: Path) -> None: + """Its key comes from the codepage reading, which a UTF-8 shard did not pick.""" + path = tmp_path / "out.jsonl" + path.write_bytes( + json.dumps({"id": 1, "author": "Jürgen"}, ensure_ascii = False).encode() + + b"\n" + + b'{"id": 99, "author": "bad \x96 byte"}\n' + ) + + writer = _load_state_store("cp1252").JsonlWriter(path) + try: + assert writer.has("id:1") + assert not writer.has("id:99") + assert writer.write({"id": 99, "author": "good byte"}) is True + finally: + writer.close() + + +def test_one_damaged_byte_does_not_relabel_a_utf8_shard(tmp_path: Path) -> None: + """A complete JSON line with a stray 0x96 parses as cp1252, but is only one vote.""" + path = tmp_path / "out.jsonl" + healthy = ["Jürgen", "Grüße", "Björn"] + path.write_bytes( + json.dumps({"id": 0, "author": healthy[0]}, ensure_ascii = False).encode() + + b"\n" + + b'{"id": 99, "author": "bad \x96 byte"}\n' + + b"".join( + json.dumps({"id": i, "author": a}, ensure_ascii = False).encode() + b"\n" + for i, a in enumerate(healthy[1:], start = 1) + ) + ) + before = path.read_bytes() + + _load_state_store("cp1252").JsonlWriter(path).close() + + # Untouched, so the healthy records were never re-read as cp1252. + assert path.read_bytes() == before + rows = [] + for line in path.read_bytes().splitlines(): + try: + rows.append(json.loads(line.decode())) + except (UnicodeDecodeError, ValueError): + continue + assert [row["author"] for row in rows] == healthy + + +def test_a_torn_line_does_not_relabel_a_utf8_shard(tmp_path: Path) -> None: + """One interrupted append must not get the whole shard read as cp1252.""" + path = tmp_path / "out.jsonl" + good = [{"id": 1, "author": "Jürgen"}, {"id": 3, "author": "Grüße"}] + torn = '{"id": 2, "author": "Jürgen"}'.encode()[:-6] # cut mid-character + path.write_bytes( + json.dumps(good[0], ensure_ascii = False).encode() + + b"\n" + + torn + + b"\n" + + json.dumps(good[1], ensure_ascii = False).encode() + + b"\n" + ) + before = path.read_bytes() + + writer = _load_state_store("cp1252").JsonlWriter(path) + try: + assert writer.has("id:1") and writer.has("id:3") + assert not writer.has("id:2") # torn line yields no key + finally: + writer.close() + + # Untouched: no rewrite, so no record was re-encoded into mojibake. + after = path.read_bytes() + assert after.startswith(before) + assert "Jürgen".encode() in after + assert "Jürgen".encode("utf-8").decode("cp1252").encode() not in after + + +def test_an_undecodable_transport_marker_reads_as_unknown(tmp_path: Path) -> None: + """Pinning the decode turns an undecodable marker into UnicodeDecodeError, + which is a ValueError and so is not an OSError. Before the pin those bytes + simply read as an unknown value and the caller safely purged and restarted + the partial download; letting the error escape aborts the transfer instead. + """ + import sys + + backend = str(Path(__file__).resolve().parent.parent) + if backend not in sys.path: + sys.path.insert(0, backend) + from hub.utils import download_registry as registry + + marker = tmp_path / ".transport" + marker.write_bytes(b"\x80\xffnative\n") + assert registry._read_marker_value(marker) is None + # A readable but unknown value takes the same path (the behaviour restored). + marker.write_text("something-else\n", encoding = "utf-8") + assert registry._read_marker_value(marker) is None + + +def test_a_torn_cache_ref_reads_as_not_cached(tmp_path: Path, monkeypatch) -> None: + """hf_cache_snapshot_dir answers "is this model already on disk", and the + offline embedding checks turn a raise into a 500. A refs/main holding a byte + the codepage used to decode into a nonsense commit simply missed the snapshot + dir before the pin; it has to keep missing it.""" + import sys + + backend = str(Path(__file__).resolve().parent.parent) + if backend not in sys.path: + sys.path.insert(0, backend) + from utils import utils as backend_utils + + good_root = tmp_path / "good" + torn_root = tmp_path / "torn" + for root, ref_bytes in ((torn_root, b"\x80\xff\n"), (good_root, b"abc123\n")): + repo = root / "models--Org--Model" + (repo / "refs").mkdir(parents = True) + (repo / "refs" / "main").write_bytes(ref_bytes) + (good_root / "models--Org--Model" / "snapshots" / "abc123").mkdir(parents = True) + + monkeypatch.setattr(backend_utils, "_hf_cache_roots", lambda: [torn_root]) + assert backend_utils.hf_cache_snapshot_dir("Org/Model") is None + # The torn root is skipped, not fatal: a healthy second root still answers. + monkeypatch.setattr(backend_utils, "_hf_cache_roots", lambda: [torn_root, good_root]) + found = backend_utils.hf_cache_snapshot_dir("Org/Model") + assert found is not None and found.name == "abc123" + + +def test_a_corrupt_pid_file_does_not_abort_shutdown(tmp_path: Path, monkeypatch) -> None: + """_remove_pid_file runs first in _graceful_shutdown, so a raise there leaves + the inference, export, training and tunnel children alive.""" + import sys + + backend = str(Path(__file__).resolve().parent.parent) + if backend not in sys.path: + sys.path.insert(0, backend) + import run as studio_run + + pid_file = tmp_path / "studio.pid" + pid_file.write_bytes(b"\x80\xff") + monkeypatch.setattr(studio_run, "_PID_FILE", pid_file) + studio_run._remove_pid_file() + # Not this process's PID, so the file stays; the point is that it returned. + assert pid_file.exists() + + pid_file.write_text(str(os.getpid()), encoding = "utf-8") + studio_run._remove_pid_file() + assert not pid_file.exists() + + +def test_the_kwargs_guard_only_judges_dicts_that_reach_a_call(tmp_path: Path) -> None: + """Only a dict splatted into a call is subprocess configuration. An unrelated + payload that happens to carry "text": True is not, and neither is one whose + encoding is filled in on a later line.""" + cases = { + "offender.py": 'kw = {"text": True}\nrun(cmd, **kw)\n', + "annotated.py": 'kw: dict = {"universal_newlines": True}\nrun(cmd, **kw)\n', + "payload.py": 'payload = {"text": True}\nrequests.post(url, json = payload)\n', + "inline.py": 'run(cmd, **{"text": True})\n', + "later.py": 'kw = {"text": True}\nkw["encoding"] = "utf-8"\nrun(cmd, **kw)\n', + "carried.py": 'kw = {"text": True, "encoding": "utf-8"}\nrun(cmd, **kw)\n', + } + flagged = set() + for name, source in cases.items(): + path = tmp_path / name + path.write_text(source, encoding = "utf-8") + if any("subprocess kwargs" in line for line in _offenders(path)): + flagged.add(name) + assert flagged == {"offender.py", "annotated.py", "inline.py"}, flagged + + +def test_the_guard_follows_subprocess_through_an_alias(tmp_path: Path) -> None: + """install_wheel() takes ``run = subprocess.run`` and calls it as a bare + name, so an attribute-only match let both of its installer calls drop their + encoding unnoticed. A name bound to something else is still not subprocess.""" + cases = { + "param_default.py": ( + "import subprocess\n" + "def install(*, run = subprocess.run):\n" + " run(cmd, text = True)\n" + ), + "assigned.py": "import subprocess\n_run = subprocess.run\n_run(cmd, text = True)\n", + "imported.py": "from subprocess import check_output\ncheck_output(cmd, text = True)\n", + "renamed.py": "from subprocess import run as _r\n_r(cmd, universal_newlines = True)\n", + "encoded.py": ( + "import subprocess\n" + "def install(*, run = subprocess.run):\n" + ' run(cmd, text = True, encoding = "utf-8")\n' + ), + "unrelated.py": "def run(cmd, text = False):\n pass\nrun(cmd, text = True)\n", + } + flagged = set() + for name, source in cases.items(): + path = tmp_path / name + path.write_text(source, encoding = "utf-8") + if any("subprocess(text = True)" in line for line in _offenders(path)): + flagged.add(name) + assert flagged == {"param_default.py", "assigned.py", "imported.py", "renamed.py"}, flagged + + +def test_the_guard_sees_os_fdopen(tmp_path: Path) -> None: + """os.fdopen(fd, mode) is open() on a descriptor and takes the same locale + default in text mode, so leaving it out let the swap lock file keep the + codepage on the write side while its reader was pinned to UTF-8.""" + cases = { + "text.py": 'import os\nos.fdopen(fd, "w")\n', + "default_mode.py": "import os\nos.fdopen(fd)\n", # defaults to "r", still text + "binary.py": 'import os\nos.fdopen(fd, "wb")\n', + "keyword.py": 'import os\nos.fdopen(fd, "w", encoding = "utf-8")\n', + "positional.py": 'import os\nos.fdopen(fd, "w", 1, "utf-8")\n', + } + flagged = set() + for name, source in cases.items(): + path = tmp_path / name + path.write_text(source, encoding = "utf-8") + if any("fdopen" in line for line in _offenders(path)): + flagged.add(name) + assert flagged == {"text.py", "default_mode.py"}, flagged + + +def test_an_undecodable_bootstrap_password_does_not_stop_startup( + tmp_path: Path, monkeypatch +) -> None: + """ensure_default_admin calls _load_bootstrap_password for every existing + admin and the lifespan calls that with no handler, so a raise here takes the + whole backend down instead of ignoring an unusable file.""" + import sys + + backend = str(Path(__file__).resolve().parent.parent) + if backend not in sys.path: + sys.path.insert(0, backend) + from auth import storage + + pw_file = tmp_path / ".bootstrap_password" + pw_file.write_bytes(b"\x80\xffnot-utf8\n") + monkeypatch.setattr(storage, "_BOOTSTRAP_PW_PATH", pw_file) + assert storage._load_bootstrap_password() is None + + # A readable one still loads, so this is a narrowing of failure, not of function. + pw_file.write_text("correct horse battery staple\n", encoding = "utf-8") + assert storage._load_bootstrap_password() == "correct horse battery staple" + + +def test_a_damaged_checkpoint_resets_instead_of_resuming_on_a_broken_cursor(tmp_path: Path) -> None: + """A checkpoint holds only base64 cursors and booleans, so a codepage reading + can only ever add non-ASCII, never recover any. Resuming on a mojibaked cursor + sends GitHub one it answers with INVALID_CURSOR_ARGUMENTS, and the empty page + that comes back marks the stream done and skips the rest of it for good. + Dropping the checkpoint only replays pages the writers already dedup.""" + module = _load_state_store("cp1252") + cursor = "Y3Vyc29yOnYyOpK0MjAxMi0wMi0xNlQwNjo1Mzo0MVrOADGL_A==" + healthy = json.dumps({"issues_cursor": cursor, "issues_done": False}, indent = 2) + path = tmp_path / "octocat__Hello-World.json" + + path.write_text(healthy, encoding = "utf-8") + assert module.StateStore(path).get("issues_cursor") == cursor + + # Written by a pre-UTF-8 release in the operator's codepage. Nothing is lost + # by reading UTF-8 only, because an all-ASCII document is the same bytes. + path.write_bytes(healthy.encode("cp1252")) + assert module.StateStore(path).get("issues_cursor") == cursor + + # One damaged byte inside the cursor: still a whole JSON document under a + # single-byte codepage, so only refusing that reading resets the checkpoint. + raw = healthy.encode() + at = raw.index(b"MjAxMi0wMi0xNlQ") + 3 + path.write_bytes(raw[:at] + b"\x96" + raw[at + 1 :]) + assert json.loads(path.read_bytes().decode("latin-1"))["issues_cursor"] != cursor + store = module.StateStore(path) + assert store.all() == {} + assert store.get("issues_cursor") is None + + +def test_a_utf8_record_is_not_parsed_a_second_time(tmp_path: Path) -> None: + """These shards reach gigabytes and every resume reads all of one, so a + record that already read as UTF-8 must not be decoded and parsed again under + the codepage. The legacy reading exists only to recover keys UTF-8 could not.""" + module = _load_state_store("cp1252") + calls: list[str] = [] + real_parse = module._parse + + def counting_parse(raw, encoding): + calls.append(encoding) + return real_parse(raw, encoding) + + module._parse = counting_parse + try: + healthy = json.dumps({"id": 1, "author": "Jürgen"}).encode("utf-8") + reading = module._read_line(healthy, "cp1252") + assert reading.as_utf8 == {"id": 1, "author": "Jürgen"} + assert calls == ["utf-8"], calls + + # A line UTF-8 cannot read still falls through to the codepage, the whole point. + calls.clear() + legacy = json.dumps({"id": 2, "author": "Jürgen"}, ensure_ascii = False).encode("cp1252") + reading = module._read_line(legacy, "cp1252") + assert reading.as_utf8 is None + assert reading.as_legacy == {"id": 2, "author": "Jürgen"} + assert calls == ["utf-8", "cp1252"], calls + finally: + module._parse = real_parse + + +def _too_deeply_nested_json() -> str: + """A JSON document nested past what this interpreter will descend into. + + Probed rather than hardcoded: the depth json.loads gives up at is bounded by + sys.getrecursionlimit() up to 3.11 and by the C recursion limit from 3.12, + which sys.setrecursionlimit no longer moves and which varies by micro + version. That is ~995 on 3.9 and ~9999 on 3.13. + """ + depth = 1 + while depth <= 1 << 17: + document = "[" * depth + "]" * depth + try: + json.loads(document) + except RecursionError: + return document + depth *= 2 + pytest.skip("this interpreter parses arbitrarily nested JSON") + + +def test_an_unparseably_nested_document_is_discarded_not_raised(tmp_path: Path) -> None: + """json.loads answers nesting it cannot descend with RecursionError, which is + a RuntimeError and so is neither a ValueError nor a UnicodeDecodeError. + _parse is called outside any other handler in both StateStore.__init__ and + JsonlWriter._scan_existing, so letting it escape aborts the scraper at + startup on a file the catch-all it replaced simply discarded.""" + module = _load_state_store("cp1252") + nested = _too_deeply_nested_json() + + checkpoint = tmp_path / "octocat__Hello-World.json" + checkpoint.write_text(nested, encoding = "utf-8") + assert module.StateStore(checkpoint).all() == {} # reset, not raised + + shard = tmp_path / "out.jsonl" + shard.write_text( + nested + "\n" + json.dumps({"id": 1}) + "\n" + json.dumps({"id": 2}) + "\n", + encoding = "utf-8", + ) + writer = module.JsonlWriter(shard) + try: + # Skipped like any other unreadable line, so its neighbours still yield the dedup + # keys that keep the resume from re-fetching them. + assert writer.has("id:1") and writer.has("id:2") + finally: + writer.close() diff --git a/studio/backend/tests/test_training_worker_flash_attn.py b/studio/backend/tests/test_training_worker_flash_attn.py index 86511987b1..d136821ea2 100644 --- a/studio/backend/tests/test_training_worker_flash_attn.py +++ b/studio/backend/tests/test_training_worker_flash_attn.py @@ -9,8 +9,28 @@ import sys from typing import Any from unittest import mock +import pytest + from core.training import worker +# The runtime install is Linux-only, so elsewhere these return before any status. +linux_only = pytest.mark.skipif( + not sys.platform.startswith("linux"), + reason = "the runtime flash-attn install is gated to Linux", +) + +# causal-conv1d and flash-linear-attention are NOT Linux-gated: both installers bail out +# on `sys.platform == "win32"` alone (no prebuilt wheel for Windows) and run everywhere +# else, macOS included. linux_only here would skip cases that legitimately pass off Linux. +not_on_windows = pytest.mark.skipif( + sys.platform == "win32", + reason = ( + "mirrors the sys.platform == 'win32' bail-out in " + "_ensure_flash_linear_attention_unconditional and " + "_ensure_causal_conv1d_fast_path" + ), +) + def _missing_flash_attn_import(): real_import = builtins.__import__ @@ -55,6 +75,7 @@ def test_should_try_runtime_flash_attn_install_threshold_and_skip(monkeypatch): assert worker._should_try_runtime_flash_attn_install(32768) is False +@linux_only def test_runtime_flash_attn_prefers_prebuilt_wheel(monkeypatch): statuses: list[str] = [] @@ -82,6 +103,7 @@ def test_runtime_flash_attn_prefers_prebuilt_wheel(monkeypatch): assert statuses == ["Installing flash-attn for faster training..."] +@linux_only def test_runtime_flash_attn_falls_back_to_pypi(monkeypatch): calls: list[list[str]] = [] statuses: list[str] = [] @@ -113,12 +135,7 @@ def test_runtime_flash_attn_falls_back_to_pypi(monkeypatch): ) monkeypatch.setattr(worker, "install_wheel", mock.Mock()) - def fake_run( - cmd, - stdout = None, - stderr = None, - text = None, - ): + def fake_run(cmd, **kwargs): calls.append(list(cmd)) return subprocess.CompletedProcess(cmd, 0, "") @@ -139,6 +156,7 @@ def test_runtime_flash_attn_skip_env_avoids_all_install_work(monkeypatch): worker._sp.run.assert_not_called() +@not_on_windows def test_causal_conv1d_fast_path_preserves_wheel_first_install_args(monkeypatch): install_mock = mock.Mock(return_value = True) monkeypatch.setattr(worker, "_install_package_wheel_first", install_mock) @@ -160,6 +178,7 @@ def test_causal_conv1d_fast_path_preserves_wheel_first_install_args(monkeypatch) ) +@not_on_windows def test_causal_conv1d_fast_path_includes_qwen3_6_variants(monkeypatch): install_mock = mock.Mock(return_value = True) monkeypatch.setattr(worker, "_install_package_wheel_first", install_mock) @@ -225,6 +244,7 @@ def _pin_fla_model_types(monkeypatch): ) +@not_on_windows def test_flash_linear_attention_installs_pinned_pair_for_qwen3_5(monkeypatch): _pin_fla_model_types(monkeypatch) monkeypatch.setattr(worker.shutil, "which", lambda name: "/usr/bin/uv") @@ -277,6 +297,7 @@ def test_flash_linear_attention_skips_for_ssm_only_models(monkeypatch): run_mock.assert_not_called() +@not_on_windows def test_flash_linear_attention_matches_full_qwen3_family(monkeypatch): monkeypatch.setattr(worker.shutil, "which", lambda name: "/usr/bin/uv") run_mock = mock.Mock(return_value = mock.Mock(returncode = 0, stdout = "")) @@ -331,6 +352,7 @@ def test_flash_linear_attention_skipped_via_env(monkeypatch): run_mock.assert_not_called() +@not_on_windows def test_flash_linear_attention_skipped_below_torch_2_7(monkeypatch): _pin_fla_model_types(monkeypatch) monkeypatch.delenv(worker._FLA_SKIP_ENV, raising = False) @@ -349,6 +371,7 @@ def test_flash_linear_attention_skipped_below_torch_2_7(monkeypatch): assert any("torch>=" in s for s in statuses) +@not_on_windows def test_flash_linear_attention_install_includes_einops(monkeypatch): _pin_fla_model_types(monkeypatch) monkeypatch.delenv(worker._FLA_SKIP_ENV, raising = False) @@ -375,6 +398,7 @@ def test_flash_linear_attention_install_includes_einops(monkeypatch): assert f"fla-core=={worker._FLA_CORE_PACKAGE_VERSION}" in args +@not_on_windows def test_flash_linear_attention_logs_post_install_import_failure(monkeypatch): """pip exits 0 but `import fla.modules` still fails (missing transitive).""" _pin_fla_model_types(monkeypatch) @@ -421,6 +445,7 @@ def test_tilelang_backend_skipped_on_unsupported_linux_arch(monkeypatch): run_mock.assert_not_called() +@linux_only def test_tilelang_backend_pins_only_binary(monkeypatch): _pin_fla_model_types(monkeypatch) monkeypatch.delenv(worker._TILELANG_SKIP_ENV, raising = False) @@ -462,6 +487,7 @@ def _force_missing_tilelang_imports(monkeypatch): monkeypatch.setattr(builtins, "__import__", fake_import) +@linux_only def test_tilelang_backend_installs_pinned_pair_for_qwen3_5(monkeypatch): _pin_fla_model_types(monkeypatch) monkeypatch.delenv(worker._TILELANG_SKIP_ENV, raising = False) @@ -486,6 +512,7 @@ def test_tilelang_backend_installs_pinned_pair_for_qwen3_5(monkeypatch): assert any("Installing TileLang" in s for s in statuses) +@linux_only def test_tilelang_backend_reinstalls_when_tvm_ffi_is_broken(monkeypatch): """Repair path issues TWO pip calls: @@ -555,6 +582,7 @@ def test_tilelang_backend_skipped_on_windows(monkeypatch): run_mock.assert_not_called() +@linux_only def test_tilelang_backend_swallows_install_timeout(monkeypatch): _pin_fla_model_types(monkeypatch) monkeypatch.delenv(worker._TILELANG_SKIP_ENV, raising = False) @@ -609,6 +637,7 @@ def test_tilelang_backend_skipped_via_env(monkeypatch): run_mock.assert_not_called() +@linux_only def test_tilelang_backend_swallows_install_failure(monkeypatch): _pin_fla_model_types(monkeypatch) monkeypatch.delenv(worker._TILELANG_SKIP_ENV, raising = False) @@ -673,6 +702,7 @@ def _patch_iu_gates(monkeypatch, fla_gate, conv_gate): monkeypatch.setattr(_iu, "is_causal_conv1d_available", conv_gate) +@not_on_windows def test_hook_installs_when_gate_returns_false(monkeypatch): _pin_fla_model_types(monkeypatch) fla_gate = _make_fake_gate(initial_return = False) @@ -976,6 +1006,7 @@ def test_hook_does_install_tilelang_for_qwen35(monkeypatch): tile_install.assert_called_once() +@linux_only def test_tilelang_repair_does_not_touch_torch_cuda_stack(monkeypatch): """Finding #2: the broken-tvm-ffi repair must use --no-deps on the forced step so --force-reinstall doesn't cascade through @@ -1119,6 +1150,7 @@ def test_hook_runs_tilelang_repair_when_fla_already_true(monkeypatch): tile_install.assert_called_once() +@not_on_windows def test_fla_installer_force_reinstalls_when_older_version_present(monkeypatch): """Finding #8: an older `flash-linear-attention` that is importable but below the pin must force a reinstall (not no-op). @@ -1583,15 +1615,10 @@ def test_install_respects_user_gcc_install_dir(monkeypatch): ) _make_hip_install_env(monkeypatch, gcc_dir = "/usr/lib/gcc/x86_64-linux-gnu/13") - captured: dict[str, str] | None = {"_called": "no"} + captured: dict[str, str] = {} def fake_run(cmd, **kwargs): - env = kwargs.get("env") - if env is not None: - captured.clear() - captured.update(env) - else: - captured["_called"] = "yes_no_env" + captured.update(kwargs.get("env") or {}) return subprocess.CompletedProcess(cmd, 0, "") monkeypatch.setattr(worker._sp, "run", fake_run) @@ -1607,14 +1634,11 @@ def test_install_respects_user_gcc_install_dir(monkeypatch): release_base_url = "https://example.com", ) - # subprocess.run invoked without env override (user already set - # HIPCC_COMPILE_FLAGS_APPEND with --gcc-install-dir, so we left the - # env alone — the existing value is inherited). - assert captured == {"_called": "yes_no_env"} + assert captured["HIPCC_COMPILE_FLAGS_APPEND"] == "--gcc-install-dir=/opt/custom/gcc-13" def test_install_does_not_inject_env_on_cuda(monkeypatch): - """CUDA path (no hip_version in env) → no env override at all.""" + """CUDA path (no hip_version in env) → no HIP flag injected.""" monkeypatch.delenv("HIPCC_COMPILE_FLAGS_APPEND", raising = False) monkeypatch.setattr(builtins, "__import__", _missing_module_import("causal_conv1d")) monkeypatch.setattr( @@ -1641,7 +1665,7 @@ def test_install_does_not_inject_env_on_cuda(monkeypatch): captured: dict[str, Any] = {} def fake_run(cmd, **kwargs): - captured["env_in_kwargs"] = "env" in kwargs + captured.update(kwargs.get("env") or {}) return subprocess.CompletedProcess(cmd, 0, "") monkeypatch.setattr(worker._sp, "run", fake_run) @@ -1657,5 +1681,5 @@ def test_install_does_not_inject_env_on_cuda(monkeypatch): release_base_url = "https://example.com", ) - # CUDA branch never sets the env, never invokes the gcc helper. - assert captured.get("env_in_kwargs") is False + # env is always passed (to force UTF-8), but never the HIP flag. + assert "HIPCC_COMPILE_FLAGS_APPEND" not in captured diff --git a/studio/backend/utils/changelog.py b/studio/backend/utils/changelog.py new file mode 100644 index 0000000000..84cd54df05 --- /dev/null +++ b/studio/backend/utils/changelog.py @@ -0,0 +1,1056 @@ +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 + +"""Release notes for the update popup, sourced from CHANGELOG.md. + +Notes are keyed to one exact version: the popup asks for the version it is +offering and gets that section or nothing, so an older release's notes can +never appear next to a newer update. + +The remote copy on the default branch wins over the bundled one, since the +offered version is newer than the installed checkout. Both reads are lazy, +cached and skipped when update checks are off. +""" + +from __future__ import annotations + +import os +import re +import threading +import time +import urllib.request +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +from packaging.version import InvalidVersion, Version + +from .update_status import DISABLE_ENV_VAR, RELEASE_NOTES_URL + +CHANGELOG_FILENAME = "CHANGELOG.md" +CHANGELOG_RAW_URL = "https://raw.githubusercontent.com/unslothai/unsloth/main/CHANGELOG.md" +CHANGELOG_URL_ENV_VAR = "UNSLOTH_CHANGELOG_URL" +CHANGELOG_PATH_ENV_VAR = "UNSLOTH_CHANGELOG_PATH" +CHANGELOG_TIMEOUT_SECONDS = 3 +CHANGELOG_MAX_BYTES = 2 * 1024 * 1024 +_CHANGELOG_CHUNK_BYTES = 64 * 1024 +_CHANGELOG_MIN_READ_SECONDS = 0.05 +CHANGELOG_SUCCESS_TTL_SECONDS = 30 * 60 +CHANGELOG_FAILURE_TTL_SECONDS = 5 * 60 +RELEASE_NOTES_MAX_CHARS = 20_000 + +# CommonMark requires a space, tab or line end after the hashes: a non-breaking +# space copied from rich text renders as text, not a heading, but a bare `##` is +# an empty heading and still ends the release above. +_HEADING_PATTERN = re.compile(r"^ {0,3}##(?:[ \t]+(?P.*?))?[ \t]*$") +_FENCE_PATTERN = re.compile(r"^ {0,3}(?P<marker>`{3,}|~{3,})(?P<rest>.*)$") +# CommonMark type 1 HTML blocks: contents are literal until a closing tag, +# which the spec says need not be the one that opened the block. +_RAW_HTML_OPEN = re.compile(r"^ {0,3}<(pre|script|style|textarea)(?=[\s>]|$)", re.IGNORECASE) +_RAW_HTML_CLOSE = re.compile(r"</(pre|script|style|textarea)\s*>", re.IGNORECASE) +# Types 3 to 5 (processing instructions, declarations, CDATA) are literal too, +# each ending on its own delimiter. Comments open mid-line, so are separate. +_RAW_BLOCKS = ( + (_RAW_HTML_OPEN, _RAW_HTML_CLOSE), + (re.compile(r"^ {0,3}<\?"), re.compile(r"\?>")), + (re.compile(r"^ {0,3}<!\[CDATA\["), re.compile(r"\]\]>")), + # A declaration needs an uppercase letter, so `<!note` stays ordinary text. + (re.compile(r"^ {0,3}<![A-Z]"), re.compile(r">")), +) +# Type 6 blocks run to the next blank line, so `<details>` only holds Markdown +# once a blank line has closed the block. Open and close tags both start one. +_HTML_BLOCK_OPEN = re.compile(r"^ {0,3}</?([a-zA-Z][a-zA-Z0-9-]*)(?=[\s/>]|$)") +# Blocks that break into an open paragraph, so none is open after them and one +# they are written below is closed rather than continued. +_INTERRUPTS = re.compile( + r"^ {0,3}(?:#{1,6}([ \t]|$)|(?:\*[ \t]*){3,}$|(?:-[ \t]*){3,}$|(?:_[ \t]*){3,}$)" +) +# A definition is a block of its own but may not interrupt a paragraph, so it +# ends the one above it only when there is none to continue. +_LINK_DEFINITION = re.compile(r"^ {0,3}\[(?:[^\[\]\\]|\\.)+\]:") +# Blocks that are not paragraph text, so a following underline is not setext. +_PARAGRAPH_TEXT = re.compile(r"^ {0,3}(?![-*+>]([ \t]|$)|\d{1,9}[.)]([ \t]|$))\S") +# A line of = or - under a paragraph line makes that line a heading. +_SETEXT_UNDERLINE = re.compile(r"^ {0,3}(=+|-+)[ \t]*$") +# A quoted paragraph continues on unmarked lines, which belong to the quote. +_BLOCK_QUOTE = re.compile(r"^ {0,3}>") +_QUOTE_MARKER = re.compile(r"^ {0,3}>[ \t]?") +# A heading at an item's content column belongs to that item, not the document. +# The marker needs whitespace after it, so `2.0` is a version, not an item. +_LIST_ITEM = re.compile(r"^[ \t]*(?P<marker>[-*+]|\d{1,9}[.)])(?P<space>[ \t]+|$)") +_THEMATIC_BREAK = re.compile(r"^ {0,3}(?:(?:\*[ \t]*){3,}|(?:-[ \t]*){3,}|(?:_[ \t]*){3,})$") +# Content indented more than this after a marker is an indented code block, so +# the item's content starts one column past the marker instead. +_MAX_ITEM_PADDING = 4 +_HTML_BLOCK_TAGS = frozenset( + """ +address article aside base basefont blockquote body caption center col colgroup +dd details dialog dir div dl dt fieldset figcaption figure footer form frame +frameset h1 h2 h3 h4 h5 h6 head header hr html iframe legend li link main menu +menuitem nav noframes ol optgroup option p param search section summary table +tbody td tfoot th thead title tr track ul +""".split() +) +# Type 7: any other complete tag alone on a line. It cannot interrupt a +# paragraph, so it only counts after a break. +_HTML_ATTRIBUTE = ( + r"""(?:\s+[a-zA-Z_:][a-zA-Z0-9_.:-]*(?:\s*=\s*(?:[^\s"'=<>`]+|'[^']*'|"[^"]*"))?)""" +) +_HTML_TAG_ONLY_LINE = re.compile( + rf"^ {{0,3}}(?:<[a-zA-Z][a-zA-Z0-9-]*{_HTML_ATTRIBUTE}*\s*/?>|</[a-zA-Z][a-zA-Z0-9-]*\s*>)\s*$" +) +# Levels above studio/ are the repo root in a checkout and site-packages in an +# install, so they are searched only when one of these markers is present. +_CHECKOUT_ONLY_LEVELS = (3, 4) +_CHECKOUT_MARKERS = ("pyproject.toml", ".git") +_COMMENT_BLOCK_OPEN = re.compile(r"^ {0,3}<!--") +_COMMENT_OPEN = "<!--" +_COMMENT_CLOSE = "-->" +# Stands in for a line the renderer hides. `#` is a block of its own, so list +# tracking reads it like a comment: never a marker, never a lazy continuation. +_HIDDEN_BLOCK = "#" +_VERSION_TOKEN_PATTERN = re.compile(r"^[\[(]?v?(?P<version>[0-9][0-9A-Za-z.!+-]*?)[\])]?$") +_SAFE_VERSION_PATTERN = re.compile(r"^[0-9A-Za-z][0-9A-Za-z.!+-]{0,63}$") + + +@dataclass(frozen = True) +class _ListState: + """The open list items, innermost last, by the column their content starts.""" + + columns: tuple[int, ...] = () + # True while the innermost item has had no content since its marker. + empty_item: bool = False + + +@dataclass(frozen = True) +class ChangelogEntry: + """One `## <version>` section of the changelog.""" + + version: str + heading: str + body: str + + +@dataclass(frozen = True) +class ChangelogSource: + text: str | None + source: str | None + error: str | None = None + + +@dataclass +class _ChangelogCacheEntry: + source: ChangelogSource + expires_at: float + + +_cache_condition = threading.Condition() +_remote_cache: _ChangelogCacheEntry | None = None +_remote_fetching = False + + +def reset_changelog_cache() -> None: + """Clear the in-process changelog cache. Intended for tests.""" + global _remote_cache, _remote_fetching + with _cache_condition: + _remote_cache = None + _remote_fetching = False + _cache_condition.notify_all() + + +def is_supported_version_query(version: str) -> bool: + """Whether `version` is shaped like something we can look up at all. + + Sections are indexed only when their version parses, so a query that does + not parse (`latest`, `main`) can never match and is rejected outright.""" + candidate = version.strip() + if not _SAFE_VERSION_PATTERN.match(candidate): + return False + return _parse_version(candidate) is not None + + +def _markdown_lines(text: str) -> list[str]: + """``text`` split the way CommonMark ends lines. + + str.splitlines also breaks on U+2028, U+2029, NEL, vertical tab and form + feed, none of which end a line in Markdown. A separator sitting in prose + before "## 9.9.9" would otherwise index a release the renderer never shows + and truncate the notes above it. + """ + return text.replace("\r\n", "\n").replace("\r", "\n").split("\n") + + +def parse_changelog(text: str) -> list[ChangelogEntry]: + """Parse `## <version>` sections, in file order. + + Headings whose first token is not a version (`## Unreleased`, `## Format`) + end the previous section but are not indexed. + """ + # A Windows editor can leave a BOM on the first line, hiding a heading. + text = text.lstrip("") + entries: list[ChangelogEntry] = [] + heading: str | None = None + version: str | None = None + body: list[str] = [] + open_fence: str | None = None + # Content column of the list item the open block belongs to, 0 at document + # level. A fence and an HTML block are scoped to their container, so the + # item's end closes them. Only one of the three is ever open. + block_column = 0 + in_comment = False + in_raw_html: int | None = None + in_html_block = False + after_paragraph = False + paragraph: list[str] = [] + in_quote = False + quoted = False + lists = _ListState() + + def flush() -> None: + if version is not None and heading is not None: + entries.append( + ChangelogEntry( + version = version, + heading = heading, + body = "\n".join(body).strip(), + ) + ) + + for line in _markdown_lines(text): + # The line as list tracking sees it: blank wherever nothing renders. + structural = "" + opened_block = False + in_block = open_fence is not None or in_html_block or in_raw_html is not None or in_comment + # A fence, comment or HTML block inside a list item runs only to the end + # of that item, so a line dedented out of the item closes both. Lazy + # continuation reaches into none of them. A raw block or comment inside an + # item also ends on a blank line: the item takes the break, so what + # follows is a block of the item's own. + leaves = ( + _indent_width(line) < block_column + if line.strip() + else in_raw_html is not None or in_comment + ) + if in_block and block_column and leaves: + open_fence = None + in_html_block = False + in_raw_html = None + in_comment = False + block_column = 0 + # The paragraph the line could have continued is block content, so + # it closes the item rather than reading as more of it. + after_paragraph = False + # A fence written as a list item's first content opens inside that item, so + # an opener is read past a marker on the same line. Only an opener: fenced + # content is literal and a closer carries no marker. + fence_line = line if open_fence else _item_content(line, after_paragraph) + # Raw HTML first: its contents are literal, so a fence in it is not one. + if in_raw_html is not None: + visible, in_raw_html = _strip_raw_html(line, in_raw_html) + elif in_html_block: + # A blank line is the only thing that ends a type 6 block. + in_html_block = line.strip() != "" + visible = "" + elif (fence := _FENCE_PATTERN.match(fence_line)) and not in_comment: + was_open = open_fence + open_fence = _next_fence_state(open_fence, fence.group("marker"), fence.group("rest")) + opened_block = was_open is None and open_fence is not None + # Hidden from heading matching, but its indent still closes items. + visible = "" + structural = line + elif open_fence: + visible = "" + else: + # A block already open owns this line, so it is content rather than a + # block written at the column it happens to start in. + hidden = in_comment or in_raw_html is not None + # A comment is an HTML block too, so one written as a list item's first + # content opens inside it exactly as a fence does: the opener is read + # past a marker on the same line. + block_open = ( + not in_comment + and _COMMENT_BLOCK_OPEN.match(_item_content(line, after_paragraph)) is not None + ) + # Commented-out sections are not rendered, so they are not releases. + visible, in_comment = _strip_comments(line, in_comment, block_open) + # An HTML block written as a list item's first content opens inside + # that item, as a fence does, so an opener is read past a marker on the + # same line. The marker stays, so its item is still tracked. A comment + # blanks its own line, so that line is read as written: the block + # renders as nothing, but the item it is content of still opens. + source = line if block_open else visible + content = _item_content(source, after_paragraph) + marker = source[: len(source) - len(content)] + # Nor is anything inside a raw HTML block such as <pre>. + stripped, in_raw_html = _strip_raw_html(content, in_raw_html) + opened_block = in_raw_html is not None or (block_open and in_comment) + # Taken before the opener is hidden: it renders as nothing, but its + # indent still closes a list item it sits left of, and a marker on its + # line still opens one. A comment or raw block keeps only those, since + # the text it hides is not Markdown and must open no list. + if block_open or stripped != content: + if not hidden: + structural = _hidden_structure(line, marker) + visible = "" + else: + visible = marker + stripped + if visible.strip(): + structural = visible + elif not hidden: + structural = _hidden_structure(line) + if stripped and _opens_html_block(stripped, after_paragraph): + in_html_block = True + opened_block = True + visible = "" + # A `##` inside a fenced block is sample markdown, not a real heading. + match = _HEADING_PATTERN.match(visible) if visible else None + # `1.0` over a line of dashes is the same heading written setext style. + setext = ( + after_paragraph + and match is None + and paragraph != [] + and _SETEXT_UNDERLINE.match(visible) is not None + and (visible.strip()[:1] == "-") + # Never a boundary inside a list item: dedented the dashes are a + # thematic break, and at the content column the heading is nested. + and not lists.columns + ) + if setext: + if version is not None: + # The whole paragraph is the heading, read as body on arrival. + del body[len(body) - len(paragraph) :] + flush() + # A wrapped heading keeps every line, so token one is the version. + heading = "\n".join(paragraph) + version = _version_from_heading(heading) + body = [] + paragraph = [] + after_paragraph = False + continue + # A dashed underline is not a list marker, so track lists after setext. + lazy_marker = _lazy_marker(structural, lists, after_paragraph, quoted) + lists = _open_lists(structural, lists, after_paragraph, quoted) + # Taken after the opening line closed the items it is dedented out of, + # so the block belongs to the item it is really written inside. + if opened_block: + block_column = lists.columns[-1] if lists.columns else 0 + elif open_fence is None and not in_html_block and in_raw_html is None and not in_comment: + block_column = 0 + # At an open item's content column a heading is nested, not a boundary. + if lists.columns and _indent_width(visible) >= lists.columns[0]: + match = None + # The line at its own nesting level: past the container's indentation + # and past a marker on the same line, so `- ## 2.0` reads as a heading. + column = lists.columns[-1] if lists.columns else 0 + content = _strip_indent(visible, column) + if (item := _LIST_ITEM.match(content)) is not None: + content = content[item.end() :] + # Only ordinary text continues a paragraph. Indented code counts four + # spaces past the container, so an item's own indent does not count. + indented_code = not after_paragraph and _indent_width(visible) - column >= 4 + # An underline ends the paragraph it underlines, so it needs one open in + # its own container: the quote above owns its own, and a row left of an + # open item is lazy text of the item's paragraph. Three dashes are a + # thematic break either way, which `_INTERRUPTS` already ends on. + underline = ( + _SETEXT_UNDERLINE.match(visible) is not None + and after_paragraph + and not quoted + and _indent_width(visible) >= column + ) + after_paragraph = ( + # Read inside its container, so an empty item and a fence written as an + # item's own content leave no paragraph open below them. A marker the + # paragraph above swallows is its text, not an item. + (bool(content.strip()) or lazy_marker) + and match is None + and _HEADING_PATTERN.match(content) is None + and _FENCE_PATTERN.match(content) is None + and not indented_code + and _INTERRUPTS.match(visible) is None + and (after_paragraph or _LINK_DEFINITION.match(visible) is None) + and not underline + ) + # A quote's paragraph runs on over plain text and owns every line of it. + # An empty quote holds none, so the line below starts the document's. + flush_left = visible.lstrip(" \t") + quote_line = _BLOCK_QUOTE.match(visible) is not None + in_quote = ( + _may_be_lazy(_quote_content(visible)) + if quote_line + else in_quote and _continues_paragraph(visible, column) + ) + if quote_line: + # The only paragraph a quote line leaves open is the quote's own, + # and a quote holding a heading or nothing at all leaves none. + after_paragraph = in_quote + # Whose paragraph the line below would continue. A quote owns the one its + # own lines hold, so a marker outside the quote is a block of its own + # rather than more of the text above it. + quoted = quote_line or in_quote + # The lines a later underline turns into one heading. A paragraph opens + # only on plain text and then runs on until something interrupts it. + continues = ( + not _interrupts_paragraph(flush_left) + if paragraph + else _PARAGRAPH_TEXT.match(flush_left) is not None + ) + # A paragraph inside an open item is that item's, and only one written + # at document level can be the heading a later underline makes of it. + if after_paragraph and not in_quote and not lists.columns and continues: + paragraph = [*paragraph, visible.strip()] + else: + paragraph = [] + if match is None: + if version is not None: + body.append(line) + continue + + flush() + # An empty heading has no title, so it ends the release above without + # indexing one: `_version_from_heading` finds no version and `flush` skips. + heading = match.group("title") or "" + version = _version_from_heading(heading) + body = [] + + flush() + return entries + + +def find_release_notes(text: str, version: str) -> ChangelogEntry | None: + """Return the section for exactly `version`, or None. + + Equality is version-aware (`2026.07.5` matches `2026.7.5`) but never fuzzy: + a near-miss returns None so the caller shows no notes, not the wrong ones. + """ + entries = parse_changelog(text) + for entry in entries: + # An exact heading wins, so `## 1.0` is never shadowed by `## 1.0.0`. + if entry.version == version: + return entry + + wanted = _parse_version(version) + for entry in entries: + if wanted is not None: + candidate = _parse_version(entry.version) + if candidate is not None and candidate == wanted: + return entry + return None + + +def get_release_notes(version: str, refresh: bool = False) -> dict[str, Any]: + """Return release notes for exactly `version` for the update popup. + + `refresh` retries a cached remote failure, so the UI's retry action is not + stuck behind the failure TTL once connectivity returns. + """ + version = version.strip() + if not is_supported_version_query(version): + return _notes_response(version = version, error = "Unsupported version.") + + local = _read_local_changelog() + remote = ChangelogSource(text = None, source = None) + if os.environ.get(DISABLE_ENV_VAR) != "1": + remote = get_remote_changelog(refresh = refresh) + + # Remote first: the offered version is newer than the local copy. + for candidate in (remote, local): + if not candidate.text: + continue + entry = find_release_notes(candidate.text, version) + if entry is not None: + return _notes_response( + version = version, + markdown = entry.body, + heading = entry.heading, + source = candidate.source, + ) + + # Nothing matched: the bundled copy cannot know a version newer than the + # install, so report a remote failure and let the UI offer a retry. + return _notes_response(version = version, error = remote.error) + + +def get_remote_changelog(refresh: bool = False) -> ChangelogSource: + """Fetch CHANGELOG.md from the repo using a small in-process TTL cache.""" + global _remote_cache, _remote_fetching + + if refresh: + # Only a cached failure is dropped, so retries cannot hammer the remote. + with _cache_condition: + if _remote_cache and _remote_cache.source.text is None: + _remote_cache = None + + # A caller waits for an in-flight fetch only as long as it may take, then + # answers locally rather than holding a worker behind a stalled upstream. + deadline = time.monotonic() + CHANGELOG_TIMEOUT_SECONDS + 1 + while True: + now = time.monotonic() + with _cache_condition: + if _remote_cache and _remote_cache.expires_at > now: + return _remote_cache.source + if not _remote_fetching: + _remote_fetching = True + break + if now >= deadline: + return ChangelogSource( + text = None, + source = None, + error = "Release notes are still loading.", + ) + _cache_condition.wait(timeout = deadline - now) + + try: + try: + source = _fetch_remote_changelog() + except Exception: + source = ChangelogSource( + text = None, + source = None, + error = "Could not fetch release notes.", + ) + + ttl = CHANGELOG_SUCCESS_TTL_SECONDS if source.text else CHANGELOG_FAILURE_TTL_SECONDS + with _cache_condition: + _remote_cache = _ChangelogCacheEntry(source = source, expires_at = time.monotonic() + ttl) + return source + finally: + # Released here, not on the Exception path: stranding the single-flight + # flag on BaseException makes every later caller wait out the deadline. + with _cache_condition: + _remote_fetching = False + _cache_condition.notify_all() + + +def _fetch_remote_changelog() -> ChangelogSource: + url = os.environ.get(CHANGELOG_URL_ENV_VAR, "").strip() or CHANGELOG_RAW_URL + if not url.startswith(("http://", "https://")): + return ChangelogSource(text = None, source = None, error = "Invalid changelog URL.") + + request = urllib.request.Request( + url, + headers = { + "User-Agent": "unsloth-studio-update-check", + # Or a compressing proxy hands back bytes we would decode as notes. + "Accept-Encoding": "identity", + }, + ) + deadline = time.monotonic() + CHANGELOG_TIMEOUT_SECONDS + try: + with urllib.request.urlopen(request, timeout = CHANGELOG_TIMEOUT_SECONDS) as response: + chunks: list[bytes] = [] + received = 0 + while received <= CHANGELOG_MAX_BYTES: + remaining = deadline - time.monotonic() + if remaining <= 0: + return ChangelogSource( + text = None, + source = None, + error = "Release notes took too long to load.", + ) + # The socket timeout is per operation, so re-cap it each read. + _limit_read(response, remaining) + chunk = response.read1(_CHANGELOG_CHUNK_BYTES) + if not chunk: + break + chunks.append(chunk) + received += len(chunk) + body = b"".join(chunks) + if len(body) > CHANGELOG_MAX_BYTES: + return ChangelogSource( + text = None, + source = None, + error = "Release notes response was too large.", + ) + return ChangelogSource(text = body.decode("utf-8", errors = "replace"), source = "remote") + except TimeoutError: + return ChangelogSource( + text = None, + source = None, + error = "Release notes took too long to load.", + ) + except OSError: + return ChangelogSource( + text = None, + source = None, + error = "Could not reach the changelog for release notes.", + ) + except UnicodeError: + return ChangelogSource(text = None, source = None, error = "Malformed changelog.") + + +def _limit_read(response: Any, remaining: float) -> None: + """Cap the next socket read at the time left in the fetch budget.""" + sock = getattr(getattr(response, "fp", None), "raw", None) + sock = getattr(sock, "_sock", None) + if sock is None: + return + try: + sock.settimeout(max(remaining, _CHANGELOG_MIN_READ_SECONDS)) + except OSError: + pass + + +def _read_local_changelog() -> ChangelogSource: + """Read the CHANGELOG.md bundled with this install, if there is one.""" + for path in _local_changelog_candidates(): + try: + if not path.is_file(): + continue + if path.stat().st_size > CHANGELOG_MAX_BYTES: + continue + return ChangelogSource( + text = path.read_text(encoding = "utf-8", errors = "replace"), + source = "local", + ) + except OSError: + continue + return ChangelogSource(text = None, source = None) + + +def _is_source_checkout(root: Path) -> bool: + """Whether `root` is this repository rather than an install directory.""" + try: + return any((root / marker).exists() for marker in _CHECKOUT_MARKERS) + except OSError: + return False + + +def _local_changelog_candidates() -> list[Path]: + override = os.environ.get(CHANGELOG_PATH_ENV_VAR, "").strip() + candidates: list[Path] = [] + if override: + candidates.append(Path(override).expanduser()) + + # changelog.py -> utils -> backend -> studio -> repo root. Repo root first + # so a checkout's editable file beats the snapshot packaging writes into + # studio/. Installed, those outer levels are site-packages, hence the marker. + parents = Path(__file__).resolve().parents + for index in (3, 2, 1, 4): + if index >= len(parents): + continue + root = parents[index] + if index in _CHECKOUT_ONLY_LEVELS and not _is_source_checkout(root): + continue + candidates.append(root / CHANGELOG_FILENAME) + + seen: set[Path] = set() + unique: list[Path] = [] + for candidate in candidates: + if candidate not in seen: + seen.add(candidate) + unique.append(candidate) + return unique + + +def _opens_fence(marker: str, rest: str) -> bool: + """A backtick fence's info string may not contain a backtick.""" + return marker[0] != "`" or "`" not in rest + + +def _next_fence_state(open_fence: str | None, marker: str, rest: str) -> str | None: + """Track the open fence marker. + + A closer must be the same character, at least as long, and carry nothing + after it. So neither a ``` sample nor a ```` line with trailing text ends + a ```` block early, while an opening fence may still have an info string. + Only spaces and tabs count as nothing: other Unicode whitespace is content. + """ + if open_fence is None: + return marker if _opens_fence(marker, rest) else None + closes = marker[0] == open_fence[0] and len(marker) >= len(open_fence) + if closes and not rest.strip(" \t"): + return None + return open_fence + + +def _code_span_ranges(line: str) -> list[tuple[int, int]]: + """Code span bounds. A run of backticks closes only on a run of its length.""" + # Collect the runs once: rescanning per opener is quadratic on a line of + # distinct unmatched runs, and notes are reparsed on every request. + runs: list[tuple[int, int]] = [] + index = 0 + while index < len(line): + if line[index] != "`" or _is_escaped(line, index): + index += 1 + continue + ticks = _run_length(line, index) + runs.append((index, ticks)) + index += ticks + + # A run closes only on a later run of its length, so one cursor per length. + by_length: dict[int, list[int]] = {} + for position, (_, ticks) in enumerate(runs): + by_length.setdefault(ticks, []).append(position) + + spans: list[tuple[int, int]] = [] + cursors: dict[int, int] = {} + current = 0 + while current < len(runs): + start, ticks = runs[current] + same = by_length[ticks] + cursor = cursors.get(ticks, 0) + while cursor < len(same) and same[cursor] <= current: + cursor += 1 + cursors[ticks] = cursor + if cursor >= len(same): + # Nothing closes this run, so it is literal text. + current += 1 + continue + closer = same[cursor] + cursors[ticks] = cursor + 1 + spans.append((start, runs[closer][0] + ticks)) + current = closer + 1 + return spans + + +def _run_length(line: str, index: int) -> int: + end = index + while end < len(line) and line[end] == "`": + end += 1 + return end - index + + +def _is_escaped(line: str, index: int) -> bool: + slashes = 0 + while index - 1 - slashes >= 0 and line[index - 1 - slashes] == "\\": + slashes += 1 + return slashes % 2 == 1 + + +def _strip_comments(line: str, in_comment: bool, block_open: bool) -> tuple[str, bool]: + """Return the line with HTML-comment spans removed, and the trailing state. + + Only a comment that starts a line opens a block and hides the lines below + it. One written mid-sentence is inline HTML: it hides the rest of its own + line at most, so a note mentioning `<!--` cannot swallow later releases. + Delimiters inside inline code are literal and hide nothing. + + "Starts a line" is read inside the container, so `block_open` is decided by + the caller from the item's content rather than from the raw line. + """ + if in_comment: + close = line.find(_COMMENT_CLOSE) + # The closing line belongs to the block, tail included. + return ("", False) if close != -1 else ("", True) + + if block_open: + # `<!-->` and `<!--->` are complete comments, so the closer may overlap + # the opener; searching past it would swallow every later release. + return ("", _COMMENT_CLOSE not in line) + + visible: list[str] = [] + index = 0 + spans = _code_span_ranges(line) + # Spans are ordered and disjoint and each opener sits at or past the one + # before, so the search resumes rather than restarts: restarting per opener is + # quadratic, and a long line of code spans is reparsed on every request. + cursor = 0 + while index < len(line): + opening = line.find(_COMMENT_OPEN, index) + if opening == -1: + visible.append(line[index:]) + break + + while cursor < len(spans) and spans[cursor][1] <= opening: + cursor += 1 + if cursor < len(spans) and spans[cursor][0] <= opening: + visible.append(line[index : spans[cursor][1]]) + index = spans[cursor][1] + continue + + visible.append(line[index:opening]) + close = line.find(_COMMENT_CLOSE, opening + len(_COMMENT_OPEN)) + if close == -1: + # Unterminated inline comment: it hides this line and no more. + break + index = close + len(_COMMENT_CLOSE) + return "".join(visible), False + + +def _hidden_structure(line: str, marker: str = "") -> str: + """`line` as list tracking sees it once the renderer hides its text. + + A comment or a raw HTML block renders nothing, but it is still a block + written at its own column, so it closes the items it sits to the left of. + Only the indentation survives: what is inside the block is not Markdown and + must not open a list of its own. `marker` is the part of the line that opens + a list item the block is the content of, which survives with it.""" + if marker: + return marker + _HIDDEN_BLOCK + if not line.strip(): + return "" + return line[: len(line) - len(line.lstrip(" \t"))] + _HIDDEN_BLOCK + + +def _indent_width(line: str) -> int: + """Columns of leading whitespace, counting a tab to the next stop of four.""" + width = 0 + for char in line: + if char == " ": + width += 1 + elif char == "\t": + width += 4 - width % 4 + else: + break + return width + + +def _strip_indent(line: str, columns: int) -> str: + """`line` with up to `columns` columns of leading whitespace removed.""" + width = 0 + index = 0 + while index < len(line) and width < columns and line[index] in " \t": + width += 1 if line[index] == " " else 4 - width % 4 + index += 1 + return line[index:] + + +def _interrupts_paragraph(line: str) -> bool: + """Whether `line` starts a block that can break into an open paragraph. + + A quote marker always can. A list item can only when it has content, and an + ordered one only when it starts at 1: anything else is text of the + paragraph it appears to interrupt.""" + if _BLOCK_QUOTE.match(line): + return True + item = None if _THEMATIC_BREAK.match(line) else _LIST_ITEM.match(line) + if item is None: + return False + marker = item.group("marker") + if not line[item.end() :].strip(): + return False + return marker[-1] not in ".)" or marker[:-1] == "1" + + +def _item_content(line: str, after_paragraph: bool) -> str: + """`line` read from the content column of a list item that opens on it. + + A block written as an item's first content sits inside that item, so + ``- ```` opens a fence even though its marker is not within three columns of + the container. The padding is capped the way `_open_lists` caps it, or + ``- ```` would read as a fence rather than the indented code it is. A + marker the paragraph above swallows opens no item, so its line is returned + whole, as is one four columns past its container. Ported to the frontend as + `itemContent` in markdown-list-columns.ts.""" + if _indent_width(line) >= 4 or (after_paragraph and not _interrupts_paragraph(line)): + return line + item = None if _THEMATIC_BREAK.match(line) else _LIST_ITEM.match(line) + if item is None: + return line + padding = _indent_width(item.group("space")) + # Over-indented content starts one column past the marker; the rest of the + # padding is the content's own indentation. + over = padding - 1 if padding > _MAX_ITEM_PADDING else 0 + return " " * over + line[item.end() :] + + +def _quote_content(line: str) -> str: + """What a blockquote line holds, with its markers stripped.""" + while (marker := _QUOTE_MARKER.match(line)) is not None: + line = line[marker.end() :] + return line + + +def _may_be_lazy(line: str) -> bool: + """Whether `line` can continue a paragraph it is indented out of. + + Only plain text can: a heading, a fence, a break or an HTML block starts a + block of its own, which closes the item instead. An underline is not one of + them: it may never be lazy, so `===` written left of an open item is read as + more of the item's paragraph. Nor is a definition, which is a block of its + own but may not interrupt a paragraph. A row of dashes still closes the + item, as `_INTERRUPTS` reads three or more as the thematic break they are.""" + return ( + _PARAGRAPH_TEXT.match(line) is not None + and _INTERRUPTS.match(line) is None + and _FENCE_PATTERN.match(line) is None + # Types 1 to 6 interrupt a paragraph, so a `<div>` left of an open item + # closes it. Type 7 cannot, and is deliberately excluded. + and not _opens_html_block(line, True) + ) + + +def _continues_paragraph(line: str, column: int) -> bool: + """Whether `line` reads as more of a paragraph open in its container. + + Measured from `column`, where that container's content starts: four columns + past it the line is an indented code block, which may not interrupt a + paragraph, so indentation alone never closes the one above it.""" + inner = _strip_indent(line, column) + return _indent_width(inner) >= 4 or _may_be_lazy(inner) + + +def _close_dedented( + columns: tuple[int, ...], line: str, indent: int, after_paragraph: bool +) -> tuple[int, ...]: + """`columns` with every item `line` is written to the left of closed. + + Read inside the container the item sits in, not from the margin: a line that + only looks indented there is lazy text of the item's paragraph, which leaves + the item open rather than closing it.""" + while columns and indent < columns[-1]: + outer = columns[-2] if len(columns) > 1 else 0 + if after_paragraph and _continues_paragraph(line, outer): + break + columns = columns[:-1] + return columns + + +def _lazy_marker(line: str, state: _ListState, after_paragraph: bool, quoted: bool) -> bool: + """Whether a marker-shaped `line` is really text of the paragraph above it. + + Only a marker inside the paragraph's own item interrupts it; one to the left + closes that item and opens a sibling. A quote owns the paragraph its lines + hold, so a marker written outside the quote opens a list of its own.""" + item = None if _THEMATIC_BREAK.match(line) else _LIST_ITEM.match(line) + columns = state.columns + return ( + item is not None + and after_paragraph + and not quoted + and (not columns or _indent_width(line) >= columns[-1]) + and not _interrupts_paragraph(line) + ) + + +def _open_lists( + line: str, + state: _ListState, + after_paragraph: bool, + quoted: bool = False, +) -> _ListState: + """The list items still open after `line`. + + A dedented line closes an item, unless it is a lazy paragraph continuation. + A new marker nests under a deeper column and replaces a sibling. `quoted` + marks a paragraph the blockquote above owns: a marker written outside the + quote is not text of it, so it opens a list of its own. + """ + columns = state.columns + if not line.strip(): + # A blank line leaves the list open, unless the item is still empty: an + # item may begin with one blank line, and later content is outside it. + return _ListState(columns[:-1] if state.empty_item else columns) + indent = _indent_width(line) + item = None if _THEMATIC_BREAK.match(line) else _LIST_ITEM.match(line) + empty = item is not None and not line[item.end() :].strip() + if _lazy_marker(line, state, after_paragraph, quoted): + # A lazy continuation or an underline, so the open items are untouched. + return state + columns = _close_dedented(columns, line, indent, after_paragraph) + # Four columns past its container the marker is an indented code block, or + # lazy text of the paragraph above it, so it opens no list of its own. + if item is None or indent - (columns[-1] if columns else 0) >= 4: + return _ListState(columns) + marker = item.group("marker") + padding = _indent_width(item.group("space")) + if padding == 0 or padding > _MAX_ITEM_PADDING: + # An empty or over-indented item still holds one column of content. + padding = 1 + while columns and columns[-1] > indent: + columns = columns[:-1] + return _ListState((*columns, indent + len(marker) + padding), empty_item = empty) + + +def _opens_html_block(line: str, after_paragraph: bool) -> bool: + """True if `line` starts a CommonMark type 6 or type 7 HTML block.""" + match = _HTML_BLOCK_OPEN.match(line) + if match is not None and match.group(1).lower() in _HTML_BLOCK_TAGS: + return True + return not after_paragraph and _HTML_TAG_ONLY_LINE.match(line) is not None + + +def _strip_raw_html(line: str, open_block: int | None) -> tuple[str, int | None]: + """Drop the parts of a line inside a raw block, and return the open block. + + The state is the index of the open block in `_RAW_BLOCKS`, or None.""" + if open_block is not None: + close = _RAW_BLOCKS[open_block][1].search(line) + return ("", None) if close else ("", open_block) + + # A block only opens at the start of a line; mid-line tags are inline HTML. + for index, (opener, closer) in enumerate(_RAW_BLOCKS): + opening = opener.match(line) + if opening is None: + continue + rest = line[opening.end() :] + close = closer.search(rest) + return ("", None) if close else ("", index) + return line, None + + +def _version_from_heading(heading: str) -> str | None: + token = heading.split()[0] if heading.split() else "" + match = _VERSION_TOKEN_PATTERN.match(token) + if match is None: + return None + version = match.group("version") + return version if _parse_version(version) is not None else None + + +def _parse_version(version: str) -> Version | None: + try: + return Version(version) + except InvalidVersion: + return None + + +def _close_open_fence(markdown: str) -> str: + """Close a fence the truncation cut in half, so the rest still renders.""" + open_fence: str | None = None + for line in _markdown_lines(markdown): + fence = _FENCE_PATTERN.match(line) + if fence: + open_fence = _next_fence_state(open_fence, fence.group("marker"), fence.group("rest")) + return f"{markdown}\n{open_fence}" if open_fence else markdown + + +def _renders_visibly(markdown: str) -> bool: + """Whether a section body renders anything at all.""" + in_comment = False + for line in _markdown_lines(markdown): + opens_raw = any(opener.match(line) for opener, _ in _RAW_BLOCKS) + if not in_comment and (_FENCE_PATTERN.match(line) or opens_raw): + # A code block or raw HTML block renders even when it is empty. + return True + # No containers are tracked here, so the opener is read at the margin. The + # answer does not turn on it: an item renders its marker whatever the block + # inside hides, so a commented-out item renders something either way. + visible, in_comment = _strip_comments( + line, in_comment, _COMMENT_BLOCK_OPEN.match(line) is not None + ) + if visible.strip(): + return True + return False + + +def _notes_response( + *, + version: str, + markdown: str | None = None, + heading: str | None = None, + source: str | None = None, + error: str | None = None, +) -> dict[str, Any]: + # A section that renders as nothing counts as unpublished, not as empty. + if markdown and not _renders_visibly(markdown): + markdown = None + source = None + + truncated = False + if markdown and len(markdown) > RELEASE_NOTES_MAX_CHARS: + markdown = _close_open_fence(markdown[:RELEASE_NOTES_MAX_CHARS].rstrip()) + truncated = True + + return { + "version": version, + "markdown": markdown or None, + "heading": heading, + # False means no notes for this exact version; the UI links out. + "matched": bool(markdown), + "truncated": truncated, + "source": source, + "release_notes_url": RELEASE_NOTES_URL, + "error": error, + } diff --git a/studio/backend/utils/child_stdio.py b/studio/backend/utils/child_stdio.py new file mode 100644 index 0000000000..4709d650df --- /dev/null +++ b/studio/backend/utils/child_stdio.py @@ -0,0 +1,22 @@ +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 + +"""Make a Python child agree with the parent that its pipes are UTF-8. + +A child's ``sys.stdout`` uses ``locale.getpreferredencoding()``, which on +Windows is the ANSI code page. Reading that pipe as UTF-8 would then mangle any +non-ASCII the child prints, so the child has to be told which encoding to emit. +Only needed for Python children; llama.cpp and node already emit UTF-8. +""" + +from __future__ import annotations + +import os +from typing import Mapping, Optional + + +def utf8_child_env(env: Optional[Mapping[str, str]] = None) -> dict[str, str]: + """Copy *env* (or the current environment) with UTF-8 stdio forced.""" + child = dict(os.environ if env is None else env) + child["PYTHONIOENCODING"] = "utf-8" + return child diff --git a/studio/backend/utils/hardware/amd.py b/studio/backend/utils/hardware/amd.py index 91a06c9a2a..318759f67d 100644 --- a/studio/backend/utils/hardware/amd.py +++ b/studio/backend/utils/hardware/amd.py @@ -144,6 +144,8 @@ def _run_amd_smi(*args: str, timeout: int = _AMD_SMI_DEFAULT_TIMEOUT) -> Optiona ["amd-smi", *args, "--json"], capture_output = True, text = True, + encoding = "utf-8", + errors = "replace", timeout = timeout, env = _amd_env, **windows_hidden_subprocess_kwargs(), diff --git a/studio/backend/utils/hardware/hardware.py b/studio/backend/utils/hardware/hardware.py index 719087c671..831bd6ca82 100644 --- a/studio/backend/utils/hardware/hardware.py +++ b/studio/backend/utils/hardware/hardware.py @@ -830,6 +830,8 @@ def _rocm_windows_perf_counter_gpu_util_pct() -> Optional[float]: ["powershell", "-NoProfile", "-NonInteractive", "-Command", ps], capture_output = True, text = True, + encoding = "utf-8", + errors = "replace", timeout = 5, ) if r.returncode != 0 or not r.stdout.strip(): @@ -1027,6 +1029,8 @@ def _rocm_windows_perf_counter_vram_by_adapter() -> Optional[list[tuple[str, flo ["powershell", "-NoProfile", "-NonInteractive", "-Command", ps], capture_output = True, text = True, + encoding = "utf-8", + errors = "replace", timeout = 5, ) if r.returncode != 0 or not r.stdout.strip(): diff --git a/studio/backend/utils/hardware/nvidia.py b/studio/backend/utils/hardware/nvidia.py index f98ca4343e..39e3652921 100644 --- a/studio/backend/utils/hardware/nvidia.py +++ b/studio/backend/utils/hardware/nvidia.py @@ -55,6 +55,8 @@ def get_physical_gpu_count() -> Optional[int]: ["nvidia-smi", "-L"], capture_output = True, text = True, + encoding = "utf-8", + errors = "replace", timeout = 5, env = child_env_without_native_path_secret(), **_windows_hidden_subprocess_kwargs(), @@ -81,6 +83,8 @@ def get_primary_gpu_utilization() -> dict[str, Any]: ], capture_output = True, text = True, + encoding = "utf-8", + errors = "replace", timeout = 5, env = child_env_without_native_path_secret(), **_windows_hidden_subprocess_kwargs(), @@ -131,6 +135,8 @@ def get_visible_gpu_utilization( ], capture_output = True, text = True, + encoding = "utf-8", + errors = "replace", timeout = 5, env = child_env_without_native_path_secret(), **_windows_hidden_subprocess_kwargs(), @@ -215,6 +221,8 @@ def get_backend_visible_gpu_info( ], capture_output = True, text = True, + encoding = "utf-8", + errors = "replace", timeout = 10, env = child_env_without_native_path_secret(), **_windows_hidden_subprocess_kwargs(), diff --git a/studio/backend/utils/llama_cpp_update.py b/studio/backend/utils/llama_cpp_update.py index 78a46a91e4..45c26d1b02 100644 --- a/studio/backend/utils/llama_cpp_update.py +++ b/studio/backend/utils/llama_cpp_update.py @@ -121,7 +121,14 @@ def _installed_build_number(binary: Optional[str]) -> Optional[int]: if not binary: return None try: - proc = subprocess.run([binary, "--version"], capture_output = True, text = True, timeout = 20) + proc = subprocess.run( + [binary, "--version"], + capture_output = True, + text = True, + encoding = "utf-8", + errors = "replace", + timeout = 20, + ) except Exception: # pragma: no cover - defensive return None m = re.search(r"version:\s*(\d+)", (proc.stderr or "") + (proc.stdout or "")) diff --git a/studio/backend/utils/mlx_repair.py b/studio/backend/utils/mlx_repair.py index 4ea1ec62f5..8e2a6a7712 100644 --- a/studio/backend/utils/mlx_repair.py +++ b/studio/backend/utils/mlx_repair.py @@ -254,7 +254,7 @@ def _transformers_constraint_args() -> tuple[list[str], str | None]: except Exception: return [], None fd, path = tempfile.mkstemp(prefix = "mlx_repair_", suffix = ".txt") - with os.fdopen(fd, "w") as fh: + with os.fdopen(fd, "w", encoding = "utf-8") as fh: fh.write(f"transformers=={transformers_version}\n") return ["--constraint", path], path @@ -290,6 +290,8 @@ def attempt_mlx_repair(*, timeout: int = _REPAIR_TIMEOUT_S) -> bool: stdout = subprocess.PIPE, stderr = subprocess.STDOUT, text = True, + encoding = "utf-8", + errors = "replace", timeout = timeout, ) except subprocess.TimeoutExpired: diff --git a/studio/backend/utils/models/checkpoints.py b/studio/backend/utils/models/checkpoints.py index 6950667bbd..eaf75140fc 100644 --- a/studio/backend/utils/models/checkpoints.py +++ b/studio/backend/utils/models/checkpoints.py @@ -129,7 +129,7 @@ def _read_checkpoint_loss(checkpoint_path: Path) -> Optional[float]: if not trainer_state.exists(): return None try: - with open(trainer_state, encoding = "utf-8") as f: + with open(trainer_state, encoding = "utf-8-sig") as f: state = json.load(f) log_history = state.get("log_history", []) if log_history: @@ -174,18 +174,18 @@ def scan_checkpoints( metadata: dict = {} try: if adapter_config.exists(): - cfg = json.loads(adapter_config.read_text(encoding = "utf-8")) + cfg = json.loads(adapter_config.read_text(encoding = "utf-8-sig")) metadata["base_model"] = cfg.get("base_model_name_or_path") metadata["peft_type"] = cfg.get("peft_type") metadata["lora_rank"] = cfg.get("r") elif config_file.exists(): - cfg = json.loads(config_file.read_text(encoding = "utf-8")) + cfg = json.loads(config_file.read_text(encoding = "utf-8-sig")) metadata["base_model"] = cfg.get("_name_or_path") # Detect BNB quantization from config.json if config_file.exists(): if "cfg" not in dir(): - cfg = json.loads(config_file.read_text(encoding = "utf-8")) + cfg = json.loads(config_file.read_text(encoding = "utf-8-sig")) quant_cfg = cfg.get("quantization_config") if ( isinstance(quant_cfg, dict) diff --git a/studio/backend/utils/models/model_config.py b/studio/backend/utils/models/model_config.py index 893b842e11..6270d9e03f 100644 --- a/studio/backend/utils/models/model_config.py +++ b/studio/backend/utils/models/model_config.py @@ -37,6 +37,7 @@ import yaml from utils.native_path_leases import child_env_without_native_path_secret +from utils.child_stdio import utf8_child_env from utils.hf_cache_settings import active_hf_hub_cache, get_hf_cache_paths from utils.subprocess_compat import ( windows_hidden_subprocess_kwargs as _windows_hidden_subprocess_kwargs, @@ -631,7 +632,7 @@ def _raw_config_has_vision_config( cache_dir = active_hf_hub_cache(), ) ) - config = json.loads(config_path.read_text(encoding = "utf-8")) + config = json.loads(config_path.read_text(encoding = "utf-8-sig")) architectures = config.get("architectures") or [] model_type = config.get("model_type") explicit_vision = ( @@ -774,8 +775,12 @@ def _is_vision_model_subprocess(model_name: str, hf_token: Optional[str] = None) ], capture_output = True, text = True, + encoding = "utf-8", + errors = "replace", timeout = 60, - env = get_hf_cache_paths().child_env(child_env_without_native_path_secret()), + env = utf8_child_env( + get_hf_cache_paths().child_env(child_env_without_native_path_secret()) + ), **_windows_hidden_subprocess_kwargs(), ) @@ -1083,7 +1088,7 @@ def _detect_audio_from_tokenizer( ]: tok_file = snapshot / tok_path if tok_file.exists(): - tok_config = json.loads(tok_file.read_text(encoding = "utf-8")) + tok_config = json.loads(tok_file.read_text(encoding = "utf-8-sig")) read_any = True result = _check_token_patterns(tok_config) if result: @@ -2283,7 +2288,7 @@ def scan_exported_models( export_meta = run_dir / "export_metadata.json" try: if export_meta.exists(): - meta = json.loads(export_meta.read_text(encoding = "utf-8")) + meta = json.loads(export_meta.read_text(encoding = "utf-8-sig")) base_model = meta.get("base_model") except Exception: pass @@ -2312,7 +2317,7 @@ def scan_exported_models( if adapter_config.exists(): export_type = "lora" try: - cfg = json.loads(adapter_config.read_text(encoding = "utf-8")) + cfg = json.loads(adapter_config.read_text(encoding = "utf-8-sig")) base_model = cfg.get("base_model_name_or_path") except Exception: pass @@ -2321,7 +2326,7 @@ def scan_exported_models( export_meta = checkpoint_dir / "export_metadata.json" try: if export_meta.exists(): - meta = json.loads(export_meta.read_text(encoding = "utf-8")) + meta = json.loads(export_meta.read_text(encoding = "utf-8-sig")) base_model = meta.get("base_model") except Exception: pass @@ -2334,7 +2339,7 @@ def scan_exported_models( export_meta = meta_dir / "export_metadata.json" try: if export_meta.exists(): - meta = json.loads(export_meta.read_text(encoding = "utf-8")) + meta = json.loads(export_meta.read_text(encoding = "utf-8-sig")) base_model = meta.get("base_model") if base_model: break @@ -2354,7 +2359,7 @@ def scan_exported_models( outputs_adapter_cfg = resolve_output_dir(run_dir.name) / "adapter_config.json" try: if outputs_adapter_cfg.exists(): - cfg = json.loads(outputs_adapter_cfg.read_text(encoding = "utf-8")) + cfg = json.loads(outputs_adapter_cfg.read_text(encoding = "utf-8-sig")) base_model = cfg.get("base_model_name_or_path") except Exception: pass @@ -2380,7 +2385,7 @@ def get_base_model_from_checkpoint(checkpoint_path: str) -> Optional[str]: adapter_config_path = checkpoint_path_obj / "adapter_config.json" if adapter_config_path.exists(): - with open(adapter_config_path, "r", encoding = "utf-8") as f: + with open(adapter_config_path, "r", encoding = "utf-8-sig") as f: config = json.load(f) base_model = config.get("base_model_name_or_path") if base_model: @@ -2389,7 +2394,7 @@ def get_base_model_from_checkpoint(checkpoint_path: str) -> Optional[str]: config_path = checkpoint_path_obj / "config.json" if config_path.exists(): - with open(config_path, "r", encoding = "utf-8") as f: + with open(config_path, "r", encoding = "utf-8-sig") as f: config = json.load(f) for key in ("model_name", "_name_or_path"): base_model = config.get(key) @@ -2445,7 +2450,7 @@ def get_base_model_from_lora(lora_path: str) -> Optional[str]: # adapter_config.json first adapter_config_path = lora_path_obj / "adapter_config.json" if adapter_config_path.exists(): - with open(adapter_config_path, "r", encoding = "utf-8") as f: + with open(adapter_config_path, "r", encoding = "utf-8-sig") as f: config = json.load(f) base_model = config.get("base_model_name_or_path") if base_model: @@ -2535,7 +2540,7 @@ def get_base_model_from_lora_identifier( last_exc = exc continue try: - with open(cfg_path, "r", encoding = "utf-8") as f: + with open(cfg_path, "r", encoding = "utf-8-sig") as f: base_model = json.load(f).get("base_model_name_or_path") except Exception as exc: logger.warning("Could not parse adapter_config.json for '%s': %s", identifier, exc) @@ -2781,7 +2786,7 @@ class ModelConfig: meta_path = gguf_dir / "export_metadata.json" if meta_path.exists(): try: - meta = json.loads(meta_path.read_text(encoding = "utf-8")) + meta = json.loads(meta_path.read_text(encoding = "utf-8-sig")) base = meta.get("base_model") if base and is_vision_model(base, hf_token = hf_token): base_is_vision = True @@ -2912,7 +2917,7 @@ class ModelConfig: token = hf_token, cache_dir = active_hf_hub_cache(), ) - with open(config_path, "r", encoding = "utf-8") as f: + with open(config_path, "r", encoding = "utf-8-sig") as f: adapter_config = json.load(f) base_model = adapter_config.get("base_model_name_or_path") if base_model: diff --git a/studio/backend/utils/node_runtime.py b/studio/backend/utils/node_runtime.py index fef2430708..697661a095 100644 --- a/studio/backend/utils/node_runtime.py +++ b/studio/backend/utils/node_runtime.py @@ -79,6 +79,8 @@ def _node_version_ok(executable: str) -> bool: [executable, "-v"], capture_output = True, text = True, + encoding = "utf-8", + errors = "replace", timeout = _NODE_VERSION_PROBE_TIMEOUT_SECONDS, **windows_hidden_subprocess_kwargs(), ) diff --git a/studio/backend/utils/paths/storage_roots.py b/studio/backend/utils/paths/storage_roots.py index ae1319d296..0b1398f6d2 100644 --- a/studio/backend/utils/paths/storage_roots.py +++ b/studio/backend/utils/paths/storage_roots.py @@ -212,7 +212,7 @@ def lmstudio_model_dirs() -> list[Path]: settings_path = Path.home() / ".lmstudio" / "settings.json" if settings_path.is_file(): try: - with open(settings_path, encoding = "utf-8") as f: + with open(settings_path, encoding = "utf-8-sig") as f: settings = json.load(f) downloads = settings.get("downloadsFolder", "") if downloads: diff --git a/studio/backend/utils/prebuilt/update_flow.py b/studio/backend/utils/prebuilt/update_flow.py index 74af0c18f9..69c1566fc3 100644 --- a/studio/backend/utils/prebuilt/update_flow.py +++ b/studio/backend/utils/prebuilt/update_flow.py @@ -24,6 +24,7 @@ from typing import Callable, Optional import structlog +from utils.child_stdio import utf8_child_env from utils.process_lifetime import child_popen_kwargs logger = structlog.get_logger(__name__) @@ -159,6 +160,8 @@ def resolve_prebuilt_for_host( cmd, capture_output = True, text = True, + encoding = "utf-8", + errors = "replace", timeout = 60, ) out = (proc.stdout or "").strip() @@ -303,7 +306,10 @@ def stream_installer( stdout = subprocess.PIPE, stderr = subprocess.STDOUT, text = True, - env = env, + encoding = "utf-8", + errors = "replace", + # Make the Python child emit the UTF-8 we decode above. + env = utf8_child_env(env), **child_popen_kwargs(), ) timed_out = threading.Event() diff --git a/studio/backend/utils/security/consent.py b/studio/backend/utils/security/consent.py index 6fee259139..9385270ee0 100644 --- a/studio/backend/utils/security/consent.py +++ b/studio/backend/utils/security/consent.py @@ -142,7 +142,7 @@ def _load_remote_code_configs(model_name: str, hf_token: Optional[str] = None) - for name in _REMOTE_CODE_CONFIG_FILES: p = root / name if p.is_file(): - configs.append(json.loads(p.read_text(encoding = "utf-8"))) + configs.append(json.loads(p.read_text(encoding = "utf-8-sig"))) return configs from huggingface_hub import hf_hub_download @@ -164,7 +164,7 @@ def _load_remote_code_configs(model_name: str, hf_token: Optional[str] = None) - # Transient/auth failure is not "absent" -> fail closed to "unknown" so # the caller scans (a tokenizer/processor-only auto_map must not slip by). return None - configs.append(json.loads(Path(p).read_text(encoding = "utf-8"))) + configs.append(json.loads(Path(p).read_text(encoding = "utf-8-sig"))) # Every config was read or a genuine 404 -> an empty list is a definitive # "no auto_map", not "unknown". return configs diff --git a/studio/backend/utils/security/file_security.py b/studio/backend/utils/security/file_security.py index 7724406e8d..4588f32b90 100644 --- a/studio/backend/utils/security/file_security.py +++ b/studio/backend/utils/security/file_security.py @@ -199,7 +199,7 @@ def _indexed_shard_paths( inconclusive = True # transient: an index that might exist could not be read continue try: - weight_map = (json.loads(open(index_path, encoding = "utf-8").read()) or {}).get( + weight_map = (json.loads(open(index_path, encoding = "utf-8-sig").read()) or {}).get( "weight_map" ) or {} for shard in weight_map.values(): @@ -328,7 +328,7 @@ def _st_load_roots(snapshot: Path) -> list: roots = [snapshot] try: import json - modules = json.loads((snapshot / "modules.json").read_text(encoding = "utf-8")) + modules = json.loads((snapshot / "modules.json").read_text(encoding = "utf-8-sig")) except (OSError, ValueError): return roots # no / invalid modules.json -> snapshot root is the only load root for module in modules or (): @@ -355,7 +355,7 @@ def _indexed_pickle_shards(index_path: Path, root: Path, snapshot: Path) -> list try: # JSON is UTF-8 by spec; pin it so a non-ASCII index is not misdecoded (and needlessly # blocked) under Windows' cp1252 default. - parsed = json.loads(index_path.read_text(encoding = "utf-8")) + parsed = json.loads(index_path.read_text(encoding = "utf-8-sig")) except (OSError, ValueError) as exc: raise OSError(f"unreadable weight index: {index_path}") from exc weight_map = parsed.get("weight_map") if isinstance(parsed, dict) else None diff --git a/studio/backend/utils/security/remote_code_approvals.py b/studio/backend/utils/security/remote_code_approvals.py index f1baac6924..d6076fd2b7 100644 --- a/studio/backend/utils/security/remote_code_approvals.py +++ b/studio/backend/utils/security/remote_code_approvals.py @@ -69,7 +69,7 @@ def approval_target_key(targets) -> str: def _load() -> dict: """Parsed store, or an empty skeleton on any error (fail-safe = re-prompt).""" try: - with open(_store_path(), encoding = "utf-8") as f: + with open(_store_path(), encoding = "utf-8-sig") as f: data = json.load(f) # Validate the shape, not just the version: a hand-edited ``subjects`` that is not a # dict (e.g. ``[]``) would otherwise crash lookup/record instead of failing safe. diff --git a/studio/backend/utils/security/remote_code_scan.py b/studio/backend/utils/security/remote_code_scan.py index d4d8003252..42f9d98efe 100644 --- a/studio/backend/utils/security/remote_code_scan.py +++ b/studio/backend/utils/security/remote_code_scan.py @@ -454,7 +454,7 @@ def repo_remote_code_files(model_name: str, hf_token: Optional[str] = None) -> d p = root / name if p.is_file(): try: - ext_refs |= _auto_map_refs(json.loads(p.read_text(encoding = "utf-8"))) + ext_refs |= _auto_map_refs(json.loads(p.read_text(encoding = "utf-8-sig"))) except Exception: pass if not _add_external_refs(files, ext_refs, hf_token, model_name): @@ -483,7 +483,7 @@ def repo_remote_code_files(model_name: str, hf_token: Optional[str] = None) -> d f"{model_name}: config {cfg_name} could not be fetched ({exc})" ) from exc try: - refs |= _auto_map_refs(json.loads(Path(cfg_path).read_text(encoding = "utf-8"))) + refs |= _auto_map_refs(json.loads(Path(cfg_path).read_text(encoding = "utf-8-sig"))) except Exception: pass own_refs = {fn for repo, fn in refs if repo is None} @@ -616,7 +616,7 @@ def external_auto_map_repos(model_name: str, hf_token: Optional[str] = None) -> if not p.is_file(): continue try: - refs = _auto_map_refs(json.loads(p.read_text(encoding = "utf-8"))) + refs = _auto_map_refs(json.loads(p.read_text(encoding = "utf-8-sig"))) except Exception: continue repos.update(repo for repo, _fn in refs if repo) @@ -638,7 +638,7 @@ def external_auto_map_repos(model_name: str, hf_token: Optional[str] = None) -> except Exception: continue try: - refs = _auto_map_refs(json.loads(Path(cfg_path).read_text(encoding = "utf-8"))) + refs = _auto_map_refs(json.loads(Path(cfg_path).read_text(encoding = "utf-8-sig"))) except Exception: continue repos.update(repo for repo, _fn in refs if repo) diff --git a/studio/backend/utils/ssm_runtime.py b/studio/backend/utils/ssm_runtime.py index ca7e2309f9..b864e78608 100644 --- a/studio/backend/utils/ssm_runtime.py +++ b/studio/backend/utils/ssm_runtime.py @@ -23,6 +23,7 @@ import threading from typing import Any, Callable, Optional from loggers import get_logger +from utils.child_stdio import utf8_child_env from utils.wheel_utils import ( direct_wheel_url, install_wheel, @@ -254,6 +255,12 @@ def _install_kernel( "stdout": subprocess.PIPE, "stderr": subprocess.STDOUT, "text": True, + # pip and the compilers it drives write UTF-8 down this pipe; the Windows + # ANSI codepage would mojibake or raise over a fine install. + "encoding": "utf-8", + "errors": "replace", + # Make the Python child emit the UTF-8 we decode above. + "env": utf8_child_env(), } if is_hip: run_kwargs["timeout"] = 1800 # ROCm builds can take 10-30 min @@ -261,7 +268,8 @@ def _install_kernel( if "--gcc-install-dir" not in existing: gcc_dir = _hipcc_gcc_install_dir() if gcc_dir: - _env = os.environ.copy() + # Extends the UTF-8 env above rather than replacing it. + _env = dict(run_kwargs["env"]) _env["HIPCC_COMPILE_FLAGS_APPEND"] = ( f"{existing} --gcc-install-dir={gcc_dir}".strip() ) diff --git a/studio/backend/utils/studio_version.py b/studio/backend/utils/studio_version.py index 82ade74bba..cfaba36a81 100644 --- a/studio/backend/utils/studio_version.py +++ b/studio/backend/utils/studio_version.py @@ -60,6 +60,8 @@ def _exact_git_studio_tag(repo_root: Path) -> str | None: stdout = subprocess.PIPE, stderr = subprocess.DEVNULL, text = True, + encoding = "utf-8", + errors = "replace", timeout = _GIT_TIMEOUT_SECONDS, ) except (OSError, subprocess.TimeoutExpired): @@ -81,6 +83,8 @@ def _git_branch(repo_root: Path) -> str | None: stdout = subprocess.PIPE, stderr = subprocess.DEVNULL, text = True, + encoding = "utf-8", + errors = "replace", timeout = _GIT_TIMEOUT_SECONDS, ) except (OSError, subprocess.TimeoutExpired): diff --git a/studio/backend/utils/transformers_version.py b/studio/backend/utils/transformers_version.py index b0a2da0e66..3774409009 100644 --- a/studio/backend/utils/transformers_version.py +++ b/studio/backend/utils/transformers_version.py @@ -44,6 +44,7 @@ import time from pathlib import Path from utils.native_path_leases import child_env_without_native_path_secret +from utils.child_stdio import utf8_child_env from utils.hf_cache_settings import get_hf_cache_paths from utils.subprocess_compat import ( windows_hidden_subprocess_kwargs as _windows_hidden_subprocess_kwargs, @@ -420,7 +421,7 @@ def _resolve_base_model(model_name: str) -> str: adapter_cfg_path = local_path / "adapter_config.json" if _safe_is_file(adapter_cfg_path): try: - with open(adapter_cfg_path, encoding = "utf-8") as f: + with open(adapter_cfg_path, encoding = "utf-8-sig") as f: cfg = json.load(f) base = cfg.get("base_model_name_or_path") if base: @@ -437,7 +438,7 @@ def _resolve_base_model(model_name: str) -> str: config_json_path = local_path / "config.json" if _safe_is_file(config_json_path): try: - with open(config_json_path, encoding = "utf-8") as f: + with open(config_json_path, encoding = "utf-8-sig") as f: cfg = json.load(f) # Unsloth writes model_name, HF writes _name_or_path; skip a self-reference. for _key in ("model_name", "_name_or_path"): @@ -544,7 +545,7 @@ def _adapter_base_from_hf_cache(model_name: str) -> str | None: ) for cfg_path in candidates: if cfg_path.is_file(): - base = json.loads(cfg_path.read_text(encoding = "utf-8")).get( + base = json.loads(cfg_path.read_text(encoding = "utf-8-sig")).get( "base_model_name_or_path" ) return base or None @@ -616,7 +617,7 @@ def _check_tokenizer_config_needs_v5(model_name: str, hf_token: str | None = Non local_tc = local_path / "tokenizer_config.json" if _safe_is_file(local_tc): try: - with open(local_tc, encoding = "utf-8") as f: + with open(local_tc, encoding = "utf-8-sig") as f: data = json.load(f) tokenizer_class = data.get("tokenizer_class", "") result = tokenizer_class in _TRANSFORMERS_5_TOKENIZER_CLASSES @@ -706,7 +707,7 @@ def _config_json_from_hf_cache(model_name: str) -> dict | None: ) for cfg_path in candidates: if cfg_path.is_file(): - with open(cfg_path, encoding = "utf-8") as f: + with open(cfg_path, encoding = "utf-8-sig") as f: return json.load(f) except Exception as exc: logger.debug("HF cache config.json lookup failed for '%s': %s", model_name, exc) @@ -731,7 +732,7 @@ def _load_config_json(model_name: str, hf_token: str | None = None) -> dict | No local_cfg = Path(model_name) / "config.json" if _safe_is_file(local_cfg): try: - with open(local_cfg, encoding = "utf-8") as f: + with open(local_cfg, encoding = "utf-8-sig") as f: cfg = json.load(f) _config_json_cache[cache_key] = cfg return cfg @@ -1271,9 +1272,10 @@ def _probe_autoconfig(target_dir: str, model_name: str, hf_token: str | None) -> [sys.executable, "-c", _PROBE_CONFIG_SCRIPT, target_dir, model_name], capture_output = True, text = True, + encoding = "utf-8", errors = "replace", timeout = _PROBE_TIMEOUT_SECS, - env = env, + env = utf8_child_env(env), **_windows_hidden_subprocess_kwargs(), ) except subprocess.TimeoutExpired: @@ -1811,7 +1813,11 @@ def _install_to_dir(pkg: str, target_dir: str) -> bool: stdout = subprocess.PIPE, stderr = subprocess.STDOUT, text = True, - env = get_hf_cache_paths().child_env(child_env_without_native_path_secret()), + encoding = "utf-8", + errors = "replace", + env = utf8_child_env( + get_hf_cache_paths().child_env(child_env_without_native_path_secret()) + ), **_windows_hidden_subprocess_kwargs(), ) if result.returncode == 0: @@ -1834,7 +1840,9 @@ def _install_to_dir(pkg: str, target_dir: str) -> bool: stdout = subprocess.PIPE, stderr = subprocess.STDOUT, text = True, - env = get_hf_cache_paths().child_env(child_env_without_native_path_secret()), + encoding = "utf-8", + errors = "replace", + env = utf8_child_env(get_hf_cache_paths().child_env(child_env_without_native_path_secret())), **_windows_hidden_subprocess_kwargs(), ) if result.returncode != 0: @@ -2079,7 +2087,7 @@ class SidecarSwapInProgress(RuntimeError): def _read_swap_lock(path: Path) -> dict | None: try: - data = json.loads(path.read_text(encoding = "utf-8")) + data = json.loads(path.read_text(encoding = "utf-8-sig")) return data if isinstance(data, dict) else {} except FileNotFoundError: return None @@ -2120,7 +2128,7 @@ def try_begin_sidecar_swap(kind: str = "install") -> bool: break if fd is not None: try: - with os.fdopen(fd, "w") as f: + with os.fdopen(fd, "w", encoding = "utf-8") as f: f.write( json.dumps( {"pid": os.getpid(), "at": time.time(), "token": token, "kind": kind} @@ -2466,7 +2474,11 @@ def _ensure_venv_llmcompressor_exists() -> bool: stdout = subprocess.PIPE, stderr = subprocess.STDOUT, text = True, - env = get_hf_cache_paths().child_env(child_env_without_native_path_secret()), + encoding = "utf-8", + errors = "replace", + env = utf8_child_env( + get_hf_cache_paths().child_env(child_env_without_native_path_secret()) + ), **_windows_hidden_subprocess_kwargs(), ) last_out = result.stdout or "" diff --git a/studio/backend/utils/update_status.py b/studio/backend/utils/update_status.py index ad9dabcf36..d4b8ca1c16 100644 --- a/studio/backend/utils/update_status.py +++ b/studio/backend/utils/update_status.py @@ -30,6 +30,7 @@ PYPI_SUCCESS_TTL_SECONDS = 12 * 60 * 60 PYPI_FAILURE_TTL_SECONDS = 60 * 60 RELEASE_NOTES_URL = "https://unsloth.ai/docs/new/changelog" DISABLE_ENV_VAR = "UNSLOTH_DISABLE_UPDATE_CHECK" +FAKE_UPDATE_ENV_VAR = "UNSLOTH_STUDIO_FAKE_UPDATE" LOCAL_INSTALL_SOURCES = {"editable", "local_path", "vcs", "local_repo"} @@ -107,11 +108,32 @@ def get_studio_install_source_status(current_version: str) -> dict[str, Any]: ) +def _is_version(value: str) -> bool: + try: + Version(value) + except InvalidVersion: + return False + return True + + def get_studio_update_status(current_version: str) -> dict[str, Any]: """Return public, read-only update status for the web UI.""" install_source = detect_install_source() + disabled = os.environ.get(DISABLE_ENV_VAR) == "1" - if os.environ.get(DISABLE_ENV_VAR) == "1": + # Dev-only: the popup is PyPI-install-only, so fake a version to review it + # from a checkout. The documented opt-out still wins. + forced_version = os.environ.get(FAKE_UPDATE_ENV_VAR, "").strip() + if forced_version and not disabled and _is_version(forced_version): + return _status_response( + current_version = current_version, + latest_version = forced_version, + install_source = "pypi", + update_available = True, + can_show_web_notification = True, + ) + + if disabled: return _status_response( current_version = current_version, latest_version = None, diff --git a/studio/backend/utils/utils.py b/studio/backend/utils/utils.py index e4964b8d04..e830ea2700 100644 --- a/studio/backend/utils/utils.py +++ b/studio/backend/utils/utils.py @@ -114,6 +114,8 @@ def hf_cache_snapshot_dir(model_name: str) -> Optional[Path]: snapshot = repo_dir / "snapshots" / commit if snapshot.is_dir(): return snapshot + # UnicodeDecodeError is a ValueError, not an OSError: a torn refs + # file must keep meaning "not cached here", not fail the offline check. except (OSError, UnicodeDecodeError): continue return None diff --git a/studio/backend/utils/wheel_utils.py b/studio/backend/utils/wheel_utils.py index 1b5926fd49..8ebdea3ac1 100644 --- a/studio/backend/utils/wheel_utils.py +++ b/studio/backend/utils/wheel_utils.py @@ -15,6 +15,7 @@ import urllib.request from typing import Callable from utils.native_path_leases import child_env_without_native_path_secret +from utils.child_stdio import utf8_child_env from utils.subprocess_compat import windows_hidden_subprocess_kwargs _logger = logging.getLogger(__name__) @@ -43,6 +44,8 @@ def has_blackwell_gpu() -> bool: stdout = subprocess.PIPE, stderr = subprocess.DEVNULL, text = True, + encoding = "utf-8", + errors = "replace", timeout = 10, env = child_env_without_native_path_secret(), ) @@ -102,8 +105,10 @@ def probe_torch_wheel_env(*, timeout: int | None = None) -> dict[str, str] | Non stdout = subprocess.PIPE, stderr = subprocess.PIPE, text = True, + encoding = "utf-8", + errors = "replace", timeout = timeout, - env = child_env_without_native_path_secret(), + env = utf8_child_env(child_env_without_native_path_secret()), **windows_hidden_subprocess_kwargs(), ) except subprocess.TimeoutExpired: @@ -201,6 +206,8 @@ def install_wheel( stdout = subprocess.PIPE, stderr = subprocess.STDOUT, text = True, + encoding = "utf-8", + errors = "replace", env = child_env_without_native_path_secret(), ) attempts.append(("uv", result)) @@ -213,7 +220,10 @@ def install_wheel( stdout = subprocess.PIPE, stderr = subprocess.STDOUT, text = True, - env = child_env_without_native_path_secret(), + encoding = "utf-8", + errors = "replace", + # Make the Python child emit the UTF-8 we decode above. + env = utf8_child_env(child_env_without_native_path_secret()), ) attempts.append(("pip", result)) return attempts diff --git a/studio/backend/utils/whisper_cpp_update.py b/studio/backend/utils/whisper_cpp_update.py index cac37c25fc..45a0faf674 100644 --- a/studio/backend/utils/whisper_cpp_update.py +++ b/studio/backend/utils/whisper_cpp_update.py @@ -121,7 +121,14 @@ def _installed_whisper_version(binary: Optional[str]) -> Optional[str]: if not binary: return None try: - proc = subprocess.run([binary, "--version"], capture_output = True, text = True, timeout = 20) + proc = subprocess.run( + [binary, "--version"], + capture_output = True, + text = True, + encoding = "utf-8", + errors = "replace", + timeout = 20, + ) except Exception: # pragma: no cover - defensive return None m = re.search(r"v?(\d+\.\d+\.\d+)", (proc.stderr or "") + (proc.stdout or "")) diff --git a/studio/frontend/src/app/provider.tsx b/studio/frontend/src/app/provider.tsx index d746ed952c..b076c8cf8d 100644 --- a/studio/frontend/src/app/provider.tsx +++ b/studio/frontend/src/app/provider.tsx @@ -214,7 +214,8 @@ function TauriUpdateLayer({ } return ( - <div className="pointer-events-none fixed bottom-4 right-4 z-[9998] flex w-[calc(100vw-2rem)] max-w-[400px] flex-col items-stretch gap-2"> + // Capped like the browser stack: the download panel shares it, so both must fit. + <div className="pointer-events-none fixed bottom-4 right-4 z-[9998] flex max-h-[calc(100dvh_-_2rem)] flex-col items-end gap-2"> <UpdateBanner status={update.status} info={update.info} @@ -223,6 +224,7 @@ function TauriUpdateLayer({ isExternalServer={isExternalServer} updatePolicyMode={update.updatePolicyMode} manualReleaseUrl={update.manualReleaseUrl} + releasePageUrl={update.releasePageUrl} positioned={false} onInstall={update.installUpdate} onDismiss={update.dismiss} @@ -379,9 +381,11 @@ function TauriWrapper({ children }: { children: ReactNode }) { return ( <> {children} - {/* One bottom-right stack so overlays never overlap; they stack with a - gap, download panel anchored at the corner with banners above. */} - <div className="pointer-events-none fixed bottom-4 right-4 z-[9998] flex w-[calc(100vw-2rem)] max-w-[400px] flex-col items-stretch gap-2"> + {/* One bottom-right stack so overlays never overlap: download panel at the + corner, banners above, each owning its width. */} + {/* Capped to the viewport, or a long download list plus expanded notes + pushes the top of the stack off screen. */} + <div className="pointer-events-none fixed bottom-4 right-4 z-[9998] flex max-h-[calc(100dvh_-_2rem)] flex-col items-end gap-2"> <WebUpdateBanner positioned={false} enabled={!WEB_UPDATE_HIDDEN_ROUTES.has(pathname)} diff --git a/studio/frontend/src/components/app-sidebar.tsx b/studio/frontend/src/components/app-sidebar.tsx index d263a6a739..ecbebba326 100644 --- a/studio/frontend/src/components/app-sidebar.tsx +++ b/studio/frontend/src/components/app-sidebar.tsx @@ -1226,7 +1226,9 @@ export function AppSidebar() { openNewChat(null); }} className={cn( - "flex items-center gap-[6px] select-none transition-opacity", + // min-w-0 so a narrow sidebar truncates the wordmark + // instead of pushing the search icon over the logo. + "flex min-w-0 items-center gap-[6px] select-none transition-opacity", chatDisabled && "pointer-events-none opacity-50", )} aria-label={t("shell.aria.home")} @@ -1238,17 +1240,17 @@ export function AppSidebar() { <img src="/circle-logo-small.png" alt="Unsloth" - className="h-[calc(26px+0.5rem*var(--ui-font-scale,1))] w-[calc(26px+0.5rem*var(--ui-font-scale,1))] rounded-full object-cover" + className="h-[calc(26px+0.5rem*var(--ui-font-scale,1))] w-[calc(26px+0.5rem*var(--ui-font-scale,1))] shrink-0 rounded-full object-cover" /> - <span className="font-heading text-[calc(13px+0.5rem*var(--ui-font-scale,1))] font-semibold tracking-[0em] leading-none text-black dark:text-white dark:tracking-[0.02em]"> + <span className="truncate font-heading text-[calc(13px+0.5rem*var(--ui-font-scale,1))] font-semibold tracking-[0em] leading-none text-black dark:text-white dark:tracking-[0.02em]"> unsloth </span> - <span className="nav-badge ml-0.5 inline-flex items-center justify-center rounded-full border border-nav-beta-border px-[5px] pt-[3px] pb-[2px] text-[calc(0.5rem*var(--ui-font-scale,1))] font-medium leading-none tracking-[0.04em] text-nav-fg-muted antialiased subpixel-antialiased shadow-[0_1px_2px_rgba(0,0,0,0.06)] dark:shadow-[0_1px_2px_rgba(0,0,0,0.35)]"> + <span className="nav-badge ml-0.5 inline-flex shrink-0 items-center justify-center rounded-full border border-nav-beta-border px-[5px] pt-[3px] pb-[2px] text-[calc(0.5rem*var(--ui-font-scale,1))] font-medium leading-none tracking-[0.04em] text-nav-fg-muted antialiased subpixel-antialiased shadow-[0_1px_2px_rgba(0,0,0,0.06)] dark:shadow-[0_1px_2px_rgba(0,0,0,0.35)]"> {t("shell.beta")} </span> </Link> )} - <div className="flex items-center gap-0.5"> + <div className="flex shrink-0 items-center gap-0.25"> <Tooltip> <TooltipPrimitive.Trigger asChild> <button @@ -1257,7 +1259,7 @@ export function AppSidebar() { useChatSearchStore.getState().open(); closeMobileIfOpen(); }} - className="inline-flex h-[33px] w-[32px] cursor-pointer items-center justify-center rounded-[10px] text-nav-icon-idle dark:text-nav-fg-muted transition-colors hover:bg-nav-surface-hover hover:text-black dark:hover:text-white focus-visible:outline-none focus-visible:ring-1 focus-visible:ring-ring" + className="inline-flex h-[33px] w-[28px] cursor-pointer items-center justify-center rounded-[10px] text-nav-icon-idle dark:text-nav-fg-muted transition-colors hover:bg-nav-surface-hover hover:text-black dark:hover:text-white focus-visible:outline-none focus-visible:ring-1 focus-visible:ring-ring" aria-label={t("shell.navigation.search")} > <HugeiconsIcon icon={Search01Icon} strokeWidth={1.75} className="size-icon" /> @@ -1281,7 +1283,7 @@ export function AppSidebar() { <button type="button" onClick={togglePinned} - className="inline-flex h-[33px] w-[32px] cursor-pointer items-center justify-center rounded-[10px] text-nav-icon-idle dark:text-nav-fg-muted transition-colors hover:bg-nav-surface-hover hover:text-black dark:hover:text-white focus-visible:outline-none focus-visible:ring-1 focus-visible:ring-ring" + className="inline-flex h-[33px] w-[28px] cursor-pointer items-center justify-center rounded-[10px] text-nav-icon-idle dark:text-nav-fg-muted transition-colors hover:bg-nav-surface-hover hover:text-black dark:hover:text-white focus-visible:outline-none focus-visible:ring-1 focus-visible:ring-ring" aria-label={t("shell.aria.closeSidebar")} > <HugeiconsIcon icon={LayoutAlignLeftIcon} strokeWidth={1.75} className="size-icon" /> @@ -1325,10 +1327,10 @@ export function AppSidebar() { )} </SidebarHeader> - {/* Uniform pl-1.5 pr-2 keeps every hover pill the same width, inset from the edge. */} + {/* Uniform pl-1.5 pr-1.75 keeps every hover pill the same width, inset from the edge. */} <SidebarGroup className={cn( - "group-data-[collapsible=icon]:px-0 pl-1.5 pr-2 shrink-0 transition-[padding]", + "group-data-[collapsible=icon]:px-0 pl-1.5 pr-1.75 shrink-0 transition-[padding]", showCompactMacBrand ? "pt-0" : "pt-[9px]", // Scrolled: New Chat is pinned, give a little gap below it. scrolled ? "pb-[5px]" : "pb-px", @@ -1417,7 +1419,7 @@ export function AppSidebar() { scrolled && "is-scrolled", )} > - <SidebarGroup className="group-data-[collapsible=icon]:px-0 pl-1.5 pr-2 py-0 shrink-0"> + <SidebarGroup className="group-data-[collapsible=icon]:px-0 pl-1.5 pr-1.75 py-0 shrink-0"> <SidebarGroupContent> <SidebarMenu> <NavItem @@ -1499,7 +1501,7 @@ export function AppSidebar() { </CollapsibleTrigger> </SidebarGroupLabel> <CollapsibleContent> - <SidebarGroupContent className="pl-1.5 pr-2"> + <SidebarGroupContent className="pl-1.5 pr-1.75"> <SidebarMenu> <NavItem icon={TestTubeOutlineIcon} @@ -1574,7 +1576,7 @@ export function AppSidebar() { </CollapsibleTrigger> </SidebarGroupLabel> <CollapsibleContent> - <SidebarGroupContent className="pl-1.5 pr-2"> + <SidebarGroupContent className="pl-1.5 pr-1.75"> <SidebarMenu> {pinnedProjectRecords.map((project) => { const projectChats = @@ -1721,7 +1723,7 @@ export function AppSidebar() { </CollapsibleTrigger> </SidebarGroupLabel> <CollapsibleContent> - <SidebarGroupContent className="pl-1.5 pr-2"> + <SidebarGroupContent className="pl-1.5 pr-1.75"> <SidebarMenu> {recentChatItems.map((item) => renderChatSidebarItem(item, "recent"), @@ -1753,7 +1755,7 @@ export function AppSidebar() { </CollapsibleTrigger> </SidebarGroupLabel> <CollapsibleContent> - <SidebarGroupContent className="pl-1.5 pr-2"> + <SidebarGroupContent className="pl-1.5 pr-1.75"> <SidebarMenu> {runItems.map((run) => { // Explicit selection wins. Otherwise highlight the active diff --git a/studio/frontend/src/components/llama-update-banner.tsx b/studio/frontend/src/components/llama-update-banner.tsx index 2729558630..5276eda858 100644 --- a/studio/frontend/src/components/llama-update-banner.tsx +++ b/studio/frontend/src/components/llama-update-banner.tsx @@ -134,7 +134,7 @@ export function LlamaUpdateBanner({ className={cn( positioned ? "fixed bottom-4 right-4 z-[9998] w-[calc(100vw-2rem)] max-w-[400px]" - : "pointer-events-auto w-full", + : "pointer-events-auto w-[calc(100vw-2rem)] max-w-[400px]", )} data-testid="llama-update-banner" > diff --git a/studio/frontend/src/components/tauri/update-banner.tsx b/studio/frontend/src/components/tauri/update-banner.tsx index 6f5e655889..49c9c44aaa 100644 --- a/studio/frontend/src/components/tauri/update-banner.tsx +++ b/studio/frontend/src/components/tauri/update-banner.tsx @@ -2,6 +2,7 @@ // Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 import { Button } from "@/components/ui/button"; +import { ReleaseNotesPanel } from "@/components/update/release-notes-panel"; import type { DesktopUpdatePolicyMode, RetainedUpdateFailure, @@ -22,6 +23,8 @@ interface UpdateBannerProps { isExternalServer?: boolean; updatePolicyMode: DesktopUpdatePolicyMode; manualReleaseUrl: string | null; + // Release page for this version, preferred over the generic changelog. + releasePageUrl?: string | null; // false fills a shared overlay stack; true self-anchors. positioned?: boolean; onInstall: () => void; @@ -30,6 +33,7 @@ interface UpdateBannerProps { } const EASE_OUT_QUART: [number, number, number, number] = [0.165, 0.84, 0.44, 1]; +const LEADING_V = /^v/; function formatVersion(version: string | null | undefined): string { if (!version) return ""; @@ -44,6 +48,7 @@ export function UpdateBanner({ isExternalServer = false, updatePolicyMode, manualReleaseUrl, + releasePageUrl = null, positioned = true, onInstall, onDismiss, @@ -52,6 +57,8 @@ export function UpdateBanner({ const [copying, setCopying] = useState(false); const [manualReport, setManualReport] = useState<string | null>(null); const [manualMessage, setManualMessage] = useState<string | null>(null); + // Version whose notes are expanded; a new offer collapses the panel. + const [notesVersion, setNotesVersion] = useState<string | null>(null); const showFailure = Boolean(lastFailure) && !dismissed; const showAvailable = status === "available" && !dismissed && !showFailure; const show = showFailure || (showAvailable && Boolean(info)); @@ -62,6 +69,11 @@ export function UpdateBanner({ const currentVersion = formatVersion(info?.currentVersion); const latestVersion = formatVersion(info?.version); const Icon = showFailure ? CircleAlert : Download; + // Keyed by the backend release, not the app's SemVer; headings drop the v. + const notesTargetVersion = + (info?.pypiVersion ?? info?.version)?.replace(LEADING_V, "") ?? null; + const notesOpen = + notesTargetVersion !== null && notesVersion === notesTargetVersion; async function handleCopyDiagnostics() { setCopying(true); @@ -94,13 +106,14 @@ export function UpdateBanner({ exit={{ opacity: 0, y: 8, scale: 0.97 }} transition={{ duration: 0.35, ease: EASE_OUT_QUART }} className={cn( + // Wider than the other overlays: notes preview plus three buttons. positioned - ? "fixed bottom-4 right-4 z-[9999] w-[calc(100vw-2rem)] max-w-[400px]" - : "pointer-events-auto w-full", + ? "fixed bottom-4 right-4 z-[9999] w-[calc(100vw-2rem)] max-w-[448px]" + : "pointer-events-auto flex min-h-0 w-[calc(100vw-2rem)] max-w-[448px] flex-col", )} data-testid="tauri-update-banner" > - <div className="relative overflow-hidden rounded-[24px] bg-white px-5 pb-4 pt-5 shadow-[0_2px_8px_-2px_rgba(0,0,0,0.16)] dark:bg-card dark:shadow-[0_8px_28px_-6px_rgba(0,0,0,0.28)]"> + <div className="relative flex max-h-[calc(100dvh_-_2rem)] flex-col overflow-hidden rounded-[24px] bg-white px-5 pb-4 pt-5 shadow-[0_2px_8px_-2px_rgba(0,0,0,0.16)] dark:bg-card dark:shadow-[0_8px_28px_-6px_rgba(0,0,0,0.28)]"> <button type="button" onClick={onDismiss} @@ -160,7 +173,40 @@ export function UpdateBanner({ </p> )} - <div className="mt-4 flex flex-wrap items-center justify-end gap-x-1 gap-y-2"> + {!showFailure && notesTargetVersion ? ( + <ReleaseNotesPanel + version={notesTargetVersion} + open={notesOpen} + // Used only if CHANGELOG.md has no section for this version. + fallbackMarkdown={info?.body ?? null} + className="min-h-0 flex-1" + releaseNotesUrl={releasePageUrl ?? manualReleaseUrl} + /> + ) : null} + + <div + className={cn( + "mt-4 flex flex-wrap items-center gap-x-1 gap-y-2", + !showFailure && notesTargetVersion + ? "justify-between" + : "justify-end", + )} + > + {!showFailure && notesTargetVersion ? ( + <Button + size="sm" + variant="ghost" + // same type size as the action buttons + className="-ml-2 h-auto whitespace-nowrap rounded-full px-2.5 py-2 text-ui-13 font-medium text-foreground" + onClick={() => + setNotesVersion(notesOpen ? null : notesTargetVersion) + } + aria-expanded={notesOpen} + data-testid="tauri-update-release-notes-toggle" + > + {notesOpen ? "Hide release notes" : "Show release notes"} + </Button> + ) : null} {showFailure ? ( <> <Button @@ -187,28 +233,31 @@ export function UpdateBanner({ onClick={onInstall} disabled={installDisabled} > - {isManualLinuxPackage ? "Open release page" : "Retry update"} + {isManualLinuxPackage + ? "Open release page" + : "Retry update"} </Button> </> ) : ( - <> + // wrap + right-align so the action pair stays together + <div className="flex flex-wrap items-center justify-end gap-x-1 gap-y-2"> <Button size="sm" variant="ghost" - className="h-auto rounded-full px-3 py-2 text-ui-13 font-medium text-foreground" + className="h-auto whitespace-nowrap rounded-full px-2.5 py-2 text-ui-13 font-medium text-foreground" onClick={onDismiss} > Remind me later </Button> <Button size="sm" - className="-mr-1 h-auto rounded-full px-3.5 py-2 text-ui-13" + className="-mr-1 h-auto whitespace-nowrap rounded-full px-3 py-2 text-ui-13" onClick={onInstall} disabled={installDisabled} > {isManualLinuxPackage ? "Open release page" : "Update"} </Button> - </> + </div> )} </div> {manualMessage && ( diff --git a/studio/frontend/src/components/tauri/window-titlebar.tsx b/studio/frontend/src/components/tauri/window-titlebar.tsx index 6a0ff8741a..db57cd9960 100644 --- a/studio/frontend/src/components/tauri/window-titlebar.tsx +++ b/studio/frontend/src/components/tauri/window-titlebar.tsx @@ -2,6 +2,7 @@ // Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 import { useSidebarPin } from "@/hooks/use-sidebar-pin"; +import { useSidebarWidth } from "@/hooks/use-sidebar-width"; import { isTauri } from "@/lib/api-base"; import { cn } from "@/lib/utils"; import { @@ -110,9 +111,13 @@ export function WindowTitlebar({ const [enabled] = useState(shouldUseCustomWindowTitlebar); const [maximized, setMaximized] = useState(false); const { pinned, togglePinned } = useSidebarPin(); + // The titlebar sits outside the sidebar wrapper, so it cannot inherit + // --sidebar-width. Read the resized width from the same store instead. + const { width } = useSidebarWidth(); const sidebarWidth = showSidebarSurface ? pinned - ? "var(--studio-sidebar-expanded-width,17.5rem)" + ? // The live value only exists mid-drag; otherwise the committed width. + `var(--studio-sidebar-live-width, ${width}px)` : "var(--studio-sidebar-collapsed-width,3rem)" : "0px"; const contentBorderLeft = pinned ? `calc(${sidebarWidth} + 12px)` : "0px"; diff --git a/studio/frontend/src/components/ui/panel-resize-handle.tsx b/studio/frontend/src/components/ui/panel-resize-handle.tsx new file mode 100644 index 0000000000..9c1fd70b6f --- /dev/null +++ b/studio/frontend/src/components/ui/panel-resize-handle.tsx @@ -0,0 +1,317 @@ +// SPDX-License-Identifier: AGPL-3.0-only +// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 + +"use client" + +import * as React from "react" + +import { cn } from "@/lib/utils" +import { + Tooltip, + TooltipContent, + TooltipTrigger, +} from "@/components/ui/tooltip" +import { getClientPlatform } from "@/components/tauri/window-titlebar" + +/** Pointer travel (px) below which a drag counts as a plain click. */ +const DRAG_SLOP = 4 +/** A compatibility click lands immediately after pointer-up. */ +const CLICK_COMPAT_WINDOW_MS = 300 +/** Arrow-key resize step for keyboard users. */ +const RESIZE_STEP = 16 + +type DragState = { + startX: number + startWidth: number + moved: boolean +} + +export type PanelResizeHandleProps = { + /** Which edge of the panel the handle sits on. */ + edge: "left" | "right" + open: boolean + width: number + /** Uncapped stored preference, so a capped drag does not lower it. */ + stored: number + min: number + max: number + clamp: (px: number) => number + setWidth: (px: number) => void + resetWidth: () => void + onToggle: () => void + /** Element to paint the live width onto, and the property to paint. */ + target: () => HTMLElement | null + cssVar: string + /** Measured to start a drag from the rendered size when collapsed. */ + measure: () => number + label: string + toggleLabel: string + /** Translated tooltip copy; the caller owns the translation layer. */ + collapseHint: string + expandHint: string + dragHint: string + /** Shown in the tooltip when the panel has a toggle shortcut. */ + shortcut?: string + dataSlot?: string + className?: string + /** Mirrors the live width onto :root for chrome outside the panel. */ + rootVar?: string +} + +/** + * A draggable panel edge: drag to resize, click to collapse or expand. Arrow + * keys resize, Home restores the default. The width is painted straight to the + * target while dragging and only persisted on release. + */ +export function PanelResizeHandle({ + edge, + open, + width, + stored, + min, + max, + clamp, + setWidth, + resetWidth, + onToggle, + target, + cssVar, + measure, + label, + toggleLabel, + collapseHint, + expandHint, + dragHint, + shortcut, + dataSlot = "panel-resize-handle", + className, + rootVar, +}: PanelResizeHandleProps) { + const ref = React.useRef<HTMLButtonElement>(null) + const dragRef = React.useRef<DragState | null>(null) + const [dragging, setDragging] = React.useState(false) + const [hovered, setHovered] = React.useState(false) + const [focused, setFocused] = React.useState(false) + const [isMacPlatform] = React.useState(() => getClientPlatform().includes("mac")) + const hint = shortcut ? shortcut.replace("Mod", isMacPlatform ? "⌘" : "Ctrl+") : null + + // Cached on pointer down so no DOM walk per move. + const targetRef = React.useRef<HTMLElement | null>(null) + const frameRef = React.useRef(0) + const pendingRef = React.useRef(0) + // What the pointer asked for, before the viewport cap. Committing the capped + // value instead would quietly downgrade a stored preference on a narrow window. + const rawRef = React.useRef(0) + // When a pointer sequence last ended. The browser's compatibility click + // lands in the same tick, so only a click that close behind is a duplicate. + // A timestamp cannot go stale the way an armed flag does: a genuine cancel + // emits no click, and a later assistive-tech click still gets through. + const handledAtRef = React.useRef(0) + const committedRef = React.useRef(width) + React.useEffect(() => { + committedRef.current = width + }, [width]) + + const paint = React.useCallback( + (value: string) => { + targetRef.current?.style.setProperty(cssVar, value) + if (rootVar) { + document.documentElement.style.setProperty(rootVar, value) + } + }, + [cssVar, rootVar], + ) + + // Resizing relayouts the whole shell, and pointermove fires faster than the + // display refreshes, so coalesce to one paint per frame. + const paintWidth = React.useCallback( + (px: number) => { + pendingRef.current = px + if (frameRef.current) return + frameRef.current = requestAnimationFrame(() => { + frameRef.current = 0 + paint(`${pendingRef.current}px`) + }) + }, + [paint], + ) + + const endDrag = React.useCallback(() => { + // Only a sequence that actually started can produce a compatibility click. + // This also runs as the effect cleanup, where no drag happened. + if (dragRef.current) handledAtRef.current = Date.now() + dragRef.current = null + if (frameRef.current) { + cancelAnimationFrame(frameRef.current) + frameRef.current = 0 + } + // Hand the property back to the committed value. A commit re-renders with + // the new width; a cancel or a no-commit drag keeps DOM and store in step. + paint(`${committedRef.current}px`) + if (rootVar) document.documentElement.style.removeProperty(rootVar) + targetRef.current?.removeAttribute("data-resizing") + document.documentElement.removeAttribute("data-panel-resizing") + targetRef.current = null + setDragging(false) + document.body.style.removeProperty("cursor") + document.body.style.removeProperty("user-select") + }, [paint, rootVar]) + + const handlePointerDown = (event: React.PointerEvent<HTMLButtonElement>) => { + if (event.button !== 0) return + event.preventDefault() + event.currentTarget.setPointerCapture(event.pointerId) + targetRef.current = target() + // Collapsed: grow from the rendered size so the edge tracks the pointer. + const start = open ? width : measure() + dragRef.current = { startX: event.clientX, startWidth: start, moved: false } + pendingRef.current = start + rawRef.current = start + targetRef.current?.setAttribute("data-resizing", "true") + document.documentElement.setAttribute("data-panel-resizing", "true") + setDragging(true) + document.body.style.setProperty("cursor", "col-resize") + document.body.style.setProperty("user-select", "none") + } + + const handlePointerMove = (event: React.PointerEvent<HTMLButtonElement>) => { + const drag = dragRef.current + if (!drag) return + // A panel whose handle is on its left edge grows as the pointer moves left. + const delta = (edge === "left" ? -1 : 1) * (event.clientX - drag.startX) + if (!drag.moved && Math.abs(delta) < DRAG_SLOP) return + drag.moved = true + + const next = drag.startWidth + delta + rawRef.current = next + if (!open) { + // Past the minimum, dragging the collapsed edge reopens it. + if (next >= min) { + paintWidth(clamp(next)) + onToggle() + } + return + } + // Dragging inward stops at the minimum. Collapsing is click or the shortcut. + paintWidth(clamp(next)) + } + + const handlePointerUp = (event: React.PointerEvent<HTMLButtonElement>) => { + const drag = dragRef.current + if (!drag) return + if (event.currentTarget.hasPointerCapture(event.pointerId)) { + event.currentTarget.releasePointerCapture(event.pointerId) + } + endDrag() + + if (!drag.moved) { + onToggle() + return + } + // A drag below the minimum leaves the stored width alone. + if (!open) return + // Capped: the visible edge is already at the cap, so an outward pull cannot + // express intent beyond it. Committing would silently lower the larger + // hidden preference. A deliberate inward drag still commits. + if (stored > max && rawRef.current >= max) return + // Commit what was asked for, not the capped paint, so a drag on a narrow + // window cannot shrink a larger stored preference. setWidth clamps. + setWidth(rawRef.current) + } + + const handleKeyDown = (event: React.KeyboardEvent<HTMLButtonElement>) => { + // The collapse/expand the label advertises, for keyboard users. Pointer-up + // handles it for the mouse; a synthesized click never reaches it. + if (event.key === "Enter" || event.key === " ") { + // preventDefault cancels the native click, so nothing follows to guard + // against; arming here would swallow the next assistive-tech click. + event.preventDefault() + onToggle() + return + } + const outward = edge === "left" ? "ArrowLeft" : "ArrowRight" + const inward = edge === "left" ? "ArrowRight" : "ArrowLeft" + if (event.key === outward || event.key === inward) { + event.preventDefault() + if (!open) { + // Collapsed there is nothing to resize, so the outward arrow reopens. + if (event.key === outward) onToggle() + return + } + if (event.key === outward && stored > max && width >= max) return + setWidth(width + (event.key === outward ? RESIZE_STEP : -RESIZE_STEP)) + return + } + if (event.key === "Home") { + event.preventDefault() + resetWidth() + } + } + + // Clear a stuck cursor override if we unmount mid-drag. + React.useEffect(() => endDrag, [endDrag]) + + return ( + <Tooltip open={(hovered || focused) && !dragging}> + <TooltipTrigger asChild> + <button + ref={ref} + type="button" + data-slot={dataSlot} + data-dragging={dragging || undefined} + aria-label={open ? label : toggleLabel} + {...(open ? { "aria-orientation": "vertical" as const } : {})} + {...(open + ? { "aria-valuenow": width, "aria-valuemin": min, "aria-valuemax": max } + : {})} + role={open ? "separator" : "button"} + onPointerDown={handlePointerDown} + onPointerMove={handlePointerMove} + onPointerUp={handlePointerUp} + onPointerCancel={endDrag} + onKeyDown={handleKeyDown} + onClick={() => { + // Switch and voice control activate by dispatching a bare click + // with no pointer or key events, which nothing else here catches. + if (Date.now() - handledAtRef.current < CLICK_COMPAT_WINDOW_MS) return + onToggle() + }} + onPointerEnter={() => setHovered(true)} + onPointerLeave={() => setHovered(false)} + onFocus={(event) => setFocused(event.target.matches(":focus-visible"))} + onBlur={() => setFocused(false)} + className={cn( + "absolute inset-y-0 z-30 hidden w-2 touch-none select-none sm:block", + edge === "left" ? "-left-1" : "-right-1", + // `!` overrides the app-wide hand cursor on buttons. + open + ? "cursor-col-resize!" + : edge === "left" + ? "cursor-w-resize!" + : "cursor-e-resize!", + // Sits exactly on the panel border so hover recolours one line. + "after:absolute after:inset-y-0 after:w-px after:bg-transparent after:transition-colors after:duration-150", + edge === "left" ? "after:left-1" : "after:right-1", + "hover:after:bg-sidebar-ring/25 data-dragging:after:bg-sidebar-ring/25", + // The app zeroes the native outline on buttons, so mark focus here. + "focus-visible:outline-none focus-visible:after:bg-sidebar-ring/60", + className, + )} + /> + </TooltipTrigger> + <TooltipContent + side={edge === "left" ? "left" : "right"} + align="center" + className="tooltip-compact" + > + <span className="flex flex-col gap-px"> + <span> + {open ? collapseHint : expandHint} + {hint ? ` ${hint}` : ""} + </span> + <span className="opacity-70">{dragHint}</span> + </span> + </TooltipContent> + </Tooltip> + ) +} diff --git a/studio/frontend/src/components/ui/sidebar.tsx b/studio/frontend/src/components/ui/sidebar.tsx index 0fe82eb428..e26a55694f 100644 --- a/studio/frontend/src/components/ui/sidebar.tsx +++ b/studio/frontend/src/components/ui/sidebar.tsx @@ -24,13 +24,21 @@ import { TooltipContent, TooltipTrigger, } from "@/components/ui/tooltip" +import { PanelResizeHandle } from "@/components/ui/panel-resize-handle" +import { useT } from "@/i18n" import { useIsMobile } from "@/hooks/use-mobile" +import { + SIDEBAR_WIDTH_DEFAULT, + SIDEBAR_WIDTH_MIN, + clampSidebarWidth, + useSidebarWidth, +} from "@/hooks/use-sidebar-width" import { HugeiconsIcon } from "@hugeicons/react" import { LayoutAlignLeftIcon } from "@hugeicons/core-free-icons" const noop = () => {} -const SIDEBAR_WIDTH = "17.5rem" +const SIDEBAR_WIDTH = `${SIDEBAR_WIDTH_DEFAULT}px` const SIDEBAR_WIDTH_ICON = "3rem" const SIDEBAR_KEYBOARD_SHORTCUT = "b" @@ -46,6 +54,11 @@ type SidebarContextProps = { pinned: boolean setPinned: (value: boolean) => void togglePinned: () => void + width: number + storedWidth: number + maxWidth: number + setWidth: (value: number) => void + resetWidth: () => void } const SidebarContext = React.createContext<SidebarContextProps | null>(null) @@ -80,6 +93,13 @@ function SidebarProvider({ }) { const isMobile = useIsMobile() const [openMobile, setOpenMobile] = React.useState(false) + const { + width, + max: maxWidth, + stored: storedWidth, + setWidth, + resetWidth, + } = useSidebarWidth() const prevIsMobileRef = React.useRef(isMobile) React.useEffect(() => { @@ -163,8 +183,13 @@ function SidebarProvider({ pinned, setPinned, togglePinned, + width, + storedWidth, + maxWidth, + setWidth, + resetWidth, }), - [state, open, setOpen, isMobile, openMobile, setOpenMobile, toggleSidebar, hasPinMode, pinned, setPinned, togglePinned] + [state, open, setOpen, isMobile, openMobile, setOpenMobile, toggleSidebar, hasPinMode, pinned, setPinned, togglePinned, width, storedWidth, maxWidth, setWidth, resetWidth] ) return ( @@ -173,7 +198,8 @@ function SidebarProvider({ data-slot="sidebar-wrapper" style={ { - "--sidebar-width": SIDEBAR_WIDTH, + // The drag handle writes this same property live while resizing. + "--sidebar-width": `${width}px`, "--sidebar-width-icon": SIDEBAR_WIDTH_ICON, ...style, } as React.CSSProperties @@ -311,11 +337,64 @@ function Sidebar({ > {children} </div> + <SidebarResizeHandle side={side} /> </div> </div> ) } +/** + * The sidebar's draggable edge, over the shared panel handle. + */ +function SidebarResizeHandle({ + className, + side = "left", +}: { + className?: string + side?: "left" | "right" +}) { + const { open, toggleSidebar, width, storedWidth, maxWidth, setWidth, resetWidth } = + useSidebar() + const ref = React.useRef<HTMLDivElement>(null) + const t = useT() + + return ( + <div ref={ref} className="contents"> + <PanelResizeHandle + edge={side === "right" ? "left" : "right"} + open={open} + width={width} + stored={storedWidth} + min={SIDEBAR_WIDTH_MIN} + max={maxWidth} + clamp={clampSidebarWidth} + setWidth={setWidth} + resetWidth={resetWidth} + onToggle={toggleSidebar} + target={() => + ref.current?.closest<HTMLElement>('[data-slot="sidebar-wrapper"]') ?? null + } + cssVar="--sidebar-width" + // The custom titlebar renders outside the wrapper and cannot inherit it. + rootVar="--studio-sidebar-live-width" + measure={() => + ref.current + ?.closest<HTMLElement>('[data-slot="sidebar-container"]') + ?.getBoundingClientRect().width ?? SIDEBAR_WIDTH_MIN + } + label={t("shell.aria.resizeSidebar")} + toggleLabel={t("shell.aria.openSidebar")} + collapseHint={t("shell.resize.collapse")} + expandHint={t("shell.resize.expand")} + dragHint={t("shell.resize.drag")} + shortcut="ModB" + dataSlot="sidebar-resize-handle" + className={className} + /> + </div> + ) +} + function SidebarTrigger({ className, onClick, @@ -777,6 +856,7 @@ export { SidebarMenuSubItem, SidebarProvider, SidebarRail, + SidebarResizeHandle, SidebarSeparator, SidebarTrigger, useSidebar, diff --git a/studio/frontend/src/components/update/release-notes-panel.tsx b/studio/frontend/src/components/update/release-notes-panel.tsx new file mode 100644 index 0000000000..d98c855daa --- /dev/null +++ b/studio/frontend/src/components/update/release-notes-panel.tsx @@ -0,0 +1,251 @@ +// SPDX-License-Identifier: AGPL-3.0-only +// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 + +import { MarkdownPreview } from "@/components/markdown/markdown-preview"; +import { useReleaseNotes } from "@/hooks/use-release-notes"; +import { resolveChangelogLinks } from "@/lib/changelog-links"; +import { releaseNotesPreview } from "@/lib/release-notes-preview"; +import { cn } from "@/lib/utils"; +import { + type ReactElement, + type ReactNode, + useEffect, + useMemo, + useRef, +} from "react"; + +interface ReleaseNotesPanelProps { + // Notes are looked up for this exact version only. + version: string; + // Collapsed previews the top bullets; expanded scrolls the full notes. + open: boolean; + // Desktop updater's body, used only if CHANGELOG.md has no section here. + fallbackMarkdown?: string | null; + releaseNotesUrl?: string | null; + className?: string; +} + +const NOTES_LINK_CLASS = + "shrink-0 whitespace-nowrap text-ui-11 font-medium text-foreground underline underline-offset-2"; + +function NotesMessage({ + children, + action, +}: { + children: ReactNode; + action?: ReactNode; +}): ReactElement { + return ( + <div className="flex items-center justify-between gap-2 px-1 py-2"> + <p className="text-ui-11 text-muted-foreground">{children}</p> + {action} + </div> + ); +} + +function ChangelogLink({ href }: { href: string }): ReactElement { + return ( + <a + href={href} + target="_blank" + rel="noopener noreferrer" + className={NOTES_LINK_CLASS} + data-testid="update-release-notes-link" + > + Open changelog + </a> + ); +} + +export function ReleaseNotesPanel({ + version, + open, + fallbackMarkdown = null, + releaseNotesUrl = null, + className, +}: ReleaseNotesPanelProps): ReactElement | null { + // Fetched with the popup: the collapsed preview needs the notes too. + const { state, notes, retry } = useReleaseNotes({ version, enabled: true }); + const scrollRef = useRef<HTMLElement | null>(null); + + // The fallback stands in for "no section in the changelog", which the hook + // reports as ready. An error is retryable, and the desktop fallback is the + // updater's static blurb, so taking it there would hide Retry until cache expiry. + const source = notes?.matched + ? notes.markdown + : state === "error" + ? null + : (fallbackMarkdown ?? null); + // Notes target the repository, so relative links must point back at it. + const markdown = useMemo( + () => (source === null ? null : resolveChangelogLinks(source)), + [source], + ); + + // Notes that are only a code block or a table preview as nothing. + const preview = useMemo( + () => (markdown === null ? null : releaseNotesPreview(markdown)), + [markdown], + ); + + // Start at the top on expand, and again once async notes land. + useEffect(() => { + if (open && markdown && scrollRef.current) { + scrollRef.current.scrollTop = 0; + } + }, [open, markdown]); + + // Caller's URL wins: the API returns only the generic changelog, while the + // desktop banner passes this version's release page. + const notesUrl = releaseNotesUrl ?? notes?.releaseNotesUrl; + const link = notesUrl ? <ChangelogLink href={notesUrl} /> : null; + + // Nothing previewable yet or ever: keep the collapsed popup compact. + if ( + !open && + (!markdown || + state === "loading" || + state === "idle" || + preview?.items.length === 0) + ) { + return null; + } + + return ( + <div + className={cn("mt-3 flex min-h-0 flex-col", className)} + data-testid="update-release-notes-panel" + data-notes-state={state} + data-notes-version={version} + data-notes-open={open} + > + {/* borderless fill, lighter than the card in dark mode */} + <div className="flex min-h-0 flex-col rounded-[14px] bg-muted/40 px-3 py-1 dark:bg-white/[0.06]"> + {markdown ? ( + open ? ( + <section + ref={scrollRef} + // biome-ignore lint/a11y/noNoninteractiveTabindex: keyboard-scrollable region + tabIndex={0} + aria-label={`Release notes for version ${version}`} + // Long notes scroll here instead of pushing the buttons off screen. + className="hover-scrollbar max-h-64 min-h-0 flex-1 overflow-y-auto overscroll-contain py-3 pr-1" + data-testid="update-release-notes-scroll" + > + <MarkdownPreview + markdown={markdown} + // Streamdown ships headings at mt-6 and code at text-sm, and + // clears max-width on descendants, so rescale and re-cap both. + className="max-h-none overflow-visible border-0 bg-transparent p-0 text-ui-11 [&_[data-streamdown=link-safety-modal]>*]:max-w-md [&_img]:h-auto [&_img]:max-w-full [&>*:first-child]:mt-0 [&>*>*:first-child]:mt-0 [&_code]:text-[0.92em] [&_h1]:mt-4 [&_h1]:font-heading [&_h1]:text-ui-13 [&_h2]:mt-4 [&_h2]:font-heading [&_h2]:text-ui-13 [&_h3]:mt-4 [&_h3]:font-heading [&_h3]:text-ui-11 [&_pre]:text-[0.92em]" + /> + {notes?.truncated ? ( + <p className="mt-2 text-ui-10 text-muted-foreground/80"> + Notes truncated. See the full changelog. + </p> + ) : null} + </section> + ) : ( + <ReleaseNotesSummary preview={preview} /> + ) + ) : ( + <NotesStatus + state={state} + version={version} + link={link} + retry={retry} + /> + )} + </div> + {open && markdown && link ? ( + <div className="mt-2 flex justify-end px-1">{link}</div> + ) : null} + </div> + ); +} + +/** Collapsed view: the first few bullets, one line each where possible. */ +function ReleaseNotesSummary({ + preview, +}: { + preview: ReturnType<typeof releaseNotesPreview> | null; +}): ReactElement | null { + if (preview === null || preview.items.length === 0) { + return null; + } + const { items, remaining } = preview; + + return ( + <ul + className="space-y-1 py-2 pr-1" + data-testid="update-release-notes-summary" + > + {items.map((item, index) => ( + <li + // Two releases can carry the same bullet text, so index is the key. + key={`${index}-${item.lead}`} + className="flex gap-1.5 text-ui-11 leading-snug text-muted-foreground" + > + <span aria-hidden="true" className="text-muted-foreground/60"> + • + </span> + <span className="line-clamp-2 min-w-0"> + {/* lead sentence carries the change */} + <span className="font-medium text-foreground">{item.lead}</span> + {item.rest ? <span> {item.rest}</span> : null} + </span> + </li> + ))} + {remaining > 0 ? ( + <li className="pl-3 text-ui-10 text-muted-foreground/70"> + +{remaining} more + </li> + ) : null} + </ul> + ); +} + +function NotesStatus({ + state, + version, + link, + retry, +}: { + state: ReturnType<typeof useReleaseNotes>["state"]; + version: string; + link: ReactNode; + retry: () => void; +}): ReactElement { + if (state === "loading" || state === "idle") { + return <NotesMessage>Loading release notes...</NotesMessage>; + } + + if (state === "error") { + return ( + <NotesMessage + action={ + // The changelog page may be reachable when the lookup is not. + <span className="flex shrink-0 items-center gap-3"> + <button + type="button" + onClick={retry} + className={NOTES_LINK_CLASS} + data-testid="update-release-notes-retry" + > + Retry + </button> + {link} + </span> + } + > + Could not load release notes. + </NotesMessage> + ); + } + + // Matched nothing: link out rather than show another release's notes. + return ( + <NotesMessage action={link}> + No release notes published for {version} yet. + </NotesMessage> + ); +} diff --git a/studio/frontend/src/components/web/update-banner.tsx b/studio/frontend/src/components/web/update-banner.tsx index d8ae92bf5f..f36f5ec3cd 100644 --- a/studio/frontend/src/components/web/update-banner.tsx +++ b/studio/frontend/src/components/web/update-banner.tsx @@ -2,6 +2,7 @@ // Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 import { Button } from "@/components/ui/button"; +import { ReleaseNotesPanel } from "@/components/update/release-notes-panel"; import { type DeviceType, usePlatformStore } from "@/config/env"; import { useWebUpdateCheck } from "@/hooks/use-web-update-check"; import { isTauri } from "@/lib/api-base"; @@ -40,6 +41,7 @@ export function WebUpdateBanner({ const deviceType = usePlatformStore((s) => s.deviceType); const installCmd = installCommandForDevice(deviceType); const [copiedVersion, setCopiedVersion] = useState<string | null>(null); + const [notesVersion, setNotesVersion] = useState<string | null>(null); const dismissTimerRef = useRef<ReturnType<typeof setTimeout> | null>(null); useEffect(() => { @@ -68,6 +70,8 @@ export function WebUpdateBanner({ } const copied = status != null && copiedVersion === status.latestVersion; + // Keyed by version so a new offer collapses the panel. + const notesOpen = status != null && notesVersion === status.latestVersion; return ( <AnimatePresence> @@ -78,13 +82,14 @@ export function WebUpdateBanner({ exit={{ opacity: 0, y: 8, scale: 0.97 }} transition={{ duration: 0.35, ease: EASE_OUT_QUART }} className={cn( + // Wider than the other overlays: notes preview plus three buttons. positioned - ? "fixed bottom-4 right-4 z-[9999] w-[calc(100vw-2rem)] max-w-[400px]" - : "pointer-events-auto w-full", + ? "fixed bottom-4 right-4 z-[9999] w-[calc(100vw-2rem)] max-w-[448px]" + : "pointer-events-auto flex min-h-0 w-[calc(100vw-2rem)] max-w-[448px] flex-col", )} data-testid="web-update-banner" > - <div className="relative overflow-hidden rounded-[24px] bg-white px-5 pb-4 pt-5 shadow-[0_2px_8px_-2px_rgba(0,0,0,0.16)] dark:bg-card dark:shadow-[0_8px_28px_-6px_rgba(0,0,0,0.28)]"> + <div className="relative flex max-h-[calc(100dvh_-_2rem)] flex-col overflow-hidden rounded-[24px] bg-white px-5 pb-4 pt-5 shadow-[0_2px_8px_-2px_rgba(0,0,0,0.16)] dark:bg-card dark:shadow-[0_8px_28px_-6px_rgba(0,0,0,0.28)]"> <button type="button" onClick={dismiss} @@ -127,22 +132,33 @@ export function WebUpdateBanner({ </div> </div> + <ReleaseNotesPanel + version={status.latestVersion} + open={notesOpen} + releaseNotesUrl={RELEASE_NOTES_URL} + className="min-h-0 flex-1" + /> + + {/* one row at one type size; wraps only on narrow viewports */} <div className="mt-4 flex flex-wrap items-center justify-between gap-y-2"> - <a - href={RELEASE_NOTES_URL} - target="_blank" - rel="noopener noreferrer" - className="-ml-2 whitespace-nowrap rounded-full px-2.5 py-2 text-ui-13 font-medium text-foreground transition-colors hover:bg-muted" - data-testid="web-update-release-notes-link" + <Button + size="sm" + variant="ghost" + className="-ml-2 h-auto whitespace-nowrap rounded-full px-2.5 py-2 text-ui-13 font-medium text-foreground" + onClick={() => + setNotesVersion(notesOpen ? null : status.latestVersion) + } + aria-expanded={notesOpen} + data-testid="web-update-release-notes-toggle" > - Release notes - </a> + {notesOpen ? "Hide release notes" : "Show release notes"} + </Button> {/* wrap + right-align so buttons stack instead of clipping on very narrow banners */} <div className="flex flex-wrap items-center justify-end gap-x-1 gap-y-2"> <Button size="sm" variant="ghost" - className="h-auto rounded-full px-3 py-2 text-ui-13 font-medium text-foreground" + className="h-auto whitespace-nowrap rounded-full px-2.5 py-2 text-ui-13 font-medium text-foreground" onClick={snooze} data-testid="web-update-snooze-button" > @@ -151,7 +167,7 @@ export function WebUpdateBanner({ <Button size="sm" // -mr optically aligns the filled pill's edge with the card padding - className="-mr-1 h-auto rounded-full px-3.5 py-2 text-ui-13" + className="-mr-1 h-auto whitespace-nowrap rounded-full px-3 py-2 text-ui-13" onClick={handleCopyCommand} data-testid="web-update-copy-button" > diff --git a/studio/frontend/src/features/chat/chat-settings-sheet.tsx b/studio/frontend/src/features/chat/chat-settings-sheet.tsx index 6070bd2e40..fd41e558f9 100644 --- a/studio/frontend/src/features/chat/chat-settings-sheet.tsx +++ b/studio/frontend/src/features/chat/chat-settings-sheet.tsx @@ -18,6 +18,7 @@ import { DropdownMenuTrigger, } from "@/components/ui/dropdown-menu"; import { InfoHint } from "@/components/ui/info-hint"; +import { PanelResizeHandle } from "@/components/ui/panel-resize-handle"; import { InputGroup, InputGroupAddon, @@ -44,7 +45,13 @@ import { Tooltip, TooltipContent } from "@/components/ui/tooltip"; import { NumericValueInput, snapToStep } from "@/features/model-picker"; import { RetrievalSettingsSection } from "@/features/rag"; import { useLlamaUpdateCheck } from "@/hooks/use-llama-update-check"; +import { + CHAT_SETTINGS_WIDTH_MIN, + clampChatSettingsWidth, + useChatSettingsWidth, +} from "@/hooks/use-chat-settings-width"; import { useIsMobile } from "@/hooks/use-mobile"; +import { useT } from "@/i18n"; import { ChevronDownStandardIcon } from "@/lib/chevron-icons"; import { toast } from "@/lib/toast"; import { cn } from "@/lib/utils"; @@ -52,7 +59,7 @@ import { Edit03Icon, LayoutAlignRightIcon } from "@hugeicons/core-free-icons"; import { HugeiconsIcon } from "@hugeicons/react"; import { Braces, ChevronDown, ExternalLink } from "lucide-react"; import { Tooltip as TooltipPrimitive } from "radix-ui"; -import { Fragment, type ReactNode } from "react"; +import { type CSSProperties, Fragment, type ReactNode } from "react"; import { useCallback, useEffect, useMemo, useRef, useState } from "react"; import { OpenAICodeExecSection } from "./components/openai-code-exec-section"; import { PermissionModeDropdown } from "./permission-mode-select"; @@ -363,6 +370,15 @@ export function ChatSettingsPanel({ onExternalProviderChange, externalProviderType = null, }: ChatSettingsPanelProps) { + const asideRef = useRef<HTMLElement>(null); + const t = useT(); + const { + width: settingsWidth, + max: settingsMax, + stored: settingsStored, + setWidth: setSettingsWidth, + resetWidth: resetSettingsWidth, + } = useChatSettingsWidth(); // Local models show every knob; providerCapabilities is only consulted when // isExternalModel. Unknown providers fall back to the OpenAI-compat shape via // getProviderCapabilities, so these flags never undercount support. @@ -461,6 +477,23 @@ export function ChatSettingsPanel({ // When the prompt overflows the inline box, clicking opens the popup editor. const systemPromptBoxRef = useRef<HTMLTextAreaElement>(null); const [systemPromptOverflows, setSystemPromptOverflows] = useState(false); + const promptObserverRef = useRef<ResizeObserver | null>(null); + const measurePromptRef = useRef<() => void>(() => {}); + // The section unmounts its textarea when collapsed, so observe through a + // callback ref: a stored observer would cling to the detached node and the + // remounted one would never be measured. + const attachPromptBox = useCallback((node: HTMLTextAreaElement | null) => { + systemPromptBoxRef.current = node; + promptObserverRef.current?.disconnect(); + promptObserverRef.current = null; + if (!node || typeof ResizeObserver === "undefined") return; + // Resizing rewraps the prompt, and a drag changes the width through a + // custom property without re-rendering, so watch the box itself. + const observer = new ResizeObserver(() => measurePromptRef.current()); + observer.observe(node); + promptObserverRef.current = observer; + measurePromptRef.current(); + }, []); const [activePresetBaseline, setActivePresetBaseline] = useState(params); const presets = useMemo(() => { return getOrderedPresets(customPresets); @@ -746,15 +779,20 @@ export function ChatSettingsPanel({ }, [open]); useEffect(() => { - const el = systemPromptBoxRef.current; - setSystemPromptOverflows( - currentSystemPrompt.length > 0 && - el != null && - el.clientHeight > 0 && - el.scrollHeight > el.clientHeight + 1, - ); + measurePromptRef.current = () => { + const el = systemPromptBoxRef.current; + setSystemPromptOverflows( + currentSystemPrompt.length > 0 && + el != null && + el.clientHeight > 0 && + el.scrollHeight > el.clientHeight + 1, + ); + }; + measurePromptRef.current(); }, [currentSystemPrompt, open]); + useEffect(() => () => promptObserverRef.current?.disconnect(), []); + const settingsScrollRef = useRef<HTMLDivElement>(null); const settingsContent = ( @@ -1124,7 +1162,7 @@ export function ChatSettingsPanel({ )} > <textarea - ref={systemPromptBoxRef} + ref={attachPromptBox} value={currentSystemPrompt} onChange={(e) => set("systemPrompt")(e.target.value)} onMouseDown={(e) => { @@ -1433,17 +1471,47 @@ export function ChatSettingsPanel({ return ( <aside + ref={asideRef} data-tour="chat-settings" + data-slot="chat-settings-panel" className={cn( - "relative z-50 shrink-0 overflow-hidden bg-panel-surface text-panel-surface-fg font-heading", - open ? "w-[17rem] border-l border-sidebar-border" : "w-0", + "relative z-50 shrink-0 bg-panel-surface text-panel-surface-fg font-heading", + open + ? "w-(--chat-settings-width) border-l border-sidebar-border" + : "w-0 overflow-hidden", )} - style={{ - height: "calc(100% - var(--studio-custom-titlebar-height, 0px))", - marginTop: "var(--studio-custom-titlebar-height, 0px)", - }} + style={ + { + "--chat-settings-width": `${settingsWidth}px`, + height: "calc(100% - var(--studio-custom-titlebar-height, 0px))", + marginTop: "var(--studio-custom-titlebar-height, 0px)", + } as CSSProperties + } > - <div className="h-full w-full">{settingsContent}</div> + {open ? ( + <PanelResizeHandle + edge="left" + open={open} + width={settingsWidth} + stored={settingsStored} + min={CHAT_SETTINGS_WIDTH_MIN} + max={settingsMax} + clamp={clampChatSettingsWidth} + setWidth={setSettingsWidth} + resetWidth={resetSettingsWidth} + onToggle={() => onOpenChange?.(!open)} + target={() => asideRef.current} + cssVar="--chat-settings-width" + measure={() => asideRef.current?.getBoundingClientRect().width ?? 0} + label={t("shell.aria.resizeRunSettings")} + toggleLabel={t("shell.aria.openRunSettings")} + collapseHint={t("shell.resize.collapse")} + expandHint={t("shell.resize.expand")} + dragHint={t("shell.resize.drag")} + dataSlot="chat-settings-resize-handle" + /> + ) : null} + <div className="h-full w-full overflow-hidden">{settingsContent}</div> </aside> ); } diff --git a/studio/frontend/src/features/chat/components/deep-research-composer-button.tsx b/studio/frontend/src/features/chat/components/deep-research-composer-button.tsx index 03a7d7cc5f..e4857d1603 100644 --- a/studio/frontend/src/features/chat/components/deep-research-composer-button.tsx +++ b/studio/frontend/src/features/chat/components/deep-research-composer-button.tsx @@ -14,7 +14,8 @@ import { } from "@/components/ui/dialog"; import { Input } from "@/components/ui/input"; import { cn } from "@/lib/utils"; -import { ChevronDownIcon, XIcon } from "lucide-react"; +import { ChevronDownStandardIcon } from "@/lib/chevron-icons"; +import { XIcon } from "lucide-react"; import { type KeyboardEvent, useState } from "react"; import { useChatRuntimeStore } from "../stores/chat-runtime-store"; import type { ResearchWebsitePolicy } from "../types/research"; @@ -24,7 +25,12 @@ function normalizeDomain(raw: string): string | null { if (!value || /[\\\s]/.test(value)) return null; try { const url = new URL(value.includes("://") ? value : `https://${value}`); - if (!/^https?:$/.test(url.protocol) || url.username || url.password || url.port) { + if ( + !/^https?:$/.test(url.protocol) || + url.username || + url.password || + url.port + ) { return null; } return url.hostname @@ -99,7 +105,9 @@ function DomainList({ type="button" className="text-muted-foreground transition-colors hover:text-foreground" aria-label={`Remove ${domain}`} - onClick={() => onChange(values.filter((value) => value !== domain))} + onClick={() => + onChange(values.filter((value) => value !== domain)) + } > <XIcon className="size-3" /> </button> @@ -129,7 +137,9 @@ export function DeepResearchComposerButton({ onConfigure: () => void; }) { const enabled = useChatRuntimeStore((state) => state.deepResearchEnabled); - const setEnabled = useChatRuntimeStore((state) => state.setDeepResearchEnabled); + const setEnabled = useChatRuntimeStore( + (state) => state.setDeepResearchEnabled, + ); if (!enabled) return null; @@ -158,9 +168,12 @@ export function DeepResearchComposerButton({ <XIcon className="composer-pill-x" /> </span> <span>Deep research</span> - <span className="composer-pill-caret flex items-center gap-0.5 text-primary/70"> - <ChevronDownIcon className="size-3" /> - </span> + {/* Same caret as the other composer pills, so the arrows match. */} + <HugeiconsIcon + icon={ChevronDownStandardIcon} + strokeWidth={1.5} + className="composer-pill-caret size-[15px] text-primary/70" + /> </button> ); } @@ -173,7 +186,9 @@ export function DeepResearchWebsiteAccessDialog({ onOpenChange: (open: boolean) => void; }) { const policy = useChatRuntimeStore((state) => state.researchWebsitePolicy); - const setPolicy = useChatRuntimeStore((state) => state.setResearchWebsitePolicy); + const setPolicy = useChatRuntimeStore( + (state) => state.setResearchWebsitePolicy, + ); return ( <Dialog open={open} onOpenChange={onOpenChange}> @@ -201,41 +216,40 @@ function DeepResearchWebsiteAccessContent({ return ( <DialogContent className="sm:max-w-lg"> - <DialogHeader> - <DialogTitle>Website access</DialogTitle> - <DialogDescription> - Control which websites the next Deep Research run can search and - read. Limits are enforced by the server and shared with the research - model. - </DialogDescription> - </DialogHeader> - <div className="space-y-6"> - <DomainList - label="Allow only" - description="When set, research can access only these domains and their subdomains." - values={draft.allowedDomains} - onChange={(allowedDomains) => setDraft({ ...draft, allowedDomains })} - /> - <DomainList - label="Always block" - description="These domains and their subdomains stay blocked. Blocking takes precedence." - values={draft.blockedDomains} - onChange={(blockedDomains) => setDraft({ ...draft, blockedDomains })} - /> - </div> - <DialogFooter> - <Button variant="ghost" onClick={onClose}> - Cancel - </Button> - <Button - onClick={() => { - setPolicy(draft); - onClose(); - }} - > - Save limits - </Button> - </DialogFooter> + <DialogHeader> + <DialogTitle>Website access</DialogTitle> + <DialogDescription> + Control which websites the next Deep Research run can search and read. + Limits are enforced by the server and shared with the research model. + </DialogDescription> + </DialogHeader> + <div className="space-y-6"> + <DomainList + label="Allow only" + description="When set, research can access only these domains and their subdomains." + values={draft.allowedDomains} + onChange={(allowedDomains) => setDraft({ ...draft, allowedDomains })} + /> + <DomainList + label="Always block" + description="These domains and their subdomains stay blocked. Blocking takes precedence." + values={draft.blockedDomains} + onChange={(blockedDomains) => setDraft({ ...draft, blockedDomains })} + /> + </div> + <DialogFooter> + <Button variant="ghost" onClick={onClose}> + Cancel + </Button> + <Button + onClick={() => { + setPolicy(draft); + onClose(); + }} + > + Save limits + </Button> + </DialogFooter> </DialogContent> ); } diff --git a/studio/frontend/src/features/hub/catalog/on-device-folders-dialog.tsx b/studio/frontend/src/features/hub/catalog/on-device-folders-dialog.tsx index 2b0f2c3a8d..bef2f79747 100644 --- a/studio/frontend/src/features/hub/catalog/on-device-folders-dialog.tsx +++ b/studio/frontend/src/features/hub/catalog/on-device-folders-dialog.tsx @@ -23,12 +23,21 @@ import { removeScanFolder, } from "@/features/hub"; import { FolderBrowser } from "@/features/model-picker"; -import { openModelsDir } from "@/features/native-intents"; +import { + openModelsDir, + pickHuggingFaceCacheDir, +} from "@/features/native-intents"; +import { + type HuggingFaceCacheSettings, + loadHuggingFaceCacheSettings, + updateHuggingFaceCacheSettings, +} from "@/features/settings"; import { isTauri } from "@/lib/api-base"; import { toast } from "@/lib/toast"; import { cn } from "@/lib/utils"; import { Delete02Icon, + DownloadCircle01Icon, FileSearchIcon, FolderAddIcon, FolderExportIcon, @@ -49,6 +58,12 @@ function formatError(error: unknown): string { return error instanceof Error ? error.message : String(error); } +function formatFreeSpace(bytes: number | null): string | null { + if (bytes === null || !Number.isFinite(bytes)) return null; + const gb = bytes / 1024 ** 3; + return gb >= 10 ? `${Math.round(gb)} GB free` : `${gb.toFixed(1)} GB free`; +} + export function OnDeviceFoldersDialog({ open, onOpenChange, @@ -68,6 +83,11 @@ export function OnDeviceFoldersDialog({ ); const refreshIdRef = useRef(0); const mutationVersionRef = useRef(0); + const [downloadCache, setDownloadCache] = + useState<HuggingFaceCacheSettings | null>(null); + const [downloadCacheLoaded, setDownloadCacheLoaded] = useState(false); + const [downloadBrowserOpen, setDownloadBrowserOpen] = useState(false); + const [downloadSaving, setDownloadSaving] = useState(false); const sortedFolders = useMemo( () => [...folders].sort((a, b) => a.path.localeCompare(b.path)), @@ -108,10 +128,66 @@ export function OnDeviceFoldersDialog({ return () => window.clearTimeout(timer); }, [open, refreshFolders]); + useEffect(() => { + if (!open) return; + let cancelled = false; + // The dialog stays mounted between opens, so re-arm the flag or a reopen + // shows the previous answer as if it were fresh. + setDownloadCacheLoaded(false); + loadHuggingFaceCacheSettings() + // Indexed locations do not depend on this. Null drops the stale path + // rather than offer Change against a location we could not confirm. + .catch(() => null) + .then((settings) => { + if (cancelled) return; + setDownloadCache(settings); + setDownloadCacheLoaded(true); + }); + return () => { + cancelled = true; + }; + }, [open]); + const handleInventoryChanged = useCallback(() => { onInventoryChange?.(); }, [onInventoryChange]); + // Relocating the cache changes which repos are on disk, but + // updateHuggingFaceCacheSettings already bumps the inventory version, which + // re-fetches every source. Refreshing here too would scan twice, since the + // two rounds carry different version keys and cannot be deduplicated. + const saveDownloadLocation = useCallback(async (nextPath: string | null) => { + setDownloadSaving(true); + try { + const settings = await updateHuggingFaceCacheSettings(nextPath); + setDownloadCache(settings); + toast.success("Download location updated", { + description: settings.cacheHome, + }); + } catch (err) { + toast.error("Couldn't update the download location", { + description: formatError(err), + }); + } finally { + setDownloadSaving(false); + } + }, []); + + const changeDownloadLocation = useCallback(async () => { + if (!isTauri) { + setDownloadBrowserOpen(true); + return; + } + try { + const picked = await pickHuggingFaceCacheDir(); + if (picked) await saveDownloadLocation(picked); + } catch (err) { + toast.error("Couldn't open the folder picker", { + description: formatError(err), + }); + } + }, [saveDownloadLocation]); + const handleAdd = useCallback( async (rawPath: string) => { const nextPath = rawPath.trim(); @@ -182,10 +258,10 @@ export function OnDeviceFoldersDialog({ <> <Dialog open={open} onOpenChange={onOpenChange}> <DialogContent - className="gap-0 overflow-hidden p-0 sm:max-w-[620px] lg:max-w-[660px] xl:max-w-[680px] [&_[data-slot=dialog-close]]:right-3 [&_[data-slot=dialog-close]]:top-3" + className="flex max-h-[90dvh] flex-col gap-0 overflow-hidden p-0 sm:max-w-[620px] lg:max-w-[660px] xl:max-w-[680px] [&_[data-slot=dialog-close]]:right-3 [&_[data-slot=dialog-close]]:top-3" overlayClassName="bg-black/20 backdrop-blur-none" > - <DialogHeader className="border-b border-border/60 px-5 py-4"> + <DialogHeader className="shrink-0 border-b border-border/60 px-5 py-4"> <DialogTitle className="text-ui-15"> On-device locations </DialogTitle> @@ -195,7 +271,78 @@ export function OnDeviceFoldersDialog({ </DialogDescription> </DialogHeader> - <div className="space-y-4 px-5 py-4"> + <div className="min-h-0 flex-1 space-y-4 overflow-y-auto px-5 py-4"> + <div className="rounded-[14px] border border-border/70 bg-muted/20 p-3"> + <div className="mb-2 flex items-center gap-2 text-ui-12 font-medium text-foreground"> + <HugeiconsIcon + icon={DownloadCircle01Icon} + strokeWidth={1.75} + className="size-3.5 text-muted-foreground" + /> + Download location + </div> + + <div className="flex flex-col gap-2 sm:flex-row sm:items-center"> + <Input + readOnly={true} + aria-label="Model download location" + value={ + downloadCache?.cacheHome ?? + (downloadCacheLoaded ? "Unknown" : "Loading...") + } + title={downloadCache?.cacheHome} + className="field-soft h-9 min-w-0 flex-1 rounded-full px-3 font-mono text-ui-12" + /> + <div className="flex shrink-0 items-center gap-2"> + <Button + type="button" + variant="outline" + size="sm" + onClick={() => void changeDownloadLocation()} + disabled={!downloadCache?.editable || downloadSaving} + className="h-9 rounded-full px-3 text-ui-12p5" + > + {downloadSaving ? ( + <Spinner className="size-3.5" /> + ) : ( + <HugeiconsIcon + icon={FolderSearchIcon} + strokeWidth={1.75} + data-icon="inline-start" + className="size-3.5" + /> + )} + Change + </Button> + {downloadCache?.isCustom ? ( + <Button + type="button" + variant="ghost" + size="sm" + onClick={() => void saveDownloadLocation(null)} + disabled={downloadSaving} + className="h-9 rounded-full px-3 text-ui-12p5 text-muted-foreground" + > + Use default + </Button> + ) : null} + </div> + </div> + + <p className="mt-2 text-ui-10p5 text-muted-foreground"> + {downloadCache?.source === "environment" + ? `Managed by the ${ + downloadCache.environmentVariable ?? "HF_HOME" + } environment variable.` + : [ + "New downloads only. Models already on disk stay where they are.", + formatFreeSpace(downloadCache?.freeBytes ?? null), + ] + .filter(Boolean) + .join(" · ")} + </p> + </div> + <div className="rounded-[14px] border border-border/70 bg-muted/20 p-3"> <div className="mb-2 flex items-center gap-2 text-ui-12 font-medium text-foreground"> <HugeiconsIcon @@ -425,6 +572,16 @@ export function OnDeviceFoldersDialog({ onOpenChange={setBrowserOpen} onSelect={(selectedPath) => void handleAdd(selectedPath)} /> + + <FolderBrowser + open={!isTauri && downloadBrowserOpen} + onOpenChange={setDownloadBrowserOpen} + onSelect={(selectedPath) => void saveDownloadLocation(selectedPath)} + initialPath={downloadCache?.cacheHome} + title="Choose model download location" + confirmLabel="Use for future downloads" + showModelHints={false} + /> </> ); } diff --git a/studio/frontend/src/features/hub/download-manager/download-manager-panel.tsx b/studio/frontend/src/features/hub/download-manager/download-manager-panel.tsx index 3e5a86a879..d68aaa5ab3 100644 --- a/studio/frontend/src/features/hub/download-manager/download-manager-panel.tsx +++ b/studio/frontend/src/features/hub/download-manager/download-manager-panel.tsx @@ -201,8 +201,10 @@ export function DownloadManagerPanel({ className={cn( // Standalone: anchor bottom-right. In a shared stack (positioned=false) // flow as a right-aligned row so overlays stack instead of overlapping. + // min-h-0 there: a flex item's min-height defaults to auto, so the capped + // stack would squeeze the update card instead of this list. "pointer-events-none", - positioned ? "fixed bottom-4 right-4 z-50" : "flex justify-end", + positioned ? "fixed bottom-4 right-4 z-50" : "flex min-h-0 justify-end", )} > {collapsed ? ( @@ -229,7 +231,7 @@ export function DownloadManagerPanel({ </TooltipContent> </Tooltip> ) : ( - <div className="hub-download-panel pointer-events-auto w-[min(400px,calc(100vw-2rem))] overflow-hidden"> + <div className="hub-download-panel pointer-events-auto flex min-h-0 w-[min(400px,calc(100vw-2rem))] flex-col overflow-hidden"> <div className="flex items-center gap-2 border-b border-foreground/[0.07] py-2 pl-4 pr-3"> <span className="min-w-0 flex-1 truncate text-ui-12p5 font-semibold text-foreground"> {headerLabel} diff --git a/studio/frontend/src/features/settings/index.ts b/studio/frontend/src/features/settings/index.ts index f27100a322..49d73dcfb3 100644 --- a/studio/frontend/src/features/settings/index.ts +++ b/studio/frontend/src/features/settings/index.ts @@ -3,6 +3,11 @@ export { SettingsDialog } from "./settings-dialog"; export { loadEmbeddingModelSettings } from "./api/embedding-model"; +export { + loadHuggingFaceCacheSettings, + updateHuggingFaceCacheSettings, +} from "./api/hugging-face-cache"; +export type { HuggingFaceCacheSettings } from "./api/hugging-face-cache"; export { loadPersonalization, savePersonalization, diff --git a/studio/frontend/src/features/settings/settings-dialog.tsx b/studio/frontend/src/features/settings/settings-dialog.tsx index f4ba98b1ce..2b46dbd55e 100644 --- a/studio/frontend/src/features/settings/settings-dialog.tsx +++ b/studio/frontend/src/features/settings/settings-dialog.tsx @@ -35,7 +35,10 @@ import { useRef, useState, } from "react"; -import { SETTINGS_SEARCH_INDEX } from "./settings-search"; +import { + SETTINGS_SEARCH_INDEX, + SETTINGS_SEARCH_KEYWORDS, +} from "./settings-search"; import { type SettingsTab, useSettingsDialogStore, @@ -157,8 +160,12 @@ export function SettingsDialog() { return TABS.map((tab) => { const tabLabel = t(tab.labelKey); const entries = SETTINGS_SEARCH_INDEX[tab.id] - .map((key) => t(key)) - .filter((label) => label.toLowerCase().includes(q)); + .filter((key) => { + if (t(key).toLowerCase().includes(q)) return true; + const keywordsKey = SETTINGS_SEARCH_KEYWORDS[key]; + return keywordsKey ? t(keywordsKey).toLowerCase().includes(q) : false; + }) + .map((key) => t(key)); const deduped = [...new Set(entries)]; return { tab, diff --git a/studio/frontend/src/features/settings/settings-search.ts b/studio/frontend/src/features/settings/settings-search.ts index a5b008579c..582602e061 100644 --- a/studio/frontend/src/features/settings/settings-search.ts +++ b/studio/frontend/src/features/settings/settings-search.ts @@ -146,3 +146,15 @@ export const SETTINGS_SEARCH_INDEX: Record<SettingsTab, TranslationKey[]> = { "settings.about.shutDownStudio", ], }; + +/** + * Extra terms a row matches on, beyond its own label. The value is a + * translation key holding space-separated synonyms; it is never rendered. + * Search matched labels only, so "models folder" or "directory" found nothing. + */ +export const SETTINGS_SEARCH_KEYWORDS: Partial< + Record<TranslationKey, TranslationKey> +> = { + "settings.resources.storage.modelsFolder": + "settings.resources.storage.modelsFolderKeywords", +}; diff --git a/studio/frontend/src/features/settings/tabs/general-tab.tsx b/studio/frontend/src/features/settings/tabs/general-tab.tsx index 1d5c533a25..11606d85af 100644 --- a/studio/frontend/src/features/settings/tabs/general-tab.tsx +++ b/studio/frontend/src/features/settings/tabs/general-tab.tsx @@ -74,6 +74,8 @@ const PREFS_KEYS: string[] = [ LOCALE_STORAGE_KEY, // UI state "sidebar_pinned", + "sidebar_width", + "chat_settings_width", "unsloth_sidebar_navigate_open", "unsloth_settings_active_tab", // Chat runtime prefs diff --git a/studio/frontend/src/hooks/use-chat-settings-width.ts b/studio/frontend/src/hooks/use-chat-settings-width.ts new file mode 100644 index 0000000000..6b2c75fe1e --- /dev/null +++ b/studio/frontend/src/hooks/use-chat-settings-width.ts @@ -0,0 +1,20 @@ +// SPDX-License-Identifier: AGPL-3.0-only +// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 + +import { createPanelWidthStore } from "./use-panel-width.ts"; + +/** The previous fixed 17rem, at a 16px root font size. */ +export const CHAT_SETTINGS_WIDTH_DEFAULT = 272; +/** Below this the sliders and their value pills start colliding. */ +export const CHAT_SETTINGS_WIDTH_MIN = 248; +export const CHAT_SETTINGS_WIDTH_MAX = 560; + +const store = createPanelWidthStore({ + key: "chat_settings_width", + min: CHAT_SETTINGS_WIDTH_MIN, + max: CHAT_SETTINGS_WIDTH_MAX, + fallback: CHAT_SETTINGS_WIDTH_DEFAULT, +}); + +export const clampChatSettingsWidth = store.clamp; +export const useChatSettingsWidth = store.useWidth; diff --git a/studio/frontend/src/hooks/use-panel-width.ts b/studio/frontend/src/hooks/use-panel-width.ts new file mode 100644 index 0000000000..d753c3a1bb --- /dev/null +++ b/studio/frontend/src/hooks/use-panel-width.ts @@ -0,0 +1,138 @@ +// SPDX-License-Identifier: AGPL-3.0-only +// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 + +import { useCallback, useSyncExternalStore } from "react"; + +/** Never let one panel eat more than this share of a narrow window. */ +const MAX_VIEWPORT_FRACTION = 0.4; + +export type PanelWidthStore = { + /** Clamps to what the current viewport allows. */ + clamp: (px: number) => number; + useWidth: () => { + width: number; + max: number; + /** The uncapped stored preference. */ + stored: number; + setWidth: (value: number) => void; + resetWidth: () => void; + }; +}; + +/** + * A persisted, viewport-aware width for a draggable panel. The preference is + * stored whole and an effective width is derived from it, so narrowing the + * window shrinks the panel without losing what the user picked. + */ +export function createPanelWidthStore({ + key, + min, + max, + fallback, +}: { + key: string; + min: number; + max: number; + fallback: number; +}): PanelWidthStore { + function maxWidth(): number { + if (typeof window === "undefined") return max; + // The floor wins on a narrow window; collapsing is the escape. + return Math.max(min, Math.min(max, window.innerWidth * MAX_VIEWPORT_FRACTION)); + } + + /** Clamps to the absolute range, ignoring the viewport. */ + function clampStored(px: number): number { + if (!Number.isFinite(px)) return fallback; + return Math.min(max, Math.max(min, Math.round(px))); + } + + function clamp(px: number): number { + return Math.min(maxWidth(), clampStored(px)); + } + + function load(): number { + if (typeof window === "undefined") return fallback; + try { + const raw = window.localStorage.getItem(key); + if (raw === null) return fallback; + return clampStored(Number.parseFloat(raw)); + } catch { + return fallback; + } + } + + let storedWidth = load(); + let effectiveWidth = clamp(storedWidth); + let effectiveMax = maxWidth(); + const listeners = new Set<() => void>(); + + let lastStored = storedWidth; + + function recompute() { + const nextWidth = clamp(storedWidth); + const nextMax = maxWidth(); + if ( + nextWidth === effectiveWidth && + nextMax === effectiveMax && + storedWidth === lastStored + ) { + return; + } + effectiveWidth = nextWidth; + effectiveMax = nextMax; + lastStored = storedWidth; + listeners.forEach((cb) => cb()); + } + + function subscribe(cb: () => void) { + // With no subscribers there is no resize listener, so the cache can be + // stale after a resize on a route that hides every panel. Refresh first; + // useSyncExternalStore re-reads the snapshot right after subscribing. + recompute(); + listeners.add(cb); + if (typeof window === "undefined") { + return () => listeners.delete(cb); + } + // Keep tabs in sync, same as the pin flag. + const onStorage = (e: StorageEvent) => { + if (e.key === key || e.key === null) { + storedWidth = load(); + effectiveWidth = clamp(storedWidth); + effectiveMax = maxWidth(); + cb(); + } + }; + window.addEventListener("storage", onStorage); + window.addEventListener("resize", recompute); + return () => { + listeners.delete(cb); + window.removeEventListener("storage", onStorage); + window.removeEventListener("resize", recompute); + }; + } + + function setWidthGlobal(next: number) { + const stored = clampStored(next); + if (stored !== storedWidth) { + storedWidth = stored; + try { + window.localStorage.setItem(key, String(stored)); + } catch {} + } + recompute(); + } + + function useWidth() { + const width = useSyncExternalStore(subscribe, () => effectiveWidth, () => fallback); + // What the viewport actually allows right now, for aria-valuemax. + const panelMax = useSyncExternalStore(subscribe, () => effectiveMax, () => max); + // The uncapped preference, so a capped drag can avoid lowering it. + const preference = useSyncExternalStore(subscribe, () => storedWidth, () => fallback); + const setWidth = useCallback((value: number) => setWidthGlobal(value), []); + const resetWidth = useCallback(() => setWidthGlobal(fallback), []); + return { width, max: panelMax, stored: preference, setWidth, resetWidth }; + } + + return { clamp, useWidth }; +} diff --git a/studio/frontend/src/hooks/use-release-notes.ts b/studio/frontend/src/hooks/use-release-notes.ts new file mode 100644 index 0000000000..7b1392fdf6 --- /dev/null +++ b/studio/frontend/src/hooks/use-release-notes.ts @@ -0,0 +1,146 @@ +// SPDX-License-Identifier: AGPL-3.0-only +// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 + +import { authFetch, hasAuthToken } from "@/features/auth"; +import { apiUrl } from "@/lib/api-base"; +import { useCallback, useEffect, useRef, useState } from "react"; + +// Keyed to one exact version, so a new update never pairs with older notes. +export interface ReleaseNotes { + version: string; + markdown: string | null; + matched: boolean; + truncated: boolean; + source: string | null; + releaseNotesUrl: string | null; + // Set when the lookup itself failed, as opposed to a version with no notes. + error: string | null; +} + +export type ReleaseNotesState = "idle" | "loading" | "ready" | "error"; + +// Desktop auto-auth installs its token after first paint, so a startup popup can +// ask before one exists. Wait briefly rather than fail. +const AUTH_POLL_MS = 250; +const AUTH_POLL_LIMIT = 40; + +interface UseReleaseNotesOptions { + version: string | null | undefined; + enabled?: boolean; +} + +type ApiObject = Record<string, unknown>; + +function stringOrNull(value: ApiObject, key: string): string | null { + const field = value[key]; + return typeof field === "string" && field.length > 0 ? field : null; +} + +function toReleaseNotes(value: unknown, version: string): ReleaseNotes | null { + if (!value || typeof value !== "object") { + return null; + } + const payload = value as ApiObject; + const notesVersion = stringOrNull(payload, "version"); + // A response for another version is not usable here. + if (notesVersion !== version) { + return null; + } + const markdown = stringOrNull(payload, "markdown"); + return { + version, + markdown, + matched: payload.matched === true && markdown !== null, + truncated: payload.truncated === true, + source: stringOrNull(payload, "source"), + releaseNotesUrl: stringOrNull(payload, "release_notes_url"), + error: stringOrNull(payload, "error"), + }; +} + +async function fetchReleaseNotes( + version: string, + refresh = false, +): Promise<ReleaseNotes | null> { + const query = `version=${encodeURIComponent(version)}${refresh ? "&refresh=true" : ""}`; + // authFetch, not fetch: an expired token is refreshed and retried. + const res = await authFetch(apiUrl(`/api/studio/release-notes?${query}`)); + if (!res.ok) { + throw new Error(`Release notes request failed: ${res.status}`); + } + + return toReleaseNotes(await res.json(), version); +} + +export function useReleaseNotes({ + version, + enabled = true, +}: UseReleaseNotesOptions) { + const [state, setState] = useState<ReleaseNotesState>("idle"); + const [notes, setNotes] = useState<ReleaseNotes | null>(null); + // Version the current state belongs to; a change invalidates it. + const requestedVersionRef = useRef<string | null>(null); + // Identifies one request, so an earlier response cannot overwrite a later one. + const requestIdRef = useRef(0); + + const load = useCallback((target: string, refresh = false) => { + requestedVersionRef.current = target; + requestIdRef.current += 1; + const requestId = requestIdRef.current; + setState("loading"); + setNotes(null); + fetchReleaseNotes(target, refresh) + .then((next) => { + // A newer request owns the state now. + if (requestIdRef.current !== requestId) { + return; + } + setNotes(next); + // A reported failure is retryable; "no notes for this version" is not. + const failed = !next || (!next.matched && next.error !== null); + setState(failed ? "error" : "ready"); + }) + .catch(() => { + if (requestIdRef.current === requestId) { + setNotes(null); + setState("error"); + } + }); + }, []); + + useEffect(() => { + if (!enabled || !version || requestedVersionRef.current === version) { + return; + } + if (hasAuthToken()) { + load(version); + return; + } + let attempts = 0; + const timer = window.setInterval(() => { + attempts += 1; + if (hasAuthToken() || attempts >= AUTH_POLL_LIMIT) { + window.clearInterval(timer); + // Out of patience: load anyway so the panel settles on retry. + load(version); + } + }, AUTH_POLL_MS); + return () => window.clearInterval(timer); + }, [enabled, version, load]); + + const retry = useCallback(() => { + if (version) { + requestedVersionRef.current = null; + // Bypass the cached remote failure, or retry waits for it to expire. + load(version, true); + } + }, [version, load]); + + // Never hand back another version's notes: state lags `version` by a render. + const matchesVersion = notes !== null && notes.version === version; + return { + state: notes !== null && !matchesVersion ? "loading" : state, + notes: matchesVersion ? notes : null, + retry, + }; +} diff --git a/studio/frontend/src/hooks/use-sidebar-width.ts b/studio/frontend/src/hooks/use-sidebar-width.ts new file mode 100644 index 0000000000..c670e254a8 --- /dev/null +++ b/studio/frontend/src/hooks/use-sidebar-width.ts @@ -0,0 +1,21 @@ +// SPDX-License-Identifier: AGPL-3.0-only +// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 + +import { createPanelWidthStore } from "./use-panel-width.ts"; + +/** The previous fixed 17.5rem, at a 16px root font size. */ +export const SIDEBAR_WIDTH_DEFAULT = 280; +/** Narrowest width that still fits the wordmark. Firefox is the constraint: + * it renders the heading ~3px wider than Chromium and WebKit. */ +export const SIDEBAR_WIDTH_MIN = 260; +export const SIDEBAR_WIDTH_MAX = 480; + +const store = createPanelWidthStore({ + key: "sidebar_width", + min: SIDEBAR_WIDTH_MIN, + max: SIDEBAR_WIDTH_MAX, + fallback: SIDEBAR_WIDTH_DEFAULT, +}); + +export const clampSidebarWidth = store.clamp; +export const useSidebarWidth = store.useWidth; diff --git a/studio/frontend/src/hooks/use-tauri-update.ts b/studio/frontend/src/hooks/use-tauri-update.ts index 8ebb4d2980..196e3cea2b 100644 --- a/studio/frontend/src/hooks/use-tauri-update.ts +++ b/studio/frontend/src/hooks/use-tauri-update.ts @@ -21,6 +21,8 @@ export type UpdateStatus = export interface UpdateInfo { version: string; currentVersion: string; + // Backend release this build pins; CHANGELOG.md is keyed by it, not the SemVer. + pypiVersion?: string; body?: string; date?: string; } @@ -42,10 +44,17 @@ interface DesktopUpdatePolicy { interface ManualUpdateInfo { version: string; currentVersion: string; + pypiVersion?: string | null; body?: string; date?: string; } +/** `pypi_version` from latest.json, which the updater passes through raw. */ +function rawPypiVersion(raw: Record<string, unknown>): string | undefined { + const value = raw.pypi_version; + return typeof value === "string" && value.length > 0 ? value : undefined; +} + export interface RetainedUpdateFailure { error: string; phase: UpdatePhase; @@ -162,6 +171,7 @@ export function useTauriUpdate(isExternalServer = false) { setInfo({ version: manualUpdate.version, currentVersion: manualUpdate.currentVersion, + pypiVersion: manualUpdate.pypiVersion ?? undefined, body: manualUpdate.body, date: manualUpdate.date, }); @@ -197,6 +207,7 @@ export function useTauriUpdate(isExternalServer = false) { setInfo({ version: update.version, currentVersion: update.currentVersion, + pypiVersion: rawPypiVersion(update.rawJson), body: update.body, date: update.date, }); @@ -384,10 +395,13 @@ export function useTauriUpdate(isExternalServer = false) { }); } + // Install target for Linux packages that cannot self-update. const manualReleaseUrl = updatePolicy.mode === "manual_linux_package" && info ? manualReleasePageUrl(updatePolicy, info.version) : null; + // Release page for the offered version, on every platform, for the notes link. + const releasePageUrl = info ? manualReleasePageUrl(updatePolicy, info.version) : null; return { status, @@ -401,6 +415,7 @@ export function useTauriUpdate(isExternalServer = false) { isExternalServer, updatePolicyMode: updatePolicy.mode, manualReleaseUrl, + releasePageUrl, installUpdate, retryUpdate, skipAndRestart, diff --git a/studio/frontend/src/i18n/locales/ar.ts b/studio/frontend/src/i18n/locales/ar.ts index 47d5032fae..a0ea2a1ee2 100644 --- a/studio/frontend/src/i18n/locales/ar.ts +++ b/studio/frontend/src/i18n/locales/ar.ts @@ -27,10 +27,18 @@ export const ar = { product: "Unsloth Studio", accountMenu: "قائمة حساب {name}", updateAvailable: "يتوفر تحديث", + resize: { + collapse: "انقر للطي", + expand: "انقر للتوسيع", + drag: "اسحب لتغيير الحجم", + }, aria: { home: "الصفحة الرئيسية لـ Unsloth", closeSidebar: "إغلاق الشريط الجانبي", openSidebar: "فتح الشريط الجانبي", + resizeSidebar: "تغيير حجم الشريط الجانبي أو طيه", + resizeRunSettings: "تغيير حجم إعدادات التشغيل أو إغلاقها", + openRunSettings: "فتح إعدادات التشغيل", chatOptions: "خيارات المحادثة", runOptions: "خيارات التدريب", }, @@ -317,6 +325,8 @@ export const ar = { diskUsage: "{used} مستخدم / {total}", diskFree: "{free} متاح", modelsFolder: "مجلد النماذج", + modelsFolderKeywords: + "النماذج مجلد دليل مسار موقع تنزيلات التنزيل ذاكرة التخزين المؤقت تخزين قرص محرك نقل تغيير models folder path hugging face", modelsFolderDescription: "المكان الذي تُخزَّن فيه النماذج المُنزَّلة.", openAction: "فتح", copyAction: "نسخ المسار", diff --git a/studio/frontend/src/i18n/locales/de.ts b/studio/frontend/src/i18n/locales/de.ts index cb7d603f42..8a5921c623 100644 --- a/studio/frontend/src/i18n/locales/de.ts +++ b/studio/frontend/src/i18n/locales/de.ts @@ -27,10 +27,18 @@ export const de = { product: "Unsloth Studio", accountMenu: "Kontomenü von {name}", updateAvailable: "Update verfügbar", + resize: { + collapse: "Zum Einklappen klicken", + expand: "Zum Ausklappen klicken", + drag: "Zum Ändern der Größe ziehen", + }, aria: { home: "Unsloth Startseite", closeSidebar: "Seitenleiste schließen", openSidebar: "Seitenleiste öffnen", + resizeSidebar: "Seitenleiste anpassen oder einklappen", + resizeRunSettings: "Ausführungseinstellungen anpassen oder schließen", + openRunSettings: "Ausführungseinstellungen öffnen", chatOptions: "Chat-Optionen", runOptions: "Trainingslauf-Optionen", }, @@ -330,9 +338,23 @@ export const de = { diskFree: "{free} frei", modelsFolder: "Modell-Ordner", modelsFolderDescription: - "Wo heruntergeladene Modelle gespeichert werden.", + "Wo heruntergeladene Modelle gespeichert werden. Ändern Sie ihn, um Modelle nicht auf dem Systemlaufwerk abzulegen.", + modelsFolderKeywords: + "Modelle Ordner Verzeichnis Pfad Speicherort Download Downloads Cache Speicher Festplatte Laufwerk verschieben ändern hugging face", + futureDownloads: "Nur neue Downloads", + environmentManaged: + "Wird über die Umgebungsvariable {variable} verwaltet.", + locationFree: "{free} frei", openAction: "Öffnen", copyAction: "Pfad kopieren", + changeAction: "Ändern", + resetAction: "Standard verwenden", + chooseTitle: "Speicherort für Modell-Downloads wählen", + chooseAction: "Für künftige Downloads verwenden", + cacheSaved: "Speicherort für Modell-Downloads aktualisiert", + cacheSaveError: + "Der Speicherort für Modell-Downloads konnte nicht geändert werden", + cachePickerError: "Die Ordnerauswahl konnte nicht geöffnet werden", copied: "Pfad kopiert", openError: "Der Ordner konnte nicht geöffnet werden", copyError: "Der Pfad konnte nicht kopiert werden", diff --git a/studio/frontend/src/i18n/locales/en.ts b/studio/frontend/src/i18n/locales/en.ts index bdfcf38231..10e571f4da 100644 --- a/studio/frontend/src/i18n/locales/en.ts +++ b/studio/frontend/src/i18n/locales/en.ts @@ -24,10 +24,18 @@ export const en = { product: "Unsloth Studio", accountMenu: "{name} account menu", updateAvailable: "Update available", + resize: { + collapse: "Click to collapse", + expand: "Click to expand", + drag: "Drag to resize", + }, aria: { home: "Unsloth home", closeSidebar: "Close sidebar", openSidebar: "Open sidebar", + resizeSidebar: "Resize or collapse sidebar", + resizeRunSettings: "Resize or close run settings", + openRunSettings: "Open run settings", chatOptions: "Chat options", runOptions: "Run options", }, @@ -556,8 +564,12 @@ export const en = { systemDisk: "System disk", diskUsage: "{used} used / {total}", diskFree: "{free} free", - modelsFolder: "Model downloads", - modelsFolderDescription: "Hugging Face cache used for model downloads.", + modelsFolder: "Models folder", + modelsFolderDescription: + "Where downloaded models are stored. Change it to keep models off your system drive.", + // Not rendered: extra terms the settings search matches this row on. + modelsFolderKeywords: + "models folder directory path location download downloads cache storage disk drive move relocate hugging face", futureDownloads: "New downloads only", environmentManaged: "Managed by the {variable} environment variable.", locationFree: "{free} free", diff --git a/studio/frontend/src/i18n/locales/es.ts b/studio/frontend/src/i18n/locales/es.ts index f7cb0e11f6..6edc33f9de 100644 --- a/studio/frontend/src/i18n/locales/es.ts +++ b/studio/frontend/src/i18n/locales/es.ts @@ -27,10 +27,18 @@ export const es = { product: "Unsloth Studio", accountMenu: "Menú de cuenta de {name}", updateAvailable: "Actualización disponible", + resize: { + collapse: "Haz clic para contraer", + expand: "Haz clic para expandir", + drag: "Arrastra para redimensionar", + }, aria: { home: "Inicio de Unsloth", closeSidebar: "Cerrar barra lateral", openSidebar: "Abrir barra lateral", + resizeSidebar: "Redimensionar o contraer la barra lateral", + resizeRunSettings: "Redimensionar o cerrar los ajustes de ejecución", + openRunSettings: "Abrir los ajustes de ejecución", chatOptions: "Opciones de chat", runOptions: "Opciones de ejecución", }, @@ -328,6 +336,8 @@ export const es = { diskUsage: "{used} en uso / {total}", diskFree: "{free} libre", modelsFolder: "Carpeta de modelos", + modelsFolderKeywords: + "modelos carpeta directorio ruta ubicacion ubicación descargas descarga cache caché almacenamiento disco unidad mover cambiar models folder path hugging face", modelsFolderDescription: "Dónde se almacenan los modelos descargados.", openAction: "Abrir", diff --git a/studio/frontend/src/i18n/locales/fr.ts b/studio/frontend/src/i18n/locales/fr.ts index 4f2838391f..789e89079c 100644 --- a/studio/frontend/src/i18n/locales/fr.ts +++ b/studio/frontend/src/i18n/locales/fr.ts @@ -27,10 +27,18 @@ export const fr = { product: "Unsloth Studio", accountMenu: "Menu du compte de {name}", updateAvailable: "Mise à jour disponible", + resize: { + collapse: "Cliquez pour réduire", + expand: "Cliquez pour développer", + drag: "Faites glisser pour redimensionner", + }, aria: { home: "Accueil Unsloth", closeSidebar: "Fermer la barre latérale", openSidebar: "Ouvrir la barre latérale", + resizeSidebar: "Redimensionner ou réduire la barre latérale", + resizeRunSettings: "Redimensionner ou fermer les paramètres d'exécution", + openRunSettings: "Ouvrir les paramètres d'exécution", chatOptions: "Options de discussion", runOptions: "Options d'exécution", }, @@ -325,6 +333,8 @@ export const fr = { diskUsage: "{used} utilisé / {total}", diskFree: "{free} libre", modelsFolder: "Dossier des modèles", + modelsFolderKeywords: + "modeles modèles dossier repertoire répertoire chemin emplacement telechargements téléchargements cache stockage disque lecteur deplacer déplacer changer models folder path hugging face", modelsFolderDescription: "Emplacement de stockage des modèles téléchargés.", openAction: "Ouvrir", copyAction: "Copier le chemin", diff --git a/studio/frontend/src/i18n/locales/hi.ts b/studio/frontend/src/i18n/locales/hi.ts index 33b827f314..410318983b 100644 --- a/studio/frontend/src/i18n/locales/hi.ts +++ b/studio/frontend/src/i18n/locales/hi.ts @@ -27,10 +27,18 @@ export const hi = { product: "Unsloth Studio", accountMenu: "{name} खाता मेनू", updateAvailable: "अपडेट उपलब्ध है", + resize: { + collapse: "छोटा करने के लिए क्लिक करें", + expand: "विस्तार के लिए क्लिक करें", + drag: "आकार बदलने के लिए खींचें", + }, aria: { home: "Unsloth होम", closeSidebar: "साइडबार बंद करें", openSidebar: "साइडबार खोलें", + resizeSidebar: "साइडबार का आकार बदलें या छोटा करें", + resizeRunSettings: "रन सेटिंग्स का आकार बदलें या बंद करें", + openRunSettings: "रन सेटिंग्स खोलें", chatOptions: "चैट विकल्प", runOptions: "रन विकल्प", }, @@ -316,6 +324,8 @@ export const hi = { diskUsage: "{used} उपयोग में / {total}", diskFree: "{free} खाली", modelsFolder: "मॉडल फ़ोल्डर", + modelsFolderKeywords: + "मॉडल फ़ोल्डर फोल्डर निर्देशिका पथ स्थान डाउनलोड कैश संग्रहण डिस्क ड्राइव स्थानांतरित बदलें models folder path hugging face", modelsFolderDescription: "जहां डाउनलोड किए गए मॉडल संग्रहीत होते हैं।", openAction: "खोलें", copyAction: "पथ कॉपी करें", diff --git a/studio/frontend/src/i18n/locales/ja.ts b/studio/frontend/src/i18n/locales/ja.ts index 978fde6281..1653bad76e 100644 --- a/studio/frontend/src/i18n/locales/ja.ts +++ b/studio/frontend/src/i18n/locales/ja.ts @@ -28,10 +28,18 @@ export const ja = { product: "Unsloth Studio", accountMenu: "{name} のアカウントメニュー", updateAvailable: "アップデートが利用可能です", + resize: { + collapse: "クリックで折りたたむ", + expand: "クリックで展開", + drag: "ドラッグでサイズ変更", + }, aria: { home: "Unsloth ホーム", closeSidebar: "サイドバーを閉じる", openSidebar: "サイドバーを開く", + resizeSidebar: "サイドバーのサイズ変更または折りたたみ", + resizeRunSettings: "実行設定のサイズ変更または閉じる", + openRunSettings: "実行設定を開く", chatOptions: "チャットオプション", runOptions: "実行オプション", }, @@ -393,6 +401,8 @@ export const ja = { diskUsage: "{used} 使用中 / {total}", diskFree: "{free} 空き", modelsFolder: "モデルフォルダ", + modelsFolderKeywords: + "モデル フォルダ ディレクトリ パス 保存先 場所 ダウンロード キャッシュ ストレージ ディスク ドライブ 移動 変更 models folder path hugging face", modelsFolderDescription: "ダウンロードしたモデルの保存先。", openAction: "開く", copyAction: "パスをコピー", diff --git a/studio/frontend/src/i18n/locales/ko.ts b/studio/frontend/src/i18n/locales/ko.ts index aa8a4fd47b..b0da314896 100644 --- a/studio/frontend/src/i18n/locales/ko.ts +++ b/studio/frontend/src/i18n/locales/ko.ts @@ -27,10 +27,18 @@ export const ko = { product: "Unsloth Studio", accountMenu: "{name} 계정 메뉴", updateAvailable: "업데이트 사용 가능", + resize: { + collapse: "클릭하여 접기", + expand: "클릭하여 펼치기", + drag: "드래그하여 크기 조절", + }, aria: { home: "Unsloth 홈", closeSidebar: "사이드바 닫기", openSidebar: "사이드바 열기", + resizeSidebar: "사이드바 크기 조절 또는 접기", + resizeRunSettings: "실행 설정 크기 조절 또는 닫기", + openRunSettings: "실행 설정 열기", chatOptions: "채팅 옵션", runOptions: "학습 옵션", }, @@ -315,6 +323,8 @@ export const ko = { diskUsage: "{used} 사용 중 / {total}", diskFree: "{free} 여유", modelsFolder: "모델 폴더", + modelsFolderKeywords: + "모델 폴더 디렉터리 디렉토리 경로 위치 저장 다운로드 캐시 저장소 디스크 드라이브 이동 변경 models folder path hugging face", modelsFolderDescription: "다운로드한 모델이 저장되는 위치입니다.", openAction: "열기", copyAction: "경로 복사", diff --git a/studio/frontend/src/i18n/locales/pt-br.ts b/studio/frontend/src/i18n/locales/pt-br.ts index 84cd3f945e..e9f23623f8 100644 --- a/studio/frontend/src/i18n/locales/pt-br.ts +++ b/studio/frontend/src/i18n/locales/pt-br.ts @@ -27,10 +27,18 @@ export const ptBR = { product: "Unsloth Studio", accountMenu: "Menu de conta {name}", updateAvailable: "Atualização disponível", + resize: { + collapse: "Clique para recolher", + expand: "Clique para expandir", + drag: "Arraste para redimensionar", + }, aria: { home: "Início do Unsloth", closeSidebar: "Fechar barra lateral", openSidebar: "Abrir barra lateral", + resizeSidebar: "Redimensionar ou recolher a barra lateral", + resizeRunSettings: "Redimensionar ou fechar as configurações de execução", + openRunSettings: "Abrir as configurações de execução", chatOptions: "Opções de chat", runOptions: "Opções de execução", }, @@ -417,6 +425,8 @@ export const ptBR = { diskUsage: "{used} usados / {total}", diskFree: "{free} livres", modelsFolder: "Pasta de modelos", + modelsFolderKeywords: + "modelos pasta diretorio diretório caminho local localizacao localização downloads baixar cache armazenamento disco unidade mover alterar models folder path hugging face", modelsFolderDescription: "Onde os modelos baixados são armazenados.", openAction: "Abrir", copyAction: "Copiar caminho", diff --git a/studio/frontend/src/i18n/locales/ru.ts b/studio/frontend/src/i18n/locales/ru.ts index 7725212e3b..364f224802 100644 --- a/studio/frontend/src/i18n/locales/ru.ts +++ b/studio/frontend/src/i18n/locales/ru.ts @@ -27,10 +27,18 @@ export const ru = { product: "Unsloth Studio", accountMenu: "Меню аккаунта {name}", updateAvailable: "Доступно обновление", + resize: { + collapse: "Нажмите, чтобы свернуть", + expand: "Нажмите, чтобы развернуть", + drag: "Потяните, чтобы изменить размер", + }, aria: { home: "Главная Unsloth", closeSidebar: "Закрыть боковую панель", openSidebar: "Открыть боковую панель", + resizeSidebar: "Изменить размер или свернуть боковую панель", + resizeRunSettings: "Изменить размер или закрыть настройки запуска", + openRunSettings: "Открыть настройки запуска", chatOptions: "Параметры чата", runOptions: "Параметры запуска", }, @@ -316,6 +324,8 @@ export const ru = { diskUsage: "{used} использовано / {total}", diskFree: "{free} свободно", modelsFolder: "Папка моделей", + modelsFolderKeywords: + "модели папка каталог путь расположение загрузки кэш хранилище диск перенести изменить models folder path hugging face", modelsFolderDescription: "Где хранятся загруженные модели.", openAction: "Открыть", copyAction: "Копировать путь", diff --git a/studio/frontend/src/i18n/locales/zh-CN.ts b/studio/frontend/src/i18n/locales/zh-CN.ts index 06326ed008..777bfbc4dd 100644 --- a/studio/frontend/src/i18n/locales/zh-CN.ts +++ b/studio/frontend/src/i18n/locales/zh-CN.ts @@ -27,10 +27,18 @@ export const zhCN = { product: "Unsloth Studio", accountMenu: "{name} 账号菜单", updateAvailable: "有可用更新", + resize: { + collapse: "点击折叠", + expand: "点击展开", + drag: "拖动调整大小", + }, aria: { home: "Unsloth 首页", closeSidebar: "关闭侧边栏", openSidebar: "打开侧边栏", + resizeSidebar: "调整或折叠侧边栏", + resizeRunSettings: "调整或关闭运行设置", + openRunSettings: "打开运行设置", chatOptions: "聊天选项", runOptions: "训练选项", }, @@ -408,6 +416,8 @@ export const zhCN = { diskUsage: "已用 {used} / {total}", diskFree: "{free} 可用", modelsFolder: "模型文件夹", + modelsFolderKeywords: + "模型 文件夹 目录 路径 位置 下载 缓存 存储 磁盘 驱动器 移动 更改 models folder path hugging face", modelsFolderDescription: "已下载模型的存储位置。", openAction: "打开", copyAction: "复制路径", diff --git a/studio/frontend/src/index.css b/studio/frontend/src/index.css index 4797401ead..4ffd29ad7c 100644 --- a/studio/frontend/src/index.css +++ b/studio/frontend/src/index.css @@ -1363,6 +1363,23 @@ html[data-chat-font] .aui-root { cursor: pointer; } + /* While a panel edge is dragged, keep the resize cursor even as the pointer + travels over buttons and text that would claim their own. */ + html[data-panel-resizing], + html[data-panel-resizing] * { + cursor: col-resize !important; + user-select: none !important; + } + + html[data-panel-resizing] + :is( + [data-slot="sidebar-inner"], + [data-slot="sidebar-inset"], + [data-slot="chat-settings-panel"] > div + ) { + pointer-events: none; + } + /* Model selector: pointer cursor on every clickable element. */ .unsloth-model-selector-trigger, .unsloth-model-selector-menu button { diff --git a/studio/frontend/src/lib/changelog-links.ts b/studio/frontend/src/lib/changelog-links.ts new file mode 100644 index 0000000000..16b3d3c8bc --- /dev/null +++ b/studio/frontend/src/lib/changelog-links.ts @@ -0,0 +1,664 @@ +// SPDX-License-Identifier: AGPL-3.0-only +// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 + +/** + * A relative link in CHANGELOG.md means "somewhere in the Unsloth repository", + * but inside Studio it would resolve against Studio's own origin. Rewriting to + * absolute repository URLs makes them behave the way GitHub renders the file. + */ + +import { + type CodeSpan, + codeSpans, + insideSpan, +} from "@/lib/markdown-code-spans"; +import { commentClosesBelow } from "@/lib/markdown-inline-comments"; +import { + EMPTY_LIST_STATE, + type ListState, + NO_QUOTE, + type QuoteState, + containerContent, + hiddenStructure, + indentWidth, + itemContent, + openLists, + quoteDepth, + quoteState, +} from "@/lib/markdown-list-columns"; + +const LINK_BASE = "https://github.com/unslothai/unsloth/blob/main/"; +const IMAGE_BASE = "https://raw.githubusercontent.com/unslothai/unsloth/main/"; + +// Inline `](dest)` plus the `[label]: dest` reference form. The destination is +// either <bracketed> or runs to whitespace or the closing paren. +const NESTED_LABEL = String.raw`((?:[^[\]\\]|\\.|\[(?:[^[\]\\]|\\.)*\])*)`; +// Only ASCII punctuation is escapable, so the backslash in `a\ b.md` is an +// ordinary character of the destination and the space still ends it. +const ESCAPABLE = String.raw`[!-/:-@[-\`{-~]`; +const DESTINATION_CHAR = String.raw`\\${ESCAPABLE}|[^\s()]`; +// A destination may hold balanced parentheses, and a path may nest them, so +// `[x](((draft)).md)` points at `((draft)).md`. An expression cannot count, so +// pairs are unrolled to the depth cmark stops at, which is what GitHub renders. +const MAX_DESTINATION_NESTING = 32; + +/** A balanced parenthesised run nested up to `depth` levels deep. */ +function nestedParens(depth: number): string { + let group = String.raw`\((?:${DESTINATION_CHAR})*\)`; + for (let left = depth - 1; left > 0; left -= 1) { + group = String.raw`\((?:${DESTINATION_CHAR}|${group})*\)`; + } + return group; +} + +const BALANCED_DESTINATION = String.raw`(?:${DESTINATION_CHAR}|${nestedParens(MAX_DESTINATION_NESTING)})*`; +const PLAIN_DESTINATION = String.raw`(?:${DESTINATION_CHAR})*`; +// A balanced pair counts only while a `)` or a title still closes the link +// after it, or swallowing it would invent a link across lines. +const CLOSES_LINK = String.raw`(?=[ \t]*[)'"])`; +// A destination that runs out of line has its closer below it, the line being +// only part of the link. One stopping short of a closer is no destination at all, +// so `[x](a b.md)` and `[x](a(b.md)` stay plain text and keep the paths they name. +const CLOSES_OR_ENDS_LINE = String.raw`(?=[ \t]*(?:[)'"]|$))`; +const INLINE_TARGET = new RegExp( + String.raw`(!?)\[${NESTED_LABEL}\]\(\s*(<[^<>\n]*>|${BALANCED_DESTINATION}${CLOSES_LINK}|${PLAIN_DESTINATION}${CLOSES_OR_ENDS_LINE})`, + "g", +); +const REFERENCE_TARGET = /^( {0,3}\[((?:[^[\]\\]|\\.)*)\]:\s*)(<[^<>\n]*>|\S+)/; +// `![alt][label]`, `![label][]` and `![label]`: a definition they point at +// has to resolve to the raw file, not to its page on GitHub. +const IMAGE_REFERENCE = + /!\[((?:[^[\]\\]|\\.)*)\](?:\[((?:[^[\]\\]|\\.)*)\]|(?!\())/g; +const FENCE = /^ {0,3}(`{3,}|~{3,})(.*)$/; +// Four columns past the container start indented code, unless a paragraph is +// open. Inside a list item that is measured from the item's content column, so a +// link indented under a bullet is prose and still resolves. +const INDENTED_CODE_INDENT = 4; +// CommonMark type 1 HTML blocks show their contents verbatim. +const RAW_HTML_OPEN = /^ {0,3}<(pre|script|style|textarea)(?=[\s>]|$)/i; +const RAW_HTML_CLOSE = /<\/(pre|script|style|textarea)\s*>/i; +// Type 6 and 7 blocks are literal too and run to the next blank line, not to a +// closing tag, so `<details>` holds Markdown only after a blank line. Type 7 (any +// other complete tag alone on a line) cannot interrupt a paragraph. +const HTML_BLOCK_OPEN = /^ {0,3}<\/?([a-zA-Z][a-zA-Z0-9-]*)(?=[\s/>]|$)/; +const HTML_ATTRIBUTE = + "(?:\\s+[a-zA-Z_:][a-zA-Z0-9_.:-]*(?:\\s*=\\s*(?:[^\\s\"'=<>`]+|'[^']*'|\"[^\"]*\"))?)"; +const HTML_TAG_ONLY_LINE = new RegExp( + `^ {0,3}(?:<[a-zA-Z][a-zA-Z0-9-]*${HTML_ATTRIBUTE}*\\s*/?>|</[a-zA-Z][a-zA-Z0-9-]*\\s*>)\\s*$`, +); +const HTML_BLOCK_TAGS = new Set( + `address article aside base basefont blockquote body caption center col colgroup + dd details dialog dir div dl dt fieldset figcaption figure footer form frame + frameset h1 h2 h3 h4 h5 h6 head header hr html iframe legend li link main menu + menuitem nav noframes ol optgroup option p param search section summary table + tbody td tfoot th thead title tr track ul`.split(/\s+/), +); +// Lines that are blocks in their own right, so no paragraph is open after. +const BLOCK_LINE = + /^ {0,3}(?:#{1,6}([ \t]|$)|(?:\*[ \t]*){3,}$|(?:-[ \t]*){3,}$|(?:_[ \t]*){3,}$|>|=+[ \t]*$)/; +// A definition is a block of its own but may not interrupt a paragraph, so it +// ends the one above only when there is none to continue. It opens none either, +// or consecutive definitions could never start (spec 0.31.2 section 4.7). Same +// rule as `_LINK_DEFINITION` in the backend's `after_paragraph`. +const LINK_DEFINITION = /^ {0,3}\[(?:[^[\]\\]|\\.)+\]:/; +const LINE_ENDINGS = /\r\n?/g; +// A scheme, a protocol-relative host, or a fragment: already absolute enough. +// `//` needs a host after it, so `///docs` stays a repository path. +const ABSOLUTE = /^(?:[a-zA-Z][a-zA-Z0-9+.-]*:|\/\/[^/]|#)/; + +const COMMENT_OPEN = "<!--"; +const COMMENT_CLOSE = "-->"; +const COMMENT_BLOCK_OPEN = /^ {0,3}<!--/; + +/** + * `line` with its commented spans blanked, and whether a comment block is still + * open below it. Commented content renders as nothing, so it holds no fence, + * block or code span. Lengths are preserved so offsets still line up. + * + * Only a comment that starts a line opens a block (CommonMark type 2), and only + * that runs on to the line holding `-->`, tail included. One written mid-sentence + * is inline raw HTML belonging to its paragraph, so its `-->` may arrive on a + * later line and only the text up to it is hidden. `closesBelow` says one does; + * without it the opener is ordinary text, so a note merely mentioning `<!--` must + * not hide the links below it. + * + * "Starts a line" is read inside the container, so `blockOpen` comes from the + * item's content rather than the raw line. + */ +function maskComments( + line: string, + inComment: boolean, + runOn: boolean, + closesBelow: boolean, + blockOpen: boolean, +): [string, boolean, boolean] { + if (inComment) { + // The closing line belongs to the block, tail included. + return [" ".repeat(line.length), !line.includes(COMMENT_CLOSE), false]; + } + if (runOn) { + const closed = line.indexOf(COMMENT_CLOSE); + if (closed < 0) { + return [" ".repeat(line.length), false, true]; + } + // Only up to the closer: the tail is the paragraph's own text again. + const resumed = closed + COMMENT_CLOSE.length; + return maskInline(line, resumed, closesBelow); + } + if (blockOpen) { + // `<!-->` and `<!--->` are complete comments, so the closer may overlap the + // opener; searching past it would blank the rest of the file. + return [" ".repeat(line.length), !line.includes(COMMENT_CLOSE), false]; + } + return maskInline(line, 0, closesBelow); +} + +/** `maskComments` from `from`, where no comment block is open. */ +function maskInline( + line: string, + from: number, + closesBelow: boolean, +): [string, boolean, boolean] { + let out = " ".repeat(from); + let index = from; + // Scanned only once an opener turns up. Spans are ordered and disjoint and each + // opener sits at or past the last, so the search resumes rather than restarts. + let spans: CodeSpan[] | null = null; + let cursor = 0; + while (index < line.length) { + const start = line.indexOf(COMMENT_OPEN, index); + if (start < 0) { + return [out + line.slice(index), false, false]; + } + spans ??= codeSpans(line); + while (cursor < spans.length && (spans[cursor]?.end ?? 0) <= start) { + cursor += 1; + } + // A delimiter inside inline code is literal, not a comment opener. + const span = spans[cursor]; + if (span !== undefined && span.start <= start) { + out += line.slice(index, span.end); + index = span.end; + continue; + } + // `<!-->` and `<!--->` are complete comments, so the closer may overlap. + const close = line.indexOf(COMMENT_CLOSE, start + 2); + if (close < 0) { + if (closesBelow) { + // The paragraph carries the comment on, so the line from the opener is + // inside it, and so is the line below. + return [ + out + line.slice(index, start) + " ".repeat(line.length - start), + false, + true, + ]; + } + // Nothing closes it at all, so the renderer shows it as ordinary text. + return [out + line.slice(index), false, false]; + } + out += line.slice(index, start); + out += " ".repeat(close + COMMENT_CLOSE.length - start); + index = close + COMMENT_CLOSE.length; + } + return [out, false, false]; +} + +/** + * Whether `line` is written outside the container an open block belongs to. A + * fence and an HTML block hold no lazy continuation line, so content left of the + * item, or outside the quote, ends the block with its container. A raw block or + * comment inside a list item ends on a blank line too: the item takes the break, + * so what follows is a block of the item's own. + */ +function leavesContainer( + line: string, + quotes: number, + column: number, + blockQuotes: number, + rawInItem: boolean, +): boolean { + if (quotes < blockQuotes) { + return true; + } + if (!line.trim()) { + return rawInItem; + } + return column > 0 && indentWidth(line) < column; +} + +/** True if `line` starts a CommonMark type 6 or type 7 HTML block. */ +function opensHtmlBlock(line: string, afterParagraph: boolean): boolean { + const named = HTML_BLOCK_OPEN.exec(line); + if (named && HTML_BLOCK_TAGS.has((named[1] ?? "").toLowerCase())) { + return true; + } + return !afterParagraph && HTML_TAG_ONLY_LINE.test(line); +} + +/** A reference label as CommonMark compares them. */ +function label(text: string): string { + return text.trim().replace(/\s+/g, " ").toLowerCase(); +} + +const NEEDS_BRACKETS = /[()\s]/; +// `\(` in a destination is a literal paren. Only ASCII punctuation is escapable, +// so the backslash in `docs\alpha.md` is part of the path and has to survive. +const ESCAPE = new RegExp(String.raw`\\(${ESCAPABLE})`, "g"); +// A URL parser reads a backslash as a path separator, so `docs\a.md` would +// resolve to `docs/a.md`. Encode it first, the way a renderer normalises it. +const BACKSLASH = /\\/g; +// Only spaces and tabs may follow a closing fence. +const NON_SPACE = /[^ \t]/; +const LEADING_SLASHES = /^\/+/; + +function absolute(target: string, image: boolean): string { + const base = image ? IMAGE_BASE : LINK_BASE; + const trimmed = target.trim().replace(ESCAPE, "$1"); + if (!trimmed || ABSOLUTE.test(trimmed)) { + return target; + } + try { + // A leading slash means the repository root, not the site root, so append + // it to the base instead of replacing the base path. + const resolved = new URL( + trimmed.replace(LEADING_SLASHES, "").replace(BACKSLASH, "%5C"), + base, + ).toString(); + // `../` can climb out of the repository: leave those alone. + return resolved.startsWith(base) ? resolved : target; + } catch { + return target; + } +} + +/** True when `index` is escaped by an odd run of backslashes. */ +function isEscaped(line: string, index: number): boolean { + let slashes = 0; + while (line[index - 1 - slashes] === "\\") { + slashes += 1; + } + return slashes % 2 === 1; +} + +function unwrap(target: string): string { + return target.startsWith("<") && target.endsWith(">") + ? target.slice(1, -1) + : target; +} + +/** The destination as it goes back into the line. */ +function wrap(resolved: string, original: string): string { + const bracketed = original.startsWith("<") && original.endsWith(">"); + return bracketed || (resolved !== original && NEEDS_BRACKETS.test(resolved)) + ? `<${resolved}>` + : resolved; +} + +/** Rewrites one line's link and image targets, leaving code spans alone. */ +function rewriteLine( + line: string, + imageLabels: Set<string>, + spans: CodeSpan[], + base: number, + isDefinition: boolean, +): string { + const reference = isDefinition ? REFERENCE_TARGET.exec(line) : null; + if (reference) { + const target = reference[3] ?? ""; + const resolved = absolute( + unwrap(target), + imageLabels.has(label(reference[2] ?? "")), + ); + const rest = line.slice(reference[0].length); + return `${reference[1]}${wrap(resolved, target)}${rest}`; + } + + INLINE_TARGET.lastIndex = 0; + return line.replace(INLINE_TARGET, (match, bang, text, target, offset) => { + // `\\[` is a literal bracket, so the expression is not a link. + const opener = offset + (bang ? 1 : 0); + if (insideSpan(spans, base + offset) || isEscaped(line, opener)) { + return match; + } + // `\\!` is a literal mark, so what follows is a link, not an image. + const image = bang === "!" && !isEscaped(line, offset); + const resolved = absolute(unwrap(target), image); + // A badge nests an image inside a link, so the label is rewritten too. + const inner = text.includes("](") + ? rewriteLine(text, imageLabels, codeSpans(text), 0, false) + : text; + return `${bang}[${inner}](${wrap(resolved, target)}`; + }); +} + +interface Classified { + // Lines the renderer shows as Markdown, by index. + text: number[]; + // Same lines, blanked where the renderer shows code, for span scanning. + masked: string; + // Lines where a `[label]: dest` definition can start. + definition: Set<number>; + // Document ranges the renderer hides inside HTML comments. + comments: CodeSpan[]; +} + +/** + * Sorts lines into Markdown and code, masking the code so a span cannot pair + * across it. Offsets are preserved, so a mask span sits where it does in the doc. + */ +function classify(lines: string[]): Classified { + const text: number[] = []; + const definition = new Set<number>(); + const masked: string[] = []; + let openFence: string | null = null; + let inRawHtml = false; + let inHtmlBlock = false; + // Where the open block was written: the content column of the item it belongs + // to, 0 at document level, plus the blockquotes it sits inside. Only one is ever + // open, and none holds a lazy continuation line, so a line left of the item or + // outside the quote ends the block with its container. + let blockColumn = 0; + let blockQuotes = 0; + let inComment = false; + // True while an inline comment opened above runs on into this line, carried by + // the paragraph holding it. + let runOn = false; + const closesBelow = commentClosesBelow(lines); + let inCode = false; + let afterParagraph = false; + let quote: QuoteState = NO_QUOTE; + let lists: ListState = EMPTY_LIST_STATE; + const comments: CodeSpan[] = []; + let offset = 0; + + // The line as list tracking sees it: blank wherever nothing renders. Taken + // with the paragraph state from the line above, as the renderer would. + const track = (structural: string, above: QuoteState): void => { + lists = openLists(structural, lists, afterParagraph, above.quoted); + }; + // Where a block just opened sits, read after the opener closed the items it + // is dedented out of, so it belongs to the container it is really in. + const startBlock = (quotes: number): void => { + blockColumn = lists.columns.at(-1) ?? 0; + blockQuotes = quotes; + }; + const endBlock = (): void => { + blockColumn = 0; + blockQuotes = 0; + }; + + lines.forEach((original, index) => { + const start = offset; + offset += original.length + 1; + // The quote state from the line above, which is what list tracking asks + // about. Only plain text below rewrites it, so every block returning early + // leaves no quoted paragraph open behind it. + const above = quote; + quote = NO_QUOTE; + // A fence, comment or HTML block runs only to the end of the container it was + // written in, so a line dedented out of that item or outside that quote + // closes both. + const quotes = quoteDepth(original); + let inBlock = openFence !== null || inRawHtml || inHtmlBlock || inComment; + if ( + inBlock && + leavesContainer( + original, + quotes, + blockColumn, + blockQuotes, + (inRawHtml || inComment) && blockColumn > 0 && blockQuotes === 0, + ) + ) { + openFence = null; + inRawHtml = false; + inHtmlBlock = false; + inComment = false; + endBlock(); + inBlock = false; + } + // Read from the container the line is written in, so a fence three columns + // past a nested bullet or behind a quote marker still opens one. A block + // already open keeps only its own quote stripped, or a deeper marker in it + // would read as a closer. + const container = containerContent( + original, + lists, + inBlock ? blockQuotes : quotes, + ); + // A comment cannot open a fence and a fence hides a comment opener, so resolve + // them in that order or a hidden delimiter opens a phantom fence. An opener is + // read past a marker on the same line too, since a fence written as an item's + // first content opens inside it. Only an opener: fenced content is literal and + // a closer carries no marker. + const fenceSource = inComment + ? null + : FENCE.exec( + openFence === null + ? itemContent(container, afterParagraph) + : container, + ); + if (inRawHtml) { + track("", above); + inRawHtml = !RAW_HTML_CLOSE.test(container); + if (!inRawHtml) { + endBlock(); + } + masked.push(" ".repeat(original.length)); + afterParagraph = false; + return; + } + if (inHtmlBlock) { + track("", above); + // Only a blank line ends a type 6 or 7 block, so nothing inside one is a + // fence or a link. A bare quote marker holds nothing, so it ends one too. + inHtmlBlock = !!container.trim(); + if (!inHtmlBlock) { + endBlock(); + } + masked.push(" ".repeat(original.length)); + afterParagraph = false; + return; + } + const fence = fenceSource; + if (fence) { + // A fence renders as nothing, but its indent still closes an item. + track(original, above); + const marker = fence[1] ?? ""; + if (openFence === null) { + // A backtick fence's info string may not contain a backtick. + openFence = + marker[0] !== "`" || !(fence[2] ?? "").includes("`") ? marker : null; + if (openFence === null) { + text.push(index); + masked.push(original); + afterParagraph = true; + return; + } + startBlock(quotes); + } else if ( + // A closer matches the opening character and carries nothing after it. + marker[0] === openFence[0] && + marker.length >= openFence.length && + !NON_SPACE.test(fence[2] ?? "") + ) { + openFence = null; + endBlock(); + } + masked.push(" ".repeat(original.length)); + afterParagraph = false; + return; + } + if (openFence !== null) { + track("", above); + // Fenced content is literal, so a comment opener in it is not one. + masked.push(" ".repeat(original.length)); + return; + } + // A block already open owns this line, so it is content rather than a block + // written at the column it happens to start in. + const hidden = inComment; + const carried = runOn; + // A comment is an HTML block too, so one written as a list item's first + // content opens inside that item exactly as a fence does: read past a marker + // on the same line and from its container's column, not the line's margin. + const opensComment = + !(hidden || carried) && + COMMENT_BLOCK_OPEN.test(itemContent(container, afterParagraph)); + // Only now, outside every fence, does a comment hide what follows. + const [line, stillInComment, stillRunOn] = maskComments( + original, + inComment, + runOn, + closesBelow[index + 1] ?? false, + opensComment, + ); + inComment = stillInComment; + runOn = stillRunOn; + // A line an inline comment runs on into is still a line of the paragraph + // that carries it: only its text is hidden, never its block structure. + const structure = carried ? original : line; + // The same container reading as above, now the comments are masked. A comment + // blanks its own line, so that line is read as written: the block renders as + // nothing, but the item it is the content of still opens. + const source = opensComment ? original : line; + const visible = containerContent(source, lists, quotes); + // An HTML block written as a list item's first content opens inside that item, + // as a fence does, so an opener is read past a marker on the same line. The + // marker survives into the structural line, so its item is still tracked. + const content = itemContent(visible, afterParagraph); + const marker = + content === visible + ? "" + : source.slice(0, source.length - content.length); + // Taken before an HTML opener is hidden: it renders as nothing, but its indent + // still closes a list item it sits left of. A comment or a <pre> keeps only its + // column and marker, since the text it hides is not Markdown and opens no list. + const opensRaw = !carried && RAW_HTML_OPEN.test(content); + track( + !(hidden || carried) && (opensRaw || !line.trim()) + ? hiddenStructure(original, marker) + : structure, + above, + ); + // Read once the opener has closed the items it is dedented out of, so the + // comment block belongs to the item it is really written inside. + if (inComment !== hidden) { + if (inComment) { + startBlock(quotes); + } else { + endBlock(); + } + } + for (let at = 0; at < line.length; at += 1) { + if (line[at] === " " && original[at] !== " ") { + const from = at; + while (at < line.length && line[at] === " " && original[at] !== " ") { + at += 1; + } + comments.push({ start: start + from, end: start + at, content: "" }); + } + } + if (opensRaw) { + inRawHtml = !RAW_HTML_CLOSE.test(content.replace(RAW_HTML_OPEN, "")); + if (inRawHtml) { + startBlock(quotes); + } + masked.push(" ".repeat(line.length)); + afterParagraph = false; + return; + } + if (!carried && content.trim() && opensHtmlBlock(content, afterParagraph)) { + inHtmlBlock = true; + startBlock(quotes); + masked.push(" ".repeat(line.length)); + afterParagraph = false; + return; + } + const blank = !structure.trim(); + // Measured from the innermost open item's content column, not the margin: + // four spaces under "- Details:" is a paragraph, not a code block. + const column = lists.columns.at(-1) ?? 0; + const indented = indentWidth(structure) - column >= INDENTED_CODE_INDENT; + // Indented code starts only outside a paragraph and runs to a dedent. + if (inCode) { + inCode = blank || indented; + } else { + inCode = !afterParagraph && !blank && indented; + } + if (inCode) { + masked.push(" ".repeat(line.length)); + afterParagraph = false; + return; + } + // A definition cannot interrupt a paragraph. + if (!afterParagraph) { + definition.add(index); + } + text.push(index); + masked.push(line); + afterParagraph = + !blank && + !BLOCK_LINE.test(structure) && + (afterParagraph || !LINK_DEFINITION.test(structure)); + quote = quoteState(structure, above.inQuote); + }); + + return { text, masked: masked.join("\n"), definition, comments }; +} + +/** Absolute repository URLs for every relative link and image in `markdown`. */ +export function resolveChangelogLinks(markdown: string): string { + // The desktop updater body arrives with CRLF, which would hide fences. + const lines = markdown.replace(LINE_ENDINGS, "\n").split("\n"); + const { text, masked, definition, comments } = classify(lines); + // Scanned over the whole document, so a span may cross a line break. Commented + // ranges join them: the renderer shows neither, so a link in one is not + // followable and rewriting it would only mutate hidden text. + const spans = [...codeSpans(masked), ...comments].sort( + (a, b) => a.start - b.start, + ); + + // Offset of each line in the document, to place matches inside it. + const offsets: number[] = []; + let cursor = 0; + for (const line of lines) { + offsets.push(cursor); + cursor += line.length + 1; + } + + // Only images resolve against the raw host, so collect the image labels + // before rewriting any definition. + const imageLabels = new Set<string>(); + for (const index of text) { + const line = lines[index] ?? ""; + IMAGE_REFERENCE.lastIndex = 0; + for ( + let match = IMAGE_REFERENCE.exec(line); + match !== null; + match = IMAGE_REFERENCE.exec(line) + ) { + // An escaped mark makes it a link, so its definition stays a page URL. + if ( + insideSpan(spans, (offsets[index] ?? 0) + match.index) || + isEscaped(line, match.index) + ) { + continue; + } + const explicit = match[2] ?? ""; + imageLabels.add(label(explicit.trim() ? explicit : (match[1] ?? ""))); + } + } + + const rewritten = [...lines]; + for (const index of text) { + rewritten[index] = rewriteLine( + lines[index] ?? "", + imageLabels, + spans, + offsets[index] ?? 0, + definition.has(index), + ); + } + return rewritten.join("\n"); +} diff --git a/studio/frontend/src/lib/markdown-code-spans.ts b/studio/frontend/src/lib/markdown-code-spans.ts new file mode 100644 index 0000000000..537aabb1ab --- /dev/null +++ b/studio/frontend/src/lib/markdown-code-spans.ts @@ -0,0 +1,123 @@ +// SPDX-License-Identifier: AGPL-3.0-only +// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 + +/** + * CommonMark code spans: a backtick run closes only on an equal-length run. + * That needs lookbehind, which older Safari rejects, so runs are scanned by hand. + */ + +export interface CodeSpan { + // Offsets of the whole span, delimiters included. + start: number; + end: number; + // Between the delimiters, with the one space of padding removed. + content: string; +} + +function runLength(text: string, index: number): number { + let end = index; + while (text[end] === "`") { + end += 1; + } + return end - index; +} + +/** True when `index` is escaped by an odd run of backslashes. */ +function escaped(text: string, index: number): boolean { + let slashes = 0; + while (text[index - 1 - slashes] === "\\") { + slashes += 1; + } + return slashes % 2 === 1; +} + +/** CommonMark drops one space of padding, so `` ` a ` `` renders as "a". */ +function stripPadding(content: string): string { + if ( + content.length > 1 && + content.startsWith(" ") && + content.endsWith(" ") && + content.trim() !== "" + ) { + return content.slice(1, -1); + } + return content; +} + +/** Every code span in `text`, in order. Unclosed runs are ordinary text. */ +export function codeSpans(text: string): CodeSpan[] { + const spans: CodeSpan[] = []; + let index = 0; + + while (index < text.length) { + if (text[index] !== "`" || escaped(text, index)) { + index += 1; + continue; + } + const ticks = runLength(text, index); + const contentStart = index + ticks; + + let cursor = contentStart; + let closed = false; + while (cursor < text.length) { + // Escapes do not apply inside a span, so a run after a backslash closes it. + if (text[cursor] !== "`") { + cursor += 1; + continue; + } + const candidate = runLength(text, cursor); + if (candidate === ticks) { + spans.push({ + start: index, + end: cursor + ticks, + content: stripPadding(text.slice(contentStart, cursor)), + }); + index = cursor + ticks; + closed = true; + break; + } + cursor += candidate; + } + if (!closed) { + // Nothing closes this run: it is literal text, carry on after it. + index = contentStart; + } + } + return spans; +} + +/** Replaces every code span with `park(content)`, leaving the rest as is. */ +export function parkCodeSpans( + text: string, + park: (content: string) => string, +): string { + const spans = codeSpans(text); + if (spans.length === 0) { + return text; + } + let out = ""; + let cursor = 0; + for (const span of spans) { + out += text.slice(cursor, span.start) + park(span.content); + cursor = span.end; + } + return out + text.slice(cursor); +} + +/** True when `index` falls inside one of `spans`, which are in order. */ +export function insideSpan(spans: CodeSpan[], index: number): boolean { + let low = 0; + let high = spans.length - 1; + while (low <= high) { + const mid = (low + high) >> 1; + const span = spans[mid]; + if (span === undefined || index < span.start) { + high = mid - 1; + } else if (index >= span.end) { + low = mid + 1; + } else { + return true; + } + } + return false; +} diff --git a/studio/frontend/src/lib/markdown-inline-comments.ts b/studio/frontend/src/lib/markdown-inline-comments.ts new file mode 100644 index 0000000000..33bbfddc31 --- /dev/null +++ b/studio/frontend/src/lib/markdown-inline-comments.ts @@ -0,0 +1,62 @@ +// SPDX-License-Identifier: AGPL-3.0-only +// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 + +/** + * An HTML comment written mid-sentence is inline raw HTML, not a block, so it + * belongs to its paragraph: the `-->` may arrive on a later line of that same + * paragraph and everything between renders as nothing, while past the paragraph + * the `<!--` is ordinary text. Both changelog scanners share that answer here. + * + * The backend needs none of it: a heading closes the paragraph it sits under, so + * no heading can ever land inside one of these comments. + */ + +import { interruptsParagraph } from "@/lib/markdown-list-columns"; + +const COMMENT_CLOSE = "-->"; +// A line that cannot be more of the paragraph above it: blank, or a block that +// may interrupt one. Leading punctuation is not one: `-->` alone is the ordinary +// multiline close and a continuation may open with emphasis, so reading either as +// a break leaves the comment unclosed and its text on show. Indented code and link +// definitions are absent: neither may interrupt a paragraph (spec 0.31.2 4.4, 4.7). +const BLANK = /^[ \t]*$/; +const ATX_HEADING = /^ {0,3}#{1,6}([ \t]|$)/; +const FENCE = /^ {0,3}(?:`{3,}|~{3,})/; +const THEMATIC_BREAK = + /^ {0,3}(?:(?:\*[ \t]*){3,}|(?:-[ \t]*){3,}|(?:_[ \t]*){3,})$/; +// A row of `=` or `-` alone makes the paragraph above it a setext heading, ending it. +const SETEXT_UNDERLINE = /^ {0,3}(?:=+|-+)[ \t]*$/; +// A tag, comment or declaration at the start of a line. HTML block types 1 to 6 +// interrupt a paragraph; type 7 does not, but reading one as a break only leaves +// the opener as plain text, which is what a leading `<` has always meant here. +const HTML_LINE = /^ {0,3}</; + +/** Whether `line` starts a block of its own rather than continuing a paragraph. */ +function startsBlock(line: string): boolean { + return ( + BLANK.test(line) || + ATX_HEADING.test(line) || + FENCE.test(line) || + THEMATIC_BREAK.test(line) || + SETEXT_UNDERLINE.test(line) || + HTML_LINE.test(line) || + // Blockquote, or a list item with content: the rule the other scanners share. + interruptsParagraph(line) + ); +} + +/** + * For each line, whether a `-->` is reachable without leaving the paragraph it + * starts in. Read at `index + 1` it answers whether an inline comment opened on + * `index` and left unclosed there is a comment at all. + */ +export function commentClosesBelow(lines: string[]): boolean[] { + const closes: boolean[] = new Array(lines.length + 1).fill(false); + for (let at = lines.length - 1; at >= 0; at -= 1) { + const line = lines[at] ?? ""; + closes[at] = + !startsBlock(line) && + (line.includes(COMMENT_CLOSE) || (closes[at + 1] ?? false)); + } + return closes; +} diff --git a/studio/frontend/src/lib/markdown-list-columns.ts b/studio/frontend/src/lib/markdown-list-columns.ts new file mode 100644 index 0000000000..761cdfac54 --- /dev/null +++ b/studio/frontend/src/lib/markdown-list-columns.ts @@ -0,0 +1,357 @@ +// SPDX-License-Identifier: AGPL-3.0-only +// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 + +/** + * CommonMark measures a block's indentation from its container, not the left + * margin: four spaces at document level and four under a bullet mean different + * things. Tracking the open items lets both changelog scanners ask "is this + * indented code?" the way a renderer would. + * + * Ported from `_open_lists` in studio/backend/utils/changelog.py so the three + * scanners classify a line the same way. + */ + +/** The open list items, innermost last, by the column their content starts. */ +export interface ListState { + columns: number[]; + // True while the innermost item has had no content since its marker. + emptyItem: boolean; +} + +export const EMPTY_LIST_STATE: ListState = { columns: [], emptyItem: false }; + +// The marker needs whitespace after it, so `2.0` is a version, not an item. +const LIST_ITEM = /^[ \t]*([-*+]|\d{1,9}[.)])([ \t]+|$)/; +const THEMATIC_BREAK = + /^ {0,3}(?:(?:\*[ \t]*){3,}|(?:-[ \t]*){3,}|(?:_[ \t]*){3,})$/; +const BLOCK_QUOTE = /^ {0,3}>/; +const QUOTE_MARKER = /^ {0,3}>[ \t]?/; +// Blocks that are not paragraph text, so they cannot continue one lazily. +const PARAGRAPH_TEXT = /^ {0,3}(?![-*+>]([ \t]|$)|\d{1,9}[.)]([ \t]|$))\S/; +// Blocks that break into an open paragraph, closing it rather than continuing +// it. A link reference definition is not one of them. +const INTERRUPTS = + /^ {0,3}(?:#{1,6}([ \t]|$)|(?:\*[ \t]*){3,}$|(?:-[ \t]*){3,}$|(?:_[ \t]*){3,}$)/; +const FENCE = /^ {0,3}(?:`{3,}|~{3,})/; +const HTML_BLOCK_OPEN = /^ {0,3}<\/?([a-zA-Z][a-zA-Z0-9-]*)(?=[\s/>]|$)/; +const HTML_BLOCK_TAGS = new Set( + `address article aside base basefont blockquote body caption center col colgroup + dd details dialog dir div dl dt fieldset figcaption figure footer form frame + frameset h1 h2 h3 h4 h5 h6 head header hr html iframe legend li link main menu + menuitem nav noframes ol optgroup option p param search section summary table + tbody td tfoot th thead title tr track ul`.split(/\s+/), +); +// Content indented more than this after a marker is an indented code block, so +// the item's content starts one column past the marker instead. +const MAX_ITEM_PADDING = 4; +// Columns past its container at which a line becomes an indented code block. +const INDENTED_CODE = 4; +// Stands in for a line the renderer hides. `#` is a block of its own, so list +// tracking reads it like a comment: never a marker, never a lazy continuation. +const HIDDEN_BLOCK = "#"; +const LEADING_SPACE = /^[ \t]*/; + +/** + * `line` as list tracking sees it once the renderer hides its text. A comment or + * raw HTML block renders nothing but is still a block at its own column, so it + * closes the items it sits left of. Only the indentation survives: what the block + * hides is not Markdown and must not open a list. `marker` is the part opening + * the item the block is content of, which survives too. Ported from + * `_hidden_structure` on the backend. + */ +export function hiddenStructure(line: string, marker = ""): string { + if (marker) { + return `${marker}${HIDDEN_BLOCK}`; + } + const indent = LEADING_SPACE.exec(line)?.[0] ?? ""; + return line.trim() ? `${indent}${HIDDEN_BLOCK}` : ""; +} + +/** Columns of leading whitespace, counting a tab to the next stop of four. */ +export function indentWidth(line: string): number { + let width = 0; + for (const char of line) { + if (char === " ") { + width += 1; + } else if (char === "\t") { + width += 4 - (width % 4); + } else { + break; + } + } + return width; +} + +/** + * Whether `line` starts a block that can break into an open paragraph. A quote + * marker always can; a list item only with content, an ordered one only at 1. + * Anything else is text of the paragraph it appears to interrupt. + */ +export function interruptsParagraph(line: string): boolean { + if (BLOCK_QUOTE.test(line)) { + return true; + } + const item = THEMATIC_BREAK.test(line) ? null : LIST_ITEM.exec(line); + if (item === null) { + return false; + } + const marker = item[1] ?? ""; + if (!line.slice(item[0].length).trim()) { + return false; + } + const ordered = marker.endsWith(".") || marker.endsWith(")"); + return !ordered || marker.slice(0, -1) === "1"; +} + +/** + * Whether a marker-shaped `line` is really text of the paragraph above. Only a + * marker inside the paragraph's own item interrupts it; one to the left closes + * that item and opens a sibling. A quote owns the paragraph its lines hold, so a + * marker outside the quote opens a list of its own. + */ +export function lazyMarker( + line: string, + state: ListState, + afterParagraph: boolean, + quoted: boolean, +): boolean { + const item = THEMATIC_BREAK.test(line) ? null : LIST_ITEM.exec(line); + const columns = state.columns; + const inside = + columns.length === 0 || indentWidth(line) >= (columns.at(-1) ?? 0); + return ( + item !== null && + afterParagraph && + !quoted && + inside && + !interruptsParagraph(line) + ); +} + +/** `columns` with every item whose content starts past `indent` closed. */ +function dropDeeper(columns: number[], indent: number): number[] { + let open = columns.length; + while (open > 0 && (columns[open - 1] ?? 0) > indent) { + open -= 1; + } + return open === columns.length ? columns : columns.slice(0, open); +} + +/** `line` with up to `columns` columns of leading whitespace removed. */ +function stripIndent(line: string, columns: number): string { + let width = 0; + let index = 0; + while (index < line.length && width < columns) { + const char = line[index]; + if (char !== " " && char !== "\t") { + break; + } + width += char === " " ? 1 : 4 - (width % 4); + index += 1; + } + return line.slice(index); +} + +/** + * Whether `line` can continue a paragraph it is indented out of. Only plain text + * can: a heading, fence, break or HTML block starts a block of its own, closing + * the item instead. An underline is not one: it may never be lazy, so `===` left + * of an open item is more of the item's paragraph. Nor is a definition, a block + * of its own that may not interrupt a paragraph. A row of dashes still closes the + * item: `INTERRUPTS` reads three or more as the thematic break they are. + */ +function mayBeLazy(line: string): boolean { + const named = HTML_BLOCK_OPEN.exec(line); + // Types 1 to 6 interrupt a paragraph, so a `<div>` left of an open item closes + // it. Type 7 cannot, and is deliberately excluded. + const htmlBlock = + named !== null && HTML_BLOCK_TAGS.has((named[1] ?? "").toLowerCase()); + return ( + PARAGRAPH_TEXT.test(line) && + !INTERRUPTS.test(line) && + !FENCE.test(line) && + !htmlBlock + ); +} + +/** + * Whether `line` reads as more of a paragraph open in its container, measured + * from `column` where that container's content starts: four columns past it the + * line is indented code, which may not interrupt a paragraph, so indentation + * alone never closes the one above. + */ +export function continuesParagraph(line: string, column: number): boolean { + const inner = stripIndent(line, column); + return indentWidth(inner) >= INDENTED_CODE || mayBeLazy(inner); +} + +/** `line` with up to `depth` blockquote markers removed, and how many went. */ +function stripQuotes(line: string, depth: number): [string, number] { + let rest = line; + let removed = 0; + let marker = removed < depth ? QUOTE_MARKER.exec(rest) : null; + while (marker !== null) { + rest = rest.slice(marker[0].length); + removed += 1; + marker = removed < depth ? QUOTE_MARKER.exec(rest) : null; + } + return [rest, removed]; +} + +/** What a blockquote line holds, with its markers stripped. */ +function quoteContent(line: string): string { + return stripQuotes(line, Number.POSITIVE_INFINITY)[0]; +} + +/** How many blockquotes `line` is written inside. */ +export function quoteDepth(line: string): number { + return stripQuotes(line, Number.POSITIVE_INFINITY)[1]; +} + +/** + * `line` as the container it is written in sees it, with `quotes` blockquote + * markers and the open item's content column removed. CommonMark measures a block + * from its container, not the margin (spec 0.31.2 sections 5.1, 5.2), so `> ~~~` + * and a fence under a nested bullet are openers despite sitting more than three + * columns in. + */ +export function containerContent( + line: string, + state: ListState, + quotes: number, +): string { + const [inner] = stripQuotes(line, quotes); + if (quotes > 0) { + // A list inside a quote is the quote's own; this tracker follows document + // level only, so its columns do not apply here. + return inner; + } + const columns = dropDeeper(state.columns, indentWidth(inner)); + return stripIndent(inner, columns.at(-1) ?? 0); +} + +/** + * `line` read from the content column of a list item that opens on it. A block + * written as an item's first content sits inside that item, so ``- ``` `` opens a + * fence even though its marker is not within three columns of the container (spec + * 0.31.2 section 5.2). Padding is capped the way `openLists` caps it, or + * ``- ``` `` would read as a fence rather than the indented code it is. A + * marker the paragraph above swallows opens no item, so its line is returned + * whole, as is one four columns past its container. + */ +export function itemContent(line: string, afterParagraph: boolean): string { + if ( + indentWidth(line) >= INDENTED_CODE || + (afterParagraph && !interruptsParagraph(line)) + ) { + return line; + } + const item = THEMATIC_BREAK.test(line) ? null : LIST_ITEM.exec(line); + if (item === null) { + return line; + } + const padding = indentWidth(item[2] ?? ""); + // Over-indented content starts one column past the marker; the rest of the + // padding is the content's own indentation. + const over = padding > MAX_ITEM_PADDING ? padding - 1 : 0; + return `${" ".repeat(over)}${line.slice(item[0].length)}`; +} + +/** Whether a blockquote owns the paragraph the line below could continue. */ +export interface QuoteState { + // True while a quoted paragraph is open, so plain text below is more of it. + inQuote: boolean; + // True whenever that paragraph is the quote's rather than the document's. + quoted: boolean; +} + +export const NO_QUOTE: QuoteState = { inQuote: false, quoted: false }; + +/** + * The quote state after `line`, given the state after the line above and the + * content column of the item `line` sits in. A quote owns the paragraph its own + * lines hold, so a marker written outside the quote opens a list of its own + * rather than reading as more of that paragraph. Ported from `in_quote` tracking + * in changelog.py. + */ +export function quoteState( + line: string, + inQuote: boolean, + column = 0, +): QuoteState { + if (BLOCK_QUOTE.test(line)) { + // An empty quote holds no paragraph, so the line below starts a new one. + return { inQuote: mayBeLazy(quoteContent(line)), quoted: true }; + } + const open = inQuote && continuesParagraph(line, column); + return { inQuote: open, quoted: open }; +} + +/** + * `columns` with every item `line` is written to the left of closed. Read inside + * the container the item sits in, not from the margin: a line that only looks + * dedented there is lazy text of the item's paragraph, leaving the item open. + */ +function closeDedented( + columns: number[], + line: string, + indent: number, + afterParagraph: boolean, +): number[] { + let open = columns.length; + while (open > 0 && (columns[open - 1] ?? 0) > indent) { + const outer = open > 1 ? (columns[open - 2] ?? 0) : 0; + if (afterParagraph && continuesParagraph(line, outer)) { + break; + } + open -= 1; + } + return open === columns.length ? columns : columns.slice(0, open); +} + +/** + * The list items still open after `line`. A dedented line closes an item unless + * it is a lazy paragraph continuation. A new marker nests under a deeper column + * and replaces a sibling. `quoted` marks a paragraph the blockquote above owns: + * a marker outside the quote is not text of it, so it opens a list of its own. + */ +export function openLists( + line: string, + state: ListState, + afterParagraph: boolean, + quoted = false, +): ListState { + let columns = state.columns; + if (!line.trim()) { + // A blank line leaves the list open, unless the item is still empty: an + // item may begin with one blank line, and later content is outside it. + return { + columns: state.emptyItem ? columns.slice(0, -1) : columns, + emptyItem: false, + }; + } + const indent = indentWidth(line); + const item = THEMATIC_BREAK.test(line) ? null : LIST_ITEM.exec(line); + const empty = item !== null && !line.slice(item[0].length).trim(); + if (lazyMarker(line, state, afterParagraph, quoted)) { + // A lazy continuation or an underline, so the open items are untouched. + return state; + } + columns = closeDedented(columns, line, indent, afterParagraph); + // Four columns past its container the marker is an indented code block, or + // lazy text of the paragraph above it, so it opens no list of its own. + if (item === null || indent - (columns.at(-1) ?? 0) >= INDENTED_CODE) { + return { columns, emptyItem: false }; + } + const marker = item[1] ?? ""; + let padding = indentWidth(item[2] ?? ""); + if (padding === 0 || padding > MAX_ITEM_PADDING) { + // An empty or over-indented item still holds one column of content. + padding = 1; + } + // A sibling marker replaces the item it lines up with. + return { + columns: [...dropDeeper(columns, indent), indent + marker.length + padding], + emptyItem: empty, + }; +} diff --git a/studio/frontend/src/lib/release-notes-preview.ts b/studio/frontend/src/lib/release-notes-preview.ts new file mode 100644 index 0000000000..97441b7cbb --- /dev/null +++ b/studio/frontend/src/lib/release-notes-preview.ts @@ -0,0 +1,1005 @@ +// SPDX-License-Identifier: AGPL-3.0-only +// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 + +// Top changelog bullets, shown in the collapsed update popup. +import { codeSpans, parkCodeSpans } from "@/lib/markdown-code-spans"; +import { commentClosesBelow } from "@/lib/markdown-inline-comments"; +import { + EMPTY_LIST_STATE, + type ListState, + NO_QUOTE, + type QuoteState, + hiddenStructure, + indentWidth, + itemContent, + openLists, + quoteState, +} from "@/lib/markdown-list-columns"; + +export const RELEASE_NOTES_PREVIEW_ITEMS = 4; +const PREVIEW_ITEM_MAX_CHARS = 120; +// Bullets indented past the shallowest one are nested detail, not headlines. +const NESTED_INDENT_TOLERANCE = 1; +const TAB_WIDTH = 4; +// Four spaces starts an indented code block in Markdown. +const INDENTED_CODE_INDENT = 4; + +// At most three leading spaces: deeper is indented code, not a fence. +const FENCE = /^ {0,3}(`{3,}|~{3,})(.*)$/; +// An ATX heading needs a space, tab or line end after the marker, as in +// _HEADING_PATTERN. `\s` would match a non-breaking space and eat prose, and a +// bare `##` is an empty heading that still ends a bullet. +const HEADING = /^#{1,6}(?:[ \t]|$)/; +const BULLET = /^(?:[-*+]|(\d{1,9})[.)])[ \t]+(.*)$/; +// At most three leading spaces, as everywhere else: deeper is indented code, +// so a quoted line inside a code sample cannot reach the collector. +const BLOCKQUOTE = /^ {0,3}>[ \t]?/; +// A GFM delimiter cell is hyphens with an optional alignment colon each side. +const TABLE_DELIMITER_CELL = /^:?-+:?$/; +// "- - -" and "***" are horizontal rules, not bullets and not notes. +const THEMATIC_BREAK = + /^ {0,3}(?:(?:\*[ \t]*){3,}|(?:-[ \t]*){3,}|(?:_[ \t]*){3,})$/; +// Destinations may escape or balance parentheses, and labels may nest one +// level so `[![alt](img)](link)` still resolves. +const DESTINATION = "\\((?:\\\\.|[^()\\\\]|\\([^()]*\\))*\\)"; +const LABEL = "((?:[^\\[\\]\\\\]|\\\\.|\\[(?:[^\\[\\]\\\\]|\\\\.)*\\])*)"; +const IMAGE = new RegExp(`!\\[${LABEL}\\]${DESTINATION}`, "g"); +const LINK = new RegExp(`\\[${LABEL}\\]${DESTINATION}`, "g"); +// Reference forms: `[text][label]`, `[text][]` and the shortcut `[text]`. +const IMAGE_REFERENCE = new RegExp(`!\\[${LABEL}\\](?:\\[([^\\]]*)\\])?`, "g"); +const LINK_REFERENCE = new RegExp(`\\[${LABEL}\\](?:\\[([^\\]]*)\\])?`, "g"); +// A definition line renders as nothing at all. +const DEFINITION = /^ {0,3}\[((?:[^\[\]\\]|\\.)+)\]:/; +// CommonMark: a backslash escapes ASCII punctuation. +const ESCAPE = /\\([!-/:-@[-`{-~])/g; +// Private-use sentinels park code spans, so document text cannot contain them. +const SENTINELS = /[\uE000\uE001]/g; +const LINE_ENDINGS = /\r\n?/g; +const TABS = /\t/g; +// Real tags only: a name character must follow "<", so a version constraint +// like "Support Python <3.15 and >3.9" keeps its operators. +const HTML_TAG = /<\/?[a-zA-Z][^>]*>/g; +// <https://x> and <a@b.c> are Markdown autolinks: keep the text they render. +const AUTOLINK = /<([a-zA-Z][a-zA-Z0-9+.-]*:[^\s<>]*|[^\s<>@]+@[^\s<>@]+)>/g; +// CommonMark type 1 HTML blocks render literally until a closing tag, which +// the spec says need not be the one that opened the block. +const RAW_HTML_OPEN = /^ {0,3}<(pre|script|style|textarea)(?=[\s>]|$)/i; +const RAW_HTML_CLOSE = /<\/(pre|script|style|textarea)\s*>/i; +// Types 3 to 5 (processing instructions, declarations, CDATA) are literal too, +// each ending on its own delimiter. Comments open mid-line, handled separately. +const RAW_BLOCKS: [RegExp, RegExp][] = [ + [RAW_HTML_OPEN, RAW_HTML_CLOSE], + [/^ {0,3}<\?/, /\?>/], + [/^ {0,3}<!\[CDATA\[/, /\]\]>/], + // A declaration needs an uppercase letter, so `<!note` stays ordinary text. + [/^ {0,3}<![A-Z]/, />/], +]; +// Type 6 and 7 blocks run to the next blank line, so `<details>` holds Markdown +// only after one. Type 7 (any complete tag alone) cannot interrupt a paragraph. +const HTML_BLOCK_OPEN = /^ {0,3}<\/?([a-zA-Z][a-zA-Z0-9-]*)(?=[\s/>]|$)/; +const HTML_ATTRIBUTE = + "(?:\\s+[a-zA-Z_:][a-zA-Z0-9_.:-]*(?:\\s*=\\s*(?:[^\\s\"'=<>`]+|'[^']*'|\"[^\"]*\"))?)"; +const HTML_TAG_ONLY_LINE = new RegExp( + `^ {0,3}(?:<[a-zA-Z][a-zA-Z0-9-]*${HTML_ATTRIBUTE}*\\s*/?>|</[a-zA-Z][a-zA-Z0-9-]*\\s*>)\\s*$`, +); +const HTML_BLOCK_TAGS = new Set( + `address article aside base basefont blockquote body caption center col colgroup + dd details dialog dir div dl dt fieldset figcaption figure footer form frame + frameset h1 h2 h3 h4 h5 h6 head header hr html iframe legend li link main menu + menuitem nav noframes ol optgroup option p param search section summary table + tbody td tfoot th thead title tr track ul`.split(/\s+/), +); +// Only spaces and tabs may follow a closing fence. +const NON_SPACE = /[^ \t]/; +const HEADING_LINE = /^ {0,3}#{1,6}(?:[ \t]|$)/; +const COMMENT_BLOCK_OPEN = /^ {0,3}<!--/; +const COMMENT_OPEN = "<!--"; +const COMMENT_CLOSE = "-->"; +// Paired emphasis only. Underscores inside identifiers are literal, so +// UNSLOTH_DISABLE_UPDATE_CHECK keeps its name. +const BOLD_STAR = /\*\*(?=\S)([\s\S]*?\S)\*\*/g; +const BOLD_UNDERSCORE = /(^|[^\w])__(?=\S)([\s\S]*?\S)__(?=[^\w]|$)/g; +const ITALIC_STAR = /\*(?=\S)([^*\n]*?\S)\*/g; +const ITALIC_UNDERSCORE = /(^|[^\w])_(?=\S)([^_\n]*?\S)_(?=[^\w]|$)/g; +const BACKTICK = /`/g; +// A closer is a run of the same length, so `` `x` `` keeps its backticks. +// Streamdown renders `AT&T` as "AT&T", so the preview decodes entities too. +const NAMED_ENTITIES: Record<string, string> = { + amp: "&", + lt: "<", + gt: ">", + quot: '"', + apos: "'", + nbsp: "\u00a0", +}; +const ENTITY = /&(#\d{1,7}|#[xX][0-9a-fA-F]{1,6}|[a-zA-Z][a-zA-Z0-9]{1,31});/g; +const PARKED = /\uE000(\d+)\uE001/g; +const WHITESPACE = /\s+/g; +// Sentence end followed by something that actually starts a sentence. +const SENTENCE_BREAK = /[.!?]\s+(?=["'“‘]?[A-Z0-9])/g; +const TRAILING_WORD = /(\S+)$/; +// A period here ends an abbreviation, not the sentence. +const ABBREVIATIONS = new Set([ + "e.g.", + "i.e.", + "etc.", + "vs.", + "cf.", + "approx.", + "no.", + "fig.", + "al.", + "dr.", + "mr.", + "mrs.", + "ms.", + "prof.", + "inc.", + "ltd.", + "st.", + "jr.", + "sr.", +]); +const INITIAL = /^[A-Za-z]\.$/; +const MIN_LEAD_CHARS = 12; + +/** Strip tags until stable, so a removal cannot re-form a tag. */ +function stripHtmlTags(text: string): string { + let out = text; + let previous: string; + do { + previous = out; + out = out.replace(HTML_TAG, ""); + } while (out !== previous); + return out; +} + +export interface ReleaseNotesPreviewItem { + // Leading sentence, highlighted in the preview. + lead: string; + // Rest of the bullet, de-emphasised. Empty for single-sentence bullets. + rest: string; +} + +export interface ReleaseNotesPreview { + items: ReleaseNotesPreviewItem[]; + // Bullets past the preview limit, for a "+N more" affordance. + remaining: number; +} + +interface Bullet { + text: string; + indent: number; +} + +/** Whether a reference points at a definition the document actually has. */ +function definedLabel( + labels: Set<string> | undefined, + reference: string | undefined, + text: string, +): boolean { + if (labels === undefined) { + return false; + } + const label = (reference?.trim() ? reference : text) + .trim() + .replace(WHITESPACE, " ") + .toLowerCase(); + return labels.has(label); +} + +/** One entity as the character it renders as, or unchanged if unknown. */ +function decodeEntity(match: string, body: string): string { + if (body.startsWith("#")) { + const hex = body[1] === "x" || body[1] === "X"; + const code = Number.parseInt( + hex ? body.slice(2) : body.slice(1), + hex ? 16 : 10, + ); + return Number.isFinite(code) && code > 0 && code <= 0x10ffff + ? String.fromCodePoint(code) + : match; + } + return NAMED_ENTITIES[body.toLowerCase()] ?? match; +} + +/** Inline markdown stripped to plain text. */ +function toPlainText(markdown: string, labels?: Set<string>): string { + // Park code spans first: their contents are literal and must survive below. + const codes: string[] = []; + const park = (text: string): string => { + codes.push(text); + return `\uE000${codes.length - 1}\uE001`; + }; + // Escaped punctuation is literal too, so `\*not italic\*` keeps its stars. + const parked = parkCodeSpans(markdown, park).replace(ESCAPE, (_match, char) => + park(char), + ); + + return stripHtmlTags( + parked + .replace(AUTOLINK, "$1") + .replace(IMAGE, "") + .replace(LINK, "$1") + .replace(IMAGE_REFERENCE, (match, text, ref) => + definedLabel(labels, ref, text) ? "" : match, + ) + .replace(LINK_REFERENCE, (match, text, ref) => + definedLabel(labels, ref, text) ? text : match, + ), + ) + .replace(BOLD_STAR, "$1") + .replace(BOLD_UNDERSCORE, "$1$2") + .replace(ITALIC_STAR, "$1") + .replace(ITALIC_UNDERSCORE, "$1$2") + .replace(BACKTICK, "") + .replace(ENTITY, decodeEntity) + .replace(PARKED, (_match, index: string) => codes[Number(index)] ?? "") + .replace(WHITESPACE, " ") + .trim(); +} + +function truncate(text: string): string { + if (text.length <= PREVIEW_ITEM_MAX_CHARS) { + return text; + } + const clipped = text.slice(0, PREVIEW_ITEM_MAX_CHARS); + const lastSpace = clipped.lastIndexOf(" "); + return `${(lastSpace > 40 ? clipped.slice(0, lastSpace) : clipped).trimEnd()}...`; +} + +interface ContentLine { + text: string; + indent: number; + // Blockquoted lines are quoted examples, not the release's own bullets. + quoted: boolean; + // Content column of the innermost open list item. CommonMark measures + // indentation from here, so `indent - column` is the real depth. + column: number; +} + +/** + * `line` with its comments removed, whether a comment block stays open, and + * whether an inline comment runs on into the line below. + * + * Only a comment starting a line opens a block, which hides whole lines to the + * one holding `-->`. One written mid-sentence is inline HTML belonging to its + * paragraph, so its `-->` may arrive on a later line and only the text up to it + * is hidden. `closesBelow` says one does; without it the opener is ordinary text + * and hides nothing below. + * + * "Starting a line" is read inside the container, so `blockOpen` comes from the + * item's content rather than the raw line. + */ +function stripCommentSpans( + line: string, + startInComment: boolean, + runOn: boolean, + closesBelow: boolean, + blockOpen: boolean, +): [string, boolean, boolean] { + if (startInComment) { + // The closing line belongs to the block, tail included. + return ["", !line.includes(COMMENT_CLOSE), false]; + } + + let visible = ""; + let index = 0; + if (runOn) { + const closed = line.indexOf(COMMENT_CLOSE); + if (closed === -1) { + return ["", false, true]; + } + // Only up to the closer: the tail is the paragraph's own text again. + index = closed + COMMENT_CLOSE.length; + } else if (blockOpen) { + // `<!-->` and `<!--->` are complete comments, so the closer may overlap the + // opener; searching past it would hide every later release. + return ["", !line.includes(COMMENT_CLOSE), false]; + } + + const spans = codeSpans(line); + while (index < line.length) { + const open = line.indexOf(COMMENT_OPEN, index); + if (open === -1) { + visible += line.slice(index); + break; + } + // A delimiter inside inline code is literal, not a comment opener. + const span = spans.find( + (candidate) => candidate.start <= open && candidate.end > open, + ); + if (span) { + visible += line.slice(index, span.end); + index = span.end; + continue; + } + const close = line.indexOf(COMMENT_CLOSE, open + COMMENT_OPEN.length); + if (close === -1) { + if (closesBelow) { + // The paragraph carries the comment on, so this line and the next are in it. + return [visible + line.slice(index, open), false, true]; + } + // Nothing closes it at all, so the renderer shows it as text. + visible += line.slice(index); + break; + } + visible += line.slice(index, open); + index = close + COMMENT_CLOSE.length; + } + return [visible, false, false]; +} + +/** Strips raw block content. State is the open block's index, or null. */ +function stripRawHtml( + line: string, + openBlock: number | null, +): [string, number | null] { + if (openBlock !== null) { + return RAW_BLOCKS[openBlock]?.[1].test(line) ? ["", null] : ["", openBlock]; + } + // A block only opens at the start of a line; mid-line tags are inline HTML. + for (const [index, [opener, closer]] of RAW_BLOCKS.entries()) { + const open = opener.exec(line); + if (!open) { + continue; + } + const rest = line.slice(open[0].length); + return closer.test(rest) ? ["", null] : ["", index]; + } + return [line, null]; +} + +/** True if `line` starts a CommonMark type 6 or type 7 HTML block. */ +function opensHtmlBlock(line: string, afterParagraph: boolean): boolean { + const named = HTML_BLOCK_OPEN.exec(line); + if (named && HTML_BLOCK_TAGS.has((named[1] ?? "").toLowerCase())) { + return true; + } + return !afterParagraph && HTML_TAG_ONLY_LINE.test(line); +} + +/** + * The line as list tracking sees it. A comment or raw block renders nothing, but + * the line opening one is still a block at its own column, so it closes a list + * item it sits left of. Only the column survives, since the text it hides is not + * Markdown. A line inside a block already open is that block's content, so it + * keeps neither. A marker the hidden block is the content of survives with the + * column, so the item it opens is still tracked. + */ +function structuralLine( + line: string, + visible: string, + hidden: boolean, + marker: string, +): string { + if (visible.trim() || hidden) { + return visible; + } + return hiddenStructure(line, marker); +} + +interface ScanState { + openFence: string | null; + // Content column of the list item the open block belongs to, 0 at document + // level. A fence and an HTML block are scoped to their container, so the item's + // end closes them. Only one of the three is ever open. + blockColumn: number; + inComment: boolean; + // True while an inline comment opened above runs on into this line, carried by + // the paragraph holding it. + runOn: boolean; + inRawHtml: number | null; + inHtmlBlock: boolean; + afterParagraph: boolean; +} + +interface ScannedLine { + // What a reader would see: "" for structure and hidden blocks, null for + // fenced content, which is skipped so it cannot split a bullet. + text: string | null; + // The same line as list tracking sees it: blank wherever nothing renders, + // but kept whole where an indent still closes an open item. + structural: string; +} + +function visibleText( + line: string, + state: ScanState, + closesBelow: boolean, +): ScannedLine { + // Raw HTML first: its contents are literal, so a fence inside it is not one. + if (state.inRawHtml !== null) { + const [after, stillInRaw] = stripRawHtml(line, state.inRawHtml); + state.inRawHtml = stillInRaw; + return { text: after, structural: "" }; + } + if (state.inHtmlBlock) { + // A blank line is the only thing that ends a type 6 or 7 block. + state.inHtmlBlock = line.trim() !== ""; + return { text: "", structural: "" }; + } + // An opener is read past a marker on the same line, since a fence written as a + // list item's first content opens inside it. Only an opener: fenced content is + // literal and a closer carries no marker. + const commented = state.inComment || state.runOn; + const fence = commented + ? null + : FENCE.exec( + state.openFence === null + ? itemContent(line, state.afterParagraph) + : line, + ); + // A backtick fence whose info string holds a backtick is prose, not a fence. + if ( + fence && + (state.openFence !== null || opensFence(fence[1] ?? "", fence[2] ?? "")) + ) { + state.openFence = nextFence( + state.openFence, + fence[1] ?? "", + fence[2] ?? "", + ); + // Hidden from the collector, but its indent still closes an item. + return { text: "", structural: line }; + } + if (state.openFence !== null) { + return { text: null, structural: "" }; + } + return visibleContent(line, state, closesBelow); +} + +/** `visibleText` for a line no fence or HTML block already owns. */ +function visibleContent( + line: string, + state: ScanState, + closesBelow: boolean, +): ScannedLine { + // A block already open owns this line, so it is content rather than a block + // written at the column it happens to start in. + const hidden = state.inComment || state.inRawHtml !== null; + const carried = state.runOn; + // A comment is an HTML block too, so one written as a list item's first content + // opens inside that item exactly as a fence does: read past a marker on the + // same line rather than from the margin. + const content = itemContent(line, state.afterParagraph); + const opensComment = + !(state.inComment || carried) && COMMENT_BLOCK_OPEN.test(content); + // Commented-out notes are not rendered, so they are not previewed either. + const [uncommented, stillInComment, stillRunOn] = stripCommentSpans( + line, + state.inComment, + state.runOn, + closesBelow, + opensComment, + ); + state.inComment = stillInComment; + state.runOn = stillRunOn; + const [visible, stillInRaw] = stripRawHtml(uncommented, state.inRawHtml); + state.inRawHtml = stillInRaw; + // Taken before the opener is hidden: it renders as nothing, but its indent still + // closes a list item it sits left of, and a marker on its line still opens one. + // A line an inline comment runs on into is still a line of the paragraph that + // carries it, so only its text is hidden, never its block structure. + const marker = opensComment + ? line.slice(0, line.length - content.length) + : ""; + const structural = carried + ? line + : structuralLine(line, visible, hidden, marker); + if ( + !carried && + stillInRaw === null && + visible.trim() && + opensHtmlBlock(visible, state.afterParagraph) + ) { + state.inHtmlBlock = true; + return { text: "", structural }; + } + return { text: visible, structural }; +} + +/** + * Marker of a fence the line scanner skipped because it is indented. Only a line + * within three columns of its item's content column is one: deeper than that it + * is an indented code block, which a dedented bullet ends. + */ +function opensDeepFence(line: ContentLine): string | null { + if ( + line.indent < INDENTED_CODE_INDENT || + line.indent - line.column >= INDENTED_CODE_INDENT + ) { + return null; + } + const fence = FENCE.exec(line.text); + return fence ? (fence[1] ?? null) : null; +} + +/** + * True when `line` is the first one outside the deep fence opened with `marker` + * at `column`. A fence inside a list item runs only to the end of that item, so a + * line left of the item's content column closes both, as `fence_column` does on + * the backend. + */ +function endsDeepFence( + marker: string, + column: number, + line: ContentLine, +): boolean { + return line.indent < column || closesDeepFence(marker, line); +} + +/** True when `line` closes the deep fence opened with `marker`. */ +function closesDeepFence(marker: string, line: ContentLine): boolean { + const fence = FENCE.exec(line.text); + if (!fence) { + return false; + } + const closer = fence[1] ?? ""; + return ( + closer[0] === marker[0] && + closer.length >= marker.length && + !NON_SPACE.test(fence[2] ?? "") + ); +} + +/** + * Cells of a GFM table row, or null when the line holds no pipe at all. The + * optional leading and trailing pipes are delimiters, not empty cells, and a + * `\|` is literal text inside one. + */ +function tableCells(text: string): string[] | null { + if (!text.includes("|")) { + return null; + } + const cells: string[] = []; + let cell = ""; + for (let at = 0; at < text.length; at += 1) { + const char = text[at]; + if (char === "\\") { + cell += char + (text[at + 1] ?? ""); + at += 1; + continue; + } + if (char === "|") { + cells.push(cell); + cell = ""; + continue; + } + cell += char; + } + cells.push(cell); + if (cells.length > 1 && text.startsWith("|")) { + cells.shift(); + } + if (cells.length > 1 && text.endsWith("|")) { + cells.pop(); + } + return cells; +} + +/** Width of a GFM delimiter row such as `| --- |:-:|`, or null if not one. */ +function delimiterWidth(text: string): number | null { + const cells = tableCells(text); + if (cells === null || cells.length === 0) { + return null; + } + return cells.every((cell) => TABLE_DELIMITER_CELL.test(cell.trim())) + ? cells.length + : null; +} + +/** + * Line indices that belong to a GFM table. A table needs a header row and a + * delimiter row of the same width, and runs to a blank line or another block. Its + * cells render as a grid, not prose, so the preview drops them like a code block. + */ +function opensTable( + header: ContentLine | undefined, + delimiter: ContentLine | undefined, +): boolean { + if (header === undefined || delimiter === undefined) { + return false; + } + if (!header.text || header.quoted) { + return false; + } + if (header.indent - header.column >= INDENTED_CODE_INDENT) { + return false; + } + const width = delimiterWidth(delimiter.text); + const cells = tableCells(header.text); + return width !== null && cells !== null && cells.length === width; +} + +/** A blank line, a heading or a list marker: where GFM breaks a table. */ +function breaksTable(line: ContentLine | undefined): boolean { + return ( + !line?.text || + line.quoted || + HEADING.test(line.text) || + BULLET.test(line.text) || + line.indent - line.column >= INDENTED_CODE_INDENT + ); +} + +function tableLines(lines: ContentLine[]): Set<number> { + const rows = new Set<number>(); + let at = 0; + while (at + 1 < lines.length) { + if (!opensTable(lines[at], lines[at + 1])) { + at += 1; + continue; + } + rows.add(at); + rows.add(at + 1); + let row = at + 2; + while (row < lines.length && !breaksTable(lines[row])) { + rows.add(row); + row += 1; + } + at = row; + } + return rows; +} + +/** A backtick fence's info string may not contain a backtick. */ +function opensFence(marker: string, rest: string): boolean { + return marker[0] !== "`" || !rest.includes("`"); +} + +function nextFence( + open: string | null, + marker: string, + rest: string, +): string | null { + if (open === null) { + return opensFence(marker, rest) ? marker : null; + } + const closes = + marker[0] === open[0] && + marker.length >= open.length && + // Only spaces or tabs may follow a closer, per CommonMark. + !NON_SPACE.test(rest); + return closes ? null : open; +} + +/** Whether a fence, a raw block, a comment or an HTML block is open. */ +function inBlock(state: ScanState): boolean { + return ( + state.openFence !== null || + state.inRawHtml !== null || + state.inHtmlBlock || + state.inComment + ); +} + +/** + * A fence, comment or HTML block inside a list item runs only to the end of that + * item, so a line dedented out of the item closes both. Lazy continuation reaches + * into none of them, so any content left of the item ends it. + */ +function closeDedentedBlock(line: string, state: ScanState): void { + if (state.blockColumn === 0 || !inBlock(state)) { + return; + } + if (line.trim() && indentWidth(line) < state.blockColumn) { + state.openFence = null; + state.inRawHtml = null; + state.inHtmlBlock = false; + state.inComment = false; + state.blockColumn = 0; + } +} + +/** Ties a block just opened to the list item it is written inside. */ +function scopeBlock( + state: ScanState, + wasInBlock: boolean, + lists: ListState, +): void { + if (!inBlock(state)) { + state.blockColumn = 0; + return; + } + if (!wasInBlock) { + // The opener closed the items it is dedented out of first, so this is the + // column of the item the block really sits in. + state.blockColumn = lists.columns.at(-1) ?? 0; + } +} + +function contentLines(markdown: string): ContentLine[] { + const lines: ContentLine[] = []; + const state: ScanState = { + openFence: null, + blockColumn: 0, + inComment: false, + runOn: false, + inRawHtml: null, + inHtmlBlock: false, + afterParagraph: false, + }; + let lists: ListState = EMPTY_LIST_STATE; + let quote: QuoteState = NO_QUOTE; + + const rawLines = markdown + .split("\n") + .map((raw) => raw.replace(TABS, " ".repeat(TAB_WIDTH))); + const closesBelow = commentClosesBelow(rawLines); + for (const [index, line] of rawLines.entries()) { + closeDedentedBlock(line, state); + const wasInBlock = inBlock(state); + const carried = state.runOn; + const { text: visible, structural } = visibleText( + line, + state, + closesBelow[index + 1] ?? false, + ); + // The quote state from the line above, which is what list tracking asks about. + // Only a line of text below rewrites it, so a fenced, blank or hidden line + // leaves no quoted paragraph open behind it. + const above = quote; + quote = NO_QUOTE; + // Taken with the paragraph state from the line above, as a renderer would. + lists = openLists(structural, lists, state.afterParagraph, above.quoted); + scopeBlock(state, wasInBlock, lists); + if (visible === null) { + continue; + } + if (carried && !visible.trim()) { + // Wholly inside a comment its paragraph carries: no text, and no break. + continue; + } + if (!visible.trim() || THEMATIC_BREAK.test(visible)) { + // A rule separates notes, so it breaks a bullet just like a blank line. + state.afterParagraph = false; + lines.push({ text: "", indent: 0, quoted: false, column: 0 }); + continue; + } + const quoted = BLOCKQUOTE.test(visible); + const stripped = visible.replace(BLOCKQUOTE, ""); + const indent = stripped.length - stripped.trimStart().length; + // A quoted line is measured inside its quote, where the document's open + // list items do not reach. + const column = quoted ? 0 : (lists.columns.at(-1) ?? 0); + // Only ordinary text continues a paragraph; a heading or indented code line + // (four columns past its container, outside a paragraph) ends one. + const startsCode = + !state.afterParagraph && indent - column >= INDENTED_CODE_INDENT; + state.afterParagraph = !HEADING_LINE.test(stripped) && !startsCode; + quote = quoteState(visible, above.inQuote); + lines.push({ text: stripped.trim(), indent, quoted, column }); + } + return lines; +} + +/** + * Split a bullet at its first sentence boundary. Conservative: the next + * sentence must start like one, so "CHANGELOG.md in the repo" is not a break. + */ +function splitLeadSentence(text: string): ReleaseNotesPreviewItem { + SENTENCE_BREAK.lastIndex = 0; + let match = SENTENCE_BREAK.exec(text); + while (match) { + const cut = match.index + 1; + const word = + TRAILING_WORD.exec(text.slice(0, cut))?.[1]?.toLowerCase() ?? ""; + const isAbbreviation = ABBREVIATIONS.has(word) || INITIAL.test(word); + if (!isAbbreviation && cut >= MIN_LEAD_CHARS) { + return { lead: text.slice(0, cut).trim(), rest: text.slice(cut).trim() }; + } + match = SENTENCE_BREAK.exec(text); + } + return { lead: text, rest: "" }; +} + +/** Bullets in document order, plus prose for changelogs written as paragraphs. */ +interface Collector { + bullets: Bullet[]; + prose: string[]; + // Wrapped bullets continue on following lines and belong to one item. + current: Bullet | null; + paragraph: string; + // True while the open paragraph is a quote's, which owns its own text: a + // marker written outside the quote opens a list rather than continuing it. + quotedParagraph: boolean; +} + +function flush(collector: Collector): void { + if (collector.current?.text) { + collector.bullets.push({ + text: truncate(collector.current.text), + indent: collector.current.indent, + }); + } + collector.current = null; + if (collector.paragraph) { + collector.prose.push(truncate(collector.paragraph)); + collector.paragraph = ""; + } + collector.quotedParagraph = false; +} + +function takeBullet( + collector: Collector, + text: string, + line: ContentLine, + labels: Set<string>, +): void { + flush(collector); + const item = toPlainText(text, labels); + // A quoted list is example output: prose at best, never a headline bullet. + if (!line.quoted) { + collector.current = { text: item, indent: line.indent }; + } else if (item) { + collector.prose.push(truncate(item)); + } +} + +function takeText( + collector: Collector, + text: string, + labels: Set<string>, + quoted: boolean, +): void { + const plain = toPlainText(text, labels); + if (!plain) { + return; + } + if (collector.current === null) { + // Wrapped paragraphs render as one block, so preview them as one item. + collector.paragraph = collector.paragraph + ? `${collector.paragraph} ${plain}` + : plain; + collector.quotedParagraph = quoted; + return; + } + collector.current = { + text: `${collector.current.text} ${plain}`, + indent: collector.current.indent, + }; +} + +function collectBullets(markdown: string): { + bullets: Bullet[]; + prose: string[]; +} { + const collector: Collector = { + bullets: [], + prose: [], + current: null, + paragraph: "", + quotedParagraph: false, + }; + + const lines = contentLines(markdown); + const labels = new Set<string>(); + // Skips the same code the pass below skips: a definition-shaped line inside + // code is literal, and a real definition never indents past three spaces. + let labelFence: string | null = null; + let labelColumn = 0; + for (const line of lines) { + if (labelFence !== null && !endsDeepFence(labelFence, labelColumn, line)) { + continue; + } + if (labelFence !== null) { + const dedented = line.indent < labelColumn; + labelFence = null; + // Its own closing line is code too; only a dedented one is a new block. + if (!dedented) { + continue; + } + } + const opener = opensDeepFence(line); + if (opener !== null) { + labelFence = opener; + labelColumn = line.column; + continue; + } + if (line.indent - line.column >= INDENTED_CODE_INDENT) { + continue; + } + const definition = DEFINITION.exec(line.text); + if (definition) { + labels.add( + (definition[1] ?? "").trim().replace(WHITESPACE, " ").toLowerCase(), + ); + } + } + + const tables = tableLines(lines); + let deepFence: string | null = null; + let deepColumn = 0; + for (const [index, line] of lines.entries()) { + if (!line.text || HEADING.test(line.text)) { + flush(collector); + continue; + } + // A table renders as a grid, no more previewable than a code block, and it + // ends whatever came before it. + if (tables.has(index)) { + flush(collector); + continue; + } + // A link reference definition renders as nothing. + if (collector.current === null && DEFINITION.test(line.text)) { + continue; + } + // A fence indented past three spaces belongs to a list item, so the line + // scanner missed it. Its contents are code either way. + if (deepFence !== null && !endsDeepFence(deepFence, deepColumn, line)) { + continue; + } + if (deepFence !== null) { + const dedented = line.indent < deepColumn; + deepFence = null; + // Its own closing line is code too; only a dedented one is a new block. + if (!dedented) { + continue; + } + } + const opener = opensDeepFence(line); + if (opener !== null) { + deepFence = opener; + deepColumn = line.column; + continue; + } + // An indented code block renders as code, so a "- cmd" line in one is not + // a bullet. Inside an open bullet or paragraph it is just a wrapped line. + const insideBlock = + collector.current !== null || collector.paragraph !== ""; + if (!insideBlock && line.indent - line.column >= INDENTED_CODE_INDENT) { + continue; + } + const bullet = BULLET.exec(line.text); + // Only an ordered list starting at 1 may interrupt a paragraph, so "2. Restart + // Studio" under prose is prose. A list item is not a paragraph. + const interrupts = + collector.current === null && + collector.paragraph !== "" && + !collector.quotedParagraph; + if ( + bullet && + !(interrupts && bullet[1] !== undefined && bullet[1] !== "1") + ) { + takeBullet(collector, bullet[2] ?? "", line, labels); + continue; + } + takeText(collector, line.text, labels, line.quoted); + } + flush(collector); + + return { bullets: collector.bullets, prose: collector.prose }; +} + +/** + * Top-level bullets of a release section, in document order. Nested bullets are + * detail and are skipped; prose is used when a release has no bullets. + */ +export function releaseNotesPreview( + markdown: string | null | undefined, + limit: number = RELEASE_NOTES_PREVIEW_ITEMS, +): ReleaseNotesPreview { + if (!markdown) { + return { items: [], remaining: 0 }; + } + + // The updater body arrives with CRLF; sentinels would collide with parking. + const text = markdown.replace(LINE_ENDINGS, "\n").replace(SENTINELS, ""); + const { bullets, prose } = collectBullets(text); + // Shallowest bullet defines top level, so a uniformly indented list previews. + const baseIndent = bullets.reduce( + (min, bullet) => Math.min(min, bullet.indent), + Number.POSITIVE_INFINITY, + ); + const topLevel = bullets + .filter((bullet) => bullet.indent <= baseIndent + NESTED_INDENT_TOLERANCE) + .map((bullet) => bullet.text); + + const source = topLevel.length > 0 ? topLevel : prose; + return { + items: source.slice(0, limit).map(splitLeadSentence), + remaining: Math.max(source.length - limit, 0), + }; +} diff --git a/studio/frontend/tests/sidebar-width.test.ts b/studio/frontend/tests/sidebar-width.test.ts new file mode 100644 index 0000000000..861bd45e3f --- /dev/null +++ b/studio/frontend/tests/sidebar-width.test.ts @@ -0,0 +1,72 @@ +// SPDX-License-Identifier: AGPL-3.0-only +// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 + +import assert from "node:assert/strict"; +import test from "node:test"; +import { readFile } from "node:fs/promises"; + +// Every localStorage key written by a panel width store. +const PANEL_WIDTH_KEYS = ["sidebar_width", "chat_settings_width"]; + +// The store reads window at import time, so stub it before importing. +const stubWindow = { + innerWidth: 1440, + localStorage: { + getItem: () => null, + setItem: () => {}, + }, + addEventListener: () => {}, + removeEventListener: () => {}, +}; +(globalThis as { window?: unknown }).window = stubWindow; + +const { + clampSidebarWidth, + SIDEBAR_WIDTH_DEFAULT, + SIDEBAR_WIDTH_MAX, + SIDEBAR_WIDTH_MIN, +} = await import("../src/hooks/use-sidebar-width.ts"); + +test("clamps to the absolute range on a roomy window", () => { + stubWindow.innerWidth = 1440; + assert.equal(clampSidebarWidth(320), 320); + assert.equal(clampSidebarWidth(SIDEBAR_WIDTH_MAX + 200), SIDEBAR_WIDTH_MAX); + assert.equal(clampSidebarWidth(10), SIDEBAR_WIDTH_MIN); + assert.equal(clampSidebarWidth(Number.NaN), SIDEBAR_WIDTH_DEFAULT); +}); + +test("caps at 40% of a narrow window", () => { + stubWindow.innerWidth = 800; + assert.equal(clampSidebarWidth(SIDEBAR_WIDTH_MAX), 320); + assert.equal(clampSidebarWidth(300), 300); +}); + +test("the floor still wins when 40% falls below it", () => { + stubWindow.innerWidth = 500; + assert.equal(clampSidebarWidth(SIDEBAR_WIDTH_MAX), SIDEBAR_WIDTH_MIN); +}); + +test("re-evaluates the cap per call, so a resize can re-clamp", () => { + stubWindow.innerWidth = 1440; + assert.equal(clampSidebarWidth(SIDEBAR_WIDTH_MAX), SIDEBAR_WIDTH_MAX); + stubWindow.innerWidth = 900; + assert.equal(clampSidebarWidth(SIDEBAR_WIDTH_MAX), 360); + stubWindow.innerWidth = 1440; + assert.equal(clampSidebarWidth(SIDEBAR_WIDTH_MAX), SIDEBAR_WIDTH_MAX); +}); + +// The reset action promises to clear every stored preference, so a persisted +// panel width that is missing from the list survives the reload. +test("persisted panel widths are cleared by the preference reset", async () => { + const source = await readFile( + new URL("../src/features/settings/tabs/general-tab.tsx", import.meta.url), + "utf8", + ); + const keys = source.slice( + source.indexOf("const PREFS_KEYS"), + source.indexOf("];", source.indexOf("const PREFS_KEYS")), + ); + for (const key of PANEL_WIDTH_KEYS) { + assert.ok(keys.includes(`"${key}"`), `${key} missing from PREFS_KEYS`); + } +}); diff --git a/studio/install_node_prebuilt.py b/studio/install_node_prebuilt.py index 82ca1d2c68..1f42729c80 100644 --- a/studio/install_node_prebuilt.py +++ b/studio/install_node_prebuilt.py @@ -707,18 +707,55 @@ def existing_install_usable(install_dir: Path, host: HostInfo) -> bool: return npm_major is not None and npm_major >= NPM_MIN_MAJOR +def _replace_with_retry( + src: Path, + dst: Path, + *, + attempts: int = 8, +) -> None: + """os.replace, retried against transient Windows sharing violations. + + A directory rename fails with WinError 5/32 while any process holds a handle inside + it, and Defender or the indexer routinely does right after extraction (seen in CI on + a fresh install, with no existing directory to conflict with). Handles clear in a + second or two, so a bounded backoff turns the failure into a pause; other errors + raise immediately rather than stalling on a real problem. + """ + delay = 0.25 + for attempt in range(attempts): + try: + os.replace(src, dst) + return + except OSError as exc: + transient = os.name == "nt" and getattr(exc, "winerror", None) in (5, 32, 145) + if not transient or attempt == attempts - 1: + raise + log( + f"rename blocked ({exc.winerror}), retrying in {delay:.2f}s " + f"-- a scanner is likely still holding the extracted files" + ) + time.sleep(delay) + delay = min(delay * 2, 4.0) + + def _swap_into_place(extracted_root: Path, install_dir: Path) -> None: """Atomically replace install_dir with extracted_root (same filesystem).""" install_dir.parent.mkdir(parents = True, exist_ok = True) backup: Path | None = None if install_dir.exists(): backup = install_dir.parent / f".{install_dir.name}.old-{os.getpid()}" - os.replace(install_dir, backup) + _replace_with_retry(install_dir, backup) try: - os.replace(extracted_root, install_dir) + _replace_with_retry(extracted_root, install_dir) except OSError: + # The forward rename retries ~16s, ample time for a scanner to grab the backup too. + # A plain os.replace would then raise over the original error and leave no + # install_dir at all, so the rollback gets the same backoff and never masks it. if backup is not None and not install_dir.exists(): - os.replace(backup, install_dir) + try: + _replace_with_retry(backup, install_dir) + except OSError as rollback_exc: + log(f"could not restore the previous Node install from {backup}: {rollback_exc}") raise if backup is not None: shutil.rmtree(backup, ignore_errors = True) diff --git a/studio/setup.ps1 b/studio/setup.ps1 index fba0e0d0f6..34ef82ee4b 100644 --- a/studio/setup.ps1 +++ b/studio/setup.ps1 @@ -869,12 +869,22 @@ function Ensure-BuildToolsForLlamaSourceBuild { } } -# Detect the VC++ 2015-2022 Redistributable that the prebuilt llama-server and -# PyTorch need (they link VCRUNTIME140_1.dll etc., which the Universal CRT lacks). -# Signal is System32\vcruntime140_1.dll (VS 2019+), registry as fallback. +# Machine arch: PROCESSOR_ARCHITECTURE describes this PROCESS, so an emulated x64 shell on +# ARM64 reports AMD64; PROCESSOR_ARCHITEW6432 is ARM64 in exactly that case. +function Get-HostMachineArch { + $osArch = "" + try { $osArch = [System.Runtime.InteropServices.RuntimeInformation]::OSArchitecture.ToString() } catch { } + foreach ($s in @([string]$env:PROCESSOR_ARCHITEW6432, [string]$env:PROCESSOR_ARCHITECTURE, $osArch)) { + if ($s.ToLowerInvariant() -eq "arm64") { return "arm64" } + } + return "other" +} + +# Detect the VC++ 2015-2022 Redistributable prebuilt llama-server and PyTorch need (they +# link VCRUNTIME140_1.dll, absent from the Universal CRT). Registry first: Runtimes\x64 is +# the only x64-specific proof; System32\vcruntime140_1.dll is arch-blind and on ARM64 may +# be the ARM64-only package, unloadable under x64 emulation. function Test-VCRedistInstalled { - $sys = $env:SystemRoot - if ($sys -and (Test-Path (Join-Path $sys 'System32\vcruntime140_1.dll'))) { return $true } foreach ($k in @( 'HKLM:\SOFTWARE\Microsoft\VisualStudio\14.0\VC\Runtimes\x64', 'HKLM:\SOFTWARE\WOW6432Node\Microsoft\VisualStudio\14.0\VC\Runtimes\x64' @@ -884,10 +894,14 @@ function Test-VCRedistInstalled { if ($r.Installed -eq 1 -and [int]$r.Major -ge 14 -and [int]$r.Minor -ge 20) { return $true } } catch { } } + if ((Get-HostMachineArch) -eq "arm64") { return $false } + $sys = $env:SystemRoot + if ($sys -and (Test-Path (Join-Path $sys 'System32\vcruntime140_1.dll'))) { return $true } return $false } -# Install the VC++ 2015-2022 runtime if missing (non-fatal; usually a no-op). +# Install the VC++ 2015-2022 runtime if missing (non-fatal; usually a no-op). Unlike CMake +# and Build Tools torch cannot import without it, and winget is absent on LTSC/Server images. function Ensure-VCRedist { if (Test-VCRedistInstalled) { step "vcredist" "present"; return } Write-Host "Microsoft Visual C++ Redistributable (2015-2022) is missing; the prebuilt llama.cpp and PyTorch need it. Installing the runtime..." -ForegroundColor Yellow @@ -897,6 +911,45 @@ function Ensure-VCRedist { Refresh-Environment } catch { substep "VCRedist install failed: $($_.Exception.Message)" "Yellow" } } + if (-not (Test-VCRedistInstalled)) { + # Evergreen link; /quiet /norestart so it never blocks or reboots an unattended run. + # Always the x64 package, deliberately: Microsoft ships it as the Arm64X superset of + # both ARM64 and X64 binaries and documents it as the one for ARM64 devices, while + # the arm64 package is ARM64-only (learn.microsoft.com/cpp/windows/latest-supported-vc-redist). + # PROCESSOR_ARCHITECTURE is wrong twice here: it reports the process, and the runtime + # must match the interpreter loading the DLLs, an emulated x64 Python not yet created. + $url = "https://aka.ms/vs/17/release/vc_redist.x64.exe" + $dst = Join-Path ([System.IO.Path]::GetTempPath()) "vc_redist.x64.exe" + substep "winget unavailable or failed; downloading the runtime directly..." + # Windows PowerShell 5.1 on an old image can carry a .NET default protocol set that + # predates TLS 1.2, which aka.ms refuses -- exactly the no-winget host this fallback + # exists for. SystemDefault (0) means "let the OS choose" and already covers TLS 1.2+, + # so only an explicit legacy set is upgraded, and it is restored afterwards. + $_prevProtocol = $null + try { + $_cur = [System.Net.ServicePointManager]::SecurityProtocol + if ([int]$_cur -ne 0 -and ([int]$_cur -band [int][System.Net.SecurityProtocolType]::Tls12) -eq 0) { + [System.Net.ServicePointManager]::SecurityProtocol = $_cur -bor [System.Net.SecurityProtocolType]::Tls12 + $_prevProtocol = $_cur + } + } catch { $_prevProtocol = $null } + try { + Invoke-WebRequest -Uri $url -OutFile $dst -UseBasicParsing -TimeoutSec 300 + $p = Start-Process -FilePath $dst -ArgumentList '/quiet', '/norestart' -Wait -PassThru + # 3010 = success, reboot required; usable either way. + if ($p.ExitCode -notin @(0, 3010)) { + substep "VC++ runtime installer exited $($p.ExitCode)" "Yellow" + } + Refresh-Environment + } catch { + substep "Direct VC++ runtime download failed: $($_.Exception.Message)" "Yellow" + } finally { + if ($null -ne $_prevProtocol) { + try { [System.Net.ServicePointManager]::SecurityProtocol = $_prevProtocol } catch { } + } + Remove-Item -LiteralPath $dst -Force -ErrorAction SilentlyContinue + } + } if (Test-VCRedistInstalled) { step "vcredist" "installed" } else { substep "Could not install the VC++ Redistributable automatically." "Yellow" @@ -1650,11 +1703,42 @@ if ($LongPathsEnabled) { } # ============================================ -# 1b. Git (required by pip for git+https:// deps and by npm) +# 1b. Git (only required for --local / source installs) # ============================================ +# Was fatal as "required by pip and npm", but the consumer path uses neither: the +# unsloth-zoo git+https URL is STUDIO_LOCAL_INSTALL only, node is a pinned prebuilt, and the +# frontend lockfile has no VCS deps. Being fatal blocked clean no-winget Windows boxes. $HasGit = $null -ne (Get-Command git -ErrorAction SilentlyContinue) if (-not $HasGit) { - Write-Host "Git not found -- installing via winget..." -ForegroundColor Yellow + # Fatal only where git is used: --local and the opt-in llama.cpp source build. A local + # llama.cpp dir overrides those opt-ins, but only once it holds a reusable binary: + # pointing at the canonical install location with nothing built there falls through to + # the normal install, so an explicit source build still needs git. The automatic + # fallback after a failed prebuilt download is not knowable here; Phase 4 handles it. + $gitNeeded = ($env:STUDIO_LOCAL_INSTALL -eq '1') + $_localLlamaDir = if ($env:UNSLOTH_LOCAL_LLAMA_CPP_DIR) { $env:UNSLOTH_LOCAL_LLAMA_CPP_DIR.Trim() } else { "" } + $_localLlamaBuilt = $false + if ($_localLlamaDir) { + # Same layout candidates as the reuse check in Phase 4. + foreach ($_c in @("llama-server.exe", "build\bin\llama-server.exe", "build\bin\Release\llama-server.exe")) { + if (Test-Path -LiteralPath (Join-Path $_localLlamaDir $_c)) { $_localLlamaBuilt = $true; break } + } + } + if (-not $_localLlamaBuilt) { + $_prForce = if ($env:UNSLOTH_LLAMA_PR_FORCE) { $env:UNSLOTH_LLAMA_PR_FORCE.Trim() } else { $DefaultLlamaPrForce } + $_llamaSrc = $DefaultLlamaSource -replace '\.git$', '' + # Same tag resolution as Phase 4. "master" is a branch, never a release, so the + # prebuilt lookup always misses and Phase 4 rebuilds it from source. + $_llamaTag = if ($env:UNSLOTH_LLAMA_TAG) { $env:UNSLOTH_LLAMA_TAG } else { $DefaultLlamaTag } + if ($_llamaTag -eq "master") { $gitNeeded = $true } + if ($env:UNSLOTH_LLAMA_FORCE_COMPILE -eq '1') { $gitNeeded = $true } + if (-not [string]::IsNullOrWhiteSpace($env:UNSLOTH_LLAMA_PR)) { $gitNeeded = $true } + # Same positive-integer predicate as the PR_FORCE promotion below: 0 or non-numeric + # never forces a source build, so it must not demand git. + if ($_prForce -match '^\d+$' -and [int]$_prForce -gt 0) { $gitNeeded = $true } + if ($_llamaSrc -ne "https://github.com/ggml-org/llama.cpp") { $gitNeeded = $true } + } + Write-Host "Git not found -- attempting install via winget..." -ForegroundColor Yellow $HasWinget = $null -ne (Get-Command winget -ErrorAction SilentlyContinue) if ($HasWinget) { try { @@ -1664,11 +1748,18 @@ if (-not $HasGit) { } catch { } } if (-not $HasGit) { - Write-Host "[ERROR] Git is required but could not be installed automatically." -ForegroundColor Red - Write-Host " Install Git from https://git-scm.com/download/win and re-run." -ForegroundColor Red - Exit-SetupFailure "Git is required but could not be installed automatically" + if ($gitNeeded) { + Write-Host "[ERROR] Git is required for --local and llama.cpp source-build installs but could not be installed." -ForegroundColor Red + Write-Host " --local clones unsloth-zoo, and a source build clones llama.cpp." -ForegroundColor Red + Write-Host " Install Git from https://git-scm.com/download/win and re-run." -ForegroundColor Red + Exit-SetupFailure "Git is required for --local / source-build installs but could not be installed" + } + step "git" "not found (not required)" "Yellow" + substep "Unsloth installs prebuilt binaries and wheels, so git is not needed." + substep "Install it only for --local/source installs: https://git-scm.com/download/win" + } else { + step "git" "$(git --version)" } - step "git" "$(git --version)" } else { step "git" "$(git --version)" } @@ -3275,18 +3366,32 @@ $PyTorchWhlBase = if ($env:UNSLOTH_PYTORCH_MIRROR) { $env:UNSLOTH_PYTORCH_MIRROR $TorchInstallIndexUrl = if ($ROCmIndexUrl) { "$PyTorchWhlBase/cpu" } elseif ($PinnedTorchIndexUrl) { $PinnedTorchIndexUrl } else { "$PyTorchWhlBase/$CuTag" } if (-not $NoTorchMode) { +# Windows on ARM has win_arm64 torch and torchvision wheels but no torchaudio on any index, +# so every branch below drops it. Ask the interpreter uv resolves for, not +# PROCESSOR_ARCHITECTURE, which describes the host process. Inside the no-torch guard +# because all three uses are, and no-torch installs nothing to skip. +$_setupPlatform = "" +try { + $_setupPlatform = (& python -c "import sysconfig; print(sysconfig.get_platform())" 2>$null | Out-String).Trim().ToLowerInvariant() +} catch { $_setupPlatform = "" } +$WinArm64NoAudio = ($_setupPlatform -eq "win-arm64") +if ($WinArm64NoAudio) { substep "windows on arm: skipping torchaudio (no win_arm64 wheel upstream)" } + $ROCmCpuFallback = $false if ($ROCmIndexUrl) { substep "installing PyTorch (AMD ROCm, $ROCmGfxArch)..." if ($ROCmTorchSpec -ne "torch") { substep " enforcing $ROCmTorchSpec $ROCmVisionSpec $ROCmAudioSpec (known _grouped_mm bug in older wheels)" "Cyan" } + # Built above the verbose branch: a splat assigned inside it is unset on the other. + $_rocmTrio = @($ROCmTorchSpec, $ROCmVisionSpec, $ROCmAudioSpec) + if ($WinArm64NoAudio) { $_rocmTrio = @($ROCmTorchSpec, $ROCmVisionSpec) } if ($script:UnslothVerbose) { - Fast-Install $ROCmTorchSpec $ROCmVisionSpec $ROCmAudioSpec --force-reinstall --index-url $ROCmIndexUrl | ForEach-Object { Redact-InstallOutput "$_" } | Out-Host + Fast-Install @_rocmTrio --force-reinstall --index-url $ROCmIndexUrl | ForEach-Object { Redact-InstallOutput "$_" } | Out-Host $torchInstallExit = $LASTEXITCODE $output = "" } else { - $output = Fast-Install $ROCmTorchSpec $ROCmVisionSpec $ROCmAudioSpec --force-reinstall --index-url $ROCmIndexUrl | Out-String + $output = Fast-Install @_rocmTrio --force-reinstall --index-url $ROCmIndexUrl | Out-String $torchInstallExit = $LASTEXITCODE } if ($torchInstallExit -ne 0) { @@ -3322,12 +3427,14 @@ if (-not $ROCmIndexUrl -and ($CuTag -eq "cpu" -or $ROCmCpuFallback)) { $cpuVisionSpec = "torchvision>=0.19,<0.27.0" $cpuAudioSpec = "torchaudio>=2.4,<2.12.0" } + $_torchTrio = @($cpuTorchSpec, $cpuVisionSpec, $cpuAudioSpec) + if ($WinArm64NoAudio) { $_torchTrio = @($cpuTorchSpec, $cpuVisionSpec) } if ($script:UnslothVerbose) { - Fast-Install $cpuTorchSpec $cpuVisionSpec $cpuAudioSpec @cpuForce --index-url $TorchInstallIndexUrl | ForEach-Object { Redact-InstallOutput "$_" } | Out-Host + Fast-Install @_torchTrio @cpuForce --index-url $TorchInstallIndexUrl | ForEach-Object { Redact-InstallOutput "$_" } | Out-Host $torchInstallExit = $LASTEXITCODE $output = "" } else { - $output = Fast-Install $cpuTorchSpec $cpuVisionSpec $cpuAudioSpec @cpuForce --index-url $TorchInstallIndexUrl | Out-String + $output = Fast-Install @_torchTrio @cpuForce --index-url $TorchInstallIndexUrl | Out-String $torchInstallExit = $LASTEXITCODE } if ($torchInstallExit -ne 0) { @@ -3354,12 +3461,16 @@ if (-not $ROCmIndexUrl -and ($CuTag -eq "cpu" -or $ROCmCpuFallback)) { $cudaVisionSpec = "torchvision>=0.19,<0.26.0" $cudaAudioSpec = "torchaudio>=2.4,<2.11.0" } + # A custom pin whose leaf is not cpu (a corporate /simple mirror) lands an ARM64 host + # here, so this branch drops torchaudio too. + $_cudaTrio = @($cudaTorchSpec, $cudaVisionSpec, $cudaAudioSpec) + if ($WinArm64NoAudio) { $_cudaTrio = @($cudaTorchSpec, $cudaVisionSpec) } if ($script:UnslothVerbose) { - Fast-Install $cudaTorchSpec $cudaVisionSpec $cudaAudioSpec @cudaForce --index-url $TorchInstallIndexUrl | ForEach-Object { Redact-InstallOutput "$_" } | Out-Host + Fast-Install @_cudaTrio @cudaForce --index-url $TorchInstallIndexUrl | ForEach-Object { Redact-InstallOutput "$_" } | Out-Host $torchInstallExit = $LASTEXITCODE $output = "" } else { - $output = Fast-Install $cudaTorchSpec $cudaVisionSpec $cudaAudioSpec @cudaForce --index-url $TorchInstallIndexUrl | Out-String + $output = Fast-Install @_cudaTrio @cudaForce --index-url $TorchInstallIndexUrl | Out-String $torchInstallExit = $LASTEXITCODE } if ($torchInstallExit -ne 0) { @@ -4107,6 +4218,7 @@ $BuildDir = Join-Path $LlamaCppDir "build" $LlamaServerBin = Join-Path $BuildDir "bin\Release\llama-server.exe" $HasCmakeForBuild = $null -ne (Get-Command cmake -ErrorAction SilentlyContinue) +$HasGitForBuild = $null -ne (Get-Command git -ErrorAction SilentlyContinue) # Check if existing llama-server matches current GPU mode. A CUDA-built binary # on a now-CPU-only machine (or vice versa) needs to be rebuilt. @@ -4132,9 +4244,27 @@ if (Test-Path -LiteralPath $LlamaServerBin) { $WillBuildLlamaFromSource = $NeedLlamaSourceBuild -and ` -not ((Test-Path -LiteralPath $LlamaServerBin) -and -not $NeedRebuild -and $RequestedLlamaTag -ne "master") if ($WillBuildLlamaFromSource) { - Ensure-BuildToolsForLlamaSourceBuild - # refresh so the chain below sees a newly installed cmake - $HasCmakeForBuild = $null -ne (Get-Command cmake -ErrorAction SilentlyContinue) + if (-not $HasGitForBuild) { + # Phase 1 keeps git optional, so only the automatic fallback after a failed prebuilt + # download arrives here without it. Last chance to install: Invoke-SetupCommand + # returns 0 for command-not-found, so a git-less clone misreports as a cmake failure. + if ($null -ne (Get-Command winget -ErrorAction SilentlyContinue)) { + try { + Invoke-SetupCommand { winget install Git.Git --source winget --accept-package-agreements --accept-source-agreements } | Out-Null + Refresh-Environment + } catch { } + } + $HasGitForBuild = $null -ne (Get-Command git -ErrorAction SilentlyContinue) + } + # Git first, then the toolchain: Ensure-BuildToolsForLlamaSourceBuild exits setup when + # Build Tools cannot be installed, so running it first made the degraded path below + # unreachable on a no-winget box, and elsewhere spent a multi-GB download on a clone + # that cannot happen. + if ($HasGitForBuild) { + Ensure-BuildToolsForLlamaSourceBuild + # refresh so the chain below sees a newly installed cmake + $HasCmakeForBuild = $null -ne (Get-Command cmake -ErrorAction SilentlyContinue) + } } if ($LocalLlamaCppLinked) { @@ -4152,6 +4282,16 @@ if ($LocalLlamaCppLinked) { # up new model architecture support (e.g. Gemma 4). Write-Host "" step "llama.cpp" "already built" +} elseif (-not $HasGitForBuild) { + # Before cmake: the toolchain install is skipped without git, so cmake may be missing + # purely as a consequence. Degrade rather than abort; the opt-in source triggers already + # required git in Phase 1, so only the automatic fallback lands here. + Write-Host "" + step "llama.cpp" "build skipped (git not available)" "Yellow" + substep "The prebuilt download failed and a source build clones llama.cpp." "Yellow" + substep "GGUF inference and export will not be available." "Yellow" + substep "Install Git from https://git-scm.com/download/win and re-run setup." "Yellow" + $script:LlamaCppDegraded = $true } elseif (-not $HasCmakeForBuild) { Write-Host "" if (-not $HasNvidiaSmi) { diff --git a/studio/src-tauri/src/desktop_update_policy.rs b/studio/src-tauri/src/desktop_update_policy.rs index c83f847eda..d186c2d01d 100644 --- a/studio/src-tauri/src/desktop_update_policy.rs +++ b/studio/src-tauri/src/desktop_update_policy.rs @@ -27,6 +27,8 @@ pub(crate) struct DesktopUpdatePolicy { pub(crate) struct ManualUpdateInfo { version: String, current_version: String, + // Backend release this desktop build pins; CHANGELOG.md is keyed by it. + pypi_version: Option<String>, body: Option<String>, date: Option<String>, } @@ -34,8 +36,12 @@ pub(crate) struct ManualUpdateInfo { #[derive(Debug, serde::Deserialize)] struct ChannelMetadata { version: String, - body: Option<String>, - date: Option<String>, + // latest.json publishes Tauri's `notes`/`pub_date`; aliases keep older metadata working. + pypi_version: Option<String>, + #[serde(alias = "body")] + notes: Option<String>, + #[serde(alias = "date")] + pub_date: Option<String>, platforms: HashMap<String, ChannelPlatform>, } @@ -99,8 +105,9 @@ pub(crate) async fn check_desktop_manual_update() -> Result<Option<ManualUpdateI Ok(Some(ManualUpdateInfo { version: latest_version, current_version: current_version.to_string(), - body: metadata.body, - date: metadata.date, + pypi_version: metadata.pypi_version, + body: metadata.notes, + date: metadata.pub_date, })) } diff --git a/tests/python/test_bitsandbytes_kernel_readiness.py b/tests/python/test_bitsandbytes_kernel_readiness.py new file mode 100644 index 0000000000..fe595f150d --- /dev/null +++ b/tests/python/test_bitsandbytes_kernel_readiness.py @@ -0,0 +1,177 @@ +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. + +"""`ALLOW_BITSANDBYTES` must follow the kernels, not the mere presence of the module. + +From bitsandbytes 0.46 a wheel whose native library never loaded still imports and +resolves every ctypes handle to a `throw_on_call` closure, so a probe made of attribute +reads alone sees a healthy wheel, the loader selects a 4bit checkpoint, and the failure +lands inside a kernel mid-run instead of degrading to 16bit. +""" + +from __future__ import annotations + +import ast +import importlib.util +import types +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parents[2] + + +def _load_probe(): + """Import by path, not as ``unsloth.bnb_availability``, which would run the package + __init__ and pull in torch. Works only because the module is a leaf - the property + that lets device_type.py, imported very early, use it without a cycle.""" + path = REPO_ROOT / "unsloth" / "bnb_availability.py" + spec = importlib.util.spec_from_file_location("_unsloth_bnb_availability", path) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def _fake_bnb(lib): + functional = types.ModuleType("bitsandbytes.functional") + functional.get_ptr = lambda tensor: None + functional.lib = lib + bnb = types.ModuleType("bitsandbytes") + bnb.__version__ = "0.50.0" + bnb.functional = functional + return bnb + + +class _DeferredFailureLib: + """What bitsandbytes >= 0.46 hands back when the native library is dead.""" + + def __getattr__(self, name): + def throw_on_call(*args, **kwargs): + raise RuntimeError(f"Method '{name}' not available in CPU-only version") + + return throw_on_call + + +class _RealHandleLib: + """ctypes caches the function object on first lookup; its handles carry restype.""" + + def __getattr__(self, name): + def handle(*args, **kwargs): + return None + + handle.restype = None + setattr(self, name, handle) + return handle + + +def test_probe_covers_every_module_scope_ctypes_bind(): + """A probe that misses one of the import-time binds lets a dead wheel through.""" + tree = ast.parse((REPO_ROOT / "unsloth" / "kernels" / "utils.py").read_text(encoding = "utf-8")) + bound = { + node.attr + for node in ast.walk(tree) + if isinstance(node, ast.Attribute) + and isinstance(node.value, ast.Attribute) + and node.value.attr == "lib" + } + probe = _load_probe() + xpu = set(probe.bitsandbytes_symbols("xpu")) + cuda = set(probe.bitsandbytes_symbols("cuda")) + assert bound == xpu | cuda, f"probe and module-scope binds differ: {bound ^ (xpu | cuda)}" + # xpu probes the gemv pair, every other device the naive gemm pair, never both. + assert xpu - cuda and cuda - xpu, "the device split collapsed" + + +def test_a_deferred_failure_handle_is_not_ready(): + probe = _load_probe() + bnb = _fake_bnb(_DeferredFailureLib()) + for device in ("cuda", "xpu"): + assert probe.native_kernels_ready(bnb, device) is False, device + + +def test_a_real_ctypes_handle_is_ready(): + probe = _load_probe() + bnb = _fake_bnb(_RealHandleLib()) + for device in ("cuda", "xpu"): + assert probe.native_kernels_ready(bnb, device) is True, device + + +def test_a_lib_that_never_loaded_is_not_ready(): + """bitsandbytes 0.45.5, the floor in pyproject.toml, sets ``functional.lib = None``.""" + probe = _load_probe() + assert probe.native_kernels_ready(_fake_bnb(None), "cuda") is False + + +def test_a_partially_exporting_library_is_not_ready(): + """One resolvable symbol is not enough: the same verdict gates the module-scope + binds, so a partial library would pass here and raise `AttributeError` at the bind.""" + + class _MissingOne(_RealHandleLib): + def __getattr__(self, name): + if name == "cgemm_4bit_inference_naive_bf16": + raise AttributeError(name) + return super().__getattr__(name) + + probe = _load_probe() + assert probe.native_kernels_ready(_fake_bnb(_MissingOne()), "cuda") is False + + +def test_one_dead_handle_among_live_ones_is_not_ready(): + """The realistic partial shape: the library loaded but one symbol is a closure.""" + + class _OneDeferred(_RealHandleLib): + def __getattr__(self, name): + if name == "cdequantize_blockwise_bf16_nf4": + return lambda *a, **k: None + return super().__getattr__(name) + + probe = _load_probe() + assert probe.native_kernels_ready(_fake_bnb(_OneDeferred()), "cuda") is False + + +def test_absent_bitsandbytes_is_not_ready(): + probe = _load_probe() + assert probe.native_kernels_ready(None, "cuda") is False + + +def test_device_type_gates_the_flags_on_the_kernels(): + """The flags must follow ``native_kernels_ready``, not the bare import.""" + head = (REPO_ROOT / "unsloth" / "device_type.py").read_text(encoding = "utf-8") + head = head.split('if DEVICE_TYPE == "hip":')[0] + assert "import bitsandbytes as _bnb_probe" in head + assert 'find_spec("bitsandbytes")' not in head, "find_spec cannot see a broken wheel" + assert "native_kernels_ready(_bnb_probe, DEVICE_TYPE)" in head + assert ( + head.count("ALLOW_BITSANDBYTES = False") >= 2 + ), "both the failed-import path and the dead-kernels path must clear the flag" + + +def test_the_ctypes_binds_are_gated_on_the_same_verdict(): + """Clearing the flag is not enough on its own: ``bnb is None`` alone let an + importable-but-dead wheel reach the binds, and 0.45.5 sets ``functional.lib = None`` + on a native-load failure, so they killed ``import unsloth`` outright instead of + degrading to 16bit.""" + source = (REPO_ROOT / "unsloth" / "kernels" / "utils.py").read_text(encoding = "utf-8") + assert "from ..bnb_availability import native_kernels_ready" in source + assert ( + "if bnb is None or not native_kernels_ready(bnb, DEVICE_TYPE):" in source + ), "the ctypes bind block must take the _bnb_required branch on a dead library too" + guarded = source.split("if bnb is None or not native_kernels_ready(bnb, DEVICE_TYPE):")[1] + # Anchor on the symbol, not the module alias: #7580 renamed the binding from + # `bnb.functional.lib` to `bnb_functional.lib`, which is exactly the kind of rename + # this assertion should survive. + assert "lib.cdequantize_blockwise_fp32" in guarded, "the binds must sit under that guard" + + +def test_the_kernel_check_reads_the_submodule_not_the_parent_attribute(): + """A part-initialised bitsandbytes leaves the parent without ``functional`` while + the submodule stays in sys.modules, which ``import bitsandbytes.functional`` reads + directly.""" + probe = _load_probe() + bnb = types.ModuleType("bitsandbytes") # zombie: parent has no `functional` + bnb.__version__ = "0.50.0" + import sys + + real = sys.modules.get("bitsandbytes.functional") + if real is None: + return # bitsandbytes not importable here; the fallback has nothing to read + # Falls back to the cached submodule instead of raising on the missing attribute. + assert probe.native_kernels_ready(bnb, "cuda") in (True, False) diff --git a/tests/python/test_cross_platform_parity.py b/tests/python/test_cross_platform_parity.py index b20e715ebc..06e444314b 100644 --- a/tests/python/test_cross_platform_parity.py +++ b/tests/python/test_cross_platform_parity.py @@ -454,9 +454,12 @@ class TestKnown211SetParity: "$_pinCuLeaf" not in text ), "install.ps1 must bound companions on every index (no cu-family exemption)" # The bounded companions must actually be passed to the install command. - assert re.search( - r'"torch>=2\.4,<2\.11\.0" \$_pinVisionSpec \$_pinAudioSpec --default-index \$TorchIndexUrl', - text, + # Specs are splatted, so check both halves: the list is built, and it is passed. + assert ( + '$_torchSpecs = @("torch>=2.4,<2.11.0", $_pinVisionSpec, $_pinAudioSpec)' in text + ), "install.ps1 custom-pin install must build the bounded spec list" + assert ( + "@_torchSpecs --default-index $TorchIndexUrl" in text ), "install.ps1 custom-pin install must pass the bounded companion specs to uv" def test_gfx_allowlist_matches_across_installers(self): @@ -704,9 +707,13 @@ class TestPinnedIndexClearsUvEnvParity: assert ( "if ($TorchIndexPinned -and -not (Test-CudaFamilyLeaf $CuTag)) {" in text ), "the custom-leaf trio bounds must be gated on a pinned non-cu-family leaf" + # Specs are splatted, so check both halves: the list is built, and it is passed. assert ( - "Fast-Install $cudaTorchSpec $cudaVisionSpec $cudaAudioSpec" in text - ), "setup.ps1's CUDA branch must install via the bounded spec variables" + "$_cudaTrio = @($cudaTorchSpec, $cudaVisionSpec, $cudaAudioSpec)" in text + ), "setup.ps1's CUDA branch must build the trio from the bounded spec variables" + assert ( + "Fast-Install @_cudaTrio @cudaForce" in text + ), "setup.ps1's CUDA branch must install the trio it built" def test_setup_ps1_bounds_pinned_cpu_torch(self): """setup.ps1's CPU branch must bound the trio under an explicit pin (parity with @@ -724,8 +731,11 @@ class TestPinnedIndexClearsUvEnvParity: "if ($TorchIndexPinned) {" in text ), "the CPU trio bounds must be gated on an explicit pin" assert ( - "Fast-Install $cpuTorchSpec $cpuVisionSpec $cpuAudioSpec @cpuForce" in text - ), "setup.ps1's CPU branch must install via the spec variables" + "$_torchTrio = @($cpuTorchSpec, $cpuVisionSpec, $cpuAudioSpec)" in text + ), "setup.ps1's CPU branch must build the trio from the spec variables" + assert ( + "Fast-Install @_torchTrio @cpuForce" in text + ), "setup.ps1's CPU branch must install the trio it built" # The ceilings mirror the Python repair spec exactly. stack = STACK_PY.read_text(encoding = "utf-8") spec_block = re.search(r"_CUDA_TORCH_PKG_SPEC[^(]*\(\s*(.*?)\)", stack, re.DOTALL) diff --git a/tests/python/test_torchcodec_torch_compat.py b/tests/python/test_torchcodec_torch_compat.py index 6ad16a73f4..728a51a321 100644 --- a/tests/python/test_torchcodec_torch_compat.py +++ b/tests/python/test_torchcodec_torch_compat.py @@ -11,12 +11,21 @@ import sys import types from pathlib import Path +import pytest + REPO_ROOT = Path(__file__).resolve().parents[2] PYPROJECT = REPO_ROOT / "pyproject.toml" IMPORT_FIXES_PATH = REPO_ROOT / "unsloth" / "import_fixes.py" +def _tomllib(): + if sys.version_info >= (3, 11): + import tomllib + return tomllib + return pytest.importorskip("tomli") + + def _load_import_fixes_module(): spec = importlib.util.spec_from_file_location( "unsloth_import_fixes_under_test", @@ -127,3 +136,41 @@ def test_import_fixes_loads_on_python39_syntax(): """Regression: module must import on 3.9 (postponed annotations for str | None).""" fixes = _load_import_fixes_module() assert callable(fixes._torchcodec_version_mismatch_hint) + + +def test_audio_extras_are_gated_to_platforms_with_a_torchcodec_wheel(): + """torchcodec publishes no sdist and no wheel for Linux aarch64, Windows ARM64 or + Intel Mac, so an ungated pin makes pip fail the whole install on those hosts instead + of just skipping audio -- and the cu*/rocm*/intel torch 2.10 extras pull it in. + The marker must match PLATFORM_LACKS_TORCHCODEC_WHEEL in install_python_stack.py. + """ + markers = pytest.importorskip("packaging.markers") + tomllib = _tomllib() + extras = tomllib.loads(PYPROJECT.read_text(encoding = "utf-8"))["project"][ + "optional-dependencies" + ] + audio = {n: d for n, d in extras.items() if n.startswith("audio-torch")} + assert audio, "expected audio-torch* extras" + + supported = [ + {"sys_platform": "linux", "platform_machine": "x86_64"}, + {"sys_platform": "win32", "platform_machine": "AMD64"}, + {"sys_platform": "darwin", "platform_machine": "arm64"}, + ] + unsupported = [ + {"sys_platform": "linux", "platform_machine": "aarch64"}, + {"sys_platform": "win32", "platform_machine": "ARM64"}, + {"sys_platform": "darwin", "platform_machine": "x86_64"}, + ] + for name, deps in audio.items(): + for dep in deps: + _, _, marker_text = dep.partition(";") + assert marker_text.strip(), f"{name}: {dep!r} has no marker" + marker = markers.Marker(marker_text.strip()) + env = {"python_version": "3.12"} + for case in supported: + assert marker.evaluate({**env, **case}), f"{name} must install on {case}" + for case in unsupported: + assert not marker.evaluate( + {**env, **case} + ), f"{name} has no wheel for {case} and must not be resolved there" diff --git a/tests/python/test_windows_arm64_python_choice.py b/tests/python/test_windows_arm64_python_choice.py new file mode 100644 index 0000000000..89546e7ac4 --- /dev/null +++ b/tests/python/test_windows_arm64_python_choice.py @@ -0,0 +1,143 @@ +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. + +"""Windows on ARM: install.ps1 must not settle for a native ARM64 interpreter. + +pyarrow (via datasets) and hf-transfer publish no win_arm64 wheels, so an ARM64 +Python source-builds both and dies minutes into the run. The resolver prefers an +x64 build of the requested minor and bootstraps one otherwise; the case pinned +here is the recovery path, where nothing can be downloaded but an x64 build of a +lower-priority supported minor is already installed. +""" + +from __future__ import annotations + +import os +import re +import shutil +import subprocess +from pathlib import Path + +import pytest + + +REPO_ROOT = Path(__file__).resolve().parents[2] +INSTALL_PS1 = REPO_ROOT / "install.ps1" + + +def _extract(pattern: str, source: str) -> str: + match = re.search(pattern, source, flags = re.DOTALL) + assert match is not None, f"install.ps1 block not found: {pattern}" + return match.group(0) + + +def _resolver_script(installed: list[tuple[str, str]], can_download: bool) -> str: + """Both production functions verbatim, over a fake set of interpreters. + + Extracted rather than reimplemented so the test cannot drift away from the + text install.ps1 actually runs. `installed` is (minor, arch) in py-launcher + order, so the first entry for a minor is what a bare `py -3.13` resolves to. + The fake interpreters are named `*.exe` and invoked through the call operator, + which resolves a string to a function, so no real binary is needed. + """ + source = INSTALL_PS1.read_text(encoding = "utf-8") + finder = _extract(r" function Find-CompatiblePython \{.*?\n \}\n", source) + installer = _extract(r" function Install-X64Python \{.*?\n \}\n", source) + + names = [f"Py{minor.replace('.', '')}{arch}.exe" for minor, arch in installed] + table = ", ".join( + f'@{{ Minor = "{minor}"; Arch = "{arch}"; Name = "{name}" }}' + for (minor, arch), name in zip(installed, names) + ) + downloaded = ( + '@{ Version = "3.13"; Path = "Downloaded.exe"; Arch = "x86_64" }' + if can_download + else "$null" + ) + version_stubs = "\n".join( + f"function {name} {{ param([Parameter(ValueFromRemainingArguments = $true)]$Rest)\n" + f' if ($Rest -contains "--version") {{ return "Python {minor}.0" }}\n' + f' return "{name}" }}' + for (minor, _arch), name in zip(installed, names) + ) + return f""" +$ErrorActionPreference = "Stop" +$PythonVersion = "3.13" +$script:WingetAvailable = $false +$script:CondaSkipPattern = 'conda' +$Interpreters = @({table}) +{version_stubs} +# `py -0p` lists every registration; `py -3.x` runs the launcher's preferred build +# for that minor, which on an ARM64 host is normally the native one. +function FakePy {{ + param([Parameter(ValueFromRemainingArguments = $true)]$Rest) + if ($Rest -contains "-0p") {{ + return @($Interpreters | ForEach-Object {{ " -V:$($_.Minor) * $($_.Name)" }}) + }} + $minor = ([string]$Rest[0]).TrimStart('-') + $hit = @($Interpreters | Where-Object {{ $_.Minor -eq $minor }}) + if ($hit.Count -eq 0) {{ return "" }} + if ($Rest -contains "--version") {{ return "Python $minor.0" }} + return $hit[0].Name +}} +function substep {{ param($a, $b) }} +function Get-HostMachineArch {{ return "arm64" }} +function Get-Command {{ + param([Parameter(Position = 0)][string]$Name, + [Parameter(ValueFromRemainingArguments = $true)]$Rest) + if ($Name -eq "py") {{ return @([pscustomobject]@{{ Source = "FakePy" }}) }} + return @() +}} +function Test-Path {{ param([Parameter(ValueFromRemainingArguments = $true)]$Rest) return $true }} +function Test-IsCondaPython {{ param([string]$Exe) return $false }} +function Get-PythonPlatformTag {{ + param([string]$Exe) + foreach ($i in $Interpreters) {{ + if ($i.Name -eq $Exe) {{ + if ($i.Arch -eq "x86_64") {{ return "win-amd64" }} else {{ return "win-arm64" }} + }} + }} + return "win-amd64" +}} +function Refresh-SessionPath {{ }} +function Install-PythonFromPythonOrg {{ param([string]$Arch = "") return {downloaded} }} +{finder} +{installer} +# The caller's ARM64 swap, condensed to what decides the interpreter. +$found = Find-CompatiblePython +if ($found -and $found.Arch -ne "x86_64") {{ + $x64 = Install-X64Python + if ($x64) {{ $found = $x64 }} +}} +if ($found) {{ Write-Output "$($found.Version)|$($found.Arch)" }} else {{ Write-Output "none" }} +""" + + +def _pwsh(script: str) -> str: + result = subprocess.run( + ["pwsh", "-NoProfile", "-NonInteractive", "-Command", script], + check = True, + capture_output = True, + text = True, + env = os.environ.copy(), + ) + return result.stdout.strip() + + +@pytest.mark.skipif(shutil.which("pwsh") is None, reason = "PowerShell is unavailable") +@pytest.mark.parametrize( + ("installed", "can_download", "expected"), + [ + # An x64 build of the requested minor wins outright, downloads irrelevant. + ([("3.13", "arm64"), ("3.13", "x86_64")], False, "3.13|x86_64"), + # Requested minor is ARM64-only: bootstrap x64 rather than take the native one. + ([("3.13", "arm64")], True, "3.13|x86_64"), + # Offline, but an x64 build of a lower-priority minor is here. Use it: the native + # 3.13 cannot resolve pyarrow or hf-transfer, and this one can. + ([("3.13", "arm64"), ("3.11", "x86_64")], False, "3.11|x86_64"), + # ARM64 everywhere: still returned, and the caller warns. + ([("3.13", "arm64"), ("3.11", "arm64")], False, "3.13|arm64"), + ], +) +def test_arm64_host_prefers_an_x64_interpreter(installed, can_download, expected): + assert _pwsh(_resolver_script(installed, can_download)) == expected diff --git a/tests/python/test_windows_git_gate.py b/tests/python/test_windows_git_gate.py new file mode 100644 index 0000000000..60de430191 --- /dev/null +++ b/tests/python/test_windows_git_gate.py @@ -0,0 +1,117 @@ +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. + +"""Git is optional on the consumer Windows path, but still required for source builds.""" + +from __future__ import annotations + +import os +import shutil +import subprocess +from pathlib import Path + +import pytest + + +REPO_ROOT = Path(__file__).resolve().parents[2] +SETUP_PS1 = REPO_ROOT / "studio" / "setup.ps1" + +_START = "$gitNeeded = ($env:STUDIO_LOCAL_INSTALL -eq '1')" +_TAIL = "if (-not $_localLlamaBuilt) {" + + +def _git_gate_block() -> str: + """Slice the real $gitNeeded computation out of setup.ps1 so the test cannot drift.""" + source = SETUP_PS1.read_text(encoding = "utf-8") + start = source.index(_START) + brace = source.index("{", source.index(_TAIL, start)) + depth = 0 + for index in range(brace, len(source)): + if source[index] == "{": + depth += 1 + elif source[index] == "}": + depth -= 1 + if depth == 0: + return source[start : index + 1] + raise AssertionError("Unclosed git gate block in setup.ps1") + + +def _script() -> str: + return f""" +$DefaultLlamaPrForce = "0" +$DefaultLlamaSource = "https://github.com/ggml-org/llama.cpp" +$DefaultLlamaTag = "latest" +{_git_gate_block()} +Write-Output $gitNeeded +""" + + +def _needs_git(env: dict[str, str]) -> bool: + merged = {k: v for k, v in os.environ.items() if not k.startswith(("UNSLOTH_", "STUDIO_"))} + merged.update(env) + result = subprocess.run( + ["pwsh", "-NoProfile", "-NonInteractive", "-Command", _script()], + check = True, + capture_output = True, + text = True, + env = merged, + ) + return result.stdout.strip() == "True" + + +pwsh_only = pytest.mark.skipif(shutil.which("pwsh") is None, reason = "PowerShell is unavailable") + + +@pwsh_only +@pytest.mark.parametrize( + ("env", "expected"), + [ + # The consumer install: prebuilt wheels and a prebuilt llama.cpp, so no git. + ({}, False), + # --local clones unsloth-zoo. + ({"STUDIO_LOCAL_INSTALL": "1"}, True), + # Opt-in source builds clone llama.cpp. + ({"UNSLOTH_LLAMA_FORCE_COMPILE": "1"}, True), + ({"UNSLOTH_LLAMA_PR": "1234"}, True), + # PR_FORCE only forces a build for a positive integer. + ({"UNSLOTH_LLAMA_PR_FORCE": "0"}, False), + ({"UNSLOTH_LLAMA_PR_FORCE": "not-a-number"}, False), + ({"UNSLOTH_LLAMA_PR_FORCE": "1234"}, True), + # "master" is a branch with no release, so Phase 4 always builds it from source. + ({"UNSLOTH_LLAMA_TAG": "master"}, True), + # A release tag resolves to a prebuilt bundle. + ({"UNSLOTH_LLAMA_TAG": "latest"}, False), + ({"UNSLOTH_LLAMA_TAG": "b8635"}, False), + ], +) +def test_git_is_required_only_for_local_and_source_builds(env, expected): + assert _needs_git(env) is expected + + +@pwsh_only +def test_a_built_local_llama_dir_drops_the_source_build_git_requirement(tmp_path): + (tmp_path / "llama-server.exe").write_text("", encoding = "utf-8") + env = { + "UNSLOTH_LOCAL_LLAMA_CPP_DIR": str(tmp_path), + "UNSLOTH_LLAMA_FORCE_COMPILE": "1", + } + # Reusing an existing binary skips both the prebuilt download and the source build. + assert _needs_git(env) is False + + +@pwsh_only +@pytest.mark.parametrize("trigger", ["UNSLOTH_LLAMA_FORCE_COMPILE", "UNSLOTH_LLAMA_PR"]) +def test_an_unbuilt_local_llama_dir_still_requires_git(tmp_path, trigger): + # Nothing built at the canonical install location falls through to the normal install, + # so the source build still runs and still needs git. Suppressing the requirement here + # let a no-git host silently degrade to a prebuilt instead. + env = { + "UNSLOTH_LOCAL_LLAMA_CPP_DIR": str(tmp_path), + trigger: "1", + } + assert _needs_git(env) is True + + +@pwsh_only +def test_an_unbuilt_local_llama_dir_alone_does_not_require_git(tmp_path): + assert _needs_git({"UNSLOTH_LOCAL_LLAMA_CPP_DIR": str(tmp_path)}) is False diff --git a/tests/python/test_windows_vcredist_download_tls.py b/tests/python/test_windows_vcredist_download_tls.py new file mode 100644 index 0000000000..9fb1c697e2 --- /dev/null +++ b/tests/python/test_windows_vcredist_download_tls.py @@ -0,0 +1,80 @@ +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. + +"""The direct VC++ runtime download must negotiate TLS 1.2 on legacy protocol defaults.""" + +from __future__ import annotations + +import shutil +import subprocess +from pathlib import Path + +import pytest + + +REPO_ROOT = Path(__file__).resolve().parents[2] +SETUP_PS1 = REPO_ROOT / "studio" / "setup.ps1" + +_START = '$url = "https://aka.ms/vs/17/release/vc_redist.x64.exe"' +_END = "Remove-Item -LiteralPath $dst -Force -ErrorAction SilentlyContinue\n }" + + +def _download_block() -> str: + """Slice the real download block out of setup.ps1 so the test cannot drift.""" + source = SETUP_PS1.read_text(encoding = "utf-8") + start = source.index(_START) + end = source.index(_END, start) + len(_END) + return source[start:end] + + +def _script(starting_protocol: str) -> str: + # Start from a non-zero set that lacks Tls12. Tls13 is the only such value modern .NET + # accepts, and it stands in for the legacy Ssl3/Tls default of Windows PowerShell 5.1. + return f""" +function substep {{ param($a, $b) }} +function Refresh-Environment {{ }} +function Invoke-WebRequest {{ + param($Uri, $OutFile, [switch]$UseBasicParsing, $TimeoutSec) + Write-Output "DURING=$([System.Net.ServicePointManager]::SecurityProtocol)" + throw "stop before Start-Process" +}} +[System.Net.ServicePointManager]::SecurityProtocol = [System.Net.SecurityProtocolType]::{starting_protocol} +{_download_block()} +Write-Output "AFTER=$([System.Net.ServicePointManager]::SecurityProtocol)" +""" + + +def _run(starting_protocol: str) -> dict[str, str]: + result = subprocess.run( + ["pwsh", "-NoProfile", "-NonInteractive", "-Command", _script(starting_protocol)], + check = True, + capture_output = True, + text = True, + ) + out = {} + for line in result.stdout.splitlines(): + if "=" in line: + key, _, value = line.partition("=") + out[key.strip()] = value.strip() + return out + + +pwsh_only = pytest.mark.skipif(shutil.which("pwsh") is None, reason = "PowerShell is unavailable") + + +@pwsh_only +def test_tls12_is_added_for_the_download_and_restored_after(): + seen = _run("Tls13") + during = {part.strip() for part in seen["DURING"].split(",")} + assert "Tls12" in during, "the download must negotiate TLS 1.2 or aka.ms refuses it" + assert "Tls13" in during, "adding TLS 1.2 must not drop protocols the host already allowed" + assert seen["AFTER"] == "Tls13", "the process-wide protocol must be restored" + + +@pwsh_only +def test_system_default_is_left_alone(): + # SystemDefault means "let the OS choose" and already covers TLS 1.2+; pinning it to + # Tls12 would strip TLS 1.3 from every later request in the process. + seen = _run("SystemDefault") + assert seen["DURING"] == "SystemDefault" + assert seen["AFTER"] == "SystemDefault" diff --git a/tests/sh/test_install_pipe_safety.sh b/tests/sh/test_install_pipe_safety.sh new file mode 100755 index 0000000000..be479dd5a5 --- /dev/null +++ b/tests/sh/test_install_pipe_safety.sh @@ -0,0 +1,89 @@ +#!/bin/bash +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 +# +# Guards that `curl ... | sh` cannot report a bogus transport error. +# +# History: install.sh was ~150KB of top-level statements. A top-level `exit` left most +# of the file unread, the write end failed, and curl appended "(56) Failure writing +# output to destination" (or "(23) Failed writing body") after our own error message, +# so users read a real diagnosis as a broken download. The fix is structural: the body +# lives in _unsloth_main, so sh parses the whole file before running anything. +# +# This pins both halves of that contract: the writer must not be killed, AND the +# installer's own exit code must still reach the caller. +set -e + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +INSTALL_SH="$SCRIPT_DIR/../../install.sh" +PASS=0 +FAIL=0 + +assert_eq() { + _label="$1"; _expected="$2"; _actual="$3" + if [ "$_actual" = "$_expected" ]; then + echo " PASS: $_label" + PASS=$((PASS + 1)) + else + echo " FAIL: $_label (expected '$_expected', got '$_actual')" + FAIL=$((FAIL + 1)) + fi +} + +echo "=== structure ===" + +# The wrapper must be invoked on the LAST executable line, or sh starts executing +# before it has drained the pipe. +if grep -q '^_unsloth_main() {' "$INSTALL_SH"; then + echo " PASS: _unsloth_main is defined at top level" + PASS=$((PASS + 1)) +else + echo " FAIL: install.sh is not wrapped in _unsloth_main -- curl-pipe safety is gone" + FAIL=$((FAIL + 1)) +fi + +_last="$(grep -vE '^\s*(#|$)' "$INSTALL_SH" | tail -1)" +assert_eq "last statement invokes the wrapper" '_unsloth_main "$@"' "$_last" + +# Below one pipe buffer the file would fit in the kernel's buffer and this test would +# prove nothing, so fail loudly instead of passing vacuously. +_bytes="$(wc -c < "$INSTALL_SH" | tr -d ' ')" +if [ "$_bytes" -gt 65536 ]; then + echo " PASS: install.sh ($_bytes bytes) exceeds a 64KiB pipe buffer, so this matters" + PASS=$((PASS + 1)) +else + echo " FAIL: install.sh is only $_bytes bytes; re-derive whether pipe safety still applies" + FAIL=$((FAIL + 1)) +fi + +echo "=== behaviour: an early exit must not kill the writer ===" + +# `--python` with no argument exits 1 from argument validation having done no work: no +# venv, no downloads, no filesystem writes. Deterministic and safe to run for real. +# +# PIPESTATUS must be read on the very next line, so drop errexit around the pipeline +# rather than appending `|| true`, which would clobber it with the status of `true`. +set +e +cat "$INSTALL_SH" | sh -s -- --python >/dev/null 2>&1 +_pipe=("${PIPESTATUS[@]}") +set -e +_writer_rc="${_pipe[0]}" +_reader_rc="${_pipe[1]}" + +# A writer rc of 141 (128 + SIGPIPE) is the failure mode curl reports as (56)/(23). +assert_eq "writer survives the early exit (not SIGPIPE)" "0" "$_writer_rc" +assert_eq "installer's own exit code still propagates" "1" "$_reader_rc" + +echo "=== behaviour: the same holds for a mid-file exit ===" +# `--package '-evil'` exits from a later validation block, still before any filesystem +# work, so the property is not specific to one early branch. +set +e +cat "$INSTALL_SH" | sh -s -- --package '-evil' >/dev/null 2>&1 +_pipe2=("${PIPESTATUS[@]}") +set -e +assert_eq "writer survives a later exit" "0" "${_pipe2[0]}" +assert_eq "later exit code propagates" "1" "${_pipe2[1]}" + +echo "" +echo "=== $PASS passed, $FAIL failed ===" +[ "$FAIL" -eq 0 ] || exit 1 diff --git a/tests/studio/install/test_install_node_prebuilt_logic.py b/tests/studio/install/test_install_node_prebuilt_logic.py index 5476702d65..5bcc9733cf 100644 --- a/tests/studio/install/test_install_node_prebuilt_logic.py +++ b/tests/studio/install/test_install_node_prebuilt_logic.py @@ -762,3 +762,94 @@ def test_pinned_target_wrong_sha_not_kept_when_download_fails(tmp_path: Path, mo monkeypatch.setattr(M, "download_file_verified", _offline) # transient download failure with pytest.raises(OSError): M.install_prebuilt(install_dir, channel = "pinned", min_major = 24, force = False) + + +# ── _replace_with_retry: transient Windows sharing violations ────────────────── +# Seen in CI: WinError 5 renaming extracted Node into place on a FRESH install, a scanner +# still holding handles inside the new files. + + +def _oserror(winerror: int) -> OSError: + exc = OSError(winerror, "mock") + exc.winerror = winerror + return exc + + +@pytest.mark.parametrize("winerror", [5, 32, 145]) +def test_replace_retries_transient_windows_errors(monkeypatch, tmp_path, winerror): + monkeypatch.setattr(M.os, "name", "nt") + monkeypatch.setattr(M.time, "sleep", lambda _s: None) # no real backoff in tests + calls = {"n": 0} + + def flaky(src, dst): + calls["n"] += 1 + if calls["n"] < 3: + raise _oserror(winerror) + + monkeypatch.setattr(M.os, "replace", flaky) + M._replace_with_retry(tmp_path / "src", tmp_path / "dst") + assert calls["n"] == 3, "should have retried until the handle was released" + + +def test_replace_gives_up_and_reports_the_real_error(monkeypatch, tmp_path): + monkeypatch.setattr(M.os, "name", "nt") + monkeypatch.setattr(M.time, "sleep", lambda _s: None) + monkeypatch.setattr(M.os, "replace", lambda s, d: (_ for _ in ()).throw(_oserror(5))) + # A scanner that never lets go must surface as a failure, not a hang. + with pytest.raises(OSError) as excinfo: + M._replace_with_retry(tmp_path / "src", tmp_path / "dst", attempts = 3) + assert excinfo.value.winerror == 5 + + +def test_replace_does_not_retry_a_genuine_error(monkeypatch, tmp_path): + # A cross-device move or real permissions problem must fail immediately. + monkeypatch.setattr(M.os, "name", "nt") + monkeypatch.setattr(M.time, "sleep", lambda _s: None) + calls = {"n": 0} + + def hard_fail(src, dst): + calls["n"] += 1 + raise _oserror(17) # ERROR_NOT_SAME_DEVICE + + monkeypatch.setattr(M.os, "replace", hard_fail) + with pytest.raises(OSError): + M._replace_with_retry(tmp_path / "src", tmp_path / "dst") + assert calls["n"] == 1 + + +def test_replace_is_a_plain_rename_on_posix(monkeypatch, tmp_path): + # POSIX has no sharing violations, so the retry must add no latency there. + monkeypatch.setattr(M.os, "name", "posix") + calls = {"n": 0} + + def once(src, dst): + calls["n"] += 1 + raise _oserror(5) + + monkeypatch.setattr(M.os, "replace", once) + with pytest.raises(OSError): + M._replace_with_retry(tmp_path / "src", tmp_path / "dst") + assert calls["n"] == 1 + + +def test_swap_into_place_survives_a_transient_lock(monkeypatch, tmp_path): + # End-to-end through the function the installer actually calls. + monkeypatch.setattr(M.os, "name", "nt") + monkeypatch.setattr(M.time, "sleep", lambda _s: None) + extracted = tmp_path / "extracted" / "node-v24" + extracted.mkdir(parents = True) + (extracted / "marker.txt").write_text("node", encoding = "utf-8") + install_dir = tmp_path / "node" + + real_replace = os.replace + state = {"failed": False} + + def flaky(src, dst): + if not state["failed"]: + state["failed"] = True + raise _oserror(32) + real_replace(src, dst) + + monkeypatch.setattr(M.os, "replace", flaky) + M._swap_into_place(extracted, install_dir) + assert (install_dir / "marker.txt").read_text(encoding = "utf-8") == "node" diff --git a/tests/studio/test_cli_studio_stop_windows.py b/tests/studio/test_cli_studio_stop_windows.py index 2267d7feda..cef7cc6db7 100644 --- a/tests/studio/test_cli_studio_stop_windows.py +++ b/tests/studio/test_cli_studio_stop_windows.py @@ -44,9 +44,12 @@ def _load_pid_alive(platform: str, fake_run = None): # ── AST: stop() must not use the broken bare liveness probe ────────────────── -def test_stop_does_not_use_bare_oskill_liveness_probe(): - """stop() must not call os.kill(pid, 0) -- it crashes on Windows.""" - stop_src = _func_source("stop") +# `stop` delegates signalling to `_signal_stop`, so guarding only `stop` would +# let os.kill(pid, 0) come back one function along and still pass. +@pytest.mark.parametrize("func", ["stop", "_signal_stop"]) +def test_stop_does_not_use_bare_oskill_liveness_probe(func): + """The signalling path must not call os.kill(pid, 0) -- WinError 87 on Windows.""" + stop_src = _func_source(func) tree = ast.parse(stop_src) for call in ast.walk(tree): if not isinstance(call, ast.Call): @@ -62,14 +65,17 @@ def test_stop_does_not_use_bare_oskill_liveness_probe(): sig = call.args[1] if isinstance(sig, ast.Constant) and sig.value == 0: raise AssertionError( - "stop() still uses os.kill(pid, 0); it raises WinError 87 on " - "Windows. Use the cross-platform _pid_alive() helper instead." + f"{func}() still uses os.kill(pid, 0); it raises WinError 87 " + "on Windows. Use the cross-platform _pid_alive() helper." ) def test_pid_alive_helper_is_defined_and_used_by_stop(): assert "def _pid_alive(" in _SOURCE, "_pid_alive helper missing" assert "_pid_alive(pid)" in _func_source("stop"), "stop() must use _pid_alive" + # The kill itself moved into _signal_stop; keep both ends of the path pinned. + assert "def _signal_stop(" in _SOURCE, "_signal_stop helper missing" + assert "taskkill" in _func_source("_signal_stop") # The helper must special-case Windows via tasklist (os.kill(pid,0) is invalid there). helper = _func_source("_pid_alive") assert 'sys.platform == "win32"' in helper diff --git a/tests/studio/test_studio_pid_file_contract.py b/tests/studio/test_studio_pid_file_contract.py new file mode 100644 index 0000000000..23ace706b5 --- /dev/null +++ b/tests/studio/test_studio_pid_file_contract.py @@ -0,0 +1,73 @@ +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. + +"""run.py writes the Studio PID files; `unsloth studio stop` globs for them. + +Nothing else ties the writer's filename to the reader's glob, and each side's own +tests hardcode the names they expect, so a rename on either side alone leaves both +suites green while `stop` silently finds nothing. `unsloth_cli/tests/` also runs +in no workflow, so this lives here, where the repo CPU job discovers it. + +AST + exec of the writer, so no backend dependency stack is imported. +""" + +import ast +import os +import sys +from pathlib import Path + +_ROOT = Path(__file__).resolve().parents[2] +if str(_ROOT) not in sys.path: + sys.path.insert(0, str(_ROOT)) + +_RUN_SRC = (_ROOT / "studio" / "backend" / "run.py").read_text(encoding = "utf-8") + + +def _func_source(source: str, name: str) -> str: + for node in ast.walk(ast.parse(source)): + if isinstance(node, ast.FunctionDef) and node.name == name: + return ast.get_source_segment(source, node) + raise AssertionError(f"function {name!r} not found") + + +def _backend_pid_path(root: Path, port: int) -> Path: + """The path run.py's own _pid_file_for_port builds, without importing run.py.""" + ns = {"os": os, "Path": Path, "_studio_root": lambda: root} + exec(_func_source(_RUN_SRC, "_pid_file_for_port"), ns) + return ns["_pid_file_for_port"](port) + + +def test_stop_finds_a_pid_file_named_the_way_the_backend_writes_it(tmp_path, monkeypatch): + from unsloth_cli.commands import studio as cli + + path = _backend_pid_path(tmp_path, 8901) + # The same three-line body _write_pid_file emits (create_time is blank when + # psutil is unavailable, and the CLI must tolerate that). + path.write_text(f"{os.getpid()}\n\n127.0.0.1", encoding = "utf-8") + + monkeypatch.setattr(cli, "STUDIO_HOME", tmp_path) + monkeypatch.setattr(cli, "_PID_FILE", tmp_path / "studio.pid") + + assert [pid for pid, _times, _files in cli._pid_file_entries()] == [os.getpid()] + + +def test_the_legacy_file_stays_a_bare_pid_an_older_cli_can_parse(tmp_path, monkeypatch): + # An older `unsloth studio stop` reads studio.pid and requires str.isdigit(), + # so the compatibility file must never gain the extra metadata lines. + ns = { + "os": os, + "Path": Path, + "_studio_root": lambda: tmp_path, + "_PID_FILE": tmp_path / "studio.pid", + "_pid_file_for_port": lambda port: _backend_pid_path(tmp_path, port), + "_process_create_time": lambda pid: None, + "_bind_addresses": lambda host, port: {host}, + # _write_pid_file consults these before taking over studio.pid. + "_read_pid_record": lambda path: None, + "_pid_alive": lambda pid: False, + "_OWN_PID_FILE": None, + } + exec(_func_source(_RUN_SRC, "_write_pid_file"), ns) + ns["_write_pid_file"](8901, "127.0.0.1") + + assert (tmp_path / "studio.pid").read_text(encoding = "utf-8").strip().isdigit() diff --git a/tests/studio/test_update_release_notes.py b/tests/studio/test_update_release_notes.py new file mode 100644 index 0000000000..765ad55321 --- /dev/null +++ b/tests/studio/test_update_release_notes.py @@ -0,0 +1,1906 @@ +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. + +"""Contracts for the update popup's release-notes preview. + +The popup renders CHANGELOG.md notes for the exact version it is offering. The +risk this file guards is showing notes from a different release: a near-miss +lookup must return nothing rather than the newest section it can find.""" + +from __future__ import annotations + +import http.server +import json +import os +import re +import shutil +import subprocess +import sys +import threading +import time +from pathlib import Path + +import pytest + +REPO = Path(__file__).resolve().parents[2] +BACKEND = REPO / "studio/backend" +FRONTEND = REPO / "studio/frontend/src" +CHANGELOG = REPO / "CHANGELOG.md" +PANEL = FRONTEND / "components/update/release-notes-panel.tsx" +NOTES_HOOK = FRONTEND / "hooks/use-release-notes.ts" +PREVIEW = FRONTEND / "lib/release-notes-preview.ts" +CODE_SPANS = FRONTEND / "lib/markdown-code-spans.ts" +LINKS = FRONTEND / "lib/changelog-links.ts" +LIST_COLUMNS = FRONTEND / "lib/markdown-list-columns.ts" +INLINE_COMMENTS = FRONTEND / "lib/markdown-inline-comments.ts" +WEB_BANNER = FRONTEND / "components/web/update-banner.tsx" +TAURI_BANNER = FRONTEND / "components/tauri/update-banner.tsx" + +# The scanners are the frontend half of the contract the parser implements, so they are +# run rather than read. Node strips the types and nothing imports a package: no install. +_TS_ALIAS = re.compile(r'"@/lib/([a-z-]+)"') +_TS_RUNNER = """ +import { resolveChangelogLinks } from "./changelog-links.ts"; +import { releaseNotesPreview } from "./release-notes-preview.ts"; + +const chunks: Buffer[] = []; +process.stdin.on("data", (chunk: Buffer) => chunks.push(chunk)); +process.stdin.on("end", () => { + const markdown = Buffer.concat(chunks).toString("utf8"); + const result = + process.argv[2] === "links" + ? resolveChangelogLinks(markdown) + : releaseNotesPreview(markdown); + process.stdout.write(JSON.stringify(result)); +}); +""" + +SAMPLE = """# Changelog + +Intro prose that belongs to no release. + +## Format + +```md +## 9999.9.9 - fenced sample, not a real section +``` + +## Unreleased + +- staged note + +## 2026.7.6 - 2026-07-22 + +### What's Changed + +- newer thing + +## 2026.7.5 + +### What's Changed + +- older thing +""" + + +@pytest.fixture(scope = "module") +def changelog_module(): + sys.path.insert(0, str(BACKEND)) + try: + from utils import changelog + finally: + sys.path.pop(0) + changelog.reset_changelog_cache() + yield changelog + changelog.reset_changelog_cache() + + +@pytest.fixture +def isolated_changelog(changelog_module, tmp_path, monkeypatch): + """Point the module at a temp file and away from the network.""" + monkeypatch.setenv(changelog_module.DISABLE_ENV_VAR, "1") + path = tmp_path / "CHANGELOG.md" + path.write_text(SAMPLE, encoding = "utf-8") + monkeypatch.setenv(changelog_module.CHANGELOG_PATH_ENV_VAR, str(path)) + changelog_module.reset_changelog_cache() + yield changelog_module + changelog_module.reset_changelog_cache() + + +def test_only_real_release_headings_become_sections(changelog_module): + versions = [entry.version for entry in changelog_module.parse_changelog(SAMPLE)] + # "Format"/"Unreleased" are not versions, and 9999.9.9 is fenced sample. + assert versions == ["2026.7.6", "2026.7.5"] + + +def test_section_body_stops_at_the_next_release(changelog_module): + entry = changelog_module.find_release_notes(SAMPLE, "2026.7.6") + assert entry is not None + assert "newer thing" in entry.body + assert "older thing" not in entry.body + + +def test_unknown_version_returns_no_notes_instead_of_a_nearby_release(changelog_module): + assert changelog_module.find_release_notes(SAMPLE, "2026.7.7") is None + assert changelog_module.find_release_notes(SAMPLE, "2026.7") is None + + +def test_version_equality_is_normalized_not_fuzzy(changelog_module): + entry = changelog_module.find_release_notes(SAMPLE, "2026.07.6") + assert entry is not None and entry.version == "2026.7.6" + + +def test_response_reports_no_match_without_markdown(isolated_changelog): + payload = isolated_changelog.get_release_notes("2026.7.7") + assert payload["matched"] is False + assert payload["markdown"] is None + assert payload["version"] == "2026.7.7" + # The UI still needs somewhere to send the user. + assert payload["release_notes_url"] + + +def test_response_matches_local_changelog_when_offline(isolated_changelog): + payload = isolated_changelog.get_release_notes("2026.7.6") + assert payload["matched"] is True + assert payload["source"] == "local" + assert "newer thing" in payload["markdown"] + + +def test_unsupported_version_query_is_rejected(isolated_changelog): + assert isolated_changelog.is_supported_version_query("2026.7.6") is True + for bad in ("../etc/passwd", "2026.7.6 OR 1", "", "a" * 80): + assert isolated_changelog.is_supported_version_query(bad) is False + assert isolated_changelog.get_release_notes("../etc/passwd")["matched"] is False + + +def test_remote_changelog_wins_over_bundled_copy(changelog_module, tmp_path, monkeypatch): + """The offered version is newer than the installed checkout, so the repo + copy has to be able to describe versions the local file has never heard of.""" + monkeypatch.delenv(changelog_module.DISABLE_ENV_VAR, raising = False) + local = tmp_path / "CHANGELOG.md" + local.write_text(SAMPLE, encoding = "utf-8") + monkeypatch.setenv(changelog_module.CHANGELOG_PATH_ENV_VAR, str(local)) + + remote_body = "# Changelog\n\n## 2026.8.0\n\n- shipped after this install\n" + + class Handler(http.server.BaseHTTPRequestHandler): + def do_GET(self): # noqa: N802 - stdlib naming + payload = remote_body.encode("utf-8") + self.send_response(200) + self.send_header("Content-Length", str(len(payload))) + self.end_headers() + self.wfile.write(payload) + + def log_message(self, *_args): + pass + + server = http.server.HTTPServer(("127.0.0.1", 0), Handler) + thread = threading.Thread(target = server.serve_forever, daemon = True) + thread.start() + try: + monkeypatch.setenv( + changelog_module.CHANGELOG_URL_ENV_VAR, + f"http://127.0.0.1:{server.server_port}/CHANGELOG.md", + ) + changelog_module.reset_changelog_cache() + payload = changelog_module.get_release_notes("2026.8.0") + assert payload["matched"] is True + assert payload["source"] == "remote" + assert "shipped after this install" in payload["markdown"] + finally: + server.shutdown() + server.server_close() + changelog_module.reset_changelog_cache() + + +def test_repo_changelog_exists_and_parses(changelog_module): + assert CHANGELOG.is_file(), "CHANGELOG.md is the editable source of release notes" + entries = changelog_module.parse_changelog(CHANGELOG.read_text(encoding = "utf-8")) + assert entries, "CHANGELOG.md needs at least one `## <version>` section" + + +def test_longer_outer_fence_does_not_leak_a_fake_section(changelog_module): + """A ``` sample inside a ```` block must not close the block and let the + sample's heading be indexed as a real release.""" + text = "## 1.0\n\n````md\n```\n## 9.9.9\n```\n````\n\n- real note\n" + assert [e.version for e in changelog_module.parse_changelog(text)] == ["1.0"] + assert changelog_module.find_release_notes(text, "9.9.9") is None + + +def test_tilde_fence_is_not_closed_by_backticks(changelog_module): + text = "## 1.0\n\n~~~\n```\n## 9.9.9\n~~~\n\n- real\n" + assert [e.version for e in changelog_module.parse_changelog(text)] == ["1.0"] + + +def test_utf8_bom_does_not_hide_the_first_section(changelog_module): + """Editors on Windows can leave a BOM on the first line.""" + assert [e.version for e in changelog_module.parse_changelog("\ufeff## 1.0\n\n- x\n")] == ["1.0"] + + +@pytest.mark.parametrize("newline", ["\r\n", "\r"]) +def test_non_unix_line_endings(changelog_module, newline): + text = f"## 1.0{newline}{newline}- windows note{newline}" + entry = changelog_module.find_release_notes(text, "1.0") + assert entry is not None and "windows note" in entry.body + assert "\r" not in entry.body + + +def test_closing_fence_must_carry_nothing_after_it(changelog_module): + """CommonMark: a closer is the delimiter plus whitespace only. A ```` line + with trailing text inside a ```` block is content, not the end.""" + text = "## 1.0\n\n````md\n```` not a closer\n## 9.9.9\n````\n\n- real\n" + assert [e.version for e in changelog_module.parse_changelog(text)] == ["1.0"] + # An opening fence may still carry an info string. + info = "## 1.0\n\n```python\n## 9.9.9\n```\n\n- real\n" + assert [e.version for e in changelog_module.parse_changelog(info)] == ["1.0"] + + +@pytest.mark.parametrize( + "text", + [ + "## 1.0\n\n- real\n\n<!--\n## 9.9.9\n\n- unpublished\n-->\n", + "## 1.0\n\n- real\n\n<!-- ## 9.9.9 -->\n", + ], +) +def test_commented_out_sections_are_not_releases(changelog_module, text): + """Markdown does not render them, so they are not published notes.""" + assert [e.version for e in changelog_module.parse_changelog(text)] == ["1.0"] + assert changelog_module.find_release_notes(text, "9.9.9") is None + + +def test_repo_root_changelog_is_preferred_over_the_build_snapshot(changelog_module): + """The build backend writes studio/CHANGELOG.md; the root file must win.""" + # Resolved paths, not name suffixes: a checkout may be renamed and Windows uses "\". + paths = [Path(p).resolve() for p in changelog_module._local_changelog_candidates()] + root = paths.index((REPO / changelog_module.CHANGELOG_FILENAME).resolve()) + packaged = paths.index((REPO / "studio" / changelog_module.CHANGELOG_FILENAME).resolve()) + assert root < packaged + build = (REPO / "build.sh").read_text(encoding = "utf-8") + assert "rm -f studio/CHANGELOG.md" in build, "snapshot must not linger after a build" + + +def test_preview_keeps_identifier_underscores(): + """UNSLOTH_DISABLE_UPDATE_CHECK must not render as UNSLOTHDISABLEUPDATECHECK.""" + src = PREVIEW.read_text(encoding = "utf-8") + assert "BOLD_UNDERSCORE" in src and "ITALIC_UNDERSCORE" in src + assert "parkCodeSpans" in src, "code spans are parked so their underscores survive" + assert "const EMPHASIS" not in src, "the blanket emphasis strip is gone" + + +def test_panel_prefers_the_callers_release_url(): + """The API only returns the generic changelog; the desktop banner passes + the exact release page for the version being offered.""" + src = PANEL.read_text(encoding = "utf-8") + assert "releaseNotesUrl ?? notes?.releaseNotesUrl" in src + + +def test_remote_failure_is_reported_so_the_ui_can_retry(changelog_module, tmp_path, monkeypatch): + """A bundled changelog cannot know a version newer than the install, so a + failed remote lookup must not read as "no notes were published".""" + monkeypatch.delenv(changelog_module.DISABLE_ENV_VAR, raising = False) + local = tmp_path / "CHANGELOG.md" + local.write_text("## 1.0\n\n- old release\n", encoding = "utf-8") + monkeypatch.setenv(changelog_module.CHANGELOG_PATH_ENV_VAR, str(local)) + # Port 9 (discard) refuses fast, standing in for an unreachable host. + monkeypatch.setenv(changelog_module.CHANGELOG_URL_ENV_VAR, "http://127.0.0.1:9/CHANGELOG.md") + changelog_module.reset_changelog_cache() + try: + payload = changelog_module.get_release_notes("2.0") + assert payload["matched"] is False + assert payload["error"], "remote failure must reach the UI" + finally: + changelog_module.reset_changelog_cache() + + +def test_preview_keeps_comparison_operators(): + """ "Support Python <3.15 and >3.9" must not lose its operators to the tag + strip, which would turn it into "Support Python 3.9".""" + src = PREVIEW.read_text(encoding = "utf-8") + assert "/<\\/?[a-zA-Z][^>]*>/g" in src, "tag strip must require a name character" + + +def test_preview_hides_commented_out_notes(): + """Unpublished notes inside <!-- --> are not rendered, so not previewed.""" + src = PREVIEW.read_text(encoding = "utf-8") + assert "stripCommentSpans" in src and "COMMENT_OPEN" in src + + +def test_hook_treats_a_reported_failure_as_retryable(): + src = NOTES_HOOK.read_text(encoding = "utf-8") + assert "next.error !== null" in src + + +def test_comment_delimiter_in_inline_code_is_literal(changelog_module): + """A note documenting `<!--` used to put the parser into comment state, + swallowing every release below it.""" + text = "## 2.0\n\n- Type `<!--` to begin a comment\n\n## 1.0\n\n- older\n" + assert [e.version for e in changelog_module.parse_changelog(text)] == ["2.0", "1.0"] + assert changelog_module.find_release_notes(text, "1.0") is not None + assert "older" not in changelog_module.find_release_notes(text, "2.0").body + + +def test_refresh_retries_a_cached_remote_failure(changelog_module, tmp_path, monkeypatch): + """Retry must reach the network again once connectivity returns, rather + than replaying the cached failure until its TTL expires.""" + monkeypatch.delenv(changelog_module.DISABLE_ENV_VAR, raising = False) + local = tmp_path / "CHANGELOG.md" + local.write_text("## 1.0\n\n- old\n", encoding = "utf-8") + monkeypatch.setenv(changelog_module.CHANGELOG_PATH_ENV_VAR, str(local)) + + hits = {"count": 0} + + class Handler(http.server.BaseHTTPRequestHandler): + def do_GET(self): # noqa: N802 - stdlib naming + hits["count"] += 1 + self.send_response(500) + self.send_header("Content-Length", "0") + self.end_headers() + + def log_message(self, *_args): + pass + + server = http.server.HTTPServer(("127.0.0.1", 0), Handler) + threading.Thread(target = server.serve_forever, daemon = True).start() + try: + monkeypatch.setenv( + changelog_module.CHANGELOG_URL_ENV_VAR, + f"http://127.0.0.1:{server.server_port}/CHANGELOG.md", + ) + changelog_module.reset_changelog_cache() + changelog_module.get_release_notes("2.0") + changelog_module.get_release_notes("2.0") + assert hits["count"] == 1, "the failure should be cached" + changelog_module.get_release_notes("2.0", refresh = True) + assert hits["count"] == 2, "refresh must bypass the cached failure" + finally: + server.shutdown() + server.server_close() + changelog_module.reset_changelog_cache() + + +def test_hook_never_returns_another_versions_notes(): + """On the render where the offered version changes, state still describes + the previous one until the effect runs.""" + src = NOTES_HOOK.read_text(encoding = "utf-8") + assert "notes.version === version" in src + assert "refresh" in src, "retry must ask the backend to bypass its cache" + + +@pytest.mark.parametrize("indent", ["", " ", " ", " "]) +def test_headings_and_fences_allow_commonmark_indentation(changelog_module, indent): + """Markdown renders up to three leading spaces, so the parser must agree + or an indented release is unreachable and its notes join the one above.""" + text = f"## 1.0\n\nOne.\n\n{indent}## 2.0\n\nTwo.\n" + assert [e.version for e in changelog_module.parse_changelog(text)] == ["1.0", "2.0"] + fenced = f"## 1.0\n\n{indent}```\n{indent}## 9.9.9\n{indent}```\n\n- real\n" + assert [e.version for e in changelog_module.parse_changelog(fenced)] == ["1.0"] + + +def test_four_space_indentation_is_code_not_structure(changelog_module): + """At four spaces Markdown switches to indented code, for both forms.""" + assert [ + e.version for e in changelog_module.parse_changelog(" ## 9.9.9\n\n## 1.0\n\n- real\n") + ] == ["1.0"] + assert [ + e.version + for e in changelog_module.parse_changelog( + "## 1.0\n\n ```\n sample\n\n## 2.0\n\n- two\n" + ) + ] == ["1.0", "2.0"] + + +def test_desktop_notes_link_to_the_release_page_on_every_platform(): + """manualReleaseUrl is Linux-package only, so in-app updates on macOS, + Windows and AppImage would otherwise link to the generic changelog.""" + hook = (FRONTEND / "hooks/use-tauri-update.ts").read_text(encoding = "utf-8") + assert "const releasePageUrl = info ?" in hook + banner = TAURI_BANNER.read_text(encoding = "utf-8") + assert "releaseNotesUrl={releasePageUrl ?? manualReleaseUrl}" in banner + provider = (FRONTEND / "app/provider.tsx").read_text(encoding = "utf-8") + assert "releasePageUrl={update.releasePageUrl}" in provider + + +def test_preview_matches_how_markdown_renders_prose_and_links(): + """Three rendering mismatches the preview must not reintroduce: wrapped + paragraphs split into fragments, autolinks eaten as tags, and a lead cut + at an abbreviation.""" + src = PREVIEW.read_text(encoding = "utf-8") + # Contiguous prose lines accumulate and flush at a paragraph boundary. + assert "collector.paragraph = collector.paragraph" in src + # <https://x> renders as link text, so it is not a tag. + assert "AUTOLINK" in src + # "e.g. GGUF" is not a sentence boundary. + assert "ABBREVIATIONS" in src and "INITIAL" in src + + +def test_preview_treats_code_as_literal(): + """Inside a code span, and inside an indented code block, Markdown renders + the text literally, so the preview must not transform or promote it.""" + src = PREVIEW.read_text(encoding = "utf-8") + # Code spans are parked before any other inline transformation. + park = src.index("parkCodeSpans(markdown") + assert park < src.index("stripHtmlTags(\n parked") + # A "- cmd" line inside an indented code block is not a headline bullet. + assert "INDENTED_CODE_INDENT" in src + + +def test_desktop_updater_metadata_maps_published_field_names(): + """latest.json publishes Tauri's `notes`/`pub_date`; the manual Linux path + must read those, not `body`/`date`, or its release notes are always empty.""" + rust = (REPO / "studio/src-tauri/src/desktop_update_policy.rs").read_text(encoding = "utf-8") + assert 'alias = "body"' in rust and "notes: Option<String>" in rust + assert 'alias = "date"' in rust and "pub_date: Option<String>" in rust + assert "body: metadata.notes" in rust and "date: metadata.pub_date" in rust + workflow = (REPO / ".github/workflows/release-desktop.yml").read_text(encoding = "utf-8") + assert "'notes': notes," in workflow, "workflow no longer publishes `notes`" + + +def test_backend_exposes_release_notes_route(): + src = (BACKEND / "main.py").read_text(encoding = "utf-8") + assert '@app.get("/api/studio/release-notes")' in src + assert "is_supported_version_query" in src + + +def test_panel_is_scrollable_and_version_scoped(): + src = PANEL.read_text(encoding = "utf-8") + assert "overflow-y-auto" in src, "release notes must scroll inside the popup" + assert "max-h-" in src, "the scroller needs a bounded height" + # Falls back to the payload's own body only, never to another version. + assert "fallbackMarkdown" in src + + +def test_notes_surface_is_borderless_and_lifts_in_dark_mode(): + src = PANEL.read_text(encoding = "utf-8") + assert "border border-border" not in src, "the notes box is a fill, not a bordered box" + # Lighter than the card behind it, rather than a darker inset. + assert "dark:bg-white/[0.06]" in src + # Streamdown's mt-6 clips the first heading against the scroller edge. + assert "[&>*>*:first-child]:mt-0" in src + # Shared utility: thumb hidden until the notes are hovered. + assert "hover-scrollbar" in src + # Streamdown renders code at text-sm, twice this panel's body size. + assert "[&_code]:text-[0.92em]" in src + + +def test_hook_discards_notes_for_a_different_version(): + src = NOTES_HOOK.read_text(encoding = "utf-8") + assert "notesVersion !== version" in src + + +def test_collapsed_panel_previews_the_top_bullets(): + """Collapsed popups show the headline changes without an extra click.""" + preview = PREVIEW.read_text(encoding = "utf-8") + assert "RELEASE_NOTES_PREVIEW_ITEMS = 4" in preview + # Wrapped bullets join into one item, or a preview ends mid-sentence. + assert "collectBullets" in preview and "flush" in preview + # Nested list items are detail, not headline changes. + assert "NESTED_INDENT_TOLERANCE" in preview + # Tag stripping repeats: one pass turns `<<b>b>` back into a live tag. + assert "while (out !== previous)" in preview + + panel = PANEL.read_text(encoding = "utf-8") + assert "releaseNotesPreview" in panel + assert 'data-testid="update-release-notes-summary"' in panel + # Fetched when the popup appears: the collapsed preview needs them too. + assert "enabled: true" in panel + + +def test_preview_highlights_the_leading_sentence(): + """Each bullet leads with its headline sentence, emphasised over the rest.""" + preview = PREVIEW.read_text(encoding = "utf-8") + assert "splitLeadSentence" in preview + # A period inside "CHANGELOG.md" or "e.g." must not read as a break. + assert "SENTENCE_BREAK" in preview and "(?=" in preview + + panel = PANEL.read_text(encoding = "utf-8") + assert '<span className="font-medium text-foreground">{item.lead}</span>' in panel + assert "item.rest" in panel + + +@pytest.mark.parametrize("banner", [WEB_BANNER, TAURI_BANNER]) +def test_update_popup_is_wider_than_the_other_overlays(banner): + """The card is sized for three same-size buttons on one row. + + Width moved from the shared overlay stack onto each overlay, so widening + the update popup does not widen the llama.cpp banner or download panel.""" + assert "max-w-[448px]" in banner.read_text(encoding = "utf-8") + provider = (FRONTEND / "app/provider.tsx").read_text(encoding = "utf-8") + assert "max-w-[400px]" not in provider, "stack must not cap overlay width" + llama = (FRONTEND / "components/llama-update-banner.tsx").read_text(encoding = "utf-8") + assert "max-w-[400px]" in llama, "unrelated overlays keep their width" + + +@pytest.mark.parametrize("banner", [WEB_BANNER, TAURI_BANNER]) +def test_banners_toggle_inline_release_notes(banner): + src = banner.read_text(encoding = "utf-8") + assert "ReleaseNotesPanel" in src + assert "Show release notes" in src and "Hide release notes" in src + # Keyed by version, so a new offer cannot leave old notes on screen. + assert "notesVersion" in src + + +@pytest.mark.parametrize( + "banner,toggle,action", + [ + (WEB_BANNER, "web-update-release-notes-toggle", "web-update-snooze-button"), + (TAURI_BANNER, "tauri-update-release-notes-toggle", "Remind me later"), + ], +) +def test_notes_toggle_shares_the_action_row(banner, toggle, action): + """The toggle sits in the same row as the actions, not on its own line.""" + src = banner.read_text(encoding = "utf-8") + row = src.index("mt-4 flex") + assert row < src.index(toggle) < src.index(action) + # Same type size as the actions beside it; nowrap keeps labels on one line. + toggle_line = next(line for line in src.splitlines() if toggle in line) + toggle_block = src[src.index("Button", row) : src.index(toggle_line)] + assert "text-ui-13" in toggle_block and "whitespace-nowrap" in toggle_block + + +def test_headings_inside_a_raw_html_block_are_not_releases(changelog_module): + """<pre> content is literal, so a sample heading in it must not become a + section and must not cut the real section's body short.""" + text = "## 1.0\n\n<pre>\n## 9.9.9\n</pre>\n\n- real note\n" + assert [e.version for e in changelog_module.parse_changelog(text)] == ["1.0"] + assert "real note" in changelog_module.find_release_notes(text, "1.0").body + assert changelog_module.find_release_notes(text, "9.9.9") is None + + +def test_details_blocks_still_contain_markdown(changelog_module): + """<details> is a CommonMark type 6 block: headings inside it still count, + so collapsible sections keep working.""" + text = "## 2.0\n\n<details>\n<summary>More</summary>\n\n- note\n\n</details>\n\n## 1.0\n\n- older\n" + assert [e.version for e in changelog_module.parse_changelog(text)] == ["2.0", "1.0"] + + +def test_inline_raw_html_tag_does_not_open_a_block(changelog_module): + """A block opens only at the start of a line. A tag named mid-sentence is + inline HTML and must not swallow the releases below it.""" + text = "## 2.0\n\n- Warn when a <script> tag is pasted\n\n## 1.0\n\n- older\n" + assert [e.version for e in changelog_module.parse_changelog(text)] == ["2.0", "1.0"] + + +def test_preview_skips_raw_html_blocks(): + src = PREVIEW.read_text(encoding = "utf-8") + assert "stripRawHtml" in src + # Anchored: only a line-leading tag opens a block, matching the parser. + assert "/^ {0,3}<(pre|script|style|textarea)" in src + + +def test_fence_inside_a_raw_html_block_is_literal(changelog_module): + """Raw HTML contents are literal, so a stray ``` in a <pre> sample is not a + fence. Treating it as one left a block open and hid every later release.""" + text = "## 2.0\n\n<pre>\n```\n</pre>\n\n## 1.0\n\n- older\n" + assert [e.version for e in changelog_module.parse_changelog(text)] == ["2.0", "1.0"] + + +def test_raw_html_block_closes_on_any_of_the_four_tags(changelog_module): + """CommonMark ends a type 1 block at the first `</pre>`, `</script>`, + `</style>` or `</textarea>`: the closer need not match the opener.""" + text = '## 1.0\n\n<script>\nconst sample = "</pre>";\n## 9.9.9\n</script>\n' + assert [e.version for e in changelog_module.parse_changelog(text)] == ["1.0", "9.9.9"] + + +@pytest.mark.parametrize("tag", ["details", "div", "table"]) +def test_type_6_blocks_run_until_a_blank_line(changelog_module, tag): + """`<details>` holds Markdown only after a blank line closes the block, so + a heading pressed against the opening tag is not a release.""" + packed = f"## 1.0\n\n<{tag}>\n## 9.9.9\n</{tag}>\n\n- note\n" + assert [e.version for e in changelog_module.parse_changelog(packed)] == ["1.0"] + spaced = f"## 1.0\n\n<{tag}>\n\n## 2.0\n\n- note\n" + assert [e.version for e in changelog_module.parse_changelog(spaced)] == ["1.0", "2.0"] + + +def test_a_tag_only_line_cannot_interrupt_a_paragraph(changelog_module): + """Type 7 blocks do not interrupt a paragraph, so prose followed by a bare + tag keeps the releases below it reachable.""" + text = "## 2.0\n\nSome prose.\n<span>\n\n## 1.0\n\n- older\n" + assert [e.version for e in changelog_module.parse_changelog(text)] == ["2.0", "1.0"] + + +def test_preview_joins_an_indented_continuation_line(): + """Four spaces only start code outside a paragraph. Inside one the line is + a wrapped continuation, so it must not be dropped from the preview.""" + src = PREVIEW.read_text(encoding = "utf-8") + # Measured from the line's container, so an item's own indent does not count. + assert "!insideBlock && line.indent - line.column >= INDENTED_CODE_INDENT" in src + # A fence indented into a list item is a block, not a wrapped line. + assert "opensDeepFence" in src + + +def test_every_packaging_path_snapshots_the_changelog(): + """`python -m build` and `pip install .` must ship the offline copy too, + so the snapshot is made by the build backend rather than by build.sh.""" + pyproject = (REPO / "pyproject.toml").read_text(encoding = "utf-8") + assert 'build_py = "_changelog_build.build_py"' in pyproject + hook = (REPO / "_changelog_build.py").read_text(encoding = "utf-8") + assert "studio" in hook and "CHANGELOG.md" in hook + # The hook has to reach the sdist, or building from one loses the snapshot. + manifest = (REPO / "MANIFEST.in").read_text(encoding = "utf-8") + assert "include _changelog_build.py" in manifest + assert "include CHANGELOG.md" in manifest + + +def test_preview_code_spans_need_a_matching_closer(): + """A closer is a run of the same length, so ``Use `` `x` `` `` keeps the + inner backticks the expanded notes show.""" + src = CODE_SPANS.read_text(encoding = "utf-8") + assert "candidate === ticks" in src, "a closer is a run of the same length" + assert "stripPadding" in src, "one space of padding is dropped, as in Markdown" + + +def test_preview_skips_thematic_breaks(): + """`- - -` renders as a rule, so it must not take a preview slot.""" + src = PREVIEW.read_text(encoding = "utf-8") + assert "THEMATIC_BREAK" in src + assert "THEMATIC_BREAK.test(visible)" in src + + +def test_preview_keeps_quoted_examples_out_of_the_headlines(): + """A quoted list is example output, not a change, so it never competes + with the release's own bullets.""" + src = PREVIEW.read_text(encoding = "utf-8") + assert "quoted: boolean" in src + assert "if (!line.quoted)" in src, "quoted bullets never become headlines" + + +def test_notes_panel_keeps_the_link_when_the_lookup_fails(): + """Retry is not the only route: the changelog page can be reachable even + when the backend lookup is not.""" + src = PANEL.read_text(encoding = "utf-8") + error_branch = src[src.index('if (state === "error")') :] + retry = error_branch.index("update-release-notes-retry") + assert error_branch.index("{link}") > retry, "link sits beside retry" + + +def test_hook_waits_for_the_desktop_auth_token(): + """The desktop popup can render before auto-auth installs its token, so a + missing token must not be recorded as a failed lookup.""" + src = NOTES_HOOK.read_text(encoding = "utf-8") + assert "hasAuthToken()" in src and "AUTH_POLL_LIMIT" in src + + +def test_installed_layout_prefers_the_bundled_changelog(tmp_path): + """Installed, the levels above studio/ are site-packages. A stray + CHANGELOG.md left there by another package must not outrank the bundled + snapshot, so those levels are only searched in a source checkout.""" + site_packages = tmp_path / "site-packages" + package = site_packages / "studio/backend/utils" + package.mkdir(parents = True) + for name in ("changelog.py", "update_status.py"): + shutil.copy(BACKEND / "utils" / name, package / name) + for parent in (site_packages / "studio", package.parent, package): + (parent / "__init__.py").write_text("", encoding = "utf-8") + (site_packages / CHANGELOG.name).write_text("## 2.0\n\n- stray\n", encoding = "utf-8") + bundled = site_packages / "studio" / CHANGELOG.name + bundled.write_text("## 2.0\n\n- bundled\n", encoding = "utf-8") + + env = {**os.environ, "PYTHONPATH": str(site_packages)} + env.pop("UNSLOTH_CHANGELOG_PATH", None) + + def served() -> str: + # cwd is outside the checkout, so this imports the installed copy. + return subprocess.run( + [ + sys.executable, + "-c", + "from studio.backend.utils import changelog\n" + "print(changelog._read_local_changelog().text)", + ], + capture_output = True, + text = True, + env = env, + cwd = tmp_path, + check = True, + ).stdout + + assert "bundled" in served() and "stray" not in served() + + # A checkout marker there means it really is a repo root, so it wins again. + (site_packages / "pyproject.toml").write_text("", encoding = "utf-8") + assert "stray" in served() + + +def test_a_section_staged_as_a_comment_reads_as_unpublished( + changelog_module, tmp_path, monkeypatch +): + """Notes staged inside <!-- --> render as nothing, so the popup must say + no notes were published rather than show an empty surface.""" + monkeypatch.setenv(changelog_module.DISABLE_ENV_VAR, "1") + local = tmp_path / "CHANGELOG.md" + local.write_text("## 2.0\n\n<!-- not ready -->\n\n## 1.0\n\n- shipped\n", encoding = "utf-8") + monkeypatch.setenv(changelog_module.CHANGELOG_PATH_ENV_VAR, str(local)) + changelog_module.reset_changelog_cache() + try: + staged = changelog_module.get_release_notes("2.0") + assert staged["matched"] is False and staged["markdown"] is None + assert changelog_module.get_release_notes("1.0")["matched"] is True + finally: + changelog_module.reset_changelog_cache() + + +@pytest.mark.parametrize( + "body,visible", + [ + ("- note", True), + ("<!-- staged -->", False), + ("```\n```", True), + ("<pre>\n</pre>", True), + (" ", False), + ], +) +def test_visibility_check_only_hides_comments(changelog_module, body, visible): + assert changelog_module._renders_visibly(body) is visible + + +@pytest.mark.parametrize( + "block", + [ + "<?php\n## 9.9.9\n?>", + "<![CDATA[\n## 9.9.9\n]]>", + "<!DOCTYPE\n## 9.9.9\n>", + ], +) +def test_processing_instructions_and_declarations_are_literal(changelog_module, block): + """Raw block types 3 to 5 render literally, like <pre>, so a heading inside + one is a sample and not a release.""" + text = f"## 1.0\n\n{block}\n\n- real note\n" + assert [e.version for e in changelog_module.parse_changelog(text)] == ["1.0"] + assert "real note" in changelog_module.find_release_notes(text, "1.0").body + + +def test_headings_need_a_space_or_tab_after_the_hashes(changelog_module): + """A non-breaking space pasted from rich text renders as ordinary text, so + the line must not end the release above it.""" + text = "## 1.0\n\n- real note\n\n## 9.9.9\n\n- not a release\n" + assert [e.version for e in changelog_module.parse_changelog(text)] == ["1.0"] + assert changelog_module.find_release_notes(text, "9.9.9") is None + # A tab is valid and still opens a heading. + tabbed = "## 1.0\n\n- one\n\n##\t2.0\n\n- two\n" + assert [e.version for e in changelog_module.parse_changelog(tabbed)] == ["1.0", "2.0"] + + +def test_preview_skips_every_raw_block_form(): + """The extractor tracks the same block forms as the parser, so a sample + bullet inside one cannot become the collapsed headline.""" + src = PREVIEW.read_text(encoding = "utf-8") + assert "RAW_BLOCKS" in src + assert "CDATA" in src and "[A-Za-z]" in src + + +@pytest.mark.parametrize("banner", [WEB_BANNER, TAURI_BANNER]) +def test_expanded_popup_fits_a_short_viewport(banner): + """A window under roughly 430px high used to push the card's title and + dismiss control above the top of the screen.""" + panel = PANEL.read_text(encoding = "utf-8") + # The notes region shrinks inside the capped card, so header and actions stay on screen. + assert "min-h-0 flex-1" in panel, "notes height must follow the viewport" + src = banner.read_text(encoding = "utf-8") + assert "max-h-[calc(100dvh_-_2rem)]" in src, "card is the backstop on tiny viewports" + + +def test_relative_changelog_links_point_at_the_repository(): + """CHANGELOG.md links are repository-relative. Rendered as-is they resolve + against Studio's origin, so the renderer blocks them.""" + src = LINKS.read_text(encoding = "utf-8") + assert "https://github.com/unslothai/unsloth/blob/main/" in src + assert "https://raw.githubusercontent.com/unslothai/unsloth/main/" in src + # Absolute targets, fragments, fenced code and code spans stay untouched. + assert "ABSOLUTE" in src and "codeSpans" in src and "FENCE" in src + panel = PANEL.read_text(encoding = "utf-8") + assert "resolveChangelogLinks" in panel + + +@pytest.mark.parametrize("query", ["latest", "main", "not-a-version", "abc"]) +def test_unparseable_versions_are_rejected(changelog_module, query): + """Sections are indexed only when their version parses, so a query that + cannot parse can never match and is a bad request, not an empty result.""" + assert changelog_module.is_supported_version_query(query) is False + + +@pytest.mark.parametrize("query", ["2026.7.5", "v2026.7.5", "2026.07.5", "1.0.0rc1"]) +def test_real_versions_are_still_accepted(changelog_module, query): + assert changelog_module.is_supported_version_query(query) is True + + +def test_reference_style_images_resolve_to_the_raw_host(): + """`![alt][arch]` with `[arch]: docs/arch.png` needs the raw file: the blob + URL is an HTML page, so the image would not load.""" + src = LINKS.read_text(encoding = "utf-8") + assert "IMAGE_REFERENCE" in src + assert "imageLabels" in src + + +def test_collapsed_notes_surface_is_hidden_when_nothing_previews(): + """Notes that are only a fenced command block preview as nothing, and an + empty muted strip is worse than no strip.""" + src = PANEL.read_text(encoding = "utf-8") + assert "preview?.items.length === 0" in src + + +def test_a_fence_closer_accepts_only_spaces_and_tabs(changelog_module): + """A delimiter followed by a non-breaking space is code content, so it must + not close the block and let a sample heading through.""" + text = "## 1.0\n\n```\n```\u00a0\n## 9.9.9\n```\n\n- real note\n" + assert [e.version for e in changelog_module.parse_changelog(text)] == ["1.0"] + plain = "## 1.0\n\n```\nx\n```\t\n\n## 2.0\n\n- two\n" + assert [e.version for e in changelog_module.parse_changelog(plain)] == ["1.0", "2.0"] + # The same rule in both frontend scanners. + for source in (PREVIEW, LINKS): + assert "/[^ \\t]/" in source.read_text(encoding = "utf-8") + + +def test_code_spans_close_on_a_run_of_equal_length(): + """`a``b [x](y.md)` is one code span, so the link inside it is literal.""" + src = CODE_SPANS.read_text(encoding = "utf-8") + assert "candidate === ticks" in src, "closer length must match the opener" + # Shared, so the preview and the link resolver cannot drift apart. + assert "markdown-code-spans" in PREVIEW.read_text(encoding = "utf-8") + assert "markdown-code-spans" in LINKS.read_text(encoding = "utf-8") + + +def test_preview_decodes_entities_like_the_renderer(): + """Streamdown renders `AT&T` as AT&T, so the collapsed preview must + not show the raw entity.""" + src = PREVIEW.read_text(encoding = "utf-8") + assert "NAMED_ENTITIES" in src and "decodeEntity" in src + # Decoded before code spans are restored, so code keeps the literal text. + assert src.index(".replace(ENTITY, decodeEntity)") < src.index(".replace(PARKED") + + +def test_release_notes_request_refreshes_an_expired_token(): + """A direct fetch cannot recover from a 401; authFetch refreshes first.""" + src = NOTES_HOOK.read_text(encoding = "utf-8") + assert "authFetch(" in src + assert "getAuthToken" not in src + + +def test_preview_handles_the_desktop_updater_line_endings(): + """The updater body arrives with CRLF, which used to hide fences from the + extractor and promote a code sample to a headline.""" + src = PREVIEW.read_text(encoding = "utf-8") + assert "LINE_ENDINGS" in src + assert "LINE_ENDINGS" in LINKS.read_text(encoding = "utf-8") + + +def test_preview_renders_reference_links_as_text(): + """`[text][label]` and `![alt][label]` render as a link and an image, so + the preview must not show their raw markup.""" + src = PREVIEW.read_text(encoding = "utf-8") + assert "LINK_REFERENCE" in src and "IMAGE_REFERENCE" in src + # A definition line renders as nothing, so it is not a preview item. + assert "DEFINITION" in src + + +def test_preview_treats_escaped_punctuation_as_literal(): + """`\\*not italic\\*` keeps its stars and an escaped backtick does not open + a code span.""" + assert "ESCAPE" in PREVIEW.read_text(encoding = "utf-8") + assert "escaped(" in CODE_SPANS.read_text(encoding = "utf-8") + + +def test_link_resolver_skips_every_code_form(): + """Indented code and code spans crossing a line render as code, so their + contents must not be rewritten.""" + src = LINKS.read_text(encoding = "utf-8") + assert "INDENTED_CODE" in src + # Spans are scanned over the whole document, not line by line. + assert "codeSpans(masked)" in src + # A definition cannot interrupt a paragraph. + assert "definition.has(index)" in src + + +def test_badge_links_resolve_both_targets(): + """`[![alt](img)](link)` is the badge idiom: the outer link used to stay + relative because the label was not allowed to nest.""" + assert "NESTED_LABEL" in LINKS.read_text(encoding = "utf-8") + + +def test_in_flight_requests_are_identified_not_just_versioned(): + """Two requests for the same version could resolve out of order and leave + the panel showing the older result.""" + assert "requestIdRef" in NOTES_HOOK.read_text(encoding = "utf-8") + + +def test_notes_repair_the_shared_previews_width_reset(): + """MarkdownPreview clears max-width on every descendant, so a wide image + and the renderer's own link dialog escape the card.""" + src = PANEL.read_text(encoding = "utf-8") + assert "[&_img]:max-w-full" in src + assert "[&_[data-streamdown=link-safety-modal]>*]:max-w-md" in src + + +@pytest.mark.parametrize("banner", [WEB_BANNER, TAURI_BANNER]) +def test_only_the_notes_region_scrolls(banner): + """The dismiss control sits inside the card, so scrolling the card itself + carried it off screen on a short viewport.""" + src = banner.read_text(encoding = "utf-8") + assert "flex max-h-[calc(100dvh_-_2rem)] flex-col overflow-hidden" in src + assert 'className="min-h-0 flex-1"' in src + panel = PANEL.read_text(encoding = "utf-8") + assert "max-h-64 min-h-0 flex-1 overflow-y-auto" in panel + + +def test_a_comment_marker_in_prose_cannot_swallow_later_releases(changelog_module): + """A note that mentions `<!--` used to put the parser into comment state + for the rest of the file: the releases below it disappeared and their + notes were served under the newer version's heading.""" + text = ( + "## 2026.8.0\n\n- Studio strips <!-- markers from pasted prompts.\n\n" + "## 2026.7.5\n\n- SECRET: an older release\n" + ) + assert [e.version for e in changelog_module.parse_changelog(text)] == [ + "2026.8.0", + "2026.7.5", + ] + assert "SECRET" not in changelog_module.find_release_notes(text, "2026.8.0").body + assert changelog_module.find_release_notes(text, "2026.7.5") is not None + # A comment that starts a line is still a block and still hides its body. + hidden = "## 2.0\n\n<!--\n## 9.9.9\n-->\n\n- note\n" + assert [e.version for e in changelog_module.parse_changelog(hidden)] == ["2.0"] + + +def test_unmatched_backtick_runs_stay_linear(changelog_module): + """Rescanning the suffix for every opener was quadratic: a line of runs of + 1, 2, 3 ... backticks, none of which ever closes, took 7.7s at 321 KB and + is reparsed on every popup request, so one malformed remote changelog could + tie up backend workers.""" + line = "".join("`" * (i + 1) + "x" for i in range(800)) + assert len(line) > 300_000 + started = time.monotonic() + assert changelog_module._code_span_ranges(line) == [] + assert time.monotonic() - started < 2.0 + + +def test_a_base_exception_releases_the_single_flight_flag(changelog_module, monkeypatch): + """The flag was cleared only after `except Exception`, so a BaseException + (KeyboardInterrupt, SystemExit, CancelledError) stranded it and every later + caller then waited out the full deadline for the life of the process.""" + changelog_module.reset_changelog_cache() + + def explode(): + raise KeyboardInterrupt + + monkeypatch.setattr(changelog_module, "_fetch_remote_changelog", explode) + with pytest.raises(KeyboardInterrupt): + changelog_module.get_remote_changelog() + assert changelog_module._remote_fetching is False + changelog_module.reset_changelog_cache() + + +@pytest.mark.parametrize("marker", ["<!-->", "<!--->"]) +def test_an_empty_comment_does_not_swallow_later_releases(changelog_module, marker): + """`<!-->` and `<!--->` are complete comments in CommonMark: the closer + overlaps the opener. Searching for `-->` past the opener missed them, so an + empty comment used as a section marker hid every release below it.""" + text = f"## 2.0\n\n- new stuff\n\n{marker}\n\n## 1.0\n\n- old stuff\n" + assert [e.version for e in changelog_module.parse_changelog(text)] == ["2.0", "1.0"] + assert changelog_module.find_release_notes(text, "1.0") is not None + assert "old stuff" not in changelog_module.find_release_notes(text, "2.0").body + # The frontend scanner has to agree, or the preview and the body disagree. + assert "!line.includes(COMMENT_CLOSE)" in PREVIEW.read_text(encoding = "utf-8") + + +def test_an_unterminated_comment_still_hides_the_rest(changelog_module): + """The fix must not turn every `<!--` line into a no-op block.""" + text = "## 2.0\n\n<!-- never closed\n\n## 1.0\n\n- old stuff\n" + assert [e.version for e in changelog_module.parse_changelog(text)] == ["2.0"] + + +def test_a_closing_delimiter_takes_its_whole_line(changelog_module): + """CommonMark keeps the closing line inside the block, so a heading glued + after `-->` or `</pre>` is not a release.""" + for text in ( + "## 1.0\n\n<!-- hidden -->## 9.9.9\n\n- note\n", + "## 1.0\n\n<pre>\nx\n</pre>## 9.9.9\n\n- note\n", + ): + assert [e.version for e in changelog_module.parse_changelog(text)] == ["1.0"] + + +def test_an_exact_heading_is_never_shadowed(changelog_module): + """PEP 440 says 1.0 == 1.0.0, so the normalised match used to win even + when the file had a section spelled exactly as asked.""" + text = "## 1.0.0\n\n- padded\n\n## 1.0\n\n- exact\n" + assert changelog_module.find_release_notes(text, "1.0").body == "- exact" + assert changelog_module.find_release_notes(text, "1.0.0").body == "- padded" + # Normalised matching still applies when there is no exact heading. + assert changelog_module.find_release_notes("## 2026.7.6\n\n- x\n", "2026.07.6") is not None + + +def test_setext_headings_are_release_boundaries(changelog_module): + """A version over a line of dashes is the same heading in setext form.""" + text = "2.0\n---\n\n- new\n\n1.0\n---\n\n- old\n" + assert [e.version for e in changelog_module.parse_changelog(text)] == ["2.0", "1.0"] + assert changelog_module.find_release_notes(text, "2.0").body == "- new" + # A rule between sections is still a rule, and a setext h1 is not a release. + assert [ + e.version + for e in changelog_module.parse_changelog("## 2.0\n\n- a\n\n---\n\n## 1.0\n\n- b\n") + ] == ["2.0", "1.0"] + + +def test_a_long_backtick_run_does_not_stall_the_parser(changelog_module): + """The code-span guard used to backtrack: 20k backticks took over a minute + and every request re-parsed the file.""" + import time + + text = "## 1.0\n\n- " + "`" * 20_000 + " <!--\n" + started = time.perf_counter() + changelog_module.parse_changelog(text) + assert time.perf_counter() - started < 1.0 + + +def test_the_remote_fetch_has_a_total_deadline(changelog_module): + """The socket timeout resets on every read, so a trickling server could + hold a worker for minutes and still be treated as a success.""" + source = (BACKEND / "utils/changelog.py").read_text(encoding = "utf-8") + assert "deadline = time.monotonic() + CHANGELOG_TIMEOUT_SECONDS" in source + # read1 returns after one socket read, so the deadline is actually checked. + assert "response.read1(" in source + # Waiters give up rather than queue behind a stalled fetch. + assert "Release notes are still loading." in source + + +def test_truncated_notes_close_their_fence(changelog_module): + """A blind slice could end inside a code block and break the rendering.""" + body = "```\n" + "x\n" * 20_000 + "```\n" + payload = changelog_module._notes_response(version = "1.0", markdown = body, source = "local") + assert payload["truncated"] is True + assert payload["markdown"].rstrip().endswith("```") + + +def test_the_opt_out_beats_the_developer_override(): + """UNSLOTH_STUDIO_FAKE_UPDATE is a dev switch; the documented kill switch + still wins, and the value has to parse as a version.""" + source = (BACKEND / "utils/update_status.py").read_text(encoding = "utf-8") + assert "forced_version and not disabled and _is_version(forced_version)" in source + + +def test_a_list_item_over_dashes_is_not_a_setext_heading(changelog_module): + """`- first` followed by `---` is a list and a rule. Reading it as a + heading discarded the bullet and the rest of the section with it.""" + text = "## 1.0\n\n- first\n---\n\n- second\n" + assert [e.version for e in changelog_module.parse_changelog(text)] == ["1.0"] + body = changelog_module.find_release_notes(text, "1.0").body + assert "first" in body and "second" in body + # Real setext headings still work. + setext = "2.0\n---\n\n- new\n\n1.0\n---\n\n- old\n" + assert [e.version for e in changelog_module.parse_changelog(setext)] == ["2.0", "1.0"] + + +def test_a_backtick_in_a_fence_info_string_is_not_a_fence(changelog_module): + """CommonMark forbids backticks in a backtick fence's info string, so such + a line is prose and must not swallow the releases below it.""" + text = "## 2.0\n\n```bad`info\n\n## 1.0\n\n- old\n" + assert [e.version for e in changelog_module.parse_changelog(text)] == ["2.0", "1.0"] + # A tilde fence may hold backticks, and a normal fence still hides samples. + assert [ + e.version + for e in changelog_module.parse_changelog( + "## 2.0\n\n```md\n## 9.9.9\n```\n\n## 1.0\n\n- old\n" + ) + ] == ["2.0", "1.0"] + for source in (PREVIEW, LINKS): + assert "info string" in source.read_text(encoding = "utf-8") + + +def test_preview_follows_commonmark_paragraph_rules(): + """Only an ordered list starting at 1 may interrupt a paragraph, and an + unresolved reference keeps its brackets. A quote owns the paragraph its own + lines hold, so a marker written outside the quote interrupts nothing.""" + src = " ".join(PREVIEW.read_text(encoding = "utf-8").split()) + assert "const interrupts = collector.current === null" in src + assert "!collector.quotedParagraph;" in src + assert "definedLabel" in src, "a reference only renders as text when defined" + # A comment written mid-sentence hides its own line at most. + assert "COMMENT_BLOCK_OPEN" in src + + +def test_link_resolver_leaves_raw_blocks_and_escapes_alone(): + src = LINKS.read_text(encoding = "utf-8") + assert "RAW_HTML_OPEN" in src and "inRawHtml" in src + assert "isEscaped(line, opener)" in src + # A heading ends a paragraph, so a definition under one is a definition. + assert "BLOCK_LINE.test(structure)" in src + + +def test_code_span_closers_ignore_backslashes(): + """Escapes are not processed inside a code span, so a run after a + backslash still closes it.""" + src = CODE_SPANS.read_text(encoding = "utf-8") + body = src[src.index("export function codeSpans") :] + assert body.count("escaped(text") == 1, "only an opener can be escaped" + + +def test_the_overlay_stack_fits_the_viewport(): + """The update card's own cap does not account for a long download list + stacked beneath it.""" + provider = (FRONTEND / "app/provider.tsx").read_text(encoding = "utf-8") + assert "max-h-[calc(100dvh_-_2rem)]" in provider + panel = (FRONTEND / "features/hub/download-manager/download-manager-panel.tsx").read_text( + encoding = "utf-8" + ) + # Both overlays scroll internally, so they can give up height. + assert "flex min-h-0" in panel + assert "flex min-h-0" in WEB_BANNER.read_text(encoding = "utf-8") + + +def test_the_desktop_stack_is_capped_like_the_browser_one(): + """The download panel shares the desktop stack, so the update card's own + cap is not enough there either.""" + provider = (FRONTEND / "app/provider.tsx").read_text(encoding = "utf-8") + assert provider.count("max-h-[calc(100dvh_-_2rem)]") == 2, "both stacks are capped" + assert "flex min-h-0" in TAURI_BANNER.read_text(encoding = "utf-8") + + +def test_desktop_notes_are_looked_up_by_the_backend_version(): + """latest.json's `version` is the app SemVer while CHANGELOG.md is keyed by + the backend release, so the desktop popup used to find no section at all + and fall back to the updater's generic text.""" + workflow = (REPO / ".github/workflows/release-desktop.yml").read_text(encoding = "utf-8") + assert "'pypi_version': os.environ['PYPI_VERSION']" in workflow + assert "PYPI_VERSION: ${{ needs.prepare-version.outputs.pypi_version }}" in workflow + rust = (REPO / "studio/src-tauri/src/desktop_update_policy.rs").read_text(encoding = "utf-8") + assert "pypi_version: Option<String>" in rust + hook = NOTES_HOOK.parent.joinpath("use-tauri-update.ts").read_text(encoding = "utf-8") + # Both desktop paths carry it: the plugin exposes the raw metadata. + assert "rawPypiVersion(update.rawJson)" in hook + assert "manualUpdate.pypiVersion" in hook + banner = TAURI_BANNER.read_text(encoding = "utf-8") + assert "info?.pypiVersion ?? info?.version" in banner + + +def test_one_slow_read_cannot_outlast_the_fetch_budget(changelog_module): + """The socket timeout is per operation, so slow headers followed by a slow + body could hold a worker for twice the advertised deadline.""" + source = (BACKEND / "utils/changelog.py").read_text(encoding = "utf-8") + assert "_limit_read(response, remaining)" in source + assert "sock.settimeout(max(remaining, _CHANGELOG_MIN_READ_SECONDS))" in source + + +def test_a_heading_indented_into_a_list_item_is_not_a_release(changelog_module): + """CommonMark keeps a heading at the item's content column inside the item. + Treating it as a boundary truncated the real release and indexed a version + that does not exist. Checked against markdown-it (commonmark preset).""" + text = "## 1.0\n\n- Example:\n ## 9.9.9\n\n- after\n" + assert [e.version for e in changelog_module.parse_changelog(text)] == ["1.0"] + body = changelog_module.find_release_notes(text, "1.0").body + assert "9.9.9" in body and "after" in body + # One space short of the content column, the list ends and it is a release. + left = "## 1.0\n\n- Example:\n ## 2.0\n" + assert [e.version for e in changelog_module.parse_changelog(left)] == ["1.0", "2.0"] + + +def test_a_closed_list_stops_holding_headings(changelog_module): + """Only an open item nests a heading, so a dedented paragraph, heading, + break or fence hands the following indentation back to the document.""" + + def versions(text): + return [e.version for e in changelog_module.parse_changelog(text)] + + assert versions("## 1.0\n\n- Example:\n\nText.\n\n ## 2.0\n") == ["1.0", "2.0"] + assert versions("## 1.0\n\n- Example:\n## 2.0\n ## 3.0\n") == ["1.0", "2.0", "3.0"] + assert versions("## 1.0\n\n- Example:\n Text.\n---\n ## 2.0\n") == ["1.0", "2.0"] + assert versions("## 1.0\n\n- Example:\n```\n```\n ## 2.0\n") == ["1.0", "2.0"] + # An item may begin with one blank line; content after that is outside it. + assert versions("## 1.0\n\n-\n\n ## 2.0\n") == ["1.0", "2.0"] + + +def test_a_version_line_is_not_an_ordered_list_marker(changelog_module): + """`2.` needs whitespace after it to be a marker, or list tracking would + read every setext version as a list item and lose the heading.""" + text = "2.0\n---\n\n- new\n\n1.0\n---\n\n- old\n" + assert [e.version for e in changelog_module.parse_changelog(text)] == ["2.0", "1.0"] + # An ordered item interrupts a paragraph only when it starts at 1. + assert [ + e.version for e in changelog_module.parse_changelog("## 1.0\n\nText.\n9) one\n ## 2.0\n") + ] == ["1.0", "2.0"] + + +def test_a_wrapped_setext_heading_is_still_a_release(changelog_module): + """CommonMark promotes the whole paragraph, so a heading that wraps keeps + the version in its first token. Reading only the last line left the release + unindexed and its notes unreachable.""" + text = "2026.7.5 - Release\nJuly 25\n---\n\n- note\n" + entries = changelog_module.parse_changelog(text) + assert [e.version for e in entries] == ["2026.7.5"] + # The heading lines are the heading, not the body. + assert entries[0].body == "- note" + assert "July 25" not in entries[0].body + + +def test_a_lowercase_declaration_is_not_a_raw_block(changelog_module): + """Only `<!` plus an uppercase letter opens one, so prose that mentions + `<!note` must not hide every release under it.""" + assert [ + e.version for e in changelog_module.parse_changelog("<!note\n\n## 1.0\n\n- real\n") + ] == ["1.0"] + # A real declaration still hides its own block. + assert [ + e.version for e in changelog_module.parse_changelog("<!DOCTYPE\n## 9.9.9\n>\n\n## 1.0\n") + ] == ["1.0"] + # The collapsed preview needs the same rule or it drops visible bullets. + assert "<![A-Z]" in PREVIEW.read_text(encoding = "utf-8") + + +def test_link_resolver_reads_html_containers_the_way_the_others_do(): + """A `<details>` or `<div>` with no blank line inside is a type 6 block, so + its contents render literally. Rewriting a link there mutates text the + reader sees verbatim, and a fence inside such a block was being taken for a + real fence, which stopped every link below it from resolving at all. The + backend parser and the collapsed preview already apply the type 6 and 7 + rules, so the resolver has to share them or the three disagree on the same + notes.""" + links = LINKS.read_text(encoding = "utf-8") + for source in (PREVIEW, LINKS): + text = source.read_text(encoding = "utf-8") + assert "HTML_BLOCK_TAGS" in text and "HTML_TAG_ONLY_LINE" in text + # A blank line ends the block, not the closing tag, and a bare quote marker counts as blank. + assert "inHtmlBlock = !!container.trim()" in links + # Type 7 cannot interrupt a paragraph, so prose above it keeps its links. + assert "return !afterParagraph && HTML_TAG_ONLY_LINE.test(line);" in links + + +def test_an_escaped_mark_makes_an_image_a_link(): + """`\\![alt](path)` renders as a link, so it resolves to the file's page on + GitHub rather than to the raw-content host.""" + links = LINKS.read_text(encoding = "utf-8") + assert 'const image = bang === "!" && !isEscaped(line, offset);' in links + # The reference pre-scan has to skip it too, or the definition flips host. + assert "isEscaped(line, match.index)" in links + + +def test_only_markdown_line_endings_split_the_changelog(changelog_module): + """str.splitlines also breaks on U+2028, U+2029, NEL, vertical tab and form + feed, none of which end a line in CommonMark. A separator sitting in prose + ahead of "## 9.9.9" made the parser index a release the renderer never shows + and truncate the notes above it.""" + text = "## 2.0\n\nnote with a separator 
## 9.9.9\n\n## 1.0\n\n- old\n" + assert [e.version for e in changelog_module.parse_changelog(text)] == ["2.0", "1.0"] + # The prose stays whole rather than being cut at the separator. + entry = changelog_module.find_release_notes(text, "2.0") + assert entry is not None and "9.9.9" in entry.body + for separator in ("
", "\x85", "\x0b", "\x0c"): + broken = f"## 2.0\n\nnote{separator}## 9.9.9\n\n## 1.0\n\n- old\n" + assert [e.version for e in changelog_module.parse_changelog(broken)] == ["2.0", "1.0"] + # The three real line endings still split. + for ending in ("\n", "\r\n", "\r"): + real = f"## 2.0{ending}{ending}- new{ending}{ending}## 1.0{ending}{ending}- old{ending}" + assert [e.version for e in changelog_module.parse_changelog(real)] == ["2.0", "1.0"] + + +def test_the_build_does_not_require_a_writable_source_tree(): + """A PEP 517 build may run against an immutable checkout (Nix, Bazel, a + read-only container mount). Writing the snapshot beside the sources raised + PermissionError before build_py started, so no wheel could be built at all. + """ + src = (REPO / "_changelog_build.py").read_text(encoding="utf-8") + # The source-tree copy is best effort. + assert "except OSError:" in src + # The wheel gets its copy from the staging directory either way. + assert 'Path(self.build_lib) / "studio" / "CHANGELOG.md"' in src + + +def test_link_resolver_reads_comments_before_fences(): + """A fence delimiter hidden inside an HTML comment is not a fence. Reading + it as one left the fence open, so every visible line below was classified as + code and none of its links were resolved, which is far worse than the + mutated-text case: the whole rest of the notes silently stops working. The + order matters both ways, so a comment opener inside a real fence is not a + comment either.""" + links = LINKS.read_text(encoding="utf-8") + # Fence state is read before comments are masked, the order the collapsed preview uses. + assert "const fenceSource = inComment\n ? null\n : FENCE.exec(" in links + # Masking happens only after the in-fence early return. + fence_return = links.index("// Fenced content is literal") + assert links.index("const [line, stillInComment, stillRunOn] = maskComments(") > fence_return + # Commented ranges join the code spans, so a hidden link is left alone. + assert "const spans = [...codeSpans(masked), ...comments].sort(" in links + + +def test_preview_heading_and_quote_markers_follow_the_backend_rule(): + """An ATX heading needs an ASCII space, a tab or the end of the line after + the marker, which is what _HEADING_PATTERN requires; `\\s` also matches a + non-breaking space, so prose beginning "## Important change" with one was + read as a heading and dropped, leaving a prose-only release with no + collapsed preview at all. A blockquote marker takes at most three leading + spaces for the same reason every other marker here does: accepting any run + let an indented code sample containing "> - sample output" shed its + indentation and be shown as the summary.""" + src = PREVIEW.read_text(encoding="utf-8") + assert "const HEADING = /^#{1,6}(?:[ \\t]|$)/;" in src + assert "const HEADING_LINE = /^ {0,3}#{1,6}(?:[ \\t]|$)/;" in src + assert "const BLOCKQUOTE = /^ {0,3}>[ \\t]?/;" in src + # The backend rule this mirrors. + backend = (BACKEND / "utils" / "changelog.py").read_text(encoding="utf-8") + assert "^ {0,3}##(?:[ \\t]+(?P<title>.*?))?[ \\t]*$" in backend + + +def test_preview_collects_labels_only_from_real_definitions(): + """A definition-shaped line inside an indented code block or a deep fence is + literal text, so CommonMark leaves a later "[Beta] support" unresolved with + its brackets showing. Recording the label anyway made toPlainText strip them + in the collapsed preview, so it disagreed with the expanded view. The + pre-scan skips the same code the collector pass skips; a real definition + takes at most three spaces of indentation, so the indent test cannot reject + one.""" + src = PREVIEW.read_text(encoding="utf-8") + scan = src.index("const labels = new Set<string>();") + collect = src.index("let deepFence: string | null = null;") + prescan = " ".join(src[scan:collect].split()) + assert "let labelFence: string | null = null;" in prescan + assert "if (line.indent - line.column >= INDENTED_CODE_INDENT) { continue; }" in prescan + assert "endsDeepFence(labelFence, labelColumn, line)" in prescan + + +def test_an_html_block_to_the_left_of_a_list_item_closes_it(changelog_module): + """Types 1 to 6 interrupt a paragraph, so an unindented <div> after "- item" + closes the item and a following one-to-three-space-indented "## 2.0" is a + real document heading. It was read as a lazy paragraph continuation, so the + item stayed open and the release below the block was swallowed.""" + text = "## 3.0\n\n- item\n<div>\nhidden\n</div>\n\n ## 2.0\n\n- two\n\n## 1.0\n\n- one\n" + assert [e.version for e in changelog_module.parse_changelog(text)] == ["3.0", "2.0", "1.0"] + # Without the block the heading really is nested, so it stays suppressed. + nested = "## 3.0\n\n- item\n\n ## 2.0\n\n- two\n\n## 1.0\n\n- one\n" + assert [e.version for e in changelog_module.parse_changelog(nested)] == ["3.0", "1.0"] + # Ordinary lazy continuation is untouched. + lazy = "## 3.0\n\n- item\ncontinued\n\n ## 2.0\n\n## 1.0\n\n- one\n" + assert [e.version for e in changelog_module.parse_changelog(lazy)] == ["3.0", "1.0"] + + +def test_the_download_panel_can_shrink_inside_the_capped_stack(): + """The bottom-right stack is capped to the viewport, and a flex item defaults + to min-height:auto, so this wrapper could not shrink below its own content. + On a short viewport the cap was then absorbed by the update card, whose + header and actions are fixed, rather than by the download list, which + scrolls. Only the shared-stack branch needs it; standalone is positioned + fixed and is not a flex item at all.""" + panel = (FRONTEND / "features/hub/download-manager/download-manager-panel.tsx").read_text( + encoding="utf-8" + ) + assert 'positioned ? "fixed bottom-4 right-4 z-50" : "flex min-h-0 justify-end"' in panel + provider = (FRONTEND / "app/provider.tsx").read_text(encoding="utf-8") + assert "max-h-[calc(100dvh_-_2rem)]" in provider, "the cap this has to absorb" + + +@pytest.fixture(scope="module") +def run_scanner(tmp_path_factory): + """Run the frontend's markdown scanners under node. + + Their job is to classify a line the way a CommonMark renderer would, which + only a real run can show. The sources are copied with their "@/lib" aliases + rewritten, because that alias resolves through Vite and not through node.""" + node = shutil.which("node") + if node is None: + pytest.skip("node is needed to run the TypeScript scanners") + work = tmp_path_factory.mktemp("release-notes-scanners") + for source in (PREVIEW, CODE_SPANS, LINKS, LIST_COLUMNS, INLINE_COMMENTS): + rewritten = _TS_ALIAS.sub(r'"./\1.ts"', source.read_text(encoding="utf-8")) + (work / source.name).write_text(rewritten, encoding="utf-8") + (work / "run.ts").write_text(_TS_RUNNER, encoding="utf-8") + + def run(kind: str, markdown: str): + result = subprocess.run( + [node, "--experimental-strip-types", "--no-warnings", str(work / "run.ts"), kind], + input=markdown, + capture_output=True, + text=True, + ) + if result.returncode != 0: + pytest.skip(f"node could not run the scanners: {result.stderr.strip()[:200]}") + return json.loads(result.stdout) + + return run + + +def preview_leads(preview) -> list[str]: + return [item["lead"] for item in preview["items"]] + + +def test_a_link_indented_under_a_bullet_still_resolves(run_scanner): + """CommonMark measures indentation from the container, not the margin + (spec 0.31.2 section 5.2, list items). Under "- Details:" the content column + is 2, so a four-space line is only two columns in: a paragraph holding a + link, which GitHub renders and follows. The scanner measured from the margin + instead, called it an indented code block (section 4.4) and left the + destination relative, so the link resolved against Studio's own origin.""" + resolved = run_scanner("links", "- Details:\n\n [guide](docs/a.md)\n") + assert "https://github.com/unslothai/unsloth/blob/main/docs/a.md" in resolved + # The same prose one column further in really is code, and stays untouched. + code = run_scanner("links", "- Added.\n\n [guide](docs/a.md)\n") + assert "[guide](docs/a.md)" in code and "github.com" not in code + # At document level four spaces is code, so that link is still left alone. + top = run_scanner("links", "Intro.\n\n [guide](docs/a.md)\n") + assert "[guide](docs/a.md)" in top and "github.com" not in top + + +def test_an_indented_fence_does_not_swallow_the_bullets_below_it(run_scanner): + """A four-space line at document level is an indented code block, and a + top-level bullet is not indented enough to continue it, so the block ends + and the list renders. Promoting the line to a list-contained fence left a + block open with no closer, so every bullet after it was skipped and the + collapsed popup lost its summary.""" + swallowed = "Example:\n\n ```\n\n- Added the exporter\n- Fixed the crash\n" + assert preview_leads(run_scanner("preview", swallowed)) == [ + "Added the exporter", + "Fixed the crash", + ] + # With nothing else to fall back on the summary disappeared entirely. + assert preview_leads(run_scanner("preview", " ```\n\n- Added the exporter\n")) == [ + "Added the exporter" + ] + # A fence that really is inside an item still hides that item's code. + nested = "- a\n - b\n ```\n - not a bullet\n ```\n\n- Added tests\n" + assert preview_leads(run_scanner("preview", nested)) == ["a", "Added tests"] + + +def test_a_table_only_release_previews_as_nothing(run_scanner): + """A release written as a GFM table renders as a grid, and the panel treats + notes that preview as nothing by staying collapsed rather than showing an + empty strip. Falling through to the prose collector put the raw + "| Change | Detail | | --- | --- |" delimiters in the popup instead.""" + table = "| Change | Detail |\n| --- | --- |\n| Exporter | Added GGUF |\n" + assert run_scanner("preview", table)["items"] == [] + # A table after prose is dropped too, rather than joined onto it. + assert preview_leads(run_scanner("preview", f"Some prose.\n\n{table}")) == ["Some prose."] + # A bullet right after the rows ends the table, so it still previews. + assert preview_leads(run_scanner("preview", f"{table}- Added tests\n")) == ["Added tests"] + # Mismatched header and delimiter widths are no table, as on GitHub, so both lines are prose. + assert preview_leads(run_scanner("preview", "| a | b |\n| --- |\n")) == ["| a | b | | --- |"] + + +def test_a_fence_inside_a_list_item_ends_with_the_item(changelog_module): + """A fence is scoped to its container: with no closer it runs to the end of + the containing block, not the document (spec 0.31.2 section 4.5). A + dedented "## 2.0" closes the list item, so it is a real release heading. + Document-wide fence state kept the block open and hid every release below + it, so one missing closing line emptied the rest of the changelog.""" + text = "## 1.0\n\n- item\n ```\n\n## 2.0\n\n- two\n" + assert [e.version for e in changelog_module.parse_changelog(text)] == ["1.0", "2.0"] + # A fence at document level still runs to the end of the file. + top = "## 1.0\n\n```\n\n## 2.0\n\n- two\n" + assert [e.version for e in changelog_module.parse_changelog(top)] == ["1.0"] + # A closed fence inside an item is unaffected, and its sample stays hidden. + closed = "## 1.0\n\n- Run:\n ```bash\n ## 9.9.9\n ```\n\n## 2.0\n\n- two\n" + assert [e.version for e in changelog_module.parse_changelog(closed)] == ["1.0", "2.0"] + # Content dedented out of the item ends the item and the fence with it. + assert changelog_module.find_release_notes(text, "2.0").body == "- two" + + +def test_stripping_comments_stays_linear_in_the_code_spans(changelog_module): + """The comment scanner restarted its code-span search at the first span for + every opener, so a line of N spans and N openers cost N squared. A 203 KiB + line is well inside the 2 MiB the fetcher accepts, and notes are reparsed on + every request, so one such line held a worker for over ten seconds.""" + line = "`a` <!--x--> " * 16_000 + assert len(line) < changelog_module.CHANGELOG_MAX_BYTES + started = time.monotonic() + visible, in_comment = changelog_module._strip_comments(line, False, False) + elapsed = time.monotonic() - started + # Roughly 40ms scanning forward against roughly 11s restarting each time. + assert elapsed < 2.0, f"comment stripping took {elapsed:.1f}s" + # Same result as before: the spans survive and the comments are gone. + assert in_comment is False + assert "<!--" not in visible and visible.count("`a`") == 16_000 + + +def test_the_three_scanners_share_one_list_column_rule(): + """The parser and both frontend scanners have to classify a line the same + way, and drifting apart on indentation is what put a paragraph link inside a + code block. The frontend pair reads its list columns from one module, ported + from the backend's own tracker.""" + shared = LIST_COLUMNS.read_text(encoding="utf-8") + assert "export function openLists(" in shared + assert "_open_lists" in shared, "the backend function this mirrors" + for source in (PREVIEW, LINKS): + src = source.read_text(encoding="utf-8") + assert 'from "@/lib/markdown-list-columns"' in src + assert "openLists(" in src + # Both sides measure indented code from the container, not from the margin. + backend = (BACKEND / "utils" / "changelog.py").read_text(encoding="utf-8") + assert "_indent_width(visible) - column >= 4" in backend + assert "indentWidth(structure) - column >= INDENTED_CODE_INDENT" in LINKS.read_text( + encoding="utf-8" + ) + + +def test_a_failed_fetch_keeps_retry_reachable(): + """The fallback stands in for "no section for this version", which the hook + reports as ready. A failed fetch is reported as error and is retryable, and on + desktop the fallback is the updater's static install blurb, so taking it there + replaced the Retry button with generic text until the cache expired.""" + src = " ".join(PANEL.read_text(encoding="utf-8").split()) + assert 'notes?.matched ? notes.markdown : state === "error" ? null' in src + # Only NotesStatus renders retry, in the else of the markdown branch: an error has no markdown. + assert "{markdown ? (" in src + assert "retry={retry}" in src + + hook = " ".join( + (FRONTEND / "hooks" / "use-release-notes.ts").read_text(encoding="utf-8").split() + ) + assert ( + "const failed = !next || (!next.matched && next.error !== null);" in hook + ), "the distinction this relies on" + + +def test_an_unclosed_comment_in_prose_cannot_hide_later_links(run_scanner): + """CommonMark opens an HTML block (spec 0.31.2 section 4.6, type 2) only + when the line itself begins with `<!--`; one written mid-sentence is inline + raw HTML and cannot outlive the block it sits in. The link resolver carried + the unclosed state to every following line instead, so a note that merely + mentions the delimiter masked the relative links under it and they resolved + against Studio's own origin.""" + repo = "https://github.com/unslothai/unsloth/blob/main/docs/a.md" + # A separate list item is a separate block, so the link below still renders. + item = run_scanner("links", "- Type <!-- to begin a comment\n- See [docs](docs/a.md)\n") + assert repo in item + # So does a paragraph the blank line already ended. + paragraph = run_scanner("links", "Type <!-- to begin\n\nSee [docs](docs/a.md)\n") + assert repo in paragraph + # A delimiter inside inline code is literal, as it is for the parser. + spanned = run_scanner("links", "- Wrap in `<!--` and `-->`\n- See [docs](docs/a.md)\n") + assert repo in spanned + # A comment starting a line is a block: it hides down to the closer's line, that line included. + block = run_scanner("links", "<!-- staged\n- See [docs](docs/a.md)\n-->\n") + assert repo not in block + closer = run_scanner("links", "<!-- staged\n--> See [docs](docs/a.md)\n") + assert repo not in closer + + +def test_a_bare_level_two_marker_ends_the_release(changelog_module, run_scanner): + """An ATX heading's opening sequence may be followed by the end of the line + (spec 0.31.2 section 4.2), so a bare `##` is an empty level-two heading. The + scanners required whitespace after the hashes, so everything below such a + line stayed inside the release above it and the popup showed unrelated notes + under that version.""" + text = "## 2.0\n\n- new thing\n\n##\n\n- SECRET: not part of 2.0\n" + entry = changelog_module.find_release_notes(text, "2.0") + assert "new thing" in entry.body + assert "SECRET" not in entry.body + # An empty heading has no version, so it ends a release without indexing one. + assert [e.version for e in changelog_module.parse_changelog(text)] == ["2.0"] + # Prose still needs a space or a tab: `##x` is a paragraph, not a heading. + prose = "## 2.0\n\n- new thing\n\n##x\n\n- still 2.0\n" + assert "still 2.0" in changelog_module.find_release_notes(prose, "2.0").body + # The preview agrees: an empty heading renders as nothing, so it ends the bullet. + preview = run_scanner("preview", "- new thing\n##\nUnrelated scratch notes\n") + assert preview_leads(preview) == ["new thing"] + + +def test_a_comment_between_bullets_closes_the_list(changelog_module, run_scanner): + """A comment is an HTML block (spec 0.31.2 section 4.6, type 2), so one + written at the margin under a bullet is not indented enough to continue that + item and closes the list. The scanners blanked the line before list tracking + saw it, which reads as a blank line and leaves the item open, so the release + heading below it looked like nested item content and the new release was + merged into the one above.""" + text = "## 1.0\n\n- old item\n<!-- separator -->\n ## 2.0\n\n- new item\n" + assert [e.version for e in changelog_module.parse_changelog(text)] == ["1.0", "2.0"] + assert "new item" not in changelog_module.find_release_notes(text, "1.0").body + assert "new item" in changelog_module.find_release_notes(text, "2.0").body + # At the item's content column the comment stays inside it, so the heading under it is nested. + nested = "## 1.0\n\n- old item\n <!-- separator -->\n ## 2.0\n\n- new item\n" + assert [e.version for e in changelog_module.parse_changelog(nested)] == ["1.0"] + # The link resolver reads the same column: list closed, four spaces is code, left untouched. + code = run_scanner("links", "- old item\n<!-- separator -->\n [guide](docs/a.md)\n") + assert "[guide](docs/a.md)" in code and "github.com" not in code + # Inside the item those four spaces are two columns in, so it is prose and the link resolves. + prose = run_scanner("links", "- old item\n <!-- separator -->\n [guide](docs/a.md)\n") + assert "https://github.com/unslothai/unsloth/blob/main/docs/a.md" in prose + # The preview agrees: the fence is indented code, not a fence swallowing the bullet below. + preview = run_scanner( + "preview", + "- Details:\n<!-- separator -->\n ```\n - hidden sample\n- Real second item\n", + ) + assert preview_leads(preview) == ["Details:", "Real second item"] + + +def test_a_parenthesised_link_destination_still_resolves(run_scanner): + """A destination may hold parentheses while they balance (spec 0.31.2 + section 6.3), so `[x]((draft).md)` points at `(draft).md`. The resolver's + destination expression stopped at the first paren, matched an empty + destination and left the markdown alone, so the link resolved against + Studio's own origin instead of the repository.""" + leading = run_scanner("links", "[details]((draft).md)\n") + assert "https://github.com/unslothai/unsloth/blob/main/(draft).md" in leading + # An image resolves against the raw host the same way. + image = run_scanner("links", "![shield]((badge).png)\n") + assert "https://raw.githubusercontent.com/unslothai/unsloth/main/(badge).png" in image + # A pair in the middle of a path balances too. + middle = run_scanner("links", "[api](docs/(v2)/api.md)\n") + assert "https://github.com/unslothai/unsloth/blob/main/docs/(v2)/api.md" in middle + # An unbalanced paren makes the destination invalid, so `[x](a(b.md)` is plain text, not a link. + unbalanced = run_scanner("links", "[x](a(b.md)\n") + assert unbalanced == "[x](a(b.md)\n" + # One more closer balances the pair, and then it is a link again. + closed = run_scanner("links", "[x](a(b.md))\n") + assert "https://github.com/unslothai/unsloth/blob/main/a(b.md)" in closed + # Pairs nest, and one level was all the expression allowed, so a path with two stayed relative. + nested = run_scanner("links", "[x](((draft)).md)\n") + assert "https://github.com/unslothai/unsloth/blob/main/((draft)).md" in nested + deep = run_scanner("links", "![shot](((((v2))))).png)\n") + assert "https://raw.githubusercontent.com/unslothai/unsloth/main/((((v2))))" in deep + # The closer must still be there: an unbalanced run below a nested pair is not a link. + across = run_scanner("links", "[x](((a).md\n[y](docs/y.md)\n") + assert "https://github.com/unslothai/unsloth/blob/main/docs/y.md" in across + assert "[x](((a).md" in across + + +def test_a_fence_inside_a_container_still_hides_its_sample(run_scanner): + """A fence is measured from its container and not from the margin (spec + 0.31.2 section 4.5), so `> ~~~` and a fence three columns under a nested + bullet open one. Reading the margin instead never saw them, so the sample + inside was treated as prose and a relative link written in a code block was + rewritten into the text the reader sees verbatim.""" + quoted = run_scanner("links", "> ~~~\n> [guide](docs/a.md)\n> ~~~\n") + assert "[guide](docs/a.md)" in quoted and "github.com" not in quoted + nested = run_scanner("links", "- a\n - b\n ~~~\n [x](docs/x.md)\n ~~~\n") + assert "[x](docs/x.md)" in nested and "github.com" not in nested + # A longer closer is still a closer, so the pair is not something a code span hid. + uneven = run_scanner("links", "> ```\n> [guide](docs/a.md)\n> ````\n") + assert "[guide](docs/a.md)" in uneven and "github.com" not in uneven + # The fence ends with its container: a line outside the quote, or left of the item, is Markdown. + left = run_scanner("links", "> ~~~\n[guide](docs/a.md)\n") + assert "https://github.com/unslothai/unsloth/blob/main/docs/a.md" in left + dedented = run_scanner("links", "- a\n ~~~\n[guide](docs/a.md)\n") + assert "https://github.com/unslothai/unsloth/blob/main/docs/a.md" in dedented + # A document-level fence owns the quoted lines below, so the marker does not undo it. + document = run_scanner("links", "~~~\n> [guide](docs/a.md)\n~~~\n") + assert "[guide](docs/a.md)" in document and "github.com" not in document + # Four columns past the item's content column it is indented code, not a fence: still literal. + code = run_scanner("links", "- Details:\n\n ~~~\n [guide](docs/a.md)\n") + assert "[guide](docs/a.md)" in code and "github.com" not in code + + +def test_an_html_block_inside_a_container_is_literal_too(run_scanner): + """Type 1 and type 6 blocks are measured from their container the same way, + so a `<details>` under a nested bullet and a `<pre>` inside a quote both + show their contents verbatim. Missing the opener treated the body as + Markdown and rewrote the literal examples in it.""" + nested = run_scanner("links", "- a\n - b\n <details>\n [x](docs/x.md)\n </details>\n") + assert "[x](docs/x.md)" in nested and "github.com" not in nested + quoted = run_scanner("links", "> <pre>\n> [x](docs/x.md)\n> </pre>\n") + assert "[x](docs/x.md)" in quoted and "github.com" not in quoted + # The block ends with its container, so a line dedented out of the item is Markdown again. + dedented = run_scanner("links", "- a\n - b\n <details>\n[x](docs/x.md)\n") + assert "https://github.com/unslothai/unsloth/blob/main/docs/x.md" in dedented + # Inside a quote a bare marker holds nothing, the blank line that ends a type 6 block. + blank = run_scanner("links", "> <details>\n>\n> [x](docs/x.md)\n") + assert "https://github.com/unslothai/unsloth/blob/main/docs/x.md" in blank + + +def test_an_underline_left_of_an_item_is_lazy_text_of_it(changelog_module, run_scanner): + """A setext underline may never be a lazy continuation line (spec 0.31.2 + section 4.3), so `===` written left of an open list item is read as more of + the item's paragraph rather than as a block that closes it. Rejecting every + underline-shaped line ended the list there, which promoted the nested + "## 2.0" below it to a document-level heading and indexed a release the + renderer never shows.""" + nested = "## 1.0\n- old note\n===\n ## 2.0\n- new\n" + assert [e.version for e in changelog_module.parse_changelog(nested)] == ["1.0"] + # A row of dashes is a thematic break, closing the item, so the heading is the next release. + broken = "## 1.0\n- old note\n---\n ## 2.0\n" + assert [e.version for e in changelog_module.parse_changelog(broken)] == ["1.0", "2.0"] + # With no paragraph above it the underline opens one, so the blank line closes the item. + apart = "## 1.0\n- old note\n\n===\n ## 2.0\n" + assert [e.version for e in changelog_module.parse_changelog(apart)] == ["1.0", "2.0"] + # The link scanner keeps the item open, so the four-space line is a paragraph and resolves. + resolved = run_scanner("links", "- Details:\n===\n\n [guide](docs/a.md)\n") + assert "https://github.com/unslothai/unsloth/blob/main/docs/a.md" in resolved + + +def test_a_quote_keeps_its_paragraph_to_itself(changelog_module, run_scanner): + """Lazy continuation runs the other way too: a marker written outside a + blockquote is not text of the quote's paragraph, so `2. item` under + `> quote` opens a list even though an ordered marker past 1 may not + interrupt a paragraph (spec 0.31.2 section 5.2). Lending the quote's + paragraph to the document left the list closed, so the heading indented to + the item's content column read as a release of its own.""" + quoted = "## 1.0\n> quote\n2. item\n ## 2.0\n- new\n" + assert [e.version for e in changelog_module.parse_changelog(quoted)] == ["1.0"] + # A quote holding a heading leaves no paragraph, nor does an empty one, so the list opens. + heading = "## 1.0\n> # inner\n2. item\n ## 2.0\n" + assert [e.version for e in changelog_module.parse_changelog(heading)] == ["1.0"] + # An unquoted line the quote's paragraph swallows keeps it open, the marker still outside. + lazy = "## 1.0\n> quote\ntext\n2. item\n ## 2.0\n" + assert [e.version for e in changelog_module.parse_changelog(lazy)] == ["1.0"] + # Under an ordinary paragraph the marker is its text, so no list opens and the heading is real. + prose = "## 1.0\nprose\n2. item\n ## 2.0\n" + assert [e.version for e in changelog_module.parse_changelog(prose)] == ["1.0", "2.0"] + # The preview reads the marker as a bullet for the same reason. + assert preview_leads(run_scanner("preview", "> quote\n2. item\n")) == ["item"] + + +def test_indented_code_before_an_ordered_marker_still_opens_a_list(changelog_module): + """An indented code block ends at the first line that is not indented enough + to continue it, and no paragraph is open for the marker below to continue, + so `2. item` opens a list whatever its start number. Reading it as text of + the code block instead would leave the list closed and index the heading at + the item's content column as a release.""" + joined = "## 1.0\n\n code\n2. item\n ## 2.0\n- new\n" + assert [e.version for e in changelog_module.parse_changelog(joined)] == ["1.0"] + # A blank line between the two changes nothing: the list opens either way. + apart = "## 1.0\n\n code\n\n2. item\n ## 2.0\n- new\n" + assert [e.version for e in changelog_module.parse_changelog(apart)] == ["1.0"] + # Four columns past its container the marker is code, so no list opens and the heading stands. + inside = "## 1.0\n\n code\n - item\n ## 2.0\n" + assert [e.version for e in changelog_module.parse_changelog(inside)] == ["1.0", "2.0"] + + +def test_a_fence_written_as_an_item_first_content_opens_in_that_item(run_scanner): + """A block written straight after a list marker is the item's own first + content, measured from the column that content starts (spec 0.31.2 section + 5.2), so "- ```md" opens a fence. Reading the whole line instead never saw + one, so the code sample below it was treated as prose: the resolver rewrote + a destination the reader sees verbatim, and the preview offered the info + string as a headline bullet.""" + sample = run_scanner("links", "- ```md\n [example](docs/a.md)\n ```\n") + assert "[example](docs/a.md)" in sample and "github.com" not in sample + ordered = run_scanner("links", "1. ~~~\n [example](docs/a.md)\n ~~~\n") + assert "[example](docs/a.md)" in ordered and "github.com" not in ordered + # The preview agrees: an item of only a code block previews as nothing; the next is a bullet. + preview = run_scanner("preview", "- ```md\n sample text\n ```\n- Added tests\n") + assert preview_leads(preview) == ["Added tests"] + # One column further in it is indented code inside the item, so the link is prose and resolves. + padded = run_scanner("links", "- ```\n [example](docs/a.md)\n") + assert "https://github.com/unslothai/unsloth/blob/main/docs/a.md" in padded + # A marker the paragraph above swallows opens no item, so no fence: ordered items open at 1. + lazy = run_scanner("links", "Intro.\n2. ```\n[guide](docs/a.md)\n") + assert "https://github.com/unslothai/unsloth/blob/main/docs/a.md" in lazy + + +def test_an_html_block_ends_with_the_item_it_was_written_in(changelog_module, run_scanner): + """An HTML block holds no lazy continuation line, so one opened on a list + item's continuation line ends where the item does, exactly as a fence there + does. Ending it only on a blank line let it run past the item and swallow + the next release heading, so those notes could never be found, and the + collapsed preview lost every bullet below it.""" + text = "## 1.0\n\n- item\n\n <div>\n## 2.0\n\n- new thing\n" + assert [e.version for e in changelog_module.parse_changelog(text)] == ["1.0", "2.0"] + assert "new thing" in changelog_module.find_release_notes(text, "2.0").body + # A raw block such as <pre> is scoped the same way. + raw = "## 1.0\n\n- item\n\n <pre>\n## 2.0\n\n- new thing\n" + assert [e.version for e in changelog_module.parse_changelog(raw)] == ["1.0", "2.0"] + # At the item's content column the block holds the heading, which is nested and indexes nothing. + nested = "## 1.0\n\n- item\n\n <div>\n ## 2.0\n" + assert [e.version for e in changelog_module.parse_changelog(nested)] == ["1.0"] + # The preview reads it the same way: the bullet below the block is a bullet. + preview = run_scanner("preview", "- item\n\n <div>\n- Added tests\n") + assert preview_leads(preview) == ["item", "Added tests"] + # An opener straight after a marker opens in that item, so the dedented heading is a release. + marked = "## 1.0\n\n- <div>\n## 2.0\n\n- new thing\n" + assert [e.version for e in changelog_module.parse_changelog(marked)] == ["1.0", "2.0"] + + +def test_a_comment_may_close_on_a_later_line_of_its_paragraph(run_scanner): + """A comment written mid-sentence is inline raw HTML belonging to the + paragraph around it, so its `-->` may arrive on a later line of that same + paragraph and everything between renders as nothing. Ending the comment at + its own line left a backtick inside it pairing with a real one below, which + hid a following link from the resolver, and left the collapsed preview + quoting text the popup body does not show.""" + carried = run_scanner("links", "Note <!-- ` open\nstill --> see [d](docs/a.md) and `x`\n") + assert "https://github.com/unslothai/unsloth/blob/main/docs/a.md" in carried + # Text inside the comment renders as nothing, so it is left alone. + inside = run_scanner("links", "Note <!-- see [c](docs/c.md)\nmore --> end\n") + assert "[c](docs/c.md)" in inside and "github.com" not in inside + # The preview hides it too, rather than quoting the comment at the reader. + preview = run_scanner( + "preview", "- Added X <!-- TODO: rewrite\n this properly -->\n- Second\n" + ) + assert preview_leads(preview) == ["Added X", "Second"] + # An opener cannot outlive its paragraph: with it closed the `<!--` is text and hides nothing. + broken = run_scanner("links", "Note <!-- open\n\nsecret --> end [d](docs/a.md)\n") + assert "https://github.com/unslothai/unsloth/blob/main/docs/a.md" in broken + # A heading breaks into the paragraph, so it ends the comment's reach too. + headed = run_scanner("links", "Note <!-- open\n## 2.0 --> end [d](docs/a.md)\n") + assert "https://github.com/unslothai/unsloth/blob/main/docs/a.md" in headed + assert preview_leads(run_scanner("preview", "Note <!-- open\n\n- Second\n")) == ["Second"] + + +def test_only_punctuation_is_escapable_in_a_link_destination(run_scanner): + """CommonMark escapes ASCII punctuation and nothing else (spec 0.31.2 + section 2.4), so the backslash in `docs\\alpha.md` is a character of the + path. Dropping every backslash rewrote it to a path that does not exist, + and a URL parser reads what is left as a separator, so a Windows or + namespaced path pointed at the wrong file either way.""" + kept = run_scanner("links", "[guide](docs\\alpha.md)\n") + assert "https://github.com/unslothai/unsloth/blob/main/docs%5Calpha.md" in kept + # An escaped backslash is one literal backslash, which survives the same. + escaped = run_scanner("links", "[guide](docs\\\\alpha.md)\n") + assert "https://github.com/unslothai/unsloth/blob/main/docs%5Calpha.md" in escaped + # A real escape is still an escape: `\\(` is a paren of the path. + paren = run_scanner("links", "[guide](a\\(b.md)\n") + assert "https://github.com/unslothai/unsloth/blob/main/a(b.md" in paren + # A space still ends the destination, escaped or not, so there is no link. + spaced = run_scanner("links", "[guide](a\\ b.md)\n") + assert spaced == "[guide](a\\ b.md)\n" + + +def test_one_definition_does_not_hide_the_next(run_scanner): + """Definitions may run consecutively (spec 0.31.2 section 4.7): a block of + them is how a changelog collects its link targets. A definition is not + paragraph text, so it opens no paragraph for the next one to be unable to + interrupt. The resolver counted one as prose, which left every definition + after the first outside the set of lines a definition may start on, so only + the first was rewritten and the rest resolved against Studio's own origin. + The backend already reads the line this way.""" + text = ( + "- AMD support is here, see [the AMD guide][amd] and the\n" + " [Intel notes][xpu].\n\n" + "[amd]: docs/basics/amd.md\n" + "[xpu]: docs/basics/xpu.md\n" + ) + resolved = run_scanner("links", text) + base = "https://github.com/unslothai/unsloth/blob/main/docs/basics/" + assert f"[amd]: {base}amd.md" in resolved + assert f"[xpu]: {base}xpu.md" in resolved + # A run of them stays a run however long it is. + run = run_scanner("links", "[a]: docs/a.md\n[b]: docs/b.md\n[c]: docs/c.md\n") + assert run.count("https://github.com/unslothai/unsloth/blob/main/docs/") == 3 + # Prose between them opens a paragraph the next line may not interrupt, so it is not one. + prose = run_scanner("links", "[a]: docs/a.md\nintro\n[b]: docs/b.md\n") + assert "[b]: docs/b.md" in prose + + +def test_a_comment_closed_on_its_own_line_still_closes(run_scanner): + """A multiline comment is ordinarily closed by a `-->` written on a line of + its own, and a wrapped line may open with emphasis. The guard asking whether + the closer is reachable read any line whose first character was punctuation + as the start of a new block, so neither shape counted as more of the + paragraph carrying the comment. The comment then never closed, and the + collapsed popup showed the author's internal note to the user.""" + closer = run_scanner( + "preview", + "- DoRA training is available in Studio. <!-- TODO confirm the exact\n" + " flag name before release\n-->\n", + ) + assert preview_leads(closer) == ["DoRA training is available in Studio."] + # A continuation may open with emphasis, which is text and not a block. + starred = run_scanner( + "preview", + "- DoRA training is available. <!-- TODO confirm the\n *before* release -->\n", + ) + assert preview_leads(starred) == ["DoRA training is available."] + underscored = run_scanner( + "preview", + "- DoRA training is available. <!-- TODO confirm the\n _draft_ note -->\n", + ) + assert preview_leads(underscored) == ["DoRA training is available."] + # A real block still ends the paragraph, so the opener below one is text and hides nothing. + broken = run_scanner("links", "Note <!-- open\n## 2.0\nsecret --> [d](docs/a.md)\n") + assert "https://github.com/unslothai/unsloth/blob/main/docs/a.md" in broken + # So does a list item with content, which may interrupt a paragraph. + item = run_scanner("links", "Note <!-- open\n- bullet\nsecret --> [d](docs/a.md)\n") + assert "https://github.com/unslothai/unsloth/blob/main/docs/a.md" in item + + +def test_a_comment_written_as_an_item_first_content_is_a_block(changelog_module, run_scanner): + """A comment is an HTML block (spec 0.31.2 section 4.6, type 2), so one + written as a list item's first content opens inside that item, exactly as a + fence written there does. The scanners looked for the opener at the margin + of the line as written, so a marker in front of it hid the block: the + resolver rewrote a destination inside raw HTML, which Streamdown then shows + the reader as a literal URL, and the preview quoted the hidden note back at + them as though the bullet were Markdown.""" + item = run_scanner("links", "- <!-- new --> AMD support, see [the guide](docs/amd.md)\n") + assert item == "- <!-- new --> AMD support, see [the guide](docs/amd.md)\n" + # Every marker opens an item, and a nested one is still an item. + for text in ( + "* <!-- new --> see [the guide](docs/amd.md)\n", + "1. <!-- new --> see [the guide](docs/amd.md)\n", + "- outer\n - <!-- new --> see [the guide](docs/amd.md)\n", + ): + assert "github.com" not in run_scanner("links", text) + # The multiline form hides lines to the closer, as a comment at the item's content column did. + multiline = run_scanner("links", "- <!-- hidden\n [a](docs/x.md)\n -->\n") + assert "[a](docs/x.md)" in multiline and "github.com" not in multiline + # Still scoped to the item it was written in, so a line dedented out of it ends the block. + dedented = run_scanner("links", "- <!-- hidden\n[a](docs/x.md)\n") + assert "https://github.com/unslothai/unsloth/blob/main/docs/x.md" in dedented + # The preview agrees: an item of only the block previews as nothing; the next is a bullet. + preview = run_scanner("preview", "- <!-- new --> hidden note\n- Real bullet\n") + assert preview_leads(preview) == ["Real bullet"] + # The parser agrees too: the item keeps its column, so a heading inside is nested, not indexed. + text = "## 1.0\n\n- <!-- hidden\n\n ## 2.0\n" + assert [e.version for e in changelog_module.parse_changelog(text)] == ["1.0"] diff --git a/tests/test_profile_startup_gate.py b/tests/test_profile_startup_gate.py new file mode 100644 index 0000000000..66e6e16a89 --- /dev/null +++ b/tests/test_profile_startup_gate.py @@ -0,0 +1,243 @@ +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. + +"""Regression coverage for the startup profiler's budget gate, teardown and triggers.""" + +from __future__ import annotations + +import ast +import fnmatch +import importlib.util +import re +import subprocess +import sys +from pathlib import Path +from types import SimpleNamespace + +import pytest +import yaml + +REPO_ROOT = Path(__file__).resolve().parents[1] +SCRIPT = REPO_ROOT / "scripts" / "profile_startup.py" +WORKFLOW = REPO_ROOT / ".github" / "workflows" / "startup-profile-ci.yml" +PROCESS_RS = REPO_ROOT / "studio" / "src-tauri" / "src" / "process.rs" + +# Checkout files that build the venv the workflow profiles. +INSTALLER_INPUTS = ( + "studio/setup.sh", + "studio/setup.ps1", + "studio/install_python_stack.py", +) +# Checkout file that defines the argv the profiler reproduces. +LAUNCH_INPUTS = ("studio/src-tauri/src/process.rs",) + + +def _load(): + spec = importlib.util.spec_from_file_location("profile_startup", SCRIPT) + mod = importlib.util.module_from_spec(spec) + spec.loader.exec_module(mod) + return mod + + +def _no_subprocesses(mod, monkeypatch): + # Keep the gate tests off the real interpreter and CLI. + monkeypatch.setattr(mod, "find_bin", lambda: None) + monkeypatch.setattr(mod, "profile_imports", lambda python, top = 15: {"ok": False, "error": ""}) + monkeypatch.setattr(mod, "python_version_of", lambda python: "3.13.0") + + +class _Proc: + """Stand-in for a still-running Popen.""" + + def __init__(self): + self.pid = 4321 + self.terminated = False + + def poll(self): + return None + + def terminate(self): + self.terminated = True + + +def _nt(mod, monkeypatch, returncode): + calls: list[list[str]] = [] + + def _run(argv, **kwargs): + calls.append(argv) + return subprocess.CompletedProcess(argv, returncode, "", "") + + # Patch the module's own references, not the real os/subprocess the session shares. + monkeypatch.setattr(mod, "os", SimpleNamespace(name = "nt")) + monkeypatch.setattr(mod, "subprocess", SimpleNamespace(run = _run)) + return calls + + +def test_budget_fails_when_no_launch_was_measured(capsys, monkeypatch): + """A requested budget must not pass just because the CLI was never found.""" + mod = _load() + _no_subprocesses(mod, monkeypatch) + rc = mod.main(["--max-healthz-seconds", "30"]) + out = capsys.readouterr().out + assert rc == 1 + assert "::error::" in out and "no healthz measurement" in out + assert "no unsloth CLI found" in out + + +def _healthy_launch( + mod, + monkeypatch, + healthz = 1.5, +): + monkeypatch.setattr(mod, "find_bin", lambda: "unsloth") + monkeypatch.setattr( + mod, + "profile_launch", + lambda bin_path, port, **kw: { + "spawn_seconds": 0.1, + "healthz_seconds": healthz, + "lifespan_ms": 100.0, + "reached_healthz": True, + "log_tail": [], + }, + ) + + +def test_budget_still_passes_when_a_launch_was_measured(monkeypatch): + """The fail-closed branch must not swallow a genuinely healthy run.""" + mod = _load() + _no_subprocesses(mod, monkeypatch) + _healthy_launch(mod, monkeypatch) + assert mod.main(["--max-healthz-seconds", "30"]) == 0 + assert mod.main(["--max-healthz-seconds", "1"]) == 1 + + +# "=" form for -inf: a bare "-inf" is an option token to argparse, not a value. +@pytest.mark.parametrize( + "bad", ["--max-healthz-seconds=nan", "--max-healthz-seconds=inf", "--max-healthz-seconds=-inf"] +) +def test_budget_rejects_non_finite_values(bad, capsys, monkeypatch): + """`med > nan` and `med > inf` are always False, so the gate would never bind.""" + mod = _load() + _no_subprocesses(mod, monkeypatch) + _healthy_launch(mod, monkeypatch) + with pytest.raises(SystemExit) as exc: + mod.main([bad]) + assert exc.value.code == 2 + assert "finite" in capsys.readouterr().err + + +def test_budget_rejects_import_only(capsys): + """--import-only launches nothing, so a budget on it could only ever pass.""" + mod = _load() + with pytest.raises(SystemExit) as exc: + mod.main(["--import-only", "--max-healthz-seconds", "30"]) + assert exc.value.code == 2 + assert "--import-only" in capsys.readouterr().err + + +def test_terminate_tree_falls_back_when_taskkill_fails(monkeypatch): + """A nonzero taskkill must still reach terminate(), not return silently.""" + mod = _load() + calls = _nt(mod, monkeypatch, returncode = 1) + proc = _Proc() + mod._terminate_tree(proc) + assert calls == [["taskkill", "/PID", "4321", "/T", "/F"]] + assert proc.terminated + + +def test_terminate_tree_falls_back_when_taskkill_raises(monkeypatch): + """A missing or hung taskkill must reach terminate() too.""" + mod = _load() + monkeypatch.setattr(mod, "os", SimpleNamespace(name = "nt")) + + def _boom(argv, **kwargs): + raise FileNotFoundError(argv) + + monkeypatch.setattr(mod, "subprocess", SimpleNamespace(run = _boom)) + proc = _Proc() + mod._terminate_tree(proc) + assert proc.terminated + + +def test_terminate_tree_returns_on_successful_taskkill(monkeypatch): + mod = _load() + _nt(mod, monkeypatch, returncode = 0) + proc = _Proc() + mod._terminate_tree(proc) + assert not proc.terminated + + +def test_terminate_tree_skips_an_exited_process(monkeypatch): + mod = _load() + calls = _nt(mod, monkeypatch, returncode = 0) + proc = _Proc() + proc.poll = lambda: 0 + mod._terminate_tree(proc) + assert calls == [] and not proc.terminated + + +def _trigger_paths(): + wf = yaml.safe_load(WORKFLOW.read_text(encoding = "utf-8")) + # YAML 1.1 turns the bare `on:` key into True. + on = wf.get("on") or wf[True] + return [p for p in on["pull_request"]["paths"] if not p.startswith("!")] + + +@pytest.mark.parametrize("rel", INSTALLER_INPUTS) +def test_workflow_triggers_on_studio_installer_inputs(rel): + """A setup script that changes the profiled venv must schedule a measurement.""" + assert (REPO_ROOT / rel).is_file(), f"{rel} moved; revisit the trigger list" + paths = _trigger_paths() + assert any(fnmatch.fnmatch(rel, p) for p in paths), f"{rel} not covered by {paths}" + + +def test_studio_installer_inputs_are_on_the_local_install_path(): + """Anchor the list above: these files are what --local actually executes.""" + # install.ps1 reaches setup.ps1 through the editable install, not by name. + assert "studio/setup.sh" in (REPO_ROOT / "install.sh").read_text(encoding = "utf-8") + for setup in ("studio/setup.sh", "studio/setup.ps1"): + text = (REPO_ROOT / setup).read_text(encoding = "utf-8", errors = "replace") + assert "install_python_stack.py" in text + + +@pytest.mark.parametrize("rel", LAUNCH_INPUTS) +def test_workflow_triggers_on_the_desktop_launch_command(rel): + """The profiler copies process.rs's argv, so a change there must be measured.""" + assert (REPO_ROOT / rel).is_file(), f"{rel} moved; revisit the trigger list" + paths = _trigger_paths() + assert any(fnmatch.fnmatch(rel, p) for p in paths), f"{rel} not covered by {paths}" + + +def _desktop_backend_argv(): + body = re.search( + r"fn backend_args\(port: u16\) -> Vec<String> \{(.*?)\n\}", + PROCESS_RS.read_text(encoding = "utf-8"), + re.S, + ) + assert body, "backend_args moved; revisit the trigger list" + return re.findall(r'"([^"]+)"', body.group(1)) + + +def _profiler_argv(): + tree = ast.parse(SCRIPT.read_text(encoding = "utf-8")) + fn = next( + n for n in ast.walk(tree) if isinstance(n, ast.FunctionDef) and n.name == "profile_launch" + ) + call = next( + n for n in ast.walk(fn) if isinstance(n, ast.Call) and ast.unparse(n.func).endswith("Popen") + ) + return [e.value for e in call.args[0].elts if isinstance(e, ast.Constant)] + + +def test_profiler_spawns_the_desktop_backend_argv(): + """Anchor the trigger above: these two argv lists must stay identical.""" + assert _profiler_argv() == _desktop_backend_argv() + + +@pytest.mark.skipif(sys.platform == "win32", reason = "posix branch") +def test_terminate_tree_posix_uses_terminate(): + mod = _load() + proc = _Proc() + mod._terminate_tree(proc) + assert proc.terminated diff --git a/unsloth/_gpu_init.py b/unsloth/_gpu_init.py index 682f3ae6c6..7e8f9ced46 100644 --- a/unsloth/_gpu_init.py +++ b/unsloth/_gpu_init.py @@ -303,13 +303,19 @@ if DEVICE_TYPE == "cuda": # Try loading bitsandbytes and triton try: import bitsandbytes as bnb + + # Bind the submodule by name: a half-imported bitsandbytes leaves the parent + # without a `functional` attribute, which would otherwise be misreported below + # as a CUDA linking failure. See unsloth/kernels/utils.py. + import bitsandbytes.functional as bnb_functional except: print( "Unsloth: `bitsandbytes` is not installed - 4bit QLoRA unallowed, but 16bit and full finetuning works!" ) bnb = None + bnb_functional = None try: - cdequantize_blockwise_fp32 = bnb.functional.lib.cdequantize_blockwise_fp32 + cdequantize_blockwise_fp32 = bnb_functional.lib.cdequantize_blockwise_fp32 libcuda_dirs() except: if hasattr(os, "geteuid") and os.geteuid() == 0: @@ -351,7 +357,7 @@ if DEVICE_TYPE == "cuda": pass else: from triton.common.build import libcuda_dirs - cdequantize_blockwise_fp32 = bnb.functional.lib.cdequantize_blockwise_fp32 + cdequantize_blockwise_fp32 = bnb_functional.lib.cdequantize_blockwise_fp32 libcuda_dirs() except: warnings.warn( diff --git a/unsloth/bnb_availability.py b/unsloth/bnb_availability.py new file mode 100644 index 0000000000..9d14bbb0f3 --- /dev/null +++ b/unsloth/bnb_availability.py @@ -0,0 +1,96 @@ +# Copyright 2023-present Daniel Han-Chen & the Unsloth team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Can bitsandbytes actually run a 4bit kernel here? A successful import does not say. + +From 0.46 a wheel whose native library never loaded still imports and hands back a +`throw_on_call` closure for every symbol, so attribute reads alone see a healthy wheel, +`ALLOW_BITSANDBYTES` stays true and 4bit dies inside a kernel instead of falling back to +16bit up front. A real handle is a ctypes function pointer and carries `restype`; a +deferred failure is a plain Python function and does not. That is the whole test, applied +to every probed handle: the same verdict gates the module-scope binds in kernels/utils.py, +where one bad symbol is the crash this exists to prevent. + +Decides the capability flags only, never importability - a CPU-only install is exactly +this shape and its Python side works. A leaf module: imports nothing from unsloth +(device_type.py imports it very early, so anything else is a cycle) and takes the +device type as an argument. +""" + +__all__ = [ + "bitsandbytes_symbols", + "check_native_kernels", + "native_kernels_ready", +] + +# The ctypes handles kernels/utils.py binds at module scope; a test asserts they match. +_C_SYMBOLS = ( + "cdequantize_blockwise_fp32", + "cdequantize_blockwise_fp16_nf4", + "cdequantize_blockwise_bf16_nf4", +) +# 4bit inference is a gemv on xpu and a naive gemm elsewhere; probing the wrong pair +# would write off a perfectly good wheel. +_C_SYMBOLS_XPU = ( + "cgemv_4bit_inference_fp16", + "cgemv_4bit_inference_bf16", +) +_C_SYMBOLS_GEMM = ( + "cgemm_4bit_inference_naive_fp16", + "cgemm_4bit_inference_naive_bf16", +) + + +def bitsandbytes_symbols(device_type): + """Names kernels/utils.py reads off `bitsandbytes.functional.lib`.""" + tail = _C_SYMBOLS_XPU if device_type == "xpu" else _C_SYMBOLS_GEMM + return _C_SYMBOLS + tail + + +def check_native_kernels(bnb, device_type): + """Raise unless every handle kernels/utils.py is about to bind is a real kernel. + + All of them: one that resolves here but not at the bind gives back the AttributeError + this prevents. Partial export costs 8bit too (`ALLOW_BITSANDBYTES` gates both), but a + wheel missing a symbol is a shape no flag makes safe. Safe to repeat - ctypes caches + each handle on first lookup, so these are the ones bound later. + """ + if bnb is None: + raise ImportError("Unsloth: `bitsandbytes` is not installed.") + functional = getattr(bnb, "functional", None) + if functional is None: + # A part-initialised bitsandbytes leaves the parent without the attribute while + # the submodule stays in sys.modules, which `import x.y as z` reads directly. + import bitsandbytes.functional as functional + + lib = functional.lib + if lib is None: + # 0.45.5, the floor in pyproject.toml, on a native-load failure. + raise AttributeError("Unsloth: `bitsandbytes.functional.lib` is None.") + for symbol in bitsandbytes_symbols(device_type): + handle = getattr(lib, symbol) # AttributeError here is itself a failed check + if not hasattr(handle, "restype"): + raise AttributeError( + f"Unsloth: `bitsandbytes.functional.lib.{symbol}` is not a native " + "function pointer - the bitsandbytes native library did not load." + ) + + +def native_kernels_ready(bnb, device_type): + """Is the bitsandbytes native library alive? Gates the flags, never the import.""" + try: + check_native_kernels(bnb, device_type) + except Exception: + return False + return True diff --git a/unsloth/device_type.py b/unsloth/device_type.py index 058e166b08..968062c7c1 100644 --- a/unsloth/device_type.py +++ b/unsloth/device_type.py @@ -27,6 +27,7 @@ import functools import inspect import os from unsloth_zoo.utils import Version +from .bnb_availability import native_kernels_ready def is_mlx_available(): @@ -117,17 +118,20 @@ DEVICE_COUNT: int = get_device_count() ALLOW_PREQUANTIZED_MODELS: bool = True # HSA_STATUS_ERROR_EXCEPTION checks - sometimes AMD fails for BnB ALLOW_BITSANDBYTES: bool = True -# Unusable bitsandbytes on any backend, not just hip: clear the flags the loader -# reads before it selects a 4bit checkpoint. Same guarded import the fallbacks in -# _gpu_init.py and kernels/utils.py use rather than a find_spec probe, so an -# installed-but-broken wheel (missing .so, wrong ROCm/CUDA build) is treated as -# unavailable by all three, not only by the ones that import it. +# Unusable bitsandbytes on any backend, not just hip: clear the flags the loader reads +# before it picks a 4bit checkpoint. A guarded import, not find_spec, since importable +# is not usable - from 0.46 a dead native library still resolves every ctypes handle to +# a closure that raises only when called, so 4bit would die mid-run, not fall back here. try: import bitsandbytes as _bnb_probe - del _bnb_probe except Exception: ALLOW_PREQUANTIZED_MODELS = False ALLOW_BITSANDBYTES = False +else: + if not native_kernels_ready(_bnb_probe, DEVICE_TYPE): + ALLOW_PREQUANTIZED_MODELS = False + ALLOW_BITSANDBYTES = False + del _bnb_probe # gfx906 (MI50 / Radeon VII / Vega 20): Dynamo/Inductor codegen is broken on this # legacy GCN arch (ROCm dropped it after 6.3) - compiled graphs crash or miscompile # while the eager path trains fine. Default compile off; setdefault so a user diff --git a/unsloth/kernels/utils.py b/unsloth/kernels/utils.py index 2118e65aef..839eb9db84 100644 --- a/unsloth/kernels/utils.py +++ b/unsloth/kernels/utils.py @@ -29,6 +29,7 @@ from ..device_type import ( DEVICE_COUNT, ALLOW_PREQUANTIZED_MODELS, ) +from ..bnb_availability import native_kernels_ready from .fp8 import weight_dequant, fp8_linear import functools @@ -135,11 +136,18 @@ def calculate_settings( HAS_CUDA_STREAM = False try: import bitsandbytes as bnb + + # If an earlier `import bitsandbytes` died inside __init__, CPython evicts only + # the parent from sys.modules and keeps its submodules, so this retry re-executes + # __init__ without rebinding `bnb.functional`. `import x.y as z` reads sys.modules + # directly and survives that, plain attribute access does not. + import bitsandbytes.functional as bnb_functional except Exception: # device_type.py already degrades to 16bit/full finetuning when bnb is missing # (e.g. gfx906, whose generic wheel has no kernels). Keep the import working and # fail only if a 4bit path is actually entered. bnb = None + bnb_functional = None def _bnb_required(*args, **kwargs): @@ -152,7 +160,7 @@ def _bnb_required(*args, **kwargs): if bnb is not None: # https://github.com/bitsandbytes-foundation/bitsandbytes/pull/1330/files HAS_CUDA_STREAM = Version(bnb.__version__) > Version("0.43.3") - get_ptr = bnb.functional.get_ptr + get_ptr = bnb_functional.get_ptr else: get_ptr = _bnb_required @@ -252,25 +260,28 @@ else: # Bitsandbytes operations ctypes_c_int = ctypes.c_int ctypes_c_int32 = ctypes.c_int32 -if bnb is None: +# Same verdict device_type.py used to clear ALLOW_BITSANDBYTES, applied to the binds +# themselves. 0.45.5 leaves `functional.lib = None` when the native library fails to +# load, so these lookups would kill `import unsloth` instead of degrading to 16bit. +if bnb is None or not native_kernels_ready(bnb, DEVICE_TYPE): cdequantize_blockwise_fp32 = _bnb_required cdequantize_blockwise_fp16_nf4 = _bnb_required cdequantize_blockwise_bf16_nf4 = _bnb_required cgemm_4bit_inference_naive_fp16 = _bnb_required cgemm_4bit_inference_naive_bf16 = _bnb_required else: - cdequantize_blockwise_fp32 = bnb.functional.lib.cdequantize_blockwise_fp32 - cdequantize_blockwise_fp16_nf4 = bnb.functional.lib.cdequantize_blockwise_fp16_nf4 - cdequantize_blockwise_bf16_nf4 = bnb.functional.lib.cdequantize_blockwise_bf16_nf4 + cdequantize_blockwise_fp32 = bnb_functional.lib.cdequantize_blockwise_fp32 + cdequantize_blockwise_fp16_nf4 = bnb_functional.lib.cdequantize_blockwise_fp16_nf4 + cdequantize_blockwise_bf16_nf4 = bnb_functional.lib.cdequantize_blockwise_bf16_nf4 if DEVICE_TYPE == "xpu": # https://github.com/bitsandbytes-foundation/bitsandbytes/blob/c3b8de268fdb55a88f92feada23fc811a1e6877a/bitsandbytes/backends/xpu/ops.py#L115 # for xpu, inference gemv using above link - cgemm_4bit_inference_naive_fp16 = bnb.functional.lib.cgemv_4bit_inference_fp16 - cgemm_4bit_inference_naive_bf16 = bnb.functional.lib.cgemv_4bit_inference_bf16 + cgemm_4bit_inference_naive_fp16 = bnb_functional.lib.cgemv_4bit_inference_fp16 + cgemm_4bit_inference_naive_bf16 = bnb_functional.lib.cgemv_4bit_inference_bf16 else: - cgemm_4bit_inference_naive_fp16 = bnb.functional.lib.cgemm_4bit_inference_naive_fp16 - cgemm_4bit_inference_naive_bf16 = bnb.functional.lib.cgemm_4bit_inference_naive_bf16 + cgemm_4bit_inference_naive_fp16 = bnb_functional.lib.cgemm_4bit_inference_naive_fp16 + cgemm_4bit_inference_naive_bf16 = bnb_functional.lib.cgemm_4bit_inference_naive_bf16 torch_device_stream = ( diff --git a/unsloth_cli/commands/studio.py b/unsloth_cli/commands/studio.py index 9fd264ddf5..560df4aea2 100644 --- a/unsloth_cli/commands/studio.py +++ b/unsloth_cli/commands/studio.py @@ -19,7 +19,7 @@ import urllib.error import urllib.request from datetime import datetime, timezone from pathlib import Path -from typing import List, Literal, Optional +from typing import List, Literal, Optional, Sequence import typer from unsloth_cli import _studio_deps @@ -483,9 +483,12 @@ def _write_auth_secret(path: Path, secret: str) -> None: os.chmod(tmp_path, 0o600) except OSError: pass - with os.fdopen(fd, "w", encoding = "utf-8") as f: + # newline pins LF: text mode writes CRLF on Windows, and `$(cat ...)` + # strips the LF but leaves the CR glued to the credential. + with os.fdopen(fd, "w", encoding = "utf-8", newline = "\n") as f: fd = -1 - f.write(secret) + # Newline so `cat` doesn't run it into the shell prompt; readers strip. + f.write(secret + "\n") os.replace(tmp_path, path) except Exception: if fd >= 0: @@ -502,6 +505,8 @@ def _connect_auth_db() -> sqlite3.Connection: auth_dir = STUDIO_HOME / "auth" auth_dir.mkdir(parents = True, exist_ok = True) conn = sqlite3.connect(auth_dir / "auth.db") + # A live server writes this DB while the CLI runs; the default lock wait is zero. + conn.execute("PRAGMA busy_timeout=5000") # Mirror backend storage.get_connection: this path can create auth/ and # auth.db (the pre-exposure gate writes here first), and sqlite3.connect # makes the DB 0644 under a 022 umask. Keep both private. @@ -529,7 +534,8 @@ def _connect_auth_db() -> sqlite3.Connection: token_hash TEXT NOT NULL, username TEXT NOT NULL, expires_at TEXT NOT NULL, - is_desktop INTEGER NOT NULL DEFAULT 0 + is_desktop INTEGER NOT NULL DEFAULT 0, + secret_gen TEXT ); """ ) @@ -564,6 +570,8 @@ def _connect_auth_db() -> sqlite3.Connection: refresh_columns = {row[1] for row in conn.execute("PRAGMA table_info(refresh_tokens)")} if "is_desktop" not in refresh_columns: conn.execute("ALTER TABLE refresh_tokens ADD COLUMN is_desktop INTEGER NOT NULL DEFAULT 0") + if "secret_gen" not in refresh_columns: + conn.execute("ALTER TABLE refresh_tokens ADD COLUMN secret_gen TEXT") conn.commit() return conn @@ -697,12 +705,30 @@ def _bootstrap_deadline_active() -> bool: return True -def _cli_update_password(conn: sqlite3.Connection, username: str, new_password: str) -> None: +def _generate_reset_password() -> str: + """Readable 4-word passphrase; the user has to type this one back in.""" + try: + import diceware + return diceware.get_passphrase( + options = diceware.handle_options(args = ["-n", "4", "-d", "", "-c"]) + ) + except Exception: + return secrets.token_urlsafe(24) + + +def _cli_update_password( + conn: sqlite3.Connection, + username: str, + new_password: str, + *, + revoke_api_keys: bool = False, +) -> None: """CLI mirror of backend update_password + change-password route effects. One transaction: rehash, rotate the JWT secret, clear must_change_password, - revoke refresh tokens (PR #6651 finding), and drop the desktop secret. File - cleanup happens after commit; a failed unlink must not roll the change back. + revoke refresh tokens (PR #6651 finding), drop the desktop secret, and (for a + reset) the API keys the old credential could have minted. File cleanup happens + after commit; a failed unlink must not roll the change back. """ password_salt, password_hash = _hash_password(new_password) with conn: @@ -719,6 +745,8 @@ def _cli_update_password(conn: sqlite3.Connection, username: str, new_password: "DELETE FROM app_secrets WHERE key IN (?, ?)", (DESKTOP_SECRET_HASH_KEY, DESKTOP_SECRET_CREATED_AT_KEY), ) + if revoke_api_keys: + conn.execute("DELETE FROM api_keys") for stale in (BOOTSTRAP_PASSWORD_FILE, DESKTOP_SECRET_FILE): stale_path = STUDIO_HOME / "auth" / stale try: @@ -728,8 +756,8 @@ def _cli_update_password(conn: sqlite3.Connection, username: str, new_password: # change back. But a locked-yet-writable file (Windows AV, read-only # auth dir) must be truncated: otherwise its stale plaintext survives # and generate_bootstrap_password() would re-validate this revoked - # credential after a later reset-password deletes auth.db. Mirrors - # backend clear_bootstrap_password(). + # credential if auth.db is ever recreated. Mirrors backend + # clear_bootstrap_password(). try: stale_path.write_text("", encoding = "utf-8") cleared = True @@ -787,8 +815,8 @@ def _apply_supplied_password_before_launch(supplied_password: "str | None") -> N if not row[2]: typer.echo( "Error: an Unsloth admin password is already set; --password only sets " - "the initial password. Run `unsloth studio reset-password` first " - "(or change it in the UI).", + "the initial password. Change it in the UI, or run `unsloth studio " + "reset-password` for a new one.", err = True, ) raise typer.Exit(1) @@ -2237,6 +2265,9 @@ def run( # Headless serving prints its own URL/API-key banner; the Tauri-only # TAURI_PORT line would corrupt that machine-parseable output. emit_tauri_port = False, + # We read the bound port back below, so a fallback past another Studio is + # safe here and keeps side-by-side model runs working. + abort_if_own_studio = False, ) # Forward the frontend validated before the gate (in-venv path). if resolved_frontend is not None: @@ -2396,6 +2427,7 @@ def run( # ── unsloth studio stop ─────────────────────────────────────────────── _PID_FILE = STUDIO_HOME / "studio.pid" +PID_FILE_GLOB = "studio-*.pid" def _pid_alive(pid: int) -> bool: @@ -2425,58 +2457,210 @@ def _pid_alive(pid: int) -> bool: return True -@studio_app.command() -def stop(): - """Stop a running Unsloth Studio server. +def _parse_pid_record(text: str) -> "tuple[int, float | None] | None": + """Parse ``pid`` / optional ``create_time`` from PID file contents.""" + lines = text.splitlines() + if not lines or not lines[0].strip().isdigit(): + return None + try: + # isdigit() is not enough: "²".isdigit() is True but int() rejects it. + pid = int(lines[0].strip()) + except ValueError: + return None + # kill(0) signals our whole process group; kill(1) is init. Never either. + if pid < 2: + return None + created = None + if len(lines) > 1: + try: + created = float(lines[1].strip()) + except ValueError: + created = None + return pid, created - Reads the PID from ~/.unsloth/studio/studio.pid and sends SIGTERM - (or TerminateProcess on Windows) to shut it down gracefully. + +def _read_pid_record(path: Path) -> "tuple[int, float | None] | None": + """Parse ``pid`` / optional ``create_time`` from a PID file.""" + try: + text = path.read_text(encoding = "utf-8") + except (OSError, UnicodeDecodeError): + return None + return _parse_pid_record(text) + + +def _unlink_quietly(path: Path) -> None: + """Drop a record without letting one bad file end the loop. + + An undeletable record must not stop us reaching the other servers -- that is + the orphan this command exists to prevent. """ + try: + path.unlink(missing_ok = True) + except OSError as e: + typer.echo(f"Could not remove PID file {path.name}: {e}", err = True) + + +def _report_unreadable(paths: "list[Path]") -> None: + """Say which servers we could not reach, since `stop` is about to exit 1.""" + names = ", ".join(sorted(p.name for p in paths)) + typer.echo( + f"Could not read {len(paths)} PID file(s): {names}. A server recorded " + f"there may still be running; re-run with permission to read " + f"{STUDIO_HOME} to stop it.", + err = True, + ) + + +def _pid_file_entries( + unreadable: "list[Path] | None" = None, +) -> "list[tuple[int, list[float | None], list[Path]]]": + """(pid, create_times, files) per recorded server, including the legacy studio.pid. + + Paths that could not be read are appended to `unreadable` when given, so the + caller can tell "nothing is running" apart from "something is running and we + could not see it". + + Grouped by PID: a server writes both its per-port file and studio.pid, and + signalling twice would hit the SIG_DFL the first SIGTERM installs, hard-killing + it mid-shutdown. Every recorded time is kept -- a stale file and a live server + can share a PID, and the stale one must not veto the live one. + """ + by_pid: "dict[int, tuple[list[float | None], list[Path]]]" = {} + try: + paths = sorted(STUDIO_HOME.glob(PID_FILE_GLOB)) + [_PID_FILE] + except OSError: + paths = [_PID_FILE] + seen = set() + for path in paths: + if path in seen or not path.is_file(): + continue + seen.add(path) + try: + text = path.read_text(encoding = "utf-8") + except (OSError, UnicodeDecodeError) as e: + # Unreadable is not the same as invalid. A root-owned record, or one + # caught mid-write, still belongs to a live server, and deleting it + # strands that server -- the bug this command exists to fix. + typer.echo(f"Cannot read PID file {path.name}: {e}", err = True) + if unreadable is not None: + unreadable.append(path) + continue + record = _parse_pid_record(text) + if record is None: + typer.echo(f"Ignoring invalid PID file {path.name}") + _unlink_quietly(path) + continue + pid, created = record + created_times, files = by_pid.setdefault(pid, ([], [])) + created_times.append(created) + files.append(path) + return [(pid, times, files) for pid, (times, files) in by_pid.items()] + + +def _pid_is_studio_server(pid: int, created_times: "Sequence[float | None]" = ()) -> bool: + """False only when a recorded start time proves this PID is a different process. + + Any recorded time matching is enough -- a stale record must not veto a live + server that reused the PID. Records with no time at all (a legacy studio.pid, + or a server started without psutil) cannot be checked, so they are trusted: + the old `stop` signalled with no checks at all, and skipping a live server is + the orphan bug this exists to fix. + + An untimed record sitting *alongside* a timed one carries no information, so + it must not cancel the timed one either. Every current server writes both a + timed per-port record and an untimed studio.pid, so letting the untimed half + win made this check inert exactly where it matters and let `stop` SIGTERM an + unrelated process that had inherited the PID. + """ + known = [c for c in created_times if c is not None] + if not known: + return True + try: + import psutil + actual = psutil.Process(pid).create_time() + except Exception: + return True + return any(abs(actual - c) < 1.0 for c in known) + + +def _signal_stop(pid: int) -> "str | None": + """SIGTERM (or taskkill) the pid. Returns an error string, or None on success.""" import signal as _signal - if not _PID_FILE.is_file(): - typer.echo("No running Unsloth server found (no PID file).") - raise typer.Exit(0) - - pid_text = _PID_FILE.read_text(encoding = "utf-8").strip() - if not pid_text.isdigit(): - typer.echo(f"Invalid PID file contents: {pid_text}") - _PID_FILE.unlink(missing_ok = True) - raise typer.Exit(1) - - pid = int(pid_text) - - # Check if still alive (os.kill(pid, 0) is invalid on Windows -- see _pid_alive). - if not _pid_alive(pid): - typer.echo(f"Unsloth server (PID {pid}) is not running. Cleaning up stale PID file.") - _PID_FILE.unlink(missing_ok = True) - raise typer.Exit(0) - - # Send SIGTERM (graceful shutdown) or TerminateProcess on Windows + if pid < 2: + return f"refusing to signal PID {pid}" try: if sys.platform == "win32": # /T also stops llama-server children, which otherwise keep GPU and port. subprocess.run(["taskkill", "/PID", str(pid), "/T", "/F"], check = True) else: os.kill(pid, _signal.SIGTERM) - typer.echo(f"Sent shutdown signal to Unsloth server (PID {pid}).") except ProcessLookupError: - typer.echo(f"Unsloth server (PID {pid}) already exited.") - _PID_FILE.unlink(missing_ok = True) - raise typer.Exit(0) + return None except Exception as e: - typer.echo(f"Failed to stop Unsloth server (PID {pid}): {e}", err = True) - raise typer.Exit(1) + return str(e) + return None - # Wait briefly for the process to exit and clean up. + +@studio_app.command() +def stop(): + """Stop every running Unsloth Studio server for this STUDIO_HOME. + + The port fallback can leave more than one running, so stop them all. + """ + unreadable: "list[Path]" = [] + entries = _pid_file_entries(unreadable) + if not entries: + if unreadable: + # Reporting success here would be a lie: the records we could not + # read are kept, and the servers behind them are still serving. + _report_unreadable(unreadable) + raise typer.Exit(1) + typer.echo("No running Unsloth server found (no PID file).") + raise typer.Exit(0) + + signalled, failed = [], [] + for pid, created_times, paths in entries: + if not _pid_alive(pid) or not _pid_is_studio_server(pid, created_times): + for path in paths: + _unlink_quietly(path) + continue + error = _signal_stop(pid) + if error is not None: + failed.append((pid, error)) + typer.echo(f"Failed to stop Unsloth server (PID {pid}): {error}", err = True) + continue + typer.echo(f"Sent shutdown signal to Unsloth server (PID {pid}).") + signalled.append((pid, paths)) + + if not signalled and not failed: + if unreadable: + _report_unreadable(unreadable) + raise typer.Exit(1) + typer.echo("No running Unsloth server found (cleaned up stale PID files).") + raise typer.Exit(0) + + pending = list(signalled) for _ in range(10): + if not pending: + break time.sleep(0.5) - if not _pid_alive(pid): - _PID_FILE.unlink(missing_ok = True) - typer.echo("Unsloth server stopped.") - raise typer.Exit(0) + for entry in list(pending): + pid, paths = entry + if not _pid_alive(pid): + for path in paths: + _unlink_quietly(path) + pending.remove(entry) - typer.echo("Unsloth server is shutting down (may take a few seconds).") + stopped = len(signalled) - len(pending) + if stopped: + typer.echo(f"Unsloth server{'s' if stopped > 1 else ''} stopped ({stopped}).") + for pid, _paths in pending: + typer.echo(f"Unsloth server (PID {pid}) is shutting down (may take a few seconds).") + if unreadable: + _report_unreadable(unreadable) + if failed or unreadable: + raise typer.Exit(1) # ── unsloth studio setup / update ───────────────────────────────────── @@ -2890,59 +3074,33 @@ def provision_desktop_auth(): def reset_password(): """Reset the Unsloth admin password. - Deletes the auth database so that a fresh admin account with a new - random password is created on the next server start. The Unsloth - server must be restarted after running this command. + Rotates the credential in place: a running Unsloth accepts the new password on + its next request, so there is nothing to restart. Shared /p preview links are + not revoked -- rotate those in Settings if the old password leaked. """ - auth_dir = STUDIO_HOME / "auth" - db_file = auth_dir / "auth.db" - stale_files = [ - auth_dir / BOOTSTRAP_PASSWORD_FILE, - auth_dir / DESKTOP_SECRET_FILE, - ] - had_db = db_file.exists() - - # Delete auth.db FIRST and prove it is gone before touching the seeded - # credential files. If it cannot be removed (a running Unsloth or Windows - # holds it open, or a read-only auth dir), abort with the credential files - # untouched: deleting them while an un-resettable DB (must_change_password=1) - # survives would lock a forgotten-password reset out of any recovery - # credential. Failing here leaves a consistent, still-recoverable state. + new_password = _generate_reset_password() try: - db_file.unlink(missing_ok = True) - except OSError as exc: + conn = _connect_auth_db() + except (OSError, sqlite3.Error) as exc: typer.echo( - f"Error: could not delete the auth database ({exc}). Stop any running " - "Unsloth and retry; no credential files were changed.", + f"Error: could not open the auth database ({exc}). Check that " + f"{STUDIO_HOME / 'auth'} is writable; if auth.db itself is unreadable, stop " + "Unsloth, delete it, and start again to re-seed.", err = True, ) raise typer.Exit(1) - # The DB is gone, so the next start re-seeds. Invalidate the seeded plaintext - # credential files so that re-seed generates a FRESH password instead of - # reusing a stale one: unlink only ignores FileNotFoundError, so a - # locked/undeletable file (Windows AV, read-only dir) would otherwise survive - # and generate_bootstrap_password() would read it back and re-validate the - # credential this reset revoked. Truncate on unlink failure; if a file can be - # neither removed nor truncated, fail closed -- the DB is already gone, so a - # surviving plaintext would be reused, and the user must remove it manually. - for path in stale_files: - try: - path.unlink(missing_ok = True) - except OSError: - try: - path.write_text("", encoding = "utf-8") - except OSError as exc: - typer.echo( - f"Error: could not remove or clear {path.name} ({exc}); delete " - "it manually before restarting Unsloth or the old password may " - "be reused.", - err = True, - ) - raise typer.Exit(1) + try: + _ensure_cli_default_admin(conn) + _cli_update_password(conn, DEFAULT_ADMIN_USERNAME, new_password, revoke_api_keys = True) + except (OSError, sqlite3.Error) as exc: + typer.echo(f"Error: could not reset the password ({exc}).", err = True) + raise typer.Exit(1) + finally: + conn.close() - if not had_db: - typer.echo("No auth database found -- nothing to reset.") - raise typer.Exit(0) - - typer.echo("Auth database deleted. Restart Unsloth Studio to get a new password.") + typer.echo(f"New password for '{DEFAULT_ADMIN_USERNAME}': {new_password}") + typer.echo( + "Sessions and API keys revoked. A running Unsloth takes it on the next request, " + "though repeated failed logins can hold the rate limit shut for up to a minute." + ) diff --git a/unsloth_cli/tests/conftest.py b/unsloth_cli/tests/conftest.py new file mode 100644 index 0000000000..bb42914e69 --- /dev/null +++ b/unsloth_cli/tests/conftest.py @@ -0,0 +1,26 @@ +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 + +"""Shared fixtures for the unsloth_cli tests.""" + +import sys +import types + +import pytest + + +@pytest.fixture +def stub_tool_policy_state(monkeypatch): + """Stub the backend's `state.tool_policy`, which run() imports in-venv. + + It lives under studio/backend, so it only imports once something has put + that directory on sys.path. Tests that reach the in-venv branch of run() + used to get that for free from whichever file ran earlier and did it as a + side effect, which made them pass only in a full-directory run. + """ + state_mod = types.ModuleType("state") + tp_mod = types.ModuleType("state.tool_policy") + tp_mod.set_tool_policy = lambda *a, **k: None + state_mod.tool_policy = tp_mod + monkeypatch.setitem(sys.modules, "state", state_mod) + monkeypatch.setitem(sys.modules, "state.tool_policy", tp_mod) diff --git a/unsloth_cli/tests/test_studio_password_prompt.py b/unsloth_cli/tests/test_studio_password_prompt.py index 6e9a2c1d52..753c22edc2 100644 --- a/unsloth_cli/tests/test_studio_password_prompt.py +++ b/unsloth_cli/tests/test_studio_password_prompt.py @@ -251,7 +251,7 @@ def test_studio_default_prompt_rejects_current_password(monkeypatch, tmp_path): studio_mod = _studio() events = _install_prompt_env(monkeypatch, tmp_path, interactive = True) _seed_auth(studio_mod) - bootstrap_pw = (tmp_path / "auth" / studio_mod.BOOTSTRAP_PASSWORD_FILE).read_text() + bootstrap_pw = (tmp_path / "auth" / studio_mod.BOOTSTRAP_PASSWORD_FILE).read_text().strip() _invoke_studio_default(monkeypatch, events, ["--secure"]) @@ -626,6 +626,12 @@ def test_studio_default_in_venv_broken_backend_exits_before_stripping_bootstrap( # Pretend we are already inside the studio venv, with a broken backend. monkeypatch.setattr(sys, "prefix", str(tmp_path / "unsloth_studio")) + # A built dist is not present in a fresh clone. The missing-frontend gate + # runs first and has its own test below; stub it so this one reaches the + # backend check it is actually about. + monkeypatch.setattr( + studio_mod, "_find_frontend_dist", lambda: Path("/fake/studio/frontend/dist") + ) def _boom(): raise ImportError("cannot import backend run.py") @@ -957,7 +963,9 @@ def test_run_reexec_forwards_resolved_frontend_on_public_launch(monkeypatch, tmp exec_argv = [argv for kind, argv in events if kind == "exec"][0] assert "--frontend" in exec_argv, exec_argv - assert exec_argv[exec_argv.index("--frontend") + 1] == "/fake/studio/frontend/dist", exec_argv + # str(Path(...)), not the literal: Windows renders it with backslashes. + expected_dist = str(Path("/fake/studio/frontend/dist")) + assert exec_argv[exec_argv.index("--frontend") + 1] == expected_dist, exec_argv def test_run_non_tty_persists_seeded_admin_on_fresh_home(monkeypatch, tmp_path): @@ -1032,50 +1040,173 @@ def test_bootstrap_deadline_active_mirrors_backend_parsing(monkeypatch, raw, exp assert studio_mod._bootstrap_deadline_active() is expected -def test_reset_password_truncates_locked_bootstrap_after_db_delete(monkeypatch, tmp_path): - # reset-password deletes auth.db first, then invalidates the seeded credential - # files. A locked/undeletable .bootstrap_password must be truncated so its - # stale plaintext cannot be re-seeded (generate_bootstrap_password reuses a - # non-empty file), while the reset still succeeds. - import pathlib - - studio_mod = _studio() - monkeypatch.setattr(studio_mod, "STUDIO_HOME", tmp_path) - _seed_auth(studio_mod) - auth_dir = tmp_path / "auth" - bootstrap_file = auth_dir / studio_mod.BOOTSTRAP_PASSWORD_FILE - db_file = auth_dir / "auth.db" - assert bootstrap_file.exists() and db_file.exists() - assert bootstrap_file.read_text().strip() - - _real_unlink = pathlib.Path.unlink - - def _boom_unlink(self, *a, **k): - if self.name == studio_mod.BOOTSTRAP_PASSWORD_FILE: - raise OSError("locked") - return _real_unlink(self, *a, **k) - - monkeypatch.setattr(pathlib.Path, "unlink", _boom_unlink) - +def _reset_password_cli(studio_mod): import typer as _typer app = _typer.Typer() app.command()(studio_mod.reset_password) - result = CliRunner().invoke(app, [], catch_exceptions = True) + return CliRunner().invoke(app, [], catch_exceptions = True) + + +def _password_works(studio_mod, candidate): + conn = studio_mod._connect_auth_db() + try: + row = conn.execute( + "SELECT password_salt, password_hash FROM auth_user WHERE username = ?", + (studio_mod.DEFAULT_ADMIN_USERNAME,), + ).fetchone() + finally: + conn.close() + return studio_mod._pbkdf2_hex(candidate, row[0].encode("utf-8")) == row[1] + + +def _printed_password(result): + line = next(l for l in result.output.splitlines() if l.startswith("New password for")) + return line.split(": ", 1)[1].strip() + + +def test_reset_password_rotates_in_place_without_deleting_the_db(monkeypatch, tmp_path): + # The DB survives, so a running server keeps its admin row and the new password. + studio_mod = _studio() + monkeypatch.setattr(studio_mod, "STUDIO_HOME", tmp_path) + _seed_auth(studio_mod) + db_file = tmp_path / "auth" / "auth.db" + before = _auth_state(studio_mod) + + result = _reset_password_cli(studio_mod) assert result.exit_code == 0, result.output - assert not db_file.exists() - # The locked file survives, but truncated -- no reusable plaintext. - assert bootstrap_file.exists() - assert bootstrap_file.read_text() == "" + assert db_file.exists() + after = _auth_state(studio_mod) + assert after["password_hash"] != before["password_hash"] + assert after["jwt_secret"] != before["jwt_secret"] + assert _password_works(studio_mod, _printed_password(result)) + + +def test_reset_password_waits_out_a_concurrent_writer(monkeypatch, tmp_path): + # The CLI now writes while the server does; without a busy_timeout this fails. + import threading + import time + + studio_mod = _studio() + monkeypatch.setattr(studio_mod, "STUDIO_HOME", tmp_path) + _seed_auth(studio_mod) + released = threading.Event() + + def hold_write_lock(): + conn = sqlite3.connect(_auth_db(tmp_path)) + conn.execute("BEGIN IMMEDIATE") + conn.execute( + "INSERT INTO refresh_tokens (token_hash, username, expires_at) " + "VALUES ('held', 'unsloth', '2099-01-01T00:00:00')" + ) + time.sleep(0.5) + conn.rollback() + conn.close() + released.set() + + holder = threading.Thread(target = hold_write_lock) + holder.start() + time.sleep(0.1) + result = _reset_password_cli(studio_mod) + holder.join() + + assert released.is_set() + assert result.exit_code == 0, result.output + assert _password_works(studio_mod, _printed_password(result)) + + +def test_reset_password_revokes_sessions_and_api_keys(monkeypatch, tmp_path): + # Deleting auth.db used to drop these implicitly. + studio_mod = _studio() + monkeypatch.setattr(studio_mod, "STUDIO_HOME", tmp_path) + _seed_auth(studio_mod) + conn = studio_mod._connect_auth_db() + conn.execute( + "INSERT INTO api_keys (username, key_prefix, key_hash, name, created_at) " + "VALUES (?, 'sk-x', 'hash', 'k', '2026-01-01T00:00:00')", + (studio_mod.DEFAULT_ADMIN_USERNAME,), + ) + conn.commit() + conn.close() + + assert _reset_password_cli(studio_mod).exit_code == 0 + + conn = studio_mod._connect_auth_db() + try: + assert conn.execute("SELECT COUNT(*) FROM api_keys").fetchone()[0] == 0 + assert conn.execute("SELECT COUNT(*) FROM refresh_tokens").fetchone()[0] == 0 + finally: + conn.close() + + +def test_reset_password_leaves_the_account_ready_to_log_in(monkeypatch, tmp_path): + # must_change_password stays 0 on purpose: at 1 a running server injects its + # startup-cached (now wrong) bootstrap password into the login page. + studio_mod = _studio() + monkeypatch.setattr(studio_mod, "STUDIO_HOME", tmp_path) + _seed_auth(studio_mod) + + assert _reset_password_cli(studio_mod).exit_code == 0 + + assert _auth_state(studio_mod)["must_change_password"] == 0 + assert not (tmp_path / "auth" / studio_mod.BOOTSTRAP_PASSWORD_FILE).exists() + + +def test_reset_password_seeds_the_admin_when_no_db_exists(monkeypatch, tmp_path): + studio_mod = _studio() + monkeypatch.setattr(studio_mod, "STUDIO_HOME", tmp_path) + + result = _reset_password_cli(studio_mod) + + assert result.exit_code == 0, result.output + assert _password_works(studio_mod, _printed_password(result)) + + +def test_reset_password_reports_an_unwritable_auth_dir(monkeypatch, tmp_path): + # _connect_auth_db creates auth/ before it opens SQLite, so a read-only Unsloth + # home raises OSError, not sqlite3.Error. + import pathlib + + studio_mod = _studio() + monkeypatch.setattr(studio_mod, "STUDIO_HOME", tmp_path) + + def _boom_mkdir(self, *a, **k): + raise PermissionError("read-only") + + monkeypatch.setattr(pathlib.Path, "mkdir", _boom_mkdir) + + result = _reset_password_cli(studio_mod) + + assert result.exit_code == 1, result.output + assert not isinstance(result.exception, OSError) + combined = (result.output or "") + (getattr(result, "stderr", "") or "") + assert "could not open the auth database" in combined.lower() + + +def test_reset_password_reports_an_unreadable_db(monkeypatch, tmp_path): + # Deleting a corrupt DB here would revive the bug: a running server would be + # left with no admin row, rejecting the correct password until restarted. + studio_mod = _studio() + monkeypatch.setattr(studio_mod, "STUDIO_HOME", tmp_path) + auth_dir = tmp_path / "auth" + auth_dir.mkdir() + (auth_dir / "auth.db").write_text("not a database") + + result = _reset_password_cli(studio_mod) + + assert result.exit_code == 1, result.output + assert (auth_dir / "auth.db").exists() + combined = (result.output or "") + (getattr(result, "stderr", "") or "") + assert "could not open the auth database" in combined.lower() def test_cli_update_password_truncates_locked_bootstrap_after_change(monkeypatch, tmp_path): # After a CLI/interactive password change the seeded .bootstrap_password is # deleted. If it cannot be unlinked but is still writable (locked file / # read-only dir), it must be TRUNCATED so its stale plaintext cannot be - # re-seeded by generate_bootstrap_password() after a later reset-password - # deletes auth.db. The change is already committed, so it must NOT roll back. + # re-seeded by generate_bootstrap_password() if auth.db is ever recreated. The + # change is already committed, so it must NOT roll back. import pathlib studio_mod = _studio() @@ -1103,88 +1234,6 @@ def test_cli_update_password_truncates_locked_bootstrap_after_change(monkeypatch assert bootstrap_file.read_text() == "" -def test_reset_password_fails_closed_when_db_cannot_be_deleted(monkeypatch, tmp_path): - # If auth.db cannot be removed (running Unsloth / Windows lock, read-only dir), - # reset must abort BEFORE touching the credential files -- deleting them while - # an un-resettable must_change_password=1 DB survives would lock a - # forgotten-password reset out with no recovery credential. - import pathlib - - studio_mod = _studio() - monkeypatch.setattr(studio_mod, "STUDIO_HOME", tmp_path) - _seed_auth(studio_mod) - auth_dir = tmp_path / "auth" - bootstrap_file = auth_dir / studio_mod.BOOTSTRAP_PASSWORD_FILE - db_file = auth_dir / "auth.db" - assert bootstrap_file.exists() and db_file.exists() - - _real_unlink = pathlib.Path.unlink - - def _boom_unlink(self, *a, **k): - if self.name == "auth.db": - raise OSError("database is locked") - return _real_unlink(self, *a, **k) - - monkeypatch.setattr(pathlib.Path, "unlink", _boom_unlink) - - import typer as _typer - - app = _typer.Typer() - app.command()(studio_mod.reset_password) - result = CliRunner().invoke(app, [], catch_exceptions = True) - - assert result.exit_code == 1, result.output - # DB still there; credential files untouched (no lockout, no half-done reset). - assert db_file.exists() - assert bootstrap_file.exists() - assert bootstrap_file.read_text().strip() - combined = (result.output or "") + (getattr(result, "stderr", "") or "") - assert "could not delete the auth database" in combined.lower() - - -def test_reset_password_fails_closed_when_credential_cannot_be_invalidated(monkeypatch, tmp_path): - # If a seeded credential file can be neither unlinked nor truncated, reset must - # fail closed: auth.db is already gone, so a surviving plaintext would be - # re-seeded and re-validate the revoked password. - import pathlib - - studio_mod = _studio() - monkeypatch.setattr(studio_mod, "STUDIO_HOME", tmp_path) - _seed_auth(studio_mod) - auth_dir = tmp_path / "auth" - bootstrap_file = auth_dir / studio_mod.BOOTSTRAP_PASSWORD_FILE - db_file = auth_dir / "auth.db" - assert bootstrap_file.exists() and db_file.exists() - - _real_unlink = pathlib.Path.unlink - _real_write_text = pathlib.Path.write_text - - def _boom_unlink(self, *a, **k): - if self.name == studio_mod.BOOTSTRAP_PASSWORD_FILE: - raise OSError("locked") - return _real_unlink(self, *a, **k) - - def _boom_write_text(self, *a, **k): - if self.name == studio_mod.BOOTSTRAP_PASSWORD_FILE: - raise OSError("read-only") - return _real_write_text(self, *a, **k) - - monkeypatch.setattr(pathlib.Path, "unlink", _boom_unlink) - monkeypatch.setattr(pathlib.Path, "write_text", _boom_write_text) - - import typer as _typer - - app = _typer.Typer() - app.command()(studio_mod.reset_password) - result = CliRunner().invoke(app, [], catch_exceptions = True) - - assert result.exit_code == 1, result.output - # auth.db was deleted first; the un-invalidatable file is reported for manual removal. - assert not db_file.exists() - combined = (result.output or "") + (getattr(result, "stderr", "") or "") - assert "delete it manually" in combined.lower() - - def test_connect_auth_db_creates_private_files(monkeypatch, tmp_path): # Fresh install: the CLI gate writes the password hash + JWT secret before # the backend ever runs, so this path must apply the same 0700/0600 modes @@ -1204,6 +1253,37 @@ def test_connect_auth_db_creates_private_files(monkeypatch, tmp_path): assert stat.S_IMODE((auth_dir / "auth.db").stat().st_mode) == 0o600 +def test_write_auth_secret_terminates_the_file_with_a_newline(monkeypatch, tmp_path): + # Shared by .bootstrap_password and .desktop_secret; every reader strips. + studio_mod = _studio() + path = tmp_path / ".desktop_secret" + + studio_mod._write_auth_secret(path, "desktop-abc123") + + # Bytes: read_text would decode CRLF back to "\n" and hide a CR. + assert path.read_bytes() == b"desktop-abc123\n" + + +def test_seeded_bootstrap_file_ends_with_a_newline(monkeypatch, tmp_path): + studio_mod = _studio() + monkeypatch.setattr(studio_mod, "STUDIO_HOME", tmp_path) + _seed_auth(studio_mod) + + raw = (tmp_path / "auth" / studio_mod.BOOTSTRAP_PASSWORD_FILE).read_bytes() + + assert raw.endswith(b"\n") and not raw.endswith(b"\r\n") + + conn = sqlite3.connect(_auth_db(tmp_path)) + try: + salt, pwd_hash = conn.execute( + "SELECT password_salt, password_hash FROM auth_user WHERE username = ?", + (studio_mod.DEFAULT_ADMIN_USERNAME,), + ).fetchone() + finally: + conn.close() + assert studio_mod._pbkdf2_hex(raw.decode("utf-8").strip(), salt.encode("utf-8")) == pwd_hash + + # ── non-interactive --password / UNSLOTH_STUDIO_PASSWORD / stdin ────── @@ -1284,7 +1364,7 @@ def test_studio_default_password_must_differ_fails_closed(monkeypatch, tmp_path) studio_mod = _studio() events = _install_prompt_env(monkeypatch, tmp_path, interactive = True) _seed_auth(studio_mod) - bootstrap_pw = (tmp_path / "auth" / studio_mod.BOOTSTRAP_PASSWORD_FILE).read_text() + bootstrap_pw = (tmp_path / "auth" / studio_mod.BOOTSTRAP_PASSWORD_FILE).read_text().strip() result = _invoke_studio_default(monkeypatch, events, ["--secure", "--password", bootstrap_pw]) @@ -1368,30 +1448,3 @@ def test_studio_default_password_applies_on_headless_wildcard_no_tunnel(monkeypa assert after["must_change_password"] == 0 assert after["password_hash"] != before["password_hash"] assert "--password" not in _exec_argv(events) - - -def test_reset_password_then_password_roundtrip(monkeypatch, tmp_path): - # After reset-password wipes the DB, the next start re-seeds a fresh admin - # that again requires a change, so --password can set a new initial password. - import typer - - studio_mod = _studio() - monkeypatch.setattr(studio_mod, "STUDIO_HOME", tmp_path) - _seed_auth(studio_mod) - conn = studio_mod._connect_auth_db() - studio_mod._cli_update_password(conn, studio_mod.DEFAULT_ADMIN_USERNAME, "first-password-1") - conn.close() - assert _auth_state(studio_mod)["must_change_password"] == 0 - - # reset-password deletes the auth DB + seeded credential files. - try: - studio_mod.reset_password() - except typer.Exit: - pass - assert not (tmp_path / "auth" / "auth.db").exists() - - # A restart re-seeds (ensure_default_admin, must_change=1); --password sets anew. - events = _install_prompt_env(monkeypatch, tmp_path, interactive = True) - _invoke_studio_default(monkeypatch, events, ["--secure", "--password", "second-password-2"]) - assert [kind for kind, _ in events] == ["exec"], events - assert _auth_state(studio_mod)["must_change_password"] == 0 diff --git a/unsloth_cli/tests/test_studio_run_parallel_flag.py b/unsloth_cli/tests/test_studio_run_parallel_flag.py index f1a4e69b81..9a3260d699 100644 --- a/unsloth_cli/tests/test_studio_run_parallel_flag.py +++ b/unsloth_cli/tests/test_studio_run_parallel_flag.py @@ -606,14 +606,14 @@ def test_studio_default_exposes_parallel_option(): assert "--parallel" in decls assert "--n-parallel" in decls assert ( - getattr(opt, "default", None) == 1 - ), "studio_default --parallel must default to 1 (pre-PR); `run` is 4" + getattr(opt, "default", None) == studio_mod._PARALLEL_DEFAULT_PLAIN + ), "studio_default --parallel must use _PARALLEL_DEFAULT_PLAIN" assert getattr(opt, "min", None) == 1 assert getattr(opt, "max", None) == 64 @pytest.mark.parametrize("value", [1, 4, 8, 64]) -def test_in_venv_path_passes_parallel_to_run_server(monkeypatch, value): +def test_in_venv_path_passes_parallel_to_run_server(monkeypatch, value, stub_tool_policy_state): """In-venv path must forward --parallel to run_server(llama_parallel_slots=N), not the old hardcoded 4.""" studio_mod = _load_run_command() @@ -679,7 +679,6 @@ def test_api_only_option_is_registered(): "extra,present", [ (["--api-only"], True), - (["--secure", "--api-only"], True), # secure headless path ([], False), ], ) @@ -691,8 +690,25 @@ def test_reexec_forwards_api_only(monkeypatch, extra, present): assert ("--api-only" in argv) is present, argv +def test_secure_api_only_is_refused_before_any_reexec(monkeypatch, tmp_path): + """`--secure --api-only` used to re-exec; the pre-exposure gate now refuses + it, because api-only has no login page and the bootstrap deadline does not + apply, so the seeded password could never be changed.""" + studio_mod = _load_run_command() + monkeypatch.setattr(studio_mod, "STUDIO_HOME", tmp_path) + + result, captured = _invoke_run(monkeypatch, _BASE + ["--secure", "--api-only"]) + + assert captured == [], captured + assert result.exit_code != 0 + combined = (result.output or "") + (getattr(result, "stderr", "") or "") + assert "default admin password was never changed" in combined.lower() + + @pytest.mark.parametrize("extra,expected", [(["--api-only"], True), ([], False)]) -def test_in_venv_path_passes_api_only_to_run_server(monkeypatch, extra, expected): +def test_in_venv_path_passes_api_only_to_run_server( + monkeypatch, extra, expected, stub_tool_policy_state +): """In-venv path must forward --api-only to run_server(api_only=...).""" studio_mod = _load_run_command() diff --git a/unsloth_cli/tests/test_studio_secure_flag.py b/unsloth_cli/tests/test_studio_secure_flag.py index 2a67aad95a..118f227949 100644 --- a/unsloth_cli/tests/test_studio_secure_flag.py +++ b/unsloth_cli/tests/test_studio_secure_flag.py @@ -244,7 +244,7 @@ class _RunServerCaptured(SystemExit): self.kwargs = dict(kwargs) -def test_run_in_venv_passes_secure_and_forces_host(monkeypatch, tmp_path): +def test_run_in_venv_passes_secure_and_forces_host(monkeypatch, tmp_path, stub_tool_policy_state): import types studio_mod = _studio() @@ -261,6 +261,11 @@ def test_run_in_venv_passes_secure_and_forces_host(monkeypatch, tmp_path): fake_venv = tmp_path / "unsloth_studio" monkeypatch.setattr(sys, "prefix", str(fake_venv)) + # A built dist is not present in a fresh clone, and without it the public + # launch gate exits before run_server is ever reached. + monkeypatch.setattr( + studio_mod, "_find_frontend_dist", lambda: Path("/fake/studio/frontend/dist") + ) from unsloth_cli import _tool_policy as _tp_mod diff --git a/unsloth_cli/tests/test_studio_stop.py b/unsloth_cli/tests/test_studio_stop.py new file mode 100644 index 0000000000..74e34d4fa1 --- /dev/null +++ b/unsloth_cli/tests/test_studio_stop.py @@ -0,0 +1,530 @@ +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 + +"""`unsloth studio stop` must stop every server it started. + +With one PID file the second launch overwrote the first entry, so stop killed +the newer server, claimed success, and left the older one serving. +""" + +from __future__ import annotations + +import sys +from pathlib import Path +from types import SimpleNamespace + +import pytest +from typer.testing import CliRunner + + +_REPO_ROOT = Path(__file__).resolve().parents[2] +if str(_REPO_ROOT) not in sys.path: + sys.path.insert(0, str(_REPO_ROOT)) + + +def _studio(): + from unsloth_cli.commands import studio as _studio_mod + return _studio_mod + + +# Captured before _install stubs it, for the tests that exercise it. +_REAL_IS_STUDIO_SERVER = _studio()._pid_is_studio_server + + +def _install( + monkeypatch, + tmp_path, + *, + alive, + killed = None, +): + """Point the CLI at tmp_path and fake process liveness.""" + studio_mod = _studio() + monkeypatch.setattr(studio_mod, "STUDIO_HOME", tmp_path) + monkeypatch.setattr(studio_mod, "_PID_FILE", tmp_path / "studio.pid") + monkeypatch.setattr(studio_mod.time, "sleep", lambda _s: None) + + live = set(alive) + killed = killed if killed is not None else [] + + monkeypatch.setattr(studio_mod, "_pid_alive", lambda pid: pid in live) + monkeypatch.setattr(studio_mod, "_pid_is_studio_server", lambda pid, created_times = (): True) + + def fake_kill(pid, _sig): + killed.append(pid) + live.discard(pid) + + monkeypatch.setattr(studio_mod.os, "kill", fake_kill) + monkeypatch.setattr(sys, "platform", "linux") + return studio_mod, live, killed + + +def _write_pid(tmp_path, name, pid): + (tmp_path / name).write_text(str(pid), encoding = "utf-8") + + +def _run_stop(studio_mod): + import typer as _typer + + app = _typer.Typer() + app.add_typer(studio_mod.studio_app, name = "studio") + return CliRunner().invoke(app, ["studio", "stop"]) + + +def test_stop_kills_every_recorded_server(monkeypatch, tmp_path): + studio_mod, _live, killed = _install(monkeypatch, tmp_path, alive = {8550, 8600}) + _write_pid(tmp_path, "studio-8901-8550.pid", 8550) + _write_pid(tmp_path, "studio-8902-8600.pid", 8600) + + result = _run_stop(studio_mod) + + assert result.exit_code == 0, result.output + assert sorted(killed) == [8550, 8600] + assert not list(tmp_path.glob("studio-*.pid")) + + +def test_stop_does_not_leave_the_older_instance_running(monkeypatch, tmp_path): + # The reported symptom: stop claimed success while instance A kept serving. + studio_mod, live, _killed = _install(monkeypatch, tmp_path, alive = {8550, 8600}) + _write_pid(tmp_path, "studio-8901-8550.pid", 8550) + _write_pid(tmp_path, "studio-8902-8600.pid", 8600) + + result = _run_stop(studio_mod) + + assert result.exit_code == 0, result.output + assert live == set() + + +def test_stop_signals_each_server_once(monkeypatch, tmp_path): + # A server writes its per-port file AND studio.pid. It stays alive while it + # shuts down gracefully, so a second SIGTERM would hit the SIG_DFL the first + # one installs and hard-kill it mid-cleanup. + studio_mod = _studio() + monkeypatch.setattr(studio_mod, "STUDIO_HOME", tmp_path) + monkeypatch.setattr(studio_mod, "_PID_FILE", tmp_path / "studio.pid") + monkeypatch.setattr(studio_mod.time, "sleep", lambda _s: None) + monkeypatch.setattr(studio_mod, "_pid_alive", lambda pid: True) + monkeypatch.setattr(studio_mod, "_pid_is_studio_server", lambda pid, created_times = (): True) + killed = [] + monkeypatch.setattr(studio_mod.os, "kill", lambda pid, _sig: killed.append(pid)) + monkeypatch.setattr(sys, "platform", "linux") + _write_pid(tmp_path, "studio-8901-8550.pid", 8550) + _write_pid(tmp_path, "studio.pid", 8550) + + result = _run_stop(studio_mod) + + assert result.exit_code == 0, result.output + assert killed == [8550] + assert result.output.lower().count("sent shutdown signal") == 1 + + +def test_stop_removes_every_stale_file_for_one_pid(monkeypatch, tmp_path): + studio_mod, _live, killed = _install(monkeypatch, tmp_path, alive = set()) + _write_pid(tmp_path, "studio-8901-8550.pid", 8550) + _write_pid(tmp_path, "studio.pid", 8550) + + result = _run_stop(studio_mod) + + assert result.exit_code == 0, result.output + assert killed == [] + assert not list(tmp_path.glob("*.pid")) + + +def test_stop_does_not_signal_a_reused_pid(monkeypatch, tmp_path): + # Crash leaves a per-port file behind, the OS hands that PID to something + # else: stop must drop the record, not SIGTERM an unrelated process. + studio_mod, _live, killed = _install(monkeypatch, tmp_path, alive = {8550}) + monkeypatch.setattr(studio_mod, "_pid_is_studio_server", lambda pid, created_times = (): False) + _write_pid(tmp_path, "studio-8901-8550.pid", 8550) + + result = _run_stop(studio_mod) + + assert result.exit_code == 0, result.output + assert killed == [] + assert not (tmp_path / "studio-8901-8550.pid").exists() + + +def test_stop_signals_a_live_server_whose_pid_has_a_stale_record(monkeypatch, tmp_path): + # Crash leaves studio-8888-8550.pid, the OS reuses 8550 for a new server on + # another port. The stale timestamp must not veto the live one. + studio_mod, _live, killed = _install(monkeypatch, tmp_path, alive = {8550}) + monkeypatch.setattr(studio_mod, "_pid_is_studio_server", _REAL_IS_STUDIO_SERVER) + + class _FakeProcess: + def __init__(self, pid): + self.pid = pid + + def create_time(self): + return 999.0 + + monkeypatch.setitem(sys.modules, "psutil", SimpleNamespace(Process = _FakeProcess)) + (tmp_path / "studio-8888-8550.pid").write_text("8550\n111.5", encoding = "utf-8") + (tmp_path / "studio-9000-8550.pid").write_text("8550\n999.0", encoding = "utf-8") + + result = _run_stop(studio_mod) + + assert result.exit_code == 0, result.output + assert killed == [8550] + assert not list(tmp_path.glob("studio-*.pid")) + + +def test_a_bare_run_py_command_line_is_not_rejected(monkeypatch): + # `cd studio/backend && python run.py --port 8901` has no "studio" or "unsloth" + # in argv. Guessing from the command line deleted its record without stopping it. + studio_mod = _studio() + + class _FakeProcess: + def __init__(self, pid): + self.pid = pid + + def cmdline(self): + return ["python", "run.py", "--port", "8901"] + + def create_time(self): + return 111.5 + + monkeypatch.setitem(sys.modules, "psutil", SimpleNamespace(Process = _FakeProcess)) + + assert studio_mod._pid_is_studio_server(8550) is True + + +def test_an_untimed_record_is_trusted(monkeypatch): + # A legacy `python run.py --port 8901` has no telltale argv, and the in-venv + # path runs in-process. Guessing from the command line rejected real servers. + studio_mod = _studio() + + assert studio_mod._pid_is_studio_server(8550) is True + assert studio_mod._pid_is_studio_server(8550, [None]) is True + + +def test_an_unverifiable_record_is_still_stopped(monkeypatch): + # psutil is not a base CLI dependency, so the CLI meets timestamped records it + # cannot check. The old `stop` signalled with no checks at all -- skipping one + # would leave a live server running, the orphan bug this exists to fix. + studio_mod = _studio() + monkeypatch.setitem(sys.modules, "psutil", None) + + assert studio_mod._pid_is_studio_server(8550, [111.5]) is True + assert studio_mod._pid_is_studio_server(8550, [None]) is True + + +def test_stop_signals_a_timestamped_record_without_psutil(monkeypatch, tmp_path): + # Multiple servers on different ports: only the newest is also in studio.pid, + # so the earlier ones are timestamp-only and must still be stopped. + studio_mod, _live, killed = _install(monkeypatch, tmp_path, alive = {8550}) + monkeypatch.setattr(studio_mod, "_pid_is_studio_server", _REAL_IS_STUDIO_SERVER) + monkeypatch.setitem(sys.modules, "psutil", None) + (tmp_path / "studio-8901-8550.pid").write_text("8550\n111.5", encoding = "utf-8") + + result = _run_stop(studio_mod) + + assert result.exit_code == 0, result.output + assert killed == [8550] + assert not (tmp_path / "studio-8901-8550.pid").exists() + + +def test_the_untimed_legacy_record_does_not_cancel_a_timed_one(monkeypatch): + # Every current server writes BOTH a timed per-port record and an untimed + # studio.pid, so letting the untimed half win made this check inert exactly + # where it matters: after a crash and a PID reuse, `stop` SIGTERMed whatever + # unrelated process had inherited the PID. An untimed record carries no + # information, so it must not overrule a start time that says "not ours". + studio_mod = _studio() + + class _FakeProcess: + def __init__(self, pid): + self.pid = pid + + def create_time(self): + return 999.0 + + monkeypatch.setitem(sys.modules, "psutil", SimpleNamespace(Process = _FakeProcess)) + + assert studio_mod._pid_is_studio_server(8550, [111.5, None]) is False + assert studio_mod._pid_is_studio_server(8550, [111.5]) is False + # A matching time still wins over a stale sibling record. + assert studio_mod._pid_is_studio_server(8550, [111.5, 999.0]) is True + assert studio_mod._pid_is_studio_server(8550, [None, None]) is True + + +def test_stop_does_not_signal_a_reused_pid_recorded_in_both_files(monkeypatch, tmp_path): + # End to end for the case above: a crashed server left studio-8901-8550.pid + # and studio.pid, and 8550 now belongs to something else entirely. + studio_mod, _live, killed = _install(monkeypatch, tmp_path, alive = {8550}) + monkeypatch.setattr(studio_mod, "_pid_is_studio_server", _REAL_IS_STUDIO_SERVER) + + class _FakeProcess: + def __init__(self, pid): + self.pid = pid + + def create_time(self): + return 999.0 + + monkeypatch.setitem(sys.modules, "psutil", SimpleNamespace(Process = _FakeProcess)) + (tmp_path / "studio-8901-8550.pid").write_text("8550\n111.5\n127.0.0.1", encoding = "utf-8") + (tmp_path / "studio.pid").write_text("8550", encoding = "utf-8") + + result = _run_stop(studio_mod) + + assert result.exit_code == 0, result.output + assert killed == [] + assert not list(tmp_path.glob("*.pid")) + + +def test_pid_identity_check_trusts_the_record_without_psutil(monkeypatch): + # No psutil: fall back to trusting the record rather than never stopping. + studio_mod = _studio() + monkeypatch.setitem(sys.modules, "psutil", None) + + assert studio_mod._pid_is_studio_server(8550) is True + + +def test_pid_identity_check_uses_the_recorded_start_time(monkeypatch): + studio_mod = _studio() + + class _FakeProcess: + def __init__(self, pid): + self.pid = pid + + def create_time(self): + return 111.5 + + monkeypatch.setitem(sys.modules, "psutil", SimpleNamespace(Process = _FakeProcess)) + + assert studio_mod._pid_is_studio_server(8550, [111.5]) is True + assert studio_mod._pid_is_studio_server(8550, [999.0]) is False + + +def test_stop_drops_a_record_whose_start_time_no_longer_matches(monkeypatch, tmp_path): + # The PID was reused: same number, different process. + studio_mod, _live, killed = _install(monkeypatch, tmp_path, alive = {8550}) + monkeypatch.setattr(studio_mod, "_pid_is_studio_server", _REAL_IS_STUDIO_SERVER) + + class _FakeProcess: + def __init__(self, pid): + self.pid = pid + + def create_time(self): + return 999.0 + + monkeypatch.setitem(sys.modules, "psutil", SimpleNamespace(Process = _FakeProcess)) + (tmp_path / "studio-8901-8550.pid").write_text("8550\n111.5", encoding = "utf-8") + + result = _run_stop(studio_mod) + + assert result.exit_code == 0, result.output + assert killed == [] + assert not (tmp_path / "studio-8901-8550.pid").exists() + # Dropped for the start-time mismatch, not because the record looked corrupt. + assert "invalid pid file" not in result.output.lower() + + +def test_stop_reads_the_legacy_single_pid_file(monkeypatch, tmp_path): + studio_mod, _live, killed = _install(monkeypatch, tmp_path, alive = {4242}) + _write_pid(tmp_path, "studio.pid", 4242) + + result = _run_stop(studio_mod) + + assert result.exit_code == 0, result.output + assert killed == [4242] + assert not (tmp_path / "studio.pid").exists() + + +def test_stop_reports_nothing_running_without_pid_files(monkeypatch, tmp_path): + studio_mod, _live, _killed = _install(monkeypatch, tmp_path, alive = set()) + + result = _run_stop(studio_mod) + + assert result.exit_code == 0, result.output + assert "no running unsloth server" in result.output.lower() + + +def test_stop_cleans_stale_pid_files_without_claiming_a_stop(monkeypatch, tmp_path): + studio_mod, _live, killed = _install(monkeypatch, tmp_path, alive = set()) + _write_pid(tmp_path, "studio-8901-8550.pid", 8550) + + result = _run_stop(studio_mod) + + assert result.exit_code == 0, result.output + assert killed == [] + assert not (tmp_path / "studio-8901-8550.pid").exists() + assert "stopped" not in result.output.lower() + + +def test_stop_does_not_claim_a_stop_while_a_server_is_still_alive(monkeypatch, tmp_path): + # SIGTERM delivered but it never exits: don't claim a stop, keep the file. + studio_mod = _studio() + monkeypatch.setattr(studio_mod, "STUDIO_HOME", tmp_path) + monkeypatch.setattr(studio_mod, "_PID_FILE", tmp_path / "studio.pid") + monkeypatch.setattr(studio_mod.time, "sleep", lambda _s: None) + monkeypatch.setattr(studio_mod, "_pid_alive", lambda pid: True) + monkeypatch.setattr(studio_mod, "_pid_is_studio_server", lambda pid, created_times = (): True) + monkeypatch.setattr(studio_mod.os, "kill", lambda pid, sig: None) + monkeypatch.setattr(sys, "platform", "linux") + _write_pid(tmp_path, "studio-8901-8550.pid", 8550) + + result = _run_stop(studio_mod) + + assert result.exit_code == 0, result.output + assert "shutting down" in result.output.lower() + assert "stopped" not in result.output.lower() + assert (tmp_path / "studio-8901-8550.pid").exists() + + +def test_stop_continues_after_one_server_fails_to_stop(monkeypatch, tmp_path): + studio_mod = _studio() + monkeypatch.setattr(studio_mod, "STUDIO_HOME", tmp_path) + monkeypatch.setattr(studio_mod, "_PID_FILE", tmp_path / "studio.pid") + monkeypatch.setattr(studio_mod.time, "sleep", lambda _s: None) + live = {8550, 8600} + monkeypatch.setattr(studio_mod, "_pid_alive", lambda pid: pid in live) + monkeypatch.setattr(studio_mod, "_pid_is_studio_server", lambda pid, created_times = (): True) + + def fake_kill(pid, _sig): + if pid == 8550: + raise PermissionError("not permitted") + live.discard(pid) + + monkeypatch.setattr(studio_mod.os, "kill", fake_kill) + monkeypatch.setattr(sys, "platform", "linux") + _write_pid(tmp_path, "studio-8901-8550.pid", 8550) + _write_pid(tmp_path, "studio-8902-8600.pid", 8600) + + result = _run_stop(studio_mod) + + combined = (result.output or "") + (getattr(result, "stderr", "") or "") + assert result.exit_code == 1, combined + assert 8600 not in live + assert "8550" in combined + + +def test_stop_never_signals_pid_zero_or_init(monkeypatch, tmp_path): + # os.kill(0, SIGTERM) hits our whole process group -- the shell and its jobs. + studio_mod, _live, killed = _install(monkeypatch, tmp_path, alive = {0, 1}) + _write_pid(tmp_path, "studio-8901-0.pid", 0) + _write_pid(tmp_path, "studio-8902-1.pid", 1) + + result = _run_stop(studio_mod) + + assert result.exit_code == 0, result.output + assert killed == [] + assert not list(tmp_path.glob("*.pid")) + + +def test_signal_stop_refuses_pid_zero_or_init(monkeypatch, tmp_path): + studio_mod, _live, killed = _install(monkeypatch, tmp_path, alive = {0, 1}) + + assert studio_mod._signal_stop(0) is not None + assert studio_mod._signal_stop(1) is not None + assert killed == [] + + +def test_stop_discards_a_corrupt_pid_file(monkeypatch, tmp_path): + studio_mod, _live, _killed = _install(monkeypatch, tmp_path, alive = set()) + (tmp_path / "studio-8901-8550.pid").write_text("not-a-pid", encoding = "utf-8") + + result = _run_stop(studio_mod) + + assert result.exit_code == 0, result.output + assert not (tmp_path / "studio-8901-8550.pid").exists() + + +def test_stop_keeps_a_record_it_cannot_read(monkeypatch, tmp_path): + # A root-owned record, or one caught mid-write, still belongs to a live + # server. Deleting it is `stop` manufacturing the orphan it exists to fix. + studio_mod, _live, killed = _install(monkeypatch, tmp_path, alive = {8550}) + path = tmp_path / "studio-8901-8550.pid" + path.write_text("8550", encoding = "utf-8") + real_read_text = Path.read_text + + def deny(self, *args, **kwargs): + if self == path: + raise PermissionError(13, "Permission denied") + return real_read_text(self, *args, **kwargs) + + monkeypatch.setattr(Path, "read_text", deny) + + result = _run_stop(studio_mod) + + assert path.exists(), "an unreadable record must not be deleted" + assert "cannot read" in (result.output + (result.stderr or "")).lower() + + +def test_stop_does_not_claim_success_when_the_only_record_is_unreadable(monkeypatch, tmp_path): + # A server started under sudo leaves a record we cannot read. Printing "no + # running server" and exiting 0 tells the user the opposite of the truth. + studio_mod, _live, killed = _install(monkeypatch, tmp_path, alive = {8550}) + path = tmp_path / "studio-8901-8550.pid" + path.write_text("8550", encoding = "utf-8") + real_read_text = Path.read_text + + def deny(self, *args, **kwargs): + if self == path: + raise PermissionError(13, "Permission denied") + return real_read_text(self, *args, **kwargs) + + monkeypatch.setattr(Path, "read_text", deny) + + result = _run_stop(studio_mod) + + assert result.exit_code == 1, "an unreachable server is not a successful stop" + output = result.output + (result.stderr or "") + assert "no running unsloth server" not in output.lower() + assert killed == [] + + +def test_stop_reports_failure_when_one_record_is_unreadable_but_another_stops( + monkeypatch, tmp_path +): + # Stopping the servers we can see is still a partial result, and exiting 0 + # would hide the one we could not. + studio_mod, _live, killed = _install(monkeypatch, tmp_path, alive = {8550, 8600}) + _write_pid(tmp_path, "studio-8901-8550.pid", 8550) + hidden = tmp_path / "studio-8902-8600.pid" + hidden.write_text("8600", encoding = "utf-8") + real_read_text = Path.read_text + + def deny(self, *args, **kwargs): + if self == hidden: + raise PermissionError(13, "Permission denied") + return real_read_text(self, *args, **kwargs) + + monkeypatch.setattr(Path, "read_text", deny) + + result = _run_stop(studio_mod) + + assert killed == [8550], "the readable server must still be stopped" + assert result.exit_code == 1 + assert hidden.exists() + + +def test_stop_reaches_every_server_when_one_record_cannot_be_removed(monkeypatch, tmp_path): + # One undeletable stale record must not end the loop before the live servers. + studio_mod, _live, killed = _install(monkeypatch, tmp_path, alive = {8600}) + _write_pid(tmp_path, "studio-8901-8550.pid", 8550) # dead -> stop prunes it + _write_pid(tmp_path, "studio-8902-8600.pid", 8600) # live -> stop signals it + real_unlink = Path.unlink + + def deny(self, *args, **kwargs): + if self.name == "studio-8901-8550.pid": + raise PermissionError(13, "Permission denied") + return real_unlink(self, *args, **kwargs) + + monkeypatch.setattr(Path, "unlink", deny) + + result = _run_stop(studio_mod) + + assert killed == [8600], "the live server must still be signalled" + assert result.exit_code == 0, result.output + + +def test_a_record_whose_pid_is_not_ascii_digits_is_discarded(monkeypatch, tmp_path): + # A superscript two passes isdigit() but int() rejects it, so that gate alone + # let a ValueError escape _read_pid_record and abort the whole command. + studio_mod, _live, _killed = _install(monkeypatch, tmp_path, alive = set()) + (tmp_path / "studio-8901-1.pid").write_text("²", encoding = "utf-8") + + result = _run_stop(studio_mod) + + assert result.exit_code == 0, result.output + assert not (tmp_path / "studio-8901-1.pid").exists()