diff --git a/.github/workflows/release-desktop.yml b/.github/workflows/release-desktop.yml index 081eda4e32..0a8d71610d 100644 --- a/.github/workflows/release-desktop.yml +++ b/.github/workflows/release-desktop.yml @@ -766,6 +766,7 @@ jobs: env: GH_REPO: ${{ github.repository }} APP_VERSION: ${{ needs.prepare-version.outputs.app_version }} + PYPI_VERSION: ${{ needs.prepare-version.outputs.pypi_version }} STUDIO_VERSION: ${{ needs.prepare-version.outputs.studio_version }} DESKTOP_RELEASE_TAG: ${{ needs.prepare-version.outputs.desktop_release_tag }} DESKTOP_PRERELEASE: ${{ needs.prepare-version.outputs.prerelease }} @@ -911,6 +912,8 @@ jobs: notes = pathlib.Path(os.environ['RUNNER_TEMP'], 'desktop-release-notes.md').read_text() metadata = { 'version': os.environ['APP_VERSION'], + # App version is SemVer; CHANGELOG.md is keyed by the backend release. + 'pypi_version': os.environ['PYPI_VERSION'], 'notes': notes, 'pub_date': datetime.datetime.now(datetime.timezone.utc).isoformat(timespec='milliseconds').replace('+00:00', 'Z'), 'platforms': { diff --git a/.github/workflows/startup-profile-ci.yml b/.github/workflows/startup-profile-ci.yml new file mode 100644 index 0000000000..fbde99836d --- /dev/null +++ b/.github/workflows/startup-profile-ci.yml @@ -0,0 +1,156 @@ +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. + +# Measures where Studio's startup time goes, on each platform. +# +# Nothing recorded a number before: main.py logs "lifespan startup completed in X ms" +# and studio_test_kit polls /healthz, but both throw the elapsed time away. A first +# local run (Linux, warm cache, 18-core server) put `import main` at 5.7-6.6s BEFORE +# the server can bind, dominated by eager module-level imports pulled in by routes: +# torch ~1.9s self, unsloth_zoo ~0.8s, routes ~0.6s, transformers ~0.5s. +# +# Not a gate yet: --max-healthz-seconds exists, but a budget should come from +# observed numbers rather than a guess. + +name: Startup profile + +on: + pull_request: + paths: + # The measured import graph is the whole backend tree: main.py imports auth, + # core, hub, loggers, models, picker, routes and utils at module scope. + - 'studio/backend/**' + - '!studio/backend/tests/**' + # The launch phase spawns `unsloth studio --api-only`, so the CLI counts too. + - 'unsloth_cli/**' + - 'studio/src-tauri/src/preflight**' + # The profiler hardcodes the desktop argv that process.rs::backend_args builds, + # so a change there must schedule a run or the two silently diverge. + - 'studio/src-tauri/src/process.rs' + - 'scripts/profile_startup.py' + - '.github/workflows/startup-profile-ci.yml' + # The job profiles whatever `install.sh --local` built: the installers pick the + # venv's Python and the dependency specs, and pyproject's include list is what + # makes --local overlay studio.backend*. + - 'install.sh' + - 'install.ps1' + - 'pyproject.toml' + # --local also runs the checkout's setup scripts (install.sh picks + # $_REPO_ROOT/studio/setup.sh, the editable install resolves setup.ps1 to the + # repo), and both call install_python_stack.py, which picks the dependencies. + - 'studio/setup.sh' + - 'studio/setup.ps1' + - 'studio/install_python_stack.py' + workflow_dispatch: + inputs: + repeats: + description: 'launch repeats per OS (median reported)' + type: string + default: '3' + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +permissions: + contents: read + +jobs: + profile: + name: startup ${{ matrix.os }} + runs-on: ${{ matrix.os }} + timeout-minutes: 60 + continue-on-error: true + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, macos-14, windows-latest] + + env: + UNSLOTH_STUDIO_HOME: ${{ github.workspace }}/.studio-home + # A wildcard bind calls ifconfig.me on the startup path; loopback times our code. + UNSLOTH_STUDIO_DISABLE_PUBLIC_CHECK: '1' + + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + + - name: Install Studio + shell: bash + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + set -o pipefail + mkdir -p logs + # --local is load-bearing: it overlays the checkout, so the profiled server + # is this diff. Without it install.sh resolves unsloth from PyPI. + if [ "${{ runner.os }}" = "Windows" ]; then + pwsh -NoProfile -File ./install.ps1 --local 2>&1 | tee logs/install.log + else + bash install.sh --local 2>&1 | tee logs/install.log + fi + + - name: Profile startup + shell: bash + run: | + BIN="$UNSLOTH_STUDIO_HOME/unsloth_studio/bin/unsloth" + [ -x "$BIN" ] || BIN="$UNSLOTH_STUDIO_HOME/unsloth_studio/Scripts/unsloth.exe" + [ -x "$BIN" ] || BIN="" + # Profile imports with the INSTALLED interpreter: that venv is what launches. + PY="$UNSLOTH_STUDIO_HOME/unsloth_studio/bin/python" + [ -x "$PY" ] || PY="$UNSLOTH_STUDIO_HOME/unsloth_studio/Scripts/python.exe" + [ -x "$PY" ] || PY="$(command -v python3 || command -v python)" + python3 scripts/profile_startup.py \ + --python "$PY" \ + ${BIN:+--bin "$BIN"} \ + --repeats "${{ inputs.repeats || '3' }}" \ + --json "startup-${{ matrix.os }}.json" 2>&1 | tee logs/profile.log + + - name: Summary + if: always() + shell: bash + run: | + f="startup-${{ matrix.os }}.json" + [ -f "$f" ] || { echo "no profile produced"; exit 0; } + python3 - "$f" >> "$GITHUB_STEP_SUMMARY" <<'PY' + import json, sys + d = json.load(open(sys.argv[1])) + print(f"### {d['platform']} / {d['machine']} (py {d['python']}, {d['cpu_count']} cpu)\n") + imp = d.get("imports", {}) + # Gate on ok: a failed `import main` still leaves rows, so a total can lie. + if imp.get("ok"): + print(f"**`import main`: {imp['total_seconds']}s**\n") + print("| package | self ms |") + print("|---|---:|") + for k, v in list(imp.get("self_by_package_ms", {}).items())[:8]: + print(f"| {k} | {v} |") + print() + else: + print("**`import main` failed - no valid import profile**\n") + print("```\n" + (imp.get("error") or "")[-1500:] + "\n```\n") + lau = d.get("launch") or {} + runs = len(lau.get("runs") or []) + failed = lau.get("failed_runs") or 0 + if lau.get("healthz_median_seconds") is not None: + # The aggregates cover only the runs that reached healthz, so flag the + # failures: bare numbers would read as a normal fast startup. + note = f" _({runs - failed} of {runs} launches; {failed} never became healthy)_" if failed else "" + print(f"**time to a healthy port: {lau['healthz_median_seconds']}s median, " + f"{lau['healthz_max_seconds']}s max**{note}\n") + elif lau.get("skipped"): + print(f"_launch phase skipped: {lau['skipped']}_\n") + elif runs: + print(f"**no launch measurement: all {runs} launches failed to become healthy**\n") + PY + + - name: Upload profile + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: startup-profile-${{ matrix.os }} + path: | + startup-*.json + logs/ + retention-days: 14 + if-no-files-found: warn diff --git a/.gitignore b/.gitignore index fafd17aa95..fa6997cb06 100644 --- a/.gitignore +++ b/.gitignore @@ -208,6 +208,9 @@ tmp/ **/node_modules/ auth.db +# Packaging snapshot of the root CHANGELOG.md (written by build.sh) +studio/CHANGELOG.md + # Tauri local build/generated output studio/src-tauri/target/ studio/src-tauri/gen/ diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000000..241e013cea --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,88 @@ +# Changelog + +Release notes for Unsloth and Unsloth Studio. + +Unsloth Studio reads this file to show release notes inside the "New Unsloth +version" update popup. Edit it here and the popup picks the change up on the +next update check, with no release or rebuild required. + +## Format + +Every release is a level-2 heading whose first token is the version, optionally +followed by a date: + +```md +## 2026.7.6 - 2026-07-22 +``` + +`## [2026.7.6] - 2026-07-22` and `## v2026.7.6` also work. Everything under a +heading, up to the next level-2 heading, is that release's notes and renders as +Markdown in the popup. + +Notes are matched to one exact version. When Studio offers an update to +`2026.7.6` it renders the `2026.7.6` section and nothing else. If that section +is missing, the popup links out to the online changelog rather than showing +notes from an unrelated release, so a new version needs its own section here +before its notes can appear. + +Keep the newest release at the top. Lead each bullet with the change itself: +the collapsed popup highlights the first sentence and dims the rest. +`## Unreleased` is ignored by the popup, so it is safe to stage notes there and +rename the heading at release time. + + + +## Unreleased + +## 2026.7.5 + +### What's Changed + +- AMD support is here. Train, run RL, chat with and deploy 500+ models on + Radeon, Instinct, Ryzen and data center GPUs across Windows, WSL and Linux, + up to 2x faster with 70% less VRAM and no accuracy loss. +- Intel XPU support lands in Studio, so Arc and Data Center GPUs run chat and + training alongside the NVIDIA, AMD and Apple paths. +- Local speech to text dictation runs fully offline, with slim Whisper bundles + and a picker for custom models. +- DoRA training is available in Studio, selectable next to LoRA and full + fine-tuning in the training tab. +- The update popup previews release notes inline, pulled from this file and + matched to the exact version being offered. + +### AMD, 23 July update + +Our AMD collaboration, custom Triton kernels and math algorithms bring local +training and inference to AMD hardware. The 23 July update builds on the +[AMD release](https://github.com/unslothai/unsloth/releases/tag/v0.1.501-beta): + +- RDNA2 and Gorgon Halo are supported, and the installer no longer fails to + detect GPUs on Strix Halo and other AMD cards. +- RDNA4 handling is better, and HIP and ROCm failures are caught and fixed + automatically instead of stopping the install. +- Unified memory safetensors loading is 2x faster, with much faster gradient + checkpointing on unified memory devices. +- Voice dictation through whisper.cpp has preliminary support. +- Rollback environments left by installs no longer eat 5GB of disk. They are + cleaned up automatically. + +Optimized ROCm builds cover GGUF and safetensors inference, and ROCm +compatibility is improved for MI300X and MI325X. Full guide: +[unsloth.ai/docs/basics/amd](https://unsloth.ai/docs/basics/amd). + +### Running larger models + +- Automatic GPU placement, or pick exactly which GPUs and layers to use. +- Move MoE expert layers into system memory so larger models fit. +- Split a model across several GPUs, or use tensor parallelism. +- Hardware settings are saved per model and quant. + +### Also in this release + +- Remote access with `unsloth studio --secure` over free HTTPS via Cloudflare. +- Web search reads PDF papers and manuals, and parallel tool calls, reasoning + output and tool retries are more reliable. +- The model download location is configurable, so weights can live on a second + drive instead of the default cache. +- Stalled Hugging Face XET downloads retry over standard HTTP, and existing + GGUF files are reused instead of downloaded again. diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 0000000000..7bce036343 --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1,2 @@ +include _changelog_build.py +include CHANGELOG.md diff --git a/_changelog_build.py b/_changelog_build.py new file mode 100644 index 0000000000..f5bcf2052c --- /dev/null +++ b/_changelog_build.py @@ -0,0 +1,36 @@ +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. + +"""Snapshot CHANGELOG.md into the studio package at build time. + +CHANGELOG.md at the repo root stays the one file to edit. Copying it here, +rather than in build.sh, means every packaging path ships it, so release notes +still render when the popup cannot reach GitHub.""" + +from __future__ import annotations + +import shutil +from pathlib import Path + +from setuptools.command.build_py import build_py as _build_py + +ROOT = Path(__file__).resolve().parent +SOURCE = ROOT / "CHANGELOG.md" +SNAPSHOT = ROOT / "studio" / "CHANGELOG.md" + + +class build_py(_build_py): + def run(self) -> None: + # Beside the sources only if writable (PEP 517 may build an immutable + # checkout); into the staging directory always. + if SOURCE.is_file(): + try: + shutil.copyfile(SOURCE, SNAPSHOT) + except OSError: + pass + super().run() + if not SOURCE.is_file(): + return + staged = Path(self.build_lib) / "studio" / "CHANGELOG.md" + staged.parent.mkdir(parents = True, exist_ok = True) + shutil.copyfile(SOURCE, staged) diff --git a/build.sh b/build.sh index 2a836e19d9..5b09a7791b 100644 --- a/build.sh +++ b/build.sh @@ -103,9 +103,13 @@ else STUDIO_STAMPED_VERSION="$(python scripts/stamp_studio_release.py)" fi -# 4. Build wheel/sdist +# 4. Build wheel/sdist. _changelog_build.py snapshots CHANGELOG.md into the studio +# package so release notes render offline. python -m build +# Drop the snapshot so a source checkout never serves a stale copy. +rm -f studio/CHANGELOG.md + if [ "${1:-}" = "publish" ]; then python scripts/stamp_studio_release.py --verify-dist dist --expected "$STUDIO_STAMPED_VERSION" fi diff --git a/install.ps1 b/install.ps1 index 8474520970..a26de28a19 100644 --- a/install.ps1 +++ b/install.ps1 @@ -57,6 +57,26 @@ function Install-UnslothStudio { } } + # Machine arch; Get-TauriDiagArch above reports the process. An emulated x64 shell on + # ARM64 reports AMD64, but PROCESSOR_ARCHITEW6432 is ARM64 in exactly that case. + function Get-HostMachineArch { + $osArch = "" + try { $osArch = [System.Runtime.InteropServices.RuntimeInformation]::OSArchitecture.ToString() } catch { $osArch = "" } + $signals = @([string]$env:PROCESSOR_ARCHITEW6432, [string]$env:PROCESSOR_ARCHITECTURE, $osArch) + foreach ($s in $signals) { + if ($s.ToLowerInvariant() -eq "arm64") { return "arm64" } + } + foreach ($s in $signals) { + if ([string]::IsNullOrWhiteSpace($s)) { continue } + switch ($s.ToLowerInvariant()) { + "amd64" { return "x86_64" } + "x64" { return "x86_64" } + "x86" { return "x86" } + } + } + return "unknown" + } + function Get-TauriTorchIndexFamily { param([string]$TorchIndexUrl) if ($SkipTorch) { return "none" } @@ -1124,10 +1144,27 @@ exit 0 return $false } + # The interpreter's own arch, asked of it: win-amd64|win-arm64|win32|"". + function Get-PythonPlatformTag { + param([string]$Exe) + try { + return (& $Exe -c "import sysconfig; print(sysconfig.get_platform())" 2>$null | Out-String).Trim().ToLowerInvariant() + } catch { return "" } + } + # Returns @{ Version = "3.13"; Path = "C:\...\python.exe" } or $null. # The resolved Path is passed to `uv venv --python` to prevent uv from # re-resolving the version string back to a conda interpreter. function Find-CompatiblePython { + # -X64Only: best installed x64 interpreter or $null, never ARM64. Last resort for + # Install-X64Python, where x64 of a lower-priority minor beats ARM64. + param([switch]$X64Only) + # Windows on ARM: prefer x64. pyarrow (via datasets) and hf-transfer ship no + # win_arm64 wheel, so a native ARM64 Python source-builds both and dies on CMake / + # Rust minutes in; x64 runs fine emulated. ARM64 is still returned when it is all + # there is, and the caller then bootstraps x64 or warns. + $preferX64 = $X64Only -or ((Get-HostMachineArch) -eq "arm64") + $candidates = @() # Try the Python Launcher first (most reliable on Windows) # py.exe resolves to the standard CPython install, not conda. # Prefer the requested $PythonVersion, then newest-first fallback. @@ -1145,7 +1182,8 @@ exit 0 # Resolve the actual executable path and verify it is not conda-based $resolvedExe = (& $pyLauncher.Source "-$minor" -c "import sys; print(sys.executable)" 2>$null | Out-String).Trim() if ($resolvedExe -and (Test-Path $resolvedExe) -and -not (Test-IsCondaPython $resolvedExe)) { - return @{ Version = $ver; Path = $resolvedExe } + if (-not $preferX64) { return @{ Version = $ver; Path = $resolvedExe; Arch = "" } } + $candidates += @{ Version = $ver; Path = $resolvedExe } } } } catch {} @@ -1166,11 +1204,53 @@ exit 0 try { $out = & $cmd.Source --version 2>&1 | Out-String if ($out -match "Python (3\.1[1-3])\.\d+") { - return @{ Version = $Matches[1]; Path = $cmd.Source } + if (-not $preferX64) { return @{ Version = $Matches[1]; Path = $cmd.Source; Arch = "" } } + $candidates += @{ Version = $Matches[1]; Path = $cmd.Source } } } catch {} } } + # `py -3.12` runs the launcher's preferred build, normally the native ARM64 one, so + # a same-minor x64 install that is neither preferred nor on PATH never becomes a + # candidate. `-3.12-64` cannot disambiguate (deprecated, it only means "not + # 32-bit"), so enumerate every registration with -0p and probe each path. + if ($preferX64) { + foreach ($pyLauncher in @(Get-Command py -All -CommandType Application -ErrorAction SilentlyContinue)) { + if ($pyLauncher.Source -match $script:CondaSkipPattern) { continue } + $listed = @() + try { $listed = @(& $pyLauncher.Source "-0p" 2>$null) } catch {} + foreach ($line in $listed) { + # " -V:3.12 * C:\...\python.exe": tag, optional default marker, path. + $m = [regex]::Match([string]$line, '(?i)^\s*-\S+\s+\*?\s*"?(?

\S.*?\.exe)"?\s*$') + if (-not $m.Success) { continue } + $exe = $m.Groups['p'].Value.Trim() + if ($candidates | Where-Object { $_.Path -eq $exe }) { continue } + if (-not (Test-Path -LiteralPath $exe)) { continue } + if (Test-IsCondaPython $exe) { continue } + try { + $out = & $exe --version 2>&1 | Out-String + if ($out -match "Python (3\.1[1-3])\.\d+") { + $candidates += @{ Version = $Matches[1]; Path = $exe } + } + } catch {} + } + } + } + # Prefer x64, but only within one minor: $minors is the caller's version preference, + # so ranking on arch alone would answer UNSLOTH_PYTHON=3.12 with an x64 3.13 and + # never bootstrap x64 3.12. Probing costs a subprocess, so non-ARM returned above. + foreach ($c in $candidates) { + $tag = Get-PythonPlatformTag $c.Path + $c.Arch = if ($tag -eq "win-amd64") { "x86_64" } elseif ($tag -eq "win-arm64") { "arm64" } else { "unknown" } + } + foreach ($minor in $minors) { + $sameMinor = @($candidates | Where-Object { $_.Version -eq $minor }) + if ($sameMinor.Count -eq 0) { continue } + $x64 = $sameMinor | Where-Object { $_.Arch -eq "x86_64" } | Select-Object -First 1 + if ($x64) { return $x64 } + if (-not $X64Only) { return $sameMinor[0] } + } + if (-not $X64Only -and $candidates.Count -gt 0) { return $candidates[0] } return $null } @@ -1181,8 +1261,11 @@ exit 0 # (no UAC), putting python.exe + the py launcher on PATH. Mirrors the uv -> # astral.sh fallback below. Returns @{ Version; Path } or $null. function Install-PythonFromPythonOrg { + # $Arch overrides the host arch, to pull x64 onto an ARM64 box. + param([string]$Arch = "") # python.org ships one installer per architecture. - $archSuffix = switch (Get-TauriDiagArch) { + $targetArch = if ($Arch) { $Arch } else { Get-TauriDiagArch } + $archSuffix = switch ($targetArch) { "x86_64" { "-amd64" } "arm64" { "-arm64" } "x86" { "" } @@ -1247,6 +1330,28 @@ exit 0 return (Find-CompatiblePython) } + # ── Windows on ARM: get an x64 CPython ── + # --architecture x64 forces winget off the ARM64 build; python.org takes the same override. + function Install-X64Python { + if ($script:WingetAvailable) { + $prevEAP = $ErrorActionPreference + $ErrorActionPreference = "Continue" + try { + winget install -e --id "Python.Python.$PythonVersion" --source winget --architecture x64 --accept-package-agreements --accept-source-agreements + } catch { } + $ErrorActionPreference = $prevEAP + Refresh-SessionPath + $found = Find-CompatiblePython + if ($found -and $found.Arch -eq "x86_64") { return $found } + substep "winget could not provide an x64 Python -- trying python.org..." "Yellow" + } + $found = Install-PythonFromPythonOrg -Arch "x86_64" + if ($found -and $found.Arch -eq "x86_64") { return $found } + # Nothing installable (offline / no winget): an x64 build of another supported minor + # still runs the wheels ARM64 cannot, so take it over the native interpreter. + return (Find-CompatiblePython -X64Only) + } + # ── Install Python if no compatible version (3.11-3.13) found ── # Find-CompatiblePython returns @{ Version = "3.13"; Path = "C:\...\python.exe" } or $null. Write-TauriLog "STEP" "Installing Python" @@ -1318,6 +1423,26 @@ exit 0 return (Exit-InstallFailure "Python installation failed") } } + # ── Windows on ARM: swap a native ARM64 interpreter for x64 ── + # pyarrow and hf-transfer publish no win_arm64 wheel, so an ARM64 Python source-builds + # both and fails deep into the run. Warn up front if x64 is unobtainable. + if ($DetectedPython -and (Get-HostMachineArch) -eq "arm64" -and $DetectedPython.Arch -ne "x86_64") { + substep "windows on arm: only a native ARM64 Python $($DetectedPython.Version) was found." "Yellow" + substep "pyarrow and hf-transfer publish no win_arm64 wheels, so installing x64 Python..." "Yellow" + $X64Python = Install-X64Python + if ($X64Python) { + $DetectedPython = $X64Python + step "python" "using x64 Python $($DetectedPython.Version) under emulation" + } else { + Write-Host "[WARN] Could not install an x64 Python on this ARM64 machine." -ForegroundColor Yellow + Write-Host " Continuing with ARM64 Python $($DetectedPython.Version), but the install is likely to fail:" -ForegroundColor Yellow + Write-Host " pyarrow (via datasets) and hf-transfer ship no win_arm64 wheels and will be" -ForegroundColor Yellow + Write-Host " built from source, which needs CMake plus the MSVC and Rust toolchains." -ForegroundColor Yellow + Write-Host " Fix: install x64 Python from https://www.python.org/downloads/windows/" -ForegroundColor Yellow + Write-Host " (choose 'Windows installer (64-bit)', not ARM64), then re-run this installer." -ForegroundColor Yellow + } + } + $DiagPythonVersion = $PythonVersion if ($DetectedPython) { $DiagPythonVersion = $DetectedPython.Version } $InitialGpuBranch = "unknown" @@ -2438,6 +2563,13 @@ exit 0 } } else { Write-TauriLog "STEP" "Installing PyTorch" + # Windows on ARM lacks only torchaudio (whl/cpu win_arm64: torch 42, + # torchvision 60, torchaudio 0), so drop that pin instead of aborting. Ask the + # interpreter, not PROCESSOR_ARCHITECTURE; reached when no x64 Python exists. + $VenvPlatform = "" + try { + $VenvPlatform = (& $VenvPython -c "import sysconfig; print(sysconfig.get_platform())" 2>$null | Out-String).Trim().ToLowerInvariant() + } catch { $VenvPlatform = "" } substep "installing PyTorch ($(Remove-IndexUrlCredentials $TorchIndexUrl))..." # Bound the companions to the capped torch on EVERY index, cu # families included: torchaudio 2.11 dropped its exact torch pin from @@ -2445,7 +2577,13 @@ exit 0 # resolve a mismatched 2.11.0 build. Mirrors install.sh. $_pinVisionSpec = "torchvision>=0.19,<0.26.0" $_pinAudioSpec = "torchaudio>=2.4,<2.11.0" - $torchInstallExit = Invoke-InstallCommandRetry -Label "install PyTorch" { uv pip install --python $VenvPython "torch>=2.4,<2.11.0" $_pinVisionSpec $_pinAudioSpec --default-index $TorchIndexUrl } + $_torchSpecs = @("torch>=2.4,<2.11.0", $_pinVisionSpec, $_pinAudioSpec) + if ($VenvPlatform -eq "win-arm64") { + substep "windows on arm: skipping torchaudio (upstream publishes no" + substep "win_arm64 wheel); torch and torchvision install normally." + $_torchSpecs = @("torch>=2.4,<2.11.0", $_pinVisionSpec) + } + $torchInstallExit = Invoke-InstallCommandRetry -Label "install PyTorch" { uv pip install --python $VenvPython @_torchSpecs --default-index $TorchIndexUrl } if ($torchInstallExit -ne 0) { Write-Host "[ERROR] Failed to install PyTorch (exit code $torchInstallExit)" -ForegroundColor Red return (Exit-InstallFailure "Failed to install PyTorch (exit code $torchInstallExit)" $torchInstallExit) diff --git a/pyproject.toml b/pyproject.toml index ce19d21399..8895bf0686 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -47,9 +47,14 @@ version = {attr = "unsloth.models._utils.__version__"} [tool.setuptools] include-package-data = true +[tool.setuptools.cmdclass] +# Snapshots CHANGELOG.md into studio/ so every build path ships it. +build_py = "_changelog_build.build_py" + [tool.setuptools.package-data] unsloth_cli = ["codex_fallback_prompt.md", "pi_subagent.ts"] studio = [ + "CHANGELOG.md", "*.sh", "*.ps1", "*.bat", diff --git a/scripts/profile_startup.py b/scripts/profile_startup.py new file mode 100644 index 0000000000..937d007ac1 --- /dev/null +++ b/scripts/profile_startup.py @@ -0,0 +1,377 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 + +"""Measure where Unsloth Studio's startup time goes, per platform. + +Nothing measured this before: the backend logs "lifespan startup completed in X ms" +but no test or CI job asserted a budget, and studio_test_kit discards the elapsed +time of its /healthz poll. A first local run (Linux, warm cache, fast server CPU) +found `import main` alone costs 6.6s before the server can bind, dominated by eager +module-level imports pulled in by the `routes` package: + + torch 1930 ms self + unsloth_zoo 914 ms self + routes 779 ms self + transformers 524 ms self + +Phases measured: + import `python -X importtime -c "import main"`, top cumulative + per-package self + spawn process start -> first byte on stdout + healthz process start -> /api/health (or /healthz) answers 200 + lifespan the backend's own "lifespan startup completed in X ms" log line + +Usage: + python scripts/profile_startup.py --repeats 3 --json out.json + python scripts/profile_startup.py --import-only # no server, no port needed + +Exit code is 0 unless --max-healthz-seconds is given and exceeded. +""" + +from __future__ import annotations + +import argparse +import json +import math +import os +import platform +import re +import shutil +import socket +import statistics +import subprocess +import sys +import threading +import time +import urllib.error +import urllib.request +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parents[1] +BACKEND = REPO_ROOT / "studio" / "backend" + +_IMPORTTIME_RE = re.compile(r"import time:\s+(\d+)\s+\|\s+(\d+)\s+\|(\s*)(\S.*)") + + +def _free_port() -> int: + with socket.socket() as s: + s.bind(("127.0.0.1", 0)) + return int(s.getsockname()[1]) + + +def profile_imports(python: str, top: int = 15) -> dict: + """Cumulative and self import cost for the backend's module graph. + + Run in a subprocess with -X importtime: the numbers are only meaningful for a + cold interpreter, and importing in-process would measure a warm sys.modules. + """ + proc = subprocess.run( + [python, "-X", "importtime", "-c", "import sys; sys.path.insert(0, '.'); import main"], + cwd = BACKEND, + capture_output = True, + text = True, + timeout = 900, + ) + rows = [] + for line in proc.stderr.splitlines(): + m = _IMPORTTIME_RE.match(line) + if m: + rows.append((int(m.group(1)), int(m.group(2)), m.group(4).strip())) + if not rows: + return {"ok": False, "error": (proc.stderr or proc.stdout)[-2000:]} + if proc.returncode != 0: + # Rows survive up to the failure, so any total from a partial graph is wrong. + return { + "ok": False, + "error": (proc.stderr or proc.stdout)[-2000:], + "partial_rows": len(rows), + } + + by_cum = sorted(rows, key = lambda r: -r[1]) + # Total comes from the `main` row, not by_cum[0]: -X importtime also prints the + # interpreter's own startup graph (`site`), which can outrank a trivial main. + main_row = next((r for r in reversed(rows) if r[2] == "main"), None) + if main_row is None: + return { + "ok": False, + "error": "no `import main` row in -X importtime output\n" + + (proc.stderr or proc.stdout)[-2000:], + } + self_by_pkg: dict[str, int] = {} + for self_us, _cum, name in rows: + pkg = name.split(".")[0] + self_by_pkg[pkg] = self_by_pkg.get(pkg, 0) + self_us + + return { + "ok": True, + "total_seconds": round(main_row[1] / 1e6, 3), + "top_cumulative": [ + {"module": n, "seconds": round(c / 1e6, 3)} for _s, c, n in by_cum[:top] + ], + "self_by_package_ms": { + k: round(v / 1000) for k, v in sorted(self_by_pkg.items(), key = lambda x: -x[1])[:top] + }, + } + + +def _terminate_tree(proc: subprocess.Popen) -> None: + """Stop the server AND its children, which on Windows are a separate process. + + CI profiles `Scripts/unsloth.exe`, a distlib launcher stub that CreateProcess's + the venv python and waits, so terminate() reaps the stub only: the real backend + keeps the inherited stdout handle, the reader thread never sees EOF, and + --repeats strands one server per iteration on the shared UNSLOTH_STUDIO_HOME. + taskkill /T walks the tree, as unsloth_cli/commands/start.py already does. + """ + if proc.poll() is not None: + return + if os.name == "nt": + try: + killed = subprocess.run( + ["taskkill", "/PID", str(proc.pid), "/T", "/F"], + capture_output = True, + timeout = 30, + check = False, + ) + if killed.returncode == 0: + return + except Exception: + # taskkill missing or timed out; fall through so the stub still dies. + pass + # check=False: a nonzero taskkill does not raise, so fall through as well. + proc.terminate() + + +def profile_launch( + bin_path: str, + port: int, + timeout_s: int = 300, +) -> dict: + """Spawn the backend the way the desktop app does and time it to first 200.""" + log_lines: list[str] = [] + first_byte: list[float] = [] + t0 = time.perf_counter() + proc = subprocess.Popen( + [bin_path, "studio", "--api-only", "-H", "127.0.0.1", "-p", str(port)], + cwd = REPO_ROOT, + stdout = subprocess.PIPE, + stderr = subprocess.STDOUT, + text = True, + bufsize = 1, + ) + + def _drain() -> None: + # Runs alongside the health polling: the first read timestamps the spawn + # phase, and an undrained pipe blocks the backend before it binds. + for line in proc.stdout: + if not first_byte: + first_byte.append(time.perf_counter() - t0) + log_lines.append(line.rstrip("\n")) + + reader = threading.Thread(target = _drain, daemon = True) + reader.start() + + t_healthz = None + deadline = t0 + timeout_s + try: + while time.perf_counter() < deadline: + if proc.poll() is not None: + break + if t_healthz is None: + for url in ( + f"http://127.0.0.1:{port}/api/health", + f"http://127.0.0.1:{port}/healthz", + ): + try: + with urllib.request.urlopen(url, timeout = 2) as r: + if r.status == 200: + t_healthz = time.perf_counter() - t0 + break + except (urllib.error.URLError, OSError, TimeoutError): + pass + if t_healthz is not None: + break + time.sleep(0.25) + finally: + _terminate_tree(proc) + try: + # Safe: the reader drains the pipe, so the child cannot block on write(). + proc.wait(timeout = 30) + except subprocess.TimeoutExpired: + proc.kill() + proc.wait() + reader.join(timeout = 10) + + t_first_byte = first_byte[0] if first_byte else None + lifespan_ms = None + for line in log_lines: + m = re.search(r"lifespan startup completed in ([\d.]+)ms", line) + if m: + lifespan_ms = float(m.group(1)) + return { + "spawn_seconds": round(t_first_byte, 3) if t_first_byte is not None else None, + "healthz_seconds": round(t_healthz, 3) if t_healthz is not None else None, + "lifespan_ms": lifespan_ms, + "reached_healthz": t_healthz is not None, + "log_tail": log_lines[-25:], + } + + +def python_version_of(python: str) -> str: + """Version of the interpreter that runs the imports, not the one running us. + + --python points at the installed Studio venv while this script runs under the + runner's system python, so platform.python_version() would label it wrong. + """ + if python == sys.executable: + return platform.python_version() + try: + proc = subprocess.run( + [python, "-c", "import platform; print(platform.python_version())"], + capture_output = True, + text = True, + timeout = 60, + ) + if proc.returncode == 0 and proc.stdout.strip(): + return proc.stdout.strip() + except (OSError, subprocess.SubprocessError): + pass + return "unknown" + + +def find_bin() -> str | None: + home = os.environ.get("UNSLOTH_STUDIO_HOME") or str(Path.home() / ".unsloth" / "studio") + names = ["unsloth.exe", "unsloth"] if platform.system() == "Windows" else ["unsloth"] + subdirs = ["unsloth_studio/Scripts", "unsloth_studio/bin", "bin", "Scripts"] + for sd in subdirs: + for n in names: + p = Path(home) / sd / n + if p.exists(): + return str(p) + return shutil.which("unsloth") + + +def main(argv: list[str]) -> int: + ap = argparse.ArgumentParser( + description = __doc__, formatter_class = argparse.RawDescriptionHelpFormatter + ) + ap.add_argument( + "--repeats", + type = int, + default = 1, + help = "launch repeats; the median is reported (imports are measured once)", + ) + ap.add_argument( + "--python", + default = sys.executable, + help = "interpreter used for the import profile (default: this one)", + ) + ap.add_argument("--bin", help = "path to the unsloth CLI (default: autodetect)") + ap.add_argument( + "--import-only", + action = "store_true", + help = "skip the server phases (no install needed beyond the deps)", + ) + ap.add_argument( + "--max-healthz-seconds", + type = float, + help = "fail if the median time to a healthy port exceeds this", + ) + ap.add_argument("--json", help = "write the full report here") + a = ap.parse_args(argv) + # range(0) launches nothing, leaving the budget check with nothing to fail on. + if a.repeats < 1: + ap.error("--repeats must be at least 1") + # Same reason: --import-only never launches anything. + if a.import_only and a.max_healthz_seconds is not None: + ap.error("--max-healthz-seconds cannot be combined with --import-only") + # nan and inf parse fine as floats but `med > budget` is then always False, + # so the gate would report success without ever bounding anything. + if a.max_healthz_seconds is not None and not math.isfinite(a.max_healthz_seconds): + ap.error("--max-healthz-seconds must be a finite number") + + report: dict = { + "platform": platform.system().lower(), + "machine": platform.machine(), + "python": python_version_of(a.python), + "cpu_count": os.cpu_count(), + } + + print("== import graph ==") + report["imports"] = profile_imports(a.python) + imp = report["imports"] + if imp.get("ok"): + print(f" import main: {imp['total_seconds']}s") + for row in imp["top_cumulative"][:8]: + print(f" {row['seconds']:7.3f}s {row['module']}") + print(" self time by package (ms):") + for k, v in list(imp["self_by_package_ms"].items())[:8]: + print(f" {v:8} ms {k}") + else: + print(f" FAILED: {imp.get('error', '')[:400]}") + + if not a.import_only: + bin_path = a.bin or find_bin() + if not bin_path: + print( + "== launch == skipped: no unsloth CLI found " + "(set UNSLOTH_STUDIO_HOME or pass --bin)" + ) + report["launch"] = {"skipped": "no unsloth CLI found"} + else: + print(f"== launch == {bin_path}") + runs = [] + for i in range(a.repeats): + r = profile_launch(bin_path, _free_port()) + runs.append(r) + print( + f" run {i + 1}: healthz={r['healthz_seconds']}s " + f"lifespan={r['lifespan_ms']}ms reached={r['reached_healthz']}" + ) + got = [r["healthz_seconds"] for r in runs if r["healthz_seconds"] is not None] + report["launch"] = { + "runs": runs, + "failed_runs": sum(1 for r in runs if not r["reached_healthz"]), + "healthz_median_seconds": round(statistics.median(got), 3) if got else None, + "healthz_max_seconds": round(max(got), 3) if got else None, + } + if got: + print( + f" median time to healthy port: {report['launch']['healthz_median_seconds']}s" + ) + + if a.json: + Path(a.json).write_text(json.dumps(report, indent = 2), encoding = "utf-8") + print(f"\nwrote {a.json}") + + if a.max_healthz_seconds is not None: + launch = report.get("launch") or {} + med = launch.get("healthz_median_seconds") + failed = launch.get("failed_runs") or 0 + if failed: + # Failed launches fail the budget; dropping them would keep only the fast ones. + print( + f"::error::startup regression: {failed} of {len(launch.get('runs') or [])} " + f"launches never became healthy within the timeout" + ) + return 1 + if med is None: + # Nothing measured: exiting 0 would pass a requested budget without a + # single health request, so fail closed. + print( + "::error::startup regression: no healthz measurement, so the " + f"{a.max_healthz_seconds}s budget was never checked " + f"({launch.get('skipped') or 'launch phase produced no runs'})" + ) + return 1 + elif med > a.max_healthz_seconds: + print( + f"::error::startup regression: {med}s median to a healthy port " + f"exceeds the {a.max_healthz_seconds}s budget" + ) + return 1 + return 0 + + +if __name__ == "__main__": + raise SystemExit(main(sys.argv[1:])) diff --git a/studio/backend/auth/storage.py b/studio/backend/auth/storage.py index 5f80ad89a3..35135b21eb 100644 --- a/studio/backend/auth/storage.py +++ b/studio/backend/auth/storage.py @@ -76,7 +76,13 @@ def _load_bootstrap_password() -> Optional[str]: global _bootstrap_password _bootstrap_password = None if _BOOTSTRAP_PW_PATH.is_file(): - bootstrap_password = _BOOTSTRAP_PW_PATH.read_text(encoding = "utf-8").strip() + # No caller handles a raise, so an unreadable file has to mean "no bootstrap + # password", not a dead backend. We write UTF-8, so bytes that will not + # decode are damage whose plaintext is worthless anyway. + try: + bootstrap_password = _BOOTSTRAP_PW_PATH.read_text(encoding = "utf-8").strip() + except (OSError, UnicodeDecodeError): + return _bootstrap_password if bootstrap_password: _bootstrap_password = bootstrap_password return _bootstrap_password diff --git a/studio/backend/cloudflare_tunnel.py b/studio/backend/cloudflare_tunnel.py index 78fce0c70a..f7967e2faa 100644 --- a/studio/backend/cloudflare_tunnel.py +++ b/studio/backend/cloudflare_tunnel.py @@ -310,6 +310,7 @@ class CloudflareTunnel: stderr = subprocess.STDOUT, stdin = subprocess.DEVNULL, text = True, + encoding = "utf-8", errors = "replace", bufsize = 1, **_windows_hidden_kwargs(), diff --git a/studio/backend/core/data_recipe/local_callable_validators.py b/studio/backend/core/data_recipe/local_callable_validators.py index ffc81669ae..143895d781 100644 --- a/studio/backend/core/data_recipe/local_callable_validators.py +++ b/studio/backend/core/data_recipe/local_callable_validators.py @@ -257,6 +257,8 @@ def _run_oxc_batch( cwd = str(_OXC_TOOL_DIR), input = json.dumps(payload), text = True, + encoding = "utf-8", + errors = "replace", capture_output = True, check = False, env = env, diff --git a/studio/backend/core/inference/inference.py b/studio/backend/core/inference/inference.py index 0af37e627f..e78bf1be8d 100644 --- a/studio/backend/core/inference/inference.py +++ b/studio/backend/core/inference/inference.py @@ -567,7 +567,7 @@ class InferenceBackend: _meta_path = Path(config.path) / "export_metadata.json" try: if _meta_path.exists(): - _meta = json.loads(_meta_path.read_text(encoding = "utf-8")) + _meta = json.loads(_meta_path.read_text(encoding = "utf-8-sig")) if _meta.get("base_model"): processor_source = _meta["base_model"] except Exception: diff --git a/studio/backend/core/inference/llama_cpp.py b/studio/backend/core/inference/llama_cpp.py index 144aa1fd37..dcfbfb3338 100644 --- a/studio/backend/core/inference/llama_cpp.py +++ b/studio/backend/core/inference/llama_cpp.py @@ -85,6 +85,7 @@ from core.tool_healing import ( strip_outside_think, ) from utils.native_path_leases import child_env_without_native_path_secret +from utils.child_stdio import utf8_child_env from utils.hf_xet_fallback import hf_hub_download_with_xet_fallback from utils.subprocess_compat import ( windows_hidden_subprocess_kwargs as _windows_hidden_subprocess_kwargs, @@ -581,7 +582,7 @@ def _load_swa_cache() -> dict: if _SWA_CACHE is not None: return _SWA_CACHE try: - with open(_swa_cache_path(), encoding = "utf-8") as f: + with open(_swa_cache_path(), encoding = "utf-8-sig") as f: _SWA_CACHE = json.load(f) if not isinstance(_SWA_CACHE, dict): _SWA_CACHE = {} @@ -632,7 +633,7 @@ def _fetch_swa_entry_from_hf(repo_id: str) -> Optional[object]: repo_type = "model", cache_dir = active_hf_hub_cache(), ) - with open(cfg_path, encoding = "utf-8") as f: + with open(cfg_path, encoding = "utf-8-sig") as f: cfg = json.load(f) except Exception: return None @@ -3046,6 +3047,7 @@ class LlamaCppBackend: [bin_path, "--help"], capture_output = True, text = True, + encoding = "utf-8", errors = "replace", timeout = 10, check = False, @@ -3618,6 +3620,8 @@ class LlamaCppBackend: ], capture_output = True, text = True, + encoding = "utf-8", + errors = "replace", timeout = 10, env = child_env_without_native_path_secret(), **_windows_hidden_subprocess_kwargs(), @@ -3732,7 +3736,7 @@ class LlamaCppBackend: encoding = "utf-8", errors = "replace", timeout = 15, - env = env, + env = utf8_child_env(env), **_windows_hidden_subprocess_kwargs(), ) if result.returncode != 0: @@ -5482,7 +5486,9 @@ class LlamaCppBackend: stdout = subprocess.PIPE, stderr = subprocess.STDOUT, text = True, - env = env, + encoding = "utf-8", + errors = "replace", + env = utf8_child_env(env), **_windows_hidden_subprocess_kwargs(), **_child_popen_kwargs(), ) @@ -6696,6 +6702,8 @@ class LlamaCppBackend: stdout = subprocess.PIPE, stderr = subprocess.STDOUT, text = True, + encoding = "utf-8", + errors = "replace", env = env, **_windows_hidden_subprocess_kwargs(), **_child_popen_kwargs(), @@ -8712,6 +8720,8 @@ class LlamaCppBackend: stdout = subprocess.PIPE, stderr = subprocess.STDOUT, text = True, + encoding = "utf-8", + errors = "replace", env = env, **_windows_hidden_subprocess_kwargs(), **_child_popen_kwargs(), @@ -10214,6 +10224,8 @@ class LlamaCppBackend: ["pgrep", "-a", "-f", "llama-server"], capture_output = True, text = True, + encoding = "utf-8", + errors = "replace", timeout = 5, env = child_env_without_native_path_secret(), ) diff --git a/studio/backend/core/inference/worker.py b/studio/backend/core/inference/worker.py index 3f32b3bd57..f208183300 100644 --- a/studio/backend/core/inference/worker.py +++ b/studio/backend/core/inference/worker.py @@ -151,7 +151,7 @@ def _resolve_lora_4bit(mc, load_in_4bit: bool) -> bool: import json try: - with open(adapter_cfg_path, encoding = "utf-8") as f: + with open(adapter_cfg_path, encoding = "utf-8-sig") as f: adapter_cfg = json.load(f) training_method = adapter_cfg.get("unsloth_training_method") if training_method == "lora" and load_in_4bit: @@ -963,7 +963,7 @@ def run_inference_process( if _local_adapter_cfg.is_file(): try: _lora_base = ( - _json.loads(_local_adapter_cfg.read_text(encoding = "utf-8")).get( + _json.loads(_local_adapter_cfg.read_text(encoding = "utf-8-sig")).get( "base_model_name_or_path" ) or None diff --git a/studio/backend/core/rag/embed_llama_server.py b/studio/backend/core/rag/embed_llama_server.py index facd989b27..b3ac62e520 100644 --- a/studio/backend/core/rag/embed_llama_server.py +++ b/studio/backend/core/rag/embed_llama_server.py @@ -103,6 +103,8 @@ class LlamaServerBackend: [binary, "--help"], capture_output = True, text = True, + encoding = "utf-8", + errors = "replace", timeout = 30, **windows_hidden_subprocess_kwargs(), ) @@ -331,6 +333,8 @@ class LlamaServerBackend: stdout = subprocess.PIPE, stderr = subprocess.STDOUT, text = True, + encoding = "utf-8", + errors = "replace", env = env, **windows_hidden_subprocess_kwargs(), **child_popen_kwargs(), diff --git a/studio/backend/core/rag/embeddings.py b/studio/backend/core/rag/embeddings.py index c86c0d3c51..95b8a866b2 100644 --- a/studio/backend/core/rag/embeddings.py +++ b/studio/backend/core/rag/embeddings.py @@ -100,7 +100,7 @@ def _st_module_subdirs(name: str, token: str | None) -> tuple[str, ...]: path = Path(normalize_path(name)).expanduser() / "modules.json" if not path.is_file(): return () - data = json.loads(path.read_text(encoding = "utf-8")) + data = json.loads(path.read_text(encoding = "utf-8-sig")) else: from huggingface_hub import hf_hub_download from huggingface_hub.utils import EntryNotFoundError @@ -115,7 +115,7 @@ def _st_module_subdirs(name: str, token: str | None) -> tuple[str, ...]: ) except EntryNotFoundError: return () - data = json.loads(open(local, encoding = "utf-8").read()) + data = json.loads(open(local, encoding = "utf-8-sig").read()) subdirs = [] for module in data or (): sub = str((module or {}).get("path", "")).strip().strip("/") diff --git a/studio/backend/core/training/worker.py b/studio/backend/core/training/worker.py index baf6329dae..b5fb5d224e 100644 --- a/studio/backend/core/training/worker.py +++ b/studio/backend/core/training/worker.py @@ -43,6 +43,7 @@ if sys.platform.startswith("linux") and "HSA_ENABLE_DXG_DETECTION" not in os.env pass logger = get_logger(__name__) +from utils.child_stdio import utf8_child_env from utils.hardware import apply_gpu_ids from utils.training_runs import build_default_output_dir_name from utils.wheel_utils import ( @@ -385,6 +386,10 @@ def _install_package_wheel_first( "stdout": _sp.PIPE, "stderr": _sp.STDOUT, "text": True, + "encoding": "utf-8", + "errors": "replace", + # Make the Python child emit the UTF-8 we decode above. + "env": utf8_child_env(), } if is_hip: _run_kwargs["timeout"] = 1800 @@ -606,6 +611,9 @@ def _ensure_flash_linear_attention_unconditional(event_queue: Any) -> bool: stdout = _sp.PIPE, stderr = _sp.STDOUT, text = True, + encoding = "utf-8", + errors = "replace", + env = utf8_child_env(), timeout = _TILELANG_INSTALL_TIMEOUT_S, ) except _sp.TimeoutExpired: @@ -849,6 +857,9 @@ def _run_pip(cmd: list[str], event_queue: Any, label: str) -> bool: stdout = _sp.PIPE, stderr = _sp.STDOUT, text = True, + encoding = "utf-8", + errors = "replace", + env = utf8_child_env(), timeout = _TILELANG_INSTALL_TIMEOUT_S, ) except _sp.TimeoutExpired: diff --git a/studio/backend/hub/services/models/ollama.py b/studio/backend/hub/services/models/ollama.py index 56275c22a9..da30f7e98c 100644 --- a/studio/backend/hub/services/models/ollama.py +++ b/studio/backend/hub/services/models/ollama.py @@ -215,7 +215,7 @@ def _ollama_model_info_from_manifest( return None try: - manifest = json.loads(tag_file.read_text(encoding = "utf-8")) + manifest = json.loads(tag_file.read_text(encoding = "utf-8-sig")) except (json.JSONDecodeError, OSError, UnicodeDecodeError) as e: logger.debug("Skipping unreadable/invalid Ollama manifest %s: %s", tag_file, e) return None @@ -228,7 +228,7 @@ def _ollama_model_info_from_manifest( config_blob = _ollama_blob_path(blobs_dir, config_digest) if config_blob is not None and _safe_is_file(config_blob): try: - cfg = json.loads(config_blob.read_text(encoding = "utf-8")) + cfg = json.loads(config_blob.read_text(encoding = "utf-8-sig")) model_type = cfg.get("model_type", "") file_type = cfg.get("file_type", "") except (json.JSONDecodeError, OSError, UnicodeDecodeError) as e: diff --git a/studio/backend/hub/utils/download_registry.py b/studio/backend/hub/utils/download_registry.py index 39c27208b1..760ef6b01c 100644 --- a/studio/backend/hub/utils/download_registry.py +++ b/studio/backend/hub/utils/download_registry.py @@ -464,6 +464,8 @@ def _read_marker_value(marker: Path) -> Optional[str]: return None value = marker.read_text(encoding = "utf-8").strip() except (OSError, UnicodeDecodeError): + # UnicodeDecodeError is a ValueError, so it would escape and abort + # prepare_cache_for_transport. An unknown value just purges and restarts. return None return value if value in VALID_TRANSPORTS else None diff --git a/studio/backend/loggers/config.py b/studio/backend/loggers/config.py index 688d3c7ebe..57cf7cecd6 100644 --- a/studio/backend/loggers/config.py +++ b/studio/backend/loggers/config.py @@ -42,8 +42,12 @@ class LogConfig: log_level_name = os.getenv("LOG_LEVEL", "INFO").upper() log_level = getattr(logging, log_level_name, logging.INFO) - if sys.platform == "win32": - for stream in (sys.stdout, sys.stderr): + # Non-ASCII on a non-UTF-8 stream raises UnicodeEncodeError (Windows, + # LANG=C), so key off the stream, not the platform. + for stream in (sys.stdout, sys.stderr): + if getattr(stream, "encoding", "") and not str(stream.encoding).lower().replace( + "-", "" + ).startswith("utf8"): if hasattr(stream, "reconfigure"): try: stream.reconfigure(encoding = "utf-8", errors = "replace") diff --git a/studio/backend/main.py b/studio/backend/main.py index 02f5a20106..9a2e598314 100644 --- a/studio/backend/main.py +++ b/studio/backend/main.py @@ -347,6 +347,7 @@ from utils.update_status import ( get_studio_install_source_status, get_studio_update_status, ) +from utils.changelog import get_release_notes, is_supported_version_query from utils.studio_version import get_studio_version from utils.api_errors import install_api_error_handlers @@ -1154,6 +1155,18 @@ def studio_update_status(_current_subject: str = Depends(get_current_subject)): return get_studio_update_status(UNSLOTH_VERSION) +@app.get("/api/studio/release-notes") +def studio_release_notes( + version: str = Query(..., max_length = 64), + refresh: bool = Query(False), + _current_subject: str = Depends(get_current_subject), +): + """Return CHANGELOG.md notes for exactly `version` (never a nearby one).""" + if not is_supported_version_query(version): + raise HTTPException(status_code = 422, detail = "Invalid version.") + return get_release_notes(version, refresh = refresh) + + @app.get( "/api/studio/download-transport-capabilities", response_model = TransportCapabilities, diff --git a/studio/backend/plugins/data-designer-github-repo-seed/src/data_designer_github_repo_seed/scraper_impl/state_store.py b/studio/backend/plugins/data-designer-github-repo-seed/src/data_designer_github_repo_seed/scraper_impl/state_store.py index b4c226136b..b059fad7ff 100644 --- a/studio/backend/plugins/data-designer-github-repo-seed/src/data_designer_github_repo_seed/scraper_impl/state_store.py +++ b/studio/backend/plugins/data-designer-github-repo-seed/src/data_designer_github_repo_seed/scraper_impl/state_store.py @@ -6,10 +6,93 @@ from __future__ import annotations import json +import locale import os import threading from pathlib import Path -from typing import Any, Dict +from typing import Any, Dict, NamedTuple + + +def _locale_encoding() -> str: + """The codepage a pre-UTF-8 release here would have written, or "". + + Empty on a UTF-8 host, where there is no codepage to attribute the file to. + """ + try: + preferred = locale.getencoding() + except AttributeError: # Python < 3.11 + preferred = locale.getpreferredencoding(False) + if preferred.lower().replace("-", "").replace("_", "") == "utf8": + return "" + return preferred + + +# Trail bytes can land on JSON punctuation, so a single-byte fallback misreads these. +_DOUBLE_BYTE_ENCODINGS = ("cp932", "cp936", "cp949", "cp950") + + +def _parse(raw: bytes, encoding: str) -> Any: + """Parse one JSON document under *encoding*, or None if it does not. + + RecursionError is a RuntimeError, so nesting json.loads will not descend is + the one parse failure the other three miss. Both callers run this outside + any further handler, so it has to answer None here or a single damaged + record aborts the scraper at startup instead of being skipped. + """ + try: + return json.loads(raw.decode(encoding)) + except (UnicodeDecodeError, LookupError, ValueError, RecursionError): + return None + + +class _Reading(NamedTuple): + as_utf8: Any + as_legacy: Any + + +def _read_line(raw: bytes, codepage: str) -> _Reading: + """Read one line as UTF-8 and as a codepage, for dedup keys only. + + Requiring valid JSON, not merely a successful decode, is what separates a + genuine legacy record from a half-written UTF-8 one: a torn multibyte + character decodes under cp1252 but leaves the JSON unterminated. Some byte + strings parse both ways, e.g. cp1251 ``Р°`` is ``D0 B0``, which is also + UTF-8 ``а``. + + The codepage reading is never authoritative, because the file's own encoding + cannot be recovered from its bytes. Reading a cp1251 shard on a cp1252 + machine turns ``Привет`` into ``Ïðèâåò`` and every byte of it decodes + cleanly, so a successful decode proves nothing about who wrote it. It is + used only to recover the dedup keys, which are ASCII ids and come back the + same under any of these, so the first reading that parses will do. + + That is also why several are tried. latin-1 alone mangles the double-byte + codepages: cp932 ``表`` is ``95 5C``, and latin-1 turns the trail byte into + a JSON backslash, so the record fails to parse and its id is forgotten. + """ + as_utf8 = _parse(raw, "utf-8") + # A record that reads as UTF-8 needs no second reading: re-parsing cost 2.8x on a + # 76 MB shard, and these reach gigabytes. Only a dict, since key lookup falls + # through to the codepage when UTF-8 yields none. + if isinstance(as_utf8, dict): + return _Reading(as_utf8, None) + for encoding in (codepage, "latin-1", *_DOUBLE_BYTE_ENCODINGS): + if not encoding: + continue + as_legacy = _parse(raw, encoding) + if as_legacy is not None: + return _Reading(as_utf8, as_legacy) + return _Reading(as_utf8, None) + + +class _Scan(NamedTuple): + """What a pass over an existing shard established about it.""" + + legacy: bool # enough evidence to trust the codepage reading's keys + readable: bool + saw_non_ascii: bool # some line's meaning depends on the encoding + utf8_keys: set # keys from lines UTF-8 could read + legacy_keys: set # keys only the codepage reading yields class StateStore: @@ -18,12 +101,19 @@ class StateStore: self.path.parent.mkdir(parents = True, exist_ok = True) self._lock = threading.Lock() self._data: Dict[str, Any] = {} + # Read whole, and UTF-8 only unlike the shards below: a checkpoint holds + # nothing but base64 cursors and booleans, so a codepage retry could only ever + # add non-ASCII. That would resume on a mojibaked cursor, which GitHub rejects + # with INVALID_CURSOR_ARGUMENTS, and the empty page it returns marks the stream + # done and skips the rest for good. Dropping a damaged checkpoint re-scrapes + # from the first page, which the writers dedup. if self.path.exists(): try: - with self.path.open(encoding = "utf-8") as f: - self._data = json.load(f) - except Exception: - self._data = {} + raw = self.path.read_bytes() + except OSError: + raw = b"" + data = _parse(raw, "utf-8") + self._data = data if isinstance(data, dict) else {} def get( self, @@ -63,24 +153,83 @@ class JsonlWriter: self.path = Path(path) self.path.parent.mkdir(parents = True, exist_ok = True) self._lock = threading.Lock() - self._fh = self.path.open("a", buffering = 1, encoding = "utf-8") self._count_seen_keys: set[str] = set() - # Preload seen keys for dedup across resumes + self._codepage = _locale_encoding() + self._ensure_ascii = False + encoding = "utf-8" if self.path.exists() and self.path.stat().st_size > 0: - try: - # No guess is safe for a file an older build wrote in the - # operator's locale, so read past whatever will not decode. - with self.path.open(encoding = "utf-8", errors = "replace") as f: - for line in f: - try: - obj = json.loads(line) - k = self._key(obj) - if k is not None: - self._count_seen_keys.add(k) - except Exception: - pass - except Exception: - pass + scan = self._scan_existing() + self._count_seen_keys = scan.utf8_keys + if scan.legacy: + self._count_seen_keys |= scan.legacy_keys + if scan.saw_non_ascii or not scan.readable: + # Never convert: the writing encoding is unrecoverable and guessing + # mojibakes the records. Pure ASCII appends store identically under + # every codepage, and json.loads turns the \uXXXX escapes back. + encoding = "ascii" + self._ensure_ascii = True + self._fh = self.path.open("a", buffering = 1, encoding = encoding, errors = "strict") + + def _scan_existing(self) -> _Scan: + """Read the shard once to recover dedup keys and judge its encoding. + + Line by line: these shards reach gigabytes on a large scrape, so neither + the bytes nor the decoded text are held whole. + + The verdict weighs the whole file. Each line with non-ASCII bytes votes: + one that parses only under the codepage is evidence of a legacy shard, + one that parses as UTF-8 is evidence against, since arbitrary codepage + text almost never forms valid multibyte UTF-8. A single corrupt byte in + a healthy shard therefore cannot outvote the records around it, and a + genuinely legacy shard has a legacy vote on every line that carries an + umlaut. + + More than one such line is required, because a single one is genuinely + undecidable: a legacy record holding one accented character and an ASCII + record holding one stray byte are the same shape. Reading it as damage + risks a duplicate; reading it as legacy marks an unreadable record seen + and blocks the retry that would replace it, losing it for good. Only one + of those is recoverable. + + The verdict only picks which reading supplies the dedup keys. The file + itself is never rewritten either way, so a wrong answer costs at most a + duplicate, never a corrupted record. + """ + legacy_votes = 0 + utf8_votes = 0 + saw_non_ascii = False + utf8_keys: set[str] = set() + legacy_keys: set[str] = set() + try: + with self.path.open("rb") as handle: + for raw in handle: + line = raw.strip() + reading = _read_line(line, self._codepage) + # ASCII reads the same everywhere: no vote, no constraint. + if not line.isascii(): + saw_non_ascii = True + if reading.as_utf8 is None and reading.as_legacy is not None: + legacy_votes += 1 + elif reading.as_utf8 is not None: + utf8_votes += 1 + # Kept apart so a damaged line does not block its own retry. + if isinstance(reading.as_utf8, dict): + key = self._key(reading.as_utf8) + if key is not None: + utf8_keys.add(key) + elif isinstance(reading.as_legacy, dict): + key = self._key(reading.as_legacy) + if key is not None: + legacy_keys.add(key) + except OSError: + return _Scan(False, False, False, utf8_keys, legacy_keys) + return _Scan( + legacy_votes > 1 and legacy_votes > utf8_votes, + True, + saw_non_ascii, + utf8_keys, + legacy_keys, + ) def _key(self, obj: dict) -> str | None: for k in ("id", "node_id", "number", "sha", "url"): @@ -99,7 +248,7 @@ class JsonlWriter: return False if k is not None: self._count_seen_keys.add(k) - self._fh.write(json.dumps(obj, default = str, ensure_ascii = False)) + self._fh.write(json.dumps(obj, default = str, ensure_ascii = self._ensure_ascii)) self._fh.write("\n") self._fh.flush() return True diff --git a/studio/backend/plugins/data-designer-unstructured-seed/src/data_designer_unstructured_seed/impl.py b/studio/backend/plugins/data-designer-unstructured-seed/src/data_designer_unstructured_seed/impl.py index ce0c88e5bf..825b050e07 100644 --- a/studio/backend/plugins/data-designer-unstructured-seed/src/data_designer_unstructured_seed/impl.py +++ b/studio/backend/plugins/data-designer-unstructured-seed/src/data_designer_unstructured_seed/impl.py @@ -30,6 +30,8 @@ class UnstructuredSeedReader(SeedReader[UnstructuredSeedSource]): meta = json_mod.loads(meta_path.read_text(encoding = "utf-8")) orig_name = meta.get("original_filename", path_obj.name) except (json_mod.JSONDecodeError, OSError, UnicodeDecodeError): + # Undecodable metadata is as malformed as invalid JSON, so + # fall back to the file's own name rather than abort the seed. pass file_entries.append((path_obj, orig_name)) diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.py index d0a2d97f74..20a5af1409 100644 --- a/studio/backend/routes/inference.py +++ b/studio/backend/routes/inference.py @@ -4434,7 +4434,7 @@ def _effective_load_in_4bit(config: ModelConfig, requested: bool) -> bool: if not adapter_cfg_path.exists(): return load_in_4bit try: - with open(adapter_cfg_path, encoding = "utf-8") as f: + with open(adapter_cfg_path, encoding = "utf-8-sig") as f: adapter_cfg = json.load(f) if not isinstance(adapter_cfg, dict): # malformed -> keep requested return load_in_4bit diff --git a/studio/backend/routes/models.py b/studio/backend/routes/models.py index 96c5b96d73..6e587c18e8 100644 --- a/studio/backend/routes/models.py +++ b/studio/backend/routes/models.py @@ -722,7 +722,7 @@ def _scan_ollama_dir(ollama_dir: Path, limit: Optional[int] = None) -> List[Loca stem_hash = hashlib.sha256(manifest_key.encode()).hexdigest()[:10] try: - manifest = json.loads(tag_file.read_text(encoding = "utf-8")) + manifest = json.loads(tag_file.read_text(encoding = "utf-8-sig")) except (json.JSONDecodeError, OSError, UnicodeDecodeError) as e: logger.debug( "Skipping unreadable/invalid Ollama manifest %s: %s", @@ -738,7 +738,7 @@ def _scan_ollama_dir(ollama_dir: Path, limit: Optional[int] = None) -> List[Loca config_blob = blobs_dir / config_digest.replace(":", "-") if config_blob.is_file(): try: - cfg = json.loads(config_blob.read_text(encoding = "utf-8")) + cfg = json.loads(config_blob.read_text(encoding = "utf-8-sig")) model_type = cfg.get("model_type", "") file_type = cfg.get("file_type", "") except (json.JSONDecodeError, OSError, UnicodeDecodeError) as e: @@ -1042,7 +1042,7 @@ def _dir_has_downloaded_model(directory: Path, max_entries: int = 4000) -> bool: if not m.is_file(): continue try: - manifest = json.loads(m.read_text(encoding = "utf-8")) + manifest = json.loads(m.read_text(encoding = "utf-8-sig")) except (json.JSONDecodeError, OSError, ValueError): continue for layer in manifest.get("layers") or []: @@ -3360,6 +3360,8 @@ def _wsl_reveal_in_explorer(path: Path) -> bool: ["wslpath", "-w", str(path)], capture_output = True, text = True, + encoding = "utf-8", + errors = "replace", check = True, timeout = 10, ).stdout.strip() diff --git a/studio/backend/run.py b/studio/backend/run.py index 08d1c5299e..ef372e004e 100644 --- a/studio/backend/run.py +++ b/studio/backend/run.py @@ -786,6 +786,8 @@ def _remove_pid_file(): stored = _PID_FILE.read_text(encoding = "utf-8").strip() if stored == str(os.getpid()): _PID_FILE.unlink(missing_ok = True) + # Runs first in _graceful_shutdown: a corrupt PID file raising here would + # abandon the children the rest of that function exists to kill. except (OSError, UnicodeDecodeError): pass diff --git a/studio/backend/tests/test_chat_text_encoding.py b/studio/backend/tests/test_chat_text_encoding.py new file mode 100644 index 0000000000..64860dab1a --- /dev/null +++ b/studio/backend/tests/test_chat_text_encoding.py @@ -0,0 +1,195 @@ +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 + +"""Model text stays intact when it carries non-ASCII. + +``open()`` and ``Path.read_text()`` fall back to ``locale.getencoding()`` when +no ``encoding`` is passed. On Windows that is the ANSI codepage, not UTF-8, so +a chat template or model config holding ``ä ö ü → 世`` mojibakes or raises +``UnicodeDecodeError``. These files are UTF-8, so the reads must say so. + +Each fixture writes raw UTF-8 (``ensure_ascii = False``), matching what +Hugging Face actually ships, rather than ASCII ``\\uXXXX`` escapes. +""" + +from __future__ import annotations + +import json +import subprocess +import sys +import textwrap +from pathlib import Path + + +BACKEND_ROOT = Path(__file__).resolve().parent.parent + + +def test_config_json_round_trips_non_ascii(tmp_path: Path) -> None: + from utils import transformers_version + + name = "Modell für Grüße 世界" + (tmp_path / "config.json").write_text( + json.dumps({"model_type": "llama", "_name_or_path": name}, ensure_ascii = False), + encoding = "utf-8", + ) + transformers_version._config_json_cache.clear() + + cfg = transformers_version._load_config_json(str(tmp_path)) + + assert cfg is not None + assert cfg["_name_or_path"] == name + + +def test_tokenizer_config_round_trips_non_ascii_chat_template(tmp_path: Path) -> None: + """Chat templates commonly hold ``→`` and smart quotes, which cp1252 mangles.""" + from utils import transformers_version + + template = "{{ '→ Grüße 世界' }}" + (tmp_path / "tokenizer_config.json").write_text( + json.dumps( + {"tokenizer_class": "TokenizersBackend", "chat_template": template}, + ensure_ascii = False, + ), + encoding = "utf-8", + ) + transformers_version._tokenizer_class_cache.clear() + + assert transformers_version._check_tokenizer_config_needs_v5(str(tmp_path)) is True + + +def test_config_json_survives_a_utf8_bom(tmp_path: Path) -> None: + """Notepad wrote "UTF-8 with BOM" by default for years, so hand-edited + configs on Windows carry one. Plain utf-8 keeps the BOM and json.load then + fails on it; utf-8-sig strips it and is identical otherwise.""" + from utils import transformers_version + + name = "Grüße 世界" + (tmp_path / "config.json").write_text( + json.dumps({"model_type": "llama", "_name_or_path": name}, ensure_ascii = False), + encoding = "utf-8-sig", + ) + transformers_version._config_json_cache.clear() + + cfg = transformers_version._load_config_json(str(tmp_path)) + + assert cfg is not None + assert cfg["_name_or_path"] == name + + +def test_remote_code_scan_reads_non_ascii_sources(tmp_path: Path) -> None: + """A German Windows profile also puts umlauts in the model sources scanned.""" + from utils.security import remote_code_scan + + source = "# Grüße über Öl\nVALUE = '世界'\n" + # newline = "" pins the bytes on disk, so Windows line end translation cannot make the + # read back differ by \r. open() because Path.write_text() only grew newline in 3.10. + with open( + tmp_path / "modeling_custom.py", + "w", + encoding = "utf-8", + newline = "", + ) as handle: + handle.write(source) + + files = remote_code_scan.repo_remote_code_files(str(tmp_path)) + + assert files["modeling_custom.py"] == source + + +def test_model_config_reads_do_not_rely_on_the_locale_encoding(tmp_path: Path) -> None: + """The reads above pass anywhere the locale is already UTF-8, which hides + the Windows bug on Linux and macOS. ``-X warn_default_encoding`` makes + CPython flag any text I/O that falls back to the locale, so this fails on + every platform if an ``encoding`` argument goes missing again.""" + # The readers swallow exceptions, so record the warnings instead of raising. + script = textwrap.dedent( + f""" + import sys, warnings + sys.path.insert(0, {str(BACKEND_ROOT)!r}) + from utils import transformers_version + + target = {str(tmp_path)!r} + with warnings.catch_warnings(record = True) as caught: + warnings.simplefilter("always") + transformers_version._config_json_cache.clear() + transformers_version._tokenizer_class_cache.clear() + assert transformers_version._load_config_json(target) is not None + assert transformers_version._check_tokenizer_config_needs_v5(target) is True + + missing = [str(w.message) for w in caught if w.category is EncodingWarning] + if missing: + sys.exit("text I/O fell back to the locale encoding: " + "; ".join(missing)) + """ + ) + for name, payload in ( + ("config.json", {"model_type": "llama", "_name_or_path": "Grüße"}), + ("tokenizer_config.json", {"tokenizer_class": "TokenizersBackend"}), + ): + (tmp_path / name).write_text(json.dumps(payload, ensure_ascii = False), encoding = "utf-8") + + result = subprocess.run( + [sys.executable, "-X", "warn_default_encoding", "-c", script], + capture_output = True, + text = True, + encoding = "utf-8", + errors = "replace", + timeout = 120, + ) + + assert result.returncode == 0, result.stderr + + +def test_utf8_child_env_round_trips_non_ascii(tmp_path: Path) -> None: + """A Python child encodes stdout with its locale unless told otherwise, so + reading its pipe as utf-8 needs the child told to emit utf-8.""" + from utils.child_stdio import utf8_child_env + + payload = "Grüße über Öl → 世界" + child = tmp_path / "child.py" + child.write_text("import sys\nsys.stdout.write(" + repr(payload) + ")\n", encoding = "utf-8") + + env = utf8_child_env() + assert env["PYTHONIOENCODING"] == "utf-8" + + proc = subprocess.run( + [sys.executable, str(child)], + capture_output = True, + text = True, + encoding = "utf-8", + errors = "replace", + env = env, + timeout = 120, + ) + + assert proc.returncode == 0, proc.stderr + assert proc.stdout == payload + + +def test_python_children_are_told_to_emit_utf8() -> None: + """Any child we decode as utf-8 must also be told to write utf-8, or a + cp1252 console silently mangles what it prints.""" + import ast + + offenders: list[str] = [] + for path in sorted(BACKEND_ROOT.rglob("*.py")): + parts = path.relative_to(BACKEND_ROOT).parts + if any(p in ("tests", "node_modules", "plugins", "__pycache__") for p in parts): + continue + source = path.read_text(encoding = "utf-8") + for node in ast.walk(ast.parse(source, filename = str(path))): + if not isinstance(node, ast.Call): + continue + func = node.func + if not (isinstance(func, ast.Attribute) and func.attr in ("run", "Popen")): + continue + segment = ast.get_source_segment(source, node) or "" + if "sys.executable" not in segment or 'encoding = "utf-8"' not in segment: + continue + if "utf8_child_env" in segment or "PYTHONIOENCODING" in segment: + continue + offenders.append(f"{path.name}:{node.lineno}") + + assert not offenders, ( + "these spawn a Python child and decode it as utf-8 without setting the " + "child's own stdio encoding; wrap env in utf8_child_env():\n " + "\n ".join(offenders) + ) diff --git a/studio/backend/tests/test_rocm_multi_gpu_vram_system_wide.py b/studio/backend/tests/test_rocm_multi_gpu_vram_system_wide.py index bdafdeae9b..db89b02003 100644 --- a/studio/backend/tests/test_rocm_multi_gpu_vram_system_wide.py +++ b/studio/backend/tests/test_rocm_multi_gpu_vram_system_wide.py @@ -45,8 +45,20 @@ def _build_structlog_stub(): _maybe_stub("loggers", _build_loggers_stub) _maybe_stub("structlog", _build_structlog_stub) +import pytest + import utils.hardware.hardware as hw # noqa: E402 +# The DRM/KFD readers below are Linux-only in production: _rocm_linux_amdgpu_cards and +# _rocm_linux_sysfs_vram_by_pci_gb return early unless platform.system() is "Linux", and +# _rocm_kfd_gpu_pci_ids only ever globs /sys/class/kfd. Their fake sysfs tree needs PCI +# addresses like "0000:00:02.0" as directory names and POSIX separators in the paths the +# readers match; Windows permits neither, so the tree cannot be represented there. +linux_only = pytest.mark.skipif( + not sys.platform.startswith("linux"), + reason = "covers Linux-only DRM/KFD sysfs parsing driven by a fake /sys tree", +) + def _device( index, @@ -99,6 +111,7 @@ def _fake_drm(tmp_path, monkeypatch, cards): return card_paths +@linux_only def test_linux_vram_keyed_by_pci_excludes_foreign_adapters(monkeypatch, tmp_path): # Foreign (non-amdgpu) adapters contribute no entry, so they cannot shift ordinals. monkeypatch.setattr(hw.platform, "system", lambda: "Linux") @@ -117,6 +130,7 @@ def test_linux_vram_keyed_by_pci_excludes_foreign_adapters(monkeypatch, tmp_path } +@linux_only def test_linux_vram_omits_bad_cards_without_shifting(monkeypatch, tmp_path): # A zero-total card has no entry; identity keying means its absence renumbers nothing. monkeypatch.setattr(hw.platform, "system", lambda: "Linux") @@ -131,6 +145,7 @@ def test_linux_vram_omits_bad_cards_without_shifting(monkeypatch, tmp_path): assert hw._rocm_linux_sysfs_vram_by_pci_gb() == {"0000:41:00.0": (2.0, 16.0)} +@linux_only def test_linux_vram_omits_amd_card_without_vram_files(monkeypatch, tmp_path): # An APU with no mem_info_vram_* files has no entry; the discrete card keeps its address. monkeypatch.setattr(hw.platform, "system", lambda: "Linux") @@ -174,6 +189,7 @@ def _fake_kfd(tmp_path, monkeypatch, nodes): return node_paths +@linux_only def test_kfd_lists_gpu_nodes_in_device_order(monkeypatch, tmp_path): # The CPU node (simd_count 0) takes no ordinal; GPU nodes in node-id order are HIP's order. monkeypatch.setattr(hw.platform, "system", lambda: "Linux") @@ -189,12 +205,14 @@ def test_kfd_lists_gpu_nodes_in_device_order(monkeypatch, tmp_path): assert hw._rocm_kfd_gpu_pci_ids() == ["0000:03:00.0", "0000:41:00.0"] +@linux_only def test_kfd_decodes_domain_device_and_function(monkeypatch, tmp_path): monkeypatch.setattr(hw.platform, "system", lambda: "Linux") _fake_kfd(tmp_path, monkeypatch, [(1, 64, (0xC1 << 8) | (0x1F << 3) | 5, 0x1234, _AMD)]) assert hw._rocm_kfd_gpu_pci_ids() == ["1234:c1:1f.5"] +@linux_only def test_kfd_skips_non_amd_gpu_nodes(monkeypatch, tmp_path): # An NVIDIA KFD node is not a HIP device: it must take no ordinal, else it # shifts every AMD GPU and ROCm device 1 resolves to AMD GPU 0. @@ -212,6 +230,7 @@ def test_kfd_skips_non_amd_gpu_nodes(monkeypatch, tmp_path): assert hw._rocm_kfd_gpu_pci_ids() == ["0000:03:00.0", "0000:41:00.0"] +@linux_only def test_kfd_fails_closed_when_a_gpu_has_no_location(monkeypatch, tmp_path): # Dropping an unplaceable AMD GPU shifts later ordinals; fail closed for the whole map. monkeypatch.setattr(hw.platform, "system", lambda: "Linux") @@ -226,6 +245,7 @@ def test_kfd_fails_closed_when_a_gpu_has_no_location(monkeypatch, tmp_path): assert hw._rocm_kfd_gpu_pci_ids() == [] +@linux_only def test_kfd_fails_closed_when_a_node_is_unreadable(monkeypatch, tmp_path): # An unreadable node could be a GPU; assuming otherwise would shift ordinals. monkeypatch.setattr(hw.platform, "system", lambda: "Linux") @@ -241,6 +261,23 @@ def test_kfd_fails_closed_when_a_node_is_unreadable(monkeypatch, tmp_path): assert hw._rocm_kfd_gpu_pci_ids() == [] +@linux_only +def test_kfd_fails_closed_when_a_node_does_not_decode(monkeypatch, tmp_path): + # UnicodeDecodeError is a ValueError, so it slips past `except OSError` and + # would shift every later HIP ordinal. + monkeypatch.setattr(hw.platform, "system", lambda: "Linux") + paths = _fake_kfd( + tmp_path, + monkeypatch, + [ + (1, 304, (0x03 << 8) | 0, 0, _AMD), + (2, 304, (0x41 << 8) | 0, 0, _AMD), + ], + ) + (Path(paths[0]) / "properties").write_bytes(b"simd_count 304\nvendor_id \x80\xff\n") + assert hw._rocm_kfd_gpu_pci_ids() == [] + + def test_kfd_absent_yields_no_device_order(monkeypatch): monkeypatch.setattr(hw.glob, "glob", lambda pattern: []) assert hw._rocm_kfd_gpu_pci_ids() == [] @@ -422,6 +459,10 @@ def test_visible_utilization_rocm_fallback_overlays(monkeypatch): ): monkeypatch.delenv(_var, raising = False) monkeypatch.setattr(hw, "IS_ROCM", True) + # No AMD adapter data on this host. On Windows this branch runs ahead of the torch + # fallback under test, and probing it imports torch, which the CI runner does not + # install. Off Windows the real function is never reached, so this changes nothing. + monkeypatch.setattr(hw, "_rocm_windows_per_device_vram", lambda ids: []) monkeypatch.setattr(hw, "get_device", lambda: hw.DeviceType.CUDA) monkeypatch.setattr(hw, "_smi_query", lambda *a, **k: None) # amd-smi unavailable monkeypatch.setattr( @@ -450,6 +491,10 @@ def test_visible_utilization_rocm_fallback_overlays(monkeypatch): def test_visible_utilization_relative_index_skips_overlay(monkeypatch): # UUID/MIG mask gives relative indices; the overlay matches physical index, so it must not run. monkeypatch.setattr(hw, "IS_ROCM", True) + # No AMD adapter data on this host. On Windows this branch runs ahead of the torch + # fallback under test, and probing it imports torch, which the CI runner does not + # install. Off Windows the real function is never reached, so this changes nothing. + monkeypatch.setattr(hw, "_rocm_windows_per_device_vram", lambda ids: []) monkeypatch.setattr(hw, "get_device", lambda: hw.DeviceType.CUDA) monkeypatch.setattr(hw, "_smi_query", lambda *a, **k: None) monkeypatch.setattr( diff --git a/studio/backend/tests/test_text_io_encoding.py b/studio/backend/tests/test_text_io_encoding.py new file mode 100644 index 0000000000..7eae3c7fef --- /dev/null +++ b/studio/backend/tests/test_text_io_encoding.py @@ -0,0 +1,809 @@ +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 + +"""Text I/O must name its encoding, or Windows silently uses the ANSI codepage. + +``open()``, ``Path.read_text()`` and ``subprocess(text = True)`` fall back to +``locale.getencoding()`` when no ``encoding`` is passed. On Windows that is +cp1252 (or cp932, cp1251, ... by system locale), not UTF-8, so a chat template, +model config or path containing ``ä ö ü → 世`` mojibakes or raises +``UnicodeDecodeError`` mid-load. Studio's files are UTF-8, so say so. +""" + +from __future__ import annotations + +import ast +import importlib.util +import json +import os +from pathlib import Path +from types import SimpleNamespace + +import pytest + + +BACKEND_ROOT = Path(__file__).resolve().parent.parent + +# Not runtime source. Shipped plugins under plugins/*/src are, so only builds are skipped. +_SKIPPED_DIRS = ("node_modules", "build", "tests", "__pycache__") + +# Path.open()'s signature is what tells it apart from other libraries' open(), +# e.g. fitz.open(stream=...) and av.open(..., metadata_errors=...). +_FILE_MODE_CHARS = set("rwxabt+") +_PATH_OPEN_ARGS = ("mode", "buffering", "encoding", "errors", "newline") +_PATH_OPEN_KWARGS = set(_PATH_OPEN_ARGS) +_PATH_OPEN_ENCODING_ARG = _PATH_OPEN_ARGS.index("encoding") + +_SUBPROCESS_CALLS = {"run", "Popen", "check_output", "check_call", "call"} + +# open(file, mode, buffering, encoding, ...), and os.fdopen forwards the same +# signature with a descriptor in place of the path. +_OPEN_ENCODING_ARG = 3 + + +def _studio_sources() -> list[Path]: + return [ + path + for path in sorted(BACKEND_ROOT.rglob("*.py")) + if not any(part in _SKIPPED_DIRS for part in path.relative_to(BACKEND_ROOT).parts) + ] + + +def _has_keyword(node: ast.Call, name: str) -> bool: + return any(keyword.arg == name for keyword in node.keywords) + + +def _mode_is_binary(node: ast.Call) -> bool: + mode: str | None = None + if len(node.args) >= 2 and isinstance(node.args[1], ast.Constant): + value = node.args[1].value + mode = value if isinstance(value, str) else None + for keyword in node.keywords: + if keyword.arg == "mode" and isinstance(keyword.value, ast.Constant): + value = keyword.value.value + if isinstance(value, str): + mode = value + return bool(mode and "b" in mode) + + +def _open_has_encoding(node: ast.Call) -> bool: + """open()/os.fdopen() also take encoding positionally: open(p, "w", 1, "utf-8").""" + return _has_keyword(node, "encoding") or len(node.args) > _OPEN_ENCODING_ARG + + +def _path_open_mode(node: ast.Call) -> str | None: + if node.args and isinstance(node.args[0], ast.Constant): + value = node.args[0].value + if isinstance(value, str): + return value + for keyword in node.keywords: + if keyword.arg == "mode" and isinstance(keyword.value, ast.Constant): + value = keyword.value.value + if isinstance(value, str): + return value + return None + + +def _is_path_open(node: ast.Call) -> bool: + """True only for calls matching ``Path.open``'s signature.""" + if len(node.args) > len(_PATH_OPEN_ARGS): + return False + if any(k.arg not in _PATH_OPEN_KWARGS for k in node.keywords): + return False + mode = _path_open_mode(node) + if mode is not None: + return bool(mode) and set(mode) <= _FILE_MODE_CHARS + return not node.args + + +def _path_open_has_encoding(node: ast.Call) -> bool: + """Path.open() also takes encoding positionally: open("w", 1, "utf-8").""" + return _has_keyword(node, "encoding") or len(node.args) > _PATH_OPEN_ENCODING_ARG + + +def _call_name(node: ast.Call) -> str | None: + func = node.func + if isinstance(func, ast.Name): + return func.id + if isinstance(func, ast.Attribute): + return func.attr + return None + + +def _subprocess_names(tree: ast.AST) -> set[str]: + """Names subprocess is reachable under here, e.g. `import subprocess as _sp`.""" + names = set() + for node in ast.walk(tree): + if isinstance(node, ast.Import): + for alias in node.names: + if alias.name == "subprocess": + names.add(alias.asname or alias.name) + return names + + +def _subprocess_aliases(tree: ast.AST, names: set[str]) -> set[str]: + """Plain names bound to a subprocess callable, called without the module. + + ``install_wheel(run = subprocess.run)`` calls its injected ``run`` as a bare + name, so matching only the attribute form leaves those installer calls + unguarded. Imports, assignments and parameter defaults all bind one. + """ + + def _is_bound(value: ast.expr | None) -> bool: + return ( + isinstance(value, ast.Attribute) + and value.attr in _SUBPROCESS_CALLS + and isinstance(value.value, ast.Name) + and value.value.id in names + ) + + aliases: set[str] = set() + for node in ast.walk(tree): + if isinstance(node, ast.ImportFrom) and node.module == "subprocess": + aliases.update(a.asname or a.name for a in node.names if a.name in _SUBPROCESS_CALLS) + elif isinstance(node, ast.Assign) and _is_bound(node.value): + aliases.update(t.id for t in node.targets if isinstance(t, ast.Name)) + elif isinstance(node, ast.AnnAssign) and _is_bound(node.value): + if isinstance(node.target, ast.Name): + aliases.add(node.target.id) + elif isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)): + args = node.args + positional = args.posonlyargs + args.args + # Defaults cover the tail of the positional parameters; kw_defaults + # is aligned with kwonlyargs already, holding None where absent. + padded = [None] * (len(positional) - len(args.defaults)) + list(args.defaults) + pairs = list(zip(positional, padded)) + list(zip(args.kwonlyargs, args.kw_defaults)) + aliases.update(arg.arg for arg, default in pairs if _is_bound(default)) + return aliases + + +def _is_subprocess_call(node: ast.Call, names: set[str], aliases: set[str]) -> bool: + func = node.func + if isinstance(func, ast.Name): + return func.id in aliases + if not isinstance(func, ast.Attribute) or func.attr not in _SUBPROCESS_CALLS: + return False + value = func.value + return isinstance(value, ast.Name) and value.id in names + + +def _text_mode_subprocess(node: ast.Call) -> bool: + for keyword in node.keywords: + if keyword.arg not in ("text", "universal_newlines"): + continue + if isinstance(keyword.value, ast.Constant) and keyword.value.value is True: + return True + return False + + +def _text_mode_dict(node: ast.Dict) -> bool: + """A ``{"text": True, ...}`` literal with no "encoding" key.""" + keys = [k.value for k in node.keys if isinstance(k, ast.Constant)] + if "encoding" in keys: + return False + for key, value in zip(node.keys, node.values): + if not isinstance(key, ast.Constant) or key.value not in ( + "text", + "universal_newlines", + ): + continue + if isinstance(value, ast.Constant) and value.value is True: + return True + return False + + +def _splatted_names(tree: ast.AST) -> set[str]: + """Names handed to a call as ``**name``.""" + names = set() + for node in ast.walk(tree): + if isinstance(node, ast.Call): + for keyword in node.keywords: + if keyword.arg is None and isinstance(keyword.value, ast.Name): + names.add(keyword.value.id) + return names + + +def _encoding_assigned_later(tree: ast.AST, name: str) -> bool: + """``name["encoding"] = ...`` somewhere, so the literal need not carry it.""" + for node in ast.walk(tree): + if not isinstance(node, ast.Subscript) or not isinstance(node.ctx, ast.Store): + continue + target, key = node.value, node.slice + if isinstance(target, ast.Name) and target.id == name: + if isinstance(key, ast.Constant) and key.value == "encoding": + return True + return False + + +def _splatted_kwargs_offenders(tree: ast.AST) -> list[ast.Dict]: + """Text-mode kwargs built in a dict and splatted into a call. + + Kwargs are collected in a dict and splatted (``run(cmd, **run_kwargs)``) + where a branch has to add a timeout or an env, and the call is often through + a helper, so neither the callee nor the keywords are visible at the call + site. Only dicts that reach a call this way are judged: an unrelated payload + that happens to carry ``"text": True`` is not subprocess configuration. + """ + found = [] + # ``run(cmd, **{...})``: the literal is at the call already. + for node in ast.walk(tree): + if not isinstance(node, ast.Call): + continue + for keyword in node.keywords: + if keyword.arg is None and isinstance(keyword.value, ast.Dict): + if _text_mode_dict(keyword.value): + found.append(keyword.value) + splatted = _splatted_names(tree) + if not splatted: + return found + for node in ast.walk(tree): + targets = [] + if isinstance(node, ast.Assign): + targets = [t for t in node.targets if isinstance(t, ast.Name)] + elif isinstance(node, ast.AnnAssign) and isinstance(node.target, ast.Name): + targets = [node.target] + if not targets or not isinstance(node.value, ast.Dict): + continue + if not _text_mode_dict(node.value): + continue + for target in targets: + if target.id in splatted and not _encoding_assigned_later(tree, target.id): + found.append(node.value) + break + return found + + +def _offenders(path: Path) -> list[str]: + source = path.read_text(encoding = "utf-8") + tree = ast.parse(source, filename = str(path)) + subprocess_names = _subprocess_names(tree) + subprocess_aliases = _subprocess_aliases(tree, subprocess_names) + found: list[str] = [] + for node in _splatted_kwargs_offenders(tree): + found.append( + f"{path.name}:{node.lineno}: subprocess kwargs with text = True and no encoding" + ) + for node in ast.walk(tree): + if not isinstance(node, ast.Call): + continue + name = _call_name(node) + + if _is_subprocess_call(node, subprocess_names, subprocess_aliases): + if _text_mode_subprocess(node) and not _has_keyword(node, "encoding"): + found.append(f"{path.name}:{node.lineno}: subprocess(text = True) without encoding") + continue + + if name == "open" and isinstance(node.func, ast.Name): + if _mode_is_binary(node) or _open_has_encoding(node): + continue + found.append(f"{path.name}:{node.lineno}: open() without encoding") + continue + + # os.fdopen(fd, "w") is open() on a descriptor, so text mode takes the + # same locale default. Its mode defaults to "r", i.e. text, like open's. + if name == "fdopen": + if _mode_is_binary(node) or _open_has_encoding(node): + continue + found.append(f"{path.name}:{node.lineno}: os.fdopen() without encoding") + continue + + if name == "open" and isinstance(node.func, ast.Attribute): + if not _is_path_open(node) or _path_open_has_encoding(node): + continue + if _path_open_mode(node) and "b" in _path_open_mode(node): + continue + found.append(f"{path.name}:{node.lineno}: Path.open() without encoding") + continue + + if name in ("read_text", "write_text") and isinstance(node.func, ast.Attribute): + if _has_keyword(node, "encoding"): + continue + # importlib.metadata Distribution.read_text() takes no encoding kwarg. + if isinstance(node.func.value, ast.Name) and node.func.value.id == "dist": + continue + found.append(f"{path.name}:{node.lineno}: {name}() without encoding") + return found + + +@pytest.mark.parametrize("path", _studio_sources(), ids = lambda p: str(p.name)) +def test_text_io_names_its_encoding(path: Path) -> None: + offenders = _offenders(path) + assert not offenders, ( + "Text I/O without an explicit encoding falls back to the Windows ANSI " + 'codepage and corrupts non-ASCII (ä ö ü → 世). Pass encoding = "utf-8":\n ' + + "\n ".join(offenders) + ) + + +_STATE_STORE = ( + BACKEND_ROOT + / "plugins/data-designer-github-repo-seed/src" + / "data_designer_github_repo_seed/scraper_impl/state_store.py" +) + + +def _load_state_store(codepage: str): + """Load state_store with the writing machine's codepage pinned.""" + spec = importlib.util.spec_from_file_location(f"state_store_{codepage}", _STATE_STORE) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + module.locale = SimpleNamespace( + getencoding = lambda: codepage, + getpreferredencoding = lambda _ = True: codepage, + ) + return module + + +@pytest.mark.parametrize( + ("codepage", "name"), [("cp1252", "Jürgen"), ("cp1251", "Юрий"), ("cp932", "田中")] +) +def test_resuming_a_legacy_jsonl_keeps_one_encoding( + tmp_path: Path, codepage: str, name: str +) -> None: + """A scrape written before UTF-8 was explicit must resume, not duplicate.""" + path = tmp_path / "out.jsonl" + records = [{"id": 1, "author": name}, {"id": 2, "author": name}] + body = "".join(json.dumps(r, ensure_ascii = False) + "\n" for r in records) + path.write_bytes(body.encode(codepage)) + before = path.read_bytes() + + writer = _load_state_store(codepage).JsonlWriter(path) + try: + # Seen keys survive the resume, so a repeat is refused, not appended. + assert writer.has("id:1") and writer.has("id:2") + assert writer.write(records[0]) is False + assert writer.write({"id": 3, "author": name}) is True + finally: + writer.close() + + # Never converted, so it still reads in its own codepage; the append is ASCII. + blob = path.read_bytes() + assert blob.startswith(before) + assert blob[len(before) :].isascii() + lines = [json.loads(x) for x in blob.decode(codepage).splitlines() if x.strip()] + assert len(lines) == 3 + assert [line["author"] for line in lines] == [name] * 3 + + +def test_a_coincidentally_utf8_legacy_line_is_left_alone(tmp_path: Path) -> None: + """cp1251 `Р°` is D0 B0, which is also UTF-8 `а`, and nothing can tell them apart.""" + path = tmp_path / "out.jsonl" + ambiguous = "Р°" + assert ambiguous.encode("cp1251").decode("utf-8") == "а" # the trap + authors = ["Привет", "Здравствуйте", "Москва", ambiguous] + path.write_bytes( + b"".join( + json.dumps({"id": i, "author": a}, ensure_ascii = False).encode("cp1251") + b"\n" + for i, a in enumerate(authors) + ) + ) + before = path.read_bytes() + + _load_state_store("cp1251").JsonlWriter(path).close() + + # Untouched, so the ambiguity never had to be resolved. + assert path.read_bytes() == before + rows = [json.loads(x) for x in path.read_text(encoding = "cp1251").splitlines() if x.strip()] + assert [row["author"] for row in rows] == authors + + +@pytest.mark.parametrize( + ("codepage", "word"), [("cp1251", "Привет"), ("cp932", "こんにちは"), ("cp1252", "Jürgen")] +) +def test_a_moved_shard_is_not_rewritten_by_guesswork( + tmp_path: Path, codepage: str, word: str +) -> None: + """Off the writing machine there is no codepage to attribute the file to.""" + path = tmp_path / "out.jsonl" + # Two records: a lone non-UTF-8 line would count as damage, not legacy. + path.write_bytes( + b"".join( + json.dumps({"id": i, "author": word}, ensure_ascii = False).encode(codepage) + b"\n" + for i in (1, 4) + ) + ) + before = path.read_bytes() + + # A UTF-8 host: latin-1 would read cp1251 `Привет` back as `Ïðèâåò`. + writer = _load_state_store("utf-8").JsonlWriter(path) + try: + assert writer.has("id:1") # ASCII keys still recover + assert writer.write({"id": 2, "author": "Grüße"}) is True + finally: + writer.close() + + blob = path.read_bytes() + assert blob.startswith(before) # never rewritten + assert blob[len(before) :].isascii() # appended as \uXXXX, so no second encoding + rows = [json.loads(x) for x in blob.decode(codepage).splitlines() if x.strip()] + assert [row["author"] for row in rows] == [word, word, "Grüße"] + + +def test_an_all_ambiguous_shard_still_gets_ascii_appends(tmp_path: Path) -> None: + """Every line valid under both readings still means the append must not pick one.""" + path = tmp_path / "out.jsonl" + ambiguous = "Р°" # cp1251 D0 B0, also valid UTF-8 for "а" + path.write_bytes( + b"".join( + json.dumps({"id": i, "a": ambiguous}, ensure_ascii = False).encode("cp1251") + b"\n" + for i in range(3) + ) + ) + before = path.read_bytes() + + writer = _load_state_store("cp1251").JsonlWriter(path) + try: + assert writer.write({"id": 9, "a": "世界"}) is True + finally: + writer.close() + + blob = path.read_bytes() + assert blob.startswith(before) + # ASCII, so the appended record survives whichever reading is chosen. + assert blob[len(before) :].isascii() + for codec in ("cp1251", "utf-8"): + rows = [json.loads(x) for x in blob.decode(codec).splitlines() if x.strip()] + assert rows[-1]["a"] == "世界" + + +def test_a_damaged_line_in_an_ascii_shard_does_not_block_its_retry(tmp_path: Path) -> None: + """With no non-ASCII records to outvote it, one damaged line is still damage.""" + path = tmp_path / "out.jsonl" + path.write_bytes( + b'{"id": 1, "author": "alice"}\n' + + b'{"id": 99, "author": "bad \x96 byte"}\n' + + b'{"id": 2, "author": "bob"}\n' + ) + + writer = _load_state_store("cp1252").JsonlWriter(path) + try: + assert writer.has("id:1") and writer.has("id:2") + assert not writer.has("id:99") + assert writer.write({"id": 99, "author": "good byte"}) is True + finally: + writer.close() + + +def test_a_damaged_line_does_not_block_its_own_retry(tmp_path: Path) -> None: + """Its key comes from the codepage reading, which a UTF-8 shard did not pick.""" + path = tmp_path / "out.jsonl" + path.write_bytes( + json.dumps({"id": 1, "author": "Jürgen"}, ensure_ascii = False).encode() + + b"\n" + + b'{"id": 99, "author": "bad \x96 byte"}\n' + ) + + writer = _load_state_store("cp1252").JsonlWriter(path) + try: + assert writer.has("id:1") + assert not writer.has("id:99") + assert writer.write({"id": 99, "author": "good byte"}) is True + finally: + writer.close() + + +def test_one_damaged_byte_does_not_relabel_a_utf8_shard(tmp_path: Path) -> None: + """A complete JSON line with a stray 0x96 parses as cp1252, but is only one vote.""" + path = tmp_path / "out.jsonl" + healthy = ["Jürgen", "Grüße", "Björn"] + path.write_bytes( + json.dumps({"id": 0, "author": healthy[0]}, ensure_ascii = False).encode() + + b"\n" + + b'{"id": 99, "author": "bad \x96 byte"}\n' + + b"".join( + json.dumps({"id": i, "author": a}, ensure_ascii = False).encode() + b"\n" + for i, a in enumerate(healthy[1:], start = 1) + ) + ) + before = path.read_bytes() + + _load_state_store("cp1252").JsonlWriter(path).close() + + # Untouched, so the healthy records were never re-read as cp1252. + assert path.read_bytes() == before + rows = [] + for line in path.read_bytes().splitlines(): + try: + rows.append(json.loads(line.decode())) + except (UnicodeDecodeError, ValueError): + continue + assert [row["author"] for row in rows] == healthy + + +def test_a_torn_line_does_not_relabel_a_utf8_shard(tmp_path: Path) -> None: + """One interrupted append must not get the whole shard read as cp1252.""" + path = tmp_path / "out.jsonl" + good = [{"id": 1, "author": "Jürgen"}, {"id": 3, "author": "Grüße"}] + torn = '{"id": 2, "author": "Jürgen"}'.encode()[:-6] # cut mid-character + path.write_bytes( + json.dumps(good[0], ensure_ascii = False).encode() + + b"\n" + + torn + + b"\n" + + json.dumps(good[1], ensure_ascii = False).encode() + + b"\n" + ) + before = path.read_bytes() + + writer = _load_state_store("cp1252").JsonlWriter(path) + try: + assert writer.has("id:1") and writer.has("id:3") + assert not writer.has("id:2") # torn line yields no key + finally: + writer.close() + + # Untouched: no rewrite, so no record was re-encoded into mojibake. + after = path.read_bytes() + assert after.startswith(before) + assert "Jürgen".encode() in after + assert "Jürgen".encode("utf-8").decode("cp1252").encode() not in after + + +def test_an_undecodable_transport_marker_reads_as_unknown(tmp_path: Path) -> None: + """Pinning the decode turns an undecodable marker into UnicodeDecodeError, + which is a ValueError and so is not an OSError. Before the pin those bytes + simply read as an unknown value and the caller safely purged and restarted + the partial download; letting the error escape aborts the transfer instead. + """ + import sys + + backend = str(Path(__file__).resolve().parent.parent) + if backend not in sys.path: + sys.path.insert(0, backend) + from hub.utils import download_registry as registry + + marker = tmp_path / ".transport" + marker.write_bytes(b"\x80\xffnative\n") + assert registry._read_marker_value(marker) is None + # A readable but unknown value takes the same path (the behaviour restored). + marker.write_text("something-else\n", encoding = "utf-8") + assert registry._read_marker_value(marker) is None + + +def test_a_torn_cache_ref_reads_as_not_cached(tmp_path: Path, monkeypatch) -> None: + """hf_cache_snapshot_dir answers "is this model already on disk", and the + offline embedding checks turn a raise into a 500. A refs/main holding a byte + the codepage used to decode into a nonsense commit simply missed the snapshot + dir before the pin; it has to keep missing it.""" + import sys + + backend = str(Path(__file__).resolve().parent.parent) + if backend not in sys.path: + sys.path.insert(0, backend) + from utils import utils as backend_utils + + good_root = tmp_path / "good" + torn_root = tmp_path / "torn" + for root, ref_bytes in ((torn_root, b"\x80\xff\n"), (good_root, b"abc123\n")): + repo = root / "models--Org--Model" + (repo / "refs").mkdir(parents = True) + (repo / "refs" / "main").write_bytes(ref_bytes) + (good_root / "models--Org--Model" / "snapshots" / "abc123").mkdir(parents = True) + + monkeypatch.setattr(backend_utils, "_hf_cache_roots", lambda: [torn_root]) + assert backend_utils.hf_cache_snapshot_dir("Org/Model") is None + # The torn root is skipped, not fatal: a healthy second root still answers. + monkeypatch.setattr(backend_utils, "_hf_cache_roots", lambda: [torn_root, good_root]) + found = backend_utils.hf_cache_snapshot_dir("Org/Model") + assert found is not None and found.name == "abc123" + + +def test_a_corrupt_pid_file_does_not_abort_shutdown(tmp_path: Path, monkeypatch) -> None: + """_remove_pid_file runs first in _graceful_shutdown, so a raise there leaves + the inference, export, training and tunnel children alive.""" + import sys + + backend = str(Path(__file__).resolve().parent.parent) + if backend not in sys.path: + sys.path.insert(0, backend) + import run as studio_run + + pid_file = tmp_path / "studio.pid" + pid_file.write_bytes(b"\x80\xff") + monkeypatch.setattr(studio_run, "_PID_FILE", pid_file) + studio_run._remove_pid_file() + # Not this process's PID, so the file stays; the point is that it returned. + assert pid_file.exists() + + pid_file.write_text(str(os.getpid()), encoding = "utf-8") + studio_run._remove_pid_file() + assert not pid_file.exists() + + +def test_the_kwargs_guard_only_judges_dicts_that_reach_a_call(tmp_path: Path) -> None: + """Only a dict splatted into a call is subprocess configuration. An unrelated + payload that happens to carry "text": True is not, and neither is one whose + encoding is filled in on a later line.""" + cases = { + "offender.py": 'kw = {"text": True}\nrun(cmd, **kw)\n', + "annotated.py": 'kw: dict = {"universal_newlines": True}\nrun(cmd, **kw)\n', + "payload.py": 'payload = {"text": True}\nrequests.post(url, json = payload)\n', + "inline.py": 'run(cmd, **{"text": True})\n', + "later.py": 'kw = {"text": True}\nkw["encoding"] = "utf-8"\nrun(cmd, **kw)\n', + "carried.py": 'kw = {"text": True, "encoding": "utf-8"}\nrun(cmd, **kw)\n', + } + flagged = set() + for name, source in cases.items(): + path = tmp_path / name + path.write_text(source, encoding = "utf-8") + if any("subprocess kwargs" in line for line in _offenders(path)): + flagged.add(name) + assert flagged == {"offender.py", "annotated.py", "inline.py"}, flagged + + +def test_the_guard_follows_subprocess_through_an_alias(tmp_path: Path) -> None: + """install_wheel() takes ``run = subprocess.run`` and calls it as a bare + name, so an attribute-only match let both of its installer calls drop their + encoding unnoticed. A name bound to something else is still not subprocess.""" + cases = { + "param_default.py": ( + "import subprocess\n" + "def install(*, run = subprocess.run):\n" + " run(cmd, text = True)\n" + ), + "assigned.py": "import subprocess\n_run = subprocess.run\n_run(cmd, text = True)\n", + "imported.py": "from subprocess import check_output\ncheck_output(cmd, text = True)\n", + "renamed.py": "from subprocess import run as _r\n_r(cmd, universal_newlines = True)\n", + "encoded.py": ( + "import subprocess\n" + "def install(*, run = subprocess.run):\n" + ' run(cmd, text = True, encoding = "utf-8")\n' + ), + "unrelated.py": "def run(cmd, text = False):\n pass\nrun(cmd, text = True)\n", + } + flagged = set() + for name, source in cases.items(): + path = tmp_path / name + path.write_text(source, encoding = "utf-8") + if any("subprocess(text = True)" in line for line in _offenders(path)): + flagged.add(name) + assert flagged == {"param_default.py", "assigned.py", "imported.py", "renamed.py"}, flagged + + +def test_the_guard_sees_os_fdopen(tmp_path: Path) -> None: + """os.fdopen(fd, mode) is open() on a descriptor and takes the same locale + default in text mode, so leaving it out let the swap lock file keep the + codepage on the write side while its reader was pinned to UTF-8.""" + cases = { + "text.py": 'import os\nos.fdopen(fd, "w")\n', + "default_mode.py": "import os\nos.fdopen(fd)\n", # defaults to "r", still text + "binary.py": 'import os\nos.fdopen(fd, "wb")\n', + "keyword.py": 'import os\nos.fdopen(fd, "w", encoding = "utf-8")\n', + "positional.py": 'import os\nos.fdopen(fd, "w", 1, "utf-8")\n', + } + flagged = set() + for name, source in cases.items(): + path = tmp_path / name + path.write_text(source, encoding = "utf-8") + if any("fdopen" in line for line in _offenders(path)): + flagged.add(name) + assert flagged == {"text.py", "default_mode.py"}, flagged + + +def test_an_undecodable_bootstrap_password_does_not_stop_startup( + tmp_path: Path, monkeypatch +) -> None: + """ensure_default_admin calls _load_bootstrap_password for every existing + admin and the lifespan calls that with no handler, so a raise here takes the + whole backend down instead of ignoring an unusable file.""" + import sys + + backend = str(Path(__file__).resolve().parent.parent) + if backend not in sys.path: + sys.path.insert(0, backend) + from auth import storage + + pw_file = tmp_path / ".bootstrap_password" + pw_file.write_bytes(b"\x80\xffnot-utf8\n") + monkeypatch.setattr(storage, "_BOOTSTRAP_PW_PATH", pw_file) + assert storage._load_bootstrap_password() is None + + # A readable one still loads, so this is a narrowing of failure, not of function. + pw_file.write_text("correct horse battery staple\n", encoding = "utf-8") + assert storage._load_bootstrap_password() == "correct horse battery staple" + + +def test_a_damaged_checkpoint_resets_instead_of_resuming_on_a_broken_cursor(tmp_path: Path) -> None: + """A checkpoint holds only base64 cursors and booleans, so a codepage reading + can only ever add non-ASCII, never recover any. Resuming on a mojibaked cursor + sends GitHub one it answers with INVALID_CURSOR_ARGUMENTS, and the empty page + that comes back marks the stream done and skips the rest of it for good. + Dropping the checkpoint only replays pages the writers already dedup.""" + module = _load_state_store("cp1252") + cursor = "Y3Vyc29yOnYyOpK0MjAxMi0wMi0xNlQwNjo1Mzo0MVrOADGL_A==" + healthy = json.dumps({"issues_cursor": cursor, "issues_done": False}, indent = 2) + path = tmp_path / "octocat__Hello-World.json" + + path.write_text(healthy, encoding = "utf-8") + assert module.StateStore(path).get("issues_cursor") == cursor + + # Written by a pre-UTF-8 release in the operator's codepage. Nothing is lost + # by reading UTF-8 only, because an all-ASCII document is the same bytes. + path.write_bytes(healthy.encode("cp1252")) + assert module.StateStore(path).get("issues_cursor") == cursor + + # One damaged byte inside the cursor: still a whole JSON document under a + # single-byte codepage, so only refusing that reading resets the checkpoint. + raw = healthy.encode() + at = raw.index(b"MjAxMi0wMi0xNlQ") + 3 + path.write_bytes(raw[:at] + b"\x96" + raw[at + 1 :]) + assert json.loads(path.read_bytes().decode("latin-1"))["issues_cursor"] != cursor + store = module.StateStore(path) + assert store.all() == {} + assert store.get("issues_cursor") is None + + +def test_a_utf8_record_is_not_parsed_a_second_time(tmp_path: Path) -> None: + """These shards reach gigabytes and every resume reads all of one, so a + record that already read as UTF-8 must not be decoded and parsed again under + the codepage. The legacy reading exists only to recover keys UTF-8 could not.""" + module = _load_state_store("cp1252") + calls: list[str] = [] + real_parse = module._parse + + def counting_parse(raw, encoding): + calls.append(encoding) + return real_parse(raw, encoding) + + module._parse = counting_parse + try: + healthy = json.dumps({"id": 1, "author": "Jürgen"}).encode("utf-8") + reading = module._read_line(healthy, "cp1252") + assert reading.as_utf8 == {"id": 1, "author": "Jürgen"} + assert calls == ["utf-8"], calls + + # A line UTF-8 cannot read still falls through to the codepage, the whole point. + calls.clear() + legacy = json.dumps({"id": 2, "author": "Jürgen"}, ensure_ascii = False).encode("cp1252") + reading = module._read_line(legacy, "cp1252") + assert reading.as_utf8 is None + assert reading.as_legacy == {"id": 2, "author": "Jürgen"} + assert calls == ["utf-8", "cp1252"], calls + finally: + module._parse = real_parse + + +def _too_deeply_nested_json() -> str: + """A JSON document nested past what this interpreter will descend into. + + Probed rather than hardcoded: the depth json.loads gives up at is bounded by + sys.getrecursionlimit() up to 3.11 and by the C recursion limit from 3.12, + which sys.setrecursionlimit no longer moves and which varies by micro + version. That is ~995 on 3.9 and ~9999 on 3.13. + """ + depth = 1 + while depth <= 1 << 17: + document = "[" * depth + "]" * depth + try: + json.loads(document) + except RecursionError: + return document + depth *= 2 + pytest.skip("this interpreter parses arbitrarily nested JSON") + + +def test_an_unparseably_nested_document_is_discarded_not_raised(tmp_path: Path) -> None: + """json.loads answers nesting it cannot descend with RecursionError, which is + a RuntimeError and so is neither a ValueError nor a UnicodeDecodeError. + _parse is called outside any other handler in both StateStore.__init__ and + JsonlWriter._scan_existing, so letting it escape aborts the scraper at + startup on a file the catch-all it replaced simply discarded.""" + module = _load_state_store("cp1252") + nested = _too_deeply_nested_json() + + checkpoint = tmp_path / "octocat__Hello-World.json" + checkpoint.write_text(nested, encoding = "utf-8") + assert module.StateStore(checkpoint).all() == {} # reset, not raised + + shard = tmp_path / "out.jsonl" + shard.write_text( + nested + "\n" + json.dumps({"id": 1}) + "\n" + json.dumps({"id": 2}) + "\n", + encoding = "utf-8", + ) + writer = module.JsonlWriter(shard) + try: + # Skipped like any other unreadable line, so its neighbours still yield the dedup + # keys that keep the resume from re-fetching them. + assert writer.has("id:1") and writer.has("id:2") + finally: + writer.close() diff --git a/studio/backend/tests/test_training_worker_flash_attn.py b/studio/backend/tests/test_training_worker_flash_attn.py index 86511987b1..d136821ea2 100644 --- a/studio/backend/tests/test_training_worker_flash_attn.py +++ b/studio/backend/tests/test_training_worker_flash_attn.py @@ -9,8 +9,28 @@ import sys from typing import Any from unittest import mock +import pytest + from core.training import worker +# The runtime install is Linux-only, so elsewhere these return before any status. +linux_only = pytest.mark.skipif( + not sys.platform.startswith("linux"), + reason = "the runtime flash-attn install is gated to Linux", +) + +# causal-conv1d and flash-linear-attention are NOT Linux-gated: both installers bail out +# on `sys.platform == "win32"` alone (no prebuilt wheel for Windows) and run everywhere +# else, macOS included. linux_only here would skip cases that legitimately pass off Linux. +not_on_windows = pytest.mark.skipif( + sys.platform == "win32", + reason = ( + "mirrors the sys.platform == 'win32' bail-out in " + "_ensure_flash_linear_attention_unconditional and " + "_ensure_causal_conv1d_fast_path" + ), +) + def _missing_flash_attn_import(): real_import = builtins.__import__ @@ -55,6 +75,7 @@ def test_should_try_runtime_flash_attn_install_threshold_and_skip(monkeypatch): assert worker._should_try_runtime_flash_attn_install(32768) is False +@linux_only def test_runtime_flash_attn_prefers_prebuilt_wheel(monkeypatch): statuses: list[str] = [] @@ -82,6 +103,7 @@ def test_runtime_flash_attn_prefers_prebuilt_wheel(monkeypatch): assert statuses == ["Installing flash-attn for faster training..."] +@linux_only def test_runtime_flash_attn_falls_back_to_pypi(monkeypatch): calls: list[list[str]] = [] statuses: list[str] = [] @@ -113,12 +135,7 @@ def test_runtime_flash_attn_falls_back_to_pypi(monkeypatch): ) monkeypatch.setattr(worker, "install_wheel", mock.Mock()) - def fake_run( - cmd, - stdout = None, - stderr = None, - text = None, - ): + def fake_run(cmd, **kwargs): calls.append(list(cmd)) return subprocess.CompletedProcess(cmd, 0, "") @@ -139,6 +156,7 @@ def test_runtime_flash_attn_skip_env_avoids_all_install_work(monkeypatch): worker._sp.run.assert_not_called() +@not_on_windows def test_causal_conv1d_fast_path_preserves_wheel_first_install_args(monkeypatch): install_mock = mock.Mock(return_value = True) monkeypatch.setattr(worker, "_install_package_wheel_first", install_mock) @@ -160,6 +178,7 @@ def test_causal_conv1d_fast_path_preserves_wheel_first_install_args(monkeypatch) ) +@not_on_windows def test_causal_conv1d_fast_path_includes_qwen3_6_variants(monkeypatch): install_mock = mock.Mock(return_value = True) monkeypatch.setattr(worker, "_install_package_wheel_first", install_mock) @@ -225,6 +244,7 @@ def _pin_fla_model_types(monkeypatch): ) +@not_on_windows def test_flash_linear_attention_installs_pinned_pair_for_qwen3_5(monkeypatch): _pin_fla_model_types(monkeypatch) monkeypatch.setattr(worker.shutil, "which", lambda name: "/usr/bin/uv") @@ -277,6 +297,7 @@ def test_flash_linear_attention_skips_for_ssm_only_models(monkeypatch): run_mock.assert_not_called() +@not_on_windows def test_flash_linear_attention_matches_full_qwen3_family(monkeypatch): monkeypatch.setattr(worker.shutil, "which", lambda name: "/usr/bin/uv") run_mock = mock.Mock(return_value = mock.Mock(returncode = 0, stdout = "")) @@ -331,6 +352,7 @@ def test_flash_linear_attention_skipped_via_env(monkeypatch): run_mock.assert_not_called() +@not_on_windows def test_flash_linear_attention_skipped_below_torch_2_7(monkeypatch): _pin_fla_model_types(monkeypatch) monkeypatch.delenv(worker._FLA_SKIP_ENV, raising = False) @@ -349,6 +371,7 @@ def test_flash_linear_attention_skipped_below_torch_2_7(monkeypatch): assert any("torch>=" in s for s in statuses) +@not_on_windows def test_flash_linear_attention_install_includes_einops(monkeypatch): _pin_fla_model_types(monkeypatch) monkeypatch.delenv(worker._FLA_SKIP_ENV, raising = False) @@ -375,6 +398,7 @@ def test_flash_linear_attention_install_includes_einops(monkeypatch): assert f"fla-core=={worker._FLA_CORE_PACKAGE_VERSION}" in args +@not_on_windows def test_flash_linear_attention_logs_post_install_import_failure(monkeypatch): """pip exits 0 but `import fla.modules` still fails (missing transitive).""" _pin_fla_model_types(monkeypatch) @@ -421,6 +445,7 @@ def test_tilelang_backend_skipped_on_unsupported_linux_arch(monkeypatch): run_mock.assert_not_called() +@linux_only def test_tilelang_backend_pins_only_binary(monkeypatch): _pin_fla_model_types(monkeypatch) monkeypatch.delenv(worker._TILELANG_SKIP_ENV, raising = False) @@ -462,6 +487,7 @@ def _force_missing_tilelang_imports(monkeypatch): monkeypatch.setattr(builtins, "__import__", fake_import) +@linux_only def test_tilelang_backend_installs_pinned_pair_for_qwen3_5(monkeypatch): _pin_fla_model_types(monkeypatch) monkeypatch.delenv(worker._TILELANG_SKIP_ENV, raising = False) @@ -486,6 +512,7 @@ def test_tilelang_backend_installs_pinned_pair_for_qwen3_5(monkeypatch): assert any("Installing TileLang" in s for s in statuses) +@linux_only def test_tilelang_backend_reinstalls_when_tvm_ffi_is_broken(monkeypatch): """Repair path issues TWO pip calls: @@ -555,6 +582,7 @@ def test_tilelang_backend_skipped_on_windows(monkeypatch): run_mock.assert_not_called() +@linux_only def test_tilelang_backend_swallows_install_timeout(monkeypatch): _pin_fla_model_types(monkeypatch) monkeypatch.delenv(worker._TILELANG_SKIP_ENV, raising = False) @@ -609,6 +637,7 @@ def test_tilelang_backend_skipped_via_env(monkeypatch): run_mock.assert_not_called() +@linux_only def test_tilelang_backend_swallows_install_failure(monkeypatch): _pin_fla_model_types(monkeypatch) monkeypatch.delenv(worker._TILELANG_SKIP_ENV, raising = False) @@ -673,6 +702,7 @@ def _patch_iu_gates(monkeypatch, fla_gate, conv_gate): monkeypatch.setattr(_iu, "is_causal_conv1d_available", conv_gate) +@not_on_windows def test_hook_installs_when_gate_returns_false(monkeypatch): _pin_fla_model_types(monkeypatch) fla_gate = _make_fake_gate(initial_return = False) @@ -976,6 +1006,7 @@ def test_hook_does_install_tilelang_for_qwen35(monkeypatch): tile_install.assert_called_once() +@linux_only def test_tilelang_repair_does_not_touch_torch_cuda_stack(monkeypatch): """Finding #2: the broken-tvm-ffi repair must use --no-deps on the forced step so --force-reinstall doesn't cascade through @@ -1119,6 +1150,7 @@ def test_hook_runs_tilelang_repair_when_fla_already_true(monkeypatch): tile_install.assert_called_once() +@not_on_windows def test_fla_installer_force_reinstalls_when_older_version_present(monkeypatch): """Finding #8: an older `flash-linear-attention` that is importable but below the pin must force a reinstall (not no-op). @@ -1583,15 +1615,10 @@ def test_install_respects_user_gcc_install_dir(monkeypatch): ) _make_hip_install_env(monkeypatch, gcc_dir = "/usr/lib/gcc/x86_64-linux-gnu/13") - captured: dict[str, str] | None = {"_called": "no"} + captured: dict[str, str] = {} def fake_run(cmd, **kwargs): - env = kwargs.get("env") - if env is not None: - captured.clear() - captured.update(env) - else: - captured["_called"] = "yes_no_env" + captured.update(kwargs.get("env") or {}) return subprocess.CompletedProcess(cmd, 0, "") monkeypatch.setattr(worker._sp, "run", fake_run) @@ -1607,14 +1634,11 @@ def test_install_respects_user_gcc_install_dir(monkeypatch): release_base_url = "https://example.com", ) - # subprocess.run invoked without env override (user already set - # HIPCC_COMPILE_FLAGS_APPEND with --gcc-install-dir, so we left the - # env alone — the existing value is inherited). - assert captured == {"_called": "yes_no_env"} + assert captured["HIPCC_COMPILE_FLAGS_APPEND"] == "--gcc-install-dir=/opt/custom/gcc-13" def test_install_does_not_inject_env_on_cuda(monkeypatch): - """CUDA path (no hip_version in env) → no env override at all.""" + """CUDA path (no hip_version in env) → no HIP flag injected.""" monkeypatch.delenv("HIPCC_COMPILE_FLAGS_APPEND", raising = False) monkeypatch.setattr(builtins, "__import__", _missing_module_import("causal_conv1d")) monkeypatch.setattr( @@ -1641,7 +1665,7 @@ def test_install_does_not_inject_env_on_cuda(monkeypatch): captured: dict[str, Any] = {} def fake_run(cmd, **kwargs): - captured["env_in_kwargs"] = "env" in kwargs + captured.update(kwargs.get("env") or {}) return subprocess.CompletedProcess(cmd, 0, "") monkeypatch.setattr(worker._sp, "run", fake_run) @@ -1657,5 +1681,5 @@ def test_install_does_not_inject_env_on_cuda(monkeypatch): release_base_url = "https://example.com", ) - # CUDA branch never sets the env, never invokes the gcc helper. - assert captured.get("env_in_kwargs") is False + # env is always passed (to force UTF-8), but never the HIP flag. + assert "HIPCC_COMPILE_FLAGS_APPEND" not in captured diff --git a/studio/backend/utils/changelog.py b/studio/backend/utils/changelog.py new file mode 100644 index 0000000000..84cd54df05 --- /dev/null +++ b/studio/backend/utils/changelog.py @@ -0,0 +1,1056 @@ +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 + +"""Release notes for the update popup, sourced from CHANGELOG.md. + +Notes are keyed to one exact version: the popup asks for the version it is +offering and gets that section or nothing, so an older release's notes can +never appear next to a newer update. + +The remote copy on the default branch wins over the bundled one, since the +offered version is newer than the installed checkout. Both reads are lazy, +cached and skipped when update checks are off. +""" + +from __future__ import annotations + +import os +import re +import threading +import time +import urllib.request +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +from packaging.version import InvalidVersion, Version + +from .update_status import DISABLE_ENV_VAR, RELEASE_NOTES_URL + +CHANGELOG_FILENAME = "CHANGELOG.md" +CHANGELOG_RAW_URL = "https://raw.githubusercontent.com/unslothai/unsloth/main/CHANGELOG.md" +CHANGELOG_URL_ENV_VAR = "UNSLOTH_CHANGELOG_URL" +CHANGELOG_PATH_ENV_VAR = "UNSLOTH_CHANGELOG_PATH" +CHANGELOG_TIMEOUT_SECONDS = 3 +CHANGELOG_MAX_BYTES = 2 * 1024 * 1024 +_CHANGELOG_CHUNK_BYTES = 64 * 1024 +_CHANGELOG_MIN_READ_SECONDS = 0.05 +CHANGELOG_SUCCESS_TTL_SECONDS = 30 * 60 +CHANGELOG_FAILURE_TTL_SECONDS = 5 * 60 +RELEASE_NOTES_MAX_CHARS = 20_000 + +# CommonMark requires a space, tab or line end after the hashes: a non-breaking +# space copied from rich text renders as text, not a heading, but a bare `##` is +# an empty heading and still ends the release above. +_HEADING_PATTERN = re.compile(r"^ {0,3}##(?:[ \t]+(?P.*?))?[ \t]*$") +_FENCE_PATTERN = re.compile(r"^ {0,3}(?P<marker>`{3,}|~{3,})(?P<rest>.*)$") +# CommonMark type 1 HTML blocks: contents are literal until a closing tag, +# which the spec says need not be the one that opened the block. +_RAW_HTML_OPEN = re.compile(r"^ {0,3}<(pre|script|style|textarea)(?=[\s>]|$)", re.IGNORECASE) +_RAW_HTML_CLOSE = re.compile(r"</(pre|script|style|textarea)\s*>", re.IGNORECASE) +# Types 3 to 5 (processing instructions, declarations, CDATA) are literal too, +# each ending on its own delimiter. Comments open mid-line, so are separate. +_RAW_BLOCKS = ( + (_RAW_HTML_OPEN, _RAW_HTML_CLOSE), + (re.compile(r"^ {0,3}<\?"), re.compile(r"\?>")), + (re.compile(r"^ {0,3}<!\[CDATA\["), re.compile(r"\]\]>")), + # A declaration needs an uppercase letter, so `<!note` stays ordinary text. + (re.compile(r"^ {0,3}<![A-Z]"), re.compile(r">")), +) +# Type 6 blocks run to the next blank line, so `<details>` only holds Markdown +# once a blank line has closed the block. Open and close tags both start one. +_HTML_BLOCK_OPEN = re.compile(r"^ {0,3}</?([a-zA-Z][a-zA-Z0-9-]*)(?=[\s/>]|$)") +# Blocks that break into an open paragraph, so none is open after them and one +# they are written below is closed rather than continued. +_INTERRUPTS = re.compile( + r"^ {0,3}(?:#{1,6}([ \t]|$)|(?:\*[ \t]*){3,}$|(?:-[ \t]*){3,}$|(?:_[ \t]*){3,}$)" +) +# A definition is a block of its own but may not interrupt a paragraph, so it +# ends the one above it only when there is none to continue. +_LINK_DEFINITION = re.compile(r"^ {0,3}\[(?:[^\[\]\\]|\\.)+\]:") +# Blocks that are not paragraph text, so a following underline is not setext. +_PARAGRAPH_TEXT = re.compile(r"^ {0,3}(?![-*+>]([ \t]|$)|\d{1,9}[.)]([ \t]|$))\S") +# A line of = or - under a paragraph line makes that line a heading. +_SETEXT_UNDERLINE = re.compile(r"^ {0,3}(=+|-+)[ \t]*$") +# A quoted paragraph continues on unmarked lines, which belong to the quote. +_BLOCK_QUOTE = re.compile(r"^ {0,3}>") +_QUOTE_MARKER = re.compile(r"^ {0,3}>[ \t]?") +# A heading at an item's content column belongs to that item, not the document. +# The marker needs whitespace after it, so `2.0` is a version, not an item. +_LIST_ITEM = re.compile(r"^[ \t]*(?P<marker>[-*+]|\d{1,9}[.)])(?P<space>[ \t]+|$)") +_THEMATIC_BREAK = re.compile(r"^ {0,3}(?:(?:\*[ \t]*){3,}|(?:-[ \t]*){3,}|(?:_[ \t]*){3,})$") +# Content indented more than this after a marker is an indented code block, so +# the item's content starts one column past the marker instead. +_MAX_ITEM_PADDING = 4 +_HTML_BLOCK_TAGS = frozenset( + """ +address article aside base basefont blockquote body caption center col colgroup +dd details dialog dir div dl dt fieldset figcaption figure footer form frame +frameset h1 h2 h3 h4 h5 h6 head header hr html iframe legend li link main menu +menuitem nav noframes ol optgroup option p param search section summary table +tbody td tfoot th thead title tr track ul +""".split() +) +# Type 7: any other complete tag alone on a line. It cannot interrupt a +# paragraph, so it only counts after a break. +_HTML_ATTRIBUTE = ( + r"""(?:\s+[a-zA-Z_:][a-zA-Z0-9_.:-]*(?:\s*=\s*(?:[^\s"'=<>`]+|'[^']*'|"[^"]*"))?)""" +) +_HTML_TAG_ONLY_LINE = re.compile( + rf"^ {{0,3}}(?:<[a-zA-Z][a-zA-Z0-9-]*{_HTML_ATTRIBUTE}*\s*/?>|</[a-zA-Z][a-zA-Z0-9-]*\s*>)\s*$" +) +# Levels above studio/ are the repo root in a checkout and site-packages in an +# install, so they are searched only when one of these markers is present. +_CHECKOUT_ONLY_LEVELS = (3, 4) +_CHECKOUT_MARKERS = ("pyproject.toml", ".git") +_COMMENT_BLOCK_OPEN = re.compile(r"^ {0,3}<!--") +_COMMENT_OPEN = "<!--" +_COMMENT_CLOSE = "-->" +# Stands in for a line the renderer hides. `#` is a block of its own, so list +# tracking reads it like a comment: never a marker, never a lazy continuation. +_HIDDEN_BLOCK = "#" +_VERSION_TOKEN_PATTERN = re.compile(r"^[\[(]?v?(?P<version>[0-9][0-9A-Za-z.!+-]*?)[\])]?$") +_SAFE_VERSION_PATTERN = re.compile(r"^[0-9A-Za-z][0-9A-Za-z.!+-]{0,63}$") + + +@dataclass(frozen = True) +class _ListState: + """The open list items, innermost last, by the column their content starts.""" + + columns: tuple[int, ...] = () + # True while the innermost item has had no content since its marker. + empty_item: bool = False + + +@dataclass(frozen = True) +class ChangelogEntry: + """One `## <version>` section of the changelog.""" + + version: str + heading: str + body: str + + +@dataclass(frozen = True) +class ChangelogSource: + text: str | None + source: str | None + error: str | None = None + + +@dataclass +class _ChangelogCacheEntry: + source: ChangelogSource + expires_at: float + + +_cache_condition = threading.Condition() +_remote_cache: _ChangelogCacheEntry | None = None +_remote_fetching = False + + +def reset_changelog_cache() -> None: + """Clear the in-process changelog cache. Intended for tests.""" + global _remote_cache, _remote_fetching + with _cache_condition: + _remote_cache = None + _remote_fetching = False + _cache_condition.notify_all() + + +def is_supported_version_query(version: str) -> bool: + """Whether `version` is shaped like something we can look up at all. + + Sections are indexed only when their version parses, so a query that does + not parse (`latest`, `main`) can never match and is rejected outright.""" + candidate = version.strip() + if not _SAFE_VERSION_PATTERN.match(candidate): + return False + return _parse_version(candidate) is not None + + +def _markdown_lines(text: str) -> list[str]: + """``text`` split the way CommonMark ends lines. + + str.splitlines also breaks on U+2028, U+2029, NEL, vertical tab and form + feed, none of which end a line in Markdown. A separator sitting in prose + before "## 9.9.9" would otherwise index a release the renderer never shows + and truncate the notes above it. + """ + return text.replace("\r\n", "\n").replace("\r", "\n").split("\n") + + +def parse_changelog(text: str) -> list[ChangelogEntry]: + """Parse `## <version>` sections, in file order. + + Headings whose first token is not a version (`## Unreleased`, `## Format`) + end the previous section but are not indexed. + """ + # A Windows editor can leave a BOM on the first line, hiding a heading. + text = text.lstrip("") + entries: list[ChangelogEntry] = [] + heading: str | None = None + version: str | None = None + body: list[str] = [] + open_fence: str | None = None + # Content column of the list item the open block belongs to, 0 at document + # level. A fence and an HTML block are scoped to their container, so the + # item's end closes them. Only one of the three is ever open. + block_column = 0 + in_comment = False + in_raw_html: int | None = None + in_html_block = False + after_paragraph = False + paragraph: list[str] = [] + in_quote = False + quoted = False + lists = _ListState() + + def flush() -> None: + if version is not None and heading is not None: + entries.append( + ChangelogEntry( + version = version, + heading = heading, + body = "\n".join(body).strip(), + ) + ) + + for line in _markdown_lines(text): + # The line as list tracking sees it: blank wherever nothing renders. + structural = "" + opened_block = False + in_block = open_fence is not None or in_html_block or in_raw_html is not None or in_comment + # A fence, comment or HTML block inside a list item runs only to the end + # of that item, so a line dedented out of the item closes both. Lazy + # continuation reaches into none of them. A raw block or comment inside an + # item also ends on a blank line: the item takes the break, so what + # follows is a block of the item's own. + leaves = ( + _indent_width(line) < block_column + if line.strip() + else in_raw_html is not None or in_comment + ) + if in_block and block_column and leaves: + open_fence = None + in_html_block = False + in_raw_html = None + in_comment = False + block_column = 0 + # The paragraph the line could have continued is block content, so + # it closes the item rather than reading as more of it. + after_paragraph = False + # A fence written as a list item's first content opens inside that item, so + # an opener is read past a marker on the same line. Only an opener: fenced + # content is literal and a closer carries no marker. + fence_line = line if open_fence else _item_content(line, after_paragraph) + # Raw HTML first: its contents are literal, so a fence in it is not one. + if in_raw_html is not None: + visible, in_raw_html = _strip_raw_html(line, in_raw_html) + elif in_html_block: + # A blank line is the only thing that ends a type 6 block. + in_html_block = line.strip() != "" + visible = "" + elif (fence := _FENCE_PATTERN.match(fence_line)) and not in_comment: + was_open = open_fence + open_fence = _next_fence_state(open_fence, fence.group("marker"), fence.group("rest")) + opened_block = was_open is None and open_fence is not None + # Hidden from heading matching, but its indent still closes items. + visible = "" + structural = line + elif open_fence: + visible = "" + else: + # A block already open owns this line, so it is content rather than a + # block written at the column it happens to start in. + hidden = in_comment or in_raw_html is not None + # A comment is an HTML block too, so one written as a list item's first + # content opens inside it exactly as a fence does: the opener is read + # past a marker on the same line. + block_open = ( + not in_comment + and _COMMENT_BLOCK_OPEN.match(_item_content(line, after_paragraph)) is not None + ) + # Commented-out sections are not rendered, so they are not releases. + visible, in_comment = _strip_comments(line, in_comment, block_open) + # An HTML block written as a list item's first content opens inside + # that item, as a fence does, so an opener is read past a marker on the + # same line. The marker stays, so its item is still tracked. A comment + # blanks its own line, so that line is read as written: the block + # renders as nothing, but the item it is content of still opens. + source = line if block_open else visible + content = _item_content(source, after_paragraph) + marker = source[: len(source) - len(content)] + # Nor is anything inside a raw HTML block such as <pre>. + stripped, in_raw_html = _strip_raw_html(content, in_raw_html) + opened_block = in_raw_html is not None or (block_open and in_comment) + # Taken before the opener is hidden: it renders as nothing, but its + # indent still closes a list item it sits left of, and a marker on its + # line still opens one. A comment or raw block keeps only those, since + # the text it hides is not Markdown and must open no list. + if block_open or stripped != content: + if not hidden: + structural = _hidden_structure(line, marker) + visible = "" + else: + visible = marker + stripped + if visible.strip(): + structural = visible + elif not hidden: + structural = _hidden_structure(line) + if stripped and _opens_html_block(stripped, after_paragraph): + in_html_block = True + opened_block = True + visible = "" + # A `##` inside a fenced block is sample markdown, not a real heading. + match = _HEADING_PATTERN.match(visible) if visible else None + # `1.0` over a line of dashes is the same heading written setext style. + setext = ( + after_paragraph + and match is None + and paragraph != [] + and _SETEXT_UNDERLINE.match(visible) is not None + and (visible.strip()[:1] == "-") + # Never a boundary inside a list item: dedented the dashes are a + # thematic break, and at the content column the heading is nested. + and not lists.columns + ) + if setext: + if version is not None: + # The whole paragraph is the heading, read as body on arrival. + del body[len(body) - len(paragraph) :] + flush() + # A wrapped heading keeps every line, so token one is the version. + heading = "\n".join(paragraph) + version = _version_from_heading(heading) + body = [] + paragraph = [] + after_paragraph = False + continue + # A dashed underline is not a list marker, so track lists after setext. + lazy_marker = _lazy_marker(structural, lists, after_paragraph, quoted) + lists = _open_lists(structural, lists, after_paragraph, quoted) + # Taken after the opening line closed the items it is dedented out of, + # so the block belongs to the item it is really written inside. + if opened_block: + block_column = lists.columns[-1] if lists.columns else 0 + elif open_fence is None and not in_html_block and in_raw_html is None and not in_comment: + block_column = 0 + # At an open item's content column a heading is nested, not a boundary. + if lists.columns and _indent_width(visible) >= lists.columns[0]: + match = None + # The line at its own nesting level: past the container's indentation + # and past a marker on the same line, so `- ## 2.0` reads as a heading. + column = lists.columns[-1] if lists.columns else 0 + content = _strip_indent(visible, column) + if (item := _LIST_ITEM.match(content)) is not None: + content = content[item.end() :] + # Only ordinary text continues a paragraph. Indented code counts four + # spaces past the container, so an item's own indent does not count. + indented_code = not after_paragraph and _indent_width(visible) - column >= 4 + # An underline ends the paragraph it underlines, so it needs one open in + # its own container: the quote above owns its own, and a row left of an + # open item is lazy text of the item's paragraph. Three dashes are a + # thematic break either way, which `_INTERRUPTS` already ends on. + underline = ( + _SETEXT_UNDERLINE.match(visible) is not None + and after_paragraph + and not quoted + and _indent_width(visible) >= column + ) + after_paragraph = ( + # Read inside its container, so an empty item and a fence written as an + # item's own content leave no paragraph open below them. A marker the + # paragraph above swallows is its text, not an item. + (bool(content.strip()) or lazy_marker) + and match is None + and _HEADING_PATTERN.match(content) is None + and _FENCE_PATTERN.match(content) is None + and not indented_code + and _INTERRUPTS.match(visible) is None + and (after_paragraph or _LINK_DEFINITION.match(visible) is None) + and not underline + ) + # A quote's paragraph runs on over plain text and owns every line of it. + # An empty quote holds none, so the line below starts the document's. + flush_left = visible.lstrip(" \t") + quote_line = _BLOCK_QUOTE.match(visible) is not None + in_quote = ( + _may_be_lazy(_quote_content(visible)) + if quote_line + else in_quote and _continues_paragraph(visible, column) + ) + if quote_line: + # The only paragraph a quote line leaves open is the quote's own, + # and a quote holding a heading or nothing at all leaves none. + after_paragraph = in_quote + # Whose paragraph the line below would continue. A quote owns the one its + # own lines hold, so a marker outside the quote is a block of its own + # rather than more of the text above it. + quoted = quote_line or in_quote + # The lines a later underline turns into one heading. A paragraph opens + # only on plain text and then runs on until something interrupts it. + continues = ( + not _interrupts_paragraph(flush_left) + if paragraph + else _PARAGRAPH_TEXT.match(flush_left) is not None + ) + # A paragraph inside an open item is that item's, and only one written + # at document level can be the heading a later underline makes of it. + if after_paragraph and not in_quote and not lists.columns and continues: + paragraph = [*paragraph, visible.strip()] + else: + paragraph = [] + if match is None: + if version is not None: + body.append(line) + continue + + flush() + # An empty heading has no title, so it ends the release above without + # indexing one: `_version_from_heading` finds no version and `flush` skips. + heading = match.group("title") or "" + version = _version_from_heading(heading) + body = [] + + flush() + return entries + + +def find_release_notes(text: str, version: str) -> ChangelogEntry | None: + """Return the section for exactly `version`, or None. + + Equality is version-aware (`2026.07.5` matches `2026.7.5`) but never fuzzy: + a near-miss returns None so the caller shows no notes, not the wrong ones. + """ + entries = parse_changelog(text) + for entry in entries: + # An exact heading wins, so `## 1.0` is never shadowed by `## 1.0.0`. + if entry.version == version: + return entry + + wanted = _parse_version(version) + for entry in entries: + if wanted is not None: + candidate = _parse_version(entry.version) + if candidate is not None and candidate == wanted: + return entry + return None + + +def get_release_notes(version: str, refresh: bool = False) -> dict[str, Any]: + """Return release notes for exactly `version` for the update popup. + + `refresh` retries a cached remote failure, so the UI's retry action is not + stuck behind the failure TTL once connectivity returns. + """ + version = version.strip() + if not is_supported_version_query(version): + return _notes_response(version = version, error = "Unsupported version.") + + local = _read_local_changelog() + remote = ChangelogSource(text = None, source = None) + if os.environ.get(DISABLE_ENV_VAR) != "1": + remote = get_remote_changelog(refresh = refresh) + + # Remote first: the offered version is newer than the local copy. + for candidate in (remote, local): + if not candidate.text: + continue + entry = find_release_notes(candidate.text, version) + if entry is not None: + return _notes_response( + version = version, + markdown = entry.body, + heading = entry.heading, + source = candidate.source, + ) + + # Nothing matched: the bundled copy cannot know a version newer than the + # install, so report a remote failure and let the UI offer a retry. + return _notes_response(version = version, error = remote.error) + + +def get_remote_changelog(refresh: bool = False) -> ChangelogSource: + """Fetch CHANGELOG.md from the repo using a small in-process TTL cache.""" + global _remote_cache, _remote_fetching + + if refresh: + # Only a cached failure is dropped, so retries cannot hammer the remote. + with _cache_condition: + if _remote_cache and _remote_cache.source.text is None: + _remote_cache = None + + # A caller waits for an in-flight fetch only as long as it may take, then + # answers locally rather than holding a worker behind a stalled upstream. + deadline = time.monotonic() + CHANGELOG_TIMEOUT_SECONDS + 1 + while True: + now = time.monotonic() + with _cache_condition: + if _remote_cache and _remote_cache.expires_at > now: + return _remote_cache.source + if not _remote_fetching: + _remote_fetching = True + break + if now >= deadline: + return ChangelogSource( + text = None, + source = None, + error = "Release notes are still loading.", + ) + _cache_condition.wait(timeout = deadline - now) + + try: + try: + source = _fetch_remote_changelog() + except Exception: + source = ChangelogSource( + text = None, + source = None, + error = "Could not fetch release notes.", + ) + + ttl = CHANGELOG_SUCCESS_TTL_SECONDS if source.text else CHANGELOG_FAILURE_TTL_SECONDS + with _cache_condition: + _remote_cache = _ChangelogCacheEntry(source = source, expires_at = time.monotonic() + ttl) + return source + finally: + # Released here, not on the Exception path: stranding the single-flight + # flag on BaseException makes every later caller wait out the deadline. + with _cache_condition: + _remote_fetching = False + _cache_condition.notify_all() + + +def _fetch_remote_changelog() -> ChangelogSource: + url = os.environ.get(CHANGELOG_URL_ENV_VAR, "").strip() or CHANGELOG_RAW_URL + if not url.startswith(("http://", "https://")): + return ChangelogSource(text = None, source = None, error = "Invalid changelog URL.") + + request = urllib.request.Request( + url, + headers = { + "User-Agent": "unsloth-studio-update-check", + # Or a compressing proxy hands back bytes we would decode as notes. + "Accept-Encoding": "identity", + }, + ) + deadline = time.monotonic() + CHANGELOG_TIMEOUT_SECONDS + try: + with urllib.request.urlopen(request, timeout = CHANGELOG_TIMEOUT_SECONDS) as response: + chunks: list[bytes] = [] + received = 0 + while received <= CHANGELOG_MAX_BYTES: + remaining = deadline - time.monotonic() + if remaining <= 0: + return ChangelogSource( + text = None, + source = None, + error = "Release notes took too long to load.", + ) + # The socket timeout is per operation, so re-cap it each read. + _limit_read(response, remaining) + chunk = response.read1(_CHANGELOG_CHUNK_BYTES) + if not chunk: + break + chunks.append(chunk) + received += len(chunk) + body = b"".join(chunks) + if len(body) > CHANGELOG_MAX_BYTES: + return ChangelogSource( + text = None, + source = None, + error = "Release notes response was too large.", + ) + return ChangelogSource(text = body.decode("utf-8", errors = "replace"), source = "remote") + except TimeoutError: + return ChangelogSource( + text = None, + source = None, + error = "Release notes took too long to load.", + ) + except OSError: + return ChangelogSource( + text = None, + source = None, + error = "Could not reach the changelog for release notes.", + ) + except UnicodeError: + return ChangelogSource(text = None, source = None, error = "Malformed changelog.") + + +def _limit_read(response: Any, remaining: float) -> None: + """Cap the next socket read at the time left in the fetch budget.""" + sock = getattr(getattr(response, "fp", None), "raw", None) + sock = getattr(sock, "_sock", None) + if sock is None: + return + try: + sock.settimeout(max(remaining, _CHANGELOG_MIN_READ_SECONDS)) + except OSError: + pass + + +def _read_local_changelog() -> ChangelogSource: + """Read the CHANGELOG.md bundled with this install, if there is one.""" + for path in _local_changelog_candidates(): + try: + if not path.is_file(): + continue + if path.stat().st_size > CHANGELOG_MAX_BYTES: + continue + return ChangelogSource( + text = path.read_text(encoding = "utf-8", errors = "replace"), + source = "local", + ) + except OSError: + continue + return ChangelogSource(text = None, source = None) + + +def _is_source_checkout(root: Path) -> bool: + """Whether `root` is this repository rather than an install directory.""" + try: + return any((root / marker).exists() for marker in _CHECKOUT_MARKERS) + except OSError: + return False + + +def _local_changelog_candidates() -> list[Path]: + override = os.environ.get(CHANGELOG_PATH_ENV_VAR, "").strip() + candidates: list[Path] = [] + if override: + candidates.append(Path(override).expanduser()) + + # changelog.py -> utils -> backend -> studio -> repo root. Repo root first + # so a checkout's editable file beats the snapshot packaging writes into + # studio/. Installed, those outer levels are site-packages, hence the marker. + parents = Path(__file__).resolve().parents + for index in (3, 2, 1, 4): + if index >= len(parents): + continue + root = parents[index] + if index in _CHECKOUT_ONLY_LEVELS and not _is_source_checkout(root): + continue + candidates.append(root / CHANGELOG_FILENAME) + + seen: set[Path] = set() + unique: list[Path] = [] + for candidate in candidates: + if candidate not in seen: + seen.add(candidate) + unique.append(candidate) + return unique + + +def _opens_fence(marker: str, rest: str) -> bool: + """A backtick fence's info string may not contain a backtick.""" + return marker[0] != "`" or "`" not in rest + + +def _next_fence_state(open_fence: str | None, marker: str, rest: str) -> str | None: + """Track the open fence marker. + + A closer must be the same character, at least as long, and carry nothing + after it. So neither a ``` sample nor a ```` line with trailing text ends + a ```` block early, while an opening fence may still have an info string. + Only spaces and tabs count as nothing: other Unicode whitespace is content. + """ + if open_fence is None: + return marker if _opens_fence(marker, rest) else None + closes = marker[0] == open_fence[0] and len(marker) >= len(open_fence) + if closes and not rest.strip(" \t"): + return None + return open_fence + + +def _code_span_ranges(line: str) -> list[tuple[int, int]]: + """Code span bounds. A run of backticks closes only on a run of its length.""" + # Collect the runs once: rescanning per opener is quadratic on a line of + # distinct unmatched runs, and notes are reparsed on every request. + runs: list[tuple[int, int]] = [] + index = 0 + while index < len(line): + if line[index] != "`" or _is_escaped(line, index): + index += 1 + continue + ticks = _run_length(line, index) + runs.append((index, ticks)) + index += ticks + + # A run closes only on a later run of its length, so one cursor per length. + by_length: dict[int, list[int]] = {} + for position, (_, ticks) in enumerate(runs): + by_length.setdefault(ticks, []).append(position) + + spans: list[tuple[int, int]] = [] + cursors: dict[int, int] = {} + current = 0 + while current < len(runs): + start, ticks = runs[current] + same = by_length[ticks] + cursor = cursors.get(ticks, 0) + while cursor < len(same) and same[cursor] <= current: + cursor += 1 + cursors[ticks] = cursor + if cursor >= len(same): + # Nothing closes this run, so it is literal text. + current += 1 + continue + closer = same[cursor] + cursors[ticks] = cursor + 1 + spans.append((start, runs[closer][0] + ticks)) + current = closer + 1 + return spans + + +def _run_length(line: str, index: int) -> int: + end = index + while end < len(line) and line[end] == "`": + end += 1 + return end - index + + +def _is_escaped(line: str, index: int) -> bool: + slashes = 0 + while index - 1 - slashes >= 0 and line[index - 1 - slashes] == "\\": + slashes += 1 + return slashes % 2 == 1 + + +def _strip_comments(line: str, in_comment: bool, block_open: bool) -> tuple[str, bool]: + """Return the line with HTML-comment spans removed, and the trailing state. + + Only a comment that starts a line opens a block and hides the lines below + it. One written mid-sentence is inline HTML: it hides the rest of its own + line at most, so a note mentioning `<!--` cannot swallow later releases. + Delimiters inside inline code are literal and hide nothing. + + "Starts a line" is read inside the container, so `block_open` is decided by + the caller from the item's content rather than from the raw line. + """ + if in_comment: + close = line.find(_COMMENT_CLOSE) + # The closing line belongs to the block, tail included. + return ("", False) if close != -1 else ("", True) + + if block_open: + # `<!-->` and `<!--->` are complete comments, so the closer may overlap + # the opener; searching past it would swallow every later release. + return ("", _COMMENT_CLOSE not in line) + + visible: list[str] = [] + index = 0 + spans = _code_span_ranges(line) + # Spans are ordered and disjoint and each opener sits at or past the one + # before, so the search resumes rather than restarts: restarting per opener is + # quadratic, and a long line of code spans is reparsed on every request. + cursor = 0 + while index < len(line): + opening = line.find(_COMMENT_OPEN, index) + if opening == -1: + visible.append(line[index:]) + break + + while cursor < len(spans) and spans[cursor][1] <= opening: + cursor += 1 + if cursor < len(spans) and spans[cursor][0] <= opening: + visible.append(line[index : spans[cursor][1]]) + index = spans[cursor][1] + continue + + visible.append(line[index:opening]) + close = line.find(_COMMENT_CLOSE, opening + len(_COMMENT_OPEN)) + if close == -1: + # Unterminated inline comment: it hides this line and no more. + break + index = close + len(_COMMENT_CLOSE) + return "".join(visible), False + + +def _hidden_structure(line: str, marker: str = "") -> str: + """`line` as list tracking sees it once the renderer hides its text. + + A comment or a raw HTML block renders nothing, but it is still a block + written at its own column, so it closes the items it sits to the left of. + Only the indentation survives: what is inside the block is not Markdown and + must not open a list of its own. `marker` is the part of the line that opens + a list item the block is the content of, which survives with it.""" + if marker: + return marker + _HIDDEN_BLOCK + if not line.strip(): + return "" + return line[: len(line) - len(line.lstrip(" \t"))] + _HIDDEN_BLOCK + + +def _indent_width(line: str) -> int: + """Columns of leading whitespace, counting a tab to the next stop of four.""" + width = 0 + for char in line: + if char == " ": + width += 1 + elif char == "\t": + width += 4 - width % 4 + else: + break + return width + + +def _strip_indent(line: str, columns: int) -> str: + """`line` with up to `columns` columns of leading whitespace removed.""" + width = 0 + index = 0 + while index < len(line) and width < columns and line[index] in " \t": + width += 1 if line[index] == " " else 4 - width % 4 + index += 1 + return line[index:] + + +def _interrupts_paragraph(line: str) -> bool: + """Whether `line` starts a block that can break into an open paragraph. + + A quote marker always can. A list item can only when it has content, and an + ordered one only when it starts at 1: anything else is text of the + paragraph it appears to interrupt.""" + if _BLOCK_QUOTE.match(line): + return True + item = None if _THEMATIC_BREAK.match(line) else _LIST_ITEM.match(line) + if item is None: + return False + marker = item.group("marker") + if not line[item.end() :].strip(): + return False + return marker[-1] not in ".)" or marker[:-1] == "1" + + +def _item_content(line: str, after_paragraph: bool) -> str: + """`line` read from the content column of a list item that opens on it. + + A block written as an item's first content sits inside that item, so + ``- ```` opens a fence even though its marker is not within three columns of + the container. The padding is capped the way `_open_lists` caps it, or + ``- ```` would read as a fence rather than the indented code it is. A + marker the paragraph above swallows opens no item, so its line is returned + whole, as is one four columns past its container. Ported to the frontend as + `itemContent` in markdown-list-columns.ts.""" + if _indent_width(line) >= 4 or (after_paragraph and not _interrupts_paragraph(line)): + return line + item = None if _THEMATIC_BREAK.match(line) else _LIST_ITEM.match(line) + if item is None: + return line + padding = _indent_width(item.group("space")) + # Over-indented content starts one column past the marker; the rest of the + # padding is the content's own indentation. + over = padding - 1 if padding > _MAX_ITEM_PADDING else 0 + return " " * over + line[item.end() :] + + +def _quote_content(line: str) -> str: + """What a blockquote line holds, with its markers stripped.""" + while (marker := _QUOTE_MARKER.match(line)) is not None: + line = line[marker.end() :] + return line + + +def _may_be_lazy(line: str) -> bool: + """Whether `line` can continue a paragraph it is indented out of. + + Only plain text can: a heading, a fence, a break or an HTML block starts a + block of its own, which closes the item instead. An underline is not one of + them: it may never be lazy, so `===` written left of an open item is read as + more of the item's paragraph. Nor is a definition, which is a block of its + own but may not interrupt a paragraph. A row of dashes still closes the + item, as `_INTERRUPTS` reads three or more as the thematic break they are.""" + return ( + _PARAGRAPH_TEXT.match(line) is not None + and _INTERRUPTS.match(line) is None + and _FENCE_PATTERN.match(line) is None + # Types 1 to 6 interrupt a paragraph, so a `<div>` left of an open item + # closes it. Type 7 cannot, and is deliberately excluded. + and not _opens_html_block(line, True) + ) + + +def _continues_paragraph(line: str, column: int) -> bool: + """Whether `line` reads as more of a paragraph open in its container. + + Measured from `column`, where that container's content starts: four columns + past it the line is an indented code block, which may not interrupt a + paragraph, so indentation alone never closes the one above it.""" + inner = _strip_indent(line, column) + return _indent_width(inner) >= 4 or _may_be_lazy(inner) + + +def _close_dedented( + columns: tuple[int, ...], line: str, indent: int, after_paragraph: bool +) -> tuple[int, ...]: + """`columns` with every item `line` is written to the left of closed. + + Read inside the container the item sits in, not from the margin: a line that + only looks indented there is lazy text of the item's paragraph, which leaves + the item open rather than closing it.""" + while columns and indent < columns[-1]: + outer = columns[-2] if len(columns) > 1 else 0 + if after_paragraph and _continues_paragraph(line, outer): + break + columns = columns[:-1] + return columns + + +def _lazy_marker(line: str, state: _ListState, after_paragraph: bool, quoted: bool) -> bool: + """Whether a marker-shaped `line` is really text of the paragraph above it. + + Only a marker inside the paragraph's own item interrupts it; one to the left + closes that item and opens a sibling. A quote owns the paragraph its lines + hold, so a marker written outside the quote opens a list of its own.""" + item = None if _THEMATIC_BREAK.match(line) else _LIST_ITEM.match(line) + columns = state.columns + return ( + item is not None + and after_paragraph + and not quoted + and (not columns or _indent_width(line) >= columns[-1]) + and not _interrupts_paragraph(line) + ) + + +def _open_lists( + line: str, + state: _ListState, + after_paragraph: bool, + quoted: bool = False, +) -> _ListState: + """The list items still open after `line`. + + A dedented line closes an item, unless it is a lazy paragraph continuation. + A new marker nests under a deeper column and replaces a sibling. `quoted` + marks a paragraph the blockquote above owns: a marker written outside the + quote is not text of it, so it opens a list of its own. + """ + columns = state.columns + if not line.strip(): + # A blank line leaves the list open, unless the item is still empty: an + # item may begin with one blank line, and later content is outside it. + return _ListState(columns[:-1] if state.empty_item else columns) + indent = _indent_width(line) + item = None if _THEMATIC_BREAK.match(line) else _LIST_ITEM.match(line) + empty = item is not None and not line[item.end() :].strip() + if _lazy_marker(line, state, after_paragraph, quoted): + # A lazy continuation or an underline, so the open items are untouched. + return state + columns = _close_dedented(columns, line, indent, after_paragraph) + # Four columns past its container the marker is an indented code block, or + # lazy text of the paragraph above it, so it opens no list of its own. + if item is None or indent - (columns[-1] if columns else 0) >= 4: + return _ListState(columns) + marker = item.group("marker") + padding = _indent_width(item.group("space")) + if padding == 0 or padding > _MAX_ITEM_PADDING: + # An empty or over-indented item still holds one column of content. + padding = 1 + while columns and columns[-1] > indent: + columns = columns[:-1] + return _ListState((*columns, indent + len(marker) + padding), empty_item = empty) + + +def _opens_html_block(line: str, after_paragraph: bool) -> bool: + """True if `line` starts a CommonMark type 6 or type 7 HTML block.""" + match = _HTML_BLOCK_OPEN.match(line) + if match is not None and match.group(1).lower() in _HTML_BLOCK_TAGS: + return True + return not after_paragraph and _HTML_TAG_ONLY_LINE.match(line) is not None + + +def _strip_raw_html(line: str, open_block: int | None) -> tuple[str, int | None]: + """Drop the parts of a line inside a raw block, and return the open block. + + The state is the index of the open block in `_RAW_BLOCKS`, or None.""" + if open_block is not None: + close = _RAW_BLOCKS[open_block][1].search(line) + return ("", None) if close else ("", open_block) + + # A block only opens at the start of a line; mid-line tags are inline HTML. + for index, (opener, closer) in enumerate(_RAW_BLOCKS): + opening = opener.match(line) + if opening is None: + continue + rest = line[opening.end() :] + close = closer.search(rest) + return ("", None) if close else ("", index) + return line, None + + +def _version_from_heading(heading: str) -> str | None: + token = heading.split()[0] if heading.split() else "" + match = _VERSION_TOKEN_PATTERN.match(token) + if match is None: + return None + version = match.group("version") + return version if _parse_version(version) is not None else None + + +def _parse_version(version: str) -> Version | None: + try: + return Version(version) + except InvalidVersion: + return None + + +def _close_open_fence(markdown: str) -> str: + """Close a fence the truncation cut in half, so the rest still renders.""" + open_fence: str | None = None + for line in _markdown_lines(markdown): + fence = _FENCE_PATTERN.match(line) + if fence: + open_fence = _next_fence_state(open_fence, fence.group("marker"), fence.group("rest")) + return f"{markdown}\n{open_fence}" if open_fence else markdown + + +def _renders_visibly(markdown: str) -> bool: + """Whether a section body renders anything at all.""" + in_comment = False + for line in _markdown_lines(markdown): + opens_raw = any(opener.match(line) for opener, _ in _RAW_BLOCKS) + if not in_comment and (_FENCE_PATTERN.match(line) or opens_raw): + # A code block or raw HTML block renders even when it is empty. + return True + # No containers are tracked here, so the opener is read at the margin. The + # answer does not turn on it: an item renders its marker whatever the block + # inside hides, so a commented-out item renders something either way. + visible, in_comment = _strip_comments( + line, in_comment, _COMMENT_BLOCK_OPEN.match(line) is not None + ) + if visible.strip(): + return True + return False + + +def _notes_response( + *, + version: str, + markdown: str | None = None, + heading: str | None = None, + source: str | None = None, + error: str | None = None, +) -> dict[str, Any]: + # A section that renders as nothing counts as unpublished, not as empty. + if markdown and not _renders_visibly(markdown): + markdown = None + source = None + + truncated = False + if markdown and len(markdown) > RELEASE_NOTES_MAX_CHARS: + markdown = _close_open_fence(markdown[:RELEASE_NOTES_MAX_CHARS].rstrip()) + truncated = True + + return { + "version": version, + "markdown": markdown or None, + "heading": heading, + # False means no notes for this exact version; the UI links out. + "matched": bool(markdown), + "truncated": truncated, + "source": source, + "release_notes_url": RELEASE_NOTES_URL, + "error": error, + } diff --git a/studio/backend/utils/child_stdio.py b/studio/backend/utils/child_stdio.py new file mode 100644 index 0000000000..4709d650df --- /dev/null +++ b/studio/backend/utils/child_stdio.py @@ -0,0 +1,22 @@ +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 + +"""Make a Python child agree with the parent that its pipes are UTF-8. + +A child's ``sys.stdout`` uses ``locale.getpreferredencoding()``, which on +Windows is the ANSI code page. Reading that pipe as UTF-8 would then mangle any +non-ASCII the child prints, so the child has to be told which encoding to emit. +Only needed for Python children; llama.cpp and node already emit UTF-8. +""" + +from __future__ import annotations + +import os +from typing import Mapping, Optional + + +def utf8_child_env(env: Optional[Mapping[str, str]] = None) -> dict[str, str]: + """Copy *env* (or the current environment) with UTF-8 stdio forced.""" + child = dict(os.environ if env is None else env) + child["PYTHONIOENCODING"] = "utf-8" + return child diff --git a/studio/backend/utils/hardware/amd.py b/studio/backend/utils/hardware/amd.py index 91a06c9a2a..318759f67d 100644 --- a/studio/backend/utils/hardware/amd.py +++ b/studio/backend/utils/hardware/amd.py @@ -144,6 +144,8 @@ def _run_amd_smi(*args: str, timeout: int = _AMD_SMI_DEFAULT_TIMEOUT) -> Optiona ["amd-smi", *args, "--json"], capture_output = True, text = True, + encoding = "utf-8", + errors = "replace", timeout = timeout, env = _amd_env, **windows_hidden_subprocess_kwargs(), diff --git a/studio/backend/utils/hardware/hardware.py b/studio/backend/utils/hardware/hardware.py index 48ba375ec5..300d26c362 100644 --- a/studio/backend/utils/hardware/hardware.py +++ b/studio/backend/utils/hardware/hardware.py @@ -830,6 +830,8 @@ def _rocm_windows_perf_counter_gpu_util_pct() -> Optional[float]: ["powershell", "-NoProfile", "-NonInteractive", "-Command", ps], capture_output = True, text = True, + encoding = "utf-8", + errors = "replace", timeout = 5, ) if r.returncode != 0 or not r.stdout.strip(): @@ -1027,6 +1029,8 @@ def _rocm_windows_perf_counter_vram_by_adapter() -> Optional[list[tuple[str, flo ["powershell", "-NoProfile", "-NonInteractive", "-Command", ps], capture_output = True, text = True, + encoding = "utf-8", + errors = "replace", timeout = 5, ) if r.returncode != 0 or not r.stdout.strip(): diff --git a/studio/backend/utils/hardware/nvidia.py b/studio/backend/utils/hardware/nvidia.py index f98ca4343e..39e3652921 100644 --- a/studio/backend/utils/hardware/nvidia.py +++ b/studio/backend/utils/hardware/nvidia.py @@ -55,6 +55,8 @@ def get_physical_gpu_count() -> Optional[int]: ["nvidia-smi", "-L"], capture_output = True, text = True, + encoding = "utf-8", + errors = "replace", timeout = 5, env = child_env_without_native_path_secret(), **_windows_hidden_subprocess_kwargs(), @@ -81,6 +83,8 @@ def get_primary_gpu_utilization() -> dict[str, Any]: ], capture_output = True, text = True, + encoding = "utf-8", + errors = "replace", timeout = 5, env = child_env_without_native_path_secret(), **_windows_hidden_subprocess_kwargs(), @@ -131,6 +135,8 @@ def get_visible_gpu_utilization( ], capture_output = True, text = True, + encoding = "utf-8", + errors = "replace", timeout = 5, env = child_env_without_native_path_secret(), **_windows_hidden_subprocess_kwargs(), @@ -215,6 +221,8 @@ def get_backend_visible_gpu_info( ], capture_output = True, text = True, + encoding = "utf-8", + errors = "replace", timeout = 10, env = child_env_without_native_path_secret(), **_windows_hidden_subprocess_kwargs(), diff --git a/studio/backend/utils/llama_cpp_update.py b/studio/backend/utils/llama_cpp_update.py index dffcddb452..5c9646f4eb 100644 --- a/studio/backend/utils/llama_cpp_update.py +++ b/studio/backend/utils/llama_cpp_update.py @@ -121,7 +121,14 @@ def _installed_build_number(binary: Optional[str]) -> Optional[int]: if not binary: return None try: - proc = subprocess.run([binary, "--version"], capture_output = True, text = True, timeout = 20) + proc = subprocess.run( + [binary, "--version"], + capture_output = True, + text = True, + encoding = "utf-8", + errors = "replace", + timeout = 20, + ) except Exception: # pragma: no cover - defensive return None m = re.search(r"version:\s*(\d+)", (proc.stderr or "") + (proc.stdout or "")) diff --git a/studio/backend/utils/mlx_repair.py b/studio/backend/utils/mlx_repair.py index 4ea1ec62f5..8e2a6a7712 100644 --- a/studio/backend/utils/mlx_repair.py +++ b/studio/backend/utils/mlx_repair.py @@ -254,7 +254,7 @@ def _transformers_constraint_args() -> tuple[list[str], str | None]: except Exception: return [], None fd, path = tempfile.mkstemp(prefix = "mlx_repair_", suffix = ".txt") - with os.fdopen(fd, "w") as fh: + with os.fdopen(fd, "w", encoding = "utf-8") as fh: fh.write(f"transformers=={transformers_version}\n") return ["--constraint", path], path @@ -290,6 +290,8 @@ def attempt_mlx_repair(*, timeout: int = _REPAIR_TIMEOUT_S) -> bool: stdout = subprocess.PIPE, stderr = subprocess.STDOUT, text = True, + encoding = "utf-8", + errors = "replace", timeout = timeout, ) except subprocess.TimeoutExpired: diff --git a/studio/backend/utils/models/checkpoints.py b/studio/backend/utils/models/checkpoints.py index 6950667bbd..eaf75140fc 100644 --- a/studio/backend/utils/models/checkpoints.py +++ b/studio/backend/utils/models/checkpoints.py @@ -129,7 +129,7 @@ def _read_checkpoint_loss(checkpoint_path: Path) -> Optional[float]: if not trainer_state.exists(): return None try: - with open(trainer_state, encoding = "utf-8") as f: + with open(trainer_state, encoding = "utf-8-sig") as f: state = json.load(f) log_history = state.get("log_history", []) if log_history: @@ -174,18 +174,18 @@ def scan_checkpoints( metadata: dict = {} try: if adapter_config.exists(): - cfg = json.loads(adapter_config.read_text(encoding = "utf-8")) + cfg = json.loads(adapter_config.read_text(encoding = "utf-8-sig")) metadata["base_model"] = cfg.get("base_model_name_or_path") metadata["peft_type"] = cfg.get("peft_type") metadata["lora_rank"] = cfg.get("r") elif config_file.exists(): - cfg = json.loads(config_file.read_text(encoding = "utf-8")) + cfg = json.loads(config_file.read_text(encoding = "utf-8-sig")) metadata["base_model"] = cfg.get("_name_or_path") # Detect BNB quantization from config.json if config_file.exists(): if "cfg" not in dir(): - cfg = json.loads(config_file.read_text(encoding = "utf-8")) + cfg = json.loads(config_file.read_text(encoding = "utf-8-sig")) quant_cfg = cfg.get("quantization_config") if ( isinstance(quant_cfg, dict) diff --git a/studio/backend/utils/models/model_config.py b/studio/backend/utils/models/model_config.py index 893b842e11..6270d9e03f 100644 --- a/studio/backend/utils/models/model_config.py +++ b/studio/backend/utils/models/model_config.py @@ -37,6 +37,7 @@ import yaml from utils.native_path_leases import child_env_without_native_path_secret +from utils.child_stdio import utf8_child_env from utils.hf_cache_settings import active_hf_hub_cache, get_hf_cache_paths from utils.subprocess_compat import ( windows_hidden_subprocess_kwargs as _windows_hidden_subprocess_kwargs, @@ -631,7 +632,7 @@ def _raw_config_has_vision_config( cache_dir = active_hf_hub_cache(), ) ) - config = json.loads(config_path.read_text(encoding = "utf-8")) + config = json.loads(config_path.read_text(encoding = "utf-8-sig")) architectures = config.get("architectures") or [] model_type = config.get("model_type") explicit_vision = ( @@ -774,8 +775,12 @@ def _is_vision_model_subprocess(model_name: str, hf_token: Optional[str] = None) ], capture_output = True, text = True, + encoding = "utf-8", + errors = "replace", timeout = 60, - env = get_hf_cache_paths().child_env(child_env_without_native_path_secret()), + env = utf8_child_env( + get_hf_cache_paths().child_env(child_env_without_native_path_secret()) + ), **_windows_hidden_subprocess_kwargs(), ) @@ -1083,7 +1088,7 @@ def _detect_audio_from_tokenizer( ]: tok_file = snapshot / tok_path if tok_file.exists(): - tok_config = json.loads(tok_file.read_text(encoding = "utf-8")) + tok_config = json.loads(tok_file.read_text(encoding = "utf-8-sig")) read_any = True result = _check_token_patterns(tok_config) if result: @@ -2283,7 +2288,7 @@ def scan_exported_models( export_meta = run_dir / "export_metadata.json" try: if export_meta.exists(): - meta = json.loads(export_meta.read_text(encoding = "utf-8")) + meta = json.loads(export_meta.read_text(encoding = "utf-8-sig")) base_model = meta.get("base_model") except Exception: pass @@ -2312,7 +2317,7 @@ def scan_exported_models( if adapter_config.exists(): export_type = "lora" try: - cfg = json.loads(adapter_config.read_text(encoding = "utf-8")) + cfg = json.loads(adapter_config.read_text(encoding = "utf-8-sig")) base_model = cfg.get("base_model_name_or_path") except Exception: pass @@ -2321,7 +2326,7 @@ def scan_exported_models( export_meta = checkpoint_dir / "export_metadata.json" try: if export_meta.exists(): - meta = json.loads(export_meta.read_text(encoding = "utf-8")) + meta = json.loads(export_meta.read_text(encoding = "utf-8-sig")) base_model = meta.get("base_model") except Exception: pass @@ -2334,7 +2339,7 @@ def scan_exported_models( export_meta = meta_dir / "export_metadata.json" try: if export_meta.exists(): - meta = json.loads(export_meta.read_text(encoding = "utf-8")) + meta = json.loads(export_meta.read_text(encoding = "utf-8-sig")) base_model = meta.get("base_model") if base_model: break @@ -2354,7 +2359,7 @@ def scan_exported_models( outputs_adapter_cfg = resolve_output_dir(run_dir.name) / "adapter_config.json" try: if outputs_adapter_cfg.exists(): - cfg = json.loads(outputs_adapter_cfg.read_text(encoding = "utf-8")) + cfg = json.loads(outputs_adapter_cfg.read_text(encoding = "utf-8-sig")) base_model = cfg.get("base_model_name_or_path") except Exception: pass @@ -2380,7 +2385,7 @@ def get_base_model_from_checkpoint(checkpoint_path: str) -> Optional[str]: adapter_config_path = checkpoint_path_obj / "adapter_config.json" if adapter_config_path.exists(): - with open(adapter_config_path, "r", encoding = "utf-8") as f: + with open(adapter_config_path, "r", encoding = "utf-8-sig") as f: config = json.load(f) base_model = config.get("base_model_name_or_path") if base_model: @@ -2389,7 +2394,7 @@ def get_base_model_from_checkpoint(checkpoint_path: str) -> Optional[str]: config_path = checkpoint_path_obj / "config.json" if config_path.exists(): - with open(config_path, "r", encoding = "utf-8") as f: + with open(config_path, "r", encoding = "utf-8-sig") as f: config = json.load(f) for key in ("model_name", "_name_or_path"): base_model = config.get(key) @@ -2445,7 +2450,7 @@ def get_base_model_from_lora(lora_path: str) -> Optional[str]: # adapter_config.json first adapter_config_path = lora_path_obj / "adapter_config.json" if adapter_config_path.exists(): - with open(adapter_config_path, "r", encoding = "utf-8") as f: + with open(adapter_config_path, "r", encoding = "utf-8-sig") as f: config = json.load(f) base_model = config.get("base_model_name_or_path") if base_model: @@ -2535,7 +2540,7 @@ def get_base_model_from_lora_identifier( last_exc = exc continue try: - with open(cfg_path, "r", encoding = "utf-8") as f: + with open(cfg_path, "r", encoding = "utf-8-sig") as f: base_model = json.load(f).get("base_model_name_or_path") except Exception as exc: logger.warning("Could not parse adapter_config.json for '%s': %s", identifier, exc) @@ -2781,7 +2786,7 @@ class ModelConfig: meta_path = gguf_dir / "export_metadata.json" if meta_path.exists(): try: - meta = json.loads(meta_path.read_text(encoding = "utf-8")) + meta = json.loads(meta_path.read_text(encoding = "utf-8-sig")) base = meta.get("base_model") if base and is_vision_model(base, hf_token = hf_token): base_is_vision = True @@ -2912,7 +2917,7 @@ class ModelConfig: token = hf_token, cache_dir = active_hf_hub_cache(), ) - with open(config_path, "r", encoding = "utf-8") as f: + with open(config_path, "r", encoding = "utf-8-sig") as f: adapter_config = json.load(f) base_model = adapter_config.get("base_model_name_or_path") if base_model: diff --git a/studio/backend/utils/node_runtime.py b/studio/backend/utils/node_runtime.py index fef2430708..697661a095 100644 --- a/studio/backend/utils/node_runtime.py +++ b/studio/backend/utils/node_runtime.py @@ -79,6 +79,8 @@ def _node_version_ok(executable: str) -> bool: [executable, "-v"], capture_output = True, text = True, + encoding = "utf-8", + errors = "replace", timeout = _NODE_VERSION_PROBE_TIMEOUT_SECONDS, **windows_hidden_subprocess_kwargs(), ) diff --git a/studio/backend/utils/paths/storage_roots.py b/studio/backend/utils/paths/storage_roots.py index ae1319d296..0b1398f6d2 100644 --- a/studio/backend/utils/paths/storage_roots.py +++ b/studio/backend/utils/paths/storage_roots.py @@ -212,7 +212,7 @@ def lmstudio_model_dirs() -> list[Path]: settings_path = Path.home() / ".lmstudio" / "settings.json" if settings_path.is_file(): try: - with open(settings_path, encoding = "utf-8") as f: + with open(settings_path, encoding = "utf-8-sig") as f: settings = json.load(f) downloads = settings.get("downloadsFolder", "") if downloads: diff --git a/studio/backend/utils/prebuilt/update_flow.py b/studio/backend/utils/prebuilt/update_flow.py index 74af0c18f9..69c1566fc3 100644 --- a/studio/backend/utils/prebuilt/update_flow.py +++ b/studio/backend/utils/prebuilt/update_flow.py @@ -24,6 +24,7 @@ from typing import Callable, Optional import structlog +from utils.child_stdio import utf8_child_env from utils.process_lifetime import child_popen_kwargs logger = structlog.get_logger(__name__) @@ -159,6 +160,8 @@ def resolve_prebuilt_for_host( cmd, capture_output = True, text = True, + encoding = "utf-8", + errors = "replace", timeout = 60, ) out = (proc.stdout or "").strip() @@ -303,7 +306,10 @@ def stream_installer( stdout = subprocess.PIPE, stderr = subprocess.STDOUT, text = True, - env = env, + encoding = "utf-8", + errors = "replace", + # Make the Python child emit the UTF-8 we decode above. + env = utf8_child_env(env), **child_popen_kwargs(), ) timed_out = threading.Event() diff --git a/studio/backend/utils/security/consent.py b/studio/backend/utils/security/consent.py index 6fee259139..9385270ee0 100644 --- a/studio/backend/utils/security/consent.py +++ b/studio/backend/utils/security/consent.py @@ -142,7 +142,7 @@ def _load_remote_code_configs(model_name: str, hf_token: Optional[str] = None) - for name in _REMOTE_CODE_CONFIG_FILES: p = root / name if p.is_file(): - configs.append(json.loads(p.read_text(encoding = "utf-8"))) + configs.append(json.loads(p.read_text(encoding = "utf-8-sig"))) return configs from huggingface_hub import hf_hub_download @@ -164,7 +164,7 @@ def _load_remote_code_configs(model_name: str, hf_token: Optional[str] = None) - # Transient/auth failure is not "absent" -> fail closed to "unknown" so # the caller scans (a tokenizer/processor-only auto_map must not slip by). return None - configs.append(json.loads(Path(p).read_text(encoding = "utf-8"))) + configs.append(json.loads(Path(p).read_text(encoding = "utf-8-sig"))) # Every config was read or a genuine 404 -> an empty list is a definitive # "no auto_map", not "unknown". return configs diff --git a/studio/backend/utils/security/file_security.py b/studio/backend/utils/security/file_security.py index 7724406e8d..4588f32b90 100644 --- a/studio/backend/utils/security/file_security.py +++ b/studio/backend/utils/security/file_security.py @@ -199,7 +199,7 @@ def _indexed_shard_paths( inconclusive = True # transient: an index that might exist could not be read continue try: - weight_map = (json.loads(open(index_path, encoding = "utf-8").read()) or {}).get( + weight_map = (json.loads(open(index_path, encoding = "utf-8-sig").read()) or {}).get( "weight_map" ) or {} for shard in weight_map.values(): @@ -328,7 +328,7 @@ def _st_load_roots(snapshot: Path) -> list: roots = [snapshot] try: import json - modules = json.loads((snapshot / "modules.json").read_text(encoding = "utf-8")) + modules = json.loads((snapshot / "modules.json").read_text(encoding = "utf-8-sig")) except (OSError, ValueError): return roots # no / invalid modules.json -> snapshot root is the only load root for module in modules or (): @@ -355,7 +355,7 @@ def _indexed_pickle_shards(index_path: Path, root: Path, snapshot: Path) -> list try: # JSON is UTF-8 by spec; pin it so a non-ASCII index is not misdecoded (and needlessly # blocked) under Windows' cp1252 default. - parsed = json.loads(index_path.read_text(encoding = "utf-8")) + parsed = json.loads(index_path.read_text(encoding = "utf-8-sig")) except (OSError, ValueError) as exc: raise OSError(f"unreadable weight index: {index_path}") from exc weight_map = parsed.get("weight_map") if isinstance(parsed, dict) else None diff --git a/studio/backend/utils/security/remote_code_approvals.py b/studio/backend/utils/security/remote_code_approvals.py index f1baac6924..d6076fd2b7 100644 --- a/studio/backend/utils/security/remote_code_approvals.py +++ b/studio/backend/utils/security/remote_code_approvals.py @@ -69,7 +69,7 @@ def approval_target_key(targets) -> str: def _load() -> dict: """Parsed store, or an empty skeleton on any error (fail-safe = re-prompt).""" try: - with open(_store_path(), encoding = "utf-8") as f: + with open(_store_path(), encoding = "utf-8-sig") as f: data = json.load(f) # Validate the shape, not just the version: a hand-edited ``subjects`` that is not a # dict (e.g. ``[]``) would otherwise crash lookup/record instead of failing safe. diff --git a/studio/backend/utils/security/remote_code_scan.py b/studio/backend/utils/security/remote_code_scan.py index d4d8003252..42f9d98efe 100644 --- a/studio/backend/utils/security/remote_code_scan.py +++ b/studio/backend/utils/security/remote_code_scan.py @@ -454,7 +454,7 @@ def repo_remote_code_files(model_name: str, hf_token: Optional[str] = None) -> d p = root / name if p.is_file(): try: - ext_refs |= _auto_map_refs(json.loads(p.read_text(encoding = "utf-8"))) + ext_refs |= _auto_map_refs(json.loads(p.read_text(encoding = "utf-8-sig"))) except Exception: pass if not _add_external_refs(files, ext_refs, hf_token, model_name): @@ -483,7 +483,7 @@ def repo_remote_code_files(model_name: str, hf_token: Optional[str] = None) -> d f"{model_name}: config {cfg_name} could not be fetched ({exc})" ) from exc try: - refs |= _auto_map_refs(json.loads(Path(cfg_path).read_text(encoding = "utf-8"))) + refs |= _auto_map_refs(json.loads(Path(cfg_path).read_text(encoding = "utf-8-sig"))) except Exception: pass own_refs = {fn for repo, fn in refs if repo is None} @@ -616,7 +616,7 @@ def external_auto_map_repos(model_name: str, hf_token: Optional[str] = None) -> if not p.is_file(): continue try: - refs = _auto_map_refs(json.loads(p.read_text(encoding = "utf-8"))) + refs = _auto_map_refs(json.loads(p.read_text(encoding = "utf-8-sig"))) except Exception: continue repos.update(repo for repo, _fn in refs if repo) @@ -638,7 +638,7 @@ def external_auto_map_repos(model_name: str, hf_token: Optional[str] = None) -> except Exception: continue try: - refs = _auto_map_refs(json.loads(Path(cfg_path).read_text(encoding = "utf-8"))) + refs = _auto_map_refs(json.loads(Path(cfg_path).read_text(encoding = "utf-8-sig"))) except Exception: continue repos.update(repo for repo, _fn in refs if repo) diff --git a/studio/backend/utils/ssm_runtime.py b/studio/backend/utils/ssm_runtime.py index ca7e2309f9..b864e78608 100644 --- a/studio/backend/utils/ssm_runtime.py +++ b/studio/backend/utils/ssm_runtime.py @@ -23,6 +23,7 @@ import threading from typing import Any, Callable, Optional from loggers import get_logger +from utils.child_stdio import utf8_child_env from utils.wheel_utils import ( direct_wheel_url, install_wheel, @@ -254,6 +255,12 @@ def _install_kernel( "stdout": subprocess.PIPE, "stderr": subprocess.STDOUT, "text": True, + # pip and the compilers it drives write UTF-8 down this pipe; the Windows + # ANSI codepage would mojibake or raise over a fine install. + "encoding": "utf-8", + "errors": "replace", + # Make the Python child emit the UTF-8 we decode above. + "env": utf8_child_env(), } if is_hip: run_kwargs["timeout"] = 1800 # ROCm builds can take 10-30 min @@ -261,7 +268,8 @@ def _install_kernel( if "--gcc-install-dir" not in existing: gcc_dir = _hipcc_gcc_install_dir() if gcc_dir: - _env = os.environ.copy() + # Extends the UTF-8 env above rather than replacing it. + _env = dict(run_kwargs["env"]) _env["HIPCC_COMPILE_FLAGS_APPEND"] = ( f"{existing} --gcc-install-dir={gcc_dir}".strip() ) diff --git a/studio/backend/utils/studio_version.py b/studio/backend/utils/studio_version.py index 82ade74bba..cfaba36a81 100644 --- a/studio/backend/utils/studio_version.py +++ b/studio/backend/utils/studio_version.py @@ -60,6 +60,8 @@ def _exact_git_studio_tag(repo_root: Path) -> str | None: stdout = subprocess.PIPE, stderr = subprocess.DEVNULL, text = True, + encoding = "utf-8", + errors = "replace", timeout = _GIT_TIMEOUT_SECONDS, ) except (OSError, subprocess.TimeoutExpired): @@ -81,6 +83,8 @@ def _git_branch(repo_root: Path) -> str | None: stdout = subprocess.PIPE, stderr = subprocess.DEVNULL, text = True, + encoding = "utf-8", + errors = "replace", timeout = _GIT_TIMEOUT_SECONDS, ) except (OSError, subprocess.TimeoutExpired): diff --git a/studio/backend/utils/transformers_version.py b/studio/backend/utils/transformers_version.py index b0a2da0e66..3774409009 100644 --- a/studio/backend/utils/transformers_version.py +++ b/studio/backend/utils/transformers_version.py @@ -44,6 +44,7 @@ import time from pathlib import Path from utils.native_path_leases import child_env_without_native_path_secret +from utils.child_stdio import utf8_child_env from utils.hf_cache_settings import get_hf_cache_paths from utils.subprocess_compat import ( windows_hidden_subprocess_kwargs as _windows_hidden_subprocess_kwargs, @@ -420,7 +421,7 @@ def _resolve_base_model(model_name: str) -> str: adapter_cfg_path = local_path / "adapter_config.json" if _safe_is_file(adapter_cfg_path): try: - with open(adapter_cfg_path, encoding = "utf-8") as f: + with open(adapter_cfg_path, encoding = "utf-8-sig") as f: cfg = json.load(f) base = cfg.get("base_model_name_or_path") if base: @@ -437,7 +438,7 @@ def _resolve_base_model(model_name: str) -> str: config_json_path = local_path / "config.json" if _safe_is_file(config_json_path): try: - with open(config_json_path, encoding = "utf-8") as f: + with open(config_json_path, encoding = "utf-8-sig") as f: cfg = json.load(f) # Unsloth writes model_name, HF writes _name_or_path; skip a self-reference. for _key in ("model_name", "_name_or_path"): @@ -544,7 +545,7 @@ def _adapter_base_from_hf_cache(model_name: str) -> str | None: ) for cfg_path in candidates: if cfg_path.is_file(): - base = json.loads(cfg_path.read_text(encoding = "utf-8")).get( + base = json.loads(cfg_path.read_text(encoding = "utf-8-sig")).get( "base_model_name_or_path" ) return base or None @@ -616,7 +617,7 @@ def _check_tokenizer_config_needs_v5(model_name: str, hf_token: str | None = Non local_tc = local_path / "tokenizer_config.json" if _safe_is_file(local_tc): try: - with open(local_tc, encoding = "utf-8") as f: + with open(local_tc, encoding = "utf-8-sig") as f: data = json.load(f) tokenizer_class = data.get("tokenizer_class", "") result = tokenizer_class in _TRANSFORMERS_5_TOKENIZER_CLASSES @@ -706,7 +707,7 @@ def _config_json_from_hf_cache(model_name: str) -> dict | None: ) for cfg_path in candidates: if cfg_path.is_file(): - with open(cfg_path, encoding = "utf-8") as f: + with open(cfg_path, encoding = "utf-8-sig") as f: return json.load(f) except Exception as exc: logger.debug("HF cache config.json lookup failed for '%s': %s", model_name, exc) @@ -731,7 +732,7 @@ def _load_config_json(model_name: str, hf_token: str | None = None) -> dict | No local_cfg = Path(model_name) / "config.json" if _safe_is_file(local_cfg): try: - with open(local_cfg, encoding = "utf-8") as f: + with open(local_cfg, encoding = "utf-8-sig") as f: cfg = json.load(f) _config_json_cache[cache_key] = cfg return cfg @@ -1271,9 +1272,10 @@ def _probe_autoconfig(target_dir: str, model_name: str, hf_token: str | None) -> [sys.executable, "-c", _PROBE_CONFIG_SCRIPT, target_dir, model_name], capture_output = True, text = True, + encoding = "utf-8", errors = "replace", timeout = _PROBE_TIMEOUT_SECS, - env = env, + env = utf8_child_env(env), **_windows_hidden_subprocess_kwargs(), ) except subprocess.TimeoutExpired: @@ -1811,7 +1813,11 @@ def _install_to_dir(pkg: str, target_dir: str) -> bool: stdout = subprocess.PIPE, stderr = subprocess.STDOUT, text = True, - env = get_hf_cache_paths().child_env(child_env_without_native_path_secret()), + encoding = "utf-8", + errors = "replace", + env = utf8_child_env( + get_hf_cache_paths().child_env(child_env_without_native_path_secret()) + ), **_windows_hidden_subprocess_kwargs(), ) if result.returncode == 0: @@ -1834,7 +1840,9 @@ def _install_to_dir(pkg: str, target_dir: str) -> bool: stdout = subprocess.PIPE, stderr = subprocess.STDOUT, text = True, - env = get_hf_cache_paths().child_env(child_env_without_native_path_secret()), + encoding = "utf-8", + errors = "replace", + env = utf8_child_env(get_hf_cache_paths().child_env(child_env_without_native_path_secret())), **_windows_hidden_subprocess_kwargs(), ) if result.returncode != 0: @@ -2079,7 +2087,7 @@ class SidecarSwapInProgress(RuntimeError): def _read_swap_lock(path: Path) -> dict | None: try: - data = json.loads(path.read_text(encoding = "utf-8")) + data = json.loads(path.read_text(encoding = "utf-8-sig")) return data if isinstance(data, dict) else {} except FileNotFoundError: return None @@ -2120,7 +2128,7 @@ def try_begin_sidecar_swap(kind: str = "install") -> bool: break if fd is not None: try: - with os.fdopen(fd, "w") as f: + with os.fdopen(fd, "w", encoding = "utf-8") as f: f.write( json.dumps( {"pid": os.getpid(), "at": time.time(), "token": token, "kind": kind} @@ -2466,7 +2474,11 @@ def _ensure_venv_llmcompressor_exists() -> bool: stdout = subprocess.PIPE, stderr = subprocess.STDOUT, text = True, - env = get_hf_cache_paths().child_env(child_env_without_native_path_secret()), + encoding = "utf-8", + errors = "replace", + env = utf8_child_env( + get_hf_cache_paths().child_env(child_env_without_native_path_secret()) + ), **_windows_hidden_subprocess_kwargs(), ) last_out = result.stdout or "" diff --git a/studio/backend/utils/update_status.py b/studio/backend/utils/update_status.py index ad9dabcf36..d4b8ca1c16 100644 --- a/studio/backend/utils/update_status.py +++ b/studio/backend/utils/update_status.py @@ -30,6 +30,7 @@ PYPI_SUCCESS_TTL_SECONDS = 12 * 60 * 60 PYPI_FAILURE_TTL_SECONDS = 60 * 60 RELEASE_NOTES_URL = "https://unsloth.ai/docs/new/changelog" DISABLE_ENV_VAR = "UNSLOTH_DISABLE_UPDATE_CHECK" +FAKE_UPDATE_ENV_VAR = "UNSLOTH_STUDIO_FAKE_UPDATE" LOCAL_INSTALL_SOURCES = {"editable", "local_path", "vcs", "local_repo"} @@ -107,11 +108,32 @@ def get_studio_install_source_status(current_version: str) -> dict[str, Any]: ) +def _is_version(value: str) -> bool: + try: + Version(value) + except InvalidVersion: + return False + return True + + def get_studio_update_status(current_version: str) -> dict[str, Any]: """Return public, read-only update status for the web UI.""" install_source = detect_install_source() + disabled = os.environ.get(DISABLE_ENV_VAR) == "1" - if os.environ.get(DISABLE_ENV_VAR) == "1": + # Dev-only: the popup is PyPI-install-only, so fake a version to review it + # from a checkout. The documented opt-out still wins. + forced_version = os.environ.get(FAKE_UPDATE_ENV_VAR, "").strip() + if forced_version and not disabled and _is_version(forced_version): + return _status_response( + current_version = current_version, + latest_version = forced_version, + install_source = "pypi", + update_available = True, + can_show_web_notification = True, + ) + + if disabled: return _status_response( current_version = current_version, latest_version = None, diff --git a/studio/backend/utils/utils.py b/studio/backend/utils/utils.py index e4964b8d04..e830ea2700 100644 --- a/studio/backend/utils/utils.py +++ b/studio/backend/utils/utils.py @@ -114,6 +114,8 @@ def hf_cache_snapshot_dir(model_name: str) -> Optional[Path]: snapshot = repo_dir / "snapshots" / commit if snapshot.is_dir(): return snapshot + # UnicodeDecodeError is a ValueError, not an OSError: a torn refs + # file must keep meaning "not cached here", not fail the offline check. except (OSError, UnicodeDecodeError): continue return None diff --git a/studio/backend/utils/wheel_utils.py b/studio/backend/utils/wheel_utils.py index 1b5926fd49..8ebdea3ac1 100644 --- a/studio/backend/utils/wheel_utils.py +++ b/studio/backend/utils/wheel_utils.py @@ -15,6 +15,7 @@ import urllib.request from typing import Callable from utils.native_path_leases import child_env_without_native_path_secret +from utils.child_stdio import utf8_child_env from utils.subprocess_compat import windows_hidden_subprocess_kwargs _logger = logging.getLogger(__name__) @@ -43,6 +44,8 @@ def has_blackwell_gpu() -> bool: stdout = subprocess.PIPE, stderr = subprocess.DEVNULL, text = True, + encoding = "utf-8", + errors = "replace", timeout = 10, env = child_env_without_native_path_secret(), ) @@ -102,8 +105,10 @@ def probe_torch_wheel_env(*, timeout: int | None = None) -> dict[str, str] | Non stdout = subprocess.PIPE, stderr = subprocess.PIPE, text = True, + encoding = "utf-8", + errors = "replace", timeout = timeout, - env = child_env_without_native_path_secret(), + env = utf8_child_env(child_env_without_native_path_secret()), **windows_hidden_subprocess_kwargs(), ) except subprocess.TimeoutExpired: @@ -201,6 +206,8 @@ def install_wheel( stdout = subprocess.PIPE, stderr = subprocess.STDOUT, text = True, + encoding = "utf-8", + errors = "replace", env = child_env_without_native_path_secret(), ) attempts.append(("uv", result)) @@ -213,7 +220,10 @@ def install_wheel( stdout = subprocess.PIPE, stderr = subprocess.STDOUT, text = True, - env = child_env_without_native_path_secret(), + encoding = "utf-8", + errors = "replace", + # Make the Python child emit the UTF-8 we decode above. + env = utf8_child_env(child_env_without_native_path_secret()), ) attempts.append(("pip", result)) return attempts diff --git a/studio/backend/utils/whisper_cpp_update.py b/studio/backend/utils/whisper_cpp_update.py index cac37c25fc..45a0faf674 100644 --- a/studio/backend/utils/whisper_cpp_update.py +++ b/studio/backend/utils/whisper_cpp_update.py @@ -121,7 +121,14 @@ def _installed_whisper_version(binary: Optional[str]) -> Optional[str]: if not binary: return None try: - proc = subprocess.run([binary, "--version"], capture_output = True, text = True, timeout = 20) + proc = subprocess.run( + [binary, "--version"], + capture_output = True, + text = True, + encoding = "utf-8", + errors = "replace", + timeout = 20, + ) except Exception: # pragma: no cover - defensive return None m = re.search(r"v?(\d+\.\d+\.\d+)", (proc.stderr or "") + (proc.stdout or "")) diff --git a/studio/frontend/src/app/provider.tsx b/studio/frontend/src/app/provider.tsx index d746ed952c..b076c8cf8d 100644 --- a/studio/frontend/src/app/provider.tsx +++ b/studio/frontend/src/app/provider.tsx @@ -214,7 +214,8 @@ function TauriUpdateLayer({ } return ( - <div className="pointer-events-none fixed bottom-4 right-4 z-[9998] flex w-[calc(100vw-2rem)] max-w-[400px] flex-col items-stretch gap-2"> + // Capped like the browser stack: the download panel shares it, so both must fit. + <div className="pointer-events-none fixed bottom-4 right-4 z-[9998] flex max-h-[calc(100dvh_-_2rem)] flex-col items-end gap-2"> <UpdateBanner status={update.status} info={update.info} @@ -223,6 +224,7 @@ function TauriUpdateLayer({ isExternalServer={isExternalServer} updatePolicyMode={update.updatePolicyMode} manualReleaseUrl={update.manualReleaseUrl} + releasePageUrl={update.releasePageUrl} positioned={false} onInstall={update.installUpdate} onDismiss={update.dismiss} @@ -379,9 +381,11 @@ function TauriWrapper({ children }: { children: ReactNode }) { return ( <> {children} - {/* One bottom-right stack so overlays never overlap; they stack with a - gap, download panel anchored at the corner with banners above. */} - <div className="pointer-events-none fixed bottom-4 right-4 z-[9998] flex w-[calc(100vw-2rem)] max-w-[400px] flex-col items-stretch gap-2"> + {/* One bottom-right stack so overlays never overlap: download panel at the + corner, banners above, each owning its width. */} + {/* Capped to the viewport, or a long download list plus expanded notes + pushes the top of the stack off screen. */} + <div className="pointer-events-none fixed bottom-4 right-4 z-[9998] flex max-h-[calc(100dvh_-_2rem)] flex-col items-end gap-2"> <WebUpdateBanner positioned={false} enabled={!WEB_UPDATE_HIDDEN_ROUTES.has(pathname)} diff --git a/studio/frontend/src/components/app-sidebar.tsx b/studio/frontend/src/components/app-sidebar.tsx index d263a6a739..ecbebba326 100644 --- a/studio/frontend/src/components/app-sidebar.tsx +++ b/studio/frontend/src/components/app-sidebar.tsx @@ -1226,7 +1226,9 @@ export function AppSidebar() { openNewChat(null); }} className={cn( - "flex items-center gap-[6px] select-none transition-opacity", + // min-w-0 so a narrow sidebar truncates the wordmark + // instead of pushing the search icon over the logo. + "flex min-w-0 items-center gap-[6px] select-none transition-opacity", chatDisabled && "pointer-events-none opacity-50", )} aria-label={t("shell.aria.home")} @@ -1238,17 +1240,17 @@ export function AppSidebar() { <img src="/circle-logo-small.png" alt="Unsloth" - className="h-[calc(26px+0.5rem*var(--ui-font-scale,1))] w-[calc(26px+0.5rem*var(--ui-font-scale,1))] rounded-full object-cover" + className="h-[calc(26px+0.5rem*var(--ui-font-scale,1))] w-[calc(26px+0.5rem*var(--ui-font-scale,1))] shrink-0 rounded-full object-cover" /> - <span className="font-heading text-[calc(13px+0.5rem*var(--ui-font-scale,1))] font-semibold tracking-[0em] leading-none text-black dark:text-white dark:tracking-[0.02em]"> + <span className="truncate font-heading text-[calc(13px+0.5rem*var(--ui-font-scale,1))] font-semibold tracking-[0em] leading-none text-black dark:text-white dark:tracking-[0.02em]"> unsloth </span> - <span className="nav-badge ml-0.5 inline-flex items-center justify-center rounded-full border border-nav-beta-border px-[5px] pt-[3px] pb-[2px] text-[calc(0.5rem*var(--ui-font-scale,1))] font-medium leading-none tracking-[0.04em] text-nav-fg-muted antialiased subpixel-antialiased shadow-[0_1px_2px_rgba(0,0,0,0.06)] dark:shadow-[0_1px_2px_rgba(0,0,0,0.35)]"> + <span className="nav-badge ml-0.5 inline-flex shrink-0 items-center justify-center rounded-full border border-nav-beta-border px-[5px] pt-[3px] pb-[2px] text-[calc(0.5rem*var(--ui-font-scale,1))] font-medium leading-none tracking-[0.04em] text-nav-fg-muted antialiased subpixel-antialiased shadow-[0_1px_2px_rgba(0,0,0,0.06)] dark:shadow-[0_1px_2px_rgba(0,0,0,0.35)]"> {t("shell.beta")} </span> </Link> )} - <div className="flex items-center gap-0.5"> + <div className="flex shrink-0 items-center gap-0.25"> <Tooltip> <TooltipPrimitive.Trigger asChild> <button @@ -1257,7 +1259,7 @@ export function AppSidebar() { useChatSearchStore.getState().open(); closeMobileIfOpen(); }} - className="inline-flex h-[33px] w-[32px] cursor-pointer items-center justify-center rounded-[10px] text-nav-icon-idle dark:text-nav-fg-muted transition-colors hover:bg-nav-surface-hover hover:text-black dark:hover:text-white focus-visible:outline-none focus-visible:ring-1 focus-visible:ring-ring" + className="inline-flex h-[33px] w-[28px] cursor-pointer items-center justify-center rounded-[10px] text-nav-icon-idle dark:text-nav-fg-muted transition-colors hover:bg-nav-surface-hover hover:text-black dark:hover:text-white focus-visible:outline-none focus-visible:ring-1 focus-visible:ring-ring" aria-label={t("shell.navigation.search")} > <HugeiconsIcon icon={Search01Icon} strokeWidth={1.75} className="size-icon" /> @@ -1281,7 +1283,7 @@ export function AppSidebar() { <button type="button" onClick={togglePinned} - className="inline-flex h-[33px] w-[32px] cursor-pointer items-center justify-center rounded-[10px] text-nav-icon-idle dark:text-nav-fg-muted transition-colors hover:bg-nav-surface-hover hover:text-black dark:hover:text-white focus-visible:outline-none focus-visible:ring-1 focus-visible:ring-ring" + className="inline-flex h-[33px] w-[28px] cursor-pointer items-center justify-center rounded-[10px] text-nav-icon-idle dark:text-nav-fg-muted transition-colors hover:bg-nav-surface-hover hover:text-black dark:hover:text-white focus-visible:outline-none focus-visible:ring-1 focus-visible:ring-ring" aria-label={t("shell.aria.closeSidebar")} > <HugeiconsIcon icon={LayoutAlignLeftIcon} strokeWidth={1.75} className="size-icon" /> @@ -1325,10 +1327,10 @@ export function AppSidebar() { )} </SidebarHeader> - {/* Uniform pl-1.5 pr-2 keeps every hover pill the same width, inset from the edge. */} + {/* Uniform pl-1.5 pr-1.75 keeps every hover pill the same width, inset from the edge. */} <SidebarGroup className={cn( - "group-data-[collapsible=icon]:px-0 pl-1.5 pr-2 shrink-0 transition-[padding]", + "group-data-[collapsible=icon]:px-0 pl-1.5 pr-1.75 shrink-0 transition-[padding]", showCompactMacBrand ? "pt-0" : "pt-[9px]", // Scrolled: New Chat is pinned, give a little gap below it. scrolled ? "pb-[5px]" : "pb-px", @@ -1417,7 +1419,7 @@ export function AppSidebar() { scrolled && "is-scrolled", )} > - <SidebarGroup className="group-data-[collapsible=icon]:px-0 pl-1.5 pr-2 py-0 shrink-0"> + <SidebarGroup className="group-data-[collapsible=icon]:px-0 pl-1.5 pr-1.75 py-0 shrink-0"> <SidebarGroupContent> <SidebarMenu> <NavItem @@ -1499,7 +1501,7 @@ export function AppSidebar() { </CollapsibleTrigger> </SidebarGroupLabel> <CollapsibleContent> - <SidebarGroupContent className="pl-1.5 pr-2"> + <SidebarGroupContent className="pl-1.5 pr-1.75"> <SidebarMenu> <NavItem icon={TestTubeOutlineIcon} @@ -1574,7 +1576,7 @@ export function AppSidebar() { </CollapsibleTrigger> </SidebarGroupLabel> <CollapsibleContent> - <SidebarGroupContent className="pl-1.5 pr-2"> + <SidebarGroupContent className="pl-1.5 pr-1.75"> <SidebarMenu> {pinnedProjectRecords.map((project) => { const projectChats = @@ -1721,7 +1723,7 @@ export function AppSidebar() { </CollapsibleTrigger> </SidebarGroupLabel> <CollapsibleContent> - <SidebarGroupContent className="pl-1.5 pr-2"> + <SidebarGroupContent className="pl-1.5 pr-1.75"> <SidebarMenu> {recentChatItems.map((item) => renderChatSidebarItem(item, "recent"), @@ -1753,7 +1755,7 @@ export function AppSidebar() { </CollapsibleTrigger> </SidebarGroupLabel> <CollapsibleContent> - <SidebarGroupContent className="pl-1.5 pr-2"> + <SidebarGroupContent className="pl-1.5 pr-1.75"> <SidebarMenu> {runItems.map((run) => { // Explicit selection wins. Otherwise highlight the active diff --git a/studio/frontend/src/components/llama-update-banner.tsx b/studio/frontend/src/components/llama-update-banner.tsx index 2729558630..5276eda858 100644 --- a/studio/frontend/src/components/llama-update-banner.tsx +++ b/studio/frontend/src/components/llama-update-banner.tsx @@ -134,7 +134,7 @@ export function LlamaUpdateBanner({ className={cn( positioned ? "fixed bottom-4 right-4 z-[9998] w-[calc(100vw-2rem)] max-w-[400px]" - : "pointer-events-auto w-full", + : "pointer-events-auto w-[calc(100vw-2rem)] max-w-[400px]", )} data-testid="llama-update-banner" > diff --git a/studio/frontend/src/components/tauri/update-banner.tsx b/studio/frontend/src/components/tauri/update-banner.tsx index 6f5e655889..49c9c44aaa 100644 --- a/studio/frontend/src/components/tauri/update-banner.tsx +++ b/studio/frontend/src/components/tauri/update-banner.tsx @@ -2,6 +2,7 @@ // Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 import { Button } from "@/components/ui/button"; +import { ReleaseNotesPanel } from "@/components/update/release-notes-panel"; import type { DesktopUpdatePolicyMode, RetainedUpdateFailure, @@ -22,6 +23,8 @@ interface UpdateBannerProps { isExternalServer?: boolean; updatePolicyMode: DesktopUpdatePolicyMode; manualReleaseUrl: string | null; + // Release page for this version, preferred over the generic changelog. + releasePageUrl?: string | null; // false fills a shared overlay stack; true self-anchors. positioned?: boolean; onInstall: () => void; @@ -30,6 +33,7 @@ interface UpdateBannerProps { } const EASE_OUT_QUART: [number, number, number, number] = [0.165, 0.84, 0.44, 1]; +const LEADING_V = /^v/; function formatVersion(version: string | null | undefined): string { if (!version) return ""; @@ -44,6 +48,7 @@ export function UpdateBanner({ isExternalServer = false, updatePolicyMode, manualReleaseUrl, + releasePageUrl = null, positioned = true, onInstall, onDismiss, @@ -52,6 +57,8 @@ export function UpdateBanner({ const [copying, setCopying] = useState(false); const [manualReport, setManualReport] = useState<string | null>(null); const [manualMessage, setManualMessage] = useState<string | null>(null); + // Version whose notes are expanded; a new offer collapses the panel. + const [notesVersion, setNotesVersion] = useState<string | null>(null); const showFailure = Boolean(lastFailure) && !dismissed; const showAvailable = status === "available" && !dismissed && !showFailure; const show = showFailure || (showAvailable && Boolean(info)); @@ -62,6 +69,11 @@ export function UpdateBanner({ const currentVersion = formatVersion(info?.currentVersion); const latestVersion = formatVersion(info?.version); const Icon = showFailure ? CircleAlert : Download; + // Keyed by the backend release, not the app's SemVer; headings drop the v. + const notesTargetVersion = + (info?.pypiVersion ?? info?.version)?.replace(LEADING_V, "") ?? null; + const notesOpen = + notesTargetVersion !== null && notesVersion === notesTargetVersion; async function handleCopyDiagnostics() { setCopying(true); @@ -94,13 +106,14 @@ export function UpdateBanner({ exit={{ opacity: 0, y: 8, scale: 0.97 }} transition={{ duration: 0.35, ease: EASE_OUT_QUART }} className={cn( + // Wider than the other overlays: notes preview plus three buttons. positioned - ? "fixed bottom-4 right-4 z-[9999] w-[calc(100vw-2rem)] max-w-[400px]" - : "pointer-events-auto w-full", + ? "fixed bottom-4 right-4 z-[9999] w-[calc(100vw-2rem)] max-w-[448px]" + : "pointer-events-auto flex min-h-0 w-[calc(100vw-2rem)] max-w-[448px] flex-col", )} data-testid="tauri-update-banner" > - <div className="relative overflow-hidden rounded-[24px] bg-white px-5 pb-4 pt-5 shadow-[0_2px_8px_-2px_rgba(0,0,0,0.16)] dark:bg-card dark:shadow-[0_8px_28px_-6px_rgba(0,0,0,0.28)]"> + <div className="relative flex max-h-[calc(100dvh_-_2rem)] flex-col overflow-hidden rounded-[24px] bg-white px-5 pb-4 pt-5 shadow-[0_2px_8px_-2px_rgba(0,0,0,0.16)] dark:bg-card dark:shadow-[0_8px_28px_-6px_rgba(0,0,0,0.28)]"> <button type="button" onClick={onDismiss} @@ -160,7 +173,40 @@ export function UpdateBanner({ </p> )} - <div className="mt-4 flex flex-wrap items-center justify-end gap-x-1 gap-y-2"> + {!showFailure && notesTargetVersion ? ( + <ReleaseNotesPanel + version={notesTargetVersion} + open={notesOpen} + // Used only if CHANGELOG.md has no section for this version. + fallbackMarkdown={info?.body ?? null} + className="min-h-0 flex-1" + releaseNotesUrl={releasePageUrl ?? manualReleaseUrl} + /> + ) : null} + + <div + className={cn( + "mt-4 flex flex-wrap items-center gap-x-1 gap-y-2", + !showFailure && notesTargetVersion + ? "justify-between" + : "justify-end", + )} + > + {!showFailure && notesTargetVersion ? ( + <Button + size="sm" + variant="ghost" + // same type size as the action buttons + className="-ml-2 h-auto whitespace-nowrap rounded-full px-2.5 py-2 text-ui-13 font-medium text-foreground" + onClick={() => + setNotesVersion(notesOpen ? null : notesTargetVersion) + } + aria-expanded={notesOpen} + data-testid="tauri-update-release-notes-toggle" + > + {notesOpen ? "Hide release notes" : "Show release notes"} + </Button> + ) : null} {showFailure ? ( <> <Button @@ -187,28 +233,31 @@ export function UpdateBanner({ onClick={onInstall} disabled={installDisabled} > - {isManualLinuxPackage ? "Open release page" : "Retry update"} + {isManualLinuxPackage + ? "Open release page" + : "Retry update"} </Button> </> ) : ( - <> + // wrap + right-align so the action pair stays together + <div className="flex flex-wrap items-center justify-end gap-x-1 gap-y-2"> <Button size="sm" variant="ghost" - className="h-auto rounded-full px-3 py-2 text-ui-13 font-medium text-foreground" + className="h-auto whitespace-nowrap rounded-full px-2.5 py-2 text-ui-13 font-medium text-foreground" onClick={onDismiss} > Remind me later </Button> <Button size="sm" - className="-mr-1 h-auto rounded-full px-3.5 py-2 text-ui-13" + className="-mr-1 h-auto whitespace-nowrap rounded-full px-3 py-2 text-ui-13" onClick={onInstall} disabled={installDisabled} > {isManualLinuxPackage ? "Open release page" : "Update"} </Button> - </> + </div> )} </div> {manualMessage && ( diff --git a/studio/frontend/src/components/tauri/window-titlebar.tsx b/studio/frontend/src/components/tauri/window-titlebar.tsx index 6a0ff8741a..db57cd9960 100644 --- a/studio/frontend/src/components/tauri/window-titlebar.tsx +++ b/studio/frontend/src/components/tauri/window-titlebar.tsx @@ -2,6 +2,7 @@ // Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 import { useSidebarPin } from "@/hooks/use-sidebar-pin"; +import { useSidebarWidth } from "@/hooks/use-sidebar-width"; import { isTauri } from "@/lib/api-base"; import { cn } from "@/lib/utils"; import { @@ -110,9 +111,13 @@ export function WindowTitlebar({ const [enabled] = useState(shouldUseCustomWindowTitlebar); const [maximized, setMaximized] = useState(false); const { pinned, togglePinned } = useSidebarPin(); + // The titlebar sits outside the sidebar wrapper, so it cannot inherit + // --sidebar-width. Read the resized width from the same store instead. + const { width } = useSidebarWidth(); const sidebarWidth = showSidebarSurface ? pinned - ? "var(--studio-sidebar-expanded-width,17.5rem)" + ? // The live value only exists mid-drag; otherwise the committed width. + `var(--studio-sidebar-live-width, ${width}px)` : "var(--studio-sidebar-collapsed-width,3rem)" : "0px"; const contentBorderLeft = pinned ? `calc(${sidebarWidth} + 12px)` : "0px"; diff --git a/studio/frontend/src/components/ui/panel-resize-handle.tsx b/studio/frontend/src/components/ui/panel-resize-handle.tsx new file mode 100644 index 0000000000..9c1fd70b6f --- /dev/null +++ b/studio/frontend/src/components/ui/panel-resize-handle.tsx @@ -0,0 +1,317 @@ +// SPDX-License-Identifier: AGPL-3.0-only +// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 + +"use client" + +import * as React from "react" + +import { cn } from "@/lib/utils" +import { + Tooltip, + TooltipContent, + TooltipTrigger, +} from "@/components/ui/tooltip" +import { getClientPlatform } from "@/components/tauri/window-titlebar" + +/** Pointer travel (px) below which a drag counts as a plain click. */ +const DRAG_SLOP = 4 +/** A compatibility click lands immediately after pointer-up. */ +const CLICK_COMPAT_WINDOW_MS = 300 +/** Arrow-key resize step for keyboard users. */ +const RESIZE_STEP = 16 + +type DragState = { + startX: number + startWidth: number + moved: boolean +} + +export type PanelResizeHandleProps = { + /** Which edge of the panel the handle sits on. */ + edge: "left" | "right" + open: boolean + width: number + /** Uncapped stored preference, so a capped drag does not lower it. */ + stored: number + min: number + max: number + clamp: (px: number) => number + setWidth: (px: number) => void + resetWidth: () => void + onToggle: () => void + /** Element to paint the live width onto, and the property to paint. */ + target: () => HTMLElement | null + cssVar: string + /** Measured to start a drag from the rendered size when collapsed. */ + measure: () => number + label: string + toggleLabel: string + /** Translated tooltip copy; the caller owns the translation layer. */ + collapseHint: string + expandHint: string + dragHint: string + /** Shown in the tooltip when the panel has a toggle shortcut. */ + shortcut?: string + dataSlot?: string + className?: string + /** Mirrors the live width onto :root for chrome outside the panel. */ + rootVar?: string +} + +/** + * A draggable panel edge: drag to resize, click to collapse or expand. Arrow + * keys resize, Home restores the default. The width is painted straight to the + * target while dragging and only persisted on release. + */ +export function PanelResizeHandle({ + edge, + open, + width, + stored, + min, + max, + clamp, + setWidth, + resetWidth, + onToggle, + target, + cssVar, + measure, + label, + toggleLabel, + collapseHint, + expandHint, + dragHint, + shortcut, + dataSlot = "panel-resize-handle", + className, + rootVar, +}: PanelResizeHandleProps) { + const ref = React.useRef<HTMLButtonElement>(null) + const dragRef = React.useRef<DragState | null>(null) + const [dragging, setDragging] = React.useState(false) + const [hovered, setHovered] = React.useState(false) + const [focused, setFocused] = React.useState(false) + const [isMacPlatform] = React.useState(() => getClientPlatform().includes("mac")) + const hint = shortcut ? shortcut.replace("Mod", isMacPlatform ? "⌘" : "Ctrl+") : null + + // Cached on pointer down so no DOM walk per move. + const targetRef = React.useRef<HTMLElement | null>(null) + const frameRef = React.useRef(0) + const pendingRef = React.useRef(0) + // What the pointer asked for, before the viewport cap. Committing the capped + // value instead would quietly downgrade a stored preference on a narrow window. + const rawRef = React.useRef(0) + // When a pointer sequence last ended. The browser's compatibility click + // lands in the same tick, so only a click that close behind is a duplicate. + // A timestamp cannot go stale the way an armed flag does: a genuine cancel + // emits no click, and a later assistive-tech click still gets through. + const handledAtRef = React.useRef(0) + const committedRef = React.useRef(width) + React.useEffect(() => { + committedRef.current = width + }, [width]) + + const paint = React.useCallback( + (value: string) => { + targetRef.current?.style.setProperty(cssVar, value) + if (rootVar) { + document.documentElement.style.setProperty(rootVar, value) + } + }, + [cssVar, rootVar], + ) + + // Resizing relayouts the whole shell, and pointermove fires faster than the + // display refreshes, so coalesce to one paint per frame. + const paintWidth = React.useCallback( + (px: number) => { + pendingRef.current = px + if (frameRef.current) return + frameRef.current = requestAnimationFrame(() => { + frameRef.current = 0 + paint(`${pendingRef.current}px`) + }) + }, + [paint], + ) + + const endDrag = React.useCallback(() => { + // Only a sequence that actually started can produce a compatibility click. + // This also runs as the effect cleanup, where no drag happened. + if (dragRef.current) handledAtRef.current = Date.now() + dragRef.current = null + if (frameRef.current) { + cancelAnimationFrame(frameRef.current) + frameRef.current = 0 + } + // Hand the property back to the committed value. A commit re-renders with + // the new width; a cancel or a no-commit drag keeps DOM and store in step. + paint(`${committedRef.current}px`) + if (rootVar) document.documentElement.style.removeProperty(rootVar) + targetRef.current?.removeAttribute("data-resizing") + document.documentElement.removeAttribute("data-panel-resizing") + targetRef.current = null + setDragging(false) + document.body.style.removeProperty("cursor") + document.body.style.removeProperty("user-select") + }, [paint, rootVar]) + + const handlePointerDown = (event: React.PointerEvent<HTMLButtonElement>) => { + if (event.button !== 0) return + event.preventDefault() + event.currentTarget.setPointerCapture(event.pointerId) + targetRef.current = target() + // Collapsed: grow from the rendered size so the edge tracks the pointer. + const start = open ? width : measure() + dragRef.current = { startX: event.clientX, startWidth: start, moved: false } + pendingRef.current = start + rawRef.current = start + targetRef.current?.setAttribute("data-resizing", "true") + document.documentElement.setAttribute("data-panel-resizing", "true") + setDragging(true) + document.body.style.setProperty("cursor", "col-resize") + document.body.style.setProperty("user-select", "none") + } + + const handlePointerMove = (event: React.PointerEvent<HTMLButtonElement>) => { + const drag = dragRef.current + if (!drag) return + // A panel whose handle is on its left edge grows as the pointer moves left. + const delta = (edge === "left" ? -1 : 1) * (event.clientX - drag.startX) + if (!drag.moved && Math.abs(delta) < DRAG_SLOP) return + drag.moved = true + + const next = drag.startWidth + delta + rawRef.current = next + if (!open) { + // Past the minimum, dragging the collapsed edge reopens it. + if (next >= min) { + paintWidth(clamp(next)) + onToggle() + } + return + } + // Dragging inward stops at the minimum. Collapsing is click or the shortcut. + paintWidth(clamp(next)) + } + + const handlePointerUp = (event: React.PointerEvent<HTMLButtonElement>) => { + const drag = dragRef.current + if (!drag) return + if (event.currentTarget.hasPointerCapture(event.pointerId)) { + event.currentTarget.releasePointerCapture(event.pointerId) + } + endDrag() + + if (!drag.moved) { + onToggle() + return + } + // A drag below the minimum leaves the stored width alone. + if (!open) return + // Capped: the visible edge is already at the cap, so an outward pull cannot + // express intent beyond it. Committing would silently lower the larger + // hidden preference. A deliberate inward drag still commits. + if (stored > max && rawRef.current >= max) return + // Commit what was asked for, not the capped paint, so a drag on a narrow + // window cannot shrink a larger stored preference. setWidth clamps. + setWidth(rawRef.current) + } + + const handleKeyDown = (event: React.KeyboardEvent<HTMLButtonElement>) => { + // The collapse/expand the label advertises, for keyboard users. Pointer-up + // handles it for the mouse; a synthesized click never reaches it. + if (event.key === "Enter" || event.key === " ") { + // preventDefault cancels the native click, so nothing follows to guard + // against; arming here would swallow the next assistive-tech click. + event.preventDefault() + onToggle() + return + } + const outward = edge === "left" ? "ArrowLeft" : "ArrowRight" + const inward = edge === "left" ? "ArrowRight" : "ArrowLeft" + if (event.key === outward || event.key === inward) { + event.preventDefault() + if (!open) { + // Collapsed there is nothing to resize, so the outward arrow reopens. + if (event.key === outward) onToggle() + return + } + if (event.key === outward && stored > max && width >= max) return + setWidth(width + (event.key === outward ? RESIZE_STEP : -RESIZE_STEP)) + return + } + if (event.key === "Home") { + event.preventDefault() + resetWidth() + } + } + + // Clear a stuck cursor override if we unmount mid-drag. + React.useEffect(() => endDrag, [endDrag]) + + return ( + <Tooltip open={(hovered || focused) && !dragging}> + <TooltipTrigger asChild> + <button + ref={ref} + type="button" + data-slot={dataSlot} + data-dragging={dragging || undefined} + aria-label={open ? label : toggleLabel} + {...(open ? { "aria-orientation": "vertical" as const } : {})} + {...(open + ? { "aria-valuenow": width, "aria-valuemin": min, "aria-valuemax": max } + : {})} + role={open ? "separator" : "button"} + onPointerDown={handlePointerDown} + onPointerMove={handlePointerMove} + onPointerUp={handlePointerUp} + onPointerCancel={endDrag} + onKeyDown={handleKeyDown} + onClick={() => { + // Switch and voice control activate by dispatching a bare click + // with no pointer or key events, which nothing else here catches. + if (Date.now() - handledAtRef.current < CLICK_COMPAT_WINDOW_MS) return + onToggle() + }} + onPointerEnter={() => setHovered(true)} + onPointerLeave={() => setHovered(false)} + onFocus={(event) => setFocused(event.target.matches(":focus-visible"))} + onBlur={() => setFocused(false)} + className={cn( + "absolute inset-y-0 z-30 hidden w-2 touch-none select-none sm:block", + edge === "left" ? "-left-1" : "-right-1", + // `!` overrides the app-wide hand cursor on buttons. + open + ? "cursor-col-resize!" + : edge === "left" + ? "cursor-w-resize!" + : "cursor-e-resize!", + // Sits exactly on the panel border so hover recolours one line. + "after:absolute after:inset-y-0 after:w-px after:bg-transparent after:transition-colors after:duration-150", + edge === "left" ? "after:left-1" : "after:right-1", + "hover:after:bg-sidebar-ring/25 data-dragging:after:bg-sidebar-ring/25", + // The app zeroes the native outline on buttons, so mark focus here. + "focus-visible:outline-none focus-visible:after:bg-sidebar-ring/60", + className, + )} + /> + </TooltipTrigger> + <TooltipContent + side={edge === "left" ? "left" : "right"} + align="center" + className="tooltip-compact" + > + <span className="flex flex-col gap-px"> + <span> + {open ? collapseHint : expandHint} + {hint ? ` ${hint}` : ""} + </span> + <span className="opacity-70">{dragHint}</span> + </span> + </TooltipContent> + </Tooltip> + ) +} diff --git a/studio/frontend/src/components/ui/sidebar.tsx b/studio/frontend/src/components/ui/sidebar.tsx index 0fe82eb428..e26a55694f 100644 --- a/studio/frontend/src/components/ui/sidebar.tsx +++ b/studio/frontend/src/components/ui/sidebar.tsx @@ -24,13 +24,21 @@ import { TooltipContent, TooltipTrigger, } from "@/components/ui/tooltip" +import { PanelResizeHandle } from "@/components/ui/panel-resize-handle" +import { useT } from "@/i18n" import { useIsMobile } from "@/hooks/use-mobile" +import { + SIDEBAR_WIDTH_DEFAULT, + SIDEBAR_WIDTH_MIN, + clampSidebarWidth, + useSidebarWidth, +} from "@/hooks/use-sidebar-width" import { HugeiconsIcon } from "@hugeicons/react" import { LayoutAlignLeftIcon } from "@hugeicons/core-free-icons" const noop = () => {} -const SIDEBAR_WIDTH = "17.5rem" +const SIDEBAR_WIDTH = `${SIDEBAR_WIDTH_DEFAULT}px` const SIDEBAR_WIDTH_ICON = "3rem" const SIDEBAR_KEYBOARD_SHORTCUT = "b" @@ -46,6 +54,11 @@ type SidebarContextProps = { pinned: boolean setPinned: (value: boolean) => void togglePinned: () => void + width: number + storedWidth: number + maxWidth: number + setWidth: (value: number) => void + resetWidth: () => void } const SidebarContext = React.createContext<SidebarContextProps | null>(null) @@ -80,6 +93,13 @@ function SidebarProvider({ }) { const isMobile = useIsMobile() const [openMobile, setOpenMobile] = React.useState(false) + const { + width, + max: maxWidth, + stored: storedWidth, + setWidth, + resetWidth, + } = useSidebarWidth() const prevIsMobileRef = React.useRef(isMobile) React.useEffect(() => { @@ -163,8 +183,13 @@ function SidebarProvider({ pinned, setPinned, togglePinned, + width, + storedWidth, + maxWidth, + setWidth, + resetWidth, }), - [state, open, setOpen, isMobile, openMobile, setOpenMobile, toggleSidebar, hasPinMode, pinned, setPinned, togglePinned] + [state, open, setOpen, isMobile, openMobile, setOpenMobile, toggleSidebar, hasPinMode, pinned, setPinned, togglePinned, width, storedWidth, maxWidth, setWidth, resetWidth] ) return ( @@ -173,7 +198,8 @@ function SidebarProvider({ data-slot="sidebar-wrapper" style={ { - "--sidebar-width": SIDEBAR_WIDTH, + // The drag handle writes this same property live while resizing. + "--sidebar-width": `${width}px`, "--sidebar-width-icon": SIDEBAR_WIDTH_ICON, ...style, } as React.CSSProperties @@ -311,11 +337,64 @@ function Sidebar({ > {children} </div> + <SidebarResizeHandle side={side} /> </div> </div> ) } +/** + * The sidebar's draggable edge, over the shared panel handle. + */ +function SidebarResizeHandle({ + className, + side = "left", +}: { + className?: string + side?: "left" | "right" +}) { + const { open, toggleSidebar, width, storedWidth, maxWidth, setWidth, resetWidth } = + useSidebar() + const ref = React.useRef<HTMLDivElement>(null) + const t = useT() + + return ( + <div ref={ref} className="contents"> + <PanelResizeHandle + edge={side === "right" ? "left" : "right"} + open={open} + width={width} + stored={storedWidth} + min={SIDEBAR_WIDTH_MIN} + max={maxWidth} + clamp={clampSidebarWidth} + setWidth={setWidth} + resetWidth={resetWidth} + onToggle={toggleSidebar} + target={() => + ref.current?.closest<HTMLElement>('[data-slot="sidebar-wrapper"]') ?? null + } + cssVar="--sidebar-width" + // The custom titlebar renders outside the wrapper and cannot inherit it. + rootVar="--studio-sidebar-live-width" + measure={() => + ref.current + ?.closest<HTMLElement>('[data-slot="sidebar-container"]') + ?.getBoundingClientRect().width ?? SIDEBAR_WIDTH_MIN + } + label={t("shell.aria.resizeSidebar")} + toggleLabel={t("shell.aria.openSidebar")} + collapseHint={t("shell.resize.collapse")} + expandHint={t("shell.resize.expand")} + dragHint={t("shell.resize.drag")} + shortcut="ModB" + dataSlot="sidebar-resize-handle" + className={className} + /> + </div> + ) +} + function SidebarTrigger({ className, onClick, @@ -777,6 +856,7 @@ export { SidebarMenuSubItem, SidebarProvider, SidebarRail, + SidebarResizeHandle, SidebarSeparator, SidebarTrigger, useSidebar, diff --git a/studio/frontend/src/components/update/release-notes-panel.tsx b/studio/frontend/src/components/update/release-notes-panel.tsx new file mode 100644 index 0000000000..d98c855daa --- /dev/null +++ b/studio/frontend/src/components/update/release-notes-panel.tsx @@ -0,0 +1,251 @@ +// SPDX-License-Identifier: AGPL-3.0-only +// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 + +import { MarkdownPreview } from "@/components/markdown/markdown-preview"; +import { useReleaseNotes } from "@/hooks/use-release-notes"; +import { resolveChangelogLinks } from "@/lib/changelog-links"; +import { releaseNotesPreview } from "@/lib/release-notes-preview"; +import { cn } from "@/lib/utils"; +import { + type ReactElement, + type ReactNode, + useEffect, + useMemo, + useRef, +} from "react"; + +interface ReleaseNotesPanelProps { + // Notes are looked up for this exact version only. + version: string; + // Collapsed previews the top bullets; expanded scrolls the full notes. + open: boolean; + // Desktop updater's body, used only if CHANGELOG.md has no section here. + fallbackMarkdown?: string | null; + releaseNotesUrl?: string | null; + className?: string; +} + +const NOTES_LINK_CLASS = + "shrink-0 whitespace-nowrap text-ui-11 font-medium text-foreground underline underline-offset-2"; + +function NotesMessage({ + children, + action, +}: { + children: ReactNode; + action?: ReactNode; +}): ReactElement { + return ( + <div className="flex items-center justify-between gap-2 px-1 py-2"> + <p className="text-ui-11 text-muted-foreground">{children}</p> + {action} + </div> + ); +} + +function ChangelogLink({ href }: { href: string }): ReactElement { + return ( + <a + href={href} + target="_blank" + rel="noopener noreferrer" + className={NOTES_LINK_CLASS} + data-testid="update-release-notes-link" + > + Open changelog + </a> + ); +} + +export function ReleaseNotesPanel({ + version, + open, + fallbackMarkdown = null, + releaseNotesUrl = null, + className, +}: ReleaseNotesPanelProps): ReactElement | null { + // Fetched with the popup: the collapsed preview needs the notes too. + const { state, notes, retry } = useReleaseNotes({ version, enabled: true }); + const scrollRef = useRef<HTMLElement | null>(null); + + // The fallback stands in for "no section in the changelog", which the hook + // reports as ready. An error is retryable, and the desktop fallback is the + // updater's static blurb, so taking it there would hide Retry until cache expiry. + const source = notes?.matched + ? notes.markdown + : state === "error" + ? null + : (fallbackMarkdown ?? null); + // Notes target the repository, so relative links must point back at it. + const markdown = useMemo( + () => (source === null ? null : resolveChangelogLinks(source)), + [source], + ); + + // Notes that are only a code block or a table preview as nothing. + const preview = useMemo( + () => (markdown === null ? null : releaseNotesPreview(markdown)), + [markdown], + ); + + // Start at the top on expand, and again once async notes land. + useEffect(() => { + if (open && markdown && scrollRef.current) { + scrollRef.current.scrollTop = 0; + } + }, [open, markdown]); + + // Caller's URL wins: the API returns only the generic changelog, while the + // desktop banner passes this version's release page. + const notesUrl = releaseNotesUrl ?? notes?.releaseNotesUrl; + const link = notesUrl ? <ChangelogLink href={notesUrl} /> : null; + + // Nothing previewable yet or ever: keep the collapsed popup compact. + if ( + !open && + (!markdown || + state === "loading" || + state === "idle" || + preview?.items.length === 0) + ) { + return null; + } + + return ( + <div + className={cn("mt-3 flex min-h-0 flex-col", className)} + data-testid="update-release-notes-panel" + data-notes-state={state} + data-notes-version={version} + data-notes-open={open} + > + {/* borderless fill, lighter than the card in dark mode */} + <div className="flex min-h-0 flex-col rounded-[14px] bg-muted/40 px-3 py-1 dark:bg-white/[0.06]"> + {markdown ? ( + open ? ( + <section + ref={scrollRef} + // biome-ignore lint/a11y/noNoninteractiveTabindex: keyboard-scrollable region + tabIndex={0} + aria-label={`Release notes for version ${version}`} + // Long notes scroll here instead of pushing the buttons off screen. + className="hover-scrollbar max-h-64 min-h-0 flex-1 overflow-y-auto overscroll-contain py-3 pr-1" + data-testid="update-release-notes-scroll" + > + <MarkdownPreview + markdown={markdown} + // Streamdown ships headings at mt-6 and code at text-sm, and + // clears max-width on descendants, so rescale and re-cap both. + className="max-h-none overflow-visible border-0 bg-transparent p-0 text-ui-11 [&_[data-streamdown=link-safety-modal]>*]:max-w-md [&_img]:h-auto [&_img]:max-w-full [&>*:first-child]:mt-0 [&>*>*:first-child]:mt-0 [&_code]:text-[0.92em] [&_h1]:mt-4 [&_h1]:font-heading [&_h1]:text-ui-13 [&_h2]:mt-4 [&_h2]:font-heading [&_h2]:text-ui-13 [&_h3]:mt-4 [&_h3]:font-heading [&_h3]:text-ui-11 [&_pre]:text-[0.92em]" + /> + {notes?.truncated ? ( + <p className="mt-2 text-ui-10 text-muted-foreground/80"> + Notes truncated. See the full changelog. + </p> + ) : null} + </section> + ) : ( + <ReleaseNotesSummary preview={preview} /> + ) + ) : ( + <NotesStatus + state={state} + version={version} + link={link} + retry={retry} + /> + )} + </div> + {open && markdown && link ? ( + <div className="mt-2 flex justify-end px-1">{link}</div> + ) : null} + </div> + ); +} + +/** Collapsed view: the first few bullets, one line each where possible. */ +function ReleaseNotesSummary({ + preview, +}: { + preview: ReturnType<typeof releaseNotesPreview> | null; +}): ReactElement | null { + if (preview === null || preview.items.length === 0) { + return null; + } + const { items, remaining } = preview; + + return ( + <ul + className="space-y-1 py-2 pr-1" + data-testid="update-release-notes-summary" + > + {items.map((item, index) => ( + <li + // Two releases can carry the same bullet text, so index is the key. + key={`${index}-${item.lead}`} + className="flex gap-1.5 text-ui-11 leading-snug text-muted-foreground" + > + <span aria-hidden="true" className="text-muted-foreground/60"> + • + </span> + <span className="line-clamp-2 min-w-0"> + {/* lead sentence carries the change */} + <span className="font-medium text-foreground">{item.lead}</span> + {item.rest ? <span> {item.rest}</span> : null} + </span> + </li> + ))} + {remaining > 0 ? ( + <li className="pl-3 text-ui-10 text-muted-foreground/70"> + +{remaining} more + </li> + ) : null} + </ul> + ); +} + +function NotesStatus({ + state, + version, + link, + retry, +}: { + state: ReturnType<typeof useReleaseNotes>["state"]; + version: string; + link: ReactNode; + retry: () => void; +}): ReactElement { + if (state === "loading" || state === "idle") { + return <NotesMessage>Loading release notes...</NotesMessage>; + } + + if (state === "error") { + return ( + <NotesMessage + action={ + // The changelog page may be reachable when the lookup is not. + <span className="flex shrink-0 items-center gap-3"> + <button + type="button" + onClick={retry} + className={NOTES_LINK_CLASS} + data-testid="update-release-notes-retry" + > + Retry + </button> + {link} + </span> + } + > + Could not load release notes. + </NotesMessage> + ); + } + + // Matched nothing: link out rather than show another release's notes. + return ( + <NotesMessage action={link}> + No release notes published for {version} yet. + </NotesMessage> + ); +} diff --git a/studio/frontend/src/components/web/update-banner.tsx b/studio/frontend/src/components/web/update-banner.tsx index d8ae92bf5f..f36f5ec3cd 100644 --- a/studio/frontend/src/components/web/update-banner.tsx +++ b/studio/frontend/src/components/web/update-banner.tsx @@ -2,6 +2,7 @@ // Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 import { Button } from "@/components/ui/button"; +import { ReleaseNotesPanel } from "@/components/update/release-notes-panel"; import { type DeviceType, usePlatformStore } from "@/config/env"; import { useWebUpdateCheck } from "@/hooks/use-web-update-check"; import { isTauri } from "@/lib/api-base"; @@ -40,6 +41,7 @@ export function WebUpdateBanner({ const deviceType = usePlatformStore((s) => s.deviceType); const installCmd = installCommandForDevice(deviceType); const [copiedVersion, setCopiedVersion] = useState<string | null>(null); + const [notesVersion, setNotesVersion] = useState<string | null>(null); const dismissTimerRef = useRef<ReturnType<typeof setTimeout> | null>(null); useEffect(() => { @@ -68,6 +70,8 @@ export function WebUpdateBanner({ } const copied = status != null && copiedVersion === status.latestVersion; + // Keyed by version so a new offer collapses the panel. + const notesOpen = status != null && notesVersion === status.latestVersion; return ( <AnimatePresence> @@ -78,13 +82,14 @@ export function WebUpdateBanner({ exit={{ opacity: 0, y: 8, scale: 0.97 }} transition={{ duration: 0.35, ease: EASE_OUT_QUART }} className={cn( + // Wider than the other overlays: notes preview plus three buttons. positioned - ? "fixed bottom-4 right-4 z-[9999] w-[calc(100vw-2rem)] max-w-[400px]" - : "pointer-events-auto w-full", + ? "fixed bottom-4 right-4 z-[9999] w-[calc(100vw-2rem)] max-w-[448px]" + : "pointer-events-auto flex min-h-0 w-[calc(100vw-2rem)] max-w-[448px] flex-col", )} data-testid="web-update-banner" > - <div className="relative overflow-hidden rounded-[24px] bg-white px-5 pb-4 pt-5 shadow-[0_2px_8px_-2px_rgba(0,0,0,0.16)] dark:bg-card dark:shadow-[0_8px_28px_-6px_rgba(0,0,0,0.28)]"> + <div className="relative flex max-h-[calc(100dvh_-_2rem)] flex-col overflow-hidden rounded-[24px] bg-white px-5 pb-4 pt-5 shadow-[0_2px_8px_-2px_rgba(0,0,0,0.16)] dark:bg-card dark:shadow-[0_8px_28px_-6px_rgba(0,0,0,0.28)]"> <button type="button" onClick={dismiss} @@ -127,22 +132,33 @@ export function WebUpdateBanner({ </div> </div> + <ReleaseNotesPanel + version={status.latestVersion} + open={notesOpen} + releaseNotesUrl={RELEASE_NOTES_URL} + className="min-h-0 flex-1" + /> + + {/* one row at one type size; wraps only on narrow viewports */} <div className="mt-4 flex flex-wrap items-center justify-between gap-y-2"> - <a - href={RELEASE_NOTES_URL} - target="_blank" - rel="noopener noreferrer" - className="-ml-2 whitespace-nowrap rounded-full px-2.5 py-2 text-ui-13 font-medium text-foreground transition-colors hover:bg-muted" - data-testid="web-update-release-notes-link" + <Button + size="sm" + variant="ghost" + className="-ml-2 h-auto whitespace-nowrap rounded-full px-2.5 py-2 text-ui-13 font-medium text-foreground" + onClick={() => + setNotesVersion(notesOpen ? null : status.latestVersion) + } + aria-expanded={notesOpen} + data-testid="web-update-release-notes-toggle" > - Release notes - </a> + {notesOpen ? "Hide release notes" : "Show release notes"} + </Button> {/* wrap + right-align so buttons stack instead of clipping on very narrow banners */} <div className="flex flex-wrap items-center justify-end gap-x-1 gap-y-2"> <Button size="sm" variant="ghost" - className="h-auto rounded-full px-3 py-2 text-ui-13 font-medium text-foreground" + className="h-auto whitespace-nowrap rounded-full px-2.5 py-2 text-ui-13 font-medium text-foreground" onClick={snooze} data-testid="web-update-snooze-button" > @@ -151,7 +167,7 @@ export function WebUpdateBanner({ <Button size="sm" // -mr optically aligns the filled pill's edge with the card padding - className="-mr-1 h-auto rounded-full px-3.5 py-2 text-ui-13" + className="-mr-1 h-auto whitespace-nowrap rounded-full px-3 py-2 text-ui-13" onClick={handleCopyCommand} data-testid="web-update-copy-button" > diff --git a/studio/frontend/src/features/chat/chat-settings-sheet.tsx b/studio/frontend/src/features/chat/chat-settings-sheet.tsx index 6070bd2e40..fd41e558f9 100644 --- a/studio/frontend/src/features/chat/chat-settings-sheet.tsx +++ b/studio/frontend/src/features/chat/chat-settings-sheet.tsx @@ -18,6 +18,7 @@ import { DropdownMenuTrigger, } from "@/components/ui/dropdown-menu"; import { InfoHint } from "@/components/ui/info-hint"; +import { PanelResizeHandle } from "@/components/ui/panel-resize-handle"; import { InputGroup, InputGroupAddon, @@ -44,7 +45,13 @@ import { Tooltip, TooltipContent } from "@/components/ui/tooltip"; import { NumericValueInput, snapToStep } from "@/features/model-picker"; import { RetrievalSettingsSection } from "@/features/rag"; import { useLlamaUpdateCheck } from "@/hooks/use-llama-update-check"; +import { + CHAT_SETTINGS_WIDTH_MIN, + clampChatSettingsWidth, + useChatSettingsWidth, +} from "@/hooks/use-chat-settings-width"; import { useIsMobile } from "@/hooks/use-mobile"; +import { useT } from "@/i18n"; import { ChevronDownStandardIcon } from "@/lib/chevron-icons"; import { toast } from "@/lib/toast"; import { cn } from "@/lib/utils"; @@ -52,7 +59,7 @@ import { Edit03Icon, LayoutAlignRightIcon } from "@hugeicons/core-free-icons"; import { HugeiconsIcon } from "@hugeicons/react"; import { Braces, ChevronDown, ExternalLink } from "lucide-react"; import { Tooltip as TooltipPrimitive } from "radix-ui"; -import { Fragment, type ReactNode } from "react"; +import { type CSSProperties, Fragment, type ReactNode } from "react"; import { useCallback, useEffect, useMemo, useRef, useState } from "react"; import { OpenAICodeExecSection } from "./components/openai-code-exec-section"; import { PermissionModeDropdown } from "./permission-mode-select"; @@ -363,6 +370,15 @@ export function ChatSettingsPanel({ onExternalProviderChange, externalProviderType = null, }: ChatSettingsPanelProps) { + const asideRef = useRef<HTMLElement>(null); + const t = useT(); + const { + width: settingsWidth, + max: settingsMax, + stored: settingsStored, + setWidth: setSettingsWidth, + resetWidth: resetSettingsWidth, + } = useChatSettingsWidth(); // Local models show every knob; providerCapabilities is only consulted when // isExternalModel. Unknown providers fall back to the OpenAI-compat shape via // getProviderCapabilities, so these flags never undercount support. @@ -461,6 +477,23 @@ export function ChatSettingsPanel({ // When the prompt overflows the inline box, clicking opens the popup editor. const systemPromptBoxRef = useRef<HTMLTextAreaElement>(null); const [systemPromptOverflows, setSystemPromptOverflows] = useState(false); + const promptObserverRef = useRef<ResizeObserver | null>(null); + const measurePromptRef = useRef<() => void>(() => {}); + // The section unmounts its textarea when collapsed, so observe through a + // callback ref: a stored observer would cling to the detached node and the + // remounted one would never be measured. + const attachPromptBox = useCallback((node: HTMLTextAreaElement | null) => { + systemPromptBoxRef.current = node; + promptObserverRef.current?.disconnect(); + promptObserverRef.current = null; + if (!node || typeof ResizeObserver === "undefined") return; + // Resizing rewraps the prompt, and a drag changes the width through a + // custom property without re-rendering, so watch the box itself. + const observer = new ResizeObserver(() => measurePromptRef.current()); + observer.observe(node); + promptObserverRef.current = observer; + measurePromptRef.current(); + }, []); const [activePresetBaseline, setActivePresetBaseline] = useState(params); const presets = useMemo(() => { return getOrderedPresets(customPresets); @@ -746,15 +779,20 @@ export function ChatSettingsPanel({ }, [open]); useEffect(() => { - const el = systemPromptBoxRef.current; - setSystemPromptOverflows( - currentSystemPrompt.length > 0 && - el != null && - el.clientHeight > 0 && - el.scrollHeight > el.clientHeight + 1, - ); + measurePromptRef.current = () => { + const el = systemPromptBoxRef.current; + setSystemPromptOverflows( + currentSystemPrompt.length > 0 && + el != null && + el.clientHeight > 0 && + el.scrollHeight > el.clientHeight + 1, + ); + }; + measurePromptRef.current(); }, [currentSystemPrompt, open]); + useEffect(() => () => promptObserverRef.current?.disconnect(), []); + const settingsScrollRef = useRef<HTMLDivElement>(null); const settingsContent = ( @@ -1124,7 +1162,7 @@ export function ChatSettingsPanel({ )} > <textarea - ref={systemPromptBoxRef} + ref={attachPromptBox} value={currentSystemPrompt} onChange={(e) => set("systemPrompt")(e.target.value)} onMouseDown={(e) => { @@ -1433,17 +1471,47 @@ export function ChatSettingsPanel({ return ( <aside + ref={asideRef} data-tour="chat-settings" + data-slot="chat-settings-panel" className={cn( - "relative z-50 shrink-0 overflow-hidden bg-panel-surface text-panel-surface-fg font-heading", - open ? "w-[17rem] border-l border-sidebar-border" : "w-0", + "relative z-50 shrink-0 bg-panel-surface text-panel-surface-fg font-heading", + open + ? "w-(--chat-settings-width) border-l border-sidebar-border" + : "w-0 overflow-hidden", )} - style={{ - height: "calc(100% - var(--studio-custom-titlebar-height, 0px))", - marginTop: "var(--studio-custom-titlebar-height, 0px)", - }} + style={ + { + "--chat-settings-width": `${settingsWidth}px`, + height: "calc(100% - var(--studio-custom-titlebar-height, 0px))", + marginTop: "var(--studio-custom-titlebar-height, 0px)", + } as CSSProperties + } > - <div className="h-full w-full">{settingsContent}</div> + {open ? ( + <PanelResizeHandle + edge="left" + open={open} + width={settingsWidth} + stored={settingsStored} + min={CHAT_SETTINGS_WIDTH_MIN} + max={settingsMax} + clamp={clampChatSettingsWidth} + setWidth={setSettingsWidth} + resetWidth={resetSettingsWidth} + onToggle={() => onOpenChange?.(!open)} + target={() => asideRef.current} + cssVar="--chat-settings-width" + measure={() => asideRef.current?.getBoundingClientRect().width ?? 0} + label={t("shell.aria.resizeRunSettings")} + toggleLabel={t("shell.aria.openRunSettings")} + collapseHint={t("shell.resize.collapse")} + expandHint={t("shell.resize.expand")} + dragHint={t("shell.resize.drag")} + dataSlot="chat-settings-resize-handle" + /> + ) : null} + <div className="h-full w-full overflow-hidden">{settingsContent}</div> </aside> ); } diff --git a/studio/frontend/src/features/hub/catalog/on-device-folders-dialog.tsx b/studio/frontend/src/features/hub/catalog/on-device-folders-dialog.tsx index 2b0f2c3a8d..bef2f79747 100644 --- a/studio/frontend/src/features/hub/catalog/on-device-folders-dialog.tsx +++ b/studio/frontend/src/features/hub/catalog/on-device-folders-dialog.tsx @@ -23,12 +23,21 @@ import { removeScanFolder, } from "@/features/hub"; import { FolderBrowser } from "@/features/model-picker"; -import { openModelsDir } from "@/features/native-intents"; +import { + openModelsDir, + pickHuggingFaceCacheDir, +} from "@/features/native-intents"; +import { + type HuggingFaceCacheSettings, + loadHuggingFaceCacheSettings, + updateHuggingFaceCacheSettings, +} from "@/features/settings"; import { isTauri } from "@/lib/api-base"; import { toast } from "@/lib/toast"; import { cn } from "@/lib/utils"; import { Delete02Icon, + DownloadCircle01Icon, FileSearchIcon, FolderAddIcon, FolderExportIcon, @@ -49,6 +58,12 @@ function formatError(error: unknown): string { return error instanceof Error ? error.message : String(error); } +function formatFreeSpace(bytes: number | null): string | null { + if (bytes === null || !Number.isFinite(bytes)) return null; + const gb = bytes / 1024 ** 3; + return gb >= 10 ? `${Math.round(gb)} GB free` : `${gb.toFixed(1)} GB free`; +} + export function OnDeviceFoldersDialog({ open, onOpenChange, @@ -68,6 +83,11 @@ export function OnDeviceFoldersDialog({ ); const refreshIdRef = useRef(0); const mutationVersionRef = useRef(0); + const [downloadCache, setDownloadCache] = + useState<HuggingFaceCacheSettings | null>(null); + const [downloadCacheLoaded, setDownloadCacheLoaded] = useState(false); + const [downloadBrowserOpen, setDownloadBrowserOpen] = useState(false); + const [downloadSaving, setDownloadSaving] = useState(false); const sortedFolders = useMemo( () => [...folders].sort((a, b) => a.path.localeCompare(b.path)), @@ -108,10 +128,66 @@ export function OnDeviceFoldersDialog({ return () => window.clearTimeout(timer); }, [open, refreshFolders]); + useEffect(() => { + if (!open) return; + let cancelled = false; + // The dialog stays mounted between opens, so re-arm the flag or a reopen + // shows the previous answer as if it were fresh. + setDownloadCacheLoaded(false); + loadHuggingFaceCacheSettings() + // Indexed locations do not depend on this. Null drops the stale path + // rather than offer Change against a location we could not confirm. + .catch(() => null) + .then((settings) => { + if (cancelled) return; + setDownloadCache(settings); + setDownloadCacheLoaded(true); + }); + return () => { + cancelled = true; + }; + }, [open]); + const handleInventoryChanged = useCallback(() => { onInventoryChange?.(); }, [onInventoryChange]); + // Relocating the cache changes which repos are on disk, but + // updateHuggingFaceCacheSettings already bumps the inventory version, which + // re-fetches every source. Refreshing here too would scan twice, since the + // two rounds carry different version keys and cannot be deduplicated. + const saveDownloadLocation = useCallback(async (nextPath: string | null) => { + setDownloadSaving(true); + try { + const settings = await updateHuggingFaceCacheSettings(nextPath); + setDownloadCache(settings); + toast.success("Download location updated", { + description: settings.cacheHome, + }); + } catch (err) { + toast.error("Couldn't update the download location", { + description: formatError(err), + }); + } finally { + setDownloadSaving(false); + } + }, []); + + const changeDownloadLocation = useCallback(async () => { + if (!isTauri) { + setDownloadBrowserOpen(true); + return; + } + try { + const picked = await pickHuggingFaceCacheDir(); + if (picked) await saveDownloadLocation(picked); + } catch (err) { + toast.error("Couldn't open the folder picker", { + description: formatError(err), + }); + } + }, [saveDownloadLocation]); + const handleAdd = useCallback( async (rawPath: string) => { const nextPath = rawPath.trim(); @@ -182,10 +258,10 @@ export function OnDeviceFoldersDialog({ <> <Dialog open={open} onOpenChange={onOpenChange}> <DialogContent - className="gap-0 overflow-hidden p-0 sm:max-w-[620px] lg:max-w-[660px] xl:max-w-[680px] [&_[data-slot=dialog-close]]:right-3 [&_[data-slot=dialog-close]]:top-3" + className="flex max-h-[90dvh] flex-col gap-0 overflow-hidden p-0 sm:max-w-[620px] lg:max-w-[660px] xl:max-w-[680px] [&_[data-slot=dialog-close]]:right-3 [&_[data-slot=dialog-close]]:top-3" overlayClassName="bg-black/20 backdrop-blur-none" > - <DialogHeader className="border-b border-border/60 px-5 py-4"> + <DialogHeader className="shrink-0 border-b border-border/60 px-5 py-4"> <DialogTitle className="text-ui-15"> On-device locations </DialogTitle> @@ -195,7 +271,78 @@ export function OnDeviceFoldersDialog({ </DialogDescription> </DialogHeader> - <div className="space-y-4 px-5 py-4"> + <div className="min-h-0 flex-1 space-y-4 overflow-y-auto px-5 py-4"> + <div className="rounded-[14px] border border-border/70 bg-muted/20 p-3"> + <div className="mb-2 flex items-center gap-2 text-ui-12 font-medium text-foreground"> + <HugeiconsIcon + icon={DownloadCircle01Icon} + strokeWidth={1.75} + className="size-3.5 text-muted-foreground" + /> + Download location + </div> + + <div className="flex flex-col gap-2 sm:flex-row sm:items-center"> + <Input + readOnly={true} + aria-label="Model download location" + value={ + downloadCache?.cacheHome ?? + (downloadCacheLoaded ? "Unknown" : "Loading...") + } + title={downloadCache?.cacheHome} + className="field-soft h-9 min-w-0 flex-1 rounded-full px-3 font-mono text-ui-12" + /> + <div className="flex shrink-0 items-center gap-2"> + <Button + type="button" + variant="outline" + size="sm" + onClick={() => void changeDownloadLocation()} + disabled={!downloadCache?.editable || downloadSaving} + className="h-9 rounded-full px-3 text-ui-12p5" + > + {downloadSaving ? ( + <Spinner className="size-3.5" /> + ) : ( + <HugeiconsIcon + icon={FolderSearchIcon} + strokeWidth={1.75} + data-icon="inline-start" + className="size-3.5" + /> + )} + Change + </Button> + {downloadCache?.isCustom ? ( + <Button + type="button" + variant="ghost" + size="sm" + onClick={() => void saveDownloadLocation(null)} + disabled={downloadSaving} + className="h-9 rounded-full px-3 text-ui-12p5 text-muted-foreground" + > + Use default + </Button> + ) : null} + </div> + </div> + + <p className="mt-2 text-ui-10p5 text-muted-foreground"> + {downloadCache?.source === "environment" + ? `Managed by the ${ + downloadCache.environmentVariable ?? "HF_HOME" + } environment variable.` + : [ + "New downloads only. Models already on disk stay where they are.", + formatFreeSpace(downloadCache?.freeBytes ?? null), + ] + .filter(Boolean) + .join(" · ")} + </p> + </div> + <div className="rounded-[14px] border border-border/70 bg-muted/20 p-3"> <div className="mb-2 flex items-center gap-2 text-ui-12 font-medium text-foreground"> <HugeiconsIcon @@ -425,6 +572,16 @@ export function OnDeviceFoldersDialog({ onOpenChange={setBrowserOpen} onSelect={(selectedPath) => void handleAdd(selectedPath)} /> + + <FolderBrowser + open={!isTauri && downloadBrowserOpen} + onOpenChange={setDownloadBrowserOpen} + onSelect={(selectedPath) => void saveDownloadLocation(selectedPath)} + initialPath={downloadCache?.cacheHome} + title="Choose model download location" + confirmLabel="Use for future downloads" + showModelHints={false} + /> </> ); } diff --git a/studio/frontend/src/features/hub/download-manager/download-manager-panel.tsx b/studio/frontend/src/features/hub/download-manager/download-manager-panel.tsx index 3e5a86a879..d68aaa5ab3 100644 --- a/studio/frontend/src/features/hub/download-manager/download-manager-panel.tsx +++ b/studio/frontend/src/features/hub/download-manager/download-manager-panel.tsx @@ -201,8 +201,10 @@ export function DownloadManagerPanel({ className={cn( // Standalone: anchor bottom-right. In a shared stack (positioned=false) // flow as a right-aligned row so overlays stack instead of overlapping. + // min-h-0 there: a flex item's min-height defaults to auto, so the capped + // stack would squeeze the update card instead of this list. "pointer-events-none", - positioned ? "fixed bottom-4 right-4 z-50" : "flex justify-end", + positioned ? "fixed bottom-4 right-4 z-50" : "flex min-h-0 justify-end", )} > {collapsed ? ( @@ -229,7 +231,7 @@ export function DownloadManagerPanel({ </TooltipContent> </Tooltip> ) : ( - <div className="hub-download-panel pointer-events-auto w-[min(400px,calc(100vw-2rem))] overflow-hidden"> + <div className="hub-download-panel pointer-events-auto flex min-h-0 w-[min(400px,calc(100vw-2rem))] flex-col overflow-hidden"> <div className="flex items-center gap-2 border-b border-foreground/[0.07] py-2 pl-4 pr-3"> <span className="min-w-0 flex-1 truncate text-ui-12p5 font-semibold text-foreground"> {headerLabel} diff --git a/studio/frontend/src/features/settings/index.ts b/studio/frontend/src/features/settings/index.ts index f27100a322..49d73dcfb3 100644 --- a/studio/frontend/src/features/settings/index.ts +++ b/studio/frontend/src/features/settings/index.ts @@ -3,6 +3,11 @@ export { SettingsDialog } from "./settings-dialog"; export { loadEmbeddingModelSettings } from "./api/embedding-model"; +export { + loadHuggingFaceCacheSettings, + updateHuggingFaceCacheSettings, +} from "./api/hugging-face-cache"; +export type { HuggingFaceCacheSettings } from "./api/hugging-face-cache"; export { loadPersonalization, savePersonalization, diff --git a/studio/frontend/src/features/settings/settings-dialog.tsx b/studio/frontend/src/features/settings/settings-dialog.tsx index f4ba98b1ce..2b46dbd55e 100644 --- a/studio/frontend/src/features/settings/settings-dialog.tsx +++ b/studio/frontend/src/features/settings/settings-dialog.tsx @@ -35,7 +35,10 @@ import { useRef, useState, } from "react"; -import { SETTINGS_SEARCH_INDEX } from "./settings-search"; +import { + SETTINGS_SEARCH_INDEX, + SETTINGS_SEARCH_KEYWORDS, +} from "./settings-search"; import { type SettingsTab, useSettingsDialogStore, @@ -157,8 +160,12 @@ export function SettingsDialog() { return TABS.map((tab) => { const tabLabel = t(tab.labelKey); const entries = SETTINGS_SEARCH_INDEX[tab.id] - .map((key) => t(key)) - .filter((label) => label.toLowerCase().includes(q)); + .filter((key) => { + if (t(key).toLowerCase().includes(q)) return true; + const keywordsKey = SETTINGS_SEARCH_KEYWORDS[key]; + return keywordsKey ? t(keywordsKey).toLowerCase().includes(q) : false; + }) + .map((key) => t(key)); const deduped = [...new Set(entries)]; return { tab, diff --git a/studio/frontend/src/features/settings/settings-search.ts b/studio/frontend/src/features/settings/settings-search.ts index a5b008579c..582602e061 100644 --- a/studio/frontend/src/features/settings/settings-search.ts +++ b/studio/frontend/src/features/settings/settings-search.ts @@ -146,3 +146,15 @@ export const SETTINGS_SEARCH_INDEX: Record<SettingsTab, TranslationKey[]> = { "settings.about.shutDownStudio", ], }; + +/** + * Extra terms a row matches on, beyond its own label. The value is a + * translation key holding space-separated synonyms; it is never rendered. + * Search matched labels only, so "models folder" or "directory" found nothing. + */ +export const SETTINGS_SEARCH_KEYWORDS: Partial< + Record<TranslationKey, TranslationKey> +> = { + "settings.resources.storage.modelsFolder": + "settings.resources.storage.modelsFolderKeywords", +}; diff --git a/studio/frontend/src/features/settings/tabs/general-tab.tsx b/studio/frontend/src/features/settings/tabs/general-tab.tsx index 1d5c533a25..11606d85af 100644 --- a/studio/frontend/src/features/settings/tabs/general-tab.tsx +++ b/studio/frontend/src/features/settings/tabs/general-tab.tsx @@ -74,6 +74,8 @@ const PREFS_KEYS: string[] = [ LOCALE_STORAGE_KEY, // UI state "sidebar_pinned", + "sidebar_width", + "chat_settings_width", "unsloth_sidebar_navigate_open", "unsloth_settings_active_tab", // Chat runtime prefs diff --git a/studio/frontend/src/hooks/use-chat-settings-width.ts b/studio/frontend/src/hooks/use-chat-settings-width.ts new file mode 100644 index 0000000000..6b2c75fe1e --- /dev/null +++ b/studio/frontend/src/hooks/use-chat-settings-width.ts @@ -0,0 +1,20 @@ +// SPDX-License-Identifier: AGPL-3.0-only +// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 + +import { createPanelWidthStore } from "./use-panel-width.ts"; + +/** The previous fixed 17rem, at a 16px root font size. */ +export const CHAT_SETTINGS_WIDTH_DEFAULT = 272; +/** Below this the sliders and their value pills start colliding. */ +export const CHAT_SETTINGS_WIDTH_MIN = 248; +export const CHAT_SETTINGS_WIDTH_MAX = 560; + +const store = createPanelWidthStore({ + key: "chat_settings_width", + min: CHAT_SETTINGS_WIDTH_MIN, + max: CHAT_SETTINGS_WIDTH_MAX, + fallback: CHAT_SETTINGS_WIDTH_DEFAULT, +}); + +export const clampChatSettingsWidth = store.clamp; +export const useChatSettingsWidth = store.useWidth; diff --git a/studio/frontend/src/hooks/use-panel-width.ts b/studio/frontend/src/hooks/use-panel-width.ts new file mode 100644 index 0000000000..d753c3a1bb --- /dev/null +++ b/studio/frontend/src/hooks/use-panel-width.ts @@ -0,0 +1,138 @@ +// SPDX-License-Identifier: AGPL-3.0-only +// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 + +import { useCallback, useSyncExternalStore } from "react"; + +/** Never let one panel eat more than this share of a narrow window. */ +const MAX_VIEWPORT_FRACTION = 0.4; + +export type PanelWidthStore = { + /** Clamps to what the current viewport allows. */ + clamp: (px: number) => number; + useWidth: () => { + width: number; + max: number; + /** The uncapped stored preference. */ + stored: number; + setWidth: (value: number) => void; + resetWidth: () => void; + }; +}; + +/** + * A persisted, viewport-aware width for a draggable panel. The preference is + * stored whole and an effective width is derived from it, so narrowing the + * window shrinks the panel without losing what the user picked. + */ +export function createPanelWidthStore({ + key, + min, + max, + fallback, +}: { + key: string; + min: number; + max: number; + fallback: number; +}): PanelWidthStore { + function maxWidth(): number { + if (typeof window === "undefined") return max; + // The floor wins on a narrow window; collapsing is the escape. + return Math.max(min, Math.min(max, window.innerWidth * MAX_VIEWPORT_FRACTION)); + } + + /** Clamps to the absolute range, ignoring the viewport. */ + function clampStored(px: number): number { + if (!Number.isFinite(px)) return fallback; + return Math.min(max, Math.max(min, Math.round(px))); + } + + function clamp(px: number): number { + return Math.min(maxWidth(), clampStored(px)); + } + + function load(): number { + if (typeof window === "undefined") return fallback; + try { + const raw = window.localStorage.getItem(key); + if (raw === null) return fallback; + return clampStored(Number.parseFloat(raw)); + } catch { + return fallback; + } + } + + let storedWidth = load(); + let effectiveWidth = clamp(storedWidth); + let effectiveMax = maxWidth(); + const listeners = new Set<() => void>(); + + let lastStored = storedWidth; + + function recompute() { + const nextWidth = clamp(storedWidth); + const nextMax = maxWidth(); + if ( + nextWidth === effectiveWidth && + nextMax === effectiveMax && + storedWidth === lastStored + ) { + return; + } + effectiveWidth = nextWidth; + effectiveMax = nextMax; + lastStored = storedWidth; + listeners.forEach((cb) => cb()); + } + + function subscribe(cb: () => void) { + // With no subscribers there is no resize listener, so the cache can be + // stale after a resize on a route that hides every panel. Refresh first; + // useSyncExternalStore re-reads the snapshot right after subscribing. + recompute(); + listeners.add(cb); + if (typeof window === "undefined") { + return () => listeners.delete(cb); + } + // Keep tabs in sync, same as the pin flag. + const onStorage = (e: StorageEvent) => { + if (e.key === key || e.key === null) { + storedWidth = load(); + effectiveWidth = clamp(storedWidth); + effectiveMax = maxWidth(); + cb(); + } + }; + window.addEventListener("storage", onStorage); + window.addEventListener("resize", recompute); + return () => { + listeners.delete(cb); + window.removeEventListener("storage", onStorage); + window.removeEventListener("resize", recompute); + }; + } + + function setWidthGlobal(next: number) { + const stored = clampStored(next); + if (stored !== storedWidth) { + storedWidth = stored; + try { + window.localStorage.setItem(key, String(stored)); + } catch {} + } + recompute(); + } + + function useWidth() { + const width = useSyncExternalStore(subscribe, () => effectiveWidth, () => fallback); + // What the viewport actually allows right now, for aria-valuemax. + const panelMax = useSyncExternalStore(subscribe, () => effectiveMax, () => max); + // The uncapped preference, so a capped drag can avoid lowering it. + const preference = useSyncExternalStore(subscribe, () => storedWidth, () => fallback); + const setWidth = useCallback((value: number) => setWidthGlobal(value), []); + const resetWidth = useCallback(() => setWidthGlobal(fallback), []); + return { width, max: panelMax, stored: preference, setWidth, resetWidth }; + } + + return { clamp, useWidth }; +} diff --git a/studio/frontend/src/hooks/use-release-notes.ts b/studio/frontend/src/hooks/use-release-notes.ts new file mode 100644 index 0000000000..7b1392fdf6 --- /dev/null +++ b/studio/frontend/src/hooks/use-release-notes.ts @@ -0,0 +1,146 @@ +// SPDX-License-Identifier: AGPL-3.0-only +// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 + +import { authFetch, hasAuthToken } from "@/features/auth"; +import { apiUrl } from "@/lib/api-base"; +import { useCallback, useEffect, useRef, useState } from "react"; + +// Keyed to one exact version, so a new update never pairs with older notes. +export interface ReleaseNotes { + version: string; + markdown: string | null; + matched: boolean; + truncated: boolean; + source: string | null; + releaseNotesUrl: string | null; + // Set when the lookup itself failed, as opposed to a version with no notes. + error: string | null; +} + +export type ReleaseNotesState = "idle" | "loading" | "ready" | "error"; + +// Desktop auto-auth installs its token after first paint, so a startup popup can +// ask before one exists. Wait briefly rather than fail. +const AUTH_POLL_MS = 250; +const AUTH_POLL_LIMIT = 40; + +interface UseReleaseNotesOptions { + version: string | null | undefined; + enabled?: boolean; +} + +type ApiObject = Record<string, unknown>; + +function stringOrNull(value: ApiObject, key: string): string | null { + const field = value[key]; + return typeof field === "string" && field.length > 0 ? field : null; +} + +function toReleaseNotes(value: unknown, version: string): ReleaseNotes | null { + if (!value || typeof value !== "object") { + return null; + } + const payload = value as ApiObject; + const notesVersion = stringOrNull(payload, "version"); + // A response for another version is not usable here. + if (notesVersion !== version) { + return null; + } + const markdown = stringOrNull(payload, "markdown"); + return { + version, + markdown, + matched: payload.matched === true && markdown !== null, + truncated: payload.truncated === true, + source: stringOrNull(payload, "source"), + releaseNotesUrl: stringOrNull(payload, "release_notes_url"), + error: stringOrNull(payload, "error"), + }; +} + +async function fetchReleaseNotes( + version: string, + refresh = false, +): Promise<ReleaseNotes | null> { + const query = `version=${encodeURIComponent(version)}${refresh ? "&refresh=true" : ""}`; + // authFetch, not fetch: an expired token is refreshed and retried. + const res = await authFetch(apiUrl(`/api/studio/release-notes?${query}`)); + if (!res.ok) { + throw new Error(`Release notes request failed: ${res.status}`); + } + + return toReleaseNotes(await res.json(), version); +} + +export function useReleaseNotes({ + version, + enabled = true, +}: UseReleaseNotesOptions) { + const [state, setState] = useState<ReleaseNotesState>("idle"); + const [notes, setNotes] = useState<ReleaseNotes | null>(null); + // Version the current state belongs to; a change invalidates it. + const requestedVersionRef = useRef<string | null>(null); + // Identifies one request, so an earlier response cannot overwrite a later one. + const requestIdRef = useRef(0); + + const load = useCallback((target: string, refresh = false) => { + requestedVersionRef.current = target; + requestIdRef.current += 1; + const requestId = requestIdRef.current; + setState("loading"); + setNotes(null); + fetchReleaseNotes(target, refresh) + .then((next) => { + // A newer request owns the state now. + if (requestIdRef.current !== requestId) { + return; + } + setNotes(next); + // A reported failure is retryable; "no notes for this version" is not. + const failed = !next || (!next.matched && next.error !== null); + setState(failed ? "error" : "ready"); + }) + .catch(() => { + if (requestIdRef.current === requestId) { + setNotes(null); + setState("error"); + } + }); + }, []); + + useEffect(() => { + if (!enabled || !version || requestedVersionRef.current === version) { + return; + } + if (hasAuthToken()) { + load(version); + return; + } + let attempts = 0; + const timer = window.setInterval(() => { + attempts += 1; + if (hasAuthToken() || attempts >= AUTH_POLL_LIMIT) { + window.clearInterval(timer); + // Out of patience: load anyway so the panel settles on retry. + load(version); + } + }, AUTH_POLL_MS); + return () => window.clearInterval(timer); + }, [enabled, version, load]); + + const retry = useCallback(() => { + if (version) { + requestedVersionRef.current = null; + // Bypass the cached remote failure, or retry waits for it to expire. + load(version, true); + } + }, [version, load]); + + // Never hand back another version's notes: state lags `version` by a render. + const matchesVersion = notes !== null && notes.version === version; + return { + state: notes !== null && !matchesVersion ? "loading" : state, + notes: matchesVersion ? notes : null, + retry, + }; +} diff --git a/studio/frontend/src/hooks/use-sidebar-width.ts b/studio/frontend/src/hooks/use-sidebar-width.ts new file mode 100644 index 0000000000..c670e254a8 --- /dev/null +++ b/studio/frontend/src/hooks/use-sidebar-width.ts @@ -0,0 +1,21 @@ +// SPDX-License-Identifier: AGPL-3.0-only +// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 + +import { createPanelWidthStore } from "./use-panel-width.ts"; + +/** The previous fixed 17.5rem, at a 16px root font size. */ +export const SIDEBAR_WIDTH_DEFAULT = 280; +/** Narrowest width that still fits the wordmark. Firefox is the constraint: + * it renders the heading ~3px wider than Chromium and WebKit. */ +export const SIDEBAR_WIDTH_MIN = 260; +export const SIDEBAR_WIDTH_MAX = 480; + +const store = createPanelWidthStore({ + key: "sidebar_width", + min: SIDEBAR_WIDTH_MIN, + max: SIDEBAR_WIDTH_MAX, + fallback: SIDEBAR_WIDTH_DEFAULT, +}); + +export const clampSidebarWidth = store.clamp; +export const useSidebarWidth = store.useWidth; diff --git a/studio/frontend/src/hooks/use-tauri-update.ts b/studio/frontend/src/hooks/use-tauri-update.ts index 8ebb4d2980..196e3cea2b 100644 --- a/studio/frontend/src/hooks/use-tauri-update.ts +++ b/studio/frontend/src/hooks/use-tauri-update.ts @@ -21,6 +21,8 @@ export type UpdateStatus = export interface UpdateInfo { version: string; currentVersion: string; + // Backend release this build pins; CHANGELOG.md is keyed by it, not the SemVer. + pypiVersion?: string; body?: string; date?: string; } @@ -42,10 +44,17 @@ interface DesktopUpdatePolicy { interface ManualUpdateInfo { version: string; currentVersion: string; + pypiVersion?: string | null; body?: string; date?: string; } +/** `pypi_version` from latest.json, which the updater passes through raw. */ +function rawPypiVersion(raw: Record<string, unknown>): string | undefined { + const value = raw.pypi_version; + return typeof value === "string" && value.length > 0 ? value : undefined; +} + export interface RetainedUpdateFailure { error: string; phase: UpdatePhase; @@ -162,6 +171,7 @@ export function useTauriUpdate(isExternalServer = false) { setInfo({ version: manualUpdate.version, currentVersion: manualUpdate.currentVersion, + pypiVersion: manualUpdate.pypiVersion ?? undefined, body: manualUpdate.body, date: manualUpdate.date, }); @@ -197,6 +207,7 @@ export function useTauriUpdate(isExternalServer = false) { setInfo({ version: update.version, currentVersion: update.currentVersion, + pypiVersion: rawPypiVersion(update.rawJson), body: update.body, date: update.date, }); @@ -384,10 +395,13 @@ export function useTauriUpdate(isExternalServer = false) { }); } + // Install target for Linux packages that cannot self-update. const manualReleaseUrl = updatePolicy.mode === "manual_linux_package" && info ? manualReleasePageUrl(updatePolicy, info.version) : null; + // Release page for the offered version, on every platform, for the notes link. + const releasePageUrl = info ? manualReleasePageUrl(updatePolicy, info.version) : null; return { status, @@ -401,6 +415,7 @@ export function useTauriUpdate(isExternalServer = false) { isExternalServer, updatePolicyMode: updatePolicy.mode, manualReleaseUrl, + releasePageUrl, installUpdate, retryUpdate, skipAndRestart, diff --git a/studio/frontend/src/i18n/locales/ar.ts b/studio/frontend/src/i18n/locales/ar.ts index 47d5032fae..a0ea2a1ee2 100644 --- a/studio/frontend/src/i18n/locales/ar.ts +++ b/studio/frontend/src/i18n/locales/ar.ts @@ -27,10 +27,18 @@ export const ar = { product: "Unsloth Studio", accountMenu: "قائمة حساب {name}", updateAvailable: "يتوفر تحديث", + resize: { + collapse: "انقر للطي", + expand: "انقر للتوسيع", + drag: "اسحب لتغيير الحجم", + }, aria: { home: "الصفحة الرئيسية لـ Unsloth", closeSidebar: "إغلاق الشريط الجانبي", openSidebar: "فتح الشريط الجانبي", + resizeSidebar: "تغيير حجم الشريط الجانبي أو طيه", + resizeRunSettings: "تغيير حجم إعدادات التشغيل أو إغلاقها", + openRunSettings: "فتح إعدادات التشغيل", chatOptions: "خيارات المحادثة", runOptions: "خيارات التدريب", }, @@ -317,6 +325,8 @@ export const ar = { diskUsage: "{used} مستخدم / {total}", diskFree: "{free} متاح", modelsFolder: "مجلد النماذج", + modelsFolderKeywords: + "النماذج مجلد دليل مسار موقع تنزيلات التنزيل ذاكرة التخزين المؤقت تخزين قرص محرك نقل تغيير models folder path hugging face", modelsFolderDescription: "المكان الذي تُخزَّن فيه النماذج المُنزَّلة.", openAction: "فتح", copyAction: "نسخ المسار", diff --git a/studio/frontend/src/i18n/locales/de.ts b/studio/frontend/src/i18n/locales/de.ts index cb7d603f42..8a5921c623 100644 --- a/studio/frontend/src/i18n/locales/de.ts +++ b/studio/frontend/src/i18n/locales/de.ts @@ -27,10 +27,18 @@ export const de = { product: "Unsloth Studio", accountMenu: "Kontomenü von {name}", updateAvailable: "Update verfügbar", + resize: { + collapse: "Zum Einklappen klicken", + expand: "Zum Ausklappen klicken", + drag: "Zum Ändern der Größe ziehen", + }, aria: { home: "Unsloth Startseite", closeSidebar: "Seitenleiste schließen", openSidebar: "Seitenleiste öffnen", + resizeSidebar: "Seitenleiste anpassen oder einklappen", + resizeRunSettings: "Ausführungseinstellungen anpassen oder schließen", + openRunSettings: "Ausführungseinstellungen öffnen", chatOptions: "Chat-Optionen", runOptions: "Trainingslauf-Optionen", }, @@ -330,9 +338,23 @@ export const de = { diskFree: "{free} frei", modelsFolder: "Modell-Ordner", modelsFolderDescription: - "Wo heruntergeladene Modelle gespeichert werden.", + "Wo heruntergeladene Modelle gespeichert werden. Ändern Sie ihn, um Modelle nicht auf dem Systemlaufwerk abzulegen.", + modelsFolderKeywords: + "Modelle Ordner Verzeichnis Pfad Speicherort Download Downloads Cache Speicher Festplatte Laufwerk verschieben ändern hugging face", + futureDownloads: "Nur neue Downloads", + environmentManaged: + "Wird über die Umgebungsvariable {variable} verwaltet.", + locationFree: "{free} frei", openAction: "Öffnen", copyAction: "Pfad kopieren", + changeAction: "Ändern", + resetAction: "Standard verwenden", + chooseTitle: "Speicherort für Modell-Downloads wählen", + chooseAction: "Für künftige Downloads verwenden", + cacheSaved: "Speicherort für Modell-Downloads aktualisiert", + cacheSaveError: + "Der Speicherort für Modell-Downloads konnte nicht geändert werden", + cachePickerError: "Die Ordnerauswahl konnte nicht geöffnet werden", copied: "Pfad kopiert", openError: "Der Ordner konnte nicht geöffnet werden", copyError: "Der Pfad konnte nicht kopiert werden", diff --git a/studio/frontend/src/i18n/locales/en.ts b/studio/frontend/src/i18n/locales/en.ts index bdfcf38231..10e571f4da 100644 --- a/studio/frontend/src/i18n/locales/en.ts +++ b/studio/frontend/src/i18n/locales/en.ts @@ -24,10 +24,18 @@ export const en = { product: "Unsloth Studio", accountMenu: "{name} account menu", updateAvailable: "Update available", + resize: { + collapse: "Click to collapse", + expand: "Click to expand", + drag: "Drag to resize", + }, aria: { home: "Unsloth home", closeSidebar: "Close sidebar", openSidebar: "Open sidebar", + resizeSidebar: "Resize or collapse sidebar", + resizeRunSettings: "Resize or close run settings", + openRunSettings: "Open run settings", chatOptions: "Chat options", runOptions: "Run options", }, @@ -556,8 +564,12 @@ export const en = { systemDisk: "System disk", diskUsage: "{used} used / {total}", diskFree: "{free} free", - modelsFolder: "Model downloads", - modelsFolderDescription: "Hugging Face cache used for model downloads.", + modelsFolder: "Models folder", + modelsFolderDescription: + "Where downloaded models are stored. Change it to keep models off your system drive.", + // Not rendered: extra terms the settings search matches this row on. + modelsFolderKeywords: + "models folder directory path location download downloads cache storage disk drive move relocate hugging face", futureDownloads: "New downloads only", environmentManaged: "Managed by the {variable} environment variable.", locationFree: "{free} free", diff --git a/studio/frontend/src/i18n/locales/es.ts b/studio/frontend/src/i18n/locales/es.ts index f7cb0e11f6..6edc33f9de 100644 --- a/studio/frontend/src/i18n/locales/es.ts +++ b/studio/frontend/src/i18n/locales/es.ts @@ -27,10 +27,18 @@ export const es = { product: "Unsloth Studio", accountMenu: "Menú de cuenta de {name}", updateAvailable: "Actualización disponible", + resize: { + collapse: "Haz clic para contraer", + expand: "Haz clic para expandir", + drag: "Arrastra para redimensionar", + }, aria: { home: "Inicio de Unsloth", closeSidebar: "Cerrar barra lateral", openSidebar: "Abrir barra lateral", + resizeSidebar: "Redimensionar o contraer la barra lateral", + resizeRunSettings: "Redimensionar o cerrar los ajustes de ejecución", + openRunSettings: "Abrir los ajustes de ejecución", chatOptions: "Opciones de chat", runOptions: "Opciones de ejecución", }, @@ -328,6 +336,8 @@ export const es = { diskUsage: "{used} en uso / {total}", diskFree: "{free} libre", modelsFolder: "Carpeta de modelos", + modelsFolderKeywords: + "modelos carpeta directorio ruta ubicacion ubicación descargas descarga cache caché almacenamiento disco unidad mover cambiar models folder path hugging face", modelsFolderDescription: "Dónde se almacenan los modelos descargados.", openAction: "Abrir", diff --git a/studio/frontend/src/i18n/locales/fr.ts b/studio/frontend/src/i18n/locales/fr.ts index 4f2838391f..789e89079c 100644 --- a/studio/frontend/src/i18n/locales/fr.ts +++ b/studio/frontend/src/i18n/locales/fr.ts @@ -27,10 +27,18 @@ export const fr = { product: "Unsloth Studio", accountMenu: "Menu du compte de {name}", updateAvailable: "Mise à jour disponible", + resize: { + collapse: "Cliquez pour réduire", + expand: "Cliquez pour développer", + drag: "Faites glisser pour redimensionner", + }, aria: { home: "Accueil Unsloth", closeSidebar: "Fermer la barre latérale", openSidebar: "Ouvrir la barre latérale", + resizeSidebar: "Redimensionner ou réduire la barre latérale", + resizeRunSettings: "Redimensionner ou fermer les paramètres d'exécution", + openRunSettings: "Ouvrir les paramètres d'exécution", chatOptions: "Options de discussion", runOptions: "Options d'exécution", }, @@ -325,6 +333,8 @@ export const fr = { diskUsage: "{used} utilisé / {total}", diskFree: "{free} libre", modelsFolder: "Dossier des modèles", + modelsFolderKeywords: + "modeles modèles dossier repertoire répertoire chemin emplacement telechargements téléchargements cache stockage disque lecteur deplacer déplacer changer models folder path hugging face", modelsFolderDescription: "Emplacement de stockage des modèles téléchargés.", openAction: "Ouvrir", copyAction: "Copier le chemin", diff --git a/studio/frontend/src/i18n/locales/hi.ts b/studio/frontend/src/i18n/locales/hi.ts index 33b827f314..410318983b 100644 --- a/studio/frontend/src/i18n/locales/hi.ts +++ b/studio/frontend/src/i18n/locales/hi.ts @@ -27,10 +27,18 @@ export const hi = { product: "Unsloth Studio", accountMenu: "{name} खाता मेनू", updateAvailable: "अपडेट उपलब्ध है", + resize: { + collapse: "छोटा करने के लिए क्लिक करें", + expand: "विस्तार के लिए क्लिक करें", + drag: "आकार बदलने के लिए खींचें", + }, aria: { home: "Unsloth होम", closeSidebar: "साइडबार बंद करें", openSidebar: "साइडबार खोलें", + resizeSidebar: "साइडबार का आकार बदलें या छोटा करें", + resizeRunSettings: "रन सेटिंग्स का आकार बदलें या बंद करें", + openRunSettings: "रन सेटिंग्स खोलें", chatOptions: "चैट विकल्प", runOptions: "रन विकल्प", }, @@ -316,6 +324,8 @@ export const hi = { diskUsage: "{used} उपयोग में / {total}", diskFree: "{free} खाली", modelsFolder: "मॉडल फ़ोल्डर", + modelsFolderKeywords: + "मॉडल फ़ोल्डर फोल्डर निर्देशिका पथ स्थान डाउनलोड कैश संग्रहण डिस्क ड्राइव स्थानांतरित बदलें models folder path hugging face", modelsFolderDescription: "जहां डाउनलोड किए गए मॉडल संग्रहीत होते हैं।", openAction: "खोलें", copyAction: "पथ कॉपी करें", diff --git a/studio/frontend/src/i18n/locales/ja.ts b/studio/frontend/src/i18n/locales/ja.ts index 978fde6281..1653bad76e 100644 --- a/studio/frontend/src/i18n/locales/ja.ts +++ b/studio/frontend/src/i18n/locales/ja.ts @@ -28,10 +28,18 @@ export const ja = { product: "Unsloth Studio", accountMenu: "{name} のアカウントメニュー", updateAvailable: "アップデートが利用可能です", + resize: { + collapse: "クリックで折りたたむ", + expand: "クリックで展開", + drag: "ドラッグでサイズ変更", + }, aria: { home: "Unsloth ホーム", closeSidebar: "サイドバーを閉じる", openSidebar: "サイドバーを開く", + resizeSidebar: "サイドバーのサイズ変更または折りたたみ", + resizeRunSettings: "実行設定のサイズ変更または閉じる", + openRunSettings: "実行設定を開く", chatOptions: "チャットオプション", runOptions: "実行オプション", }, @@ -393,6 +401,8 @@ export const ja = { diskUsage: "{used} 使用中 / {total}", diskFree: "{free} 空き", modelsFolder: "モデルフォルダ", + modelsFolderKeywords: + "モデル フォルダ ディレクトリ パス 保存先 場所 ダウンロード キャッシュ ストレージ ディスク ドライブ 移動 変更 models folder path hugging face", modelsFolderDescription: "ダウンロードしたモデルの保存先。", openAction: "開く", copyAction: "パスをコピー", diff --git a/studio/frontend/src/i18n/locales/ko.ts b/studio/frontend/src/i18n/locales/ko.ts index aa8a4fd47b..b0da314896 100644 --- a/studio/frontend/src/i18n/locales/ko.ts +++ b/studio/frontend/src/i18n/locales/ko.ts @@ -27,10 +27,18 @@ export const ko = { product: "Unsloth Studio", accountMenu: "{name} 계정 메뉴", updateAvailable: "업데이트 사용 가능", + resize: { + collapse: "클릭하여 접기", + expand: "클릭하여 펼치기", + drag: "드래그하여 크기 조절", + }, aria: { home: "Unsloth 홈", closeSidebar: "사이드바 닫기", openSidebar: "사이드바 열기", + resizeSidebar: "사이드바 크기 조절 또는 접기", + resizeRunSettings: "실행 설정 크기 조절 또는 닫기", + openRunSettings: "실행 설정 열기", chatOptions: "채팅 옵션", runOptions: "학습 옵션", }, @@ -315,6 +323,8 @@ export const ko = { diskUsage: "{used} 사용 중 / {total}", diskFree: "{free} 여유", modelsFolder: "모델 폴더", + modelsFolderKeywords: + "모델 폴더 디렉터리 디렉토리 경로 위치 저장 다운로드 캐시 저장소 디스크 드라이브 이동 변경 models folder path hugging face", modelsFolderDescription: "다운로드한 모델이 저장되는 위치입니다.", openAction: "열기", copyAction: "경로 복사", diff --git a/studio/frontend/src/i18n/locales/pt-br.ts b/studio/frontend/src/i18n/locales/pt-br.ts index 84cd3f945e..e9f23623f8 100644 --- a/studio/frontend/src/i18n/locales/pt-br.ts +++ b/studio/frontend/src/i18n/locales/pt-br.ts @@ -27,10 +27,18 @@ export const ptBR = { product: "Unsloth Studio", accountMenu: "Menu de conta {name}", updateAvailable: "Atualização disponível", + resize: { + collapse: "Clique para recolher", + expand: "Clique para expandir", + drag: "Arraste para redimensionar", + }, aria: { home: "Início do Unsloth", closeSidebar: "Fechar barra lateral", openSidebar: "Abrir barra lateral", + resizeSidebar: "Redimensionar ou recolher a barra lateral", + resizeRunSettings: "Redimensionar ou fechar as configurações de execução", + openRunSettings: "Abrir as configurações de execução", chatOptions: "Opções de chat", runOptions: "Opções de execução", }, @@ -417,6 +425,8 @@ export const ptBR = { diskUsage: "{used} usados / {total}", diskFree: "{free} livres", modelsFolder: "Pasta de modelos", + modelsFolderKeywords: + "modelos pasta diretorio diretório caminho local localizacao localização downloads baixar cache armazenamento disco unidade mover alterar models folder path hugging face", modelsFolderDescription: "Onde os modelos baixados são armazenados.", openAction: "Abrir", copyAction: "Copiar caminho", diff --git a/studio/frontend/src/i18n/locales/ru.ts b/studio/frontend/src/i18n/locales/ru.ts index 7725212e3b..364f224802 100644 --- a/studio/frontend/src/i18n/locales/ru.ts +++ b/studio/frontend/src/i18n/locales/ru.ts @@ -27,10 +27,18 @@ export const ru = { product: "Unsloth Studio", accountMenu: "Меню аккаунта {name}", updateAvailable: "Доступно обновление", + resize: { + collapse: "Нажмите, чтобы свернуть", + expand: "Нажмите, чтобы развернуть", + drag: "Потяните, чтобы изменить размер", + }, aria: { home: "Главная Unsloth", closeSidebar: "Закрыть боковую панель", openSidebar: "Открыть боковую панель", + resizeSidebar: "Изменить размер или свернуть боковую панель", + resizeRunSettings: "Изменить размер или закрыть настройки запуска", + openRunSettings: "Открыть настройки запуска", chatOptions: "Параметры чата", runOptions: "Параметры запуска", }, @@ -316,6 +324,8 @@ export const ru = { diskUsage: "{used} использовано / {total}", diskFree: "{free} свободно", modelsFolder: "Папка моделей", + modelsFolderKeywords: + "модели папка каталог путь расположение загрузки кэш хранилище диск перенести изменить models folder path hugging face", modelsFolderDescription: "Где хранятся загруженные модели.", openAction: "Открыть", copyAction: "Копировать путь", diff --git a/studio/frontend/src/i18n/locales/zh-CN.ts b/studio/frontend/src/i18n/locales/zh-CN.ts index 06326ed008..777bfbc4dd 100644 --- a/studio/frontend/src/i18n/locales/zh-CN.ts +++ b/studio/frontend/src/i18n/locales/zh-CN.ts @@ -27,10 +27,18 @@ export const zhCN = { product: "Unsloth Studio", accountMenu: "{name} 账号菜单", updateAvailable: "有可用更新", + resize: { + collapse: "点击折叠", + expand: "点击展开", + drag: "拖动调整大小", + }, aria: { home: "Unsloth 首页", closeSidebar: "关闭侧边栏", openSidebar: "打开侧边栏", + resizeSidebar: "调整或折叠侧边栏", + resizeRunSettings: "调整或关闭运行设置", + openRunSettings: "打开运行设置", chatOptions: "聊天选项", runOptions: "训练选项", }, @@ -408,6 +416,8 @@ export const zhCN = { diskUsage: "已用 {used} / {total}", diskFree: "{free} 可用", modelsFolder: "模型文件夹", + modelsFolderKeywords: + "模型 文件夹 目录 路径 位置 下载 缓存 存储 磁盘 驱动器 移动 更改 models folder path hugging face", modelsFolderDescription: "已下载模型的存储位置。", openAction: "打开", copyAction: "复制路径", diff --git a/studio/frontend/src/index.css b/studio/frontend/src/index.css index 4797401ead..4ffd29ad7c 100644 --- a/studio/frontend/src/index.css +++ b/studio/frontend/src/index.css @@ -1363,6 +1363,23 @@ html[data-chat-font] .aui-root { cursor: pointer; } + /* While a panel edge is dragged, keep the resize cursor even as the pointer + travels over buttons and text that would claim their own. */ + html[data-panel-resizing], + html[data-panel-resizing] * { + cursor: col-resize !important; + user-select: none !important; + } + + html[data-panel-resizing] + :is( + [data-slot="sidebar-inner"], + [data-slot="sidebar-inset"], + [data-slot="chat-settings-panel"] > div + ) { + pointer-events: none; + } + /* Model selector: pointer cursor on every clickable element. */ .unsloth-model-selector-trigger, .unsloth-model-selector-menu button { diff --git a/studio/frontend/src/lib/changelog-links.ts b/studio/frontend/src/lib/changelog-links.ts new file mode 100644 index 0000000000..16b3d3c8bc --- /dev/null +++ b/studio/frontend/src/lib/changelog-links.ts @@ -0,0 +1,664 @@ +// SPDX-License-Identifier: AGPL-3.0-only +// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 + +/** + * A relative link in CHANGELOG.md means "somewhere in the Unsloth repository", + * but inside Studio it would resolve against Studio's own origin. Rewriting to + * absolute repository URLs makes them behave the way GitHub renders the file. + */ + +import { + type CodeSpan, + codeSpans, + insideSpan, +} from "@/lib/markdown-code-spans"; +import { commentClosesBelow } from "@/lib/markdown-inline-comments"; +import { + EMPTY_LIST_STATE, + type ListState, + NO_QUOTE, + type QuoteState, + containerContent, + hiddenStructure, + indentWidth, + itemContent, + openLists, + quoteDepth, + quoteState, +} from "@/lib/markdown-list-columns"; + +const LINK_BASE = "https://github.com/unslothai/unsloth/blob/main/"; +const IMAGE_BASE = "https://raw.githubusercontent.com/unslothai/unsloth/main/"; + +// Inline `](dest)` plus the `[label]: dest` reference form. The destination is +// either <bracketed> or runs to whitespace or the closing paren. +const NESTED_LABEL = String.raw`((?:[^[\]\\]|\\.|\[(?:[^[\]\\]|\\.)*\])*)`; +// Only ASCII punctuation is escapable, so the backslash in `a\ b.md` is an +// ordinary character of the destination and the space still ends it. +const ESCAPABLE = String.raw`[!-/:-@[-\`{-~]`; +const DESTINATION_CHAR = String.raw`\\${ESCAPABLE}|[^\s()]`; +// A destination may hold balanced parentheses, and a path may nest them, so +// `[x](((draft)).md)` points at `((draft)).md`. An expression cannot count, so +// pairs are unrolled to the depth cmark stops at, which is what GitHub renders. +const MAX_DESTINATION_NESTING = 32; + +/** A balanced parenthesised run nested up to `depth` levels deep. */ +function nestedParens(depth: number): string { + let group = String.raw`\((?:${DESTINATION_CHAR})*\)`; + for (let left = depth - 1; left > 0; left -= 1) { + group = String.raw`\((?:${DESTINATION_CHAR}|${group})*\)`; + } + return group; +} + +const BALANCED_DESTINATION = String.raw`(?:${DESTINATION_CHAR}|${nestedParens(MAX_DESTINATION_NESTING)})*`; +const PLAIN_DESTINATION = String.raw`(?:${DESTINATION_CHAR})*`; +// A balanced pair counts only while a `)` or a title still closes the link +// after it, or swallowing it would invent a link across lines. +const CLOSES_LINK = String.raw`(?=[ \t]*[)'"])`; +// A destination that runs out of line has its closer below it, the line being +// only part of the link. One stopping short of a closer is no destination at all, +// so `[x](a b.md)` and `[x](a(b.md)` stay plain text and keep the paths they name. +const CLOSES_OR_ENDS_LINE = String.raw`(?=[ \t]*(?:[)'"]|$))`; +const INLINE_TARGET = new RegExp( + String.raw`(!?)\[${NESTED_LABEL}\]\(\s*(<[^<>\n]*>|${BALANCED_DESTINATION}${CLOSES_LINK}|${PLAIN_DESTINATION}${CLOSES_OR_ENDS_LINE})`, + "g", +); +const REFERENCE_TARGET = /^( {0,3}\[((?:[^[\]\\]|\\.)*)\]:\s*)(<[^<>\n]*>|\S+)/; +// `![alt][label]`, `![label][]` and `![label]`: a definition they point at +// has to resolve to the raw file, not to its page on GitHub. +const IMAGE_REFERENCE = + /!\[((?:[^[\]\\]|\\.)*)\](?:\[((?:[^[\]\\]|\\.)*)\]|(?!\())/g; +const FENCE = /^ {0,3}(`{3,}|~{3,})(.*)$/; +// Four columns past the container start indented code, unless a paragraph is +// open. Inside a list item that is measured from the item's content column, so a +// link indented under a bullet is prose and still resolves. +const INDENTED_CODE_INDENT = 4; +// CommonMark type 1 HTML blocks show their contents verbatim. +const RAW_HTML_OPEN = /^ {0,3}<(pre|script|style|textarea)(?=[\s>]|$)/i; +const RAW_HTML_CLOSE = /<\/(pre|script|style|textarea)\s*>/i; +// Type 6 and 7 blocks are literal too and run to the next blank line, not to a +// closing tag, so `<details>` holds Markdown only after a blank line. Type 7 (any +// other complete tag alone on a line) cannot interrupt a paragraph. +const HTML_BLOCK_OPEN = /^ {0,3}<\/?([a-zA-Z][a-zA-Z0-9-]*)(?=[\s/>]|$)/; +const HTML_ATTRIBUTE = + "(?:\\s+[a-zA-Z_:][a-zA-Z0-9_.:-]*(?:\\s*=\\s*(?:[^\\s\"'=<>`]+|'[^']*'|\"[^\"]*\"))?)"; +const HTML_TAG_ONLY_LINE = new RegExp( + `^ {0,3}(?:<[a-zA-Z][a-zA-Z0-9-]*${HTML_ATTRIBUTE}*\\s*/?>|</[a-zA-Z][a-zA-Z0-9-]*\\s*>)\\s*$`, +); +const HTML_BLOCK_TAGS = new Set( + `address article aside base basefont blockquote body caption center col colgroup + dd details dialog dir div dl dt fieldset figcaption figure footer form frame + frameset h1 h2 h3 h4 h5 h6 head header hr html iframe legend li link main menu + menuitem nav noframes ol optgroup option p param search section summary table + tbody td tfoot th thead title tr track ul`.split(/\s+/), +); +// Lines that are blocks in their own right, so no paragraph is open after. +const BLOCK_LINE = + /^ {0,3}(?:#{1,6}([ \t]|$)|(?:\*[ \t]*){3,}$|(?:-[ \t]*){3,}$|(?:_[ \t]*){3,}$|>|=+[ \t]*$)/; +// A definition is a block of its own but may not interrupt a paragraph, so it +// ends the one above only when there is none to continue. It opens none either, +// or consecutive definitions could never start (spec 0.31.2 section 4.7). Same +// rule as `_LINK_DEFINITION` in the backend's `after_paragraph`. +const LINK_DEFINITION = /^ {0,3}\[(?:[^[\]\\]|\\.)+\]:/; +const LINE_ENDINGS = /\r\n?/g; +// A scheme, a protocol-relative host, or a fragment: already absolute enough. +// `//` needs a host after it, so `///docs` stays a repository path. +const ABSOLUTE = /^(?:[a-zA-Z][a-zA-Z0-9+.-]*:|\/\/[^/]|#)/; + +const COMMENT_OPEN = "<!--"; +const COMMENT_CLOSE = "-->"; +const COMMENT_BLOCK_OPEN = /^ {0,3}<!--/; + +/** + * `line` with its commented spans blanked, and whether a comment block is still + * open below it. Commented content renders as nothing, so it holds no fence, + * block or code span. Lengths are preserved so offsets still line up. + * + * Only a comment that starts a line opens a block (CommonMark type 2), and only + * that runs on to the line holding `-->`, tail included. One written mid-sentence + * is inline raw HTML belonging to its paragraph, so its `-->` may arrive on a + * later line and only the text up to it is hidden. `closesBelow` says one does; + * without it the opener is ordinary text, so a note merely mentioning `<!--` must + * not hide the links below it. + * + * "Starts a line" is read inside the container, so `blockOpen` comes from the + * item's content rather than the raw line. + */ +function maskComments( + line: string, + inComment: boolean, + runOn: boolean, + closesBelow: boolean, + blockOpen: boolean, +): [string, boolean, boolean] { + if (inComment) { + // The closing line belongs to the block, tail included. + return [" ".repeat(line.length), !line.includes(COMMENT_CLOSE), false]; + } + if (runOn) { + const closed = line.indexOf(COMMENT_CLOSE); + if (closed < 0) { + return [" ".repeat(line.length), false, true]; + } + // Only up to the closer: the tail is the paragraph's own text again. + const resumed = closed + COMMENT_CLOSE.length; + return maskInline(line, resumed, closesBelow); + } + if (blockOpen) { + // `<!-->` and `<!--->` are complete comments, so the closer may overlap the + // opener; searching past it would blank the rest of the file. + return [" ".repeat(line.length), !line.includes(COMMENT_CLOSE), false]; + } + return maskInline(line, 0, closesBelow); +} + +/** `maskComments` from `from`, where no comment block is open. */ +function maskInline( + line: string, + from: number, + closesBelow: boolean, +): [string, boolean, boolean] { + let out = " ".repeat(from); + let index = from; + // Scanned only once an opener turns up. Spans are ordered and disjoint and each + // opener sits at or past the last, so the search resumes rather than restarts. + let spans: CodeSpan[] | null = null; + let cursor = 0; + while (index < line.length) { + const start = line.indexOf(COMMENT_OPEN, index); + if (start < 0) { + return [out + line.slice(index), false, false]; + } + spans ??= codeSpans(line); + while (cursor < spans.length && (spans[cursor]?.end ?? 0) <= start) { + cursor += 1; + } + // A delimiter inside inline code is literal, not a comment opener. + const span = spans[cursor]; + if (span !== undefined && span.start <= start) { + out += line.slice(index, span.end); + index = span.end; + continue; + } + // `<!-->` and `<!--->` are complete comments, so the closer may overlap. + const close = line.indexOf(COMMENT_CLOSE, start + 2); + if (close < 0) { + if (closesBelow) { + // The paragraph carries the comment on, so the line from the opener is + // inside it, and so is the line below. + return [ + out + line.slice(index, start) + " ".repeat(line.length - start), + false, + true, + ]; + } + // Nothing closes it at all, so the renderer shows it as ordinary text. + return [out + line.slice(index), false, false]; + } + out += line.slice(index, start); + out += " ".repeat(close + COMMENT_CLOSE.length - start); + index = close + COMMENT_CLOSE.length; + } + return [out, false, false]; +} + +/** + * Whether `line` is written outside the container an open block belongs to. A + * fence and an HTML block hold no lazy continuation line, so content left of the + * item, or outside the quote, ends the block with its container. A raw block or + * comment inside a list item ends on a blank line too: the item takes the break, + * so what follows is a block of the item's own. + */ +function leavesContainer( + line: string, + quotes: number, + column: number, + blockQuotes: number, + rawInItem: boolean, +): boolean { + if (quotes < blockQuotes) { + return true; + } + if (!line.trim()) { + return rawInItem; + } + return column > 0 && indentWidth(line) < column; +} + +/** True if `line` starts a CommonMark type 6 or type 7 HTML block. */ +function opensHtmlBlock(line: string, afterParagraph: boolean): boolean { + const named = HTML_BLOCK_OPEN.exec(line); + if (named && HTML_BLOCK_TAGS.has((named[1] ?? "").toLowerCase())) { + return true; + } + return !afterParagraph && HTML_TAG_ONLY_LINE.test(line); +} + +/** A reference label as CommonMark compares them. */ +function label(text: string): string { + return text.trim().replace(/\s+/g, " ").toLowerCase(); +} + +const NEEDS_BRACKETS = /[()\s]/; +// `\(` in a destination is a literal paren. Only ASCII punctuation is escapable, +// so the backslash in `docs\alpha.md` is part of the path and has to survive. +const ESCAPE = new RegExp(String.raw`\\(${ESCAPABLE})`, "g"); +// A URL parser reads a backslash as a path separator, so `docs\a.md` would +// resolve to `docs/a.md`. Encode it first, the way a renderer normalises it. +const BACKSLASH = /\\/g; +// Only spaces and tabs may follow a closing fence. +const NON_SPACE = /[^ \t]/; +const LEADING_SLASHES = /^\/+/; + +function absolute(target: string, image: boolean): string { + const base = image ? IMAGE_BASE : LINK_BASE; + const trimmed = target.trim().replace(ESCAPE, "$1"); + if (!trimmed || ABSOLUTE.test(trimmed)) { + return target; + } + try { + // A leading slash means the repository root, not the site root, so append + // it to the base instead of replacing the base path. + const resolved = new URL( + trimmed.replace(LEADING_SLASHES, "").replace(BACKSLASH, "%5C"), + base, + ).toString(); + // `../` can climb out of the repository: leave those alone. + return resolved.startsWith(base) ? resolved : target; + } catch { + return target; + } +} + +/** True when `index` is escaped by an odd run of backslashes. */ +function isEscaped(line: string, index: number): boolean { + let slashes = 0; + while (line[index - 1 - slashes] === "\\") { + slashes += 1; + } + return slashes % 2 === 1; +} + +function unwrap(target: string): string { + return target.startsWith("<") && target.endsWith(">") + ? target.slice(1, -1) + : target; +} + +/** The destination as it goes back into the line. */ +function wrap(resolved: string, original: string): string { + const bracketed = original.startsWith("<") && original.endsWith(">"); + return bracketed || (resolved !== original && NEEDS_BRACKETS.test(resolved)) + ? `<${resolved}>` + : resolved; +} + +/** Rewrites one line's link and image targets, leaving code spans alone. */ +function rewriteLine( + line: string, + imageLabels: Set<string>, + spans: CodeSpan[], + base: number, + isDefinition: boolean, +): string { + const reference = isDefinition ? REFERENCE_TARGET.exec(line) : null; + if (reference) { + const target = reference[3] ?? ""; + const resolved = absolute( + unwrap(target), + imageLabels.has(label(reference[2] ?? "")), + ); + const rest = line.slice(reference[0].length); + return `${reference[1]}${wrap(resolved, target)}${rest}`; + } + + INLINE_TARGET.lastIndex = 0; + return line.replace(INLINE_TARGET, (match, bang, text, target, offset) => { + // `\\[` is a literal bracket, so the expression is not a link. + const opener = offset + (bang ? 1 : 0); + if (insideSpan(spans, base + offset) || isEscaped(line, opener)) { + return match; + } + // `\\!` is a literal mark, so what follows is a link, not an image. + const image = bang === "!" && !isEscaped(line, offset); + const resolved = absolute(unwrap(target), image); + // A badge nests an image inside a link, so the label is rewritten too. + const inner = text.includes("](") + ? rewriteLine(text, imageLabels, codeSpans(text), 0, false) + : text; + return `${bang}[${inner}](${wrap(resolved, target)}`; + }); +} + +interface Classified { + // Lines the renderer shows as Markdown, by index. + text: number[]; + // Same lines, blanked where the renderer shows code, for span scanning. + masked: string; + // Lines where a `[label]: dest` definition can start. + definition: Set<number>; + // Document ranges the renderer hides inside HTML comments. + comments: CodeSpan[]; +} + +/** + * Sorts lines into Markdown and code, masking the code so a span cannot pair + * across it. Offsets are preserved, so a mask span sits where it does in the doc. + */ +function classify(lines: string[]): Classified { + const text: number[] = []; + const definition = new Set<number>(); + const masked: string[] = []; + let openFence: string | null = null; + let inRawHtml = false; + let inHtmlBlock = false; + // Where the open block was written: the content column of the item it belongs + // to, 0 at document level, plus the blockquotes it sits inside. Only one is ever + // open, and none holds a lazy continuation line, so a line left of the item or + // outside the quote ends the block with its container. + let blockColumn = 0; + let blockQuotes = 0; + let inComment = false; + // True while an inline comment opened above runs on into this line, carried by + // the paragraph holding it. + let runOn = false; + const closesBelow = commentClosesBelow(lines); + let inCode = false; + let afterParagraph = false; + let quote: QuoteState = NO_QUOTE; + let lists: ListState = EMPTY_LIST_STATE; + const comments: CodeSpan[] = []; + let offset = 0; + + // The line as list tracking sees it: blank wherever nothing renders. Taken + // with the paragraph state from the line above, as the renderer would. + const track = (structural: string, above: QuoteState): void => { + lists = openLists(structural, lists, afterParagraph, above.quoted); + }; + // Where a block just opened sits, read after the opener closed the items it + // is dedented out of, so it belongs to the container it is really in. + const startBlock = (quotes: number): void => { + blockColumn = lists.columns.at(-1) ?? 0; + blockQuotes = quotes; + }; + const endBlock = (): void => { + blockColumn = 0; + blockQuotes = 0; + }; + + lines.forEach((original, index) => { + const start = offset; + offset += original.length + 1; + // The quote state from the line above, which is what list tracking asks + // about. Only plain text below rewrites it, so every block returning early + // leaves no quoted paragraph open behind it. + const above = quote; + quote = NO_QUOTE; + // A fence, comment or HTML block runs only to the end of the container it was + // written in, so a line dedented out of that item or outside that quote + // closes both. + const quotes = quoteDepth(original); + let inBlock = openFence !== null || inRawHtml || inHtmlBlock || inComment; + if ( + inBlock && + leavesContainer( + original, + quotes, + blockColumn, + blockQuotes, + (inRawHtml || inComment) && blockColumn > 0 && blockQuotes === 0, + ) + ) { + openFence = null; + inRawHtml = false; + inHtmlBlock = false; + inComment = false; + endBlock(); + inBlock = false; + } + // Read from the container the line is written in, so a fence three columns + // past a nested bullet or behind a quote marker still opens one. A block + // already open keeps only its own quote stripped, or a deeper marker in it + // would read as a closer. + const container = containerContent( + original, + lists, + inBlock ? blockQuotes : quotes, + ); + // A comment cannot open a fence and a fence hides a comment opener, so resolve + // them in that order or a hidden delimiter opens a phantom fence. An opener is + // read past a marker on the same line too, since a fence written as an item's + // first content opens inside it. Only an opener: fenced content is literal and + // a closer carries no marker. + const fenceSource = inComment + ? null + : FENCE.exec( + openFence === null + ? itemContent(container, afterParagraph) + : container, + ); + if (inRawHtml) { + track("", above); + inRawHtml = !RAW_HTML_CLOSE.test(container); + if (!inRawHtml) { + endBlock(); + } + masked.push(" ".repeat(original.length)); + afterParagraph = false; + return; + } + if (inHtmlBlock) { + track("", above); + // Only a blank line ends a type 6 or 7 block, so nothing inside one is a + // fence or a link. A bare quote marker holds nothing, so it ends one too. + inHtmlBlock = !!container.trim(); + if (!inHtmlBlock) { + endBlock(); + } + masked.push(" ".repeat(original.length)); + afterParagraph = false; + return; + } + const fence = fenceSource; + if (fence) { + // A fence renders as nothing, but its indent still closes an item. + track(original, above); + const marker = fence[1] ?? ""; + if (openFence === null) { + // A backtick fence's info string may not contain a backtick. + openFence = + marker[0] !== "`" || !(fence[2] ?? "").includes("`") ? marker : null; + if (openFence === null) { + text.push(index); + masked.push(original); + afterParagraph = true; + return; + } + startBlock(quotes); + } else if ( + // A closer matches the opening character and carries nothing after it. + marker[0] === openFence[0] && + marker.length >= openFence.length && + !NON_SPACE.test(fence[2] ?? "") + ) { + openFence = null; + endBlock(); + } + masked.push(" ".repeat(original.length)); + afterParagraph = false; + return; + } + if (openFence !== null) { + track("", above); + // Fenced content is literal, so a comment opener in it is not one. + masked.push(" ".repeat(original.length)); + return; + } + // A block already open owns this line, so it is content rather than a block + // written at the column it happens to start in. + const hidden = inComment; + const carried = runOn; + // A comment is an HTML block too, so one written as a list item's first + // content opens inside that item exactly as a fence does: read past a marker + // on the same line and from its container's column, not the line's margin. + const opensComment = + !(hidden || carried) && + COMMENT_BLOCK_OPEN.test(itemContent(container, afterParagraph)); + // Only now, outside every fence, does a comment hide what follows. + const [line, stillInComment, stillRunOn] = maskComments( + original, + inComment, + runOn, + closesBelow[index + 1] ?? false, + opensComment, + ); + inComment = stillInComment; + runOn = stillRunOn; + // A line an inline comment runs on into is still a line of the paragraph + // that carries it: only its text is hidden, never its block structure. + const structure = carried ? original : line; + // The same container reading as above, now the comments are masked. A comment + // blanks its own line, so that line is read as written: the block renders as + // nothing, but the item it is the content of still opens. + const source = opensComment ? original : line; + const visible = containerContent(source, lists, quotes); + // An HTML block written as a list item's first content opens inside that item, + // as a fence does, so an opener is read past a marker on the same line. The + // marker survives into the structural line, so its item is still tracked. + const content = itemContent(visible, afterParagraph); + const marker = + content === visible + ? "" + : source.slice(0, source.length - content.length); + // Taken before an HTML opener is hidden: it renders as nothing, but its indent + // still closes a list item it sits left of. A comment or a <pre> keeps only its + // column and marker, since the text it hides is not Markdown and opens no list. + const opensRaw = !carried && RAW_HTML_OPEN.test(content); + track( + !(hidden || carried) && (opensRaw || !line.trim()) + ? hiddenStructure(original, marker) + : structure, + above, + ); + // Read once the opener has closed the items it is dedented out of, so the + // comment block belongs to the item it is really written inside. + if (inComment !== hidden) { + if (inComment) { + startBlock(quotes); + } else { + endBlock(); + } + } + for (let at = 0; at < line.length; at += 1) { + if (line[at] === " " && original[at] !== " ") { + const from = at; + while (at < line.length && line[at] === " " && original[at] !== " ") { + at += 1; + } + comments.push({ start: start + from, end: start + at, content: "" }); + } + } + if (opensRaw) { + inRawHtml = !RAW_HTML_CLOSE.test(content.replace(RAW_HTML_OPEN, "")); + if (inRawHtml) { + startBlock(quotes); + } + masked.push(" ".repeat(line.length)); + afterParagraph = false; + return; + } + if (!carried && content.trim() && opensHtmlBlock(content, afterParagraph)) { + inHtmlBlock = true; + startBlock(quotes); + masked.push(" ".repeat(line.length)); + afterParagraph = false; + return; + } + const blank = !structure.trim(); + // Measured from the innermost open item's content column, not the margin: + // four spaces under "- Details:" is a paragraph, not a code block. + const column = lists.columns.at(-1) ?? 0; + const indented = indentWidth(structure) - column >= INDENTED_CODE_INDENT; + // Indented code starts only outside a paragraph and runs to a dedent. + if (inCode) { + inCode = blank || indented; + } else { + inCode = !afterParagraph && !blank && indented; + } + if (inCode) { + masked.push(" ".repeat(line.length)); + afterParagraph = false; + return; + } + // A definition cannot interrupt a paragraph. + if (!afterParagraph) { + definition.add(index); + } + text.push(index); + masked.push(line); + afterParagraph = + !blank && + !BLOCK_LINE.test(structure) && + (afterParagraph || !LINK_DEFINITION.test(structure)); + quote = quoteState(structure, above.inQuote); + }); + + return { text, masked: masked.join("\n"), definition, comments }; +} + +/** Absolute repository URLs for every relative link and image in `markdown`. */ +export function resolveChangelogLinks(markdown: string): string { + // The desktop updater body arrives with CRLF, which would hide fences. + const lines = markdown.replace(LINE_ENDINGS, "\n").split("\n"); + const { text, masked, definition, comments } = classify(lines); + // Scanned over the whole document, so a span may cross a line break. Commented + // ranges join them: the renderer shows neither, so a link in one is not + // followable and rewriting it would only mutate hidden text. + const spans = [...codeSpans(masked), ...comments].sort( + (a, b) => a.start - b.start, + ); + + // Offset of each line in the document, to place matches inside it. + const offsets: number[] = []; + let cursor = 0; + for (const line of lines) { + offsets.push(cursor); + cursor += line.length + 1; + } + + // Only images resolve against the raw host, so collect the image labels + // before rewriting any definition. + const imageLabels = new Set<string>(); + for (const index of text) { + const line = lines[index] ?? ""; + IMAGE_REFERENCE.lastIndex = 0; + for ( + let match = IMAGE_REFERENCE.exec(line); + match !== null; + match = IMAGE_REFERENCE.exec(line) + ) { + // An escaped mark makes it a link, so its definition stays a page URL. + if ( + insideSpan(spans, (offsets[index] ?? 0) + match.index) || + isEscaped(line, match.index) + ) { + continue; + } + const explicit = match[2] ?? ""; + imageLabels.add(label(explicit.trim() ? explicit : (match[1] ?? ""))); + } + } + + const rewritten = [...lines]; + for (const index of text) { + rewritten[index] = rewriteLine( + lines[index] ?? "", + imageLabels, + spans, + offsets[index] ?? 0, + definition.has(index), + ); + } + return rewritten.join("\n"); +} diff --git a/studio/frontend/src/lib/markdown-code-spans.ts b/studio/frontend/src/lib/markdown-code-spans.ts new file mode 100644 index 0000000000..537aabb1ab --- /dev/null +++ b/studio/frontend/src/lib/markdown-code-spans.ts @@ -0,0 +1,123 @@ +// SPDX-License-Identifier: AGPL-3.0-only +// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 + +/** + * CommonMark code spans: a backtick run closes only on an equal-length run. + * That needs lookbehind, which older Safari rejects, so runs are scanned by hand. + */ + +export interface CodeSpan { + // Offsets of the whole span, delimiters included. + start: number; + end: number; + // Between the delimiters, with the one space of padding removed. + content: string; +} + +function runLength(text: string, index: number): number { + let end = index; + while (text[end] === "`") { + end += 1; + } + return end - index; +} + +/** True when `index` is escaped by an odd run of backslashes. */ +function escaped(text: string, index: number): boolean { + let slashes = 0; + while (text[index - 1 - slashes] === "\\") { + slashes += 1; + } + return slashes % 2 === 1; +} + +/** CommonMark drops one space of padding, so `` ` a ` `` renders as "a". */ +function stripPadding(content: string): string { + if ( + content.length > 1 && + content.startsWith(" ") && + content.endsWith(" ") && + content.trim() !== "" + ) { + return content.slice(1, -1); + } + return content; +} + +/** Every code span in `text`, in order. Unclosed runs are ordinary text. */ +export function codeSpans(text: string): CodeSpan[] { + const spans: CodeSpan[] = []; + let index = 0; + + while (index < text.length) { + if (text[index] !== "`" || escaped(text, index)) { + index += 1; + continue; + } + const ticks = runLength(text, index); + const contentStart = index + ticks; + + let cursor = contentStart; + let closed = false; + while (cursor < text.length) { + // Escapes do not apply inside a span, so a run after a backslash closes it. + if (text[cursor] !== "`") { + cursor += 1; + continue; + } + const candidate = runLength(text, cursor); + if (candidate === ticks) { + spans.push({ + start: index, + end: cursor + ticks, + content: stripPadding(text.slice(contentStart, cursor)), + }); + index = cursor + ticks; + closed = true; + break; + } + cursor += candidate; + } + if (!closed) { + // Nothing closes this run: it is literal text, carry on after it. + index = contentStart; + } + } + return spans; +} + +/** Replaces every code span with `park(content)`, leaving the rest as is. */ +export function parkCodeSpans( + text: string, + park: (content: string) => string, +): string { + const spans = codeSpans(text); + if (spans.length === 0) { + return text; + } + let out = ""; + let cursor = 0; + for (const span of spans) { + out += text.slice(cursor, span.start) + park(span.content); + cursor = span.end; + } + return out + text.slice(cursor); +} + +/** True when `index` falls inside one of `spans`, which are in order. */ +export function insideSpan(spans: CodeSpan[], index: number): boolean { + let low = 0; + let high = spans.length - 1; + while (low <= high) { + const mid = (low + high) >> 1; + const span = spans[mid]; + if (span === undefined || index < span.start) { + high = mid - 1; + } else if (index >= span.end) { + low = mid + 1; + } else { + return true; + } + } + return false; +} diff --git a/studio/frontend/src/lib/markdown-inline-comments.ts b/studio/frontend/src/lib/markdown-inline-comments.ts new file mode 100644 index 0000000000..33bbfddc31 --- /dev/null +++ b/studio/frontend/src/lib/markdown-inline-comments.ts @@ -0,0 +1,62 @@ +// SPDX-License-Identifier: AGPL-3.0-only +// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 + +/** + * An HTML comment written mid-sentence is inline raw HTML, not a block, so it + * belongs to its paragraph: the `-->` may arrive on a later line of that same + * paragraph and everything between renders as nothing, while past the paragraph + * the `<!--` is ordinary text. Both changelog scanners share that answer here. + * + * The backend needs none of it: a heading closes the paragraph it sits under, so + * no heading can ever land inside one of these comments. + */ + +import { interruptsParagraph } from "@/lib/markdown-list-columns"; + +const COMMENT_CLOSE = "-->"; +// A line that cannot be more of the paragraph above it: blank, or a block that +// may interrupt one. Leading punctuation is not one: `-->` alone is the ordinary +// multiline close and a continuation may open with emphasis, so reading either as +// a break leaves the comment unclosed and its text on show. Indented code and link +// definitions are absent: neither may interrupt a paragraph (spec 0.31.2 4.4, 4.7). +const BLANK = /^[ \t]*$/; +const ATX_HEADING = /^ {0,3}#{1,6}([ \t]|$)/; +const FENCE = /^ {0,3}(?:`{3,}|~{3,})/; +const THEMATIC_BREAK = + /^ {0,3}(?:(?:\*[ \t]*){3,}|(?:-[ \t]*){3,}|(?:_[ \t]*){3,})$/; +// A row of `=` or `-` alone makes the paragraph above it a setext heading, ending it. +const SETEXT_UNDERLINE = /^ {0,3}(?:=+|-+)[ \t]*$/; +// A tag, comment or declaration at the start of a line. HTML block types 1 to 6 +// interrupt a paragraph; type 7 does not, but reading one as a break only leaves +// the opener as plain text, which is what a leading `<` has always meant here. +const HTML_LINE = /^ {0,3}</; + +/** Whether `line` starts a block of its own rather than continuing a paragraph. */ +function startsBlock(line: string): boolean { + return ( + BLANK.test(line) || + ATX_HEADING.test(line) || + FENCE.test(line) || + THEMATIC_BREAK.test(line) || + SETEXT_UNDERLINE.test(line) || + HTML_LINE.test(line) || + // Blockquote, or a list item with content: the rule the other scanners share. + interruptsParagraph(line) + ); +} + +/** + * For each line, whether a `-->` is reachable without leaving the paragraph it + * starts in. Read at `index + 1` it answers whether an inline comment opened on + * `index` and left unclosed there is a comment at all. + */ +export function commentClosesBelow(lines: string[]): boolean[] { + const closes: boolean[] = new Array(lines.length + 1).fill(false); + for (let at = lines.length - 1; at >= 0; at -= 1) { + const line = lines[at] ?? ""; + closes[at] = + !startsBlock(line) && + (line.includes(COMMENT_CLOSE) || (closes[at + 1] ?? false)); + } + return closes; +} diff --git a/studio/frontend/src/lib/markdown-list-columns.ts b/studio/frontend/src/lib/markdown-list-columns.ts new file mode 100644 index 0000000000..761cdfac54 --- /dev/null +++ b/studio/frontend/src/lib/markdown-list-columns.ts @@ -0,0 +1,357 @@ +// SPDX-License-Identifier: AGPL-3.0-only +// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 + +/** + * CommonMark measures a block's indentation from its container, not the left + * margin: four spaces at document level and four under a bullet mean different + * things. Tracking the open items lets both changelog scanners ask "is this + * indented code?" the way a renderer would. + * + * Ported from `_open_lists` in studio/backend/utils/changelog.py so the three + * scanners classify a line the same way. + */ + +/** The open list items, innermost last, by the column their content starts. */ +export interface ListState { + columns: number[]; + // True while the innermost item has had no content since its marker. + emptyItem: boolean; +} + +export const EMPTY_LIST_STATE: ListState = { columns: [], emptyItem: false }; + +// The marker needs whitespace after it, so `2.0` is a version, not an item. +const LIST_ITEM = /^[ \t]*([-*+]|\d{1,9}[.)])([ \t]+|$)/; +const THEMATIC_BREAK = + /^ {0,3}(?:(?:\*[ \t]*){3,}|(?:-[ \t]*){3,}|(?:_[ \t]*){3,})$/; +const BLOCK_QUOTE = /^ {0,3}>/; +const QUOTE_MARKER = /^ {0,3}>[ \t]?/; +// Blocks that are not paragraph text, so they cannot continue one lazily. +const PARAGRAPH_TEXT = /^ {0,3}(?![-*+>]([ \t]|$)|\d{1,9}[.)]([ \t]|$))\S/; +// Blocks that break into an open paragraph, closing it rather than continuing +// it. A link reference definition is not one of them. +const INTERRUPTS = + /^ {0,3}(?:#{1,6}([ \t]|$)|(?:\*[ \t]*){3,}$|(?:-[ \t]*){3,}$|(?:_[ \t]*){3,}$)/; +const FENCE = /^ {0,3}(?:`{3,}|~{3,})/; +const HTML_BLOCK_OPEN = /^ {0,3}<\/?([a-zA-Z][a-zA-Z0-9-]*)(?=[\s/>]|$)/; +const HTML_BLOCK_TAGS = new Set( + `address article aside base basefont blockquote body caption center col colgroup + dd details dialog dir div dl dt fieldset figcaption figure footer form frame + frameset h1 h2 h3 h4 h5 h6 head header hr html iframe legend li link main menu + menuitem nav noframes ol optgroup option p param search section summary table + tbody td tfoot th thead title tr track ul`.split(/\s+/), +); +// Content indented more than this after a marker is an indented code block, so +// the item's content starts one column past the marker instead. +const MAX_ITEM_PADDING = 4; +// Columns past its container at which a line becomes an indented code block. +const INDENTED_CODE = 4; +// Stands in for a line the renderer hides. `#` is a block of its own, so list +// tracking reads it like a comment: never a marker, never a lazy continuation. +const HIDDEN_BLOCK = "#"; +const LEADING_SPACE = /^[ \t]*/; + +/** + * `line` as list tracking sees it once the renderer hides its text. A comment or + * raw HTML block renders nothing but is still a block at its own column, so it + * closes the items it sits left of. Only the indentation survives: what the block + * hides is not Markdown and must not open a list. `marker` is the part opening + * the item the block is content of, which survives too. Ported from + * `_hidden_structure` on the backend. + */ +export function hiddenStructure(line: string, marker = ""): string { + if (marker) { + return `${marker}${HIDDEN_BLOCK}`; + } + const indent = LEADING_SPACE.exec(line)?.[0] ?? ""; + return line.trim() ? `${indent}${HIDDEN_BLOCK}` : ""; +} + +/** Columns of leading whitespace, counting a tab to the next stop of four. */ +export function indentWidth(line: string): number { + let width = 0; + for (const char of line) { + if (char === " ") { + width += 1; + } else if (char === "\t") { + width += 4 - (width % 4); + } else { + break; + } + } + return width; +} + +/** + * Whether `line` starts a block that can break into an open paragraph. A quote + * marker always can; a list item only with content, an ordered one only at 1. + * Anything else is text of the paragraph it appears to interrupt. + */ +export function interruptsParagraph(line: string): boolean { + if (BLOCK_QUOTE.test(line)) { + return true; + } + const item = THEMATIC_BREAK.test(line) ? null : LIST_ITEM.exec(line); + if (item === null) { + return false; + } + const marker = item[1] ?? ""; + if (!line.slice(item[0].length).trim()) { + return false; + } + const ordered = marker.endsWith(".") || marker.endsWith(")"); + return !ordered || marker.slice(0, -1) === "1"; +} + +/** + * Whether a marker-shaped `line` is really text of the paragraph above. Only a + * marker inside the paragraph's own item interrupts it; one to the left closes + * that item and opens a sibling. A quote owns the paragraph its lines hold, so a + * marker outside the quote opens a list of its own. + */ +export function lazyMarker( + line: string, + state: ListState, + afterParagraph: boolean, + quoted: boolean, +): boolean { + const item = THEMATIC_BREAK.test(line) ? null : LIST_ITEM.exec(line); + const columns = state.columns; + const inside = + columns.length === 0 || indentWidth(line) >= (columns.at(-1) ?? 0); + return ( + item !== null && + afterParagraph && + !quoted && + inside && + !interruptsParagraph(line) + ); +} + +/** `columns` with every item whose content starts past `indent` closed. */ +function dropDeeper(columns: number[], indent: number): number[] { + let open = columns.length; + while (open > 0 && (columns[open - 1] ?? 0) > indent) { + open -= 1; + } + return open === columns.length ? columns : columns.slice(0, open); +} + +/** `line` with up to `columns` columns of leading whitespace removed. */ +function stripIndent(line: string, columns: number): string { + let width = 0; + let index = 0; + while (index < line.length && width < columns) { + const char = line[index]; + if (char !== " " && char !== "\t") { + break; + } + width += char === " " ? 1 : 4 - (width % 4); + index += 1; + } + return line.slice(index); +} + +/** + * Whether `line` can continue a paragraph it is indented out of. Only plain text + * can: a heading, fence, break or HTML block starts a block of its own, closing + * the item instead. An underline is not one: it may never be lazy, so `===` left + * of an open item is more of the item's paragraph. Nor is a definition, a block + * of its own that may not interrupt a paragraph. A row of dashes still closes the + * item: `INTERRUPTS` reads three or more as the thematic break they are. + */ +function mayBeLazy(line: string): boolean { + const named = HTML_BLOCK_OPEN.exec(line); + // Types 1 to 6 interrupt a paragraph, so a `<div>` left of an open item closes + // it. Type 7 cannot, and is deliberately excluded. + const htmlBlock = + named !== null && HTML_BLOCK_TAGS.has((named[1] ?? "").toLowerCase()); + return ( + PARAGRAPH_TEXT.test(line) && + !INTERRUPTS.test(line) && + !FENCE.test(line) && + !htmlBlock + ); +} + +/** + * Whether `line` reads as more of a paragraph open in its container, measured + * from `column` where that container's content starts: four columns past it the + * line is indented code, which may not interrupt a paragraph, so indentation + * alone never closes the one above. + */ +export function continuesParagraph(line: string, column: number): boolean { + const inner = stripIndent(line, column); + return indentWidth(inner) >= INDENTED_CODE || mayBeLazy(inner); +} + +/** `line` with up to `depth` blockquote markers removed, and how many went. */ +function stripQuotes(line: string, depth: number): [string, number] { + let rest = line; + let removed = 0; + let marker = removed < depth ? QUOTE_MARKER.exec(rest) : null; + while (marker !== null) { + rest = rest.slice(marker[0].length); + removed += 1; + marker = removed < depth ? QUOTE_MARKER.exec(rest) : null; + } + return [rest, removed]; +} + +/** What a blockquote line holds, with its markers stripped. */ +function quoteContent(line: string): string { + return stripQuotes(line, Number.POSITIVE_INFINITY)[0]; +} + +/** How many blockquotes `line` is written inside. */ +export function quoteDepth(line: string): number { + return stripQuotes(line, Number.POSITIVE_INFINITY)[1]; +} + +/** + * `line` as the container it is written in sees it, with `quotes` blockquote + * markers and the open item's content column removed. CommonMark measures a block + * from its container, not the margin (spec 0.31.2 sections 5.1, 5.2), so `> ~~~` + * and a fence under a nested bullet are openers despite sitting more than three + * columns in. + */ +export function containerContent( + line: string, + state: ListState, + quotes: number, +): string { + const [inner] = stripQuotes(line, quotes); + if (quotes > 0) { + // A list inside a quote is the quote's own; this tracker follows document + // level only, so its columns do not apply here. + return inner; + } + const columns = dropDeeper(state.columns, indentWidth(inner)); + return stripIndent(inner, columns.at(-1) ?? 0); +} + +/** + * `line` read from the content column of a list item that opens on it. A block + * written as an item's first content sits inside that item, so ``- ``` `` opens a + * fence even though its marker is not within three columns of the container (spec + * 0.31.2 section 5.2). Padding is capped the way `openLists` caps it, or + * ``- ``` `` would read as a fence rather than the indented code it is. A + * marker the paragraph above swallows opens no item, so its line is returned + * whole, as is one four columns past its container. + */ +export function itemContent(line: string, afterParagraph: boolean): string { + if ( + indentWidth(line) >= INDENTED_CODE || + (afterParagraph && !interruptsParagraph(line)) + ) { + return line; + } + const item = THEMATIC_BREAK.test(line) ? null : LIST_ITEM.exec(line); + if (item === null) { + return line; + } + const padding = indentWidth(item[2] ?? ""); + // Over-indented content starts one column past the marker; the rest of the + // padding is the content's own indentation. + const over = padding > MAX_ITEM_PADDING ? padding - 1 : 0; + return `${" ".repeat(over)}${line.slice(item[0].length)}`; +} + +/** Whether a blockquote owns the paragraph the line below could continue. */ +export interface QuoteState { + // True while a quoted paragraph is open, so plain text below is more of it. + inQuote: boolean; + // True whenever that paragraph is the quote's rather than the document's. + quoted: boolean; +} + +export const NO_QUOTE: QuoteState = { inQuote: false, quoted: false }; + +/** + * The quote state after `line`, given the state after the line above and the + * content column of the item `line` sits in. A quote owns the paragraph its own + * lines hold, so a marker written outside the quote opens a list of its own + * rather than reading as more of that paragraph. Ported from `in_quote` tracking + * in changelog.py. + */ +export function quoteState( + line: string, + inQuote: boolean, + column = 0, +): QuoteState { + if (BLOCK_QUOTE.test(line)) { + // An empty quote holds no paragraph, so the line below starts a new one. + return { inQuote: mayBeLazy(quoteContent(line)), quoted: true }; + } + const open = inQuote && continuesParagraph(line, column); + return { inQuote: open, quoted: open }; +} + +/** + * `columns` with every item `line` is written to the left of closed. Read inside + * the container the item sits in, not from the margin: a line that only looks + * dedented there is lazy text of the item's paragraph, leaving the item open. + */ +function closeDedented( + columns: number[], + line: string, + indent: number, + afterParagraph: boolean, +): number[] { + let open = columns.length; + while (open > 0 && (columns[open - 1] ?? 0) > indent) { + const outer = open > 1 ? (columns[open - 2] ?? 0) : 0; + if (afterParagraph && continuesParagraph(line, outer)) { + break; + } + open -= 1; + } + return open === columns.length ? columns : columns.slice(0, open); +} + +/** + * The list items still open after `line`. A dedented line closes an item unless + * it is a lazy paragraph continuation. A new marker nests under a deeper column + * and replaces a sibling. `quoted` marks a paragraph the blockquote above owns: + * a marker outside the quote is not text of it, so it opens a list of its own. + */ +export function openLists( + line: string, + state: ListState, + afterParagraph: boolean, + quoted = false, +): ListState { + let columns = state.columns; + if (!line.trim()) { + // A blank line leaves the list open, unless the item is still empty: an + // item may begin with one blank line, and later content is outside it. + return { + columns: state.emptyItem ? columns.slice(0, -1) : columns, + emptyItem: false, + }; + } + const indent = indentWidth(line); + const item = THEMATIC_BREAK.test(line) ? null : LIST_ITEM.exec(line); + const empty = item !== null && !line.slice(item[0].length).trim(); + if (lazyMarker(line, state, afterParagraph, quoted)) { + // A lazy continuation or an underline, so the open items are untouched. + return state; + } + columns = closeDedented(columns, line, indent, afterParagraph); + // Four columns past its container the marker is an indented code block, or + // lazy text of the paragraph above it, so it opens no list of its own. + if (item === null || indent - (columns.at(-1) ?? 0) >= INDENTED_CODE) { + return { columns, emptyItem: false }; + } + const marker = item[1] ?? ""; + let padding = indentWidth(item[2] ?? ""); + if (padding === 0 || padding > MAX_ITEM_PADDING) { + // An empty or over-indented item still holds one column of content. + padding = 1; + } + // A sibling marker replaces the item it lines up with. + return { + columns: [...dropDeeper(columns, indent), indent + marker.length + padding], + emptyItem: empty, + }; +} diff --git a/studio/frontend/src/lib/release-notes-preview.ts b/studio/frontend/src/lib/release-notes-preview.ts new file mode 100644 index 0000000000..97441b7cbb --- /dev/null +++ b/studio/frontend/src/lib/release-notes-preview.ts @@ -0,0 +1,1005 @@ +// SPDX-License-Identifier: AGPL-3.0-only +// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 + +// Top changelog bullets, shown in the collapsed update popup. +import { codeSpans, parkCodeSpans } from "@/lib/markdown-code-spans"; +import { commentClosesBelow } from "@/lib/markdown-inline-comments"; +import { + EMPTY_LIST_STATE, + type ListState, + NO_QUOTE, + type QuoteState, + hiddenStructure, + indentWidth, + itemContent, + openLists, + quoteState, +} from "@/lib/markdown-list-columns"; + +export const RELEASE_NOTES_PREVIEW_ITEMS = 4; +const PREVIEW_ITEM_MAX_CHARS = 120; +// Bullets indented past the shallowest one are nested detail, not headlines. +const NESTED_INDENT_TOLERANCE = 1; +const TAB_WIDTH = 4; +// Four spaces starts an indented code block in Markdown. +const INDENTED_CODE_INDENT = 4; + +// At most three leading spaces: deeper is indented code, not a fence. +const FENCE = /^ {0,3}(`{3,}|~{3,})(.*)$/; +// An ATX heading needs a space, tab or line end after the marker, as in +// _HEADING_PATTERN. `\s` would match a non-breaking space and eat prose, and a +// bare `##` is an empty heading that still ends a bullet. +const HEADING = /^#{1,6}(?:[ \t]|$)/; +const BULLET = /^(?:[-*+]|(\d{1,9})[.)])[ \t]+(.*)$/; +// At most three leading spaces, as everywhere else: deeper is indented code, +// so a quoted line inside a code sample cannot reach the collector. +const BLOCKQUOTE = /^ {0,3}>[ \t]?/; +// A GFM delimiter cell is hyphens with an optional alignment colon each side. +const TABLE_DELIMITER_CELL = /^:?-+:?$/; +// "- - -" and "***" are horizontal rules, not bullets and not notes. +const THEMATIC_BREAK = + /^ {0,3}(?:(?:\*[ \t]*){3,}|(?:-[ \t]*){3,}|(?:_[ \t]*){3,})$/; +// Destinations may escape or balance parentheses, and labels may nest one +// level so `[![alt](img)](link)` still resolves. +const DESTINATION = "\\((?:\\\\.|[^()\\\\]|\\([^()]*\\))*\\)"; +const LABEL = "((?:[^\\[\\]\\\\]|\\\\.|\\[(?:[^\\[\\]\\\\]|\\\\.)*\\])*)"; +const IMAGE = new RegExp(`!\\[${LABEL}\\]${DESTINATION}`, "g"); +const LINK = new RegExp(`\\[${LABEL}\\]${DESTINATION}`, "g"); +// Reference forms: `[text][label]`, `[text][]` and the shortcut `[text]`. +const IMAGE_REFERENCE = new RegExp(`!\\[${LABEL}\\](?:\\[([^\\]]*)\\])?`, "g"); +const LINK_REFERENCE = new RegExp(`\\[${LABEL}\\](?:\\[([^\\]]*)\\])?`, "g"); +// A definition line renders as nothing at all. +const DEFINITION = /^ {0,3}\[((?:[^\[\]\\]|\\.)+)\]:/; +// CommonMark: a backslash escapes ASCII punctuation. +const ESCAPE = /\\([!-/:-@[-`{-~])/g; +// Private-use sentinels park code spans, so document text cannot contain them. +const SENTINELS = /[\uE000\uE001]/g; +const LINE_ENDINGS = /\r\n?/g; +const TABS = /\t/g; +// Real tags only: a name character must follow "<", so a version constraint +// like "Support Python <3.15 and >3.9" keeps its operators. +const HTML_TAG = /<\/?[a-zA-Z][^>]*>/g; +// <https://x> and <a@b.c> are Markdown autolinks: keep the text they render. +const AUTOLINK = /<([a-zA-Z][a-zA-Z0-9+.-]*:[^\s<>]*|[^\s<>@]+@[^\s<>@]+)>/g; +// CommonMark type 1 HTML blocks render literally until a closing tag, which +// the spec says need not be the one that opened the block. +const RAW_HTML_OPEN = /^ {0,3}<(pre|script|style|textarea)(?=[\s>]|$)/i; +const RAW_HTML_CLOSE = /<\/(pre|script|style|textarea)\s*>/i; +// Types 3 to 5 (processing instructions, declarations, CDATA) are literal too, +// each ending on its own delimiter. Comments open mid-line, handled separately. +const RAW_BLOCKS: [RegExp, RegExp][] = [ + [RAW_HTML_OPEN, RAW_HTML_CLOSE], + [/^ {0,3}<\?/, /\?>/], + [/^ {0,3}<!\[CDATA\[/, /\]\]>/], + // A declaration needs an uppercase letter, so `<!note` stays ordinary text. + [/^ {0,3}<![A-Z]/, />/], +]; +// Type 6 and 7 blocks run to the next blank line, so `<details>` holds Markdown +// only after one. Type 7 (any complete tag alone) cannot interrupt a paragraph. +const HTML_BLOCK_OPEN = /^ {0,3}<\/?([a-zA-Z][a-zA-Z0-9-]*)(?=[\s/>]|$)/; +const HTML_ATTRIBUTE = + "(?:\\s+[a-zA-Z_:][a-zA-Z0-9_.:-]*(?:\\s*=\\s*(?:[^\\s\"'=<>`]+|'[^']*'|\"[^\"]*\"))?)"; +const HTML_TAG_ONLY_LINE = new RegExp( + `^ {0,3}(?:<[a-zA-Z][a-zA-Z0-9-]*${HTML_ATTRIBUTE}*\\s*/?>|</[a-zA-Z][a-zA-Z0-9-]*\\s*>)\\s*$`, +); +const HTML_BLOCK_TAGS = new Set( + `address article aside base basefont blockquote body caption center col colgroup + dd details dialog dir div dl dt fieldset figcaption figure footer form frame + frameset h1 h2 h3 h4 h5 h6 head header hr html iframe legend li link main menu + menuitem nav noframes ol optgroup option p param search section summary table + tbody td tfoot th thead title tr track ul`.split(/\s+/), +); +// Only spaces and tabs may follow a closing fence. +const NON_SPACE = /[^ \t]/; +const HEADING_LINE = /^ {0,3}#{1,6}(?:[ \t]|$)/; +const COMMENT_BLOCK_OPEN = /^ {0,3}<!--/; +const COMMENT_OPEN = "<!--"; +const COMMENT_CLOSE = "-->"; +// Paired emphasis only. Underscores inside identifiers are literal, so +// UNSLOTH_DISABLE_UPDATE_CHECK keeps its name. +const BOLD_STAR = /\*\*(?=\S)([\s\S]*?\S)\*\*/g; +const BOLD_UNDERSCORE = /(^|[^\w])__(?=\S)([\s\S]*?\S)__(?=[^\w]|$)/g; +const ITALIC_STAR = /\*(?=\S)([^*\n]*?\S)\*/g; +const ITALIC_UNDERSCORE = /(^|[^\w])_(?=\S)([^_\n]*?\S)_(?=[^\w]|$)/g; +const BACKTICK = /`/g; +// A closer is a run of the same length, so `` `x` `` keeps its backticks. +// Streamdown renders `AT&T` as "AT&T", so the preview decodes entities too. +const NAMED_ENTITIES: Record<string, string> = { + amp: "&", + lt: "<", + gt: ">", + quot: '"', + apos: "'", + nbsp: "\u00a0", +}; +const ENTITY = /&(#\d{1,7}|#[xX][0-9a-fA-F]{1,6}|[a-zA-Z][a-zA-Z0-9]{1,31});/g; +const PARKED = /\uE000(\d+)\uE001/g; +const WHITESPACE = /\s+/g; +// Sentence end followed by something that actually starts a sentence. +const SENTENCE_BREAK = /[.!?]\s+(?=["'“‘]?[A-Z0-9])/g; +const TRAILING_WORD = /(\S+)$/; +// A period here ends an abbreviation, not the sentence. +const ABBREVIATIONS = new Set([ + "e.g.", + "i.e.", + "etc.", + "vs.", + "cf.", + "approx.", + "no.", + "fig.", + "al.", + "dr.", + "mr.", + "mrs.", + "ms.", + "prof.", + "inc.", + "ltd.", + "st.", + "jr.", + "sr.", +]); +const INITIAL = /^[A-Za-z]\.$/; +const MIN_LEAD_CHARS = 12; + +/** Strip tags until stable, so a removal cannot re-form a tag. */ +function stripHtmlTags(text: string): string { + let out = text; + let previous: string; + do { + previous = out; + out = out.replace(HTML_TAG, ""); + } while (out !== previous); + return out; +} + +export interface ReleaseNotesPreviewItem { + // Leading sentence, highlighted in the preview. + lead: string; + // Rest of the bullet, de-emphasised. Empty for single-sentence bullets. + rest: string; +} + +export interface ReleaseNotesPreview { + items: ReleaseNotesPreviewItem[]; + // Bullets past the preview limit, for a "+N more" affordance. + remaining: number; +} + +interface Bullet { + text: string; + indent: number; +} + +/** Whether a reference points at a definition the document actually has. */ +function definedLabel( + labels: Set<string> | undefined, + reference: string | undefined, + text: string, +): boolean { + if (labels === undefined) { + return false; + } + const label = (reference?.trim() ? reference : text) + .trim() + .replace(WHITESPACE, " ") + .toLowerCase(); + return labels.has(label); +} + +/** One entity as the character it renders as, or unchanged if unknown. */ +function decodeEntity(match: string, body: string): string { + if (body.startsWith("#")) { + const hex = body[1] === "x" || body[1] === "X"; + const code = Number.parseInt( + hex ? body.slice(2) : body.slice(1), + hex ? 16 : 10, + ); + return Number.isFinite(code) && code > 0 && code <= 0x10ffff + ? String.fromCodePoint(code) + : match; + } + return NAMED_ENTITIES[body.toLowerCase()] ?? match; +} + +/** Inline markdown stripped to plain text. */ +function toPlainText(markdown: string, labels?: Set<string>): string { + // Park code spans first: their contents are literal and must survive below. + const codes: string[] = []; + const park = (text: string): string => { + codes.push(text); + return `\uE000${codes.length - 1}\uE001`; + }; + // Escaped punctuation is literal too, so `\*not italic\*` keeps its stars. + const parked = parkCodeSpans(markdown, park).replace(ESCAPE, (_match, char) => + park(char), + ); + + return stripHtmlTags( + parked + .replace(AUTOLINK, "$1") + .replace(IMAGE, "") + .replace(LINK, "$1") + .replace(IMAGE_REFERENCE, (match, text, ref) => + definedLabel(labels, ref, text) ? "" : match, + ) + .replace(LINK_REFERENCE, (match, text, ref) => + definedLabel(labels, ref, text) ? text : match, + ), + ) + .replace(BOLD_STAR, "$1") + .replace(BOLD_UNDERSCORE, "$1$2") + .replace(ITALIC_STAR, "$1") + .replace(ITALIC_UNDERSCORE, "$1$2") + .replace(BACKTICK, "") + .replace(ENTITY, decodeEntity) + .replace(PARKED, (_match, index: string) => codes[Number(index)] ?? "") + .replace(WHITESPACE, " ") + .trim(); +} + +function truncate(text: string): string { + if (text.length <= PREVIEW_ITEM_MAX_CHARS) { + return text; + } + const clipped = text.slice(0, PREVIEW_ITEM_MAX_CHARS); + const lastSpace = clipped.lastIndexOf(" "); + return `${(lastSpace > 40 ? clipped.slice(0, lastSpace) : clipped).trimEnd()}...`; +} + +interface ContentLine { + text: string; + indent: number; + // Blockquoted lines are quoted examples, not the release's own bullets. + quoted: boolean; + // Content column of the innermost open list item. CommonMark measures + // indentation from here, so `indent - column` is the real depth. + column: number; +} + +/** + * `line` with its comments removed, whether a comment block stays open, and + * whether an inline comment runs on into the line below. + * + * Only a comment starting a line opens a block, which hides whole lines to the + * one holding `-->`. One written mid-sentence is inline HTML belonging to its + * paragraph, so its `-->` may arrive on a later line and only the text up to it + * is hidden. `closesBelow` says one does; without it the opener is ordinary text + * and hides nothing below. + * + * "Starting a line" is read inside the container, so `blockOpen` comes from the + * item's content rather than the raw line. + */ +function stripCommentSpans( + line: string, + startInComment: boolean, + runOn: boolean, + closesBelow: boolean, + blockOpen: boolean, +): [string, boolean, boolean] { + if (startInComment) { + // The closing line belongs to the block, tail included. + return ["", !line.includes(COMMENT_CLOSE), false]; + } + + let visible = ""; + let index = 0; + if (runOn) { + const closed = line.indexOf(COMMENT_CLOSE); + if (closed === -1) { + return ["", false, true]; + } + // Only up to the closer: the tail is the paragraph's own text again. + index = closed + COMMENT_CLOSE.length; + } else if (blockOpen) { + // `<!-->` and `<!--->` are complete comments, so the closer may overlap the + // opener; searching past it would hide every later release. + return ["", !line.includes(COMMENT_CLOSE), false]; + } + + const spans = codeSpans(line); + while (index < line.length) { + const open = line.indexOf(COMMENT_OPEN, index); + if (open === -1) { + visible += line.slice(index); + break; + } + // A delimiter inside inline code is literal, not a comment opener. + const span = spans.find( + (candidate) => candidate.start <= open && candidate.end > open, + ); + if (span) { + visible += line.slice(index, span.end); + index = span.end; + continue; + } + const close = line.indexOf(COMMENT_CLOSE, open + COMMENT_OPEN.length); + if (close === -1) { + if (closesBelow) { + // The paragraph carries the comment on, so this line and the next are in it. + return [visible + line.slice(index, open), false, true]; + } + // Nothing closes it at all, so the renderer shows it as text. + visible += line.slice(index); + break; + } + visible += line.slice(index, open); + index = close + COMMENT_CLOSE.length; + } + return [visible, false, false]; +} + +/** Strips raw block content. State is the open block's index, or null. */ +function stripRawHtml( + line: string, + openBlock: number | null, +): [string, number | null] { + if (openBlock !== null) { + return RAW_BLOCKS[openBlock]?.[1].test(line) ? ["", null] : ["", openBlock]; + } + // A block only opens at the start of a line; mid-line tags are inline HTML. + for (const [index, [opener, closer]] of RAW_BLOCKS.entries()) { + const open = opener.exec(line); + if (!open) { + continue; + } + const rest = line.slice(open[0].length); + return closer.test(rest) ? ["", null] : ["", index]; + } + return [line, null]; +} + +/** True if `line` starts a CommonMark type 6 or type 7 HTML block. */ +function opensHtmlBlock(line: string, afterParagraph: boolean): boolean { + const named = HTML_BLOCK_OPEN.exec(line); + if (named && HTML_BLOCK_TAGS.has((named[1] ?? "").toLowerCase())) { + return true; + } + return !afterParagraph && HTML_TAG_ONLY_LINE.test(line); +} + +/** + * The line as list tracking sees it. A comment or raw block renders nothing, but + * the line opening one is still a block at its own column, so it closes a list + * item it sits left of. Only the column survives, since the text it hides is not + * Markdown. A line inside a block already open is that block's content, so it + * keeps neither. A marker the hidden block is the content of survives with the + * column, so the item it opens is still tracked. + */ +function structuralLine( + line: string, + visible: string, + hidden: boolean, + marker: string, +): string { + if (visible.trim() || hidden) { + return visible; + } + return hiddenStructure(line, marker); +} + +interface ScanState { + openFence: string | null; + // Content column of the list item the open block belongs to, 0 at document + // level. A fence and an HTML block are scoped to their container, so the item's + // end closes them. Only one of the three is ever open. + blockColumn: number; + inComment: boolean; + // True while an inline comment opened above runs on into this line, carried by + // the paragraph holding it. + runOn: boolean; + inRawHtml: number | null; + inHtmlBlock: boolean; + afterParagraph: boolean; +} + +interface ScannedLine { + // What a reader would see: "" for structure and hidden blocks, null for + // fenced content, which is skipped so it cannot split a bullet. + text: string | null; + // The same line as list tracking sees it: blank wherever nothing renders, + // but kept whole where an indent still closes an open item. + structural: string; +} + +function visibleText( + line: string, + state: ScanState, + closesBelow: boolean, +): ScannedLine { + // Raw HTML first: its contents are literal, so a fence inside it is not one. + if (state.inRawHtml !== null) { + const [after, stillInRaw] = stripRawHtml(line, state.inRawHtml); + state.inRawHtml = stillInRaw; + return { text: after, structural: "" }; + } + if (state.inHtmlBlock) { + // A blank line is the only thing that ends a type 6 or 7 block. + state.inHtmlBlock = line.trim() !== ""; + return { text: "", structural: "" }; + } + // An opener is read past a marker on the same line, since a fence written as a + // list item's first content opens inside it. Only an opener: fenced content is + // literal and a closer carries no marker. + const commented = state.inComment || state.runOn; + const fence = commented + ? null + : FENCE.exec( + state.openFence === null + ? itemContent(line, state.afterParagraph) + : line, + ); + // A backtick fence whose info string holds a backtick is prose, not a fence. + if ( + fence && + (state.openFence !== null || opensFence(fence[1] ?? "", fence[2] ?? "")) + ) { + state.openFence = nextFence( + state.openFence, + fence[1] ?? "", + fence[2] ?? "", + ); + // Hidden from the collector, but its indent still closes an item. + return { text: "", structural: line }; + } + if (state.openFence !== null) { + return { text: null, structural: "" }; + } + return visibleContent(line, state, closesBelow); +} + +/** `visibleText` for a line no fence or HTML block already owns. */ +function visibleContent( + line: string, + state: ScanState, + closesBelow: boolean, +): ScannedLine { + // A block already open owns this line, so it is content rather than a block + // written at the column it happens to start in. + const hidden = state.inComment || state.inRawHtml !== null; + const carried = state.runOn; + // A comment is an HTML block too, so one written as a list item's first content + // opens inside that item exactly as a fence does: read past a marker on the + // same line rather than from the margin. + const content = itemContent(line, state.afterParagraph); + const opensComment = + !(state.inComment || carried) && COMMENT_BLOCK_OPEN.test(content); + // Commented-out notes are not rendered, so they are not previewed either. + const [uncommented, stillInComment, stillRunOn] = stripCommentSpans( + line, + state.inComment, + state.runOn, + closesBelow, + opensComment, + ); + state.inComment = stillInComment; + state.runOn = stillRunOn; + const [visible, stillInRaw] = stripRawHtml(uncommented, state.inRawHtml); + state.inRawHtml = stillInRaw; + // Taken before the opener is hidden: it renders as nothing, but its indent still + // closes a list item it sits left of, and a marker on its line still opens one. + // A line an inline comment runs on into is still a line of the paragraph that + // carries it, so only its text is hidden, never its block structure. + const marker = opensComment + ? line.slice(0, line.length - content.length) + : ""; + const structural = carried + ? line + : structuralLine(line, visible, hidden, marker); + if ( + !carried && + stillInRaw === null && + visible.trim() && + opensHtmlBlock(visible, state.afterParagraph) + ) { + state.inHtmlBlock = true; + return { text: "", structural }; + } + return { text: visible, structural }; +} + +/** + * Marker of a fence the line scanner skipped because it is indented. Only a line + * within three columns of its item's content column is one: deeper than that it + * is an indented code block, which a dedented bullet ends. + */ +function opensDeepFence(line: ContentLine): string | null { + if ( + line.indent < INDENTED_CODE_INDENT || + line.indent - line.column >= INDENTED_CODE_INDENT + ) { + return null; + } + const fence = FENCE.exec(line.text); + return fence ? (fence[1] ?? null) : null; +} + +/** + * True when `line` is the first one outside the deep fence opened with `marker` + * at `column`. A fence inside a list item runs only to the end of that item, so a + * line left of the item's content column closes both, as `fence_column` does on + * the backend. + */ +function endsDeepFence( + marker: string, + column: number, + line: ContentLine, +): boolean { + return line.indent < column || closesDeepFence(marker, line); +} + +/** True when `line` closes the deep fence opened with `marker`. */ +function closesDeepFence(marker: string, line: ContentLine): boolean { + const fence = FENCE.exec(line.text); + if (!fence) { + return false; + } + const closer = fence[1] ?? ""; + return ( + closer[0] === marker[0] && + closer.length >= marker.length && + !NON_SPACE.test(fence[2] ?? "") + ); +} + +/** + * Cells of a GFM table row, or null when the line holds no pipe at all. The + * optional leading and trailing pipes are delimiters, not empty cells, and a + * `\|` is literal text inside one. + */ +function tableCells(text: string): string[] | null { + if (!text.includes("|")) { + return null; + } + const cells: string[] = []; + let cell = ""; + for (let at = 0; at < text.length; at += 1) { + const char = text[at]; + if (char === "\\") { + cell += char + (text[at + 1] ?? ""); + at += 1; + continue; + } + if (char === "|") { + cells.push(cell); + cell = ""; + continue; + } + cell += char; + } + cells.push(cell); + if (cells.length > 1 && text.startsWith("|")) { + cells.shift(); + } + if (cells.length > 1 && text.endsWith("|")) { + cells.pop(); + } + return cells; +} + +/** Width of a GFM delimiter row such as `| --- |:-:|`, or null if not one. */ +function delimiterWidth(text: string): number | null { + const cells = tableCells(text); + if (cells === null || cells.length === 0) { + return null; + } + return cells.every((cell) => TABLE_DELIMITER_CELL.test(cell.trim())) + ? cells.length + : null; +} + +/** + * Line indices that belong to a GFM table. A table needs a header row and a + * delimiter row of the same width, and runs to a blank line or another block. Its + * cells render as a grid, not prose, so the preview drops them like a code block. + */ +function opensTable( + header: ContentLine | undefined, + delimiter: ContentLine | undefined, +): boolean { + if (header === undefined || delimiter === undefined) { + return false; + } + if (!header.text || header.quoted) { + return false; + } + if (header.indent - header.column >= INDENTED_CODE_INDENT) { + return false; + } + const width = delimiterWidth(delimiter.text); + const cells = tableCells(header.text); + return width !== null && cells !== null && cells.length === width; +} + +/** A blank line, a heading or a list marker: where GFM breaks a table. */ +function breaksTable(line: ContentLine | undefined): boolean { + return ( + !line?.text || + line.quoted || + HEADING.test(line.text) || + BULLET.test(line.text) || + line.indent - line.column >= INDENTED_CODE_INDENT + ); +} + +function tableLines(lines: ContentLine[]): Set<number> { + const rows = new Set<number>(); + let at = 0; + while (at + 1 < lines.length) { + if (!opensTable(lines[at], lines[at + 1])) { + at += 1; + continue; + } + rows.add(at); + rows.add(at + 1); + let row = at + 2; + while (row < lines.length && !breaksTable(lines[row])) { + rows.add(row); + row += 1; + } + at = row; + } + return rows; +} + +/** A backtick fence's info string may not contain a backtick. */ +function opensFence(marker: string, rest: string): boolean { + return marker[0] !== "`" || !rest.includes("`"); +} + +function nextFence( + open: string | null, + marker: string, + rest: string, +): string | null { + if (open === null) { + return opensFence(marker, rest) ? marker : null; + } + const closes = + marker[0] === open[0] && + marker.length >= open.length && + // Only spaces or tabs may follow a closer, per CommonMark. + !NON_SPACE.test(rest); + return closes ? null : open; +} + +/** Whether a fence, a raw block, a comment or an HTML block is open. */ +function inBlock(state: ScanState): boolean { + return ( + state.openFence !== null || + state.inRawHtml !== null || + state.inHtmlBlock || + state.inComment + ); +} + +/** + * A fence, comment or HTML block inside a list item runs only to the end of that + * item, so a line dedented out of the item closes both. Lazy continuation reaches + * into none of them, so any content left of the item ends it. + */ +function closeDedentedBlock(line: string, state: ScanState): void { + if (state.blockColumn === 0 || !inBlock(state)) { + return; + } + if (line.trim() && indentWidth(line) < state.blockColumn) { + state.openFence = null; + state.inRawHtml = null; + state.inHtmlBlock = false; + state.inComment = false; + state.blockColumn = 0; + } +} + +/** Ties a block just opened to the list item it is written inside. */ +function scopeBlock( + state: ScanState, + wasInBlock: boolean, + lists: ListState, +): void { + if (!inBlock(state)) { + state.blockColumn = 0; + return; + } + if (!wasInBlock) { + // The opener closed the items it is dedented out of first, so this is the + // column of the item the block really sits in. + state.blockColumn = lists.columns.at(-1) ?? 0; + } +} + +function contentLines(markdown: string): ContentLine[] { + const lines: ContentLine[] = []; + const state: ScanState = { + openFence: null, + blockColumn: 0, + inComment: false, + runOn: false, + inRawHtml: null, + inHtmlBlock: false, + afterParagraph: false, + }; + let lists: ListState = EMPTY_LIST_STATE; + let quote: QuoteState = NO_QUOTE; + + const rawLines = markdown + .split("\n") + .map((raw) => raw.replace(TABS, " ".repeat(TAB_WIDTH))); + const closesBelow = commentClosesBelow(rawLines); + for (const [index, line] of rawLines.entries()) { + closeDedentedBlock(line, state); + const wasInBlock = inBlock(state); + const carried = state.runOn; + const { text: visible, structural } = visibleText( + line, + state, + closesBelow[index + 1] ?? false, + ); + // The quote state from the line above, which is what list tracking asks about. + // Only a line of text below rewrites it, so a fenced, blank or hidden line + // leaves no quoted paragraph open behind it. + const above = quote; + quote = NO_QUOTE; + // Taken with the paragraph state from the line above, as a renderer would. + lists = openLists(structural, lists, state.afterParagraph, above.quoted); + scopeBlock(state, wasInBlock, lists); + if (visible === null) { + continue; + } + if (carried && !visible.trim()) { + // Wholly inside a comment its paragraph carries: no text, and no break. + continue; + } + if (!visible.trim() || THEMATIC_BREAK.test(visible)) { + // A rule separates notes, so it breaks a bullet just like a blank line. + state.afterParagraph = false; + lines.push({ text: "", indent: 0, quoted: false, column: 0 }); + continue; + } + const quoted = BLOCKQUOTE.test(visible); + const stripped = visible.replace(BLOCKQUOTE, ""); + const indent = stripped.length - stripped.trimStart().length; + // A quoted line is measured inside its quote, where the document's open + // list items do not reach. + const column = quoted ? 0 : (lists.columns.at(-1) ?? 0); + // Only ordinary text continues a paragraph; a heading or indented code line + // (four columns past its container, outside a paragraph) ends one. + const startsCode = + !state.afterParagraph && indent - column >= INDENTED_CODE_INDENT; + state.afterParagraph = !HEADING_LINE.test(stripped) && !startsCode; + quote = quoteState(visible, above.inQuote); + lines.push({ text: stripped.trim(), indent, quoted, column }); + } + return lines; +} + +/** + * Split a bullet at its first sentence boundary. Conservative: the next + * sentence must start like one, so "CHANGELOG.md in the repo" is not a break. + */ +function splitLeadSentence(text: string): ReleaseNotesPreviewItem { + SENTENCE_BREAK.lastIndex = 0; + let match = SENTENCE_BREAK.exec(text); + while (match) { + const cut = match.index + 1; + const word = + TRAILING_WORD.exec(text.slice(0, cut))?.[1]?.toLowerCase() ?? ""; + const isAbbreviation = ABBREVIATIONS.has(word) || INITIAL.test(word); + if (!isAbbreviation && cut >= MIN_LEAD_CHARS) { + return { lead: text.slice(0, cut).trim(), rest: text.slice(cut).trim() }; + } + match = SENTENCE_BREAK.exec(text); + } + return { lead: text, rest: "" }; +} + +/** Bullets in document order, plus prose for changelogs written as paragraphs. */ +interface Collector { + bullets: Bullet[]; + prose: string[]; + // Wrapped bullets continue on following lines and belong to one item. + current: Bullet | null; + paragraph: string; + // True while the open paragraph is a quote's, which owns its own text: a + // marker written outside the quote opens a list rather than continuing it. + quotedParagraph: boolean; +} + +function flush(collector: Collector): void { + if (collector.current?.text) { + collector.bullets.push({ + text: truncate(collector.current.text), + indent: collector.current.indent, + }); + } + collector.current = null; + if (collector.paragraph) { + collector.prose.push(truncate(collector.paragraph)); + collector.paragraph = ""; + } + collector.quotedParagraph = false; +} + +function takeBullet( + collector: Collector, + text: string, + line: ContentLine, + labels: Set<string>, +): void { + flush(collector); + const item = toPlainText(text, labels); + // A quoted list is example output: prose at best, never a headline bullet. + if (!line.quoted) { + collector.current = { text: item, indent: line.indent }; + } else if (item) { + collector.prose.push(truncate(item)); + } +} + +function takeText( + collector: Collector, + text: string, + labels: Set<string>, + quoted: boolean, +): void { + const plain = toPlainText(text, labels); + if (!plain) { + return; + } + if (collector.current === null) { + // Wrapped paragraphs render as one block, so preview them as one item. + collector.paragraph = collector.paragraph + ? `${collector.paragraph} ${plain}` + : plain; + collector.quotedParagraph = quoted; + return; + } + collector.current = { + text: `${collector.current.text} ${plain}`, + indent: collector.current.indent, + }; +} + +function collectBullets(markdown: string): { + bullets: Bullet[]; + prose: string[]; +} { + const collector: Collector = { + bullets: [], + prose: [], + current: null, + paragraph: "", + quotedParagraph: false, + }; + + const lines = contentLines(markdown); + const labels = new Set<string>(); + // Skips the same code the pass below skips: a definition-shaped line inside + // code is literal, and a real definition never indents past three spaces. + let labelFence: string | null = null; + let labelColumn = 0; + for (const line of lines) { + if (labelFence !== null && !endsDeepFence(labelFence, labelColumn, line)) { + continue; + } + if (labelFence !== null) { + const dedented = line.indent < labelColumn; + labelFence = null; + // Its own closing line is code too; only a dedented one is a new block. + if (!dedented) { + continue; + } + } + const opener = opensDeepFence(line); + if (opener !== null) { + labelFence = opener; + labelColumn = line.column; + continue; + } + if (line.indent - line.column >= INDENTED_CODE_INDENT) { + continue; + } + const definition = DEFINITION.exec(line.text); + if (definition) { + labels.add( + (definition[1] ?? "").trim().replace(WHITESPACE, " ").toLowerCase(), + ); + } + } + + const tables = tableLines(lines); + let deepFence: string | null = null; + let deepColumn = 0; + for (const [index, line] of lines.entries()) { + if (!line.text || HEADING.test(line.text)) { + flush(collector); + continue; + } + // A table renders as a grid, no more previewable than a code block, and it + // ends whatever came before it. + if (tables.has(index)) { + flush(collector); + continue; + } + // A link reference definition renders as nothing. + if (collector.current === null && DEFINITION.test(line.text)) { + continue; + } + // A fence indented past three spaces belongs to a list item, so the line + // scanner missed it. Its contents are code either way. + if (deepFence !== null && !endsDeepFence(deepFence, deepColumn, line)) { + continue; + } + if (deepFence !== null) { + const dedented = line.indent < deepColumn; + deepFence = null; + // Its own closing line is code too; only a dedented one is a new block. + if (!dedented) { + continue; + } + } + const opener = opensDeepFence(line); + if (opener !== null) { + deepFence = opener; + deepColumn = line.column; + continue; + } + // An indented code block renders as code, so a "- cmd" line in one is not + // a bullet. Inside an open bullet or paragraph it is just a wrapped line. + const insideBlock = + collector.current !== null || collector.paragraph !== ""; + if (!insideBlock && line.indent - line.column >= INDENTED_CODE_INDENT) { + continue; + } + const bullet = BULLET.exec(line.text); + // Only an ordered list starting at 1 may interrupt a paragraph, so "2. Restart + // Studio" under prose is prose. A list item is not a paragraph. + const interrupts = + collector.current === null && + collector.paragraph !== "" && + !collector.quotedParagraph; + if ( + bullet && + !(interrupts && bullet[1] !== undefined && bullet[1] !== "1") + ) { + takeBullet(collector, bullet[2] ?? "", line, labels); + continue; + } + takeText(collector, line.text, labels, line.quoted); + } + flush(collector); + + return { bullets: collector.bullets, prose: collector.prose }; +} + +/** + * Top-level bullets of a release section, in document order. Nested bullets are + * detail and are skipped; prose is used when a release has no bullets. + */ +export function releaseNotesPreview( + markdown: string | null | undefined, + limit: number = RELEASE_NOTES_PREVIEW_ITEMS, +): ReleaseNotesPreview { + if (!markdown) { + return { items: [], remaining: 0 }; + } + + // The updater body arrives with CRLF; sentinels would collide with parking. + const text = markdown.replace(LINE_ENDINGS, "\n").replace(SENTINELS, ""); + const { bullets, prose } = collectBullets(text); + // Shallowest bullet defines top level, so a uniformly indented list previews. + const baseIndent = bullets.reduce( + (min, bullet) => Math.min(min, bullet.indent), + Number.POSITIVE_INFINITY, + ); + const topLevel = bullets + .filter((bullet) => bullet.indent <= baseIndent + NESTED_INDENT_TOLERANCE) + .map((bullet) => bullet.text); + + const source = topLevel.length > 0 ? topLevel : prose; + return { + items: source.slice(0, limit).map(splitLeadSentence), + remaining: Math.max(source.length - limit, 0), + }; +} diff --git a/studio/frontend/tests/sidebar-width.test.ts b/studio/frontend/tests/sidebar-width.test.ts new file mode 100644 index 0000000000..861bd45e3f --- /dev/null +++ b/studio/frontend/tests/sidebar-width.test.ts @@ -0,0 +1,72 @@ +// SPDX-License-Identifier: AGPL-3.0-only +// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 + +import assert from "node:assert/strict"; +import test from "node:test"; +import { readFile } from "node:fs/promises"; + +// Every localStorage key written by a panel width store. +const PANEL_WIDTH_KEYS = ["sidebar_width", "chat_settings_width"]; + +// The store reads window at import time, so stub it before importing. +const stubWindow = { + innerWidth: 1440, + localStorage: { + getItem: () => null, + setItem: () => {}, + }, + addEventListener: () => {}, + removeEventListener: () => {}, +}; +(globalThis as { window?: unknown }).window = stubWindow; + +const { + clampSidebarWidth, + SIDEBAR_WIDTH_DEFAULT, + SIDEBAR_WIDTH_MAX, + SIDEBAR_WIDTH_MIN, +} = await import("../src/hooks/use-sidebar-width.ts"); + +test("clamps to the absolute range on a roomy window", () => { + stubWindow.innerWidth = 1440; + assert.equal(clampSidebarWidth(320), 320); + assert.equal(clampSidebarWidth(SIDEBAR_WIDTH_MAX + 200), SIDEBAR_WIDTH_MAX); + assert.equal(clampSidebarWidth(10), SIDEBAR_WIDTH_MIN); + assert.equal(clampSidebarWidth(Number.NaN), SIDEBAR_WIDTH_DEFAULT); +}); + +test("caps at 40% of a narrow window", () => { + stubWindow.innerWidth = 800; + assert.equal(clampSidebarWidth(SIDEBAR_WIDTH_MAX), 320); + assert.equal(clampSidebarWidth(300), 300); +}); + +test("the floor still wins when 40% falls below it", () => { + stubWindow.innerWidth = 500; + assert.equal(clampSidebarWidth(SIDEBAR_WIDTH_MAX), SIDEBAR_WIDTH_MIN); +}); + +test("re-evaluates the cap per call, so a resize can re-clamp", () => { + stubWindow.innerWidth = 1440; + assert.equal(clampSidebarWidth(SIDEBAR_WIDTH_MAX), SIDEBAR_WIDTH_MAX); + stubWindow.innerWidth = 900; + assert.equal(clampSidebarWidth(SIDEBAR_WIDTH_MAX), 360); + stubWindow.innerWidth = 1440; + assert.equal(clampSidebarWidth(SIDEBAR_WIDTH_MAX), SIDEBAR_WIDTH_MAX); +}); + +// The reset action promises to clear every stored preference, so a persisted +// panel width that is missing from the list survives the reload. +test("persisted panel widths are cleared by the preference reset", async () => { + const source = await readFile( + new URL("../src/features/settings/tabs/general-tab.tsx", import.meta.url), + "utf8", + ); + const keys = source.slice( + source.indexOf("const PREFS_KEYS"), + source.indexOf("];", source.indexOf("const PREFS_KEYS")), + ); + for (const key of PANEL_WIDTH_KEYS) { + assert.ok(keys.includes(`"${key}"`), `${key} missing from PREFS_KEYS`); + } +}); diff --git a/studio/install_node_prebuilt.py b/studio/install_node_prebuilt.py index 82ca1d2c68..1f42729c80 100644 --- a/studio/install_node_prebuilt.py +++ b/studio/install_node_prebuilt.py @@ -707,18 +707,55 @@ def existing_install_usable(install_dir: Path, host: HostInfo) -> bool: return npm_major is not None and npm_major >= NPM_MIN_MAJOR +def _replace_with_retry( + src: Path, + dst: Path, + *, + attempts: int = 8, +) -> None: + """os.replace, retried against transient Windows sharing violations. + + A directory rename fails with WinError 5/32 while any process holds a handle inside + it, and Defender or the indexer routinely does right after extraction (seen in CI on + a fresh install, with no existing directory to conflict with). Handles clear in a + second or two, so a bounded backoff turns the failure into a pause; other errors + raise immediately rather than stalling on a real problem. + """ + delay = 0.25 + for attempt in range(attempts): + try: + os.replace(src, dst) + return + except OSError as exc: + transient = os.name == "nt" and getattr(exc, "winerror", None) in (5, 32, 145) + if not transient or attempt == attempts - 1: + raise + log( + f"rename blocked ({exc.winerror}), retrying in {delay:.2f}s " + f"-- a scanner is likely still holding the extracted files" + ) + time.sleep(delay) + delay = min(delay * 2, 4.0) + + def _swap_into_place(extracted_root: Path, install_dir: Path) -> None: """Atomically replace install_dir with extracted_root (same filesystem).""" install_dir.parent.mkdir(parents = True, exist_ok = True) backup: Path | None = None if install_dir.exists(): backup = install_dir.parent / f".{install_dir.name}.old-{os.getpid()}" - os.replace(install_dir, backup) + _replace_with_retry(install_dir, backup) try: - os.replace(extracted_root, install_dir) + _replace_with_retry(extracted_root, install_dir) except OSError: + # The forward rename retries ~16s, ample time for a scanner to grab the backup too. + # A plain os.replace would then raise over the original error and leave no + # install_dir at all, so the rollback gets the same backoff and never masks it. if backup is not None and not install_dir.exists(): - os.replace(backup, install_dir) + try: + _replace_with_retry(backup, install_dir) + except OSError as rollback_exc: + log(f"could not restore the previous Node install from {backup}: {rollback_exc}") raise if backup is not None: shutil.rmtree(backup, ignore_errors = True) diff --git a/studio/setup.ps1 b/studio/setup.ps1 index a4eb54a9ef..0b6cf292c2 100644 --- a/studio/setup.ps1 +++ b/studio/setup.ps1 @@ -869,12 +869,22 @@ function Ensure-BuildToolsForLlamaSourceBuild { } } -# Detect the VC++ 2015-2022 Redistributable that the prebuilt llama-server and -# PyTorch need (they link VCRUNTIME140_1.dll etc., which the Universal CRT lacks). -# Signal is System32\vcruntime140_1.dll (VS 2019+), registry as fallback. +# Machine arch: PROCESSOR_ARCHITECTURE describes this PROCESS, so an emulated x64 shell on +# ARM64 reports AMD64; PROCESSOR_ARCHITEW6432 is ARM64 in exactly that case. +function Get-HostMachineArch { + $osArch = "" + try { $osArch = [System.Runtime.InteropServices.RuntimeInformation]::OSArchitecture.ToString() } catch { } + foreach ($s in @([string]$env:PROCESSOR_ARCHITEW6432, [string]$env:PROCESSOR_ARCHITECTURE, $osArch)) { + if ($s.ToLowerInvariant() -eq "arm64") { return "arm64" } + } + return "other" +} + +# Detect the VC++ 2015-2022 Redistributable prebuilt llama-server and PyTorch need (they +# link VCRUNTIME140_1.dll, absent from the Universal CRT). Registry first: Runtimes\x64 is +# the only x64-specific proof; System32\vcruntime140_1.dll is arch-blind and on ARM64 may +# be the ARM64-only package, unloadable under x64 emulation. function Test-VCRedistInstalled { - $sys = $env:SystemRoot - if ($sys -and (Test-Path (Join-Path $sys 'System32\vcruntime140_1.dll'))) { return $true } foreach ($k in @( 'HKLM:\SOFTWARE\Microsoft\VisualStudio\14.0\VC\Runtimes\x64', 'HKLM:\SOFTWARE\WOW6432Node\Microsoft\VisualStudio\14.0\VC\Runtimes\x64' @@ -884,10 +894,14 @@ function Test-VCRedistInstalled { if ($r.Installed -eq 1 -and [int]$r.Major -ge 14 -and [int]$r.Minor -ge 20) { return $true } } catch { } } + if ((Get-HostMachineArch) -eq "arm64") { return $false } + $sys = $env:SystemRoot + if ($sys -and (Test-Path (Join-Path $sys 'System32\vcruntime140_1.dll'))) { return $true } return $false } -# Install the VC++ 2015-2022 runtime if missing (non-fatal; usually a no-op). +# Install the VC++ 2015-2022 runtime if missing (non-fatal; usually a no-op). Unlike CMake +# and Build Tools torch cannot import without it, and winget is absent on LTSC/Server images. function Ensure-VCRedist { if (Test-VCRedistInstalled) { step "vcredist" "present"; return } Write-Host "Microsoft Visual C++ Redistributable (2015-2022) is missing; the prebuilt llama.cpp and PyTorch need it. Installing the runtime..." -ForegroundColor Yellow @@ -897,6 +911,45 @@ function Ensure-VCRedist { Refresh-Environment } catch { substep "VCRedist install failed: $($_.Exception.Message)" "Yellow" } } + if (-not (Test-VCRedistInstalled)) { + # Evergreen link; /quiet /norestart so it never blocks or reboots an unattended run. + # Always the x64 package, deliberately: Microsoft ships it as the Arm64X superset of + # both ARM64 and X64 binaries and documents it as the one for ARM64 devices, while + # the arm64 package is ARM64-only (learn.microsoft.com/cpp/windows/latest-supported-vc-redist). + # PROCESSOR_ARCHITECTURE is wrong twice here: it reports the process, and the runtime + # must match the interpreter loading the DLLs, an emulated x64 Python not yet created. + $url = "https://aka.ms/vs/17/release/vc_redist.x64.exe" + $dst = Join-Path ([System.IO.Path]::GetTempPath()) "vc_redist.x64.exe" + substep "winget unavailable or failed; downloading the runtime directly..." + # Windows PowerShell 5.1 on an old image can carry a .NET default protocol set that + # predates TLS 1.2, which aka.ms refuses -- exactly the no-winget host this fallback + # exists for. SystemDefault (0) means "let the OS choose" and already covers TLS 1.2+, + # so only an explicit legacy set is upgraded, and it is restored afterwards. + $_prevProtocol = $null + try { + $_cur = [System.Net.ServicePointManager]::SecurityProtocol + if ([int]$_cur -ne 0 -and ([int]$_cur -band [int][System.Net.SecurityProtocolType]::Tls12) -eq 0) { + [System.Net.ServicePointManager]::SecurityProtocol = $_cur -bor [System.Net.SecurityProtocolType]::Tls12 + $_prevProtocol = $_cur + } + } catch { $_prevProtocol = $null } + try { + Invoke-WebRequest -Uri $url -OutFile $dst -UseBasicParsing -TimeoutSec 300 + $p = Start-Process -FilePath $dst -ArgumentList '/quiet', '/norestart' -Wait -PassThru + # 3010 = success, reboot required; usable either way. + if ($p.ExitCode -notin @(0, 3010)) { + substep "VC++ runtime installer exited $($p.ExitCode)" "Yellow" + } + Refresh-Environment + } catch { + substep "Direct VC++ runtime download failed: $($_.Exception.Message)" "Yellow" + } finally { + if ($null -ne $_prevProtocol) { + try { [System.Net.ServicePointManager]::SecurityProtocol = $_prevProtocol } catch { } + } + Remove-Item -LiteralPath $dst -Force -ErrorAction SilentlyContinue + } + } if (Test-VCRedistInstalled) { step "vcredist" "installed" } else { substep "Could not install the VC++ Redistributable automatically." "Yellow" @@ -1650,11 +1703,42 @@ if ($LongPathsEnabled) { } # ============================================ -# 1b. Git (required by pip for git+https:// deps and by npm) +# 1b. Git (only required for --local / source installs) # ============================================ +# Was fatal as "required by pip and npm", but the consumer path uses neither: the +# unsloth-zoo git+https URL is STUDIO_LOCAL_INSTALL only, node is a pinned prebuilt, and the +# frontend lockfile has no VCS deps. Being fatal blocked clean no-winget Windows boxes. $HasGit = $null -ne (Get-Command git -ErrorAction SilentlyContinue) if (-not $HasGit) { - Write-Host "Git not found -- installing via winget..." -ForegroundColor Yellow + # Fatal only where git is used: --local and the opt-in llama.cpp source build. A local + # llama.cpp dir overrides those opt-ins, but only once it holds a reusable binary: + # pointing at the canonical install location with nothing built there falls through to + # the normal install, so an explicit source build still needs git. The automatic + # fallback after a failed prebuilt download is not knowable here; Phase 4 handles it. + $gitNeeded = ($env:STUDIO_LOCAL_INSTALL -eq '1') + $_localLlamaDir = if ($env:UNSLOTH_LOCAL_LLAMA_CPP_DIR) { $env:UNSLOTH_LOCAL_LLAMA_CPP_DIR.Trim() } else { "" } + $_localLlamaBuilt = $false + if ($_localLlamaDir) { + # Same layout candidates as the reuse check in Phase 4. + foreach ($_c in @("llama-server.exe", "build\bin\llama-server.exe", "build\bin\Release\llama-server.exe")) { + if (Test-Path -LiteralPath (Join-Path $_localLlamaDir $_c)) { $_localLlamaBuilt = $true; break } + } + } + if (-not $_localLlamaBuilt) { + $_prForce = if ($env:UNSLOTH_LLAMA_PR_FORCE) { $env:UNSLOTH_LLAMA_PR_FORCE.Trim() } else { $DefaultLlamaPrForce } + $_llamaSrc = $DefaultLlamaSource -replace '\.git$', '' + # Same tag resolution as Phase 4. "master" is a branch, never a release, so the + # prebuilt lookup always misses and Phase 4 rebuilds it from source. + $_llamaTag = if ($env:UNSLOTH_LLAMA_TAG) { $env:UNSLOTH_LLAMA_TAG } else { $DefaultLlamaTag } + if ($_llamaTag -eq "master") { $gitNeeded = $true } + if ($env:UNSLOTH_LLAMA_FORCE_COMPILE -eq '1') { $gitNeeded = $true } + if (-not [string]::IsNullOrWhiteSpace($env:UNSLOTH_LLAMA_PR)) { $gitNeeded = $true } + # Same positive-integer predicate as the PR_FORCE promotion below: 0 or non-numeric + # never forces a source build, so it must not demand git. + if ($_prForce -match '^\d+$' -and [int]$_prForce -gt 0) { $gitNeeded = $true } + if ($_llamaSrc -ne "https://github.com/ggml-org/llama.cpp") { $gitNeeded = $true } + } + Write-Host "Git not found -- attempting install via winget..." -ForegroundColor Yellow $HasWinget = $null -ne (Get-Command winget -ErrorAction SilentlyContinue) if ($HasWinget) { try { @@ -1664,11 +1748,18 @@ if (-not $HasGit) { } catch { } } if (-not $HasGit) { - Write-Host "[ERROR] Git is required but could not be installed automatically." -ForegroundColor Red - Write-Host " Install Git from https://git-scm.com/download/win and re-run." -ForegroundColor Red - Exit-SetupFailure "Git is required but could not be installed automatically" + if ($gitNeeded) { + Write-Host "[ERROR] Git is required for --local and llama.cpp source-build installs but could not be installed." -ForegroundColor Red + Write-Host " --local clones unsloth-zoo, and a source build clones llama.cpp." -ForegroundColor Red + Write-Host " Install Git from https://git-scm.com/download/win and re-run." -ForegroundColor Red + Exit-SetupFailure "Git is required for --local / source-build installs but could not be installed" + } + step "git" "not found (not required)" "Yellow" + substep "Unsloth installs prebuilt binaries and wheels, so git is not needed." + substep "Install it only for --local/source installs: https://git-scm.com/download/win" + } else { + step "git" "$(git --version)" } - step "git" "$(git --version)" } else { step "git" "$(git --version)" } @@ -3275,18 +3366,32 @@ $PyTorchWhlBase = if ($env:UNSLOTH_PYTORCH_MIRROR) { $env:UNSLOTH_PYTORCH_MIRROR $TorchInstallIndexUrl = if ($ROCmIndexUrl) { "$PyTorchWhlBase/cpu" } elseif ($PinnedTorchIndexUrl) { $PinnedTorchIndexUrl } else { "$PyTorchWhlBase/$CuTag" } if (-not $NoTorchMode) { +# Windows on ARM has win_arm64 torch and torchvision wheels but no torchaudio on any index, +# so every branch below drops it. Ask the interpreter uv resolves for, not +# PROCESSOR_ARCHITECTURE, which describes the host process. Inside the no-torch guard +# because all three uses are, and no-torch installs nothing to skip. +$_setupPlatform = "" +try { + $_setupPlatform = (& python -c "import sysconfig; print(sysconfig.get_platform())" 2>$null | Out-String).Trim().ToLowerInvariant() +} catch { $_setupPlatform = "" } +$WinArm64NoAudio = ($_setupPlatform -eq "win-arm64") +if ($WinArm64NoAudio) { substep "windows on arm: skipping torchaudio (no win_arm64 wheel upstream)" } + $ROCmCpuFallback = $false if ($ROCmIndexUrl) { substep "installing PyTorch (AMD ROCm, $ROCmGfxArch)..." if ($ROCmTorchSpec -ne "torch") { substep " enforcing $ROCmTorchSpec $ROCmVisionSpec $ROCmAudioSpec (known _grouped_mm bug in older wheels)" "Cyan" } + # Built above the verbose branch: a splat assigned inside it is unset on the other. + $_rocmTrio = @($ROCmTorchSpec, $ROCmVisionSpec, $ROCmAudioSpec) + if ($WinArm64NoAudio) { $_rocmTrio = @($ROCmTorchSpec, $ROCmVisionSpec) } if ($script:UnslothVerbose) { - Fast-Install $ROCmTorchSpec $ROCmVisionSpec $ROCmAudioSpec --force-reinstall --index-url $ROCmIndexUrl | ForEach-Object { Redact-InstallOutput "$_" } | Out-Host + Fast-Install @_rocmTrio --force-reinstall --index-url $ROCmIndexUrl | ForEach-Object { Redact-InstallOutput "$_" } | Out-Host $torchInstallExit = $LASTEXITCODE $output = "" } else { - $output = Fast-Install $ROCmTorchSpec $ROCmVisionSpec $ROCmAudioSpec --force-reinstall --index-url $ROCmIndexUrl | Out-String + $output = Fast-Install @_rocmTrio --force-reinstall --index-url $ROCmIndexUrl | Out-String $torchInstallExit = $LASTEXITCODE } if ($torchInstallExit -ne 0) { @@ -3322,12 +3427,14 @@ if (-not $ROCmIndexUrl -and ($CuTag -eq "cpu" -or $ROCmCpuFallback)) { $cpuVisionSpec = "torchvision>=0.19,<0.27.0" $cpuAudioSpec = "torchaudio>=2.4,<2.12.0" } + $_torchTrio = @($cpuTorchSpec, $cpuVisionSpec, $cpuAudioSpec) + if ($WinArm64NoAudio) { $_torchTrio = @($cpuTorchSpec, $cpuVisionSpec) } if ($script:UnslothVerbose) { - Fast-Install $cpuTorchSpec $cpuVisionSpec $cpuAudioSpec @cpuForce --index-url $TorchInstallIndexUrl | ForEach-Object { Redact-InstallOutput "$_" } | Out-Host + Fast-Install @_torchTrio @cpuForce --index-url $TorchInstallIndexUrl | ForEach-Object { Redact-InstallOutput "$_" } | Out-Host $torchInstallExit = $LASTEXITCODE $output = "" } else { - $output = Fast-Install $cpuTorchSpec $cpuVisionSpec $cpuAudioSpec @cpuForce --index-url $TorchInstallIndexUrl | Out-String + $output = Fast-Install @_torchTrio @cpuForce --index-url $TorchInstallIndexUrl | Out-String $torchInstallExit = $LASTEXITCODE } if ($torchInstallExit -ne 0) { @@ -3354,12 +3461,16 @@ if (-not $ROCmIndexUrl -and ($CuTag -eq "cpu" -or $ROCmCpuFallback)) { $cudaVisionSpec = "torchvision>=0.19,<0.26.0" $cudaAudioSpec = "torchaudio>=2.4,<2.11.0" } + # A custom pin whose leaf is not cpu (a corporate /simple mirror) lands an ARM64 host + # here, so this branch drops torchaudio too. + $_cudaTrio = @($cudaTorchSpec, $cudaVisionSpec, $cudaAudioSpec) + if ($WinArm64NoAudio) { $_cudaTrio = @($cudaTorchSpec, $cudaVisionSpec) } if ($script:UnslothVerbose) { - Fast-Install $cudaTorchSpec $cudaVisionSpec $cudaAudioSpec @cudaForce --index-url $TorchInstallIndexUrl | ForEach-Object { Redact-InstallOutput "$_" } | Out-Host + Fast-Install @_cudaTrio @cudaForce --index-url $TorchInstallIndexUrl | ForEach-Object { Redact-InstallOutput "$_" } | Out-Host $torchInstallExit = $LASTEXITCODE $output = "" } else { - $output = Fast-Install $cudaTorchSpec $cudaVisionSpec $cudaAudioSpec @cudaForce --index-url $TorchInstallIndexUrl | Out-String + $output = Fast-Install @_cudaTrio @cudaForce --index-url $TorchInstallIndexUrl | Out-String $torchInstallExit = $LASTEXITCODE } if ($torchInstallExit -ne 0) { @@ -4048,6 +4159,7 @@ $BuildDir = Join-Path $LlamaCppDir "build" $LlamaServerBin = Join-Path $BuildDir "bin\Release\llama-server.exe" $HasCmakeForBuild = $null -ne (Get-Command cmake -ErrorAction SilentlyContinue) +$HasGitForBuild = $null -ne (Get-Command git -ErrorAction SilentlyContinue) # Check if existing llama-server matches current GPU mode. A CUDA-built binary # on a now-CPU-only machine (or vice versa) needs to be rebuilt. @@ -4073,9 +4185,27 @@ if (Test-Path -LiteralPath $LlamaServerBin) { $WillBuildLlamaFromSource = $NeedLlamaSourceBuild -and ` -not ((Test-Path -LiteralPath $LlamaServerBin) -and -not $NeedRebuild -and $RequestedLlamaTag -ne "master") if ($WillBuildLlamaFromSource) { - Ensure-BuildToolsForLlamaSourceBuild - # refresh so the chain below sees a newly installed cmake - $HasCmakeForBuild = $null -ne (Get-Command cmake -ErrorAction SilentlyContinue) + if (-not $HasGitForBuild) { + # Phase 1 keeps git optional, so only the automatic fallback after a failed prebuilt + # download arrives here without it. Last chance to install: Invoke-SetupCommand + # returns 0 for command-not-found, so a git-less clone misreports as a cmake failure. + if ($null -ne (Get-Command winget -ErrorAction SilentlyContinue)) { + try { + Invoke-SetupCommand { winget install Git.Git --source winget --accept-package-agreements --accept-source-agreements } | Out-Null + Refresh-Environment + } catch { } + } + $HasGitForBuild = $null -ne (Get-Command git -ErrorAction SilentlyContinue) + } + # Git first, then the toolchain: Ensure-BuildToolsForLlamaSourceBuild exits setup when + # Build Tools cannot be installed, so running it first made the degraded path below + # unreachable on a no-winget box, and elsewhere spent a multi-GB download on a clone + # that cannot happen. + if ($HasGitForBuild) { + Ensure-BuildToolsForLlamaSourceBuild + # refresh so the chain below sees a newly installed cmake + $HasCmakeForBuild = $null -ne (Get-Command cmake -ErrorAction SilentlyContinue) + } } if ($LocalLlamaCppLinked) { @@ -4093,6 +4223,16 @@ if ($LocalLlamaCppLinked) { # up new model architecture support (e.g. Gemma 4). Write-Host "" step "llama.cpp" "already built" +} elseif (-not $HasGitForBuild) { + # Before cmake: the toolchain install is skipped without git, so cmake may be missing + # purely as a consequence. Degrade rather than abort; the opt-in source triggers already + # required git in Phase 1, so only the automatic fallback lands here. + Write-Host "" + step "llama.cpp" "build skipped (git not available)" "Yellow" + substep "The prebuilt download failed and a source build clones llama.cpp." "Yellow" + substep "GGUF inference and export will not be available." "Yellow" + substep "Install Git from https://git-scm.com/download/win and re-run setup." "Yellow" + $script:LlamaCppDegraded = $true } elseif (-not $HasCmakeForBuild) { Write-Host "" if (-not $HasNvidiaSmi) { diff --git a/studio/src-tauri/src/desktop_update_policy.rs b/studio/src-tauri/src/desktop_update_policy.rs index c83f847eda..d186c2d01d 100644 --- a/studio/src-tauri/src/desktop_update_policy.rs +++ b/studio/src-tauri/src/desktop_update_policy.rs @@ -27,6 +27,8 @@ pub(crate) struct DesktopUpdatePolicy { pub(crate) struct ManualUpdateInfo { version: String, current_version: String, + // Backend release this desktop build pins; CHANGELOG.md is keyed by it. + pypi_version: Option<String>, body: Option<String>, date: Option<String>, } @@ -34,8 +36,12 @@ pub(crate) struct ManualUpdateInfo { #[derive(Debug, serde::Deserialize)] struct ChannelMetadata { version: String, - body: Option<String>, - date: Option<String>, + // latest.json publishes Tauri's `notes`/`pub_date`; aliases keep older metadata working. + pypi_version: Option<String>, + #[serde(alias = "body")] + notes: Option<String>, + #[serde(alias = "date")] + pub_date: Option<String>, platforms: HashMap<String, ChannelPlatform>, } @@ -99,8 +105,9 @@ pub(crate) async fn check_desktop_manual_update() -> Result<Option<ManualUpdateI Ok(Some(ManualUpdateInfo { version: latest_version, current_version: current_version.to_string(), - body: metadata.body, - date: metadata.date, + pypi_version: metadata.pypi_version, + body: metadata.notes, + date: metadata.pub_date, })) } diff --git a/tests/python/test_bitsandbytes_kernel_readiness.py b/tests/python/test_bitsandbytes_kernel_readiness.py index db6ec74e57..fe595f150d 100644 --- a/tests/python/test_bitsandbytes_kernel_readiness.py +++ b/tests/python/test_bitsandbytes_kernel_readiness.py @@ -155,7 +155,10 @@ def test_the_ctypes_binds_are_gated_on_the_same_verdict(): "if bnb is None or not native_kernels_ready(bnb, DEVICE_TYPE):" in source ), "the ctypes bind block must take the _bnb_required branch on a dead library too" guarded = source.split("if bnb is None or not native_kernels_ready(bnb, DEVICE_TYPE):")[1] - assert "bnb.functional.lib" in guarded, "the binds must sit under that guard" + # Anchor on the symbol, not the module alias: #7580 renamed the binding from + # `bnb.functional.lib` to `bnb_functional.lib`, which is exactly the kind of rename + # this assertion should survive. + assert "lib.cdequantize_blockwise_fp32" in guarded, "the binds must sit under that guard" def test_the_kernel_check_reads_the_submodule_not_the_parent_attribute(): diff --git a/tests/python/test_cross_platform_parity.py b/tests/python/test_cross_platform_parity.py index b20e715ebc..06e444314b 100644 --- a/tests/python/test_cross_platform_parity.py +++ b/tests/python/test_cross_platform_parity.py @@ -454,9 +454,12 @@ class TestKnown211SetParity: "$_pinCuLeaf" not in text ), "install.ps1 must bound companions on every index (no cu-family exemption)" # The bounded companions must actually be passed to the install command. - assert re.search( - r'"torch>=2\.4,<2\.11\.0" \$_pinVisionSpec \$_pinAudioSpec --default-index \$TorchIndexUrl', - text, + # Specs are splatted, so check both halves: the list is built, and it is passed. + assert ( + '$_torchSpecs = @("torch>=2.4,<2.11.0", $_pinVisionSpec, $_pinAudioSpec)' in text + ), "install.ps1 custom-pin install must build the bounded spec list" + assert ( + "@_torchSpecs --default-index $TorchIndexUrl" in text ), "install.ps1 custom-pin install must pass the bounded companion specs to uv" def test_gfx_allowlist_matches_across_installers(self): @@ -704,9 +707,13 @@ class TestPinnedIndexClearsUvEnvParity: assert ( "if ($TorchIndexPinned -and -not (Test-CudaFamilyLeaf $CuTag)) {" in text ), "the custom-leaf trio bounds must be gated on a pinned non-cu-family leaf" + # Specs are splatted, so check both halves: the list is built, and it is passed. assert ( - "Fast-Install $cudaTorchSpec $cudaVisionSpec $cudaAudioSpec" in text - ), "setup.ps1's CUDA branch must install via the bounded spec variables" + "$_cudaTrio = @($cudaTorchSpec, $cudaVisionSpec, $cudaAudioSpec)" in text + ), "setup.ps1's CUDA branch must build the trio from the bounded spec variables" + assert ( + "Fast-Install @_cudaTrio @cudaForce" in text + ), "setup.ps1's CUDA branch must install the trio it built" def test_setup_ps1_bounds_pinned_cpu_torch(self): """setup.ps1's CPU branch must bound the trio under an explicit pin (parity with @@ -724,8 +731,11 @@ class TestPinnedIndexClearsUvEnvParity: "if ($TorchIndexPinned) {" in text ), "the CPU trio bounds must be gated on an explicit pin" assert ( - "Fast-Install $cpuTorchSpec $cpuVisionSpec $cpuAudioSpec @cpuForce" in text - ), "setup.ps1's CPU branch must install via the spec variables" + "$_torchTrio = @($cpuTorchSpec, $cpuVisionSpec, $cpuAudioSpec)" in text + ), "setup.ps1's CPU branch must build the trio from the spec variables" + assert ( + "Fast-Install @_torchTrio @cpuForce" in text + ), "setup.ps1's CPU branch must install the trio it built" # The ceilings mirror the Python repair spec exactly. stack = STACK_PY.read_text(encoding = "utf-8") spec_block = re.search(r"_CUDA_TORCH_PKG_SPEC[^(]*\(\s*(.*?)\)", stack, re.DOTALL) diff --git a/tests/python/test_windows_arm64_python_choice.py b/tests/python/test_windows_arm64_python_choice.py new file mode 100644 index 0000000000..89546e7ac4 --- /dev/null +++ b/tests/python/test_windows_arm64_python_choice.py @@ -0,0 +1,143 @@ +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. + +"""Windows on ARM: install.ps1 must not settle for a native ARM64 interpreter. + +pyarrow (via datasets) and hf-transfer publish no win_arm64 wheels, so an ARM64 +Python source-builds both and dies minutes into the run. The resolver prefers an +x64 build of the requested minor and bootstraps one otherwise; the case pinned +here is the recovery path, where nothing can be downloaded but an x64 build of a +lower-priority supported minor is already installed. +""" + +from __future__ import annotations + +import os +import re +import shutil +import subprocess +from pathlib import Path + +import pytest + + +REPO_ROOT = Path(__file__).resolve().parents[2] +INSTALL_PS1 = REPO_ROOT / "install.ps1" + + +def _extract(pattern: str, source: str) -> str: + match = re.search(pattern, source, flags = re.DOTALL) + assert match is not None, f"install.ps1 block not found: {pattern}" + return match.group(0) + + +def _resolver_script(installed: list[tuple[str, str]], can_download: bool) -> str: + """Both production functions verbatim, over a fake set of interpreters. + + Extracted rather than reimplemented so the test cannot drift away from the + text install.ps1 actually runs. `installed` is (minor, arch) in py-launcher + order, so the first entry for a minor is what a bare `py -3.13` resolves to. + The fake interpreters are named `*.exe` and invoked through the call operator, + which resolves a string to a function, so no real binary is needed. + """ + source = INSTALL_PS1.read_text(encoding = "utf-8") + finder = _extract(r" function Find-CompatiblePython \{.*?\n \}\n", source) + installer = _extract(r" function Install-X64Python \{.*?\n \}\n", source) + + names = [f"Py{minor.replace('.', '')}{arch}.exe" for minor, arch in installed] + table = ", ".join( + f'@{{ Minor = "{minor}"; Arch = "{arch}"; Name = "{name}" }}' + for (minor, arch), name in zip(installed, names) + ) + downloaded = ( + '@{ Version = "3.13"; Path = "Downloaded.exe"; Arch = "x86_64" }' + if can_download + else "$null" + ) + version_stubs = "\n".join( + f"function {name} {{ param([Parameter(ValueFromRemainingArguments = $true)]$Rest)\n" + f' if ($Rest -contains "--version") {{ return "Python {minor}.0" }}\n' + f' return "{name}" }}' + for (minor, _arch), name in zip(installed, names) + ) + return f""" +$ErrorActionPreference = "Stop" +$PythonVersion = "3.13" +$script:WingetAvailable = $false +$script:CondaSkipPattern = 'conda' +$Interpreters = @({table}) +{version_stubs} +# `py -0p` lists every registration; `py -3.x` runs the launcher's preferred build +# for that minor, which on an ARM64 host is normally the native one. +function FakePy {{ + param([Parameter(ValueFromRemainingArguments = $true)]$Rest) + if ($Rest -contains "-0p") {{ + return @($Interpreters | ForEach-Object {{ " -V:$($_.Minor) * $($_.Name)" }}) + }} + $minor = ([string]$Rest[0]).TrimStart('-') + $hit = @($Interpreters | Where-Object {{ $_.Minor -eq $minor }}) + if ($hit.Count -eq 0) {{ return "" }} + if ($Rest -contains "--version") {{ return "Python $minor.0" }} + return $hit[0].Name +}} +function substep {{ param($a, $b) }} +function Get-HostMachineArch {{ return "arm64" }} +function Get-Command {{ + param([Parameter(Position = 0)][string]$Name, + [Parameter(ValueFromRemainingArguments = $true)]$Rest) + if ($Name -eq "py") {{ return @([pscustomobject]@{{ Source = "FakePy" }}) }} + return @() +}} +function Test-Path {{ param([Parameter(ValueFromRemainingArguments = $true)]$Rest) return $true }} +function Test-IsCondaPython {{ param([string]$Exe) return $false }} +function Get-PythonPlatformTag {{ + param([string]$Exe) + foreach ($i in $Interpreters) {{ + if ($i.Name -eq $Exe) {{ + if ($i.Arch -eq "x86_64") {{ return "win-amd64" }} else {{ return "win-arm64" }} + }} + }} + return "win-amd64" +}} +function Refresh-SessionPath {{ }} +function Install-PythonFromPythonOrg {{ param([string]$Arch = "") return {downloaded} }} +{finder} +{installer} +# The caller's ARM64 swap, condensed to what decides the interpreter. +$found = Find-CompatiblePython +if ($found -and $found.Arch -ne "x86_64") {{ + $x64 = Install-X64Python + if ($x64) {{ $found = $x64 }} +}} +if ($found) {{ Write-Output "$($found.Version)|$($found.Arch)" }} else {{ Write-Output "none" }} +""" + + +def _pwsh(script: str) -> str: + result = subprocess.run( + ["pwsh", "-NoProfile", "-NonInteractive", "-Command", script], + check = True, + capture_output = True, + text = True, + env = os.environ.copy(), + ) + return result.stdout.strip() + + +@pytest.mark.skipif(shutil.which("pwsh") is None, reason = "PowerShell is unavailable") +@pytest.mark.parametrize( + ("installed", "can_download", "expected"), + [ + # An x64 build of the requested minor wins outright, downloads irrelevant. + ([("3.13", "arm64"), ("3.13", "x86_64")], False, "3.13|x86_64"), + # Requested minor is ARM64-only: bootstrap x64 rather than take the native one. + ([("3.13", "arm64")], True, "3.13|x86_64"), + # Offline, but an x64 build of a lower-priority minor is here. Use it: the native + # 3.13 cannot resolve pyarrow or hf-transfer, and this one can. + ([("3.13", "arm64"), ("3.11", "x86_64")], False, "3.11|x86_64"), + # ARM64 everywhere: still returned, and the caller warns. + ([("3.13", "arm64"), ("3.11", "arm64")], False, "3.13|arm64"), + ], +) +def test_arm64_host_prefers_an_x64_interpreter(installed, can_download, expected): + assert _pwsh(_resolver_script(installed, can_download)) == expected diff --git a/tests/python/test_windows_git_gate.py b/tests/python/test_windows_git_gate.py new file mode 100644 index 0000000000..60de430191 --- /dev/null +++ b/tests/python/test_windows_git_gate.py @@ -0,0 +1,117 @@ +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. + +"""Git is optional on the consumer Windows path, but still required for source builds.""" + +from __future__ import annotations + +import os +import shutil +import subprocess +from pathlib import Path + +import pytest + + +REPO_ROOT = Path(__file__).resolve().parents[2] +SETUP_PS1 = REPO_ROOT / "studio" / "setup.ps1" + +_START = "$gitNeeded = ($env:STUDIO_LOCAL_INSTALL -eq '1')" +_TAIL = "if (-not $_localLlamaBuilt) {" + + +def _git_gate_block() -> str: + """Slice the real $gitNeeded computation out of setup.ps1 so the test cannot drift.""" + source = SETUP_PS1.read_text(encoding = "utf-8") + start = source.index(_START) + brace = source.index("{", source.index(_TAIL, start)) + depth = 0 + for index in range(brace, len(source)): + if source[index] == "{": + depth += 1 + elif source[index] == "}": + depth -= 1 + if depth == 0: + return source[start : index + 1] + raise AssertionError("Unclosed git gate block in setup.ps1") + + +def _script() -> str: + return f""" +$DefaultLlamaPrForce = "0" +$DefaultLlamaSource = "https://github.com/ggml-org/llama.cpp" +$DefaultLlamaTag = "latest" +{_git_gate_block()} +Write-Output $gitNeeded +""" + + +def _needs_git(env: dict[str, str]) -> bool: + merged = {k: v for k, v in os.environ.items() if not k.startswith(("UNSLOTH_", "STUDIO_"))} + merged.update(env) + result = subprocess.run( + ["pwsh", "-NoProfile", "-NonInteractive", "-Command", _script()], + check = True, + capture_output = True, + text = True, + env = merged, + ) + return result.stdout.strip() == "True" + + +pwsh_only = pytest.mark.skipif(shutil.which("pwsh") is None, reason = "PowerShell is unavailable") + + +@pwsh_only +@pytest.mark.parametrize( + ("env", "expected"), + [ + # The consumer install: prebuilt wheels and a prebuilt llama.cpp, so no git. + ({}, False), + # --local clones unsloth-zoo. + ({"STUDIO_LOCAL_INSTALL": "1"}, True), + # Opt-in source builds clone llama.cpp. + ({"UNSLOTH_LLAMA_FORCE_COMPILE": "1"}, True), + ({"UNSLOTH_LLAMA_PR": "1234"}, True), + # PR_FORCE only forces a build for a positive integer. + ({"UNSLOTH_LLAMA_PR_FORCE": "0"}, False), + ({"UNSLOTH_LLAMA_PR_FORCE": "not-a-number"}, False), + ({"UNSLOTH_LLAMA_PR_FORCE": "1234"}, True), + # "master" is a branch with no release, so Phase 4 always builds it from source. + ({"UNSLOTH_LLAMA_TAG": "master"}, True), + # A release tag resolves to a prebuilt bundle. + ({"UNSLOTH_LLAMA_TAG": "latest"}, False), + ({"UNSLOTH_LLAMA_TAG": "b8635"}, False), + ], +) +def test_git_is_required_only_for_local_and_source_builds(env, expected): + assert _needs_git(env) is expected + + +@pwsh_only +def test_a_built_local_llama_dir_drops_the_source_build_git_requirement(tmp_path): + (tmp_path / "llama-server.exe").write_text("", encoding = "utf-8") + env = { + "UNSLOTH_LOCAL_LLAMA_CPP_DIR": str(tmp_path), + "UNSLOTH_LLAMA_FORCE_COMPILE": "1", + } + # Reusing an existing binary skips both the prebuilt download and the source build. + assert _needs_git(env) is False + + +@pwsh_only +@pytest.mark.parametrize("trigger", ["UNSLOTH_LLAMA_FORCE_COMPILE", "UNSLOTH_LLAMA_PR"]) +def test_an_unbuilt_local_llama_dir_still_requires_git(tmp_path, trigger): + # Nothing built at the canonical install location falls through to the normal install, + # so the source build still runs and still needs git. Suppressing the requirement here + # let a no-git host silently degrade to a prebuilt instead. + env = { + "UNSLOTH_LOCAL_LLAMA_CPP_DIR": str(tmp_path), + trigger: "1", + } + assert _needs_git(env) is True + + +@pwsh_only +def test_an_unbuilt_local_llama_dir_alone_does_not_require_git(tmp_path): + assert _needs_git({"UNSLOTH_LOCAL_LLAMA_CPP_DIR": str(tmp_path)}) is False diff --git a/tests/python/test_windows_vcredist_download_tls.py b/tests/python/test_windows_vcredist_download_tls.py new file mode 100644 index 0000000000..9fb1c697e2 --- /dev/null +++ b/tests/python/test_windows_vcredist_download_tls.py @@ -0,0 +1,80 @@ +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. + +"""The direct VC++ runtime download must negotiate TLS 1.2 on legacy protocol defaults.""" + +from __future__ import annotations + +import shutil +import subprocess +from pathlib import Path + +import pytest + + +REPO_ROOT = Path(__file__).resolve().parents[2] +SETUP_PS1 = REPO_ROOT / "studio" / "setup.ps1" + +_START = '$url = "https://aka.ms/vs/17/release/vc_redist.x64.exe"' +_END = "Remove-Item -LiteralPath $dst -Force -ErrorAction SilentlyContinue\n }" + + +def _download_block() -> str: + """Slice the real download block out of setup.ps1 so the test cannot drift.""" + source = SETUP_PS1.read_text(encoding = "utf-8") + start = source.index(_START) + end = source.index(_END, start) + len(_END) + return source[start:end] + + +def _script(starting_protocol: str) -> str: + # Start from a non-zero set that lacks Tls12. Tls13 is the only such value modern .NET + # accepts, and it stands in for the legacy Ssl3/Tls default of Windows PowerShell 5.1. + return f""" +function substep {{ param($a, $b) }} +function Refresh-Environment {{ }} +function Invoke-WebRequest {{ + param($Uri, $OutFile, [switch]$UseBasicParsing, $TimeoutSec) + Write-Output "DURING=$([System.Net.ServicePointManager]::SecurityProtocol)" + throw "stop before Start-Process" +}} +[System.Net.ServicePointManager]::SecurityProtocol = [System.Net.SecurityProtocolType]::{starting_protocol} +{_download_block()} +Write-Output "AFTER=$([System.Net.ServicePointManager]::SecurityProtocol)" +""" + + +def _run(starting_protocol: str) -> dict[str, str]: + result = subprocess.run( + ["pwsh", "-NoProfile", "-NonInteractive", "-Command", _script(starting_protocol)], + check = True, + capture_output = True, + text = True, + ) + out = {} + for line in result.stdout.splitlines(): + if "=" in line: + key, _, value = line.partition("=") + out[key.strip()] = value.strip() + return out + + +pwsh_only = pytest.mark.skipif(shutil.which("pwsh") is None, reason = "PowerShell is unavailable") + + +@pwsh_only +def test_tls12_is_added_for_the_download_and_restored_after(): + seen = _run("Tls13") + during = {part.strip() for part in seen["DURING"].split(",")} + assert "Tls12" in during, "the download must negotiate TLS 1.2 or aka.ms refuses it" + assert "Tls13" in during, "adding TLS 1.2 must not drop protocols the host already allowed" + assert seen["AFTER"] == "Tls13", "the process-wide protocol must be restored" + + +@pwsh_only +def test_system_default_is_left_alone(): + # SystemDefault means "let the OS choose" and already covers TLS 1.2+; pinning it to + # Tls12 would strip TLS 1.3 from every later request in the process. + seen = _run("SystemDefault") + assert seen["DURING"] == "SystemDefault" + assert seen["AFTER"] == "SystemDefault" diff --git a/tests/studio/install/test_install_node_prebuilt_logic.py b/tests/studio/install/test_install_node_prebuilt_logic.py index 5476702d65..5bcc9733cf 100644 --- a/tests/studio/install/test_install_node_prebuilt_logic.py +++ b/tests/studio/install/test_install_node_prebuilt_logic.py @@ -762,3 +762,94 @@ def test_pinned_target_wrong_sha_not_kept_when_download_fails(tmp_path: Path, mo monkeypatch.setattr(M, "download_file_verified", _offline) # transient download failure with pytest.raises(OSError): M.install_prebuilt(install_dir, channel = "pinned", min_major = 24, force = False) + + +# ── _replace_with_retry: transient Windows sharing violations ────────────────── +# Seen in CI: WinError 5 renaming extracted Node into place on a FRESH install, a scanner +# still holding handles inside the new files. + + +def _oserror(winerror: int) -> OSError: + exc = OSError(winerror, "mock") + exc.winerror = winerror + return exc + + +@pytest.mark.parametrize("winerror", [5, 32, 145]) +def test_replace_retries_transient_windows_errors(monkeypatch, tmp_path, winerror): + monkeypatch.setattr(M.os, "name", "nt") + monkeypatch.setattr(M.time, "sleep", lambda _s: None) # no real backoff in tests + calls = {"n": 0} + + def flaky(src, dst): + calls["n"] += 1 + if calls["n"] < 3: + raise _oserror(winerror) + + monkeypatch.setattr(M.os, "replace", flaky) + M._replace_with_retry(tmp_path / "src", tmp_path / "dst") + assert calls["n"] == 3, "should have retried until the handle was released" + + +def test_replace_gives_up_and_reports_the_real_error(monkeypatch, tmp_path): + monkeypatch.setattr(M.os, "name", "nt") + monkeypatch.setattr(M.time, "sleep", lambda _s: None) + monkeypatch.setattr(M.os, "replace", lambda s, d: (_ for _ in ()).throw(_oserror(5))) + # A scanner that never lets go must surface as a failure, not a hang. + with pytest.raises(OSError) as excinfo: + M._replace_with_retry(tmp_path / "src", tmp_path / "dst", attempts = 3) + assert excinfo.value.winerror == 5 + + +def test_replace_does_not_retry_a_genuine_error(monkeypatch, tmp_path): + # A cross-device move or real permissions problem must fail immediately. + monkeypatch.setattr(M.os, "name", "nt") + monkeypatch.setattr(M.time, "sleep", lambda _s: None) + calls = {"n": 0} + + def hard_fail(src, dst): + calls["n"] += 1 + raise _oserror(17) # ERROR_NOT_SAME_DEVICE + + monkeypatch.setattr(M.os, "replace", hard_fail) + with pytest.raises(OSError): + M._replace_with_retry(tmp_path / "src", tmp_path / "dst") + assert calls["n"] == 1 + + +def test_replace_is_a_plain_rename_on_posix(monkeypatch, tmp_path): + # POSIX has no sharing violations, so the retry must add no latency there. + monkeypatch.setattr(M.os, "name", "posix") + calls = {"n": 0} + + def once(src, dst): + calls["n"] += 1 + raise _oserror(5) + + monkeypatch.setattr(M.os, "replace", once) + with pytest.raises(OSError): + M._replace_with_retry(tmp_path / "src", tmp_path / "dst") + assert calls["n"] == 1 + + +def test_swap_into_place_survives_a_transient_lock(monkeypatch, tmp_path): + # End-to-end through the function the installer actually calls. + monkeypatch.setattr(M.os, "name", "nt") + monkeypatch.setattr(M.time, "sleep", lambda _s: None) + extracted = tmp_path / "extracted" / "node-v24" + extracted.mkdir(parents = True) + (extracted / "marker.txt").write_text("node", encoding = "utf-8") + install_dir = tmp_path / "node" + + real_replace = os.replace + state = {"failed": False} + + def flaky(src, dst): + if not state["failed"]: + state["failed"] = True + raise _oserror(32) + real_replace(src, dst) + + monkeypatch.setattr(M.os, "replace", flaky) + M._swap_into_place(extracted, install_dir) + assert (install_dir / "marker.txt").read_text(encoding = "utf-8") == "node" diff --git a/tests/studio/test_update_release_notes.py b/tests/studio/test_update_release_notes.py new file mode 100644 index 0000000000..765ad55321 --- /dev/null +++ b/tests/studio/test_update_release_notes.py @@ -0,0 +1,1906 @@ +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. + +"""Contracts for the update popup's release-notes preview. + +The popup renders CHANGELOG.md notes for the exact version it is offering. The +risk this file guards is showing notes from a different release: a near-miss +lookup must return nothing rather than the newest section it can find.""" + +from __future__ import annotations + +import http.server +import json +import os +import re +import shutil +import subprocess +import sys +import threading +import time +from pathlib import Path + +import pytest + +REPO = Path(__file__).resolve().parents[2] +BACKEND = REPO / "studio/backend" +FRONTEND = REPO / "studio/frontend/src" +CHANGELOG = REPO / "CHANGELOG.md" +PANEL = FRONTEND / "components/update/release-notes-panel.tsx" +NOTES_HOOK = FRONTEND / "hooks/use-release-notes.ts" +PREVIEW = FRONTEND / "lib/release-notes-preview.ts" +CODE_SPANS = FRONTEND / "lib/markdown-code-spans.ts" +LINKS = FRONTEND / "lib/changelog-links.ts" +LIST_COLUMNS = FRONTEND / "lib/markdown-list-columns.ts" +INLINE_COMMENTS = FRONTEND / "lib/markdown-inline-comments.ts" +WEB_BANNER = FRONTEND / "components/web/update-banner.tsx" +TAURI_BANNER = FRONTEND / "components/tauri/update-banner.tsx" + +# The scanners are the frontend half of the contract the parser implements, so they are +# run rather than read. Node strips the types and nothing imports a package: no install. +_TS_ALIAS = re.compile(r'"@/lib/([a-z-]+)"') +_TS_RUNNER = """ +import { resolveChangelogLinks } from "./changelog-links.ts"; +import { releaseNotesPreview } from "./release-notes-preview.ts"; + +const chunks: Buffer[] = []; +process.stdin.on("data", (chunk: Buffer) => chunks.push(chunk)); +process.stdin.on("end", () => { + const markdown = Buffer.concat(chunks).toString("utf8"); + const result = + process.argv[2] === "links" + ? resolveChangelogLinks(markdown) + : releaseNotesPreview(markdown); + process.stdout.write(JSON.stringify(result)); +}); +""" + +SAMPLE = """# Changelog + +Intro prose that belongs to no release. + +## Format + +```md +## 9999.9.9 - fenced sample, not a real section +``` + +## Unreleased + +- staged note + +## 2026.7.6 - 2026-07-22 + +### What's Changed + +- newer thing + +## 2026.7.5 + +### What's Changed + +- older thing +""" + + +@pytest.fixture(scope = "module") +def changelog_module(): + sys.path.insert(0, str(BACKEND)) + try: + from utils import changelog + finally: + sys.path.pop(0) + changelog.reset_changelog_cache() + yield changelog + changelog.reset_changelog_cache() + + +@pytest.fixture +def isolated_changelog(changelog_module, tmp_path, monkeypatch): + """Point the module at a temp file and away from the network.""" + monkeypatch.setenv(changelog_module.DISABLE_ENV_VAR, "1") + path = tmp_path / "CHANGELOG.md" + path.write_text(SAMPLE, encoding = "utf-8") + monkeypatch.setenv(changelog_module.CHANGELOG_PATH_ENV_VAR, str(path)) + changelog_module.reset_changelog_cache() + yield changelog_module + changelog_module.reset_changelog_cache() + + +def test_only_real_release_headings_become_sections(changelog_module): + versions = [entry.version for entry in changelog_module.parse_changelog(SAMPLE)] + # "Format"/"Unreleased" are not versions, and 9999.9.9 is fenced sample. + assert versions == ["2026.7.6", "2026.7.5"] + + +def test_section_body_stops_at_the_next_release(changelog_module): + entry = changelog_module.find_release_notes(SAMPLE, "2026.7.6") + assert entry is not None + assert "newer thing" in entry.body + assert "older thing" not in entry.body + + +def test_unknown_version_returns_no_notes_instead_of_a_nearby_release(changelog_module): + assert changelog_module.find_release_notes(SAMPLE, "2026.7.7") is None + assert changelog_module.find_release_notes(SAMPLE, "2026.7") is None + + +def test_version_equality_is_normalized_not_fuzzy(changelog_module): + entry = changelog_module.find_release_notes(SAMPLE, "2026.07.6") + assert entry is not None and entry.version == "2026.7.6" + + +def test_response_reports_no_match_without_markdown(isolated_changelog): + payload = isolated_changelog.get_release_notes("2026.7.7") + assert payload["matched"] is False + assert payload["markdown"] is None + assert payload["version"] == "2026.7.7" + # The UI still needs somewhere to send the user. + assert payload["release_notes_url"] + + +def test_response_matches_local_changelog_when_offline(isolated_changelog): + payload = isolated_changelog.get_release_notes("2026.7.6") + assert payload["matched"] is True + assert payload["source"] == "local" + assert "newer thing" in payload["markdown"] + + +def test_unsupported_version_query_is_rejected(isolated_changelog): + assert isolated_changelog.is_supported_version_query("2026.7.6") is True + for bad in ("../etc/passwd", "2026.7.6 OR 1", "", "a" * 80): + assert isolated_changelog.is_supported_version_query(bad) is False + assert isolated_changelog.get_release_notes("../etc/passwd")["matched"] is False + + +def test_remote_changelog_wins_over_bundled_copy(changelog_module, tmp_path, monkeypatch): + """The offered version is newer than the installed checkout, so the repo + copy has to be able to describe versions the local file has never heard of.""" + monkeypatch.delenv(changelog_module.DISABLE_ENV_VAR, raising = False) + local = tmp_path / "CHANGELOG.md" + local.write_text(SAMPLE, encoding = "utf-8") + monkeypatch.setenv(changelog_module.CHANGELOG_PATH_ENV_VAR, str(local)) + + remote_body = "# Changelog\n\n## 2026.8.0\n\n- shipped after this install\n" + + class Handler(http.server.BaseHTTPRequestHandler): + def do_GET(self): # noqa: N802 - stdlib naming + payload = remote_body.encode("utf-8") + self.send_response(200) + self.send_header("Content-Length", str(len(payload))) + self.end_headers() + self.wfile.write(payload) + + def log_message(self, *_args): + pass + + server = http.server.HTTPServer(("127.0.0.1", 0), Handler) + thread = threading.Thread(target = server.serve_forever, daemon = True) + thread.start() + try: + monkeypatch.setenv( + changelog_module.CHANGELOG_URL_ENV_VAR, + f"http://127.0.0.1:{server.server_port}/CHANGELOG.md", + ) + changelog_module.reset_changelog_cache() + payload = changelog_module.get_release_notes("2026.8.0") + assert payload["matched"] is True + assert payload["source"] == "remote" + assert "shipped after this install" in payload["markdown"] + finally: + server.shutdown() + server.server_close() + changelog_module.reset_changelog_cache() + + +def test_repo_changelog_exists_and_parses(changelog_module): + assert CHANGELOG.is_file(), "CHANGELOG.md is the editable source of release notes" + entries = changelog_module.parse_changelog(CHANGELOG.read_text(encoding = "utf-8")) + assert entries, "CHANGELOG.md needs at least one `## <version>` section" + + +def test_longer_outer_fence_does_not_leak_a_fake_section(changelog_module): + """A ``` sample inside a ```` block must not close the block and let the + sample's heading be indexed as a real release.""" + text = "## 1.0\n\n````md\n```\n## 9.9.9\n```\n````\n\n- real note\n" + assert [e.version for e in changelog_module.parse_changelog(text)] == ["1.0"] + assert changelog_module.find_release_notes(text, "9.9.9") is None + + +def test_tilde_fence_is_not_closed_by_backticks(changelog_module): + text = "## 1.0\n\n~~~\n```\n## 9.9.9\n~~~\n\n- real\n" + assert [e.version for e in changelog_module.parse_changelog(text)] == ["1.0"] + + +def test_utf8_bom_does_not_hide_the_first_section(changelog_module): + """Editors on Windows can leave a BOM on the first line.""" + assert [e.version for e in changelog_module.parse_changelog("\ufeff## 1.0\n\n- x\n")] == ["1.0"] + + +@pytest.mark.parametrize("newline", ["\r\n", "\r"]) +def test_non_unix_line_endings(changelog_module, newline): + text = f"## 1.0{newline}{newline}- windows note{newline}" + entry = changelog_module.find_release_notes(text, "1.0") + assert entry is not None and "windows note" in entry.body + assert "\r" not in entry.body + + +def test_closing_fence_must_carry_nothing_after_it(changelog_module): + """CommonMark: a closer is the delimiter plus whitespace only. A ```` line + with trailing text inside a ```` block is content, not the end.""" + text = "## 1.0\n\n````md\n```` not a closer\n## 9.9.9\n````\n\n- real\n" + assert [e.version for e in changelog_module.parse_changelog(text)] == ["1.0"] + # An opening fence may still carry an info string. + info = "## 1.0\n\n```python\n## 9.9.9\n```\n\n- real\n" + assert [e.version for e in changelog_module.parse_changelog(info)] == ["1.0"] + + +@pytest.mark.parametrize( + "text", + [ + "## 1.0\n\n- real\n\n<!--\n## 9.9.9\n\n- unpublished\n-->\n", + "## 1.0\n\n- real\n\n<!-- ## 9.9.9 -->\n", + ], +) +def test_commented_out_sections_are_not_releases(changelog_module, text): + """Markdown does not render them, so they are not published notes.""" + assert [e.version for e in changelog_module.parse_changelog(text)] == ["1.0"] + assert changelog_module.find_release_notes(text, "9.9.9") is None + + +def test_repo_root_changelog_is_preferred_over_the_build_snapshot(changelog_module): + """The build backend writes studio/CHANGELOG.md; the root file must win.""" + # Resolved paths, not name suffixes: a checkout may be renamed and Windows uses "\". + paths = [Path(p).resolve() for p in changelog_module._local_changelog_candidates()] + root = paths.index((REPO / changelog_module.CHANGELOG_FILENAME).resolve()) + packaged = paths.index((REPO / "studio" / changelog_module.CHANGELOG_FILENAME).resolve()) + assert root < packaged + build = (REPO / "build.sh").read_text(encoding = "utf-8") + assert "rm -f studio/CHANGELOG.md" in build, "snapshot must not linger after a build" + + +def test_preview_keeps_identifier_underscores(): + """UNSLOTH_DISABLE_UPDATE_CHECK must not render as UNSLOTHDISABLEUPDATECHECK.""" + src = PREVIEW.read_text(encoding = "utf-8") + assert "BOLD_UNDERSCORE" in src and "ITALIC_UNDERSCORE" in src + assert "parkCodeSpans" in src, "code spans are parked so their underscores survive" + assert "const EMPHASIS" not in src, "the blanket emphasis strip is gone" + + +def test_panel_prefers_the_callers_release_url(): + """The API only returns the generic changelog; the desktop banner passes + the exact release page for the version being offered.""" + src = PANEL.read_text(encoding = "utf-8") + assert "releaseNotesUrl ?? notes?.releaseNotesUrl" in src + + +def test_remote_failure_is_reported_so_the_ui_can_retry(changelog_module, tmp_path, monkeypatch): + """A bundled changelog cannot know a version newer than the install, so a + failed remote lookup must not read as "no notes were published".""" + monkeypatch.delenv(changelog_module.DISABLE_ENV_VAR, raising = False) + local = tmp_path / "CHANGELOG.md" + local.write_text("## 1.0\n\n- old release\n", encoding = "utf-8") + monkeypatch.setenv(changelog_module.CHANGELOG_PATH_ENV_VAR, str(local)) + # Port 9 (discard) refuses fast, standing in for an unreachable host. + monkeypatch.setenv(changelog_module.CHANGELOG_URL_ENV_VAR, "http://127.0.0.1:9/CHANGELOG.md") + changelog_module.reset_changelog_cache() + try: + payload = changelog_module.get_release_notes("2.0") + assert payload["matched"] is False + assert payload["error"], "remote failure must reach the UI" + finally: + changelog_module.reset_changelog_cache() + + +def test_preview_keeps_comparison_operators(): + """ "Support Python <3.15 and >3.9" must not lose its operators to the tag + strip, which would turn it into "Support Python 3.9".""" + src = PREVIEW.read_text(encoding = "utf-8") + assert "/<\\/?[a-zA-Z][^>]*>/g" in src, "tag strip must require a name character" + + +def test_preview_hides_commented_out_notes(): + """Unpublished notes inside <!-- --> are not rendered, so not previewed.""" + src = PREVIEW.read_text(encoding = "utf-8") + assert "stripCommentSpans" in src and "COMMENT_OPEN" in src + + +def test_hook_treats_a_reported_failure_as_retryable(): + src = NOTES_HOOK.read_text(encoding = "utf-8") + assert "next.error !== null" in src + + +def test_comment_delimiter_in_inline_code_is_literal(changelog_module): + """A note documenting `<!--` used to put the parser into comment state, + swallowing every release below it.""" + text = "## 2.0\n\n- Type `<!--` to begin a comment\n\n## 1.0\n\n- older\n" + assert [e.version for e in changelog_module.parse_changelog(text)] == ["2.0", "1.0"] + assert changelog_module.find_release_notes(text, "1.0") is not None + assert "older" not in changelog_module.find_release_notes(text, "2.0").body + + +def test_refresh_retries_a_cached_remote_failure(changelog_module, tmp_path, monkeypatch): + """Retry must reach the network again once connectivity returns, rather + than replaying the cached failure until its TTL expires.""" + monkeypatch.delenv(changelog_module.DISABLE_ENV_VAR, raising = False) + local = tmp_path / "CHANGELOG.md" + local.write_text("## 1.0\n\n- old\n", encoding = "utf-8") + monkeypatch.setenv(changelog_module.CHANGELOG_PATH_ENV_VAR, str(local)) + + hits = {"count": 0} + + class Handler(http.server.BaseHTTPRequestHandler): + def do_GET(self): # noqa: N802 - stdlib naming + hits["count"] += 1 + self.send_response(500) + self.send_header("Content-Length", "0") + self.end_headers() + + def log_message(self, *_args): + pass + + server = http.server.HTTPServer(("127.0.0.1", 0), Handler) + threading.Thread(target = server.serve_forever, daemon = True).start() + try: + monkeypatch.setenv( + changelog_module.CHANGELOG_URL_ENV_VAR, + f"http://127.0.0.1:{server.server_port}/CHANGELOG.md", + ) + changelog_module.reset_changelog_cache() + changelog_module.get_release_notes("2.0") + changelog_module.get_release_notes("2.0") + assert hits["count"] == 1, "the failure should be cached" + changelog_module.get_release_notes("2.0", refresh = True) + assert hits["count"] == 2, "refresh must bypass the cached failure" + finally: + server.shutdown() + server.server_close() + changelog_module.reset_changelog_cache() + + +def test_hook_never_returns_another_versions_notes(): + """On the render where the offered version changes, state still describes + the previous one until the effect runs.""" + src = NOTES_HOOK.read_text(encoding = "utf-8") + assert "notes.version === version" in src + assert "refresh" in src, "retry must ask the backend to bypass its cache" + + +@pytest.mark.parametrize("indent", ["", " ", " ", " "]) +def test_headings_and_fences_allow_commonmark_indentation(changelog_module, indent): + """Markdown renders up to three leading spaces, so the parser must agree + or an indented release is unreachable and its notes join the one above.""" + text = f"## 1.0\n\nOne.\n\n{indent}## 2.0\n\nTwo.\n" + assert [e.version for e in changelog_module.parse_changelog(text)] == ["1.0", "2.0"] + fenced = f"## 1.0\n\n{indent}```\n{indent}## 9.9.9\n{indent}```\n\n- real\n" + assert [e.version for e in changelog_module.parse_changelog(fenced)] == ["1.0"] + + +def test_four_space_indentation_is_code_not_structure(changelog_module): + """At four spaces Markdown switches to indented code, for both forms.""" + assert [ + e.version for e in changelog_module.parse_changelog(" ## 9.9.9\n\n## 1.0\n\n- real\n") + ] == ["1.0"] + assert [ + e.version + for e in changelog_module.parse_changelog( + "## 1.0\n\n ```\n sample\n\n## 2.0\n\n- two\n" + ) + ] == ["1.0", "2.0"] + + +def test_desktop_notes_link_to_the_release_page_on_every_platform(): + """manualReleaseUrl is Linux-package only, so in-app updates on macOS, + Windows and AppImage would otherwise link to the generic changelog.""" + hook = (FRONTEND / "hooks/use-tauri-update.ts").read_text(encoding = "utf-8") + assert "const releasePageUrl = info ?" in hook + banner = TAURI_BANNER.read_text(encoding = "utf-8") + assert "releaseNotesUrl={releasePageUrl ?? manualReleaseUrl}" in banner + provider = (FRONTEND / "app/provider.tsx").read_text(encoding = "utf-8") + assert "releasePageUrl={update.releasePageUrl}" in provider + + +def test_preview_matches_how_markdown_renders_prose_and_links(): + """Three rendering mismatches the preview must not reintroduce: wrapped + paragraphs split into fragments, autolinks eaten as tags, and a lead cut + at an abbreviation.""" + src = PREVIEW.read_text(encoding = "utf-8") + # Contiguous prose lines accumulate and flush at a paragraph boundary. + assert "collector.paragraph = collector.paragraph" in src + # <https://x> renders as link text, so it is not a tag. + assert "AUTOLINK" in src + # "e.g. GGUF" is not a sentence boundary. + assert "ABBREVIATIONS" in src and "INITIAL" in src + + +def test_preview_treats_code_as_literal(): + """Inside a code span, and inside an indented code block, Markdown renders + the text literally, so the preview must not transform or promote it.""" + src = PREVIEW.read_text(encoding = "utf-8") + # Code spans are parked before any other inline transformation. + park = src.index("parkCodeSpans(markdown") + assert park < src.index("stripHtmlTags(\n parked") + # A "- cmd" line inside an indented code block is not a headline bullet. + assert "INDENTED_CODE_INDENT" in src + + +def test_desktop_updater_metadata_maps_published_field_names(): + """latest.json publishes Tauri's `notes`/`pub_date`; the manual Linux path + must read those, not `body`/`date`, or its release notes are always empty.""" + rust = (REPO / "studio/src-tauri/src/desktop_update_policy.rs").read_text(encoding = "utf-8") + assert 'alias = "body"' in rust and "notes: Option<String>" in rust + assert 'alias = "date"' in rust and "pub_date: Option<String>" in rust + assert "body: metadata.notes" in rust and "date: metadata.pub_date" in rust + workflow = (REPO / ".github/workflows/release-desktop.yml").read_text(encoding = "utf-8") + assert "'notes': notes," in workflow, "workflow no longer publishes `notes`" + + +def test_backend_exposes_release_notes_route(): + src = (BACKEND / "main.py").read_text(encoding = "utf-8") + assert '@app.get("/api/studio/release-notes")' in src + assert "is_supported_version_query" in src + + +def test_panel_is_scrollable_and_version_scoped(): + src = PANEL.read_text(encoding = "utf-8") + assert "overflow-y-auto" in src, "release notes must scroll inside the popup" + assert "max-h-" in src, "the scroller needs a bounded height" + # Falls back to the payload's own body only, never to another version. + assert "fallbackMarkdown" in src + + +def test_notes_surface_is_borderless_and_lifts_in_dark_mode(): + src = PANEL.read_text(encoding = "utf-8") + assert "border border-border" not in src, "the notes box is a fill, not a bordered box" + # Lighter than the card behind it, rather than a darker inset. + assert "dark:bg-white/[0.06]" in src + # Streamdown's mt-6 clips the first heading against the scroller edge. + assert "[&>*>*:first-child]:mt-0" in src + # Shared utility: thumb hidden until the notes are hovered. + assert "hover-scrollbar" in src + # Streamdown renders code at text-sm, twice this panel's body size. + assert "[&_code]:text-[0.92em]" in src + + +def test_hook_discards_notes_for_a_different_version(): + src = NOTES_HOOK.read_text(encoding = "utf-8") + assert "notesVersion !== version" in src + + +def test_collapsed_panel_previews_the_top_bullets(): + """Collapsed popups show the headline changes without an extra click.""" + preview = PREVIEW.read_text(encoding = "utf-8") + assert "RELEASE_NOTES_PREVIEW_ITEMS = 4" in preview + # Wrapped bullets join into one item, or a preview ends mid-sentence. + assert "collectBullets" in preview and "flush" in preview + # Nested list items are detail, not headline changes. + assert "NESTED_INDENT_TOLERANCE" in preview + # Tag stripping repeats: one pass turns `<<b>b>` back into a live tag. + assert "while (out !== previous)" in preview + + panel = PANEL.read_text(encoding = "utf-8") + assert "releaseNotesPreview" in panel + assert 'data-testid="update-release-notes-summary"' in panel + # Fetched when the popup appears: the collapsed preview needs them too. + assert "enabled: true" in panel + + +def test_preview_highlights_the_leading_sentence(): + """Each bullet leads with its headline sentence, emphasised over the rest.""" + preview = PREVIEW.read_text(encoding = "utf-8") + assert "splitLeadSentence" in preview + # A period inside "CHANGELOG.md" or "e.g." must not read as a break. + assert "SENTENCE_BREAK" in preview and "(?=" in preview + + panel = PANEL.read_text(encoding = "utf-8") + assert '<span className="font-medium text-foreground">{item.lead}</span>' in panel + assert "item.rest" in panel + + +@pytest.mark.parametrize("banner", [WEB_BANNER, TAURI_BANNER]) +def test_update_popup_is_wider_than_the_other_overlays(banner): + """The card is sized for three same-size buttons on one row. + + Width moved from the shared overlay stack onto each overlay, so widening + the update popup does not widen the llama.cpp banner or download panel.""" + assert "max-w-[448px]" in banner.read_text(encoding = "utf-8") + provider = (FRONTEND / "app/provider.tsx").read_text(encoding = "utf-8") + assert "max-w-[400px]" not in provider, "stack must not cap overlay width" + llama = (FRONTEND / "components/llama-update-banner.tsx").read_text(encoding = "utf-8") + assert "max-w-[400px]" in llama, "unrelated overlays keep their width" + + +@pytest.mark.parametrize("banner", [WEB_BANNER, TAURI_BANNER]) +def test_banners_toggle_inline_release_notes(banner): + src = banner.read_text(encoding = "utf-8") + assert "ReleaseNotesPanel" in src + assert "Show release notes" in src and "Hide release notes" in src + # Keyed by version, so a new offer cannot leave old notes on screen. + assert "notesVersion" in src + + +@pytest.mark.parametrize( + "banner,toggle,action", + [ + (WEB_BANNER, "web-update-release-notes-toggle", "web-update-snooze-button"), + (TAURI_BANNER, "tauri-update-release-notes-toggle", "Remind me later"), + ], +) +def test_notes_toggle_shares_the_action_row(banner, toggle, action): + """The toggle sits in the same row as the actions, not on its own line.""" + src = banner.read_text(encoding = "utf-8") + row = src.index("mt-4 flex") + assert row < src.index(toggle) < src.index(action) + # Same type size as the actions beside it; nowrap keeps labels on one line. + toggle_line = next(line for line in src.splitlines() if toggle in line) + toggle_block = src[src.index("Button", row) : src.index(toggle_line)] + assert "text-ui-13" in toggle_block and "whitespace-nowrap" in toggle_block + + +def test_headings_inside_a_raw_html_block_are_not_releases(changelog_module): + """<pre> content is literal, so a sample heading in it must not become a + section and must not cut the real section's body short.""" + text = "## 1.0\n\n<pre>\n## 9.9.9\n</pre>\n\n- real note\n" + assert [e.version for e in changelog_module.parse_changelog(text)] == ["1.0"] + assert "real note" in changelog_module.find_release_notes(text, "1.0").body + assert changelog_module.find_release_notes(text, "9.9.9") is None + + +def test_details_blocks_still_contain_markdown(changelog_module): + """<details> is a CommonMark type 6 block: headings inside it still count, + so collapsible sections keep working.""" + text = "## 2.0\n\n<details>\n<summary>More</summary>\n\n- note\n\n</details>\n\n## 1.0\n\n- older\n" + assert [e.version for e in changelog_module.parse_changelog(text)] == ["2.0", "1.0"] + + +def test_inline_raw_html_tag_does_not_open_a_block(changelog_module): + """A block opens only at the start of a line. A tag named mid-sentence is + inline HTML and must not swallow the releases below it.""" + text = "## 2.0\n\n- Warn when a <script> tag is pasted\n\n## 1.0\n\n- older\n" + assert [e.version for e in changelog_module.parse_changelog(text)] == ["2.0", "1.0"] + + +def test_preview_skips_raw_html_blocks(): + src = PREVIEW.read_text(encoding = "utf-8") + assert "stripRawHtml" in src + # Anchored: only a line-leading tag opens a block, matching the parser. + assert "/^ {0,3}<(pre|script|style|textarea)" in src + + +def test_fence_inside_a_raw_html_block_is_literal(changelog_module): + """Raw HTML contents are literal, so a stray ``` in a <pre> sample is not a + fence. Treating it as one left a block open and hid every later release.""" + text = "## 2.0\n\n<pre>\n```\n</pre>\n\n## 1.0\n\n- older\n" + assert [e.version for e in changelog_module.parse_changelog(text)] == ["2.0", "1.0"] + + +def test_raw_html_block_closes_on_any_of_the_four_tags(changelog_module): + """CommonMark ends a type 1 block at the first `</pre>`, `</script>`, + `</style>` or `</textarea>`: the closer need not match the opener.""" + text = '## 1.0\n\n<script>\nconst sample = "</pre>";\n## 9.9.9\n</script>\n' + assert [e.version for e in changelog_module.parse_changelog(text)] == ["1.0", "9.9.9"] + + +@pytest.mark.parametrize("tag", ["details", "div", "table"]) +def test_type_6_blocks_run_until_a_blank_line(changelog_module, tag): + """`<details>` holds Markdown only after a blank line closes the block, so + a heading pressed against the opening tag is not a release.""" + packed = f"## 1.0\n\n<{tag}>\n## 9.9.9\n</{tag}>\n\n- note\n" + assert [e.version for e in changelog_module.parse_changelog(packed)] == ["1.0"] + spaced = f"## 1.0\n\n<{tag}>\n\n## 2.0\n\n- note\n" + assert [e.version for e in changelog_module.parse_changelog(spaced)] == ["1.0", "2.0"] + + +def test_a_tag_only_line_cannot_interrupt_a_paragraph(changelog_module): + """Type 7 blocks do not interrupt a paragraph, so prose followed by a bare + tag keeps the releases below it reachable.""" + text = "## 2.0\n\nSome prose.\n<span>\n\n## 1.0\n\n- older\n" + assert [e.version for e in changelog_module.parse_changelog(text)] == ["2.0", "1.0"] + + +def test_preview_joins_an_indented_continuation_line(): + """Four spaces only start code outside a paragraph. Inside one the line is + a wrapped continuation, so it must not be dropped from the preview.""" + src = PREVIEW.read_text(encoding = "utf-8") + # Measured from the line's container, so an item's own indent does not count. + assert "!insideBlock && line.indent - line.column >= INDENTED_CODE_INDENT" in src + # A fence indented into a list item is a block, not a wrapped line. + assert "opensDeepFence" in src + + +def test_every_packaging_path_snapshots_the_changelog(): + """`python -m build` and `pip install .` must ship the offline copy too, + so the snapshot is made by the build backend rather than by build.sh.""" + pyproject = (REPO / "pyproject.toml").read_text(encoding = "utf-8") + assert 'build_py = "_changelog_build.build_py"' in pyproject + hook = (REPO / "_changelog_build.py").read_text(encoding = "utf-8") + assert "studio" in hook and "CHANGELOG.md" in hook + # The hook has to reach the sdist, or building from one loses the snapshot. + manifest = (REPO / "MANIFEST.in").read_text(encoding = "utf-8") + assert "include _changelog_build.py" in manifest + assert "include CHANGELOG.md" in manifest + + +def test_preview_code_spans_need_a_matching_closer(): + """A closer is a run of the same length, so ``Use `` `x` `` `` keeps the + inner backticks the expanded notes show.""" + src = CODE_SPANS.read_text(encoding = "utf-8") + assert "candidate === ticks" in src, "a closer is a run of the same length" + assert "stripPadding" in src, "one space of padding is dropped, as in Markdown" + + +def test_preview_skips_thematic_breaks(): + """`- - -` renders as a rule, so it must not take a preview slot.""" + src = PREVIEW.read_text(encoding = "utf-8") + assert "THEMATIC_BREAK" in src + assert "THEMATIC_BREAK.test(visible)" in src + + +def test_preview_keeps_quoted_examples_out_of_the_headlines(): + """A quoted list is example output, not a change, so it never competes + with the release's own bullets.""" + src = PREVIEW.read_text(encoding = "utf-8") + assert "quoted: boolean" in src + assert "if (!line.quoted)" in src, "quoted bullets never become headlines" + + +def test_notes_panel_keeps_the_link_when_the_lookup_fails(): + """Retry is not the only route: the changelog page can be reachable even + when the backend lookup is not.""" + src = PANEL.read_text(encoding = "utf-8") + error_branch = src[src.index('if (state === "error")') :] + retry = error_branch.index("update-release-notes-retry") + assert error_branch.index("{link}") > retry, "link sits beside retry" + + +def test_hook_waits_for_the_desktop_auth_token(): + """The desktop popup can render before auto-auth installs its token, so a + missing token must not be recorded as a failed lookup.""" + src = NOTES_HOOK.read_text(encoding = "utf-8") + assert "hasAuthToken()" in src and "AUTH_POLL_LIMIT" in src + + +def test_installed_layout_prefers_the_bundled_changelog(tmp_path): + """Installed, the levels above studio/ are site-packages. A stray + CHANGELOG.md left there by another package must not outrank the bundled + snapshot, so those levels are only searched in a source checkout.""" + site_packages = tmp_path / "site-packages" + package = site_packages / "studio/backend/utils" + package.mkdir(parents = True) + for name in ("changelog.py", "update_status.py"): + shutil.copy(BACKEND / "utils" / name, package / name) + for parent in (site_packages / "studio", package.parent, package): + (parent / "__init__.py").write_text("", encoding = "utf-8") + (site_packages / CHANGELOG.name).write_text("## 2.0\n\n- stray\n", encoding = "utf-8") + bundled = site_packages / "studio" / CHANGELOG.name + bundled.write_text("## 2.0\n\n- bundled\n", encoding = "utf-8") + + env = {**os.environ, "PYTHONPATH": str(site_packages)} + env.pop("UNSLOTH_CHANGELOG_PATH", None) + + def served() -> str: + # cwd is outside the checkout, so this imports the installed copy. + return subprocess.run( + [ + sys.executable, + "-c", + "from studio.backend.utils import changelog\n" + "print(changelog._read_local_changelog().text)", + ], + capture_output = True, + text = True, + env = env, + cwd = tmp_path, + check = True, + ).stdout + + assert "bundled" in served() and "stray" not in served() + + # A checkout marker there means it really is a repo root, so it wins again. + (site_packages / "pyproject.toml").write_text("", encoding = "utf-8") + assert "stray" in served() + + +def test_a_section_staged_as_a_comment_reads_as_unpublished( + changelog_module, tmp_path, monkeypatch +): + """Notes staged inside <!-- --> render as nothing, so the popup must say + no notes were published rather than show an empty surface.""" + monkeypatch.setenv(changelog_module.DISABLE_ENV_VAR, "1") + local = tmp_path / "CHANGELOG.md" + local.write_text("## 2.0\n\n<!-- not ready -->\n\n## 1.0\n\n- shipped\n", encoding = "utf-8") + monkeypatch.setenv(changelog_module.CHANGELOG_PATH_ENV_VAR, str(local)) + changelog_module.reset_changelog_cache() + try: + staged = changelog_module.get_release_notes("2.0") + assert staged["matched"] is False and staged["markdown"] is None + assert changelog_module.get_release_notes("1.0")["matched"] is True + finally: + changelog_module.reset_changelog_cache() + + +@pytest.mark.parametrize( + "body,visible", + [ + ("- note", True), + ("<!-- staged -->", False), + ("```\n```", True), + ("<pre>\n</pre>", True), + (" ", False), + ], +) +def test_visibility_check_only_hides_comments(changelog_module, body, visible): + assert changelog_module._renders_visibly(body) is visible + + +@pytest.mark.parametrize( + "block", + [ + "<?php\n## 9.9.9\n?>", + "<![CDATA[\n## 9.9.9\n]]>", + "<!DOCTYPE\n## 9.9.9\n>", + ], +) +def test_processing_instructions_and_declarations_are_literal(changelog_module, block): + """Raw block types 3 to 5 render literally, like <pre>, so a heading inside + one is a sample and not a release.""" + text = f"## 1.0\n\n{block}\n\n- real note\n" + assert [e.version for e in changelog_module.parse_changelog(text)] == ["1.0"] + assert "real note" in changelog_module.find_release_notes(text, "1.0").body + + +def test_headings_need_a_space_or_tab_after_the_hashes(changelog_module): + """A non-breaking space pasted from rich text renders as ordinary text, so + the line must not end the release above it.""" + text = "## 1.0\n\n- real note\n\n## 9.9.9\n\n- not a release\n" + assert [e.version for e in changelog_module.parse_changelog(text)] == ["1.0"] + assert changelog_module.find_release_notes(text, "9.9.9") is None + # A tab is valid and still opens a heading. + tabbed = "## 1.0\n\n- one\n\n##\t2.0\n\n- two\n" + assert [e.version for e in changelog_module.parse_changelog(tabbed)] == ["1.0", "2.0"] + + +def test_preview_skips_every_raw_block_form(): + """The extractor tracks the same block forms as the parser, so a sample + bullet inside one cannot become the collapsed headline.""" + src = PREVIEW.read_text(encoding = "utf-8") + assert "RAW_BLOCKS" in src + assert "CDATA" in src and "[A-Za-z]" in src + + +@pytest.mark.parametrize("banner", [WEB_BANNER, TAURI_BANNER]) +def test_expanded_popup_fits_a_short_viewport(banner): + """A window under roughly 430px high used to push the card's title and + dismiss control above the top of the screen.""" + panel = PANEL.read_text(encoding = "utf-8") + # The notes region shrinks inside the capped card, so header and actions stay on screen. + assert "min-h-0 flex-1" in panel, "notes height must follow the viewport" + src = banner.read_text(encoding = "utf-8") + assert "max-h-[calc(100dvh_-_2rem)]" in src, "card is the backstop on tiny viewports" + + +def test_relative_changelog_links_point_at_the_repository(): + """CHANGELOG.md links are repository-relative. Rendered as-is they resolve + against Studio's origin, so the renderer blocks them.""" + src = LINKS.read_text(encoding = "utf-8") + assert "https://github.com/unslothai/unsloth/blob/main/" in src + assert "https://raw.githubusercontent.com/unslothai/unsloth/main/" in src + # Absolute targets, fragments, fenced code and code spans stay untouched. + assert "ABSOLUTE" in src and "codeSpans" in src and "FENCE" in src + panel = PANEL.read_text(encoding = "utf-8") + assert "resolveChangelogLinks" in panel + + +@pytest.mark.parametrize("query", ["latest", "main", "not-a-version", "abc"]) +def test_unparseable_versions_are_rejected(changelog_module, query): + """Sections are indexed only when their version parses, so a query that + cannot parse can never match and is a bad request, not an empty result.""" + assert changelog_module.is_supported_version_query(query) is False + + +@pytest.mark.parametrize("query", ["2026.7.5", "v2026.7.5", "2026.07.5", "1.0.0rc1"]) +def test_real_versions_are_still_accepted(changelog_module, query): + assert changelog_module.is_supported_version_query(query) is True + + +def test_reference_style_images_resolve_to_the_raw_host(): + """`![alt][arch]` with `[arch]: docs/arch.png` needs the raw file: the blob + URL is an HTML page, so the image would not load.""" + src = LINKS.read_text(encoding = "utf-8") + assert "IMAGE_REFERENCE" in src + assert "imageLabels" in src + + +def test_collapsed_notes_surface_is_hidden_when_nothing_previews(): + """Notes that are only a fenced command block preview as nothing, and an + empty muted strip is worse than no strip.""" + src = PANEL.read_text(encoding = "utf-8") + assert "preview?.items.length === 0" in src + + +def test_a_fence_closer_accepts_only_spaces_and_tabs(changelog_module): + """A delimiter followed by a non-breaking space is code content, so it must + not close the block and let a sample heading through.""" + text = "## 1.0\n\n```\n```\u00a0\n## 9.9.9\n```\n\n- real note\n" + assert [e.version for e in changelog_module.parse_changelog(text)] == ["1.0"] + plain = "## 1.0\n\n```\nx\n```\t\n\n## 2.0\n\n- two\n" + assert [e.version for e in changelog_module.parse_changelog(plain)] == ["1.0", "2.0"] + # The same rule in both frontend scanners. + for source in (PREVIEW, LINKS): + assert "/[^ \\t]/" in source.read_text(encoding = "utf-8") + + +def test_code_spans_close_on_a_run_of_equal_length(): + """`a``b [x](y.md)` is one code span, so the link inside it is literal.""" + src = CODE_SPANS.read_text(encoding = "utf-8") + assert "candidate === ticks" in src, "closer length must match the opener" + # Shared, so the preview and the link resolver cannot drift apart. + assert "markdown-code-spans" in PREVIEW.read_text(encoding = "utf-8") + assert "markdown-code-spans" in LINKS.read_text(encoding = "utf-8") + + +def test_preview_decodes_entities_like_the_renderer(): + """Streamdown renders `AT&T` as AT&T, so the collapsed preview must + not show the raw entity.""" + src = PREVIEW.read_text(encoding = "utf-8") + assert "NAMED_ENTITIES" in src and "decodeEntity" in src + # Decoded before code spans are restored, so code keeps the literal text. + assert src.index(".replace(ENTITY, decodeEntity)") < src.index(".replace(PARKED") + + +def test_release_notes_request_refreshes_an_expired_token(): + """A direct fetch cannot recover from a 401; authFetch refreshes first.""" + src = NOTES_HOOK.read_text(encoding = "utf-8") + assert "authFetch(" in src + assert "getAuthToken" not in src + + +def test_preview_handles_the_desktop_updater_line_endings(): + """The updater body arrives with CRLF, which used to hide fences from the + extractor and promote a code sample to a headline.""" + src = PREVIEW.read_text(encoding = "utf-8") + assert "LINE_ENDINGS" in src + assert "LINE_ENDINGS" in LINKS.read_text(encoding = "utf-8") + + +def test_preview_renders_reference_links_as_text(): + """`[text][label]` and `![alt][label]` render as a link and an image, so + the preview must not show their raw markup.""" + src = PREVIEW.read_text(encoding = "utf-8") + assert "LINK_REFERENCE" in src and "IMAGE_REFERENCE" in src + # A definition line renders as nothing, so it is not a preview item. + assert "DEFINITION" in src + + +def test_preview_treats_escaped_punctuation_as_literal(): + """`\\*not italic\\*` keeps its stars and an escaped backtick does not open + a code span.""" + assert "ESCAPE" in PREVIEW.read_text(encoding = "utf-8") + assert "escaped(" in CODE_SPANS.read_text(encoding = "utf-8") + + +def test_link_resolver_skips_every_code_form(): + """Indented code and code spans crossing a line render as code, so their + contents must not be rewritten.""" + src = LINKS.read_text(encoding = "utf-8") + assert "INDENTED_CODE" in src + # Spans are scanned over the whole document, not line by line. + assert "codeSpans(masked)" in src + # A definition cannot interrupt a paragraph. + assert "definition.has(index)" in src + + +def test_badge_links_resolve_both_targets(): + """`[![alt](img)](link)` is the badge idiom: the outer link used to stay + relative because the label was not allowed to nest.""" + assert "NESTED_LABEL" in LINKS.read_text(encoding = "utf-8") + + +def test_in_flight_requests_are_identified_not_just_versioned(): + """Two requests for the same version could resolve out of order and leave + the panel showing the older result.""" + assert "requestIdRef" in NOTES_HOOK.read_text(encoding = "utf-8") + + +def test_notes_repair_the_shared_previews_width_reset(): + """MarkdownPreview clears max-width on every descendant, so a wide image + and the renderer's own link dialog escape the card.""" + src = PANEL.read_text(encoding = "utf-8") + assert "[&_img]:max-w-full" in src + assert "[&_[data-streamdown=link-safety-modal]>*]:max-w-md" in src + + +@pytest.mark.parametrize("banner", [WEB_BANNER, TAURI_BANNER]) +def test_only_the_notes_region_scrolls(banner): + """The dismiss control sits inside the card, so scrolling the card itself + carried it off screen on a short viewport.""" + src = banner.read_text(encoding = "utf-8") + assert "flex max-h-[calc(100dvh_-_2rem)] flex-col overflow-hidden" in src + assert 'className="min-h-0 flex-1"' in src + panel = PANEL.read_text(encoding = "utf-8") + assert "max-h-64 min-h-0 flex-1 overflow-y-auto" in panel + + +def test_a_comment_marker_in_prose_cannot_swallow_later_releases(changelog_module): + """A note that mentions `<!--` used to put the parser into comment state + for the rest of the file: the releases below it disappeared and their + notes were served under the newer version's heading.""" + text = ( + "## 2026.8.0\n\n- Studio strips <!-- markers from pasted prompts.\n\n" + "## 2026.7.5\n\n- SECRET: an older release\n" + ) + assert [e.version for e in changelog_module.parse_changelog(text)] == [ + "2026.8.0", + "2026.7.5", + ] + assert "SECRET" not in changelog_module.find_release_notes(text, "2026.8.0").body + assert changelog_module.find_release_notes(text, "2026.7.5") is not None + # A comment that starts a line is still a block and still hides its body. + hidden = "## 2.0\n\n<!--\n## 9.9.9\n-->\n\n- note\n" + assert [e.version for e in changelog_module.parse_changelog(hidden)] == ["2.0"] + + +def test_unmatched_backtick_runs_stay_linear(changelog_module): + """Rescanning the suffix for every opener was quadratic: a line of runs of + 1, 2, 3 ... backticks, none of which ever closes, took 7.7s at 321 KB and + is reparsed on every popup request, so one malformed remote changelog could + tie up backend workers.""" + line = "".join("`" * (i + 1) + "x" for i in range(800)) + assert len(line) > 300_000 + started = time.monotonic() + assert changelog_module._code_span_ranges(line) == [] + assert time.monotonic() - started < 2.0 + + +def test_a_base_exception_releases_the_single_flight_flag(changelog_module, monkeypatch): + """The flag was cleared only after `except Exception`, so a BaseException + (KeyboardInterrupt, SystemExit, CancelledError) stranded it and every later + caller then waited out the full deadline for the life of the process.""" + changelog_module.reset_changelog_cache() + + def explode(): + raise KeyboardInterrupt + + monkeypatch.setattr(changelog_module, "_fetch_remote_changelog", explode) + with pytest.raises(KeyboardInterrupt): + changelog_module.get_remote_changelog() + assert changelog_module._remote_fetching is False + changelog_module.reset_changelog_cache() + + +@pytest.mark.parametrize("marker", ["<!-->", "<!--->"]) +def test_an_empty_comment_does_not_swallow_later_releases(changelog_module, marker): + """`<!-->` and `<!--->` are complete comments in CommonMark: the closer + overlaps the opener. Searching for `-->` past the opener missed them, so an + empty comment used as a section marker hid every release below it.""" + text = f"## 2.0\n\n- new stuff\n\n{marker}\n\n## 1.0\n\n- old stuff\n" + assert [e.version for e in changelog_module.parse_changelog(text)] == ["2.0", "1.0"] + assert changelog_module.find_release_notes(text, "1.0") is not None + assert "old stuff" not in changelog_module.find_release_notes(text, "2.0").body + # The frontend scanner has to agree, or the preview and the body disagree. + assert "!line.includes(COMMENT_CLOSE)" in PREVIEW.read_text(encoding = "utf-8") + + +def test_an_unterminated_comment_still_hides_the_rest(changelog_module): + """The fix must not turn every `<!--` line into a no-op block.""" + text = "## 2.0\n\n<!-- never closed\n\n## 1.0\n\n- old stuff\n" + assert [e.version for e in changelog_module.parse_changelog(text)] == ["2.0"] + + +def test_a_closing_delimiter_takes_its_whole_line(changelog_module): + """CommonMark keeps the closing line inside the block, so a heading glued + after `-->` or `</pre>` is not a release.""" + for text in ( + "## 1.0\n\n<!-- hidden -->## 9.9.9\n\n- note\n", + "## 1.0\n\n<pre>\nx\n</pre>## 9.9.9\n\n- note\n", + ): + assert [e.version for e in changelog_module.parse_changelog(text)] == ["1.0"] + + +def test_an_exact_heading_is_never_shadowed(changelog_module): + """PEP 440 says 1.0 == 1.0.0, so the normalised match used to win even + when the file had a section spelled exactly as asked.""" + text = "## 1.0.0\n\n- padded\n\n## 1.0\n\n- exact\n" + assert changelog_module.find_release_notes(text, "1.0").body == "- exact" + assert changelog_module.find_release_notes(text, "1.0.0").body == "- padded" + # Normalised matching still applies when there is no exact heading. + assert changelog_module.find_release_notes("## 2026.7.6\n\n- x\n", "2026.07.6") is not None + + +def test_setext_headings_are_release_boundaries(changelog_module): + """A version over a line of dashes is the same heading in setext form.""" + text = "2.0\n---\n\n- new\n\n1.0\n---\n\n- old\n" + assert [e.version for e in changelog_module.parse_changelog(text)] == ["2.0", "1.0"] + assert changelog_module.find_release_notes(text, "2.0").body == "- new" + # A rule between sections is still a rule, and a setext h1 is not a release. + assert [ + e.version + for e in changelog_module.parse_changelog("## 2.0\n\n- a\n\n---\n\n## 1.0\n\n- b\n") + ] == ["2.0", "1.0"] + + +def test_a_long_backtick_run_does_not_stall_the_parser(changelog_module): + """The code-span guard used to backtrack: 20k backticks took over a minute + and every request re-parsed the file.""" + import time + + text = "## 1.0\n\n- " + "`" * 20_000 + " <!--\n" + started = time.perf_counter() + changelog_module.parse_changelog(text) + assert time.perf_counter() - started < 1.0 + + +def test_the_remote_fetch_has_a_total_deadline(changelog_module): + """The socket timeout resets on every read, so a trickling server could + hold a worker for minutes and still be treated as a success.""" + source = (BACKEND / "utils/changelog.py").read_text(encoding = "utf-8") + assert "deadline = time.monotonic() + CHANGELOG_TIMEOUT_SECONDS" in source + # read1 returns after one socket read, so the deadline is actually checked. + assert "response.read1(" in source + # Waiters give up rather than queue behind a stalled fetch. + assert "Release notes are still loading." in source + + +def test_truncated_notes_close_their_fence(changelog_module): + """A blind slice could end inside a code block and break the rendering.""" + body = "```\n" + "x\n" * 20_000 + "```\n" + payload = changelog_module._notes_response(version = "1.0", markdown = body, source = "local") + assert payload["truncated"] is True + assert payload["markdown"].rstrip().endswith("```") + + +def test_the_opt_out_beats_the_developer_override(): + """UNSLOTH_STUDIO_FAKE_UPDATE is a dev switch; the documented kill switch + still wins, and the value has to parse as a version.""" + source = (BACKEND / "utils/update_status.py").read_text(encoding = "utf-8") + assert "forced_version and not disabled and _is_version(forced_version)" in source + + +def test_a_list_item_over_dashes_is_not_a_setext_heading(changelog_module): + """`- first` followed by `---` is a list and a rule. Reading it as a + heading discarded the bullet and the rest of the section with it.""" + text = "## 1.0\n\n- first\n---\n\n- second\n" + assert [e.version for e in changelog_module.parse_changelog(text)] == ["1.0"] + body = changelog_module.find_release_notes(text, "1.0").body + assert "first" in body and "second" in body + # Real setext headings still work. + setext = "2.0\n---\n\n- new\n\n1.0\n---\n\n- old\n" + assert [e.version for e in changelog_module.parse_changelog(setext)] == ["2.0", "1.0"] + + +def test_a_backtick_in_a_fence_info_string_is_not_a_fence(changelog_module): + """CommonMark forbids backticks in a backtick fence's info string, so such + a line is prose and must not swallow the releases below it.""" + text = "## 2.0\n\n```bad`info\n\n## 1.0\n\n- old\n" + assert [e.version for e in changelog_module.parse_changelog(text)] == ["2.0", "1.0"] + # A tilde fence may hold backticks, and a normal fence still hides samples. + assert [ + e.version + for e in changelog_module.parse_changelog( + "## 2.0\n\n```md\n## 9.9.9\n```\n\n## 1.0\n\n- old\n" + ) + ] == ["2.0", "1.0"] + for source in (PREVIEW, LINKS): + assert "info string" in source.read_text(encoding = "utf-8") + + +def test_preview_follows_commonmark_paragraph_rules(): + """Only an ordered list starting at 1 may interrupt a paragraph, and an + unresolved reference keeps its brackets. A quote owns the paragraph its own + lines hold, so a marker written outside the quote interrupts nothing.""" + src = " ".join(PREVIEW.read_text(encoding = "utf-8").split()) + assert "const interrupts = collector.current === null" in src + assert "!collector.quotedParagraph;" in src + assert "definedLabel" in src, "a reference only renders as text when defined" + # A comment written mid-sentence hides its own line at most. + assert "COMMENT_BLOCK_OPEN" in src + + +def test_link_resolver_leaves_raw_blocks_and_escapes_alone(): + src = LINKS.read_text(encoding = "utf-8") + assert "RAW_HTML_OPEN" in src and "inRawHtml" in src + assert "isEscaped(line, opener)" in src + # A heading ends a paragraph, so a definition under one is a definition. + assert "BLOCK_LINE.test(structure)" in src + + +def test_code_span_closers_ignore_backslashes(): + """Escapes are not processed inside a code span, so a run after a + backslash still closes it.""" + src = CODE_SPANS.read_text(encoding = "utf-8") + body = src[src.index("export function codeSpans") :] + assert body.count("escaped(text") == 1, "only an opener can be escaped" + + +def test_the_overlay_stack_fits_the_viewport(): + """The update card's own cap does not account for a long download list + stacked beneath it.""" + provider = (FRONTEND / "app/provider.tsx").read_text(encoding = "utf-8") + assert "max-h-[calc(100dvh_-_2rem)]" in provider + panel = (FRONTEND / "features/hub/download-manager/download-manager-panel.tsx").read_text( + encoding = "utf-8" + ) + # Both overlays scroll internally, so they can give up height. + assert "flex min-h-0" in panel + assert "flex min-h-0" in WEB_BANNER.read_text(encoding = "utf-8") + + +def test_the_desktop_stack_is_capped_like_the_browser_one(): + """The download panel shares the desktop stack, so the update card's own + cap is not enough there either.""" + provider = (FRONTEND / "app/provider.tsx").read_text(encoding = "utf-8") + assert provider.count("max-h-[calc(100dvh_-_2rem)]") == 2, "both stacks are capped" + assert "flex min-h-0" in TAURI_BANNER.read_text(encoding = "utf-8") + + +def test_desktop_notes_are_looked_up_by_the_backend_version(): + """latest.json's `version` is the app SemVer while CHANGELOG.md is keyed by + the backend release, so the desktop popup used to find no section at all + and fall back to the updater's generic text.""" + workflow = (REPO / ".github/workflows/release-desktop.yml").read_text(encoding = "utf-8") + assert "'pypi_version': os.environ['PYPI_VERSION']" in workflow + assert "PYPI_VERSION: ${{ needs.prepare-version.outputs.pypi_version }}" in workflow + rust = (REPO / "studio/src-tauri/src/desktop_update_policy.rs").read_text(encoding = "utf-8") + assert "pypi_version: Option<String>" in rust + hook = NOTES_HOOK.parent.joinpath("use-tauri-update.ts").read_text(encoding = "utf-8") + # Both desktop paths carry it: the plugin exposes the raw metadata. + assert "rawPypiVersion(update.rawJson)" in hook + assert "manualUpdate.pypiVersion" in hook + banner = TAURI_BANNER.read_text(encoding = "utf-8") + assert "info?.pypiVersion ?? info?.version" in banner + + +def test_one_slow_read_cannot_outlast_the_fetch_budget(changelog_module): + """The socket timeout is per operation, so slow headers followed by a slow + body could hold a worker for twice the advertised deadline.""" + source = (BACKEND / "utils/changelog.py").read_text(encoding = "utf-8") + assert "_limit_read(response, remaining)" in source + assert "sock.settimeout(max(remaining, _CHANGELOG_MIN_READ_SECONDS))" in source + + +def test_a_heading_indented_into_a_list_item_is_not_a_release(changelog_module): + """CommonMark keeps a heading at the item's content column inside the item. + Treating it as a boundary truncated the real release and indexed a version + that does not exist. Checked against markdown-it (commonmark preset).""" + text = "## 1.0\n\n- Example:\n ## 9.9.9\n\n- after\n" + assert [e.version for e in changelog_module.parse_changelog(text)] == ["1.0"] + body = changelog_module.find_release_notes(text, "1.0").body + assert "9.9.9" in body and "after" in body + # One space short of the content column, the list ends and it is a release. + left = "## 1.0\n\n- Example:\n ## 2.0\n" + assert [e.version for e in changelog_module.parse_changelog(left)] == ["1.0", "2.0"] + + +def test_a_closed_list_stops_holding_headings(changelog_module): + """Only an open item nests a heading, so a dedented paragraph, heading, + break or fence hands the following indentation back to the document.""" + + def versions(text): + return [e.version for e in changelog_module.parse_changelog(text)] + + assert versions("## 1.0\n\n- Example:\n\nText.\n\n ## 2.0\n") == ["1.0", "2.0"] + assert versions("## 1.0\n\n- Example:\n## 2.0\n ## 3.0\n") == ["1.0", "2.0", "3.0"] + assert versions("## 1.0\n\n- Example:\n Text.\n---\n ## 2.0\n") == ["1.0", "2.0"] + assert versions("## 1.0\n\n- Example:\n```\n```\n ## 2.0\n") == ["1.0", "2.0"] + # An item may begin with one blank line; content after that is outside it. + assert versions("## 1.0\n\n-\n\n ## 2.0\n") == ["1.0", "2.0"] + + +def test_a_version_line_is_not_an_ordered_list_marker(changelog_module): + """`2.` needs whitespace after it to be a marker, or list tracking would + read every setext version as a list item and lose the heading.""" + text = "2.0\n---\n\n- new\n\n1.0\n---\n\n- old\n" + assert [e.version for e in changelog_module.parse_changelog(text)] == ["2.0", "1.0"] + # An ordered item interrupts a paragraph only when it starts at 1. + assert [ + e.version for e in changelog_module.parse_changelog("## 1.0\n\nText.\n9) one\n ## 2.0\n") + ] == ["1.0", "2.0"] + + +def test_a_wrapped_setext_heading_is_still_a_release(changelog_module): + """CommonMark promotes the whole paragraph, so a heading that wraps keeps + the version in its first token. Reading only the last line left the release + unindexed and its notes unreachable.""" + text = "2026.7.5 - Release\nJuly 25\n---\n\n- note\n" + entries = changelog_module.parse_changelog(text) + assert [e.version for e in entries] == ["2026.7.5"] + # The heading lines are the heading, not the body. + assert entries[0].body == "- note" + assert "July 25" not in entries[0].body + + +def test_a_lowercase_declaration_is_not_a_raw_block(changelog_module): + """Only `<!` plus an uppercase letter opens one, so prose that mentions + `<!note` must not hide every release under it.""" + assert [ + e.version for e in changelog_module.parse_changelog("<!note\n\n## 1.0\n\n- real\n") + ] == ["1.0"] + # A real declaration still hides its own block. + assert [ + e.version for e in changelog_module.parse_changelog("<!DOCTYPE\n## 9.9.9\n>\n\n## 1.0\n") + ] == ["1.0"] + # The collapsed preview needs the same rule or it drops visible bullets. + assert "<![A-Z]" in PREVIEW.read_text(encoding = "utf-8") + + +def test_link_resolver_reads_html_containers_the_way_the_others_do(): + """A `<details>` or `<div>` with no blank line inside is a type 6 block, so + its contents render literally. Rewriting a link there mutates text the + reader sees verbatim, and a fence inside such a block was being taken for a + real fence, which stopped every link below it from resolving at all. The + backend parser and the collapsed preview already apply the type 6 and 7 + rules, so the resolver has to share them or the three disagree on the same + notes.""" + links = LINKS.read_text(encoding = "utf-8") + for source in (PREVIEW, LINKS): + text = source.read_text(encoding = "utf-8") + assert "HTML_BLOCK_TAGS" in text and "HTML_TAG_ONLY_LINE" in text + # A blank line ends the block, not the closing tag, and a bare quote marker counts as blank. + assert "inHtmlBlock = !!container.trim()" in links + # Type 7 cannot interrupt a paragraph, so prose above it keeps its links. + assert "return !afterParagraph && HTML_TAG_ONLY_LINE.test(line);" in links + + +def test_an_escaped_mark_makes_an_image_a_link(): + """`\\![alt](path)` renders as a link, so it resolves to the file's page on + GitHub rather than to the raw-content host.""" + links = LINKS.read_text(encoding = "utf-8") + assert 'const image = bang === "!" && !isEscaped(line, offset);' in links + # The reference pre-scan has to skip it too, or the definition flips host. + assert "isEscaped(line, match.index)" in links + + +def test_only_markdown_line_endings_split_the_changelog(changelog_module): + """str.splitlines also breaks on U+2028, U+2029, NEL, vertical tab and form + feed, none of which end a line in CommonMark. A separator sitting in prose + ahead of "## 9.9.9" made the parser index a release the renderer never shows + and truncate the notes above it.""" + text = "## 2.0\n\nnote with a separator 
## 9.9.9\n\n## 1.0\n\n- old\n" + assert [e.version for e in changelog_module.parse_changelog(text)] == ["2.0", "1.0"] + # The prose stays whole rather than being cut at the separator. + entry = changelog_module.find_release_notes(text, "2.0") + assert entry is not None and "9.9.9" in entry.body + for separator in ("
", "\x85", "\x0b", "\x0c"): + broken = f"## 2.0\n\nnote{separator}## 9.9.9\n\n## 1.0\n\n- old\n" + assert [e.version for e in changelog_module.parse_changelog(broken)] == ["2.0", "1.0"] + # The three real line endings still split. + for ending in ("\n", "\r\n", "\r"): + real = f"## 2.0{ending}{ending}- new{ending}{ending}## 1.0{ending}{ending}- old{ending}" + assert [e.version for e in changelog_module.parse_changelog(real)] == ["2.0", "1.0"] + + +def test_the_build_does_not_require_a_writable_source_tree(): + """A PEP 517 build may run against an immutable checkout (Nix, Bazel, a + read-only container mount). Writing the snapshot beside the sources raised + PermissionError before build_py started, so no wheel could be built at all. + """ + src = (REPO / "_changelog_build.py").read_text(encoding="utf-8") + # The source-tree copy is best effort. + assert "except OSError:" in src + # The wheel gets its copy from the staging directory either way. + assert 'Path(self.build_lib) / "studio" / "CHANGELOG.md"' in src + + +def test_link_resolver_reads_comments_before_fences(): + """A fence delimiter hidden inside an HTML comment is not a fence. Reading + it as one left the fence open, so every visible line below was classified as + code and none of its links were resolved, which is far worse than the + mutated-text case: the whole rest of the notes silently stops working. The + order matters both ways, so a comment opener inside a real fence is not a + comment either.""" + links = LINKS.read_text(encoding="utf-8") + # Fence state is read before comments are masked, the order the collapsed preview uses. + assert "const fenceSource = inComment\n ? null\n : FENCE.exec(" in links + # Masking happens only after the in-fence early return. + fence_return = links.index("// Fenced content is literal") + assert links.index("const [line, stillInComment, stillRunOn] = maskComments(") > fence_return + # Commented ranges join the code spans, so a hidden link is left alone. + assert "const spans = [...codeSpans(masked), ...comments].sort(" in links + + +def test_preview_heading_and_quote_markers_follow_the_backend_rule(): + """An ATX heading needs an ASCII space, a tab or the end of the line after + the marker, which is what _HEADING_PATTERN requires; `\\s` also matches a + non-breaking space, so prose beginning "## Important change" with one was + read as a heading and dropped, leaving a prose-only release with no + collapsed preview at all. A blockquote marker takes at most three leading + spaces for the same reason every other marker here does: accepting any run + let an indented code sample containing "> - sample output" shed its + indentation and be shown as the summary.""" + src = PREVIEW.read_text(encoding="utf-8") + assert "const HEADING = /^#{1,6}(?:[ \\t]|$)/;" in src + assert "const HEADING_LINE = /^ {0,3}#{1,6}(?:[ \\t]|$)/;" in src + assert "const BLOCKQUOTE = /^ {0,3}>[ \\t]?/;" in src + # The backend rule this mirrors. + backend = (BACKEND / "utils" / "changelog.py").read_text(encoding="utf-8") + assert "^ {0,3}##(?:[ \\t]+(?P<title>.*?))?[ \\t]*$" in backend + + +def test_preview_collects_labels_only_from_real_definitions(): + """A definition-shaped line inside an indented code block or a deep fence is + literal text, so CommonMark leaves a later "[Beta] support" unresolved with + its brackets showing. Recording the label anyway made toPlainText strip them + in the collapsed preview, so it disagreed with the expanded view. The + pre-scan skips the same code the collector pass skips; a real definition + takes at most three spaces of indentation, so the indent test cannot reject + one.""" + src = PREVIEW.read_text(encoding="utf-8") + scan = src.index("const labels = new Set<string>();") + collect = src.index("let deepFence: string | null = null;") + prescan = " ".join(src[scan:collect].split()) + assert "let labelFence: string | null = null;" in prescan + assert "if (line.indent - line.column >= INDENTED_CODE_INDENT) { continue; }" in prescan + assert "endsDeepFence(labelFence, labelColumn, line)" in prescan + + +def test_an_html_block_to_the_left_of_a_list_item_closes_it(changelog_module): + """Types 1 to 6 interrupt a paragraph, so an unindented <div> after "- item" + closes the item and a following one-to-three-space-indented "## 2.0" is a + real document heading. It was read as a lazy paragraph continuation, so the + item stayed open and the release below the block was swallowed.""" + text = "## 3.0\n\n- item\n<div>\nhidden\n</div>\n\n ## 2.0\n\n- two\n\n## 1.0\n\n- one\n" + assert [e.version for e in changelog_module.parse_changelog(text)] == ["3.0", "2.0", "1.0"] + # Without the block the heading really is nested, so it stays suppressed. + nested = "## 3.0\n\n- item\n\n ## 2.0\n\n- two\n\n## 1.0\n\n- one\n" + assert [e.version for e in changelog_module.parse_changelog(nested)] == ["3.0", "1.0"] + # Ordinary lazy continuation is untouched. + lazy = "## 3.0\n\n- item\ncontinued\n\n ## 2.0\n\n## 1.0\n\n- one\n" + assert [e.version for e in changelog_module.parse_changelog(lazy)] == ["3.0", "1.0"] + + +def test_the_download_panel_can_shrink_inside_the_capped_stack(): + """The bottom-right stack is capped to the viewport, and a flex item defaults + to min-height:auto, so this wrapper could not shrink below its own content. + On a short viewport the cap was then absorbed by the update card, whose + header and actions are fixed, rather than by the download list, which + scrolls. Only the shared-stack branch needs it; standalone is positioned + fixed and is not a flex item at all.""" + panel = (FRONTEND / "features/hub/download-manager/download-manager-panel.tsx").read_text( + encoding="utf-8" + ) + assert 'positioned ? "fixed bottom-4 right-4 z-50" : "flex min-h-0 justify-end"' in panel + provider = (FRONTEND / "app/provider.tsx").read_text(encoding="utf-8") + assert "max-h-[calc(100dvh_-_2rem)]" in provider, "the cap this has to absorb" + + +@pytest.fixture(scope="module") +def run_scanner(tmp_path_factory): + """Run the frontend's markdown scanners under node. + + Their job is to classify a line the way a CommonMark renderer would, which + only a real run can show. The sources are copied with their "@/lib" aliases + rewritten, because that alias resolves through Vite and not through node.""" + node = shutil.which("node") + if node is None: + pytest.skip("node is needed to run the TypeScript scanners") + work = tmp_path_factory.mktemp("release-notes-scanners") + for source in (PREVIEW, CODE_SPANS, LINKS, LIST_COLUMNS, INLINE_COMMENTS): + rewritten = _TS_ALIAS.sub(r'"./\1.ts"', source.read_text(encoding="utf-8")) + (work / source.name).write_text(rewritten, encoding="utf-8") + (work / "run.ts").write_text(_TS_RUNNER, encoding="utf-8") + + def run(kind: str, markdown: str): + result = subprocess.run( + [node, "--experimental-strip-types", "--no-warnings", str(work / "run.ts"), kind], + input=markdown, + capture_output=True, + text=True, + ) + if result.returncode != 0: + pytest.skip(f"node could not run the scanners: {result.stderr.strip()[:200]}") + return json.loads(result.stdout) + + return run + + +def preview_leads(preview) -> list[str]: + return [item["lead"] for item in preview["items"]] + + +def test_a_link_indented_under_a_bullet_still_resolves(run_scanner): + """CommonMark measures indentation from the container, not the margin + (spec 0.31.2 section 5.2, list items). Under "- Details:" the content column + is 2, so a four-space line is only two columns in: a paragraph holding a + link, which GitHub renders and follows. The scanner measured from the margin + instead, called it an indented code block (section 4.4) and left the + destination relative, so the link resolved against Studio's own origin.""" + resolved = run_scanner("links", "- Details:\n\n [guide](docs/a.md)\n") + assert "https://github.com/unslothai/unsloth/blob/main/docs/a.md" in resolved + # The same prose one column further in really is code, and stays untouched. + code = run_scanner("links", "- Added.\n\n [guide](docs/a.md)\n") + assert "[guide](docs/a.md)" in code and "github.com" not in code + # At document level four spaces is code, so that link is still left alone. + top = run_scanner("links", "Intro.\n\n [guide](docs/a.md)\n") + assert "[guide](docs/a.md)" in top and "github.com" not in top + + +def test_an_indented_fence_does_not_swallow_the_bullets_below_it(run_scanner): + """A four-space line at document level is an indented code block, and a + top-level bullet is not indented enough to continue it, so the block ends + and the list renders. Promoting the line to a list-contained fence left a + block open with no closer, so every bullet after it was skipped and the + collapsed popup lost its summary.""" + swallowed = "Example:\n\n ```\n\n- Added the exporter\n- Fixed the crash\n" + assert preview_leads(run_scanner("preview", swallowed)) == [ + "Added the exporter", + "Fixed the crash", + ] + # With nothing else to fall back on the summary disappeared entirely. + assert preview_leads(run_scanner("preview", " ```\n\n- Added the exporter\n")) == [ + "Added the exporter" + ] + # A fence that really is inside an item still hides that item's code. + nested = "- a\n - b\n ```\n - not a bullet\n ```\n\n- Added tests\n" + assert preview_leads(run_scanner("preview", nested)) == ["a", "Added tests"] + + +def test_a_table_only_release_previews_as_nothing(run_scanner): + """A release written as a GFM table renders as a grid, and the panel treats + notes that preview as nothing by staying collapsed rather than showing an + empty strip. Falling through to the prose collector put the raw + "| Change | Detail | | --- | --- |" delimiters in the popup instead.""" + table = "| Change | Detail |\n| --- | --- |\n| Exporter | Added GGUF |\n" + assert run_scanner("preview", table)["items"] == [] + # A table after prose is dropped too, rather than joined onto it. + assert preview_leads(run_scanner("preview", f"Some prose.\n\n{table}")) == ["Some prose."] + # A bullet right after the rows ends the table, so it still previews. + assert preview_leads(run_scanner("preview", f"{table}- Added tests\n")) == ["Added tests"] + # Mismatched header and delimiter widths are no table, as on GitHub, so both lines are prose. + assert preview_leads(run_scanner("preview", "| a | b |\n| --- |\n")) == ["| a | b | | --- |"] + + +def test_a_fence_inside_a_list_item_ends_with_the_item(changelog_module): + """A fence is scoped to its container: with no closer it runs to the end of + the containing block, not the document (spec 0.31.2 section 4.5). A + dedented "## 2.0" closes the list item, so it is a real release heading. + Document-wide fence state kept the block open and hid every release below + it, so one missing closing line emptied the rest of the changelog.""" + text = "## 1.0\n\n- item\n ```\n\n## 2.0\n\n- two\n" + assert [e.version for e in changelog_module.parse_changelog(text)] == ["1.0", "2.0"] + # A fence at document level still runs to the end of the file. + top = "## 1.0\n\n```\n\n## 2.0\n\n- two\n" + assert [e.version for e in changelog_module.parse_changelog(top)] == ["1.0"] + # A closed fence inside an item is unaffected, and its sample stays hidden. + closed = "## 1.0\n\n- Run:\n ```bash\n ## 9.9.9\n ```\n\n## 2.0\n\n- two\n" + assert [e.version for e in changelog_module.parse_changelog(closed)] == ["1.0", "2.0"] + # Content dedented out of the item ends the item and the fence with it. + assert changelog_module.find_release_notes(text, "2.0").body == "- two" + + +def test_stripping_comments_stays_linear_in_the_code_spans(changelog_module): + """The comment scanner restarted its code-span search at the first span for + every opener, so a line of N spans and N openers cost N squared. A 203 KiB + line is well inside the 2 MiB the fetcher accepts, and notes are reparsed on + every request, so one such line held a worker for over ten seconds.""" + line = "`a` <!--x--> " * 16_000 + assert len(line) < changelog_module.CHANGELOG_MAX_BYTES + started = time.monotonic() + visible, in_comment = changelog_module._strip_comments(line, False, False) + elapsed = time.monotonic() - started + # Roughly 40ms scanning forward against roughly 11s restarting each time. + assert elapsed < 2.0, f"comment stripping took {elapsed:.1f}s" + # Same result as before: the spans survive and the comments are gone. + assert in_comment is False + assert "<!--" not in visible and visible.count("`a`") == 16_000 + + +def test_the_three_scanners_share_one_list_column_rule(): + """The parser and both frontend scanners have to classify a line the same + way, and drifting apart on indentation is what put a paragraph link inside a + code block. The frontend pair reads its list columns from one module, ported + from the backend's own tracker.""" + shared = LIST_COLUMNS.read_text(encoding="utf-8") + assert "export function openLists(" in shared + assert "_open_lists" in shared, "the backend function this mirrors" + for source in (PREVIEW, LINKS): + src = source.read_text(encoding="utf-8") + assert 'from "@/lib/markdown-list-columns"' in src + assert "openLists(" in src + # Both sides measure indented code from the container, not from the margin. + backend = (BACKEND / "utils" / "changelog.py").read_text(encoding="utf-8") + assert "_indent_width(visible) - column >= 4" in backend + assert "indentWidth(structure) - column >= INDENTED_CODE_INDENT" in LINKS.read_text( + encoding="utf-8" + ) + + +def test_a_failed_fetch_keeps_retry_reachable(): + """The fallback stands in for "no section for this version", which the hook + reports as ready. A failed fetch is reported as error and is retryable, and on + desktop the fallback is the updater's static install blurb, so taking it there + replaced the Retry button with generic text until the cache expired.""" + src = " ".join(PANEL.read_text(encoding="utf-8").split()) + assert 'notes?.matched ? notes.markdown : state === "error" ? null' in src + # Only NotesStatus renders retry, in the else of the markdown branch: an error has no markdown. + assert "{markdown ? (" in src + assert "retry={retry}" in src + + hook = " ".join( + (FRONTEND / "hooks" / "use-release-notes.ts").read_text(encoding="utf-8").split() + ) + assert ( + "const failed = !next || (!next.matched && next.error !== null);" in hook + ), "the distinction this relies on" + + +def test_an_unclosed_comment_in_prose_cannot_hide_later_links(run_scanner): + """CommonMark opens an HTML block (spec 0.31.2 section 4.6, type 2) only + when the line itself begins with `<!--`; one written mid-sentence is inline + raw HTML and cannot outlive the block it sits in. The link resolver carried + the unclosed state to every following line instead, so a note that merely + mentions the delimiter masked the relative links under it and they resolved + against Studio's own origin.""" + repo = "https://github.com/unslothai/unsloth/blob/main/docs/a.md" + # A separate list item is a separate block, so the link below still renders. + item = run_scanner("links", "- Type <!-- to begin a comment\n- See [docs](docs/a.md)\n") + assert repo in item + # So does a paragraph the blank line already ended. + paragraph = run_scanner("links", "Type <!-- to begin\n\nSee [docs](docs/a.md)\n") + assert repo in paragraph + # A delimiter inside inline code is literal, as it is for the parser. + spanned = run_scanner("links", "- Wrap in `<!--` and `-->`\n- See [docs](docs/a.md)\n") + assert repo in spanned + # A comment starting a line is a block: it hides down to the closer's line, that line included. + block = run_scanner("links", "<!-- staged\n- See [docs](docs/a.md)\n-->\n") + assert repo not in block + closer = run_scanner("links", "<!-- staged\n--> See [docs](docs/a.md)\n") + assert repo not in closer + + +def test_a_bare_level_two_marker_ends_the_release(changelog_module, run_scanner): + """An ATX heading's opening sequence may be followed by the end of the line + (spec 0.31.2 section 4.2), so a bare `##` is an empty level-two heading. The + scanners required whitespace after the hashes, so everything below such a + line stayed inside the release above it and the popup showed unrelated notes + under that version.""" + text = "## 2.0\n\n- new thing\n\n##\n\n- SECRET: not part of 2.0\n" + entry = changelog_module.find_release_notes(text, "2.0") + assert "new thing" in entry.body + assert "SECRET" not in entry.body + # An empty heading has no version, so it ends a release without indexing one. + assert [e.version for e in changelog_module.parse_changelog(text)] == ["2.0"] + # Prose still needs a space or a tab: `##x` is a paragraph, not a heading. + prose = "## 2.0\n\n- new thing\n\n##x\n\n- still 2.0\n" + assert "still 2.0" in changelog_module.find_release_notes(prose, "2.0").body + # The preview agrees: an empty heading renders as nothing, so it ends the bullet. + preview = run_scanner("preview", "- new thing\n##\nUnrelated scratch notes\n") + assert preview_leads(preview) == ["new thing"] + + +def test_a_comment_between_bullets_closes_the_list(changelog_module, run_scanner): + """A comment is an HTML block (spec 0.31.2 section 4.6, type 2), so one + written at the margin under a bullet is not indented enough to continue that + item and closes the list. The scanners blanked the line before list tracking + saw it, which reads as a blank line and leaves the item open, so the release + heading below it looked like nested item content and the new release was + merged into the one above.""" + text = "## 1.0\n\n- old item\n<!-- separator -->\n ## 2.0\n\n- new item\n" + assert [e.version for e in changelog_module.parse_changelog(text)] == ["1.0", "2.0"] + assert "new item" not in changelog_module.find_release_notes(text, "1.0").body + assert "new item" in changelog_module.find_release_notes(text, "2.0").body + # At the item's content column the comment stays inside it, so the heading under it is nested. + nested = "## 1.0\n\n- old item\n <!-- separator -->\n ## 2.0\n\n- new item\n" + assert [e.version for e in changelog_module.parse_changelog(nested)] == ["1.0"] + # The link resolver reads the same column: list closed, four spaces is code, left untouched. + code = run_scanner("links", "- old item\n<!-- separator -->\n [guide](docs/a.md)\n") + assert "[guide](docs/a.md)" in code and "github.com" not in code + # Inside the item those four spaces are two columns in, so it is prose and the link resolves. + prose = run_scanner("links", "- old item\n <!-- separator -->\n [guide](docs/a.md)\n") + assert "https://github.com/unslothai/unsloth/blob/main/docs/a.md" in prose + # The preview agrees: the fence is indented code, not a fence swallowing the bullet below. + preview = run_scanner( + "preview", + "- Details:\n<!-- separator -->\n ```\n - hidden sample\n- Real second item\n", + ) + assert preview_leads(preview) == ["Details:", "Real second item"] + + +def test_a_parenthesised_link_destination_still_resolves(run_scanner): + """A destination may hold parentheses while they balance (spec 0.31.2 + section 6.3), so `[x]((draft).md)` points at `(draft).md`. The resolver's + destination expression stopped at the first paren, matched an empty + destination and left the markdown alone, so the link resolved against + Studio's own origin instead of the repository.""" + leading = run_scanner("links", "[details]((draft).md)\n") + assert "https://github.com/unslothai/unsloth/blob/main/(draft).md" in leading + # An image resolves against the raw host the same way. + image = run_scanner("links", "![shield]((badge).png)\n") + assert "https://raw.githubusercontent.com/unslothai/unsloth/main/(badge).png" in image + # A pair in the middle of a path balances too. + middle = run_scanner("links", "[api](docs/(v2)/api.md)\n") + assert "https://github.com/unslothai/unsloth/blob/main/docs/(v2)/api.md" in middle + # An unbalanced paren makes the destination invalid, so `[x](a(b.md)` is plain text, not a link. + unbalanced = run_scanner("links", "[x](a(b.md)\n") + assert unbalanced == "[x](a(b.md)\n" + # One more closer balances the pair, and then it is a link again. + closed = run_scanner("links", "[x](a(b.md))\n") + assert "https://github.com/unslothai/unsloth/blob/main/a(b.md)" in closed + # Pairs nest, and one level was all the expression allowed, so a path with two stayed relative. + nested = run_scanner("links", "[x](((draft)).md)\n") + assert "https://github.com/unslothai/unsloth/blob/main/((draft)).md" in nested + deep = run_scanner("links", "![shot](((((v2))))).png)\n") + assert "https://raw.githubusercontent.com/unslothai/unsloth/main/((((v2))))" in deep + # The closer must still be there: an unbalanced run below a nested pair is not a link. + across = run_scanner("links", "[x](((a).md\n[y](docs/y.md)\n") + assert "https://github.com/unslothai/unsloth/blob/main/docs/y.md" in across + assert "[x](((a).md" in across + + +def test_a_fence_inside_a_container_still_hides_its_sample(run_scanner): + """A fence is measured from its container and not from the margin (spec + 0.31.2 section 4.5), so `> ~~~` and a fence three columns under a nested + bullet open one. Reading the margin instead never saw them, so the sample + inside was treated as prose and a relative link written in a code block was + rewritten into the text the reader sees verbatim.""" + quoted = run_scanner("links", "> ~~~\n> [guide](docs/a.md)\n> ~~~\n") + assert "[guide](docs/a.md)" in quoted and "github.com" not in quoted + nested = run_scanner("links", "- a\n - b\n ~~~\n [x](docs/x.md)\n ~~~\n") + assert "[x](docs/x.md)" in nested and "github.com" not in nested + # A longer closer is still a closer, so the pair is not something a code span hid. + uneven = run_scanner("links", "> ```\n> [guide](docs/a.md)\n> ````\n") + assert "[guide](docs/a.md)" in uneven and "github.com" not in uneven + # The fence ends with its container: a line outside the quote, or left of the item, is Markdown. + left = run_scanner("links", "> ~~~\n[guide](docs/a.md)\n") + assert "https://github.com/unslothai/unsloth/blob/main/docs/a.md" in left + dedented = run_scanner("links", "- a\n ~~~\n[guide](docs/a.md)\n") + assert "https://github.com/unslothai/unsloth/blob/main/docs/a.md" in dedented + # A document-level fence owns the quoted lines below, so the marker does not undo it. + document = run_scanner("links", "~~~\n> [guide](docs/a.md)\n~~~\n") + assert "[guide](docs/a.md)" in document and "github.com" not in document + # Four columns past the item's content column it is indented code, not a fence: still literal. + code = run_scanner("links", "- Details:\n\n ~~~\n [guide](docs/a.md)\n") + assert "[guide](docs/a.md)" in code and "github.com" not in code + + +def test_an_html_block_inside_a_container_is_literal_too(run_scanner): + """Type 1 and type 6 blocks are measured from their container the same way, + so a `<details>` under a nested bullet and a `<pre>` inside a quote both + show their contents verbatim. Missing the opener treated the body as + Markdown and rewrote the literal examples in it.""" + nested = run_scanner("links", "- a\n - b\n <details>\n [x](docs/x.md)\n </details>\n") + assert "[x](docs/x.md)" in nested and "github.com" not in nested + quoted = run_scanner("links", "> <pre>\n> [x](docs/x.md)\n> </pre>\n") + assert "[x](docs/x.md)" in quoted and "github.com" not in quoted + # The block ends with its container, so a line dedented out of the item is Markdown again. + dedented = run_scanner("links", "- a\n - b\n <details>\n[x](docs/x.md)\n") + assert "https://github.com/unslothai/unsloth/blob/main/docs/x.md" in dedented + # Inside a quote a bare marker holds nothing, the blank line that ends a type 6 block. + blank = run_scanner("links", "> <details>\n>\n> [x](docs/x.md)\n") + assert "https://github.com/unslothai/unsloth/blob/main/docs/x.md" in blank + + +def test_an_underline_left_of_an_item_is_lazy_text_of_it(changelog_module, run_scanner): + """A setext underline may never be a lazy continuation line (spec 0.31.2 + section 4.3), so `===` written left of an open list item is read as more of + the item's paragraph rather than as a block that closes it. Rejecting every + underline-shaped line ended the list there, which promoted the nested + "## 2.0" below it to a document-level heading and indexed a release the + renderer never shows.""" + nested = "## 1.0\n- old note\n===\n ## 2.0\n- new\n" + assert [e.version for e in changelog_module.parse_changelog(nested)] == ["1.0"] + # A row of dashes is a thematic break, closing the item, so the heading is the next release. + broken = "## 1.0\n- old note\n---\n ## 2.0\n" + assert [e.version for e in changelog_module.parse_changelog(broken)] == ["1.0", "2.0"] + # With no paragraph above it the underline opens one, so the blank line closes the item. + apart = "## 1.0\n- old note\n\n===\n ## 2.0\n" + assert [e.version for e in changelog_module.parse_changelog(apart)] == ["1.0", "2.0"] + # The link scanner keeps the item open, so the four-space line is a paragraph and resolves. + resolved = run_scanner("links", "- Details:\n===\n\n [guide](docs/a.md)\n") + assert "https://github.com/unslothai/unsloth/blob/main/docs/a.md" in resolved + + +def test_a_quote_keeps_its_paragraph_to_itself(changelog_module, run_scanner): + """Lazy continuation runs the other way too: a marker written outside a + blockquote is not text of the quote's paragraph, so `2. item` under + `> quote` opens a list even though an ordered marker past 1 may not + interrupt a paragraph (spec 0.31.2 section 5.2). Lending the quote's + paragraph to the document left the list closed, so the heading indented to + the item's content column read as a release of its own.""" + quoted = "## 1.0\n> quote\n2. item\n ## 2.0\n- new\n" + assert [e.version for e in changelog_module.parse_changelog(quoted)] == ["1.0"] + # A quote holding a heading leaves no paragraph, nor does an empty one, so the list opens. + heading = "## 1.0\n> # inner\n2. item\n ## 2.0\n" + assert [e.version for e in changelog_module.parse_changelog(heading)] == ["1.0"] + # An unquoted line the quote's paragraph swallows keeps it open, the marker still outside. + lazy = "## 1.0\n> quote\ntext\n2. item\n ## 2.0\n" + assert [e.version for e in changelog_module.parse_changelog(lazy)] == ["1.0"] + # Under an ordinary paragraph the marker is its text, so no list opens and the heading is real. + prose = "## 1.0\nprose\n2. item\n ## 2.0\n" + assert [e.version for e in changelog_module.parse_changelog(prose)] == ["1.0", "2.0"] + # The preview reads the marker as a bullet for the same reason. + assert preview_leads(run_scanner("preview", "> quote\n2. item\n")) == ["item"] + + +def test_indented_code_before_an_ordered_marker_still_opens_a_list(changelog_module): + """An indented code block ends at the first line that is not indented enough + to continue it, and no paragraph is open for the marker below to continue, + so `2. item` opens a list whatever its start number. Reading it as text of + the code block instead would leave the list closed and index the heading at + the item's content column as a release.""" + joined = "## 1.0\n\n code\n2. item\n ## 2.0\n- new\n" + assert [e.version for e in changelog_module.parse_changelog(joined)] == ["1.0"] + # A blank line between the two changes nothing: the list opens either way. + apart = "## 1.0\n\n code\n\n2. item\n ## 2.0\n- new\n" + assert [e.version for e in changelog_module.parse_changelog(apart)] == ["1.0"] + # Four columns past its container the marker is code, so no list opens and the heading stands. + inside = "## 1.0\n\n code\n - item\n ## 2.0\n" + assert [e.version for e in changelog_module.parse_changelog(inside)] == ["1.0", "2.0"] + + +def test_a_fence_written_as_an_item_first_content_opens_in_that_item(run_scanner): + """A block written straight after a list marker is the item's own first + content, measured from the column that content starts (spec 0.31.2 section + 5.2), so "- ```md" opens a fence. Reading the whole line instead never saw + one, so the code sample below it was treated as prose: the resolver rewrote + a destination the reader sees verbatim, and the preview offered the info + string as a headline bullet.""" + sample = run_scanner("links", "- ```md\n [example](docs/a.md)\n ```\n") + assert "[example](docs/a.md)" in sample and "github.com" not in sample + ordered = run_scanner("links", "1. ~~~\n [example](docs/a.md)\n ~~~\n") + assert "[example](docs/a.md)" in ordered and "github.com" not in ordered + # The preview agrees: an item of only a code block previews as nothing; the next is a bullet. + preview = run_scanner("preview", "- ```md\n sample text\n ```\n- Added tests\n") + assert preview_leads(preview) == ["Added tests"] + # One column further in it is indented code inside the item, so the link is prose and resolves. + padded = run_scanner("links", "- ```\n [example](docs/a.md)\n") + assert "https://github.com/unslothai/unsloth/blob/main/docs/a.md" in padded + # A marker the paragraph above swallows opens no item, so no fence: ordered items open at 1. + lazy = run_scanner("links", "Intro.\n2. ```\n[guide](docs/a.md)\n") + assert "https://github.com/unslothai/unsloth/blob/main/docs/a.md" in lazy + + +def test_an_html_block_ends_with_the_item_it_was_written_in(changelog_module, run_scanner): + """An HTML block holds no lazy continuation line, so one opened on a list + item's continuation line ends where the item does, exactly as a fence there + does. Ending it only on a blank line let it run past the item and swallow + the next release heading, so those notes could never be found, and the + collapsed preview lost every bullet below it.""" + text = "## 1.0\n\n- item\n\n <div>\n## 2.0\n\n- new thing\n" + assert [e.version for e in changelog_module.parse_changelog(text)] == ["1.0", "2.0"] + assert "new thing" in changelog_module.find_release_notes(text, "2.0").body + # A raw block such as <pre> is scoped the same way. + raw = "## 1.0\n\n- item\n\n <pre>\n## 2.0\n\n- new thing\n" + assert [e.version for e in changelog_module.parse_changelog(raw)] == ["1.0", "2.0"] + # At the item's content column the block holds the heading, which is nested and indexes nothing. + nested = "## 1.0\n\n- item\n\n <div>\n ## 2.0\n" + assert [e.version for e in changelog_module.parse_changelog(nested)] == ["1.0"] + # The preview reads it the same way: the bullet below the block is a bullet. + preview = run_scanner("preview", "- item\n\n <div>\n- Added tests\n") + assert preview_leads(preview) == ["item", "Added tests"] + # An opener straight after a marker opens in that item, so the dedented heading is a release. + marked = "## 1.0\n\n- <div>\n## 2.0\n\n- new thing\n" + assert [e.version for e in changelog_module.parse_changelog(marked)] == ["1.0", "2.0"] + + +def test_a_comment_may_close_on_a_later_line_of_its_paragraph(run_scanner): + """A comment written mid-sentence is inline raw HTML belonging to the + paragraph around it, so its `-->` may arrive on a later line of that same + paragraph and everything between renders as nothing. Ending the comment at + its own line left a backtick inside it pairing with a real one below, which + hid a following link from the resolver, and left the collapsed preview + quoting text the popup body does not show.""" + carried = run_scanner("links", "Note <!-- ` open\nstill --> see [d](docs/a.md) and `x`\n") + assert "https://github.com/unslothai/unsloth/blob/main/docs/a.md" in carried + # Text inside the comment renders as nothing, so it is left alone. + inside = run_scanner("links", "Note <!-- see [c](docs/c.md)\nmore --> end\n") + assert "[c](docs/c.md)" in inside and "github.com" not in inside + # The preview hides it too, rather than quoting the comment at the reader. + preview = run_scanner( + "preview", "- Added X <!-- TODO: rewrite\n this properly -->\n- Second\n" + ) + assert preview_leads(preview) == ["Added X", "Second"] + # An opener cannot outlive its paragraph: with it closed the `<!--` is text and hides nothing. + broken = run_scanner("links", "Note <!-- open\n\nsecret --> end [d](docs/a.md)\n") + assert "https://github.com/unslothai/unsloth/blob/main/docs/a.md" in broken + # A heading breaks into the paragraph, so it ends the comment's reach too. + headed = run_scanner("links", "Note <!-- open\n## 2.0 --> end [d](docs/a.md)\n") + assert "https://github.com/unslothai/unsloth/blob/main/docs/a.md" in headed + assert preview_leads(run_scanner("preview", "Note <!-- open\n\n- Second\n")) == ["Second"] + + +def test_only_punctuation_is_escapable_in_a_link_destination(run_scanner): + """CommonMark escapes ASCII punctuation and nothing else (spec 0.31.2 + section 2.4), so the backslash in `docs\\alpha.md` is a character of the + path. Dropping every backslash rewrote it to a path that does not exist, + and a URL parser reads what is left as a separator, so a Windows or + namespaced path pointed at the wrong file either way.""" + kept = run_scanner("links", "[guide](docs\\alpha.md)\n") + assert "https://github.com/unslothai/unsloth/blob/main/docs%5Calpha.md" in kept + # An escaped backslash is one literal backslash, which survives the same. + escaped = run_scanner("links", "[guide](docs\\\\alpha.md)\n") + assert "https://github.com/unslothai/unsloth/blob/main/docs%5Calpha.md" in escaped + # A real escape is still an escape: `\\(` is a paren of the path. + paren = run_scanner("links", "[guide](a\\(b.md)\n") + assert "https://github.com/unslothai/unsloth/blob/main/a(b.md" in paren + # A space still ends the destination, escaped or not, so there is no link. + spaced = run_scanner("links", "[guide](a\\ b.md)\n") + assert spaced == "[guide](a\\ b.md)\n" + + +def test_one_definition_does_not_hide_the_next(run_scanner): + """Definitions may run consecutively (spec 0.31.2 section 4.7): a block of + them is how a changelog collects its link targets. A definition is not + paragraph text, so it opens no paragraph for the next one to be unable to + interrupt. The resolver counted one as prose, which left every definition + after the first outside the set of lines a definition may start on, so only + the first was rewritten and the rest resolved against Studio's own origin. + The backend already reads the line this way.""" + text = ( + "- AMD support is here, see [the AMD guide][amd] and the\n" + " [Intel notes][xpu].\n\n" + "[amd]: docs/basics/amd.md\n" + "[xpu]: docs/basics/xpu.md\n" + ) + resolved = run_scanner("links", text) + base = "https://github.com/unslothai/unsloth/blob/main/docs/basics/" + assert f"[amd]: {base}amd.md" in resolved + assert f"[xpu]: {base}xpu.md" in resolved + # A run of them stays a run however long it is. + run = run_scanner("links", "[a]: docs/a.md\n[b]: docs/b.md\n[c]: docs/c.md\n") + assert run.count("https://github.com/unslothai/unsloth/blob/main/docs/") == 3 + # Prose between them opens a paragraph the next line may not interrupt, so it is not one. + prose = run_scanner("links", "[a]: docs/a.md\nintro\n[b]: docs/b.md\n") + assert "[b]: docs/b.md" in prose + + +def test_a_comment_closed_on_its_own_line_still_closes(run_scanner): + """A multiline comment is ordinarily closed by a `-->` written on a line of + its own, and a wrapped line may open with emphasis. The guard asking whether + the closer is reachable read any line whose first character was punctuation + as the start of a new block, so neither shape counted as more of the + paragraph carrying the comment. The comment then never closed, and the + collapsed popup showed the author's internal note to the user.""" + closer = run_scanner( + "preview", + "- DoRA training is available in Studio. <!-- TODO confirm the exact\n" + " flag name before release\n-->\n", + ) + assert preview_leads(closer) == ["DoRA training is available in Studio."] + # A continuation may open with emphasis, which is text and not a block. + starred = run_scanner( + "preview", + "- DoRA training is available. <!-- TODO confirm the\n *before* release -->\n", + ) + assert preview_leads(starred) == ["DoRA training is available."] + underscored = run_scanner( + "preview", + "- DoRA training is available. <!-- TODO confirm the\n _draft_ note -->\n", + ) + assert preview_leads(underscored) == ["DoRA training is available."] + # A real block still ends the paragraph, so the opener below one is text and hides nothing. + broken = run_scanner("links", "Note <!-- open\n## 2.0\nsecret --> [d](docs/a.md)\n") + assert "https://github.com/unslothai/unsloth/blob/main/docs/a.md" in broken + # So does a list item with content, which may interrupt a paragraph. + item = run_scanner("links", "Note <!-- open\n- bullet\nsecret --> [d](docs/a.md)\n") + assert "https://github.com/unslothai/unsloth/blob/main/docs/a.md" in item + + +def test_a_comment_written_as_an_item_first_content_is_a_block(changelog_module, run_scanner): + """A comment is an HTML block (spec 0.31.2 section 4.6, type 2), so one + written as a list item's first content opens inside that item, exactly as a + fence written there does. The scanners looked for the opener at the margin + of the line as written, so a marker in front of it hid the block: the + resolver rewrote a destination inside raw HTML, which Streamdown then shows + the reader as a literal URL, and the preview quoted the hidden note back at + them as though the bullet were Markdown.""" + item = run_scanner("links", "- <!-- new --> AMD support, see [the guide](docs/amd.md)\n") + assert item == "- <!-- new --> AMD support, see [the guide](docs/amd.md)\n" + # Every marker opens an item, and a nested one is still an item. + for text in ( + "* <!-- new --> see [the guide](docs/amd.md)\n", + "1. <!-- new --> see [the guide](docs/amd.md)\n", + "- outer\n - <!-- new --> see [the guide](docs/amd.md)\n", + ): + assert "github.com" not in run_scanner("links", text) + # The multiline form hides lines to the closer, as a comment at the item's content column did. + multiline = run_scanner("links", "- <!-- hidden\n [a](docs/x.md)\n -->\n") + assert "[a](docs/x.md)" in multiline and "github.com" not in multiline + # Still scoped to the item it was written in, so a line dedented out of it ends the block. + dedented = run_scanner("links", "- <!-- hidden\n[a](docs/x.md)\n") + assert "https://github.com/unslothai/unsloth/blob/main/docs/x.md" in dedented + # The preview agrees: an item of only the block previews as nothing; the next is a bullet. + preview = run_scanner("preview", "- <!-- new --> hidden note\n- Real bullet\n") + assert preview_leads(preview) == ["Real bullet"] + # The parser agrees too: the item keeps its column, so a heading inside is nested, not indexed. + text = "## 1.0\n\n- <!-- hidden\n\n ## 2.0\n" + assert [e.version for e in changelog_module.parse_changelog(text)] == ["1.0"] diff --git a/tests/test_profile_startup_gate.py b/tests/test_profile_startup_gate.py new file mode 100644 index 0000000000..66e6e16a89 --- /dev/null +++ b/tests/test_profile_startup_gate.py @@ -0,0 +1,243 @@ +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. + +"""Regression coverage for the startup profiler's budget gate, teardown and triggers.""" + +from __future__ import annotations + +import ast +import fnmatch +import importlib.util +import re +import subprocess +import sys +from pathlib import Path +from types import SimpleNamespace + +import pytest +import yaml + +REPO_ROOT = Path(__file__).resolve().parents[1] +SCRIPT = REPO_ROOT / "scripts" / "profile_startup.py" +WORKFLOW = REPO_ROOT / ".github" / "workflows" / "startup-profile-ci.yml" +PROCESS_RS = REPO_ROOT / "studio" / "src-tauri" / "src" / "process.rs" + +# Checkout files that build the venv the workflow profiles. +INSTALLER_INPUTS = ( + "studio/setup.sh", + "studio/setup.ps1", + "studio/install_python_stack.py", +) +# Checkout file that defines the argv the profiler reproduces. +LAUNCH_INPUTS = ("studio/src-tauri/src/process.rs",) + + +def _load(): + spec = importlib.util.spec_from_file_location("profile_startup", SCRIPT) + mod = importlib.util.module_from_spec(spec) + spec.loader.exec_module(mod) + return mod + + +def _no_subprocesses(mod, monkeypatch): + # Keep the gate tests off the real interpreter and CLI. + monkeypatch.setattr(mod, "find_bin", lambda: None) + monkeypatch.setattr(mod, "profile_imports", lambda python, top = 15: {"ok": False, "error": ""}) + monkeypatch.setattr(mod, "python_version_of", lambda python: "3.13.0") + + +class _Proc: + """Stand-in for a still-running Popen.""" + + def __init__(self): + self.pid = 4321 + self.terminated = False + + def poll(self): + return None + + def terminate(self): + self.terminated = True + + +def _nt(mod, monkeypatch, returncode): + calls: list[list[str]] = [] + + def _run(argv, **kwargs): + calls.append(argv) + return subprocess.CompletedProcess(argv, returncode, "", "") + + # Patch the module's own references, not the real os/subprocess the session shares. + monkeypatch.setattr(mod, "os", SimpleNamespace(name = "nt")) + monkeypatch.setattr(mod, "subprocess", SimpleNamespace(run = _run)) + return calls + + +def test_budget_fails_when_no_launch_was_measured(capsys, monkeypatch): + """A requested budget must not pass just because the CLI was never found.""" + mod = _load() + _no_subprocesses(mod, monkeypatch) + rc = mod.main(["--max-healthz-seconds", "30"]) + out = capsys.readouterr().out + assert rc == 1 + assert "::error::" in out and "no healthz measurement" in out + assert "no unsloth CLI found" in out + + +def _healthy_launch( + mod, + monkeypatch, + healthz = 1.5, +): + monkeypatch.setattr(mod, "find_bin", lambda: "unsloth") + monkeypatch.setattr( + mod, + "profile_launch", + lambda bin_path, port, **kw: { + "spawn_seconds": 0.1, + "healthz_seconds": healthz, + "lifespan_ms": 100.0, + "reached_healthz": True, + "log_tail": [], + }, + ) + + +def test_budget_still_passes_when_a_launch_was_measured(monkeypatch): + """The fail-closed branch must not swallow a genuinely healthy run.""" + mod = _load() + _no_subprocesses(mod, monkeypatch) + _healthy_launch(mod, monkeypatch) + assert mod.main(["--max-healthz-seconds", "30"]) == 0 + assert mod.main(["--max-healthz-seconds", "1"]) == 1 + + +# "=" form for -inf: a bare "-inf" is an option token to argparse, not a value. +@pytest.mark.parametrize( + "bad", ["--max-healthz-seconds=nan", "--max-healthz-seconds=inf", "--max-healthz-seconds=-inf"] +) +def test_budget_rejects_non_finite_values(bad, capsys, monkeypatch): + """`med > nan` and `med > inf` are always False, so the gate would never bind.""" + mod = _load() + _no_subprocesses(mod, monkeypatch) + _healthy_launch(mod, monkeypatch) + with pytest.raises(SystemExit) as exc: + mod.main([bad]) + assert exc.value.code == 2 + assert "finite" in capsys.readouterr().err + + +def test_budget_rejects_import_only(capsys): + """--import-only launches nothing, so a budget on it could only ever pass.""" + mod = _load() + with pytest.raises(SystemExit) as exc: + mod.main(["--import-only", "--max-healthz-seconds", "30"]) + assert exc.value.code == 2 + assert "--import-only" in capsys.readouterr().err + + +def test_terminate_tree_falls_back_when_taskkill_fails(monkeypatch): + """A nonzero taskkill must still reach terminate(), not return silently.""" + mod = _load() + calls = _nt(mod, monkeypatch, returncode = 1) + proc = _Proc() + mod._terminate_tree(proc) + assert calls == [["taskkill", "/PID", "4321", "/T", "/F"]] + assert proc.terminated + + +def test_terminate_tree_falls_back_when_taskkill_raises(monkeypatch): + """A missing or hung taskkill must reach terminate() too.""" + mod = _load() + monkeypatch.setattr(mod, "os", SimpleNamespace(name = "nt")) + + def _boom(argv, **kwargs): + raise FileNotFoundError(argv) + + monkeypatch.setattr(mod, "subprocess", SimpleNamespace(run = _boom)) + proc = _Proc() + mod._terminate_tree(proc) + assert proc.terminated + + +def test_terminate_tree_returns_on_successful_taskkill(monkeypatch): + mod = _load() + _nt(mod, monkeypatch, returncode = 0) + proc = _Proc() + mod._terminate_tree(proc) + assert not proc.terminated + + +def test_terminate_tree_skips_an_exited_process(monkeypatch): + mod = _load() + calls = _nt(mod, monkeypatch, returncode = 0) + proc = _Proc() + proc.poll = lambda: 0 + mod._terminate_tree(proc) + assert calls == [] and not proc.terminated + + +def _trigger_paths(): + wf = yaml.safe_load(WORKFLOW.read_text(encoding = "utf-8")) + # YAML 1.1 turns the bare `on:` key into True. + on = wf.get("on") or wf[True] + return [p for p in on["pull_request"]["paths"] if not p.startswith("!")] + + +@pytest.mark.parametrize("rel", INSTALLER_INPUTS) +def test_workflow_triggers_on_studio_installer_inputs(rel): + """A setup script that changes the profiled venv must schedule a measurement.""" + assert (REPO_ROOT / rel).is_file(), f"{rel} moved; revisit the trigger list" + paths = _trigger_paths() + assert any(fnmatch.fnmatch(rel, p) for p in paths), f"{rel} not covered by {paths}" + + +def test_studio_installer_inputs_are_on_the_local_install_path(): + """Anchor the list above: these files are what --local actually executes.""" + # install.ps1 reaches setup.ps1 through the editable install, not by name. + assert "studio/setup.sh" in (REPO_ROOT / "install.sh").read_text(encoding = "utf-8") + for setup in ("studio/setup.sh", "studio/setup.ps1"): + text = (REPO_ROOT / setup).read_text(encoding = "utf-8", errors = "replace") + assert "install_python_stack.py" in text + + +@pytest.mark.parametrize("rel", LAUNCH_INPUTS) +def test_workflow_triggers_on_the_desktop_launch_command(rel): + """The profiler copies process.rs's argv, so a change there must be measured.""" + assert (REPO_ROOT / rel).is_file(), f"{rel} moved; revisit the trigger list" + paths = _trigger_paths() + assert any(fnmatch.fnmatch(rel, p) for p in paths), f"{rel} not covered by {paths}" + + +def _desktop_backend_argv(): + body = re.search( + r"fn backend_args\(port: u16\) -> Vec<String> \{(.*?)\n\}", + PROCESS_RS.read_text(encoding = "utf-8"), + re.S, + ) + assert body, "backend_args moved; revisit the trigger list" + return re.findall(r'"([^"]+)"', body.group(1)) + + +def _profiler_argv(): + tree = ast.parse(SCRIPT.read_text(encoding = "utf-8")) + fn = next( + n for n in ast.walk(tree) if isinstance(n, ast.FunctionDef) and n.name == "profile_launch" + ) + call = next( + n for n in ast.walk(fn) if isinstance(n, ast.Call) and ast.unparse(n.func).endswith("Popen") + ) + return [e.value for e in call.args[0].elts if isinstance(e, ast.Constant)] + + +def test_profiler_spawns_the_desktop_backend_argv(): + """Anchor the trigger above: these two argv lists must stay identical.""" + assert _profiler_argv() == _desktop_backend_argv() + + +@pytest.mark.skipif(sys.platform == "win32", reason = "posix branch") +def test_terminate_tree_posix_uses_terminate(): + mod = _load() + proc = _Proc() + mod._terminate_tree(proc) + assert proc.terminated