Merge remote-tracking branch 'origin/main' into r7551

This commit is contained in:
Daniel Han 2026-07-29 05:32:38 +00:00
commit 94056e0c09
105 changed files with 10016 additions and 224 deletions

View file

@ -766,6 +766,7 @@ jobs:
env:
GH_REPO: ${{ github.repository }}
APP_VERSION: ${{ needs.prepare-version.outputs.app_version }}
PYPI_VERSION: ${{ needs.prepare-version.outputs.pypi_version }}
STUDIO_VERSION: ${{ needs.prepare-version.outputs.studio_version }}
DESKTOP_RELEASE_TAG: ${{ needs.prepare-version.outputs.desktop_release_tag }}
DESKTOP_PRERELEASE: ${{ needs.prepare-version.outputs.prerelease }}
@ -911,6 +912,8 @@ jobs:
notes = pathlib.Path(os.environ['RUNNER_TEMP'], 'desktop-release-notes.md').read_text()
metadata = {
'version': os.environ['APP_VERSION'],
# App version is SemVer; CHANGELOG.md is keyed by the backend release.
'pypi_version': os.environ['PYPI_VERSION'],
'notes': notes,
'pub_date': datetime.datetime.now(datetime.timezone.utc).isoformat(timespec='milliseconds').replace('+00:00', 'Z'),
'platforms': {

156
.github/workflows/startup-profile-ci.yml vendored Normal file
View file

@ -0,0 +1,156 @@
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
# Measures where Studio's startup time goes, on each platform.
#
# Nothing recorded a number before: main.py logs "lifespan startup completed in X ms"
# and studio_test_kit polls /healthz, but both throw the elapsed time away. A first
# local run (Linux, warm cache, 18-core server) put `import main` at 5.7-6.6s BEFORE
# the server can bind, dominated by eager module-level imports pulled in by routes:
# torch ~1.9s self, unsloth_zoo ~0.8s, routes ~0.6s, transformers ~0.5s.
#
# Not a gate yet: --max-healthz-seconds exists, but a budget should come from
# observed numbers rather than a guess.
name: Startup profile
on:
pull_request:
paths:
# The measured import graph is the whole backend tree: main.py imports auth,
# core, hub, loggers, models, picker, routes and utils at module scope.
- 'studio/backend/**'
- '!studio/backend/tests/**'
# The launch phase spawns `unsloth studio --api-only`, so the CLI counts too.
- 'unsloth_cli/**'
- 'studio/src-tauri/src/preflight**'
# The profiler hardcodes the desktop argv that process.rs::backend_args builds,
# so a change there must schedule a run or the two silently diverge.
- 'studio/src-tauri/src/process.rs'
- 'scripts/profile_startup.py'
- '.github/workflows/startup-profile-ci.yml'
# The job profiles whatever `install.sh --local` built: the installers pick the
# venv's Python and the dependency specs, and pyproject's include list is what
# makes --local overlay studio.backend*.
- 'install.sh'
- 'install.ps1'
- 'pyproject.toml'
# --local also runs the checkout's setup scripts (install.sh picks
# $_REPO_ROOT/studio/setup.sh, the editable install resolves setup.ps1 to the
# repo), and both call install_python_stack.py, which picks the dependencies.
- 'studio/setup.sh'
- 'studio/setup.ps1'
- 'studio/install_python_stack.py'
workflow_dispatch:
inputs:
repeats:
description: 'launch repeats per OS (median reported)'
type: string
default: '3'
concurrency:
group: ${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: true
permissions:
contents: read
jobs:
profile:
name: startup ${{ matrix.os }}
runs-on: ${{ matrix.os }}
timeout-minutes: 60
continue-on-error: true
strategy:
fail-fast: false
matrix:
os: [ubuntu-latest, macos-14, windows-latest]
env:
UNSLOTH_STUDIO_HOME: ${{ github.workspace }}/.studio-home
# A wildcard bind calls ifconfig.me on the startup path; loopback times our code.
UNSLOTH_STUDIO_DISABLE_PUBLIC_CHECK: '1'
steps:
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
with:
persist-credentials: false
- name: Install Studio
shell: bash
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
run: |
set -o pipefail
mkdir -p logs
# --local is load-bearing: it overlays the checkout, so the profiled server
# is this diff. Without it install.sh resolves unsloth from PyPI.
if [ "${{ runner.os }}" = "Windows" ]; then
pwsh -NoProfile -File ./install.ps1 --local 2>&1 | tee logs/install.log
else
bash install.sh --local 2>&1 | tee logs/install.log
fi
- name: Profile startup
shell: bash
run: |
BIN="$UNSLOTH_STUDIO_HOME/unsloth_studio/bin/unsloth"
[ -x "$BIN" ] || BIN="$UNSLOTH_STUDIO_HOME/unsloth_studio/Scripts/unsloth.exe"
[ -x "$BIN" ] || BIN=""
# Profile imports with the INSTALLED interpreter: that venv is what launches.
PY="$UNSLOTH_STUDIO_HOME/unsloth_studio/bin/python"
[ -x "$PY" ] || PY="$UNSLOTH_STUDIO_HOME/unsloth_studio/Scripts/python.exe"
[ -x "$PY" ] || PY="$(command -v python3 || command -v python)"
python3 scripts/profile_startup.py \
--python "$PY" \
${BIN:+--bin "$BIN"} \
--repeats "${{ inputs.repeats || '3' }}" \
--json "startup-${{ matrix.os }}.json" 2>&1 | tee logs/profile.log
- name: Summary
if: always()
shell: bash
run: |
f="startup-${{ matrix.os }}.json"
[ -f "$f" ] || { echo "no profile produced"; exit 0; }
python3 - "$f" >> "$GITHUB_STEP_SUMMARY" <<'PY'
import json, sys
d = json.load(open(sys.argv[1]))
print(f"### {d['platform']} / {d['machine']} (py {d['python']}, {d['cpu_count']} cpu)\n")
imp = d.get("imports", {})
# Gate on ok: a failed `import main` still leaves rows, so a total can lie.
if imp.get("ok"):
print(f"**`import main`: {imp['total_seconds']}s**\n")
print("| package | self ms |")
print("|---|---:|")
for k, v in list(imp.get("self_by_package_ms", {}).items())[:8]:
print(f"| {k} | {v} |")
print()
else:
print("**`import main` failed - no valid import profile**\n")
print("```\n" + (imp.get("error") or "")[-1500:] + "\n```\n")
lau = d.get("launch") or {}
runs = len(lau.get("runs") or [])
failed = lau.get("failed_runs") or 0
if lau.get("healthz_median_seconds") is not None:
# The aggregates cover only the runs that reached healthz, so flag the
# failures: bare numbers would read as a normal fast startup.
note = f" _({runs - failed} of {runs} launches; {failed} never became healthy)_" if failed else ""
print(f"**time to a healthy port: {lau['healthz_median_seconds']}s median, "
f"{lau['healthz_max_seconds']}s max**{note}\n")
elif lau.get("skipped"):
print(f"_launch phase skipped: {lau['skipped']}_\n")
elif runs:
print(f"**no launch measurement: all {runs} launches failed to become healthy**\n")
PY
- name: Upload profile
if: always()
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
with:
name: startup-profile-${{ matrix.os }}
path: |
startup-*.json
logs/
retention-days: 14
if-no-files-found: warn

3
.gitignore vendored
View file

@ -208,6 +208,9 @@ tmp/
**/node_modules/
auth.db
# Packaging snapshot of the root CHANGELOG.md (written by build.sh)
studio/CHANGELOG.md
# Tauri local build/generated output
studio/src-tauri/target/
studio/src-tauri/gen/

88
CHANGELOG.md Normal file
View file

@ -0,0 +1,88 @@
# Changelog
Release notes for Unsloth and Unsloth Studio.
Unsloth Studio reads this file to show release notes inside the "New Unsloth
version" update popup. Edit it here and the popup picks the change up on the
next update check, with no release or rebuild required.
## Format
Every release is a level-2 heading whose first token is the version, optionally
followed by a date:
```md
## 2026.7.6 - 2026-07-22
```
`## [2026.7.6] - 2026-07-22` and `## v2026.7.6` also work. Everything under a
heading, up to the next level-2 heading, is that release's notes and renders as
Markdown in the popup.
Notes are matched to one exact version. When Studio offers an update to
`2026.7.6` it renders the `2026.7.6` section and nothing else. If that section
is missing, the popup links out to the online changelog rather than showing
notes from an unrelated release, so a new version needs its own section here
before its notes can appear.
Keep the newest release at the top. Lead each bullet with the change itself:
the collapsed popup highlights the first sentence and dims the rest.
`## Unreleased` is ignored by the popup, so it is safe to stage notes there and
rename the heading at release time.
<!-- Add new releases directly below this line. -->
## Unreleased
## 2026.7.5
### What's Changed
- AMD support is here. Train, run RL, chat with and deploy 500+ models on
Radeon, Instinct, Ryzen and data center GPUs across Windows, WSL and Linux,
up to 2x faster with 70% less VRAM and no accuracy loss.
- Intel XPU support lands in Studio, so Arc and Data Center GPUs run chat and
training alongside the NVIDIA, AMD and Apple paths.
- Local speech to text dictation runs fully offline, with slim Whisper bundles
and a picker for custom models.
- DoRA training is available in Studio, selectable next to LoRA and full
fine-tuning in the training tab.
- The update popup previews release notes inline, pulled from this file and
matched to the exact version being offered.
### AMD, 23 July update
Our AMD collaboration, custom Triton kernels and math algorithms bring local
training and inference to AMD hardware. The 23 July update builds on the
[AMD release](https://github.com/unslothai/unsloth/releases/tag/v0.1.501-beta):
- RDNA2 and Gorgon Halo are supported, and the installer no longer fails to
detect GPUs on Strix Halo and other AMD cards.
- RDNA4 handling is better, and HIP and ROCm failures are caught and fixed
automatically instead of stopping the install.
- Unified memory safetensors loading is 2x faster, with much faster gradient
checkpointing on unified memory devices.
- Voice dictation through whisper.cpp has preliminary support.
- Rollback environments left by installs no longer eat 5GB of disk. They are
cleaned up automatically.
Optimized ROCm builds cover GGUF and safetensors inference, and ROCm
compatibility is improved for MI300X and MI325X. Full guide:
[unsloth.ai/docs/basics/amd](https://unsloth.ai/docs/basics/amd).
### Running larger models
- Automatic GPU placement, or pick exactly which GPUs and layers to use.
- Move MoE expert layers into system memory so larger models fit.
- Split a model across several GPUs, or use tensor parallelism.
- Hardware settings are saved per model and quant.
### Also in this release
- Remote access with `unsloth studio --secure` over free HTTPS via Cloudflare.
- Web search reads PDF papers and manuals, and parallel tool calls, reasoning
output and tool retries are more reliable.
- The model download location is configurable, so weights can live on a second
drive instead of the default cache.
- Stalled Hugging Face XET downloads retry over standard HTTP, and existing
GGUF files are reused instead of downloaded again.

2
MANIFEST.in Normal file
View file

@ -0,0 +1,2 @@
include _changelog_build.py
include CHANGELOG.md

36
_changelog_build.py Normal file
View file

@ -0,0 +1,36 @@
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
"""Snapshot CHANGELOG.md into the studio package at build time.
CHANGELOG.md at the repo root stays the one file to edit. Copying it here,
rather than in build.sh, means every packaging path ships it, so release notes
still render when the popup cannot reach GitHub."""
from __future__ import annotations
import shutil
from pathlib import Path
from setuptools.command.build_py import build_py as _build_py
ROOT = Path(__file__).resolve().parent
SOURCE = ROOT / "CHANGELOG.md"
SNAPSHOT = ROOT / "studio" / "CHANGELOG.md"
class build_py(_build_py):
def run(self) -> None:
# Beside the sources only if writable (PEP 517 may build an immutable
# checkout); into the staging directory always.
if SOURCE.is_file():
try:
shutil.copyfile(SOURCE, SNAPSHOT)
except OSError:
pass
super().run()
if not SOURCE.is_file():
return
staged = Path(self.build_lib) / "studio" / "CHANGELOG.md"
staged.parent.mkdir(parents = True, exist_ok = True)
shutil.copyfile(SOURCE, staged)

View file

@ -103,9 +103,13 @@ else
STUDIO_STAMPED_VERSION="$(python scripts/stamp_studio_release.py)"
fi
# 4. Build wheel/sdist
# 4. Build wheel/sdist. _changelog_build.py snapshots CHANGELOG.md into the studio
# package so release notes render offline.
python -m build
# Drop the snapshot so a source checkout never serves a stale copy.
rm -f studio/CHANGELOG.md
if [ "${1:-}" = "publish" ]; then
python scripts/stamp_studio_release.py --verify-dist dist --expected "$STUDIO_STAMPED_VERSION"
fi

View file

@ -57,6 +57,26 @@ function Install-UnslothStudio {
}
}
# Machine arch; Get-TauriDiagArch above reports the process. An emulated x64 shell on
# ARM64 reports AMD64, but PROCESSOR_ARCHITEW6432 is ARM64 in exactly that case.
function Get-HostMachineArch {
$osArch = ""
try { $osArch = [System.Runtime.InteropServices.RuntimeInformation]::OSArchitecture.ToString() } catch { $osArch = "" }
$signals = @([string]$env:PROCESSOR_ARCHITEW6432, [string]$env:PROCESSOR_ARCHITECTURE, $osArch)
foreach ($s in $signals) {
if ($s.ToLowerInvariant() -eq "arm64") { return "arm64" }
}
foreach ($s in $signals) {
if ([string]::IsNullOrWhiteSpace($s)) { continue }
switch ($s.ToLowerInvariant()) {
"amd64" { return "x86_64" }
"x64" { return "x86_64" }
"x86" { return "x86" }
}
}
return "unknown"
}
function Get-TauriTorchIndexFamily {
param([string]$TorchIndexUrl)
if ($SkipTorch) { return "none" }
@ -1124,10 +1144,27 @@ exit 0
return $false
}
# The interpreter's own arch, asked of it: win-amd64|win-arm64|win32|"".
function Get-PythonPlatformTag {
param([string]$Exe)
try {
return (& $Exe -c "import sysconfig; print(sysconfig.get_platform())" 2>$null | Out-String).Trim().ToLowerInvariant()
} catch { return "" }
}
# Returns @{ Version = "3.13"; Path = "C:\...\python.exe" } or $null.
# The resolved Path is passed to `uv venv --python` to prevent uv from
# re-resolving the version string back to a conda interpreter.
function Find-CompatiblePython {
# -X64Only: best installed x64 interpreter or $null, never ARM64. Last resort for
# Install-X64Python, where x64 of a lower-priority minor beats ARM64.
param([switch]$X64Only)
# Windows on ARM: prefer x64. pyarrow (via datasets) and hf-transfer ship no
# win_arm64 wheel, so a native ARM64 Python source-builds both and dies on CMake /
# Rust minutes in; x64 runs fine emulated. ARM64 is still returned when it is all
# there is, and the caller then bootstraps x64 or warns.
$preferX64 = $X64Only -or ((Get-HostMachineArch) -eq "arm64")
$candidates = @()
# Try the Python Launcher first (most reliable on Windows)
# py.exe resolves to the standard CPython install, not conda.
# Prefer the requested $PythonVersion, then newest-first fallback.
@ -1145,7 +1182,8 @@ exit 0
# Resolve the actual executable path and verify it is not conda-based
$resolvedExe = (& $pyLauncher.Source "-$minor" -c "import sys; print(sys.executable)" 2>$null | Out-String).Trim()
if ($resolvedExe -and (Test-Path $resolvedExe) -and -not (Test-IsCondaPython $resolvedExe)) {
return @{ Version = $ver; Path = $resolvedExe }
if (-not $preferX64) { return @{ Version = $ver; Path = $resolvedExe; Arch = "" } }
$candidates += @{ Version = $ver; Path = $resolvedExe }
}
}
} catch {}
@ -1166,11 +1204,53 @@ exit 0
try {
$out = & $cmd.Source --version 2>&1 | Out-String
if ($out -match "Python (3\.1[1-3])\.\d+") {
return @{ Version = $Matches[1]; Path = $cmd.Source }
if (-not $preferX64) { return @{ Version = $Matches[1]; Path = $cmd.Source; Arch = "" } }
$candidates += @{ Version = $Matches[1]; Path = $cmd.Source }
}
} catch {}
}
}
# `py -3.12` runs the launcher's preferred build, normally the native ARM64 one, so
# a same-minor x64 install that is neither preferred nor on PATH never becomes a
# candidate. `-3.12-64` cannot disambiguate (deprecated, it only means "not
# 32-bit"), so enumerate every registration with -0p and probe each path.
if ($preferX64) {
foreach ($pyLauncher in @(Get-Command py -All -CommandType Application -ErrorAction SilentlyContinue)) {
if ($pyLauncher.Source -match $script:CondaSkipPattern) { continue }
$listed = @()
try { $listed = @(& $pyLauncher.Source "-0p" 2>$null) } catch {}
foreach ($line in $listed) {
# " -V:3.12 * C:\...\python.exe": tag, optional default marker, path.
$m = [regex]::Match([string]$line, '(?i)^\s*-\S+\s+\*?\s*"?(?<p>\S.*?\.exe)"?\s*$')
if (-not $m.Success) { continue }
$exe = $m.Groups['p'].Value.Trim()
if ($candidates | Where-Object { $_.Path -eq $exe }) { continue }
if (-not (Test-Path -LiteralPath $exe)) { continue }
if (Test-IsCondaPython $exe) { continue }
try {
$out = & $exe --version 2>&1 | Out-String
if ($out -match "Python (3\.1[1-3])\.\d+") {
$candidates += @{ Version = $Matches[1]; Path = $exe }
}
} catch {}
}
}
}
# Prefer x64, but only within one minor: $minors is the caller's version preference,
# so ranking on arch alone would answer UNSLOTH_PYTHON=3.12 with an x64 3.13 and
# never bootstrap x64 3.12. Probing costs a subprocess, so non-ARM returned above.
foreach ($c in $candidates) {
$tag = Get-PythonPlatformTag $c.Path
$c.Arch = if ($tag -eq "win-amd64") { "x86_64" } elseif ($tag -eq "win-arm64") { "arm64" } else { "unknown" }
}
foreach ($minor in $minors) {
$sameMinor = @($candidates | Where-Object { $_.Version -eq $minor })
if ($sameMinor.Count -eq 0) { continue }
$x64 = $sameMinor | Where-Object { $_.Arch -eq "x86_64" } | Select-Object -First 1
if ($x64) { return $x64 }
if (-not $X64Only) { return $sameMinor[0] }
}
if (-not $X64Only -and $candidates.Count -gt 0) { return $candidates[0] }
return $null
}
@ -1181,8 +1261,11 @@ exit 0
# (no UAC), putting python.exe + the py launcher on PATH. Mirrors the uv ->
# astral.sh fallback below. Returns @{ Version; Path } or $null.
function Install-PythonFromPythonOrg {
# $Arch overrides the host arch, to pull x64 onto an ARM64 box.
param([string]$Arch = "")
# python.org ships one installer per architecture.
$archSuffix = switch (Get-TauriDiagArch) {
$targetArch = if ($Arch) { $Arch } else { Get-TauriDiagArch }
$archSuffix = switch ($targetArch) {
"x86_64" { "-amd64" }
"arm64" { "-arm64" }
"x86" { "" }
@ -1247,6 +1330,28 @@ exit 0
return (Find-CompatiblePython)
}
# ── Windows on ARM: get an x64 CPython ──
# --architecture x64 forces winget off the ARM64 build; python.org takes the same override.
function Install-X64Python {
if ($script:WingetAvailable) {
$prevEAP = $ErrorActionPreference
$ErrorActionPreference = "Continue"
try {
winget install -e --id "Python.Python.$PythonVersion" --source winget --architecture x64 --accept-package-agreements --accept-source-agreements
} catch { }
$ErrorActionPreference = $prevEAP
Refresh-SessionPath
$found = Find-CompatiblePython
if ($found -and $found.Arch -eq "x86_64") { return $found }
substep "winget could not provide an x64 Python -- trying python.org..." "Yellow"
}
$found = Install-PythonFromPythonOrg -Arch "x86_64"
if ($found -and $found.Arch -eq "x86_64") { return $found }
# Nothing installable (offline / no winget): an x64 build of another supported minor
# still runs the wheels ARM64 cannot, so take it over the native interpreter.
return (Find-CompatiblePython -X64Only)
}
# ── Install Python if no compatible version (3.11-3.13) found ──
# Find-CompatiblePython returns @{ Version = "3.13"; Path = "C:\...\python.exe" } or $null.
Write-TauriLog "STEP" "Installing Python"
@ -1318,6 +1423,26 @@ exit 0
return (Exit-InstallFailure "Python installation failed")
}
}
# ── Windows on ARM: swap a native ARM64 interpreter for x64 ──
# pyarrow and hf-transfer publish no win_arm64 wheel, so an ARM64 Python source-builds
# both and fails deep into the run. Warn up front if x64 is unobtainable.
if ($DetectedPython -and (Get-HostMachineArch) -eq "arm64" -and $DetectedPython.Arch -ne "x86_64") {
substep "windows on arm: only a native ARM64 Python $($DetectedPython.Version) was found." "Yellow"
substep "pyarrow and hf-transfer publish no win_arm64 wheels, so installing x64 Python..." "Yellow"
$X64Python = Install-X64Python
if ($X64Python) {
$DetectedPython = $X64Python
step "python" "using x64 Python $($DetectedPython.Version) under emulation"
} else {
Write-Host "[WARN] Could not install an x64 Python on this ARM64 machine." -ForegroundColor Yellow
Write-Host " Continuing with ARM64 Python $($DetectedPython.Version), but the install is likely to fail:" -ForegroundColor Yellow
Write-Host " pyarrow (via datasets) and hf-transfer ship no win_arm64 wheels and will be" -ForegroundColor Yellow
Write-Host " built from source, which needs CMake plus the MSVC and Rust toolchains." -ForegroundColor Yellow
Write-Host " Fix: install x64 Python from https://www.python.org/downloads/windows/" -ForegroundColor Yellow
Write-Host " (choose 'Windows installer (64-bit)', not ARM64), then re-run this installer." -ForegroundColor Yellow
}
}
$DiagPythonVersion = $PythonVersion
if ($DetectedPython) { $DiagPythonVersion = $DetectedPython.Version }
$InitialGpuBranch = "unknown"
@ -2438,6 +2563,13 @@ exit 0
}
} else {
Write-TauriLog "STEP" "Installing PyTorch"
# Windows on ARM lacks only torchaudio (whl/cpu win_arm64: torch 42,
# torchvision 60, torchaudio 0), so drop that pin instead of aborting. Ask the
# interpreter, not PROCESSOR_ARCHITECTURE; reached when no x64 Python exists.
$VenvPlatform = ""
try {
$VenvPlatform = (& $VenvPython -c "import sysconfig; print(sysconfig.get_platform())" 2>$null | Out-String).Trim().ToLowerInvariant()
} catch { $VenvPlatform = "" }
substep "installing PyTorch ($(Remove-IndexUrlCredentials $TorchIndexUrl))..."
# Bound the companions to the capped torch on EVERY index, cu<digits>
# families included: torchaudio 2.11 dropped its exact torch pin from
@ -2445,7 +2577,13 @@ exit 0
# resolve a mismatched 2.11.0 build. Mirrors install.sh.
$_pinVisionSpec = "torchvision>=0.19,<0.26.0"
$_pinAudioSpec = "torchaudio>=2.4,<2.11.0"
$torchInstallExit = Invoke-InstallCommandRetry -Label "install PyTorch" { uv pip install --python $VenvPython "torch>=2.4,<2.11.0" $_pinVisionSpec $_pinAudioSpec --default-index $TorchIndexUrl }
$_torchSpecs = @("torch>=2.4,<2.11.0", $_pinVisionSpec, $_pinAudioSpec)
if ($VenvPlatform -eq "win-arm64") {
substep "windows on arm: skipping torchaudio (upstream publishes no"
substep "win_arm64 wheel); torch and torchvision install normally."
$_torchSpecs = @("torch>=2.4,<2.11.0", $_pinVisionSpec)
}
$torchInstallExit = Invoke-InstallCommandRetry -Label "install PyTorch" { uv pip install --python $VenvPython @_torchSpecs --default-index $TorchIndexUrl }
if ($torchInstallExit -ne 0) {
Write-Host "[ERROR] Failed to install PyTorch (exit code $torchInstallExit)" -ForegroundColor Red
return (Exit-InstallFailure "Failed to install PyTorch (exit code $torchInstallExit)" $torchInstallExit)

View file

@ -47,9 +47,14 @@ version = {attr = "unsloth.models._utils.__version__"}
[tool.setuptools]
include-package-data = true
[tool.setuptools.cmdclass]
# Snapshots CHANGELOG.md into studio/ so every build path ships it.
build_py = "_changelog_build.build_py"
[tool.setuptools.package-data]
unsloth_cli = ["codex_fallback_prompt.md", "pi_subagent.ts"]
studio = [
"CHANGELOG.md",
"*.sh",
"*.ps1",
"*.bat",

377
scripts/profile_startup.py Normal file
View file

@ -0,0 +1,377 @@
#!/usr/bin/env python3
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
"""Measure where Unsloth Studio's startup time goes, per platform.
Nothing measured this before: the backend logs "lifespan startup completed in X ms"
but no test or CI job asserted a budget, and studio_test_kit discards the elapsed
time of its /healthz poll. A first local run (Linux, warm cache, fast server CPU)
found `import main` alone costs 6.6s before the server can bind, dominated by eager
module-level imports pulled in by the `routes` package:
torch 1930 ms self
unsloth_zoo 914 ms self
routes 779 ms self
transformers 524 ms self
Phases measured:
import `python -X importtime -c "import main"`, top cumulative + per-package self
spawn process start -> first byte on stdout
healthz process start -> /api/health (or /healthz) answers 200
lifespan the backend's own "lifespan startup completed in X ms" log line
Usage:
python scripts/profile_startup.py --repeats 3 --json out.json
python scripts/profile_startup.py --import-only # no server, no port needed
Exit code is 0 unless --max-healthz-seconds is given and exceeded.
"""
from __future__ import annotations
import argparse
import json
import math
import os
import platform
import re
import shutil
import socket
import statistics
import subprocess
import sys
import threading
import time
import urllib.error
import urllib.request
from pathlib import Path
REPO_ROOT = Path(__file__).resolve().parents[1]
BACKEND = REPO_ROOT / "studio" / "backend"
_IMPORTTIME_RE = re.compile(r"import time:\s+(\d+)\s+\|\s+(\d+)\s+\|(\s*)(\S.*)")
def _free_port() -> int:
with socket.socket() as s:
s.bind(("127.0.0.1", 0))
return int(s.getsockname()[1])
def profile_imports(python: str, top: int = 15) -> dict:
"""Cumulative and self import cost for the backend's module graph.
Run in a subprocess with -X importtime: the numbers are only meaningful for a
cold interpreter, and importing in-process would measure a warm sys.modules.
"""
proc = subprocess.run(
[python, "-X", "importtime", "-c", "import sys; sys.path.insert(0, '.'); import main"],
cwd = BACKEND,
capture_output = True,
text = True,
timeout = 900,
)
rows = []
for line in proc.stderr.splitlines():
m = _IMPORTTIME_RE.match(line)
if m:
rows.append((int(m.group(1)), int(m.group(2)), m.group(4).strip()))
if not rows:
return {"ok": False, "error": (proc.stderr or proc.stdout)[-2000:]}
if proc.returncode != 0:
# Rows survive up to the failure, so any total from a partial graph is wrong.
return {
"ok": False,
"error": (proc.stderr or proc.stdout)[-2000:],
"partial_rows": len(rows),
}
by_cum = sorted(rows, key = lambda r: -r[1])
# Total comes from the `main` row, not by_cum[0]: -X importtime also prints the
# interpreter's own startup graph (`site`), which can outrank a trivial main.
main_row = next((r for r in reversed(rows) if r[2] == "main"), None)
if main_row is None:
return {
"ok": False,
"error": "no `import main` row in -X importtime output\n"
+ (proc.stderr or proc.stdout)[-2000:],
}
self_by_pkg: dict[str, int] = {}
for self_us, _cum, name in rows:
pkg = name.split(".")[0]
self_by_pkg[pkg] = self_by_pkg.get(pkg, 0) + self_us
return {
"ok": True,
"total_seconds": round(main_row[1] / 1e6, 3),
"top_cumulative": [
{"module": n, "seconds": round(c / 1e6, 3)} for _s, c, n in by_cum[:top]
],
"self_by_package_ms": {
k: round(v / 1000) for k, v in sorted(self_by_pkg.items(), key = lambda x: -x[1])[:top]
},
}
def _terminate_tree(proc: subprocess.Popen) -> None:
"""Stop the server AND its children, which on Windows are a separate process.
CI profiles `Scripts/unsloth.exe`, a distlib launcher stub that CreateProcess's
the venv python and waits, so terminate() reaps the stub only: the real backend
keeps the inherited stdout handle, the reader thread never sees EOF, and
--repeats strands one server per iteration on the shared UNSLOTH_STUDIO_HOME.
taskkill /T walks the tree, as unsloth_cli/commands/start.py already does.
"""
if proc.poll() is not None:
return
if os.name == "nt":
try:
killed = subprocess.run(
["taskkill", "/PID", str(proc.pid), "/T", "/F"],
capture_output = True,
timeout = 30,
check = False,
)
if killed.returncode == 0:
return
except Exception:
# taskkill missing or timed out; fall through so the stub still dies.
pass
# check=False: a nonzero taskkill does not raise, so fall through as well.
proc.terminate()
def profile_launch(
bin_path: str,
port: int,
timeout_s: int = 300,
) -> dict:
"""Spawn the backend the way the desktop app does and time it to first 200."""
log_lines: list[str] = []
first_byte: list[float] = []
t0 = time.perf_counter()
proc = subprocess.Popen(
[bin_path, "studio", "--api-only", "-H", "127.0.0.1", "-p", str(port)],
cwd = REPO_ROOT,
stdout = subprocess.PIPE,
stderr = subprocess.STDOUT,
text = True,
bufsize = 1,
)
def _drain() -> None:
# Runs alongside the health polling: the first read timestamps the spawn
# phase, and an undrained pipe blocks the backend before it binds.
for line in proc.stdout:
if not first_byte:
first_byte.append(time.perf_counter() - t0)
log_lines.append(line.rstrip("\n"))
reader = threading.Thread(target = _drain, daemon = True)
reader.start()
t_healthz = None
deadline = t0 + timeout_s
try:
while time.perf_counter() < deadline:
if proc.poll() is not None:
break
if t_healthz is None:
for url in (
f"http://127.0.0.1:{port}/api/health",
f"http://127.0.0.1:{port}/healthz",
):
try:
with urllib.request.urlopen(url, timeout = 2) as r:
if r.status == 200:
t_healthz = time.perf_counter() - t0
break
except (urllib.error.URLError, OSError, TimeoutError):
pass
if t_healthz is not None:
break
time.sleep(0.25)
finally:
_terminate_tree(proc)
try:
# Safe: the reader drains the pipe, so the child cannot block on write().
proc.wait(timeout = 30)
except subprocess.TimeoutExpired:
proc.kill()
proc.wait()
reader.join(timeout = 10)
t_first_byte = first_byte[0] if first_byte else None
lifespan_ms = None
for line in log_lines:
m = re.search(r"lifespan startup completed in ([\d.]+)ms", line)
if m:
lifespan_ms = float(m.group(1))
return {
"spawn_seconds": round(t_first_byte, 3) if t_first_byte is not None else None,
"healthz_seconds": round(t_healthz, 3) if t_healthz is not None else None,
"lifespan_ms": lifespan_ms,
"reached_healthz": t_healthz is not None,
"log_tail": log_lines[-25:],
}
def python_version_of(python: str) -> str:
"""Version of the interpreter that runs the imports, not the one running us.
--python points at the installed Studio venv while this script runs under the
runner's system python, so platform.python_version() would label it wrong.
"""
if python == sys.executable:
return platform.python_version()
try:
proc = subprocess.run(
[python, "-c", "import platform; print(platform.python_version())"],
capture_output = True,
text = True,
timeout = 60,
)
if proc.returncode == 0 and proc.stdout.strip():
return proc.stdout.strip()
except (OSError, subprocess.SubprocessError):
pass
return "unknown"
def find_bin() -> str | None:
home = os.environ.get("UNSLOTH_STUDIO_HOME") or str(Path.home() / ".unsloth" / "studio")
names = ["unsloth.exe", "unsloth"] if platform.system() == "Windows" else ["unsloth"]
subdirs = ["unsloth_studio/Scripts", "unsloth_studio/bin", "bin", "Scripts"]
for sd in subdirs:
for n in names:
p = Path(home) / sd / n
if p.exists():
return str(p)
return shutil.which("unsloth")
def main(argv: list[str]) -> int:
ap = argparse.ArgumentParser(
description = __doc__, formatter_class = argparse.RawDescriptionHelpFormatter
)
ap.add_argument(
"--repeats",
type = int,
default = 1,
help = "launch repeats; the median is reported (imports are measured once)",
)
ap.add_argument(
"--python",
default = sys.executable,
help = "interpreter used for the import profile (default: this one)",
)
ap.add_argument("--bin", help = "path to the unsloth CLI (default: autodetect)")
ap.add_argument(
"--import-only",
action = "store_true",
help = "skip the server phases (no install needed beyond the deps)",
)
ap.add_argument(
"--max-healthz-seconds",
type = float,
help = "fail if the median time to a healthy port exceeds this",
)
ap.add_argument("--json", help = "write the full report here")
a = ap.parse_args(argv)
# range(0) launches nothing, leaving the budget check with nothing to fail on.
if a.repeats < 1:
ap.error("--repeats must be at least 1")
# Same reason: --import-only never launches anything.
if a.import_only and a.max_healthz_seconds is not None:
ap.error("--max-healthz-seconds cannot be combined with --import-only")
# nan and inf parse fine as floats but `med > budget` is then always False,
# so the gate would report success without ever bounding anything.
if a.max_healthz_seconds is not None and not math.isfinite(a.max_healthz_seconds):
ap.error("--max-healthz-seconds must be a finite number")
report: dict = {
"platform": platform.system().lower(),
"machine": platform.machine(),
"python": python_version_of(a.python),
"cpu_count": os.cpu_count(),
}
print("== import graph ==")
report["imports"] = profile_imports(a.python)
imp = report["imports"]
if imp.get("ok"):
print(f" import main: {imp['total_seconds']}s")
for row in imp["top_cumulative"][:8]:
print(f" {row['seconds']:7.3f}s {row['module']}")
print(" self time by package (ms):")
for k, v in list(imp["self_by_package_ms"].items())[:8]:
print(f" {v:8} ms {k}")
else:
print(f" FAILED: {imp.get('error', '')[:400]}")
if not a.import_only:
bin_path = a.bin or find_bin()
if not bin_path:
print(
"== launch == skipped: no unsloth CLI found "
"(set UNSLOTH_STUDIO_HOME or pass --bin)"
)
report["launch"] = {"skipped": "no unsloth CLI found"}
else:
print(f"== launch == {bin_path}")
runs = []
for i in range(a.repeats):
r = profile_launch(bin_path, _free_port())
runs.append(r)
print(
f" run {i + 1}: healthz={r['healthz_seconds']}s "
f"lifespan={r['lifespan_ms']}ms reached={r['reached_healthz']}"
)
got = [r["healthz_seconds"] for r in runs if r["healthz_seconds"] is not None]
report["launch"] = {
"runs": runs,
"failed_runs": sum(1 for r in runs if not r["reached_healthz"]),
"healthz_median_seconds": round(statistics.median(got), 3) if got else None,
"healthz_max_seconds": round(max(got), 3) if got else None,
}
if got:
print(
f" median time to healthy port: {report['launch']['healthz_median_seconds']}s"
)
if a.json:
Path(a.json).write_text(json.dumps(report, indent = 2), encoding = "utf-8")
print(f"\nwrote {a.json}")
if a.max_healthz_seconds is not None:
launch = report.get("launch") or {}
med = launch.get("healthz_median_seconds")
failed = launch.get("failed_runs") or 0
if failed:
# Failed launches fail the budget; dropping them would keep only the fast ones.
print(
f"::error::startup regression: {failed} of {len(launch.get('runs') or [])} "
f"launches never became healthy within the timeout"
)
return 1
if med is None:
# Nothing measured: exiting 0 would pass a requested budget without a
# single health request, so fail closed.
print(
"::error::startup regression: no healthz measurement, so the "
f"{a.max_healthz_seconds}s budget was never checked "
f"({launch.get('skipped') or 'launch phase produced no runs'})"
)
return 1
elif med > a.max_healthz_seconds:
print(
f"::error::startup regression: {med}s median to a healthy port "
f"exceeds the {a.max_healthz_seconds}s budget"
)
return 1
return 0
if __name__ == "__main__":
raise SystemExit(main(sys.argv[1:]))

View file

@ -76,7 +76,13 @@ def _load_bootstrap_password() -> Optional[str]:
global _bootstrap_password
_bootstrap_password = None
if _BOOTSTRAP_PW_PATH.is_file():
bootstrap_password = _BOOTSTRAP_PW_PATH.read_text(encoding = "utf-8").strip()
# No caller handles a raise, so an unreadable file has to mean "no bootstrap
# password", not a dead backend. We write UTF-8, so bytes that will not
# decode are damage whose plaintext is worthless anyway.
try:
bootstrap_password = _BOOTSTRAP_PW_PATH.read_text(encoding = "utf-8").strip()
except (OSError, UnicodeDecodeError):
return _bootstrap_password
if bootstrap_password:
_bootstrap_password = bootstrap_password
return _bootstrap_password

View file

@ -310,6 +310,7 @@ class CloudflareTunnel:
stderr = subprocess.STDOUT,
stdin = subprocess.DEVNULL,
text = True,
encoding = "utf-8",
errors = "replace",
bufsize = 1,
**_windows_hidden_kwargs(),

View file

@ -257,6 +257,8 @@ def _run_oxc_batch(
cwd = str(_OXC_TOOL_DIR),
input = json.dumps(payload),
text = True,
encoding = "utf-8",
errors = "replace",
capture_output = True,
check = False,
env = env,

View file

@ -567,7 +567,7 @@ class InferenceBackend:
_meta_path = Path(config.path) / "export_metadata.json"
try:
if _meta_path.exists():
_meta = json.loads(_meta_path.read_text(encoding = "utf-8"))
_meta = json.loads(_meta_path.read_text(encoding = "utf-8-sig"))
if _meta.get("base_model"):
processor_source = _meta["base_model"]
except Exception:

View file

@ -85,6 +85,7 @@ from core.tool_healing import (
strip_outside_think,
)
from utils.native_path_leases import child_env_without_native_path_secret
from utils.child_stdio import utf8_child_env
from utils.hf_xet_fallback import hf_hub_download_with_xet_fallback
from utils.subprocess_compat import (
windows_hidden_subprocess_kwargs as _windows_hidden_subprocess_kwargs,
@ -581,7 +582,7 @@ def _load_swa_cache() -> dict:
if _SWA_CACHE is not None:
return _SWA_CACHE
try:
with open(_swa_cache_path(), encoding = "utf-8") as f:
with open(_swa_cache_path(), encoding = "utf-8-sig") as f:
_SWA_CACHE = json.load(f)
if not isinstance(_SWA_CACHE, dict):
_SWA_CACHE = {}
@ -632,7 +633,7 @@ def _fetch_swa_entry_from_hf(repo_id: str) -> Optional[object]:
repo_type = "model",
cache_dir = active_hf_hub_cache(),
)
with open(cfg_path, encoding = "utf-8") as f:
with open(cfg_path, encoding = "utf-8-sig") as f:
cfg = json.load(f)
except Exception:
return None
@ -3046,6 +3047,7 @@ class LlamaCppBackend:
[bin_path, "--help"],
capture_output = True,
text = True,
encoding = "utf-8",
errors = "replace",
timeout = 10,
check = False,
@ -3618,6 +3620,8 @@ class LlamaCppBackend:
],
capture_output = True,
text = True,
encoding = "utf-8",
errors = "replace",
timeout = 10,
env = child_env_without_native_path_secret(),
**_windows_hidden_subprocess_kwargs(),
@ -3732,7 +3736,7 @@ class LlamaCppBackend:
encoding = "utf-8",
errors = "replace",
timeout = 15,
env = env,
env = utf8_child_env(env),
**_windows_hidden_subprocess_kwargs(),
)
if result.returncode != 0:
@ -5482,7 +5486,9 @@ class LlamaCppBackend:
stdout = subprocess.PIPE,
stderr = subprocess.STDOUT,
text = True,
env = env,
encoding = "utf-8",
errors = "replace",
env = utf8_child_env(env),
**_windows_hidden_subprocess_kwargs(),
**_child_popen_kwargs(),
)
@ -6696,6 +6702,8 @@ class LlamaCppBackend:
stdout = subprocess.PIPE,
stderr = subprocess.STDOUT,
text = True,
encoding = "utf-8",
errors = "replace",
env = env,
**_windows_hidden_subprocess_kwargs(),
**_child_popen_kwargs(),
@ -8712,6 +8720,8 @@ class LlamaCppBackend:
stdout = subprocess.PIPE,
stderr = subprocess.STDOUT,
text = True,
encoding = "utf-8",
errors = "replace",
env = env,
**_windows_hidden_subprocess_kwargs(),
**_child_popen_kwargs(),
@ -10214,6 +10224,8 @@ class LlamaCppBackend:
["pgrep", "-a", "-f", "llama-server"],
capture_output = True,
text = True,
encoding = "utf-8",
errors = "replace",
timeout = 5,
env = child_env_without_native_path_secret(),
)

View file

@ -151,7 +151,7 @@ def _resolve_lora_4bit(mc, load_in_4bit: bool) -> bool:
import json
try:
with open(adapter_cfg_path, encoding = "utf-8") as f:
with open(adapter_cfg_path, encoding = "utf-8-sig") as f:
adapter_cfg = json.load(f)
training_method = adapter_cfg.get("unsloth_training_method")
if training_method == "lora" and load_in_4bit:
@ -963,7 +963,7 @@ def run_inference_process(
if _local_adapter_cfg.is_file():
try:
_lora_base = (
_json.loads(_local_adapter_cfg.read_text(encoding = "utf-8")).get(
_json.loads(_local_adapter_cfg.read_text(encoding = "utf-8-sig")).get(
"base_model_name_or_path"
)
or None

View file

@ -103,6 +103,8 @@ class LlamaServerBackend:
[binary, "--help"],
capture_output = True,
text = True,
encoding = "utf-8",
errors = "replace",
timeout = 30,
**windows_hidden_subprocess_kwargs(),
)
@ -331,6 +333,8 @@ class LlamaServerBackend:
stdout = subprocess.PIPE,
stderr = subprocess.STDOUT,
text = True,
encoding = "utf-8",
errors = "replace",
env = env,
**windows_hidden_subprocess_kwargs(),
**child_popen_kwargs(),

View file

@ -100,7 +100,7 @@ def _st_module_subdirs(name: str, token: str | None) -> tuple[str, ...]:
path = Path(normalize_path(name)).expanduser() / "modules.json"
if not path.is_file():
return ()
data = json.loads(path.read_text(encoding = "utf-8"))
data = json.loads(path.read_text(encoding = "utf-8-sig"))
else:
from huggingface_hub import hf_hub_download
from huggingface_hub.utils import EntryNotFoundError
@ -115,7 +115,7 @@ def _st_module_subdirs(name: str, token: str | None) -> tuple[str, ...]:
)
except EntryNotFoundError:
return ()
data = json.loads(open(local, encoding = "utf-8").read())
data = json.loads(open(local, encoding = "utf-8-sig").read())
subdirs = []
for module in data or ():
sub = str((module or {}).get("path", "")).strip().strip("/")

View file

@ -43,6 +43,7 @@ if sys.platform.startswith("linux") and "HSA_ENABLE_DXG_DETECTION" not in os.env
pass
logger = get_logger(__name__)
from utils.child_stdio import utf8_child_env
from utils.hardware import apply_gpu_ids
from utils.training_runs import build_default_output_dir_name
from utils.wheel_utils import (
@ -385,6 +386,10 @@ def _install_package_wheel_first(
"stdout": _sp.PIPE,
"stderr": _sp.STDOUT,
"text": True,
"encoding": "utf-8",
"errors": "replace",
# Make the Python child emit the UTF-8 we decode above.
"env": utf8_child_env(),
}
if is_hip:
_run_kwargs["timeout"] = 1800
@ -606,6 +611,9 @@ def _ensure_flash_linear_attention_unconditional(event_queue: Any) -> bool:
stdout = _sp.PIPE,
stderr = _sp.STDOUT,
text = True,
encoding = "utf-8",
errors = "replace",
env = utf8_child_env(),
timeout = _TILELANG_INSTALL_TIMEOUT_S,
)
except _sp.TimeoutExpired:
@ -849,6 +857,9 @@ def _run_pip(cmd: list[str], event_queue: Any, label: str) -> bool:
stdout = _sp.PIPE,
stderr = _sp.STDOUT,
text = True,
encoding = "utf-8",
errors = "replace",
env = utf8_child_env(),
timeout = _TILELANG_INSTALL_TIMEOUT_S,
)
except _sp.TimeoutExpired:

View file

@ -215,7 +215,7 @@ def _ollama_model_info_from_manifest(
return None
try:
manifest = json.loads(tag_file.read_text(encoding = "utf-8"))
manifest = json.loads(tag_file.read_text(encoding = "utf-8-sig"))
except (json.JSONDecodeError, OSError, UnicodeDecodeError) as e:
logger.debug("Skipping unreadable/invalid Ollama manifest %s: %s", tag_file, e)
return None
@ -228,7 +228,7 @@ def _ollama_model_info_from_manifest(
config_blob = _ollama_blob_path(blobs_dir, config_digest)
if config_blob is not None and _safe_is_file(config_blob):
try:
cfg = json.loads(config_blob.read_text(encoding = "utf-8"))
cfg = json.loads(config_blob.read_text(encoding = "utf-8-sig"))
model_type = cfg.get("model_type", "")
file_type = cfg.get("file_type", "")
except (json.JSONDecodeError, OSError, UnicodeDecodeError) as e:

View file

@ -464,6 +464,8 @@ def _read_marker_value(marker: Path) -> Optional[str]:
return None
value = marker.read_text(encoding = "utf-8").strip()
except (OSError, UnicodeDecodeError):
# UnicodeDecodeError is a ValueError, so it would escape and abort
# prepare_cache_for_transport. An unknown value just purges and restarts.
return None
return value if value in VALID_TRANSPORTS else None

View file

@ -42,8 +42,12 @@ class LogConfig:
log_level_name = os.getenv("LOG_LEVEL", "INFO").upper()
log_level = getattr(logging, log_level_name, logging.INFO)
if sys.platform == "win32":
for stream in (sys.stdout, sys.stderr):
# Non-ASCII on a non-UTF-8 stream raises UnicodeEncodeError (Windows,
# LANG=C), so key off the stream, not the platform.
for stream in (sys.stdout, sys.stderr):
if getattr(stream, "encoding", "") and not str(stream.encoding).lower().replace(
"-", ""
).startswith("utf8"):
if hasattr(stream, "reconfigure"):
try:
stream.reconfigure(encoding = "utf-8", errors = "replace")

View file

@ -347,6 +347,7 @@ from utils.update_status import (
get_studio_install_source_status,
get_studio_update_status,
)
from utils.changelog import get_release_notes, is_supported_version_query
from utils.studio_version import get_studio_version
from utils.api_errors import install_api_error_handlers
@ -1154,6 +1155,18 @@ def studio_update_status(_current_subject: str = Depends(get_current_subject)):
return get_studio_update_status(UNSLOTH_VERSION)
@app.get("/api/studio/release-notes")
def studio_release_notes(
version: str = Query(..., max_length = 64),
refresh: bool = Query(False),
_current_subject: str = Depends(get_current_subject),
):
"""Return CHANGELOG.md notes for exactly `version` (never a nearby one)."""
if not is_supported_version_query(version):
raise HTTPException(status_code = 422, detail = "Invalid version.")
return get_release_notes(version, refresh = refresh)
@app.get(
"/api/studio/download-transport-capabilities",
response_model = TransportCapabilities,

View file

@ -6,10 +6,93 @@
from __future__ import annotations
import json
import locale
import os
import threading
from pathlib import Path
from typing import Any, Dict
from typing import Any, Dict, NamedTuple
def _locale_encoding() -> str:
"""The codepage a pre-UTF-8 release here would have written, or "".
Empty on a UTF-8 host, where there is no codepage to attribute the file to.
"""
try:
preferred = locale.getencoding()
except AttributeError: # Python < 3.11
preferred = locale.getpreferredencoding(False)
if preferred.lower().replace("-", "").replace("_", "") == "utf8":
return ""
return preferred
# Trail bytes can land on JSON punctuation, so a single-byte fallback misreads these.
_DOUBLE_BYTE_ENCODINGS = ("cp932", "cp936", "cp949", "cp950")
def _parse(raw: bytes, encoding: str) -> Any:
"""Parse one JSON document under *encoding*, or None if it does not.
RecursionError is a RuntimeError, so nesting json.loads will not descend is
the one parse failure the other three miss. Both callers run this outside
any further handler, so it has to answer None here or a single damaged
record aborts the scraper at startup instead of being skipped.
"""
try:
return json.loads(raw.decode(encoding))
except (UnicodeDecodeError, LookupError, ValueError, RecursionError):
return None
class _Reading(NamedTuple):
as_utf8: Any
as_legacy: Any
def _read_line(raw: bytes, codepage: str) -> _Reading:
"""Read one line as UTF-8 and as a codepage, for dedup keys only.
Requiring valid JSON, not merely a successful decode, is what separates a
genuine legacy record from a half-written UTF-8 one: a torn multibyte
character decodes under cp1252 but leaves the JSON unterminated. Some byte
strings parse both ways, e.g. cp1251 ``Р°`` is ``D0 B0``, which is also
UTF-8 ``а``.
The codepage reading is never authoritative, because the file's own encoding
cannot be recovered from its bytes. Reading a cp1251 shard on a cp1252
machine turns ``Привет`` into ``Ïðèâåò`` and every byte of it decodes
cleanly, so a successful decode proves nothing about who wrote it. It is
used only to recover the dedup keys, which are ASCII ids and come back the
same under any of these, so the first reading that parses will do.
That is also why several are tried. latin-1 alone mangles the double-byte
codepages: cp932 ```` is ``95 5C``, and latin-1 turns the trail byte into
a JSON backslash, so the record fails to parse and its id is forgotten.
"""
as_utf8 = _parse(raw, "utf-8")
# A record that reads as UTF-8 needs no second reading: re-parsing cost 2.8x on a
# 76 MB shard, and these reach gigabytes. Only a dict, since key lookup falls
# through to the codepage when UTF-8 yields none.
if isinstance(as_utf8, dict):
return _Reading(as_utf8, None)
for encoding in (codepage, "latin-1", *_DOUBLE_BYTE_ENCODINGS):
if not encoding:
continue
as_legacy = _parse(raw, encoding)
if as_legacy is not None:
return _Reading(as_utf8, as_legacy)
return _Reading(as_utf8, None)
class _Scan(NamedTuple):
"""What a pass over an existing shard established about it."""
legacy: bool # enough evidence to trust the codepage reading's keys
readable: bool
saw_non_ascii: bool # some line's meaning depends on the encoding
utf8_keys: set # keys from lines UTF-8 could read
legacy_keys: set # keys only the codepage reading yields
class StateStore:
@ -18,12 +101,19 @@ class StateStore:
self.path.parent.mkdir(parents = True, exist_ok = True)
self._lock = threading.Lock()
self._data: Dict[str, Any] = {}
# Read whole, and UTF-8 only unlike the shards below: a checkpoint holds
# nothing but base64 cursors and booleans, so a codepage retry could only ever
# add non-ASCII. That would resume on a mojibaked cursor, which GitHub rejects
# with INVALID_CURSOR_ARGUMENTS, and the empty page it returns marks the stream
# done and skips the rest for good. Dropping a damaged checkpoint re-scrapes
# from the first page, which the writers dedup.
if self.path.exists():
try:
with self.path.open(encoding = "utf-8") as f:
self._data = json.load(f)
except Exception:
self._data = {}
raw = self.path.read_bytes()
except OSError:
raw = b""
data = _parse(raw, "utf-8")
self._data = data if isinstance(data, dict) else {}
def get(
self,
@ -63,24 +153,83 @@ class JsonlWriter:
self.path = Path(path)
self.path.parent.mkdir(parents = True, exist_ok = True)
self._lock = threading.Lock()
self._fh = self.path.open("a", buffering = 1, encoding = "utf-8")
self._count_seen_keys: set[str] = set()
# Preload seen keys for dedup across resumes
self._codepage = _locale_encoding()
self._ensure_ascii = False
encoding = "utf-8"
if self.path.exists() and self.path.stat().st_size > 0:
try:
# No guess is safe for a file an older build wrote in the
# operator's locale, so read past whatever will not decode.
with self.path.open(encoding = "utf-8", errors = "replace") as f:
for line in f:
try:
obj = json.loads(line)
k = self._key(obj)
if k is not None:
self._count_seen_keys.add(k)
except Exception:
pass
except Exception:
pass
scan = self._scan_existing()
self._count_seen_keys = scan.utf8_keys
if scan.legacy:
self._count_seen_keys |= scan.legacy_keys
if scan.saw_non_ascii or not scan.readable:
# Never convert: the writing encoding is unrecoverable and guessing
# mojibakes the records. Pure ASCII appends store identically under
# every codepage, and json.loads turns the \uXXXX escapes back.
encoding = "ascii"
self._ensure_ascii = True
self._fh = self.path.open("a", buffering = 1, encoding = encoding, errors = "strict")
def _scan_existing(self) -> _Scan:
"""Read the shard once to recover dedup keys and judge its encoding.
Line by line: these shards reach gigabytes on a large scrape, so neither
the bytes nor the decoded text are held whole.
The verdict weighs the whole file. Each line with non-ASCII bytes votes:
one that parses only under the codepage is evidence of a legacy shard,
one that parses as UTF-8 is evidence against, since arbitrary codepage
text almost never forms valid multibyte UTF-8. A single corrupt byte in
a healthy shard therefore cannot outvote the records around it, and a
genuinely legacy shard has a legacy vote on every line that carries an
umlaut.
More than one such line is required, because a single one is genuinely
undecidable: a legacy record holding one accented character and an ASCII
record holding one stray byte are the same shape. Reading it as damage
risks a duplicate; reading it as legacy marks an unreadable record seen
and blocks the retry that would replace it, losing it for good. Only one
of those is recoverable.
The verdict only picks which reading supplies the dedup keys. The file
itself is never rewritten either way, so a wrong answer costs at most a
duplicate, never a corrupted record.
"""
legacy_votes = 0
utf8_votes = 0
saw_non_ascii = False
utf8_keys: set[str] = set()
legacy_keys: set[str] = set()
try:
with self.path.open("rb") as handle:
for raw in handle:
line = raw.strip()
reading = _read_line(line, self._codepage)
# ASCII reads the same everywhere: no vote, no constraint.
if not line.isascii():
saw_non_ascii = True
if reading.as_utf8 is None and reading.as_legacy is not None:
legacy_votes += 1
elif reading.as_utf8 is not None:
utf8_votes += 1
# Kept apart so a damaged line does not block its own retry.
if isinstance(reading.as_utf8, dict):
key = self._key(reading.as_utf8)
if key is not None:
utf8_keys.add(key)
elif isinstance(reading.as_legacy, dict):
key = self._key(reading.as_legacy)
if key is not None:
legacy_keys.add(key)
except OSError:
return _Scan(False, False, False, utf8_keys, legacy_keys)
return _Scan(
legacy_votes > 1 and legacy_votes > utf8_votes,
True,
saw_non_ascii,
utf8_keys,
legacy_keys,
)
def _key(self, obj: dict) -> str | None:
for k in ("id", "node_id", "number", "sha", "url"):
@ -99,7 +248,7 @@ class JsonlWriter:
return False
if k is not None:
self._count_seen_keys.add(k)
self._fh.write(json.dumps(obj, default = str, ensure_ascii = False))
self._fh.write(json.dumps(obj, default = str, ensure_ascii = self._ensure_ascii))
self._fh.write("\n")
self._fh.flush()
return True

View file

@ -30,6 +30,8 @@ class UnstructuredSeedReader(SeedReader[UnstructuredSeedSource]):
meta = json_mod.loads(meta_path.read_text(encoding = "utf-8"))
orig_name = meta.get("original_filename", path_obj.name)
except (json_mod.JSONDecodeError, OSError, UnicodeDecodeError):
# Undecodable metadata is as malformed as invalid JSON, so
# fall back to the file's own name rather than abort the seed.
pass
file_entries.append((path_obj, orig_name))

View file

@ -4434,7 +4434,7 @@ def _effective_load_in_4bit(config: ModelConfig, requested: bool) -> bool:
if not adapter_cfg_path.exists():
return load_in_4bit
try:
with open(adapter_cfg_path, encoding = "utf-8") as f:
with open(adapter_cfg_path, encoding = "utf-8-sig") as f:
adapter_cfg = json.load(f)
if not isinstance(adapter_cfg, dict): # malformed -> keep requested
return load_in_4bit

View file

@ -722,7 +722,7 @@ def _scan_ollama_dir(ollama_dir: Path, limit: Optional[int] = None) -> List[Loca
stem_hash = hashlib.sha256(manifest_key.encode()).hexdigest()[:10]
try:
manifest = json.loads(tag_file.read_text(encoding = "utf-8"))
manifest = json.loads(tag_file.read_text(encoding = "utf-8-sig"))
except (json.JSONDecodeError, OSError, UnicodeDecodeError) as e:
logger.debug(
"Skipping unreadable/invalid Ollama manifest %s: %s",
@ -738,7 +738,7 @@ def _scan_ollama_dir(ollama_dir: Path, limit: Optional[int] = None) -> List[Loca
config_blob = blobs_dir / config_digest.replace(":", "-")
if config_blob.is_file():
try:
cfg = json.loads(config_blob.read_text(encoding = "utf-8"))
cfg = json.loads(config_blob.read_text(encoding = "utf-8-sig"))
model_type = cfg.get("model_type", "")
file_type = cfg.get("file_type", "")
except (json.JSONDecodeError, OSError, UnicodeDecodeError) as e:
@ -1042,7 +1042,7 @@ def _dir_has_downloaded_model(directory: Path, max_entries: int = 4000) -> bool:
if not m.is_file():
continue
try:
manifest = json.loads(m.read_text(encoding = "utf-8"))
manifest = json.loads(m.read_text(encoding = "utf-8-sig"))
except (json.JSONDecodeError, OSError, ValueError):
continue
for layer in manifest.get("layers") or []:
@ -3360,6 +3360,8 @@ def _wsl_reveal_in_explorer(path: Path) -> bool:
["wslpath", "-w", str(path)],
capture_output = True,
text = True,
encoding = "utf-8",
errors = "replace",
check = True,
timeout = 10,
).stdout.strip()

View file

@ -786,6 +786,8 @@ def _remove_pid_file():
stored = _PID_FILE.read_text(encoding = "utf-8").strip()
if stored == str(os.getpid()):
_PID_FILE.unlink(missing_ok = True)
# Runs first in _graceful_shutdown: a corrupt PID file raising here would
# abandon the children the rest of that function exists to kill.
except (OSError, UnicodeDecodeError):
pass

View file

@ -0,0 +1,195 @@
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
"""Model text stays intact when it carries non-ASCII.
``open()`` and ``Path.read_text()`` fall back to ``locale.getencoding()`` when
no ``encoding`` is passed. On Windows that is the ANSI codepage, not UTF-8, so
a chat template or model config holding ``ä ö ü `` mojibakes or raises
``UnicodeDecodeError``. These files are UTF-8, so the reads must say so.
Each fixture writes raw UTF-8 (``ensure_ascii = False``), matching what
Hugging Face actually ships, rather than ASCII ``\\uXXXX`` escapes.
"""
from __future__ import annotations
import json
import subprocess
import sys
import textwrap
from pathlib import Path
BACKEND_ROOT = Path(__file__).resolve().parent.parent
def test_config_json_round_trips_non_ascii(tmp_path: Path) -> None:
from utils import transformers_version
name = "Modell für Grüße 世界"
(tmp_path / "config.json").write_text(
json.dumps({"model_type": "llama", "_name_or_path": name}, ensure_ascii = False),
encoding = "utf-8",
)
transformers_version._config_json_cache.clear()
cfg = transformers_version._load_config_json(str(tmp_path))
assert cfg is not None
assert cfg["_name_or_path"] == name
def test_tokenizer_config_round_trips_non_ascii_chat_template(tmp_path: Path) -> None:
"""Chat templates commonly hold ``→`` and smart quotes, which cp1252 mangles."""
from utils import transformers_version
template = "{{ '→ Grüße 世界' }}"
(tmp_path / "tokenizer_config.json").write_text(
json.dumps(
{"tokenizer_class": "TokenizersBackend", "chat_template": template},
ensure_ascii = False,
),
encoding = "utf-8",
)
transformers_version._tokenizer_class_cache.clear()
assert transformers_version._check_tokenizer_config_needs_v5(str(tmp_path)) is True
def test_config_json_survives_a_utf8_bom(tmp_path: Path) -> None:
"""Notepad wrote "UTF-8 with BOM" by default for years, so hand-edited
configs on Windows carry one. Plain utf-8 keeps the BOM and json.load then
fails on it; utf-8-sig strips it and is identical otherwise."""
from utils import transformers_version
name = "Grüße 世界"
(tmp_path / "config.json").write_text(
json.dumps({"model_type": "llama", "_name_or_path": name}, ensure_ascii = False),
encoding = "utf-8-sig",
)
transformers_version._config_json_cache.clear()
cfg = transformers_version._load_config_json(str(tmp_path))
assert cfg is not None
assert cfg["_name_or_path"] == name
def test_remote_code_scan_reads_non_ascii_sources(tmp_path: Path) -> None:
"""A German Windows profile also puts umlauts in the model sources scanned."""
from utils.security import remote_code_scan
source = "# Grüße über Öl\nVALUE = '世界'\n"
# newline = "" pins the bytes on disk, so Windows line end translation cannot make the
# read back differ by \r. open() because Path.write_text() only grew newline in 3.10.
with open(
tmp_path / "modeling_custom.py",
"w",
encoding = "utf-8",
newline = "",
) as handle:
handle.write(source)
files = remote_code_scan.repo_remote_code_files(str(tmp_path))
assert files["modeling_custom.py"] == source
def test_model_config_reads_do_not_rely_on_the_locale_encoding(tmp_path: Path) -> None:
"""The reads above pass anywhere the locale is already UTF-8, which hides
the Windows bug on Linux and macOS. ``-X warn_default_encoding`` makes
CPython flag any text I/O that falls back to the locale, so this fails on
every platform if an ``encoding`` argument goes missing again."""
# The readers swallow exceptions, so record the warnings instead of raising.
script = textwrap.dedent(
f"""
import sys, warnings
sys.path.insert(0, {str(BACKEND_ROOT)!r})
from utils import transformers_version
target = {str(tmp_path)!r}
with warnings.catch_warnings(record = True) as caught:
warnings.simplefilter("always")
transformers_version._config_json_cache.clear()
transformers_version._tokenizer_class_cache.clear()
assert transformers_version._load_config_json(target) is not None
assert transformers_version._check_tokenizer_config_needs_v5(target) is True
missing = [str(w.message) for w in caught if w.category is EncodingWarning]
if missing:
sys.exit("text I/O fell back to the locale encoding: " + "; ".join(missing))
"""
)
for name, payload in (
("config.json", {"model_type": "llama", "_name_or_path": "Grüße"}),
("tokenizer_config.json", {"tokenizer_class": "TokenizersBackend"}),
):
(tmp_path / name).write_text(json.dumps(payload, ensure_ascii = False), encoding = "utf-8")
result = subprocess.run(
[sys.executable, "-X", "warn_default_encoding", "-c", script],
capture_output = True,
text = True,
encoding = "utf-8",
errors = "replace",
timeout = 120,
)
assert result.returncode == 0, result.stderr
def test_utf8_child_env_round_trips_non_ascii(tmp_path: Path) -> None:
"""A Python child encodes stdout with its locale unless told otherwise, so
reading its pipe as utf-8 needs the child told to emit utf-8."""
from utils.child_stdio import utf8_child_env
payload = "Grüße über Öl → 世界"
child = tmp_path / "child.py"
child.write_text("import sys\nsys.stdout.write(" + repr(payload) + ")\n", encoding = "utf-8")
env = utf8_child_env()
assert env["PYTHONIOENCODING"] == "utf-8"
proc = subprocess.run(
[sys.executable, str(child)],
capture_output = True,
text = True,
encoding = "utf-8",
errors = "replace",
env = env,
timeout = 120,
)
assert proc.returncode == 0, proc.stderr
assert proc.stdout == payload
def test_python_children_are_told_to_emit_utf8() -> None:
"""Any child we decode as utf-8 must also be told to write utf-8, or a
cp1252 console silently mangles what it prints."""
import ast
offenders: list[str] = []
for path in sorted(BACKEND_ROOT.rglob("*.py")):
parts = path.relative_to(BACKEND_ROOT).parts
if any(p in ("tests", "node_modules", "plugins", "__pycache__") for p in parts):
continue
source = path.read_text(encoding = "utf-8")
for node in ast.walk(ast.parse(source, filename = str(path))):
if not isinstance(node, ast.Call):
continue
func = node.func
if not (isinstance(func, ast.Attribute) and func.attr in ("run", "Popen")):
continue
segment = ast.get_source_segment(source, node) or ""
if "sys.executable" not in segment or 'encoding = "utf-8"' not in segment:
continue
if "utf8_child_env" in segment or "PYTHONIOENCODING" in segment:
continue
offenders.append(f"{path.name}:{node.lineno}")
assert not offenders, (
"these spawn a Python child and decode it as utf-8 without setting the "
"child's own stdio encoding; wrap env in utf8_child_env():\n " + "\n ".join(offenders)
)

View file

@ -45,8 +45,20 @@ def _build_structlog_stub():
_maybe_stub("loggers", _build_loggers_stub)
_maybe_stub("structlog", _build_structlog_stub)
import pytest
import utils.hardware.hardware as hw # noqa: E402
# The DRM/KFD readers below are Linux-only in production: _rocm_linux_amdgpu_cards and
# _rocm_linux_sysfs_vram_by_pci_gb return early unless platform.system() is "Linux", and
# _rocm_kfd_gpu_pci_ids only ever globs /sys/class/kfd. Their fake sysfs tree needs PCI
# addresses like "0000:00:02.0" as directory names and POSIX separators in the paths the
# readers match; Windows permits neither, so the tree cannot be represented there.
linux_only = pytest.mark.skipif(
not sys.platform.startswith("linux"),
reason = "covers Linux-only DRM/KFD sysfs parsing driven by a fake /sys tree",
)
def _device(
index,
@ -99,6 +111,7 @@ def _fake_drm(tmp_path, monkeypatch, cards):
return card_paths
@linux_only
def test_linux_vram_keyed_by_pci_excludes_foreign_adapters(monkeypatch, tmp_path):
# Foreign (non-amdgpu) adapters contribute no entry, so they cannot shift ordinals.
monkeypatch.setattr(hw.platform, "system", lambda: "Linux")
@ -117,6 +130,7 @@ def test_linux_vram_keyed_by_pci_excludes_foreign_adapters(monkeypatch, tmp_path
}
@linux_only
def test_linux_vram_omits_bad_cards_without_shifting(monkeypatch, tmp_path):
# A zero-total card has no entry; identity keying means its absence renumbers nothing.
monkeypatch.setattr(hw.platform, "system", lambda: "Linux")
@ -131,6 +145,7 @@ def test_linux_vram_omits_bad_cards_without_shifting(monkeypatch, tmp_path):
assert hw._rocm_linux_sysfs_vram_by_pci_gb() == {"0000:41:00.0": (2.0, 16.0)}
@linux_only
def test_linux_vram_omits_amd_card_without_vram_files(monkeypatch, tmp_path):
# An APU with no mem_info_vram_* files has no entry; the discrete card keeps its address.
monkeypatch.setattr(hw.platform, "system", lambda: "Linux")
@ -174,6 +189,7 @@ def _fake_kfd(tmp_path, monkeypatch, nodes):
return node_paths
@linux_only
def test_kfd_lists_gpu_nodes_in_device_order(monkeypatch, tmp_path):
# The CPU node (simd_count 0) takes no ordinal; GPU nodes in node-id order are HIP's order.
monkeypatch.setattr(hw.platform, "system", lambda: "Linux")
@ -189,12 +205,14 @@ def test_kfd_lists_gpu_nodes_in_device_order(monkeypatch, tmp_path):
assert hw._rocm_kfd_gpu_pci_ids() == ["0000:03:00.0", "0000:41:00.0"]
@linux_only
def test_kfd_decodes_domain_device_and_function(monkeypatch, tmp_path):
monkeypatch.setattr(hw.platform, "system", lambda: "Linux")
_fake_kfd(tmp_path, monkeypatch, [(1, 64, (0xC1 << 8) | (0x1F << 3) | 5, 0x1234, _AMD)])
assert hw._rocm_kfd_gpu_pci_ids() == ["1234:c1:1f.5"]
@linux_only
def test_kfd_skips_non_amd_gpu_nodes(monkeypatch, tmp_path):
# An NVIDIA KFD node is not a HIP device: it must take no ordinal, else it
# shifts every AMD GPU and ROCm device 1 resolves to AMD GPU 0.
@ -212,6 +230,7 @@ def test_kfd_skips_non_amd_gpu_nodes(monkeypatch, tmp_path):
assert hw._rocm_kfd_gpu_pci_ids() == ["0000:03:00.0", "0000:41:00.0"]
@linux_only
def test_kfd_fails_closed_when_a_gpu_has_no_location(monkeypatch, tmp_path):
# Dropping an unplaceable AMD GPU shifts later ordinals; fail closed for the whole map.
monkeypatch.setattr(hw.platform, "system", lambda: "Linux")
@ -226,6 +245,7 @@ def test_kfd_fails_closed_when_a_gpu_has_no_location(monkeypatch, tmp_path):
assert hw._rocm_kfd_gpu_pci_ids() == []
@linux_only
def test_kfd_fails_closed_when_a_node_is_unreadable(monkeypatch, tmp_path):
# An unreadable node could be a GPU; assuming otherwise would shift ordinals.
monkeypatch.setattr(hw.platform, "system", lambda: "Linux")
@ -241,6 +261,23 @@ def test_kfd_fails_closed_when_a_node_is_unreadable(monkeypatch, tmp_path):
assert hw._rocm_kfd_gpu_pci_ids() == []
@linux_only
def test_kfd_fails_closed_when_a_node_does_not_decode(monkeypatch, tmp_path):
# UnicodeDecodeError is a ValueError, so it slips past `except OSError` and
# would shift every later HIP ordinal.
monkeypatch.setattr(hw.platform, "system", lambda: "Linux")
paths = _fake_kfd(
tmp_path,
monkeypatch,
[
(1, 304, (0x03 << 8) | 0, 0, _AMD),
(2, 304, (0x41 << 8) | 0, 0, _AMD),
],
)
(Path(paths[0]) / "properties").write_bytes(b"simd_count 304\nvendor_id \x80\xff\n")
assert hw._rocm_kfd_gpu_pci_ids() == []
def test_kfd_absent_yields_no_device_order(monkeypatch):
monkeypatch.setattr(hw.glob, "glob", lambda pattern: [])
assert hw._rocm_kfd_gpu_pci_ids() == []
@ -422,6 +459,10 @@ def test_visible_utilization_rocm_fallback_overlays(monkeypatch):
):
monkeypatch.delenv(_var, raising = False)
monkeypatch.setattr(hw, "IS_ROCM", True)
# No AMD adapter data on this host. On Windows this branch runs ahead of the torch
# fallback under test, and probing it imports torch, which the CI runner does not
# install. Off Windows the real function is never reached, so this changes nothing.
monkeypatch.setattr(hw, "_rocm_windows_per_device_vram", lambda ids: [])
monkeypatch.setattr(hw, "get_device", lambda: hw.DeviceType.CUDA)
monkeypatch.setattr(hw, "_smi_query", lambda *a, **k: None) # amd-smi unavailable
monkeypatch.setattr(
@ -450,6 +491,10 @@ def test_visible_utilization_rocm_fallback_overlays(monkeypatch):
def test_visible_utilization_relative_index_skips_overlay(monkeypatch):
# UUID/MIG mask gives relative indices; the overlay matches physical index, so it must not run.
monkeypatch.setattr(hw, "IS_ROCM", True)
# No AMD adapter data on this host. On Windows this branch runs ahead of the torch
# fallback under test, and probing it imports torch, which the CI runner does not
# install. Off Windows the real function is never reached, so this changes nothing.
monkeypatch.setattr(hw, "_rocm_windows_per_device_vram", lambda ids: [])
monkeypatch.setattr(hw, "get_device", lambda: hw.DeviceType.CUDA)
monkeypatch.setattr(hw, "_smi_query", lambda *a, **k: None)
monkeypatch.setattr(

View file

@ -0,0 +1,809 @@
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
"""Text I/O must name its encoding, or Windows silently uses the ANSI codepage.
``open()``, ``Path.read_text()`` and ``subprocess(text = True)`` fall back to
``locale.getencoding()`` when no ``encoding`` is passed. On Windows that is
cp1252 (or cp932, cp1251, ... by system locale), not UTF-8, so a chat template,
model config or path containing ``ä ö ü `` mojibakes or raises
``UnicodeDecodeError`` mid-load. Studio's files are UTF-8, so say so.
"""
from __future__ import annotations
import ast
import importlib.util
import json
import os
from pathlib import Path
from types import SimpleNamespace
import pytest
BACKEND_ROOT = Path(__file__).resolve().parent.parent
# Not runtime source. Shipped plugins under plugins/*/src are, so only builds are skipped.
_SKIPPED_DIRS = ("node_modules", "build", "tests", "__pycache__")
# Path.open()'s signature is what tells it apart from other libraries' open(),
# e.g. fitz.open(stream=...) and av.open(..., metadata_errors=...).
_FILE_MODE_CHARS = set("rwxabt+")
_PATH_OPEN_ARGS = ("mode", "buffering", "encoding", "errors", "newline")
_PATH_OPEN_KWARGS = set(_PATH_OPEN_ARGS)
_PATH_OPEN_ENCODING_ARG = _PATH_OPEN_ARGS.index("encoding")
_SUBPROCESS_CALLS = {"run", "Popen", "check_output", "check_call", "call"}
# open(file, mode, buffering, encoding, ...), and os.fdopen forwards the same
# signature with a descriptor in place of the path.
_OPEN_ENCODING_ARG = 3
def _studio_sources() -> list[Path]:
return [
path
for path in sorted(BACKEND_ROOT.rglob("*.py"))
if not any(part in _SKIPPED_DIRS for part in path.relative_to(BACKEND_ROOT).parts)
]
def _has_keyword(node: ast.Call, name: str) -> bool:
return any(keyword.arg == name for keyword in node.keywords)
def _mode_is_binary(node: ast.Call) -> bool:
mode: str | None = None
if len(node.args) >= 2 and isinstance(node.args[1], ast.Constant):
value = node.args[1].value
mode = value if isinstance(value, str) else None
for keyword in node.keywords:
if keyword.arg == "mode" and isinstance(keyword.value, ast.Constant):
value = keyword.value.value
if isinstance(value, str):
mode = value
return bool(mode and "b" in mode)
def _open_has_encoding(node: ast.Call) -> bool:
"""open()/os.fdopen() also take encoding positionally: open(p, "w", 1, "utf-8")."""
return _has_keyword(node, "encoding") or len(node.args) > _OPEN_ENCODING_ARG
def _path_open_mode(node: ast.Call) -> str | None:
if node.args and isinstance(node.args[0], ast.Constant):
value = node.args[0].value
if isinstance(value, str):
return value
for keyword in node.keywords:
if keyword.arg == "mode" and isinstance(keyword.value, ast.Constant):
value = keyword.value.value
if isinstance(value, str):
return value
return None
def _is_path_open(node: ast.Call) -> bool:
"""True only for calls matching ``Path.open``'s signature."""
if len(node.args) > len(_PATH_OPEN_ARGS):
return False
if any(k.arg not in _PATH_OPEN_KWARGS for k in node.keywords):
return False
mode = _path_open_mode(node)
if mode is not None:
return bool(mode) and set(mode) <= _FILE_MODE_CHARS
return not node.args
def _path_open_has_encoding(node: ast.Call) -> bool:
"""Path.open() also takes encoding positionally: open("w", 1, "utf-8")."""
return _has_keyword(node, "encoding") or len(node.args) > _PATH_OPEN_ENCODING_ARG
def _call_name(node: ast.Call) -> str | None:
func = node.func
if isinstance(func, ast.Name):
return func.id
if isinstance(func, ast.Attribute):
return func.attr
return None
def _subprocess_names(tree: ast.AST) -> set[str]:
"""Names subprocess is reachable under here, e.g. `import subprocess as _sp`."""
names = set()
for node in ast.walk(tree):
if isinstance(node, ast.Import):
for alias in node.names:
if alias.name == "subprocess":
names.add(alias.asname or alias.name)
return names
def _subprocess_aliases(tree: ast.AST, names: set[str]) -> set[str]:
"""Plain names bound to a subprocess callable, called without the module.
``install_wheel(run = subprocess.run)`` calls its injected ``run`` as a bare
name, so matching only the attribute form leaves those installer calls
unguarded. Imports, assignments and parameter defaults all bind one.
"""
def _is_bound(value: ast.expr | None) -> bool:
return (
isinstance(value, ast.Attribute)
and value.attr in _SUBPROCESS_CALLS
and isinstance(value.value, ast.Name)
and value.value.id in names
)
aliases: set[str] = set()
for node in ast.walk(tree):
if isinstance(node, ast.ImportFrom) and node.module == "subprocess":
aliases.update(a.asname or a.name for a in node.names if a.name in _SUBPROCESS_CALLS)
elif isinstance(node, ast.Assign) and _is_bound(node.value):
aliases.update(t.id for t in node.targets if isinstance(t, ast.Name))
elif isinstance(node, ast.AnnAssign) and _is_bound(node.value):
if isinstance(node.target, ast.Name):
aliases.add(node.target.id)
elif isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)):
args = node.args
positional = args.posonlyargs + args.args
# Defaults cover the tail of the positional parameters; kw_defaults
# is aligned with kwonlyargs already, holding None where absent.
padded = [None] * (len(positional) - len(args.defaults)) + list(args.defaults)
pairs = list(zip(positional, padded)) + list(zip(args.kwonlyargs, args.kw_defaults))
aliases.update(arg.arg for arg, default in pairs if _is_bound(default))
return aliases
def _is_subprocess_call(node: ast.Call, names: set[str], aliases: set[str]) -> bool:
func = node.func
if isinstance(func, ast.Name):
return func.id in aliases
if not isinstance(func, ast.Attribute) or func.attr not in _SUBPROCESS_CALLS:
return False
value = func.value
return isinstance(value, ast.Name) and value.id in names
def _text_mode_subprocess(node: ast.Call) -> bool:
for keyword in node.keywords:
if keyword.arg not in ("text", "universal_newlines"):
continue
if isinstance(keyword.value, ast.Constant) and keyword.value.value is True:
return True
return False
def _text_mode_dict(node: ast.Dict) -> bool:
"""A ``{"text": True, ...}`` literal with no "encoding" key."""
keys = [k.value for k in node.keys if isinstance(k, ast.Constant)]
if "encoding" in keys:
return False
for key, value in zip(node.keys, node.values):
if not isinstance(key, ast.Constant) or key.value not in (
"text",
"universal_newlines",
):
continue
if isinstance(value, ast.Constant) and value.value is True:
return True
return False
def _splatted_names(tree: ast.AST) -> set[str]:
"""Names handed to a call as ``**name``."""
names = set()
for node in ast.walk(tree):
if isinstance(node, ast.Call):
for keyword in node.keywords:
if keyword.arg is None and isinstance(keyword.value, ast.Name):
names.add(keyword.value.id)
return names
def _encoding_assigned_later(tree: ast.AST, name: str) -> bool:
"""``name["encoding"] = ...`` somewhere, so the literal need not carry it."""
for node in ast.walk(tree):
if not isinstance(node, ast.Subscript) or not isinstance(node.ctx, ast.Store):
continue
target, key = node.value, node.slice
if isinstance(target, ast.Name) and target.id == name:
if isinstance(key, ast.Constant) and key.value == "encoding":
return True
return False
def _splatted_kwargs_offenders(tree: ast.AST) -> list[ast.Dict]:
"""Text-mode kwargs built in a dict and splatted into a call.
Kwargs are collected in a dict and splatted (``run(cmd, **run_kwargs)``)
where a branch has to add a timeout or an env, and the call is often through
a helper, so neither the callee nor the keywords are visible at the call
site. Only dicts that reach a call this way are judged: an unrelated payload
that happens to carry ``"text": True`` is not subprocess configuration.
"""
found = []
# ``run(cmd, **{...})``: the literal is at the call already.
for node in ast.walk(tree):
if not isinstance(node, ast.Call):
continue
for keyword in node.keywords:
if keyword.arg is None and isinstance(keyword.value, ast.Dict):
if _text_mode_dict(keyword.value):
found.append(keyword.value)
splatted = _splatted_names(tree)
if not splatted:
return found
for node in ast.walk(tree):
targets = []
if isinstance(node, ast.Assign):
targets = [t for t in node.targets if isinstance(t, ast.Name)]
elif isinstance(node, ast.AnnAssign) and isinstance(node.target, ast.Name):
targets = [node.target]
if not targets or not isinstance(node.value, ast.Dict):
continue
if not _text_mode_dict(node.value):
continue
for target in targets:
if target.id in splatted and not _encoding_assigned_later(tree, target.id):
found.append(node.value)
break
return found
def _offenders(path: Path) -> list[str]:
source = path.read_text(encoding = "utf-8")
tree = ast.parse(source, filename = str(path))
subprocess_names = _subprocess_names(tree)
subprocess_aliases = _subprocess_aliases(tree, subprocess_names)
found: list[str] = []
for node in _splatted_kwargs_offenders(tree):
found.append(
f"{path.name}:{node.lineno}: subprocess kwargs with text = True and no encoding"
)
for node in ast.walk(tree):
if not isinstance(node, ast.Call):
continue
name = _call_name(node)
if _is_subprocess_call(node, subprocess_names, subprocess_aliases):
if _text_mode_subprocess(node) and not _has_keyword(node, "encoding"):
found.append(f"{path.name}:{node.lineno}: subprocess(text = True) without encoding")
continue
if name == "open" and isinstance(node.func, ast.Name):
if _mode_is_binary(node) or _open_has_encoding(node):
continue
found.append(f"{path.name}:{node.lineno}: open() without encoding")
continue
# os.fdopen(fd, "w") is open() on a descriptor, so text mode takes the
# same locale default. Its mode defaults to "r", i.e. text, like open's.
if name == "fdopen":
if _mode_is_binary(node) or _open_has_encoding(node):
continue
found.append(f"{path.name}:{node.lineno}: os.fdopen() without encoding")
continue
if name == "open" and isinstance(node.func, ast.Attribute):
if not _is_path_open(node) or _path_open_has_encoding(node):
continue
if _path_open_mode(node) and "b" in _path_open_mode(node):
continue
found.append(f"{path.name}:{node.lineno}: Path.open() without encoding")
continue
if name in ("read_text", "write_text") and isinstance(node.func, ast.Attribute):
if _has_keyword(node, "encoding"):
continue
# importlib.metadata Distribution.read_text() takes no encoding kwarg.
if isinstance(node.func.value, ast.Name) and node.func.value.id == "dist":
continue
found.append(f"{path.name}:{node.lineno}: {name}() without encoding")
return found
@pytest.mark.parametrize("path", _studio_sources(), ids = lambda p: str(p.name))
def test_text_io_names_its_encoding(path: Path) -> None:
offenders = _offenders(path)
assert not offenders, (
"Text I/O without an explicit encoding falls back to the Windows ANSI "
'codepage and corrupts non-ASCII (ä ö ü → 世). Pass encoding = "utf-8":\n '
+ "\n ".join(offenders)
)
_STATE_STORE = (
BACKEND_ROOT
/ "plugins/data-designer-github-repo-seed/src"
/ "data_designer_github_repo_seed/scraper_impl/state_store.py"
)
def _load_state_store(codepage: str):
"""Load state_store with the writing machine's codepage pinned."""
spec = importlib.util.spec_from_file_location(f"state_store_{codepage}", _STATE_STORE)
module = importlib.util.module_from_spec(spec)
spec.loader.exec_module(module)
module.locale = SimpleNamespace(
getencoding = lambda: codepage,
getpreferredencoding = lambda _ = True: codepage,
)
return module
@pytest.mark.parametrize(
("codepage", "name"), [("cp1252", "Jürgen"), ("cp1251", "Юрий"), ("cp932", "田中")]
)
def test_resuming_a_legacy_jsonl_keeps_one_encoding(
tmp_path: Path, codepage: str, name: str
) -> None:
"""A scrape written before UTF-8 was explicit must resume, not duplicate."""
path = tmp_path / "out.jsonl"
records = [{"id": 1, "author": name}, {"id": 2, "author": name}]
body = "".join(json.dumps(r, ensure_ascii = False) + "\n" for r in records)
path.write_bytes(body.encode(codepage))
before = path.read_bytes()
writer = _load_state_store(codepage).JsonlWriter(path)
try:
# Seen keys survive the resume, so a repeat is refused, not appended.
assert writer.has("id:1") and writer.has("id:2")
assert writer.write(records[0]) is False
assert writer.write({"id": 3, "author": name}) is True
finally:
writer.close()
# Never converted, so it still reads in its own codepage; the append is ASCII.
blob = path.read_bytes()
assert blob.startswith(before)
assert blob[len(before) :].isascii()
lines = [json.loads(x) for x in blob.decode(codepage).splitlines() if x.strip()]
assert len(lines) == 3
assert [line["author"] for line in lines] == [name] * 3
def test_a_coincidentally_utf8_legacy_line_is_left_alone(tmp_path: Path) -> None:
"""cp1251 `Р°` is D0 B0, which is also UTF-8 `а`, and nothing can tell them apart."""
path = tmp_path / "out.jsonl"
ambiguous = "Р°"
assert ambiguous.encode("cp1251").decode("utf-8") == "а" # the trap
authors = ["Привет", "Здравствуйте", "Москва", ambiguous]
path.write_bytes(
b"".join(
json.dumps({"id": i, "author": a}, ensure_ascii = False).encode("cp1251") + b"\n"
for i, a in enumerate(authors)
)
)
before = path.read_bytes()
_load_state_store("cp1251").JsonlWriter(path).close()
# Untouched, so the ambiguity never had to be resolved.
assert path.read_bytes() == before
rows = [json.loads(x) for x in path.read_text(encoding = "cp1251").splitlines() if x.strip()]
assert [row["author"] for row in rows] == authors
@pytest.mark.parametrize(
("codepage", "word"), [("cp1251", "Привет"), ("cp932", "こんにちは"), ("cp1252", "Jürgen")]
)
def test_a_moved_shard_is_not_rewritten_by_guesswork(
tmp_path: Path, codepage: str, word: str
) -> None:
"""Off the writing machine there is no codepage to attribute the file to."""
path = tmp_path / "out.jsonl"
# Two records: a lone non-UTF-8 line would count as damage, not legacy.
path.write_bytes(
b"".join(
json.dumps({"id": i, "author": word}, ensure_ascii = False).encode(codepage) + b"\n"
for i in (1, 4)
)
)
before = path.read_bytes()
# A UTF-8 host: latin-1 would read cp1251 `Привет` back as `Ïðèâåò`.
writer = _load_state_store("utf-8").JsonlWriter(path)
try:
assert writer.has("id:1") # ASCII keys still recover
assert writer.write({"id": 2, "author": "Grüße"}) is True
finally:
writer.close()
blob = path.read_bytes()
assert blob.startswith(before) # never rewritten
assert blob[len(before) :].isascii() # appended as \uXXXX, so no second encoding
rows = [json.loads(x) for x in blob.decode(codepage).splitlines() if x.strip()]
assert [row["author"] for row in rows] == [word, word, "Grüße"]
def test_an_all_ambiguous_shard_still_gets_ascii_appends(tmp_path: Path) -> None:
"""Every line valid under both readings still means the append must not pick one."""
path = tmp_path / "out.jsonl"
ambiguous = "Р°" # cp1251 D0 B0, also valid UTF-8 for "а"
path.write_bytes(
b"".join(
json.dumps({"id": i, "a": ambiguous}, ensure_ascii = False).encode("cp1251") + b"\n"
for i in range(3)
)
)
before = path.read_bytes()
writer = _load_state_store("cp1251").JsonlWriter(path)
try:
assert writer.write({"id": 9, "a": "世界"}) is True
finally:
writer.close()
blob = path.read_bytes()
assert blob.startswith(before)
# ASCII, so the appended record survives whichever reading is chosen.
assert blob[len(before) :].isascii()
for codec in ("cp1251", "utf-8"):
rows = [json.loads(x) for x in blob.decode(codec).splitlines() if x.strip()]
assert rows[-1]["a"] == "世界"
def test_a_damaged_line_in_an_ascii_shard_does_not_block_its_retry(tmp_path: Path) -> None:
"""With no non-ASCII records to outvote it, one damaged line is still damage."""
path = tmp_path / "out.jsonl"
path.write_bytes(
b'{"id": 1, "author": "alice"}\n'
+ b'{"id": 99, "author": "bad \x96 byte"}\n'
+ b'{"id": 2, "author": "bob"}\n'
)
writer = _load_state_store("cp1252").JsonlWriter(path)
try:
assert writer.has("id:1") and writer.has("id:2")
assert not writer.has("id:99")
assert writer.write({"id": 99, "author": "good byte"}) is True
finally:
writer.close()
def test_a_damaged_line_does_not_block_its_own_retry(tmp_path: Path) -> None:
"""Its key comes from the codepage reading, which a UTF-8 shard did not pick."""
path = tmp_path / "out.jsonl"
path.write_bytes(
json.dumps({"id": 1, "author": "Jürgen"}, ensure_ascii = False).encode()
+ b"\n"
+ b'{"id": 99, "author": "bad \x96 byte"}\n'
)
writer = _load_state_store("cp1252").JsonlWriter(path)
try:
assert writer.has("id:1")
assert not writer.has("id:99")
assert writer.write({"id": 99, "author": "good byte"}) is True
finally:
writer.close()
def test_one_damaged_byte_does_not_relabel_a_utf8_shard(tmp_path: Path) -> None:
"""A complete JSON line with a stray 0x96 parses as cp1252, but is only one vote."""
path = tmp_path / "out.jsonl"
healthy = ["Jürgen", "Grüße", "Björn"]
path.write_bytes(
json.dumps({"id": 0, "author": healthy[0]}, ensure_ascii = False).encode()
+ b"\n"
+ b'{"id": 99, "author": "bad \x96 byte"}\n'
+ b"".join(
json.dumps({"id": i, "author": a}, ensure_ascii = False).encode() + b"\n"
for i, a in enumerate(healthy[1:], start = 1)
)
)
before = path.read_bytes()
_load_state_store("cp1252").JsonlWriter(path).close()
# Untouched, so the healthy records were never re-read as cp1252.
assert path.read_bytes() == before
rows = []
for line in path.read_bytes().splitlines():
try:
rows.append(json.loads(line.decode()))
except (UnicodeDecodeError, ValueError):
continue
assert [row["author"] for row in rows] == healthy
def test_a_torn_line_does_not_relabel_a_utf8_shard(tmp_path: Path) -> None:
"""One interrupted append must not get the whole shard read as cp1252."""
path = tmp_path / "out.jsonl"
good = [{"id": 1, "author": "Jürgen"}, {"id": 3, "author": "Grüße"}]
torn = '{"id": 2, "author": "Jürgen"}'.encode()[:-6] # cut mid-character
path.write_bytes(
json.dumps(good[0], ensure_ascii = False).encode()
+ b"\n"
+ torn
+ b"\n"
+ json.dumps(good[1], ensure_ascii = False).encode()
+ b"\n"
)
before = path.read_bytes()
writer = _load_state_store("cp1252").JsonlWriter(path)
try:
assert writer.has("id:1") and writer.has("id:3")
assert not writer.has("id:2") # torn line yields no key
finally:
writer.close()
# Untouched: no rewrite, so no record was re-encoded into mojibake.
after = path.read_bytes()
assert after.startswith(before)
assert "Jürgen".encode() in after
assert "Jürgen".encode("utf-8").decode("cp1252").encode() not in after
def test_an_undecodable_transport_marker_reads_as_unknown(tmp_path: Path) -> None:
"""Pinning the decode turns an undecodable marker into UnicodeDecodeError,
which is a ValueError and so is not an OSError. Before the pin those bytes
simply read as an unknown value and the caller safely purged and restarted
the partial download; letting the error escape aborts the transfer instead.
"""
import sys
backend = str(Path(__file__).resolve().parent.parent)
if backend not in sys.path:
sys.path.insert(0, backend)
from hub.utils import download_registry as registry
marker = tmp_path / ".transport"
marker.write_bytes(b"\x80\xffnative\n")
assert registry._read_marker_value(marker) is None
# A readable but unknown value takes the same path (the behaviour restored).
marker.write_text("something-else\n", encoding = "utf-8")
assert registry._read_marker_value(marker) is None
def test_a_torn_cache_ref_reads_as_not_cached(tmp_path: Path, monkeypatch) -> None:
"""hf_cache_snapshot_dir answers "is this model already on disk", and the
offline embedding checks turn a raise into a 500. A refs/main holding a byte
the codepage used to decode into a nonsense commit simply missed the snapshot
dir before the pin; it has to keep missing it."""
import sys
backend = str(Path(__file__).resolve().parent.parent)
if backend not in sys.path:
sys.path.insert(0, backend)
from utils import utils as backend_utils
good_root = tmp_path / "good"
torn_root = tmp_path / "torn"
for root, ref_bytes in ((torn_root, b"\x80\xff\n"), (good_root, b"abc123\n")):
repo = root / "models--Org--Model"
(repo / "refs").mkdir(parents = True)
(repo / "refs" / "main").write_bytes(ref_bytes)
(good_root / "models--Org--Model" / "snapshots" / "abc123").mkdir(parents = True)
monkeypatch.setattr(backend_utils, "_hf_cache_roots", lambda: [torn_root])
assert backend_utils.hf_cache_snapshot_dir("Org/Model") is None
# The torn root is skipped, not fatal: a healthy second root still answers.
monkeypatch.setattr(backend_utils, "_hf_cache_roots", lambda: [torn_root, good_root])
found = backend_utils.hf_cache_snapshot_dir("Org/Model")
assert found is not None and found.name == "abc123"
def test_a_corrupt_pid_file_does_not_abort_shutdown(tmp_path: Path, monkeypatch) -> None:
"""_remove_pid_file runs first in _graceful_shutdown, so a raise there leaves
the inference, export, training and tunnel children alive."""
import sys
backend = str(Path(__file__).resolve().parent.parent)
if backend not in sys.path:
sys.path.insert(0, backend)
import run as studio_run
pid_file = tmp_path / "studio.pid"
pid_file.write_bytes(b"\x80\xff")
monkeypatch.setattr(studio_run, "_PID_FILE", pid_file)
studio_run._remove_pid_file()
# Not this process's PID, so the file stays; the point is that it returned.
assert pid_file.exists()
pid_file.write_text(str(os.getpid()), encoding = "utf-8")
studio_run._remove_pid_file()
assert not pid_file.exists()
def test_the_kwargs_guard_only_judges_dicts_that_reach_a_call(tmp_path: Path) -> None:
"""Only a dict splatted into a call is subprocess configuration. An unrelated
payload that happens to carry "text": True is not, and neither is one whose
encoding is filled in on a later line."""
cases = {
"offender.py": 'kw = {"text": True}\nrun(cmd, **kw)\n',
"annotated.py": 'kw: dict = {"universal_newlines": True}\nrun(cmd, **kw)\n',
"payload.py": 'payload = {"text": True}\nrequests.post(url, json = payload)\n',
"inline.py": 'run(cmd, **{"text": True})\n',
"later.py": 'kw = {"text": True}\nkw["encoding"] = "utf-8"\nrun(cmd, **kw)\n',
"carried.py": 'kw = {"text": True, "encoding": "utf-8"}\nrun(cmd, **kw)\n',
}
flagged = set()
for name, source in cases.items():
path = tmp_path / name
path.write_text(source, encoding = "utf-8")
if any("subprocess kwargs" in line for line in _offenders(path)):
flagged.add(name)
assert flagged == {"offender.py", "annotated.py", "inline.py"}, flagged
def test_the_guard_follows_subprocess_through_an_alias(tmp_path: Path) -> None:
"""install_wheel() takes ``run = subprocess.run`` and calls it as a bare
name, so an attribute-only match let both of its installer calls drop their
encoding unnoticed. A name bound to something else is still not subprocess."""
cases = {
"param_default.py": (
"import subprocess\n"
"def install(*, run = subprocess.run):\n"
" run(cmd, text = True)\n"
),
"assigned.py": "import subprocess\n_run = subprocess.run\n_run(cmd, text = True)\n",
"imported.py": "from subprocess import check_output\ncheck_output(cmd, text = True)\n",
"renamed.py": "from subprocess import run as _r\n_r(cmd, universal_newlines = True)\n",
"encoded.py": (
"import subprocess\n"
"def install(*, run = subprocess.run):\n"
' run(cmd, text = True, encoding = "utf-8")\n'
),
"unrelated.py": "def run(cmd, text = False):\n pass\nrun(cmd, text = True)\n",
}
flagged = set()
for name, source in cases.items():
path = tmp_path / name
path.write_text(source, encoding = "utf-8")
if any("subprocess(text = True)" in line for line in _offenders(path)):
flagged.add(name)
assert flagged == {"param_default.py", "assigned.py", "imported.py", "renamed.py"}, flagged
def test_the_guard_sees_os_fdopen(tmp_path: Path) -> None:
"""os.fdopen(fd, mode) is open() on a descriptor and takes the same locale
default in text mode, so leaving it out let the swap lock file keep the
codepage on the write side while its reader was pinned to UTF-8."""
cases = {
"text.py": 'import os\nos.fdopen(fd, "w")\n',
"default_mode.py": "import os\nos.fdopen(fd)\n", # defaults to "r", still text
"binary.py": 'import os\nos.fdopen(fd, "wb")\n',
"keyword.py": 'import os\nos.fdopen(fd, "w", encoding = "utf-8")\n',
"positional.py": 'import os\nos.fdopen(fd, "w", 1, "utf-8")\n',
}
flagged = set()
for name, source in cases.items():
path = tmp_path / name
path.write_text(source, encoding = "utf-8")
if any("fdopen" in line for line in _offenders(path)):
flagged.add(name)
assert flagged == {"text.py", "default_mode.py"}, flagged
def test_an_undecodable_bootstrap_password_does_not_stop_startup(
tmp_path: Path, monkeypatch
) -> None:
"""ensure_default_admin calls _load_bootstrap_password for every existing
admin and the lifespan calls that with no handler, so a raise here takes the
whole backend down instead of ignoring an unusable file."""
import sys
backend = str(Path(__file__).resolve().parent.parent)
if backend not in sys.path:
sys.path.insert(0, backend)
from auth import storage
pw_file = tmp_path / ".bootstrap_password"
pw_file.write_bytes(b"\x80\xffnot-utf8\n")
monkeypatch.setattr(storage, "_BOOTSTRAP_PW_PATH", pw_file)
assert storage._load_bootstrap_password() is None
# A readable one still loads, so this is a narrowing of failure, not of function.
pw_file.write_text("correct horse battery staple\n", encoding = "utf-8")
assert storage._load_bootstrap_password() == "correct horse battery staple"
def test_a_damaged_checkpoint_resets_instead_of_resuming_on_a_broken_cursor(tmp_path: Path) -> None:
"""A checkpoint holds only base64 cursors and booleans, so a codepage reading
can only ever add non-ASCII, never recover any. Resuming on a mojibaked cursor
sends GitHub one it answers with INVALID_CURSOR_ARGUMENTS, and the empty page
that comes back marks the stream done and skips the rest of it for good.
Dropping the checkpoint only replays pages the writers already dedup."""
module = _load_state_store("cp1252")
cursor = "Y3Vyc29yOnYyOpK0MjAxMi0wMi0xNlQwNjo1Mzo0MVrOADGL_A=="
healthy = json.dumps({"issues_cursor": cursor, "issues_done": False}, indent = 2)
path = tmp_path / "octocat__Hello-World.json"
path.write_text(healthy, encoding = "utf-8")
assert module.StateStore(path).get("issues_cursor") == cursor
# Written by a pre-UTF-8 release in the operator's codepage. Nothing is lost
# by reading UTF-8 only, because an all-ASCII document is the same bytes.
path.write_bytes(healthy.encode("cp1252"))
assert module.StateStore(path).get("issues_cursor") == cursor
# One damaged byte inside the cursor: still a whole JSON document under a
# single-byte codepage, so only refusing that reading resets the checkpoint.
raw = healthy.encode()
at = raw.index(b"MjAxMi0wMi0xNlQ") + 3
path.write_bytes(raw[:at] + b"\x96" + raw[at + 1 :])
assert json.loads(path.read_bytes().decode("latin-1"))["issues_cursor"] != cursor
store = module.StateStore(path)
assert store.all() == {}
assert store.get("issues_cursor") is None
def test_a_utf8_record_is_not_parsed_a_second_time(tmp_path: Path) -> None:
"""These shards reach gigabytes and every resume reads all of one, so a
record that already read as UTF-8 must not be decoded and parsed again under
the codepage. The legacy reading exists only to recover keys UTF-8 could not."""
module = _load_state_store("cp1252")
calls: list[str] = []
real_parse = module._parse
def counting_parse(raw, encoding):
calls.append(encoding)
return real_parse(raw, encoding)
module._parse = counting_parse
try:
healthy = json.dumps({"id": 1, "author": "Jürgen"}).encode("utf-8")
reading = module._read_line(healthy, "cp1252")
assert reading.as_utf8 == {"id": 1, "author": "Jürgen"}
assert calls == ["utf-8"], calls
# A line UTF-8 cannot read still falls through to the codepage, the whole point.
calls.clear()
legacy = json.dumps({"id": 2, "author": "Jürgen"}, ensure_ascii = False).encode("cp1252")
reading = module._read_line(legacy, "cp1252")
assert reading.as_utf8 is None
assert reading.as_legacy == {"id": 2, "author": "Jürgen"}
assert calls == ["utf-8", "cp1252"], calls
finally:
module._parse = real_parse
def _too_deeply_nested_json() -> str:
"""A JSON document nested past what this interpreter will descend into.
Probed rather than hardcoded: the depth json.loads gives up at is bounded by
sys.getrecursionlimit() up to 3.11 and by the C recursion limit from 3.12,
which sys.setrecursionlimit no longer moves and which varies by micro
version. That is ~995 on 3.9 and ~9999 on 3.13.
"""
depth = 1
while depth <= 1 << 17:
document = "[" * depth + "]" * depth
try:
json.loads(document)
except RecursionError:
return document
depth *= 2
pytest.skip("this interpreter parses arbitrarily nested JSON")
def test_an_unparseably_nested_document_is_discarded_not_raised(tmp_path: Path) -> None:
"""json.loads answers nesting it cannot descend with RecursionError, which is
a RuntimeError and so is neither a ValueError nor a UnicodeDecodeError.
_parse is called outside any other handler in both StateStore.__init__ and
JsonlWriter._scan_existing, so letting it escape aborts the scraper at
startup on a file the catch-all it replaced simply discarded."""
module = _load_state_store("cp1252")
nested = _too_deeply_nested_json()
checkpoint = tmp_path / "octocat__Hello-World.json"
checkpoint.write_text(nested, encoding = "utf-8")
assert module.StateStore(checkpoint).all() == {} # reset, not raised
shard = tmp_path / "out.jsonl"
shard.write_text(
nested + "\n" + json.dumps({"id": 1}) + "\n" + json.dumps({"id": 2}) + "\n",
encoding = "utf-8",
)
writer = module.JsonlWriter(shard)
try:
# Skipped like any other unreadable line, so its neighbours still yield the dedup
# keys that keep the resume from re-fetching them.
assert writer.has("id:1") and writer.has("id:2")
finally:
writer.close()

View file

@ -9,8 +9,28 @@ import sys
from typing import Any
from unittest import mock
import pytest
from core.training import worker
# The runtime install is Linux-only, so elsewhere these return before any status.
linux_only = pytest.mark.skipif(
not sys.platform.startswith("linux"),
reason = "the runtime flash-attn install is gated to Linux",
)
# causal-conv1d and flash-linear-attention are NOT Linux-gated: both installers bail out
# on `sys.platform == "win32"` alone (no prebuilt wheel for Windows) and run everywhere
# else, macOS included. linux_only here would skip cases that legitimately pass off Linux.
not_on_windows = pytest.mark.skipif(
sys.platform == "win32",
reason = (
"mirrors the sys.platform == 'win32' bail-out in "
"_ensure_flash_linear_attention_unconditional and "
"_ensure_causal_conv1d_fast_path"
),
)
def _missing_flash_attn_import():
real_import = builtins.__import__
@ -55,6 +75,7 @@ def test_should_try_runtime_flash_attn_install_threshold_and_skip(monkeypatch):
assert worker._should_try_runtime_flash_attn_install(32768) is False
@linux_only
def test_runtime_flash_attn_prefers_prebuilt_wheel(monkeypatch):
statuses: list[str] = []
@ -82,6 +103,7 @@ def test_runtime_flash_attn_prefers_prebuilt_wheel(monkeypatch):
assert statuses == ["Installing flash-attn for faster training..."]
@linux_only
def test_runtime_flash_attn_falls_back_to_pypi(monkeypatch):
calls: list[list[str]] = []
statuses: list[str] = []
@ -113,12 +135,7 @@ def test_runtime_flash_attn_falls_back_to_pypi(monkeypatch):
)
monkeypatch.setattr(worker, "install_wheel", mock.Mock())
def fake_run(
cmd,
stdout = None,
stderr = None,
text = None,
):
def fake_run(cmd, **kwargs):
calls.append(list(cmd))
return subprocess.CompletedProcess(cmd, 0, "")
@ -139,6 +156,7 @@ def test_runtime_flash_attn_skip_env_avoids_all_install_work(monkeypatch):
worker._sp.run.assert_not_called()
@not_on_windows
def test_causal_conv1d_fast_path_preserves_wheel_first_install_args(monkeypatch):
install_mock = mock.Mock(return_value = True)
monkeypatch.setattr(worker, "_install_package_wheel_first", install_mock)
@ -160,6 +178,7 @@ def test_causal_conv1d_fast_path_preserves_wheel_first_install_args(monkeypatch)
)
@not_on_windows
def test_causal_conv1d_fast_path_includes_qwen3_6_variants(monkeypatch):
install_mock = mock.Mock(return_value = True)
monkeypatch.setattr(worker, "_install_package_wheel_first", install_mock)
@ -225,6 +244,7 @@ def _pin_fla_model_types(monkeypatch):
)
@not_on_windows
def test_flash_linear_attention_installs_pinned_pair_for_qwen3_5(monkeypatch):
_pin_fla_model_types(monkeypatch)
monkeypatch.setattr(worker.shutil, "which", lambda name: "/usr/bin/uv")
@ -277,6 +297,7 @@ def test_flash_linear_attention_skips_for_ssm_only_models(monkeypatch):
run_mock.assert_not_called()
@not_on_windows
def test_flash_linear_attention_matches_full_qwen3_family(monkeypatch):
monkeypatch.setattr(worker.shutil, "which", lambda name: "/usr/bin/uv")
run_mock = mock.Mock(return_value = mock.Mock(returncode = 0, stdout = ""))
@ -331,6 +352,7 @@ def test_flash_linear_attention_skipped_via_env(monkeypatch):
run_mock.assert_not_called()
@not_on_windows
def test_flash_linear_attention_skipped_below_torch_2_7(monkeypatch):
_pin_fla_model_types(monkeypatch)
monkeypatch.delenv(worker._FLA_SKIP_ENV, raising = False)
@ -349,6 +371,7 @@ def test_flash_linear_attention_skipped_below_torch_2_7(monkeypatch):
assert any("torch>=" in s for s in statuses)
@not_on_windows
def test_flash_linear_attention_install_includes_einops(monkeypatch):
_pin_fla_model_types(monkeypatch)
monkeypatch.delenv(worker._FLA_SKIP_ENV, raising = False)
@ -375,6 +398,7 @@ def test_flash_linear_attention_install_includes_einops(monkeypatch):
assert f"fla-core=={worker._FLA_CORE_PACKAGE_VERSION}" in args
@not_on_windows
def test_flash_linear_attention_logs_post_install_import_failure(monkeypatch):
"""pip exits 0 but `import fla.modules` still fails (missing transitive)."""
_pin_fla_model_types(monkeypatch)
@ -421,6 +445,7 @@ def test_tilelang_backend_skipped_on_unsupported_linux_arch(monkeypatch):
run_mock.assert_not_called()
@linux_only
def test_tilelang_backend_pins_only_binary(monkeypatch):
_pin_fla_model_types(monkeypatch)
monkeypatch.delenv(worker._TILELANG_SKIP_ENV, raising = False)
@ -462,6 +487,7 @@ def _force_missing_tilelang_imports(monkeypatch):
monkeypatch.setattr(builtins, "__import__", fake_import)
@linux_only
def test_tilelang_backend_installs_pinned_pair_for_qwen3_5(monkeypatch):
_pin_fla_model_types(monkeypatch)
monkeypatch.delenv(worker._TILELANG_SKIP_ENV, raising = False)
@ -486,6 +512,7 @@ def test_tilelang_backend_installs_pinned_pair_for_qwen3_5(monkeypatch):
assert any("Installing TileLang" in s for s in statuses)
@linux_only
def test_tilelang_backend_reinstalls_when_tvm_ffi_is_broken(monkeypatch):
"""Repair path issues TWO pip calls:
@ -555,6 +582,7 @@ def test_tilelang_backend_skipped_on_windows(monkeypatch):
run_mock.assert_not_called()
@linux_only
def test_tilelang_backend_swallows_install_timeout(monkeypatch):
_pin_fla_model_types(monkeypatch)
monkeypatch.delenv(worker._TILELANG_SKIP_ENV, raising = False)
@ -609,6 +637,7 @@ def test_tilelang_backend_skipped_via_env(monkeypatch):
run_mock.assert_not_called()
@linux_only
def test_tilelang_backend_swallows_install_failure(monkeypatch):
_pin_fla_model_types(monkeypatch)
monkeypatch.delenv(worker._TILELANG_SKIP_ENV, raising = False)
@ -673,6 +702,7 @@ def _patch_iu_gates(monkeypatch, fla_gate, conv_gate):
monkeypatch.setattr(_iu, "is_causal_conv1d_available", conv_gate)
@not_on_windows
def test_hook_installs_when_gate_returns_false(monkeypatch):
_pin_fla_model_types(monkeypatch)
fla_gate = _make_fake_gate(initial_return = False)
@ -976,6 +1006,7 @@ def test_hook_does_install_tilelang_for_qwen35(monkeypatch):
tile_install.assert_called_once()
@linux_only
def test_tilelang_repair_does_not_touch_torch_cuda_stack(monkeypatch):
"""Finding #2: the broken-tvm-ffi repair must use --no-deps on the
forced step so --force-reinstall doesn't cascade through
@ -1119,6 +1150,7 @@ def test_hook_runs_tilelang_repair_when_fla_already_true(monkeypatch):
tile_install.assert_called_once()
@not_on_windows
def test_fla_installer_force_reinstalls_when_older_version_present(monkeypatch):
"""Finding #8: an older `flash-linear-attention` that is importable
but below the pin must force a reinstall (not no-op).
@ -1583,15 +1615,10 @@ def test_install_respects_user_gcc_install_dir(monkeypatch):
)
_make_hip_install_env(monkeypatch, gcc_dir = "/usr/lib/gcc/x86_64-linux-gnu/13")
captured: dict[str, str] | None = {"_called": "no"}
captured: dict[str, str] = {}
def fake_run(cmd, **kwargs):
env = kwargs.get("env")
if env is not None:
captured.clear()
captured.update(env)
else:
captured["_called"] = "yes_no_env"
captured.update(kwargs.get("env") or {})
return subprocess.CompletedProcess(cmd, 0, "")
monkeypatch.setattr(worker._sp, "run", fake_run)
@ -1607,14 +1634,11 @@ def test_install_respects_user_gcc_install_dir(monkeypatch):
release_base_url = "https://example.com",
)
# subprocess.run invoked without env override (user already set
# HIPCC_COMPILE_FLAGS_APPEND with --gcc-install-dir, so we left the
# env alone — the existing value is inherited).
assert captured == {"_called": "yes_no_env"}
assert captured["HIPCC_COMPILE_FLAGS_APPEND"] == "--gcc-install-dir=/opt/custom/gcc-13"
def test_install_does_not_inject_env_on_cuda(monkeypatch):
"""CUDA path (no hip_version in env) → no env override at all."""
"""CUDA path (no hip_version in env) → no HIP flag injected."""
monkeypatch.delenv("HIPCC_COMPILE_FLAGS_APPEND", raising = False)
monkeypatch.setattr(builtins, "__import__", _missing_module_import("causal_conv1d"))
monkeypatch.setattr(
@ -1641,7 +1665,7 @@ def test_install_does_not_inject_env_on_cuda(monkeypatch):
captured: dict[str, Any] = {}
def fake_run(cmd, **kwargs):
captured["env_in_kwargs"] = "env" in kwargs
captured.update(kwargs.get("env") or {})
return subprocess.CompletedProcess(cmd, 0, "")
monkeypatch.setattr(worker._sp, "run", fake_run)
@ -1657,5 +1681,5 @@ def test_install_does_not_inject_env_on_cuda(monkeypatch):
release_base_url = "https://example.com",
)
# CUDA branch never sets the env, never invokes the gcc helper.
assert captured.get("env_in_kwargs") is False
# env is always passed (to force UTF-8), but never the HIP flag.
assert "HIPCC_COMPILE_FLAGS_APPEND" not in captured

File diff suppressed because it is too large Load diff

View file

@ -0,0 +1,22 @@
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
"""Make a Python child agree with the parent that its pipes are UTF-8.
A child's ``sys.stdout`` uses ``locale.getpreferredencoding()``, which on
Windows is the ANSI code page. Reading that pipe as UTF-8 would then mangle any
non-ASCII the child prints, so the child has to be told which encoding to emit.
Only needed for Python children; llama.cpp and node already emit UTF-8.
"""
from __future__ import annotations
import os
from typing import Mapping, Optional
def utf8_child_env(env: Optional[Mapping[str, str]] = None) -> dict[str, str]:
"""Copy *env* (or the current environment) with UTF-8 stdio forced."""
child = dict(os.environ if env is None else env)
child["PYTHONIOENCODING"] = "utf-8"
return child

View file

@ -144,6 +144,8 @@ def _run_amd_smi(*args: str, timeout: int = _AMD_SMI_DEFAULT_TIMEOUT) -> Optiona
["amd-smi", *args, "--json"],
capture_output = True,
text = True,
encoding = "utf-8",
errors = "replace",
timeout = timeout,
env = _amd_env,
**windows_hidden_subprocess_kwargs(),

View file

@ -830,6 +830,8 @@ def _rocm_windows_perf_counter_gpu_util_pct() -> Optional[float]:
["powershell", "-NoProfile", "-NonInteractive", "-Command", ps],
capture_output = True,
text = True,
encoding = "utf-8",
errors = "replace",
timeout = 5,
)
if r.returncode != 0 or not r.stdout.strip():
@ -1027,6 +1029,8 @@ def _rocm_windows_perf_counter_vram_by_adapter() -> Optional[list[tuple[str, flo
["powershell", "-NoProfile", "-NonInteractive", "-Command", ps],
capture_output = True,
text = True,
encoding = "utf-8",
errors = "replace",
timeout = 5,
)
if r.returncode != 0 or not r.stdout.strip():

View file

@ -55,6 +55,8 @@ def get_physical_gpu_count() -> Optional[int]:
["nvidia-smi", "-L"],
capture_output = True,
text = True,
encoding = "utf-8",
errors = "replace",
timeout = 5,
env = child_env_without_native_path_secret(),
**_windows_hidden_subprocess_kwargs(),
@ -81,6 +83,8 @@ def get_primary_gpu_utilization() -> dict[str, Any]:
],
capture_output = True,
text = True,
encoding = "utf-8",
errors = "replace",
timeout = 5,
env = child_env_without_native_path_secret(),
**_windows_hidden_subprocess_kwargs(),
@ -131,6 +135,8 @@ def get_visible_gpu_utilization(
],
capture_output = True,
text = True,
encoding = "utf-8",
errors = "replace",
timeout = 5,
env = child_env_without_native_path_secret(),
**_windows_hidden_subprocess_kwargs(),
@ -215,6 +221,8 @@ def get_backend_visible_gpu_info(
],
capture_output = True,
text = True,
encoding = "utf-8",
errors = "replace",
timeout = 10,
env = child_env_without_native_path_secret(),
**_windows_hidden_subprocess_kwargs(),

View file

@ -121,7 +121,14 @@ def _installed_build_number(binary: Optional[str]) -> Optional[int]:
if not binary:
return None
try:
proc = subprocess.run([binary, "--version"], capture_output = True, text = True, timeout = 20)
proc = subprocess.run(
[binary, "--version"],
capture_output = True,
text = True,
encoding = "utf-8",
errors = "replace",
timeout = 20,
)
except Exception: # pragma: no cover - defensive
return None
m = re.search(r"version:\s*(\d+)", (proc.stderr or "") + (proc.stdout or ""))

View file

@ -254,7 +254,7 @@ def _transformers_constraint_args() -> tuple[list[str], str | None]:
except Exception:
return [], None
fd, path = tempfile.mkstemp(prefix = "mlx_repair_", suffix = ".txt")
with os.fdopen(fd, "w") as fh:
with os.fdopen(fd, "w", encoding = "utf-8") as fh:
fh.write(f"transformers=={transformers_version}\n")
return ["--constraint", path], path
@ -290,6 +290,8 @@ def attempt_mlx_repair(*, timeout: int = _REPAIR_TIMEOUT_S) -> bool:
stdout = subprocess.PIPE,
stderr = subprocess.STDOUT,
text = True,
encoding = "utf-8",
errors = "replace",
timeout = timeout,
)
except subprocess.TimeoutExpired:

View file

@ -129,7 +129,7 @@ def _read_checkpoint_loss(checkpoint_path: Path) -> Optional[float]:
if not trainer_state.exists():
return None
try:
with open(trainer_state, encoding = "utf-8") as f:
with open(trainer_state, encoding = "utf-8-sig") as f:
state = json.load(f)
log_history = state.get("log_history", [])
if log_history:
@ -174,18 +174,18 @@ def scan_checkpoints(
metadata: dict = {}
try:
if adapter_config.exists():
cfg = json.loads(adapter_config.read_text(encoding = "utf-8"))
cfg = json.loads(adapter_config.read_text(encoding = "utf-8-sig"))
metadata["base_model"] = cfg.get("base_model_name_or_path")
metadata["peft_type"] = cfg.get("peft_type")
metadata["lora_rank"] = cfg.get("r")
elif config_file.exists():
cfg = json.loads(config_file.read_text(encoding = "utf-8"))
cfg = json.loads(config_file.read_text(encoding = "utf-8-sig"))
metadata["base_model"] = cfg.get("_name_or_path")
# Detect BNB quantization from config.json
if config_file.exists():
if "cfg" not in dir():
cfg = json.loads(config_file.read_text(encoding = "utf-8"))
cfg = json.loads(config_file.read_text(encoding = "utf-8-sig"))
quant_cfg = cfg.get("quantization_config")
if (
isinstance(quant_cfg, dict)

View file

@ -37,6 +37,7 @@ import yaml
from utils.native_path_leases import child_env_without_native_path_secret
from utils.child_stdio import utf8_child_env
from utils.hf_cache_settings import active_hf_hub_cache, get_hf_cache_paths
from utils.subprocess_compat import (
windows_hidden_subprocess_kwargs as _windows_hidden_subprocess_kwargs,
@ -631,7 +632,7 @@ def _raw_config_has_vision_config(
cache_dir = active_hf_hub_cache(),
)
)
config = json.loads(config_path.read_text(encoding = "utf-8"))
config = json.loads(config_path.read_text(encoding = "utf-8-sig"))
architectures = config.get("architectures") or []
model_type = config.get("model_type")
explicit_vision = (
@ -774,8 +775,12 @@ def _is_vision_model_subprocess(model_name: str, hf_token: Optional[str] = None)
],
capture_output = True,
text = True,
encoding = "utf-8",
errors = "replace",
timeout = 60,
env = get_hf_cache_paths().child_env(child_env_without_native_path_secret()),
env = utf8_child_env(
get_hf_cache_paths().child_env(child_env_without_native_path_secret())
),
**_windows_hidden_subprocess_kwargs(),
)
@ -1083,7 +1088,7 @@ def _detect_audio_from_tokenizer(
]:
tok_file = snapshot / tok_path
if tok_file.exists():
tok_config = json.loads(tok_file.read_text(encoding = "utf-8"))
tok_config = json.loads(tok_file.read_text(encoding = "utf-8-sig"))
read_any = True
result = _check_token_patterns(tok_config)
if result:
@ -2283,7 +2288,7 @@ def scan_exported_models(
export_meta = run_dir / "export_metadata.json"
try:
if export_meta.exists():
meta = json.loads(export_meta.read_text(encoding = "utf-8"))
meta = json.loads(export_meta.read_text(encoding = "utf-8-sig"))
base_model = meta.get("base_model")
except Exception:
pass
@ -2312,7 +2317,7 @@ def scan_exported_models(
if adapter_config.exists():
export_type = "lora"
try:
cfg = json.loads(adapter_config.read_text(encoding = "utf-8"))
cfg = json.loads(adapter_config.read_text(encoding = "utf-8-sig"))
base_model = cfg.get("base_model_name_or_path")
except Exception:
pass
@ -2321,7 +2326,7 @@ def scan_exported_models(
export_meta = checkpoint_dir / "export_metadata.json"
try:
if export_meta.exists():
meta = json.loads(export_meta.read_text(encoding = "utf-8"))
meta = json.loads(export_meta.read_text(encoding = "utf-8-sig"))
base_model = meta.get("base_model")
except Exception:
pass
@ -2334,7 +2339,7 @@ def scan_exported_models(
export_meta = meta_dir / "export_metadata.json"
try:
if export_meta.exists():
meta = json.loads(export_meta.read_text(encoding = "utf-8"))
meta = json.loads(export_meta.read_text(encoding = "utf-8-sig"))
base_model = meta.get("base_model")
if base_model:
break
@ -2354,7 +2359,7 @@ def scan_exported_models(
outputs_adapter_cfg = resolve_output_dir(run_dir.name) / "adapter_config.json"
try:
if outputs_adapter_cfg.exists():
cfg = json.loads(outputs_adapter_cfg.read_text(encoding = "utf-8"))
cfg = json.loads(outputs_adapter_cfg.read_text(encoding = "utf-8-sig"))
base_model = cfg.get("base_model_name_or_path")
except Exception:
pass
@ -2380,7 +2385,7 @@ def get_base_model_from_checkpoint(checkpoint_path: str) -> Optional[str]:
adapter_config_path = checkpoint_path_obj / "adapter_config.json"
if adapter_config_path.exists():
with open(adapter_config_path, "r", encoding = "utf-8") as f:
with open(adapter_config_path, "r", encoding = "utf-8-sig") as f:
config = json.load(f)
base_model = config.get("base_model_name_or_path")
if base_model:
@ -2389,7 +2394,7 @@ def get_base_model_from_checkpoint(checkpoint_path: str) -> Optional[str]:
config_path = checkpoint_path_obj / "config.json"
if config_path.exists():
with open(config_path, "r", encoding = "utf-8") as f:
with open(config_path, "r", encoding = "utf-8-sig") as f:
config = json.load(f)
for key in ("model_name", "_name_or_path"):
base_model = config.get(key)
@ -2445,7 +2450,7 @@ def get_base_model_from_lora(lora_path: str) -> Optional[str]:
# adapter_config.json first
adapter_config_path = lora_path_obj / "adapter_config.json"
if adapter_config_path.exists():
with open(adapter_config_path, "r", encoding = "utf-8") as f:
with open(adapter_config_path, "r", encoding = "utf-8-sig") as f:
config = json.load(f)
base_model = config.get("base_model_name_or_path")
if base_model:
@ -2535,7 +2540,7 @@ def get_base_model_from_lora_identifier(
last_exc = exc
continue
try:
with open(cfg_path, "r", encoding = "utf-8") as f:
with open(cfg_path, "r", encoding = "utf-8-sig") as f:
base_model = json.load(f).get("base_model_name_or_path")
except Exception as exc:
logger.warning("Could not parse adapter_config.json for '%s': %s", identifier, exc)
@ -2781,7 +2786,7 @@ class ModelConfig:
meta_path = gguf_dir / "export_metadata.json"
if meta_path.exists():
try:
meta = json.loads(meta_path.read_text(encoding = "utf-8"))
meta = json.loads(meta_path.read_text(encoding = "utf-8-sig"))
base = meta.get("base_model")
if base and is_vision_model(base, hf_token = hf_token):
base_is_vision = True
@ -2912,7 +2917,7 @@ class ModelConfig:
token = hf_token,
cache_dir = active_hf_hub_cache(),
)
with open(config_path, "r", encoding = "utf-8") as f:
with open(config_path, "r", encoding = "utf-8-sig") as f:
adapter_config = json.load(f)
base_model = adapter_config.get("base_model_name_or_path")
if base_model:

View file

@ -79,6 +79,8 @@ def _node_version_ok(executable: str) -> bool:
[executable, "-v"],
capture_output = True,
text = True,
encoding = "utf-8",
errors = "replace",
timeout = _NODE_VERSION_PROBE_TIMEOUT_SECONDS,
**windows_hidden_subprocess_kwargs(),
)

View file

@ -212,7 +212,7 @@ def lmstudio_model_dirs() -> list[Path]:
settings_path = Path.home() / ".lmstudio" / "settings.json"
if settings_path.is_file():
try:
with open(settings_path, encoding = "utf-8") as f:
with open(settings_path, encoding = "utf-8-sig") as f:
settings = json.load(f)
downloads = settings.get("downloadsFolder", "")
if downloads:

View file

@ -24,6 +24,7 @@ from typing import Callable, Optional
import structlog
from utils.child_stdio import utf8_child_env
from utils.process_lifetime import child_popen_kwargs
logger = structlog.get_logger(__name__)
@ -159,6 +160,8 @@ def resolve_prebuilt_for_host(
cmd,
capture_output = True,
text = True,
encoding = "utf-8",
errors = "replace",
timeout = 60,
)
out = (proc.stdout or "").strip()
@ -303,7 +306,10 @@ def stream_installer(
stdout = subprocess.PIPE,
stderr = subprocess.STDOUT,
text = True,
env = env,
encoding = "utf-8",
errors = "replace",
# Make the Python child emit the UTF-8 we decode above.
env = utf8_child_env(env),
**child_popen_kwargs(),
)
timed_out = threading.Event()

View file

@ -142,7 +142,7 @@ def _load_remote_code_configs(model_name: str, hf_token: Optional[str] = None) -
for name in _REMOTE_CODE_CONFIG_FILES:
p = root / name
if p.is_file():
configs.append(json.loads(p.read_text(encoding = "utf-8")))
configs.append(json.loads(p.read_text(encoding = "utf-8-sig")))
return configs
from huggingface_hub import hf_hub_download
@ -164,7 +164,7 @@ def _load_remote_code_configs(model_name: str, hf_token: Optional[str] = None) -
# Transient/auth failure is not "absent" -> fail closed to "unknown" so
# the caller scans (a tokenizer/processor-only auto_map must not slip by).
return None
configs.append(json.loads(Path(p).read_text(encoding = "utf-8")))
configs.append(json.loads(Path(p).read_text(encoding = "utf-8-sig")))
# Every config was read or a genuine 404 -> an empty list is a definitive
# "no auto_map", not "unknown".
return configs

View file

@ -199,7 +199,7 @@ def _indexed_shard_paths(
inconclusive = True # transient: an index that might exist could not be read
continue
try:
weight_map = (json.loads(open(index_path, encoding = "utf-8").read()) or {}).get(
weight_map = (json.loads(open(index_path, encoding = "utf-8-sig").read()) or {}).get(
"weight_map"
) or {}
for shard in weight_map.values():
@ -328,7 +328,7 @@ def _st_load_roots(snapshot: Path) -> list:
roots = [snapshot]
try:
import json
modules = json.loads((snapshot / "modules.json").read_text(encoding = "utf-8"))
modules = json.loads((snapshot / "modules.json").read_text(encoding = "utf-8-sig"))
except (OSError, ValueError):
return roots # no / invalid modules.json -> snapshot root is the only load root
for module in modules or ():
@ -355,7 +355,7 @@ def _indexed_pickle_shards(index_path: Path, root: Path, snapshot: Path) -> list
try:
# JSON is UTF-8 by spec; pin it so a non-ASCII index is not misdecoded (and needlessly
# blocked) under Windows' cp1252 default.
parsed = json.loads(index_path.read_text(encoding = "utf-8"))
parsed = json.loads(index_path.read_text(encoding = "utf-8-sig"))
except (OSError, ValueError) as exc:
raise OSError(f"unreadable weight index: {index_path}") from exc
weight_map = parsed.get("weight_map") if isinstance(parsed, dict) else None

View file

@ -69,7 +69,7 @@ def approval_target_key(targets) -> str:
def _load() -> dict:
"""Parsed store, or an empty skeleton on any error (fail-safe = re-prompt)."""
try:
with open(_store_path(), encoding = "utf-8") as f:
with open(_store_path(), encoding = "utf-8-sig") as f:
data = json.load(f)
# Validate the shape, not just the version: a hand-edited ``subjects`` that is not a
# dict (e.g. ``[]``) would otherwise crash lookup/record instead of failing safe.

View file

@ -454,7 +454,7 @@ def repo_remote_code_files(model_name: str, hf_token: Optional[str] = None) -> d
p = root / name
if p.is_file():
try:
ext_refs |= _auto_map_refs(json.loads(p.read_text(encoding = "utf-8")))
ext_refs |= _auto_map_refs(json.loads(p.read_text(encoding = "utf-8-sig")))
except Exception:
pass
if not _add_external_refs(files, ext_refs, hf_token, model_name):
@ -483,7 +483,7 @@ def repo_remote_code_files(model_name: str, hf_token: Optional[str] = None) -> d
f"{model_name}: config {cfg_name} could not be fetched ({exc})"
) from exc
try:
refs |= _auto_map_refs(json.loads(Path(cfg_path).read_text(encoding = "utf-8")))
refs |= _auto_map_refs(json.loads(Path(cfg_path).read_text(encoding = "utf-8-sig")))
except Exception:
pass
own_refs = {fn for repo, fn in refs if repo is None}
@ -616,7 +616,7 @@ def external_auto_map_repos(model_name: str, hf_token: Optional[str] = None) ->
if not p.is_file():
continue
try:
refs = _auto_map_refs(json.loads(p.read_text(encoding = "utf-8")))
refs = _auto_map_refs(json.loads(p.read_text(encoding = "utf-8-sig")))
except Exception:
continue
repos.update(repo for repo, _fn in refs if repo)
@ -638,7 +638,7 @@ def external_auto_map_repos(model_name: str, hf_token: Optional[str] = None) ->
except Exception:
continue
try:
refs = _auto_map_refs(json.loads(Path(cfg_path).read_text(encoding = "utf-8")))
refs = _auto_map_refs(json.loads(Path(cfg_path).read_text(encoding = "utf-8-sig")))
except Exception:
continue
repos.update(repo for repo, _fn in refs if repo)

View file

@ -23,6 +23,7 @@ import threading
from typing import Any, Callable, Optional
from loggers import get_logger
from utils.child_stdio import utf8_child_env
from utils.wheel_utils import (
direct_wheel_url,
install_wheel,
@ -254,6 +255,12 @@ def _install_kernel(
"stdout": subprocess.PIPE,
"stderr": subprocess.STDOUT,
"text": True,
# pip and the compilers it drives write UTF-8 down this pipe; the Windows
# ANSI codepage would mojibake or raise over a fine install.
"encoding": "utf-8",
"errors": "replace",
# Make the Python child emit the UTF-8 we decode above.
"env": utf8_child_env(),
}
if is_hip:
run_kwargs["timeout"] = 1800 # ROCm builds can take 10-30 min
@ -261,7 +268,8 @@ def _install_kernel(
if "--gcc-install-dir" not in existing:
gcc_dir = _hipcc_gcc_install_dir()
if gcc_dir:
_env = os.environ.copy()
# Extends the UTF-8 env above rather than replacing it.
_env = dict(run_kwargs["env"])
_env["HIPCC_COMPILE_FLAGS_APPEND"] = (
f"{existing} --gcc-install-dir={gcc_dir}".strip()
)

View file

@ -60,6 +60,8 @@ def _exact_git_studio_tag(repo_root: Path) -> str | None:
stdout = subprocess.PIPE,
stderr = subprocess.DEVNULL,
text = True,
encoding = "utf-8",
errors = "replace",
timeout = _GIT_TIMEOUT_SECONDS,
)
except (OSError, subprocess.TimeoutExpired):
@ -81,6 +83,8 @@ def _git_branch(repo_root: Path) -> str | None:
stdout = subprocess.PIPE,
stderr = subprocess.DEVNULL,
text = True,
encoding = "utf-8",
errors = "replace",
timeout = _GIT_TIMEOUT_SECONDS,
)
except (OSError, subprocess.TimeoutExpired):

View file

@ -44,6 +44,7 @@ import time
from pathlib import Path
from utils.native_path_leases import child_env_without_native_path_secret
from utils.child_stdio import utf8_child_env
from utils.hf_cache_settings import get_hf_cache_paths
from utils.subprocess_compat import (
windows_hidden_subprocess_kwargs as _windows_hidden_subprocess_kwargs,
@ -420,7 +421,7 @@ def _resolve_base_model(model_name: str) -> str:
adapter_cfg_path = local_path / "adapter_config.json"
if _safe_is_file(adapter_cfg_path):
try:
with open(adapter_cfg_path, encoding = "utf-8") as f:
with open(adapter_cfg_path, encoding = "utf-8-sig") as f:
cfg = json.load(f)
base = cfg.get("base_model_name_or_path")
if base:
@ -437,7 +438,7 @@ def _resolve_base_model(model_name: str) -> str:
config_json_path = local_path / "config.json"
if _safe_is_file(config_json_path):
try:
with open(config_json_path, encoding = "utf-8") as f:
with open(config_json_path, encoding = "utf-8-sig") as f:
cfg = json.load(f)
# Unsloth writes model_name, HF writes _name_or_path; skip a self-reference.
for _key in ("model_name", "_name_or_path"):
@ -544,7 +545,7 @@ def _adapter_base_from_hf_cache(model_name: str) -> str | None:
)
for cfg_path in candidates:
if cfg_path.is_file():
base = json.loads(cfg_path.read_text(encoding = "utf-8")).get(
base = json.loads(cfg_path.read_text(encoding = "utf-8-sig")).get(
"base_model_name_or_path"
)
return base or None
@ -616,7 +617,7 @@ def _check_tokenizer_config_needs_v5(model_name: str, hf_token: str | None = Non
local_tc = local_path / "tokenizer_config.json"
if _safe_is_file(local_tc):
try:
with open(local_tc, encoding = "utf-8") as f:
with open(local_tc, encoding = "utf-8-sig") as f:
data = json.load(f)
tokenizer_class = data.get("tokenizer_class", "")
result = tokenizer_class in _TRANSFORMERS_5_TOKENIZER_CLASSES
@ -706,7 +707,7 @@ def _config_json_from_hf_cache(model_name: str) -> dict | None:
)
for cfg_path in candidates:
if cfg_path.is_file():
with open(cfg_path, encoding = "utf-8") as f:
with open(cfg_path, encoding = "utf-8-sig") as f:
return json.load(f)
except Exception as exc:
logger.debug("HF cache config.json lookup failed for '%s': %s", model_name, exc)
@ -731,7 +732,7 @@ def _load_config_json(model_name: str, hf_token: str | None = None) -> dict | No
local_cfg = Path(model_name) / "config.json"
if _safe_is_file(local_cfg):
try:
with open(local_cfg, encoding = "utf-8") as f:
with open(local_cfg, encoding = "utf-8-sig") as f:
cfg = json.load(f)
_config_json_cache[cache_key] = cfg
return cfg
@ -1271,9 +1272,10 @@ def _probe_autoconfig(target_dir: str, model_name: str, hf_token: str | None) ->
[sys.executable, "-c", _PROBE_CONFIG_SCRIPT, target_dir, model_name],
capture_output = True,
text = True,
encoding = "utf-8",
errors = "replace",
timeout = _PROBE_TIMEOUT_SECS,
env = env,
env = utf8_child_env(env),
**_windows_hidden_subprocess_kwargs(),
)
except subprocess.TimeoutExpired:
@ -1811,7 +1813,11 @@ def _install_to_dir(pkg: str, target_dir: str) -> bool:
stdout = subprocess.PIPE,
stderr = subprocess.STDOUT,
text = True,
env = get_hf_cache_paths().child_env(child_env_without_native_path_secret()),
encoding = "utf-8",
errors = "replace",
env = utf8_child_env(
get_hf_cache_paths().child_env(child_env_without_native_path_secret())
),
**_windows_hidden_subprocess_kwargs(),
)
if result.returncode == 0:
@ -1834,7 +1840,9 @@ def _install_to_dir(pkg: str, target_dir: str) -> bool:
stdout = subprocess.PIPE,
stderr = subprocess.STDOUT,
text = True,
env = get_hf_cache_paths().child_env(child_env_without_native_path_secret()),
encoding = "utf-8",
errors = "replace",
env = utf8_child_env(get_hf_cache_paths().child_env(child_env_without_native_path_secret())),
**_windows_hidden_subprocess_kwargs(),
)
if result.returncode != 0:
@ -2079,7 +2087,7 @@ class SidecarSwapInProgress(RuntimeError):
def _read_swap_lock(path: Path) -> dict | None:
try:
data = json.loads(path.read_text(encoding = "utf-8"))
data = json.loads(path.read_text(encoding = "utf-8-sig"))
return data if isinstance(data, dict) else {}
except FileNotFoundError:
return None
@ -2120,7 +2128,7 @@ def try_begin_sidecar_swap(kind: str = "install") -> bool:
break
if fd is not None:
try:
with os.fdopen(fd, "w") as f:
with os.fdopen(fd, "w", encoding = "utf-8") as f:
f.write(
json.dumps(
{"pid": os.getpid(), "at": time.time(), "token": token, "kind": kind}
@ -2466,7 +2474,11 @@ def _ensure_venv_llmcompressor_exists() -> bool:
stdout = subprocess.PIPE,
stderr = subprocess.STDOUT,
text = True,
env = get_hf_cache_paths().child_env(child_env_without_native_path_secret()),
encoding = "utf-8",
errors = "replace",
env = utf8_child_env(
get_hf_cache_paths().child_env(child_env_without_native_path_secret())
),
**_windows_hidden_subprocess_kwargs(),
)
last_out = result.stdout or ""

View file

@ -30,6 +30,7 @@ PYPI_SUCCESS_TTL_SECONDS = 12 * 60 * 60
PYPI_FAILURE_TTL_SECONDS = 60 * 60
RELEASE_NOTES_URL = "https://unsloth.ai/docs/new/changelog"
DISABLE_ENV_VAR = "UNSLOTH_DISABLE_UPDATE_CHECK"
FAKE_UPDATE_ENV_VAR = "UNSLOTH_STUDIO_FAKE_UPDATE"
LOCAL_INSTALL_SOURCES = {"editable", "local_path", "vcs", "local_repo"}
@ -107,11 +108,32 @@ def get_studio_install_source_status(current_version: str) -> dict[str, Any]:
)
def _is_version(value: str) -> bool:
try:
Version(value)
except InvalidVersion:
return False
return True
def get_studio_update_status(current_version: str) -> dict[str, Any]:
"""Return public, read-only update status for the web UI."""
install_source = detect_install_source()
disabled = os.environ.get(DISABLE_ENV_VAR) == "1"
if os.environ.get(DISABLE_ENV_VAR) == "1":
# Dev-only: the popup is PyPI-install-only, so fake a version to review it
# from a checkout. The documented opt-out still wins.
forced_version = os.environ.get(FAKE_UPDATE_ENV_VAR, "").strip()
if forced_version and not disabled and _is_version(forced_version):
return _status_response(
current_version = current_version,
latest_version = forced_version,
install_source = "pypi",
update_available = True,
can_show_web_notification = True,
)
if disabled:
return _status_response(
current_version = current_version,
latest_version = None,

View file

@ -114,6 +114,8 @@ def hf_cache_snapshot_dir(model_name: str) -> Optional[Path]:
snapshot = repo_dir / "snapshots" / commit
if snapshot.is_dir():
return snapshot
# UnicodeDecodeError is a ValueError, not an OSError: a torn refs
# file must keep meaning "not cached here", not fail the offline check.
except (OSError, UnicodeDecodeError):
continue
return None

View file

@ -15,6 +15,7 @@ import urllib.request
from typing import Callable
from utils.native_path_leases import child_env_without_native_path_secret
from utils.child_stdio import utf8_child_env
from utils.subprocess_compat import windows_hidden_subprocess_kwargs
_logger = logging.getLogger(__name__)
@ -43,6 +44,8 @@ def has_blackwell_gpu() -> bool:
stdout = subprocess.PIPE,
stderr = subprocess.DEVNULL,
text = True,
encoding = "utf-8",
errors = "replace",
timeout = 10,
env = child_env_without_native_path_secret(),
)
@ -102,8 +105,10 @@ def probe_torch_wheel_env(*, timeout: int | None = None) -> dict[str, str] | Non
stdout = subprocess.PIPE,
stderr = subprocess.PIPE,
text = True,
encoding = "utf-8",
errors = "replace",
timeout = timeout,
env = child_env_without_native_path_secret(),
env = utf8_child_env(child_env_without_native_path_secret()),
**windows_hidden_subprocess_kwargs(),
)
except subprocess.TimeoutExpired:
@ -201,6 +206,8 @@ def install_wheel(
stdout = subprocess.PIPE,
stderr = subprocess.STDOUT,
text = True,
encoding = "utf-8",
errors = "replace",
env = child_env_without_native_path_secret(),
)
attempts.append(("uv", result))
@ -213,7 +220,10 @@ def install_wheel(
stdout = subprocess.PIPE,
stderr = subprocess.STDOUT,
text = True,
env = child_env_without_native_path_secret(),
encoding = "utf-8",
errors = "replace",
# Make the Python child emit the UTF-8 we decode above.
env = utf8_child_env(child_env_without_native_path_secret()),
)
attempts.append(("pip", result))
return attempts

View file

@ -121,7 +121,14 @@ def _installed_whisper_version(binary: Optional[str]) -> Optional[str]:
if not binary:
return None
try:
proc = subprocess.run([binary, "--version"], capture_output = True, text = True, timeout = 20)
proc = subprocess.run(
[binary, "--version"],
capture_output = True,
text = True,
encoding = "utf-8",
errors = "replace",
timeout = 20,
)
except Exception: # pragma: no cover - defensive
return None
m = re.search(r"v?(\d+\.\d+\.\d+)", (proc.stderr or "") + (proc.stdout or ""))

View file

@ -214,7 +214,8 @@ function TauriUpdateLayer({
}
return (
<div className="pointer-events-none fixed bottom-4 right-4 z-[9998] flex w-[calc(100vw-2rem)] max-w-[400px] flex-col items-stretch gap-2">
// Capped like the browser stack: the download panel shares it, so both must fit.
<div className="pointer-events-none fixed bottom-4 right-4 z-[9998] flex max-h-[calc(100dvh_-_2rem)] flex-col items-end gap-2">
<UpdateBanner
status={update.status}
info={update.info}
@ -223,6 +224,7 @@ function TauriUpdateLayer({
isExternalServer={isExternalServer}
updatePolicyMode={update.updatePolicyMode}
manualReleaseUrl={update.manualReleaseUrl}
releasePageUrl={update.releasePageUrl}
positioned={false}
onInstall={update.installUpdate}
onDismiss={update.dismiss}
@ -379,9 +381,11 @@ function TauriWrapper({ children }: { children: ReactNode }) {
return (
<>
{children}
{/* One bottom-right stack so overlays never overlap; they stack with a
gap, download panel anchored at the corner with banners above. */}
<div className="pointer-events-none fixed bottom-4 right-4 z-[9998] flex w-[calc(100vw-2rem)] max-w-[400px] flex-col items-stretch gap-2">
{/* One bottom-right stack so overlays never overlap: download panel at the
corner, banners above, each owning its width. */}
{/* Capped to the viewport, or a long download list plus expanded notes
pushes the top of the stack off screen. */}
<div className="pointer-events-none fixed bottom-4 right-4 z-[9998] flex max-h-[calc(100dvh_-_2rem)] flex-col items-end gap-2">
<WebUpdateBanner
positioned={false}
enabled={!WEB_UPDATE_HIDDEN_ROUTES.has(pathname)}

View file

@ -1226,7 +1226,9 @@ export function AppSidebar() {
openNewChat(null);
}}
className={cn(
"flex items-center gap-[6px] select-none transition-opacity",
// min-w-0 so a narrow sidebar truncates the wordmark
// instead of pushing the search icon over the logo.
"flex min-w-0 items-center gap-[6px] select-none transition-opacity",
chatDisabled && "pointer-events-none opacity-50",
)}
aria-label={t("shell.aria.home")}
@ -1238,17 +1240,17 @@ export function AppSidebar() {
<img
src="/circle-logo-small.png"
alt="Unsloth"
className="h-[calc(26px+0.5rem*var(--ui-font-scale,1))] w-[calc(26px+0.5rem*var(--ui-font-scale,1))] rounded-full object-cover"
className="h-[calc(26px+0.5rem*var(--ui-font-scale,1))] w-[calc(26px+0.5rem*var(--ui-font-scale,1))] shrink-0 rounded-full object-cover"
/>
<span className="font-heading text-[calc(13px+0.5rem*var(--ui-font-scale,1))] font-semibold tracking-[0em] leading-none text-black dark:text-white dark:tracking-[0.02em]">
<span className="truncate font-heading text-[calc(13px+0.5rem*var(--ui-font-scale,1))] font-semibold tracking-[0em] leading-none text-black dark:text-white dark:tracking-[0.02em]">
unsloth
</span>
<span className="nav-badge ml-0.5 inline-flex items-center justify-center rounded-full border border-nav-beta-border px-[5px] pt-[3px] pb-[2px] text-[calc(0.5rem*var(--ui-font-scale,1))] font-medium leading-none tracking-[0.04em] text-nav-fg-muted antialiased subpixel-antialiased shadow-[0_1px_2px_rgba(0,0,0,0.06)] dark:shadow-[0_1px_2px_rgba(0,0,0,0.35)]">
<span className="nav-badge ml-0.5 inline-flex shrink-0 items-center justify-center rounded-full border border-nav-beta-border px-[5px] pt-[3px] pb-[2px] text-[calc(0.5rem*var(--ui-font-scale,1))] font-medium leading-none tracking-[0.04em] text-nav-fg-muted antialiased subpixel-antialiased shadow-[0_1px_2px_rgba(0,0,0,0.06)] dark:shadow-[0_1px_2px_rgba(0,0,0,0.35)]">
{t("shell.beta")}
</span>
</Link>
)}
<div className="flex items-center gap-0.5">
<div className="flex shrink-0 items-center gap-0.25">
<Tooltip>
<TooltipPrimitive.Trigger asChild>
<button
@ -1257,7 +1259,7 @@ export function AppSidebar() {
useChatSearchStore.getState().open();
closeMobileIfOpen();
}}
className="inline-flex h-[33px] w-[32px] cursor-pointer items-center justify-center rounded-[10px] text-nav-icon-idle dark:text-nav-fg-muted transition-colors hover:bg-nav-surface-hover hover:text-black dark:hover:text-white focus-visible:outline-none focus-visible:ring-1 focus-visible:ring-ring"
className="inline-flex h-[33px] w-[28px] cursor-pointer items-center justify-center rounded-[10px] text-nav-icon-idle dark:text-nav-fg-muted transition-colors hover:bg-nav-surface-hover hover:text-black dark:hover:text-white focus-visible:outline-none focus-visible:ring-1 focus-visible:ring-ring"
aria-label={t("shell.navigation.search")}
>
<HugeiconsIcon icon={Search01Icon} strokeWidth={1.75} className="size-icon" />
@ -1281,7 +1283,7 @@ export function AppSidebar() {
<button
type="button"
onClick={togglePinned}
className="inline-flex h-[33px] w-[32px] cursor-pointer items-center justify-center rounded-[10px] text-nav-icon-idle dark:text-nav-fg-muted transition-colors hover:bg-nav-surface-hover hover:text-black dark:hover:text-white focus-visible:outline-none focus-visible:ring-1 focus-visible:ring-ring"
className="inline-flex h-[33px] w-[28px] cursor-pointer items-center justify-center rounded-[10px] text-nav-icon-idle dark:text-nav-fg-muted transition-colors hover:bg-nav-surface-hover hover:text-black dark:hover:text-white focus-visible:outline-none focus-visible:ring-1 focus-visible:ring-ring"
aria-label={t("shell.aria.closeSidebar")}
>
<HugeiconsIcon icon={LayoutAlignLeftIcon} strokeWidth={1.75} className="size-icon" />
@ -1325,10 +1327,10 @@ export function AppSidebar() {
)}
</SidebarHeader>
{/* Uniform pl-1.5 pr-2 keeps every hover pill the same width, inset from the edge. */}
{/* Uniform pl-1.5 pr-1.75 keeps every hover pill the same width, inset from the edge. */}
<SidebarGroup
className={cn(
"group-data-[collapsible=icon]:px-0 pl-1.5 pr-2 shrink-0 transition-[padding]",
"group-data-[collapsible=icon]:px-0 pl-1.5 pr-1.75 shrink-0 transition-[padding]",
showCompactMacBrand ? "pt-0" : "pt-[9px]",
// Scrolled: New Chat is pinned, give a little gap below it.
scrolled ? "pb-[5px]" : "pb-px",
@ -1417,7 +1419,7 @@ export function AppSidebar() {
scrolled && "is-scrolled",
)}
>
<SidebarGroup className="group-data-[collapsible=icon]:px-0 pl-1.5 pr-2 py-0 shrink-0">
<SidebarGroup className="group-data-[collapsible=icon]:px-0 pl-1.5 pr-1.75 py-0 shrink-0">
<SidebarGroupContent>
<SidebarMenu>
<NavItem
@ -1499,7 +1501,7 @@ export function AppSidebar() {
</CollapsibleTrigger>
</SidebarGroupLabel>
<CollapsibleContent>
<SidebarGroupContent className="pl-1.5 pr-2">
<SidebarGroupContent className="pl-1.5 pr-1.75">
<SidebarMenu>
<NavItem
icon={TestTubeOutlineIcon}
@ -1574,7 +1576,7 @@ export function AppSidebar() {
</CollapsibleTrigger>
</SidebarGroupLabel>
<CollapsibleContent>
<SidebarGroupContent className="pl-1.5 pr-2">
<SidebarGroupContent className="pl-1.5 pr-1.75">
<SidebarMenu>
{pinnedProjectRecords.map((project) => {
const projectChats =
@ -1721,7 +1723,7 @@ export function AppSidebar() {
</CollapsibleTrigger>
</SidebarGroupLabel>
<CollapsibleContent>
<SidebarGroupContent className="pl-1.5 pr-2">
<SidebarGroupContent className="pl-1.5 pr-1.75">
<SidebarMenu>
{recentChatItems.map((item) =>
renderChatSidebarItem(item, "recent"),
@ -1753,7 +1755,7 @@ export function AppSidebar() {
</CollapsibleTrigger>
</SidebarGroupLabel>
<CollapsibleContent>
<SidebarGroupContent className="pl-1.5 pr-2">
<SidebarGroupContent className="pl-1.5 pr-1.75">
<SidebarMenu>
{runItems.map((run) => {
// Explicit selection wins. Otherwise highlight the active

View file

@ -134,7 +134,7 @@ export function LlamaUpdateBanner({
className={cn(
positioned
? "fixed bottom-4 right-4 z-[9998] w-[calc(100vw-2rem)] max-w-[400px]"
: "pointer-events-auto w-full",
: "pointer-events-auto w-[calc(100vw-2rem)] max-w-[400px]",
)}
data-testid="llama-update-banner"
>

View file

@ -2,6 +2,7 @@
// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
import { Button } from "@/components/ui/button";
import { ReleaseNotesPanel } from "@/components/update/release-notes-panel";
import type {
DesktopUpdatePolicyMode,
RetainedUpdateFailure,
@ -22,6 +23,8 @@ interface UpdateBannerProps {
isExternalServer?: boolean;
updatePolicyMode: DesktopUpdatePolicyMode;
manualReleaseUrl: string | null;
// Release page for this version, preferred over the generic changelog.
releasePageUrl?: string | null;
// false fills a shared overlay stack; true self-anchors.
positioned?: boolean;
onInstall: () => void;
@ -30,6 +33,7 @@ interface UpdateBannerProps {
}
const EASE_OUT_QUART: [number, number, number, number] = [0.165, 0.84, 0.44, 1];
const LEADING_V = /^v/;
function formatVersion(version: string | null | undefined): string {
if (!version) return "";
@ -44,6 +48,7 @@ export function UpdateBanner({
isExternalServer = false,
updatePolicyMode,
manualReleaseUrl,
releasePageUrl = null,
positioned = true,
onInstall,
onDismiss,
@ -52,6 +57,8 @@ export function UpdateBanner({
const [copying, setCopying] = useState(false);
const [manualReport, setManualReport] = useState<string | null>(null);
const [manualMessage, setManualMessage] = useState<string | null>(null);
// Version whose notes are expanded; a new offer collapses the panel.
const [notesVersion, setNotesVersion] = useState<string | null>(null);
const showFailure = Boolean(lastFailure) && !dismissed;
const showAvailable = status === "available" && !dismissed && !showFailure;
const show = showFailure || (showAvailable && Boolean(info));
@ -62,6 +69,11 @@ export function UpdateBanner({
const currentVersion = formatVersion(info?.currentVersion);
const latestVersion = formatVersion(info?.version);
const Icon = showFailure ? CircleAlert : Download;
// Keyed by the backend release, not the app's SemVer; headings drop the v.
const notesTargetVersion =
(info?.pypiVersion ?? info?.version)?.replace(LEADING_V, "") ?? null;
const notesOpen =
notesTargetVersion !== null && notesVersion === notesTargetVersion;
async function handleCopyDiagnostics() {
setCopying(true);
@ -94,13 +106,14 @@ export function UpdateBanner({
exit={{ opacity: 0, y: 8, scale: 0.97 }}
transition={{ duration: 0.35, ease: EASE_OUT_QUART }}
className={cn(
// Wider than the other overlays: notes preview plus three buttons.
positioned
? "fixed bottom-4 right-4 z-[9999] w-[calc(100vw-2rem)] max-w-[400px]"
: "pointer-events-auto w-full",
? "fixed bottom-4 right-4 z-[9999] w-[calc(100vw-2rem)] max-w-[448px]"
: "pointer-events-auto flex min-h-0 w-[calc(100vw-2rem)] max-w-[448px] flex-col",
)}
data-testid="tauri-update-banner"
>
<div className="relative overflow-hidden rounded-[24px] bg-white px-5 pb-4 pt-5 shadow-[0_2px_8px_-2px_rgba(0,0,0,0.16)] dark:bg-card dark:shadow-[0_8px_28px_-6px_rgba(0,0,0,0.28)]">
<div className="relative flex max-h-[calc(100dvh_-_2rem)] flex-col overflow-hidden rounded-[24px] bg-white px-5 pb-4 pt-5 shadow-[0_2px_8px_-2px_rgba(0,0,0,0.16)] dark:bg-card dark:shadow-[0_8px_28px_-6px_rgba(0,0,0,0.28)]">
<button
type="button"
onClick={onDismiss}
@ -160,7 +173,40 @@ export function UpdateBanner({
</p>
)}
<div className="mt-4 flex flex-wrap items-center justify-end gap-x-1 gap-y-2">
{!showFailure && notesTargetVersion ? (
<ReleaseNotesPanel
version={notesTargetVersion}
open={notesOpen}
// Used only if CHANGELOG.md has no section for this version.
fallbackMarkdown={info?.body ?? null}
className="min-h-0 flex-1"
releaseNotesUrl={releasePageUrl ?? manualReleaseUrl}
/>
) : null}
<div
className={cn(
"mt-4 flex flex-wrap items-center gap-x-1 gap-y-2",
!showFailure && notesTargetVersion
? "justify-between"
: "justify-end",
)}
>
{!showFailure && notesTargetVersion ? (
<Button
size="sm"
variant="ghost"
// same type size as the action buttons
className="-ml-2 h-auto whitespace-nowrap rounded-full px-2.5 py-2 text-ui-13 font-medium text-foreground"
onClick={() =>
setNotesVersion(notesOpen ? null : notesTargetVersion)
}
aria-expanded={notesOpen}
data-testid="tauri-update-release-notes-toggle"
>
{notesOpen ? "Hide release notes" : "Show release notes"}
</Button>
) : null}
{showFailure ? (
<>
<Button
@ -187,28 +233,31 @@ export function UpdateBanner({
onClick={onInstall}
disabled={installDisabled}
>
{isManualLinuxPackage ? "Open release page" : "Retry update"}
{isManualLinuxPackage
? "Open release page"
: "Retry update"}
</Button>
</>
) : (
<>
// wrap + right-align so the action pair stays together
<div className="flex flex-wrap items-center justify-end gap-x-1 gap-y-2">
<Button
size="sm"
variant="ghost"
className="h-auto rounded-full px-3 py-2 text-ui-13 font-medium text-foreground"
className="h-auto whitespace-nowrap rounded-full px-2.5 py-2 text-ui-13 font-medium text-foreground"
onClick={onDismiss}
>
Remind me later
</Button>
<Button
size="sm"
className="-mr-1 h-auto rounded-full px-3.5 py-2 text-ui-13"
className="-mr-1 h-auto whitespace-nowrap rounded-full px-3 py-2 text-ui-13"
onClick={onInstall}
disabled={installDisabled}
>
{isManualLinuxPackage ? "Open release page" : "Update"}
</Button>
</>
</div>
)}
</div>
{manualMessage && (

View file

@ -2,6 +2,7 @@
// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
import { useSidebarPin } from "@/hooks/use-sidebar-pin";
import { useSidebarWidth } from "@/hooks/use-sidebar-width";
import { isTauri } from "@/lib/api-base";
import { cn } from "@/lib/utils";
import {
@ -110,9 +111,13 @@ export function WindowTitlebar({
const [enabled] = useState(shouldUseCustomWindowTitlebar);
const [maximized, setMaximized] = useState(false);
const { pinned, togglePinned } = useSidebarPin();
// The titlebar sits outside the sidebar wrapper, so it cannot inherit
// --sidebar-width. Read the resized width from the same store instead.
const { width } = useSidebarWidth();
const sidebarWidth = showSidebarSurface
? pinned
? "var(--studio-sidebar-expanded-width,17.5rem)"
? // The live value only exists mid-drag; otherwise the committed width.
`var(--studio-sidebar-live-width, ${width}px)`
: "var(--studio-sidebar-collapsed-width,3rem)"
: "0px";
const contentBorderLeft = pinned ? `calc(${sidebarWidth} + 12px)` : "0px";

View file

@ -0,0 +1,317 @@
// SPDX-License-Identifier: AGPL-3.0-only
// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
"use client"
import * as React from "react"
import { cn } from "@/lib/utils"
import {
Tooltip,
TooltipContent,
TooltipTrigger,
} from "@/components/ui/tooltip"
import { getClientPlatform } from "@/components/tauri/window-titlebar"
/** Pointer travel (px) below which a drag counts as a plain click. */
const DRAG_SLOP = 4
/** A compatibility click lands immediately after pointer-up. */
const CLICK_COMPAT_WINDOW_MS = 300
/** Arrow-key resize step for keyboard users. */
const RESIZE_STEP = 16
type DragState = {
startX: number
startWidth: number
moved: boolean
}
export type PanelResizeHandleProps = {
/** Which edge of the panel the handle sits on. */
edge: "left" | "right"
open: boolean
width: number
/** Uncapped stored preference, so a capped drag does not lower it. */
stored: number
min: number
max: number
clamp: (px: number) => number
setWidth: (px: number) => void
resetWidth: () => void
onToggle: () => void
/** Element to paint the live width onto, and the property to paint. */
target: () => HTMLElement | null
cssVar: string
/** Measured to start a drag from the rendered size when collapsed. */
measure: () => number
label: string
toggleLabel: string
/** Translated tooltip copy; the caller owns the translation layer. */
collapseHint: string
expandHint: string
dragHint: string
/** Shown in the tooltip when the panel has a toggle shortcut. */
shortcut?: string
dataSlot?: string
className?: string
/** Mirrors the live width onto :root for chrome outside the panel. */
rootVar?: string
}
/**
* A draggable panel edge: drag to resize, click to collapse or expand. Arrow
* keys resize, Home restores the default. The width is painted straight to the
* target while dragging and only persisted on release.
*/
export function PanelResizeHandle({
edge,
open,
width,
stored,
min,
max,
clamp,
setWidth,
resetWidth,
onToggle,
target,
cssVar,
measure,
label,
toggleLabel,
collapseHint,
expandHint,
dragHint,
shortcut,
dataSlot = "panel-resize-handle",
className,
rootVar,
}: PanelResizeHandleProps) {
const ref = React.useRef<HTMLButtonElement>(null)
const dragRef = React.useRef<DragState | null>(null)
const [dragging, setDragging] = React.useState(false)
const [hovered, setHovered] = React.useState(false)
const [focused, setFocused] = React.useState(false)
const [isMacPlatform] = React.useState(() => getClientPlatform().includes("mac"))
const hint = shortcut ? shortcut.replace("Mod", isMacPlatform ? "⌘" : "Ctrl+") : null
// Cached on pointer down so no DOM walk per move.
const targetRef = React.useRef<HTMLElement | null>(null)
const frameRef = React.useRef(0)
const pendingRef = React.useRef(0)
// What the pointer asked for, before the viewport cap. Committing the capped
// value instead would quietly downgrade a stored preference on a narrow window.
const rawRef = React.useRef(0)
// When a pointer sequence last ended. The browser's compatibility click
// lands in the same tick, so only a click that close behind is a duplicate.
// A timestamp cannot go stale the way an armed flag does: a genuine cancel
// emits no click, and a later assistive-tech click still gets through.
const handledAtRef = React.useRef(0)
const committedRef = React.useRef(width)
React.useEffect(() => {
committedRef.current = width
}, [width])
const paint = React.useCallback(
(value: string) => {
targetRef.current?.style.setProperty(cssVar, value)
if (rootVar) {
document.documentElement.style.setProperty(rootVar, value)
}
},
[cssVar, rootVar],
)
// Resizing relayouts the whole shell, and pointermove fires faster than the
// display refreshes, so coalesce to one paint per frame.
const paintWidth = React.useCallback(
(px: number) => {
pendingRef.current = px
if (frameRef.current) return
frameRef.current = requestAnimationFrame(() => {
frameRef.current = 0
paint(`${pendingRef.current}px`)
})
},
[paint],
)
const endDrag = React.useCallback(() => {
// Only a sequence that actually started can produce a compatibility click.
// This also runs as the effect cleanup, where no drag happened.
if (dragRef.current) handledAtRef.current = Date.now()
dragRef.current = null
if (frameRef.current) {
cancelAnimationFrame(frameRef.current)
frameRef.current = 0
}
// Hand the property back to the committed value. A commit re-renders with
// the new width; a cancel or a no-commit drag keeps DOM and store in step.
paint(`${committedRef.current}px`)
if (rootVar) document.documentElement.style.removeProperty(rootVar)
targetRef.current?.removeAttribute("data-resizing")
document.documentElement.removeAttribute("data-panel-resizing")
targetRef.current = null
setDragging(false)
document.body.style.removeProperty("cursor")
document.body.style.removeProperty("user-select")
}, [paint, rootVar])
const handlePointerDown = (event: React.PointerEvent<HTMLButtonElement>) => {
if (event.button !== 0) return
event.preventDefault()
event.currentTarget.setPointerCapture(event.pointerId)
targetRef.current = target()
// Collapsed: grow from the rendered size so the edge tracks the pointer.
const start = open ? width : measure()
dragRef.current = { startX: event.clientX, startWidth: start, moved: false }
pendingRef.current = start
rawRef.current = start
targetRef.current?.setAttribute("data-resizing", "true")
document.documentElement.setAttribute("data-panel-resizing", "true")
setDragging(true)
document.body.style.setProperty("cursor", "col-resize")
document.body.style.setProperty("user-select", "none")
}
const handlePointerMove = (event: React.PointerEvent<HTMLButtonElement>) => {
const drag = dragRef.current
if (!drag) return
// A panel whose handle is on its left edge grows as the pointer moves left.
const delta = (edge === "left" ? -1 : 1) * (event.clientX - drag.startX)
if (!drag.moved && Math.abs(delta) < DRAG_SLOP) return
drag.moved = true
const next = drag.startWidth + delta
rawRef.current = next
if (!open) {
// Past the minimum, dragging the collapsed edge reopens it.
if (next >= min) {
paintWidth(clamp(next))
onToggle()
}
return
}
// Dragging inward stops at the minimum. Collapsing is click or the shortcut.
paintWidth(clamp(next))
}
const handlePointerUp = (event: React.PointerEvent<HTMLButtonElement>) => {
const drag = dragRef.current
if (!drag) return
if (event.currentTarget.hasPointerCapture(event.pointerId)) {
event.currentTarget.releasePointerCapture(event.pointerId)
}
endDrag()
if (!drag.moved) {
onToggle()
return
}
// A drag below the minimum leaves the stored width alone.
if (!open) return
// Capped: the visible edge is already at the cap, so an outward pull cannot
// express intent beyond it. Committing would silently lower the larger
// hidden preference. A deliberate inward drag still commits.
if (stored > max && rawRef.current >= max) return
// Commit what was asked for, not the capped paint, so a drag on a narrow
// window cannot shrink a larger stored preference. setWidth clamps.
setWidth(rawRef.current)
}
const handleKeyDown = (event: React.KeyboardEvent<HTMLButtonElement>) => {
// The collapse/expand the label advertises, for keyboard users. Pointer-up
// handles it for the mouse; a synthesized click never reaches it.
if (event.key === "Enter" || event.key === " ") {
// preventDefault cancels the native click, so nothing follows to guard
// against; arming here would swallow the next assistive-tech click.
event.preventDefault()
onToggle()
return
}
const outward = edge === "left" ? "ArrowLeft" : "ArrowRight"
const inward = edge === "left" ? "ArrowRight" : "ArrowLeft"
if (event.key === outward || event.key === inward) {
event.preventDefault()
if (!open) {
// Collapsed there is nothing to resize, so the outward arrow reopens.
if (event.key === outward) onToggle()
return
}
if (event.key === outward && stored > max && width >= max) return
setWidth(width + (event.key === outward ? RESIZE_STEP : -RESIZE_STEP))
return
}
if (event.key === "Home") {
event.preventDefault()
resetWidth()
}
}
// Clear a stuck cursor override if we unmount mid-drag.
React.useEffect(() => endDrag, [endDrag])
return (
<Tooltip open={(hovered || focused) && !dragging}>
<TooltipTrigger asChild>
<button
ref={ref}
type="button"
data-slot={dataSlot}
data-dragging={dragging || undefined}
aria-label={open ? label : toggleLabel}
{...(open ? { "aria-orientation": "vertical" as const } : {})}
{...(open
? { "aria-valuenow": width, "aria-valuemin": min, "aria-valuemax": max }
: {})}
role={open ? "separator" : "button"}
onPointerDown={handlePointerDown}
onPointerMove={handlePointerMove}
onPointerUp={handlePointerUp}
onPointerCancel={endDrag}
onKeyDown={handleKeyDown}
onClick={() => {
// Switch and voice control activate by dispatching a bare click
// with no pointer or key events, which nothing else here catches.
if (Date.now() - handledAtRef.current < CLICK_COMPAT_WINDOW_MS) return
onToggle()
}}
onPointerEnter={() => setHovered(true)}
onPointerLeave={() => setHovered(false)}
onFocus={(event) => setFocused(event.target.matches(":focus-visible"))}
onBlur={() => setFocused(false)}
className={cn(
"absolute inset-y-0 z-30 hidden w-2 touch-none select-none sm:block",
edge === "left" ? "-left-1" : "-right-1",
// `!` overrides the app-wide hand cursor on buttons.
open
? "cursor-col-resize!"
: edge === "left"
? "cursor-w-resize!"
: "cursor-e-resize!",
// Sits exactly on the panel border so hover recolours one line.
"after:absolute after:inset-y-0 after:w-px after:bg-transparent after:transition-colors after:duration-150",
edge === "left" ? "after:left-1" : "after:right-1",
"hover:after:bg-sidebar-ring/25 data-dragging:after:bg-sidebar-ring/25",
// The app zeroes the native outline on buttons, so mark focus here.
"focus-visible:outline-none focus-visible:after:bg-sidebar-ring/60",
className,
)}
/>
</TooltipTrigger>
<TooltipContent
side={edge === "left" ? "left" : "right"}
align="center"
className="tooltip-compact"
>
<span className="flex flex-col gap-px">
<span>
{open ? collapseHint : expandHint}
{hint ? ` ${hint}` : ""}
</span>
<span className="opacity-70">{dragHint}</span>
</span>
</TooltipContent>
</Tooltip>
)
}

View file

@ -24,13 +24,21 @@ import {
TooltipContent,
TooltipTrigger,
} from "@/components/ui/tooltip"
import { PanelResizeHandle } from "@/components/ui/panel-resize-handle"
import { useT } from "@/i18n"
import { useIsMobile } from "@/hooks/use-mobile"
import {
SIDEBAR_WIDTH_DEFAULT,
SIDEBAR_WIDTH_MIN,
clampSidebarWidth,
useSidebarWidth,
} from "@/hooks/use-sidebar-width"
import { HugeiconsIcon } from "@hugeicons/react"
import { LayoutAlignLeftIcon } from "@hugeicons/core-free-icons"
const noop = () => {}
const SIDEBAR_WIDTH = "17.5rem"
const SIDEBAR_WIDTH = `${SIDEBAR_WIDTH_DEFAULT}px`
const SIDEBAR_WIDTH_ICON = "3rem"
const SIDEBAR_KEYBOARD_SHORTCUT = "b"
@ -46,6 +54,11 @@ type SidebarContextProps = {
pinned: boolean
setPinned: (value: boolean) => void
togglePinned: () => void
width: number
storedWidth: number
maxWidth: number
setWidth: (value: number) => void
resetWidth: () => void
}
const SidebarContext = React.createContext<SidebarContextProps | null>(null)
@ -80,6 +93,13 @@ function SidebarProvider({
}) {
const isMobile = useIsMobile()
const [openMobile, setOpenMobile] = React.useState(false)
const {
width,
max: maxWidth,
stored: storedWidth,
setWidth,
resetWidth,
} = useSidebarWidth()
const prevIsMobileRef = React.useRef(isMobile)
React.useEffect(() => {
@ -163,8 +183,13 @@ function SidebarProvider({
pinned,
setPinned,
togglePinned,
width,
storedWidth,
maxWidth,
setWidth,
resetWidth,
}),
[state, open, setOpen, isMobile, openMobile, setOpenMobile, toggleSidebar, hasPinMode, pinned, setPinned, togglePinned]
[state, open, setOpen, isMobile, openMobile, setOpenMobile, toggleSidebar, hasPinMode, pinned, setPinned, togglePinned, width, storedWidth, maxWidth, setWidth, resetWidth]
)
return (
@ -173,7 +198,8 @@ function SidebarProvider({
data-slot="sidebar-wrapper"
style={
{
"--sidebar-width": SIDEBAR_WIDTH,
// The drag handle writes this same property live while resizing.
"--sidebar-width": `${width}px`,
"--sidebar-width-icon": SIDEBAR_WIDTH_ICON,
...style,
} as React.CSSProperties
@ -311,11 +337,64 @@ function Sidebar({
>
{children}
</div>
<SidebarResizeHandle side={side} />
</div>
</div>
)
}
/**
* The sidebar's draggable edge, over the shared panel handle.
*/
function SidebarResizeHandle({
className,
side = "left",
}: {
className?: string
side?: "left" | "right"
}) {
const { open, toggleSidebar, width, storedWidth, maxWidth, setWidth, resetWidth } =
useSidebar()
const ref = React.useRef<HTMLDivElement>(null)
const t = useT()
return (
<div ref={ref} className="contents">
<PanelResizeHandle
edge={side === "right" ? "left" : "right"}
open={open}
width={width}
stored={storedWidth}
min={SIDEBAR_WIDTH_MIN}
max={maxWidth}
clamp={clampSidebarWidth}
setWidth={setWidth}
resetWidth={resetWidth}
onToggle={toggleSidebar}
target={() =>
ref.current?.closest<HTMLElement>('[data-slot="sidebar-wrapper"]') ?? null
}
cssVar="--sidebar-width"
// The custom titlebar renders outside the wrapper and cannot inherit it.
rootVar="--studio-sidebar-live-width"
measure={() =>
ref.current
?.closest<HTMLElement>('[data-slot="sidebar-container"]')
?.getBoundingClientRect().width ?? SIDEBAR_WIDTH_MIN
}
label={t("shell.aria.resizeSidebar")}
toggleLabel={t("shell.aria.openSidebar")}
collapseHint={t("shell.resize.collapse")}
expandHint={t("shell.resize.expand")}
dragHint={t("shell.resize.drag")}
shortcut="ModB"
dataSlot="sidebar-resize-handle"
className={className}
/>
</div>
)
}
function SidebarTrigger({
className,
onClick,
@ -777,6 +856,7 @@ export {
SidebarMenuSubItem,
SidebarProvider,
SidebarRail,
SidebarResizeHandle,
SidebarSeparator,
SidebarTrigger,
useSidebar,

View file

@ -0,0 +1,251 @@
// SPDX-License-Identifier: AGPL-3.0-only
// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
import { MarkdownPreview } from "@/components/markdown/markdown-preview";
import { useReleaseNotes } from "@/hooks/use-release-notes";
import { resolveChangelogLinks } from "@/lib/changelog-links";
import { releaseNotesPreview } from "@/lib/release-notes-preview";
import { cn } from "@/lib/utils";
import {
type ReactElement,
type ReactNode,
useEffect,
useMemo,
useRef,
} from "react";
interface ReleaseNotesPanelProps {
// Notes are looked up for this exact version only.
version: string;
// Collapsed previews the top bullets; expanded scrolls the full notes.
open: boolean;
// Desktop updater's body, used only if CHANGELOG.md has no section here.
fallbackMarkdown?: string | null;
releaseNotesUrl?: string | null;
className?: string;
}
const NOTES_LINK_CLASS =
"shrink-0 whitespace-nowrap text-ui-11 font-medium text-foreground underline underline-offset-2";
function NotesMessage({
children,
action,
}: {
children: ReactNode;
action?: ReactNode;
}): ReactElement {
return (
<div className="flex items-center justify-between gap-2 px-1 py-2">
<p className="text-ui-11 text-muted-foreground">{children}</p>
{action}
</div>
);
}
function ChangelogLink({ href }: { href: string }): ReactElement {
return (
<a
href={href}
target="_blank"
rel="noopener noreferrer"
className={NOTES_LINK_CLASS}
data-testid="update-release-notes-link"
>
Open changelog
</a>
);
}
export function ReleaseNotesPanel({
version,
open,
fallbackMarkdown = null,
releaseNotesUrl = null,
className,
}: ReleaseNotesPanelProps): ReactElement | null {
// Fetched with the popup: the collapsed preview needs the notes too.
const { state, notes, retry } = useReleaseNotes({ version, enabled: true });
const scrollRef = useRef<HTMLElement | null>(null);
// The fallback stands in for "no section in the changelog", which the hook
// reports as ready. An error is retryable, and the desktop fallback is the
// updater's static blurb, so taking it there would hide Retry until cache expiry.
const source = notes?.matched
? notes.markdown
: state === "error"
? null
: (fallbackMarkdown ?? null);
// Notes target the repository, so relative links must point back at it.
const markdown = useMemo(
() => (source === null ? null : resolveChangelogLinks(source)),
[source],
);
// Notes that are only a code block or a table preview as nothing.
const preview = useMemo(
() => (markdown === null ? null : releaseNotesPreview(markdown)),
[markdown],
);
// Start at the top on expand, and again once async notes land.
useEffect(() => {
if (open && markdown && scrollRef.current) {
scrollRef.current.scrollTop = 0;
}
}, [open, markdown]);
// Caller's URL wins: the API returns only the generic changelog, while the
// desktop banner passes this version's release page.
const notesUrl = releaseNotesUrl ?? notes?.releaseNotesUrl;
const link = notesUrl ? <ChangelogLink href={notesUrl} /> : null;
// Nothing previewable yet or ever: keep the collapsed popup compact.
if (
!open &&
(!markdown ||
state === "loading" ||
state === "idle" ||
preview?.items.length === 0)
) {
return null;
}
return (
<div
className={cn("mt-3 flex min-h-0 flex-col", className)}
data-testid="update-release-notes-panel"
data-notes-state={state}
data-notes-version={version}
data-notes-open={open}
>
{/* borderless fill, lighter than the card in dark mode */}
<div className="flex min-h-0 flex-col rounded-[14px] bg-muted/40 px-3 py-1 dark:bg-white/[0.06]">
{markdown ? (
open ? (
<section
ref={scrollRef}
// biome-ignore lint/a11y/noNoninteractiveTabindex: keyboard-scrollable region
tabIndex={0}
aria-label={`Release notes for version ${version}`}
// Long notes scroll here instead of pushing the buttons off screen.
className="hover-scrollbar max-h-64 min-h-0 flex-1 overflow-y-auto overscroll-contain py-3 pr-1"
data-testid="update-release-notes-scroll"
>
<MarkdownPreview
markdown={markdown}
// Streamdown ships headings at mt-6 and code at text-sm, and
// clears max-width on descendants, so rescale and re-cap both.
className="max-h-none overflow-visible border-0 bg-transparent p-0 text-ui-11 [&_[data-streamdown=link-safety-modal]>*]:max-w-md [&_img]:h-auto [&_img]:max-w-full [&>*:first-child]:mt-0 [&>*>*:first-child]:mt-0 [&_code]:text-[0.92em] [&_h1]:mt-4 [&_h1]:font-heading [&_h1]:text-ui-13 [&_h2]:mt-4 [&_h2]:font-heading [&_h2]:text-ui-13 [&_h3]:mt-4 [&_h3]:font-heading [&_h3]:text-ui-11 [&_pre]:text-[0.92em]"
/>
{notes?.truncated ? (
<p className="mt-2 text-ui-10 text-muted-foreground/80">
Notes truncated. See the full changelog.
</p>
) : null}
</section>
) : (
<ReleaseNotesSummary preview={preview} />
)
) : (
<NotesStatus
state={state}
version={version}
link={link}
retry={retry}
/>
)}
</div>
{open && markdown && link ? (
<div className="mt-2 flex justify-end px-1">{link}</div>
) : null}
</div>
);
}
/** Collapsed view: the first few bullets, one line each where possible. */
function ReleaseNotesSummary({
preview,
}: {
preview: ReturnType<typeof releaseNotesPreview> | null;
}): ReactElement | null {
if (preview === null || preview.items.length === 0) {
return null;
}
const { items, remaining } = preview;
return (
<ul
className="space-y-1 py-2 pr-1"
data-testid="update-release-notes-summary"
>
{items.map((item, index) => (
<li
// Two releases can carry the same bullet text, so index is the key.
key={`${index}-${item.lead}`}
className="flex gap-1.5 text-ui-11 leading-snug text-muted-foreground"
>
<span aria-hidden="true" className="text-muted-foreground/60">
&bull;
</span>
<span className="line-clamp-2 min-w-0">
{/* lead sentence carries the change */}
<span className="font-medium text-foreground">{item.lead}</span>
{item.rest ? <span> {item.rest}</span> : null}
</span>
</li>
))}
{remaining > 0 ? (
<li className="pl-3 text-ui-10 text-muted-foreground/70">
+{remaining} more
</li>
) : null}
</ul>
);
}
function NotesStatus({
state,
version,
link,
retry,
}: {
state: ReturnType<typeof useReleaseNotes>["state"];
version: string;
link: ReactNode;
retry: () => void;
}): ReactElement {
if (state === "loading" || state === "idle") {
return <NotesMessage>Loading release notes...</NotesMessage>;
}
if (state === "error") {
return (
<NotesMessage
action={
// The changelog page may be reachable when the lookup is not.
<span className="flex shrink-0 items-center gap-3">
<button
type="button"
onClick={retry}
className={NOTES_LINK_CLASS}
data-testid="update-release-notes-retry"
>
Retry
</button>
{link}
</span>
}
>
Could not load release notes.
</NotesMessage>
);
}
// Matched nothing: link out rather than show another release's notes.
return (
<NotesMessage action={link}>
No release notes published for {version} yet.
</NotesMessage>
);
}

View file

@ -2,6 +2,7 @@
// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
import { Button } from "@/components/ui/button";
import { ReleaseNotesPanel } from "@/components/update/release-notes-panel";
import { type DeviceType, usePlatformStore } from "@/config/env";
import { useWebUpdateCheck } from "@/hooks/use-web-update-check";
import { isTauri } from "@/lib/api-base";
@ -40,6 +41,7 @@ export function WebUpdateBanner({
const deviceType = usePlatformStore((s) => s.deviceType);
const installCmd = installCommandForDevice(deviceType);
const [copiedVersion, setCopiedVersion] = useState<string | null>(null);
const [notesVersion, setNotesVersion] = useState<string | null>(null);
const dismissTimerRef = useRef<ReturnType<typeof setTimeout> | null>(null);
useEffect(() => {
@ -68,6 +70,8 @@ export function WebUpdateBanner({
}
const copied = status != null && copiedVersion === status.latestVersion;
// Keyed by version so a new offer collapses the panel.
const notesOpen = status != null && notesVersion === status.latestVersion;
return (
<AnimatePresence>
@ -78,13 +82,14 @@ export function WebUpdateBanner({
exit={{ opacity: 0, y: 8, scale: 0.97 }}
transition={{ duration: 0.35, ease: EASE_OUT_QUART }}
className={cn(
// Wider than the other overlays: notes preview plus three buttons.
positioned
? "fixed bottom-4 right-4 z-[9999] w-[calc(100vw-2rem)] max-w-[400px]"
: "pointer-events-auto w-full",
? "fixed bottom-4 right-4 z-[9999] w-[calc(100vw-2rem)] max-w-[448px]"
: "pointer-events-auto flex min-h-0 w-[calc(100vw-2rem)] max-w-[448px] flex-col",
)}
data-testid="web-update-banner"
>
<div className="relative overflow-hidden rounded-[24px] bg-white px-5 pb-4 pt-5 shadow-[0_2px_8px_-2px_rgba(0,0,0,0.16)] dark:bg-card dark:shadow-[0_8px_28px_-6px_rgba(0,0,0,0.28)]">
<div className="relative flex max-h-[calc(100dvh_-_2rem)] flex-col overflow-hidden rounded-[24px] bg-white px-5 pb-4 pt-5 shadow-[0_2px_8px_-2px_rgba(0,0,0,0.16)] dark:bg-card dark:shadow-[0_8px_28px_-6px_rgba(0,0,0,0.28)]">
<button
type="button"
onClick={dismiss}
@ -127,22 +132,33 @@ export function WebUpdateBanner({
</div>
</div>
<ReleaseNotesPanel
version={status.latestVersion}
open={notesOpen}
releaseNotesUrl={RELEASE_NOTES_URL}
className="min-h-0 flex-1"
/>
{/* one row at one type size; wraps only on narrow viewports */}
<div className="mt-4 flex flex-wrap items-center justify-between gap-y-2">
<a
href={RELEASE_NOTES_URL}
target="_blank"
rel="noopener noreferrer"
className="-ml-2 whitespace-nowrap rounded-full px-2.5 py-2 text-ui-13 font-medium text-foreground transition-colors hover:bg-muted"
data-testid="web-update-release-notes-link"
<Button
size="sm"
variant="ghost"
className="-ml-2 h-auto whitespace-nowrap rounded-full px-2.5 py-2 text-ui-13 font-medium text-foreground"
onClick={() =>
setNotesVersion(notesOpen ? null : status.latestVersion)
}
aria-expanded={notesOpen}
data-testid="web-update-release-notes-toggle"
>
Release notes
</a>
{notesOpen ? "Hide release notes" : "Show release notes"}
</Button>
{/* wrap + right-align so buttons stack instead of clipping on very narrow banners */}
<div className="flex flex-wrap items-center justify-end gap-x-1 gap-y-2">
<Button
size="sm"
variant="ghost"
className="h-auto rounded-full px-3 py-2 text-ui-13 font-medium text-foreground"
className="h-auto whitespace-nowrap rounded-full px-2.5 py-2 text-ui-13 font-medium text-foreground"
onClick={snooze}
data-testid="web-update-snooze-button"
>
@ -151,7 +167,7 @@ export function WebUpdateBanner({
<Button
size="sm"
// -mr optically aligns the filled pill's edge with the card padding
className="-mr-1 h-auto rounded-full px-3.5 py-2 text-ui-13"
className="-mr-1 h-auto whitespace-nowrap rounded-full px-3 py-2 text-ui-13"
onClick={handleCopyCommand}
data-testid="web-update-copy-button"
>

View file

@ -18,6 +18,7 @@ import {
DropdownMenuTrigger,
} from "@/components/ui/dropdown-menu";
import { InfoHint } from "@/components/ui/info-hint";
import { PanelResizeHandle } from "@/components/ui/panel-resize-handle";
import {
InputGroup,
InputGroupAddon,
@ -44,7 +45,13 @@ import { Tooltip, TooltipContent } from "@/components/ui/tooltip";
import { NumericValueInput, snapToStep } from "@/features/model-picker";
import { RetrievalSettingsSection } from "@/features/rag";
import { useLlamaUpdateCheck } from "@/hooks/use-llama-update-check";
import {
CHAT_SETTINGS_WIDTH_MIN,
clampChatSettingsWidth,
useChatSettingsWidth,
} from "@/hooks/use-chat-settings-width";
import { useIsMobile } from "@/hooks/use-mobile";
import { useT } from "@/i18n";
import { ChevronDownStandardIcon } from "@/lib/chevron-icons";
import { toast } from "@/lib/toast";
import { cn } from "@/lib/utils";
@ -52,7 +59,7 @@ import { Edit03Icon, LayoutAlignRightIcon } from "@hugeicons/core-free-icons";
import { HugeiconsIcon } from "@hugeicons/react";
import { Braces, ChevronDown, ExternalLink } from "lucide-react";
import { Tooltip as TooltipPrimitive } from "radix-ui";
import { Fragment, type ReactNode } from "react";
import { type CSSProperties, Fragment, type ReactNode } from "react";
import { useCallback, useEffect, useMemo, useRef, useState } from "react";
import { OpenAICodeExecSection } from "./components/openai-code-exec-section";
import { PermissionModeDropdown } from "./permission-mode-select";
@ -363,6 +370,15 @@ export function ChatSettingsPanel({
onExternalProviderChange,
externalProviderType = null,
}: ChatSettingsPanelProps) {
const asideRef = useRef<HTMLElement>(null);
const t = useT();
const {
width: settingsWidth,
max: settingsMax,
stored: settingsStored,
setWidth: setSettingsWidth,
resetWidth: resetSettingsWidth,
} = useChatSettingsWidth();
// Local models show every knob; providerCapabilities is only consulted when
// isExternalModel. Unknown providers fall back to the OpenAI-compat shape via
// getProviderCapabilities, so these flags never undercount support.
@ -461,6 +477,23 @@ export function ChatSettingsPanel({
// When the prompt overflows the inline box, clicking opens the popup editor.
const systemPromptBoxRef = useRef<HTMLTextAreaElement>(null);
const [systemPromptOverflows, setSystemPromptOverflows] = useState(false);
const promptObserverRef = useRef<ResizeObserver | null>(null);
const measurePromptRef = useRef<() => void>(() => {});
// The section unmounts its textarea when collapsed, so observe through a
// callback ref: a stored observer would cling to the detached node and the
// remounted one would never be measured.
const attachPromptBox = useCallback((node: HTMLTextAreaElement | null) => {
systemPromptBoxRef.current = node;
promptObserverRef.current?.disconnect();
promptObserverRef.current = null;
if (!node || typeof ResizeObserver === "undefined") return;
// Resizing rewraps the prompt, and a drag changes the width through a
// custom property without re-rendering, so watch the box itself.
const observer = new ResizeObserver(() => measurePromptRef.current());
observer.observe(node);
promptObserverRef.current = observer;
measurePromptRef.current();
}, []);
const [activePresetBaseline, setActivePresetBaseline] = useState(params);
const presets = useMemo(() => {
return getOrderedPresets(customPresets);
@ -746,15 +779,20 @@ export function ChatSettingsPanel({
}, [open]);
useEffect(() => {
const el = systemPromptBoxRef.current;
setSystemPromptOverflows(
currentSystemPrompt.length > 0 &&
el != null &&
el.clientHeight > 0 &&
el.scrollHeight > el.clientHeight + 1,
);
measurePromptRef.current = () => {
const el = systemPromptBoxRef.current;
setSystemPromptOverflows(
currentSystemPrompt.length > 0 &&
el != null &&
el.clientHeight > 0 &&
el.scrollHeight > el.clientHeight + 1,
);
};
measurePromptRef.current();
}, [currentSystemPrompt, open]);
useEffect(() => () => promptObserverRef.current?.disconnect(), []);
const settingsScrollRef = useRef<HTMLDivElement>(null);
const settingsContent = (
@ -1124,7 +1162,7 @@ export function ChatSettingsPanel({
)}
>
<textarea
ref={systemPromptBoxRef}
ref={attachPromptBox}
value={currentSystemPrompt}
onChange={(e) => set("systemPrompt")(e.target.value)}
onMouseDown={(e) => {
@ -1433,17 +1471,47 @@ export function ChatSettingsPanel({
return (
<aside
ref={asideRef}
data-tour="chat-settings"
data-slot="chat-settings-panel"
className={cn(
"relative z-50 shrink-0 overflow-hidden bg-panel-surface text-panel-surface-fg font-heading",
open ? "w-[17rem] border-l border-sidebar-border" : "w-0",
"relative z-50 shrink-0 bg-panel-surface text-panel-surface-fg font-heading",
open
? "w-(--chat-settings-width) border-l border-sidebar-border"
: "w-0 overflow-hidden",
)}
style={{
height: "calc(100% - var(--studio-custom-titlebar-height, 0px))",
marginTop: "var(--studio-custom-titlebar-height, 0px)",
}}
style={
{
"--chat-settings-width": `${settingsWidth}px`,
height: "calc(100% - var(--studio-custom-titlebar-height, 0px))",
marginTop: "var(--studio-custom-titlebar-height, 0px)",
} as CSSProperties
}
>
<div className="h-full w-full">{settingsContent}</div>
{open ? (
<PanelResizeHandle
edge="left"
open={open}
width={settingsWidth}
stored={settingsStored}
min={CHAT_SETTINGS_WIDTH_MIN}
max={settingsMax}
clamp={clampChatSettingsWidth}
setWidth={setSettingsWidth}
resetWidth={resetSettingsWidth}
onToggle={() => onOpenChange?.(!open)}
target={() => asideRef.current}
cssVar="--chat-settings-width"
measure={() => asideRef.current?.getBoundingClientRect().width ?? 0}
label={t("shell.aria.resizeRunSettings")}
toggleLabel={t("shell.aria.openRunSettings")}
collapseHint={t("shell.resize.collapse")}
expandHint={t("shell.resize.expand")}
dragHint={t("shell.resize.drag")}
dataSlot="chat-settings-resize-handle"
/>
) : null}
<div className="h-full w-full overflow-hidden">{settingsContent}</div>
</aside>
);
}

View file

@ -23,12 +23,21 @@ import {
removeScanFolder,
} from "@/features/hub";
import { FolderBrowser } from "@/features/model-picker";
import { openModelsDir } from "@/features/native-intents";
import {
openModelsDir,
pickHuggingFaceCacheDir,
} from "@/features/native-intents";
import {
type HuggingFaceCacheSettings,
loadHuggingFaceCacheSettings,
updateHuggingFaceCacheSettings,
} from "@/features/settings";
import { isTauri } from "@/lib/api-base";
import { toast } from "@/lib/toast";
import { cn } from "@/lib/utils";
import {
Delete02Icon,
DownloadCircle01Icon,
FileSearchIcon,
FolderAddIcon,
FolderExportIcon,
@ -49,6 +58,12 @@ function formatError(error: unknown): string {
return error instanceof Error ? error.message : String(error);
}
function formatFreeSpace(bytes: number | null): string | null {
if (bytes === null || !Number.isFinite(bytes)) return null;
const gb = bytes / 1024 ** 3;
return gb >= 10 ? `${Math.round(gb)} GB free` : `${gb.toFixed(1)} GB free`;
}
export function OnDeviceFoldersDialog({
open,
onOpenChange,
@ -68,6 +83,11 @@ export function OnDeviceFoldersDialog({
);
const refreshIdRef = useRef(0);
const mutationVersionRef = useRef(0);
const [downloadCache, setDownloadCache] =
useState<HuggingFaceCacheSettings | null>(null);
const [downloadCacheLoaded, setDownloadCacheLoaded] = useState(false);
const [downloadBrowserOpen, setDownloadBrowserOpen] = useState(false);
const [downloadSaving, setDownloadSaving] = useState(false);
const sortedFolders = useMemo(
() => [...folders].sort((a, b) => a.path.localeCompare(b.path)),
@ -108,10 +128,66 @@ export function OnDeviceFoldersDialog({
return () => window.clearTimeout(timer);
}, [open, refreshFolders]);
useEffect(() => {
if (!open) return;
let cancelled = false;
// The dialog stays mounted between opens, so re-arm the flag or a reopen
// shows the previous answer as if it were fresh.
setDownloadCacheLoaded(false);
loadHuggingFaceCacheSettings()
// Indexed locations do not depend on this. Null drops the stale path
// rather than offer Change against a location we could not confirm.
.catch(() => null)
.then((settings) => {
if (cancelled) return;
setDownloadCache(settings);
setDownloadCacheLoaded(true);
});
return () => {
cancelled = true;
};
}, [open]);
const handleInventoryChanged = useCallback(() => {
onInventoryChange?.();
}, [onInventoryChange]);
// Relocating the cache changes which repos are on disk, but
// updateHuggingFaceCacheSettings already bumps the inventory version, which
// re-fetches every source. Refreshing here too would scan twice, since the
// two rounds carry different version keys and cannot be deduplicated.
const saveDownloadLocation = useCallback(async (nextPath: string | null) => {
setDownloadSaving(true);
try {
const settings = await updateHuggingFaceCacheSettings(nextPath);
setDownloadCache(settings);
toast.success("Download location updated", {
description: settings.cacheHome,
});
} catch (err) {
toast.error("Couldn't update the download location", {
description: formatError(err),
});
} finally {
setDownloadSaving(false);
}
}, []);
const changeDownloadLocation = useCallback(async () => {
if (!isTauri) {
setDownloadBrowserOpen(true);
return;
}
try {
const picked = await pickHuggingFaceCacheDir();
if (picked) await saveDownloadLocation(picked);
} catch (err) {
toast.error("Couldn't open the folder picker", {
description: formatError(err),
});
}
}, [saveDownloadLocation]);
const handleAdd = useCallback(
async (rawPath: string) => {
const nextPath = rawPath.trim();
@ -182,10 +258,10 @@ export function OnDeviceFoldersDialog({
<>
<Dialog open={open} onOpenChange={onOpenChange}>
<DialogContent
className="gap-0 overflow-hidden p-0 sm:max-w-[620px] lg:max-w-[660px] xl:max-w-[680px] [&_[data-slot=dialog-close]]:right-3 [&_[data-slot=dialog-close]]:top-3"
className="flex max-h-[90dvh] flex-col gap-0 overflow-hidden p-0 sm:max-w-[620px] lg:max-w-[660px] xl:max-w-[680px] [&_[data-slot=dialog-close]]:right-3 [&_[data-slot=dialog-close]]:top-3"
overlayClassName="bg-black/20 backdrop-blur-none"
>
<DialogHeader className="border-b border-border/60 px-5 py-4">
<DialogHeader className="shrink-0 border-b border-border/60 px-5 py-4">
<DialogTitle className="text-ui-15">
On-device locations
</DialogTitle>
@ -195,7 +271,78 @@ export function OnDeviceFoldersDialog({
</DialogDescription>
</DialogHeader>
<div className="space-y-4 px-5 py-4">
<div className="min-h-0 flex-1 space-y-4 overflow-y-auto px-5 py-4">
<div className="rounded-[14px] border border-border/70 bg-muted/20 p-3">
<div className="mb-2 flex items-center gap-2 text-ui-12 font-medium text-foreground">
<HugeiconsIcon
icon={DownloadCircle01Icon}
strokeWidth={1.75}
className="size-3.5 text-muted-foreground"
/>
Download location
</div>
<div className="flex flex-col gap-2 sm:flex-row sm:items-center">
<Input
readOnly={true}
aria-label="Model download location"
value={
downloadCache?.cacheHome ??
(downloadCacheLoaded ? "Unknown" : "Loading...")
}
title={downloadCache?.cacheHome}
className="field-soft h-9 min-w-0 flex-1 rounded-full px-3 font-mono text-ui-12"
/>
<div className="flex shrink-0 items-center gap-2">
<Button
type="button"
variant="outline"
size="sm"
onClick={() => void changeDownloadLocation()}
disabled={!downloadCache?.editable || downloadSaving}
className="h-9 rounded-full px-3 text-ui-12p5"
>
{downloadSaving ? (
<Spinner className="size-3.5" />
) : (
<HugeiconsIcon
icon={FolderSearchIcon}
strokeWidth={1.75}
data-icon="inline-start"
className="size-3.5"
/>
)}
Change
</Button>
{downloadCache?.isCustom ? (
<Button
type="button"
variant="ghost"
size="sm"
onClick={() => void saveDownloadLocation(null)}
disabled={downloadSaving}
className="h-9 rounded-full px-3 text-ui-12p5 text-muted-foreground"
>
Use default
</Button>
) : null}
</div>
</div>
<p className="mt-2 text-ui-10p5 text-muted-foreground">
{downloadCache?.source === "environment"
? `Managed by the ${
downloadCache.environmentVariable ?? "HF_HOME"
} environment variable.`
: [
"New downloads only. Models already on disk stay where they are.",
formatFreeSpace(downloadCache?.freeBytes ?? null),
]
.filter(Boolean)
.join(" · ")}
</p>
</div>
<div className="rounded-[14px] border border-border/70 bg-muted/20 p-3">
<div className="mb-2 flex items-center gap-2 text-ui-12 font-medium text-foreground">
<HugeiconsIcon
@ -425,6 +572,16 @@ export function OnDeviceFoldersDialog({
onOpenChange={setBrowserOpen}
onSelect={(selectedPath) => void handleAdd(selectedPath)}
/>
<FolderBrowser
open={!isTauri && downloadBrowserOpen}
onOpenChange={setDownloadBrowserOpen}
onSelect={(selectedPath) => void saveDownloadLocation(selectedPath)}
initialPath={downloadCache?.cacheHome}
title="Choose model download location"
confirmLabel="Use for future downloads"
showModelHints={false}
/>
</>
);
}

View file

@ -201,8 +201,10 @@ export function DownloadManagerPanel({
className={cn(
// Standalone: anchor bottom-right. In a shared stack (positioned=false)
// flow as a right-aligned row so overlays stack instead of overlapping.
// min-h-0 there: a flex item's min-height defaults to auto, so the capped
// stack would squeeze the update card instead of this list.
"pointer-events-none",
positioned ? "fixed bottom-4 right-4 z-50" : "flex justify-end",
positioned ? "fixed bottom-4 right-4 z-50" : "flex min-h-0 justify-end",
)}
>
{collapsed ? (
@ -229,7 +231,7 @@ export function DownloadManagerPanel({
</TooltipContent>
</Tooltip>
) : (
<div className="hub-download-panel pointer-events-auto w-[min(400px,calc(100vw-2rem))] overflow-hidden">
<div className="hub-download-panel pointer-events-auto flex min-h-0 w-[min(400px,calc(100vw-2rem))] flex-col overflow-hidden">
<div className="flex items-center gap-2 border-b border-foreground/[0.07] py-2 pl-4 pr-3">
<span className="min-w-0 flex-1 truncate text-ui-12p5 font-semibold text-foreground">
{headerLabel}

View file

@ -3,6 +3,11 @@
export { SettingsDialog } from "./settings-dialog";
export { loadEmbeddingModelSettings } from "./api/embedding-model";
export {
loadHuggingFaceCacheSettings,
updateHuggingFaceCacheSettings,
} from "./api/hugging-face-cache";
export type { HuggingFaceCacheSettings } from "./api/hugging-face-cache";
export {
loadPersonalization,
savePersonalization,

View file

@ -35,7 +35,10 @@ import {
useRef,
useState,
} from "react";
import { SETTINGS_SEARCH_INDEX } from "./settings-search";
import {
SETTINGS_SEARCH_INDEX,
SETTINGS_SEARCH_KEYWORDS,
} from "./settings-search";
import {
type SettingsTab,
useSettingsDialogStore,
@ -157,8 +160,12 @@ export function SettingsDialog() {
return TABS.map((tab) => {
const tabLabel = t(tab.labelKey);
const entries = SETTINGS_SEARCH_INDEX[tab.id]
.map((key) => t(key))
.filter((label) => label.toLowerCase().includes(q));
.filter((key) => {
if (t(key).toLowerCase().includes(q)) return true;
const keywordsKey = SETTINGS_SEARCH_KEYWORDS[key];
return keywordsKey ? t(keywordsKey).toLowerCase().includes(q) : false;
})
.map((key) => t(key));
const deduped = [...new Set(entries)];
return {
tab,

View file

@ -146,3 +146,15 @@ export const SETTINGS_SEARCH_INDEX: Record<SettingsTab, TranslationKey[]> = {
"settings.about.shutDownStudio",
],
};
/**
* Extra terms a row matches on, beyond its own label. The value is a
* translation key holding space-separated synonyms; it is never rendered.
* Search matched labels only, so "models folder" or "directory" found nothing.
*/
export const SETTINGS_SEARCH_KEYWORDS: Partial<
Record<TranslationKey, TranslationKey>
> = {
"settings.resources.storage.modelsFolder":
"settings.resources.storage.modelsFolderKeywords",
};

View file

@ -74,6 +74,8 @@ const PREFS_KEYS: string[] = [
LOCALE_STORAGE_KEY,
// UI state
"sidebar_pinned",
"sidebar_width",
"chat_settings_width",
"unsloth_sidebar_navigate_open",
"unsloth_settings_active_tab",
// Chat runtime prefs

View file

@ -0,0 +1,20 @@
// SPDX-License-Identifier: AGPL-3.0-only
// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
import { createPanelWidthStore } from "./use-panel-width.ts";
/** The previous fixed 17rem, at a 16px root font size. */
export const CHAT_SETTINGS_WIDTH_DEFAULT = 272;
/** Below this the sliders and their value pills start colliding. */
export const CHAT_SETTINGS_WIDTH_MIN = 248;
export const CHAT_SETTINGS_WIDTH_MAX = 560;
const store = createPanelWidthStore({
key: "chat_settings_width",
min: CHAT_SETTINGS_WIDTH_MIN,
max: CHAT_SETTINGS_WIDTH_MAX,
fallback: CHAT_SETTINGS_WIDTH_DEFAULT,
});
export const clampChatSettingsWidth = store.clamp;
export const useChatSettingsWidth = store.useWidth;

View file

@ -0,0 +1,138 @@
// SPDX-License-Identifier: AGPL-3.0-only
// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
import { useCallback, useSyncExternalStore } from "react";
/** Never let one panel eat more than this share of a narrow window. */
const MAX_VIEWPORT_FRACTION = 0.4;
export type PanelWidthStore = {
/** Clamps to what the current viewport allows. */
clamp: (px: number) => number;
useWidth: () => {
width: number;
max: number;
/** The uncapped stored preference. */
stored: number;
setWidth: (value: number) => void;
resetWidth: () => void;
};
};
/**
* A persisted, viewport-aware width for a draggable panel. The preference is
* stored whole and an effective width is derived from it, so narrowing the
* window shrinks the panel without losing what the user picked.
*/
export function createPanelWidthStore({
key,
min,
max,
fallback,
}: {
key: string;
min: number;
max: number;
fallback: number;
}): PanelWidthStore {
function maxWidth(): number {
if (typeof window === "undefined") return max;
// The floor wins on a narrow window; collapsing is the escape.
return Math.max(min, Math.min(max, window.innerWidth * MAX_VIEWPORT_FRACTION));
}
/** Clamps to the absolute range, ignoring the viewport. */
function clampStored(px: number): number {
if (!Number.isFinite(px)) return fallback;
return Math.min(max, Math.max(min, Math.round(px)));
}
function clamp(px: number): number {
return Math.min(maxWidth(), clampStored(px));
}
function load(): number {
if (typeof window === "undefined") return fallback;
try {
const raw = window.localStorage.getItem(key);
if (raw === null) return fallback;
return clampStored(Number.parseFloat(raw));
} catch {
return fallback;
}
}
let storedWidth = load();
let effectiveWidth = clamp(storedWidth);
let effectiveMax = maxWidth();
const listeners = new Set<() => void>();
let lastStored = storedWidth;
function recompute() {
const nextWidth = clamp(storedWidth);
const nextMax = maxWidth();
if (
nextWidth === effectiveWidth &&
nextMax === effectiveMax &&
storedWidth === lastStored
) {
return;
}
effectiveWidth = nextWidth;
effectiveMax = nextMax;
lastStored = storedWidth;
listeners.forEach((cb) => cb());
}
function subscribe(cb: () => void) {
// With no subscribers there is no resize listener, so the cache can be
// stale after a resize on a route that hides every panel. Refresh first;
// useSyncExternalStore re-reads the snapshot right after subscribing.
recompute();
listeners.add(cb);
if (typeof window === "undefined") {
return () => listeners.delete(cb);
}
// Keep tabs in sync, same as the pin flag.
const onStorage = (e: StorageEvent) => {
if (e.key === key || e.key === null) {
storedWidth = load();
effectiveWidth = clamp(storedWidth);
effectiveMax = maxWidth();
cb();
}
};
window.addEventListener("storage", onStorage);
window.addEventListener("resize", recompute);
return () => {
listeners.delete(cb);
window.removeEventListener("storage", onStorage);
window.removeEventListener("resize", recompute);
};
}
function setWidthGlobal(next: number) {
const stored = clampStored(next);
if (stored !== storedWidth) {
storedWidth = stored;
try {
window.localStorage.setItem(key, String(stored));
} catch {}
}
recompute();
}
function useWidth() {
const width = useSyncExternalStore(subscribe, () => effectiveWidth, () => fallback);
// What the viewport actually allows right now, for aria-valuemax.
const panelMax = useSyncExternalStore(subscribe, () => effectiveMax, () => max);
// The uncapped preference, so a capped drag can avoid lowering it.
const preference = useSyncExternalStore(subscribe, () => storedWidth, () => fallback);
const setWidth = useCallback((value: number) => setWidthGlobal(value), []);
const resetWidth = useCallback(() => setWidthGlobal(fallback), []);
return { width, max: panelMax, stored: preference, setWidth, resetWidth };
}
return { clamp, useWidth };
}

View file

@ -0,0 +1,146 @@
// SPDX-License-Identifier: AGPL-3.0-only
// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
import { authFetch, hasAuthToken } from "@/features/auth";
import { apiUrl } from "@/lib/api-base";
import { useCallback, useEffect, useRef, useState } from "react";
// Keyed to one exact version, so a new update never pairs with older notes.
export interface ReleaseNotes {
version: string;
markdown: string | null;
matched: boolean;
truncated: boolean;
source: string | null;
releaseNotesUrl: string | null;
// Set when the lookup itself failed, as opposed to a version with no notes.
error: string | null;
}
export type ReleaseNotesState = "idle" | "loading" | "ready" | "error";
// Desktop auto-auth installs its token after first paint, so a startup popup can
// ask before one exists. Wait briefly rather than fail.
const AUTH_POLL_MS = 250;
const AUTH_POLL_LIMIT = 40;
interface UseReleaseNotesOptions {
version: string | null | undefined;
enabled?: boolean;
}
type ApiObject = Record<string, unknown>;
function stringOrNull(value: ApiObject, key: string): string | null {
const field = value[key];
return typeof field === "string" && field.length > 0 ? field : null;
}
function toReleaseNotes(value: unknown, version: string): ReleaseNotes | null {
if (!value || typeof value !== "object") {
return null;
}
const payload = value as ApiObject;
const notesVersion = stringOrNull(payload, "version");
// A response for another version is not usable here.
if (notesVersion !== version) {
return null;
}
const markdown = stringOrNull(payload, "markdown");
return {
version,
markdown,
matched: payload.matched === true && markdown !== null,
truncated: payload.truncated === true,
source: stringOrNull(payload, "source"),
releaseNotesUrl: stringOrNull(payload, "release_notes_url"),
error: stringOrNull(payload, "error"),
};
}
async function fetchReleaseNotes(
version: string,
refresh = false,
): Promise<ReleaseNotes | null> {
const query = `version=${encodeURIComponent(version)}${refresh ? "&refresh=true" : ""}`;
// authFetch, not fetch: an expired token is refreshed and retried.
const res = await authFetch(apiUrl(`/api/studio/release-notes?${query}`));
if (!res.ok) {
throw new Error(`Release notes request failed: ${res.status}`);
}
return toReleaseNotes(await res.json(), version);
}
export function useReleaseNotes({
version,
enabled = true,
}: UseReleaseNotesOptions) {
const [state, setState] = useState<ReleaseNotesState>("idle");
const [notes, setNotes] = useState<ReleaseNotes | null>(null);
// Version the current state belongs to; a change invalidates it.
const requestedVersionRef = useRef<string | null>(null);
// Identifies one request, so an earlier response cannot overwrite a later one.
const requestIdRef = useRef(0);
const load = useCallback((target: string, refresh = false) => {
requestedVersionRef.current = target;
requestIdRef.current += 1;
const requestId = requestIdRef.current;
setState("loading");
setNotes(null);
fetchReleaseNotes(target, refresh)
.then((next) => {
// A newer request owns the state now.
if (requestIdRef.current !== requestId) {
return;
}
setNotes(next);
// A reported failure is retryable; "no notes for this version" is not.
const failed = !next || (!next.matched && next.error !== null);
setState(failed ? "error" : "ready");
})
.catch(() => {
if (requestIdRef.current === requestId) {
setNotes(null);
setState("error");
}
});
}, []);
useEffect(() => {
if (!enabled || !version || requestedVersionRef.current === version) {
return;
}
if (hasAuthToken()) {
load(version);
return;
}
let attempts = 0;
const timer = window.setInterval(() => {
attempts += 1;
if (hasAuthToken() || attempts >= AUTH_POLL_LIMIT) {
window.clearInterval(timer);
// Out of patience: load anyway so the panel settles on retry.
load(version);
}
}, AUTH_POLL_MS);
return () => window.clearInterval(timer);
}, [enabled, version, load]);
const retry = useCallback(() => {
if (version) {
requestedVersionRef.current = null;
// Bypass the cached remote failure, or retry waits for it to expire.
load(version, true);
}
}, [version, load]);
// Never hand back another version's notes: state lags `version` by a render.
const matchesVersion = notes !== null && notes.version === version;
return {
state: notes !== null && !matchesVersion ? "loading" : state,
notes: matchesVersion ? notes : null,
retry,
};
}

View file

@ -0,0 +1,21 @@
// SPDX-License-Identifier: AGPL-3.0-only
// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
import { createPanelWidthStore } from "./use-panel-width.ts";
/** The previous fixed 17.5rem, at a 16px root font size. */
export const SIDEBAR_WIDTH_DEFAULT = 280;
/** Narrowest width that still fits the wordmark. Firefox is the constraint:
* it renders the heading ~3px wider than Chromium and WebKit. */
export const SIDEBAR_WIDTH_MIN = 260;
export const SIDEBAR_WIDTH_MAX = 480;
const store = createPanelWidthStore({
key: "sidebar_width",
min: SIDEBAR_WIDTH_MIN,
max: SIDEBAR_WIDTH_MAX,
fallback: SIDEBAR_WIDTH_DEFAULT,
});
export const clampSidebarWidth = store.clamp;
export const useSidebarWidth = store.useWidth;

View file

@ -21,6 +21,8 @@ export type UpdateStatus =
export interface UpdateInfo {
version: string;
currentVersion: string;
// Backend release this build pins; CHANGELOG.md is keyed by it, not the SemVer.
pypiVersion?: string;
body?: string;
date?: string;
}
@ -42,10 +44,17 @@ interface DesktopUpdatePolicy {
interface ManualUpdateInfo {
version: string;
currentVersion: string;
pypiVersion?: string | null;
body?: string;
date?: string;
}
/** `pypi_version` from latest.json, which the updater passes through raw. */
function rawPypiVersion(raw: Record<string, unknown>): string | undefined {
const value = raw.pypi_version;
return typeof value === "string" && value.length > 0 ? value : undefined;
}
export interface RetainedUpdateFailure {
error: string;
phase: UpdatePhase;
@ -162,6 +171,7 @@ export function useTauriUpdate(isExternalServer = false) {
setInfo({
version: manualUpdate.version,
currentVersion: manualUpdate.currentVersion,
pypiVersion: manualUpdate.pypiVersion ?? undefined,
body: manualUpdate.body,
date: manualUpdate.date,
});
@ -197,6 +207,7 @@ export function useTauriUpdate(isExternalServer = false) {
setInfo({
version: update.version,
currentVersion: update.currentVersion,
pypiVersion: rawPypiVersion(update.rawJson),
body: update.body,
date: update.date,
});
@ -384,10 +395,13 @@ export function useTauriUpdate(isExternalServer = false) {
});
}
// Install target for Linux packages that cannot self-update.
const manualReleaseUrl =
updatePolicy.mode === "manual_linux_package" && info
? manualReleasePageUrl(updatePolicy, info.version)
: null;
// Release page for the offered version, on every platform, for the notes link.
const releasePageUrl = info ? manualReleasePageUrl(updatePolicy, info.version) : null;
return {
status,
@ -401,6 +415,7 @@ export function useTauriUpdate(isExternalServer = false) {
isExternalServer,
updatePolicyMode: updatePolicy.mode,
manualReleaseUrl,
releasePageUrl,
installUpdate,
retryUpdate,
skipAndRestart,

View file

@ -27,10 +27,18 @@ export const ar = {
product: "Unsloth Studio",
accountMenu: "قائمة حساب {name}",
updateAvailable: "يتوفر تحديث",
resize: {
collapse: "انقر للطي",
expand: "انقر للتوسيع",
drag: "اسحب لتغيير الحجم",
},
aria: {
home: "الصفحة الرئيسية لـ Unsloth",
closeSidebar: "إغلاق الشريط الجانبي",
openSidebar: "فتح الشريط الجانبي",
resizeSidebar: "تغيير حجم الشريط الجانبي أو طيه",
resizeRunSettings: "تغيير حجم إعدادات التشغيل أو إغلاقها",
openRunSettings: "فتح إعدادات التشغيل",
chatOptions: "خيارات المحادثة",
runOptions: "خيارات التدريب",
},
@ -317,6 +325,8 @@ export const ar = {
diskUsage: "{used} مستخدم / {total}",
diskFree: "{free} متاح",
modelsFolder: "مجلد النماذج",
modelsFolderKeywords:
"النماذج مجلد دليل مسار موقع تنزيلات التنزيل ذاكرة التخزين المؤقت تخزين قرص محرك نقل تغيير models folder path hugging face",
modelsFolderDescription: "المكان الذي تُخزَّن فيه النماذج المُنزَّلة.",
openAction: "فتح",
copyAction: "نسخ المسار",

View file

@ -27,10 +27,18 @@ export const de = {
product: "Unsloth Studio",
accountMenu: "Kontomenü von {name}",
updateAvailable: "Update verfügbar",
resize: {
collapse: "Zum Einklappen klicken",
expand: "Zum Ausklappen klicken",
drag: "Zum Ändern der Größe ziehen",
},
aria: {
home: "Unsloth Startseite",
closeSidebar: "Seitenleiste schließen",
openSidebar: "Seitenleiste öffnen",
resizeSidebar: "Seitenleiste anpassen oder einklappen",
resizeRunSettings: "Ausführungseinstellungen anpassen oder schließen",
openRunSettings: "Ausführungseinstellungen öffnen",
chatOptions: "Chat-Optionen",
runOptions: "Trainingslauf-Optionen",
},
@ -330,9 +338,23 @@ export const de = {
diskFree: "{free} frei",
modelsFolder: "Modell-Ordner",
modelsFolderDescription:
"Wo heruntergeladene Modelle gespeichert werden.",
"Wo heruntergeladene Modelle gespeichert werden. Ändern Sie ihn, um Modelle nicht auf dem Systemlaufwerk abzulegen.",
modelsFolderKeywords:
"Modelle Ordner Verzeichnis Pfad Speicherort Download Downloads Cache Speicher Festplatte Laufwerk verschieben ändern hugging face",
futureDownloads: "Nur neue Downloads",
environmentManaged:
"Wird über die Umgebungsvariable {variable} verwaltet.",
locationFree: "{free} frei",
openAction: "Öffnen",
copyAction: "Pfad kopieren",
changeAction: "Ändern",
resetAction: "Standard verwenden",
chooseTitle: "Speicherort für Modell-Downloads wählen",
chooseAction: "Für künftige Downloads verwenden",
cacheSaved: "Speicherort für Modell-Downloads aktualisiert",
cacheSaveError:
"Der Speicherort für Modell-Downloads konnte nicht geändert werden",
cachePickerError: "Die Ordnerauswahl konnte nicht geöffnet werden",
copied: "Pfad kopiert",
openError: "Der Ordner konnte nicht geöffnet werden",
copyError: "Der Pfad konnte nicht kopiert werden",

View file

@ -24,10 +24,18 @@ export const en = {
product: "Unsloth Studio",
accountMenu: "{name} account menu",
updateAvailable: "Update available",
resize: {
collapse: "Click to collapse",
expand: "Click to expand",
drag: "Drag to resize",
},
aria: {
home: "Unsloth home",
closeSidebar: "Close sidebar",
openSidebar: "Open sidebar",
resizeSidebar: "Resize or collapse sidebar",
resizeRunSettings: "Resize or close run settings",
openRunSettings: "Open run settings",
chatOptions: "Chat options",
runOptions: "Run options",
},
@ -556,8 +564,12 @@ export const en = {
systemDisk: "System disk",
diskUsage: "{used} used / {total}",
diskFree: "{free} free",
modelsFolder: "Model downloads",
modelsFolderDescription: "Hugging Face cache used for model downloads.",
modelsFolder: "Models folder",
modelsFolderDescription:
"Where downloaded models are stored. Change it to keep models off your system drive.",
// Not rendered: extra terms the settings search matches this row on.
modelsFolderKeywords:
"models folder directory path location download downloads cache storage disk drive move relocate hugging face",
futureDownloads: "New downloads only",
environmentManaged: "Managed by the {variable} environment variable.",
locationFree: "{free} free",

View file

@ -27,10 +27,18 @@ export const es = {
product: "Unsloth Studio",
accountMenu: "Menú de cuenta de {name}",
updateAvailable: "Actualización disponible",
resize: {
collapse: "Haz clic para contraer",
expand: "Haz clic para expandir",
drag: "Arrastra para redimensionar",
},
aria: {
home: "Inicio de Unsloth",
closeSidebar: "Cerrar barra lateral",
openSidebar: "Abrir barra lateral",
resizeSidebar: "Redimensionar o contraer la barra lateral",
resizeRunSettings: "Redimensionar o cerrar los ajustes de ejecución",
openRunSettings: "Abrir los ajustes de ejecución",
chatOptions: "Opciones de chat",
runOptions: "Opciones de ejecución",
},
@ -328,6 +336,8 @@ export const es = {
diskUsage: "{used} en uso / {total}",
diskFree: "{free} libre",
modelsFolder: "Carpeta de modelos",
modelsFolderKeywords:
"modelos carpeta directorio ruta ubicacion ubicación descargas descarga cache caché almacenamiento disco unidad mover cambiar models folder path hugging face",
modelsFolderDescription:
"Dónde se almacenan los modelos descargados.",
openAction: "Abrir",

View file

@ -27,10 +27,18 @@ export const fr = {
product: "Unsloth Studio",
accountMenu: "Menu du compte de {name}",
updateAvailable: "Mise à jour disponible",
resize: {
collapse: "Cliquez pour réduire",
expand: "Cliquez pour développer",
drag: "Faites glisser pour redimensionner",
},
aria: {
home: "Accueil Unsloth",
closeSidebar: "Fermer la barre latérale",
openSidebar: "Ouvrir la barre latérale",
resizeSidebar: "Redimensionner ou réduire la barre latérale",
resizeRunSettings: "Redimensionner ou fermer les paramètres d'exécution",
openRunSettings: "Ouvrir les paramètres d'exécution",
chatOptions: "Options de discussion",
runOptions: "Options d'exécution",
},
@ -325,6 +333,8 @@ export const fr = {
diskUsage: "{used} utilisé / {total}",
diskFree: "{free} libre",
modelsFolder: "Dossier des modèles",
modelsFolderKeywords:
"modeles modèles dossier repertoire répertoire chemin emplacement telechargements téléchargements cache stockage disque lecteur deplacer déplacer changer models folder path hugging face",
modelsFolderDescription: "Emplacement de stockage des modèles téléchargés.",
openAction: "Ouvrir",
copyAction: "Copier le chemin",

View file

@ -27,10 +27,18 @@ export const hi = {
product: "Unsloth Studio",
accountMenu: "{name} खाता मेनू",
updateAvailable: "अपडेट उपलब्ध है",
resize: {
collapse: "छोटा करने के लिए क्लिक करें",
expand: "विस्तार के लिए क्लिक करें",
drag: "आकार बदलने के लिए खींचें",
},
aria: {
home: "Unsloth होम",
closeSidebar: "साइडबार बंद करें",
openSidebar: "साइडबार खोलें",
resizeSidebar: "साइडबार का आकार बदलें या छोटा करें",
resizeRunSettings: "रन सेटिंग्स का आकार बदलें या बंद करें",
openRunSettings: "रन सेटिंग्स खोलें",
chatOptions: "चैट विकल्प",
runOptions: "रन विकल्प",
},
@ -316,6 +324,8 @@ export const hi = {
diskUsage: "{used} उपयोग में / {total}",
diskFree: "{free} खाली",
modelsFolder: "मॉडल फ़ोल्डर",
modelsFolderKeywords:
"मॉडल फ़ोल्डर फोल्डर निर्देशिका पथ स्थान डाउनलोड कैश संग्रहण डिस्क ड्राइव स्थानांतरित बदलें models folder path hugging face",
modelsFolderDescription: "जहां डाउनलोड किए गए मॉडल संग्रहीत होते हैं।",
openAction: "खोलें",
copyAction: "पथ कॉपी करें",

View file

@ -28,10 +28,18 @@ export const ja = {
product: "Unsloth Studio",
accountMenu: "{name} のアカウントメニュー",
updateAvailable: "アップデートが利用可能です",
resize: {
collapse: "クリックで折りたたむ",
expand: "クリックで展開",
drag: "ドラッグでサイズ変更",
},
aria: {
home: "Unsloth ホーム",
closeSidebar: "サイドバーを閉じる",
openSidebar: "サイドバーを開く",
resizeSidebar: "サイドバーのサイズ変更または折りたたみ",
resizeRunSettings: "実行設定のサイズ変更または閉じる",
openRunSettings: "実行設定を開く",
chatOptions: "チャットオプション",
runOptions: "実行オプション",
},
@ -393,6 +401,8 @@ export const ja = {
diskUsage: "{used} 使用中 / {total}",
diskFree: "{free} 空き",
modelsFolder: "モデルフォルダ",
modelsFolderKeywords:
"モデル フォルダ ディレクトリ パス 保存先 場所 ダウンロード キャッシュ ストレージ ディスク ドライブ 移動 変更 models folder path hugging face",
modelsFolderDescription: "ダウンロードしたモデルの保存先。",
openAction: "開く",
copyAction: "パスをコピー",

View file

@ -27,10 +27,18 @@ export const ko = {
product: "Unsloth Studio",
accountMenu: "{name} 계정 메뉴",
updateAvailable: "업데이트 사용 가능",
resize: {
collapse: "클릭하여 접기",
expand: "클릭하여 펼치기",
drag: "드래그하여 크기 조절",
},
aria: {
home: "Unsloth 홈",
closeSidebar: "사이드바 닫기",
openSidebar: "사이드바 열기",
resizeSidebar: "사이드바 크기 조절 또는 접기",
resizeRunSettings: "실행 설정 크기 조절 또는 닫기",
openRunSettings: "실행 설정 열기",
chatOptions: "채팅 옵션",
runOptions: "학습 옵션",
},
@ -315,6 +323,8 @@ export const ko = {
diskUsage: "{used} 사용 중 / {total}",
diskFree: "{free} 여유",
modelsFolder: "모델 폴더",
modelsFolderKeywords:
"모델 폴더 디렉터리 디렉토리 경로 위치 저장 다운로드 캐시 저장소 디스크 드라이브 이동 변경 models folder path hugging face",
modelsFolderDescription: "다운로드한 모델이 저장되는 위치입니다.",
openAction: "열기",
copyAction: "경로 복사",

View file

@ -27,10 +27,18 @@ export const ptBR = {
product: "Unsloth Studio",
accountMenu: "Menu de conta {name}",
updateAvailable: "Atualização disponível",
resize: {
collapse: "Clique para recolher",
expand: "Clique para expandir",
drag: "Arraste para redimensionar",
},
aria: {
home: "Início do Unsloth",
closeSidebar: "Fechar barra lateral",
openSidebar: "Abrir barra lateral",
resizeSidebar: "Redimensionar ou recolher a barra lateral",
resizeRunSettings: "Redimensionar ou fechar as configurações de execução",
openRunSettings: "Abrir as configurações de execução",
chatOptions: "Opções de chat",
runOptions: "Opções de execução",
},
@ -417,6 +425,8 @@ export const ptBR = {
diskUsage: "{used} usados / {total}",
diskFree: "{free} livres",
modelsFolder: "Pasta de modelos",
modelsFolderKeywords:
"modelos pasta diretorio diretório caminho local localizacao localização downloads baixar cache armazenamento disco unidade mover alterar models folder path hugging face",
modelsFolderDescription: "Onde os modelos baixados são armazenados.",
openAction: "Abrir",
copyAction: "Copiar caminho",

View file

@ -27,10 +27,18 @@ export const ru = {
product: "Unsloth Studio",
accountMenu: "Меню аккаунта {name}",
updateAvailable: "Доступно обновление",
resize: {
collapse: "Нажмите, чтобы свернуть",
expand: "Нажмите, чтобы развернуть",
drag: "Потяните, чтобы изменить размер",
},
aria: {
home: "Главная Unsloth",
closeSidebar: "Закрыть боковую панель",
openSidebar: "Открыть боковую панель",
resizeSidebar: "Изменить размер или свернуть боковую панель",
resizeRunSettings: "Изменить размер или закрыть настройки запуска",
openRunSettings: "Открыть настройки запуска",
chatOptions: "Параметры чата",
runOptions: "Параметры запуска",
},
@ -316,6 +324,8 @@ export const ru = {
diskUsage: "{used} использовано / {total}",
diskFree: "{free} свободно",
modelsFolder: "Папка моделей",
modelsFolderKeywords:
"модели папка каталог путь расположение загрузки кэш хранилище диск перенести изменить models folder path hugging face",
modelsFolderDescription: "Где хранятся загруженные модели.",
openAction: "Открыть",
copyAction: "Копировать путь",

View file

@ -27,10 +27,18 @@ export const zhCN = {
product: "Unsloth Studio",
accountMenu: "{name} 账号菜单",
updateAvailable: "有可用更新",
resize: {
collapse: "点击折叠",
expand: "点击展开",
drag: "拖动调整大小",
},
aria: {
home: "Unsloth 首页",
closeSidebar: "关闭侧边栏",
openSidebar: "打开侧边栏",
resizeSidebar: "调整或折叠侧边栏",
resizeRunSettings: "调整或关闭运行设置",
openRunSettings: "打开运行设置",
chatOptions: "聊天选项",
runOptions: "训练选项",
},
@ -408,6 +416,8 @@ export const zhCN = {
diskUsage: "已用 {used} / {total}",
diskFree: "{free} 可用",
modelsFolder: "模型文件夹",
modelsFolderKeywords:
"模型 文件夹 目录 路径 位置 下载 缓存 存储 磁盘 驱动器 移动 更改 models folder path hugging face",
modelsFolderDescription: "已下载模型的存储位置。",
openAction: "打开",
copyAction: "复制路径",

View file

@ -1363,6 +1363,23 @@ html[data-chat-font] .aui-root {
cursor: pointer;
}
/* While a panel edge is dragged, keep the resize cursor even as the pointer
travels over buttons and text that would claim their own. */
html[data-panel-resizing],
html[data-panel-resizing] * {
cursor: col-resize !important;
user-select: none !important;
}
html[data-panel-resizing]
:is(
[data-slot="sidebar-inner"],
[data-slot="sidebar-inset"],
[data-slot="chat-settings-panel"] > div
) {
pointer-events: none;
}
/* Model selector: pointer cursor on every clickable element. */
.unsloth-model-selector-trigger,
.unsloth-model-selector-menu button {

View file

@ -0,0 +1,664 @@
// SPDX-License-Identifier: AGPL-3.0-only
// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
/**
* A relative link in CHANGELOG.md means "somewhere in the Unsloth repository",
* but inside Studio it would resolve against Studio's own origin. Rewriting to
* absolute repository URLs makes them behave the way GitHub renders the file.
*/
import {
type CodeSpan,
codeSpans,
insideSpan,
} from "@/lib/markdown-code-spans";
import { commentClosesBelow } from "@/lib/markdown-inline-comments";
import {
EMPTY_LIST_STATE,
type ListState,
NO_QUOTE,
type QuoteState,
containerContent,
hiddenStructure,
indentWidth,
itemContent,
openLists,
quoteDepth,
quoteState,
} from "@/lib/markdown-list-columns";
const LINK_BASE = "https://github.com/unslothai/unsloth/blob/main/";
const IMAGE_BASE = "https://raw.githubusercontent.com/unslothai/unsloth/main/";
// Inline `](dest)` plus the `[label]: dest` reference form. The destination is
// either <bracketed> or runs to whitespace or the closing paren.
const NESTED_LABEL = String.raw`((?:[^[\]\\]|\\.|\[(?:[^[\]\\]|\\.)*\])*)`;
// Only ASCII punctuation is escapable, so the backslash in `a\ b.md` is an
// ordinary character of the destination and the space still ends it.
const ESCAPABLE = String.raw`[!-/:-@[-\`{-~]`;
const DESTINATION_CHAR = String.raw`\\${ESCAPABLE}|[^\s()]`;
// A destination may hold balanced parentheses, and a path may nest them, so
// `[x](((draft)).md)` points at `((draft)).md`. An expression cannot count, so
// pairs are unrolled to the depth cmark stops at, which is what GitHub renders.
const MAX_DESTINATION_NESTING = 32;
/** A balanced parenthesised run nested up to `depth` levels deep. */
function nestedParens(depth: number): string {
let group = String.raw`\((?:${DESTINATION_CHAR})*\)`;
for (let left = depth - 1; left > 0; left -= 1) {
group = String.raw`\((?:${DESTINATION_CHAR}|${group})*\)`;
}
return group;
}
const BALANCED_DESTINATION = String.raw`(?:${DESTINATION_CHAR}|${nestedParens(MAX_DESTINATION_NESTING)})*`;
const PLAIN_DESTINATION = String.raw`(?:${DESTINATION_CHAR})*`;
// A balanced pair counts only while a `)` or a title still closes the link
// after it, or swallowing it would invent a link across lines.
const CLOSES_LINK = String.raw`(?=[ \t]*[)'"])`;
// A destination that runs out of line has its closer below it, the line being
// only part of the link. One stopping short of a closer is no destination at all,
// so `[x](a b.md)` and `[x](a(b.md)` stay plain text and keep the paths they name.
const CLOSES_OR_ENDS_LINE = String.raw`(?=[ \t]*(?:[)'"]|$))`;
const INLINE_TARGET = new RegExp(
String.raw`(!?)\[${NESTED_LABEL}\]\(\s*(<[^<>\n]*>|${BALANCED_DESTINATION}${CLOSES_LINK}|${PLAIN_DESTINATION}${CLOSES_OR_ENDS_LINE})`,
"g",
);
const REFERENCE_TARGET = /^( {0,3}\[((?:[^[\]\\]|\\.)*)\]:\s*)(<[^<>\n]*>|\S+)/;
// `![alt][label]`, `![label][]` and `![label]`: a definition they point at
// has to resolve to the raw file, not to its page on GitHub.
const IMAGE_REFERENCE =
/!\[((?:[^[\]\\]|\\.)*)\](?:\[((?:[^[\]\\]|\\.)*)\]|(?!\())/g;
const FENCE = /^ {0,3}(`{3,}|~{3,})(.*)$/;
// Four columns past the container start indented code, unless a paragraph is
// open. Inside a list item that is measured from the item's content column, so a
// link indented under a bullet is prose and still resolves.
const INDENTED_CODE_INDENT = 4;
// CommonMark type 1 HTML blocks show their contents verbatim.
const RAW_HTML_OPEN = /^ {0,3}<(pre|script|style|textarea)(?=[\s>]|$)/i;
const RAW_HTML_CLOSE = /<\/(pre|script|style|textarea)\s*>/i;
// Type 6 and 7 blocks are literal too and run to the next blank line, not to a
// closing tag, so `<details>` holds Markdown only after a blank line. Type 7 (any
// other complete tag alone on a line) cannot interrupt a paragraph.
const HTML_BLOCK_OPEN = /^ {0,3}<\/?([a-zA-Z][a-zA-Z0-9-]*)(?=[\s/>]|$)/;
const HTML_ATTRIBUTE =
"(?:\\s+[a-zA-Z_:][a-zA-Z0-9_.:-]*(?:\\s*=\\s*(?:[^\\s\"'=<>`]+|'[^']*'|\"[^\"]*\"))?)";
const HTML_TAG_ONLY_LINE = new RegExp(
`^ {0,3}(?:<[a-zA-Z][a-zA-Z0-9-]*${HTML_ATTRIBUTE}*\\s*/?>|</[a-zA-Z][a-zA-Z0-9-]*\\s*>)\\s*$`,
);
const HTML_BLOCK_TAGS = new Set(
`address article aside base basefont blockquote body caption center col colgroup
dd details dialog dir div dl dt fieldset figcaption figure footer form frame
frameset h1 h2 h3 h4 h5 h6 head header hr html iframe legend li link main menu
menuitem nav noframes ol optgroup option p param search section summary table
tbody td tfoot th thead title tr track ul`.split(/\s+/),
);
// Lines that are blocks in their own right, so no paragraph is open after.
const BLOCK_LINE =
/^ {0,3}(?:#{1,6}([ \t]|$)|(?:\*[ \t]*){3,}$|(?:-[ \t]*){3,}$|(?:_[ \t]*){3,}$|>|=+[ \t]*$)/;
// A definition is a block of its own but may not interrupt a paragraph, so it
// ends the one above only when there is none to continue. It opens none either,
// or consecutive definitions could never start (spec 0.31.2 section 4.7). Same
// rule as `_LINK_DEFINITION` in the backend's `after_paragraph`.
const LINK_DEFINITION = /^ {0,3}\[(?:[^[\]\\]|\\.)+\]:/;
const LINE_ENDINGS = /\r\n?/g;
// A scheme, a protocol-relative host, or a fragment: already absolute enough.
// `//` needs a host after it, so `///docs` stays a repository path.
const ABSOLUTE = /^(?:[a-zA-Z][a-zA-Z0-9+.-]*:|\/\/[^/]|#)/;
const COMMENT_OPEN = "<!--";
const COMMENT_CLOSE = "-->";
const COMMENT_BLOCK_OPEN = /^ {0,3}<!--/;
/**
* `line` with its commented spans blanked, and whether a comment block is still
* open below it. Commented content renders as nothing, so it holds no fence,
* block or code span. Lengths are preserved so offsets still line up.
*
* Only a comment that starts a line opens a block (CommonMark type 2), and only
* that runs on to the line holding `-->`, tail included. One written mid-sentence
* is inline raw HTML belonging to its paragraph, so its `-->` may arrive on a
* later line and only the text up to it is hidden. `closesBelow` says one does;
* without it the opener is ordinary text, so a note merely mentioning `<!--` must
* not hide the links below it.
*
* "Starts a line" is read inside the container, so `blockOpen` comes from the
* item's content rather than the raw line.
*/
function maskComments(
line: string,
inComment: boolean,
runOn: boolean,
closesBelow: boolean,
blockOpen: boolean,
): [string, boolean, boolean] {
if (inComment) {
// The closing line belongs to the block, tail included.
return [" ".repeat(line.length), !line.includes(COMMENT_CLOSE), false];
}
if (runOn) {
const closed = line.indexOf(COMMENT_CLOSE);
if (closed < 0) {
return [" ".repeat(line.length), false, true];
}
// Only up to the closer: the tail is the paragraph's own text again.
const resumed = closed + COMMENT_CLOSE.length;
return maskInline(line, resumed, closesBelow);
}
if (blockOpen) {
// `<!-->` and `<!--->` are complete comments, so the closer may overlap the
// opener; searching past it would blank the rest of the file.
return [" ".repeat(line.length), !line.includes(COMMENT_CLOSE), false];
}
return maskInline(line, 0, closesBelow);
}
/** `maskComments` from `from`, where no comment block is open. */
function maskInline(
line: string,
from: number,
closesBelow: boolean,
): [string, boolean, boolean] {
let out = " ".repeat(from);
let index = from;
// Scanned only once an opener turns up. Spans are ordered and disjoint and each
// opener sits at or past the last, so the search resumes rather than restarts.
let spans: CodeSpan[] | null = null;
let cursor = 0;
while (index < line.length) {
const start = line.indexOf(COMMENT_OPEN, index);
if (start < 0) {
return [out + line.slice(index), false, false];
}
spans ??= codeSpans(line);
while (cursor < spans.length && (spans[cursor]?.end ?? 0) <= start) {
cursor += 1;
}
// A delimiter inside inline code is literal, not a comment opener.
const span = spans[cursor];
if (span !== undefined && span.start <= start) {
out += line.slice(index, span.end);
index = span.end;
continue;
}
// `<!-->` and `<!--->` are complete comments, so the closer may overlap.
const close = line.indexOf(COMMENT_CLOSE, start + 2);
if (close < 0) {
if (closesBelow) {
// The paragraph carries the comment on, so the line from the opener is
// inside it, and so is the line below.
return [
out + line.slice(index, start) + " ".repeat(line.length - start),
false,
true,
];
}
// Nothing closes it at all, so the renderer shows it as ordinary text.
return [out + line.slice(index), false, false];
}
out += line.slice(index, start);
out += " ".repeat(close + COMMENT_CLOSE.length - start);
index = close + COMMENT_CLOSE.length;
}
return [out, false, false];
}
/**
* Whether `line` is written outside the container an open block belongs to. A
* fence and an HTML block hold no lazy continuation line, so content left of the
* item, or outside the quote, ends the block with its container. A raw block or
* comment inside a list item ends on a blank line too: the item takes the break,
* so what follows is a block of the item's own.
*/
function leavesContainer(
line: string,
quotes: number,
column: number,
blockQuotes: number,
rawInItem: boolean,
): boolean {
if (quotes < blockQuotes) {
return true;
}
if (!line.trim()) {
return rawInItem;
}
return column > 0 && indentWidth(line) < column;
}
/** True if `line` starts a CommonMark type 6 or type 7 HTML block. */
function opensHtmlBlock(line: string, afterParagraph: boolean): boolean {
const named = HTML_BLOCK_OPEN.exec(line);
if (named && HTML_BLOCK_TAGS.has((named[1] ?? "").toLowerCase())) {
return true;
}
return !afterParagraph && HTML_TAG_ONLY_LINE.test(line);
}
/** A reference label as CommonMark compares them. */
function label(text: string): string {
return text.trim().replace(/\s+/g, " ").toLowerCase();
}
const NEEDS_BRACKETS = /[()\s]/;
// `\(` in a destination is a literal paren. Only ASCII punctuation is escapable,
// so the backslash in `docs\alpha.md` is part of the path and has to survive.
const ESCAPE = new RegExp(String.raw`\\(${ESCAPABLE})`, "g");
// A URL parser reads a backslash as a path separator, so `docs\a.md` would
// resolve to `docs/a.md`. Encode it first, the way a renderer normalises it.
const BACKSLASH = /\\/g;
// Only spaces and tabs may follow a closing fence.
const NON_SPACE = /[^ \t]/;
const LEADING_SLASHES = /^\/+/;
function absolute(target: string, image: boolean): string {
const base = image ? IMAGE_BASE : LINK_BASE;
const trimmed = target.trim().replace(ESCAPE, "$1");
if (!trimmed || ABSOLUTE.test(trimmed)) {
return target;
}
try {
// A leading slash means the repository root, not the site root, so append
// it to the base instead of replacing the base path.
const resolved = new URL(
trimmed.replace(LEADING_SLASHES, "").replace(BACKSLASH, "%5C"),
base,
).toString();
// `../` can climb out of the repository: leave those alone.
return resolved.startsWith(base) ? resolved : target;
} catch {
return target;
}
}
/** True when `index` is escaped by an odd run of backslashes. */
function isEscaped(line: string, index: number): boolean {
let slashes = 0;
while (line[index - 1 - slashes] === "\\") {
slashes += 1;
}
return slashes % 2 === 1;
}
function unwrap(target: string): string {
return target.startsWith("<") && target.endsWith(">")
? target.slice(1, -1)
: target;
}
/** The destination as it goes back into the line. */
function wrap(resolved: string, original: string): string {
const bracketed = original.startsWith("<") && original.endsWith(">");
return bracketed || (resolved !== original && NEEDS_BRACKETS.test(resolved))
? `<${resolved}>`
: resolved;
}
/** Rewrites one line's link and image targets, leaving code spans alone. */
function rewriteLine(
line: string,
imageLabels: Set<string>,
spans: CodeSpan[],
base: number,
isDefinition: boolean,
): string {
const reference = isDefinition ? REFERENCE_TARGET.exec(line) : null;
if (reference) {
const target = reference[3] ?? "";
const resolved = absolute(
unwrap(target),
imageLabels.has(label(reference[2] ?? "")),
);
const rest = line.slice(reference[0].length);
return `${reference[1]}${wrap(resolved, target)}${rest}`;
}
INLINE_TARGET.lastIndex = 0;
return line.replace(INLINE_TARGET, (match, bang, text, target, offset) => {
// `\\[` is a literal bracket, so the expression is not a link.
const opener = offset + (bang ? 1 : 0);
if (insideSpan(spans, base + offset) || isEscaped(line, opener)) {
return match;
}
// `\\!` is a literal mark, so what follows is a link, not an image.
const image = bang === "!" && !isEscaped(line, offset);
const resolved = absolute(unwrap(target), image);
// A badge nests an image inside a link, so the label is rewritten too.
const inner = text.includes("](")
? rewriteLine(text, imageLabels, codeSpans(text), 0, false)
: text;
return `${bang}[${inner}](${wrap(resolved, target)}`;
});
}
interface Classified {
// Lines the renderer shows as Markdown, by index.
text: number[];
// Same lines, blanked where the renderer shows code, for span scanning.
masked: string;
// Lines where a `[label]: dest` definition can start.
definition: Set<number>;
// Document ranges the renderer hides inside HTML comments.
comments: CodeSpan[];
}
/**
* Sorts lines into Markdown and code, masking the code so a span cannot pair
* across it. Offsets are preserved, so a mask span sits where it does in the doc.
*/
function classify(lines: string[]): Classified {
const text: number[] = [];
const definition = new Set<number>();
const masked: string[] = [];
let openFence: string | null = null;
let inRawHtml = false;
let inHtmlBlock = false;
// Where the open block was written: the content column of the item it belongs
// to, 0 at document level, plus the blockquotes it sits inside. Only one is ever
// open, and none holds a lazy continuation line, so a line left of the item or
// outside the quote ends the block with its container.
let blockColumn = 0;
let blockQuotes = 0;
let inComment = false;
// True while an inline comment opened above runs on into this line, carried by
// the paragraph holding it.
let runOn = false;
const closesBelow = commentClosesBelow(lines);
let inCode = false;
let afterParagraph = false;
let quote: QuoteState = NO_QUOTE;
let lists: ListState = EMPTY_LIST_STATE;
const comments: CodeSpan[] = [];
let offset = 0;
// The line as list tracking sees it: blank wherever nothing renders. Taken
// with the paragraph state from the line above, as the renderer would.
const track = (structural: string, above: QuoteState): void => {
lists = openLists(structural, lists, afterParagraph, above.quoted);
};
// Where a block just opened sits, read after the opener closed the items it
// is dedented out of, so it belongs to the container it is really in.
const startBlock = (quotes: number): void => {
blockColumn = lists.columns.at(-1) ?? 0;
blockQuotes = quotes;
};
const endBlock = (): void => {
blockColumn = 0;
blockQuotes = 0;
};
lines.forEach((original, index) => {
const start = offset;
offset += original.length + 1;
// The quote state from the line above, which is what list tracking asks
// about. Only plain text below rewrites it, so every block returning early
// leaves no quoted paragraph open behind it.
const above = quote;
quote = NO_QUOTE;
// A fence, comment or HTML block runs only to the end of the container it was
// written in, so a line dedented out of that item or outside that quote
// closes both.
const quotes = quoteDepth(original);
let inBlock = openFence !== null || inRawHtml || inHtmlBlock || inComment;
if (
inBlock &&
leavesContainer(
original,
quotes,
blockColumn,
blockQuotes,
(inRawHtml || inComment) && blockColumn > 0 && blockQuotes === 0,
)
) {
openFence = null;
inRawHtml = false;
inHtmlBlock = false;
inComment = false;
endBlock();
inBlock = false;
}
// Read from the container the line is written in, so a fence three columns
// past a nested bullet or behind a quote marker still opens one. A block
// already open keeps only its own quote stripped, or a deeper marker in it
// would read as a closer.
const container = containerContent(
original,
lists,
inBlock ? blockQuotes : quotes,
);
// A comment cannot open a fence and a fence hides a comment opener, so resolve
// them in that order or a hidden delimiter opens a phantom fence. An opener is
// read past a marker on the same line too, since a fence written as an item's
// first content opens inside it. Only an opener: fenced content is literal and
// a closer carries no marker.
const fenceSource = inComment
? null
: FENCE.exec(
openFence === null
? itemContent(container, afterParagraph)
: container,
);
if (inRawHtml) {
track("", above);
inRawHtml = !RAW_HTML_CLOSE.test(container);
if (!inRawHtml) {
endBlock();
}
masked.push(" ".repeat(original.length));
afterParagraph = false;
return;
}
if (inHtmlBlock) {
track("", above);
// Only a blank line ends a type 6 or 7 block, so nothing inside one is a
// fence or a link. A bare quote marker holds nothing, so it ends one too.
inHtmlBlock = !!container.trim();
if (!inHtmlBlock) {
endBlock();
}
masked.push(" ".repeat(original.length));
afterParagraph = false;
return;
}
const fence = fenceSource;
if (fence) {
// A fence renders as nothing, but its indent still closes an item.
track(original, above);
const marker = fence[1] ?? "";
if (openFence === null) {
// A backtick fence's info string may not contain a backtick.
openFence =
marker[0] !== "`" || !(fence[2] ?? "").includes("`") ? marker : null;
if (openFence === null) {
text.push(index);
masked.push(original);
afterParagraph = true;
return;
}
startBlock(quotes);
} else if (
// A closer matches the opening character and carries nothing after it.
marker[0] === openFence[0] &&
marker.length >= openFence.length &&
!NON_SPACE.test(fence[2] ?? "")
) {
openFence = null;
endBlock();
}
masked.push(" ".repeat(original.length));
afterParagraph = false;
return;
}
if (openFence !== null) {
track("", above);
// Fenced content is literal, so a comment opener in it is not one.
masked.push(" ".repeat(original.length));
return;
}
// A block already open owns this line, so it is content rather than a block
// written at the column it happens to start in.
const hidden = inComment;
const carried = runOn;
// A comment is an HTML block too, so one written as a list item's first
// content opens inside that item exactly as a fence does: read past a marker
// on the same line and from its container's column, not the line's margin.
const opensComment =
!(hidden || carried) &&
COMMENT_BLOCK_OPEN.test(itemContent(container, afterParagraph));
// Only now, outside every fence, does a comment hide what follows.
const [line, stillInComment, stillRunOn] = maskComments(
original,
inComment,
runOn,
closesBelow[index + 1] ?? false,
opensComment,
);
inComment = stillInComment;
runOn = stillRunOn;
// A line an inline comment runs on into is still a line of the paragraph
// that carries it: only its text is hidden, never its block structure.
const structure = carried ? original : line;
// The same container reading as above, now the comments are masked. A comment
// blanks its own line, so that line is read as written: the block renders as
// nothing, but the item it is the content of still opens.
const source = opensComment ? original : line;
const visible = containerContent(source, lists, quotes);
// An HTML block written as a list item's first content opens inside that item,
// as a fence does, so an opener is read past a marker on the same line. The
// marker survives into the structural line, so its item is still tracked.
const content = itemContent(visible, afterParagraph);
const marker =
content === visible
? ""
: source.slice(0, source.length - content.length);
// Taken before an HTML opener is hidden: it renders as nothing, but its indent
// still closes a list item it sits left of. A comment or a <pre> keeps only its
// column and marker, since the text it hides is not Markdown and opens no list.
const opensRaw = !carried && RAW_HTML_OPEN.test(content);
track(
!(hidden || carried) && (opensRaw || !line.trim())
? hiddenStructure(original, marker)
: structure,
above,
);
// Read once the opener has closed the items it is dedented out of, so the
// comment block belongs to the item it is really written inside.
if (inComment !== hidden) {
if (inComment) {
startBlock(quotes);
} else {
endBlock();
}
}
for (let at = 0; at < line.length; at += 1) {
if (line[at] === " " && original[at] !== " ") {
const from = at;
while (at < line.length && line[at] === " " && original[at] !== " ") {
at += 1;
}
comments.push({ start: start + from, end: start + at, content: "" });
}
}
if (opensRaw) {
inRawHtml = !RAW_HTML_CLOSE.test(content.replace(RAW_HTML_OPEN, ""));
if (inRawHtml) {
startBlock(quotes);
}
masked.push(" ".repeat(line.length));
afterParagraph = false;
return;
}
if (!carried && content.trim() && opensHtmlBlock(content, afterParagraph)) {
inHtmlBlock = true;
startBlock(quotes);
masked.push(" ".repeat(line.length));
afterParagraph = false;
return;
}
const blank = !structure.trim();
// Measured from the innermost open item's content column, not the margin:
// four spaces under "- Details:" is a paragraph, not a code block.
const column = lists.columns.at(-1) ?? 0;
const indented = indentWidth(structure) - column >= INDENTED_CODE_INDENT;
// Indented code starts only outside a paragraph and runs to a dedent.
if (inCode) {
inCode = blank || indented;
} else {
inCode = !afterParagraph && !blank && indented;
}
if (inCode) {
masked.push(" ".repeat(line.length));
afterParagraph = false;
return;
}
// A definition cannot interrupt a paragraph.
if (!afterParagraph) {
definition.add(index);
}
text.push(index);
masked.push(line);
afterParagraph =
!blank &&
!BLOCK_LINE.test(structure) &&
(afterParagraph || !LINK_DEFINITION.test(structure));
quote = quoteState(structure, above.inQuote);
});
return { text, masked: masked.join("\n"), definition, comments };
}
/** Absolute repository URLs for every relative link and image in `markdown`. */
export function resolveChangelogLinks(markdown: string): string {
// The desktop updater body arrives with CRLF, which would hide fences.
const lines = markdown.replace(LINE_ENDINGS, "\n").split("\n");
const { text, masked, definition, comments } = classify(lines);
// Scanned over the whole document, so a span may cross a line break. Commented
// ranges join them: the renderer shows neither, so a link in one is not
// followable and rewriting it would only mutate hidden text.
const spans = [...codeSpans(masked), ...comments].sort(
(a, b) => a.start - b.start,
);
// Offset of each line in the document, to place matches inside it.
const offsets: number[] = [];
let cursor = 0;
for (const line of lines) {
offsets.push(cursor);
cursor += line.length + 1;
}
// Only images resolve against the raw host, so collect the image labels
// before rewriting any definition.
const imageLabels = new Set<string>();
for (const index of text) {
const line = lines[index] ?? "";
IMAGE_REFERENCE.lastIndex = 0;
for (
let match = IMAGE_REFERENCE.exec(line);
match !== null;
match = IMAGE_REFERENCE.exec(line)
) {
// An escaped mark makes it a link, so its definition stays a page URL.
if (
insideSpan(spans, (offsets[index] ?? 0) + match.index) ||
isEscaped(line, match.index)
) {
continue;
}
const explicit = match[2] ?? "";
imageLabels.add(label(explicit.trim() ? explicit : (match[1] ?? "")));
}
}
const rewritten = [...lines];
for (const index of text) {
rewritten[index] = rewriteLine(
lines[index] ?? "",
imageLabels,
spans,
offsets[index] ?? 0,
definition.has(index),
);
}
return rewritten.join("\n");
}

View file

@ -0,0 +1,123 @@
// SPDX-License-Identifier: AGPL-3.0-only
// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
/**
* CommonMark code spans: a backtick run closes only on an equal-length run.
* That needs lookbehind, which older Safari rejects, so runs are scanned by hand.
*/
export interface CodeSpan {
// Offsets of the whole span, delimiters included.
start: number;
end: number;
// Between the delimiters, with the one space of padding removed.
content: string;
}
function runLength(text: string, index: number): number {
let end = index;
while (text[end] === "`") {
end += 1;
}
return end - index;
}
/** True when `index` is escaped by an odd run of backslashes. */
function escaped(text: string, index: number): boolean {
let slashes = 0;
while (text[index - 1 - slashes] === "\\") {
slashes += 1;
}
return slashes % 2 === 1;
}
/** CommonMark drops one space of padding, so `` ` a ` `` renders as "a". */
function stripPadding(content: string): string {
if (
content.length > 1 &&
content.startsWith(" ") &&
content.endsWith(" ") &&
content.trim() !== ""
) {
return content.slice(1, -1);
}
return content;
}
/** Every code span in `text`, in order. Unclosed runs are ordinary text. */
export function codeSpans(text: string): CodeSpan[] {
const spans: CodeSpan[] = [];
let index = 0;
while (index < text.length) {
if (text[index] !== "`" || escaped(text, index)) {
index += 1;
continue;
}
const ticks = runLength(text, index);
const contentStart = index + ticks;
let cursor = contentStart;
let closed = false;
while (cursor < text.length) {
// Escapes do not apply inside a span, so a run after a backslash closes it.
if (text[cursor] !== "`") {
cursor += 1;
continue;
}
const candidate = runLength(text, cursor);
if (candidate === ticks) {
spans.push({
start: index,
end: cursor + ticks,
content: stripPadding(text.slice(contentStart, cursor)),
});
index = cursor + ticks;
closed = true;
break;
}
cursor += candidate;
}
if (!closed) {
// Nothing closes this run: it is literal text, carry on after it.
index = contentStart;
}
}
return spans;
}
/** Replaces every code span with `park(content)`, leaving the rest as is. */
export function parkCodeSpans(
text: string,
park: (content: string) => string,
): string {
const spans = codeSpans(text);
if (spans.length === 0) {
return text;
}
let out = "";
let cursor = 0;
for (const span of spans) {
out += text.slice(cursor, span.start) + park(span.content);
cursor = span.end;
}
return out + text.slice(cursor);
}
/** True when `index` falls inside one of `spans`, which are in order. */
export function insideSpan(spans: CodeSpan[], index: number): boolean {
let low = 0;
let high = spans.length - 1;
while (low <= high) {
const mid = (low + high) >> 1;
const span = spans[mid];
if (span === undefined || index < span.start) {
high = mid - 1;
} else if (index >= span.end) {
low = mid + 1;
} else {
return true;
}
}
return false;
}

View file

@ -0,0 +1,62 @@
// SPDX-License-Identifier: AGPL-3.0-only
// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
/**
* An HTML comment written mid-sentence is inline raw HTML, not a block, so it
* belongs to its paragraph: the `-->` may arrive on a later line of that same
* paragraph and everything between renders as nothing, while past the paragraph
* the `<!--` is ordinary text. Both changelog scanners share that answer here.
*
* The backend needs none of it: a heading closes the paragraph it sits under, so
* no heading can ever land inside one of these comments.
*/
import { interruptsParagraph } from "@/lib/markdown-list-columns";
const COMMENT_CLOSE = "-->";
// A line that cannot be more of the paragraph above it: blank, or a block that
// may interrupt one. Leading punctuation is not one: `-->` alone is the ordinary
// multiline close and a continuation may open with emphasis, so reading either as
// a break leaves the comment unclosed and its text on show. Indented code and link
// definitions are absent: neither may interrupt a paragraph (spec 0.31.2 4.4, 4.7).
const BLANK = /^[ \t]*$/;
const ATX_HEADING = /^ {0,3}#{1,6}([ \t]|$)/;
const FENCE = /^ {0,3}(?:`{3,}|~{3,})/;
const THEMATIC_BREAK =
/^ {0,3}(?:(?:\*[ \t]*){3,}|(?:-[ \t]*){3,}|(?:_[ \t]*){3,})$/;
// A row of `=` or `-` alone makes the paragraph above it a setext heading, ending it.
const SETEXT_UNDERLINE = /^ {0,3}(?:=+|-+)[ \t]*$/;
// A tag, comment or declaration at the start of a line. HTML block types 1 to 6
// interrupt a paragraph; type 7 does not, but reading one as a break only leaves
// the opener as plain text, which is what a leading `<` has always meant here.
const HTML_LINE = /^ {0,3}</;
/** Whether `line` starts a block of its own rather than continuing a paragraph. */
function startsBlock(line: string): boolean {
return (
BLANK.test(line) ||
ATX_HEADING.test(line) ||
FENCE.test(line) ||
THEMATIC_BREAK.test(line) ||
SETEXT_UNDERLINE.test(line) ||
HTML_LINE.test(line) ||
// Blockquote, or a list item with content: the rule the other scanners share.
interruptsParagraph(line)
);
}
/**
* For each line, whether a `-->` is reachable without leaving the paragraph it
* starts in. Read at `index + 1` it answers whether an inline comment opened on
* `index` and left unclosed there is a comment at all.
*/
export function commentClosesBelow(lines: string[]): boolean[] {
const closes: boolean[] = new Array(lines.length + 1).fill(false);
for (let at = lines.length - 1; at >= 0; at -= 1) {
const line = lines[at] ?? "";
closes[at] =
!startsBlock(line) &&
(line.includes(COMMENT_CLOSE) || (closes[at + 1] ?? false));
}
return closes;
}

View file

@ -0,0 +1,357 @@
// SPDX-License-Identifier: AGPL-3.0-only
// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
/**
* CommonMark measures a block's indentation from its container, not the left
* margin: four spaces at document level and four under a bullet mean different
* things. Tracking the open items lets both changelog scanners ask "is this
* indented code?" the way a renderer would.
*
* Ported from `_open_lists` in studio/backend/utils/changelog.py so the three
* scanners classify a line the same way.
*/
/** The open list items, innermost last, by the column their content starts. */
export interface ListState {
columns: number[];
// True while the innermost item has had no content since its marker.
emptyItem: boolean;
}
export const EMPTY_LIST_STATE: ListState = { columns: [], emptyItem: false };
// The marker needs whitespace after it, so `2.0` is a version, not an item.
const LIST_ITEM = /^[ \t]*([-*+]|\d{1,9}[.)])([ \t]+|$)/;
const THEMATIC_BREAK =
/^ {0,3}(?:(?:\*[ \t]*){3,}|(?:-[ \t]*){3,}|(?:_[ \t]*){3,})$/;
const BLOCK_QUOTE = /^ {0,3}>/;
const QUOTE_MARKER = /^ {0,3}>[ \t]?/;
// Blocks that are not paragraph text, so they cannot continue one lazily.
const PARAGRAPH_TEXT = /^ {0,3}(?![-*+>]([ \t]|$)|\d{1,9}[.)]([ \t]|$))\S/;
// Blocks that break into an open paragraph, closing it rather than continuing
// it. A link reference definition is not one of them.
const INTERRUPTS =
/^ {0,3}(?:#{1,6}([ \t]|$)|(?:\*[ \t]*){3,}$|(?:-[ \t]*){3,}$|(?:_[ \t]*){3,}$)/;
const FENCE = /^ {0,3}(?:`{3,}|~{3,})/;
const HTML_BLOCK_OPEN = /^ {0,3}<\/?([a-zA-Z][a-zA-Z0-9-]*)(?=[\s/>]|$)/;
const HTML_BLOCK_TAGS = new Set(
`address article aside base basefont blockquote body caption center col colgroup
dd details dialog dir div dl dt fieldset figcaption figure footer form frame
frameset h1 h2 h3 h4 h5 h6 head header hr html iframe legend li link main menu
menuitem nav noframes ol optgroup option p param search section summary table
tbody td tfoot th thead title tr track ul`.split(/\s+/),
);
// Content indented more than this after a marker is an indented code block, so
// the item's content starts one column past the marker instead.
const MAX_ITEM_PADDING = 4;
// Columns past its container at which a line becomes an indented code block.
const INDENTED_CODE = 4;
// Stands in for a line the renderer hides. `#` is a block of its own, so list
// tracking reads it like a comment: never a marker, never a lazy continuation.
const HIDDEN_BLOCK = "#";
const LEADING_SPACE = /^[ \t]*/;
/**
* `line` as list tracking sees it once the renderer hides its text. A comment or
* raw HTML block renders nothing but is still a block at its own column, so it
* closes the items it sits left of. Only the indentation survives: what the block
* hides is not Markdown and must not open a list. `marker` is the part opening
* the item the block is content of, which survives too. Ported from
* `_hidden_structure` on the backend.
*/
export function hiddenStructure(line: string, marker = ""): string {
if (marker) {
return `${marker}${HIDDEN_BLOCK}`;
}
const indent = LEADING_SPACE.exec(line)?.[0] ?? "";
return line.trim() ? `${indent}${HIDDEN_BLOCK}` : "";
}
/** Columns of leading whitespace, counting a tab to the next stop of four. */
export function indentWidth(line: string): number {
let width = 0;
for (const char of line) {
if (char === " ") {
width += 1;
} else if (char === "\t") {
width += 4 - (width % 4);
} else {
break;
}
}
return width;
}
/**
* Whether `line` starts a block that can break into an open paragraph. A quote
* marker always can; a list item only with content, an ordered one only at 1.
* Anything else is text of the paragraph it appears to interrupt.
*/
export function interruptsParagraph(line: string): boolean {
if (BLOCK_QUOTE.test(line)) {
return true;
}
const item = THEMATIC_BREAK.test(line) ? null : LIST_ITEM.exec(line);
if (item === null) {
return false;
}
const marker = item[1] ?? "";
if (!line.slice(item[0].length).trim()) {
return false;
}
const ordered = marker.endsWith(".") || marker.endsWith(")");
return !ordered || marker.slice(0, -1) === "1";
}
/**
* Whether a marker-shaped `line` is really text of the paragraph above. Only a
* marker inside the paragraph's own item interrupts it; one to the left closes
* that item and opens a sibling. A quote owns the paragraph its lines hold, so a
* marker outside the quote opens a list of its own.
*/
export function lazyMarker(
line: string,
state: ListState,
afterParagraph: boolean,
quoted: boolean,
): boolean {
const item = THEMATIC_BREAK.test(line) ? null : LIST_ITEM.exec(line);
const columns = state.columns;
const inside =
columns.length === 0 || indentWidth(line) >= (columns.at(-1) ?? 0);
return (
item !== null &&
afterParagraph &&
!quoted &&
inside &&
!interruptsParagraph(line)
);
}
/** `columns` with every item whose content starts past `indent` closed. */
function dropDeeper(columns: number[], indent: number): number[] {
let open = columns.length;
while (open > 0 && (columns[open - 1] ?? 0) > indent) {
open -= 1;
}
return open === columns.length ? columns : columns.slice(0, open);
}
/** `line` with up to `columns` columns of leading whitespace removed. */
function stripIndent(line: string, columns: number): string {
let width = 0;
let index = 0;
while (index < line.length && width < columns) {
const char = line[index];
if (char !== " " && char !== "\t") {
break;
}
width += char === " " ? 1 : 4 - (width % 4);
index += 1;
}
return line.slice(index);
}
/**
* Whether `line` can continue a paragraph it is indented out of. Only plain text
* can: a heading, fence, break or HTML block starts a block of its own, closing
* the item instead. An underline is not one: it may never be lazy, so `===` left
* of an open item is more of the item's paragraph. Nor is a definition, a block
* of its own that may not interrupt a paragraph. A row of dashes still closes the
* item: `INTERRUPTS` reads three or more as the thematic break they are.
*/
function mayBeLazy(line: string): boolean {
const named = HTML_BLOCK_OPEN.exec(line);
// Types 1 to 6 interrupt a paragraph, so a `<div>` left of an open item closes
// it. Type 7 cannot, and is deliberately excluded.
const htmlBlock =
named !== null && HTML_BLOCK_TAGS.has((named[1] ?? "").toLowerCase());
return (
PARAGRAPH_TEXT.test(line) &&
!INTERRUPTS.test(line) &&
!FENCE.test(line) &&
!htmlBlock
);
}
/**
* Whether `line` reads as more of a paragraph open in its container, measured
* from `column` where that container's content starts: four columns past it the
* line is indented code, which may not interrupt a paragraph, so indentation
* alone never closes the one above.
*/
export function continuesParagraph(line: string, column: number): boolean {
const inner = stripIndent(line, column);
return indentWidth(inner) >= INDENTED_CODE || mayBeLazy(inner);
}
/** `line` with up to `depth` blockquote markers removed, and how many went. */
function stripQuotes(line: string, depth: number): [string, number] {
let rest = line;
let removed = 0;
let marker = removed < depth ? QUOTE_MARKER.exec(rest) : null;
while (marker !== null) {
rest = rest.slice(marker[0].length);
removed += 1;
marker = removed < depth ? QUOTE_MARKER.exec(rest) : null;
}
return [rest, removed];
}
/** What a blockquote line holds, with its markers stripped. */
function quoteContent(line: string): string {
return stripQuotes(line, Number.POSITIVE_INFINITY)[0];
}
/** How many blockquotes `line` is written inside. */
export function quoteDepth(line: string): number {
return stripQuotes(line, Number.POSITIVE_INFINITY)[1];
}
/**
* `line` as the container it is written in sees it, with `quotes` blockquote
* markers and the open item's content column removed. CommonMark measures a block
* from its container, not the margin (spec 0.31.2 sections 5.1, 5.2), so `> ~~~`
* and a fence under a nested bullet are openers despite sitting more than three
* columns in.
*/
export function containerContent(
line: string,
state: ListState,
quotes: number,
): string {
const [inner] = stripQuotes(line, quotes);
if (quotes > 0) {
// A list inside a quote is the quote's own; this tracker follows document
// level only, so its columns do not apply here.
return inner;
}
const columns = dropDeeper(state.columns, indentWidth(inner));
return stripIndent(inner, columns.at(-1) ?? 0);
}
/**
* `line` read from the content column of a list item that opens on it. A block
* written as an item's first content sits inside that item, so ``- ``` `` opens a
* fence even though its marker is not within three columns of the container (spec
* 0.31.2 section 5.2). Padding is capped the way `openLists` caps it, or
* ``- ``` `` would read as a fence rather than the indented code it is. A
* marker the paragraph above swallows opens no item, so its line is returned
* whole, as is one four columns past its container.
*/
export function itemContent(line: string, afterParagraph: boolean): string {
if (
indentWidth(line) >= INDENTED_CODE ||
(afterParagraph && !interruptsParagraph(line))
) {
return line;
}
const item = THEMATIC_BREAK.test(line) ? null : LIST_ITEM.exec(line);
if (item === null) {
return line;
}
const padding = indentWidth(item[2] ?? "");
// Over-indented content starts one column past the marker; the rest of the
// padding is the content's own indentation.
const over = padding > MAX_ITEM_PADDING ? padding - 1 : 0;
return `${" ".repeat(over)}${line.slice(item[0].length)}`;
}
/** Whether a blockquote owns the paragraph the line below could continue. */
export interface QuoteState {
// True while a quoted paragraph is open, so plain text below is more of it.
inQuote: boolean;
// True whenever that paragraph is the quote's rather than the document's.
quoted: boolean;
}
export const NO_QUOTE: QuoteState = { inQuote: false, quoted: false };
/**
* The quote state after `line`, given the state after the line above and the
* content column of the item `line` sits in. A quote owns the paragraph its own
* lines hold, so a marker written outside the quote opens a list of its own
* rather than reading as more of that paragraph. Ported from `in_quote` tracking
* in changelog.py.
*/
export function quoteState(
line: string,
inQuote: boolean,
column = 0,
): QuoteState {
if (BLOCK_QUOTE.test(line)) {
// An empty quote holds no paragraph, so the line below starts a new one.
return { inQuote: mayBeLazy(quoteContent(line)), quoted: true };
}
const open = inQuote && continuesParagraph(line, column);
return { inQuote: open, quoted: open };
}
/**
* `columns` with every item `line` is written to the left of closed. Read inside
* the container the item sits in, not from the margin: a line that only looks
* dedented there is lazy text of the item's paragraph, leaving the item open.
*/
function closeDedented(
columns: number[],
line: string,
indent: number,
afterParagraph: boolean,
): number[] {
let open = columns.length;
while (open > 0 && (columns[open - 1] ?? 0) > indent) {
const outer = open > 1 ? (columns[open - 2] ?? 0) : 0;
if (afterParagraph && continuesParagraph(line, outer)) {
break;
}
open -= 1;
}
return open === columns.length ? columns : columns.slice(0, open);
}
/**
* The list items still open after `line`. A dedented line closes an item unless
* it is a lazy paragraph continuation. A new marker nests under a deeper column
* and replaces a sibling. `quoted` marks a paragraph the blockquote above owns:
* a marker outside the quote is not text of it, so it opens a list of its own.
*/
export function openLists(
line: string,
state: ListState,
afterParagraph: boolean,
quoted = false,
): ListState {
let columns = state.columns;
if (!line.trim()) {
// A blank line leaves the list open, unless the item is still empty: an
// item may begin with one blank line, and later content is outside it.
return {
columns: state.emptyItem ? columns.slice(0, -1) : columns,
emptyItem: false,
};
}
const indent = indentWidth(line);
const item = THEMATIC_BREAK.test(line) ? null : LIST_ITEM.exec(line);
const empty = item !== null && !line.slice(item[0].length).trim();
if (lazyMarker(line, state, afterParagraph, quoted)) {
// A lazy continuation or an underline, so the open items are untouched.
return state;
}
columns = closeDedented(columns, line, indent, afterParagraph);
// Four columns past its container the marker is an indented code block, or
// lazy text of the paragraph above it, so it opens no list of its own.
if (item === null || indent - (columns.at(-1) ?? 0) >= INDENTED_CODE) {
return { columns, emptyItem: false };
}
const marker = item[1] ?? "";
let padding = indentWidth(item[2] ?? "");
if (padding === 0 || padding > MAX_ITEM_PADDING) {
// An empty or over-indented item still holds one column of content.
padding = 1;
}
// A sibling marker replaces the item it lines up with.
return {
columns: [...dropDeeper(columns, indent), indent + marker.length + padding],
emptyItem: empty,
};
}

File diff suppressed because it is too large Load diff

View file

@ -0,0 +1,72 @@
// SPDX-License-Identifier: AGPL-3.0-only
// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
import assert from "node:assert/strict";
import test from "node:test";
import { readFile } from "node:fs/promises";
// Every localStorage key written by a panel width store.
const PANEL_WIDTH_KEYS = ["sidebar_width", "chat_settings_width"];
// The store reads window at import time, so stub it before importing.
const stubWindow = {
innerWidth: 1440,
localStorage: {
getItem: () => null,
setItem: () => {},
},
addEventListener: () => {},
removeEventListener: () => {},
};
(globalThis as { window?: unknown }).window = stubWindow;
const {
clampSidebarWidth,
SIDEBAR_WIDTH_DEFAULT,
SIDEBAR_WIDTH_MAX,
SIDEBAR_WIDTH_MIN,
} = await import("../src/hooks/use-sidebar-width.ts");
test("clamps to the absolute range on a roomy window", () => {
stubWindow.innerWidth = 1440;
assert.equal(clampSidebarWidth(320), 320);
assert.equal(clampSidebarWidth(SIDEBAR_WIDTH_MAX + 200), SIDEBAR_WIDTH_MAX);
assert.equal(clampSidebarWidth(10), SIDEBAR_WIDTH_MIN);
assert.equal(clampSidebarWidth(Number.NaN), SIDEBAR_WIDTH_DEFAULT);
});
test("caps at 40% of a narrow window", () => {
stubWindow.innerWidth = 800;
assert.equal(clampSidebarWidth(SIDEBAR_WIDTH_MAX), 320);
assert.equal(clampSidebarWidth(300), 300);
});
test("the floor still wins when 40% falls below it", () => {
stubWindow.innerWidth = 500;
assert.equal(clampSidebarWidth(SIDEBAR_WIDTH_MAX), SIDEBAR_WIDTH_MIN);
});
test("re-evaluates the cap per call, so a resize can re-clamp", () => {
stubWindow.innerWidth = 1440;
assert.equal(clampSidebarWidth(SIDEBAR_WIDTH_MAX), SIDEBAR_WIDTH_MAX);
stubWindow.innerWidth = 900;
assert.equal(clampSidebarWidth(SIDEBAR_WIDTH_MAX), 360);
stubWindow.innerWidth = 1440;
assert.equal(clampSidebarWidth(SIDEBAR_WIDTH_MAX), SIDEBAR_WIDTH_MAX);
});
// The reset action promises to clear every stored preference, so a persisted
// panel width that is missing from the list survives the reload.
test("persisted panel widths are cleared by the preference reset", async () => {
const source = await readFile(
new URL("../src/features/settings/tabs/general-tab.tsx", import.meta.url),
"utf8",
);
const keys = source.slice(
source.indexOf("const PREFS_KEYS"),
source.indexOf("];", source.indexOf("const PREFS_KEYS")),
);
for (const key of PANEL_WIDTH_KEYS) {
assert.ok(keys.includes(`"${key}"`), `${key} missing from PREFS_KEYS`);
}
});

View file

@ -707,18 +707,55 @@ def existing_install_usable(install_dir: Path, host: HostInfo) -> bool:
return npm_major is not None and npm_major >= NPM_MIN_MAJOR
def _replace_with_retry(
src: Path,
dst: Path,
*,
attempts: int = 8,
) -> None:
"""os.replace, retried against transient Windows sharing violations.
A directory rename fails with WinError 5/32 while any process holds a handle inside
it, and Defender or the indexer routinely does right after extraction (seen in CI on
a fresh install, with no existing directory to conflict with). Handles clear in a
second or two, so a bounded backoff turns the failure into a pause; other errors
raise immediately rather than stalling on a real problem.
"""
delay = 0.25
for attempt in range(attempts):
try:
os.replace(src, dst)
return
except OSError as exc:
transient = os.name == "nt" and getattr(exc, "winerror", None) in (5, 32, 145)
if not transient or attempt == attempts - 1:
raise
log(
f"rename blocked ({exc.winerror}), retrying in {delay:.2f}s "
f"-- a scanner is likely still holding the extracted files"
)
time.sleep(delay)
delay = min(delay * 2, 4.0)
def _swap_into_place(extracted_root: Path, install_dir: Path) -> None:
"""Atomically replace install_dir with extracted_root (same filesystem)."""
install_dir.parent.mkdir(parents = True, exist_ok = True)
backup: Path | None = None
if install_dir.exists():
backup = install_dir.parent / f".{install_dir.name}.old-{os.getpid()}"
os.replace(install_dir, backup)
_replace_with_retry(install_dir, backup)
try:
os.replace(extracted_root, install_dir)
_replace_with_retry(extracted_root, install_dir)
except OSError:
# The forward rename retries ~16s, ample time for a scanner to grab the backup too.
# A plain os.replace would then raise over the original error and leave no
# install_dir at all, so the rollback gets the same backoff and never masks it.
if backup is not None and not install_dir.exists():
os.replace(backup, install_dir)
try:
_replace_with_retry(backup, install_dir)
except OSError as rollback_exc:
log(f"could not restore the previous Node install from {backup}: {rollback_exc}")
raise
if backup is not None:
shutil.rmtree(backup, ignore_errors = True)

View file

@ -869,12 +869,22 @@ function Ensure-BuildToolsForLlamaSourceBuild {
}
}
# Detect the VC++ 2015-2022 Redistributable that the prebuilt llama-server and
# PyTorch need (they link VCRUNTIME140_1.dll etc., which the Universal CRT lacks).
# Signal is System32\vcruntime140_1.dll (VS 2019+), registry as fallback.
# Machine arch: PROCESSOR_ARCHITECTURE describes this PROCESS, so an emulated x64 shell on
# ARM64 reports AMD64; PROCESSOR_ARCHITEW6432 is ARM64 in exactly that case.
function Get-HostMachineArch {
$osArch = ""
try { $osArch = [System.Runtime.InteropServices.RuntimeInformation]::OSArchitecture.ToString() } catch { }
foreach ($s in @([string]$env:PROCESSOR_ARCHITEW6432, [string]$env:PROCESSOR_ARCHITECTURE, $osArch)) {
if ($s.ToLowerInvariant() -eq "arm64") { return "arm64" }
}
return "other"
}
# Detect the VC++ 2015-2022 Redistributable prebuilt llama-server and PyTorch need (they
# link VCRUNTIME140_1.dll, absent from the Universal CRT). Registry first: Runtimes\x64 is
# the only x64-specific proof; System32\vcruntime140_1.dll is arch-blind and on ARM64 may
# be the ARM64-only package, unloadable under x64 emulation.
function Test-VCRedistInstalled {
$sys = $env:SystemRoot
if ($sys -and (Test-Path (Join-Path $sys 'System32\vcruntime140_1.dll'))) { return $true }
foreach ($k in @(
'HKLM:\SOFTWARE\Microsoft\VisualStudio\14.0\VC\Runtimes\x64',
'HKLM:\SOFTWARE\WOW6432Node\Microsoft\VisualStudio\14.0\VC\Runtimes\x64'
@ -884,10 +894,14 @@ function Test-VCRedistInstalled {
if ($r.Installed -eq 1 -and [int]$r.Major -ge 14 -and [int]$r.Minor -ge 20) { return $true }
} catch { }
}
if ((Get-HostMachineArch) -eq "arm64") { return $false }
$sys = $env:SystemRoot
if ($sys -and (Test-Path (Join-Path $sys 'System32\vcruntime140_1.dll'))) { return $true }
return $false
}
# Install the VC++ 2015-2022 runtime if missing (non-fatal; usually a no-op).
# Install the VC++ 2015-2022 runtime if missing (non-fatal; usually a no-op). Unlike CMake
# and Build Tools torch cannot import without it, and winget is absent on LTSC/Server images.
function Ensure-VCRedist {
if (Test-VCRedistInstalled) { step "vcredist" "present"; return }
Write-Host "Microsoft Visual C++ Redistributable (2015-2022) is missing; the prebuilt llama.cpp and PyTorch need it. Installing the runtime..." -ForegroundColor Yellow
@ -897,6 +911,45 @@ function Ensure-VCRedist {
Refresh-Environment
} catch { substep "VCRedist install failed: $($_.Exception.Message)" "Yellow" }
}
if (-not (Test-VCRedistInstalled)) {
# Evergreen link; /quiet /norestart so it never blocks or reboots an unattended run.
# Always the x64 package, deliberately: Microsoft ships it as the Arm64X superset of
# both ARM64 and X64 binaries and documents it as the one for ARM64 devices, while
# the arm64 package is ARM64-only (learn.microsoft.com/cpp/windows/latest-supported-vc-redist).
# PROCESSOR_ARCHITECTURE is wrong twice here: it reports the process, and the runtime
# must match the interpreter loading the DLLs, an emulated x64 Python not yet created.
$url = "https://aka.ms/vs/17/release/vc_redist.x64.exe"
$dst = Join-Path ([System.IO.Path]::GetTempPath()) "vc_redist.x64.exe"
substep "winget unavailable or failed; downloading the runtime directly..."
# Windows PowerShell 5.1 on an old image can carry a .NET default protocol set that
# predates TLS 1.2, which aka.ms refuses -- exactly the no-winget host this fallback
# exists for. SystemDefault (0) means "let the OS choose" and already covers TLS 1.2+,
# so only an explicit legacy set is upgraded, and it is restored afterwards.
$_prevProtocol = $null
try {
$_cur = [System.Net.ServicePointManager]::SecurityProtocol
if ([int]$_cur -ne 0 -and ([int]$_cur -band [int][System.Net.SecurityProtocolType]::Tls12) -eq 0) {
[System.Net.ServicePointManager]::SecurityProtocol = $_cur -bor [System.Net.SecurityProtocolType]::Tls12
$_prevProtocol = $_cur
}
} catch { $_prevProtocol = $null }
try {
Invoke-WebRequest -Uri $url -OutFile $dst -UseBasicParsing -TimeoutSec 300
$p = Start-Process -FilePath $dst -ArgumentList '/quiet', '/norestart' -Wait -PassThru
# 3010 = success, reboot required; usable either way.
if ($p.ExitCode -notin @(0, 3010)) {
substep "VC++ runtime installer exited $($p.ExitCode)" "Yellow"
}
Refresh-Environment
} catch {
substep "Direct VC++ runtime download failed: $($_.Exception.Message)" "Yellow"
} finally {
if ($null -ne $_prevProtocol) {
try { [System.Net.ServicePointManager]::SecurityProtocol = $_prevProtocol } catch { }
}
Remove-Item -LiteralPath $dst -Force -ErrorAction SilentlyContinue
}
}
if (Test-VCRedistInstalled) { step "vcredist" "installed" }
else {
substep "Could not install the VC++ Redistributable automatically." "Yellow"
@ -1650,11 +1703,42 @@ if ($LongPathsEnabled) {
}
# ============================================
# 1b. Git (required by pip for git+https:// deps and by npm)
# 1b. Git (only required for --local / source installs)
# ============================================
# Was fatal as "required by pip and npm", but the consumer path uses neither: the
# unsloth-zoo git+https URL is STUDIO_LOCAL_INSTALL only, node is a pinned prebuilt, and the
# frontend lockfile has no VCS deps. Being fatal blocked clean no-winget Windows boxes.
$HasGit = $null -ne (Get-Command git -ErrorAction SilentlyContinue)
if (-not $HasGit) {
Write-Host "Git not found -- installing via winget..." -ForegroundColor Yellow
# Fatal only where git is used: --local and the opt-in llama.cpp source build. A local
# llama.cpp dir overrides those opt-ins, but only once it holds a reusable binary:
# pointing at the canonical install location with nothing built there falls through to
# the normal install, so an explicit source build still needs git. The automatic
# fallback after a failed prebuilt download is not knowable here; Phase 4 handles it.
$gitNeeded = ($env:STUDIO_LOCAL_INSTALL -eq '1')
$_localLlamaDir = if ($env:UNSLOTH_LOCAL_LLAMA_CPP_DIR) { $env:UNSLOTH_LOCAL_LLAMA_CPP_DIR.Trim() } else { "" }
$_localLlamaBuilt = $false
if ($_localLlamaDir) {
# Same layout candidates as the reuse check in Phase 4.
foreach ($_c in @("llama-server.exe", "build\bin\llama-server.exe", "build\bin\Release\llama-server.exe")) {
if (Test-Path -LiteralPath (Join-Path $_localLlamaDir $_c)) { $_localLlamaBuilt = $true; break }
}
}
if (-not $_localLlamaBuilt) {
$_prForce = if ($env:UNSLOTH_LLAMA_PR_FORCE) { $env:UNSLOTH_LLAMA_PR_FORCE.Trim() } else { $DefaultLlamaPrForce }
$_llamaSrc = $DefaultLlamaSource -replace '\.git$', ''
# Same tag resolution as Phase 4. "master" is a branch, never a release, so the
# prebuilt lookup always misses and Phase 4 rebuilds it from source.
$_llamaTag = if ($env:UNSLOTH_LLAMA_TAG) { $env:UNSLOTH_LLAMA_TAG } else { $DefaultLlamaTag }
if ($_llamaTag -eq "master") { $gitNeeded = $true }
if ($env:UNSLOTH_LLAMA_FORCE_COMPILE -eq '1') { $gitNeeded = $true }
if (-not [string]::IsNullOrWhiteSpace($env:UNSLOTH_LLAMA_PR)) { $gitNeeded = $true }
# Same positive-integer predicate as the PR_FORCE promotion below: 0 or non-numeric
# never forces a source build, so it must not demand git.
if ($_prForce -match '^\d+$' -and [int]$_prForce -gt 0) { $gitNeeded = $true }
if ($_llamaSrc -ne "https://github.com/ggml-org/llama.cpp") { $gitNeeded = $true }
}
Write-Host "Git not found -- attempting install via winget..." -ForegroundColor Yellow
$HasWinget = $null -ne (Get-Command winget -ErrorAction SilentlyContinue)
if ($HasWinget) {
try {
@ -1664,11 +1748,18 @@ if (-not $HasGit) {
} catch { }
}
if (-not $HasGit) {
Write-Host "[ERROR] Git is required but could not be installed automatically." -ForegroundColor Red
Write-Host " Install Git from https://git-scm.com/download/win and re-run." -ForegroundColor Red
Exit-SetupFailure "Git is required but could not be installed automatically"
if ($gitNeeded) {
Write-Host "[ERROR] Git is required for --local and llama.cpp source-build installs but could not be installed." -ForegroundColor Red
Write-Host " --local clones unsloth-zoo, and a source build clones llama.cpp." -ForegroundColor Red
Write-Host " Install Git from https://git-scm.com/download/win and re-run." -ForegroundColor Red
Exit-SetupFailure "Git is required for --local / source-build installs but could not be installed"
}
step "git" "not found (not required)" "Yellow"
substep "Unsloth installs prebuilt binaries and wheels, so git is not needed."
substep "Install it only for --local/source installs: https://git-scm.com/download/win"
} else {
step "git" "$(git --version)"
}
step "git" "$(git --version)"
} else {
step "git" "$(git --version)"
}
@ -3275,18 +3366,32 @@ $PyTorchWhlBase = if ($env:UNSLOTH_PYTORCH_MIRROR) { $env:UNSLOTH_PYTORCH_MIRROR
$TorchInstallIndexUrl = if ($ROCmIndexUrl) { "$PyTorchWhlBase/cpu" } elseif ($PinnedTorchIndexUrl) { $PinnedTorchIndexUrl } else { "$PyTorchWhlBase/$CuTag" }
if (-not $NoTorchMode) {
# Windows on ARM has win_arm64 torch and torchvision wheels but no torchaudio on any index,
# so every branch below drops it. Ask the interpreter uv resolves for, not
# PROCESSOR_ARCHITECTURE, which describes the host process. Inside the no-torch guard
# because all three uses are, and no-torch installs nothing to skip.
$_setupPlatform = ""
try {
$_setupPlatform = (& python -c "import sysconfig; print(sysconfig.get_platform())" 2>$null | Out-String).Trim().ToLowerInvariant()
} catch { $_setupPlatform = "" }
$WinArm64NoAudio = ($_setupPlatform -eq "win-arm64")
if ($WinArm64NoAudio) { substep "windows on arm: skipping torchaudio (no win_arm64 wheel upstream)" }
$ROCmCpuFallback = $false
if ($ROCmIndexUrl) {
substep "installing PyTorch (AMD ROCm, $ROCmGfxArch)..."
if ($ROCmTorchSpec -ne "torch") {
substep " enforcing $ROCmTorchSpec $ROCmVisionSpec $ROCmAudioSpec (known _grouped_mm bug in older wheels)" "Cyan"
}
# Built above the verbose branch: a splat assigned inside it is unset on the other.
$_rocmTrio = @($ROCmTorchSpec, $ROCmVisionSpec, $ROCmAudioSpec)
if ($WinArm64NoAudio) { $_rocmTrio = @($ROCmTorchSpec, $ROCmVisionSpec) }
if ($script:UnslothVerbose) {
Fast-Install $ROCmTorchSpec $ROCmVisionSpec $ROCmAudioSpec --force-reinstall --index-url $ROCmIndexUrl | ForEach-Object { Redact-InstallOutput "$_" } | Out-Host
Fast-Install @_rocmTrio --force-reinstall --index-url $ROCmIndexUrl | ForEach-Object { Redact-InstallOutput "$_" } | Out-Host
$torchInstallExit = $LASTEXITCODE
$output = ""
} else {
$output = Fast-Install $ROCmTorchSpec $ROCmVisionSpec $ROCmAudioSpec --force-reinstall --index-url $ROCmIndexUrl | Out-String
$output = Fast-Install @_rocmTrio --force-reinstall --index-url $ROCmIndexUrl | Out-String
$torchInstallExit = $LASTEXITCODE
}
if ($torchInstallExit -ne 0) {
@ -3322,12 +3427,14 @@ if (-not $ROCmIndexUrl -and ($CuTag -eq "cpu" -or $ROCmCpuFallback)) {
$cpuVisionSpec = "torchvision>=0.19,<0.27.0"
$cpuAudioSpec = "torchaudio>=2.4,<2.12.0"
}
$_torchTrio = @($cpuTorchSpec, $cpuVisionSpec, $cpuAudioSpec)
if ($WinArm64NoAudio) { $_torchTrio = @($cpuTorchSpec, $cpuVisionSpec) }
if ($script:UnslothVerbose) {
Fast-Install $cpuTorchSpec $cpuVisionSpec $cpuAudioSpec @cpuForce --index-url $TorchInstallIndexUrl | ForEach-Object { Redact-InstallOutput "$_" } | Out-Host
Fast-Install @_torchTrio @cpuForce --index-url $TorchInstallIndexUrl | ForEach-Object { Redact-InstallOutput "$_" } | Out-Host
$torchInstallExit = $LASTEXITCODE
$output = ""
} else {
$output = Fast-Install $cpuTorchSpec $cpuVisionSpec $cpuAudioSpec @cpuForce --index-url $TorchInstallIndexUrl | Out-String
$output = Fast-Install @_torchTrio @cpuForce --index-url $TorchInstallIndexUrl | Out-String
$torchInstallExit = $LASTEXITCODE
}
if ($torchInstallExit -ne 0) {
@ -3354,12 +3461,16 @@ if (-not $ROCmIndexUrl -and ($CuTag -eq "cpu" -or $ROCmCpuFallback)) {
$cudaVisionSpec = "torchvision>=0.19,<0.26.0"
$cudaAudioSpec = "torchaudio>=2.4,<2.11.0"
}
# A custom pin whose leaf is not cpu (a corporate /simple mirror) lands an ARM64 host
# here, so this branch drops torchaudio too.
$_cudaTrio = @($cudaTorchSpec, $cudaVisionSpec, $cudaAudioSpec)
if ($WinArm64NoAudio) { $_cudaTrio = @($cudaTorchSpec, $cudaVisionSpec) }
if ($script:UnslothVerbose) {
Fast-Install $cudaTorchSpec $cudaVisionSpec $cudaAudioSpec @cudaForce --index-url $TorchInstallIndexUrl | ForEach-Object { Redact-InstallOutput "$_" } | Out-Host
Fast-Install @_cudaTrio @cudaForce --index-url $TorchInstallIndexUrl | ForEach-Object { Redact-InstallOutput "$_" } | Out-Host
$torchInstallExit = $LASTEXITCODE
$output = ""
} else {
$output = Fast-Install $cudaTorchSpec $cudaVisionSpec $cudaAudioSpec @cudaForce --index-url $TorchInstallIndexUrl | Out-String
$output = Fast-Install @_cudaTrio @cudaForce --index-url $TorchInstallIndexUrl | Out-String
$torchInstallExit = $LASTEXITCODE
}
if ($torchInstallExit -ne 0) {
@ -4048,6 +4159,7 @@ $BuildDir = Join-Path $LlamaCppDir "build"
$LlamaServerBin = Join-Path $BuildDir "bin\Release\llama-server.exe"
$HasCmakeForBuild = $null -ne (Get-Command cmake -ErrorAction SilentlyContinue)
$HasGitForBuild = $null -ne (Get-Command git -ErrorAction SilentlyContinue)
# Check if existing llama-server matches current GPU mode. A CUDA-built binary
# on a now-CPU-only machine (or vice versa) needs to be rebuilt.
@ -4073,9 +4185,27 @@ if (Test-Path -LiteralPath $LlamaServerBin) {
$WillBuildLlamaFromSource = $NeedLlamaSourceBuild -and `
-not ((Test-Path -LiteralPath $LlamaServerBin) -and -not $NeedRebuild -and $RequestedLlamaTag -ne "master")
if ($WillBuildLlamaFromSource) {
Ensure-BuildToolsForLlamaSourceBuild
# refresh so the chain below sees a newly installed cmake
$HasCmakeForBuild = $null -ne (Get-Command cmake -ErrorAction SilentlyContinue)
if (-not $HasGitForBuild) {
# Phase 1 keeps git optional, so only the automatic fallback after a failed prebuilt
# download arrives here without it. Last chance to install: Invoke-SetupCommand
# returns 0 for command-not-found, so a git-less clone misreports as a cmake failure.
if ($null -ne (Get-Command winget -ErrorAction SilentlyContinue)) {
try {
Invoke-SetupCommand { winget install Git.Git --source winget --accept-package-agreements --accept-source-agreements } | Out-Null
Refresh-Environment
} catch { }
}
$HasGitForBuild = $null -ne (Get-Command git -ErrorAction SilentlyContinue)
}
# Git first, then the toolchain: Ensure-BuildToolsForLlamaSourceBuild exits setup when
# Build Tools cannot be installed, so running it first made the degraded path below
# unreachable on a no-winget box, and elsewhere spent a multi-GB download on a clone
# that cannot happen.
if ($HasGitForBuild) {
Ensure-BuildToolsForLlamaSourceBuild
# refresh so the chain below sees a newly installed cmake
$HasCmakeForBuild = $null -ne (Get-Command cmake -ErrorAction SilentlyContinue)
}
}
if ($LocalLlamaCppLinked) {
@ -4093,6 +4223,16 @@ if ($LocalLlamaCppLinked) {
# up new model architecture support (e.g. Gemma 4).
Write-Host ""
step "llama.cpp" "already built"
} elseif (-not $HasGitForBuild) {
# Before cmake: the toolchain install is skipped without git, so cmake may be missing
# purely as a consequence. Degrade rather than abort; the opt-in source triggers already
# required git in Phase 1, so only the automatic fallback lands here.
Write-Host ""
step "llama.cpp" "build skipped (git not available)" "Yellow"
substep "The prebuilt download failed and a source build clones llama.cpp." "Yellow"
substep "GGUF inference and export will not be available." "Yellow"
substep "Install Git from https://git-scm.com/download/win and re-run setup." "Yellow"
$script:LlamaCppDegraded = $true
} elseif (-not $HasCmakeForBuild) {
Write-Host ""
if (-not $HasNvidiaSmi) {

View file

@ -27,6 +27,8 @@ pub(crate) struct DesktopUpdatePolicy {
pub(crate) struct ManualUpdateInfo {
version: String,
current_version: String,
// Backend release this desktop build pins; CHANGELOG.md is keyed by it.
pypi_version: Option<String>,
body: Option<String>,
date: Option<String>,
}
@ -34,8 +36,12 @@ pub(crate) struct ManualUpdateInfo {
#[derive(Debug, serde::Deserialize)]
struct ChannelMetadata {
version: String,
body: Option<String>,
date: Option<String>,
// latest.json publishes Tauri's `notes`/`pub_date`; aliases keep older metadata working.
pypi_version: Option<String>,
#[serde(alias = "body")]
notes: Option<String>,
#[serde(alias = "date")]
pub_date: Option<String>,
platforms: HashMap<String, ChannelPlatform>,
}
@ -99,8 +105,9 @@ pub(crate) async fn check_desktop_manual_update() -> Result<Option<ManualUpdateI
Ok(Some(ManualUpdateInfo {
version: latest_version,
current_version: current_version.to_string(),
body: metadata.body,
date: metadata.date,
pypi_version: metadata.pypi_version,
body: metadata.notes,
date: metadata.pub_date,
}))
}

View file

@ -155,7 +155,10 @@ def test_the_ctypes_binds_are_gated_on_the_same_verdict():
"if bnb is None or not native_kernels_ready(bnb, DEVICE_TYPE):" in source
), "the ctypes bind block must take the _bnb_required branch on a dead library too"
guarded = source.split("if bnb is None or not native_kernels_ready(bnb, DEVICE_TYPE):")[1]
assert "bnb.functional.lib" in guarded, "the binds must sit under that guard"
# Anchor on the symbol, not the module alias: #7580 renamed the binding from
# `bnb.functional.lib` to `bnb_functional.lib`, which is exactly the kind of rename
# this assertion should survive.
assert "lib.cdequantize_blockwise_fp32" in guarded, "the binds must sit under that guard"
def test_the_kernel_check_reads_the_submodule_not_the_parent_attribute():

View file

@ -454,9 +454,12 @@ class TestKnown211SetParity:
"$_pinCuLeaf" not in text
), "install.ps1 must bound companions on every index (no cu-family exemption)"
# The bounded companions must actually be passed to the install command.
assert re.search(
r'"torch>=2\.4,<2\.11\.0" \$_pinVisionSpec \$_pinAudioSpec --default-index \$TorchIndexUrl',
text,
# Specs are splatted, so check both halves: the list is built, and it is passed.
assert (
'$_torchSpecs = @("torch>=2.4,<2.11.0", $_pinVisionSpec, $_pinAudioSpec)' in text
), "install.ps1 custom-pin install must build the bounded spec list"
assert (
"@_torchSpecs --default-index $TorchIndexUrl" in text
), "install.ps1 custom-pin install must pass the bounded companion specs to uv"
def test_gfx_allowlist_matches_across_installers(self):
@ -704,9 +707,13 @@ class TestPinnedIndexClearsUvEnvParity:
assert (
"if ($TorchIndexPinned -and -not (Test-CudaFamilyLeaf $CuTag)) {" in text
), "the custom-leaf trio bounds must be gated on a pinned non-cu-family leaf"
# Specs are splatted, so check both halves: the list is built, and it is passed.
assert (
"Fast-Install $cudaTorchSpec $cudaVisionSpec $cudaAudioSpec" in text
), "setup.ps1's CUDA branch must install via the bounded spec variables"
"$_cudaTrio = @($cudaTorchSpec, $cudaVisionSpec, $cudaAudioSpec)" in text
), "setup.ps1's CUDA branch must build the trio from the bounded spec variables"
assert (
"Fast-Install @_cudaTrio @cudaForce" in text
), "setup.ps1's CUDA branch must install the trio it built"
def test_setup_ps1_bounds_pinned_cpu_torch(self):
"""setup.ps1's CPU branch must bound the trio under an explicit pin (parity with
@ -724,8 +731,11 @@ class TestPinnedIndexClearsUvEnvParity:
"if ($TorchIndexPinned) {" in text
), "the CPU trio bounds must be gated on an explicit pin"
assert (
"Fast-Install $cpuTorchSpec $cpuVisionSpec $cpuAudioSpec @cpuForce" in text
), "setup.ps1's CPU branch must install via the spec variables"
"$_torchTrio = @($cpuTorchSpec, $cpuVisionSpec, $cpuAudioSpec)" in text
), "setup.ps1's CPU branch must build the trio from the spec variables"
assert (
"Fast-Install @_torchTrio @cpuForce" in text
), "setup.ps1's CPU branch must install the trio it built"
# The ceilings mirror the Python repair spec exactly.
stack = STACK_PY.read_text(encoding = "utf-8")
spec_block = re.search(r"_CUDA_TORCH_PKG_SPEC[^(]*\(\s*(.*?)\)", stack, re.DOTALL)

View file

@ -0,0 +1,143 @@
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
"""Windows on ARM: install.ps1 must not settle for a native ARM64 interpreter.
pyarrow (via datasets) and hf-transfer publish no win_arm64 wheels, so an ARM64
Python source-builds both and dies minutes into the run. The resolver prefers an
x64 build of the requested minor and bootstraps one otherwise; the case pinned
here is the recovery path, where nothing can be downloaded but an x64 build of a
lower-priority supported minor is already installed.
"""
from __future__ import annotations
import os
import re
import shutil
import subprocess
from pathlib import Path
import pytest
REPO_ROOT = Path(__file__).resolve().parents[2]
INSTALL_PS1 = REPO_ROOT / "install.ps1"
def _extract(pattern: str, source: str) -> str:
match = re.search(pattern, source, flags = re.DOTALL)
assert match is not None, f"install.ps1 block not found: {pattern}"
return match.group(0)
def _resolver_script(installed: list[tuple[str, str]], can_download: bool) -> str:
"""Both production functions verbatim, over a fake set of interpreters.
Extracted rather than reimplemented so the test cannot drift away from the
text install.ps1 actually runs. `installed` is (minor, arch) in py-launcher
order, so the first entry for a minor is what a bare `py -3.13` resolves to.
The fake interpreters are named `*.exe` and invoked through the call operator,
which resolves a string to a function, so no real binary is needed.
"""
source = INSTALL_PS1.read_text(encoding = "utf-8")
finder = _extract(r" function Find-CompatiblePython \{.*?\n \}\n", source)
installer = _extract(r" function Install-X64Python \{.*?\n \}\n", source)
names = [f"Py{minor.replace('.', '')}{arch}.exe" for minor, arch in installed]
table = ", ".join(
f'@{{ Minor = "{minor}"; Arch = "{arch}"; Name = "{name}" }}'
for (minor, arch), name in zip(installed, names)
)
downloaded = (
'@{ Version = "3.13"; Path = "Downloaded.exe"; Arch = "x86_64" }'
if can_download
else "$null"
)
version_stubs = "\n".join(
f"function {name} {{ param([Parameter(ValueFromRemainingArguments = $true)]$Rest)\n"
f' if ($Rest -contains "--version") {{ return "Python {minor}.0" }}\n'
f' return "{name}" }}'
for (minor, _arch), name in zip(installed, names)
)
return f"""
$ErrorActionPreference = "Stop"
$PythonVersion = "3.13"
$script:WingetAvailable = $false
$script:CondaSkipPattern = 'conda'
$Interpreters = @({table})
{version_stubs}
# `py -0p` lists every registration; `py -3.x` runs the launcher's preferred build
# for that minor, which on an ARM64 host is normally the native one.
function FakePy {{
param([Parameter(ValueFromRemainingArguments = $true)]$Rest)
if ($Rest -contains "-0p") {{
return @($Interpreters | ForEach-Object {{ " -V:$($_.Minor) * $($_.Name)" }})
}}
$minor = ([string]$Rest[0]).TrimStart('-')
$hit = @($Interpreters | Where-Object {{ $_.Minor -eq $minor }})
if ($hit.Count -eq 0) {{ return "" }}
if ($Rest -contains "--version") {{ return "Python $minor.0" }}
return $hit[0].Name
}}
function substep {{ param($a, $b) }}
function Get-HostMachineArch {{ return "arm64" }}
function Get-Command {{
param([Parameter(Position = 0)][string]$Name,
[Parameter(ValueFromRemainingArguments = $true)]$Rest)
if ($Name -eq "py") {{ return @([pscustomobject]@{{ Source = "FakePy" }}) }}
return @()
}}
function Test-Path {{ param([Parameter(ValueFromRemainingArguments = $true)]$Rest) return $true }}
function Test-IsCondaPython {{ param([string]$Exe) return $false }}
function Get-PythonPlatformTag {{
param([string]$Exe)
foreach ($i in $Interpreters) {{
if ($i.Name -eq $Exe) {{
if ($i.Arch -eq "x86_64") {{ return "win-amd64" }} else {{ return "win-arm64" }}
}}
}}
return "win-amd64"
}}
function Refresh-SessionPath {{ }}
function Install-PythonFromPythonOrg {{ param([string]$Arch = "") return {downloaded} }}
{finder}
{installer}
# The caller's ARM64 swap, condensed to what decides the interpreter.
$found = Find-CompatiblePython
if ($found -and $found.Arch -ne "x86_64") {{
$x64 = Install-X64Python
if ($x64) {{ $found = $x64 }}
}}
if ($found) {{ Write-Output "$($found.Version)|$($found.Arch)" }} else {{ Write-Output "none" }}
"""
def _pwsh(script: str) -> str:
result = subprocess.run(
["pwsh", "-NoProfile", "-NonInteractive", "-Command", script],
check = True,
capture_output = True,
text = True,
env = os.environ.copy(),
)
return result.stdout.strip()
@pytest.mark.skipif(shutil.which("pwsh") is None, reason = "PowerShell is unavailable")
@pytest.mark.parametrize(
("installed", "can_download", "expected"),
[
# An x64 build of the requested minor wins outright, downloads irrelevant.
([("3.13", "arm64"), ("3.13", "x86_64")], False, "3.13|x86_64"),
# Requested minor is ARM64-only: bootstrap x64 rather than take the native one.
([("3.13", "arm64")], True, "3.13|x86_64"),
# Offline, but an x64 build of a lower-priority minor is here. Use it: the native
# 3.13 cannot resolve pyarrow or hf-transfer, and this one can.
([("3.13", "arm64"), ("3.11", "x86_64")], False, "3.11|x86_64"),
# ARM64 everywhere: still returned, and the caller warns.
([("3.13", "arm64"), ("3.11", "arm64")], False, "3.13|arm64"),
],
)
def test_arm64_host_prefers_an_x64_interpreter(installed, can_download, expected):
assert _pwsh(_resolver_script(installed, can_download)) == expected

Some files were not shown because too many files have changed in this diff Show more