Merge remote-tracking branch 'origin/main' into r7551
This commit is contained in:
commit
94056e0c09
105 changed files with 10016 additions and 224 deletions
3
.github/workflows/release-desktop.yml
vendored
3
.github/workflows/release-desktop.yml
vendored
|
|
@ -766,6 +766,7 @@ jobs:
|
|||
env:
|
||||
GH_REPO: ${{ github.repository }}
|
||||
APP_VERSION: ${{ needs.prepare-version.outputs.app_version }}
|
||||
PYPI_VERSION: ${{ needs.prepare-version.outputs.pypi_version }}
|
||||
STUDIO_VERSION: ${{ needs.prepare-version.outputs.studio_version }}
|
||||
DESKTOP_RELEASE_TAG: ${{ needs.prepare-version.outputs.desktop_release_tag }}
|
||||
DESKTOP_PRERELEASE: ${{ needs.prepare-version.outputs.prerelease }}
|
||||
|
|
@ -911,6 +912,8 @@ jobs:
|
|||
notes = pathlib.Path(os.environ['RUNNER_TEMP'], 'desktop-release-notes.md').read_text()
|
||||
metadata = {
|
||||
'version': os.environ['APP_VERSION'],
|
||||
# App version is SemVer; CHANGELOG.md is keyed by the backend release.
|
||||
'pypi_version': os.environ['PYPI_VERSION'],
|
||||
'notes': notes,
|
||||
'pub_date': datetime.datetime.now(datetime.timezone.utc).isoformat(timespec='milliseconds').replace('+00:00', 'Z'),
|
||||
'platforms': {
|
||||
|
|
|
|||
156
.github/workflows/startup-profile-ci.yml
vendored
Normal file
156
.github/workflows/startup-profile-ci.yml
vendored
Normal file
|
|
@ -0,0 +1,156 @@
|
|||
# SPDX-License-Identifier: AGPL-3.0-only
|
||||
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
|
||||
|
||||
# Measures where Studio's startup time goes, on each platform.
|
||||
#
|
||||
# Nothing recorded a number before: main.py logs "lifespan startup completed in X ms"
|
||||
# and studio_test_kit polls /healthz, but both throw the elapsed time away. A first
|
||||
# local run (Linux, warm cache, 18-core server) put `import main` at 5.7-6.6s BEFORE
|
||||
# the server can bind, dominated by eager module-level imports pulled in by routes:
|
||||
# torch ~1.9s self, unsloth_zoo ~0.8s, routes ~0.6s, transformers ~0.5s.
|
||||
#
|
||||
# Not a gate yet: --max-healthz-seconds exists, but a budget should come from
|
||||
# observed numbers rather than a guess.
|
||||
|
||||
name: Startup profile
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
paths:
|
||||
# The measured import graph is the whole backend tree: main.py imports auth,
|
||||
# core, hub, loggers, models, picker, routes and utils at module scope.
|
||||
- 'studio/backend/**'
|
||||
- '!studio/backend/tests/**'
|
||||
# The launch phase spawns `unsloth studio --api-only`, so the CLI counts too.
|
||||
- 'unsloth_cli/**'
|
||||
- 'studio/src-tauri/src/preflight**'
|
||||
# The profiler hardcodes the desktop argv that process.rs::backend_args builds,
|
||||
# so a change there must schedule a run or the two silently diverge.
|
||||
- 'studio/src-tauri/src/process.rs'
|
||||
- 'scripts/profile_startup.py'
|
||||
- '.github/workflows/startup-profile-ci.yml'
|
||||
# The job profiles whatever `install.sh --local` built: the installers pick the
|
||||
# venv's Python and the dependency specs, and pyproject's include list is what
|
||||
# makes --local overlay studio.backend*.
|
||||
- 'install.sh'
|
||||
- 'install.ps1'
|
||||
- 'pyproject.toml'
|
||||
# --local also runs the checkout's setup scripts (install.sh picks
|
||||
# $_REPO_ROOT/studio/setup.sh, the editable install resolves setup.ps1 to the
|
||||
# repo), and both call install_python_stack.py, which picks the dependencies.
|
||||
- 'studio/setup.sh'
|
||||
- 'studio/setup.ps1'
|
||||
- 'studio/install_python_stack.py'
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
repeats:
|
||||
description: 'launch repeats per OS (median reported)'
|
||||
type: string
|
||||
default: '3'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
profile:
|
||||
name: startup ${{ matrix.os }}
|
||||
runs-on: ${{ matrix.os }}
|
||||
timeout-minutes: 60
|
||||
continue-on-error: true
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
os: [ubuntu-latest, macos-14, windows-latest]
|
||||
|
||||
env:
|
||||
UNSLOTH_STUDIO_HOME: ${{ github.workspace }}/.studio-home
|
||||
# A wildcard bind calls ifconfig.me on the startup path; loopback times our code.
|
||||
UNSLOTH_STUDIO_DISABLE_PUBLIC_CHECK: '1'
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Install Studio
|
||||
shell: bash
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
run: |
|
||||
set -o pipefail
|
||||
mkdir -p logs
|
||||
# --local is load-bearing: it overlays the checkout, so the profiled server
|
||||
# is this diff. Without it install.sh resolves unsloth from PyPI.
|
||||
if [ "${{ runner.os }}" = "Windows" ]; then
|
||||
pwsh -NoProfile -File ./install.ps1 --local 2>&1 | tee logs/install.log
|
||||
else
|
||||
bash install.sh --local 2>&1 | tee logs/install.log
|
||||
fi
|
||||
|
||||
- name: Profile startup
|
||||
shell: bash
|
||||
run: |
|
||||
BIN="$UNSLOTH_STUDIO_HOME/unsloth_studio/bin/unsloth"
|
||||
[ -x "$BIN" ] || BIN="$UNSLOTH_STUDIO_HOME/unsloth_studio/Scripts/unsloth.exe"
|
||||
[ -x "$BIN" ] || BIN=""
|
||||
# Profile imports with the INSTALLED interpreter: that venv is what launches.
|
||||
PY="$UNSLOTH_STUDIO_HOME/unsloth_studio/bin/python"
|
||||
[ -x "$PY" ] || PY="$UNSLOTH_STUDIO_HOME/unsloth_studio/Scripts/python.exe"
|
||||
[ -x "$PY" ] || PY="$(command -v python3 || command -v python)"
|
||||
python3 scripts/profile_startup.py \
|
||||
--python "$PY" \
|
||||
${BIN:+--bin "$BIN"} \
|
||||
--repeats "${{ inputs.repeats || '3' }}" \
|
||||
--json "startup-${{ matrix.os }}.json" 2>&1 | tee logs/profile.log
|
||||
|
||||
- name: Summary
|
||||
if: always()
|
||||
shell: bash
|
||||
run: |
|
||||
f="startup-${{ matrix.os }}.json"
|
||||
[ -f "$f" ] || { echo "no profile produced"; exit 0; }
|
||||
python3 - "$f" >> "$GITHUB_STEP_SUMMARY" <<'PY'
|
||||
import json, sys
|
||||
d = json.load(open(sys.argv[1]))
|
||||
print(f"### {d['platform']} / {d['machine']} (py {d['python']}, {d['cpu_count']} cpu)\n")
|
||||
imp = d.get("imports", {})
|
||||
# Gate on ok: a failed `import main` still leaves rows, so a total can lie.
|
||||
if imp.get("ok"):
|
||||
print(f"**`import main`: {imp['total_seconds']}s**\n")
|
||||
print("| package | self ms |")
|
||||
print("|---|---:|")
|
||||
for k, v in list(imp.get("self_by_package_ms", {}).items())[:8]:
|
||||
print(f"| {k} | {v} |")
|
||||
print()
|
||||
else:
|
||||
print("**`import main` failed - no valid import profile**\n")
|
||||
print("```\n" + (imp.get("error") or "")[-1500:] + "\n```\n")
|
||||
lau = d.get("launch") or {}
|
||||
runs = len(lau.get("runs") or [])
|
||||
failed = lau.get("failed_runs") or 0
|
||||
if lau.get("healthz_median_seconds") is not None:
|
||||
# The aggregates cover only the runs that reached healthz, so flag the
|
||||
# failures: bare numbers would read as a normal fast startup.
|
||||
note = f" _({runs - failed} of {runs} launches; {failed} never became healthy)_" if failed else ""
|
||||
print(f"**time to a healthy port: {lau['healthz_median_seconds']}s median, "
|
||||
f"{lau['healthz_max_seconds']}s max**{note}\n")
|
||||
elif lau.get("skipped"):
|
||||
print(f"_launch phase skipped: {lau['skipped']}_\n")
|
||||
elif runs:
|
||||
print(f"**no launch measurement: all {runs} launches failed to become healthy**\n")
|
||||
PY
|
||||
|
||||
- name: Upload profile
|
||||
if: always()
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
name: startup-profile-${{ matrix.os }}
|
||||
path: |
|
||||
startup-*.json
|
||||
logs/
|
||||
retention-days: 14
|
||||
if-no-files-found: warn
|
||||
3
.gitignore
vendored
3
.gitignore
vendored
|
|
@ -208,6 +208,9 @@ tmp/
|
|||
**/node_modules/
|
||||
auth.db
|
||||
|
||||
# Packaging snapshot of the root CHANGELOG.md (written by build.sh)
|
||||
studio/CHANGELOG.md
|
||||
|
||||
# Tauri local build/generated output
|
||||
studio/src-tauri/target/
|
||||
studio/src-tauri/gen/
|
||||
|
|
|
|||
88
CHANGELOG.md
Normal file
88
CHANGELOG.md
Normal file
|
|
@ -0,0 +1,88 @@
|
|||
# Changelog
|
||||
|
||||
Release notes for Unsloth and Unsloth Studio.
|
||||
|
||||
Unsloth Studio reads this file to show release notes inside the "New Unsloth
|
||||
version" update popup. Edit it here and the popup picks the change up on the
|
||||
next update check, with no release or rebuild required.
|
||||
|
||||
## Format
|
||||
|
||||
Every release is a level-2 heading whose first token is the version, optionally
|
||||
followed by a date:
|
||||
|
||||
```md
|
||||
## 2026.7.6 - 2026-07-22
|
||||
```
|
||||
|
||||
`## [2026.7.6] - 2026-07-22` and `## v2026.7.6` also work. Everything under a
|
||||
heading, up to the next level-2 heading, is that release's notes and renders as
|
||||
Markdown in the popup.
|
||||
|
||||
Notes are matched to one exact version. When Studio offers an update to
|
||||
`2026.7.6` it renders the `2026.7.6` section and nothing else. If that section
|
||||
is missing, the popup links out to the online changelog rather than showing
|
||||
notes from an unrelated release, so a new version needs its own section here
|
||||
before its notes can appear.
|
||||
|
||||
Keep the newest release at the top. Lead each bullet with the change itself:
|
||||
the collapsed popup highlights the first sentence and dims the rest.
|
||||
`## Unreleased` is ignored by the popup, so it is safe to stage notes there and
|
||||
rename the heading at release time.
|
||||
|
||||
<!-- Add new releases directly below this line. -->
|
||||
|
||||
## Unreleased
|
||||
|
||||
## 2026.7.5
|
||||
|
||||
### What's Changed
|
||||
|
||||
- AMD support is here. Train, run RL, chat with and deploy 500+ models on
|
||||
Radeon, Instinct, Ryzen and data center GPUs across Windows, WSL and Linux,
|
||||
up to 2x faster with 70% less VRAM and no accuracy loss.
|
||||
- Intel XPU support lands in Studio, so Arc and Data Center GPUs run chat and
|
||||
training alongside the NVIDIA, AMD and Apple paths.
|
||||
- Local speech to text dictation runs fully offline, with slim Whisper bundles
|
||||
and a picker for custom models.
|
||||
- DoRA training is available in Studio, selectable next to LoRA and full
|
||||
fine-tuning in the training tab.
|
||||
- The update popup previews release notes inline, pulled from this file and
|
||||
matched to the exact version being offered.
|
||||
|
||||
### AMD, 23 July update
|
||||
|
||||
Our AMD collaboration, custom Triton kernels and math algorithms bring local
|
||||
training and inference to AMD hardware. The 23 July update builds on the
|
||||
[AMD release](https://github.com/unslothai/unsloth/releases/tag/v0.1.501-beta):
|
||||
|
||||
- RDNA2 and Gorgon Halo are supported, and the installer no longer fails to
|
||||
detect GPUs on Strix Halo and other AMD cards.
|
||||
- RDNA4 handling is better, and HIP and ROCm failures are caught and fixed
|
||||
automatically instead of stopping the install.
|
||||
- Unified memory safetensors loading is 2x faster, with much faster gradient
|
||||
checkpointing on unified memory devices.
|
||||
- Voice dictation through whisper.cpp has preliminary support.
|
||||
- Rollback environments left by installs no longer eat 5GB of disk. They are
|
||||
cleaned up automatically.
|
||||
|
||||
Optimized ROCm builds cover GGUF and safetensors inference, and ROCm
|
||||
compatibility is improved for MI300X and MI325X. Full guide:
|
||||
[unsloth.ai/docs/basics/amd](https://unsloth.ai/docs/basics/amd).
|
||||
|
||||
### Running larger models
|
||||
|
||||
- Automatic GPU placement, or pick exactly which GPUs and layers to use.
|
||||
- Move MoE expert layers into system memory so larger models fit.
|
||||
- Split a model across several GPUs, or use tensor parallelism.
|
||||
- Hardware settings are saved per model and quant.
|
||||
|
||||
### Also in this release
|
||||
|
||||
- Remote access with `unsloth studio --secure` over free HTTPS via Cloudflare.
|
||||
- Web search reads PDF papers and manuals, and parallel tool calls, reasoning
|
||||
output and tool retries are more reliable.
|
||||
- The model download location is configurable, so weights can live on a second
|
||||
drive instead of the default cache.
|
||||
- Stalled Hugging Face XET downloads retry over standard HTTP, and existing
|
||||
GGUF files are reused instead of downloaded again.
|
||||
2
MANIFEST.in
Normal file
2
MANIFEST.in
Normal file
|
|
@ -0,0 +1,2 @@
|
|||
include _changelog_build.py
|
||||
include CHANGELOG.md
|
||||
36
_changelog_build.py
Normal file
36
_changelog_build.py
Normal file
|
|
@ -0,0 +1,36 @@
|
|||
# SPDX-License-Identifier: AGPL-3.0-only
|
||||
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
|
||||
|
||||
"""Snapshot CHANGELOG.md into the studio package at build time.
|
||||
|
||||
CHANGELOG.md at the repo root stays the one file to edit. Copying it here,
|
||||
rather than in build.sh, means every packaging path ships it, so release notes
|
||||
still render when the popup cannot reach GitHub."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import shutil
|
||||
from pathlib import Path
|
||||
|
||||
from setuptools.command.build_py import build_py as _build_py
|
||||
|
||||
ROOT = Path(__file__).resolve().parent
|
||||
SOURCE = ROOT / "CHANGELOG.md"
|
||||
SNAPSHOT = ROOT / "studio" / "CHANGELOG.md"
|
||||
|
||||
|
||||
class build_py(_build_py):
|
||||
def run(self) -> None:
|
||||
# Beside the sources only if writable (PEP 517 may build an immutable
|
||||
# checkout); into the staging directory always.
|
||||
if SOURCE.is_file():
|
||||
try:
|
||||
shutil.copyfile(SOURCE, SNAPSHOT)
|
||||
except OSError:
|
||||
pass
|
||||
super().run()
|
||||
if not SOURCE.is_file():
|
||||
return
|
||||
staged = Path(self.build_lib) / "studio" / "CHANGELOG.md"
|
||||
staged.parent.mkdir(parents = True, exist_ok = True)
|
||||
shutil.copyfile(SOURCE, staged)
|
||||
6
build.sh
6
build.sh
|
|
@ -103,9 +103,13 @@ else
|
|||
STUDIO_STAMPED_VERSION="$(python scripts/stamp_studio_release.py)"
|
||||
fi
|
||||
|
||||
# 4. Build wheel/sdist
|
||||
# 4. Build wheel/sdist. _changelog_build.py snapshots CHANGELOG.md into the studio
|
||||
# package so release notes render offline.
|
||||
python -m build
|
||||
|
||||
# Drop the snapshot so a source checkout never serves a stale copy.
|
||||
rm -f studio/CHANGELOG.md
|
||||
|
||||
if [ "${1:-}" = "publish" ]; then
|
||||
python scripts/stamp_studio_release.py --verify-dist dist --expected "$STUDIO_STAMPED_VERSION"
|
||||
fi
|
||||
|
|
|
|||
146
install.ps1
146
install.ps1
|
|
@ -57,6 +57,26 @@ function Install-UnslothStudio {
|
|||
}
|
||||
}
|
||||
|
||||
# Machine arch; Get-TauriDiagArch above reports the process. An emulated x64 shell on
|
||||
# ARM64 reports AMD64, but PROCESSOR_ARCHITEW6432 is ARM64 in exactly that case.
|
||||
function Get-HostMachineArch {
|
||||
$osArch = ""
|
||||
try { $osArch = [System.Runtime.InteropServices.RuntimeInformation]::OSArchitecture.ToString() } catch { $osArch = "" }
|
||||
$signals = @([string]$env:PROCESSOR_ARCHITEW6432, [string]$env:PROCESSOR_ARCHITECTURE, $osArch)
|
||||
foreach ($s in $signals) {
|
||||
if ($s.ToLowerInvariant() -eq "arm64") { return "arm64" }
|
||||
}
|
||||
foreach ($s in $signals) {
|
||||
if ([string]::IsNullOrWhiteSpace($s)) { continue }
|
||||
switch ($s.ToLowerInvariant()) {
|
||||
"amd64" { return "x86_64" }
|
||||
"x64" { return "x86_64" }
|
||||
"x86" { return "x86" }
|
||||
}
|
||||
}
|
||||
return "unknown"
|
||||
}
|
||||
|
||||
function Get-TauriTorchIndexFamily {
|
||||
param([string]$TorchIndexUrl)
|
||||
if ($SkipTorch) { return "none" }
|
||||
|
|
@ -1124,10 +1144,27 @@ exit 0
|
|||
return $false
|
||||
}
|
||||
|
||||
# The interpreter's own arch, asked of it: win-amd64|win-arm64|win32|"".
|
||||
function Get-PythonPlatformTag {
|
||||
param([string]$Exe)
|
||||
try {
|
||||
return (& $Exe -c "import sysconfig; print(sysconfig.get_platform())" 2>$null | Out-String).Trim().ToLowerInvariant()
|
||||
} catch { return "" }
|
||||
}
|
||||
|
||||
# Returns @{ Version = "3.13"; Path = "C:\...\python.exe" } or $null.
|
||||
# The resolved Path is passed to `uv venv --python` to prevent uv from
|
||||
# re-resolving the version string back to a conda interpreter.
|
||||
function Find-CompatiblePython {
|
||||
# -X64Only: best installed x64 interpreter or $null, never ARM64. Last resort for
|
||||
# Install-X64Python, where x64 of a lower-priority minor beats ARM64.
|
||||
param([switch]$X64Only)
|
||||
# Windows on ARM: prefer x64. pyarrow (via datasets) and hf-transfer ship no
|
||||
# win_arm64 wheel, so a native ARM64 Python source-builds both and dies on CMake /
|
||||
# Rust minutes in; x64 runs fine emulated. ARM64 is still returned when it is all
|
||||
# there is, and the caller then bootstraps x64 or warns.
|
||||
$preferX64 = $X64Only -or ((Get-HostMachineArch) -eq "arm64")
|
||||
$candidates = @()
|
||||
# Try the Python Launcher first (most reliable on Windows)
|
||||
# py.exe resolves to the standard CPython install, not conda.
|
||||
# Prefer the requested $PythonVersion, then newest-first fallback.
|
||||
|
|
@ -1145,7 +1182,8 @@ exit 0
|
|||
# Resolve the actual executable path and verify it is not conda-based
|
||||
$resolvedExe = (& $pyLauncher.Source "-$minor" -c "import sys; print(sys.executable)" 2>$null | Out-String).Trim()
|
||||
if ($resolvedExe -and (Test-Path $resolvedExe) -and -not (Test-IsCondaPython $resolvedExe)) {
|
||||
return @{ Version = $ver; Path = $resolvedExe }
|
||||
if (-not $preferX64) { return @{ Version = $ver; Path = $resolvedExe; Arch = "" } }
|
||||
$candidates += @{ Version = $ver; Path = $resolvedExe }
|
||||
}
|
||||
}
|
||||
} catch {}
|
||||
|
|
@ -1166,11 +1204,53 @@ exit 0
|
|||
try {
|
||||
$out = & $cmd.Source --version 2>&1 | Out-String
|
||||
if ($out -match "Python (3\.1[1-3])\.\d+") {
|
||||
return @{ Version = $Matches[1]; Path = $cmd.Source }
|
||||
if (-not $preferX64) { return @{ Version = $Matches[1]; Path = $cmd.Source; Arch = "" } }
|
||||
$candidates += @{ Version = $Matches[1]; Path = $cmd.Source }
|
||||
}
|
||||
} catch {}
|
||||
}
|
||||
}
|
||||
# `py -3.12` runs the launcher's preferred build, normally the native ARM64 one, so
|
||||
# a same-minor x64 install that is neither preferred nor on PATH never becomes a
|
||||
# candidate. `-3.12-64` cannot disambiguate (deprecated, it only means "not
|
||||
# 32-bit"), so enumerate every registration with -0p and probe each path.
|
||||
if ($preferX64) {
|
||||
foreach ($pyLauncher in @(Get-Command py -All -CommandType Application -ErrorAction SilentlyContinue)) {
|
||||
if ($pyLauncher.Source -match $script:CondaSkipPattern) { continue }
|
||||
$listed = @()
|
||||
try { $listed = @(& $pyLauncher.Source "-0p" 2>$null) } catch {}
|
||||
foreach ($line in $listed) {
|
||||
# " -V:3.12 * C:\...\python.exe": tag, optional default marker, path.
|
||||
$m = [regex]::Match([string]$line, '(?i)^\s*-\S+\s+\*?\s*"?(?<p>\S.*?\.exe)"?\s*$')
|
||||
if (-not $m.Success) { continue }
|
||||
$exe = $m.Groups['p'].Value.Trim()
|
||||
if ($candidates | Where-Object { $_.Path -eq $exe }) { continue }
|
||||
if (-not (Test-Path -LiteralPath $exe)) { continue }
|
||||
if (Test-IsCondaPython $exe) { continue }
|
||||
try {
|
||||
$out = & $exe --version 2>&1 | Out-String
|
||||
if ($out -match "Python (3\.1[1-3])\.\d+") {
|
||||
$candidates += @{ Version = $Matches[1]; Path = $exe }
|
||||
}
|
||||
} catch {}
|
||||
}
|
||||
}
|
||||
}
|
||||
# Prefer x64, but only within one minor: $minors is the caller's version preference,
|
||||
# so ranking on arch alone would answer UNSLOTH_PYTHON=3.12 with an x64 3.13 and
|
||||
# never bootstrap x64 3.12. Probing costs a subprocess, so non-ARM returned above.
|
||||
foreach ($c in $candidates) {
|
||||
$tag = Get-PythonPlatformTag $c.Path
|
||||
$c.Arch = if ($tag -eq "win-amd64") { "x86_64" } elseif ($tag -eq "win-arm64") { "arm64" } else { "unknown" }
|
||||
}
|
||||
foreach ($minor in $minors) {
|
||||
$sameMinor = @($candidates | Where-Object { $_.Version -eq $minor })
|
||||
if ($sameMinor.Count -eq 0) { continue }
|
||||
$x64 = $sameMinor | Where-Object { $_.Arch -eq "x86_64" } | Select-Object -First 1
|
||||
if ($x64) { return $x64 }
|
||||
if (-not $X64Only) { return $sameMinor[0] }
|
||||
}
|
||||
if (-not $X64Only -and $candidates.Count -gt 0) { return $candidates[0] }
|
||||
return $null
|
||||
}
|
||||
|
||||
|
|
@ -1181,8 +1261,11 @@ exit 0
|
|||
# (no UAC), putting python.exe + the py launcher on PATH. Mirrors the uv ->
|
||||
# astral.sh fallback below. Returns @{ Version; Path } or $null.
|
||||
function Install-PythonFromPythonOrg {
|
||||
# $Arch overrides the host arch, to pull x64 onto an ARM64 box.
|
||||
param([string]$Arch = "")
|
||||
# python.org ships one installer per architecture.
|
||||
$archSuffix = switch (Get-TauriDiagArch) {
|
||||
$targetArch = if ($Arch) { $Arch } else { Get-TauriDiagArch }
|
||||
$archSuffix = switch ($targetArch) {
|
||||
"x86_64" { "-amd64" }
|
||||
"arm64" { "-arm64" }
|
||||
"x86" { "" }
|
||||
|
|
@ -1247,6 +1330,28 @@ exit 0
|
|||
return (Find-CompatiblePython)
|
||||
}
|
||||
|
||||
# ── Windows on ARM: get an x64 CPython ──
|
||||
# --architecture x64 forces winget off the ARM64 build; python.org takes the same override.
|
||||
function Install-X64Python {
|
||||
if ($script:WingetAvailable) {
|
||||
$prevEAP = $ErrorActionPreference
|
||||
$ErrorActionPreference = "Continue"
|
||||
try {
|
||||
winget install -e --id "Python.Python.$PythonVersion" --source winget --architecture x64 --accept-package-agreements --accept-source-agreements
|
||||
} catch { }
|
||||
$ErrorActionPreference = $prevEAP
|
||||
Refresh-SessionPath
|
||||
$found = Find-CompatiblePython
|
||||
if ($found -and $found.Arch -eq "x86_64") { return $found }
|
||||
substep "winget could not provide an x64 Python -- trying python.org..." "Yellow"
|
||||
}
|
||||
$found = Install-PythonFromPythonOrg -Arch "x86_64"
|
||||
if ($found -and $found.Arch -eq "x86_64") { return $found }
|
||||
# Nothing installable (offline / no winget): an x64 build of another supported minor
|
||||
# still runs the wheels ARM64 cannot, so take it over the native interpreter.
|
||||
return (Find-CompatiblePython -X64Only)
|
||||
}
|
||||
|
||||
# ── Install Python if no compatible version (3.11-3.13) found ──
|
||||
# Find-CompatiblePython returns @{ Version = "3.13"; Path = "C:\...\python.exe" } or $null.
|
||||
Write-TauriLog "STEP" "Installing Python"
|
||||
|
|
@ -1318,6 +1423,26 @@ exit 0
|
|||
return (Exit-InstallFailure "Python installation failed")
|
||||
}
|
||||
}
|
||||
# ── Windows on ARM: swap a native ARM64 interpreter for x64 ──
|
||||
# pyarrow and hf-transfer publish no win_arm64 wheel, so an ARM64 Python source-builds
|
||||
# both and fails deep into the run. Warn up front if x64 is unobtainable.
|
||||
if ($DetectedPython -and (Get-HostMachineArch) -eq "arm64" -and $DetectedPython.Arch -ne "x86_64") {
|
||||
substep "windows on arm: only a native ARM64 Python $($DetectedPython.Version) was found." "Yellow"
|
||||
substep "pyarrow and hf-transfer publish no win_arm64 wheels, so installing x64 Python..." "Yellow"
|
||||
$X64Python = Install-X64Python
|
||||
if ($X64Python) {
|
||||
$DetectedPython = $X64Python
|
||||
step "python" "using x64 Python $($DetectedPython.Version) under emulation"
|
||||
} else {
|
||||
Write-Host "[WARN] Could not install an x64 Python on this ARM64 machine." -ForegroundColor Yellow
|
||||
Write-Host " Continuing with ARM64 Python $($DetectedPython.Version), but the install is likely to fail:" -ForegroundColor Yellow
|
||||
Write-Host " pyarrow (via datasets) and hf-transfer ship no win_arm64 wheels and will be" -ForegroundColor Yellow
|
||||
Write-Host " built from source, which needs CMake plus the MSVC and Rust toolchains." -ForegroundColor Yellow
|
||||
Write-Host " Fix: install x64 Python from https://www.python.org/downloads/windows/" -ForegroundColor Yellow
|
||||
Write-Host " (choose 'Windows installer (64-bit)', not ARM64), then re-run this installer." -ForegroundColor Yellow
|
||||
}
|
||||
}
|
||||
|
||||
$DiagPythonVersion = $PythonVersion
|
||||
if ($DetectedPython) { $DiagPythonVersion = $DetectedPython.Version }
|
||||
$InitialGpuBranch = "unknown"
|
||||
|
|
@ -2438,6 +2563,13 @@ exit 0
|
|||
}
|
||||
} else {
|
||||
Write-TauriLog "STEP" "Installing PyTorch"
|
||||
# Windows on ARM lacks only torchaudio (whl/cpu win_arm64: torch 42,
|
||||
# torchvision 60, torchaudio 0), so drop that pin instead of aborting. Ask the
|
||||
# interpreter, not PROCESSOR_ARCHITECTURE; reached when no x64 Python exists.
|
||||
$VenvPlatform = ""
|
||||
try {
|
||||
$VenvPlatform = (& $VenvPython -c "import sysconfig; print(sysconfig.get_platform())" 2>$null | Out-String).Trim().ToLowerInvariant()
|
||||
} catch { $VenvPlatform = "" }
|
||||
substep "installing PyTorch ($(Remove-IndexUrlCredentials $TorchIndexUrl))..."
|
||||
# Bound the companions to the capped torch on EVERY index, cu<digits>
|
||||
# families included: torchaudio 2.11 dropped its exact torch pin from
|
||||
|
|
@ -2445,7 +2577,13 @@ exit 0
|
|||
# resolve a mismatched 2.11.0 build. Mirrors install.sh.
|
||||
$_pinVisionSpec = "torchvision>=0.19,<0.26.0"
|
||||
$_pinAudioSpec = "torchaudio>=2.4,<2.11.0"
|
||||
$torchInstallExit = Invoke-InstallCommandRetry -Label "install PyTorch" { uv pip install --python $VenvPython "torch>=2.4,<2.11.0" $_pinVisionSpec $_pinAudioSpec --default-index $TorchIndexUrl }
|
||||
$_torchSpecs = @("torch>=2.4,<2.11.0", $_pinVisionSpec, $_pinAudioSpec)
|
||||
if ($VenvPlatform -eq "win-arm64") {
|
||||
substep "windows on arm: skipping torchaudio (upstream publishes no"
|
||||
substep "win_arm64 wheel); torch and torchvision install normally."
|
||||
$_torchSpecs = @("torch>=2.4,<2.11.0", $_pinVisionSpec)
|
||||
}
|
||||
$torchInstallExit = Invoke-InstallCommandRetry -Label "install PyTorch" { uv pip install --python $VenvPython @_torchSpecs --default-index $TorchIndexUrl }
|
||||
if ($torchInstallExit -ne 0) {
|
||||
Write-Host "[ERROR] Failed to install PyTorch (exit code $torchInstallExit)" -ForegroundColor Red
|
||||
return (Exit-InstallFailure "Failed to install PyTorch (exit code $torchInstallExit)" $torchInstallExit)
|
||||
|
|
|
|||
|
|
@ -47,9 +47,14 @@ version = {attr = "unsloth.models._utils.__version__"}
|
|||
[tool.setuptools]
|
||||
include-package-data = true
|
||||
|
||||
[tool.setuptools.cmdclass]
|
||||
# Snapshots CHANGELOG.md into studio/ so every build path ships it.
|
||||
build_py = "_changelog_build.build_py"
|
||||
|
||||
[tool.setuptools.package-data]
|
||||
unsloth_cli = ["codex_fallback_prompt.md", "pi_subagent.ts"]
|
||||
studio = [
|
||||
"CHANGELOG.md",
|
||||
"*.sh",
|
||||
"*.ps1",
|
||||
"*.bat",
|
||||
|
|
|
|||
377
scripts/profile_startup.py
Normal file
377
scripts/profile_startup.py
Normal file
|
|
@ -0,0 +1,377 @@
|
|||
#!/usr/bin/env python3
|
||||
# SPDX-License-Identifier: AGPL-3.0-only
|
||||
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
|
||||
|
||||
"""Measure where Unsloth Studio's startup time goes, per platform.
|
||||
|
||||
Nothing measured this before: the backend logs "lifespan startup completed in X ms"
|
||||
but no test or CI job asserted a budget, and studio_test_kit discards the elapsed
|
||||
time of its /healthz poll. A first local run (Linux, warm cache, fast server CPU)
|
||||
found `import main` alone costs 6.6s before the server can bind, dominated by eager
|
||||
module-level imports pulled in by the `routes` package:
|
||||
|
||||
torch 1930 ms self
|
||||
unsloth_zoo 914 ms self
|
||||
routes 779 ms self
|
||||
transformers 524 ms self
|
||||
|
||||
Phases measured:
|
||||
import `python -X importtime -c "import main"`, top cumulative + per-package self
|
||||
spawn process start -> first byte on stdout
|
||||
healthz process start -> /api/health (or /healthz) answers 200
|
||||
lifespan the backend's own "lifespan startup completed in X ms" log line
|
||||
|
||||
Usage:
|
||||
python scripts/profile_startup.py --repeats 3 --json out.json
|
||||
python scripts/profile_startup.py --import-only # no server, no port needed
|
||||
|
||||
Exit code is 0 unless --max-healthz-seconds is given and exceeded.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import math
|
||||
import os
|
||||
import platform
|
||||
import re
|
||||
import shutil
|
||||
import socket
|
||||
import statistics
|
||||
import subprocess
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
from pathlib import Path
|
||||
|
||||
REPO_ROOT = Path(__file__).resolve().parents[1]
|
||||
BACKEND = REPO_ROOT / "studio" / "backend"
|
||||
|
||||
_IMPORTTIME_RE = re.compile(r"import time:\s+(\d+)\s+\|\s+(\d+)\s+\|(\s*)(\S.*)")
|
||||
|
||||
|
||||
def _free_port() -> int:
|
||||
with socket.socket() as s:
|
||||
s.bind(("127.0.0.1", 0))
|
||||
return int(s.getsockname()[1])
|
||||
|
||||
|
||||
def profile_imports(python: str, top: int = 15) -> dict:
|
||||
"""Cumulative and self import cost for the backend's module graph.
|
||||
|
||||
Run in a subprocess with -X importtime: the numbers are only meaningful for a
|
||||
cold interpreter, and importing in-process would measure a warm sys.modules.
|
||||
"""
|
||||
proc = subprocess.run(
|
||||
[python, "-X", "importtime", "-c", "import sys; sys.path.insert(0, '.'); import main"],
|
||||
cwd = BACKEND,
|
||||
capture_output = True,
|
||||
text = True,
|
||||
timeout = 900,
|
||||
)
|
||||
rows = []
|
||||
for line in proc.stderr.splitlines():
|
||||
m = _IMPORTTIME_RE.match(line)
|
||||
if m:
|
||||
rows.append((int(m.group(1)), int(m.group(2)), m.group(4).strip()))
|
||||
if not rows:
|
||||
return {"ok": False, "error": (proc.stderr or proc.stdout)[-2000:]}
|
||||
if proc.returncode != 0:
|
||||
# Rows survive up to the failure, so any total from a partial graph is wrong.
|
||||
return {
|
||||
"ok": False,
|
||||
"error": (proc.stderr or proc.stdout)[-2000:],
|
||||
"partial_rows": len(rows),
|
||||
}
|
||||
|
||||
by_cum = sorted(rows, key = lambda r: -r[1])
|
||||
# Total comes from the `main` row, not by_cum[0]: -X importtime also prints the
|
||||
# interpreter's own startup graph (`site`), which can outrank a trivial main.
|
||||
main_row = next((r for r in reversed(rows) if r[2] == "main"), None)
|
||||
if main_row is None:
|
||||
return {
|
||||
"ok": False,
|
||||
"error": "no `import main` row in -X importtime output\n"
|
||||
+ (proc.stderr or proc.stdout)[-2000:],
|
||||
}
|
||||
self_by_pkg: dict[str, int] = {}
|
||||
for self_us, _cum, name in rows:
|
||||
pkg = name.split(".")[0]
|
||||
self_by_pkg[pkg] = self_by_pkg.get(pkg, 0) + self_us
|
||||
|
||||
return {
|
||||
"ok": True,
|
||||
"total_seconds": round(main_row[1] / 1e6, 3),
|
||||
"top_cumulative": [
|
||||
{"module": n, "seconds": round(c / 1e6, 3)} for _s, c, n in by_cum[:top]
|
||||
],
|
||||
"self_by_package_ms": {
|
||||
k: round(v / 1000) for k, v in sorted(self_by_pkg.items(), key = lambda x: -x[1])[:top]
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def _terminate_tree(proc: subprocess.Popen) -> None:
|
||||
"""Stop the server AND its children, which on Windows are a separate process.
|
||||
|
||||
CI profiles `Scripts/unsloth.exe`, a distlib launcher stub that CreateProcess's
|
||||
the venv python and waits, so terminate() reaps the stub only: the real backend
|
||||
keeps the inherited stdout handle, the reader thread never sees EOF, and
|
||||
--repeats strands one server per iteration on the shared UNSLOTH_STUDIO_HOME.
|
||||
taskkill /T walks the tree, as unsloth_cli/commands/start.py already does.
|
||||
"""
|
||||
if proc.poll() is not None:
|
||||
return
|
||||
if os.name == "nt":
|
||||
try:
|
||||
killed = subprocess.run(
|
||||
["taskkill", "/PID", str(proc.pid), "/T", "/F"],
|
||||
capture_output = True,
|
||||
timeout = 30,
|
||||
check = False,
|
||||
)
|
||||
if killed.returncode == 0:
|
||||
return
|
||||
except Exception:
|
||||
# taskkill missing or timed out; fall through so the stub still dies.
|
||||
pass
|
||||
# check=False: a nonzero taskkill does not raise, so fall through as well.
|
||||
proc.terminate()
|
||||
|
||||
|
||||
def profile_launch(
|
||||
bin_path: str,
|
||||
port: int,
|
||||
timeout_s: int = 300,
|
||||
) -> dict:
|
||||
"""Spawn the backend the way the desktop app does and time it to first 200."""
|
||||
log_lines: list[str] = []
|
||||
first_byte: list[float] = []
|
||||
t0 = time.perf_counter()
|
||||
proc = subprocess.Popen(
|
||||
[bin_path, "studio", "--api-only", "-H", "127.0.0.1", "-p", str(port)],
|
||||
cwd = REPO_ROOT,
|
||||
stdout = subprocess.PIPE,
|
||||
stderr = subprocess.STDOUT,
|
||||
text = True,
|
||||
bufsize = 1,
|
||||
)
|
||||
|
||||
def _drain() -> None:
|
||||
# Runs alongside the health polling: the first read timestamps the spawn
|
||||
# phase, and an undrained pipe blocks the backend before it binds.
|
||||
for line in proc.stdout:
|
||||
if not first_byte:
|
||||
first_byte.append(time.perf_counter() - t0)
|
||||
log_lines.append(line.rstrip("\n"))
|
||||
|
||||
reader = threading.Thread(target = _drain, daemon = True)
|
||||
reader.start()
|
||||
|
||||
t_healthz = None
|
||||
deadline = t0 + timeout_s
|
||||
try:
|
||||
while time.perf_counter() < deadline:
|
||||
if proc.poll() is not None:
|
||||
break
|
||||
if t_healthz is None:
|
||||
for url in (
|
||||
f"http://127.0.0.1:{port}/api/health",
|
||||
f"http://127.0.0.1:{port}/healthz",
|
||||
):
|
||||
try:
|
||||
with urllib.request.urlopen(url, timeout = 2) as r:
|
||||
if r.status == 200:
|
||||
t_healthz = time.perf_counter() - t0
|
||||
break
|
||||
except (urllib.error.URLError, OSError, TimeoutError):
|
||||
pass
|
||||
if t_healthz is not None:
|
||||
break
|
||||
time.sleep(0.25)
|
||||
finally:
|
||||
_terminate_tree(proc)
|
||||
try:
|
||||
# Safe: the reader drains the pipe, so the child cannot block on write().
|
||||
proc.wait(timeout = 30)
|
||||
except subprocess.TimeoutExpired:
|
||||
proc.kill()
|
||||
proc.wait()
|
||||
reader.join(timeout = 10)
|
||||
|
||||
t_first_byte = first_byte[0] if first_byte else None
|
||||
lifespan_ms = None
|
||||
for line in log_lines:
|
||||
m = re.search(r"lifespan startup completed in ([\d.]+)ms", line)
|
||||
if m:
|
||||
lifespan_ms = float(m.group(1))
|
||||
return {
|
||||
"spawn_seconds": round(t_first_byte, 3) if t_first_byte is not None else None,
|
||||
"healthz_seconds": round(t_healthz, 3) if t_healthz is not None else None,
|
||||
"lifespan_ms": lifespan_ms,
|
||||
"reached_healthz": t_healthz is not None,
|
||||
"log_tail": log_lines[-25:],
|
||||
}
|
||||
|
||||
|
||||
def python_version_of(python: str) -> str:
|
||||
"""Version of the interpreter that runs the imports, not the one running us.
|
||||
|
||||
--python points at the installed Studio venv while this script runs under the
|
||||
runner's system python, so platform.python_version() would label it wrong.
|
||||
"""
|
||||
if python == sys.executable:
|
||||
return platform.python_version()
|
||||
try:
|
||||
proc = subprocess.run(
|
||||
[python, "-c", "import platform; print(platform.python_version())"],
|
||||
capture_output = True,
|
||||
text = True,
|
||||
timeout = 60,
|
||||
)
|
||||
if proc.returncode == 0 and proc.stdout.strip():
|
||||
return proc.stdout.strip()
|
||||
except (OSError, subprocess.SubprocessError):
|
||||
pass
|
||||
return "unknown"
|
||||
|
||||
|
||||
def find_bin() -> str | None:
|
||||
home = os.environ.get("UNSLOTH_STUDIO_HOME") or str(Path.home() / ".unsloth" / "studio")
|
||||
names = ["unsloth.exe", "unsloth"] if platform.system() == "Windows" else ["unsloth"]
|
||||
subdirs = ["unsloth_studio/Scripts", "unsloth_studio/bin", "bin", "Scripts"]
|
||||
for sd in subdirs:
|
||||
for n in names:
|
||||
p = Path(home) / sd / n
|
||||
if p.exists():
|
||||
return str(p)
|
||||
return shutil.which("unsloth")
|
||||
|
||||
|
||||
def main(argv: list[str]) -> int:
|
||||
ap = argparse.ArgumentParser(
|
||||
description = __doc__, formatter_class = argparse.RawDescriptionHelpFormatter
|
||||
)
|
||||
ap.add_argument(
|
||||
"--repeats",
|
||||
type = int,
|
||||
default = 1,
|
||||
help = "launch repeats; the median is reported (imports are measured once)",
|
||||
)
|
||||
ap.add_argument(
|
||||
"--python",
|
||||
default = sys.executable,
|
||||
help = "interpreter used for the import profile (default: this one)",
|
||||
)
|
||||
ap.add_argument("--bin", help = "path to the unsloth CLI (default: autodetect)")
|
||||
ap.add_argument(
|
||||
"--import-only",
|
||||
action = "store_true",
|
||||
help = "skip the server phases (no install needed beyond the deps)",
|
||||
)
|
||||
ap.add_argument(
|
||||
"--max-healthz-seconds",
|
||||
type = float,
|
||||
help = "fail if the median time to a healthy port exceeds this",
|
||||
)
|
||||
ap.add_argument("--json", help = "write the full report here")
|
||||
a = ap.parse_args(argv)
|
||||
# range(0) launches nothing, leaving the budget check with nothing to fail on.
|
||||
if a.repeats < 1:
|
||||
ap.error("--repeats must be at least 1")
|
||||
# Same reason: --import-only never launches anything.
|
||||
if a.import_only and a.max_healthz_seconds is not None:
|
||||
ap.error("--max-healthz-seconds cannot be combined with --import-only")
|
||||
# nan and inf parse fine as floats but `med > budget` is then always False,
|
||||
# so the gate would report success without ever bounding anything.
|
||||
if a.max_healthz_seconds is not None and not math.isfinite(a.max_healthz_seconds):
|
||||
ap.error("--max-healthz-seconds must be a finite number")
|
||||
|
||||
report: dict = {
|
||||
"platform": platform.system().lower(),
|
||||
"machine": platform.machine(),
|
||||
"python": python_version_of(a.python),
|
||||
"cpu_count": os.cpu_count(),
|
||||
}
|
||||
|
||||
print("== import graph ==")
|
||||
report["imports"] = profile_imports(a.python)
|
||||
imp = report["imports"]
|
||||
if imp.get("ok"):
|
||||
print(f" import main: {imp['total_seconds']}s")
|
||||
for row in imp["top_cumulative"][:8]:
|
||||
print(f" {row['seconds']:7.3f}s {row['module']}")
|
||||
print(" self time by package (ms):")
|
||||
for k, v in list(imp["self_by_package_ms"].items())[:8]:
|
||||
print(f" {v:8} ms {k}")
|
||||
else:
|
||||
print(f" FAILED: {imp.get('error', '')[:400]}")
|
||||
|
||||
if not a.import_only:
|
||||
bin_path = a.bin or find_bin()
|
||||
if not bin_path:
|
||||
print(
|
||||
"== launch == skipped: no unsloth CLI found "
|
||||
"(set UNSLOTH_STUDIO_HOME or pass --bin)"
|
||||
)
|
||||
report["launch"] = {"skipped": "no unsloth CLI found"}
|
||||
else:
|
||||
print(f"== launch == {bin_path}")
|
||||
runs = []
|
||||
for i in range(a.repeats):
|
||||
r = profile_launch(bin_path, _free_port())
|
||||
runs.append(r)
|
||||
print(
|
||||
f" run {i + 1}: healthz={r['healthz_seconds']}s "
|
||||
f"lifespan={r['lifespan_ms']}ms reached={r['reached_healthz']}"
|
||||
)
|
||||
got = [r["healthz_seconds"] for r in runs if r["healthz_seconds"] is not None]
|
||||
report["launch"] = {
|
||||
"runs": runs,
|
||||
"failed_runs": sum(1 for r in runs if not r["reached_healthz"]),
|
||||
"healthz_median_seconds": round(statistics.median(got), 3) if got else None,
|
||||
"healthz_max_seconds": round(max(got), 3) if got else None,
|
||||
}
|
||||
if got:
|
||||
print(
|
||||
f" median time to healthy port: {report['launch']['healthz_median_seconds']}s"
|
||||
)
|
||||
|
||||
if a.json:
|
||||
Path(a.json).write_text(json.dumps(report, indent = 2), encoding = "utf-8")
|
||||
print(f"\nwrote {a.json}")
|
||||
|
||||
if a.max_healthz_seconds is not None:
|
||||
launch = report.get("launch") or {}
|
||||
med = launch.get("healthz_median_seconds")
|
||||
failed = launch.get("failed_runs") or 0
|
||||
if failed:
|
||||
# Failed launches fail the budget; dropping them would keep only the fast ones.
|
||||
print(
|
||||
f"::error::startup regression: {failed} of {len(launch.get('runs') or [])} "
|
||||
f"launches never became healthy within the timeout"
|
||||
)
|
||||
return 1
|
||||
if med is None:
|
||||
# Nothing measured: exiting 0 would pass a requested budget without a
|
||||
# single health request, so fail closed.
|
||||
print(
|
||||
"::error::startup regression: no healthz measurement, so the "
|
||||
f"{a.max_healthz_seconds}s budget was never checked "
|
||||
f"({launch.get('skipped') or 'launch phase produced no runs'})"
|
||||
)
|
||||
return 1
|
||||
elif med > a.max_healthz_seconds:
|
||||
print(
|
||||
f"::error::startup regression: {med}s median to a healthy port "
|
||||
f"exceeds the {a.max_healthz_seconds}s budget"
|
||||
)
|
||||
return 1
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main(sys.argv[1:]))
|
||||
|
|
@ -76,7 +76,13 @@ def _load_bootstrap_password() -> Optional[str]:
|
|||
global _bootstrap_password
|
||||
_bootstrap_password = None
|
||||
if _BOOTSTRAP_PW_PATH.is_file():
|
||||
bootstrap_password = _BOOTSTRAP_PW_PATH.read_text(encoding = "utf-8").strip()
|
||||
# No caller handles a raise, so an unreadable file has to mean "no bootstrap
|
||||
# password", not a dead backend. We write UTF-8, so bytes that will not
|
||||
# decode are damage whose plaintext is worthless anyway.
|
||||
try:
|
||||
bootstrap_password = _BOOTSTRAP_PW_PATH.read_text(encoding = "utf-8").strip()
|
||||
except (OSError, UnicodeDecodeError):
|
||||
return _bootstrap_password
|
||||
if bootstrap_password:
|
||||
_bootstrap_password = bootstrap_password
|
||||
return _bootstrap_password
|
||||
|
|
|
|||
|
|
@ -310,6 +310,7 @@ class CloudflareTunnel:
|
|||
stderr = subprocess.STDOUT,
|
||||
stdin = subprocess.DEVNULL,
|
||||
text = True,
|
||||
encoding = "utf-8",
|
||||
errors = "replace",
|
||||
bufsize = 1,
|
||||
**_windows_hidden_kwargs(),
|
||||
|
|
|
|||
|
|
@ -257,6 +257,8 @@ def _run_oxc_batch(
|
|||
cwd = str(_OXC_TOOL_DIR),
|
||||
input = json.dumps(payload),
|
||||
text = True,
|
||||
encoding = "utf-8",
|
||||
errors = "replace",
|
||||
capture_output = True,
|
||||
check = False,
|
||||
env = env,
|
||||
|
|
|
|||
|
|
@ -567,7 +567,7 @@ class InferenceBackend:
|
|||
_meta_path = Path(config.path) / "export_metadata.json"
|
||||
try:
|
||||
if _meta_path.exists():
|
||||
_meta = json.loads(_meta_path.read_text(encoding = "utf-8"))
|
||||
_meta = json.loads(_meta_path.read_text(encoding = "utf-8-sig"))
|
||||
if _meta.get("base_model"):
|
||||
processor_source = _meta["base_model"]
|
||||
except Exception:
|
||||
|
|
|
|||
|
|
@ -85,6 +85,7 @@ from core.tool_healing import (
|
|||
strip_outside_think,
|
||||
)
|
||||
from utils.native_path_leases import child_env_without_native_path_secret
|
||||
from utils.child_stdio import utf8_child_env
|
||||
from utils.hf_xet_fallback import hf_hub_download_with_xet_fallback
|
||||
from utils.subprocess_compat import (
|
||||
windows_hidden_subprocess_kwargs as _windows_hidden_subprocess_kwargs,
|
||||
|
|
@ -581,7 +582,7 @@ def _load_swa_cache() -> dict:
|
|||
if _SWA_CACHE is not None:
|
||||
return _SWA_CACHE
|
||||
try:
|
||||
with open(_swa_cache_path(), encoding = "utf-8") as f:
|
||||
with open(_swa_cache_path(), encoding = "utf-8-sig") as f:
|
||||
_SWA_CACHE = json.load(f)
|
||||
if not isinstance(_SWA_CACHE, dict):
|
||||
_SWA_CACHE = {}
|
||||
|
|
@ -632,7 +633,7 @@ def _fetch_swa_entry_from_hf(repo_id: str) -> Optional[object]:
|
|||
repo_type = "model",
|
||||
cache_dir = active_hf_hub_cache(),
|
||||
)
|
||||
with open(cfg_path, encoding = "utf-8") as f:
|
||||
with open(cfg_path, encoding = "utf-8-sig") as f:
|
||||
cfg = json.load(f)
|
||||
except Exception:
|
||||
return None
|
||||
|
|
@ -3046,6 +3047,7 @@ class LlamaCppBackend:
|
|||
[bin_path, "--help"],
|
||||
capture_output = True,
|
||||
text = True,
|
||||
encoding = "utf-8",
|
||||
errors = "replace",
|
||||
timeout = 10,
|
||||
check = False,
|
||||
|
|
@ -3618,6 +3620,8 @@ class LlamaCppBackend:
|
|||
],
|
||||
capture_output = True,
|
||||
text = True,
|
||||
encoding = "utf-8",
|
||||
errors = "replace",
|
||||
timeout = 10,
|
||||
env = child_env_without_native_path_secret(),
|
||||
**_windows_hidden_subprocess_kwargs(),
|
||||
|
|
@ -3732,7 +3736,7 @@ class LlamaCppBackend:
|
|||
encoding = "utf-8",
|
||||
errors = "replace",
|
||||
timeout = 15,
|
||||
env = env,
|
||||
env = utf8_child_env(env),
|
||||
**_windows_hidden_subprocess_kwargs(),
|
||||
)
|
||||
if result.returncode != 0:
|
||||
|
|
@ -5482,7 +5486,9 @@ class LlamaCppBackend:
|
|||
stdout = subprocess.PIPE,
|
||||
stderr = subprocess.STDOUT,
|
||||
text = True,
|
||||
env = env,
|
||||
encoding = "utf-8",
|
||||
errors = "replace",
|
||||
env = utf8_child_env(env),
|
||||
**_windows_hidden_subprocess_kwargs(),
|
||||
**_child_popen_kwargs(),
|
||||
)
|
||||
|
|
@ -6696,6 +6702,8 @@ class LlamaCppBackend:
|
|||
stdout = subprocess.PIPE,
|
||||
stderr = subprocess.STDOUT,
|
||||
text = True,
|
||||
encoding = "utf-8",
|
||||
errors = "replace",
|
||||
env = env,
|
||||
**_windows_hidden_subprocess_kwargs(),
|
||||
**_child_popen_kwargs(),
|
||||
|
|
@ -8712,6 +8720,8 @@ class LlamaCppBackend:
|
|||
stdout = subprocess.PIPE,
|
||||
stderr = subprocess.STDOUT,
|
||||
text = True,
|
||||
encoding = "utf-8",
|
||||
errors = "replace",
|
||||
env = env,
|
||||
**_windows_hidden_subprocess_kwargs(),
|
||||
**_child_popen_kwargs(),
|
||||
|
|
@ -10214,6 +10224,8 @@ class LlamaCppBackend:
|
|||
["pgrep", "-a", "-f", "llama-server"],
|
||||
capture_output = True,
|
||||
text = True,
|
||||
encoding = "utf-8",
|
||||
errors = "replace",
|
||||
timeout = 5,
|
||||
env = child_env_without_native_path_secret(),
|
||||
)
|
||||
|
|
|
|||
|
|
@ -151,7 +151,7 @@ def _resolve_lora_4bit(mc, load_in_4bit: bool) -> bool:
|
|||
import json
|
||||
|
||||
try:
|
||||
with open(adapter_cfg_path, encoding = "utf-8") as f:
|
||||
with open(adapter_cfg_path, encoding = "utf-8-sig") as f:
|
||||
adapter_cfg = json.load(f)
|
||||
training_method = adapter_cfg.get("unsloth_training_method")
|
||||
if training_method == "lora" and load_in_4bit:
|
||||
|
|
@ -963,7 +963,7 @@ def run_inference_process(
|
|||
if _local_adapter_cfg.is_file():
|
||||
try:
|
||||
_lora_base = (
|
||||
_json.loads(_local_adapter_cfg.read_text(encoding = "utf-8")).get(
|
||||
_json.loads(_local_adapter_cfg.read_text(encoding = "utf-8-sig")).get(
|
||||
"base_model_name_or_path"
|
||||
)
|
||||
or None
|
||||
|
|
|
|||
|
|
@ -103,6 +103,8 @@ class LlamaServerBackend:
|
|||
[binary, "--help"],
|
||||
capture_output = True,
|
||||
text = True,
|
||||
encoding = "utf-8",
|
||||
errors = "replace",
|
||||
timeout = 30,
|
||||
**windows_hidden_subprocess_kwargs(),
|
||||
)
|
||||
|
|
@ -331,6 +333,8 @@ class LlamaServerBackend:
|
|||
stdout = subprocess.PIPE,
|
||||
stderr = subprocess.STDOUT,
|
||||
text = True,
|
||||
encoding = "utf-8",
|
||||
errors = "replace",
|
||||
env = env,
|
||||
**windows_hidden_subprocess_kwargs(),
|
||||
**child_popen_kwargs(),
|
||||
|
|
|
|||
|
|
@ -100,7 +100,7 @@ def _st_module_subdirs(name: str, token: str | None) -> tuple[str, ...]:
|
|||
path = Path(normalize_path(name)).expanduser() / "modules.json"
|
||||
if not path.is_file():
|
||||
return ()
|
||||
data = json.loads(path.read_text(encoding = "utf-8"))
|
||||
data = json.loads(path.read_text(encoding = "utf-8-sig"))
|
||||
else:
|
||||
from huggingface_hub import hf_hub_download
|
||||
from huggingface_hub.utils import EntryNotFoundError
|
||||
|
|
@ -115,7 +115,7 @@ def _st_module_subdirs(name: str, token: str | None) -> tuple[str, ...]:
|
|||
)
|
||||
except EntryNotFoundError:
|
||||
return ()
|
||||
data = json.loads(open(local, encoding = "utf-8").read())
|
||||
data = json.loads(open(local, encoding = "utf-8-sig").read())
|
||||
subdirs = []
|
||||
for module in data or ():
|
||||
sub = str((module or {}).get("path", "")).strip().strip("/")
|
||||
|
|
|
|||
|
|
@ -43,6 +43,7 @@ if sys.platform.startswith("linux") and "HSA_ENABLE_DXG_DETECTION" not in os.env
|
|||
pass
|
||||
|
||||
logger = get_logger(__name__)
|
||||
from utils.child_stdio import utf8_child_env
|
||||
from utils.hardware import apply_gpu_ids
|
||||
from utils.training_runs import build_default_output_dir_name
|
||||
from utils.wheel_utils import (
|
||||
|
|
@ -385,6 +386,10 @@ def _install_package_wheel_first(
|
|||
"stdout": _sp.PIPE,
|
||||
"stderr": _sp.STDOUT,
|
||||
"text": True,
|
||||
"encoding": "utf-8",
|
||||
"errors": "replace",
|
||||
# Make the Python child emit the UTF-8 we decode above.
|
||||
"env": utf8_child_env(),
|
||||
}
|
||||
if is_hip:
|
||||
_run_kwargs["timeout"] = 1800
|
||||
|
|
@ -606,6 +611,9 @@ def _ensure_flash_linear_attention_unconditional(event_queue: Any) -> bool:
|
|||
stdout = _sp.PIPE,
|
||||
stderr = _sp.STDOUT,
|
||||
text = True,
|
||||
encoding = "utf-8",
|
||||
errors = "replace",
|
||||
env = utf8_child_env(),
|
||||
timeout = _TILELANG_INSTALL_TIMEOUT_S,
|
||||
)
|
||||
except _sp.TimeoutExpired:
|
||||
|
|
@ -849,6 +857,9 @@ def _run_pip(cmd: list[str], event_queue: Any, label: str) -> bool:
|
|||
stdout = _sp.PIPE,
|
||||
stderr = _sp.STDOUT,
|
||||
text = True,
|
||||
encoding = "utf-8",
|
||||
errors = "replace",
|
||||
env = utf8_child_env(),
|
||||
timeout = _TILELANG_INSTALL_TIMEOUT_S,
|
||||
)
|
||||
except _sp.TimeoutExpired:
|
||||
|
|
|
|||
|
|
@ -215,7 +215,7 @@ def _ollama_model_info_from_manifest(
|
|||
return None
|
||||
|
||||
try:
|
||||
manifest = json.loads(tag_file.read_text(encoding = "utf-8"))
|
||||
manifest = json.loads(tag_file.read_text(encoding = "utf-8-sig"))
|
||||
except (json.JSONDecodeError, OSError, UnicodeDecodeError) as e:
|
||||
logger.debug("Skipping unreadable/invalid Ollama manifest %s: %s", tag_file, e)
|
||||
return None
|
||||
|
|
@ -228,7 +228,7 @@ def _ollama_model_info_from_manifest(
|
|||
config_blob = _ollama_blob_path(blobs_dir, config_digest)
|
||||
if config_blob is not None and _safe_is_file(config_blob):
|
||||
try:
|
||||
cfg = json.loads(config_blob.read_text(encoding = "utf-8"))
|
||||
cfg = json.loads(config_blob.read_text(encoding = "utf-8-sig"))
|
||||
model_type = cfg.get("model_type", "")
|
||||
file_type = cfg.get("file_type", "")
|
||||
except (json.JSONDecodeError, OSError, UnicodeDecodeError) as e:
|
||||
|
|
|
|||
|
|
@ -464,6 +464,8 @@ def _read_marker_value(marker: Path) -> Optional[str]:
|
|||
return None
|
||||
value = marker.read_text(encoding = "utf-8").strip()
|
||||
except (OSError, UnicodeDecodeError):
|
||||
# UnicodeDecodeError is a ValueError, so it would escape and abort
|
||||
# prepare_cache_for_transport. An unknown value just purges and restarts.
|
||||
return None
|
||||
return value if value in VALID_TRANSPORTS else None
|
||||
|
||||
|
|
|
|||
|
|
@ -42,8 +42,12 @@ class LogConfig:
|
|||
log_level_name = os.getenv("LOG_LEVEL", "INFO").upper()
|
||||
log_level = getattr(logging, log_level_name, logging.INFO)
|
||||
|
||||
if sys.platform == "win32":
|
||||
for stream in (sys.stdout, sys.stderr):
|
||||
# Non-ASCII on a non-UTF-8 stream raises UnicodeEncodeError (Windows,
|
||||
# LANG=C), so key off the stream, not the platform.
|
||||
for stream in (sys.stdout, sys.stderr):
|
||||
if getattr(stream, "encoding", "") and not str(stream.encoding).lower().replace(
|
||||
"-", ""
|
||||
).startswith("utf8"):
|
||||
if hasattr(stream, "reconfigure"):
|
||||
try:
|
||||
stream.reconfigure(encoding = "utf-8", errors = "replace")
|
||||
|
|
|
|||
|
|
@ -347,6 +347,7 @@ from utils.update_status import (
|
|||
get_studio_install_source_status,
|
||||
get_studio_update_status,
|
||||
)
|
||||
from utils.changelog import get_release_notes, is_supported_version_query
|
||||
from utils.studio_version import get_studio_version
|
||||
from utils.api_errors import install_api_error_handlers
|
||||
|
||||
|
|
@ -1154,6 +1155,18 @@ def studio_update_status(_current_subject: str = Depends(get_current_subject)):
|
|||
return get_studio_update_status(UNSLOTH_VERSION)
|
||||
|
||||
|
||||
@app.get("/api/studio/release-notes")
|
||||
def studio_release_notes(
|
||||
version: str = Query(..., max_length = 64),
|
||||
refresh: bool = Query(False),
|
||||
_current_subject: str = Depends(get_current_subject),
|
||||
):
|
||||
"""Return CHANGELOG.md notes for exactly `version` (never a nearby one)."""
|
||||
if not is_supported_version_query(version):
|
||||
raise HTTPException(status_code = 422, detail = "Invalid version.")
|
||||
return get_release_notes(version, refresh = refresh)
|
||||
|
||||
|
||||
@app.get(
|
||||
"/api/studio/download-transport-capabilities",
|
||||
response_model = TransportCapabilities,
|
||||
|
|
|
|||
|
|
@ -6,10 +6,93 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import locale
|
||||
import os
|
||||
import threading
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict
|
||||
from typing import Any, Dict, NamedTuple
|
||||
|
||||
|
||||
def _locale_encoding() -> str:
|
||||
"""The codepage a pre-UTF-8 release here would have written, or "".
|
||||
|
||||
Empty on a UTF-8 host, where there is no codepage to attribute the file to.
|
||||
"""
|
||||
try:
|
||||
preferred = locale.getencoding()
|
||||
except AttributeError: # Python < 3.11
|
||||
preferred = locale.getpreferredencoding(False)
|
||||
if preferred.lower().replace("-", "").replace("_", "") == "utf8":
|
||||
return ""
|
||||
return preferred
|
||||
|
||||
|
||||
# Trail bytes can land on JSON punctuation, so a single-byte fallback misreads these.
|
||||
_DOUBLE_BYTE_ENCODINGS = ("cp932", "cp936", "cp949", "cp950")
|
||||
|
||||
|
||||
def _parse(raw: bytes, encoding: str) -> Any:
|
||||
"""Parse one JSON document under *encoding*, or None if it does not.
|
||||
|
||||
RecursionError is a RuntimeError, so nesting json.loads will not descend is
|
||||
the one parse failure the other three miss. Both callers run this outside
|
||||
any further handler, so it has to answer None here or a single damaged
|
||||
record aborts the scraper at startup instead of being skipped.
|
||||
"""
|
||||
try:
|
||||
return json.loads(raw.decode(encoding))
|
||||
except (UnicodeDecodeError, LookupError, ValueError, RecursionError):
|
||||
return None
|
||||
|
||||
|
||||
class _Reading(NamedTuple):
|
||||
as_utf8: Any
|
||||
as_legacy: Any
|
||||
|
||||
|
||||
def _read_line(raw: bytes, codepage: str) -> _Reading:
|
||||
"""Read one line as UTF-8 and as a codepage, for dedup keys only.
|
||||
|
||||
Requiring valid JSON, not merely a successful decode, is what separates a
|
||||
genuine legacy record from a half-written UTF-8 one: a torn multibyte
|
||||
character decodes under cp1252 but leaves the JSON unterminated. Some byte
|
||||
strings parse both ways, e.g. cp1251 ``Р°`` is ``D0 B0``, which is also
|
||||
UTF-8 ``а``.
|
||||
|
||||
The codepage reading is never authoritative, because the file's own encoding
|
||||
cannot be recovered from its bytes. Reading a cp1251 shard on a cp1252
|
||||
machine turns ``Привет`` into ``Ïðèâåò`` and every byte of it decodes
|
||||
cleanly, so a successful decode proves nothing about who wrote it. It is
|
||||
used only to recover the dedup keys, which are ASCII ids and come back the
|
||||
same under any of these, so the first reading that parses will do.
|
||||
|
||||
That is also why several are tried. latin-1 alone mangles the double-byte
|
||||
codepages: cp932 ``表`` is ``95 5C``, and latin-1 turns the trail byte into
|
||||
a JSON backslash, so the record fails to parse and its id is forgotten.
|
||||
"""
|
||||
as_utf8 = _parse(raw, "utf-8")
|
||||
# A record that reads as UTF-8 needs no second reading: re-parsing cost 2.8x on a
|
||||
# 76 MB shard, and these reach gigabytes. Only a dict, since key lookup falls
|
||||
# through to the codepage when UTF-8 yields none.
|
||||
if isinstance(as_utf8, dict):
|
||||
return _Reading(as_utf8, None)
|
||||
for encoding in (codepage, "latin-1", *_DOUBLE_BYTE_ENCODINGS):
|
||||
if not encoding:
|
||||
continue
|
||||
as_legacy = _parse(raw, encoding)
|
||||
if as_legacy is not None:
|
||||
return _Reading(as_utf8, as_legacy)
|
||||
return _Reading(as_utf8, None)
|
||||
|
||||
|
||||
class _Scan(NamedTuple):
|
||||
"""What a pass over an existing shard established about it."""
|
||||
|
||||
legacy: bool # enough evidence to trust the codepage reading's keys
|
||||
readable: bool
|
||||
saw_non_ascii: bool # some line's meaning depends on the encoding
|
||||
utf8_keys: set # keys from lines UTF-8 could read
|
||||
legacy_keys: set # keys only the codepage reading yields
|
||||
|
||||
|
||||
class StateStore:
|
||||
|
|
@ -18,12 +101,19 @@ class StateStore:
|
|||
self.path.parent.mkdir(parents = True, exist_ok = True)
|
||||
self._lock = threading.Lock()
|
||||
self._data: Dict[str, Any] = {}
|
||||
# Read whole, and UTF-8 only unlike the shards below: a checkpoint holds
|
||||
# nothing but base64 cursors and booleans, so a codepage retry could only ever
|
||||
# add non-ASCII. That would resume on a mojibaked cursor, which GitHub rejects
|
||||
# with INVALID_CURSOR_ARGUMENTS, and the empty page it returns marks the stream
|
||||
# done and skips the rest for good. Dropping a damaged checkpoint re-scrapes
|
||||
# from the first page, which the writers dedup.
|
||||
if self.path.exists():
|
||||
try:
|
||||
with self.path.open(encoding = "utf-8") as f:
|
||||
self._data = json.load(f)
|
||||
except Exception:
|
||||
self._data = {}
|
||||
raw = self.path.read_bytes()
|
||||
except OSError:
|
||||
raw = b""
|
||||
data = _parse(raw, "utf-8")
|
||||
self._data = data if isinstance(data, dict) else {}
|
||||
|
||||
def get(
|
||||
self,
|
||||
|
|
@ -63,24 +153,83 @@ class JsonlWriter:
|
|||
self.path = Path(path)
|
||||
self.path.parent.mkdir(parents = True, exist_ok = True)
|
||||
self._lock = threading.Lock()
|
||||
self._fh = self.path.open("a", buffering = 1, encoding = "utf-8")
|
||||
self._count_seen_keys: set[str] = set()
|
||||
# Preload seen keys for dedup across resumes
|
||||
self._codepage = _locale_encoding()
|
||||
self._ensure_ascii = False
|
||||
encoding = "utf-8"
|
||||
if self.path.exists() and self.path.stat().st_size > 0:
|
||||
try:
|
||||
# No guess is safe for a file an older build wrote in the
|
||||
# operator's locale, so read past whatever will not decode.
|
||||
with self.path.open(encoding = "utf-8", errors = "replace") as f:
|
||||
for line in f:
|
||||
try:
|
||||
obj = json.loads(line)
|
||||
k = self._key(obj)
|
||||
if k is not None:
|
||||
self._count_seen_keys.add(k)
|
||||
except Exception:
|
||||
pass
|
||||
except Exception:
|
||||
pass
|
||||
scan = self._scan_existing()
|
||||
self._count_seen_keys = scan.utf8_keys
|
||||
if scan.legacy:
|
||||
self._count_seen_keys |= scan.legacy_keys
|
||||
if scan.saw_non_ascii or not scan.readable:
|
||||
# Never convert: the writing encoding is unrecoverable and guessing
|
||||
# mojibakes the records. Pure ASCII appends store identically under
|
||||
# every codepage, and json.loads turns the \uXXXX escapes back.
|
||||
encoding = "ascii"
|
||||
self._ensure_ascii = True
|
||||
self._fh = self.path.open("a", buffering = 1, encoding = encoding, errors = "strict")
|
||||
|
||||
def _scan_existing(self) -> _Scan:
|
||||
"""Read the shard once to recover dedup keys and judge its encoding.
|
||||
|
||||
Line by line: these shards reach gigabytes on a large scrape, so neither
|
||||
the bytes nor the decoded text are held whole.
|
||||
|
||||
The verdict weighs the whole file. Each line with non-ASCII bytes votes:
|
||||
one that parses only under the codepage is evidence of a legacy shard,
|
||||
one that parses as UTF-8 is evidence against, since arbitrary codepage
|
||||
text almost never forms valid multibyte UTF-8. A single corrupt byte in
|
||||
a healthy shard therefore cannot outvote the records around it, and a
|
||||
genuinely legacy shard has a legacy vote on every line that carries an
|
||||
umlaut.
|
||||
|
||||
More than one such line is required, because a single one is genuinely
|
||||
undecidable: a legacy record holding one accented character and an ASCII
|
||||
record holding one stray byte are the same shape. Reading it as damage
|
||||
risks a duplicate; reading it as legacy marks an unreadable record seen
|
||||
and blocks the retry that would replace it, losing it for good. Only one
|
||||
of those is recoverable.
|
||||
|
||||
The verdict only picks which reading supplies the dedup keys. The file
|
||||
itself is never rewritten either way, so a wrong answer costs at most a
|
||||
duplicate, never a corrupted record.
|
||||
"""
|
||||
legacy_votes = 0
|
||||
utf8_votes = 0
|
||||
saw_non_ascii = False
|
||||
utf8_keys: set[str] = set()
|
||||
legacy_keys: set[str] = set()
|
||||
try:
|
||||
with self.path.open("rb") as handle:
|
||||
for raw in handle:
|
||||
line = raw.strip()
|
||||
reading = _read_line(line, self._codepage)
|
||||
# ASCII reads the same everywhere: no vote, no constraint.
|
||||
if not line.isascii():
|
||||
saw_non_ascii = True
|
||||
if reading.as_utf8 is None and reading.as_legacy is not None:
|
||||
legacy_votes += 1
|
||||
elif reading.as_utf8 is not None:
|
||||
utf8_votes += 1
|
||||
# Kept apart so a damaged line does not block its own retry.
|
||||
if isinstance(reading.as_utf8, dict):
|
||||
key = self._key(reading.as_utf8)
|
||||
if key is not None:
|
||||
utf8_keys.add(key)
|
||||
elif isinstance(reading.as_legacy, dict):
|
||||
key = self._key(reading.as_legacy)
|
||||
if key is not None:
|
||||
legacy_keys.add(key)
|
||||
except OSError:
|
||||
return _Scan(False, False, False, utf8_keys, legacy_keys)
|
||||
return _Scan(
|
||||
legacy_votes > 1 and legacy_votes > utf8_votes,
|
||||
True,
|
||||
saw_non_ascii,
|
||||
utf8_keys,
|
||||
legacy_keys,
|
||||
)
|
||||
|
||||
def _key(self, obj: dict) -> str | None:
|
||||
for k in ("id", "node_id", "number", "sha", "url"):
|
||||
|
|
@ -99,7 +248,7 @@ class JsonlWriter:
|
|||
return False
|
||||
if k is not None:
|
||||
self._count_seen_keys.add(k)
|
||||
self._fh.write(json.dumps(obj, default = str, ensure_ascii = False))
|
||||
self._fh.write(json.dumps(obj, default = str, ensure_ascii = self._ensure_ascii))
|
||||
self._fh.write("\n")
|
||||
self._fh.flush()
|
||||
return True
|
||||
|
|
|
|||
|
|
@ -30,6 +30,8 @@ class UnstructuredSeedReader(SeedReader[UnstructuredSeedSource]):
|
|||
meta = json_mod.loads(meta_path.read_text(encoding = "utf-8"))
|
||||
orig_name = meta.get("original_filename", path_obj.name)
|
||||
except (json_mod.JSONDecodeError, OSError, UnicodeDecodeError):
|
||||
# Undecodable metadata is as malformed as invalid JSON, so
|
||||
# fall back to the file's own name rather than abort the seed.
|
||||
pass
|
||||
file_entries.append((path_obj, orig_name))
|
||||
|
||||
|
|
|
|||
|
|
@ -4434,7 +4434,7 @@ def _effective_load_in_4bit(config: ModelConfig, requested: bool) -> bool:
|
|||
if not adapter_cfg_path.exists():
|
||||
return load_in_4bit
|
||||
try:
|
||||
with open(adapter_cfg_path, encoding = "utf-8") as f:
|
||||
with open(adapter_cfg_path, encoding = "utf-8-sig") as f:
|
||||
adapter_cfg = json.load(f)
|
||||
if not isinstance(adapter_cfg, dict): # malformed -> keep requested
|
||||
return load_in_4bit
|
||||
|
|
|
|||
|
|
@ -722,7 +722,7 @@ def _scan_ollama_dir(ollama_dir: Path, limit: Optional[int] = None) -> List[Loca
|
|||
stem_hash = hashlib.sha256(manifest_key.encode()).hexdigest()[:10]
|
||||
|
||||
try:
|
||||
manifest = json.loads(tag_file.read_text(encoding = "utf-8"))
|
||||
manifest = json.loads(tag_file.read_text(encoding = "utf-8-sig"))
|
||||
except (json.JSONDecodeError, OSError, UnicodeDecodeError) as e:
|
||||
logger.debug(
|
||||
"Skipping unreadable/invalid Ollama manifest %s: %s",
|
||||
|
|
@ -738,7 +738,7 @@ def _scan_ollama_dir(ollama_dir: Path, limit: Optional[int] = None) -> List[Loca
|
|||
config_blob = blobs_dir / config_digest.replace(":", "-")
|
||||
if config_blob.is_file():
|
||||
try:
|
||||
cfg = json.loads(config_blob.read_text(encoding = "utf-8"))
|
||||
cfg = json.loads(config_blob.read_text(encoding = "utf-8-sig"))
|
||||
model_type = cfg.get("model_type", "")
|
||||
file_type = cfg.get("file_type", "")
|
||||
except (json.JSONDecodeError, OSError, UnicodeDecodeError) as e:
|
||||
|
|
@ -1042,7 +1042,7 @@ def _dir_has_downloaded_model(directory: Path, max_entries: int = 4000) -> bool:
|
|||
if not m.is_file():
|
||||
continue
|
||||
try:
|
||||
manifest = json.loads(m.read_text(encoding = "utf-8"))
|
||||
manifest = json.loads(m.read_text(encoding = "utf-8-sig"))
|
||||
except (json.JSONDecodeError, OSError, ValueError):
|
||||
continue
|
||||
for layer in manifest.get("layers") or []:
|
||||
|
|
@ -3360,6 +3360,8 @@ def _wsl_reveal_in_explorer(path: Path) -> bool:
|
|||
["wslpath", "-w", str(path)],
|
||||
capture_output = True,
|
||||
text = True,
|
||||
encoding = "utf-8",
|
||||
errors = "replace",
|
||||
check = True,
|
||||
timeout = 10,
|
||||
).stdout.strip()
|
||||
|
|
|
|||
|
|
@ -786,6 +786,8 @@ def _remove_pid_file():
|
|||
stored = _PID_FILE.read_text(encoding = "utf-8").strip()
|
||||
if stored == str(os.getpid()):
|
||||
_PID_FILE.unlink(missing_ok = True)
|
||||
# Runs first in _graceful_shutdown: a corrupt PID file raising here would
|
||||
# abandon the children the rest of that function exists to kill.
|
||||
except (OSError, UnicodeDecodeError):
|
||||
pass
|
||||
|
||||
|
|
|
|||
195
studio/backend/tests/test_chat_text_encoding.py
Normal file
195
studio/backend/tests/test_chat_text_encoding.py
Normal file
|
|
@ -0,0 +1,195 @@
|
|||
# SPDX-License-Identifier: AGPL-3.0-only
|
||||
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
|
||||
|
||||
"""Model text stays intact when it carries non-ASCII.
|
||||
|
||||
``open()`` and ``Path.read_text()`` fall back to ``locale.getencoding()`` when
|
||||
no ``encoding`` is passed. On Windows that is the ANSI codepage, not UTF-8, so
|
||||
a chat template or model config holding ``ä ö ü → 世`` mojibakes or raises
|
||||
``UnicodeDecodeError``. These files are UTF-8, so the reads must say so.
|
||||
|
||||
Each fixture writes raw UTF-8 (``ensure_ascii = False``), matching what
|
||||
Hugging Face actually ships, rather than ASCII ``\\uXXXX`` escapes.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import subprocess
|
||||
import sys
|
||||
import textwrap
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
BACKEND_ROOT = Path(__file__).resolve().parent.parent
|
||||
|
||||
|
||||
def test_config_json_round_trips_non_ascii(tmp_path: Path) -> None:
|
||||
from utils import transformers_version
|
||||
|
||||
name = "Modell für Grüße 世界"
|
||||
(tmp_path / "config.json").write_text(
|
||||
json.dumps({"model_type": "llama", "_name_or_path": name}, ensure_ascii = False),
|
||||
encoding = "utf-8",
|
||||
)
|
||||
transformers_version._config_json_cache.clear()
|
||||
|
||||
cfg = transformers_version._load_config_json(str(tmp_path))
|
||||
|
||||
assert cfg is not None
|
||||
assert cfg["_name_or_path"] == name
|
||||
|
||||
|
||||
def test_tokenizer_config_round_trips_non_ascii_chat_template(tmp_path: Path) -> None:
|
||||
"""Chat templates commonly hold ``→`` and smart quotes, which cp1252 mangles."""
|
||||
from utils import transformers_version
|
||||
|
||||
template = "{{ '→ Grüße 世界' }}"
|
||||
(tmp_path / "tokenizer_config.json").write_text(
|
||||
json.dumps(
|
||||
{"tokenizer_class": "TokenizersBackend", "chat_template": template},
|
||||
ensure_ascii = False,
|
||||
),
|
||||
encoding = "utf-8",
|
||||
)
|
||||
transformers_version._tokenizer_class_cache.clear()
|
||||
|
||||
assert transformers_version._check_tokenizer_config_needs_v5(str(tmp_path)) is True
|
||||
|
||||
|
||||
def test_config_json_survives_a_utf8_bom(tmp_path: Path) -> None:
|
||||
"""Notepad wrote "UTF-8 with BOM" by default for years, so hand-edited
|
||||
configs on Windows carry one. Plain utf-8 keeps the BOM and json.load then
|
||||
fails on it; utf-8-sig strips it and is identical otherwise."""
|
||||
from utils import transformers_version
|
||||
|
||||
name = "Grüße 世界"
|
||||
(tmp_path / "config.json").write_text(
|
||||
json.dumps({"model_type": "llama", "_name_or_path": name}, ensure_ascii = False),
|
||||
encoding = "utf-8-sig",
|
||||
)
|
||||
transformers_version._config_json_cache.clear()
|
||||
|
||||
cfg = transformers_version._load_config_json(str(tmp_path))
|
||||
|
||||
assert cfg is not None
|
||||
assert cfg["_name_or_path"] == name
|
||||
|
||||
|
||||
def test_remote_code_scan_reads_non_ascii_sources(tmp_path: Path) -> None:
|
||||
"""A German Windows profile also puts umlauts in the model sources scanned."""
|
||||
from utils.security import remote_code_scan
|
||||
|
||||
source = "# Grüße über Öl\nVALUE = '世界'\n"
|
||||
# newline = "" pins the bytes on disk, so Windows line end translation cannot make the
|
||||
# read back differ by \r. open() because Path.write_text() only grew newline in 3.10.
|
||||
with open(
|
||||
tmp_path / "modeling_custom.py",
|
||||
"w",
|
||||
encoding = "utf-8",
|
||||
newline = "",
|
||||
) as handle:
|
||||
handle.write(source)
|
||||
|
||||
files = remote_code_scan.repo_remote_code_files(str(tmp_path))
|
||||
|
||||
assert files["modeling_custom.py"] == source
|
||||
|
||||
|
||||
def test_model_config_reads_do_not_rely_on_the_locale_encoding(tmp_path: Path) -> None:
|
||||
"""The reads above pass anywhere the locale is already UTF-8, which hides
|
||||
the Windows bug on Linux and macOS. ``-X warn_default_encoding`` makes
|
||||
CPython flag any text I/O that falls back to the locale, so this fails on
|
||||
every platform if an ``encoding`` argument goes missing again."""
|
||||
# The readers swallow exceptions, so record the warnings instead of raising.
|
||||
script = textwrap.dedent(
|
||||
f"""
|
||||
import sys, warnings
|
||||
sys.path.insert(0, {str(BACKEND_ROOT)!r})
|
||||
from utils import transformers_version
|
||||
|
||||
target = {str(tmp_path)!r}
|
||||
with warnings.catch_warnings(record = True) as caught:
|
||||
warnings.simplefilter("always")
|
||||
transformers_version._config_json_cache.clear()
|
||||
transformers_version._tokenizer_class_cache.clear()
|
||||
assert transformers_version._load_config_json(target) is not None
|
||||
assert transformers_version._check_tokenizer_config_needs_v5(target) is True
|
||||
|
||||
missing = [str(w.message) for w in caught if w.category is EncodingWarning]
|
||||
if missing:
|
||||
sys.exit("text I/O fell back to the locale encoding: " + "; ".join(missing))
|
||||
"""
|
||||
)
|
||||
for name, payload in (
|
||||
("config.json", {"model_type": "llama", "_name_or_path": "Grüße"}),
|
||||
("tokenizer_config.json", {"tokenizer_class": "TokenizersBackend"}),
|
||||
):
|
||||
(tmp_path / name).write_text(json.dumps(payload, ensure_ascii = False), encoding = "utf-8")
|
||||
|
||||
result = subprocess.run(
|
||||
[sys.executable, "-X", "warn_default_encoding", "-c", script],
|
||||
capture_output = True,
|
||||
text = True,
|
||||
encoding = "utf-8",
|
||||
errors = "replace",
|
||||
timeout = 120,
|
||||
)
|
||||
|
||||
assert result.returncode == 0, result.stderr
|
||||
|
||||
|
||||
def test_utf8_child_env_round_trips_non_ascii(tmp_path: Path) -> None:
|
||||
"""A Python child encodes stdout with its locale unless told otherwise, so
|
||||
reading its pipe as utf-8 needs the child told to emit utf-8."""
|
||||
from utils.child_stdio import utf8_child_env
|
||||
|
||||
payload = "Grüße über Öl → 世界"
|
||||
child = tmp_path / "child.py"
|
||||
child.write_text("import sys\nsys.stdout.write(" + repr(payload) + ")\n", encoding = "utf-8")
|
||||
|
||||
env = utf8_child_env()
|
||||
assert env["PYTHONIOENCODING"] == "utf-8"
|
||||
|
||||
proc = subprocess.run(
|
||||
[sys.executable, str(child)],
|
||||
capture_output = True,
|
||||
text = True,
|
||||
encoding = "utf-8",
|
||||
errors = "replace",
|
||||
env = env,
|
||||
timeout = 120,
|
||||
)
|
||||
|
||||
assert proc.returncode == 0, proc.stderr
|
||||
assert proc.stdout == payload
|
||||
|
||||
|
||||
def test_python_children_are_told_to_emit_utf8() -> None:
|
||||
"""Any child we decode as utf-8 must also be told to write utf-8, or a
|
||||
cp1252 console silently mangles what it prints."""
|
||||
import ast
|
||||
|
||||
offenders: list[str] = []
|
||||
for path in sorted(BACKEND_ROOT.rglob("*.py")):
|
||||
parts = path.relative_to(BACKEND_ROOT).parts
|
||||
if any(p in ("tests", "node_modules", "plugins", "__pycache__") for p in parts):
|
||||
continue
|
||||
source = path.read_text(encoding = "utf-8")
|
||||
for node in ast.walk(ast.parse(source, filename = str(path))):
|
||||
if not isinstance(node, ast.Call):
|
||||
continue
|
||||
func = node.func
|
||||
if not (isinstance(func, ast.Attribute) and func.attr in ("run", "Popen")):
|
||||
continue
|
||||
segment = ast.get_source_segment(source, node) or ""
|
||||
if "sys.executable" not in segment or 'encoding = "utf-8"' not in segment:
|
||||
continue
|
||||
if "utf8_child_env" in segment or "PYTHONIOENCODING" in segment:
|
||||
continue
|
||||
offenders.append(f"{path.name}:{node.lineno}")
|
||||
|
||||
assert not offenders, (
|
||||
"these spawn a Python child and decode it as utf-8 without setting the "
|
||||
"child's own stdio encoding; wrap env in utf8_child_env():\n " + "\n ".join(offenders)
|
||||
)
|
||||
|
|
@ -45,8 +45,20 @@ def _build_structlog_stub():
|
|||
_maybe_stub("loggers", _build_loggers_stub)
|
||||
_maybe_stub("structlog", _build_structlog_stub)
|
||||
|
||||
import pytest
|
||||
|
||||
import utils.hardware.hardware as hw # noqa: E402
|
||||
|
||||
# The DRM/KFD readers below are Linux-only in production: _rocm_linux_amdgpu_cards and
|
||||
# _rocm_linux_sysfs_vram_by_pci_gb return early unless platform.system() is "Linux", and
|
||||
# _rocm_kfd_gpu_pci_ids only ever globs /sys/class/kfd. Their fake sysfs tree needs PCI
|
||||
# addresses like "0000:00:02.0" as directory names and POSIX separators in the paths the
|
||||
# readers match; Windows permits neither, so the tree cannot be represented there.
|
||||
linux_only = pytest.mark.skipif(
|
||||
not sys.platform.startswith("linux"),
|
||||
reason = "covers Linux-only DRM/KFD sysfs parsing driven by a fake /sys tree",
|
||||
)
|
||||
|
||||
|
||||
def _device(
|
||||
index,
|
||||
|
|
@ -99,6 +111,7 @@ def _fake_drm(tmp_path, monkeypatch, cards):
|
|||
return card_paths
|
||||
|
||||
|
||||
@linux_only
|
||||
def test_linux_vram_keyed_by_pci_excludes_foreign_adapters(monkeypatch, tmp_path):
|
||||
# Foreign (non-amdgpu) adapters contribute no entry, so they cannot shift ordinals.
|
||||
monkeypatch.setattr(hw.platform, "system", lambda: "Linux")
|
||||
|
|
@ -117,6 +130,7 @@ def test_linux_vram_keyed_by_pci_excludes_foreign_adapters(monkeypatch, tmp_path
|
|||
}
|
||||
|
||||
|
||||
@linux_only
|
||||
def test_linux_vram_omits_bad_cards_without_shifting(monkeypatch, tmp_path):
|
||||
# A zero-total card has no entry; identity keying means its absence renumbers nothing.
|
||||
monkeypatch.setattr(hw.platform, "system", lambda: "Linux")
|
||||
|
|
@ -131,6 +145,7 @@ def test_linux_vram_omits_bad_cards_without_shifting(monkeypatch, tmp_path):
|
|||
assert hw._rocm_linux_sysfs_vram_by_pci_gb() == {"0000:41:00.0": (2.0, 16.0)}
|
||||
|
||||
|
||||
@linux_only
|
||||
def test_linux_vram_omits_amd_card_without_vram_files(monkeypatch, tmp_path):
|
||||
# An APU with no mem_info_vram_* files has no entry; the discrete card keeps its address.
|
||||
monkeypatch.setattr(hw.platform, "system", lambda: "Linux")
|
||||
|
|
@ -174,6 +189,7 @@ def _fake_kfd(tmp_path, monkeypatch, nodes):
|
|||
return node_paths
|
||||
|
||||
|
||||
@linux_only
|
||||
def test_kfd_lists_gpu_nodes_in_device_order(monkeypatch, tmp_path):
|
||||
# The CPU node (simd_count 0) takes no ordinal; GPU nodes in node-id order are HIP's order.
|
||||
monkeypatch.setattr(hw.platform, "system", lambda: "Linux")
|
||||
|
|
@ -189,12 +205,14 @@ def test_kfd_lists_gpu_nodes_in_device_order(monkeypatch, tmp_path):
|
|||
assert hw._rocm_kfd_gpu_pci_ids() == ["0000:03:00.0", "0000:41:00.0"]
|
||||
|
||||
|
||||
@linux_only
|
||||
def test_kfd_decodes_domain_device_and_function(monkeypatch, tmp_path):
|
||||
monkeypatch.setattr(hw.platform, "system", lambda: "Linux")
|
||||
_fake_kfd(tmp_path, monkeypatch, [(1, 64, (0xC1 << 8) | (0x1F << 3) | 5, 0x1234, _AMD)])
|
||||
assert hw._rocm_kfd_gpu_pci_ids() == ["1234:c1:1f.5"]
|
||||
|
||||
|
||||
@linux_only
|
||||
def test_kfd_skips_non_amd_gpu_nodes(monkeypatch, tmp_path):
|
||||
# An NVIDIA KFD node is not a HIP device: it must take no ordinal, else it
|
||||
# shifts every AMD GPU and ROCm device 1 resolves to AMD GPU 0.
|
||||
|
|
@ -212,6 +230,7 @@ def test_kfd_skips_non_amd_gpu_nodes(monkeypatch, tmp_path):
|
|||
assert hw._rocm_kfd_gpu_pci_ids() == ["0000:03:00.0", "0000:41:00.0"]
|
||||
|
||||
|
||||
@linux_only
|
||||
def test_kfd_fails_closed_when_a_gpu_has_no_location(monkeypatch, tmp_path):
|
||||
# Dropping an unplaceable AMD GPU shifts later ordinals; fail closed for the whole map.
|
||||
monkeypatch.setattr(hw.platform, "system", lambda: "Linux")
|
||||
|
|
@ -226,6 +245,7 @@ def test_kfd_fails_closed_when_a_gpu_has_no_location(monkeypatch, tmp_path):
|
|||
assert hw._rocm_kfd_gpu_pci_ids() == []
|
||||
|
||||
|
||||
@linux_only
|
||||
def test_kfd_fails_closed_when_a_node_is_unreadable(monkeypatch, tmp_path):
|
||||
# An unreadable node could be a GPU; assuming otherwise would shift ordinals.
|
||||
monkeypatch.setattr(hw.platform, "system", lambda: "Linux")
|
||||
|
|
@ -241,6 +261,23 @@ def test_kfd_fails_closed_when_a_node_is_unreadable(monkeypatch, tmp_path):
|
|||
assert hw._rocm_kfd_gpu_pci_ids() == []
|
||||
|
||||
|
||||
@linux_only
|
||||
def test_kfd_fails_closed_when_a_node_does_not_decode(monkeypatch, tmp_path):
|
||||
# UnicodeDecodeError is a ValueError, so it slips past `except OSError` and
|
||||
# would shift every later HIP ordinal.
|
||||
monkeypatch.setattr(hw.platform, "system", lambda: "Linux")
|
||||
paths = _fake_kfd(
|
||||
tmp_path,
|
||||
monkeypatch,
|
||||
[
|
||||
(1, 304, (0x03 << 8) | 0, 0, _AMD),
|
||||
(2, 304, (0x41 << 8) | 0, 0, _AMD),
|
||||
],
|
||||
)
|
||||
(Path(paths[0]) / "properties").write_bytes(b"simd_count 304\nvendor_id \x80\xff\n")
|
||||
assert hw._rocm_kfd_gpu_pci_ids() == []
|
||||
|
||||
|
||||
def test_kfd_absent_yields_no_device_order(monkeypatch):
|
||||
monkeypatch.setattr(hw.glob, "glob", lambda pattern: [])
|
||||
assert hw._rocm_kfd_gpu_pci_ids() == []
|
||||
|
|
@ -422,6 +459,10 @@ def test_visible_utilization_rocm_fallback_overlays(monkeypatch):
|
|||
):
|
||||
monkeypatch.delenv(_var, raising = False)
|
||||
monkeypatch.setattr(hw, "IS_ROCM", True)
|
||||
# No AMD adapter data on this host. On Windows this branch runs ahead of the torch
|
||||
# fallback under test, and probing it imports torch, which the CI runner does not
|
||||
# install. Off Windows the real function is never reached, so this changes nothing.
|
||||
monkeypatch.setattr(hw, "_rocm_windows_per_device_vram", lambda ids: [])
|
||||
monkeypatch.setattr(hw, "get_device", lambda: hw.DeviceType.CUDA)
|
||||
monkeypatch.setattr(hw, "_smi_query", lambda *a, **k: None) # amd-smi unavailable
|
||||
monkeypatch.setattr(
|
||||
|
|
@ -450,6 +491,10 @@ def test_visible_utilization_rocm_fallback_overlays(monkeypatch):
|
|||
def test_visible_utilization_relative_index_skips_overlay(monkeypatch):
|
||||
# UUID/MIG mask gives relative indices; the overlay matches physical index, so it must not run.
|
||||
monkeypatch.setattr(hw, "IS_ROCM", True)
|
||||
# No AMD adapter data on this host. On Windows this branch runs ahead of the torch
|
||||
# fallback under test, and probing it imports torch, which the CI runner does not
|
||||
# install. Off Windows the real function is never reached, so this changes nothing.
|
||||
monkeypatch.setattr(hw, "_rocm_windows_per_device_vram", lambda ids: [])
|
||||
monkeypatch.setattr(hw, "get_device", lambda: hw.DeviceType.CUDA)
|
||||
monkeypatch.setattr(hw, "_smi_query", lambda *a, **k: None)
|
||||
monkeypatch.setattr(
|
||||
|
|
|
|||
809
studio/backend/tests/test_text_io_encoding.py
Normal file
809
studio/backend/tests/test_text_io_encoding.py
Normal file
|
|
@ -0,0 +1,809 @@
|
|||
# SPDX-License-Identifier: AGPL-3.0-only
|
||||
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
|
||||
|
||||
"""Text I/O must name its encoding, or Windows silently uses the ANSI codepage.
|
||||
|
||||
``open()``, ``Path.read_text()`` and ``subprocess(text = True)`` fall back to
|
||||
``locale.getencoding()`` when no ``encoding`` is passed. On Windows that is
|
||||
cp1252 (or cp932, cp1251, ... by system locale), not UTF-8, so a chat template,
|
||||
model config or path containing ``ä ö ü → 世`` mojibakes or raises
|
||||
``UnicodeDecodeError`` mid-load. Studio's files are UTF-8, so say so.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import ast
|
||||
import importlib.util
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
BACKEND_ROOT = Path(__file__).resolve().parent.parent
|
||||
|
||||
# Not runtime source. Shipped plugins under plugins/*/src are, so only builds are skipped.
|
||||
_SKIPPED_DIRS = ("node_modules", "build", "tests", "__pycache__")
|
||||
|
||||
# Path.open()'s signature is what tells it apart from other libraries' open(),
|
||||
# e.g. fitz.open(stream=...) and av.open(..., metadata_errors=...).
|
||||
_FILE_MODE_CHARS = set("rwxabt+")
|
||||
_PATH_OPEN_ARGS = ("mode", "buffering", "encoding", "errors", "newline")
|
||||
_PATH_OPEN_KWARGS = set(_PATH_OPEN_ARGS)
|
||||
_PATH_OPEN_ENCODING_ARG = _PATH_OPEN_ARGS.index("encoding")
|
||||
|
||||
_SUBPROCESS_CALLS = {"run", "Popen", "check_output", "check_call", "call"}
|
||||
|
||||
# open(file, mode, buffering, encoding, ...), and os.fdopen forwards the same
|
||||
# signature with a descriptor in place of the path.
|
||||
_OPEN_ENCODING_ARG = 3
|
||||
|
||||
|
||||
def _studio_sources() -> list[Path]:
|
||||
return [
|
||||
path
|
||||
for path in sorted(BACKEND_ROOT.rglob("*.py"))
|
||||
if not any(part in _SKIPPED_DIRS for part in path.relative_to(BACKEND_ROOT).parts)
|
||||
]
|
||||
|
||||
|
||||
def _has_keyword(node: ast.Call, name: str) -> bool:
|
||||
return any(keyword.arg == name for keyword in node.keywords)
|
||||
|
||||
|
||||
def _mode_is_binary(node: ast.Call) -> bool:
|
||||
mode: str | None = None
|
||||
if len(node.args) >= 2 and isinstance(node.args[1], ast.Constant):
|
||||
value = node.args[1].value
|
||||
mode = value if isinstance(value, str) else None
|
||||
for keyword in node.keywords:
|
||||
if keyword.arg == "mode" and isinstance(keyword.value, ast.Constant):
|
||||
value = keyword.value.value
|
||||
if isinstance(value, str):
|
||||
mode = value
|
||||
return bool(mode and "b" in mode)
|
||||
|
||||
|
||||
def _open_has_encoding(node: ast.Call) -> bool:
|
||||
"""open()/os.fdopen() also take encoding positionally: open(p, "w", 1, "utf-8")."""
|
||||
return _has_keyword(node, "encoding") or len(node.args) > _OPEN_ENCODING_ARG
|
||||
|
||||
|
||||
def _path_open_mode(node: ast.Call) -> str | None:
|
||||
if node.args and isinstance(node.args[0], ast.Constant):
|
||||
value = node.args[0].value
|
||||
if isinstance(value, str):
|
||||
return value
|
||||
for keyword in node.keywords:
|
||||
if keyword.arg == "mode" and isinstance(keyword.value, ast.Constant):
|
||||
value = keyword.value.value
|
||||
if isinstance(value, str):
|
||||
return value
|
||||
return None
|
||||
|
||||
|
||||
def _is_path_open(node: ast.Call) -> bool:
|
||||
"""True only for calls matching ``Path.open``'s signature."""
|
||||
if len(node.args) > len(_PATH_OPEN_ARGS):
|
||||
return False
|
||||
if any(k.arg not in _PATH_OPEN_KWARGS for k in node.keywords):
|
||||
return False
|
||||
mode = _path_open_mode(node)
|
||||
if mode is not None:
|
||||
return bool(mode) and set(mode) <= _FILE_MODE_CHARS
|
||||
return not node.args
|
||||
|
||||
|
||||
def _path_open_has_encoding(node: ast.Call) -> bool:
|
||||
"""Path.open() also takes encoding positionally: open("w", 1, "utf-8")."""
|
||||
return _has_keyword(node, "encoding") or len(node.args) > _PATH_OPEN_ENCODING_ARG
|
||||
|
||||
|
||||
def _call_name(node: ast.Call) -> str | None:
|
||||
func = node.func
|
||||
if isinstance(func, ast.Name):
|
||||
return func.id
|
||||
if isinstance(func, ast.Attribute):
|
||||
return func.attr
|
||||
return None
|
||||
|
||||
|
||||
def _subprocess_names(tree: ast.AST) -> set[str]:
|
||||
"""Names subprocess is reachable under here, e.g. `import subprocess as _sp`."""
|
||||
names = set()
|
||||
for node in ast.walk(tree):
|
||||
if isinstance(node, ast.Import):
|
||||
for alias in node.names:
|
||||
if alias.name == "subprocess":
|
||||
names.add(alias.asname or alias.name)
|
||||
return names
|
||||
|
||||
|
||||
def _subprocess_aliases(tree: ast.AST, names: set[str]) -> set[str]:
|
||||
"""Plain names bound to a subprocess callable, called without the module.
|
||||
|
||||
``install_wheel(run = subprocess.run)`` calls its injected ``run`` as a bare
|
||||
name, so matching only the attribute form leaves those installer calls
|
||||
unguarded. Imports, assignments and parameter defaults all bind one.
|
||||
"""
|
||||
|
||||
def _is_bound(value: ast.expr | None) -> bool:
|
||||
return (
|
||||
isinstance(value, ast.Attribute)
|
||||
and value.attr in _SUBPROCESS_CALLS
|
||||
and isinstance(value.value, ast.Name)
|
||||
and value.value.id in names
|
||||
)
|
||||
|
||||
aliases: set[str] = set()
|
||||
for node in ast.walk(tree):
|
||||
if isinstance(node, ast.ImportFrom) and node.module == "subprocess":
|
||||
aliases.update(a.asname or a.name for a in node.names if a.name in _SUBPROCESS_CALLS)
|
||||
elif isinstance(node, ast.Assign) and _is_bound(node.value):
|
||||
aliases.update(t.id for t in node.targets if isinstance(t, ast.Name))
|
||||
elif isinstance(node, ast.AnnAssign) and _is_bound(node.value):
|
||||
if isinstance(node.target, ast.Name):
|
||||
aliases.add(node.target.id)
|
||||
elif isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)):
|
||||
args = node.args
|
||||
positional = args.posonlyargs + args.args
|
||||
# Defaults cover the tail of the positional parameters; kw_defaults
|
||||
# is aligned with kwonlyargs already, holding None where absent.
|
||||
padded = [None] * (len(positional) - len(args.defaults)) + list(args.defaults)
|
||||
pairs = list(zip(positional, padded)) + list(zip(args.kwonlyargs, args.kw_defaults))
|
||||
aliases.update(arg.arg for arg, default in pairs if _is_bound(default))
|
||||
return aliases
|
||||
|
||||
|
||||
def _is_subprocess_call(node: ast.Call, names: set[str], aliases: set[str]) -> bool:
|
||||
func = node.func
|
||||
if isinstance(func, ast.Name):
|
||||
return func.id in aliases
|
||||
if not isinstance(func, ast.Attribute) or func.attr not in _SUBPROCESS_CALLS:
|
||||
return False
|
||||
value = func.value
|
||||
return isinstance(value, ast.Name) and value.id in names
|
||||
|
||||
|
||||
def _text_mode_subprocess(node: ast.Call) -> bool:
|
||||
for keyword in node.keywords:
|
||||
if keyword.arg not in ("text", "universal_newlines"):
|
||||
continue
|
||||
if isinstance(keyword.value, ast.Constant) and keyword.value.value is True:
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def _text_mode_dict(node: ast.Dict) -> bool:
|
||||
"""A ``{"text": True, ...}`` literal with no "encoding" key."""
|
||||
keys = [k.value for k in node.keys if isinstance(k, ast.Constant)]
|
||||
if "encoding" in keys:
|
||||
return False
|
||||
for key, value in zip(node.keys, node.values):
|
||||
if not isinstance(key, ast.Constant) or key.value not in (
|
||||
"text",
|
||||
"universal_newlines",
|
||||
):
|
||||
continue
|
||||
if isinstance(value, ast.Constant) and value.value is True:
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def _splatted_names(tree: ast.AST) -> set[str]:
|
||||
"""Names handed to a call as ``**name``."""
|
||||
names = set()
|
||||
for node in ast.walk(tree):
|
||||
if isinstance(node, ast.Call):
|
||||
for keyword in node.keywords:
|
||||
if keyword.arg is None and isinstance(keyword.value, ast.Name):
|
||||
names.add(keyword.value.id)
|
||||
return names
|
||||
|
||||
|
||||
def _encoding_assigned_later(tree: ast.AST, name: str) -> bool:
|
||||
"""``name["encoding"] = ...`` somewhere, so the literal need not carry it."""
|
||||
for node in ast.walk(tree):
|
||||
if not isinstance(node, ast.Subscript) or not isinstance(node.ctx, ast.Store):
|
||||
continue
|
||||
target, key = node.value, node.slice
|
||||
if isinstance(target, ast.Name) and target.id == name:
|
||||
if isinstance(key, ast.Constant) and key.value == "encoding":
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def _splatted_kwargs_offenders(tree: ast.AST) -> list[ast.Dict]:
|
||||
"""Text-mode kwargs built in a dict and splatted into a call.
|
||||
|
||||
Kwargs are collected in a dict and splatted (``run(cmd, **run_kwargs)``)
|
||||
where a branch has to add a timeout or an env, and the call is often through
|
||||
a helper, so neither the callee nor the keywords are visible at the call
|
||||
site. Only dicts that reach a call this way are judged: an unrelated payload
|
||||
that happens to carry ``"text": True`` is not subprocess configuration.
|
||||
"""
|
||||
found = []
|
||||
# ``run(cmd, **{...})``: the literal is at the call already.
|
||||
for node in ast.walk(tree):
|
||||
if not isinstance(node, ast.Call):
|
||||
continue
|
||||
for keyword in node.keywords:
|
||||
if keyword.arg is None and isinstance(keyword.value, ast.Dict):
|
||||
if _text_mode_dict(keyword.value):
|
||||
found.append(keyword.value)
|
||||
splatted = _splatted_names(tree)
|
||||
if not splatted:
|
||||
return found
|
||||
for node in ast.walk(tree):
|
||||
targets = []
|
||||
if isinstance(node, ast.Assign):
|
||||
targets = [t for t in node.targets if isinstance(t, ast.Name)]
|
||||
elif isinstance(node, ast.AnnAssign) and isinstance(node.target, ast.Name):
|
||||
targets = [node.target]
|
||||
if not targets or not isinstance(node.value, ast.Dict):
|
||||
continue
|
||||
if not _text_mode_dict(node.value):
|
||||
continue
|
||||
for target in targets:
|
||||
if target.id in splatted and not _encoding_assigned_later(tree, target.id):
|
||||
found.append(node.value)
|
||||
break
|
||||
return found
|
||||
|
||||
|
||||
def _offenders(path: Path) -> list[str]:
|
||||
source = path.read_text(encoding = "utf-8")
|
||||
tree = ast.parse(source, filename = str(path))
|
||||
subprocess_names = _subprocess_names(tree)
|
||||
subprocess_aliases = _subprocess_aliases(tree, subprocess_names)
|
||||
found: list[str] = []
|
||||
for node in _splatted_kwargs_offenders(tree):
|
||||
found.append(
|
||||
f"{path.name}:{node.lineno}: subprocess kwargs with text = True and no encoding"
|
||||
)
|
||||
for node in ast.walk(tree):
|
||||
if not isinstance(node, ast.Call):
|
||||
continue
|
||||
name = _call_name(node)
|
||||
|
||||
if _is_subprocess_call(node, subprocess_names, subprocess_aliases):
|
||||
if _text_mode_subprocess(node) and not _has_keyword(node, "encoding"):
|
||||
found.append(f"{path.name}:{node.lineno}: subprocess(text = True) without encoding")
|
||||
continue
|
||||
|
||||
if name == "open" and isinstance(node.func, ast.Name):
|
||||
if _mode_is_binary(node) or _open_has_encoding(node):
|
||||
continue
|
||||
found.append(f"{path.name}:{node.lineno}: open() without encoding")
|
||||
continue
|
||||
|
||||
# os.fdopen(fd, "w") is open() on a descriptor, so text mode takes the
|
||||
# same locale default. Its mode defaults to "r", i.e. text, like open's.
|
||||
if name == "fdopen":
|
||||
if _mode_is_binary(node) or _open_has_encoding(node):
|
||||
continue
|
||||
found.append(f"{path.name}:{node.lineno}: os.fdopen() without encoding")
|
||||
continue
|
||||
|
||||
if name == "open" and isinstance(node.func, ast.Attribute):
|
||||
if not _is_path_open(node) or _path_open_has_encoding(node):
|
||||
continue
|
||||
if _path_open_mode(node) and "b" in _path_open_mode(node):
|
||||
continue
|
||||
found.append(f"{path.name}:{node.lineno}: Path.open() without encoding")
|
||||
continue
|
||||
|
||||
if name in ("read_text", "write_text") and isinstance(node.func, ast.Attribute):
|
||||
if _has_keyword(node, "encoding"):
|
||||
continue
|
||||
# importlib.metadata Distribution.read_text() takes no encoding kwarg.
|
||||
if isinstance(node.func.value, ast.Name) and node.func.value.id == "dist":
|
||||
continue
|
||||
found.append(f"{path.name}:{node.lineno}: {name}() without encoding")
|
||||
return found
|
||||
|
||||
|
||||
@pytest.mark.parametrize("path", _studio_sources(), ids = lambda p: str(p.name))
|
||||
def test_text_io_names_its_encoding(path: Path) -> None:
|
||||
offenders = _offenders(path)
|
||||
assert not offenders, (
|
||||
"Text I/O without an explicit encoding falls back to the Windows ANSI "
|
||||
'codepage and corrupts non-ASCII (ä ö ü → 世). Pass encoding = "utf-8":\n '
|
||||
+ "\n ".join(offenders)
|
||||
)
|
||||
|
||||
|
||||
_STATE_STORE = (
|
||||
BACKEND_ROOT
|
||||
/ "plugins/data-designer-github-repo-seed/src"
|
||||
/ "data_designer_github_repo_seed/scraper_impl/state_store.py"
|
||||
)
|
||||
|
||||
|
||||
def _load_state_store(codepage: str):
|
||||
"""Load state_store with the writing machine's codepage pinned."""
|
||||
spec = importlib.util.spec_from_file_location(f"state_store_{codepage}", _STATE_STORE)
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(module)
|
||||
module.locale = SimpleNamespace(
|
||||
getencoding = lambda: codepage,
|
||||
getpreferredencoding = lambda _ = True: codepage,
|
||||
)
|
||||
return module
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("codepage", "name"), [("cp1252", "Jürgen"), ("cp1251", "Юрий"), ("cp932", "田中")]
|
||||
)
|
||||
def test_resuming_a_legacy_jsonl_keeps_one_encoding(
|
||||
tmp_path: Path, codepage: str, name: str
|
||||
) -> None:
|
||||
"""A scrape written before UTF-8 was explicit must resume, not duplicate."""
|
||||
path = tmp_path / "out.jsonl"
|
||||
records = [{"id": 1, "author": name}, {"id": 2, "author": name}]
|
||||
body = "".join(json.dumps(r, ensure_ascii = False) + "\n" for r in records)
|
||||
path.write_bytes(body.encode(codepage))
|
||||
before = path.read_bytes()
|
||||
|
||||
writer = _load_state_store(codepage).JsonlWriter(path)
|
||||
try:
|
||||
# Seen keys survive the resume, so a repeat is refused, not appended.
|
||||
assert writer.has("id:1") and writer.has("id:2")
|
||||
assert writer.write(records[0]) is False
|
||||
assert writer.write({"id": 3, "author": name}) is True
|
||||
finally:
|
||||
writer.close()
|
||||
|
||||
# Never converted, so it still reads in its own codepage; the append is ASCII.
|
||||
blob = path.read_bytes()
|
||||
assert blob.startswith(before)
|
||||
assert blob[len(before) :].isascii()
|
||||
lines = [json.loads(x) for x in blob.decode(codepage).splitlines() if x.strip()]
|
||||
assert len(lines) == 3
|
||||
assert [line["author"] for line in lines] == [name] * 3
|
||||
|
||||
|
||||
def test_a_coincidentally_utf8_legacy_line_is_left_alone(tmp_path: Path) -> None:
|
||||
"""cp1251 `Р°` is D0 B0, which is also UTF-8 `а`, and nothing can tell them apart."""
|
||||
path = tmp_path / "out.jsonl"
|
||||
ambiguous = "Р°"
|
||||
assert ambiguous.encode("cp1251").decode("utf-8") == "а" # the trap
|
||||
authors = ["Привет", "Здравствуйте", "Москва", ambiguous]
|
||||
path.write_bytes(
|
||||
b"".join(
|
||||
json.dumps({"id": i, "author": a}, ensure_ascii = False).encode("cp1251") + b"\n"
|
||||
for i, a in enumerate(authors)
|
||||
)
|
||||
)
|
||||
before = path.read_bytes()
|
||||
|
||||
_load_state_store("cp1251").JsonlWriter(path).close()
|
||||
|
||||
# Untouched, so the ambiguity never had to be resolved.
|
||||
assert path.read_bytes() == before
|
||||
rows = [json.loads(x) for x in path.read_text(encoding = "cp1251").splitlines() if x.strip()]
|
||||
assert [row["author"] for row in rows] == authors
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("codepage", "word"), [("cp1251", "Привет"), ("cp932", "こんにちは"), ("cp1252", "Jürgen")]
|
||||
)
|
||||
def test_a_moved_shard_is_not_rewritten_by_guesswork(
|
||||
tmp_path: Path, codepage: str, word: str
|
||||
) -> None:
|
||||
"""Off the writing machine there is no codepage to attribute the file to."""
|
||||
path = tmp_path / "out.jsonl"
|
||||
# Two records: a lone non-UTF-8 line would count as damage, not legacy.
|
||||
path.write_bytes(
|
||||
b"".join(
|
||||
json.dumps({"id": i, "author": word}, ensure_ascii = False).encode(codepage) + b"\n"
|
||||
for i in (1, 4)
|
||||
)
|
||||
)
|
||||
before = path.read_bytes()
|
||||
|
||||
# A UTF-8 host: latin-1 would read cp1251 `Привет` back as `Ïðèâåò`.
|
||||
writer = _load_state_store("utf-8").JsonlWriter(path)
|
||||
try:
|
||||
assert writer.has("id:1") # ASCII keys still recover
|
||||
assert writer.write({"id": 2, "author": "Grüße"}) is True
|
||||
finally:
|
||||
writer.close()
|
||||
|
||||
blob = path.read_bytes()
|
||||
assert blob.startswith(before) # never rewritten
|
||||
assert blob[len(before) :].isascii() # appended as \uXXXX, so no second encoding
|
||||
rows = [json.loads(x) for x in blob.decode(codepage).splitlines() if x.strip()]
|
||||
assert [row["author"] for row in rows] == [word, word, "Grüße"]
|
||||
|
||||
|
||||
def test_an_all_ambiguous_shard_still_gets_ascii_appends(tmp_path: Path) -> None:
|
||||
"""Every line valid under both readings still means the append must not pick one."""
|
||||
path = tmp_path / "out.jsonl"
|
||||
ambiguous = "Р°" # cp1251 D0 B0, also valid UTF-8 for "а"
|
||||
path.write_bytes(
|
||||
b"".join(
|
||||
json.dumps({"id": i, "a": ambiguous}, ensure_ascii = False).encode("cp1251") + b"\n"
|
||||
for i in range(3)
|
||||
)
|
||||
)
|
||||
before = path.read_bytes()
|
||||
|
||||
writer = _load_state_store("cp1251").JsonlWriter(path)
|
||||
try:
|
||||
assert writer.write({"id": 9, "a": "世界"}) is True
|
||||
finally:
|
||||
writer.close()
|
||||
|
||||
blob = path.read_bytes()
|
||||
assert blob.startswith(before)
|
||||
# ASCII, so the appended record survives whichever reading is chosen.
|
||||
assert blob[len(before) :].isascii()
|
||||
for codec in ("cp1251", "utf-8"):
|
||||
rows = [json.loads(x) for x in blob.decode(codec).splitlines() if x.strip()]
|
||||
assert rows[-1]["a"] == "世界"
|
||||
|
||||
|
||||
def test_a_damaged_line_in_an_ascii_shard_does_not_block_its_retry(tmp_path: Path) -> None:
|
||||
"""With no non-ASCII records to outvote it, one damaged line is still damage."""
|
||||
path = tmp_path / "out.jsonl"
|
||||
path.write_bytes(
|
||||
b'{"id": 1, "author": "alice"}\n'
|
||||
+ b'{"id": 99, "author": "bad \x96 byte"}\n'
|
||||
+ b'{"id": 2, "author": "bob"}\n'
|
||||
)
|
||||
|
||||
writer = _load_state_store("cp1252").JsonlWriter(path)
|
||||
try:
|
||||
assert writer.has("id:1") and writer.has("id:2")
|
||||
assert not writer.has("id:99")
|
||||
assert writer.write({"id": 99, "author": "good byte"}) is True
|
||||
finally:
|
||||
writer.close()
|
||||
|
||||
|
||||
def test_a_damaged_line_does_not_block_its_own_retry(tmp_path: Path) -> None:
|
||||
"""Its key comes from the codepage reading, which a UTF-8 shard did not pick."""
|
||||
path = tmp_path / "out.jsonl"
|
||||
path.write_bytes(
|
||||
json.dumps({"id": 1, "author": "Jürgen"}, ensure_ascii = False).encode()
|
||||
+ b"\n"
|
||||
+ b'{"id": 99, "author": "bad \x96 byte"}\n'
|
||||
)
|
||||
|
||||
writer = _load_state_store("cp1252").JsonlWriter(path)
|
||||
try:
|
||||
assert writer.has("id:1")
|
||||
assert not writer.has("id:99")
|
||||
assert writer.write({"id": 99, "author": "good byte"}) is True
|
||||
finally:
|
||||
writer.close()
|
||||
|
||||
|
||||
def test_one_damaged_byte_does_not_relabel_a_utf8_shard(tmp_path: Path) -> None:
|
||||
"""A complete JSON line with a stray 0x96 parses as cp1252, but is only one vote."""
|
||||
path = tmp_path / "out.jsonl"
|
||||
healthy = ["Jürgen", "Grüße", "Björn"]
|
||||
path.write_bytes(
|
||||
json.dumps({"id": 0, "author": healthy[0]}, ensure_ascii = False).encode()
|
||||
+ b"\n"
|
||||
+ b'{"id": 99, "author": "bad \x96 byte"}\n'
|
||||
+ b"".join(
|
||||
json.dumps({"id": i, "author": a}, ensure_ascii = False).encode() + b"\n"
|
||||
for i, a in enumerate(healthy[1:], start = 1)
|
||||
)
|
||||
)
|
||||
before = path.read_bytes()
|
||||
|
||||
_load_state_store("cp1252").JsonlWriter(path).close()
|
||||
|
||||
# Untouched, so the healthy records were never re-read as cp1252.
|
||||
assert path.read_bytes() == before
|
||||
rows = []
|
||||
for line in path.read_bytes().splitlines():
|
||||
try:
|
||||
rows.append(json.loads(line.decode()))
|
||||
except (UnicodeDecodeError, ValueError):
|
||||
continue
|
||||
assert [row["author"] for row in rows] == healthy
|
||||
|
||||
|
||||
def test_a_torn_line_does_not_relabel_a_utf8_shard(tmp_path: Path) -> None:
|
||||
"""One interrupted append must not get the whole shard read as cp1252."""
|
||||
path = tmp_path / "out.jsonl"
|
||||
good = [{"id": 1, "author": "Jürgen"}, {"id": 3, "author": "Grüße"}]
|
||||
torn = '{"id": 2, "author": "Jürgen"}'.encode()[:-6] # cut mid-character
|
||||
path.write_bytes(
|
||||
json.dumps(good[0], ensure_ascii = False).encode()
|
||||
+ b"\n"
|
||||
+ torn
|
||||
+ b"\n"
|
||||
+ json.dumps(good[1], ensure_ascii = False).encode()
|
||||
+ b"\n"
|
||||
)
|
||||
before = path.read_bytes()
|
||||
|
||||
writer = _load_state_store("cp1252").JsonlWriter(path)
|
||||
try:
|
||||
assert writer.has("id:1") and writer.has("id:3")
|
||||
assert not writer.has("id:2") # torn line yields no key
|
||||
finally:
|
||||
writer.close()
|
||||
|
||||
# Untouched: no rewrite, so no record was re-encoded into mojibake.
|
||||
after = path.read_bytes()
|
||||
assert after.startswith(before)
|
||||
assert "Jürgen".encode() in after
|
||||
assert "Jürgen".encode("utf-8").decode("cp1252").encode() not in after
|
||||
|
||||
|
||||
def test_an_undecodable_transport_marker_reads_as_unknown(tmp_path: Path) -> None:
|
||||
"""Pinning the decode turns an undecodable marker into UnicodeDecodeError,
|
||||
which is a ValueError and so is not an OSError. Before the pin those bytes
|
||||
simply read as an unknown value and the caller safely purged and restarted
|
||||
the partial download; letting the error escape aborts the transfer instead.
|
||||
"""
|
||||
import sys
|
||||
|
||||
backend = str(Path(__file__).resolve().parent.parent)
|
||||
if backend not in sys.path:
|
||||
sys.path.insert(0, backend)
|
||||
from hub.utils import download_registry as registry
|
||||
|
||||
marker = tmp_path / ".transport"
|
||||
marker.write_bytes(b"\x80\xffnative\n")
|
||||
assert registry._read_marker_value(marker) is None
|
||||
# A readable but unknown value takes the same path (the behaviour restored).
|
||||
marker.write_text("something-else\n", encoding = "utf-8")
|
||||
assert registry._read_marker_value(marker) is None
|
||||
|
||||
|
||||
def test_a_torn_cache_ref_reads_as_not_cached(tmp_path: Path, monkeypatch) -> None:
|
||||
"""hf_cache_snapshot_dir answers "is this model already on disk", and the
|
||||
offline embedding checks turn a raise into a 500. A refs/main holding a byte
|
||||
the codepage used to decode into a nonsense commit simply missed the snapshot
|
||||
dir before the pin; it has to keep missing it."""
|
||||
import sys
|
||||
|
||||
backend = str(Path(__file__).resolve().parent.parent)
|
||||
if backend not in sys.path:
|
||||
sys.path.insert(0, backend)
|
||||
from utils import utils as backend_utils
|
||||
|
||||
good_root = tmp_path / "good"
|
||||
torn_root = tmp_path / "torn"
|
||||
for root, ref_bytes in ((torn_root, b"\x80\xff\n"), (good_root, b"abc123\n")):
|
||||
repo = root / "models--Org--Model"
|
||||
(repo / "refs").mkdir(parents = True)
|
||||
(repo / "refs" / "main").write_bytes(ref_bytes)
|
||||
(good_root / "models--Org--Model" / "snapshots" / "abc123").mkdir(parents = True)
|
||||
|
||||
monkeypatch.setattr(backend_utils, "_hf_cache_roots", lambda: [torn_root])
|
||||
assert backend_utils.hf_cache_snapshot_dir("Org/Model") is None
|
||||
# The torn root is skipped, not fatal: a healthy second root still answers.
|
||||
monkeypatch.setattr(backend_utils, "_hf_cache_roots", lambda: [torn_root, good_root])
|
||||
found = backend_utils.hf_cache_snapshot_dir("Org/Model")
|
||||
assert found is not None and found.name == "abc123"
|
||||
|
||||
|
||||
def test_a_corrupt_pid_file_does_not_abort_shutdown(tmp_path: Path, monkeypatch) -> None:
|
||||
"""_remove_pid_file runs first in _graceful_shutdown, so a raise there leaves
|
||||
the inference, export, training and tunnel children alive."""
|
||||
import sys
|
||||
|
||||
backend = str(Path(__file__).resolve().parent.parent)
|
||||
if backend not in sys.path:
|
||||
sys.path.insert(0, backend)
|
||||
import run as studio_run
|
||||
|
||||
pid_file = tmp_path / "studio.pid"
|
||||
pid_file.write_bytes(b"\x80\xff")
|
||||
monkeypatch.setattr(studio_run, "_PID_FILE", pid_file)
|
||||
studio_run._remove_pid_file()
|
||||
# Not this process's PID, so the file stays; the point is that it returned.
|
||||
assert pid_file.exists()
|
||||
|
||||
pid_file.write_text(str(os.getpid()), encoding = "utf-8")
|
||||
studio_run._remove_pid_file()
|
||||
assert not pid_file.exists()
|
||||
|
||||
|
||||
def test_the_kwargs_guard_only_judges_dicts_that_reach_a_call(tmp_path: Path) -> None:
|
||||
"""Only a dict splatted into a call is subprocess configuration. An unrelated
|
||||
payload that happens to carry "text": True is not, and neither is one whose
|
||||
encoding is filled in on a later line."""
|
||||
cases = {
|
||||
"offender.py": 'kw = {"text": True}\nrun(cmd, **kw)\n',
|
||||
"annotated.py": 'kw: dict = {"universal_newlines": True}\nrun(cmd, **kw)\n',
|
||||
"payload.py": 'payload = {"text": True}\nrequests.post(url, json = payload)\n',
|
||||
"inline.py": 'run(cmd, **{"text": True})\n',
|
||||
"later.py": 'kw = {"text": True}\nkw["encoding"] = "utf-8"\nrun(cmd, **kw)\n',
|
||||
"carried.py": 'kw = {"text": True, "encoding": "utf-8"}\nrun(cmd, **kw)\n',
|
||||
}
|
||||
flagged = set()
|
||||
for name, source in cases.items():
|
||||
path = tmp_path / name
|
||||
path.write_text(source, encoding = "utf-8")
|
||||
if any("subprocess kwargs" in line for line in _offenders(path)):
|
||||
flagged.add(name)
|
||||
assert flagged == {"offender.py", "annotated.py", "inline.py"}, flagged
|
||||
|
||||
|
||||
def test_the_guard_follows_subprocess_through_an_alias(tmp_path: Path) -> None:
|
||||
"""install_wheel() takes ``run = subprocess.run`` and calls it as a bare
|
||||
name, so an attribute-only match let both of its installer calls drop their
|
||||
encoding unnoticed. A name bound to something else is still not subprocess."""
|
||||
cases = {
|
||||
"param_default.py": (
|
||||
"import subprocess\n"
|
||||
"def install(*, run = subprocess.run):\n"
|
||||
" run(cmd, text = True)\n"
|
||||
),
|
||||
"assigned.py": "import subprocess\n_run = subprocess.run\n_run(cmd, text = True)\n",
|
||||
"imported.py": "from subprocess import check_output\ncheck_output(cmd, text = True)\n",
|
||||
"renamed.py": "from subprocess import run as _r\n_r(cmd, universal_newlines = True)\n",
|
||||
"encoded.py": (
|
||||
"import subprocess\n"
|
||||
"def install(*, run = subprocess.run):\n"
|
||||
' run(cmd, text = True, encoding = "utf-8")\n'
|
||||
),
|
||||
"unrelated.py": "def run(cmd, text = False):\n pass\nrun(cmd, text = True)\n",
|
||||
}
|
||||
flagged = set()
|
||||
for name, source in cases.items():
|
||||
path = tmp_path / name
|
||||
path.write_text(source, encoding = "utf-8")
|
||||
if any("subprocess(text = True)" in line for line in _offenders(path)):
|
||||
flagged.add(name)
|
||||
assert flagged == {"param_default.py", "assigned.py", "imported.py", "renamed.py"}, flagged
|
||||
|
||||
|
||||
def test_the_guard_sees_os_fdopen(tmp_path: Path) -> None:
|
||||
"""os.fdopen(fd, mode) is open() on a descriptor and takes the same locale
|
||||
default in text mode, so leaving it out let the swap lock file keep the
|
||||
codepage on the write side while its reader was pinned to UTF-8."""
|
||||
cases = {
|
||||
"text.py": 'import os\nos.fdopen(fd, "w")\n',
|
||||
"default_mode.py": "import os\nos.fdopen(fd)\n", # defaults to "r", still text
|
||||
"binary.py": 'import os\nos.fdopen(fd, "wb")\n',
|
||||
"keyword.py": 'import os\nos.fdopen(fd, "w", encoding = "utf-8")\n',
|
||||
"positional.py": 'import os\nos.fdopen(fd, "w", 1, "utf-8")\n',
|
||||
}
|
||||
flagged = set()
|
||||
for name, source in cases.items():
|
||||
path = tmp_path / name
|
||||
path.write_text(source, encoding = "utf-8")
|
||||
if any("fdopen" in line for line in _offenders(path)):
|
||||
flagged.add(name)
|
||||
assert flagged == {"text.py", "default_mode.py"}, flagged
|
||||
|
||||
|
||||
def test_an_undecodable_bootstrap_password_does_not_stop_startup(
|
||||
tmp_path: Path, monkeypatch
|
||||
) -> None:
|
||||
"""ensure_default_admin calls _load_bootstrap_password for every existing
|
||||
admin and the lifespan calls that with no handler, so a raise here takes the
|
||||
whole backend down instead of ignoring an unusable file."""
|
||||
import sys
|
||||
|
||||
backend = str(Path(__file__).resolve().parent.parent)
|
||||
if backend not in sys.path:
|
||||
sys.path.insert(0, backend)
|
||||
from auth import storage
|
||||
|
||||
pw_file = tmp_path / ".bootstrap_password"
|
||||
pw_file.write_bytes(b"\x80\xffnot-utf8\n")
|
||||
monkeypatch.setattr(storage, "_BOOTSTRAP_PW_PATH", pw_file)
|
||||
assert storage._load_bootstrap_password() is None
|
||||
|
||||
# A readable one still loads, so this is a narrowing of failure, not of function.
|
||||
pw_file.write_text("correct horse battery staple\n", encoding = "utf-8")
|
||||
assert storage._load_bootstrap_password() == "correct horse battery staple"
|
||||
|
||||
|
||||
def test_a_damaged_checkpoint_resets_instead_of_resuming_on_a_broken_cursor(tmp_path: Path) -> None:
|
||||
"""A checkpoint holds only base64 cursors and booleans, so a codepage reading
|
||||
can only ever add non-ASCII, never recover any. Resuming on a mojibaked cursor
|
||||
sends GitHub one it answers with INVALID_CURSOR_ARGUMENTS, and the empty page
|
||||
that comes back marks the stream done and skips the rest of it for good.
|
||||
Dropping the checkpoint only replays pages the writers already dedup."""
|
||||
module = _load_state_store("cp1252")
|
||||
cursor = "Y3Vyc29yOnYyOpK0MjAxMi0wMi0xNlQwNjo1Mzo0MVrOADGL_A=="
|
||||
healthy = json.dumps({"issues_cursor": cursor, "issues_done": False}, indent = 2)
|
||||
path = tmp_path / "octocat__Hello-World.json"
|
||||
|
||||
path.write_text(healthy, encoding = "utf-8")
|
||||
assert module.StateStore(path).get("issues_cursor") == cursor
|
||||
|
||||
# Written by a pre-UTF-8 release in the operator's codepage. Nothing is lost
|
||||
# by reading UTF-8 only, because an all-ASCII document is the same bytes.
|
||||
path.write_bytes(healthy.encode("cp1252"))
|
||||
assert module.StateStore(path).get("issues_cursor") == cursor
|
||||
|
||||
# One damaged byte inside the cursor: still a whole JSON document under a
|
||||
# single-byte codepage, so only refusing that reading resets the checkpoint.
|
||||
raw = healthy.encode()
|
||||
at = raw.index(b"MjAxMi0wMi0xNlQ") + 3
|
||||
path.write_bytes(raw[:at] + b"\x96" + raw[at + 1 :])
|
||||
assert json.loads(path.read_bytes().decode("latin-1"))["issues_cursor"] != cursor
|
||||
store = module.StateStore(path)
|
||||
assert store.all() == {}
|
||||
assert store.get("issues_cursor") is None
|
||||
|
||||
|
||||
def test_a_utf8_record_is_not_parsed_a_second_time(tmp_path: Path) -> None:
|
||||
"""These shards reach gigabytes and every resume reads all of one, so a
|
||||
record that already read as UTF-8 must not be decoded and parsed again under
|
||||
the codepage. The legacy reading exists only to recover keys UTF-8 could not."""
|
||||
module = _load_state_store("cp1252")
|
||||
calls: list[str] = []
|
||||
real_parse = module._parse
|
||||
|
||||
def counting_parse(raw, encoding):
|
||||
calls.append(encoding)
|
||||
return real_parse(raw, encoding)
|
||||
|
||||
module._parse = counting_parse
|
||||
try:
|
||||
healthy = json.dumps({"id": 1, "author": "Jürgen"}).encode("utf-8")
|
||||
reading = module._read_line(healthy, "cp1252")
|
||||
assert reading.as_utf8 == {"id": 1, "author": "Jürgen"}
|
||||
assert calls == ["utf-8"], calls
|
||||
|
||||
# A line UTF-8 cannot read still falls through to the codepage, the whole point.
|
||||
calls.clear()
|
||||
legacy = json.dumps({"id": 2, "author": "Jürgen"}, ensure_ascii = False).encode("cp1252")
|
||||
reading = module._read_line(legacy, "cp1252")
|
||||
assert reading.as_utf8 is None
|
||||
assert reading.as_legacy == {"id": 2, "author": "Jürgen"}
|
||||
assert calls == ["utf-8", "cp1252"], calls
|
||||
finally:
|
||||
module._parse = real_parse
|
||||
|
||||
|
||||
def _too_deeply_nested_json() -> str:
|
||||
"""A JSON document nested past what this interpreter will descend into.
|
||||
|
||||
Probed rather than hardcoded: the depth json.loads gives up at is bounded by
|
||||
sys.getrecursionlimit() up to 3.11 and by the C recursion limit from 3.12,
|
||||
which sys.setrecursionlimit no longer moves and which varies by micro
|
||||
version. That is ~995 on 3.9 and ~9999 on 3.13.
|
||||
"""
|
||||
depth = 1
|
||||
while depth <= 1 << 17:
|
||||
document = "[" * depth + "]" * depth
|
||||
try:
|
||||
json.loads(document)
|
||||
except RecursionError:
|
||||
return document
|
||||
depth *= 2
|
||||
pytest.skip("this interpreter parses arbitrarily nested JSON")
|
||||
|
||||
|
||||
def test_an_unparseably_nested_document_is_discarded_not_raised(tmp_path: Path) -> None:
|
||||
"""json.loads answers nesting it cannot descend with RecursionError, which is
|
||||
a RuntimeError and so is neither a ValueError nor a UnicodeDecodeError.
|
||||
_parse is called outside any other handler in both StateStore.__init__ and
|
||||
JsonlWriter._scan_existing, so letting it escape aborts the scraper at
|
||||
startup on a file the catch-all it replaced simply discarded."""
|
||||
module = _load_state_store("cp1252")
|
||||
nested = _too_deeply_nested_json()
|
||||
|
||||
checkpoint = tmp_path / "octocat__Hello-World.json"
|
||||
checkpoint.write_text(nested, encoding = "utf-8")
|
||||
assert module.StateStore(checkpoint).all() == {} # reset, not raised
|
||||
|
||||
shard = tmp_path / "out.jsonl"
|
||||
shard.write_text(
|
||||
nested + "\n" + json.dumps({"id": 1}) + "\n" + json.dumps({"id": 2}) + "\n",
|
||||
encoding = "utf-8",
|
||||
)
|
||||
writer = module.JsonlWriter(shard)
|
||||
try:
|
||||
# Skipped like any other unreadable line, so its neighbours still yield the dedup
|
||||
# keys that keep the resume from re-fetching them.
|
||||
assert writer.has("id:1") and writer.has("id:2")
|
||||
finally:
|
||||
writer.close()
|
||||
|
|
@ -9,8 +9,28 @@ import sys
|
|||
from typing import Any
|
||||
from unittest import mock
|
||||
|
||||
import pytest
|
||||
|
||||
from core.training import worker
|
||||
|
||||
# The runtime install is Linux-only, so elsewhere these return before any status.
|
||||
linux_only = pytest.mark.skipif(
|
||||
not sys.platform.startswith("linux"),
|
||||
reason = "the runtime flash-attn install is gated to Linux",
|
||||
)
|
||||
|
||||
# causal-conv1d and flash-linear-attention are NOT Linux-gated: both installers bail out
|
||||
# on `sys.platform == "win32"` alone (no prebuilt wheel for Windows) and run everywhere
|
||||
# else, macOS included. linux_only here would skip cases that legitimately pass off Linux.
|
||||
not_on_windows = pytest.mark.skipif(
|
||||
sys.platform == "win32",
|
||||
reason = (
|
||||
"mirrors the sys.platform == 'win32' bail-out in "
|
||||
"_ensure_flash_linear_attention_unconditional and "
|
||||
"_ensure_causal_conv1d_fast_path"
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def _missing_flash_attn_import():
|
||||
real_import = builtins.__import__
|
||||
|
|
@ -55,6 +75,7 @@ def test_should_try_runtime_flash_attn_install_threshold_and_skip(monkeypatch):
|
|||
assert worker._should_try_runtime_flash_attn_install(32768) is False
|
||||
|
||||
|
||||
@linux_only
|
||||
def test_runtime_flash_attn_prefers_prebuilt_wheel(monkeypatch):
|
||||
statuses: list[str] = []
|
||||
|
||||
|
|
@ -82,6 +103,7 @@ def test_runtime_flash_attn_prefers_prebuilt_wheel(monkeypatch):
|
|||
assert statuses == ["Installing flash-attn for faster training..."]
|
||||
|
||||
|
||||
@linux_only
|
||||
def test_runtime_flash_attn_falls_back_to_pypi(monkeypatch):
|
||||
calls: list[list[str]] = []
|
||||
statuses: list[str] = []
|
||||
|
|
@ -113,12 +135,7 @@ def test_runtime_flash_attn_falls_back_to_pypi(monkeypatch):
|
|||
)
|
||||
monkeypatch.setattr(worker, "install_wheel", mock.Mock())
|
||||
|
||||
def fake_run(
|
||||
cmd,
|
||||
stdout = None,
|
||||
stderr = None,
|
||||
text = None,
|
||||
):
|
||||
def fake_run(cmd, **kwargs):
|
||||
calls.append(list(cmd))
|
||||
return subprocess.CompletedProcess(cmd, 0, "")
|
||||
|
||||
|
|
@ -139,6 +156,7 @@ def test_runtime_flash_attn_skip_env_avoids_all_install_work(monkeypatch):
|
|||
worker._sp.run.assert_not_called()
|
||||
|
||||
|
||||
@not_on_windows
|
||||
def test_causal_conv1d_fast_path_preserves_wheel_first_install_args(monkeypatch):
|
||||
install_mock = mock.Mock(return_value = True)
|
||||
monkeypatch.setattr(worker, "_install_package_wheel_first", install_mock)
|
||||
|
|
@ -160,6 +178,7 @@ def test_causal_conv1d_fast_path_preserves_wheel_first_install_args(monkeypatch)
|
|||
)
|
||||
|
||||
|
||||
@not_on_windows
|
||||
def test_causal_conv1d_fast_path_includes_qwen3_6_variants(monkeypatch):
|
||||
install_mock = mock.Mock(return_value = True)
|
||||
monkeypatch.setattr(worker, "_install_package_wheel_first", install_mock)
|
||||
|
|
@ -225,6 +244,7 @@ def _pin_fla_model_types(monkeypatch):
|
|||
)
|
||||
|
||||
|
||||
@not_on_windows
|
||||
def test_flash_linear_attention_installs_pinned_pair_for_qwen3_5(monkeypatch):
|
||||
_pin_fla_model_types(monkeypatch)
|
||||
monkeypatch.setattr(worker.shutil, "which", lambda name: "/usr/bin/uv")
|
||||
|
|
@ -277,6 +297,7 @@ def test_flash_linear_attention_skips_for_ssm_only_models(monkeypatch):
|
|||
run_mock.assert_not_called()
|
||||
|
||||
|
||||
@not_on_windows
|
||||
def test_flash_linear_attention_matches_full_qwen3_family(monkeypatch):
|
||||
monkeypatch.setattr(worker.shutil, "which", lambda name: "/usr/bin/uv")
|
||||
run_mock = mock.Mock(return_value = mock.Mock(returncode = 0, stdout = ""))
|
||||
|
|
@ -331,6 +352,7 @@ def test_flash_linear_attention_skipped_via_env(monkeypatch):
|
|||
run_mock.assert_not_called()
|
||||
|
||||
|
||||
@not_on_windows
|
||||
def test_flash_linear_attention_skipped_below_torch_2_7(monkeypatch):
|
||||
_pin_fla_model_types(monkeypatch)
|
||||
monkeypatch.delenv(worker._FLA_SKIP_ENV, raising = False)
|
||||
|
|
@ -349,6 +371,7 @@ def test_flash_linear_attention_skipped_below_torch_2_7(monkeypatch):
|
|||
assert any("torch>=" in s for s in statuses)
|
||||
|
||||
|
||||
@not_on_windows
|
||||
def test_flash_linear_attention_install_includes_einops(monkeypatch):
|
||||
_pin_fla_model_types(monkeypatch)
|
||||
monkeypatch.delenv(worker._FLA_SKIP_ENV, raising = False)
|
||||
|
|
@ -375,6 +398,7 @@ def test_flash_linear_attention_install_includes_einops(monkeypatch):
|
|||
assert f"fla-core=={worker._FLA_CORE_PACKAGE_VERSION}" in args
|
||||
|
||||
|
||||
@not_on_windows
|
||||
def test_flash_linear_attention_logs_post_install_import_failure(monkeypatch):
|
||||
"""pip exits 0 but `import fla.modules` still fails (missing transitive)."""
|
||||
_pin_fla_model_types(monkeypatch)
|
||||
|
|
@ -421,6 +445,7 @@ def test_tilelang_backend_skipped_on_unsupported_linux_arch(monkeypatch):
|
|||
run_mock.assert_not_called()
|
||||
|
||||
|
||||
@linux_only
|
||||
def test_tilelang_backend_pins_only_binary(monkeypatch):
|
||||
_pin_fla_model_types(monkeypatch)
|
||||
monkeypatch.delenv(worker._TILELANG_SKIP_ENV, raising = False)
|
||||
|
|
@ -462,6 +487,7 @@ def _force_missing_tilelang_imports(monkeypatch):
|
|||
monkeypatch.setattr(builtins, "__import__", fake_import)
|
||||
|
||||
|
||||
@linux_only
|
||||
def test_tilelang_backend_installs_pinned_pair_for_qwen3_5(monkeypatch):
|
||||
_pin_fla_model_types(monkeypatch)
|
||||
monkeypatch.delenv(worker._TILELANG_SKIP_ENV, raising = False)
|
||||
|
|
@ -486,6 +512,7 @@ def test_tilelang_backend_installs_pinned_pair_for_qwen3_5(monkeypatch):
|
|||
assert any("Installing TileLang" in s for s in statuses)
|
||||
|
||||
|
||||
@linux_only
|
||||
def test_tilelang_backend_reinstalls_when_tvm_ffi_is_broken(monkeypatch):
|
||||
"""Repair path issues TWO pip calls:
|
||||
|
||||
|
|
@ -555,6 +582,7 @@ def test_tilelang_backend_skipped_on_windows(monkeypatch):
|
|||
run_mock.assert_not_called()
|
||||
|
||||
|
||||
@linux_only
|
||||
def test_tilelang_backend_swallows_install_timeout(monkeypatch):
|
||||
_pin_fla_model_types(monkeypatch)
|
||||
monkeypatch.delenv(worker._TILELANG_SKIP_ENV, raising = False)
|
||||
|
|
@ -609,6 +637,7 @@ def test_tilelang_backend_skipped_via_env(monkeypatch):
|
|||
run_mock.assert_not_called()
|
||||
|
||||
|
||||
@linux_only
|
||||
def test_tilelang_backend_swallows_install_failure(monkeypatch):
|
||||
_pin_fla_model_types(monkeypatch)
|
||||
monkeypatch.delenv(worker._TILELANG_SKIP_ENV, raising = False)
|
||||
|
|
@ -673,6 +702,7 @@ def _patch_iu_gates(monkeypatch, fla_gate, conv_gate):
|
|||
monkeypatch.setattr(_iu, "is_causal_conv1d_available", conv_gate)
|
||||
|
||||
|
||||
@not_on_windows
|
||||
def test_hook_installs_when_gate_returns_false(monkeypatch):
|
||||
_pin_fla_model_types(monkeypatch)
|
||||
fla_gate = _make_fake_gate(initial_return = False)
|
||||
|
|
@ -976,6 +1006,7 @@ def test_hook_does_install_tilelang_for_qwen35(monkeypatch):
|
|||
tile_install.assert_called_once()
|
||||
|
||||
|
||||
@linux_only
|
||||
def test_tilelang_repair_does_not_touch_torch_cuda_stack(monkeypatch):
|
||||
"""Finding #2: the broken-tvm-ffi repair must use --no-deps on the
|
||||
forced step so --force-reinstall doesn't cascade through
|
||||
|
|
@ -1119,6 +1150,7 @@ def test_hook_runs_tilelang_repair_when_fla_already_true(monkeypatch):
|
|||
tile_install.assert_called_once()
|
||||
|
||||
|
||||
@not_on_windows
|
||||
def test_fla_installer_force_reinstalls_when_older_version_present(monkeypatch):
|
||||
"""Finding #8: an older `flash-linear-attention` that is importable
|
||||
but below the pin must force a reinstall (not no-op).
|
||||
|
|
@ -1583,15 +1615,10 @@ def test_install_respects_user_gcc_install_dir(monkeypatch):
|
|||
)
|
||||
_make_hip_install_env(monkeypatch, gcc_dir = "/usr/lib/gcc/x86_64-linux-gnu/13")
|
||||
|
||||
captured: dict[str, str] | None = {"_called": "no"}
|
||||
captured: dict[str, str] = {}
|
||||
|
||||
def fake_run(cmd, **kwargs):
|
||||
env = kwargs.get("env")
|
||||
if env is not None:
|
||||
captured.clear()
|
||||
captured.update(env)
|
||||
else:
|
||||
captured["_called"] = "yes_no_env"
|
||||
captured.update(kwargs.get("env") or {})
|
||||
return subprocess.CompletedProcess(cmd, 0, "")
|
||||
|
||||
monkeypatch.setattr(worker._sp, "run", fake_run)
|
||||
|
|
@ -1607,14 +1634,11 @@ def test_install_respects_user_gcc_install_dir(monkeypatch):
|
|||
release_base_url = "https://example.com",
|
||||
)
|
||||
|
||||
# subprocess.run invoked without env override (user already set
|
||||
# HIPCC_COMPILE_FLAGS_APPEND with --gcc-install-dir, so we left the
|
||||
# env alone — the existing value is inherited).
|
||||
assert captured == {"_called": "yes_no_env"}
|
||||
assert captured["HIPCC_COMPILE_FLAGS_APPEND"] == "--gcc-install-dir=/opt/custom/gcc-13"
|
||||
|
||||
|
||||
def test_install_does_not_inject_env_on_cuda(monkeypatch):
|
||||
"""CUDA path (no hip_version in env) → no env override at all."""
|
||||
"""CUDA path (no hip_version in env) → no HIP flag injected."""
|
||||
monkeypatch.delenv("HIPCC_COMPILE_FLAGS_APPEND", raising = False)
|
||||
monkeypatch.setattr(builtins, "__import__", _missing_module_import("causal_conv1d"))
|
||||
monkeypatch.setattr(
|
||||
|
|
@ -1641,7 +1665,7 @@ def test_install_does_not_inject_env_on_cuda(monkeypatch):
|
|||
captured: dict[str, Any] = {}
|
||||
|
||||
def fake_run(cmd, **kwargs):
|
||||
captured["env_in_kwargs"] = "env" in kwargs
|
||||
captured.update(kwargs.get("env") or {})
|
||||
return subprocess.CompletedProcess(cmd, 0, "")
|
||||
|
||||
monkeypatch.setattr(worker._sp, "run", fake_run)
|
||||
|
|
@ -1657,5 +1681,5 @@ def test_install_does_not_inject_env_on_cuda(monkeypatch):
|
|||
release_base_url = "https://example.com",
|
||||
)
|
||||
|
||||
# CUDA branch never sets the env, never invokes the gcc helper.
|
||||
assert captured.get("env_in_kwargs") is False
|
||||
# env is always passed (to force UTF-8), but never the HIP flag.
|
||||
assert "HIPCC_COMPILE_FLAGS_APPEND" not in captured
|
||||
|
|
|
|||
1056
studio/backend/utils/changelog.py
Normal file
1056
studio/backend/utils/changelog.py
Normal file
File diff suppressed because it is too large
Load diff
22
studio/backend/utils/child_stdio.py
Normal file
22
studio/backend/utils/child_stdio.py
Normal file
|
|
@ -0,0 +1,22 @@
|
|||
# SPDX-License-Identifier: AGPL-3.0-only
|
||||
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
|
||||
|
||||
"""Make a Python child agree with the parent that its pipes are UTF-8.
|
||||
|
||||
A child's ``sys.stdout`` uses ``locale.getpreferredencoding()``, which on
|
||||
Windows is the ANSI code page. Reading that pipe as UTF-8 would then mangle any
|
||||
non-ASCII the child prints, so the child has to be told which encoding to emit.
|
||||
Only needed for Python children; llama.cpp and node already emit UTF-8.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
from typing import Mapping, Optional
|
||||
|
||||
|
||||
def utf8_child_env(env: Optional[Mapping[str, str]] = None) -> dict[str, str]:
|
||||
"""Copy *env* (or the current environment) with UTF-8 stdio forced."""
|
||||
child = dict(os.environ if env is None else env)
|
||||
child["PYTHONIOENCODING"] = "utf-8"
|
||||
return child
|
||||
|
|
@ -144,6 +144,8 @@ def _run_amd_smi(*args: str, timeout: int = _AMD_SMI_DEFAULT_TIMEOUT) -> Optiona
|
|||
["amd-smi", *args, "--json"],
|
||||
capture_output = True,
|
||||
text = True,
|
||||
encoding = "utf-8",
|
||||
errors = "replace",
|
||||
timeout = timeout,
|
||||
env = _amd_env,
|
||||
**windows_hidden_subprocess_kwargs(),
|
||||
|
|
|
|||
|
|
@ -830,6 +830,8 @@ def _rocm_windows_perf_counter_gpu_util_pct() -> Optional[float]:
|
|||
["powershell", "-NoProfile", "-NonInteractive", "-Command", ps],
|
||||
capture_output = True,
|
||||
text = True,
|
||||
encoding = "utf-8",
|
||||
errors = "replace",
|
||||
timeout = 5,
|
||||
)
|
||||
if r.returncode != 0 or not r.stdout.strip():
|
||||
|
|
@ -1027,6 +1029,8 @@ def _rocm_windows_perf_counter_vram_by_adapter() -> Optional[list[tuple[str, flo
|
|||
["powershell", "-NoProfile", "-NonInteractive", "-Command", ps],
|
||||
capture_output = True,
|
||||
text = True,
|
||||
encoding = "utf-8",
|
||||
errors = "replace",
|
||||
timeout = 5,
|
||||
)
|
||||
if r.returncode != 0 or not r.stdout.strip():
|
||||
|
|
|
|||
|
|
@ -55,6 +55,8 @@ def get_physical_gpu_count() -> Optional[int]:
|
|||
["nvidia-smi", "-L"],
|
||||
capture_output = True,
|
||||
text = True,
|
||||
encoding = "utf-8",
|
||||
errors = "replace",
|
||||
timeout = 5,
|
||||
env = child_env_without_native_path_secret(),
|
||||
**_windows_hidden_subprocess_kwargs(),
|
||||
|
|
@ -81,6 +83,8 @@ def get_primary_gpu_utilization() -> dict[str, Any]:
|
|||
],
|
||||
capture_output = True,
|
||||
text = True,
|
||||
encoding = "utf-8",
|
||||
errors = "replace",
|
||||
timeout = 5,
|
||||
env = child_env_without_native_path_secret(),
|
||||
**_windows_hidden_subprocess_kwargs(),
|
||||
|
|
@ -131,6 +135,8 @@ def get_visible_gpu_utilization(
|
|||
],
|
||||
capture_output = True,
|
||||
text = True,
|
||||
encoding = "utf-8",
|
||||
errors = "replace",
|
||||
timeout = 5,
|
||||
env = child_env_without_native_path_secret(),
|
||||
**_windows_hidden_subprocess_kwargs(),
|
||||
|
|
@ -215,6 +221,8 @@ def get_backend_visible_gpu_info(
|
|||
],
|
||||
capture_output = True,
|
||||
text = True,
|
||||
encoding = "utf-8",
|
||||
errors = "replace",
|
||||
timeout = 10,
|
||||
env = child_env_without_native_path_secret(),
|
||||
**_windows_hidden_subprocess_kwargs(),
|
||||
|
|
|
|||
|
|
@ -121,7 +121,14 @@ def _installed_build_number(binary: Optional[str]) -> Optional[int]:
|
|||
if not binary:
|
||||
return None
|
||||
try:
|
||||
proc = subprocess.run([binary, "--version"], capture_output = True, text = True, timeout = 20)
|
||||
proc = subprocess.run(
|
||||
[binary, "--version"],
|
||||
capture_output = True,
|
||||
text = True,
|
||||
encoding = "utf-8",
|
||||
errors = "replace",
|
||||
timeout = 20,
|
||||
)
|
||||
except Exception: # pragma: no cover - defensive
|
||||
return None
|
||||
m = re.search(r"version:\s*(\d+)", (proc.stderr or "") + (proc.stdout or ""))
|
||||
|
|
|
|||
|
|
@ -254,7 +254,7 @@ def _transformers_constraint_args() -> tuple[list[str], str | None]:
|
|||
except Exception:
|
||||
return [], None
|
||||
fd, path = tempfile.mkstemp(prefix = "mlx_repair_", suffix = ".txt")
|
||||
with os.fdopen(fd, "w") as fh:
|
||||
with os.fdopen(fd, "w", encoding = "utf-8") as fh:
|
||||
fh.write(f"transformers=={transformers_version}\n")
|
||||
return ["--constraint", path], path
|
||||
|
||||
|
|
@ -290,6 +290,8 @@ def attempt_mlx_repair(*, timeout: int = _REPAIR_TIMEOUT_S) -> bool:
|
|||
stdout = subprocess.PIPE,
|
||||
stderr = subprocess.STDOUT,
|
||||
text = True,
|
||||
encoding = "utf-8",
|
||||
errors = "replace",
|
||||
timeout = timeout,
|
||||
)
|
||||
except subprocess.TimeoutExpired:
|
||||
|
|
|
|||
|
|
@ -129,7 +129,7 @@ def _read_checkpoint_loss(checkpoint_path: Path) -> Optional[float]:
|
|||
if not trainer_state.exists():
|
||||
return None
|
||||
try:
|
||||
with open(trainer_state, encoding = "utf-8") as f:
|
||||
with open(trainer_state, encoding = "utf-8-sig") as f:
|
||||
state = json.load(f)
|
||||
log_history = state.get("log_history", [])
|
||||
if log_history:
|
||||
|
|
@ -174,18 +174,18 @@ def scan_checkpoints(
|
|||
metadata: dict = {}
|
||||
try:
|
||||
if adapter_config.exists():
|
||||
cfg = json.loads(adapter_config.read_text(encoding = "utf-8"))
|
||||
cfg = json.loads(adapter_config.read_text(encoding = "utf-8-sig"))
|
||||
metadata["base_model"] = cfg.get("base_model_name_or_path")
|
||||
metadata["peft_type"] = cfg.get("peft_type")
|
||||
metadata["lora_rank"] = cfg.get("r")
|
||||
elif config_file.exists():
|
||||
cfg = json.loads(config_file.read_text(encoding = "utf-8"))
|
||||
cfg = json.loads(config_file.read_text(encoding = "utf-8-sig"))
|
||||
metadata["base_model"] = cfg.get("_name_or_path")
|
||||
|
||||
# Detect BNB quantization from config.json
|
||||
if config_file.exists():
|
||||
if "cfg" not in dir():
|
||||
cfg = json.loads(config_file.read_text(encoding = "utf-8"))
|
||||
cfg = json.loads(config_file.read_text(encoding = "utf-8-sig"))
|
||||
quant_cfg = cfg.get("quantization_config")
|
||||
if (
|
||||
isinstance(quant_cfg, dict)
|
||||
|
|
|
|||
|
|
@ -37,6 +37,7 @@ import yaml
|
|||
|
||||
|
||||
from utils.native_path_leases import child_env_without_native_path_secret
|
||||
from utils.child_stdio import utf8_child_env
|
||||
from utils.hf_cache_settings import active_hf_hub_cache, get_hf_cache_paths
|
||||
from utils.subprocess_compat import (
|
||||
windows_hidden_subprocess_kwargs as _windows_hidden_subprocess_kwargs,
|
||||
|
|
@ -631,7 +632,7 @@ def _raw_config_has_vision_config(
|
|||
cache_dir = active_hf_hub_cache(),
|
||||
)
|
||||
)
|
||||
config = json.loads(config_path.read_text(encoding = "utf-8"))
|
||||
config = json.loads(config_path.read_text(encoding = "utf-8-sig"))
|
||||
architectures = config.get("architectures") or []
|
||||
model_type = config.get("model_type")
|
||||
explicit_vision = (
|
||||
|
|
@ -774,8 +775,12 @@ def _is_vision_model_subprocess(model_name: str, hf_token: Optional[str] = None)
|
|||
],
|
||||
capture_output = True,
|
||||
text = True,
|
||||
encoding = "utf-8",
|
||||
errors = "replace",
|
||||
timeout = 60,
|
||||
env = get_hf_cache_paths().child_env(child_env_without_native_path_secret()),
|
||||
env = utf8_child_env(
|
||||
get_hf_cache_paths().child_env(child_env_without_native_path_secret())
|
||||
),
|
||||
**_windows_hidden_subprocess_kwargs(),
|
||||
)
|
||||
|
||||
|
|
@ -1083,7 +1088,7 @@ def _detect_audio_from_tokenizer(
|
|||
]:
|
||||
tok_file = snapshot / tok_path
|
||||
if tok_file.exists():
|
||||
tok_config = json.loads(tok_file.read_text(encoding = "utf-8"))
|
||||
tok_config = json.loads(tok_file.read_text(encoding = "utf-8-sig"))
|
||||
read_any = True
|
||||
result = _check_token_patterns(tok_config)
|
||||
if result:
|
||||
|
|
@ -2283,7 +2288,7 @@ def scan_exported_models(
|
|||
export_meta = run_dir / "export_metadata.json"
|
||||
try:
|
||||
if export_meta.exists():
|
||||
meta = json.loads(export_meta.read_text(encoding = "utf-8"))
|
||||
meta = json.loads(export_meta.read_text(encoding = "utf-8-sig"))
|
||||
base_model = meta.get("base_model")
|
||||
except Exception:
|
||||
pass
|
||||
|
|
@ -2312,7 +2317,7 @@ def scan_exported_models(
|
|||
if adapter_config.exists():
|
||||
export_type = "lora"
|
||||
try:
|
||||
cfg = json.loads(adapter_config.read_text(encoding = "utf-8"))
|
||||
cfg = json.loads(adapter_config.read_text(encoding = "utf-8-sig"))
|
||||
base_model = cfg.get("base_model_name_or_path")
|
||||
except Exception:
|
||||
pass
|
||||
|
|
@ -2321,7 +2326,7 @@ def scan_exported_models(
|
|||
export_meta = checkpoint_dir / "export_metadata.json"
|
||||
try:
|
||||
if export_meta.exists():
|
||||
meta = json.loads(export_meta.read_text(encoding = "utf-8"))
|
||||
meta = json.loads(export_meta.read_text(encoding = "utf-8-sig"))
|
||||
base_model = meta.get("base_model")
|
||||
except Exception:
|
||||
pass
|
||||
|
|
@ -2334,7 +2339,7 @@ def scan_exported_models(
|
|||
export_meta = meta_dir / "export_metadata.json"
|
||||
try:
|
||||
if export_meta.exists():
|
||||
meta = json.loads(export_meta.read_text(encoding = "utf-8"))
|
||||
meta = json.loads(export_meta.read_text(encoding = "utf-8-sig"))
|
||||
base_model = meta.get("base_model")
|
||||
if base_model:
|
||||
break
|
||||
|
|
@ -2354,7 +2359,7 @@ def scan_exported_models(
|
|||
outputs_adapter_cfg = resolve_output_dir(run_dir.name) / "adapter_config.json"
|
||||
try:
|
||||
if outputs_adapter_cfg.exists():
|
||||
cfg = json.loads(outputs_adapter_cfg.read_text(encoding = "utf-8"))
|
||||
cfg = json.loads(outputs_adapter_cfg.read_text(encoding = "utf-8-sig"))
|
||||
base_model = cfg.get("base_model_name_or_path")
|
||||
except Exception:
|
||||
pass
|
||||
|
|
@ -2380,7 +2385,7 @@ def get_base_model_from_checkpoint(checkpoint_path: str) -> Optional[str]:
|
|||
|
||||
adapter_config_path = checkpoint_path_obj / "adapter_config.json"
|
||||
if adapter_config_path.exists():
|
||||
with open(adapter_config_path, "r", encoding = "utf-8") as f:
|
||||
with open(adapter_config_path, "r", encoding = "utf-8-sig") as f:
|
||||
config = json.load(f)
|
||||
base_model = config.get("base_model_name_or_path")
|
||||
if base_model:
|
||||
|
|
@ -2389,7 +2394,7 @@ def get_base_model_from_checkpoint(checkpoint_path: str) -> Optional[str]:
|
|||
|
||||
config_path = checkpoint_path_obj / "config.json"
|
||||
if config_path.exists():
|
||||
with open(config_path, "r", encoding = "utf-8") as f:
|
||||
with open(config_path, "r", encoding = "utf-8-sig") as f:
|
||||
config = json.load(f)
|
||||
for key in ("model_name", "_name_or_path"):
|
||||
base_model = config.get(key)
|
||||
|
|
@ -2445,7 +2450,7 @@ def get_base_model_from_lora(lora_path: str) -> Optional[str]:
|
|||
# adapter_config.json first
|
||||
adapter_config_path = lora_path_obj / "adapter_config.json"
|
||||
if adapter_config_path.exists():
|
||||
with open(adapter_config_path, "r", encoding = "utf-8") as f:
|
||||
with open(adapter_config_path, "r", encoding = "utf-8-sig") as f:
|
||||
config = json.load(f)
|
||||
base_model = config.get("base_model_name_or_path")
|
||||
if base_model:
|
||||
|
|
@ -2535,7 +2540,7 @@ def get_base_model_from_lora_identifier(
|
|||
last_exc = exc
|
||||
continue
|
||||
try:
|
||||
with open(cfg_path, "r", encoding = "utf-8") as f:
|
||||
with open(cfg_path, "r", encoding = "utf-8-sig") as f:
|
||||
base_model = json.load(f).get("base_model_name_or_path")
|
||||
except Exception as exc:
|
||||
logger.warning("Could not parse adapter_config.json for '%s': %s", identifier, exc)
|
||||
|
|
@ -2781,7 +2786,7 @@ class ModelConfig:
|
|||
meta_path = gguf_dir / "export_metadata.json"
|
||||
if meta_path.exists():
|
||||
try:
|
||||
meta = json.loads(meta_path.read_text(encoding = "utf-8"))
|
||||
meta = json.loads(meta_path.read_text(encoding = "utf-8-sig"))
|
||||
base = meta.get("base_model")
|
||||
if base and is_vision_model(base, hf_token = hf_token):
|
||||
base_is_vision = True
|
||||
|
|
@ -2912,7 +2917,7 @@ class ModelConfig:
|
|||
token = hf_token,
|
||||
cache_dir = active_hf_hub_cache(),
|
||||
)
|
||||
with open(config_path, "r", encoding = "utf-8") as f:
|
||||
with open(config_path, "r", encoding = "utf-8-sig") as f:
|
||||
adapter_config = json.load(f)
|
||||
base_model = adapter_config.get("base_model_name_or_path")
|
||||
if base_model:
|
||||
|
|
|
|||
|
|
@ -79,6 +79,8 @@ def _node_version_ok(executable: str) -> bool:
|
|||
[executable, "-v"],
|
||||
capture_output = True,
|
||||
text = True,
|
||||
encoding = "utf-8",
|
||||
errors = "replace",
|
||||
timeout = _NODE_VERSION_PROBE_TIMEOUT_SECONDS,
|
||||
**windows_hidden_subprocess_kwargs(),
|
||||
)
|
||||
|
|
|
|||
|
|
@ -212,7 +212,7 @@ def lmstudio_model_dirs() -> list[Path]:
|
|||
settings_path = Path.home() / ".lmstudio" / "settings.json"
|
||||
if settings_path.is_file():
|
||||
try:
|
||||
with open(settings_path, encoding = "utf-8") as f:
|
||||
with open(settings_path, encoding = "utf-8-sig") as f:
|
||||
settings = json.load(f)
|
||||
downloads = settings.get("downloadsFolder", "")
|
||||
if downloads:
|
||||
|
|
|
|||
|
|
@ -24,6 +24,7 @@ from typing import Callable, Optional
|
|||
|
||||
import structlog
|
||||
|
||||
from utils.child_stdio import utf8_child_env
|
||||
from utils.process_lifetime import child_popen_kwargs
|
||||
|
||||
logger = structlog.get_logger(__name__)
|
||||
|
|
@ -159,6 +160,8 @@ def resolve_prebuilt_for_host(
|
|||
cmd,
|
||||
capture_output = True,
|
||||
text = True,
|
||||
encoding = "utf-8",
|
||||
errors = "replace",
|
||||
timeout = 60,
|
||||
)
|
||||
out = (proc.stdout or "").strip()
|
||||
|
|
@ -303,7 +306,10 @@ def stream_installer(
|
|||
stdout = subprocess.PIPE,
|
||||
stderr = subprocess.STDOUT,
|
||||
text = True,
|
||||
env = env,
|
||||
encoding = "utf-8",
|
||||
errors = "replace",
|
||||
# Make the Python child emit the UTF-8 we decode above.
|
||||
env = utf8_child_env(env),
|
||||
**child_popen_kwargs(),
|
||||
)
|
||||
timed_out = threading.Event()
|
||||
|
|
|
|||
|
|
@ -142,7 +142,7 @@ def _load_remote_code_configs(model_name: str, hf_token: Optional[str] = None) -
|
|||
for name in _REMOTE_CODE_CONFIG_FILES:
|
||||
p = root / name
|
||||
if p.is_file():
|
||||
configs.append(json.loads(p.read_text(encoding = "utf-8")))
|
||||
configs.append(json.loads(p.read_text(encoding = "utf-8-sig")))
|
||||
return configs
|
||||
|
||||
from huggingface_hub import hf_hub_download
|
||||
|
|
@ -164,7 +164,7 @@ def _load_remote_code_configs(model_name: str, hf_token: Optional[str] = None) -
|
|||
# Transient/auth failure is not "absent" -> fail closed to "unknown" so
|
||||
# the caller scans (a tokenizer/processor-only auto_map must not slip by).
|
||||
return None
|
||||
configs.append(json.loads(Path(p).read_text(encoding = "utf-8")))
|
||||
configs.append(json.loads(Path(p).read_text(encoding = "utf-8-sig")))
|
||||
# Every config was read or a genuine 404 -> an empty list is a definitive
|
||||
# "no auto_map", not "unknown".
|
||||
return configs
|
||||
|
|
|
|||
|
|
@ -199,7 +199,7 @@ def _indexed_shard_paths(
|
|||
inconclusive = True # transient: an index that might exist could not be read
|
||||
continue
|
||||
try:
|
||||
weight_map = (json.loads(open(index_path, encoding = "utf-8").read()) or {}).get(
|
||||
weight_map = (json.loads(open(index_path, encoding = "utf-8-sig").read()) or {}).get(
|
||||
"weight_map"
|
||||
) or {}
|
||||
for shard in weight_map.values():
|
||||
|
|
@ -328,7 +328,7 @@ def _st_load_roots(snapshot: Path) -> list:
|
|||
roots = [snapshot]
|
||||
try:
|
||||
import json
|
||||
modules = json.loads((snapshot / "modules.json").read_text(encoding = "utf-8"))
|
||||
modules = json.loads((snapshot / "modules.json").read_text(encoding = "utf-8-sig"))
|
||||
except (OSError, ValueError):
|
||||
return roots # no / invalid modules.json -> snapshot root is the only load root
|
||||
for module in modules or ():
|
||||
|
|
@ -355,7 +355,7 @@ def _indexed_pickle_shards(index_path: Path, root: Path, snapshot: Path) -> list
|
|||
try:
|
||||
# JSON is UTF-8 by spec; pin it so a non-ASCII index is not misdecoded (and needlessly
|
||||
# blocked) under Windows' cp1252 default.
|
||||
parsed = json.loads(index_path.read_text(encoding = "utf-8"))
|
||||
parsed = json.loads(index_path.read_text(encoding = "utf-8-sig"))
|
||||
except (OSError, ValueError) as exc:
|
||||
raise OSError(f"unreadable weight index: {index_path}") from exc
|
||||
weight_map = parsed.get("weight_map") if isinstance(parsed, dict) else None
|
||||
|
|
|
|||
|
|
@ -69,7 +69,7 @@ def approval_target_key(targets) -> str:
|
|||
def _load() -> dict:
|
||||
"""Parsed store, or an empty skeleton on any error (fail-safe = re-prompt)."""
|
||||
try:
|
||||
with open(_store_path(), encoding = "utf-8") as f:
|
||||
with open(_store_path(), encoding = "utf-8-sig") as f:
|
||||
data = json.load(f)
|
||||
# Validate the shape, not just the version: a hand-edited ``subjects`` that is not a
|
||||
# dict (e.g. ``[]``) would otherwise crash lookup/record instead of failing safe.
|
||||
|
|
|
|||
|
|
@ -454,7 +454,7 @@ def repo_remote_code_files(model_name: str, hf_token: Optional[str] = None) -> d
|
|||
p = root / name
|
||||
if p.is_file():
|
||||
try:
|
||||
ext_refs |= _auto_map_refs(json.loads(p.read_text(encoding = "utf-8")))
|
||||
ext_refs |= _auto_map_refs(json.loads(p.read_text(encoding = "utf-8-sig")))
|
||||
except Exception:
|
||||
pass
|
||||
if not _add_external_refs(files, ext_refs, hf_token, model_name):
|
||||
|
|
@ -483,7 +483,7 @@ def repo_remote_code_files(model_name: str, hf_token: Optional[str] = None) -> d
|
|||
f"{model_name}: config {cfg_name} could not be fetched ({exc})"
|
||||
) from exc
|
||||
try:
|
||||
refs |= _auto_map_refs(json.loads(Path(cfg_path).read_text(encoding = "utf-8")))
|
||||
refs |= _auto_map_refs(json.loads(Path(cfg_path).read_text(encoding = "utf-8-sig")))
|
||||
except Exception:
|
||||
pass
|
||||
own_refs = {fn for repo, fn in refs if repo is None}
|
||||
|
|
@ -616,7 +616,7 @@ def external_auto_map_repos(model_name: str, hf_token: Optional[str] = None) ->
|
|||
if not p.is_file():
|
||||
continue
|
||||
try:
|
||||
refs = _auto_map_refs(json.loads(p.read_text(encoding = "utf-8")))
|
||||
refs = _auto_map_refs(json.loads(p.read_text(encoding = "utf-8-sig")))
|
||||
except Exception:
|
||||
continue
|
||||
repos.update(repo for repo, _fn in refs if repo)
|
||||
|
|
@ -638,7 +638,7 @@ def external_auto_map_repos(model_name: str, hf_token: Optional[str] = None) ->
|
|||
except Exception:
|
||||
continue
|
||||
try:
|
||||
refs = _auto_map_refs(json.loads(Path(cfg_path).read_text(encoding = "utf-8")))
|
||||
refs = _auto_map_refs(json.loads(Path(cfg_path).read_text(encoding = "utf-8-sig")))
|
||||
except Exception:
|
||||
continue
|
||||
repos.update(repo for repo, _fn in refs if repo)
|
||||
|
|
|
|||
|
|
@ -23,6 +23,7 @@ import threading
|
|||
from typing import Any, Callable, Optional
|
||||
|
||||
from loggers import get_logger
|
||||
from utils.child_stdio import utf8_child_env
|
||||
from utils.wheel_utils import (
|
||||
direct_wheel_url,
|
||||
install_wheel,
|
||||
|
|
@ -254,6 +255,12 @@ def _install_kernel(
|
|||
"stdout": subprocess.PIPE,
|
||||
"stderr": subprocess.STDOUT,
|
||||
"text": True,
|
||||
# pip and the compilers it drives write UTF-8 down this pipe; the Windows
|
||||
# ANSI codepage would mojibake or raise over a fine install.
|
||||
"encoding": "utf-8",
|
||||
"errors": "replace",
|
||||
# Make the Python child emit the UTF-8 we decode above.
|
||||
"env": utf8_child_env(),
|
||||
}
|
||||
if is_hip:
|
||||
run_kwargs["timeout"] = 1800 # ROCm builds can take 10-30 min
|
||||
|
|
@ -261,7 +268,8 @@ def _install_kernel(
|
|||
if "--gcc-install-dir" not in existing:
|
||||
gcc_dir = _hipcc_gcc_install_dir()
|
||||
if gcc_dir:
|
||||
_env = os.environ.copy()
|
||||
# Extends the UTF-8 env above rather than replacing it.
|
||||
_env = dict(run_kwargs["env"])
|
||||
_env["HIPCC_COMPILE_FLAGS_APPEND"] = (
|
||||
f"{existing} --gcc-install-dir={gcc_dir}".strip()
|
||||
)
|
||||
|
|
|
|||
|
|
@ -60,6 +60,8 @@ def _exact_git_studio_tag(repo_root: Path) -> str | None:
|
|||
stdout = subprocess.PIPE,
|
||||
stderr = subprocess.DEVNULL,
|
||||
text = True,
|
||||
encoding = "utf-8",
|
||||
errors = "replace",
|
||||
timeout = _GIT_TIMEOUT_SECONDS,
|
||||
)
|
||||
except (OSError, subprocess.TimeoutExpired):
|
||||
|
|
@ -81,6 +83,8 @@ def _git_branch(repo_root: Path) -> str | None:
|
|||
stdout = subprocess.PIPE,
|
||||
stderr = subprocess.DEVNULL,
|
||||
text = True,
|
||||
encoding = "utf-8",
|
||||
errors = "replace",
|
||||
timeout = _GIT_TIMEOUT_SECONDS,
|
||||
)
|
||||
except (OSError, subprocess.TimeoutExpired):
|
||||
|
|
|
|||
|
|
@ -44,6 +44,7 @@ import time
|
|||
from pathlib import Path
|
||||
|
||||
from utils.native_path_leases import child_env_without_native_path_secret
|
||||
from utils.child_stdio import utf8_child_env
|
||||
from utils.hf_cache_settings import get_hf_cache_paths
|
||||
from utils.subprocess_compat import (
|
||||
windows_hidden_subprocess_kwargs as _windows_hidden_subprocess_kwargs,
|
||||
|
|
@ -420,7 +421,7 @@ def _resolve_base_model(model_name: str) -> str:
|
|||
adapter_cfg_path = local_path / "adapter_config.json"
|
||||
if _safe_is_file(adapter_cfg_path):
|
||||
try:
|
||||
with open(adapter_cfg_path, encoding = "utf-8") as f:
|
||||
with open(adapter_cfg_path, encoding = "utf-8-sig") as f:
|
||||
cfg = json.load(f)
|
||||
base = cfg.get("base_model_name_or_path")
|
||||
if base:
|
||||
|
|
@ -437,7 +438,7 @@ def _resolve_base_model(model_name: str) -> str:
|
|||
config_json_path = local_path / "config.json"
|
||||
if _safe_is_file(config_json_path):
|
||||
try:
|
||||
with open(config_json_path, encoding = "utf-8") as f:
|
||||
with open(config_json_path, encoding = "utf-8-sig") as f:
|
||||
cfg = json.load(f)
|
||||
# Unsloth writes model_name, HF writes _name_or_path; skip a self-reference.
|
||||
for _key in ("model_name", "_name_or_path"):
|
||||
|
|
@ -544,7 +545,7 @@ def _adapter_base_from_hf_cache(model_name: str) -> str | None:
|
|||
)
|
||||
for cfg_path in candidates:
|
||||
if cfg_path.is_file():
|
||||
base = json.loads(cfg_path.read_text(encoding = "utf-8")).get(
|
||||
base = json.loads(cfg_path.read_text(encoding = "utf-8-sig")).get(
|
||||
"base_model_name_or_path"
|
||||
)
|
||||
return base or None
|
||||
|
|
@ -616,7 +617,7 @@ def _check_tokenizer_config_needs_v5(model_name: str, hf_token: str | None = Non
|
|||
local_tc = local_path / "tokenizer_config.json"
|
||||
if _safe_is_file(local_tc):
|
||||
try:
|
||||
with open(local_tc, encoding = "utf-8") as f:
|
||||
with open(local_tc, encoding = "utf-8-sig") as f:
|
||||
data = json.load(f)
|
||||
tokenizer_class = data.get("tokenizer_class", "")
|
||||
result = tokenizer_class in _TRANSFORMERS_5_TOKENIZER_CLASSES
|
||||
|
|
@ -706,7 +707,7 @@ def _config_json_from_hf_cache(model_name: str) -> dict | None:
|
|||
)
|
||||
for cfg_path in candidates:
|
||||
if cfg_path.is_file():
|
||||
with open(cfg_path, encoding = "utf-8") as f:
|
||||
with open(cfg_path, encoding = "utf-8-sig") as f:
|
||||
return json.load(f)
|
||||
except Exception as exc:
|
||||
logger.debug("HF cache config.json lookup failed for '%s': %s", model_name, exc)
|
||||
|
|
@ -731,7 +732,7 @@ def _load_config_json(model_name: str, hf_token: str | None = None) -> dict | No
|
|||
local_cfg = Path(model_name) / "config.json"
|
||||
if _safe_is_file(local_cfg):
|
||||
try:
|
||||
with open(local_cfg, encoding = "utf-8") as f:
|
||||
with open(local_cfg, encoding = "utf-8-sig") as f:
|
||||
cfg = json.load(f)
|
||||
_config_json_cache[cache_key] = cfg
|
||||
return cfg
|
||||
|
|
@ -1271,9 +1272,10 @@ def _probe_autoconfig(target_dir: str, model_name: str, hf_token: str | None) ->
|
|||
[sys.executable, "-c", _PROBE_CONFIG_SCRIPT, target_dir, model_name],
|
||||
capture_output = True,
|
||||
text = True,
|
||||
encoding = "utf-8",
|
||||
errors = "replace",
|
||||
timeout = _PROBE_TIMEOUT_SECS,
|
||||
env = env,
|
||||
env = utf8_child_env(env),
|
||||
**_windows_hidden_subprocess_kwargs(),
|
||||
)
|
||||
except subprocess.TimeoutExpired:
|
||||
|
|
@ -1811,7 +1813,11 @@ def _install_to_dir(pkg: str, target_dir: str) -> bool:
|
|||
stdout = subprocess.PIPE,
|
||||
stderr = subprocess.STDOUT,
|
||||
text = True,
|
||||
env = get_hf_cache_paths().child_env(child_env_without_native_path_secret()),
|
||||
encoding = "utf-8",
|
||||
errors = "replace",
|
||||
env = utf8_child_env(
|
||||
get_hf_cache_paths().child_env(child_env_without_native_path_secret())
|
||||
),
|
||||
**_windows_hidden_subprocess_kwargs(),
|
||||
)
|
||||
if result.returncode == 0:
|
||||
|
|
@ -1834,7 +1840,9 @@ def _install_to_dir(pkg: str, target_dir: str) -> bool:
|
|||
stdout = subprocess.PIPE,
|
||||
stderr = subprocess.STDOUT,
|
||||
text = True,
|
||||
env = get_hf_cache_paths().child_env(child_env_without_native_path_secret()),
|
||||
encoding = "utf-8",
|
||||
errors = "replace",
|
||||
env = utf8_child_env(get_hf_cache_paths().child_env(child_env_without_native_path_secret())),
|
||||
**_windows_hidden_subprocess_kwargs(),
|
||||
)
|
||||
if result.returncode != 0:
|
||||
|
|
@ -2079,7 +2087,7 @@ class SidecarSwapInProgress(RuntimeError):
|
|||
|
||||
def _read_swap_lock(path: Path) -> dict | None:
|
||||
try:
|
||||
data = json.loads(path.read_text(encoding = "utf-8"))
|
||||
data = json.loads(path.read_text(encoding = "utf-8-sig"))
|
||||
return data if isinstance(data, dict) else {}
|
||||
except FileNotFoundError:
|
||||
return None
|
||||
|
|
@ -2120,7 +2128,7 @@ def try_begin_sidecar_swap(kind: str = "install") -> bool:
|
|||
break
|
||||
if fd is not None:
|
||||
try:
|
||||
with os.fdopen(fd, "w") as f:
|
||||
with os.fdopen(fd, "w", encoding = "utf-8") as f:
|
||||
f.write(
|
||||
json.dumps(
|
||||
{"pid": os.getpid(), "at": time.time(), "token": token, "kind": kind}
|
||||
|
|
@ -2466,7 +2474,11 @@ def _ensure_venv_llmcompressor_exists() -> bool:
|
|||
stdout = subprocess.PIPE,
|
||||
stderr = subprocess.STDOUT,
|
||||
text = True,
|
||||
env = get_hf_cache_paths().child_env(child_env_without_native_path_secret()),
|
||||
encoding = "utf-8",
|
||||
errors = "replace",
|
||||
env = utf8_child_env(
|
||||
get_hf_cache_paths().child_env(child_env_without_native_path_secret())
|
||||
),
|
||||
**_windows_hidden_subprocess_kwargs(),
|
||||
)
|
||||
last_out = result.stdout or ""
|
||||
|
|
|
|||
|
|
@ -30,6 +30,7 @@ PYPI_SUCCESS_TTL_SECONDS = 12 * 60 * 60
|
|||
PYPI_FAILURE_TTL_SECONDS = 60 * 60
|
||||
RELEASE_NOTES_URL = "https://unsloth.ai/docs/new/changelog"
|
||||
DISABLE_ENV_VAR = "UNSLOTH_DISABLE_UPDATE_CHECK"
|
||||
FAKE_UPDATE_ENV_VAR = "UNSLOTH_STUDIO_FAKE_UPDATE"
|
||||
|
||||
LOCAL_INSTALL_SOURCES = {"editable", "local_path", "vcs", "local_repo"}
|
||||
|
||||
|
|
@ -107,11 +108,32 @@ def get_studio_install_source_status(current_version: str) -> dict[str, Any]:
|
|||
)
|
||||
|
||||
|
||||
def _is_version(value: str) -> bool:
|
||||
try:
|
||||
Version(value)
|
||||
except InvalidVersion:
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def get_studio_update_status(current_version: str) -> dict[str, Any]:
|
||||
"""Return public, read-only update status for the web UI."""
|
||||
install_source = detect_install_source()
|
||||
disabled = os.environ.get(DISABLE_ENV_VAR) == "1"
|
||||
|
||||
if os.environ.get(DISABLE_ENV_VAR) == "1":
|
||||
# Dev-only: the popup is PyPI-install-only, so fake a version to review it
|
||||
# from a checkout. The documented opt-out still wins.
|
||||
forced_version = os.environ.get(FAKE_UPDATE_ENV_VAR, "").strip()
|
||||
if forced_version and not disabled and _is_version(forced_version):
|
||||
return _status_response(
|
||||
current_version = current_version,
|
||||
latest_version = forced_version,
|
||||
install_source = "pypi",
|
||||
update_available = True,
|
||||
can_show_web_notification = True,
|
||||
)
|
||||
|
||||
if disabled:
|
||||
return _status_response(
|
||||
current_version = current_version,
|
||||
latest_version = None,
|
||||
|
|
|
|||
|
|
@ -114,6 +114,8 @@ def hf_cache_snapshot_dir(model_name: str) -> Optional[Path]:
|
|||
snapshot = repo_dir / "snapshots" / commit
|
||||
if snapshot.is_dir():
|
||||
return snapshot
|
||||
# UnicodeDecodeError is a ValueError, not an OSError: a torn refs
|
||||
# file must keep meaning "not cached here", not fail the offline check.
|
||||
except (OSError, UnicodeDecodeError):
|
||||
continue
|
||||
return None
|
||||
|
|
|
|||
|
|
@ -15,6 +15,7 @@ import urllib.request
|
|||
from typing import Callable
|
||||
|
||||
from utils.native_path_leases import child_env_without_native_path_secret
|
||||
from utils.child_stdio import utf8_child_env
|
||||
from utils.subprocess_compat import windows_hidden_subprocess_kwargs
|
||||
|
||||
_logger = logging.getLogger(__name__)
|
||||
|
|
@ -43,6 +44,8 @@ def has_blackwell_gpu() -> bool:
|
|||
stdout = subprocess.PIPE,
|
||||
stderr = subprocess.DEVNULL,
|
||||
text = True,
|
||||
encoding = "utf-8",
|
||||
errors = "replace",
|
||||
timeout = 10,
|
||||
env = child_env_without_native_path_secret(),
|
||||
)
|
||||
|
|
@ -102,8 +105,10 @@ def probe_torch_wheel_env(*, timeout: int | None = None) -> dict[str, str] | Non
|
|||
stdout = subprocess.PIPE,
|
||||
stderr = subprocess.PIPE,
|
||||
text = True,
|
||||
encoding = "utf-8",
|
||||
errors = "replace",
|
||||
timeout = timeout,
|
||||
env = child_env_without_native_path_secret(),
|
||||
env = utf8_child_env(child_env_without_native_path_secret()),
|
||||
**windows_hidden_subprocess_kwargs(),
|
||||
)
|
||||
except subprocess.TimeoutExpired:
|
||||
|
|
@ -201,6 +206,8 @@ def install_wheel(
|
|||
stdout = subprocess.PIPE,
|
||||
stderr = subprocess.STDOUT,
|
||||
text = True,
|
||||
encoding = "utf-8",
|
||||
errors = "replace",
|
||||
env = child_env_without_native_path_secret(),
|
||||
)
|
||||
attempts.append(("uv", result))
|
||||
|
|
@ -213,7 +220,10 @@ def install_wheel(
|
|||
stdout = subprocess.PIPE,
|
||||
stderr = subprocess.STDOUT,
|
||||
text = True,
|
||||
env = child_env_without_native_path_secret(),
|
||||
encoding = "utf-8",
|
||||
errors = "replace",
|
||||
# Make the Python child emit the UTF-8 we decode above.
|
||||
env = utf8_child_env(child_env_without_native_path_secret()),
|
||||
)
|
||||
attempts.append(("pip", result))
|
||||
return attempts
|
||||
|
|
|
|||
|
|
@ -121,7 +121,14 @@ def _installed_whisper_version(binary: Optional[str]) -> Optional[str]:
|
|||
if not binary:
|
||||
return None
|
||||
try:
|
||||
proc = subprocess.run([binary, "--version"], capture_output = True, text = True, timeout = 20)
|
||||
proc = subprocess.run(
|
||||
[binary, "--version"],
|
||||
capture_output = True,
|
||||
text = True,
|
||||
encoding = "utf-8",
|
||||
errors = "replace",
|
||||
timeout = 20,
|
||||
)
|
||||
except Exception: # pragma: no cover - defensive
|
||||
return None
|
||||
m = re.search(r"v?(\d+\.\d+\.\d+)", (proc.stderr or "") + (proc.stdout or ""))
|
||||
|
|
|
|||
|
|
@ -214,7 +214,8 @@ function TauriUpdateLayer({
|
|||
}
|
||||
|
||||
return (
|
||||
<div className="pointer-events-none fixed bottom-4 right-4 z-[9998] flex w-[calc(100vw-2rem)] max-w-[400px] flex-col items-stretch gap-2">
|
||||
// Capped like the browser stack: the download panel shares it, so both must fit.
|
||||
<div className="pointer-events-none fixed bottom-4 right-4 z-[9998] flex max-h-[calc(100dvh_-_2rem)] flex-col items-end gap-2">
|
||||
<UpdateBanner
|
||||
status={update.status}
|
||||
info={update.info}
|
||||
|
|
@ -223,6 +224,7 @@ function TauriUpdateLayer({
|
|||
isExternalServer={isExternalServer}
|
||||
updatePolicyMode={update.updatePolicyMode}
|
||||
manualReleaseUrl={update.manualReleaseUrl}
|
||||
releasePageUrl={update.releasePageUrl}
|
||||
positioned={false}
|
||||
onInstall={update.installUpdate}
|
||||
onDismiss={update.dismiss}
|
||||
|
|
@ -379,9 +381,11 @@ function TauriWrapper({ children }: { children: ReactNode }) {
|
|||
return (
|
||||
<>
|
||||
{children}
|
||||
{/* One bottom-right stack so overlays never overlap; they stack with a
|
||||
gap, download panel anchored at the corner with banners above. */}
|
||||
<div className="pointer-events-none fixed bottom-4 right-4 z-[9998] flex w-[calc(100vw-2rem)] max-w-[400px] flex-col items-stretch gap-2">
|
||||
{/* One bottom-right stack so overlays never overlap: download panel at the
|
||||
corner, banners above, each owning its width. */}
|
||||
{/* Capped to the viewport, or a long download list plus expanded notes
|
||||
pushes the top of the stack off screen. */}
|
||||
<div className="pointer-events-none fixed bottom-4 right-4 z-[9998] flex max-h-[calc(100dvh_-_2rem)] flex-col items-end gap-2">
|
||||
<WebUpdateBanner
|
||||
positioned={false}
|
||||
enabled={!WEB_UPDATE_HIDDEN_ROUTES.has(pathname)}
|
||||
|
|
|
|||
|
|
@ -1226,7 +1226,9 @@ export function AppSidebar() {
|
|||
openNewChat(null);
|
||||
}}
|
||||
className={cn(
|
||||
"flex items-center gap-[6px] select-none transition-opacity",
|
||||
// min-w-0 so a narrow sidebar truncates the wordmark
|
||||
// instead of pushing the search icon over the logo.
|
||||
"flex min-w-0 items-center gap-[6px] select-none transition-opacity",
|
||||
chatDisabled && "pointer-events-none opacity-50",
|
||||
)}
|
||||
aria-label={t("shell.aria.home")}
|
||||
|
|
@ -1238,17 +1240,17 @@ export function AppSidebar() {
|
|||
<img
|
||||
src="/circle-logo-small.png"
|
||||
alt="Unsloth"
|
||||
className="h-[calc(26px+0.5rem*var(--ui-font-scale,1))] w-[calc(26px+0.5rem*var(--ui-font-scale,1))] rounded-full object-cover"
|
||||
className="h-[calc(26px+0.5rem*var(--ui-font-scale,1))] w-[calc(26px+0.5rem*var(--ui-font-scale,1))] shrink-0 rounded-full object-cover"
|
||||
/>
|
||||
<span className="font-heading text-[calc(13px+0.5rem*var(--ui-font-scale,1))] font-semibold tracking-[0em] leading-none text-black dark:text-white dark:tracking-[0.02em]">
|
||||
<span className="truncate font-heading text-[calc(13px+0.5rem*var(--ui-font-scale,1))] font-semibold tracking-[0em] leading-none text-black dark:text-white dark:tracking-[0.02em]">
|
||||
unsloth
|
||||
</span>
|
||||
<span className="nav-badge ml-0.5 inline-flex items-center justify-center rounded-full border border-nav-beta-border px-[5px] pt-[3px] pb-[2px] text-[calc(0.5rem*var(--ui-font-scale,1))] font-medium leading-none tracking-[0.04em] text-nav-fg-muted antialiased subpixel-antialiased shadow-[0_1px_2px_rgba(0,0,0,0.06)] dark:shadow-[0_1px_2px_rgba(0,0,0,0.35)]">
|
||||
<span className="nav-badge ml-0.5 inline-flex shrink-0 items-center justify-center rounded-full border border-nav-beta-border px-[5px] pt-[3px] pb-[2px] text-[calc(0.5rem*var(--ui-font-scale,1))] font-medium leading-none tracking-[0.04em] text-nav-fg-muted antialiased subpixel-antialiased shadow-[0_1px_2px_rgba(0,0,0,0.06)] dark:shadow-[0_1px_2px_rgba(0,0,0,0.35)]">
|
||||
{t("shell.beta")}
|
||||
</span>
|
||||
</Link>
|
||||
)}
|
||||
<div className="flex items-center gap-0.5">
|
||||
<div className="flex shrink-0 items-center gap-0.25">
|
||||
<Tooltip>
|
||||
<TooltipPrimitive.Trigger asChild>
|
||||
<button
|
||||
|
|
@ -1257,7 +1259,7 @@ export function AppSidebar() {
|
|||
useChatSearchStore.getState().open();
|
||||
closeMobileIfOpen();
|
||||
}}
|
||||
className="inline-flex h-[33px] w-[32px] cursor-pointer items-center justify-center rounded-[10px] text-nav-icon-idle dark:text-nav-fg-muted transition-colors hover:bg-nav-surface-hover hover:text-black dark:hover:text-white focus-visible:outline-none focus-visible:ring-1 focus-visible:ring-ring"
|
||||
className="inline-flex h-[33px] w-[28px] cursor-pointer items-center justify-center rounded-[10px] text-nav-icon-idle dark:text-nav-fg-muted transition-colors hover:bg-nav-surface-hover hover:text-black dark:hover:text-white focus-visible:outline-none focus-visible:ring-1 focus-visible:ring-ring"
|
||||
aria-label={t("shell.navigation.search")}
|
||||
>
|
||||
<HugeiconsIcon icon={Search01Icon} strokeWidth={1.75} className="size-icon" />
|
||||
|
|
@ -1281,7 +1283,7 @@ export function AppSidebar() {
|
|||
<button
|
||||
type="button"
|
||||
onClick={togglePinned}
|
||||
className="inline-flex h-[33px] w-[32px] cursor-pointer items-center justify-center rounded-[10px] text-nav-icon-idle dark:text-nav-fg-muted transition-colors hover:bg-nav-surface-hover hover:text-black dark:hover:text-white focus-visible:outline-none focus-visible:ring-1 focus-visible:ring-ring"
|
||||
className="inline-flex h-[33px] w-[28px] cursor-pointer items-center justify-center rounded-[10px] text-nav-icon-idle dark:text-nav-fg-muted transition-colors hover:bg-nav-surface-hover hover:text-black dark:hover:text-white focus-visible:outline-none focus-visible:ring-1 focus-visible:ring-ring"
|
||||
aria-label={t("shell.aria.closeSidebar")}
|
||||
>
|
||||
<HugeiconsIcon icon={LayoutAlignLeftIcon} strokeWidth={1.75} className="size-icon" />
|
||||
|
|
@ -1325,10 +1327,10 @@ export function AppSidebar() {
|
|||
)}
|
||||
</SidebarHeader>
|
||||
|
||||
{/* Uniform pl-1.5 pr-2 keeps every hover pill the same width, inset from the edge. */}
|
||||
{/* Uniform pl-1.5 pr-1.75 keeps every hover pill the same width, inset from the edge. */}
|
||||
<SidebarGroup
|
||||
className={cn(
|
||||
"group-data-[collapsible=icon]:px-0 pl-1.5 pr-2 shrink-0 transition-[padding]",
|
||||
"group-data-[collapsible=icon]:px-0 pl-1.5 pr-1.75 shrink-0 transition-[padding]",
|
||||
showCompactMacBrand ? "pt-0" : "pt-[9px]",
|
||||
// Scrolled: New Chat is pinned, give a little gap below it.
|
||||
scrolled ? "pb-[5px]" : "pb-px",
|
||||
|
|
@ -1417,7 +1419,7 @@ export function AppSidebar() {
|
|||
scrolled && "is-scrolled",
|
||||
)}
|
||||
>
|
||||
<SidebarGroup className="group-data-[collapsible=icon]:px-0 pl-1.5 pr-2 py-0 shrink-0">
|
||||
<SidebarGroup className="group-data-[collapsible=icon]:px-0 pl-1.5 pr-1.75 py-0 shrink-0">
|
||||
<SidebarGroupContent>
|
||||
<SidebarMenu>
|
||||
<NavItem
|
||||
|
|
@ -1499,7 +1501,7 @@ export function AppSidebar() {
|
|||
</CollapsibleTrigger>
|
||||
</SidebarGroupLabel>
|
||||
<CollapsibleContent>
|
||||
<SidebarGroupContent className="pl-1.5 pr-2">
|
||||
<SidebarGroupContent className="pl-1.5 pr-1.75">
|
||||
<SidebarMenu>
|
||||
<NavItem
|
||||
icon={TestTubeOutlineIcon}
|
||||
|
|
@ -1574,7 +1576,7 @@ export function AppSidebar() {
|
|||
</CollapsibleTrigger>
|
||||
</SidebarGroupLabel>
|
||||
<CollapsibleContent>
|
||||
<SidebarGroupContent className="pl-1.5 pr-2">
|
||||
<SidebarGroupContent className="pl-1.5 pr-1.75">
|
||||
<SidebarMenu>
|
||||
{pinnedProjectRecords.map((project) => {
|
||||
const projectChats =
|
||||
|
|
@ -1721,7 +1723,7 @@ export function AppSidebar() {
|
|||
</CollapsibleTrigger>
|
||||
</SidebarGroupLabel>
|
||||
<CollapsibleContent>
|
||||
<SidebarGroupContent className="pl-1.5 pr-2">
|
||||
<SidebarGroupContent className="pl-1.5 pr-1.75">
|
||||
<SidebarMenu>
|
||||
{recentChatItems.map((item) =>
|
||||
renderChatSidebarItem(item, "recent"),
|
||||
|
|
@ -1753,7 +1755,7 @@ export function AppSidebar() {
|
|||
</CollapsibleTrigger>
|
||||
</SidebarGroupLabel>
|
||||
<CollapsibleContent>
|
||||
<SidebarGroupContent className="pl-1.5 pr-2">
|
||||
<SidebarGroupContent className="pl-1.5 pr-1.75">
|
||||
<SidebarMenu>
|
||||
{runItems.map((run) => {
|
||||
// Explicit selection wins. Otherwise highlight the active
|
||||
|
|
|
|||
|
|
@ -134,7 +134,7 @@ export function LlamaUpdateBanner({
|
|||
className={cn(
|
||||
positioned
|
||||
? "fixed bottom-4 right-4 z-[9998] w-[calc(100vw-2rem)] max-w-[400px]"
|
||||
: "pointer-events-auto w-full",
|
||||
: "pointer-events-auto w-[calc(100vw-2rem)] max-w-[400px]",
|
||||
)}
|
||||
data-testid="llama-update-banner"
|
||||
>
|
||||
|
|
|
|||
|
|
@ -2,6 +2,7 @@
|
|||
// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
|
||||
|
||||
import { Button } from "@/components/ui/button";
|
||||
import { ReleaseNotesPanel } from "@/components/update/release-notes-panel";
|
||||
import type {
|
||||
DesktopUpdatePolicyMode,
|
||||
RetainedUpdateFailure,
|
||||
|
|
@ -22,6 +23,8 @@ interface UpdateBannerProps {
|
|||
isExternalServer?: boolean;
|
||||
updatePolicyMode: DesktopUpdatePolicyMode;
|
||||
manualReleaseUrl: string | null;
|
||||
// Release page for this version, preferred over the generic changelog.
|
||||
releasePageUrl?: string | null;
|
||||
// false fills a shared overlay stack; true self-anchors.
|
||||
positioned?: boolean;
|
||||
onInstall: () => void;
|
||||
|
|
@ -30,6 +33,7 @@ interface UpdateBannerProps {
|
|||
}
|
||||
|
||||
const EASE_OUT_QUART: [number, number, number, number] = [0.165, 0.84, 0.44, 1];
|
||||
const LEADING_V = /^v/;
|
||||
|
||||
function formatVersion(version: string | null | undefined): string {
|
||||
if (!version) return "";
|
||||
|
|
@ -44,6 +48,7 @@ export function UpdateBanner({
|
|||
isExternalServer = false,
|
||||
updatePolicyMode,
|
||||
manualReleaseUrl,
|
||||
releasePageUrl = null,
|
||||
positioned = true,
|
||||
onInstall,
|
||||
onDismiss,
|
||||
|
|
@ -52,6 +57,8 @@ export function UpdateBanner({
|
|||
const [copying, setCopying] = useState(false);
|
||||
const [manualReport, setManualReport] = useState<string | null>(null);
|
||||
const [manualMessage, setManualMessage] = useState<string | null>(null);
|
||||
// Version whose notes are expanded; a new offer collapses the panel.
|
||||
const [notesVersion, setNotesVersion] = useState<string | null>(null);
|
||||
const showFailure = Boolean(lastFailure) && !dismissed;
|
||||
const showAvailable = status === "available" && !dismissed && !showFailure;
|
||||
const show = showFailure || (showAvailable && Boolean(info));
|
||||
|
|
@ -62,6 +69,11 @@ export function UpdateBanner({
|
|||
const currentVersion = formatVersion(info?.currentVersion);
|
||||
const latestVersion = formatVersion(info?.version);
|
||||
const Icon = showFailure ? CircleAlert : Download;
|
||||
// Keyed by the backend release, not the app's SemVer; headings drop the v.
|
||||
const notesTargetVersion =
|
||||
(info?.pypiVersion ?? info?.version)?.replace(LEADING_V, "") ?? null;
|
||||
const notesOpen =
|
||||
notesTargetVersion !== null && notesVersion === notesTargetVersion;
|
||||
|
||||
async function handleCopyDiagnostics() {
|
||||
setCopying(true);
|
||||
|
|
@ -94,13 +106,14 @@ export function UpdateBanner({
|
|||
exit={{ opacity: 0, y: 8, scale: 0.97 }}
|
||||
transition={{ duration: 0.35, ease: EASE_OUT_QUART }}
|
||||
className={cn(
|
||||
// Wider than the other overlays: notes preview plus three buttons.
|
||||
positioned
|
||||
? "fixed bottom-4 right-4 z-[9999] w-[calc(100vw-2rem)] max-w-[400px]"
|
||||
: "pointer-events-auto w-full",
|
||||
? "fixed bottom-4 right-4 z-[9999] w-[calc(100vw-2rem)] max-w-[448px]"
|
||||
: "pointer-events-auto flex min-h-0 w-[calc(100vw-2rem)] max-w-[448px] flex-col",
|
||||
)}
|
||||
data-testid="tauri-update-banner"
|
||||
>
|
||||
<div className="relative overflow-hidden rounded-[24px] bg-white px-5 pb-4 pt-5 shadow-[0_2px_8px_-2px_rgba(0,0,0,0.16)] dark:bg-card dark:shadow-[0_8px_28px_-6px_rgba(0,0,0,0.28)]">
|
||||
<div className="relative flex max-h-[calc(100dvh_-_2rem)] flex-col overflow-hidden rounded-[24px] bg-white px-5 pb-4 pt-5 shadow-[0_2px_8px_-2px_rgba(0,0,0,0.16)] dark:bg-card dark:shadow-[0_8px_28px_-6px_rgba(0,0,0,0.28)]">
|
||||
<button
|
||||
type="button"
|
||||
onClick={onDismiss}
|
||||
|
|
@ -160,7 +173,40 @@ export function UpdateBanner({
|
|||
</p>
|
||||
)}
|
||||
|
||||
<div className="mt-4 flex flex-wrap items-center justify-end gap-x-1 gap-y-2">
|
||||
{!showFailure && notesTargetVersion ? (
|
||||
<ReleaseNotesPanel
|
||||
version={notesTargetVersion}
|
||||
open={notesOpen}
|
||||
// Used only if CHANGELOG.md has no section for this version.
|
||||
fallbackMarkdown={info?.body ?? null}
|
||||
className="min-h-0 flex-1"
|
||||
releaseNotesUrl={releasePageUrl ?? manualReleaseUrl}
|
||||
/>
|
||||
) : null}
|
||||
|
||||
<div
|
||||
className={cn(
|
||||
"mt-4 flex flex-wrap items-center gap-x-1 gap-y-2",
|
||||
!showFailure && notesTargetVersion
|
||||
? "justify-between"
|
||||
: "justify-end",
|
||||
)}
|
||||
>
|
||||
{!showFailure && notesTargetVersion ? (
|
||||
<Button
|
||||
size="sm"
|
||||
variant="ghost"
|
||||
// same type size as the action buttons
|
||||
className="-ml-2 h-auto whitespace-nowrap rounded-full px-2.5 py-2 text-ui-13 font-medium text-foreground"
|
||||
onClick={() =>
|
||||
setNotesVersion(notesOpen ? null : notesTargetVersion)
|
||||
}
|
||||
aria-expanded={notesOpen}
|
||||
data-testid="tauri-update-release-notes-toggle"
|
||||
>
|
||||
{notesOpen ? "Hide release notes" : "Show release notes"}
|
||||
</Button>
|
||||
) : null}
|
||||
{showFailure ? (
|
||||
<>
|
||||
<Button
|
||||
|
|
@ -187,28 +233,31 @@ export function UpdateBanner({
|
|||
onClick={onInstall}
|
||||
disabled={installDisabled}
|
||||
>
|
||||
{isManualLinuxPackage ? "Open release page" : "Retry update"}
|
||||
{isManualLinuxPackage
|
||||
? "Open release page"
|
||||
: "Retry update"}
|
||||
</Button>
|
||||
</>
|
||||
) : (
|
||||
<>
|
||||
// wrap + right-align so the action pair stays together
|
||||
<div className="flex flex-wrap items-center justify-end gap-x-1 gap-y-2">
|
||||
<Button
|
||||
size="sm"
|
||||
variant="ghost"
|
||||
className="h-auto rounded-full px-3 py-2 text-ui-13 font-medium text-foreground"
|
||||
className="h-auto whitespace-nowrap rounded-full px-2.5 py-2 text-ui-13 font-medium text-foreground"
|
||||
onClick={onDismiss}
|
||||
>
|
||||
Remind me later
|
||||
</Button>
|
||||
<Button
|
||||
size="sm"
|
||||
className="-mr-1 h-auto rounded-full px-3.5 py-2 text-ui-13"
|
||||
className="-mr-1 h-auto whitespace-nowrap rounded-full px-3 py-2 text-ui-13"
|
||||
onClick={onInstall}
|
||||
disabled={installDisabled}
|
||||
>
|
||||
{isManualLinuxPackage ? "Open release page" : "Update"}
|
||||
</Button>
|
||||
</>
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
{manualMessage && (
|
||||
|
|
|
|||
|
|
@ -2,6 +2,7 @@
|
|||
// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
|
||||
|
||||
import { useSidebarPin } from "@/hooks/use-sidebar-pin";
|
||||
import { useSidebarWidth } from "@/hooks/use-sidebar-width";
|
||||
import { isTauri } from "@/lib/api-base";
|
||||
import { cn } from "@/lib/utils";
|
||||
import {
|
||||
|
|
@ -110,9 +111,13 @@ export function WindowTitlebar({
|
|||
const [enabled] = useState(shouldUseCustomWindowTitlebar);
|
||||
const [maximized, setMaximized] = useState(false);
|
||||
const { pinned, togglePinned } = useSidebarPin();
|
||||
// The titlebar sits outside the sidebar wrapper, so it cannot inherit
|
||||
// --sidebar-width. Read the resized width from the same store instead.
|
||||
const { width } = useSidebarWidth();
|
||||
const sidebarWidth = showSidebarSurface
|
||||
? pinned
|
||||
? "var(--studio-sidebar-expanded-width,17.5rem)"
|
||||
? // The live value only exists mid-drag; otherwise the committed width.
|
||||
`var(--studio-sidebar-live-width, ${width}px)`
|
||||
: "var(--studio-sidebar-collapsed-width,3rem)"
|
||||
: "0px";
|
||||
const contentBorderLeft = pinned ? `calc(${sidebarWidth} + 12px)` : "0px";
|
||||
|
|
|
|||
317
studio/frontend/src/components/ui/panel-resize-handle.tsx
Normal file
317
studio/frontend/src/components/ui/panel-resize-handle.tsx
Normal file
|
|
@ -0,0 +1,317 @@
|
|||
// SPDX-License-Identifier: AGPL-3.0-only
|
||||
// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
|
||||
|
||||
"use client"
|
||||
|
||||
import * as React from "react"
|
||||
|
||||
import { cn } from "@/lib/utils"
|
||||
import {
|
||||
Tooltip,
|
||||
TooltipContent,
|
||||
TooltipTrigger,
|
||||
} from "@/components/ui/tooltip"
|
||||
import { getClientPlatform } from "@/components/tauri/window-titlebar"
|
||||
|
||||
/** Pointer travel (px) below which a drag counts as a plain click. */
|
||||
const DRAG_SLOP = 4
|
||||
/** A compatibility click lands immediately after pointer-up. */
|
||||
const CLICK_COMPAT_WINDOW_MS = 300
|
||||
/** Arrow-key resize step for keyboard users. */
|
||||
const RESIZE_STEP = 16
|
||||
|
||||
type DragState = {
|
||||
startX: number
|
||||
startWidth: number
|
||||
moved: boolean
|
||||
}
|
||||
|
||||
export type PanelResizeHandleProps = {
|
||||
/** Which edge of the panel the handle sits on. */
|
||||
edge: "left" | "right"
|
||||
open: boolean
|
||||
width: number
|
||||
/** Uncapped stored preference, so a capped drag does not lower it. */
|
||||
stored: number
|
||||
min: number
|
||||
max: number
|
||||
clamp: (px: number) => number
|
||||
setWidth: (px: number) => void
|
||||
resetWidth: () => void
|
||||
onToggle: () => void
|
||||
/** Element to paint the live width onto, and the property to paint. */
|
||||
target: () => HTMLElement | null
|
||||
cssVar: string
|
||||
/** Measured to start a drag from the rendered size when collapsed. */
|
||||
measure: () => number
|
||||
label: string
|
||||
toggleLabel: string
|
||||
/** Translated tooltip copy; the caller owns the translation layer. */
|
||||
collapseHint: string
|
||||
expandHint: string
|
||||
dragHint: string
|
||||
/** Shown in the tooltip when the panel has a toggle shortcut. */
|
||||
shortcut?: string
|
||||
dataSlot?: string
|
||||
className?: string
|
||||
/** Mirrors the live width onto :root for chrome outside the panel. */
|
||||
rootVar?: string
|
||||
}
|
||||
|
||||
/**
|
||||
* A draggable panel edge: drag to resize, click to collapse or expand. Arrow
|
||||
* keys resize, Home restores the default. The width is painted straight to the
|
||||
* target while dragging and only persisted on release.
|
||||
*/
|
||||
export function PanelResizeHandle({
|
||||
edge,
|
||||
open,
|
||||
width,
|
||||
stored,
|
||||
min,
|
||||
max,
|
||||
clamp,
|
||||
setWidth,
|
||||
resetWidth,
|
||||
onToggle,
|
||||
target,
|
||||
cssVar,
|
||||
measure,
|
||||
label,
|
||||
toggleLabel,
|
||||
collapseHint,
|
||||
expandHint,
|
||||
dragHint,
|
||||
shortcut,
|
||||
dataSlot = "panel-resize-handle",
|
||||
className,
|
||||
rootVar,
|
||||
}: PanelResizeHandleProps) {
|
||||
const ref = React.useRef<HTMLButtonElement>(null)
|
||||
const dragRef = React.useRef<DragState | null>(null)
|
||||
const [dragging, setDragging] = React.useState(false)
|
||||
const [hovered, setHovered] = React.useState(false)
|
||||
const [focused, setFocused] = React.useState(false)
|
||||
const [isMacPlatform] = React.useState(() => getClientPlatform().includes("mac"))
|
||||
const hint = shortcut ? shortcut.replace("Mod", isMacPlatform ? "⌘" : "Ctrl+") : null
|
||||
|
||||
// Cached on pointer down so no DOM walk per move.
|
||||
const targetRef = React.useRef<HTMLElement | null>(null)
|
||||
const frameRef = React.useRef(0)
|
||||
const pendingRef = React.useRef(0)
|
||||
// What the pointer asked for, before the viewport cap. Committing the capped
|
||||
// value instead would quietly downgrade a stored preference on a narrow window.
|
||||
const rawRef = React.useRef(0)
|
||||
// When a pointer sequence last ended. The browser's compatibility click
|
||||
// lands in the same tick, so only a click that close behind is a duplicate.
|
||||
// A timestamp cannot go stale the way an armed flag does: a genuine cancel
|
||||
// emits no click, and a later assistive-tech click still gets through.
|
||||
const handledAtRef = React.useRef(0)
|
||||
const committedRef = React.useRef(width)
|
||||
React.useEffect(() => {
|
||||
committedRef.current = width
|
||||
}, [width])
|
||||
|
||||
const paint = React.useCallback(
|
||||
(value: string) => {
|
||||
targetRef.current?.style.setProperty(cssVar, value)
|
||||
if (rootVar) {
|
||||
document.documentElement.style.setProperty(rootVar, value)
|
||||
}
|
||||
},
|
||||
[cssVar, rootVar],
|
||||
)
|
||||
|
||||
// Resizing relayouts the whole shell, and pointermove fires faster than the
|
||||
// display refreshes, so coalesce to one paint per frame.
|
||||
const paintWidth = React.useCallback(
|
||||
(px: number) => {
|
||||
pendingRef.current = px
|
||||
if (frameRef.current) return
|
||||
frameRef.current = requestAnimationFrame(() => {
|
||||
frameRef.current = 0
|
||||
paint(`${pendingRef.current}px`)
|
||||
})
|
||||
},
|
||||
[paint],
|
||||
)
|
||||
|
||||
const endDrag = React.useCallback(() => {
|
||||
// Only a sequence that actually started can produce a compatibility click.
|
||||
// This also runs as the effect cleanup, where no drag happened.
|
||||
if (dragRef.current) handledAtRef.current = Date.now()
|
||||
dragRef.current = null
|
||||
if (frameRef.current) {
|
||||
cancelAnimationFrame(frameRef.current)
|
||||
frameRef.current = 0
|
||||
}
|
||||
// Hand the property back to the committed value. A commit re-renders with
|
||||
// the new width; a cancel or a no-commit drag keeps DOM and store in step.
|
||||
paint(`${committedRef.current}px`)
|
||||
if (rootVar) document.documentElement.style.removeProperty(rootVar)
|
||||
targetRef.current?.removeAttribute("data-resizing")
|
||||
document.documentElement.removeAttribute("data-panel-resizing")
|
||||
targetRef.current = null
|
||||
setDragging(false)
|
||||
document.body.style.removeProperty("cursor")
|
||||
document.body.style.removeProperty("user-select")
|
||||
}, [paint, rootVar])
|
||||
|
||||
const handlePointerDown = (event: React.PointerEvent<HTMLButtonElement>) => {
|
||||
if (event.button !== 0) return
|
||||
event.preventDefault()
|
||||
event.currentTarget.setPointerCapture(event.pointerId)
|
||||
targetRef.current = target()
|
||||
// Collapsed: grow from the rendered size so the edge tracks the pointer.
|
||||
const start = open ? width : measure()
|
||||
dragRef.current = { startX: event.clientX, startWidth: start, moved: false }
|
||||
pendingRef.current = start
|
||||
rawRef.current = start
|
||||
targetRef.current?.setAttribute("data-resizing", "true")
|
||||
document.documentElement.setAttribute("data-panel-resizing", "true")
|
||||
setDragging(true)
|
||||
document.body.style.setProperty("cursor", "col-resize")
|
||||
document.body.style.setProperty("user-select", "none")
|
||||
}
|
||||
|
||||
const handlePointerMove = (event: React.PointerEvent<HTMLButtonElement>) => {
|
||||
const drag = dragRef.current
|
||||
if (!drag) return
|
||||
// A panel whose handle is on its left edge grows as the pointer moves left.
|
||||
const delta = (edge === "left" ? -1 : 1) * (event.clientX - drag.startX)
|
||||
if (!drag.moved && Math.abs(delta) < DRAG_SLOP) return
|
||||
drag.moved = true
|
||||
|
||||
const next = drag.startWidth + delta
|
||||
rawRef.current = next
|
||||
if (!open) {
|
||||
// Past the minimum, dragging the collapsed edge reopens it.
|
||||
if (next >= min) {
|
||||
paintWidth(clamp(next))
|
||||
onToggle()
|
||||
}
|
||||
return
|
||||
}
|
||||
// Dragging inward stops at the minimum. Collapsing is click or the shortcut.
|
||||
paintWidth(clamp(next))
|
||||
}
|
||||
|
||||
const handlePointerUp = (event: React.PointerEvent<HTMLButtonElement>) => {
|
||||
const drag = dragRef.current
|
||||
if (!drag) return
|
||||
if (event.currentTarget.hasPointerCapture(event.pointerId)) {
|
||||
event.currentTarget.releasePointerCapture(event.pointerId)
|
||||
}
|
||||
endDrag()
|
||||
|
||||
if (!drag.moved) {
|
||||
onToggle()
|
||||
return
|
||||
}
|
||||
// A drag below the minimum leaves the stored width alone.
|
||||
if (!open) return
|
||||
// Capped: the visible edge is already at the cap, so an outward pull cannot
|
||||
// express intent beyond it. Committing would silently lower the larger
|
||||
// hidden preference. A deliberate inward drag still commits.
|
||||
if (stored > max && rawRef.current >= max) return
|
||||
// Commit what was asked for, not the capped paint, so a drag on a narrow
|
||||
// window cannot shrink a larger stored preference. setWidth clamps.
|
||||
setWidth(rawRef.current)
|
||||
}
|
||||
|
||||
const handleKeyDown = (event: React.KeyboardEvent<HTMLButtonElement>) => {
|
||||
// The collapse/expand the label advertises, for keyboard users. Pointer-up
|
||||
// handles it for the mouse; a synthesized click never reaches it.
|
||||
if (event.key === "Enter" || event.key === " ") {
|
||||
// preventDefault cancels the native click, so nothing follows to guard
|
||||
// against; arming here would swallow the next assistive-tech click.
|
||||
event.preventDefault()
|
||||
onToggle()
|
||||
return
|
||||
}
|
||||
const outward = edge === "left" ? "ArrowLeft" : "ArrowRight"
|
||||
const inward = edge === "left" ? "ArrowRight" : "ArrowLeft"
|
||||
if (event.key === outward || event.key === inward) {
|
||||
event.preventDefault()
|
||||
if (!open) {
|
||||
// Collapsed there is nothing to resize, so the outward arrow reopens.
|
||||
if (event.key === outward) onToggle()
|
||||
return
|
||||
}
|
||||
if (event.key === outward && stored > max && width >= max) return
|
||||
setWidth(width + (event.key === outward ? RESIZE_STEP : -RESIZE_STEP))
|
||||
return
|
||||
}
|
||||
if (event.key === "Home") {
|
||||
event.preventDefault()
|
||||
resetWidth()
|
||||
}
|
||||
}
|
||||
|
||||
// Clear a stuck cursor override if we unmount mid-drag.
|
||||
React.useEffect(() => endDrag, [endDrag])
|
||||
|
||||
return (
|
||||
<Tooltip open={(hovered || focused) && !dragging}>
|
||||
<TooltipTrigger asChild>
|
||||
<button
|
||||
ref={ref}
|
||||
type="button"
|
||||
data-slot={dataSlot}
|
||||
data-dragging={dragging || undefined}
|
||||
aria-label={open ? label : toggleLabel}
|
||||
{...(open ? { "aria-orientation": "vertical" as const } : {})}
|
||||
{...(open
|
||||
? { "aria-valuenow": width, "aria-valuemin": min, "aria-valuemax": max }
|
||||
: {})}
|
||||
role={open ? "separator" : "button"}
|
||||
onPointerDown={handlePointerDown}
|
||||
onPointerMove={handlePointerMove}
|
||||
onPointerUp={handlePointerUp}
|
||||
onPointerCancel={endDrag}
|
||||
onKeyDown={handleKeyDown}
|
||||
onClick={() => {
|
||||
// Switch and voice control activate by dispatching a bare click
|
||||
// with no pointer or key events, which nothing else here catches.
|
||||
if (Date.now() - handledAtRef.current < CLICK_COMPAT_WINDOW_MS) return
|
||||
onToggle()
|
||||
}}
|
||||
onPointerEnter={() => setHovered(true)}
|
||||
onPointerLeave={() => setHovered(false)}
|
||||
onFocus={(event) => setFocused(event.target.matches(":focus-visible"))}
|
||||
onBlur={() => setFocused(false)}
|
||||
className={cn(
|
||||
"absolute inset-y-0 z-30 hidden w-2 touch-none select-none sm:block",
|
||||
edge === "left" ? "-left-1" : "-right-1",
|
||||
// `!` overrides the app-wide hand cursor on buttons.
|
||||
open
|
||||
? "cursor-col-resize!"
|
||||
: edge === "left"
|
||||
? "cursor-w-resize!"
|
||||
: "cursor-e-resize!",
|
||||
// Sits exactly on the panel border so hover recolours one line.
|
||||
"after:absolute after:inset-y-0 after:w-px after:bg-transparent after:transition-colors after:duration-150",
|
||||
edge === "left" ? "after:left-1" : "after:right-1",
|
||||
"hover:after:bg-sidebar-ring/25 data-dragging:after:bg-sidebar-ring/25",
|
||||
// The app zeroes the native outline on buttons, so mark focus here.
|
||||
"focus-visible:outline-none focus-visible:after:bg-sidebar-ring/60",
|
||||
className,
|
||||
)}
|
||||
/>
|
||||
</TooltipTrigger>
|
||||
<TooltipContent
|
||||
side={edge === "left" ? "left" : "right"}
|
||||
align="center"
|
||||
className="tooltip-compact"
|
||||
>
|
||||
<span className="flex flex-col gap-px">
|
||||
<span>
|
||||
{open ? collapseHint : expandHint}
|
||||
{hint ? ` ${hint}` : ""}
|
||||
</span>
|
||||
<span className="opacity-70">{dragHint}</span>
|
||||
</span>
|
||||
</TooltipContent>
|
||||
</Tooltip>
|
||||
)
|
||||
}
|
||||
|
|
@ -24,13 +24,21 @@ import {
|
|||
TooltipContent,
|
||||
TooltipTrigger,
|
||||
} from "@/components/ui/tooltip"
|
||||
import { PanelResizeHandle } from "@/components/ui/panel-resize-handle"
|
||||
import { useT } from "@/i18n"
|
||||
import { useIsMobile } from "@/hooks/use-mobile"
|
||||
import {
|
||||
SIDEBAR_WIDTH_DEFAULT,
|
||||
SIDEBAR_WIDTH_MIN,
|
||||
clampSidebarWidth,
|
||||
useSidebarWidth,
|
||||
} from "@/hooks/use-sidebar-width"
|
||||
import { HugeiconsIcon } from "@hugeicons/react"
|
||||
import { LayoutAlignLeftIcon } from "@hugeicons/core-free-icons"
|
||||
|
||||
const noop = () => {}
|
||||
|
||||
const SIDEBAR_WIDTH = "17.5rem"
|
||||
const SIDEBAR_WIDTH = `${SIDEBAR_WIDTH_DEFAULT}px`
|
||||
const SIDEBAR_WIDTH_ICON = "3rem"
|
||||
const SIDEBAR_KEYBOARD_SHORTCUT = "b"
|
||||
|
||||
|
|
@ -46,6 +54,11 @@ type SidebarContextProps = {
|
|||
pinned: boolean
|
||||
setPinned: (value: boolean) => void
|
||||
togglePinned: () => void
|
||||
width: number
|
||||
storedWidth: number
|
||||
maxWidth: number
|
||||
setWidth: (value: number) => void
|
||||
resetWidth: () => void
|
||||
}
|
||||
|
||||
const SidebarContext = React.createContext<SidebarContextProps | null>(null)
|
||||
|
|
@ -80,6 +93,13 @@ function SidebarProvider({
|
|||
}) {
|
||||
const isMobile = useIsMobile()
|
||||
const [openMobile, setOpenMobile] = React.useState(false)
|
||||
const {
|
||||
width,
|
||||
max: maxWidth,
|
||||
stored: storedWidth,
|
||||
setWidth,
|
||||
resetWidth,
|
||||
} = useSidebarWidth()
|
||||
|
||||
const prevIsMobileRef = React.useRef(isMobile)
|
||||
React.useEffect(() => {
|
||||
|
|
@ -163,8 +183,13 @@ function SidebarProvider({
|
|||
pinned,
|
||||
setPinned,
|
||||
togglePinned,
|
||||
width,
|
||||
storedWidth,
|
||||
maxWidth,
|
||||
setWidth,
|
||||
resetWidth,
|
||||
}),
|
||||
[state, open, setOpen, isMobile, openMobile, setOpenMobile, toggleSidebar, hasPinMode, pinned, setPinned, togglePinned]
|
||||
[state, open, setOpen, isMobile, openMobile, setOpenMobile, toggleSidebar, hasPinMode, pinned, setPinned, togglePinned, width, storedWidth, maxWidth, setWidth, resetWidth]
|
||||
)
|
||||
|
||||
return (
|
||||
|
|
@ -173,7 +198,8 @@ function SidebarProvider({
|
|||
data-slot="sidebar-wrapper"
|
||||
style={
|
||||
{
|
||||
"--sidebar-width": SIDEBAR_WIDTH,
|
||||
// The drag handle writes this same property live while resizing.
|
||||
"--sidebar-width": `${width}px`,
|
||||
"--sidebar-width-icon": SIDEBAR_WIDTH_ICON,
|
||||
...style,
|
||||
} as React.CSSProperties
|
||||
|
|
@ -311,11 +337,64 @@ function Sidebar({
|
|||
>
|
||||
{children}
|
||||
</div>
|
||||
<SidebarResizeHandle side={side} />
|
||||
</div>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
/**
|
||||
* The sidebar's draggable edge, over the shared panel handle.
|
||||
*/
|
||||
function SidebarResizeHandle({
|
||||
className,
|
||||
side = "left",
|
||||
}: {
|
||||
className?: string
|
||||
side?: "left" | "right"
|
||||
}) {
|
||||
const { open, toggleSidebar, width, storedWidth, maxWidth, setWidth, resetWidth } =
|
||||
useSidebar()
|
||||
const ref = React.useRef<HTMLDivElement>(null)
|
||||
const t = useT()
|
||||
|
||||
return (
|
||||
<div ref={ref} className="contents">
|
||||
<PanelResizeHandle
|
||||
edge={side === "right" ? "left" : "right"}
|
||||
open={open}
|
||||
width={width}
|
||||
stored={storedWidth}
|
||||
min={SIDEBAR_WIDTH_MIN}
|
||||
max={maxWidth}
|
||||
clamp={clampSidebarWidth}
|
||||
setWidth={setWidth}
|
||||
resetWidth={resetWidth}
|
||||
onToggle={toggleSidebar}
|
||||
target={() =>
|
||||
ref.current?.closest<HTMLElement>('[data-slot="sidebar-wrapper"]') ?? null
|
||||
}
|
||||
cssVar="--sidebar-width"
|
||||
// The custom titlebar renders outside the wrapper and cannot inherit it.
|
||||
rootVar="--studio-sidebar-live-width"
|
||||
measure={() =>
|
||||
ref.current
|
||||
?.closest<HTMLElement>('[data-slot="sidebar-container"]')
|
||||
?.getBoundingClientRect().width ?? SIDEBAR_WIDTH_MIN
|
||||
}
|
||||
label={t("shell.aria.resizeSidebar")}
|
||||
toggleLabel={t("shell.aria.openSidebar")}
|
||||
collapseHint={t("shell.resize.collapse")}
|
||||
expandHint={t("shell.resize.expand")}
|
||||
dragHint={t("shell.resize.drag")}
|
||||
shortcut="ModB"
|
||||
dataSlot="sidebar-resize-handle"
|
||||
className={className}
|
||||
/>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
function SidebarTrigger({
|
||||
className,
|
||||
onClick,
|
||||
|
|
@ -777,6 +856,7 @@ export {
|
|||
SidebarMenuSubItem,
|
||||
SidebarProvider,
|
||||
SidebarRail,
|
||||
SidebarResizeHandle,
|
||||
SidebarSeparator,
|
||||
SidebarTrigger,
|
||||
useSidebar,
|
||||
|
|
|
|||
251
studio/frontend/src/components/update/release-notes-panel.tsx
Normal file
251
studio/frontend/src/components/update/release-notes-panel.tsx
Normal file
|
|
@ -0,0 +1,251 @@
|
|||
// SPDX-License-Identifier: AGPL-3.0-only
|
||||
// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
|
||||
|
||||
import { MarkdownPreview } from "@/components/markdown/markdown-preview";
|
||||
import { useReleaseNotes } from "@/hooks/use-release-notes";
|
||||
import { resolveChangelogLinks } from "@/lib/changelog-links";
|
||||
import { releaseNotesPreview } from "@/lib/release-notes-preview";
|
||||
import { cn } from "@/lib/utils";
|
||||
import {
|
||||
type ReactElement,
|
||||
type ReactNode,
|
||||
useEffect,
|
||||
useMemo,
|
||||
useRef,
|
||||
} from "react";
|
||||
|
||||
interface ReleaseNotesPanelProps {
|
||||
// Notes are looked up for this exact version only.
|
||||
version: string;
|
||||
// Collapsed previews the top bullets; expanded scrolls the full notes.
|
||||
open: boolean;
|
||||
// Desktop updater's body, used only if CHANGELOG.md has no section here.
|
||||
fallbackMarkdown?: string | null;
|
||||
releaseNotesUrl?: string | null;
|
||||
className?: string;
|
||||
}
|
||||
|
||||
const NOTES_LINK_CLASS =
|
||||
"shrink-0 whitespace-nowrap text-ui-11 font-medium text-foreground underline underline-offset-2";
|
||||
|
||||
function NotesMessage({
|
||||
children,
|
||||
action,
|
||||
}: {
|
||||
children: ReactNode;
|
||||
action?: ReactNode;
|
||||
}): ReactElement {
|
||||
return (
|
||||
<div className="flex items-center justify-between gap-2 px-1 py-2">
|
||||
<p className="text-ui-11 text-muted-foreground">{children}</p>
|
||||
{action}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
function ChangelogLink({ href }: { href: string }): ReactElement {
|
||||
return (
|
||||
<a
|
||||
href={href}
|
||||
target="_blank"
|
||||
rel="noopener noreferrer"
|
||||
className={NOTES_LINK_CLASS}
|
||||
data-testid="update-release-notes-link"
|
||||
>
|
||||
Open changelog
|
||||
</a>
|
||||
);
|
||||
}
|
||||
|
||||
export function ReleaseNotesPanel({
|
||||
version,
|
||||
open,
|
||||
fallbackMarkdown = null,
|
||||
releaseNotesUrl = null,
|
||||
className,
|
||||
}: ReleaseNotesPanelProps): ReactElement | null {
|
||||
// Fetched with the popup: the collapsed preview needs the notes too.
|
||||
const { state, notes, retry } = useReleaseNotes({ version, enabled: true });
|
||||
const scrollRef = useRef<HTMLElement | null>(null);
|
||||
|
||||
// The fallback stands in for "no section in the changelog", which the hook
|
||||
// reports as ready. An error is retryable, and the desktop fallback is the
|
||||
// updater's static blurb, so taking it there would hide Retry until cache expiry.
|
||||
const source = notes?.matched
|
||||
? notes.markdown
|
||||
: state === "error"
|
||||
? null
|
||||
: (fallbackMarkdown ?? null);
|
||||
// Notes target the repository, so relative links must point back at it.
|
||||
const markdown = useMemo(
|
||||
() => (source === null ? null : resolveChangelogLinks(source)),
|
||||
[source],
|
||||
);
|
||||
|
||||
// Notes that are only a code block or a table preview as nothing.
|
||||
const preview = useMemo(
|
||||
() => (markdown === null ? null : releaseNotesPreview(markdown)),
|
||||
[markdown],
|
||||
);
|
||||
|
||||
// Start at the top on expand, and again once async notes land.
|
||||
useEffect(() => {
|
||||
if (open && markdown && scrollRef.current) {
|
||||
scrollRef.current.scrollTop = 0;
|
||||
}
|
||||
}, [open, markdown]);
|
||||
|
||||
// Caller's URL wins: the API returns only the generic changelog, while the
|
||||
// desktop banner passes this version's release page.
|
||||
const notesUrl = releaseNotesUrl ?? notes?.releaseNotesUrl;
|
||||
const link = notesUrl ? <ChangelogLink href={notesUrl} /> : null;
|
||||
|
||||
// Nothing previewable yet or ever: keep the collapsed popup compact.
|
||||
if (
|
||||
!open &&
|
||||
(!markdown ||
|
||||
state === "loading" ||
|
||||
state === "idle" ||
|
||||
preview?.items.length === 0)
|
||||
) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return (
|
||||
<div
|
||||
className={cn("mt-3 flex min-h-0 flex-col", className)}
|
||||
data-testid="update-release-notes-panel"
|
||||
data-notes-state={state}
|
||||
data-notes-version={version}
|
||||
data-notes-open={open}
|
||||
>
|
||||
{/* borderless fill, lighter than the card in dark mode */}
|
||||
<div className="flex min-h-0 flex-col rounded-[14px] bg-muted/40 px-3 py-1 dark:bg-white/[0.06]">
|
||||
{markdown ? (
|
||||
open ? (
|
||||
<section
|
||||
ref={scrollRef}
|
||||
// biome-ignore lint/a11y/noNoninteractiveTabindex: keyboard-scrollable region
|
||||
tabIndex={0}
|
||||
aria-label={`Release notes for version ${version}`}
|
||||
// Long notes scroll here instead of pushing the buttons off screen.
|
||||
className="hover-scrollbar max-h-64 min-h-0 flex-1 overflow-y-auto overscroll-contain py-3 pr-1"
|
||||
data-testid="update-release-notes-scroll"
|
||||
>
|
||||
<MarkdownPreview
|
||||
markdown={markdown}
|
||||
// Streamdown ships headings at mt-6 and code at text-sm, and
|
||||
// clears max-width on descendants, so rescale and re-cap both.
|
||||
className="max-h-none overflow-visible border-0 bg-transparent p-0 text-ui-11 [&_[data-streamdown=link-safety-modal]>*]:max-w-md [&_img]:h-auto [&_img]:max-w-full [&>*:first-child]:mt-0 [&>*>*:first-child]:mt-0 [&_code]:text-[0.92em] [&_h1]:mt-4 [&_h1]:font-heading [&_h1]:text-ui-13 [&_h2]:mt-4 [&_h2]:font-heading [&_h2]:text-ui-13 [&_h3]:mt-4 [&_h3]:font-heading [&_h3]:text-ui-11 [&_pre]:text-[0.92em]"
|
||||
/>
|
||||
{notes?.truncated ? (
|
||||
<p className="mt-2 text-ui-10 text-muted-foreground/80">
|
||||
Notes truncated. See the full changelog.
|
||||
</p>
|
||||
) : null}
|
||||
</section>
|
||||
) : (
|
||||
<ReleaseNotesSummary preview={preview} />
|
||||
)
|
||||
) : (
|
||||
<NotesStatus
|
||||
state={state}
|
||||
version={version}
|
||||
link={link}
|
||||
retry={retry}
|
||||
/>
|
||||
)}
|
||||
</div>
|
||||
{open && markdown && link ? (
|
||||
<div className="mt-2 flex justify-end px-1">{link}</div>
|
||||
) : null}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
/** Collapsed view: the first few bullets, one line each where possible. */
|
||||
function ReleaseNotesSummary({
|
||||
preview,
|
||||
}: {
|
||||
preview: ReturnType<typeof releaseNotesPreview> | null;
|
||||
}): ReactElement | null {
|
||||
if (preview === null || preview.items.length === 0) {
|
||||
return null;
|
||||
}
|
||||
const { items, remaining } = preview;
|
||||
|
||||
return (
|
||||
<ul
|
||||
className="space-y-1 py-2 pr-1"
|
||||
data-testid="update-release-notes-summary"
|
||||
>
|
||||
{items.map((item, index) => (
|
||||
<li
|
||||
// Two releases can carry the same bullet text, so index is the key.
|
||||
key={`${index}-${item.lead}`}
|
||||
className="flex gap-1.5 text-ui-11 leading-snug text-muted-foreground"
|
||||
>
|
||||
<span aria-hidden="true" className="text-muted-foreground/60">
|
||||
•
|
||||
</span>
|
||||
<span className="line-clamp-2 min-w-0">
|
||||
{/* lead sentence carries the change */}
|
||||
<span className="font-medium text-foreground">{item.lead}</span>
|
||||
{item.rest ? <span> {item.rest}</span> : null}
|
||||
</span>
|
||||
</li>
|
||||
))}
|
||||
{remaining > 0 ? (
|
||||
<li className="pl-3 text-ui-10 text-muted-foreground/70">
|
||||
+{remaining} more
|
||||
</li>
|
||||
) : null}
|
||||
</ul>
|
||||
);
|
||||
}
|
||||
|
||||
function NotesStatus({
|
||||
state,
|
||||
version,
|
||||
link,
|
||||
retry,
|
||||
}: {
|
||||
state: ReturnType<typeof useReleaseNotes>["state"];
|
||||
version: string;
|
||||
link: ReactNode;
|
||||
retry: () => void;
|
||||
}): ReactElement {
|
||||
if (state === "loading" || state === "idle") {
|
||||
return <NotesMessage>Loading release notes...</NotesMessage>;
|
||||
}
|
||||
|
||||
if (state === "error") {
|
||||
return (
|
||||
<NotesMessage
|
||||
action={
|
||||
// The changelog page may be reachable when the lookup is not.
|
||||
<span className="flex shrink-0 items-center gap-3">
|
||||
<button
|
||||
type="button"
|
||||
onClick={retry}
|
||||
className={NOTES_LINK_CLASS}
|
||||
data-testid="update-release-notes-retry"
|
||||
>
|
||||
Retry
|
||||
</button>
|
||||
{link}
|
||||
</span>
|
||||
}
|
||||
>
|
||||
Could not load release notes.
|
||||
</NotesMessage>
|
||||
);
|
||||
}
|
||||
|
||||
// Matched nothing: link out rather than show another release's notes.
|
||||
return (
|
||||
<NotesMessage action={link}>
|
||||
No release notes published for {version} yet.
|
||||
</NotesMessage>
|
||||
);
|
||||
}
|
||||
|
|
@ -2,6 +2,7 @@
|
|||
// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
|
||||
|
||||
import { Button } from "@/components/ui/button";
|
||||
import { ReleaseNotesPanel } from "@/components/update/release-notes-panel";
|
||||
import { type DeviceType, usePlatformStore } from "@/config/env";
|
||||
import { useWebUpdateCheck } from "@/hooks/use-web-update-check";
|
||||
import { isTauri } from "@/lib/api-base";
|
||||
|
|
@ -40,6 +41,7 @@ export function WebUpdateBanner({
|
|||
const deviceType = usePlatformStore((s) => s.deviceType);
|
||||
const installCmd = installCommandForDevice(deviceType);
|
||||
const [copiedVersion, setCopiedVersion] = useState<string | null>(null);
|
||||
const [notesVersion, setNotesVersion] = useState<string | null>(null);
|
||||
const dismissTimerRef = useRef<ReturnType<typeof setTimeout> | null>(null);
|
||||
|
||||
useEffect(() => {
|
||||
|
|
@ -68,6 +70,8 @@ export function WebUpdateBanner({
|
|||
}
|
||||
|
||||
const copied = status != null && copiedVersion === status.latestVersion;
|
||||
// Keyed by version so a new offer collapses the panel.
|
||||
const notesOpen = status != null && notesVersion === status.latestVersion;
|
||||
|
||||
return (
|
||||
<AnimatePresence>
|
||||
|
|
@ -78,13 +82,14 @@ export function WebUpdateBanner({
|
|||
exit={{ opacity: 0, y: 8, scale: 0.97 }}
|
||||
transition={{ duration: 0.35, ease: EASE_OUT_QUART }}
|
||||
className={cn(
|
||||
// Wider than the other overlays: notes preview plus three buttons.
|
||||
positioned
|
||||
? "fixed bottom-4 right-4 z-[9999] w-[calc(100vw-2rem)] max-w-[400px]"
|
||||
: "pointer-events-auto w-full",
|
||||
? "fixed bottom-4 right-4 z-[9999] w-[calc(100vw-2rem)] max-w-[448px]"
|
||||
: "pointer-events-auto flex min-h-0 w-[calc(100vw-2rem)] max-w-[448px] flex-col",
|
||||
)}
|
||||
data-testid="web-update-banner"
|
||||
>
|
||||
<div className="relative overflow-hidden rounded-[24px] bg-white px-5 pb-4 pt-5 shadow-[0_2px_8px_-2px_rgba(0,0,0,0.16)] dark:bg-card dark:shadow-[0_8px_28px_-6px_rgba(0,0,0,0.28)]">
|
||||
<div className="relative flex max-h-[calc(100dvh_-_2rem)] flex-col overflow-hidden rounded-[24px] bg-white px-5 pb-4 pt-5 shadow-[0_2px_8px_-2px_rgba(0,0,0,0.16)] dark:bg-card dark:shadow-[0_8px_28px_-6px_rgba(0,0,0,0.28)]">
|
||||
<button
|
||||
type="button"
|
||||
onClick={dismiss}
|
||||
|
|
@ -127,22 +132,33 @@ export function WebUpdateBanner({
|
|||
</div>
|
||||
</div>
|
||||
|
||||
<ReleaseNotesPanel
|
||||
version={status.latestVersion}
|
||||
open={notesOpen}
|
||||
releaseNotesUrl={RELEASE_NOTES_URL}
|
||||
className="min-h-0 flex-1"
|
||||
/>
|
||||
|
||||
{/* one row at one type size; wraps only on narrow viewports */}
|
||||
<div className="mt-4 flex flex-wrap items-center justify-between gap-y-2">
|
||||
<a
|
||||
href={RELEASE_NOTES_URL}
|
||||
target="_blank"
|
||||
rel="noopener noreferrer"
|
||||
className="-ml-2 whitespace-nowrap rounded-full px-2.5 py-2 text-ui-13 font-medium text-foreground transition-colors hover:bg-muted"
|
||||
data-testid="web-update-release-notes-link"
|
||||
<Button
|
||||
size="sm"
|
||||
variant="ghost"
|
||||
className="-ml-2 h-auto whitespace-nowrap rounded-full px-2.5 py-2 text-ui-13 font-medium text-foreground"
|
||||
onClick={() =>
|
||||
setNotesVersion(notesOpen ? null : status.latestVersion)
|
||||
}
|
||||
aria-expanded={notesOpen}
|
||||
data-testid="web-update-release-notes-toggle"
|
||||
>
|
||||
Release notes
|
||||
</a>
|
||||
{notesOpen ? "Hide release notes" : "Show release notes"}
|
||||
</Button>
|
||||
{/* wrap + right-align so buttons stack instead of clipping on very narrow banners */}
|
||||
<div className="flex flex-wrap items-center justify-end gap-x-1 gap-y-2">
|
||||
<Button
|
||||
size="sm"
|
||||
variant="ghost"
|
||||
className="h-auto rounded-full px-3 py-2 text-ui-13 font-medium text-foreground"
|
||||
className="h-auto whitespace-nowrap rounded-full px-2.5 py-2 text-ui-13 font-medium text-foreground"
|
||||
onClick={snooze}
|
||||
data-testid="web-update-snooze-button"
|
||||
>
|
||||
|
|
@ -151,7 +167,7 @@ export function WebUpdateBanner({
|
|||
<Button
|
||||
size="sm"
|
||||
// -mr optically aligns the filled pill's edge with the card padding
|
||||
className="-mr-1 h-auto rounded-full px-3.5 py-2 text-ui-13"
|
||||
className="-mr-1 h-auto whitespace-nowrap rounded-full px-3 py-2 text-ui-13"
|
||||
onClick={handleCopyCommand}
|
||||
data-testid="web-update-copy-button"
|
||||
>
|
||||
|
|
|
|||
|
|
@ -18,6 +18,7 @@ import {
|
|||
DropdownMenuTrigger,
|
||||
} from "@/components/ui/dropdown-menu";
|
||||
import { InfoHint } from "@/components/ui/info-hint";
|
||||
import { PanelResizeHandle } from "@/components/ui/panel-resize-handle";
|
||||
import {
|
||||
InputGroup,
|
||||
InputGroupAddon,
|
||||
|
|
@ -44,7 +45,13 @@ import { Tooltip, TooltipContent } from "@/components/ui/tooltip";
|
|||
import { NumericValueInput, snapToStep } from "@/features/model-picker";
|
||||
import { RetrievalSettingsSection } from "@/features/rag";
|
||||
import { useLlamaUpdateCheck } from "@/hooks/use-llama-update-check";
|
||||
import {
|
||||
CHAT_SETTINGS_WIDTH_MIN,
|
||||
clampChatSettingsWidth,
|
||||
useChatSettingsWidth,
|
||||
} from "@/hooks/use-chat-settings-width";
|
||||
import { useIsMobile } from "@/hooks/use-mobile";
|
||||
import { useT } from "@/i18n";
|
||||
import { ChevronDownStandardIcon } from "@/lib/chevron-icons";
|
||||
import { toast } from "@/lib/toast";
|
||||
import { cn } from "@/lib/utils";
|
||||
|
|
@ -52,7 +59,7 @@ import { Edit03Icon, LayoutAlignRightIcon } from "@hugeicons/core-free-icons";
|
|||
import { HugeiconsIcon } from "@hugeicons/react";
|
||||
import { Braces, ChevronDown, ExternalLink } from "lucide-react";
|
||||
import { Tooltip as TooltipPrimitive } from "radix-ui";
|
||||
import { Fragment, type ReactNode } from "react";
|
||||
import { type CSSProperties, Fragment, type ReactNode } from "react";
|
||||
import { useCallback, useEffect, useMemo, useRef, useState } from "react";
|
||||
import { OpenAICodeExecSection } from "./components/openai-code-exec-section";
|
||||
import { PermissionModeDropdown } from "./permission-mode-select";
|
||||
|
|
@ -363,6 +370,15 @@ export function ChatSettingsPanel({
|
|||
onExternalProviderChange,
|
||||
externalProviderType = null,
|
||||
}: ChatSettingsPanelProps) {
|
||||
const asideRef = useRef<HTMLElement>(null);
|
||||
const t = useT();
|
||||
const {
|
||||
width: settingsWidth,
|
||||
max: settingsMax,
|
||||
stored: settingsStored,
|
||||
setWidth: setSettingsWidth,
|
||||
resetWidth: resetSettingsWidth,
|
||||
} = useChatSettingsWidth();
|
||||
// Local models show every knob; providerCapabilities is only consulted when
|
||||
// isExternalModel. Unknown providers fall back to the OpenAI-compat shape via
|
||||
// getProviderCapabilities, so these flags never undercount support.
|
||||
|
|
@ -461,6 +477,23 @@ export function ChatSettingsPanel({
|
|||
// When the prompt overflows the inline box, clicking opens the popup editor.
|
||||
const systemPromptBoxRef = useRef<HTMLTextAreaElement>(null);
|
||||
const [systemPromptOverflows, setSystemPromptOverflows] = useState(false);
|
||||
const promptObserverRef = useRef<ResizeObserver | null>(null);
|
||||
const measurePromptRef = useRef<() => void>(() => {});
|
||||
// The section unmounts its textarea when collapsed, so observe through a
|
||||
// callback ref: a stored observer would cling to the detached node and the
|
||||
// remounted one would never be measured.
|
||||
const attachPromptBox = useCallback((node: HTMLTextAreaElement | null) => {
|
||||
systemPromptBoxRef.current = node;
|
||||
promptObserverRef.current?.disconnect();
|
||||
promptObserverRef.current = null;
|
||||
if (!node || typeof ResizeObserver === "undefined") return;
|
||||
// Resizing rewraps the prompt, and a drag changes the width through a
|
||||
// custom property without re-rendering, so watch the box itself.
|
||||
const observer = new ResizeObserver(() => measurePromptRef.current());
|
||||
observer.observe(node);
|
||||
promptObserverRef.current = observer;
|
||||
measurePromptRef.current();
|
||||
}, []);
|
||||
const [activePresetBaseline, setActivePresetBaseline] = useState(params);
|
||||
const presets = useMemo(() => {
|
||||
return getOrderedPresets(customPresets);
|
||||
|
|
@ -746,15 +779,20 @@ export function ChatSettingsPanel({
|
|||
}, [open]);
|
||||
|
||||
useEffect(() => {
|
||||
const el = systemPromptBoxRef.current;
|
||||
setSystemPromptOverflows(
|
||||
currentSystemPrompt.length > 0 &&
|
||||
el != null &&
|
||||
el.clientHeight > 0 &&
|
||||
el.scrollHeight > el.clientHeight + 1,
|
||||
);
|
||||
measurePromptRef.current = () => {
|
||||
const el = systemPromptBoxRef.current;
|
||||
setSystemPromptOverflows(
|
||||
currentSystemPrompt.length > 0 &&
|
||||
el != null &&
|
||||
el.clientHeight > 0 &&
|
||||
el.scrollHeight > el.clientHeight + 1,
|
||||
);
|
||||
};
|
||||
measurePromptRef.current();
|
||||
}, [currentSystemPrompt, open]);
|
||||
|
||||
useEffect(() => () => promptObserverRef.current?.disconnect(), []);
|
||||
|
||||
const settingsScrollRef = useRef<HTMLDivElement>(null);
|
||||
|
||||
const settingsContent = (
|
||||
|
|
@ -1124,7 +1162,7 @@ export function ChatSettingsPanel({
|
|||
)}
|
||||
>
|
||||
<textarea
|
||||
ref={systemPromptBoxRef}
|
||||
ref={attachPromptBox}
|
||||
value={currentSystemPrompt}
|
||||
onChange={(e) => set("systemPrompt")(e.target.value)}
|
||||
onMouseDown={(e) => {
|
||||
|
|
@ -1433,17 +1471,47 @@ export function ChatSettingsPanel({
|
|||
|
||||
return (
|
||||
<aside
|
||||
ref={asideRef}
|
||||
data-tour="chat-settings"
|
||||
data-slot="chat-settings-panel"
|
||||
className={cn(
|
||||
"relative z-50 shrink-0 overflow-hidden bg-panel-surface text-panel-surface-fg font-heading",
|
||||
open ? "w-[17rem] border-l border-sidebar-border" : "w-0",
|
||||
"relative z-50 shrink-0 bg-panel-surface text-panel-surface-fg font-heading",
|
||||
open
|
||||
? "w-(--chat-settings-width) border-l border-sidebar-border"
|
||||
: "w-0 overflow-hidden",
|
||||
)}
|
||||
style={{
|
||||
height: "calc(100% - var(--studio-custom-titlebar-height, 0px))",
|
||||
marginTop: "var(--studio-custom-titlebar-height, 0px)",
|
||||
}}
|
||||
style={
|
||||
{
|
||||
"--chat-settings-width": `${settingsWidth}px`,
|
||||
height: "calc(100% - var(--studio-custom-titlebar-height, 0px))",
|
||||
marginTop: "var(--studio-custom-titlebar-height, 0px)",
|
||||
} as CSSProperties
|
||||
}
|
||||
>
|
||||
<div className="h-full w-full">{settingsContent}</div>
|
||||
{open ? (
|
||||
<PanelResizeHandle
|
||||
edge="left"
|
||||
open={open}
|
||||
width={settingsWidth}
|
||||
stored={settingsStored}
|
||||
min={CHAT_SETTINGS_WIDTH_MIN}
|
||||
max={settingsMax}
|
||||
clamp={clampChatSettingsWidth}
|
||||
setWidth={setSettingsWidth}
|
||||
resetWidth={resetSettingsWidth}
|
||||
onToggle={() => onOpenChange?.(!open)}
|
||||
target={() => asideRef.current}
|
||||
cssVar="--chat-settings-width"
|
||||
measure={() => asideRef.current?.getBoundingClientRect().width ?? 0}
|
||||
label={t("shell.aria.resizeRunSettings")}
|
||||
toggleLabel={t("shell.aria.openRunSettings")}
|
||||
collapseHint={t("shell.resize.collapse")}
|
||||
expandHint={t("shell.resize.expand")}
|
||||
dragHint={t("shell.resize.drag")}
|
||||
dataSlot="chat-settings-resize-handle"
|
||||
/>
|
||||
) : null}
|
||||
<div className="h-full w-full overflow-hidden">{settingsContent}</div>
|
||||
</aside>
|
||||
);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -23,12 +23,21 @@ import {
|
|||
removeScanFolder,
|
||||
} from "@/features/hub";
|
||||
import { FolderBrowser } from "@/features/model-picker";
|
||||
import { openModelsDir } from "@/features/native-intents";
|
||||
import {
|
||||
openModelsDir,
|
||||
pickHuggingFaceCacheDir,
|
||||
} from "@/features/native-intents";
|
||||
import {
|
||||
type HuggingFaceCacheSettings,
|
||||
loadHuggingFaceCacheSettings,
|
||||
updateHuggingFaceCacheSettings,
|
||||
} from "@/features/settings";
|
||||
import { isTauri } from "@/lib/api-base";
|
||||
import { toast } from "@/lib/toast";
|
||||
import { cn } from "@/lib/utils";
|
||||
import {
|
||||
Delete02Icon,
|
||||
DownloadCircle01Icon,
|
||||
FileSearchIcon,
|
||||
FolderAddIcon,
|
||||
FolderExportIcon,
|
||||
|
|
@ -49,6 +58,12 @@ function formatError(error: unknown): string {
|
|||
return error instanceof Error ? error.message : String(error);
|
||||
}
|
||||
|
||||
function formatFreeSpace(bytes: number | null): string | null {
|
||||
if (bytes === null || !Number.isFinite(bytes)) return null;
|
||||
const gb = bytes / 1024 ** 3;
|
||||
return gb >= 10 ? `${Math.round(gb)} GB free` : `${gb.toFixed(1)} GB free`;
|
||||
}
|
||||
|
||||
export function OnDeviceFoldersDialog({
|
||||
open,
|
||||
onOpenChange,
|
||||
|
|
@ -68,6 +83,11 @@ export function OnDeviceFoldersDialog({
|
|||
);
|
||||
const refreshIdRef = useRef(0);
|
||||
const mutationVersionRef = useRef(0);
|
||||
const [downloadCache, setDownloadCache] =
|
||||
useState<HuggingFaceCacheSettings | null>(null);
|
||||
const [downloadCacheLoaded, setDownloadCacheLoaded] = useState(false);
|
||||
const [downloadBrowserOpen, setDownloadBrowserOpen] = useState(false);
|
||||
const [downloadSaving, setDownloadSaving] = useState(false);
|
||||
|
||||
const sortedFolders = useMemo(
|
||||
() => [...folders].sort((a, b) => a.path.localeCompare(b.path)),
|
||||
|
|
@ -108,10 +128,66 @@ export function OnDeviceFoldersDialog({
|
|||
return () => window.clearTimeout(timer);
|
||||
}, [open, refreshFolders]);
|
||||
|
||||
useEffect(() => {
|
||||
if (!open) return;
|
||||
let cancelled = false;
|
||||
// The dialog stays mounted between opens, so re-arm the flag or a reopen
|
||||
// shows the previous answer as if it were fresh.
|
||||
setDownloadCacheLoaded(false);
|
||||
loadHuggingFaceCacheSettings()
|
||||
// Indexed locations do not depend on this. Null drops the stale path
|
||||
// rather than offer Change against a location we could not confirm.
|
||||
.catch(() => null)
|
||||
.then((settings) => {
|
||||
if (cancelled) return;
|
||||
setDownloadCache(settings);
|
||||
setDownloadCacheLoaded(true);
|
||||
});
|
||||
return () => {
|
||||
cancelled = true;
|
||||
};
|
||||
}, [open]);
|
||||
|
||||
const handleInventoryChanged = useCallback(() => {
|
||||
onInventoryChange?.();
|
||||
}, [onInventoryChange]);
|
||||
|
||||
// Relocating the cache changes which repos are on disk, but
|
||||
// updateHuggingFaceCacheSettings already bumps the inventory version, which
|
||||
// re-fetches every source. Refreshing here too would scan twice, since the
|
||||
// two rounds carry different version keys and cannot be deduplicated.
|
||||
const saveDownloadLocation = useCallback(async (nextPath: string | null) => {
|
||||
setDownloadSaving(true);
|
||||
try {
|
||||
const settings = await updateHuggingFaceCacheSettings(nextPath);
|
||||
setDownloadCache(settings);
|
||||
toast.success("Download location updated", {
|
||||
description: settings.cacheHome,
|
||||
});
|
||||
} catch (err) {
|
||||
toast.error("Couldn't update the download location", {
|
||||
description: formatError(err),
|
||||
});
|
||||
} finally {
|
||||
setDownloadSaving(false);
|
||||
}
|
||||
}, []);
|
||||
|
||||
const changeDownloadLocation = useCallback(async () => {
|
||||
if (!isTauri) {
|
||||
setDownloadBrowserOpen(true);
|
||||
return;
|
||||
}
|
||||
try {
|
||||
const picked = await pickHuggingFaceCacheDir();
|
||||
if (picked) await saveDownloadLocation(picked);
|
||||
} catch (err) {
|
||||
toast.error("Couldn't open the folder picker", {
|
||||
description: formatError(err),
|
||||
});
|
||||
}
|
||||
}, [saveDownloadLocation]);
|
||||
|
||||
const handleAdd = useCallback(
|
||||
async (rawPath: string) => {
|
||||
const nextPath = rawPath.trim();
|
||||
|
|
@ -182,10 +258,10 @@ export function OnDeviceFoldersDialog({
|
|||
<>
|
||||
<Dialog open={open} onOpenChange={onOpenChange}>
|
||||
<DialogContent
|
||||
className="gap-0 overflow-hidden p-0 sm:max-w-[620px] lg:max-w-[660px] xl:max-w-[680px] [&_[data-slot=dialog-close]]:right-3 [&_[data-slot=dialog-close]]:top-3"
|
||||
className="flex max-h-[90dvh] flex-col gap-0 overflow-hidden p-0 sm:max-w-[620px] lg:max-w-[660px] xl:max-w-[680px] [&_[data-slot=dialog-close]]:right-3 [&_[data-slot=dialog-close]]:top-3"
|
||||
overlayClassName="bg-black/20 backdrop-blur-none"
|
||||
>
|
||||
<DialogHeader className="border-b border-border/60 px-5 py-4">
|
||||
<DialogHeader className="shrink-0 border-b border-border/60 px-5 py-4">
|
||||
<DialogTitle className="text-ui-15">
|
||||
On-device locations
|
||||
</DialogTitle>
|
||||
|
|
@ -195,7 +271,78 @@ export function OnDeviceFoldersDialog({
|
|||
</DialogDescription>
|
||||
</DialogHeader>
|
||||
|
||||
<div className="space-y-4 px-5 py-4">
|
||||
<div className="min-h-0 flex-1 space-y-4 overflow-y-auto px-5 py-4">
|
||||
<div className="rounded-[14px] border border-border/70 bg-muted/20 p-3">
|
||||
<div className="mb-2 flex items-center gap-2 text-ui-12 font-medium text-foreground">
|
||||
<HugeiconsIcon
|
||||
icon={DownloadCircle01Icon}
|
||||
strokeWidth={1.75}
|
||||
className="size-3.5 text-muted-foreground"
|
||||
/>
|
||||
Download location
|
||||
</div>
|
||||
|
||||
<div className="flex flex-col gap-2 sm:flex-row sm:items-center">
|
||||
<Input
|
||||
readOnly={true}
|
||||
aria-label="Model download location"
|
||||
value={
|
||||
downloadCache?.cacheHome ??
|
||||
(downloadCacheLoaded ? "Unknown" : "Loading...")
|
||||
}
|
||||
title={downloadCache?.cacheHome}
|
||||
className="field-soft h-9 min-w-0 flex-1 rounded-full px-3 font-mono text-ui-12"
|
||||
/>
|
||||
<div className="flex shrink-0 items-center gap-2">
|
||||
<Button
|
||||
type="button"
|
||||
variant="outline"
|
||||
size="sm"
|
||||
onClick={() => void changeDownloadLocation()}
|
||||
disabled={!downloadCache?.editable || downloadSaving}
|
||||
className="h-9 rounded-full px-3 text-ui-12p5"
|
||||
>
|
||||
{downloadSaving ? (
|
||||
<Spinner className="size-3.5" />
|
||||
) : (
|
||||
<HugeiconsIcon
|
||||
icon={FolderSearchIcon}
|
||||
strokeWidth={1.75}
|
||||
data-icon="inline-start"
|
||||
className="size-3.5"
|
||||
/>
|
||||
)}
|
||||
Change
|
||||
</Button>
|
||||
{downloadCache?.isCustom ? (
|
||||
<Button
|
||||
type="button"
|
||||
variant="ghost"
|
||||
size="sm"
|
||||
onClick={() => void saveDownloadLocation(null)}
|
||||
disabled={downloadSaving}
|
||||
className="h-9 rounded-full px-3 text-ui-12p5 text-muted-foreground"
|
||||
>
|
||||
Use default
|
||||
</Button>
|
||||
) : null}
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<p className="mt-2 text-ui-10p5 text-muted-foreground">
|
||||
{downloadCache?.source === "environment"
|
||||
? `Managed by the ${
|
||||
downloadCache.environmentVariable ?? "HF_HOME"
|
||||
} environment variable.`
|
||||
: [
|
||||
"New downloads only. Models already on disk stay where they are.",
|
||||
formatFreeSpace(downloadCache?.freeBytes ?? null),
|
||||
]
|
||||
.filter(Boolean)
|
||||
.join(" · ")}
|
||||
</p>
|
||||
</div>
|
||||
|
||||
<div className="rounded-[14px] border border-border/70 bg-muted/20 p-3">
|
||||
<div className="mb-2 flex items-center gap-2 text-ui-12 font-medium text-foreground">
|
||||
<HugeiconsIcon
|
||||
|
|
@ -425,6 +572,16 @@ export function OnDeviceFoldersDialog({
|
|||
onOpenChange={setBrowserOpen}
|
||||
onSelect={(selectedPath) => void handleAdd(selectedPath)}
|
||||
/>
|
||||
|
||||
<FolderBrowser
|
||||
open={!isTauri && downloadBrowserOpen}
|
||||
onOpenChange={setDownloadBrowserOpen}
|
||||
onSelect={(selectedPath) => void saveDownloadLocation(selectedPath)}
|
||||
initialPath={downloadCache?.cacheHome}
|
||||
title="Choose model download location"
|
||||
confirmLabel="Use for future downloads"
|
||||
showModelHints={false}
|
||||
/>
|
||||
</>
|
||||
);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -201,8 +201,10 @@ export function DownloadManagerPanel({
|
|||
className={cn(
|
||||
// Standalone: anchor bottom-right. In a shared stack (positioned=false)
|
||||
// flow as a right-aligned row so overlays stack instead of overlapping.
|
||||
// min-h-0 there: a flex item's min-height defaults to auto, so the capped
|
||||
// stack would squeeze the update card instead of this list.
|
||||
"pointer-events-none",
|
||||
positioned ? "fixed bottom-4 right-4 z-50" : "flex justify-end",
|
||||
positioned ? "fixed bottom-4 right-4 z-50" : "flex min-h-0 justify-end",
|
||||
)}
|
||||
>
|
||||
{collapsed ? (
|
||||
|
|
@ -229,7 +231,7 @@ export function DownloadManagerPanel({
|
|||
</TooltipContent>
|
||||
</Tooltip>
|
||||
) : (
|
||||
<div className="hub-download-panel pointer-events-auto w-[min(400px,calc(100vw-2rem))] overflow-hidden">
|
||||
<div className="hub-download-panel pointer-events-auto flex min-h-0 w-[min(400px,calc(100vw-2rem))] flex-col overflow-hidden">
|
||||
<div className="flex items-center gap-2 border-b border-foreground/[0.07] py-2 pl-4 pr-3">
|
||||
<span className="min-w-0 flex-1 truncate text-ui-12p5 font-semibold text-foreground">
|
||||
{headerLabel}
|
||||
|
|
|
|||
|
|
@ -3,6 +3,11 @@
|
|||
|
||||
export { SettingsDialog } from "./settings-dialog";
|
||||
export { loadEmbeddingModelSettings } from "./api/embedding-model";
|
||||
export {
|
||||
loadHuggingFaceCacheSettings,
|
||||
updateHuggingFaceCacheSettings,
|
||||
} from "./api/hugging-face-cache";
|
||||
export type { HuggingFaceCacheSettings } from "./api/hugging-face-cache";
|
||||
export {
|
||||
loadPersonalization,
|
||||
savePersonalization,
|
||||
|
|
|
|||
|
|
@ -35,7 +35,10 @@ import {
|
|||
useRef,
|
||||
useState,
|
||||
} from "react";
|
||||
import { SETTINGS_SEARCH_INDEX } from "./settings-search";
|
||||
import {
|
||||
SETTINGS_SEARCH_INDEX,
|
||||
SETTINGS_SEARCH_KEYWORDS,
|
||||
} from "./settings-search";
|
||||
import {
|
||||
type SettingsTab,
|
||||
useSettingsDialogStore,
|
||||
|
|
@ -157,8 +160,12 @@ export function SettingsDialog() {
|
|||
return TABS.map((tab) => {
|
||||
const tabLabel = t(tab.labelKey);
|
||||
const entries = SETTINGS_SEARCH_INDEX[tab.id]
|
||||
.map((key) => t(key))
|
||||
.filter((label) => label.toLowerCase().includes(q));
|
||||
.filter((key) => {
|
||||
if (t(key).toLowerCase().includes(q)) return true;
|
||||
const keywordsKey = SETTINGS_SEARCH_KEYWORDS[key];
|
||||
return keywordsKey ? t(keywordsKey).toLowerCase().includes(q) : false;
|
||||
})
|
||||
.map((key) => t(key));
|
||||
const deduped = [...new Set(entries)];
|
||||
return {
|
||||
tab,
|
||||
|
|
|
|||
|
|
@ -146,3 +146,15 @@ export const SETTINGS_SEARCH_INDEX: Record<SettingsTab, TranslationKey[]> = {
|
|||
"settings.about.shutDownStudio",
|
||||
],
|
||||
};
|
||||
|
||||
/**
|
||||
* Extra terms a row matches on, beyond its own label. The value is a
|
||||
* translation key holding space-separated synonyms; it is never rendered.
|
||||
* Search matched labels only, so "models folder" or "directory" found nothing.
|
||||
*/
|
||||
export const SETTINGS_SEARCH_KEYWORDS: Partial<
|
||||
Record<TranslationKey, TranslationKey>
|
||||
> = {
|
||||
"settings.resources.storage.modelsFolder":
|
||||
"settings.resources.storage.modelsFolderKeywords",
|
||||
};
|
||||
|
|
|
|||
|
|
@ -74,6 +74,8 @@ const PREFS_KEYS: string[] = [
|
|||
LOCALE_STORAGE_KEY,
|
||||
// UI state
|
||||
"sidebar_pinned",
|
||||
"sidebar_width",
|
||||
"chat_settings_width",
|
||||
"unsloth_sidebar_navigate_open",
|
||||
"unsloth_settings_active_tab",
|
||||
// Chat runtime prefs
|
||||
|
|
|
|||
20
studio/frontend/src/hooks/use-chat-settings-width.ts
Normal file
20
studio/frontend/src/hooks/use-chat-settings-width.ts
Normal file
|
|
@ -0,0 +1,20 @@
|
|||
// SPDX-License-Identifier: AGPL-3.0-only
|
||||
// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
|
||||
|
||||
import { createPanelWidthStore } from "./use-panel-width.ts";
|
||||
|
||||
/** The previous fixed 17rem, at a 16px root font size. */
|
||||
export const CHAT_SETTINGS_WIDTH_DEFAULT = 272;
|
||||
/** Below this the sliders and their value pills start colliding. */
|
||||
export const CHAT_SETTINGS_WIDTH_MIN = 248;
|
||||
export const CHAT_SETTINGS_WIDTH_MAX = 560;
|
||||
|
||||
const store = createPanelWidthStore({
|
||||
key: "chat_settings_width",
|
||||
min: CHAT_SETTINGS_WIDTH_MIN,
|
||||
max: CHAT_SETTINGS_WIDTH_MAX,
|
||||
fallback: CHAT_SETTINGS_WIDTH_DEFAULT,
|
||||
});
|
||||
|
||||
export const clampChatSettingsWidth = store.clamp;
|
||||
export const useChatSettingsWidth = store.useWidth;
|
||||
138
studio/frontend/src/hooks/use-panel-width.ts
Normal file
138
studio/frontend/src/hooks/use-panel-width.ts
Normal file
|
|
@ -0,0 +1,138 @@
|
|||
// SPDX-License-Identifier: AGPL-3.0-only
|
||||
// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
|
||||
|
||||
import { useCallback, useSyncExternalStore } from "react";
|
||||
|
||||
/** Never let one panel eat more than this share of a narrow window. */
|
||||
const MAX_VIEWPORT_FRACTION = 0.4;
|
||||
|
||||
export type PanelWidthStore = {
|
||||
/** Clamps to what the current viewport allows. */
|
||||
clamp: (px: number) => number;
|
||||
useWidth: () => {
|
||||
width: number;
|
||||
max: number;
|
||||
/** The uncapped stored preference. */
|
||||
stored: number;
|
||||
setWidth: (value: number) => void;
|
||||
resetWidth: () => void;
|
||||
};
|
||||
};
|
||||
|
||||
/**
|
||||
* A persisted, viewport-aware width for a draggable panel. The preference is
|
||||
* stored whole and an effective width is derived from it, so narrowing the
|
||||
* window shrinks the panel without losing what the user picked.
|
||||
*/
|
||||
export function createPanelWidthStore({
|
||||
key,
|
||||
min,
|
||||
max,
|
||||
fallback,
|
||||
}: {
|
||||
key: string;
|
||||
min: number;
|
||||
max: number;
|
||||
fallback: number;
|
||||
}): PanelWidthStore {
|
||||
function maxWidth(): number {
|
||||
if (typeof window === "undefined") return max;
|
||||
// The floor wins on a narrow window; collapsing is the escape.
|
||||
return Math.max(min, Math.min(max, window.innerWidth * MAX_VIEWPORT_FRACTION));
|
||||
}
|
||||
|
||||
/** Clamps to the absolute range, ignoring the viewport. */
|
||||
function clampStored(px: number): number {
|
||||
if (!Number.isFinite(px)) return fallback;
|
||||
return Math.min(max, Math.max(min, Math.round(px)));
|
||||
}
|
||||
|
||||
function clamp(px: number): number {
|
||||
return Math.min(maxWidth(), clampStored(px));
|
||||
}
|
||||
|
||||
function load(): number {
|
||||
if (typeof window === "undefined") return fallback;
|
||||
try {
|
||||
const raw = window.localStorage.getItem(key);
|
||||
if (raw === null) return fallback;
|
||||
return clampStored(Number.parseFloat(raw));
|
||||
} catch {
|
||||
return fallback;
|
||||
}
|
||||
}
|
||||
|
||||
let storedWidth = load();
|
||||
let effectiveWidth = clamp(storedWidth);
|
||||
let effectiveMax = maxWidth();
|
||||
const listeners = new Set<() => void>();
|
||||
|
||||
let lastStored = storedWidth;
|
||||
|
||||
function recompute() {
|
||||
const nextWidth = clamp(storedWidth);
|
||||
const nextMax = maxWidth();
|
||||
if (
|
||||
nextWidth === effectiveWidth &&
|
||||
nextMax === effectiveMax &&
|
||||
storedWidth === lastStored
|
||||
) {
|
||||
return;
|
||||
}
|
||||
effectiveWidth = nextWidth;
|
||||
effectiveMax = nextMax;
|
||||
lastStored = storedWidth;
|
||||
listeners.forEach((cb) => cb());
|
||||
}
|
||||
|
||||
function subscribe(cb: () => void) {
|
||||
// With no subscribers there is no resize listener, so the cache can be
|
||||
// stale after a resize on a route that hides every panel. Refresh first;
|
||||
// useSyncExternalStore re-reads the snapshot right after subscribing.
|
||||
recompute();
|
||||
listeners.add(cb);
|
||||
if (typeof window === "undefined") {
|
||||
return () => listeners.delete(cb);
|
||||
}
|
||||
// Keep tabs in sync, same as the pin flag.
|
||||
const onStorage = (e: StorageEvent) => {
|
||||
if (e.key === key || e.key === null) {
|
||||
storedWidth = load();
|
||||
effectiveWidth = clamp(storedWidth);
|
||||
effectiveMax = maxWidth();
|
||||
cb();
|
||||
}
|
||||
};
|
||||
window.addEventListener("storage", onStorage);
|
||||
window.addEventListener("resize", recompute);
|
||||
return () => {
|
||||
listeners.delete(cb);
|
||||
window.removeEventListener("storage", onStorage);
|
||||
window.removeEventListener("resize", recompute);
|
||||
};
|
||||
}
|
||||
|
||||
function setWidthGlobal(next: number) {
|
||||
const stored = clampStored(next);
|
||||
if (stored !== storedWidth) {
|
||||
storedWidth = stored;
|
||||
try {
|
||||
window.localStorage.setItem(key, String(stored));
|
||||
} catch {}
|
||||
}
|
||||
recompute();
|
||||
}
|
||||
|
||||
function useWidth() {
|
||||
const width = useSyncExternalStore(subscribe, () => effectiveWidth, () => fallback);
|
||||
// What the viewport actually allows right now, for aria-valuemax.
|
||||
const panelMax = useSyncExternalStore(subscribe, () => effectiveMax, () => max);
|
||||
// The uncapped preference, so a capped drag can avoid lowering it.
|
||||
const preference = useSyncExternalStore(subscribe, () => storedWidth, () => fallback);
|
||||
const setWidth = useCallback((value: number) => setWidthGlobal(value), []);
|
||||
const resetWidth = useCallback(() => setWidthGlobal(fallback), []);
|
||||
return { width, max: panelMax, stored: preference, setWidth, resetWidth };
|
||||
}
|
||||
|
||||
return { clamp, useWidth };
|
||||
}
|
||||
146
studio/frontend/src/hooks/use-release-notes.ts
Normal file
146
studio/frontend/src/hooks/use-release-notes.ts
Normal file
|
|
@ -0,0 +1,146 @@
|
|||
// SPDX-License-Identifier: AGPL-3.0-only
|
||||
// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
|
||||
|
||||
import { authFetch, hasAuthToken } from "@/features/auth";
|
||||
import { apiUrl } from "@/lib/api-base";
|
||||
import { useCallback, useEffect, useRef, useState } from "react";
|
||||
|
||||
// Keyed to one exact version, so a new update never pairs with older notes.
|
||||
export interface ReleaseNotes {
|
||||
version: string;
|
||||
markdown: string | null;
|
||||
matched: boolean;
|
||||
truncated: boolean;
|
||||
source: string | null;
|
||||
releaseNotesUrl: string | null;
|
||||
// Set when the lookup itself failed, as opposed to a version with no notes.
|
||||
error: string | null;
|
||||
}
|
||||
|
||||
export type ReleaseNotesState = "idle" | "loading" | "ready" | "error";
|
||||
|
||||
// Desktop auto-auth installs its token after first paint, so a startup popup can
|
||||
// ask before one exists. Wait briefly rather than fail.
|
||||
const AUTH_POLL_MS = 250;
|
||||
const AUTH_POLL_LIMIT = 40;
|
||||
|
||||
interface UseReleaseNotesOptions {
|
||||
version: string | null | undefined;
|
||||
enabled?: boolean;
|
||||
}
|
||||
|
||||
type ApiObject = Record<string, unknown>;
|
||||
|
||||
function stringOrNull(value: ApiObject, key: string): string | null {
|
||||
const field = value[key];
|
||||
return typeof field === "string" && field.length > 0 ? field : null;
|
||||
}
|
||||
|
||||
function toReleaseNotes(value: unknown, version: string): ReleaseNotes | null {
|
||||
if (!value || typeof value !== "object") {
|
||||
return null;
|
||||
}
|
||||
const payload = value as ApiObject;
|
||||
const notesVersion = stringOrNull(payload, "version");
|
||||
// A response for another version is not usable here.
|
||||
if (notesVersion !== version) {
|
||||
return null;
|
||||
}
|
||||
const markdown = stringOrNull(payload, "markdown");
|
||||
return {
|
||||
version,
|
||||
markdown,
|
||||
matched: payload.matched === true && markdown !== null,
|
||||
truncated: payload.truncated === true,
|
||||
source: stringOrNull(payload, "source"),
|
||||
releaseNotesUrl: stringOrNull(payload, "release_notes_url"),
|
||||
error: stringOrNull(payload, "error"),
|
||||
};
|
||||
}
|
||||
|
||||
async function fetchReleaseNotes(
|
||||
version: string,
|
||||
refresh = false,
|
||||
): Promise<ReleaseNotes | null> {
|
||||
const query = `version=${encodeURIComponent(version)}${refresh ? "&refresh=true" : ""}`;
|
||||
// authFetch, not fetch: an expired token is refreshed and retried.
|
||||
const res = await authFetch(apiUrl(`/api/studio/release-notes?${query}`));
|
||||
if (!res.ok) {
|
||||
throw new Error(`Release notes request failed: ${res.status}`);
|
||||
}
|
||||
|
||||
return toReleaseNotes(await res.json(), version);
|
||||
}
|
||||
|
||||
export function useReleaseNotes({
|
||||
version,
|
||||
enabled = true,
|
||||
}: UseReleaseNotesOptions) {
|
||||
const [state, setState] = useState<ReleaseNotesState>("idle");
|
||||
const [notes, setNotes] = useState<ReleaseNotes | null>(null);
|
||||
// Version the current state belongs to; a change invalidates it.
|
||||
const requestedVersionRef = useRef<string | null>(null);
|
||||
// Identifies one request, so an earlier response cannot overwrite a later one.
|
||||
const requestIdRef = useRef(0);
|
||||
|
||||
const load = useCallback((target: string, refresh = false) => {
|
||||
requestedVersionRef.current = target;
|
||||
requestIdRef.current += 1;
|
||||
const requestId = requestIdRef.current;
|
||||
setState("loading");
|
||||
setNotes(null);
|
||||
fetchReleaseNotes(target, refresh)
|
||||
.then((next) => {
|
||||
// A newer request owns the state now.
|
||||
if (requestIdRef.current !== requestId) {
|
||||
return;
|
||||
}
|
||||
setNotes(next);
|
||||
// A reported failure is retryable; "no notes for this version" is not.
|
||||
const failed = !next || (!next.matched && next.error !== null);
|
||||
setState(failed ? "error" : "ready");
|
||||
})
|
||||
.catch(() => {
|
||||
if (requestIdRef.current === requestId) {
|
||||
setNotes(null);
|
||||
setState("error");
|
||||
}
|
||||
});
|
||||
}, []);
|
||||
|
||||
useEffect(() => {
|
||||
if (!enabled || !version || requestedVersionRef.current === version) {
|
||||
return;
|
||||
}
|
||||
if (hasAuthToken()) {
|
||||
load(version);
|
||||
return;
|
||||
}
|
||||
let attempts = 0;
|
||||
const timer = window.setInterval(() => {
|
||||
attempts += 1;
|
||||
if (hasAuthToken() || attempts >= AUTH_POLL_LIMIT) {
|
||||
window.clearInterval(timer);
|
||||
// Out of patience: load anyway so the panel settles on retry.
|
||||
load(version);
|
||||
}
|
||||
}, AUTH_POLL_MS);
|
||||
return () => window.clearInterval(timer);
|
||||
}, [enabled, version, load]);
|
||||
|
||||
const retry = useCallback(() => {
|
||||
if (version) {
|
||||
requestedVersionRef.current = null;
|
||||
// Bypass the cached remote failure, or retry waits for it to expire.
|
||||
load(version, true);
|
||||
}
|
||||
}, [version, load]);
|
||||
|
||||
// Never hand back another version's notes: state lags `version` by a render.
|
||||
const matchesVersion = notes !== null && notes.version === version;
|
||||
return {
|
||||
state: notes !== null && !matchesVersion ? "loading" : state,
|
||||
notes: matchesVersion ? notes : null,
|
||||
retry,
|
||||
};
|
||||
}
|
||||
21
studio/frontend/src/hooks/use-sidebar-width.ts
Normal file
21
studio/frontend/src/hooks/use-sidebar-width.ts
Normal file
|
|
@ -0,0 +1,21 @@
|
|||
// SPDX-License-Identifier: AGPL-3.0-only
|
||||
// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
|
||||
|
||||
import { createPanelWidthStore } from "./use-panel-width.ts";
|
||||
|
||||
/** The previous fixed 17.5rem, at a 16px root font size. */
|
||||
export const SIDEBAR_WIDTH_DEFAULT = 280;
|
||||
/** Narrowest width that still fits the wordmark. Firefox is the constraint:
|
||||
* it renders the heading ~3px wider than Chromium and WebKit. */
|
||||
export const SIDEBAR_WIDTH_MIN = 260;
|
||||
export const SIDEBAR_WIDTH_MAX = 480;
|
||||
|
||||
const store = createPanelWidthStore({
|
||||
key: "sidebar_width",
|
||||
min: SIDEBAR_WIDTH_MIN,
|
||||
max: SIDEBAR_WIDTH_MAX,
|
||||
fallback: SIDEBAR_WIDTH_DEFAULT,
|
||||
});
|
||||
|
||||
export const clampSidebarWidth = store.clamp;
|
||||
export const useSidebarWidth = store.useWidth;
|
||||
|
|
@ -21,6 +21,8 @@ export type UpdateStatus =
|
|||
export interface UpdateInfo {
|
||||
version: string;
|
||||
currentVersion: string;
|
||||
// Backend release this build pins; CHANGELOG.md is keyed by it, not the SemVer.
|
||||
pypiVersion?: string;
|
||||
body?: string;
|
||||
date?: string;
|
||||
}
|
||||
|
|
@ -42,10 +44,17 @@ interface DesktopUpdatePolicy {
|
|||
interface ManualUpdateInfo {
|
||||
version: string;
|
||||
currentVersion: string;
|
||||
pypiVersion?: string | null;
|
||||
body?: string;
|
||||
date?: string;
|
||||
}
|
||||
|
||||
/** `pypi_version` from latest.json, which the updater passes through raw. */
|
||||
function rawPypiVersion(raw: Record<string, unknown>): string | undefined {
|
||||
const value = raw.pypi_version;
|
||||
return typeof value === "string" && value.length > 0 ? value : undefined;
|
||||
}
|
||||
|
||||
export interface RetainedUpdateFailure {
|
||||
error: string;
|
||||
phase: UpdatePhase;
|
||||
|
|
@ -162,6 +171,7 @@ export function useTauriUpdate(isExternalServer = false) {
|
|||
setInfo({
|
||||
version: manualUpdate.version,
|
||||
currentVersion: manualUpdate.currentVersion,
|
||||
pypiVersion: manualUpdate.pypiVersion ?? undefined,
|
||||
body: manualUpdate.body,
|
||||
date: manualUpdate.date,
|
||||
});
|
||||
|
|
@ -197,6 +207,7 @@ export function useTauriUpdate(isExternalServer = false) {
|
|||
setInfo({
|
||||
version: update.version,
|
||||
currentVersion: update.currentVersion,
|
||||
pypiVersion: rawPypiVersion(update.rawJson),
|
||||
body: update.body,
|
||||
date: update.date,
|
||||
});
|
||||
|
|
@ -384,10 +395,13 @@ export function useTauriUpdate(isExternalServer = false) {
|
|||
});
|
||||
}
|
||||
|
||||
// Install target for Linux packages that cannot self-update.
|
||||
const manualReleaseUrl =
|
||||
updatePolicy.mode === "manual_linux_package" && info
|
||||
? manualReleasePageUrl(updatePolicy, info.version)
|
||||
: null;
|
||||
// Release page for the offered version, on every platform, for the notes link.
|
||||
const releasePageUrl = info ? manualReleasePageUrl(updatePolicy, info.version) : null;
|
||||
|
||||
return {
|
||||
status,
|
||||
|
|
@ -401,6 +415,7 @@ export function useTauriUpdate(isExternalServer = false) {
|
|||
isExternalServer,
|
||||
updatePolicyMode: updatePolicy.mode,
|
||||
manualReleaseUrl,
|
||||
releasePageUrl,
|
||||
installUpdate,
|
||||
retryUpdate,
|
||||
skipAndRestart,
|
||||
|
|
|
|||
|
|
@ -27,10 +27,18 @@ export const ar = {
|
|||
product: "Unsloth Studio",
|
||||
accountMenu: "قائمة حساب {name}",
|
||||
updateAvailable: "يتوفر تحديث",
|
||||
resize: {
|
||||
collapse: "انقر للطي",
|
||||
expand: "انقر للتوسيع",
|
||||
drag: "اسحب لتغيير الحجم",
|
||||
},
|
||||
aria: {
|
||||
home: "الصفحة الرئيسية لـ Unsloth",
|
||||
closeSidebar: "إغلاق الشريط الجانبي",
|
||||
openSidebar: "فتح الشريط الجانبي",
|
||||
resizeSidebar: "تغيير حجم الشريط الجانبي أو طيه",
|
||||
resizeRunSettings: "تغيير حجم إعدادات التشغيل أو إغلاقها",
|
||||
openRunSettings: "فتح إعدادات التشغيل",
|
||||
chatOptions: "خيارات المحادثة",
|
||||
runOptions: "خيارات التدريب",
|
||||
},
|
||||
|
|
@ -317,6 +325,8 @@ export const ar = {
|
|||
diskUsage: "{used} مستخدم / {total}",
|
||||
diskFree: "{free} متاح",
|
||||
modelsFolder: "مجلد النماذج",
|
||||
modelsFolderKeywords:
|
||||
"النماذج مجلد دليل مسار موقع تنزيلات التنزيل ذاكرة التخزين المؤقت تخزين قرص محرك نقل تغيير models folder path hugging face",
|
||||
modelsFolderDescription: "المكان الذي تُخزَّن فيه النماذج المُنزَّلة.",
|
||||
openAction: "فتح",
|
||||
copyAction: "نسخ المسار",
|
||||
|
|
|
|||
|
|
@ -27,10 +27,18 @@ export const de = {
|
|||
product: "Unsloth Studio",
|
||||
accountMenu: "Kontomenü von {name}",
|
||||
updateAvailable: "Update verfügbar",
|
||||
resize: {
|
||||
collapse: "Zum Einklappen klicken",
|
||||
expand: "Zum Ausklappen klicken",
|
||||
drag: "Zum Ändern der Größe ziehen",
|
||||
},
|
||||
aria: {
|
||||
home: "Unsloth Startseite",
|
||||
closeSidebar: "Seitenleiste schließen",
|
||||
openSidebar: "Seitenleiste öffnen",
|
||||
resizeSidebar: "Seitenleiste anpassen oder einklappen",
|
||||
resizeRunSettings: "Ausführungseinstellungen anpassen oder schließen",
|
||||
openRunSettings: "Ausführungseinstellungen öffnen",
|
||||
chatOptions: "Chat-Optionen",
|
||||
runOptions: "Trainingslauf-Optionen",
|
||||
},
|
||||
|
|
@ -330,9 +338,23 @@ export const de = {
|
|||
diskFree: "{free} frei",
|
||||
modelsFolder: "Modell-Ordner",
|
||||
modelsFolderDescription:
|
||||
"Wo heruntergeladene Modelle gespeichert werden.",
|
||||
"Wo heruntergeladene Modelle gespeichert werden. Ändern Sie ihn, um Modelle nicht auf dem Systemlaufwerk abzulegen.",
|
||||
modelsFolderKeywords:
|
||||
"Modelle Ordner Verzeichnis Pfad Speicherort Download Downloads Cache Speicher Festplatte Laufwerk verschieben ändern hugging face",
|
||||
futureDownloads: "Nur neue Downloads",
|
||||
environmentManaged:
|
||||
"Wird über die Umgebungsvariable {variable} verwaltet.",
|
||||
locationFree: "{free} frei",
|
||||
openAction: "Öffnen",
|
||||
copyAction: "Pfad kopieren",
|
||||
changeAction: "Ändern",
|
||||
resetAction: "Standard verwenden",
|
||||
chooseTitle: "Speicherort für Modell-Downloads wählen",
|
||||
chooseAction: "Für künftige Downloads verwenden",
|
||||
cacheSaved: "Speicherort für Modell-Downloads aktualisiert",
|
||||
cacheSaveError:
|
||||
"Der Speicherort für Modell-Downloads konnte nicht geändert werden",
|
||||
cachePickerError: "Die Ordnerauswahl konnte nicht geöffnet werden",
|
||||
copied: "Pfad kopiert",
|
||||
openError: "Der Ordner konnte nicht geöffnet werden",
|
||||
copyError: "Der Pfad konnte nicht kopiert werden",
|
||||
|
|
|
|||
|
|
@ -24,10 +24,18 @@ export const en = {
|
|||
product: "Unsloth Studio",
|
||||
accountMenu: "{name} account menu",
|
||||
updateAvailable: "Update available",
|
||||
resize: {
|
||||
collapse: "Click to collapse",
|
||||
expand: "Click to expand",
|
||||
drag: "Drag to resize",
|
||||
},
|
||||
aria: {
|
||||
home: "Unsloth home",
|
||||
closeSidebar: "Close sidebar",
|
||||
openSidebar: "Open sidebar",
|
||||
resizeSidebar: "Resize or collapse sidebar",
|
||||
resizeRunSettings: "Resize or close run settings",
|
||||
openRunSettings: "Open run settings",
|
||||
chatOptions: "Chat options",
|
||||
runOptions: "Run options",
|
||||
},
|
||||
|
|
@ -556,8 +564,12 @@ export const en = {
|
|||
systemDisk: "System disk",
|
||||
diskUsage: "{used} used / {total}",
|
||||
diskFree: "{free} free",
|
||||
modelsFolder: "Model downloads",
|
||||
modelsFolderDescription: "Hugging Face cache used for model downloads.",
|
||||
modelsFolder: "Models folder",
|
||||
modelsFolderDescription:
|
||||
"Where downloaded models are stored. Change it to keep models off your system drive.",
|
||||
// Not rendered: extra terms the settings search matches this row on.
|
||||
modelsFolderKeywords:
|
||||
"models folder directory path location download downloads cache storage disk drive move relocate hugging face",
|
||||
futureDownloads: "New downloads only",
|
||||
environmentManaged: "Managed by the {variable} environment variable.",
|
||||
locationFree: "{free} free",
|
||||
|
|
|
|||
|
|
@ -27,10 +27,18 @@ export const es = {
|
|||
product: "Unsloth Studio",
|
||||
accountMenu: "Menú de cuenta de {name}",
|
||||
updateAvailable: "Actualización disponible",
|
||||
resize: {
|
||||
collapse: "Haz clic para contraer",
|
||||
expand: "Haz clic para expandir",
|
||||
drag: "Arrastra para redimensionar",
|
||||
},
|
||||
aria: {
|
||||
home: "Inicio de Unsloth",
|
||||
closeSidebar: "Cerrar barra lateral",
|
||||
openSidebar: "Abrir barra lateral",
|
||||
resizeSidebar: "Redimensionar o contraer la barra lateral",
|
||||
resizeRunSettings: "Redimensionar o cerrar los ajustes de ejecución",
|
||||
openRunSettings: "Abrir los ajustes de ejecución",
|
||||
chatOptions: "Opciones de chat",
|
||||
runOptions: "Opciones de ejecución",
|
||||
},
|
||||
|
|
@ -328,6 +336,8 @@ export const es = {
|
|||
diskUsage: "{used} en uso / {total}",
|
||||
diskFree: "{free} libre",
|
||||
modelsFolder: "Carpeta de modelos",
|
||||
modelsFolderKeywords:
|
||||
"modelos carpeta directorio ruta ubicacion ubicación descargas descarga cache caché almacenamiento disco unidad mover cambiar models folder path hugging face",
|
||||
modelsFolderDescription:
|
||||
"Dónde se almacenan los modelos descargados.",
|
||||
openAction: "Abrir",
|
||||
|
|
|
|||
|
|
@ -27,10 +27,18 @@ export const fr = {
|
|||
product: "Unsloth Studio",
|
||||
accountMenu: "Menu du compte de {name}",
|
||||
updateAvailable: "Mise à jour disponible",
|
||||
resize: {
|
||||
collapse: "Cliquez pour réduire",
|
||||
expand: "Cliquez pour développer",
|
||||
drag: "Faites glisser pour redimensionner",
|
||||
},
|
||||
aria: {
|
||||
home: "Accueil Unsloth",
|
||||
closeSidebar: "Fermer la barre latérale",
|
||||
openSidebar: "Ouvrir la barre latérale",
|
||||
resizeSidebar: "Redimensionner ou réduire la barre latérale",
|
||||
resizeRunSettings: "Redimensionner ou fermer les paramètres d'exécution",
|
||||
openRunSettings: "Ouvrir les paramètres d'exécution",
|
||||
chatOptions: "Options de discussion",
|
||||
runOptions: "Options d'exécution",
|
||||
},
|
||||
|
|
@ -325,6 +333,8 @@ export const fr = {
|
|||
diskUsage: "{used} utilisé / {total}",
|
||||
diskFree: "{free} libre",
|
||||
modelsFolder: "Dossier des modèles",
|
||||
modelsFolderKeywords:
|
||||
"modeles modèles dossier repertoire répertoire chemin emplacement telechargements téléchargements cache stockage disque lecteur deplacer déplacer changer models folder path hugging face",
|
||||
modelsFolderDescription: "Emplacement de stockage des modèles téléchargés.",
|
||||
openAction: "Ouvrir",
|
||||
copyAction: "Copier le chemin",
|
||||
|
|
|
|||
|
|
@ -27,10 +27,18 @@ export const hi = {
|
|||
product: "Unsloth Studio",
|
||||
accountMenu: "{name} खाता मेनू",
|
||||
updateAvailable: "अपडेट उपलब्ध है",
|
||||
resize: {
|
||||
collapse: "छोटा करने के लिए क्लिक करें",
|
||||
expand: "विस्तार के लिए क्लिक करें",
|
||||
drag: "आकार बदलने के लिए खींचें",
|
||||
},
|
||||
aria: {
|
||||
home: "Unsloth होम",
|
||||
closeSidebar: "साइडबार बंद करें",
|
||||
openSidebar: "साइडबार खोलें",
|
||||
resizeSidebar: "साइडबार का आकार बदलें या छोटा करें",
|
||||
resizeRunSettings: "रन सेटिंग्स का आकार बदलें या बंद करें",
|
||||
openRunSettings: "रन सेटिंग्स खोलें",
|
||||
chatOptions: "चैट विकल्प",
|
||||
runOptions: "रन विकल्प",
|
||||
},
|
||||
|
|
@ -316,6 +324,8 @@ export const hi = {
|
|||
diskUsage: "{used} उपयोग में / {total}",
|
||||
diskFree: "{free} खाली",
|
||||
modelsFolder: "मॉडल फ़ोल्डर",
|
||||
modelsFolderKeywords:
|
||||
"मॉडल फ़ोल्डर फोल्डर निर्देशिका पथ स्थान डाउनलोड कैश संग्रहण डिस्क ड्राइव स्थानांतरित बदलें models folder path hugging face",
|
||||
modelsFolderDescription: "जहां डाउनलोड किए गए मॉडल संग्रहीत होते हैं।",
|
||||
openAction: "खोलें",
|
||||
copyAction: "पथ कॉपी करें",
|
||||
|
|
|
|||
|
|
@ -28,10 +28,18 @@ export const ja = {
|
|||
product: "Unsloth Studio",
|
||||
accountMenu: "{name} のアカウントメニュー",
|
||||
updateAvailable: "アップデートが利用可能です",
|
||||
resize: {
|
||||
collapse: "クリックで折りたたむ",
|
||||
expand: "クリックで展開",
|
||||
drag: "ドラッグでサイズ変更",
|
||||
},
|
||||
aria: {
|
||||
home: "Unsloth ホーム",
|
||||
closeSidebar: "サイドバーを閉じる",
|
||||
openSidebar: "サイドバーを開く",
|
||||
resizeSidebar: "サイドバーのサイズ変更または折りたたみ",
|
||||
resizeRunSettings: "実行設定のサイズ変更または閉じる",
|
||||
openRunSettings: "実行設定を開く",
|
||||
chatOptions: "チャットオプション",
|
||||
runOptions: "実行オプション",
|
||||
},
|
||||
|
|
@ -393,6 +401,8 @@ export const ja = {
|
|||
diskUsage: "{used} 使用中 / {total}",
|
||||
diskFree: "{free} 空き",
|
||||
modelsFolder: "モデルフォルダ",
|
||||
modelsFolderKeywords:
|
||||
"モデル フォルダ ディレクトリ パス 保存先 場所 ダウンロード キャッシュ ストレージ ディスク ドライブ 移動 変更 models folder path hugging face",
|
||||
modelsFolderDescription: "ダウンロードしたモデルの保存先。",
|
||||
openAction: "開く",
|
||||
copyAction: "パスをコピー",
|
||||
|
|
|
|||
|
|
@ -27,10 +27,18 @@ export const ko = {
|
|||
product: "Unsloth Studio",
|
||||
accountMenu: "{name} 계정 메뉴",
|
||||
updateAvailable: "업데이트 사용 가능",
|
||||
resize: {
|
||||
collapse: "클릭하여 접기",
|
||||
expand: "클릭하여 펼치기",
|
||||
drag: "드래그하여 크기 조절",
|
||||
},
|
||||
aria: {
|
||||
home: "Unsloth 홈",
|
||||
closeSidebar: "사이드바 닫기",
|
||||
openSidebar: "사이드바 열기",
|
||||
resizeSidebar: "사이드바 크기 조절 또는 접기",
|
||||
resizeRunSettings: "실행 설정 크기 조절 또는 닫기",
|
||||
openRunSettings: "실행 설정 열기",
|
||||
chatOptions: "채팅 옵션",
|
||||
runOptions: "학습 옵션",
|
||||
},
|
||||
|
|
@ -315,6 +323,8 @@ export const ko = {
|
|||
diskUsage: "{used} 사용 중 / {total}",
|
||||
diskFree: "{free} 여유",
|
||||
modelsFolder: "모델 폴더",
|
||||
modelsFolderKeywords:
|
||||
"모델 폴더 디렉터리 디렉토리 경로 위치 저장 다운로드 캐시 저장소 디스크 드라이브 이동 변경 models folder path hugging face",
|
||||
modelsFolderDescription: "다운로드한 모델이 저장되는 위치입니다.",
|
||||
openAction: "열기",
|
||||
copyAction: "경로 복사",
|
||||
|
|
|
|||
|
|
@ -27,10 +27,18 @@ export const ptBR = {
|
|||
product: "Unsloth Studio",
|
||||
accountMenu: "Menu de conta {name}",
|
||||
updateAvailable: "Atualização disponível",
|
||||
resize: {
|
||||
collapse: "Clique para recolher",
|
||||
expand: "Clique para expandir",
|
||||
drag: "Arraste para redimensionar",
|
||||
},
|
||||
aria: {
|
||||
home: "Início do Unsloth",
|
||||
closeSidebar: "Fechar barra lateral",
|
||||
openSidebar: "Abrir barra lateral",
|
||||
resizeSidebar: "Redimensionar ou recolher a barra lateral",
|
||||
resizeRunSettings: "Redimensionar ou fechar as configurações de execução",
|
||||
openRunSettings: "Abrir as configurações de execução",
|
||||
chatOptions: "Opções de chat",
|
||||
runOptions: "Opções de execução",
|
||||
},
|
||||
|
|
@ -417,6 +425,8 @@ export const ptBR = {
|
|||
diskUsage: "{used} usados / {total}",
|
||||
diskFree: "{free} livres",
|
||||
modelsFolder: "Pasta de modelos",
|
||||
modelsFolderKeywords:
|
||||
"modelos pasta diretorio diretório caminho local localizacao localização downloads baixar cache armazenamento disco unidade mover alterar models folder path hugging face",
|
||||
modelsFolderDescription: "Onde os modelos baixados são armazenados.",
|
||||
openAction: "Abrir",
|
||||
copyAction: "Copiar caminho",
|
||||
|
|
|
|||
|
|
@ -27,10 +27,18 @@ export const ru = {
|
|||
product: "Unsloth Studio",
|
||||
accountMenu: "Меню аккаунта {name}",
|
||||
updateAvailable: "Доступно обновление",
|
||||
resize: {
|
||||
collapse: "Нажмите, чтобы свернуть",
|
||||
expand: "Нажмите, чтобы развернуть",
|
||||
drag: "Потяните, чтобы изменить размер",
|
||||
},
|
||||
aria: {
|
||||
home: "Главная Unsloth",
|
||||
closeSidebar: "Закрыть боковую панель",
|
||||
openSidebar: "Открыть боковую панель",
|
||||
resizeSidebar: "Изменить размер или свернуть боковую панель",
|
||||
resizeRunSettings: "Изменить размер или закрыть настройки запуска",
|
||||
openRunSettings: "Открыть настройки запуска",
|
||||
chatOptions: "Параметры чата",
|
||||
runOptions: "Параметры запуска",
|
||||
},
|
||||
|
|
@ -316,6 +324,8 @@ export const ru = {
|
|||
diskUsage: "{used} использовано / {total}",
|
||||
diskFree: "{free} свободно",
|
||||
modelsFolder: "Папка моделей",
|
||||
modelsFolderKeywords:
|
||||
"модели папка каталог путь расположение загрузки кэш хранилище диск перенести изменить models folder path hugging face",
|
||||
modelsFolderDescription: "Где хранятся загруженные модели.",
|
||||
openAction: "Открыть",
|
||||
copyAction: "Копировать путь",
|
||||
|
|
|
|||
|
|
@ -27,10 +27,18 @@ export const zhCN = {
|
|||
product: "Unsloth Studio",
|
||||
accountMenu: "{name} 账号菜单",
|
||||
updateAvailable: "有可用更新",
|
||||
resize: {
|
||||
collapse: "点击折叠",
|
||||
expand: "点击展开",
|
||||
drag: "拖动调整大小",
|
||||
},
|
||||
aria: {
|
||||
home: "Unsloth 首页",
|
||||
closeSidebar: "关闭侧边栏",
|
||||
openSidebar: "打开侧边栏",
|
||||
resizeSidebar: "调整或折叠侧边栏",
|
||||
resizeRunSettings: "调整或关闭运行设置",
|
||||
openRunSettings: "打开运行设置",
|
||||
chatOptions: "聊天选项",
|
||||
runOptions: "训练选项",
|
||||
},
|
||||
|
|
@ -408,6 +416,8 @@ export const zhCN = {
|
|||
diskUsage: "已用 {used} / {total}",
|
||||
diskFree: "{free} 可用",
|
||||
modelsFolder: "模型文件夹",
|
||||
modelsFolderKeywords:
|
||||
"模型 文件夹 目录 路径 位置 下载 缓存 存储 磁盘 驱动器 移动 更改 models folder path hugging face",
|
||||
modelsFolderDescription: "已下载模型的存储位置。",
|
||||
openAction: "打开",
|
||||
copyAction: "复制路径",
|
||||
|
|
|
|||
|
|
@ -1363,6 +1363,23 @@ html[data-chat-font] .aui-root {
|
|||
cursor: pointer;
|
||||
}
|
||||
|
||||
/* While a panel edge is dragged, keep the resize cursor even as the pointer
|
||||
travels over buttons and text that would claim their own. */
|
||||
html[data-panel-resizing],
|
||||
html[data-panel-resizing] * {
|
||||
cursor: col-resize !important;
|
||||
user-select: none !important;
|
||||
}
|
||||
|
||||
html[data-panel-resizing]
|
||||
:is(
|
||||
[data-slot="sidebar-inner"],
|
||||
[data-slot="sidebar-inset"],
|
||||
[data-slot="chat-settings-panel"] > div
|
||||
) {
|
||||
pointer-events: none;
|
||||
}
|
||||
|
||||
/* Model selector: pointer cursor on every clickable element. */
|
||||
.unsloth-model-selector-trigger,
|
||||
.unsloth-model-selector-menu button {
|
||||
|
|
|
|||
664
studio/frontend/src/lib/changelog-links.ts
Normal file
664
studio/frontend/src/lib/changelog-links.ts
Normal file
|
|
@ -0,0 +1,664 @@
|
|||
// SPDX-License-Identifier: AGPL-3.0-only
|
||||
// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
|
||||
|
||||
/**
|
||||
* A relative link in CHANGELOG.md means "somewhere in the Unsloth repository",
|
||||
* but inside Studio it would resolve against Studio's own origin. Rewriting to
|
||||
* absolute repository URLs makes them behave the way GitHub renders the file.
|
||||
*/
|
||||
|
||||
import {
|
||||
type CodeSpan,
|
||||
codeSpans,
|
||||
insideSpan,
|
||||
} from "@/lib/markdown-code-spans";
|
||||
import { commentClosesBelow } from "@/lib/markdown-inline-comments";
|
||||
import {
|
||||
EMPTY_LIST_STATE,
|
||||
type ListState,
|
||||
NO_QUOTE,
|
||||
type QuoteState,
|
||||
containerContent,
|
||||
hiddenStructure,
|
||||
indentWidth,
|
||||
itemContent,
|
||||
openLists,
|
||||
quoteDepth,
|
||||
quoteState,
|
||||
} from "@/lib/markdown-list-columns";
|
||||
|
||||
const LINK_BASE = "https://github.com/unslothai/unsloth/blob/main/";
|
||||
const IMAGE_BASE = "https://raw.githubusercontent.com/unslothai/unsloth/main/";
|
||||
|
||||
// Inline `](dest)` plus the `[label]: dest` reference form. The destination is
|
||||
// either <bracketed> or runs to whitespace or the closing paren.
|
||||
const NESTED_LABEL = String.raw`((?:[^[\]\\]|\\.|\[(?:[^[\]\\]|\\.)*\])*)`;
|
||||
// Only ASCII punctuation is escapable, so the backslash in `a\ b.md` is an
|
||||
// ordinary character of the destination and the space still ends it.
|
||||
const ESCAPABLE = String.raw`[!-/:-@[-\`{-~]`;
|
||||
const DESTINATION_CHAR = String.raw`\\${ESCAPABLE}|[^\s()]`;
|
||||
// A destination may hold balanced parentheses, and a path may nest them, so
|
||||
// `[x](((draft)).md)` points at `((draft)).md`. An expression cannot count, so
|
||||
// pairs are unrolled to the depth cmark stops at, which is what GitHub renders.
|
||||
const MAX_DESTINATION_NESTING = 32;
|
||||
|
||||
/** A balanced parenthesised run nested up to `depth` levels deep. */
|
||||
function nestedParens(depth: number): string {
|
||||
let group = String.raw`\((?:${DESTINATION_CHAR})*\)`;
|
||||
for (let left = depth - 1; left > 0; left -= 1) {
|
||||
group = String.raw`\((?:${DESTINATION_CHAR}|${group})*\)`;
|
||||
}
|
||||
return group;
|
||||
}
|
||||
|
||||
const BALANCED_DESTINATION = String.raw`(?:${DESTINATION_CHAR}|${nestedParens(MAX_DESTINATION_NESTING)})*`;
|
||||
const PLAIN_DESTINATION = String.raw`(?:${DESTINATION_CHAR})*`;
|
||||
// A balanced pair counts only while a `)` or a title still closes the link
|
||||
// after it, or swallowing it would invent a link across lines.
|
||||
const CLOSES_LINK = String.raw`(?=[ \t]*[)'"])`;
|
||||
// A destination that runs out of line has its closer below it, the line being
|
||||
// only part of the link. One stopping short of a closer is no destination at all,
|
||||
// so `[x](a b.md)` and `[x](a(b.md)` stay plain text and keep the paths they name.
|
||||
const CLOSES_OR_ENDS_LINE = String.raw`(?=[ \t]*(?:[)'"]|$))`;
|
||||
const INLINE_TARGET = new RegExp(
|
||||
String.raw`(!?)\[${NESTED_LABEL}\]\(\s*(<[^<>\n]*>|${BALANCED_DESTINATION}${CLOSES_LINK}|${PLAIN_DESTINATION}${CLOSES_OR_ENDS_LINE})`,
|
||||
"g",
|
||||
);
|
||||
const REFERENCE_TARGET = /^( {0,3}\[((?:[^[\]\\]|\\.)*)\]:\s*)(<[^<>\n]*>|\S+)/;
|
||||
// `![alt][label]`, `![label][]` and `![label]`: a definition they point at
|
||||
// has to resolve to the raw file, not to its page on GitHub.
|
||||
const IMAGE_REFERENCE =
|
||||
/!\[((?:[^[\]\\]|\\.)*)\](?:\[((?:[^[\]\\]|\\.)*)\]|(?!\())/g;
|
||||
const FENCE = /^ {0,3}(`{3,}|~{3,})(.*)$/;
|
||||
// Four columns past the container start indented code, unless a paragraph is
|
||||
// open. Inside a list item that is measured from the item's content column, so a
|
||||
// link indented under a bullet is prose and still resolves.
|
||||
const INDENTED_CODE_INDENT = 4;
|
||||
// CommonMark type 1 HTML blocks show their contents verbatim.
|
||||
const RAW_HTML_OPEN = /^ {0,3}<(pre|script|style|textarea)(?=[\s>]|$)/i;
|
||||
const RAW_HTML_CLOSE = /<\/(pre|script|style|textarea)\s*>/i;
|
||||
// Type 6 and 7 blocks are literal too and run to the next blank line, not to a
|
||||
// closing tag, so `<details>` holds Markdown only after a blank line. Type 7 (any
|
||||
// other complete tag alone on a line) cannot interrupt a paragraph.
|
||||
const HTML_BLOCK_OPEN = /^ {0,3}<\/?([a-zA-Z][a-zA-Z0-9-]*)(?=[\s/>]|$)/;
|
||||
const HTML_ATTRIBUTE =
|
||||
"(?:\\s+[a-zA-Z_:][a-zA-Z0-9_.:-]*(?:\\s*=\\s*(?:[^\\s\"'=<>`]+|'[^']*'|\"[^\"]*\"))?)";
|
||||
const HTML_TAG_ONLY_LINE = new RegExp(
|
||||
`^ {0,3}(?:<[a-zA-Z][a-zA-Z0-9-]*${HTML_ATTRIBUTE}*\\s*/?>|</[a-zA-Z][a-zA-Z0-9-]*\\s*>)\\s*$`,
|
||||
);
|
||||
const HTML_BLOCK_TAGS = new Set(
|
||||
`address article aside base basefont blockquote body caption center col colgroup
|
||||
dd details dialog dir div dl dt fieldset figcaption figure footer form frame
|
||||
frameset h1 h2 h3 h4 h5 h6 head header hr html iframe legend li link main menu
|
||||
menuitem nav noframes ol optgroup option p param search section summary table
|
||||
tbody td tfoot th thead title tr track ul`.split(/\s+/),
|
||||
);
|
||||
// Lines that are blocks in their own right, so no paragraph is open after.
|
||||
const BLOCK_LINE =
|
||||
/^ {0,3}(?:#{1,6}([ \t]|$)|(?:\*[ \t]*){3,}$|(?:-[ \t]*){3,}$|(?:_[ \t]*){3,}$|>|=+[ \t]*$)/;
|
||||
// A definition is a block of its own but may not interrupt a paragraph, so it
|
||||
// ends the one above only when there is none to continue. It opens none either,
|
||||
// or consecutive definitions could never start (spec 0.31.2 section 4.7). Same
|
||||
// rule as `_LINK_DEFINITION` in the backend's `after_paragraph`.
|
||||
const LINK_DEFINITION = /^ {0,3}\[(?:[^[\]\\]|\\.)+\]:/;
|
||||
const LINE_ENDINGS = /\r\n?/g;
|
||||
// A scheme, a protocol-relative host, or a fragment: already absolute enough.
|
||||
// `//` needs a host after it, so `///docs` stays a repository path.
|
||||
const ABSOLUTE = /^(?:[a-zA-Z][a-zA-Z0-9+.-]*:|\/\/[^/]|#)/;
|
||||
|
||||
const COMMENT_OPEN = "<!--";
|
||||
const COMMENT_CLOSE = "-->";
|
||||
const COMMENT_BLOCK_OPEN = /^ {0,3}<!--/;
|
||||
|
||||
/**
|
||||
* `line` with its commented spans blanked, and whether a comment block is still
|
||||
* open below it. Commented content renders as nothing, so it holds no fence,
|
||||
* block or code span. Lengths are preserved so offsets still line up.
|
||||
*
|
||||
* Only a comment that starts a line opens a block (CommonMark type 2), and only
|
||||
* that runs on to the line holding `-->`, tail included. One written mid-sentence
|
||||
* is inline raw HTML belonging to its paragraph, so its `-->` may arrive on a
|
||||
* later line and only the text up to it is hidden. `closesBelow` says one does;
|
||||
* without it the opener is ordinary text, so a note merely mentioning `<!--` must
|
||||
* not hide the links below it.
|
||||
*
|
||||
* "Starts a line" is read inside the container, so `blockOpen` comes from the
|
||||
* item's content rather than the raw line.
|
||||
*/
|
||||
function maskComments(
|
||||
line: string,
|
||||
inComment: boolean,
|
||||
runOn: boolean,
|
||||
closesBelow: boolean,
|
||||
blockOpen: boolean,
|
||||
): [string, boolean, boolean] {
|
||||
if (inComment) {
|
||||
// The closing line belongs to the block, tail included.
|
||||
return [" ".repeat(line.length), !line.includes(COMMENT_CLOSE), false];
|
||||
}
|
||||
if (runOn) {
|
||||
const closed = line.indexOf(COMMENT_CLOSE);
|
||||
if (closed < 0) {
|
||||
return [" ".repeat(line.length), false, true];
|
||||
}
|
||||
// Only up to the closer: the tail is the paragraph's own text again.
|
||||
const resumed = closed + COMMENT_CLOSE.length;
|
||||
return maskInline(line, resumed, closesBelow);
|
||||
}
|
||||
if (blockOpen) {
|
||||
// `<!-->` and `<!--->` are complete comments, so the closer may overlap the
|
||||
// opener; searching past it would blank the rest of the file.
|
||||
return [" ".repeat(line.length), !line.includes(COMMENT_CLOSE), false];
|
||||
}
|
||||
return maskInline(line, 0, closesBelow);
|
||||
}
|
||||
|
||||
/** `maskComments` from `from`, where no comment block is open. */
|
||||
function maskInline(
|
||||
line: string,
|
||||
from: number,
|
||||
closesBelow: boolean,
|
||||
): [string, boolean, boolean] {
|
||||
let out = " ".repeat(from);
|
||||
let index = from;
|
||||
// Scanned only once an opener turns up. Spans are ordered and disjoint and each
|
||||
// opener sits at or past the last, so the search resumes rather than restarts.
|
||||
let spans: CodeSpan[] | null = null;
|
||||
let cursor = 0;
|
||||
while (index < line.length) {
|
||||
const start = line.indexOf(COMMENT_OPEN, index);
|
||||
if (start < 0) {
|
||||
return [out + line.slice(index), false, false];
|
||||
}
|
||||
spans ??= codeSpans(line);
|
||||
while (cursor < spans.length && (spans[cursor]?.end ?? 0) <= start) {
|
||||
cursor += 1;
|
||||
}
|
||||
// A delimiter inside inline code is literal, not a comment opener.
|
||||
const span = spans[cursor];
|
||||
if (span !== undefined && span.start <= start) {
|
||||
out += line.slice(index, span.end);
|
||||
index = span.end;
|
||||
continue;
|
||||
}
|
||||
// `<!-->` and `<!--->` are complete comments, so the closer may overlap.
|
||||
const close = line.indexOf(COMMENT_CLOSE, start + 2);
|
||||
if (close < 0) {
|
||||
if (closesBelow) {
|
||||
// The paragraph carries the comment on, so the line from the opener is
|
||||
// inside it, and so is the line below.
|
||||
return [
|
||||
out + line.slice(index, start) + " ".repeat(line.length - start),
|
||||
false,
|
||||
true,
|
||||
];
|
||||
}
|
||||
// Nothing closes it at all, so the renderer shows it as ordinary text.
|
||||
return [out + line.slice(index), false, false];
|
||||
}
|
||||
out += line.slice(index, start);
|
||||
out += " ".repeat(close + COMMENT_CLOSE.length - start);
|
||||
index = close + COMMENT_CLOSE.length;
|
||||
}
|
||||
return [out, false, false];
|
||||
}
|
||||
|
||||
/**
|
||||
* Whether `line` is written outside the container an open block belongs to. A
|
||||
* fence and an HTML block hold no lazy continuation line, so content left of the
|
||||
* item, or outside the quote, ends the block with its container. A raw block or
|
||||
* comment inside a list item ends on a blank line too: the item takes the break,
|
||||
* so what follows is a block of the item's own.
|
||||
*/
|
||||
function leavesContainer(
|
||||
line: string,
|
||||
quotes: number,
|
||||
column: number,
|
||||
blockQuotes: number,
|
||||
rawInItem: boolean,
|
||||
): boolean {
|
||||
if (quotes < blockQuotes) {
|
||||
return true;
|
||||
}
|
||||
if (!line.trim()) {
|
||||
return rawInItem;
|
||||
}
|
||||
return column > 0 && indentWidth(line) < column;
|
||||
}
|
||||
|
||||
/** True if `line` starts a CommonMark type 6 or type 7 HTML block. */
|
||||
function opensHtmlBlock(line: string, afterParagraph: boolean): boolean {
|
||||
const named = HTML_BLOCK_OPEN.exec(line);
|
||||
if (named && HTML_BLOCK_TAGS.has((named[1] ?? "").toLowerCase())) {
|
||||
return true;
|
||||
}
|
||||
return !afterParagraph && HTML_TAG_ONLY_LINE.test(line);
|
||||
}
|
||||
|
||||
/** A reference label as CommonMark compares them. */
|
||||
function label(text: string): string {
|
||||
return text.trim().replace(/\s+/g, " ").toLowerCase();
|
||||
}
|
||||
|
||||
const NEEDS_BRACKETS = /[()\s]/;
|
||||
// `\(` in a destination is a literal paren. Only ASCII punctuation is escapable,
|
||||
// so the backslash in `docs\alpha.md` is part of the path and has to survive.
|
||||
const ESCAPE = new RegExp(String.raw`\\(${ESCAPABLE})`, "g");
|
||||
// A URL parser reads a backslash as a path separator, so `docs\a.md` would
|
||||
// resolve to `docs/a.md`. Encode it first, the way a renderer normalises it.
|
||||
const BACKSLASH = /\\/g;
|
||||
// Only spaces and tabs may follow a closing fence.
|
||||
const NON_SPACE = /[^ \t]/;
|
||||
const LEADING_SLASHES = /^\/+/;
|
||||
|
||||
function absolute(target: string, image: boolean): string {
|
||||
const base = image ? IMAGE_BASE : LINK_BASE;
|
||||
const trimmed = target.trim().replace(ESCAPE, "$1");
|
||||
if (!trimmed || ABSOLUTE.test(trimmed)) {
|
||||
return target;
|
||||
}
|
||||
try {
|
||||
// A leading slash means the repository root, not the site root, so append
|
||||
// it to the base instead of replacing the base path.
|
||||
const resolved = new URL(
|
||||
trimmed.replace(LEADING_SLASHES, "").replace(BACKSLASH, "%5C"),
|
||||
base,
|
||||
).toString();
|
||||
// `../` can climb out of the repository: leave those alone.
|
||||
return resolved.startsWith(base) ? resolved : target;
|
||||
} catch {
|
||||
return target;
|
||||
}
|
||||
}
|
||||
|
||||
/** True when `index` is escaped by an odd run of backslashes. */
|
||||
function isEscaped(line: string, index: number): boolean {
|
||||
let slashes = 0;
|
||||
while (line[index - 1 - slashes] === "\\") {
|
||||
slashes += 1;
|
||||
}
|
||||
return slashes % 2 === 1;
|
||||
}
|
||||
|
||||
function unwrap(target: string): string {
|
||||
return target.startsWith("<") && target.endsWith(">")
|
||||
? target.slice(1, -1)
|
||||
: target;
|
||||
}
|
||||
|
||||
/** The destination as it goes back into the line. */
|
||||
function wrap(resolved: string, original: string): string {
|
||||
const bracketed = original.startsWith("<") && original.endsWith(">");
|
||||
return bracketed || (resolved !== original && NEEDS_BRACKETS.test(resolved))
|
||||
? `<${resolved}>`
|
||||
: resolved;
|
||||
}
|
||||
|
||||
/** Rewrites one line's link and image targets, leaving code spans alone. */
|
||||
function rewriteLine(
|
||||
line: string,
|
||||
imageLabels: Set<string>,
|
||||
spans: CodeSpan[],
|
||||
base: number,
|
||||
isDefinition: boolean,
|
||||
): string {
|
||||
const reference = isDefinition ? REFERENCE_TARGET.exec(line) : null;
|
||||
if (reference) {
|
||||
const target = reference[3] ?? "";
|
||||
const resolved = absolute(
|
||||
unwrap(target),
|
||||
imageLabels.has(label(reference[2] ?? "")),
|
||||
);
|
||||
const rest = line.slice(reference[0].length);
|
||||
return `${reference[1]}${wrap(resolved, target)}${rest}`;
|
||||
}
|
||||
|
||||
INLINE_TARGET.lastIndex = 0;
|
||||
return line.replace(INLINE_TARGET, (match, bang, text, target, offset) => {
|
||||
// `\\[` is a literal bracket, so the expression is not a link.
|
||||
const opener = offset + (bang ? 1 : 0);
|
||||
if (insideSpan(spans, base + offset) || isEscaped(line, opener)) {
|
||||
return match;
|
||||
}
|
||||
// `\\!` is a literal mark, so what follows is a link, not an image.
|
||||
const image = bang === "!" && !isEscaped(line, offset);
|
||||
const resolved = absolute(unwrap(target), image);
|
||||
// A badge nests an image inside a link, so the label is rewritten too.
|
||||
const inner = text.includes("](")
|
||||
? rewriteLine(text, imageLabels, codeSpans(text), 0, false)
|
||||
: text;
|
||||
return `${bang}[${inner}](${wrap(resolved, target)}`;
|
||||
});
|
||||
}
|
||||
|
||||
interface Classified {
|
||||
// Lines the renderer shows as Markdown, by index.
|
||||
text: number[];
|
||||
// Same lines, blanked where the renderer shows code, for span scanning.
|
||||
masked: string;
|
||||
// Lines where a `[label]: dest` definition can start.
|
||||
definition: Set<number>;
|
||||
// Document ranges the renderer hides inside HTML comments.
|
||||
comments: CodeSpan[];
|
||||
}
|
||||
|
||||
/**
|
||||
* Sorts lines into Markdown and code, masking the code so a span cannot pair
|
||||
* across it. Offsets are preserved, so a mask span sits where it does in the doc.
|
||||
*/
|
||||
function classify(lines: string[]): Classified {
|
||||
const text: number[] = [];
|
||||
const definition = new Set<number>();
|
||||
const masked: string[] = [];
|
||||
let openFence: string | null = null;
|
||||
let inRawHtml = false;
|
||||
let inHtmlBlock = false;
|
||||
// Where the open block was written: the content column of the item it belongs
|
||||
// to, 0 at document level, plus the blockquotes it sits inside. Only one is ever
|
||||
// open, and none holds a lazy continuation line, so a line left of the item or
|
||||
// outside the quote ends the block with its container.
|
||||
let blockColumn = 0;
|
||||
let blockQuotes = 0;
|
||||
let inComment = false;
|
||||
// True while an inline comment opened above runs on into this line, carried by
|
||||
// the paragraph holding it.
|
||||
let runOn = false;
|
||||
const closesBelow = commentClosesBelow(lines);
|
||||
let inCode = false;
|
||||
let afterParagraph = false;
|
||||
let quote: QuoteState = NO_QUOTE;
|
||||
let lists: ListState = EMPTY_LIST_STATE;
|
||||
const comments: CodeSpan[] = [];
|
||||
let offset = 0;
|
||||
|
||||
// The line as list tracking sees it: blank wherever nothing renders. Taken
|
||||
// with the paragraph state from the line above, as the renderer would.
|
||||
const track = (structural: string, above: QuoteState): void => {
|
||||
lists = openLists(structural, lists, afterParagraph, above.quoted);
|
||||
};
|
||||
// Where a block just opened sits, read after the opener closed the items it
|
||||
// is dedented out of, so it belongs to the container it is really in.
|
||||
const startBlock = (quotes: number): void => {
|
||||
blockColumn = lists.columns.at(-1) ?? 0;
|
||||
blockQuotes = quotes;
|
||||
};
|
||||
const endBlock = (): void => {
|
||||
blockColumn = 0;
|
||||
blockQuotes = 0;
|
||||
};
|
||||
|
||||
lines.forEach((original, index) => {
|
||||
const start = offset;
|
||||
offset += original.length + 1;
|
||||
// The quote state from the line above, which is what list tracking asks
|
||||
// about. Only plain text below rewrites it, so every block returning early
|
||||
// leaves no quoted paragraph open behind it.
|
||||
const above = quote;
|
||||
quote = NO_QUOTE;
|
||||
// A fence, comment or HTML block runs only to the end of the container it was
|
||||
// written in, so a line dedented out of that item or outside that quote
|
||||
// closes both.
|
||||
const quotes = quoteDepth(original);
|
||||
let inBlock = openFence !== null || inRawHtml || inHtmlBlock || inComment;
|
||||
if (
|
||||
inBlock &&
|
||||
leavesContainer(
|
||||
original,
|
||||
quotes,
|
||||
blockColumn,
|
||||
blockQuotes,
|
||||
(inRawHtml || inComment) && blockColumn > 0 && blockQuotes === 0,
|
||||
)
|
||||
) {
|
||||
openFence = null;
|
||||
inRawHtml = false;
|
||||
inHtmlBlock = false;
|
||||
inComment = false;
|
||||
endBlock();
|
||||
inBlock = false;
|
||||
}
|
||||
// Read from the container the line is written in, so a fence three columns
|
||||
// past a nested bullet or behind a quote marker still opens one. A block
|
||||
// already open keeps only its own quote stripped, or a deeper marker in it
|
||||
// would read as a closer.
|
||||
const container = containerContent(
|
||||
original,
|
||||
lists,
|
||||
inBlock ? blockQuotes : quotes,
|
||||
);
|
||||
// A comment cannot open a fence and a fence hides a comment opener, so resolve
|
||||
// them in that order or a hidden delimiter opens a phantom fence. An opener is
|
||||
// read past a marker on the same line too, since a fence written as an item's
|
||||
// first content opens inside it. Only an opener: fenced content is literal and
|
||||
// a closer carries no marker.
|
||||
const fenceSource = inComment
|
||||
? null
|
||||
: FENCE.exec(
|
||||
openFence === null
|
||||
? itemContent(container, afterParagraph)
|
||||
: container,
|
||||
);
|
||||
if (inRawHtml) {
|
||||
track("", above);
|
||||
inRawHtml = !RAW_HTML_CLOSE.test(container);
|
||||
if (!inRawHtml) {
|
||||
endBlock();
|
||||
}
|
||||
masked.push(" ".repeat(original.length));
|
||||
afterParagraph = false;
|
||||
return;
|
||||
}
|
||||
if (inHtmlBlock) {
|
||||
track("", above);
|
||||
// Only a blank line ends a type 6 or 7 block, so nothing inside one is a
|
||||
// fence or a link. A bare quote marker holds nothing, so it ends one too.
|
||||
inHtmlBlock = !!container.trim();
|
||||
if (!inHtmlBlock) {
|
||||
endBlock();
|
||||
}
|
||||
masked.push(" ".repeat(original.length));
|
||||
afterParagraph = false;
|
||||
return;
|
||||
}
|
||||
const fence = fenceSource;
|
||||
if (fence) {
|
||||
// A fence renders as nothing, but its indent still closes an item.
|
||||
track(original, above);
|
||||
const marker = fence[1] ?? "";
|
||||
if (openFence === null) {
|
||||
// A backtick fence's info string may not contain a backtick.
|
||||
openFence =
|
||||
marker[0] !== "`" || !(fence[2] ?? "").includes("`") ? marker : null;
|
||||
if (openFence === null) {
|
||||
text.push(index);
|
||||
masked.push(original);
|
||||
afterParagraph = true;
|
||||
return;
|
||||
}
|
||||
startBlock(quotes);
|
||||
} else if (
|
||||
// A closer matches the opening character and carries nothing after it.
|
||||
marker[0] === openFence[0] &&
|
||||
marker.length >= openFence.length &&
|
||||
!NON_SPACE.test(fence[2] ?? "")
|
||||
) {
|
||||
openFence = null;
|
||||
endBlock();
|
||||
}
|
||||
masked.push(" ".repeat(original.length));
|
||||
afterParagraph = false;
|
||||
return;
|
||||
}
|
||||
if (openFence !== null) {
|
||||
track("", above);
|
||||
// Fenced content is literal, so a comment opener in it is not one.
|
||||
masked.push(" ".repeat(original.length));
|
||||
return;
|
||||
}
|
||||
// A block already open owns this line, so it is content rather than a block
|
||||
// written at the column it happens to start in.
|
||||
const hidden = inComment;
|
||||
const carried = runOn;
|
||||
// A comment is an HTML block too, so one written as a list item's first
|
||||
// content opens inside that item exactly as a fence does: read past a marker
|
||||
// on the same line and from its container's column, not the line's margin.
|
||||
const opensComment =
|
||||
!(hidden || carried) &&
|
||||
COMMENT_BLOCK_OPEN.test(itemContent(container, afterParagraph));
|
||||
// Only now, outside every fence, does a comment hide what follows.
|
||||
const [line, stillInComment, stillRunOn] = maskComments(
|
||||
original,
|
||||
inComment,
|
||||
runOn,
|
||||
closesBelow[index + 1] ?? false,
|
||||
opensComment,
|
||||
);
|
||||
inComment = stillInComment;
|
||||
runOn = stillRunOn;
|
||||
// A line an inline comment runs on into is still a line of the paragraph
|
||||
// that carries it: only its text is hidden, never its block structure.
|
||||
const structure = carried ? original : line;
|
||||
// The same container reading as above, now the comments are masked. A comment
|
||||
// blanks its own line, so that line is read as written: the block renders as
|
||||
// nothing, but the item it is the content of still opens.
|
||||
const source = opensComment ? original : line;
|
||||
const visible = containerContent(source, lists, quotes);
|
||||
// An HTML block written as a list item's first content opens inside that item,
|
||||
// as a fence does, so an opener is read past a marker on the same line. The
|
||||
// marker survives into the structural line, so its item is still tracked.
|
||||
const content = itemContent(visible, afterParagraph);
|
||||
const marker =
|
||||
content === visible
|
||||
? ""
|
||||
: source.slice(0, source.length - content.length);
|
||||
// Taken before an HTML opener is hidden: it renders as nothing, but its indent
|
||||
// still closes a list item it sits left of. A comment or a <pre> keeps only its
|
||||
// column and marker, since the text it hides is not Markdown and opens no list.
|
||||
const opensRaw = !carried && RAW_HTML_OPEN.test(content);
|
||||
track(
|
||||
!(hidden || carried) && (opensRaw || !line.trim())
|
||||
? hiddenStructure(original, marker)
|
||||
: structure,
|
||||
above,
|
||||
);
|
||||
// Read once the opener has closed the items it is dedented out of, so the
|
||||
// comment block belongs to the item it is really written inside.
|
||||
if (inComment !== hidden) {
|
||||
if (inComment) {
|
||||
startBlock(quotes);
|
||||
} else {
|
||||
endBlock();
|
||||
}
|
||||
}
|
||||
for (let at = 0; at < line.length; at += 1) {
|
||||
if (line[at] === " " && original[at] !== " ") {
|
||||
const from = at;
|
||||
while (at < line.length && line[at] === " " && original[at] !== " ") {
|
||||
at += 1;
|
||||
}
|
||||
comments.push({ start: start + from, end: start + at, content: "" });
|
||||
}
|
||||
}
|
||||
if (opensRaw) {
|
||||
inRawHtml = !RAW_HTML_CLOSE.test(content.replace(RAW_HTML_OPEN, ""));
|
||||
if (inRawHtml) {
|
||||
startBlock(quotes);
|
||||
}
|
||||
masked.push(" ".repeat(line.length));
|
||||
afterParagraph = false;
|
||||
return;
|
||||
}
|
||||
if (!carried && content.trim() && opensHtmlBlock(content, afterParagraph)) {
|
||||
inHtmlBlock = true;
|
||||
startBlock(quotes);
|
||||
masked.push(" ".repeat(line.length));
|
||||
afterParagraph = false;
|
||||
return;
|
||||
}
|
||||
const blank = !structure.trim();
|
||||
// Measured from the innermost open item's content column, not the margin:
|
||||
// four spaces under "- Details:" is a paragraph, not a code block.
|
||||
const column = lists.columns.at(-1) ?? 0;
|
||||
const indented = indentWidth(structure) - column >= INDENTED_CODE_INDENT;
|
||||
// Indented code starts only outside a paragraph and runs to a dedent.
|
||||
if (inCode) {
|
||||
inCode = blank || indented;
|
||||
} else {
|
||||
inCode = !afterParagraph && !blank && indented;
|
||||
}
|
||||
if (inCode) {
|
||||
masked.push(" ".repeat(line.length));
|
||||
afterParagraph = false;
|
||||
return;
|
||||
}
|
||||
// A definition cannot interrupt a paragraph.
|
||||
if (!afterParagraph) {
|
||||
definition.add(index);
|
||||
}
|
||||
text.push(index);
|
||||
masked.push(line);
|
||||
afterParagraph =
|
||||
!blank &&
|
||||
!BLOCK_LINE.test(structure) &&
|
||||
(afterParagraph || !LINK_DEFINITION.test(structure));
|
||||
quote = quoteState(structure, above.inQuote);
|
||||
});
|
||||
|
||||
return { text, masked: masked.join("\n"), definition, comments };
|
||||
}
|
||||
|
||||
/** Absolute repository URLs for every relative link and image in `markdown`. */
|
||||
export function resolveChangelogLinks(markdown: string): string {
|
||||
// The desktop updater body arrives with CRLF, which would hide fences.
|
||||
const lines = markdown.replace(LINE_ENDINGS, "\n").split("\n");
|
||||
const { text, masked, definition, comments } = classify(lines);
|
||||
// Scanned over the whole document, so a span may cross a line break. Commented
|
||||
// ranges join them: the renderer shows neither, so a link in one is not
|
||||
// followable and rewriting it would only mutate hidden text.
|
||||
const spans = [...codeSpans(masked), ...comments].sort(
|
||||
(a, b) => a.start - b.start,
|
||||
);
|
||||
|
||||
// Offset of each line in the document, to place matches inside it.
|
||||
const offsets: number[] = [];
|
||||
let cursor = 0;
|
||||
for (const line of lines) {
|
||||
offsets.push(cursor);
|
||||
cursor += line.length + 1;
|
||||
}
|
||||
|
||||
// Only images resolve against the raw host, so collect the image labels
|
||||
// before rewriting any definition.
|
||||
const imageLabels = new Set<string>();
|
||||
for (const index of text) {
|
||||
const line = lines[index] ?? "";
|
||||
IMAGE_REFERENCE.lastIndex = 0;
|
||||
for (
|
||||
let match = IMAGE_REFERENCE.exec(line);
|
||||
match !== null;
|
||||
match = IMAGE_REFERENCE.exec(line)
|
||||
) {
|
||||
// An escaped mark makes it a link, so its definition stays a page URL.
|
||||
if (
|
||||
insideSpan(spans, (offsets[index] ?? 0) + match.index) ||
|
||||
isEscaped(line, match.index)
|
||||
) {
|
||||
continue;
|
||||
}
|
||||
const explicit = match[2] ?? "";
|
||||
imageLabels.add(label(explicit.trim() ? explicit : (match[1] ?? "")));
|
||||
}
|
||||
}
|
||||
|
||||
const rewritten = [...lines];
|
||||
for (const index of text) {
|
||||
rewritten[index] = rewriteLine(
|
||||
lines[index] ?? "",
|
||||
imageLabels,
|
||||
spans,
|
||||
offsets[index] ?? 0,
|
||||
definition.has(index),
|
||||
);
|
||||
}
|
||||
return rewritten.join("\n");
|
||||
}
|
||||
123
studio/frontend/src/lib/markdown-code-spans.ts
Normal file
123
studio/frontend/src/lib/markdown-code-spans.ts
Normal file
|
|
@ -0,0 +1,123 @@
|
|||
// SPDX-License-Identifier: AGPL-3.0-only
|
||||
// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
|
||||
|
||||
/**
|
||||
* CommonMark code spans: a backtick run closes only on an equal-length run.
|
||||
* That needs lookbehind, which older Safari rejects, so runs are scanned by hand.
|
||||
*/
|
||||
|
||||
export interface CodeSpan {
|
||||
// Offsets of the whole span, delimiters included.
|
||||
start: number;
|
||||
end: number;
|
||||
// Between the delimiters, with the one space of padding removed.
|
||||
content: string;
|
||||
}
|
||||
|
||||
function runLength(text: string, index: number): number {
|
||||
let end = index;
|
||||
while (text[end] === "`") {
|
||||
end += 1;
|
||||
}
|
||||
return end - index;
|
||||
}
|
||||
|
||||
/** True when `index` is escaped by an odd run of backslashes. */
|
||||
function escaped(text: string, index: number): boolean {
|
||||
let slashes = 0;
|
||||
while (text[index - 1 - slashes] === "\\") {
|
||||
slashes += 1;
|
||||
}
|
||||
return slashes % 2 === 1;
|
||||
}
|
||||
|
||||
/** CommonMark drops one space of padding, so `` ` a ` `` renders as "a". */
|
||||
function stripPadding(content: string): string {
|
||||
if (
|
||||
content.length > 1 &&
|
||||
content.startsWith(" ") &&
|
||||
content.endsWith(" ") &&
|
||||
content.trim() !== ""
|
||||
) {
|
||||
return content.slice(1, -1);
|
||||
}
|
||||
return content;
|
||||
}
|
||||
|
||||
/** Every code span in `text`, in order. Unclosed runs are ordinary text. */
|
||||
export function codeSpans(text: string): CodeSpan[] {
|
||||
const spans: CodeSpan[] = [];
|
||||
let index = 0;
|
||||
|
||||
while (index < text.length) {
|
||||
if (text[index] !== "`" || escaped(text, index)) {
|
||||
index += 1;
|
||||
continue;
|
||||
}
|
||||
const ticks = runLength(text, index);
|
||||
const contentStart = index + ticks;
|
||||
|
||||
let cursor = contentStart;
|
||||
let closed = false;
|
||||
while (cursor < text.length) {
|
||||
// Escapes do not apply inside a span, so a run after a backslash closes it.
|
||||
if (text[cursor] !== "`") {
|
||||
cursor += 1;
|
||||
continue;
|
||||
}
|
||||
const candidate = runLength(text, cursor);
|
||||
if (candidate === ticks) {
|
||||
spans.push({
|
||||
start: index,
|
||||
end: cursor + ticks,
|
||||
content: stripPadding(text.slice(contentStart, cursor)),
|
||||
});
|
||||
index = cursor + ticks;
|
||||
closed = true;
|
||||
break;
|
||||
}
|
||||
cursor += candidate;
|
||||
}
|
||||
if (!closed) {
|
||||
// Nothing closes this run: it is literal text, carry on after it.
|
||||
index = contentStart;
|
||||
}
|
||||
}
|
||||
return spans;
|
||||
}
|
||||
|
||||
/** Replaces every code span with `park(content)`, leaving the rest as is. */
|
||||
export function parkCodeSpans(
|
||||
text: string,
|
||||
park: (content: string) => string,
|
||||
): string {
|
||||
const spans = codeSpans(text);
|
||||
if (spans.length === 0) {
|
||||
return text;
|
||||
}
|
||||
let out = "";
|
||||
let cursor = 0;
|
||||
for (const span of spans) {
|
||||
out += text.slice(cursor, span.start) + park(span.content);
|
||||
cursor = span.end;
|
||||
}
|
||||
return out + text.slice(cursor);
|
||||
}
|
||||
|
||||
/** True when `index` falls inside one of `spans`, which are in order. */
|
||||
export function insideSpan(spans: CodeSpan[], index: number): boolean {
|
||||
let low = 0;
|
||||
let high = spans.length - 1;
|
||||
while (low <= high) {
|
||||
const mid = (low + high) >> 1;
|
||||
const span = spans[mid];
|
||||
if (span === undefined || index < span.start) {
|
||||
high = mid - 1;
|
||||
} else if (index >= span.end) {
|
||||
low = mid + 1;
|
||||
} else {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
62
studio/frontend/src/lib/markdown-inline-comments.ts
Normal file
62
studio/frontend/src/lib/markdown-inline-comments.ts
Normal file
|
|
@ -0,0 +1,62 @@
|
|||
// SPDX-License-Identifier: AGPL-3.0-only
|
||||
// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
|
||||
|
||||
/**
|
||||
* An HTML comment written mid-sentence is inline raw HTML, not a block, so it
|
||||
* belongs to its paragraph: the `-->` may arrive on a later line of that same
|
||||
* paragraph and everything between renders as nothing, while past the paragraph
|
||||
* the `<!--` is ordinary text. Both changelog scanners share that answer here.
|
||||
*
|
||||
* The backend needs none of it: a heading closes the paragraph it sits under, so
|
||||
* no heading can ever land inside one of these comments.
|
||||
*/
|
||||
|
||||
import { interruptsParagraph } from "@/lib/markdown-list-columns";
|
||||
|
||||
const COMMENT_CLOSE = "-->";
|
||||
// A line that cannot be more of the paragraph above it: blank, or a block that
|
||||
// may interrupt one. Leading punctuation is not one: `-->` alone is the ordinary
|
||||
// multiline close and a continuation may open with emphasis, so reading either as
|
||||
// a break leaves the comment unclosed and its text on show. Indented code and link
|
||||
// definitions are absent: neither may interrupt a paragraph (spec 0.31.2 4.4, 4.7).
|
||||
const BLANK = /^[ \t]*$/;
|
||||
const ATX_HEADING = /^ {0,3}#{1,6}([ \t]|$)/;
|
||||
const FENCE = /^ {0,3}(?:`{3,}|~{3,})/;
|
||||
const THEMATIC_BREAK =
|
||||
/^ {0,3}(?:(?:\*[ \t]*){3,}|(?:-[ \t]*){3,}|(?:_[ \t]*){3,})$/;
|
||||
// A row of `=` or `-` alone makes the paragraph above it a setext heading, ending it.
|
||||
const SETEXT_UNDERLINE = /^ {0,3}(?:=+|-+)[ \t]*$/;
|
||||
// A tag, comment or declaration at the start of a line. HTML block types 1 to 6
|
||||
// interrupt a paragraph; type 7 does not, but reading one as a break only leaves
|
||||
// the opener as plain text, which is what a leading `<` has always meant here.
|
||||
const HTML_LINE = /^ {0,3}</;
|
||||
|
||||
/** Whether `line` starts a block of its own rather than continuing a paragraph. */
|
||||
function startsBlock(line: string): boolean {
|
||||
return (
|
||||
BLANK.test(line) ||
|
||||
ATX_HEADING.test(line) ||
|
||||
FENCE.test(line) ||
|
||||
THEMATIC_BREAK.test(line) ||
|
||||
SETEXT_UNDERLINE.test(line) ||
|
||||
HTML_LINE.test(line) ||
|
||||
// Blockquote, or a list item with content: the rule the other scanners share.
|
||||
interruptsParagraph(line)
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* For each line, whether a `-->` is reachable without leaving the paragraph it
|
||||
* starts in. Read at `index + 1` it answers whether an inline comment opened on
|
||||
* `index` and left unclosed there is a comment at all.
|
||||
*/
|
||||
export function commentClosesBelow(lines: string[]): boolean[] {
|
||||
const closes: boolean[] = new Array(lines.length + 1).fill(false);
|
||||
for (let at = lines.length - 1; at >= 0; at -= 1) {
|
||||
const line = lines[at] ?? "";
|
||||
closes[at] =
|
||||
!startsBlock(line) &&
|
||||
(line.includes(COMMENT_CLOSE) || (closes[at + 1] ?? false));
|
||||
}
|
||||
return closes;
|
||||
}
|
||||
357
studio/frontend/src/lib/markdown-list-columns.ts
Normal file
357
studio/frontend/src/lib/markdown-list-columns.ts
Normal file
|
|
@ -0,0 +1,357 @@
|
|||
// SPDX-License-Identifier: AGPL-3.0-only
|
||||
// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
|
||||
|
||||
/**
|
||||
* CommonMark measures a block's indentation from its container, not the left
|
||||
* margin: four spaces at document level and four under a bullet mean different
|
||||
* things. Tracking the open items lets both changelog scanners ask "is this
|
||||
* indented code?" the way a renderer would.
|
||||
*
|
||||
* Ported from `_open_lists` in studio/backend/utils/changelog.py so the three
|
||||
* scanners classify a line the same way.
|
||||
*/
|
||||
|
||||
/** The open list items, innermost last, by the column their content starts. */
|
||||
export interface ListState {
|
||||
columns: number[];
|
||||
// True while the innermost item has had no content since its marker.
|
||||
emptyItem: boolean;
|
||||
}
|
||||
|
||||
export const EMPTY_LIST_STATE: ListState = { columns: [], emptyItem: false };
|
||||
|
||||
// The marker needs whitespace after it, so `2.0` is a version, not an item.
|
||||
const LIST_ITEM = /^[ \t]*([-*+]|\d{1,9}[.)])([ \t]+|$)/;
|
||||
const THEMATIC_BREAK =
|
||||
/^ {0,3}(?:(?:\*[ \t]*){3,}|(?:-[ \t]*){3,}|(?:_[ \t]*){3,})$/;
|
||||
const BLOCK_QUOTE = /^ {0,3}>/;
|
||||
const QUOTE_MARKER = /^ {0,3}>[ \t]?/;
|
||||
// Blocks that are not paragraph text, so they cannot continue one lazily.
|
||||
const PARAGRAPH_TEXT = /^ {0,3}(?|\d{1,9}[.)]([ \t]|$))\S/;
|
||||
// Blocks that break into an open paragraph, closing it rather than continuing
|
||||
// it. A link reference definition is not one of them.
|
||||
const INTERRUPTS =
|
||||
/^ {0,3}(?:#{1,6}([ \t]|$)|(?:\*[ \t]*){3,}$|(?:-[ \t]*){3,}$|(?:_[ \t]*){3,}$)/;
|
||||
const FENCE = /^ {0,3}(?:`{3,}|~{3,})/;
|
||||
const HTML_BLOCK_OPEN = /^ {0,3}<\/?([a-zA-Z][a-zA-Z0-9-]*)(?=[\s/>]|$)/;
|
||||
const HTML_BLOCK_TAGS = new Set(
|
||||
`address article aside base basefont blockquote body caption center col colgroup
|
||||
dd details dialog dir div dl dt fieldset figcaption figure footer form frame
|
||||
frameset h1 h2 h3 h4 h5 h6 head header hr html iframe legend li link main menu
|
||||
menuitem nav noframes ol optgroup option p param search section summary table
|
||||
tbody td tfoot th thead title tr track ul`.split(/\s+/),
|
||||
);
|
||||
// Content indented more than this after a marker is an indented code block, so
|
||||
// the item's content starts one column past the marker instead.
|
||||
const MAX_ITEM_PADDING = 4;
|
||||
// Columns past its container at which a line becomes an indented code block.
|
||||
const INDENTED_CODE = 4;
|
||||
// Stands in for a line the renderer hides. `#` is a block of its own, so list
|
||||
// tracking reads it like a comment: never a marker, never a lazy continuation.
|
||||
const HIDDEN_BLOCK = "#";
|
||||
const LEADING_SPACE = /^[ \t]*/;
|
||||
|
||||
/**
|
||||
* `line` as list tracking sees it once the renderer hides its text. A comment or
|
||||
* raw HTML block renders nothing but is still a block at its own column, so it
|
||||
* closes the items it sits left of. Only the indentation survives: what the block
|
||||
* hides is not Markdown and must not open a list. `marker` is the part opening
|
||||
* the item the block is content of, which survives too. Ported from
|
||||
* `_hidden_structure` on the backend.
|
||||
*/
|
||||
export function hiddenStructure(line: string, marker = ""): string {
|
||||
if (marker) {
|
||||
return `${marker}${HIDDEN_BLOCK}`;
|
||||
}
|
||||
const indent = LEADING_SPACE.exec(line)?.[0] ?? "";
|
||||
return line.trim() ? `${indent}${HIDDEN_BLOCK}` : "";
|
||||
}
|
||||
|
||||
/** Columns of leading whitespace, counting a tab to the next stop of four. */
|
||||
export function indentWidth(line: string): number {
|
||||
let width = 0;
|
||||
for (const char of line) {
|
||||
if (char === " ") {
|
||||
width += 1;
|
||||
} else if (char === "\t") {
|
||||
width += 4 - (width % 4);
|
||||
} else {
|
||||
break;
|
||||
}
|
||||
}
|
||||
return width;
|
||||
}
|
||||
|
||||
/**
|
||||
* Whether `line` starts a block that can break into an open paragraph. A quote
|
||||
* marker always can; a list item only with content, an ordered one only at 1.
|
||||
* Anything else is text of the paragraph it appears to interrupt.
|
||||
*/
|
||||
export function interruptsParagraph(line: string): boolean {
|
||||
if (BLOCK_QUOTE.test(line)) {
|
||||
return true;
|
||||
}
|
||||
const item = THEMATIC_BREAK.test(line) ? null : LIST_ITEM.exec(line);
|
||||
if (item === null) {
|
||||
return false;
|
||||
}
|
||||
const marker = item[1] ?? "";
|
||||
if (!line.slice(item[0].length).trim()) {
|
||||
return false;
|
||||
}
|
||||
const ordered = marker.endsWith(".") || marker.endsWith(")");
|
||||
return !ordered || marker.slice(0, -1) === "1";
|
||||
}
|
||||
|
||||
/**
|
||||
* Whether a marker-shaped `line` is really text of the paragraph above. Only a
|
||||
* marker inside the paragraph's own item interrupts it; one to the left closes
|
||||
* that item and opens a sibling. A quote owns the paragraph its lines hold, so a
|
||||
* marker outside the quote opens a list of its own.
|
||||
*/
|
||||
export function lazyMarker(
|
||||
line: string,
|
||||
state: ListState,
|
||||
afterParagraph: boolean,
|
||||
quoted: boolean,
|
||||
): boolean {
|
||||
const item = THEMATIC_BREAK.test(line) ? null : LIST_ITEM.exec(line);
|
||||
const columns = state.columns;
|
||||
const inside =
|
||||
columns.length === 0 || indentWidth(line) >= (columns.at(-1) ?? 0);
|
||||
return (
|
||||
item !== null &&
|
||||
afterParagraph &&
|
||||
!quoted &&
|
||||
inside &&
|
||||
!interruptsParagraph(line)
|
||||
);
|
||||
}
|
||||
|
||||
/** `columns` with every item whose content starts past `indent` closed. */
|
||||
function dropDeeper(columns: number[], indent: number): number[] {
|
||||
let open = columns.length;
|
||||
while (open > 0 && (columns[open - 1] ?? 0) > indent) {
|
||||
open -= 1;
|
||||
}
|
||||
return open === columns.length ? columns : columns.slice(0, open);
|
||||
}
|
||||
|
||||
/** `line` with up to `columns` columns of leading whitespace removed. */
|
||||
function stripIndent(line: string, columns: number): string {
|
||||
let width = 0;
|
||||
let index = 0;
|
||||
while (index < line.length && width < columns) {
|
||||
const char = line[index];
|
||||
if (char !== " " && char !== "\t") {
|
||||
break;
|
||||
}
|
||||
width += char === " " ? 1 : 4 - (width % 4);
|
||||
index += 1;
|
||||
}
|
||||
return line.slice(index);
|
||||
}
|
||||
|
||||
/**
|
||||
* Whether `line` can continue a paragraph it is indented out of. Only plain text
|
||||
* can: a heading, fence, break or HTML block starts a block of its own, closing
|
||||
* the item instead. An underline is not one: it may never be lazy, so `===` left
|
||||
* of an open item is more of the item's paragraph. Nor is a definition, a block
|
||||
* of its own that may not interrupt a paragraph. A row of dashes still closes the
|
||||
* item: `INTERRUPTS` reads three or more as the thematic break they are.
|
||||
*/
|
||||
function mayBeLazy(line: string): boolean {
|
||||
const named = HTML_BLOCK_OPEN.exec(line);
|
||||
// Types 1 to 6 interrupt a paragraph, so a `<div>` left of an open item closes
|
||||
// it. Type 7 cannot, and is deliberately excluded.
|
||||
const htmlBlock =
|
||||
named !== null && HTML_BLOCK_TAGS.has((named[1] ?? "").toLowerCase());
|
||||
return (
|
||||
PARAGRAPH_TEXT.test(line) &&
|
||||
!INTERRUPTS.test(line) &&
|
||||
!FENCE.test(line) &&
|
||||
!htmlBlock
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Whether `line` reads as more of a paragraph open in its container, measured
|
||||
* from `column` where that container's content starts: four columns past it the
|
||||
* line is indented code, which may not interrupt a paragraph, so indentation
|
||||
* alone never closes the one above.
|
||||
*/
|
||||
export function continuesParagraph(line: string, column: number): boolean {
|
||||
const inner = stripIndent(line, column);
|
||||
return indentWidth(inner) >= INDENTED_CODE || mayBeLazy(inner);
|
||||
}
|
||||
|
||||
/** `line` with up to `depth` blockquote markers removed, and how many went. */
|
||||
function stripQuotes(line: string, depth: number): [string, number] {
|
||||
let rest = line;
|
||||
let removed = 0;
|
||||
let marker = removed < depth ? QUOTE_MARKER.exec(rest) : null;
|
||||
while (marker !== null) {
|
||||
rest = rest.slice(marker[0].length);
|
||||
removed += 1;
|
||||
marker = removed < depth ? QUOTE_MARKER.exec(rest) : null;
|
||||
}
|
||||
return [rest, removed];
|
||||
}
|
||||
|
||||
/** What a blockquote line holds, with its markers stripped. */
|
||||
function quoteContent(line: string): string {
|
||||
return stripQuotes(line, Number.POSITIVE_INFINITY)[0];
|
||||
}
|
||||
|
||||
/** How many blockquotes `line` is written inside. */
|
||||
export function quoteDepth(line: string): number {
|
||||
return stripQuotes(line, Number.POSITIVE_INFINITY)[1];
|
||||
}
|
||||
|
||||
/**
|
||||
* `line` as the container it is written in sees it, with `quotes` blockquote
|
||||
* markers and the open item's content column removed. CommonMark measures a block
|
||||
* from its container, not the margin (spec 0.31.2 sections 5.1, 5.2), so `> ~~~`
|
||||
* and a fence under a nested bullet are openers despite sitting more than three
|
||||
* columns in.
|
||||
*/
|
||||
export function containerContent(
|
||||
line: string,
|
||||
state: ListState,
|
||||
quotes: number,
|
||||
): string {
|
||||
const [inner] = stripQuotes(line, quotes);
|
||||
if (quotes > 0) {
|
||||
// A list inside a quote is the quote's own; this tracker follows document
|
||||
// level only, so its columns do not apply here.
|
||||
return inner;
|
||||
}
|
||||
const columns = dropDeeper(state.columns, indentWidth(inner));
|
||||
return stripIndent(inner, columns.at(-1) ?? 0);
|
||||
}
|
||||
|
||||
/**
|
||||
* `line` read from the content column of a list item that opens on it. A block
|
||||
* written as an item's first content sits inside that item, so ``- ``` `` opens a
|
||||
* fence even though its marker is not within three columns of the container (spec
|
||||
* 0.31.2 section 5.2). Padding is capped the way `openLists` caps it, or
|
||||
* ``- ``` `` would read as a fence rather than the indented code it is. A
|
||||
* marker the paragraph above swallows opens no item, so its line is returned
|
||||
* whole, as is one four columns past its container.
|
||||
*/
|
||||
export function itemContent(line: string, afterParagraph: boolean): string {
|
||||
if (
|
||||
indentWidth(line) >= INDENTED_CODE ||
|
||||
(afterParagraph && !interruptsParagraph(line))
|
||||
) {
|
||||
return line;
|
||||
}
|
||||
const item = THEMATIC_BREAK.test(line) ? null : LIST_ITEM.exec(line);
|
||||
if (item === null) {
|
||||
return line;
|
||||
}
|
||||
const padding = indentWidth(item[2] ?? "");
|
||||
// Over-indented content starts one column past the marker; the rest of the
|
||||
// padding is the content's own indentation.
|
||||
const over = padding > MAX_ITEM_PADDING ? padding - 1 : 0;
|
||||
return `${" ".repeat(over)}${line.slice(item[0].length)}`;
|
||||
}
|
||||
|
||||
/** Whether a blockquote owns the paragraph the line below could continue. */
|
||||
export interface QuoteState {
|
||||
// True while a quoted paragraph is open, so plain text below is more of it.
|
||||
inQuote: boolean;
|
||||
// True whenever that paragraph is the quote's rather than the document's.
|
||||
quoted: boolean;
|
||||
}
|
||||
|
||||
export const NO_QUOTE: QuoteState = { inQuote: false, quoted: false };
|
||||
|
||||
/**
|
||||
* The quote state after `line`, given the state after the line above and the
|
||||
* content column of the item `line` sits in. A quote owns the paragraph its own
|
||||
* lines hold, so a marker written outside the quote opens a list of its own
|
||||
* rather than reading as more of that paragraph. Ported from `in_quote` tracking
|
||||
* in changelog.py.
|
||||
*/
|
||||
export function quoteState(
|
||||
line: string,
|
||||
inQuote: boolean,
|
||||
column = 0,
|
||||
): QuoteState {
|
||||
if (BLOCK_QUOTE.test(line)) {
|
||||
// An empty quote holds no paragraph, so the line below starts a new one.
|
||||
return { inQuote: mayBeLazy(quoteContent(line)), quoted: true };
|
||||
}
|
||||
const open = inQuote && continuesParagraph(line, column);
|
||||
return { inQuote: open, quoted: open };
|
||||
}
|
||||
|
||||
/**
|
||||
* `columns` with every item `line` is written to the left of closed. Read inside
|
||||
* the container the item sits in, not from the margin: a line that only looks
|
||||
* dedented there is lazy text of the item's paragraph, leaving the item open.
|
||||
*/
|
||||
function closeDedented(
|
||||
columns: number[],
|
||||
line: string,
|
||||
indent: number,
|
||||
afterParagraph: boolean,
|
||||
): number[] {
|
||||
let open = columns.length;
|
||||
while (open > 0 && (columns[open - 1] ?? 0) > indent) {
|
||||
const outer = open > 1 ? (columns[open - 2] ?? 0) : 0;
|
||||
if (afterParagraph && continuesParagraph(line, outer)) {
|
||||
break;
|
||||
}
|
||||
open -= 1;
|
||||
}
|
||||
return open === columns.length ? columns : columns.slice(0, open);
|
||||
}
|
||||
|
||||
/**
|
||||
* The list items still open after `line`. A dedented line closes an item unless
|
||||
* it is a lazy paragraph continuation. A new marker nests under a deeper column
|
||||
* and replaces a sibling. `quoted` marks a paragraph the blockquote above owns:
|
||||
* a marker outside the quote is not text of it, so it opens a list of its own.
|
||||
*/
|
||||
export function openLists(
|
||||
line: string,
|
||||
state: ListState,
|
||||
afterParagraph: boolean,
|
||||
quoted = false,
|
||||
): ListState {
|
||||
let columns = state.columns;
|
||||
if (!line.trim()) {
|
||||
// A blank line leaves the list open, unless the item is still empty: an
|
||||
// item may begin with one blank line, and later content is outside it.
|
||||
return {
|
||||
columns: state.emptyItem ? columns.slice(0, -1) : columns,
|
||||
emptyItem: false,
|
||||
};
|
||||
}
|
||||
const indent = indentWidth(line);
|
||||
const item = THEMATIC_BREAK.test(line) ? null : LIST_ITEM.exec(line);
|
||||
const empty = item !== null && !line.slice(item[0].length).trim();
|
||||
if (lazyMarker(line, state, afterParagraph, quoted)) {
|
||||
// A lazy continuation or an underline, so the open items are untouched.
|
||||
return state;
|
||||
}
|
||||
columns = closeDedented(columns, line, indent, afterParagraph);
|
||||
// Four columns past its container the marker is an indented code block, or
|
||||
// lazy text of the paragraph above it, so it opens no list of its own.
|
||||
if (item === null || indent - (columns.at(-1) ?? 0) >= INDENTED_CODE) {
|
||||
return { columns, emptyItem: false };
|
||||
}
|
||||
const marker = item[1] ?? "";
|
||||
let padding = indentWidth(item[2] ?? "");
|
||||
if (padding === 0 || padding > MAX_ITEM_PADDING) {
|
||||
// An empty or over-indented item still holds one column of content.
|
||||
padding = 1;
|
||||
}
|
||||
// A sibling marker replaces the item it lines up with.
|
||||
return {
|
||||
columns: [...dropDeeper(columns, indent), indent + marker.length + padding],
|
||||
emptyItem: empty,
|
||||
};
|
||||
}
|
||||
1005
studio/frontend/src/lib/release-notes-preview.ts
Normal file
1005
studio/frontend/src/lib/release-notes-preview.ts
Normal file
File diff suppressed because it is too large
Load diff
72
studio/frontend/tests/sidebar-width.test.ts
Normal file
72
studio/frontend/tests/sidebar-width.test.ts
Normal file
|
|
@ -0,0 +1,72 @@
|
|||
// SPDX-License-Identifier: AGPL-3.0-only
|
||||
// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
|
||||
|
||||
import assert from "node:assert/strict";
|
||||
import test from "node:test";
|
||||
import { readFile } from "node:fs/promises";
|
||||
|
||||
// Every localStorage key written by a panel width store.
|
||||
const PANEL_WIDTH_KEYS = ["sidebar_width", "chat_settings_width"];
|
||||
|
||||
// The store reads window at import time, so stub it before importing.
|
||||
const stubWindow = {
|
||||
innerWidth: 1440,
|
||||
localStorage: {
|
||||
getItem: () => null,
|
||||
setItem: () => {},
|
||||
},
|
||||
addEventListener: () => {},
|
||||
removeEventListener: () => {},
|
||||
};
|
||||
(globalThis as { window?: unknown }).window = stubWindow;
|
||||
|
||||
const {
|
||||
clampSidebarWidth,
|
||||
SIDEBAR_WIDTH_DEFAULT,
|
||||
SIDEBAR_WIDTH_MAX,
|
||||
SIDEBAR_WIDTH_MIN,
|
||||
} = await import("../src/hooks/use-sidebar-width.ts");
|
||||
|
||||
test("clamps to the absolute range on a roomy window", () => {
|
||||
stubWindow.innerWidth = 1440;
|
||||
assert.equal(clampSidebarWidth(320), 320);
|
||||
assert.equal(clampSidebarWidth(SIDEBAR_WIDTH_MAX + 200), SIDEBAR_WIDTH_MAX);
|
||||
assert.equal(clampSidebarWidth(10), SIDEBAR_WIDTH_MIN);
|
||||
assert.equal(clampSidebarWidth(Number.NaN), SIDEBAR_WIDTH_DEFAULT);
|
||||
});
|
||||
|
||||
test("caps at 40% of a narrow window", () => {
|
||||
stubWindow.innerWidth = 800;
|
||||
assert.equal(clampSidebarWidth(SIDEBAR_WIDTH_MAX), 320);
|
||||
assert.equal(clampSidebarWidth(300), 300);
|
||||
});
|
||||
|
||||
test("the floor still wins when 40% falls below it", () => {
|
||||
stubWindow.innerWidth = 500;
|
||||
assert.equal(clampSidebarWidth(SIDEBAR_WIDTH_MAX), SIDEBAR_WIDTH_MIN);
|
||||
});
|
||||
|
||||
test("re-evaluates the cap per call, so a resize can re-clamp", () => {
|
||||
stubWindow.innerWidth = 1440;
|
||||
assert.equal(clampSidebarWidth(SIDEBAR_WIDTH_MAX), SIDEBAR_WIDTH_MAX);
|
||||
stubWindow.innerWidth = 900;
|
||||
assert.equal(clampSidebarWidth(SIDEBAR_WIDTH_MAX), 360);
|
||||
stubWindow.innerWidth = 1440;
|
||||
assert.equal(clampSidebarWidth(SIDEBAR_WIDTH_MAX), SIDEBAR_WIDTH_MAX);
|
||||
});
|
||||
|
||||
// The reset action promises to clear every stored preference, so a persisted
|
||||
// panel width that is missing from the list survives the reload.
|
||||
test("persisted panel widths are cleared by the preference reset", async () => {
|
||||
const source = await readFile(
|
||||
new URL("../src/features/settings/tabs/general-tab.tsx", import.meta.url),
|
||||
"utf8",
|
||||
);
|
||||
const keys = source.slice(
|
||||
source.indexOf("const PREFS_KEYS"),
|
||||
source.indexOf("];", source.indexOf("const PREFS_KEYS")),
|
||||
);
|
||||
for (const key of PANEL_WIDTH_KEYS) {
|
||||
assert.ok(keys.includes(`"${key}"`), `${key} missing from PREFS_KEYS`);
|
||||
}
|
||||
});
|
||||
|
|
@ -707,18 +707,55 @@ def existing_install_usable(install_dir: Path, host: HostInfo) -> bool:
|
|||
return npm_major is not None and npm_major >= NPM_MIN_MAJOR
|
||||
|
||||
|
||||
def _replace_with_retry(
|
||||
src: Path,
|
||||
dst: Path,
|
||||
*,
|
||||
attempts: int = 8,
|
||||
) -> None:
|
||||
"""os.replace, retried against transient Windows sharing violations.
|
||||
|
||||
A directory rename fails with WinError 5/32 while any process holds a handle inside
|
||||
it, and Defender or the indexer routinely does right after extraction (seen in CI on
|
||||
a fresh install, with no existing directory to conflict with). Handles clear in a
|
||||
second or two, so a bounded backoff turns the failure into a pause; other errors
|
||||
raise immediately rather than stalling on a real problem.
|
||||
"""
|
||||
delay = 0.25
|
||||
for attempt in range(attempts):
|
||||
try:
|
||||
os.replace(src, dst)
|
||||
return
|
||||
except OSError as exc:
|
||||
transient = os.name == "nt" and getattr(exc, "winerror", None) in (5, 32, 145)
|
||||
if not transient or attempt == attempts - 1:
|
||||
raise
|
||||
log(
|
||||
f"rename blocked ({exc.winerror}), retrying in {delay:.2f}s "
|
||||
f"-- a scanner is likely still holding the extracted files"
|
||||
)
|
||||
time.sleep(delay)
|
||||
delay = min(delay * 2, 4.0)
|
||||
|
||||
|
||||
def _swap_into_place(extracted_root: Path, install_dir: Path) -> None:
|
||||
"""Atomically replace install_dir with extracted_root (same filesystem)."""
|
||||
install_dir.parent.mkdir(parents = True, exist_ok = True)
|
||||
backup: Path | None = None
|
||||
if install_dir.exists():
|
||||
backup = install_dir.parent / f".{install_dir.name}.old-{os.getpid()}"
|
||||
os.replace(install_dir, backup)
|
||||
_replace_with_retry(install_dir, backup)
|
||||
try:
|
||||
os.replace(extracted_root, install_dir)
|
||||
_replace_with_retry(extracted_root, install_dir)
|
||||
except OSError:
|
||||
# The forward rename retries ~16s, ample time for a scanner to grab the backup too.
|
||||
# A plain os.replace would then raise over the original error and leave no
|
||||
# install_dir at all, so the rollback gets the same backoff and never masks it.
|
||||
if backup is not None and not install_dir.exists():
|
||||
os.replace(backup, install_dir)
|
||||
try:
|
||||
_replace_with_retry(backup, install_dir)
|
||||
except OSError as rollback_exc:
|
||||
log(f"could not restore the previous Node install from {backup}: {rollback_exc}")
|
||||
raise
|
||||
if backup is not None:
|
||||
shutil.rmtree(backup, ignore_errors = True)
|
||||
|
|
|
|||
182
studio/setup.ps1
182
studio/setup.ps1
|
|
@ -869,12 +869,22 @@ function Ensure-BuildToolsForLlamaSourceBuild {
|
|||
}
|
||||
}
|
||||
|
||||
# Detect the VC++ 2015-2022 Redistributable that the prebuilt llama-server and
|
||||
# PyTorch need (they link VCRUNTIME140_1.dll etc., which the Universal CRT lacks).
|
||||
# Signal is System32\vcruntime140_1.dll (VS 2019+), registry as fallback.
|
||||
# Machine arch: PROCESSOR_ARCHITECTURE describes this PROCESS, so an emulated x64 shell on
|
||||
# ARM64 reports AMD64; PROCESSOR_ARCHITEW6432 is ARM64 in exactly that case.
|
||||
function Get-HostMachineArch {
|
||||
$osArch = ""
|
||||
try { $osArch = [System.Runtime.InteropServices.RuntimeInformation]::OSArchitecture.ToString() } catch { }
|
||||
foreach ($s in @([string]$env:PROCESSOR_ARCHITEW6432, [string]$env:PROCESSOR_ARCHITECTURE, $osArch)) {
|
||||
if ($s.ToLowerInvariant() -eq "arm64") { return "arm64" }
|
||||
}
|
||||
return "other"
|
||||
}
|
||||
|
||||
# Detect the VC++ 2015-2022 Redistributable prebuilt llama-server and PyTorch need (they
|
||||
# link VCRUNTIME140_1.dll, absent from the Universal CRT). Registry first: Runtimes\x64 is
|
||||
# the only x64-specific proof; System32\vcruntime140_1.dll is arch-blind and on ARM64 may
|
||||
# be the ARM64-only package, unloadable under x64 emulation.
|
||||
function Test-VCRedistInstalled {
|
||||
$sys = $env:SystemRoot
|
||||
if ($sys -and (Test-Path (Join-Path $sys 'System32\vcruntime140_1.dll'))) { return $true }
|
||||
foreach ($k in @(
|
||||
'HKLM:\SOFTWARE\Microsoft\VisualStudio\14.0\VC\Runtimes\x64',
|
||||
'HKLM:\SOFTWARE\WOW6432Node\Microsoft\VisualStudio\14.0\VC\Runtimes\x64'
|
||||
|
|
@ -884,10 +894,14 @@ function Test-VCRedistInstalled {
|
|||
if ($r.Installed -eq 1 -and [int]$r.Major -ge 14 -and [int]$r.Minor -ge 20) { return $true }
|
||||
} catch { }
|
||||
}
|
||||
if ((Get-HostMachineArch) -eq "arm64") { return $false }
|
||||
$sys = $env:SystemRoot
|
||||
if ($sys -and (Test-Path (Join-Path $sys 'System32\vcruntime140_1.dll'))) { return $true }
|
||||
return $false
|
||||
}
|
||||
|
||||
# Install the VC++ 2015-2022 runtime if missing (non-fatal; usually a no-op).
|
||||
# Install the VC++ 2015-2022 runtime if missing (non-fatal; usually a no-op). Unlike CMake
|
||||
# and Build Tools torch cannot import without it, and winget is absent on LTSC/Server images.
|
||||
function Ensure-VCRedist {
|
||||
if (Test-VCRedistInstalled) { step "vcredist" "present"; return }
|
||||
Write-Host "Microsoft Visual C++ Redistributable (2015-2022) is missing; the prebuilt llama.cpp and PyTorch need it. Installing the runtime..." -ForegroundColor Yellow
|
||||
|
|
@ -897,6 +911,45 @@ function Ensure-VCRedist {
|
|||
Refresh-Environment
|
||||
} catch { substep "VCRedist install failed: $($_.Exception.Message)" "Yellow" }
|
||||
}
|
||||
if (-not (Test-VCRedistInstalled)) {
|
||||
# Evergreen link; /quiet /norestart so it never blocks or reboots an unattended run.
|
||||
# Always the x64 package, deliberately: Microsoft ships it as the Arm64X superset of
|
||||
# both ARM64 and X64 binaries and documents it as the one for ARM64 devices, while
|
||||
# the arm64 package is ARM64-only (learn.microsoft.com/cpp/windows/latest-supported-vc-redist).
|
||||
# PROCESSOR_ARCHITECTURE is wrong twice here: it reports the process, and the runtime
|
||||
# must match the interpreter loading the DLLs, an emulated x64 Python not yet created.
|
||||
$url = "https://aka.ms/vs/17/release/vc_redist.x64.exe"
|
||||
$dst = Join-Path ([System.IO.Path]::GetTempPath()) "vc_redist.x64.exe"
|
||||
substep "winget unavailable or failed; downloading the runtime directly..."
|
||||
# Windows PowerShell 5.1 on an old image can carry a .NET default protocol set that
|
||||
# predates TLS 1.2, which aka.ms refuses -- exactly the no-winget host this fallback
|
||||
# exists for. SystemDefault (0) means "let the OS choose" and already covers TLS 1.2+,
|
||||
# so only an explicit legacy set is upgraded, and it is restored afterwards.
|
||||
$_prevProtocol = $null
|
||||
try {
|
||||
$_cur = [System.Net.ServicePointManager]::SecurityProtocol
|
||||
if ([int]$_cur -ne 0 -and ([int]$_cur -band [int][System.Net.SecurityProtocolType]::Tls12) -eq 0) {
|
||||
[System.Net.ServicePointManager]::SecurityProtocol = $_cur -bor [System.Net.SecurityProtocolType]::Tls12
|
||||
$_prevProtocol = $_cur
|
||||
}
|
||||
} catch { $_prevProtocol = $null }
|
||||
try {
|
||||
Invoke-WebRequest -Uri $url -OutFile $dst -UseBasicParsing -TimeoutSec 300
|
||||
$p = Start-Process -FilePath $dst -ArgumentList '/quiet', '/norestart' -Wait -PassThru
|
||||
# 3010 = success, reboot required; usable either way.
|
||||
if ($p.ExitCode -notin @(0, 3010)) {
|
||||
substep "VC++ runtime installer exited $($p.ExitCode)" "Yellow"
|
||||
}
|
||||
Refresh-Environment
|
||||
} catch {
|
||||
substep "Direct VC++ runtime download failed: $($_.Exception.Message)" "Yellow"
|
||||
} finally {
|
||||
if ($null -ne $_prevProtocol) {
|
||||
try { [System.Net.ServicePointManager]::SecurityProtocol = $_prevProtocol } catch { }
|
||||
}
|
||||
Remove-Item -LiteralPath $dst -Force -ErrorAction SilentlyContinue
|
||||
}
|
||||
}
|
||||
if (Test-VCRedistInstalled) { step "vcredist" "installed" }
|
||||
else {
|
||||
substep "Could not install the VC++ Redistributable automatically." "Yellow"
|
||||
|
|
@ -1650,11 +1703,42 @@ if ($LongPathsEnabled) {
|
|||
}
|
||||
|
||||
# ============================================
|
||||
# 1b. Git (required by pip for git+https:// deps and by npm)
|
||||
# 1b. Git (only required for --local / source installs)
|
||||
# ============================================
|
||||
# Was fatal as "required by pip and npm", but the consumer path uses neither: the
|
||||
# unsloth-zoo git+https URL is STUDIO_LOCAL_INSTALL only, node is a pinned prebuilt, and the
|
||||
# frontend lockfile has no VCS deps. Being fatal blocked clean no-winget Windows boxes.
|
||||
$HasGit = $null -ne (Get-Command git -ErrorAction SilentlyContinue)
|
||||
if (-not $HasGit) {
|
||||
Write-Host "Git not found -- installing via winget..." -ForegroundColor Yellow
|
||||
# Fatal only where git is used: --local and the opt-in llama.cpp source build. A local
|
||||
# llama.cpp dir overrides those opt-ins, but only once it holds a reusable binary:
|
||||
# pointing at the canonical install location with nothing built there falls through to
|
||||
# the normal install, so an explicit source build still needs git. The automatic
|
||||
# fallback after a failed prebuilt download is not knowable here; Phase 4 handles it.
|
||||
$gitNeeded = ($env:STUDIO_LOCAL_INSTALL -eq '1')
|
||||
$_localLlamaDir = if ($env:UNSLOTH_LOCAL_LLAMA_CPP_DIR) { $env:UNSLOTH_LOCAL_LLAMA_CPP_DIR.Trim() } else { "" }
|
||||
$_localLlamaBuilt = $false
|
||||
if ($_localLlamaDir) {
|
||||
# Same layout candidates as the reuse check in Phase 4.
|
||||
foreach ($_c in @("llama-server.exe", "build\bin\llama-server.exe", "build\bin\Release\llama-server.exe")) {
|
||||
if (Test-Path -LiteralPath (Join-Path $_localLlamaDir $_c)) { $_localLlamaBuilt = $true; break }
|
||||
}
|
||||
}
|
||||
if (-not $_localLlamaBuilt) {
|
||||
$_prForce = if ($env:UNSLOTH_LLAMA_PR_FORCE) { $env:UNSLOTH_LLAMA_PR_FORCE.Trim() } else { $DefaultLlamaPrForce }
|
||||
$_llamaSrc = $DefaultLlamaSource -replace '\.git$', ''
|
||||
# Same tag resolution as Phase 4. "master" is a branch, never a release, so the
|
||||
# prebuilt lookup always misses and Phase 4 rebuilds it from source.
|
||||
$_llamaTag = if ($env:UNSLOTH_LLAMA_TAG) { $env:UNSLOTH_LLAMA_TAG } else { $DefaultLlamaTag }
|
||||
if ($_llamaTag -eq "master") { $gitNeeded = $true }
|
||||
if ($env:UNSLOTH_LLAMA_FORCE_COMPILE -eq '1') { $gitNeeded = $true }
|
||||
if (-not [string]::IsNullOrWhiteSpace($env:UNSLOTH_LLAMA_PR)) { $gitNeeded = $true }
|
||||
# Same positive-integer predicate as the PR_FORCE promotion below: 0 or non-numeric
|
||||
# never forces a source build, so it must not demand git.
|
||||
if ($_prForce -match '^\d+$' -and [int]$_prForce -gt 0) { $gitNeeded = $true }
|
||||
if ($_llamaSrc -ne "https://github.com/ggml-org/llama.cpp") { $gitNeeded = $true }
|
||||
}
|
||||
Write-Host "Git not found -- attempting install via winget..." -ForegroundColor Yellow
|
||||
$HasWinget = $null -ne (Get-Command winget -ErrorAction SilentlyContinue)
|
||||
if ($HasWinget) {
|
||||
try {
|
||||
|
|
@ -1664,11 +1748,18 @@ if (-not $HasGit) {
|
|||
} catch { }
|
||||
}
|
||||
if (-not $HasGit) {
|
||||
Write-Host "[ERROR] Git is required but could not be installed automatically." -ForegroundColor Red
|
||||
Write-Host " Install Git from https://git-scm.com/download/win and re-run." -ForegroundColor Red
|
||||
Exit-SetupFailure "Git is required but could not be installed automatically"
|
||||
if ($gitNeeded) {
|
||||
Write-Host "[ERROR] Git is required for --local and llama.cpp source-build installs but could not be installed." -ForegroundColor Red
|
||||
Write-Host " --local clones unsloth-zoo, and a source build clones llama.cpp." -ForegroundColor Red
|
||||
Write-Host " Install Git from https://git-scm.com/download/win and re-run." -ForegroundColor Red
|
||||
Exit-SetupFailure "Git is required for --local / source-build installs but could not be installed"
|
||||
}
|
||||
step "git" "not found (not required)" "Yellow"
|
||||
substep "Unsloth installs prebuilt binaries and wheels, so git is not needed."
|
||||
substep "Install it only for --local/source installs: https://git-scm.com/download/win"
|
||||
} else {
|
||||
step "git" "$(git --version)"
|
||||
}
|
||||
step "git" "$(git --version)"
|
||||
} else {
|
||||
step "git" "$(git --version)"
|
||||
}
|
||||
|
|
@ -3275,18 +3366,32 @@ $PyTorchWhlBase = if ($env:UNSLOTH_PYTORCH_MIRROR) { $env:UNSLOTH_PYTORCH_MIRROR
|
|||
$TorchInstallIndexUrl = if ($ROCmIndexUrl) { "$PyTorchWhlBase/cpu" } elseif ($PinnedTorchIndexUrl) { $PinnedTorchIndexUrl } else { "$PyTorchWhlBase/$CuTag" }
|
||||
|
||||
if (-not $NoTorchMode) {
|
||||
# Windows on ARM has win_arm64 torch and torchvision wheels but no torchaudio on any index,
|
||||
# so every branch below drops it. Ask the interpreter uv resolves for, not
|
||||
# PROCESSOR_ARCHITECTURE, which describes the host process. Inside the no-torch guard
|
||||
# because all three uses are, and no-torch installs nothing to skip.
|
||||
$_setupPlatform = ""
|
||||
try {
|
||||
$_setupPlatform = (& python -c "import sysconfig; print(sysconfig.get_platform())" 2>$null | Out-String).Trim().ToLowerInvariant()
|
||||
} catch { $_setupPlatform = "" }
|
||||
$WinArm64NoAudio = ($_setupPlatform -eq "win-arm64")
|
||||
if ($WinArm64NoAudio) { substep "windows on arm: skipping torchaudio (no win_arm64 wheel upstream)" }
|
||||
|
||||
$ROCmCpuFallback = $false
|
||||
if ($ROCmIndexUrl) {
|
||||
substep "installing PyTorch (AMD ROCm, $ROCmGfxArch)..."
|
||||
if ($ROCmTorchSpec -ne "torch") {
|
||||
substep " enforcing $ROCmTorchSpec $ROCmVisionSpec $ROCmAudioSpec (known _grouped_mm bug in older wheels)" "Cyan"
|
||||
}
|
||||
# Built above the verbose branch: a splat assigned inside it is unset on the other.
|
||||
$_rocmTrio = @($ROCmTorchSpec, $ROCmVisionSpec, $ROCmAudioSpec)
|
||||
if ($WinArm64NoAudio) { $_rocmTrio = @($ROCmTorchSpec, $ROCmVisionSpec) }
|
||||
if ($script:UnslothVerbose) {
|
||||
Fast-Install $ROCmTorchSpec $ROCmVisionSpec $ROCmAudioSpec --force-reinstall --index-url $ROCmIndexUrl | ForEach-Object { Redact-InstallOutput "$_" } | Out-Host
|
||||
Fast-Install @_rocmTrio --force-reinstall --index-url $ROCmIndexUrl | ForEach-Object { Redact-InstallOutput "$_" } | Out-Host
|
||||
$torchInstallExit = $LASTEXITCODE
|
||||
$output = ""
|
||||
} else {
|
||||
$output = Fast-Install $ROCmTorchSpec $ROCmVisionSpec $ROCmAudioSpec --force-reinstall --index-url $ROCmIndexUrl | Out-String
|
||||
$output = Fast-Install @_rocmTrio --force-reinstall --index-url $ROCmIndexUrl | Out-String
|
||||
$torchInstallExit = $LASTEXITCODE
|
||||
}
|
||||
if ($torchInstallExit -ne 0) {
|
||||
|
|
@ -3322,12 +3427,14 @@ if (-not $ROCmIndexUrl -and ($CuTag -eq "cpu" -or $ROCmCpuFallback)) {
|
|||
$cpuVisionSpec = "torchvision>=0.19,<0.27.0"
|
||||
$cpuAudioSpec = "torchaudio>=2.4,<2.12.0"
|
||||
}
|
||||
$_torchTrio = @($cpuTorchSpec, $cpuVisionSpec, $cpuAudioSpec)
|
||||
if ($WinArm64NoAudio) { $_torchTrio = @($cpuTorchSpec, $cpuVisionSpec) }
|
||||
if ($script:UnslothVerbose) {
|
||||
Fast-Install $cpuTorchSpec $cpuVisionSpec $cpuAudioSpec @cpuForce --index-url $TorchInstallIndexUrl | ForEach-Object { Redact-InstallOutput "$_" } | Out-Host
|
||||
Fast-Install @_torchTrio @cpuForce --index-url $TorchInstallIndexUrl | ForEach-Object { Redact-InstallOutput "$_" } | Out-Host
|
||||
$torchInstallExit = $LASTEXITCODE
|
||||
$output = ""
|
||||
} else {
|
||||
$output = Fast-Install $cpuTorchSpec $cpuVisionSpec $cpuAudioSpec @cpuForce --index-url $TorchInstallIndexUrl | Out-String
|
||||
$output = Fast-Install @_torchTrio @cpuForce --index-url $TorchInstallIndexUrl | Out-String
|
||||
$torchInstallExit = $LASTEXITCODE
|
||||
}
|
||||
if ($torchInstallExit -ne 0) {
|
||||
|
|
@ -3354,12 +3461,16 @@ if (-not $ROCmIndexUrl -and ($CuTag -eq "cpu" -or $ROCmCpuFallback)) {
|
|||
$cudaVisionSpec = "torchvision>=0.19,<0.26.0"
|
||||
$cudaAudioSpec = "torchaudio>=2.4,<2.11.0"
|
||||
}
|
||||
# A custom pin whose leaf is not cpu (a corporate /simple mirror) lands an ARM64 host
|
||||
# here, so this branch drops torchaudio too.
|
||||
$_cudaTrio = @($cudaTorchSpec, $cudaVisionSpec, $cudaAudioSpec)
|
||||
if ($WinArm64NoAudio) { $_cudaTrio = @($cudaTorchSpec, $cudaVisionSpec) }
|
||||
if ($script:UnslothVerbose) {
|
||||
Fast-Install $cudaTorchSpec $cudaVisionSpec $cudaAudioSpec @cudaForce --index-url $TorchInstallIndexUrl | ForEach-Object { Redact-InstallOutput "$_" } | Out-Host
|
||||
Fast-Install @_cudaTrio @cudaForce --index-url $TorchInstallIndexUrl | ForEach-Object { Redact-InstallOutput "$_" } | Out-Host
|
||||
$torchInstallExit = $LASTEXITCODE
|
||||
$output = ""
|
||||
} else {
|
||||
$output = Fast-Install $cudaTorchSpec $cudaVisionSpec $cudaAudioSpec @cudaForce --index-url $TorchInstallIndexUrl | Out-String
|
||||
$output = Fast-Install @_cudaTrio @cudaForce --index-url $TorchInstallIndexUrl | Out-String
|
||||
$torchInstallExit = $LASTEXITCODE
|
||||
}
|
||||
if ($torchInstallExit -ne 0) {
|
||||
|
|
@ -4048,6 +4159,7 @@ $BuildDir = Join-Path $LlamaCppDir "build"
|
|||
$LlamaServerBin = Join-Path $BuildDir "bin\Release\llama-server.exe"
|
||||
|
||||
$HasCmakeForBuild = $null -ne (Get-Command cmake -ErrorAction SilentlyContinue)
|
||||
$HasGitForBuild = $null -ne (Get-Command git -ErrorAction SilentlyContinue)
|
||||
|
||||
# Check if existing llama-server matches current GPU mode. A CUDA-built binary
|
||||
# on a now-CPU-only machine (or vice versa) needs to be rebuilt.
|
||||
|
|
@ -4073,9 +4185,27 @@ if (Test-Path -LiteralPath $LlamaServerBin) {
|
|||
$WillBuildLlamaFromSource = $NeedLlamaSourceBuild -and `
|
||||
-not ((Test-Path -LiteralPath $LlamaServerBin) -and -not $NeedRebuild -and $RequestedLlamaTag -ne "master")
|
||||
if ($WillBuildLlamaFromSource) {
|
||||
Ensure-BuildToolsForLlamaSourceBuild
|
||||
# refresh so the chain below sees a newly installed cmake
|
||||
$HasCmakeForBuild = $null -ne (Get-Command cmake -ErrorAction SilentlyContinue)
|
||||
if (-not $HasGitForBuild) {
|
||||
# Phase 1 keeps git optional, so only the automatic fallback after a failed prebuilt
|
||||
# download arrives here without it. Last chance to install: Invoke-SetupCommand
|
||||
# returns 0 for command-not-found, so a git-less clone misreports as a cmake failure.
|
||||
if ($null -ne (Get-Command winget -ErrorAction SilentlyContinue)) {
|
||||
try {
|
||||
Invoke-SetupCommand { winget install Git.Git --source winget --accept-package-agreements --accept-source-agreements } | Out-Null
|
||||
Refresh-Environment
|
||||
} catch { }
|
||||
}
|
||||
$HasGitForBuild = $null -ne (Get-Command git -ErrorAction SilentlyContinue)
|
||||
}
|
||||
# Git first, then the toolchain: Ensure-BuildToolsForLlamaSourceBuild exits setup when
|
||||
# Build Tools cannot be installed, so running it first made the degraded path below
|
||||
# unreachable on a no-winget box, and elsewhere spent a multi-GB download on a clone
|
||||
# that cannot happen.
|
||||
if ($HasGitForBuild) {
|
||||
Ensure-BuildToolsForLlamaSourceBuild
|
||||
# refresh so the chain below sees a newly installed cmake
|
||||
$HasCmakeForBuild = $null -ne (Get-Command cmake -ErrorAction SilentlyContinue)
|
||||
}
|
||||
}
|
||||
|
||||
if ($LocalLlamaCppLinked) {
|
||||
|
|
@ -4093,6 +4223,16 @@ if ($LocalLlamaCppLinked) {
|
|||
# up new model architecture support (e.g. Gemma 4).
|
||||
Write-Host ""
|
||||
step "llama.cpp" "already built"
|
||||
} elseif (-not $HasGitForBuild) {
|
||||
# Before cmake: the toolchain install is skipped without git, so cmake may be missing
|
||||
# purely as a consequence. Degrade rather than abort; the opt-in source triggers already
|
||||
# required git in Phase 1, so only the automatic fallback lands here.
|
||||
Write-Host ""
|
||||
step "llama.cpp" "build skipped (git not available)" "Yellow"
|
||||
substep "The prebuilt download failed and a source build clones llama.cpp." "Yellow"
|
||||
substep "GGUF inference and export will not be available." "Yellow"
|
||||
substep "Install Git from https://git-scm.com/download/win and re-run setup." "Yellow"
|
||||
$script:LlamaCppDegraded = $true
|
||||
} elseif (-not $HasCmakeForBuild) {
|
||||
Write-Host ""
|
||||
if (-not $HasNvidiaSmi) {
|
||||
|
|
|
|||
|
|
@ -27,6 +27,8 @@ pub(crate) struct DesktopUpdatePolicy {
|
|||
pub(crate) struct ManualUpdateInfo {
|
||||
version: String,
|
||||
current_version: String,
|
||||
// Backend release this desktop build pins; CHANGELOG.md is keyed by it.
|
||||
pypi_version: Option<String>,
|
||||
body: Option<String>,
|
||||
date: Option<String>,
|
||||
}
|
||||
|
|
@ -34,8 +36,12 @@ pub(crate) struct ManualUpdateInfo {
|
|||
#[derive(Debug, serde::Deserialize)]
|
||||
struct ChannelMetadata {
|
||||
version: String,
|
||||
body: Option<String>,
|
||||
date: Option<String>,
|
||||
// latest.json publishes Tauri's `notes`/`pub_date`; aliases keep older metadata working.
|
||||
pypi_version: Option<String>,
|
||||
#[serde(alias = "body")]
|
||||
notes: Option<String>,
|
||||
#[serde(alias = "date")]
|
||||
pub_date: Option<String>,
|
||||
platforms: HashMap<String, ChannelPlatform>,
|
||||
}
|
||||
|
||||
|
|
@ -99,8 +105,9 @@ pub(crate) async fn check_desktop_manual_update() -> Result<Option<ManualUpdateI
|
|||
Ok(Some(ManualUpdateInfo {
|
||||
version: latest_version,
|
||||
current_version: current_version.to_string(),
|
||||
body: metadata.body,
|
||||
date: metadata.date,
|
||||
pypi_version: metadata.pypi_version,
|
||||
body: metadata.notes,
|
||||
date: metadata.pub_date,
|
||||
}))
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -155,7 +155,10 @@ def test_the_ctypes_binds_are_gated_on_the_same_verdict():
|
|||
"if bnb is None or not native_kernels_ready(bnb, DEVICE_TYPE):" in source
|
||||
), "the ctypes bind block must take the _bnb_required branch on a dead library too"
|
||||
guarded = source.split("if bnb is None or not native_kernels_ready(bnb, DEVICE_TYPE):")[1]
|
||||
assert "bnb.functional.lib" in guarded, "the binds must sit under that guard"
|
||||
# Anchor on the symbol, not the module alias: #7580 renamed the binding from
|
||||
# `bnb.functional.lib` to `bnb_functional.lib`, which is exactly the kind of rename
|
||||
# this assertion should survive.
|
||||
assert "lib.cdequantize_blockwise_fp32" in guarded, "the binds must sit under that guard"
|
||||
|
||||
|
||||
def test_the_kernel_check_reads_the_submodule_not_the_parent_attribute():
|
||||
|
|
|
|||
|
|
@ -454,9 +454,12 @@ class TestKnown211SetParity:
|
|||
"$_pinCuLeaf" not in text
|
||||
), "install.ps1 must bound companions on every index (no cu-family exemption)"
|
||||
# The bounded companions must actually be passed to the install command.
|
||||
assert re.search(
|
||||
r'"torch>=2\.4,<2\.11\.0" \$_pinVisionSpec \$_pinAudioSpec --default-index \$TorchIndexUrl',
|
||||
text,
|
||||
# Specs are splatted, so check both halves: the list is built, and it is passed.
|
||||
assert (
|
||||
'$_torchSpecs = @("torch>=2.4,<2.11.0", $_pinVisionSpec, $_pinAudioSpec)' in text
|
||||
), "install.ps1 custom-pin install must build the bounded spec list"
|
||||
assert (
|
||||
"@_torchSpecs --default-index $TorchIndexUrl" in text
|
||||
), "install.ps1 custom-pin install must pass the bounded companion specs to uv"
|
||||
|
||||
def test_gfx_allowlist_matches_across_installers(self):
|
||||
|
|
@ -704,9 +707,13 @@ class TestPinnedIndexClearsUvEnvParity:
|
|||
assert (
|
||||
"if ($TorchIndexPinned -and -not (Test-CudaFamilyLeaf $CuTag)) {" in text
|
||||
), "the custom-leaf trio bounds must be gated on a pinned non-cu-family leaf"
|
||||
# Specs are splatted, so check both halves: the list is built, and it is passed.
|
||||
assert (
|
||||
"Fast-Install $cudaTorchSpec $cudaVisionSpec $cudaAudioSpec" in text
|
||||
), "setup.ps1's CUDA branch must install via the bounded spec variables"
|
||||
"$_cudaTrio = @($cudaTorchSpec, $cudaVisionSpec, $cudaAudioSpec)" in text
|
||||
), "setup.ps1's CUDA branch must build the trio from the bounded spec variables"
|
||||
assert (
|
||||
"Fast-Install @_cudaTrio @cudaForce" in text
|
||||
), "setup.ps1's CUDA branch must install the trio it built"
|
||||
|
||||
def test_setup_ps1_bounds_pinned_cpu_torch(self):
|
||||
"""setup.ps1's CPU branch must bound the trio under an explicit pin (parity with
|
||||
|
|
@ -724,8 +731,11 @@ class TestPinnedIndexClearsUvEnvParity:
|
|||
"if ($TorchIndexPinned) {" in text
|
||||
), "the CPU trio bounds must be gated on an explicit pin"
|
||||
assert (
|
||||
"Fast-Install $cpuTorchSpec $cpuVisionSpec $cpuAudioSpec @cpuForce" in text
|
||||
), "setup.ps1's CPU branch must install via the spec variables"
|
||||
"$_torchTrio = @($cpuTorchSpec, $cpuVisionSpec, $cpuAudioSpec)" in text
|
||||
), "setup.ps1's CPU branch must build the trio from the spec variables"
|
||||
assert (
|
||||
"Fast-Install @_torchTrio @cpuForce" in text
|
||||
), "setup.ps1's CPU branch must install the trio it built"
|
||||
# The ceilings mirror the Python repair spec exactly.
|
||||
stack = STACK_PY.read_text(encoding = "utf-8")
|
||||
spec_block = re.search(r"_CUDA_TORCH_PKG_SPEC[^(]*\(\s*(.*?)\)", stack, re.DOTALL)
|
||||
|
|
|
|||
143
tests/python/test_windows_arm64_python_choice.py
Normal file
143
tests/python/test_windows_arm64_python_choice.py
Normal file
|
|
@ -0,0 +1,143 @@
|
|||
# SPDX-License-Identifier: AGPL-3.0-only
|
||||
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
|
||||
|
||||
"""Windows on ARM: install.ps1 must not settle for a native ARM64 interpreter.
|
||||
|
||||
pyarrow (via datasets) and hf-transfer publish no win_arm64 wheels, so an ARM64
|
||||
Python source-builds both and dies minutes into the run. The resolver prefers an
|
||||
x64 build of the requested minor and bootstraps one otherwise; the case pinned
|
||||
here is the recovery path, where nothing can be downloaded but an x64 build of a
|
||||
lower-priority supported minor is already installed.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import re
|
||||
import shutil
|
||||
import subprocess
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
REPO_ROOT = Path(__file__).resolve().parents[2]
|
||||
INSTALL_PS1 = REPO_ROOT / "install.ps1"
|
||||
|
||||
|
||||
def _extract(pattern: str, source: str) -> str:
|
||||
match = re.search(pattern, source, flags = re.DOTALL)
|
||||
assert match is not None, f"install.ps1 block not found: {pattern}"
|
||||
return match.group(0)
|
||||
|
||||
|
||||
def _resolver_script(installed: list[tuple[str, str]], can_download: bool) -> str:
|
||||
"""Both production functions verbatim, over a fake set of interpreters.
|
||||
|
||||
Extracted rather than reimplemented so the test cannot drift away from the
|
||||
text install.ps1 actually runs. `installed` is (minor, arch) in py-launcher
|
||||
order, so the first entry for a minor is what a bare `py -3.13` resolves to.
|
||||
The fake interpreters are named `*.exe` and invoked through the call operator,
|
||||
which resolves a string to a function, so no real binary is needed.
|
||||
"""
|
||||
source = INSTALL_PS1.read_text(encoding = "utf-8")
|
||||
finder = _extract(r" function Find-CompatiblePython \{.*?\n \}\n", source)
|
||||
installer = _extract(r" function Install-X64Python \{.*?\n \}\n", source)
|
||||
|
||||
names = [f"Py{minor.replace('.', '')}{arch}.exe" for minor, arch in installed]
|
||||
table = ", ".join(
|
||||
f'@{{ Minor = "{minor}"; Arch = "{arch}"; Name = "{name}" }}'
|
||||
for (minor, arch), name in zip(installed, names)
|
||||
)
|
||||
downloaded = (
|
||||
'@{ Version = "3.13"; Path = "Downloaded.exe"; Arch = "x86_64" }'
|
||||
if can_download
|
||||
else "$null"
|
||||
)
|
||||
version_stubs = "\n".join(
|
||||
f"function {name} {{ param([Parameter(ValueFromRemainingArguments = $true)]$Rest)\n"
|
||||
f' if ($Rest -contains "--version") {{ return "Python {minor}.0" }}\n'
|
||||
f' return "{name}" }}'
|
||||
for (minor, _arch), name in zip(installed, names)
|
||||
)
|
||||
return f"""
|
||||
$ErrorActionPreference = "Stop"
|
||||
$PythonVersion = "3.13"
|
||||
$script:WingetAvailable = $false
|
||||
$script:CondaSkipPattern = 'conda'
|
||||
$Interpreters = @({table})
|
||||
{version_stubs}
|
||||
# `py -0p` lists every registration; `py -3.x` runs the launcher's preferred build
|
||||
# for that minor, which on an ARM64 host is normally the native one.
|
||||
function FakePy {{
|
||||
param([Parameter(ValueFromRemainingArguments = $true)]$Rest)
|
||||
if ($Rest -contains "-0p") {{
|
||||
return @($Interpreters | ForEach-Object {{ " -V:$($_.Minor) * $($_.Name)" }})
|
||||
}}
|
||||
$minor = ([string]$Rest[0]).TrimStart('-')
|
||||
$hit = @($Interpreters | Where-Object {{ $_.Minor -eq $minor }})
|
||||
if ($hit.Count -eq 0) {{ return "" }}
|
||||
if ($Rest -contains "--version") {{ return "Python $minor.0" }}
|
||||
return $hit[0].Name
|
||||
}}
|
||||
function substep {{ param($a, $b) }}
|
||||
function Get-HostMachineArch {{ return "arm64" }}
|
||||
function Get-Command {{
|
||||
param([Parameter(Position = 0)][string]$Name,
|
||||
[Parameter(ValueFromRemainingArguments = $true)]$Rest)
|
||||
if ($Name -eq "py") {{ return @([pscustomobject]@{{ Source = "FakePy" }}) }}
|
||||
return @()
|
||||
}}
|
||||
function Test-Path {{ param([Parameter(ValueFromRemainingArguments = $true)]$Rest) return $true }}
|
||||
function Test-IsCondaPython {{ param([string]$Exe) return $false }}
|
||||
function Get-PythonPlatformTag {{
|
||||
param([string]$Exe)
|
||||
foreach ($i in $Interpreters) {{
|
||||
if ($i.Name -eq $Exe) {{
|
||||
if ($i.Arch -eq "x86_64") {{ return "win-amd64" }} else {{ return "win-arm64" }}
|
||||
}}
|
||||
}}
|
||||
return "win-amd64"
|
||||
}}
|
||||
function Refresh-SessionPath {{ }}
|
||||
function Install-PythonFromPythonOrg {{ param([string]$Arch = "") return {downloaded} }}
|
||||
{finder}
|
||||
{installer}
|
||||
# The caller's ARM64 swap, condensed to what decides the interpreter.
|
||||
$found = Find-CompatiblePython
|
||||
if ($found -and $found.Arch -ne "x86_64") {{
|
||||
$x64 = Install-X64Python
|
||||
if ($x64) {{ $found = $x64 }}
|
||||
}}
|
||||
if ($found) {{ Write-Output "$($found.Version)|$($found.Arch)" }} else {{ Write-Output "none" }}
|
||||
"""
|
||||
|
||||
|
||||
def _pwsh(script: str) -> str:
|
||||
result = subprocess.run(
|
||||
["pwsh", "-NoProfile", "-NonInteractive", "-Command", script],
|
||||
check = True,
|
||||
capture_output = True,
|
||||
text = True,
|
||||
env = os.environ.copy(),
|
||||
)
|
||||
return result.stdout.strip()
|
||||
|
||||
|
||||
@pytest.mark.skipif(shutil.which("pwsh") is None, reason = "PowerShell is unavailable")
|
||||
@pytest.mark.parametrize(
|
||||
("installed", "can_download", "expected"),
|
||||
[
|
||||
# An x64 build of the requested minor wins outright, downloads irrelevant.
|
||||
([("3.13", "arm64"), ("3.13", "x86_64")], False, "3.13|x86_64"),
|
||||
# Requested minor is ARM64-only: bootstrap x64 rather than take the native one.
|
||||
([("3.13", "arm64")], True, "3.13|x86_64"),
|
||||
# Offline, but an x64 build of a lower-priority minor is here. Use it: the native
|
||||
# 3.13 cannot resolve pyarrow or hf-transfer, and this one can.
|
||||
([("3.13", "arm64"), ("3.11", "x86_64")], False, "3.11|x86_64"),
|
||||
# ARM64 everywhere: still returned, and the caller warns.
|
||||
([("3.13", "arm64"), ("3.11", "arm64")], False, "3.13|arm64"),
|
||||
],
|
||||
)
|
||||
def test_arm64_host_prefers_an_x64_interpreter(installed, can_download, expected):
|
||||
assert _pwsh(_resolver_script(installed, can_download)) == expected
|
||||
Some files were not shown because too many files have changed in this diff Show more
Loading…
Add table
Add a link
Reference in a new issue