diff --git a/.github/workflows/lockfile-audit.yml b/.github/workflows/lockfile-audit.yml new file mode 100644 index 0000000000..9c28e21672 --- /dev/null +++ b/.github/workflows/lockfile-audit.yml @@ -0,0 +1,79 @@ +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. +# +# Fast, focused supply-chain audit of every checked-in lockfile. +# +# Runs scripts/lockfile_supply_chain_audit.py on PRs that touch any +# npm or cargo lockfile, on push to main, and on a daily schedule so +# newly-published IOCs surface even when no PR opens. +# +# Default behavior is "advisory": only public indicator-of-compromise +# strings, known-malicious pinned versions, and structurally broken +# lockfiles fail the build. Structural anomalies (missing integrity, +# non-default registry, etc.) are emitted as GitHub Actions warnings +# but do not block merges. This deliberately keeps the noise floor +# low while still failing the moment a checked-in lockfile starts +# pointing at known-bad bytes. +# +# This workflow is intentionally separate from security-audit.yml: +# - security-audit.yml is the umbrella job (pip-audit + npm audit + +# cargo audit + OSV + Semgrep + secret scanning + SBOM + ...); +# it takes ~25 minutes and runs only when dep manifests change. +# - lockfile-audit.yml is a ~30 second pure-Python parse + grep on +# the lockfiles themselves; it runs on every PR that even nudges +# a lockfile so reviewers always see the audit result inline. + +name: Lockfile supply-chain audit + +on: + pull_request: + paths: + - 'studio/frontend/package-lock.json' + - 'studio/backend/core/data_recipe/oxc-validator/package-lock.json' + - 'studio/package-lock.json' + - 'studio/src-tauri/Cargo.lock' + - 'scripts/lockfile_supply_chain_audit.py' + - '.github/workflows/lockfile-audit.yml' + push: + branches: [main] + paths: + - 'studio/frontend/package-lock.json' + - 'studio/backend/core/data_recipe/oxc-validator/package-lock.json' + - 'studio/package-lock.json' + - 'studio/src-tauri/Cargo.lock' + - 'scripts/lockfile_supply_chain_audit.py' + - '.github/workflows/lockfile-audit.yml' + schedule: + - cron: '37 5 * * *' + workflow_dispatch: + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +permissions: + contents: read + +jobs: + audit: + name: lockfile supply-chain audit + runs-on: ubuntu-latest + timeout-minutes: 5 + steps: + - uses: actions/checkout@v4 + with: + persist-credentials: false + + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + + - name: Verify audit script parses + run: python3 -c "import ast; ast.parse(open('scripts/lockfile_supply_chain_audit.py').read())" + + - name: Run lockfile supply-chain audit + # Default mode: only known-malicious pinned versions, known IOC + # strings, and structurally broken lockfiles fail the build. + # Missing-integrity and other structural anomalies are emitted + # as ::warning:: annotations and do not gate merges. + run: python3 scripts/lockfile_supply_chain_audit.py diff --git a/.gitignore b/.gitignore index bc7d59316d..a839633790 100644 --- a/.gitignore +++ b/.gitignore @@ -229,5 +229,9 @@ log.txt setup_leo.sh server.pid *.log +# Ignore stray lockfiles; real npm projects opt back in below (npm ci needs them). package-lock.json +!studio/frontend/package-lock.json +!studio/backend/core/data_recipe/oxc-validator/package-lock.json +!studio/package-lock.json llama.cpp/ diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index a2a4995d62..9d80fe6ff5 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,6 +1,6 @@ repos: - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.15.12 + rev: v0.15.13 hooks: - id: ruff args: diff --git a/install.ps1 b/install.ps1 index 35951d7ee2..5e3d4b6a50 100644 --- a/install.ps1 +++ b/install.ps1 @@ -92,6 +92,7 @@ function Install-UnslothStudio { $RepoRoot = "" $TauriMode = $false $SkipTorch = $false + $ShortcutsOnly = $false $argList = $args for ($i = 0; $i -lt $argList.Count; $i++) { switch ($argList[$i]) { @@ -100,6 +101,7 @@ function Install-UnslothStudio { "--no-torch" { $SkipTorch = $true } "--verbose" { $script:UnslothVerbose = $true } "-v" { $script:UnslothVerbose = $true } + "--shortcuts-only" { $ShortcutsOnly = $true } "--package" { $i++ if ($i -ge $argList.Count) { @@ -871,6 +873,19 @@ shell.Run cmd, 0, False } } + # Regen .lnk + launcher only; used by `unsloth studio update`. + if ($ShortcutsOnly) { + if ($TauriMode) { return } + $UnslothExe = Join-Path $VenvDir "Scripts\unsloth.exe" + if (-not (Test-Path -LiteralPath $UnslothExe)) { + Write-Host "[ERROR] unsloth.exe missing at $UnslothExe; run install.ps1 first." -ForegroundColor Red + # throw (not Exit-InstallFailure) so non-Tauri callers see rc != 0. + throw "unsloth.exe missing" + } + New-StudioShortcuts -UnslothExePath $UnslothExe + return + } + # ── Check winget ── Write-TauriLog "STEP" "Checking system dependencies" if (-not (Get-Command winget -ErrorAction SilentlyContinue)) { @@ -1285,7 +1300,7 @@ shell.Run cmd, 0, False if ($SkipTorch) { # No-torch: install unsloth + unsloth-zoo with --no-deps, then # runtime deps (typer, safetensors, transformers, etc.) with --no-deps. - $baseInstallExit = Invoke-InstallCommand { uv pip install --python $VenvPython --no-deps --reinstall-package unsloth --reinstall-package unsloth-zoo "unsloth>=2026.5.3" unsloth-zoo } + $baseInstallExit = Invoke-InstallCommand { uv pip install --python $VenvPython --no-deps --reinstall-package unsloth --reinstall-package unsloth-zoo "unsloth>=2026.5.4" unsloth-zoo } if ($baseInstallExit -eq 0) { $NoTorchReq = Find-NoTorchRuntimeFile if ($NoTorchReq) { @@ -1293,7 +1308,7 @@ shell.Run cmd, 0, False } } } else { - $baseInstallExit = Invoke-InstallCommand { uv pip install --python $VenvPython --reinstall-package unsloth --reinstall-package unsloth-zoo "unsloth>=2026.5.3" unsloth-zoo } + $baseInstallExit = Invoke-InstallCommand { uv pip install --python $VenvPython --reinstall-package unsloth --reinstall-package unsloth-zoo "unsloth>=2026.5.4" unsloth-zoo } } if ($baseInstallExit -ne 0) { Write-Host "[ERROR] Failed to install unsloth (exit code $baseInstallExit)" -ForegroundColor Red @@ -1331,7 +1346,7 @@ shell.Run cmd, 0, False if ($SkipTorch) { # No-torch: install unsloth + unsloth-zoo with --no-deps, then # runtime deps (typer, safetensors, transformers, etc.) with --no-deps. - $baseInstallExit = Invoke-InstallCommand { uv pip install --python $VenvPython --no-deps --upgrade-package unsloth --upgrade-package unsloth-zoo "unsloth>=2026.5.3" unsloth-zoo } + $baseInstallExit = Invoke-InstallCommand { uv pip install --python $VenvPython --no-deps --upgrade-package unsloth --upgrade-package unsloth-zoo "unsloth>=2026.5.4" unsloth-zoo } if ($baseInstallExit -eq 0) { $NoTorchReq = Find-NoTorchRuntimeFile if ($NoTorchReq) { @@ -1339,7 +1354,7 @@ shell.Run cmd, 0, False } } } elseif ($StudioLocalInstall) { - $baseInstallExit = Invoke-InstallCommand { uv pip install --python $VenvPython --upgrade-package unsloth "unsloth>=2026.5.3" unsloth-zoo } + $baseInstallExit = Invoke-InstallCommand { uv pip install --python $VenvPython --upgrade-package unsloth "unsloth>=2026.5.4" unsloth-zoo } } else { $baseInstallExit = Invoke-InstallCommand { uv pip install --python $VenvPython --upgrade-package unsloth -- "$PackageName" } } @@ -1367,7 +1382,7 @@ shell.Run cmd, 0, False Write-TauriLog "STEP" "Installing unsloth" substep "installing unsloth (this may take a few minutes)..." if ($StudioLocalInstall) { - $baseInstallExit = Invoke-InstallCommand { uv pip install --python $VenvPython unsloth-zoo "unsloth>=2026.5.3" --torch-backend=auto } + $baseInstallExit = Invoke-InstallCommand { uv pip install --python $VenvPython unsloth-zoo "unsloth>=2026.5.4" --torch-backend=auto } if ($baseInstallExit -ne 0) { Write-Host "[ERROR] Failed to install unsloth (exit code $baseInstallExit)" -ForegroundColor Red return (Exit-InstallFailure "Failed to install unsloth (exit code $baseInstallExit)" $baseInstallExit) diff --git a/install.sh b/install.sh index d59605b6a5..cfd76fa945 100755 --- a/install.sh +++ b/install.sh @@ -45,6 +45,7 @@ TAURI_MODE=false _USER_PYTHON="" _NO_TORCH_FLAG=false _VERBOSE=false +_SHORTCUTS_ONLY=false _next_is_package=false _next_is_python=false for arg in "$@"; do @@ -65,6 +66,7 @@ for arg in "$@"; do --python) _next_is_python=true ;; --no-torch) _NO_TORCH_FLAG=true ;; --verbose|-v) _VERBOSE=true ;; + --shortcuts-only) _SHORTCUTS_ONLY=true ;; esac done @@ -1233,6 +1235,20 @@ elif grep -qi microsoft /proc/version 2>/dev/null; then fi step "platform" "$OS" +# Regen launcher/shortcuts only; used by `unsloth studio update`. +if [ "$_SHORTCUTS_ONLY" = true ]; then + # Tauri owns its own shortcuts. + if [ "$TAURI_MODE" != true ]; then + VENV_ABS_BIN="$VENV_DIR/bin" + if [ ! -x "$VENV_ABS_BIN/unsloth" ]; then + echo "ERROR: unsloth binary missing at '$VENV_ABS_BIN/unsloth'; run install.sh first." >&2 + exit 1 + fi + create_studio_shortcuts "$VENV_ABS_BIN/unsloth" "$OS" + fi + exit 0 +fi + # ── Architecture detection & Python version ── _ARCH=$(uname -m) MAC_INTEL=false @@ -1849,7 +1865,7 @@ if [ "$_MIGRATED" = true ]; then # to prevent transitive torch resolution. run_install_cmd "install unsloth (migrated no-torch)" uv pip install --python "$_VENV_PY" --no-deps \ --reinstall-package unsloth --reinstall-package unsloth-zoo \ - "unsloth>=2026.5.3" unsloth-zoo + "unsloth>=2026.5.4" unsloth-zoo _NO_TORCH_RT="$(_find_no_torch_runtime)" if [ -n "$_NO_TORCH_RT" ]; then run_install_cmd "install no-torch runtime deps" uv pip install --python "$_VENV_PY" --no-deps -r "$_NO_TORCH_RT" @@ -1857,7 +1873,7 @@ if [ "$_MIGRATED" = true ]; then else run_install_cmd "install unsloth (migrated)" uv pip install --python "$_VENV_PY" \ --reinstall-package unsloth --reinstall-package unsloth-zoo \ - "unsloth>=2026.5.3" unsloth-zoo + "unsloth>=2026.5.4" unsloth-zoo fi if [ "$STUDIO_LOCAL_INSTALL" = true ]; then substep "overlaying local repo (editable)..." @@ -2025,7 +2041,7 @@ elif [ -n "$TORCH_INDEX_URL" ]; then # runtime deps (typer, safetensors, transformers, etc.) with --no-deps. run_install_cmd "install unsloth (no-torch)" uv pip install --python "$_VENV_PY" --no-deps \ --upgrade-package unsloth --upgrade-package unsloth-zoo \ - "unsloth>=2026.5.3" unsloth-zoo + "unsloth>=2026.5.4" unsloth-zoo _NO_TORCH_RT="$(_find_no_torch_runtime)" if [ -n "$_NO_TORCH_RT" ]; then run_install_cmd "install no-torch runtime deps" uv pip install --python "$_VENV_PY" --no-deps -r "$_NO_TORCH_RT" @@ -2040,7 +2056,7 @@ elif [ -n "$TORCH_INDEX_URL" ]; then fi elif [ "$STUDIO_LOCAL_INSTALL" = true ]; then run_install_cmd "install unsloth (local)" uv pip install --python "$_VENV_PY" \ - --upgrade-package unsloth "unsloth>=2026.5.3" unsloth-zoo + --upgrade-package unsloth "unsloth>=2026.5.4" unsloth-zoo substep "overlaying local repo (editable)..." run_install_cmd "overlay local repo" uv pip install --python "$_VENV_PY" -e "$_REPO_ROOT" --no-deps substep "overlaying unsloth-zoo from git main..." @@ -2072,7 +2088,7 @@ else tauri_log "STEP" "Installing Unsloth" substep "installing unsloth (this may take a few minutes)..." if [ "$STUDIO_LOCAL_INSTALL" = true ]; then - run_install_cmd "install unsloth (auto torch backend)" uv pip install --python "$_VENV_PY" unsloth-zoo "unsloth>=2026.5.3" --torch-backend=auto + run_install_cmd "install unsloth (auto torch backend)" uv pip install --python "$_VENV_PY" unsloth-zoo "unsloth>=2026.5.4" --torch-backend=auto substep "overlaying local repo (editable)..." run_install_cmd "overlay local repo" uv pip install --python "$_VENV_PY" -e "$_REPO_ROOT" --no-deps substep "overlaying unsloth-zoo from git main..." diff --git a/scripts/lockfile_supply_chain_audit.py b/scripts/lockfile_supply_chain_audit.py index ae215bf344..ffeea51c23 100644 --- a/scripts/lockfile_supply_chain_audit.py +++ b/scripts/lockfile_supply_chain_audit.py @@ -389,6 +389,18 @@ class Finding: ) +def _gha_escape(text: str) -> str: + """Escape a string for use in a GitHub Actions `::warning::` / + `::error::` workflow command message. GH Actions truncates + annotation messages at the first newline unless `\\n` is + escaped as `%0A`; carriage returns and the percent sign need + matching escapes per the workflow-commands spec. Order matters: + `%` must be replaced first so the subsequent `%0A` / `%0D` + sequences are not double-encoded. + """ + return text.replace("%", "%25").replace("\r", "%0D").replace("\n", "%0A") + + # ───────────────────────────────────────────────────────────────────── # package-lock.json audit. # ───────────────────────────────────────────────────────────────────── @@ -397,9 +409,35 @@ class Finding: def audit_npm_lockfile(path: Path) -> list[Finding]: findings: list[Finding] = [] if not path.exists(): + # A missing requested lockfile is a config error, not a clean + # audit; surface it so a deleted default cannot pass silently. + findings.append( + Finding( + path = str(path), + package = "", + kind = "missing-lockfile", + detail = ( + "expected lockfile not found; refusing to silently " + "report a clean audit for a path that was not scanned" + ), + ) + ) return findings - raw = path.read_text(encoding = "utf-8") + try: + raw = path.read_text(encoding = "utf-8") + except OSError as exc: + # Permission denied, is-a-directory, broken-pipe etc. -- surface + # as a finding instead of crashing CI with a raw traceback. + findings.append( + Finding( + path = str(path), + package = "", + kind = "unreadable-lockfile", + detail = f"could not read file: {exc}", + ) + ) + return findings try: lock = json.loads(raw) except json.JSONDecodeError as exc: @@ -553,9 +591,32 @@ _PACKAGE_HEADER = re.compile(r"^\[\[package\]\]\s*$") def audit_cargo_lockfile(path: Path) -> list[Finding]: findings: list[Finding] = [] if not path.exists(): + # See audit_npm_lockfile: missing lockfile is a finding. + findings.append( + Finding( + path = str(path), + package = "", + kind = "missing-lockfile", + detail = ( + "expected lockfile not found; refusing to silently " + "report a clean audit for a path that was not scanned" + ), + ) + ) return findings - raw = path.read_text(encoding = "utf-8") + try: + raw = path.read_text(encoding = "utf-8") + except OSError as exc: + findings.append( + Finding( + path = str(path), + package = "", + kind = "unreadable-lockfile", + detail = f"could not read file: {exc}", + ) + ) + return findings try: import tomllib # type: ignore[import-not-found] except ImportError: @@ -652,7 +713,31 @@ def audit_cargo_lockfile(path: Path) -> list[Finding]: # ───────────────────────────────────────────────────────────────────── -DEFAULT_NPM_LOCKFILES = ("studio/frontend/package-lock.json",) +# Finding kinds split into BLOCKING vs ADVISORY for the default run mode. +# Blocking findings come from public supply-chain attack indicators (a +# version we know is malicious, a string an attacker would have to embed +# for an attack to work). Advisory findings are structural lockfile +# anomalies (missing integrity, non-default registry, etc.) -- they +# WARN the maintainer but do not block merges. Pass --strict to make +# every finding blocking (PR-5479-style behavior for opt-in adopters). +BLOCKING_KINDS: frozenset[str] = frozenset( + { + "blocked-known-malicious", + "known-ioc-string", + # Internal-failure kinds: a structurally broken lockfile MIGHT + # be hiding a real attack, so we keep these blocking too. + "malformed-lockfile", + "missing-lockfile", + "unreadable-lockfile", + "missing-toml-parser", + } +) + +DEFAULT_NPM_LOCKFILES = ( + "studio/frontend/package-lock.json", + "studio/backend/core/data_recipe/oxc-validator/package-lock.json", + "studio/package-lock.json", +) DEFAULT_CARGO_LOCKFILES = ("studio/src-tauri/Cargo.lock",) @@ -671,7 +756,9 @@ def main(argv: list[str] | None = None) -> int: default = None, help = ( "Path to a package-lock.json (repeatable). " - "Default: studio/frontend/package-lock.json." + "Default: studio/frontend/package-lock.json, " + "studio/backend/core/data_recipe/oxc-validator/package-lock.json, " + "and studio/package-lock.json (Tauri CLI for desktop release)." ), ) parser.add_argument( @@ -683,6 +770,18 @@ def main(argv: list[str] | None = None) -> int: "Default: studio/src-tauri/Cargo.lock." ), ) + parser.add_argument( + "--strict", + action = "store_true", + help = ( + "Treat every finding as blocking (exit 1). " + "Default mode only blocks on known-malicious versions, " + "indicator-of-compromise strings, or structurally broken " + "lockfiles; everything else is printed as an advisory " + "warning with exit 0. CI should use the default; local " + "audits aiming for zero noise can opt in via --strict." + ), + ) args = parser.parse_args(argv) # SF4: require a real justification (e.g. JIRA ticket id) for the @@ -715,8 +814,15 @@ def main(argv: list[str] | None = None) -> int: return 0 root = Path(args.root).resolve() - npm_paths = [root / p for p in (args.npm_lockfile or DEFAULT_NPM_LOCKFILES)] - cargo_paths = [root / p for p in (args.cargo_lockfile or DEFAULT_CARGO_LOCKFILES)] + # Explicit --npm-lockfile/--cargo-lockfile scopes the scan to those + # paths; defaults apply only to the no-args CI invocation. + _user_explicit = args.npm_lockfile is not None or args.cargo_lockfile is not None + if _user_explicit: + npm_paths = [root / p for p in (args.npm_lockfile or ())] + cargo_paths = [root / p for p in (args.cargo_lockfile or ())] + else: + npm_paths = [root / p for p in DEFAULT_NPM_LOCKFILES] + cargo_paths = [root / p for p in DEFAULT_CARGO_LOCKFILES] all_findings: list[Finding] = [] for p in npm_paths: @@ -734,17 +840,57 @@ def main(argv: list[str] | None = None) -> int: ) return 0 + # Split findings into blocking (known-malicious / IOC / structurally + # broken) and advisory (everything else, e.g. missing integrity on a + # registry-published tarball). In default mode advisory findings are + # printed but do not change the exit code; --strict treats every + # finding as blocking. + blocking = [f for f in all_findings if f.kind in BLOCKING_KINDS] + advisory = [f for f in all_findings if f.kind not in BLOCKING_KINDS] + + if args.strict: + blocking = list(all_findings) + advisory = [] + + if advisory: + print( + f"\n[lockfile-audit] {len(advisory)} advisory finding(s) " + "(non-blocking; pass --strict to fail the build on these):\n", + file = sys.stderr, + ) + for f in advisory: + # Surface in GitHub Actions UI as a warning annotation when run + # under Actions; harmless prefix elsewhere. GH Actions + # truncates annotation messages at the first newline unless + # newlines are escaped as `%0A`, so the full multi-line + # Finding (kind + path + package + detail) only renders in + # the UI after _gha_escape collapses it onto one line. + print(f"::warning::{_gha_escape(str(f))}", file = sys.stderr) + print(file = sys.stderr) + + if not blocking: + print( + f"[lockfile-audit] OK: {len(advisory)} advisory finding(s), " + "0 blocking. Run with --strict to escalate advisory findings.", + flush = True, + ) + return 0 + print( - f"\n[lockfile-audit] FAIL: {len(all_findings)} finding(s):\n", + f"\n[lockfile-audit] FAIL: {len(blocking)} blocking finding(s):\n", file = sys.stderr, ) - for f in all_findings: - print(str(f), file = sys.stderr) + for f in blocking: + # Same %-encoding rationale as the advisory branch above: the + # GH Actions annotation is truncated at the first newline + # unless the message is escaped. + print(f"::error::{_gha_escape(str(f))}", file = sys.stderr) print(file = sys.stderr) print( - "[lockfile-audit] Refusing to proceed. Each finding above is " - "either a structural lockfile anomaly or a public indicator-of-" - "compromise. Investigate before running `npm ci` or `cargo fetch`.", + "[lockfile-audit] Refusing to proceed. Each blocking finding " + "above is either a public indicator-of-compromise, a known-" + "malicious pinned version, or a structurally broken lockfile. " + "Investigate before running `npm ci` or `cargo fetch`.", file = sys.stderr, ) return 1 diff --git a/studio/backend/core/data_recipe/oxc-validator/package-lock.json b/studio/backend/core/data_recipe/oxc-validator/package-lock.json new file mode 100644 index 0000000000..bb2ae29b23 --- /dev/null +++ b/studio/backend/core/data_recipe/oxc-validator/package-lock.json @@ -0,0 +1,799 @@ +{ + "name": "unsloth-oxc-validator-runtime", + "version": "0.0.1", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "unsloth-oxc-validator-runtime", + "version": "0.0.1", + "dependencies": { + "oxc-parser": "^0.123.0", + "oxlint": "^1.51.0" + } + }, + "node_modules/@emnapi/core": { + "version": "1.10.0", + "resolved": "https://registry.npmjs.org/@emnapi/core/-/core-1.10.0.tgz", + "integrity": "sha512-yq6OkJ4p82CAfPl0u9mQebQHKPJkY7WrIuk205cTYnYe+k2Z8YBh11FrbRG/H6ihirqcacOgl2BIO8oyMQLeXw==", + "license": "MIT", + "optional": true, + "peer": true, + "dependencies": { + "@emnapi/wasi-threads": "1.2.1", + "tslib": "^2.4.0" + } + }, + "node_modules/@emnapi/runtime": { + "version": "1.10.0", + "resolved": "https://registry.npmjs.org/@emnapi/runtime/-/runtime-1.10.0.tgz", + "integrity": "sha512-ewvYlk86xUoGI0zQRNq/mC+16R1QeDlKQy21Ki3oSYXNgLb45GV1P6A0M+/s6nyCuNDqe5VpaY84BzXGwVbwFA==", + "license": "MIT", + "optional": true, + "peer": true, + "dependencies": { + "tslib": "^2.4.0" + } + }, + "node_modules/@emnapi/wasi-threads": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/@emnapi/wasi-threads/-/wasi-threads-1.2.1.tgz", + "integrity": "sha512-uTII7OYF+/Mes/MrcIOYp5yOtSMLBWSIoLPpcgwipoiKbli6k322tcoFsxoIIxPDqW01SQGAgko4EzZi2BNv2w==", + "license": "MIT", + "optional": true, + "peer": true, + "dependencies": { + "tslib": "^2.4.0" + } + }, + "node_modules/@napi-rs/wasm-runtime": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/@napi-rs/wasm-runtime/-/wasm-runtime-1.1.4.tgz", + "integrity": "sha512-3NQNNgA1YSlJb/kMH1ildASP9HW7/7kYnRI2szWJaofaS1hWmbGI4H+d3+22aGzXXN9IJ+n+GiFVcGipJP18ow==", + "license": "MIT", + "optional": true, + "dependencies": { + "@tybys/wasm-util": "^0.10.1" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/Brooooooklyn" + }, + "peerDependencies": { + "@emnapi/core": "^1.7.1", + "@emnapi/runtime": "^1.7.1" + } + }, + "node_modules/@oxc-parser/binding-android-arm-eabi": { + "version": "0.123.0", + "resolved": "https://registry.npmjs.org/@oxc-parser/binding-android-arm-eabi/-/binding-android-arm-eabi-0.123.0.tgz", + "integrity": "sha512-EHQ58z+6DbZWokMOKg5AB1KuwrXVgfbBLuuLFfzdc7bI5A4igvdvjKMhUv1VBV+0FABiUCOjNKUmMF7ugprwbQ==", + "cpu": [ + "arm" + ], + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@oxc-parser/binding-android-arm64": { + "version": "0.123.0", + "resolved": "https://registry.npmjs.org/@oxc-parser/binding-android-arm64/-/binding-android-arm64-0.123.0.tgz", + "integrity": "sha512-BK1E0zqNoHf38nTHjnGZ+olKHSKNHh65pChjY06yhaWYP8X7yNDqhQDA4neMPRqnPBgpN4/OW1oSMrdJgDi2aw==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@oxc-parser/binding-darwin-arm64": { + "version": "0.123.0", + "resolved": "https://registry.npmjs.org/@oxc-parser/binding-darwin-arm64/-/binding-darwin-arm64-0.123.0.tgz", + "integrity": "sha512-dkMPbtTbqU+cm+k4YGOBs4zAuq3Xu+wqjbGQvLAuVO7qHhNY4p5LBNudOmOoi0jxS8h1W6Jmlzv8MAKGpK+iDg==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@oxc-parser/binding-darwin-x64": { + "version": "0.123.0", + "resolved": "https://registry.npmjs.org/@oxc-parser/binding-darwin-x64/-/binding-darwin-x64-0.123.0.tgz", + "integrity": "sha512-85pic0rCd59DGdM69jI9xE/Snb2KtrfiU48QigjJXjzxUOenGvH4SAFIjFpO/2ZnI3Kz50D8pht4jKN3t2022Q==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@oxc-parser/binding-freebsd-x64": { + "version": "0.123.0", + "resolved": "https://registry.npmjs.org/@oxc-parser/binding-freebsd-x64/-/binding-freebsd-x64-0.123.0.tgz", + "integrity": "sha512-mjEiW6z7JtaiHMK/8aJic1lfjkKpzFwK2XFNmm187BFbtDamjGVuKNr2TEyrFEYJyZc217wokR1wrYeZGBQo4Q==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@oxc-parser/binding-linux-arm-gnueabihf": { + "version": "0.123.0", + "resolved": "https://registry.npmjs.org/@oxc-parser/binding-linux-arm-gnueabihf/-/binding-linux-arm-gnueabihf-0.123.0.tgz", + "integrity": "sha512-mYxigPtGt6SZfhNZBIJfuDM92cLo8XUW08WuKxzHvcmWu6xndLqwLp99Vg4uHke1AXicQEHU3Wri2X9bHF0Vlw==", + "cpu": [ + "arm" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@oxc-parser/binding-linux-arm-musleabihf": { + "version": "0.123.0", + "resolved": "https://registry.npmjs.org/@oxc-parser/binding-linux-arm-musleabihf/-/binding-linux-arm-musleabihf-0.123.0.tgz", + "integrity": "sha512-ttWirDC9eUBn0R4Tzz3aeDaLrx9drPdNiLJ8MXeDBFxd6cwLfTIC27qjsdfGpn942tkVIZY3sjWAnvbwDDjX7g==", + "cpu": [ + "arm" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@oxc-parser/binding-linux-arm64-gnu": { + "version": "0.123.0", + "resolved": "https://registry.npmjs.org/@oxc-parser/binding-linux-arm64-gnu/-/binding-linux-arm64-gnu-0.123.0.tgz", + "integrity": "sha512-apAHyoMNRYT+2G98Y14caZmsr5LD9PsWpGI7nXmSwK26LGiQneCU6HvHQ+d+AX+RJ5TTWZtEb2RD7OLqAC0cYQ==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@oxc-parser/binding-linux-arm64-musl": { + "version": "0.123.0", + "resolved": "https://registry.npmjs.org/@oxc-parser/binding-linux-arm64-musl/-/binding-linux-arm64-musl-0.123.0.tgz", + "integrity": "sha512-3r99Qa4egjO/iXUBxTlN6Ddt1YkLifG6olzvj8gkoKEK2U/MOW7mQfXRyBmuoMgmZ7O4vk41gO3d21c6VcN3yQ==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@oxc-parser/binding-linux-ppc64-gnu": { + "version": "0.123.0", + "resolved": "https://registry.npmjs.org/@oxc-parser/binding-linux-ppc64-gnu/-/binding-linux-ppc64-gnu-0.123.0.tgz", + "integrity": "sha512-Hr/Z24kUE4pjJs346g80WDwjyJGrxiw6hExJuOiME/76ZFz68y5L11UzprRkW9FN4HxBB7tLZ/fytczV2fEsiA==", + "cpu": [ + "ppc64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@oxc-parser/binding-linux-riscv64-gnu": { + "version": "0.123.0", + "resolved": "https://registry.npmjs.org/@oxc-parser/binding-linux-riscv64-gnu/-/binding-linux-riscv64-gnu-0.123.0.tgz", + "integrity": "sha512-sxjbhs+8WXeuoLnZ2rBmQ96gPdq3SCmz24reIltsKLUt1EDMgdaQsr7RqwBphw3QAImkMtlPQfAWDWwZyo0xDg==", + "cpu": [ + "riscv64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@oxc-parser/binding-linux-riscv64-musl": { + "version": "0.123.0", + "resolved": "https://registry.npmjs.org/@oxc-parser/binding-linux-riscv64-musl/-/binding-linux-riscv64-musl-0.123.0.tgz", + "integrity": "sha512-d6xHHhqldA/W+VC7v8uHs24zM69Ad3HnHQ45h+uuBhCsbZx3d0E0wL2K3uJ5mYKTR6UPMFk9VMXcHWwvg1PRZQ==", + "cpu": [ + "riscv64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@oxc-parser/binding-linux-s390x-gnu": { + "version": "0.123.0", + "resolved": "https://registry.npmjs.org/@oxc-parser/binding-linux-s390x-gnu/-/binding-linux-s390x-gnu-0.123.0.tgz", + "integrity": "sha512-+di9A5wJQlv0VodyhADjJ2rC4geyHY+uhJDl3TFjMgYhhlgLZchi9uHD5mfiUEDWHt1x7/eU2u1ge3LLazZmFw==", + "cpu": [ + "s390x" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@oxc-parser/binding-linux-x64-gnu": { + "version": "0.123.0", + "resolved": "https://registry.npmjs.org/@oxc-parser/binding-linux-x64-gnu/-/binding-linux-x64-gnu-0.123.0.tgz", + "integrity": "sha512-sh7pw2g/u6LE1TaRRQsV9Kv9+1y+CywaaNwWWP+3bnEPk/L692oTG0hmEviUlawI8v3OGC+AhbjtAD+HXWQAkg==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@oxc-parser/binding-linux-x64-musl": { + "version": "0.123.0", + "resolved": "https://registry.npmjs.org/@oxc-parser/binding-linux-x64-musl/-/binding-linux-x64-musl-0.123.0.tgz", + "integrity": "sha512-S+LoD8PiJ639JwIqK1knIeqAyYkeCbLHtAgfapszKX0yVCaYP+aer8dJxL25de9qcDjvYWVrYCkuDZzHmOl2Xw==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@oxc-parser/binding-openharmony-arm64": { + "version": "0.123.0", + "resolved": "https://registry.npmjs.org/@oxc-parser/binding-openharmony-arm64/-/binding-openharmony-arm64-0.123.0.tgz", + "integrity": "sha512-/65vryK11q1I+k+7ukDlwZOxUFCLYsoZBZPGZHyet5bIP5e3D8mV3uCuvpWZ9Hoe6vUZFw/nAfCrX59MeuJPgw==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "openharmony" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@oxc-parser/binding-wasm32-wasi": { + "version": "0.123.0", + "resolved": "https://registry.npmjs.org/@oxc-parser/binding-wasm32-wasi/-/binding-wasm32-wasi-0.123.0.tgz", + "integrity": "sha512-y4OsMGQiAbZzj2Rq0LEfvhR48rQDvbvqsl/dPdn4tdf+z3H79nZuR+lQ/+KUGjD30vpVGem138sBWHFj9UR+Vg==", + "cpu": [ + "wasm32" + ], + "license": "MIT", + "optional": true, + "dependencies": { + "@napi-rs/wasm-runtime": "^1.1.2" + }, + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/@oxc-parser/binding-win32-arm64-msvc": { + "version": "0.123.0", + "resolved": "https://registry.npmjs.org/@oxc-parser/binding-win32-arm64-msvc/-/binding-win32-arm64-msvc-0.123.0.tgz", + "integrity": "sha512-9lBqI6AXAkjYavkdpizNU3Q51uoVYfp9FJPx19hnCEdPku1jSgzSnvgmCvhCue0GziIvIvIdWgZ41wXQ3EOoBw==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@oxc-parser/binding-win32-ia32-msvc": { + "version": "0.123.0", + "resolved": "https://registry.npmjs.org/@oxc-parser/binding-win32-ia32-msvc/-/binding-win32-ia32-msvc-0.123.0.tgz", + "integrity": "sha512-zJbqBHwSUB7CyvAONy9ewGtQwcQj+ylOhYGETvUPp3KIYx7lolj4Gayof7iA22SU5eMSjO5COL0c8wYhmn9agA==", + "cpu": [ + "ia32" + ], + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@oxc-parser/binding-win32-x64-msvc": { + "version": "0.123.0", + "resolved": "https://registry.npmjs.org/@oxc-parser/binding-win32-x64-msvc/-/binding-win32-x64-msvc-0.123.0.tgz", + "integrity": "sha512-q7RZvglQvGo3RX5ljtcGSabu2B2c0oDU/6xC3sBMhsV5KRo0PvyxLdordbEN31NTfuZu4Sgl86C76cAURZIHWA==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@oxc-project/types": { + "version": "0.123.0", + "resolved": "https://registry.npmjs.org/@oxc-project/types/-/types-0.123.0.tgz", + "integrity": "sha512-YtECP/y8Mj1lSHiUWGSRzy/C6teUKlS87dEfuVKT09LgQbUsBW1rNg+MiJ4buGu3yuADV60gbIvo9/HplA56Ew==", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/Boshen" + } + }, + "node_modules/@oxlint/binding-android-arm-eabi": { + "version": "1.64.0", + "resolved": "https://registry.npmjs.org/@oxlint/binding-android-arm-eabi/-/binding-android-arm-eabi-1.64.0.tgz", + "integrity": "sha512-2r6Nq3XXGLHEXKkSj8JtmJ6N4gDw431DPFOg0ZoJHlNjnG6HVMm/ksQ10m0HJ8WBvwgMe1L50UHPaYZutCRPCw==", + "cpu": [ + "arm" + ], + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@oxlint/binding-android-arm64": { + "version": "1.64.0", + "resolved": "https://registry.npmjs.org/@oxlint/binding-android-arm64/-/binding-android-arm64-1.64.0.tgz", + "integrity": "sha512-ePJMpePgg7fBv+L/hVx1xXRU5/5gd5m0obLA6hPEfLXF3GjpR8idIDbY1dhQYhyz1ms2wdTccSboo6KEd2Oxtg==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@oxlint/binding-darwin-arm64": { + "version": "1.64.0", + "resolved": "https://registry.npmjs.org/@oxlint/binding-darwin-arm64/-/binding-darwin-arm64-1.64.0.tgz", + "integrity": "sha512-U4DMLQd10gJLuoSTLSGbfv3bGjTlUNsScm9Dgb8wwBqmCzidf1pE1pXV4doGNxqwH3KtVng1AGTINA0NvkGLvQ==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@oxlint/binding-darwin-x64": { + "version": "1.64.0", + "resolved": "https://registry.npmjs.org/@oxlint/binding-darwin-x64/-/binding-darwin-x64-1.64.0.tgz", + "integrity": "sha512-GoRIL48QWm4/TAvjN8pB1nAG+1/uqc9EdnWT9zqHeb6wsmjZtywj8VRe5aGW47Fdb64YtLOsdLqVxOvQuz98Wg==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@oxlint/binding-freebsd-x64": { + "version": "1.64.0", + "resolved": "https://registry.npmjs.org/@oxlint/binding-freebsd-x64/-/binding-freebsd-x64-1.64.0.tgz", + "integrity": "sha512-5dFkv4tkg7PxJJGS9/OjrJwjhuHczrd3OQOkRE0wHcLM+ncUnULtzEPWjqGOxTXxZnLWcB91bGiIznx89TVXyQ==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@oxlint/binding-linux-arm-gnueabihf": { + "version": "1.64.0", + "resolved": "https://registry.npmjs.org/@oxlint/binding-linux-arm-gnueabihf/-/binding-linux-arm-gnueabihf-1.64.0.tgz", + "integrity": "sha512-jsBqMLl/uOL5+Kq/+BtK9FrmiNGUbx8SiyZXv+WlUxA45KuwcLu9BfiSIL3I3DBDgWM3yZizDITnTK9BcqNBQg==", + "cpu": [ + "arm" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@oxlint/binding-linux-arm-musleabihf": { + "version": "1.64.0", + "resolved": "https://registry.npmjs.org/@oxlint/binding-linux-arm-musleabihf/-/binding-linux-arm-musleabihf-1.64.0.tgz", + "integrity": "sha512-1lrj8At/Uuc9GhjrVFBQo0NEjfBrTkzpmtHIGAhNnIXqn1CAyGL+qrztUsXb2GIluJrpl9Q7qRLJOb/NqydacQ==", + "cpu": [ + "arm" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@oxlint/binding-linux-arm64-gnu": { + "version": "1.64.0", + "resolved": "https://registry.npmjs.org/@oxlint/binding-linux-arm64-gnu/-/binding-linux-arm64-gnu-1.64.0.tgz", + "integrity": "sha512-HpSQbubwh03mMhAdy2BYtad/fsY8vDFHDAb6bUwuCYg2VD3xCQgn6ArKcO0oZyLCheacKTv4PrF3Mfu5hgoE2g==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@oxlint/binding-linux-arm64-musl": { + "version": "1.64.0", + "resolved": "https://registry.npmjs.org/@oxlint/binding-linux-arm64-musl/-/binding-linux-arm64-musl-1.64.0.tgz", + "integrity": "sha512-00QQ0h0Y7u0G69BgiH3+ky2aaq/QvkDL6DYok8htIuJHxybiux5aQ8jwmg8qIk9wha6UagUP2BAwAzbemcJbpg==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@oxlint/binding-linux-ppc64-gnu": { + "version": "1.64.0", + "resolved": "https://registry.npmjs.org/@oxlint/binding-linux-ppc64-gnu/-/binding-linux-ppc64-gnu-1.64.0.tgz", + "integrity": "sha512-2GaimTV6EMW+s5HS0An3oGbQme3BgHswvfVdGk3EB57Xe9+/gyT+Qd7lNVzb3rtir52vbIPzXfaYArzs5b5zcw==", + "cpu": [ + "ppc64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@oxlint/binding-linux-riscv64-gnu": { + "version": "1.64.0", + "resolved": "https://registry.npmjs.org/@oxlint/binding-linux-riscv64-gnu/-/binding-linux-riscv64-gnu-1.64.0.tgz", + "integrity": "sha512-H46AtFb9wypjoVwGdlxrm0DsD809NGmtiK9HiyPKTxkSte2YjhC4S+00rOIrwCaxcyPiGid3Y3OMXp5KMAkGZw==", + "cpu": [ + "riscv64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@oxlint/binding-linux-riscv64-musl": { + "version": "1.64.0", + "resolved": "https://registry.npmjs.org/@oxlint/binding-linux-riscv64-musl/-/binding-linux-riscv64-musl-1.64.0.tgz", + "integrity": "sha512-HEgsidjjvvyzdg82icYkuFCf7REDV7B9JFwbIMbVwrKLBY0MrXX+bku3POn/hduZ2yW91IyVDUMq0Bf02KwXQw==", + "cpu": [ + "riscv64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@oxlint/binding-linux-s390x-gnu": { + "version": "1.64.0", + "resolved": "https://registry.npmjs.org/@oxlint/binding-linux-s390x-gnu/-/binding-linux-s390x-gnu-1.64.0.tgz", + "integrity": "sha512-Axvm8qryotmKN00P5w4JapaSjvP2LOSbdbBJiX+2SuHd3QzhW7TUc8skqgw+ahQZ5DmzEYeHCqauvW8f32Ns6Q==", + "cpu": [ + "s390x" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@oxlint/binding-linux-x64-gnu": { + "version": "1.64.0", + "resolved": "https://registry.npmjs.org/@oxlint/binding-linux-x64-gnu/-/binding-linux-x64-gnu-1.64.0.tgz", + "integrity": "sha512-cR60vSd7+m+KRZ3GQGfDxWwahW5RMXg0qlGvAluZr0fTUYvw0H9N9AXAF/M/PMqgytyqvVNmBAkJG9l7U30Y1g==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@oxlint/binding-linux-x64-musl": { + "version": "1.64.0", + "resolved": "https://registry.npmjs.org/@oxlint/binding-linux-x64-musl/-/binding-linux-x64-musl-1.64.0.tgz", + "integrity": "sha512-2u/aPZ9pEg7HnvZPDsHxUGNnrpr4qaHi+mCgLgpt+LYRzPrS4Px4wPfkIdRdr2GvKnaYyt+XSlto0Vm5sbStTg==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@oxlint/binding-openharmony-arm64": { + "version": "1.64.0", + "resolved": "https://registry.npmjs.org/@oxlint/binding-openharmony-arm64/-/binding-openharmony-arm64-1.64.0.tgz", + "integrity": "sha512-kfhkGfCdoXLSxEkrhDlJrvBYajGmq+ma4EMc53dsOWTq+rIBOlI0vTBmpZNnM5oH2LY/K/w1HAK+UQEgjgpVUg==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "openharmony" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@oxlint/binding-win32-arm64-msvc": { + "version": "1.64.0", + "resolved": "https://registry.npmjs.org/@oxlint/binding-win32-arm64-msvc/-/binding-win32-arm64-msvc-1.64.0.tgz", + "integrity": "sha512-r/cNKBFieONoVu2bb1KkVouq9W+edDUgHumXJGphCRRj+U0xaD4nanrw8ZOqo0IsutPkEM4vCcGBpak6x5aXMg==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@oxlint/binding-win32-ia32-msvc": { + "version": "1.64.0", + "resolved": "https://registry.npmjs.org/@oxlint/binding-win32-ia32-msvc/-/binding-win32-ia32-msvc-1.64.0.tgz", + "integrity": "sha512-tUw0xUUwEFVZbpJoeCblkv8SJA4Xz3CdXCJbAnBsiNLyxDrk2tLcxEAS6M73Q7hHHDg3OtwI8vZVK3t5RJt4Gw==", + "cpu": [ + "ia32" + ], + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@oxlint/binding-win32-x64-msvc": { + "version": "1.64.0", + "resolved": "https://registry.npmjs.org/@oxlint/binding-win32-x64-msvc/-/binding-win32-x64-msvc-1.64.0.tgz", + "integrity": "sha512-9CBR+LO0JVST87fNTzzNxS5I29jIUO5gxT9i9+M3SDHHALElj9sY1Prf12tad3vIRC6OD7Ehtvvh+sn13vSwHw==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@tybys/wasm-util": { + "version": "0.10.2", + "resolved": "https://registry.npmjs.org/@tybys/wasm-util/-/wasm-util-0.10.2.tgz", + "integrity": "sha512-RoBvJ2X0wuKlWFIjrwffGw1IqZHKQqzIchKaadZZfnNpsAYp2mM0h36JtPCjNDAHGgYez/15uMBpfGwchhiMgg==", + "license": "MIT", + "optional": true, + "dependencies": { + "tslib": "^2.4.0" + } + }, + "node_modules/oxc-parser": { + "version": "0.123.0", + "resolved": "https://registry.npmjs.org/oxc-parser/-/oxc-parser-0.123.0.tgz", + "integrity": "sha512-F6ak0tFc01ZGbl5KxvLDQ2K005Z086mp3ByCQBDhUjqXLkapGUkMuJSsYixncdEpkLlcRDcruHR71LD339ADUA==", + "license": "MIT", + "dependencies": { + "@oxc-project/types": "^0.123.0" + }, + "engines": { + "node": "^20.19.0 || >=22.12.0" + }, + "funding": { + "url": "https://github.com/sponsors/Boshen" + }, + "optionalDependencies": { + "@oxc-parser/binding-android-arm-eabi": "0.123.0", + "@oxc-parser/binding-android-arm64": "0.123.0", + "@oxc-parser/binding-darwin-arm64": "0.123.0", + "@oxc-parser/binding-darwin-x64": "0.123.0", + "@oxc-parser/binding-freebsd-x64": "0.123.0", + "@oxc-parser/binding-linux-arm-gnueabihf": "0.123.0", + "@oxc-parser/binding-linux-arm-musleabihf": "0.123.0", + "@oxc-parser/binding-linux-arm64-gnu": "0.123.0", + "@oxc-parser/binding-linux-arm64-musl": "0.123.0", + "@oxc-parser/binding-linux-ppc64-gnu": "0.123.0", + "@oxc-parser/binding-linux-riscv64-gnu": "0.123.0", + "@oxc-parser/binding-linux-riscv64-musl": "0.123.0", + "@oxc-parser/binding-linux-s390x-gnu": "0.123.0", + "@oxc-parser/binding-linux-x64-gnu": "0.123.0", + "@oxc-parser/binding-linux-x64-musl": "0.123.0", + "@oxc-parser/binding-openharmony-arm64": "0.123.0", + "@oxc-parser/binding-wasm32-wasi": "0.123.0", + "@oxc-parser/binding-win32-arm64-msvc": "0.123.0", + "@oxc-parser/binding-win32-ia32-msvc": "0.123.0", + "@oxc-parser/binding-win32-x64-msvc": "0.123.0" + } + }, + "node_modules/oxlint": { + "version": "1.64.0", + "resolved": "https://registry.npmjs.org/oxlint/-/oxlint-1.64.0.tgz", + "integrity": "sha512-Star3SNpWPeWFPw7kRXIhXUSn6fdiAl25q15CQzH/9WaOtG6e9CWTc25vNZOCr4PE1yEP1GtKJKIKglhj3OmEQ==", + "license": "MIT", + "bin": { + "oxlint": "bin/oxlint" + }, + "engines": { + "node": "^20.19.0 || >=22.12.0" + }, + "funding": { + "url": "https://github.com/sponsors/Boshen" + }, + "optionalDependencies": { + "@oxlint/binding-android-arm-eabi": "1.64.0", + "@oxlint/binding-android-arm64": "1.64.0", + "@oxlint/binding-darwin-arm64": "1.64.0", + "@oxlint/binding-darwin-x64": "1.64.0", + "@oxlint/binding-freebsd-x64": "1.64.0", + "@oxlint/binding-linux-arm-gnueabihf": "1.64.0", + "@oxlint/binding-linux-arm-musleabihf": "1.64.0", + "@oxlint/binding-linux-arm64-gnu": "1.64.0", + "@oxlint/binding-linux-arm64-musl": "1.64.0", + "@oxlint/binding-linux-ppc64-gnu": "1.64.0", + "@oxlint/binding-linux-riscv64-gnu": "1.64.0", + "@oxlint/binding-linux-riscv64-musl": "1.64.0", + "@oxlint/binding-linux-s390x-gnu": "1.64.0", + "@oxlint/binding-linux-x64-gnu": "1.64.0", + "@oxlint/binding-linux-x64-musl": "1.64.0", + "@oxlint/binding-openharmony-arm64": "1.64.0", + "@oxlint/binding-win32-arm64-msvc": "1.64.0", + "@oxlint/binding-win32-ia32-msvc": "1.64.0", + "@oxlint/binding-win32-x64-msvc": "1.64.0" + }, + "peerDependencies": { + "oxlint-tsgolint": ">=0.22.1" + }, + "peerDependenciesMeta": { + "oxlint-tsgolint": { + "optional": true + } + } + }, + "node_modules/tslib": { + "version": "2.8.1", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.8.1.tgz", + "integrity": "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==", + "license": "0BSD", + "optional": true + } + } +} diff --git a/studio/backend/core/inference/chat_template_helpers.py b/studio/backend/core/inference/chat_template_helpers.py new file mode 100644 index 0000000000..833a714ee4 --- /dev/null +++ b/studio/backend/core/inference/chat_template_helpers.py @@ -0,0 +1,60 @@ +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 + +""" +Dependency-light wrapper around tokenizer.apply_chat_template with a +kwarg fallback for templates that reject reasoning/tools args. +""" + +from typing import Optional + + +def apply_chat_template_for_generation( + tokenizer, + messages: list, + *, + tools: Optional[list] = None, + enable_thinking: Optional[bool] = None, + reasoning_effort: Optional[str] = None, + preserve_thinking: Optional[bool] = None, +) -> str: + """Render the chat prompt. Try richest kwargs first; drop one + group at a time on TypeError. Jinja / missing-variable errors + propagate.""" + reasoning_kwargs: dict = {} + if enable_thinking is not None: + reasoning_kwargs["enable_thinking"] = enable_thinking + if reasoning_effort is not None: + reasoning_kwargs["reasoning_effort"] = reasoning_effort + if preserve_thinking is not None: + reasoning_kwargs["preserve_thinking"] = preserve_thinking + + attempts: list[dict] = [] + if tools and reasoning_kwargs: + attempts.append({"tools": tools, **reasoning_kwargs}) + if tools: + attempts.append({"tools": tools}) + if reasoning_kwargs: + attempts.append(dict(reasoning_kwargs)) + attempts.append({}) + + last_exc: Optional[Exception] = None + for kwargs in attempts: + try: + return tokenizer.apply_chat_template( + messages, + tokenize = False, + add_generation_prompt = True, + **kwargs, + ) + except TypeError as e: + last_exc = e + continue + except Exception as e: + last_exc = e + break + if last_exc is not None: + raise last_exc + raise RuntimeError( + "apply_chat_template_for_generation: no attempt produced a result" + ) diff --git a/studio/backend/core/inference/inference.py b/studio/backend/core/inference/inference.py index 4c140013a0..e1620f5ca3 100644 --- a/studio/backend/core/inference/inference.py +++ b/studio/backend/core/inference/inference.py @@ -839,6 +839,74 @@ class InferenceBackend: cancel_event = cancel_event, _adapter_state = use_adapter, **gen_kwargs ) + def generate_chat_completion_with_tools( + self, + messages: list, + tools: list, + system_prompt: str = "", + temperature: float = 0.7, + top_p: float = 0.9, + top_k: int = 40, + min_p: float = 0.0, + max_new_tokens: int = 2048, + repetition_penalty: float = 1.0, + cancel_event = None, + enable_thinking: Optional[bool] = None, + reasoning_effort: Optional[str] = None, + preserve_thinking: Optional[bool] = None, + max_tool_iterations: int = 25, + auto_heal_tool_calls: bool = True, + tool_call_timeout: int = 300, + session_id: Optional[str] = None, + ): + """Run an agentic tool loop on top of ``generate_chat_response``. + + Yields the same event-dict protocol used by the GGUF path so + the route layer can stream both backends through one helper. + Each event is one of: + + * ``{"type": "status", "text": ...}`` + * ``{"type": "content", "text": cumulative_text}`` + * ``{"type": "tool_start", "tool_name", "tool_call_id", "arguments"}`` + * ``{"type": "tool_end", "tool_name", "tool_call_id", "result"}`` + """ + from core.inference.safetensors_agentic import run_safetensors_tool_loop + from core.inference.tools import execute_tool + + def _single_turn(conv: list): + # conv already has the system message -- avoid double-prepend. + yield from self._generate_chat_response_inner( + messages = conv, + system_prompt = "", + temperature = temperature, + top_p = top_p, + top_k = top_k, + min_p = min_p, + max_new_tokens = max_new_tokens, + repetition_penalty = repetition_penalty, + cancel_event = cancel_event, + tools = tools, + enable_thinking = enable_thinking, + reasoning_effort = reasoning_effort, + preserve_thinking = preserve_thinking, + ) + + initial = list(messages) + if system_prompt: + initial = [{"role": "system", "content": system_prompt}] + initial + + yield from run_safetensors_tool_loop( + single_turn = _single_turn, + messages = initial, + tools = tools, + execute_tool = execute_tool, + cancel_event = cancel_event, + auto_heal_tool_calls = auto_heal_tool_calls, + max_tool_iterations = max_tool_iterations, + tool_call_timeout = tool_call_timeout, + session_id = session_id, + ) + def generate_chat_response( self, messages: list, @@ -851,10 +919,20 @@ class InferenceBackend: max_new_tokens: int = 256, repetition_penalty: float = 1.0, cancel_event = None, + tools: Optional[list] = None, + enable_thinking: Optional[bool] = None, + reasoning_effort: Optional[str] = None, + preserve_thinking: Optional[bool] = None, ) -> Generator[str, None, None]: """ Generate response for text or vision models. The generation lock is acquired by the background generation thread. + + ``tools`` / ``enable_thinking`` / ``reasoning_effort`` / + ``preserve_thinking`` are forwarded into + ``tokenizer.apply_chat_template`` so templates that understand + these kwargs (Qwen3, Llama 3.1+, gpt-oss harmony, ...) advertise + the tool schemas and reasoning controls to the model. """ yield from self._generate_chat_response_inner( messages = messages, @@ -867,6 +945,10 @@ class InferenceBackend: max_new_tokens = max_new_tokens, repetition_penalty = repetition_penalty, cancel_event = cancel_event, + tools = tools, + enable_thinking = enable_thinking, + reasoning_effort = reasoning_effort, + preserve_thinking = preserve_thinking, ) def _generate_chat_response_inner( @@ -882,6 +964,10 @@ class InferenceBackend: repetition_penalty: float = 1.0, cancel_event = None, _adapter_state = None, + tools: Optional[list] = None, + enable_thinking: Optional[bool] = None, + reasoning_effort: Optional[str] = None, + preserve_thinking: Optional[bool] = None, ) -> Generator[str, None, None]: """ Inner generation logic. Called by both generate_chat_response @@ -981,8 +1067,13 @@ class InferenceBackend: f"Please use a model that includes a chat template, or manually set " f"one via tokenizer.chat_template before inference." ) - formatted_prompt = tokenizer.apply_chat_template( - template_messages, tokenize = False, add_generation_prompt = True + formatted_prompt = self._apply_chat_template_for_generation( + tokenizer, + template_messages, + tools = tools, + enable_thinking = enable_thinking, + reasoning_effort = reasoning_effort, + preserve_thinking = preserve_thinking, ) logger.debug(f"Formatted prompt: {formatted_prompt[:200]}...") except Exception as e: @@ -1319,20 +1410,9 @@ class InferenceBackend: def _is_gpt_oss_model(self, model_name: str = None) -> bool: """Check if the given (or active) model uses the gpt-oss harmony protocol.""" - name = (model_name or self.active_model_name or "").lower() - try: - from utils.datasets import MODEL_TO_TEMPLATE_MAPPER + from utils.datasets import is_gpt_oss_model_name - # Exact match - if MODEL_TO_TEMPLATE_MAPPER.get(name) == "gpt-oss": - return True - # Partial match (e.g. name-bnb-4bit variants) - for key, tmpl in MODEL_TO_TEMPLATE_MAPPER.items(): - if tmpl == "gpt-oss" and (key in name or name in key): - return True - except Exception: - pass - return "gpt-oss" in name + return is_gpt_oss_model_name(model_name or self.active_model_name or "") def generate_stream( self, @@ -1715,6 +1795,34 @@ class InferenceBackend: "Patched RepetitionPenaltyLogitsProcessor with 64-token window for OuteTTS" ) + def _apply_chat_template_for_generation( + self, + tokenizer, + messages: list, + *, + tools: Optional[list] = None, + enable_thinking: Optional[bool] = None, + reasoning_effort: Optional[str] = None, + preserve_thinking: Optional[bool] = None, + ) -> str: + """Render the chat prompt, peeling kwargs the template does not + understand. Delegates to the dependency-light helper module so + the fallback chain can be unit-tested without pulling unsloth / + torch into the test sandbox. + """ + from core.inference.chat_template_helpers import ( + apply_chat_template_for_generation, + ) + + return apply_chat_template_for_generation( + tokenizer, + messages, + tools = tools, + enable_thinking = enable_thinking, + reasoning_effort = reasoning_effort, + preserve_thinking = preserve_thinking, + ) + def format_chat_prompt(self, messages: list, system_prompt: str = None) -> str: if not self.active_model_name or self.active_model_name not in self.models: logger.error("No active model available") diff --git a/studio/backend/core/inference/llama_cpp.py b/studio/backend/core/inference/llama_cpp.py index 21f2fe71b5..260e675a73 100644 --- a/studio/backend/core/inference/llama_cpp.py +++ b/studio/backend/core/inference/llama_cpp.py @@ -28,10 +28,25 @@ from urllib.parse import urlparse import httpx +from core.tool_healing import ( + _TC_END_TAG_RE, + _TC_FUNC_CLOSE_RE, + _TC_FUNC_START_RE, + _TC_JSON_START_RE, + _TC_PARAM_CLOSE_RE, + _TC_PARAM_START_RE, + _TOOL_ALL_PATS, + _TOOL_CLOSED_PATS, + parse_tool_calls_from_text, + strip_tool_call_markup, +) from utils.native_path_leases import child_env_without_native_path_secret from utils.subprocess_compat import ( windows_hidden_subprocess_kwargs as _windows_hidden_subprocess_kwargs, ) +from core.inference.tool_call_parser import ( + parse_tool_calls_from_text as _shared_parse_tool_calls_from_text, +) logger = get_logger(__name__) @@ -362,25 +377,6 @@ def _extract_model_size_b(model_id: str): return extract_model_size_b(model_id) -# ── Pre-compiled patterns for tool XML stripping ───────────── -_TOOL_CLOSED_PATS = [ - re.compile(r".*?", re.DOTALL), - re.compile(r".*?", re.DOTALL), -] -_TOOL_ALL_PATS = _TOOL_CLOSED_PATS + [ - re.compile(r".*$", re.DOTALL), - re.compile(r".*$", re.DOTALL), -] - -# ── Pre-compiled patterns for tool-call XML parsing ────────── -_TC_JSON_START_RE = re.compile(r"\s*\{") -_TC_FUNC_START_RE = re.compile(r"\s*") -_TC_END_TAG_RE = re.compile(r"") -_TC_FUNC_CLOSE_RE = re.compile(r"\s*\s*$") -_TC_PARAM_START_RE = re.compile(r"\s*") -_TC_PARAM_CLOSE_RE = re.compile(r"\s*\s*$") - - _TOOL_TEMPLATE_MARKERS = ( "{%- if tools %}", "{%- if tools -%}", @@ -471,8 +467,9 @@ def _is_mtp_model_name( def _extra_args_set_spec_type(extra_args: Optional[Iterable[str]]) -> bool: - """User passed --spec-type / --spec-default? llama-server accumulates - repeated --spec-type, so we suppress auto-emit when this is true.""" + """User passed --spec-type / --spec-default? llama-server takes a + single --spec-type (comma-separated to chain), so suppress + auto-emit when this is true.""" if not extra_args: return False for raw in extra_args: @@ -485,6 +482,122 @@ def _extra_args_set_spec_type(extra_args: Optional[Iterable[str]]) -> bool: return False +def _build_ngram_mod_flags( + caps: Optional[dict], + n_match: int = 24, + n_min: int = 48, + n_max: int = 64, +) -> list[str]: + """Emit the right ngram-mod knob flags for the running llama-server. + + Post-rename builds expose ``--spec-ngram-mod-n-{match,min,max}``; + pre-rename builds expose the legacy ``--spec-ngram-size-n`` / + ``--draft-min`` / ``--draft-max``. ``caps`` comes from + ``probe_server_capabilities``; ``ngram_mod_flavor`` tells us which + set is real (vs a removal-stub entry). Returns ``[]`` when neither + set is available so the caller can drop ngram-mod entirely. + """ + flavor = caps.get("ngram_mod_flavor") if caps else None + if flavor == "new": + return [ + "--spec-ngram-mod-n-match", + str(n_match), + "--spec-ngram-mod-n-min", + str(n_min), + "--spec-ngram-mod-n-max", + str(n_max), + ] + if flavor == "legacy": + # Legacy llama.cpp before the spec arg rename: same knobs lived + # under --spec-ngram-size-n (lookup length) and the generic + # --draft-min / --draft-max (ngram size N range). + return [ + "--spec-ngram-size-n", + str(n_match), + "--draft-min", + str(n_min), + "--draft-max", + str(n_max), + ] + return [] + + +# Canonical Speculative Decoding modes exposed by the Studio chat UI. +# The dropdown renders five options (auto, mtp, ngram, mtp+ngram, off); +# the load API also accepts legacy values that the original Switch and +# external callers emit (default, draft-mtp, ngram-mod, ngram-simple). +_CANONICAL_SPEC_MODES = {"auto", "mtp", "ngram", "mtp+ngram", "off", "ngram-simple"} +_LEGACY_SPEC_MODE_MAP = { + "default": "auto", + "draft-mtp": "mtp", + "ngram-mod": "ngram", +} + + +def _canonicalize_spec_mode(value): + """Map any accepted ``speculative_type`` input onto a canonical mode. + + Returns one of ``auto``, ``mtp``, ``ngram``, ``mtp+ngram``, ``off``, + ``ngram-simple``, or ``None`` (callers treat ``None`` as ``auto``). + Unknown strings collapse to ``auto`` so a stale UI value or typo + falls back to the safe platform-aware path. + """ + if value is None: + return None + if not isinstance(value, str): + return None + stripped = value.strip().lower() + if not stripped: + return None + if stripped in _CANONICAL_SPEC_MODES: + return stripped + if stripped in _LEGACY_SPEC_MODE_MAP: + return _LEGACY_SPEC_MODE_MAP[stripped] + # llama.cpp comma-chains are emitted by old persisted state e.g. + # "ngram-mod,draft-mtp"; collapse the most common one explicitly. + pieces = [p.strip() for p in stripped.split(",") if p.strip()] + has_mtp = any(p in ("mtp", "draft-mtp") for p in pieces) + has_ngram = any(p in ("ngram", "ngram-mod") for p in pieces) + if has_mtp and has_ngram: + return "mtp+ngram" + if has_mtp: + return "mtp" + if has_ngram: + return "ngram" + return "auto" + + +def _backfill_usage_from_timings(usage, timings): + """Synthesize ``usage`` from llama-server's ``timings`` when the + OpenAI-style usage block is missing or reports zero tokens. + + The Studio chat UI computes generation t/s from + ``meta.usage.completion_tokens / totalStreamTime``. llama-server + always populates ``timings.predicted_n`` (true decoded count) and + ``timings.prompt_n``, but the ``usage`` field on the final SSE chunk + can be absent or zero on some server builds / streaming + configurations, which makes the UI fall back to wall-clock t/s and + dilute speculative-decoding speedups. + """ + if not timings: + return usage + if usage and usage.get("completion_tokens"): + return usage + predicted_n = timings.get("predicted_n") + prompt_n = timings.get("prompt_n") + if predicted_n is None and prompt_n is None: + return usage + out = dict(usage or {}) + if not out.get("completion_tokens") and predicted_n is not None: + out["completion_tokens"] = predicted_n + if not out.get("prompt_tokens") and prompt_n is not None: + out["prompt_tokens"] = prompt_n + out["total_tokens"] = int(out.get("prompt_tokens") or 0) + int( + out.get("completion_tokens") or 0 + ) + return out + + class LlamaCppBackend: """ Manages a llama-server subprocess for GGUF model inference. @@ -519,6 +632,15 @@ class LlamaCppBackend: self._cache_type_kv: Optional[str] = None self._reasoning_default: bool = True self._speculative_type: Optional[str] = None + # Canonical UI-facing mode the user requested: one of + # ``auto``/``mtp``/``ngram``/``mtp+ngram``/``off``/``ngram-simple``. + # Round-tripped through the status API so the dropdown reflects + # the picked mode rather than the resolved internal flag set + # (auto on a 27B MTP GGUF resolves to draft-mtp but the dropdown + # should still read "Auto"). + self._requested_spec_mode: Optional[str] = None + # User-supplied --spec-draft-n-max override (None = platform default). + self._spec_draft_n_max: Optional[int] = None # KV-cache estimation fields (populated by _read_gguf_metadata) self._n_layers: Optional[int] = None self._n_kv_heads: Optional[int] = None @@ -799,6 +921,17 @@ class LlamaCppBackend: def speculative_type(self) -> Optional[str]: return self._speculative_type + @property + def requested_spec_mode(self) -> Optional[str]: + """Canonical UI-facing mode the user requested (see field doc).""" + return self._requested_spec_mode + + @property + def spec_draft_n_max(self) -> Optional[int]: + """User --spec-draft-n-max override active on the load, or None + when the platform default (6 GPU / 3 CPU) is in effect.""" + return self._spec_draft_n_max + # ── Binary discovery ────────────────────────────────────────── @staticmethod @@ -925,11 +1058,31 @@ class LlamaCppBackend: cls, binary: Optional[str] = None ) -> dict[str, object]: """Parse `llama-server --help` for feature flags. Returns - {found, mtp_token, supports_mtp}. mtp_token is "draft-mtp" - (older) or "mtp" (renamed upstream), or None.""" + {found, mtp_token, supports_mtp, ngram_mod_flavor, + supports_ngram_mod, spec_draft_n_max_flag}. + + ``ngram_mod_flavor`` is ``"new"`` when the binary exposes the + post-rename ``--spec-ngram-mod-n-match / -n-min / -n-max`` as + real args, ``"legacy"`` when only the pre-rename + ``--spec-ngram-size-n / --draft-min / --draft-max`` are real + (the rename ships with stub removal entries for the legacy + names; we tell stubs apart by the "argument has been removed" + description), or ``None`` if neither set is usable. + + ``spec_draft_n_max_flag`` is the actual flag name the binary + accepts: ``--spec-draft-n-max`` on post-rename builds, or + ``--draft-max`` on legacy. ``None`` means n_max cannot be set. + """ bin_path = binary or cls._find_llama_server_binary() if not bin_path or not Path(bin_path).is_file(): - return {"found": False, "mtp_token": None, "supports_mtp": False} + return { + "found": False, + "mtp_token": None, + "supports_mtp": False, + "ngram_mod_flavor": None, + "supports_ngram_mod": False, + "spec_draft_n_max_flag": None, + } try: mtime = int(Path(bin_path).stat().st_mtime) except OSError: @@ -940,6 +1093,8 @@ class LlamaCppBackend: return cached mtp_token: Optional[str] = None + ngram_mod_flavor: Optional[str] = None + spec_draft_n_max_flag: Optional[str] = None try: result = subprocess.run( [bin_path, "--help"], @@ -949,6 +1104,52 @@ class LlamaCppBackend: check = False, ) help_text = (result.stdout or "") + "\n" + (result.stderr or "") + # Split into per-flag blocks: each --flag line plus its + # indented continuation lines, so the "argument has been + # removed" description sits with its flag. + blocks: dict[str, str] = {} + current_flags: list[str] = [] + current_desc: list[str] = [] + for line in help_text.splitlines(): + stripped = line.strip() + if stripped.startswith("-") and not line.startswith(" "): + # New flag line; flush previous. + if current_flags: + desc = " ".join(current_desc) + for f in current_flags: + blocks[f] = desc + current_flags = [] + current_desc = [stripped] + # Extract long-form flag tokens from the DECLARATION + # prefix only (comma-separated aliases). Stop at the + # first token that isn't itself a flag, so flag + # references inside descriptions are ignored. + for tok in re.split(r"[,\s]+", stripped): + if tok.startswith("--") and re.match( + r"--[A-Za-z][A-Za-z0-9_-]*$", tok + ): + current_flags.append(tok) + elif tok.startswith("-") and len(tok) > 1: + # short alias like -fa; keep scanning aliases. + continue + else: + # First non-flag token marks end of decl. + break + else: + current_desc.append(stripped) + if current_flags: + desc = " ".join(current_desc) + for f in current_flags: + blocks[f] = desc + + def _is_real(flag: str) -> bool: + """True if the flag exists AND is not a removal stub.""" + desc = blocks.get(flag) + if desc is None: + return False + return "argument has been removed" not in desc + + # MTP token detection from --spec-type line. spec_line = "" for line in help_text.splitlines(): if "--spec-type" in line: @@ -959,6 +1160,30 @@ class LlamaCppBackend: mtp_token = "draft-mtp" elif re.search(r"[|,\[]mtp[|,\]]", spec_line): mtp_token = "mtp" + + # ngram-mod flag flavor. Post-rename builds advertise both + # the new args (real) and the legacy ones (stubs); pre-rename + # builds only have the legacy ones as real. + new_ngram_real = ( + _is_real("--spec-ngram-mod-n-match") + and _is_real("--spec-ngram-mod-n-min") + and _is_real("--spec-ngram-mod-n-max") + ) + legacy_ngram_real = ( + _is_real("--spec-ngram-size-n") + and _is_real("--draft-max") + and _is_real("--draft-min") + ) + if new_ngram_real: + ngram_mod_flavor = "new" + elif legacy_ngram_real: + ngram_mod_flavor = "legacy" + + # n_max flag: prefer post-rename, fall back to legacy. + if _is_real("--spec-draft-n-max"): + spec_draft_n_max_flag = "--spec-draft-n-max" + elif _is_real("--draft-max"): + spec_draft_n_max_flag = "--draft-max" except (OSError, subprocess.SubprocessError) as exc: logger.debug(f"llama-server --help probe failed: {exc}") @@ -966,6 +1191,9 @@ class LlamaCppBackend: "found": True, "mtp_token": mtp_token, "supports_mtp": mtp_token is not None, + "ngram_mod_flavor": ngram_mod_flavor, + "supports_ngram_mod": ngram_mod_flavor is not None, + "spec_draft_n_max_flag": spec_draft_n_max_flag, } cls._capability_cache[cache_key] = info return info @@ -1470,6 +1698,7 @@ class LlamaCppBackend: kv_unified: bool = True, ctx_checkpoints: int = 0, kv_on_gpu: bool = True, + mtp_engaged: bool = False, ) -> int: """Return the largest context length that fits in GPU VRAM. @@ -1483,6 +1712,12 @@ class LlamaCppBackend: the KV cache lives in CPU RAM and doesn't compete with weights for VRAM; the requested context is honored verbatim. The other keyword args mirror ``_estimate_kv_cache_bytes``. + + ``mtp_engaged`` reserves extra VRAM for the MTP draft model's + KV cache + compute graph buffers. llama.cpp's MTP path keeps a + secondary cache sized off the target's KV; on tight VRAM tiers + (e.g. 32 GB) auto-fit at native context would otherwise spill + and force llama-server into a slower partial-offload path. """ if not self._can_estimate_kv(): logger.debug( @@ -1503,7 +1738,9 @@ class LlamaCppBackend: ctx_checkpoints = ctx_checkpoints, ) - budget_bytes = available_mib * 1024 * 1024 * 0.90 + # MTP needs a tighter budget; drop from 0.90 to 0.85. + budget_frac = 0.85 if mtp_engaged else 0.90 + budget_bytes = available_mib * 1024 * 1024 * budget_frac model_footprint = model_size_bytes # Check if requested context already fits @@ -2264,6 +2501,7 @@ class LlamaCppBackend: chat_template_override: Optional[str] = None, cache_type_kv: Optional[str] = None, speculative_type: Optional[str] = None, + spec_draft_n_max: Optional[int] = None, n_threads: Optional[int] = None, n_gpu_layers: Optional[int] = None, # Accepted for caller compat, unused n_parallel: int = 1, @@ -2295,6 +2533,7 @@ class LlamaCppBackend: n_ctx = n_ctx, cache_type_kv = cache_type_kv, speculative_type = speculative_type, + spec_draft_n_max = spec_draft_n_max, chat_template_override = chat_template_override, extra_args = extra_args, is_vision = is_vision, @@ -2387,6 +2626,35 @@ class LlamaCppBackend: # GPU/VRAM-fit logic below may shrink this if hardware is limited. max_available_ctx = self._context_length or effective_ctx + # Will MTP engage on this load? If so, the auto-fit + # budget needs to reserve extra VRAM for the draft + # model's KV cache + compute graph. Mirrors the + # canonical-mode resolver in _build_speculative_flags: + # forced mtp / mtp+ngram always engage; auto only + # engages on an MTP GGUF >= 3B (sub-3B auto falls + # back to ngram-mod which doesn't need headroom); + # ngram / ngram-simple / off never engage MTP. + _mtp_canonical = _canonicalize_spec_mode(speculative_type) + _mtp_effective = _mtp_canonical or "auto" + _mtp_size_for_fit = _extract_model_size_b(model_identifier) + _mtp_sub_3b_for_fit = ( + _mtp_size_for_fit is not None and _mtp_size_for_fit < 3.0 + ) + _mtp_will_engage = bool( + not _extra_args_set_spec_type(extra_args) + and ( + _mtp_effective in ("mtp", "mtp+ngram") + or ( + _mtp_effective == "auto" + and ( + bool(self._nextn_predict_layers) + or _is_mtp_model_name(model_identifier, model_path) + ) + and not _mtp_sub_3b_for_fit + ) + ) + ) + # Auto-cap context to fit in GPU VRAM and select GPUs. # # Two policies depending on whether the user set n_ctx: @@ -2422,6 +2690,7 @@ class LlamaCppBackend: model_size, cache_type_kv, n_parallel = n_parallel, + mtp_engaged = _mtp_will_engage, ) kv = self._estimate_kv_cache_bytes( capped, cache_type_kv, n_parallel = n_parallel @@ -2473,6 +2742,7 @@ class LlamaCppBackend: model_size, cache_type_kv, n_parallel = n_parallel, + mtp_engaged = _mtp_will_engage, ) kv = self._estimate_kv_cache_bytes( capped, cache_type_kv, n_parallel = n_parallel @@ -2631,10 +2901,10 @@ class LlamaCppBackend: # Qwen3-235B offloaded | 12 t/s | 21 t/s | 1.8x # gpt-oss-120b repeat (92% accept)| 181 t/s | 814 t/s | 4.5x # - # Params from llama.cpp docs (docs/speculative.md): - # --spec-ngram-size-n 24 (small n not recommended) - # --draft-min 48 --draft-max 64 (MoEs need long drafts; - # dense models can reduce these) + # Params from llama.cpp server README: + # --spec-ngram-mod-n-match 24 (lookup length) + # --spec-ngram-mod-n-min 48 --spec-ngram-mod-n-max 64 + # (MoEs need long drafts; dense models can reduce these) # ref: https://github.com/ggml-org/llama.cpp/blob/master/docs/speculative.md # ref: https://github.com/ggml-org/llama.cpp/pull/19164 # ref: https://github.com/ggml-org/llama.cpp/pull/18471 @@ -2642,94 +2912,16 @@ class LlamaCppBackend: # (llama.cpp #22673). Auto-enabled via nextn_predict_layers, # fallback to -MTP in name. GPU: MTP-only. CPU/Mac: chain # with ngram-mod. See unsloth.ai/docs/models/qwen3.6#mtp-guide. - _valid_spec_types = {"ngram-simple", "ngram-mod", "draft-mtp"} - normalized_spec = ( - speculative_type.lower().strip() if speculative_type else None + spec_flags = self._build_speculative_flags( + speculative_type = speculative_type, + spec_draft_n_max = spec_draft_n_max, + extra_args = extra_args, + model_identifier = model_identifier, + model_path = model_path, + gpus = bool(gpus), + binary = binary, ) - is_mtp_model = bool(self._nextn_predict_layers) or ( - _is_mtp_model_name(model_identifier, model_path) - ) - user_owns_spec_type = _extra_args_set_spec_type(extra_args) - # Auto-promote unset/"default" to draft-mtp on MTP GGUFs. - # llama.cpp #22673: MTP is compatible with mmproj, so the - # vision gate previously here was wrong. - if ( - is_mtp_model - and not user_owns_spec_type - and normalized_spec in (None, "", "default") - ): - normalized_spec = "draft-mtp" - if user_owns_spec_type: - # User --spec-type wins (it accumulates if repeated). - normalized_spec = None - self._speculative_type = None - if normalized_spec and normalized_spec != "off": - if normalized_spec == "default": - cmd.append("--spec-default") - self._speculative_type = "default" - elif normalized_spec == "draft-mtp": - # Probe binary; fail gracefully on outdated prebuilts. - # Use whichever token the binary advertises - # (older: draft-mtp; renamed upstream: mtp). - caps = self.probe_server_capabilities(binary) - mtp_token = caps.get("mtp_token") if caps else None - if not mtp_token: - logger.warning( - "MTP GGUF detected but llama-server lacks " - "--spec-type mtp/draft-mtp; run " - "`unsloth studio update`. Loading without " - "speculative decoding." - ) - self._speculative_type = None - else: - if gpus: - cmd.extend( - [ - "--spec-type", - mtp_token, - "--spec-draft-n-max", - "6", - ] - ) - else: - cmd.extend( - [ - "--spec-type", - mtp_token, - "--spec-draft-n-max", - "3", - "--spec-type", - "ngram-mod", - "--spec-ngram-mod-n-match", - "24", - "--spec-ngram-mod-n-min", - "48", - "--spec-ngram-mod-n-max", - "6", - ] - ) - self._speculative_type = "draft-mtp" - logger.info( - f"Spec decoding: {mtp_token} ({'GPU' if gpus else 'CPU/Mac'})" - ) - elif normalized_spec in _valid_spec_types: - cmd.extend(["--spec-type", normalized_spec]) - if normalized_spec == "ngram-mod": - cmd.extend( - [ - "--spec-ngram-size-n", - "24", - "--draft-min", - "48", - "--draft-max", - "64", - ] - ) - self._speculative_type = normalized_spec - else: - self._speculative_type = None - else: - self._speculative_type = None + cmd.extend(spec_flags) # Apply custom chat template override if provided self._chat_template_override = chat_template_override @@ -3061,6 +3253,220 @@ class LlamaCppBackend: ) return True + def _build_speculative_flags( + self, + *, + speculative_type: Optional[str], + spec_draft_n_max: Optional[int], + extra_args: Optional[List[str]], + model_identifier: str, + model_path: Optional[str], + gpus: bool, + binary: Optional[str], + ) -> List[str]: + """Return the llama-server flag list for the requested spec mode. + + Side effects: sets ``self._speculative_type`` (resolved internal + emit), ``self._requested_spec_mode`` (canonical UI mode for the + status round-trip), and ``self._spec_draft_n_max`` (user override + only; None when the platform default applies). + + Speculative decoding (n-gram self-speculation, zero VRAM cost): + ngram-mod uses a ~16 MB shared hash pool, constant memory / + complexity, variable draft lengths. Helps most when the model + repeats existing text (code refactor, summarisation, reasoning). + For general chat with low repetition, overhead is ~5 ms. + + Benchmarks from upstream llama.cpp speculative-decoding PRs: + Scenario | Without | With | Speedup + gpt-oss-120b code refactor | 181 t/s | 446 t/s | 2.5x + Qwen3-235B offloaded | 12 t/s | 21 t/s | 1.8x + gpt-oss-120b repeat (92% accept)| 181 t/s | 814 t/s | 4.5x + + Sub-3B dense MTP regresses vs spec-off because the draft head's + per-token cost exceeds the acceptance savings at this scale. + Q4_K_XL clean bench (each prompt once after an unrelated warmup) + on B200 + x86 CPU: + 0.8B GPU: draft-mtp n=2 = 0.58x vs OFF; ngram-only = 1.10x + 2B GPU: draft-mtp n=2 = 0.82x vs OFF; OFF or ngram = 1.00x + 0.8B CPU: chained n=2 = 0.86x vs OFF; ngram-only = 1.19x + 2B CPU: chained n=2 = 0.83x vs OFF; ngram-only = 1.01x + 4B+ GPU/CPU: spec on is a net win (1.08x-1.46x). + Auto falls back to ngram-mod (zero-VRAM, near-zero idle cost on + diverse content); forced MTP variants engage anyway and just log + a warning per the user's choice. + """ + flags: List[str] = [] + # Reset; emit branches re-set on the resolved emission. + self._spec_draft_n_max = None + self._speculative_type = None + + # Canonical UI-facing requested mode: auto / mtp / ngram / + # mtp+ngram / off / ngram-simple. Legacy values are mapped via + # _canonicalize_spec_mode (default->auto, draft-mtp->mtp, + # ngram-mod->ngram, "ngram-mod,draft-mtp"->mtp+ngram). + canonical_mode = _canonicalize_spec_mode(speculative_type) + is_mtp_model = bool(self._nextn_predict_layers) or ( + _is_mtp_model_name(model_identifier, model_path) + ) + user_owns_spec_type = _extra_args_set_spec_type(extra_args) + _mtp_size_b = _extract_model_size_b(model_identifier) + _mtp_too_small = _mtp_size_b is not None and _mtp_size_b < 3.0 + + if user_owns_spec_type: + # User --spec-type in extra_args wins outright; suppress + # auto-emit so we don't emit a duplicate / conflicting + # spec block. Record requested mode as None. + self._requested_spec_mode = None + return flags + + effective_mode = canonical_mode or "auto" + self._requested_spec_mode = effective_mode + + def _resolved_draft_n_max() -> int: + # User override wins; else platform default (the B200 / x86 + # clean-sweep sweet spot from PR #5582 is n=2 GPU, n=3 CPU; + # raising past 3 starts to regress on essay-style + # low-acceptance prompts). + if spec_draft_n_max is not None: + n = int(spec_draft_n_max) + self._spec_draft_n_max = n + return n + return 2 if gpus else 3 + + def _emit_mtp(*, chain_ngram: bool) -> bool: + """Append --spec-type mtp[/draft-mtp][,ngram-mod] + n-max.""" + caps = self.probe_server_capabilities(binary) + mtp_token = caps.get("mtp_token") if caps else None + if not mtp_token: + logger.warning( + "Requested MTP speculative decoding but " + "llama-server lacks --spec-type mtp/draft-mtp; " + "run `unsloth studio update`. Loading without " + "speculative decoding." + ) + return False + draft_n_max = _resolved_draft_n_max() + n_max_flag = caps.get("spec_draft_n_max_flag") or "--spec-draft-n-max" + if chain_ngram: + ngram_knobs = _build_ngram_mod_flags(caps) + if ngram_knobs: + spec_value = f"ngram-mod,{mtp_token}" + else: + logger.warning( + "llama-server lacks ngram-mod tuning " + "flags; loading MTP only (no ngram chain)" + ) + spec_value = mtp_token + flags.extend( + [ + "--spec-type", + spec_value, + n_max_flag, + str(draft_n_max), + ] + ) + flags.extend(ngram_knobs) + else: + flags.extend( + [ + "--spec-type", + mtp_token, + n_max_flag, + str(draft_n_max), + ] + ) + self._speculative_type = "draft-mtp" + chain_label = "chained ngram-mod" if chain_ngram else "MTP-only" + logger.info(f"Spec decoding: {mtp_token} ({chain_label})") + return True + + def _emit_ngram_mod() -> bool: + """Append --spec-type ngram-mod + flag-set knobs.""" + ngram_caps = self.probe_server_capabilities(binary) + ngram_knobs = _build_ngram_mod_flags(ngram_caps) + flags.extend(["--spec-type", "ngram-mod"]) + if not ngram_knobs: + logger.warning( + "llama-server lacks ngram-mod tuning " + "flags; loading without --spec-ngram-mod-* knobs" + ) + flags.extend(ngram_knobs) + self._speculative_type = "ngram-mod" + logger.info("Spec decoding: ngram-mod") + return True + + if effective_mode == "off": + return flags # nothing to emit + if effective_mode == "ngram-simple": + flags.extend(["--spec-type", "ngram-simple"]) + self._speculative_type = "ngram-simple" + return flags + if effective_mode == "ngram": + _emit_ngram_mod() + return flags + if effective_mode == "mtp": + if _mtp_too_small: + logger.warning( + f"Forcing MTP on a {_mtp_size_b:.1f}B model; " + "the bench shows draft-mtp regresses below 3B. " + "Engaging anyway (user override)." + ) + elif not is_mtp_model: + logger.warning( + "Forcing MTP on a non-MTP GGUF; llama-server may " + "fall back to spec-off if no nextn head is present. " + "Engaging anyway (user override)." + ) + _emit_mtp(chain_ngram = False) + return flags + if effective_mode == "mtp+ngram": + if _mtp_too_small: + logger.warning( + f"Forcing MTP+Ngram on a {_mtp_size_b:.1f}B model; " + "the bench shows the chain regresses below 3B. " + "Engaging anyway (user override)." + ) + elif not is_mtp_model: + logger.warning( + "Forcing MTP+Ngram on a non-MTP GGUF; llama-server " + "may fall back to ngram-only if no nextn head is " + "present. Engaging anyway (user override)." + ) + _emit_mtp(chain_ngram = True) + return flags + + # effective_mode == "auto": today's promotion path. llama.cpp + # #22673: MTP is compatible with mmproj, so there's no vision gate. + if is_mtp_model and not _mtp_too_small: + # GPU: MTP-only. CPU/Mac: chain ngram-mod + MTP. + _emit_mtp(chain_ngram = not gpus) + elif is_mtp_model and _mtp_too_small: + # Sub-3B fallback: drop the MTP draft head, keep ngram-mod + # when the binary supports it. + _small_caps = self.probe_server_capabilities(binary) + if _small_caps.get("supports_ngram_mod"): + logger.info( + f"MTP GGUF detected but model size {_mtp_size_b:.1f}B " + "is below the 3B speedup threshold; using ngram-mod " + "only (zero-VRAM, no draft head). Override via " + "--spec-type or the Studio Speculative Decoding " + "dropdown." + ) + _emit_ngram_mod() + else: + logger.info( + f"MTP GGUF detected but model size {_mtp_size_b:.1f}B " + "is below the 3B speedup threshold and the bundled " + "llama-server does not advertise ngram-mod; " + "auto-disabling speculative decoding." + ) + else: + # Non-MTP model: let llama-server choose its default strategy. + flags.append("--spec-default") + self._speculative_type = "default" + return flags + def _already_in_target_state( self, *, @@ -3073,6 +3479,7 @@ class LlamaCppBackend: extra_args: Optional[List[str]], is_vision: bool, gguf_path: Optional[str] = None, + spec_draft_n_max: Optional[int] = None, ) -> bool: """True iff the live server already satisfies these load kwargs. @@ -3109,18 +3516,28 @@ class LlamaCppBackend: if _norm(self._cache_type_kv) != _norm(cache_type_kv): return False - # Mirror load_model's auto-promotion. Vision is no longer a - # spec blocker (llama.cpp #22673: MTP is compatible with mmproj). - raw_spec = _norm(speculative_type) - req_spec = raw_spec or "off" + # Compare on the canonical UI-facing mode the user requested. + # When extra_args carries --spec-type, the route-layer code paths + # bypass the dropdown anyway and the backend stores + # _requested_spec_mode = None; the request mirrors that by + # canonicalising to None. + if _extra_args_set_spec_type(extra_args): + req_mode = None + else: + req_mode = _canonicalize_spec_mode(speculative_type) or "auto" + backend_mode = self._requested_spec_mode + if req_mode != backend_mode: + return False + + # spec_draft_n_max only matters when an MTP variant is actually + # engaged. Compare on the resolved spec rather than the requested + # mode so an Auto request that auto-promoted to draft-mtp under + # the hood still bounces a reload when the user changes n_max. if ( - raw_spec in (None, "default") - and _is_mtp_model_name(model_identifier, gguf_path) - and not _extra_args_set_spec_type(extra_args) + self._speculative_type == "draft-mtp" + and spec_draft_n_max is not None + and int(spec_draft_n_max) != (self._spec_draft_n_max or 0) ): - req_spec = "draft-mtp" - backend_spec = _norm(self._speculative_type) or "off" - if req_spec != backend_spec: return False if (self._chat_template_override or None) != (chat_template_override or None): @@ -3189,6 +3606,8 @@ class LlamaCppBackend: self._supports_tools = False self._cache_type_kv = None self._speculative_type = None + self._requested_spec_mode = None + self._spec_draft_n_max = None self._n_layers = None self._n_kv_heads = None self._n_kv_heads_by_layer = None @@ -3488,128 +3907,9 @@ class LlamaCppBackend: @staticmethod def _parse_tool_calls_from_text(content: str) -> list[dict]: - """ - Parse tool calls from XML markup in content text. - - Handles formats like: - {"name":"web_search","arguments":{"query":"..."}} - ... - Closing tags (, , ) are all optional - since models frequently omit them. - """ - tool_calls = [] - - # Pattern 1: JSON inside tags. - # Use balanced-brace extraction that skips braces inside JSON strings. - for m in _TC_JSON_START_RE.finditer(content): - brace_start = m.end() - 1 # position of the opening { - depth, i = 0, brace_start - in_string = False - while i < len(content): - ch = content[i] - if in_string: - if ch == "\\" and i + 1 < len(content): - i += 2 # skip escaped character - continue - if ch == '"': - in_string = False - elif ch == '"': - in_string = True - elif ch == "{": - depth += 1 - elif ch == "}": - depth -= 1 - if depth == 0: - break - i += 1 - if depth == 0: - json_str = content[brace_start : i + 1] - try: - obj = json.loads(json_str) - tc = { - "id": f"call_{len(tool_calls)}", - "type": "function", - "function": { - "name": obj.get("name", ""), - "arguments": obj.get("arguments", {}), - }, - } - if isinstance(tc["function"]["arguments"], dict): - tc["function"]["arguments"] = json.dumps( - tc["function"]["arguments"] - ) - tool_calls.append(tc) - except (json.JSONDecodeError, ValueError): - pass - - # Pattern 2: XML-style value - # All closing tags optional -- models frequently omit , - # , and/or . - if not tool_calls: - # Step 1: Find all positions and extract their bodies. - # Body boundary: use only or next as a boundary because - # code parameter values can contain that literal string. - # After extracting, we trim a trailing if present. - func_starts = list(_TC_FUNC_START_RE.finditer(content)) - for idx, fm in enumerate(func_starts): - func_name = fm.group(1) - body_start = fm.end() - # Hard boundaries: next - next_func = ( - func_starts[idx + 1].start() - if idx + 1 < len(func_starts) - else len(content) - ) - end_tag = _TC_END_TAG_RE.search(content[body_start:]) - if end_tag: - body_end = body_start + end_tag.start() - else: - body_end = len(content) - body_end = min(body_end, next_func) - body = content[body_start:body_end] - # Trim trailing if present (it's the real closing tag) - body = _TC_FUNC_CLOSE_RE.sub("", body) - - # Step 2: Extract parameters from body. - # For single-parameter functions (the common case: code, command, - # query), use body end as the only boundary to avoid false matches - # on inside code strings. - arguments = {} - param_starts = list(_TC_PARAM_START_RE.finditer(body)) - if len(param_starts) == 1: - # Single parameter: value is everything from after the tag - # to end of body, trimming any trailing . - pm = param_starts[0] - val = body[pm.end() :] - val = _TC_PARAM_CLOSE_RE.sub("", val) - arguments[pm.group(1)] = val.strip() - else: - for pidx, pm in enumerate(param_starts): - param_name = pm.group(1) - val_start = pm.end() - # Value ends at next if present - val = _TC_PARAM_CLOSE_RE.sub("", val) - arguments[param_name] = val.strip() - - tc = { - "id": f"call_{len(tool_calls)}", - "type": "function", - "function": { - "name": func_name, - "arguments": json.dumps(arguments), - }, - } - tool_calls.append(tc) - - return tool_calls + """Thin wrapper around the shared parser in tool_call_parser + so safetensors and llama_cpp pick up the same fixes.""" + return _shared_parse_tool_calls_from_text(content) @staticmethod def _build_openai_messages( @@ -3930,6 +4230,9 @@ class LlamaCppBackend: if _stream_done: break # exit outer for if _metadata_usage or _metadata_timings: + _metadata_usage = _backfill_usage_from_timings( + _metadata_usage, _metadata_timings + ) yield { "type": "metadata", "usage": _metadata_usage, @@ -3988,10 +4291,7 @@ class LlamaCppBackend: def _strip_tool_markup(text: str, *, final: bool = False) -> str: if not auto_heal_tool_calls: return text - patterns = _TOOL_ALL_PATS if final else _TOOL_CLOSED_PATS - for pat in patterns: - text = pat.sub("", text) - return text.strip() if final else text + return strip_tool_call_markup(text, final = final) # XML prefixes that signal a tool call in content. # Empty when auto_heal is disabled so the buffer never @@ -4382,7 +4682,10 @@ class LlamaCppBackend: } ) # Accumulate tokens and timing from this iteration - _fu_r = _iter_usage or {} + _fu_r = ( + _backfill_usage_from_timings(_iter_usage, _iter_timings) + or {} + ) _accumulated_completion_tokens += _fu_r.get( "completion_tokens", 0 ) @@ -4394,7 +4697,10 @@ class LlamaCppBackend: # Content was already streamed. Yield metadata. yield {"type": "status", "text": ""} - _fu = _iter_usage or {} + _fu = ( + _backfill_usage_from_timings(_iter_usage, _iter_timings) + or {} + ) _fc = _fu.get("completion_tokens", 0) _fp = _fu.get("prompt_tokens", 0) _tc = _fc + _accumulated_completion_tokens @@ -4484,7 +4790,10 @@ class LlamaCppBackend: ) if content_accum: yield {"type": "content", "text": content_accum} - _fu = _iter_usage or {} + _fu = ( + _backfill_usage_from_timings(_iter_usage, _iter_timings) + or {} + ) _fc = _fu.get("completion_tokens", 0) _fp = _fu.get("prompt_tokens", 0) _tc = _fc + _accumulated_completion_tokens @@ -4518,9 +4827,9 @@ class LlamaCppBackend: return # ── Execute tool calls ── - _accumulated_completion_tokens += (_iter_usage or {}).get( - "completion_tokens", 0 - ) + _accumulated_completion_tokens += ( + _backfill_usage_from_timings(_iter_usage, _iter_timings) or {} + ).get("completion_tokens", 0) _it = _iter_timings or {} _accumulated_predicted_ms += _it.get("predicted_ms", 0) _accumulated_predicted_n += _it.get("predicted_n", 0) diff --git a/studio/backend/core/inference/llama_server_args.py b/studio/backend/core/inference/llama_server_args.py index 572ac2ceda..d8b7eb383e 100644 --- a/studio/backend/core/inference/llama_server_args.py +++ b/studio/backend/core/inference/llama_server_args.py @@ -148,6 +148,8 @@ _SPEC_FLAGS: frozenset[str] = frozenset( # MTP path (llama.cpp #22673). "--spec-draft-n-max", "--spec-draft-n-min", + "--spec-draft-p-min", + "--spec-draft-p-split", "--spec-ngram-mod-n-match", "--spec-ngram-mod-n-min", "--spec-ngram-mod-n-max", diff --git a/studio/backend/core/inference/mlx_inference.py b/studio/backend/core/inference/mlx_inference.py index e7bce2d33e..716e4c27a2 100644 --- a/studio/backend/core/inference/mlx_inference.py +++ b/studio/backend/core/inference/mlx_inference.py @@ -157,10 +157,59 @@ class MLXInferenceBackend: "audio_type": None, "has_audio_input": False, } + # Capture chat_template_info so the worker IPC reply can ship + # it back to the parent and the route layer classifies + # capabilities the same way as the transformers / GGUF paths. + self._populate_chat_template_info(model_name) logger.info("Model %s loaded successfully", model_name) return True + def _populate_chat_template_info(self, model_name: str) -> None: + """Mirror InferenceBackend._load_chat_template_info for MLX. + + Stores ``chat_template_info`` on ``self.models[model_name]`` + with the resolved ``tokenizer.chat_template`` so + ``_detect_safetensors_features`` (route layer) sees the same + template the model actually uses.""" + entry = self.models.get(model_name) + if not entry: + return + tok = entry.get("tokenizer") + if tok is None: + proc = entry.get("processor") + tok = getattr(proc, "tokenizer", None) if proc else None + info = { + "has_template": False, + "template": None, + "format_type": "generic", + "special_tokens": {}, + "template_name": None, + } + try: + tpl = getattr(tok, "chat_template", None) + if tpl: + info["has_template"] = True + info["template"] = tpl + lower = tpl.lower() + if "start_header_id" in lower and "end_header_id" in lower: + info["format_type"] = "llama3" + elif "[inst]" in lower and "[/inst]" in lower: + info["format_type"] = "mistral" + elif "<|im_start|>" in lower and "<|im_end|>" in lower: + info["format_type"] = "chatml" + else: + info["format_type"] = "custom" + special = {} + for attr in ("bos_token", "eos_token", "pad_token"): + val = getattr(tok, attr, None) + if val: + special[attr] = val + info["special_tokens"] = special + except Exception as exc: + logger.warning("MLX chat_template_info capture failed: %s", exc) + entry["chat_template_info"] = info + def unload_model(self, model_name: str) -> bool: import mlx.core as mx import gc @@ -197,6 +246,14 @@ class MLXInferenceBackend: max_new_tokens = 256, repetition_penalty = 1.0, cancel_event = None, + # Reasoning / tool kwargs forwarded by the route + worker -- the + # MLX path renders the template via apply_chat_template_for_ + # generation so these are honoured the same way as the + # transformers path. + tools = None, + enable_thinking = None, + reasoning_effort = None, + preserve_thinking = None, ) -> Generator[str, None, None]: if self._model is None: raise RuntimeError("No model loaded") @@ -239,6 +296,10 @@ class MLXInferenceBackend: max_new_tokens, repetition_penalty, cancel_event, + tools = tools, + enable_thinking = enable_thinking, + reasoning_effort = reasoning_effort, + preserve_thinking = preserve_thinking, ) else: yield from self._generate_text( @@ -250,6 +311,10 @@ class MLXInferenceBackend: max_new_tokens, repetition_penalty, cancel_event, + tools = tools, + enable_thinking = enable_thinking, + reasoning_effort = reasoning_effort, + preserve_thinking = preserve_thinking, ) def _generate_text( @@ -262,14 +327,26 @@ class MLXInferenceBackend: max_new_tokens, repetition_penalty, cancel_event, + *, + tools = None, + enable_thinking = None, + reasoning_effort = None, + preserve_thinking = None, ): from mlx_lm import stream_generate from mlx_lm.sample_utils import make_sampler, make_logits_processors - prompt = self._tokenizer.apply_chat_template( + from core.inference.chat_template_helpers import ( + apply_chat_template_for_generation, + ) + + prompt = apply_chat_template_for_generation( + self._tokenizer, messages, - tokenize = False, - add_generation_prompt = True, + tools = tools, + enable_thinking = enable_thinking, + reasoning_effort = reasoning_effort, + preserve_thinking = preserve_thinking, ) if prompt is None: raise RuntimeError( @@ -343,20 +420,38 @@ class MLXInferenceBackend: max_new_tokens, repetition_penalty, cancel_event, + *, + tools = None, + enable_thinking = None, + reasoning_effort = None, + preserve_thinking = None, ): from mlx_vlm import stream_generate as vlm_stream - # Apply chat template - chat_fn = getattr(self._processor, "apply_chat_template", None) + from core.inference.chat_template_helpers import ( + apply_chat_template_for_generation, + ) + + # Pick the chat-template-aware caller: processors that expose + # their own apply_chat_template + chat_template attr (e.g. + # Qwen2.5-VL) use it directly; otherwise fall back to the + # nested tokenizer. + chat_target = self._processor if ( - chat_fn is None + getattr(self._processor, "apply_chat_template", None) is None or not hasattr(self._processor, "chat_template") or self._processor.chat_template is None ): - tok = getattr(self._processor, "tokenizer", self._processor) - chat_fn = tok.apply_chat_template + chat_target = getattr(self._processor, "tokenizer", self._processor) - prompt = chat_fn(messages, tokenize = False, add_generation_prompt = True) + prompt = apply_chat_template_for_generation( + chat_target, + messages, + tools = tools, + enable_thinking = enable_thinking, + reasoning_effort = reasoning_effort, + preserve_thinking = preserve_thinking, + ) # For VLM: always use mlx_vlm's stream_generate which handles # pixel_values properly (passes None for text-only, image for VLM) diff --git a/studio/backend/core/inference/orchestrator.py b/studio/backend/core/inference/orchestrator.py index 5562820f49..7e7d7026f6 100644 --- a/studio/backend/core/inference/orchestrator.py +++ b/studio/backend/core/inference/orchestrator.py @@ -449,6 +449,10 @@ class InferenceOrchestrator: repetition_penalty: float = 1.0, cancel_event = None, use_adapter = None, + tools: Optional[list] = None, + enable_thinking: Optional[bool] = None, + reasoning_effort: Optional[str] = None, + preserve_thinking: Optional[bool] = None, ) -> Generator[str, None, None]: """Dispatched generation — sends command without holding _gen_lock. @@ -494,6 +498,14 @@ class InferenceOrchestrator: if use_adapter is not None: cmd["use_adapter"] = use_adapter + if tools is not None: + cmd["tools"] = tools + if enable_thinking is not None: + cmd["enable_thinking"] = enable_thinking + if reasoning_effort is not None: + cmd["reasoning_effort"] = reasoning_effort + if preserve_thinking is not None: + cmd["preserve_thinking"] = preserve_thinking # Create mailbox BEFORE sending command mailbox: queue.Queue = queue.Queue() @@ -695,6 +707,13 @@ class InferenceOrchestrator: "audio_type": model_info.get("audio_type"), "has_audio_input": model_info.get("has_audio_input", False), } + # Mirror chat_template_info so routes can classify + # capabilities without re-entering the subprocess. + _tpl_info = model_info.get("chat_template_info") + if isinstance(_tpl_info, dict): + self.models[self.active_model_name]["chat_template_info"] = ( + _tpl_info + ) self.loading_models.discard(model_name) logger.info( "Model '%s' loaded successfully in subprocess", model_name @@ -770,8 +789,18 @@ class InferenceOrchestrator: max_new_tokens: int = 256, repetition_penalty: float = 1.0, cancel_event = None, + tools: Optional[list] = None, + enable_thinking: Optional[bool] = None, + reasoning_effort: Optional[str] = None, + preserve_thinking: Optional[bool] = None, ) -> Generator[str, None, None]: - """Generate response, streaming tokens from subprocess.""" + """Generate response, streaming tokens from subprocess. + + Optional ``tools`` / ``enable_thinking`` / ``reasoning_effort`` / + ``preserve_thinking`` kwargs are forwarded into the worker so + ``tokenizer.apply_chat_template`` can render tool schemas and + reasoning controls when the template understands them. + """ yield from self._generate_inner( messages = messages, system_prompt = system_prompt, @@ -784,6 +813,88 @@ class InferenceOrchestrator: repetition_penalty = repetition_penalty, cancel_event = cancel_event, use_adapter = None, + tools = tools, + enable_thinking = enable_thinking, + reasoning_effort = reasoning_effort, + preserve_thinking = preserve_thinking, + ) + + def generate_chat_completion_with_tools( + self, + messages: list, + tools: list, + system_prompt: str = "", + temperature: float = 0.7, + top_p: float = 0.9, + top_k: int = 40, + min_p: float = 0.0, + max_tokens: Optional[int] = None, + repetition_penalty: float = 1.0, + cancel_event = None, + enable_thinking: Optional[bool] = None, + reasoning_effort: Optional[str] = None, + preserve_thinking: Optional[bool] = None, + max_tool_iterations: int = 25, + auto_heal_tool_calls: bool = True, + tool_call_timeout: int = 300, + session_id: Optional[str] = None, + use_adapter: Optional[Union[bool, str]] = None, + **_unused, + ): + """Run the safetensors agentic tool loop in this (parent) + process, calling the worker for each generation turn. + + Yields the same event dicts as the GGUF tool loop so the route + layer can stream both backends through one helper. See + ``safetensors_agentic.run_safetensors_tool_loop`` for the + event protocol. + """ + from core.inference.safetensors_agentic import run_safetensors_tool_loop + from core.inference.tools import execute_tool + + max_new_tokens = max_tokens if max_tokens and max_tokens > 0 else 2048 + + def _single_turn(conv: list): + # ``conv`` already carries any system message because the + # loop appends to a list seeded with system+user above. + common_kwargs = dict( + messages = conv, + system_prompt = "", + image = None, + temperature = temperature, + top_p = top_p, + top_k = top_k, + min_p = min_p, + max_new_tokens = max_new_tokens, + repetition_penalty = repetition_penalty, + cancel_event = cancel_event, + tools = tools, + enable_thinking = enable_thinking, + reasoning_effort = reasoning_effort, + preserve_thinking = preserve_thinking, + ) + if use_adapter is not None: + yield from self.generate_with_adapter_control( + use_adapter = use_adapter, + **common_kwargs, + ) + else: + yield from self.generate_chat_response(**common_kwargs) + + initial = list(messages) + if system_prompt: + initial = [{"role": "system", "content": system_prompt}] + initial + + yield from run_safetensors_tool_loop( + single_turn = _single_turn, + messages = initial, + tools = tools, + execute_tool = execute_tool, + cancel_event = cancel_event, + auto_heal_tool_calls = auto_heal_tool_calls, + max_tool_iterations = max_tool_iterations, + tool_call_timeout = tool_call_timeout, + session_id = session_id, ) def generate_with_adapter_control( @@ -817,6 +928,10 @@ class InferenceOrchestrator: repetition_penalty: float = 1.0, cancel_event = None, use_adapter = None, + tools: Optional[list] = None, + enable_thinking: Optional[bool] = None, + reasoning_effort: Optional[str] = None, + preserve_thinking: Optional[bool] = None, ) -> Generator[str, None, None]: """Inner generation logic — sends command to subprocess, yields tokens. @@ -853,6 +968,10 @@ class InferenceOrchestrator: repetition_penalty = repetition_penalty, cancel_event = cancel_event, use_adapter = use_adapter, + tools = tools, + enable_thinking = enable_thinking, + reasoning_effort = reasoning_effort, + preserve_thinking = preserve_thinking, ) def _generate_locked( @@ -868,6 +987,10 @@ class InferenceOrchestrator: repetition_penalty: float = 1.0, cancel_event = None, use_adapter = None, + tools: Optional[list] = None, + enable_thinking: Optional[bool] = None, + reasoning_effort: Optional[str] = None, + preserve_thinking: Optional[bool] = None, ) -> Generator[str, None, None]: """Actual generation logic — must be called under _gen_lock.""" request_id = str(uuid.uuid4()) @@ -893,6 +1016,16 @@ class InferenceOrchestrator: if use_adapter is not None: cmd["use_adapter"] = use_adapter + # Only forward template kwargs the caller actually set so older + # workers that ignore unknown keys still work. + if tools is not None: + cmd["tools"] = tools + if enable_thinking is not None: + cmd["enable_thinking"] = enable_thinking + if reasoning_effort is not None: + cmd["reasoning_effort"] = reasoning_effort + if preserve_thinking is not None: + cmd["preserve_thinking"] = preserve_thinking try: self._send_cmd(cmd) @@ -1200,6 +1333,13 @@ class InferenceOrchestrator: return self.models[self.active_model_name].get("is_vision", False) return False + def _is_gpt_oss_model(self, model_name: str = None) -> bool: + """Parent-side gpt-oss detection so the safetensors route can run + the same guard without an IPC round-trip to the subprocess.""" + from utils.datasets import is_gpt_oss_model_name + + return is_gpt_oss_model_name(model_name or self.active_model_name or "") + # ========== GLOBAL INSTANCE ========== _inference_backend = None diff --git a/studio/backend/core/inference/safetensors_agentic.py b/studio/backend/core/inference/safetensors_agentic.py new file mode 100644 index 0000000000..73bb3d090a --- /dev/null +++ b/studio/backend/core/inference/safetensors_agentic.py @@ -0,0 +1,392 @@ +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 + +""" +Safetensors/transformers agentic tool loop. + +Wraps a single-turn cumulative-text generator (the existing +``InferenceOrchestrator.generate_chat_response`` pipeline that streams +from a worker subprocess) with the tool-calling, thinking-block, +status, and metadata event protocol used by the GGUF path. Keeps the +front-end SSE shape identical across backends so the chat UI does not +care which engine actually ran the model. + +The GGUF path lives in ``llama_cpp.py`` and talks to llama-server's +structured ``delta.tool_calls`` directly. Native transformers has no +such structured channel, so this loop parses tool calls from the +cumulative text and dispatches them via ``core.inference.tools``. +""" + +import json +import threading +from typing import Callable, Generator, Optional +from urllib.parse import urlparse + +from loggers import get_logger + +from core.inference.tool_call_parser import ( + BUDGET_EXHAUSTED_NUDGE, + DUPLICATE_CALL_NUDGE, + TOOL_ERROR_NUDGE, + TOOL_ERROR_PREFIXES, + TOOL_XML_SIGNALS, + has_tool_signal, + parse_tool_calls_from_text, + strip_tool_markup, +) + + +logger = get_logger(__name__) + + +# Buffer cap while waiting to disambiguate a possible tool-call prefix. +_MAX_BUFFER_CHARS = 32 + + +def _status_for_tool(tool_name: str, arguments: dict) -> str: + """Return a human-readable status line matching the GGUF path.""" + if tool_name == "web_search": + url = (arguments.get("url") or "").strip() + if url: + parsed = urlparse(url) + if parsed.scheme in ("http", "https") and parsed.hostname: + host = parsed.hostname + if host.startswith("www."): + host = host[4:] + return f"Reading: {host}" + return "Reading page..." + query = arguments.get("query", "") + return f"Searching: {query}" + if tool_name == "python": + preview = (arguments.get("code") or "").strip().split("\n")[0][:60] + return f"Running Python: {preview}" if preview else "Running Python..." + if tool_name == "terminal": + preview = (arguments.get("command") or "")[:60] + return f"Running: {preview}" if preview else "Running command..." + return f"Calling: {tool_name}" + + +_CANONICAL_HEAL_ARG = {"python": "code", "terminal": "command"} + + +def _coerce_arguments(raw_args, *, heal: bool, tool_name: str = "") -> dict: + """Normalise tool ``arguments`` to a dict. + + Some templates emit a JSON string, others a bare query string. With + ``heal=True`` we accept a bare string as ``{: ...}`` + so a Hermes-style call without proper JSON still runs the tool. The + canonical key is picked per tool: ``code`` for python, ``command`` + for terminal, ``query`` for everything else (e.g. web_search). + """ + if isinstance(raw_args, dict): + return raw_args + if isinstance(raw_args, str): + try: + parsed = json.loads(raw_args) + if isinstance(parsed, dict): + return parsed + except (json.JSONDecodeError, ValueError): + pass + if heal: + key = _CANONICAL_HEAL_ARG.get(tool_name, "query") + return {key: raw_args} + return {"raw": raw_args} + return {} + + +def run_safetensors_tool_loop( + *, + single_turn: Callable[[list], Generator[str, None, None]], + messages: list[dict], + tools: list[dict], + execute_tool: Callable[..., str], + cancel_event: Optional[threading.Event] = None, + auto_heal_tool_calls: bool = True, + max_tool_iterations: int = 25, + tool_call_timeout: int = 300, + session_id: Optional[str] = None, +) -> Generator[dict, None, None]: + """Drive an agentic tool loop on top of a cumulative-text generator. + + ``single_turn(messages)`` must yield cumulative assistant text + (each yield is a snapshot including all previously emitted tokens). + The loop: + + * Buffers the leading characters of every turn so it can decide + whether the model is about to emit a tool call. Plain content + starts streaming as soon as the buffer rules it out. + * On detecting ```` or ``= 0 and (signal_pos < 0 or p < signal_pos): + signal_pos = p + if signal_pos >= 0: + before_tool = candidate[:signal_pos] + cleaned_before = strip_tool_markup(before_tool) + if len(cleaned_before) > len(last_emitted): + last_emitted = cleaned_before + yield {"type": "content", "text": cleaned_before} + cumulative_display = candidate + detect_state = _state_draining + continue + cumulative_display = candidate + cleaned = strip_tool_markup(cumulative_display) + if len(cleaned) > len(last_emitted): + last_emitted = cleaned + yield {"type": "content", "text": cleaned} + continue + + # BUFFERING: hold until we know it is not a tool call. + content_buffer += delta + stripped = content_buffer.lstrip() + if not stripped: + continue + + is_match = False + is_prefix = False + for sig in TOOL_XML_SIGNALS: + if stripped.startswith(sig): + is_match = True + break + if sig.startswith(stripped): + is_prefix = True + break + + if is_match: + detect_state = _state_draining + elif is_prefix and len(stripped) < _MAX_BUFFER_CHARS: + continue + else: + detect_state = _state_streaming + cumulative_display += content_buffer + cleaned = strip_tool_markup(cumulative_display) + if len(cleaned) > len(last_emitted): + last_emitted = cleaned + yield {"type": "content", "text": cleaned} + + # Stream finished -- resolve what we collected. + if cancel_event is not None and cancel_event.is_set(): + return + + if detect_state == _state_buffering: + # Buffer never resolved -- tool XML or plain content. + stripped = content_buffer.lstrip() + if stripped and has_tool_signal(stripped): + detect_state = _state_draining + else: + if content_buffer: + cumulative_display += content_buffer + yield { + "type": "content", + "text": strip_tool_markup(cumulative_display, final = True), + } + yield {"type": "status", "text": ""} + return + + if detect_state == _state_streaming: + # No tool detected mid-stream -- check for late tool XML. + safety_tc = None + if has_tool_signal(content_accum): + safety_tc = parse_tool_calls_from_text( + content_accum, + id_offset = next_call_id, + ) + if not safety_tc: + # Final answer: streaming already emitted content. + # Skip a final=True re-strip so literal "" + # in prose survives when no real tool call parsed. + yield {"type": "status", "text": ""} + return + tool_calls = safety_tc + content_text = strip_tool_markup(content_accum, final = True) + logger.info( + "Safetensors safety net: parsed %d tool call(s) from streamed content", + len(tool_calls), + ) + else: + # DRAINING: parse tool calls out of full content. + tool_calls = parse_tool_calls_from_text( + content_accum, + id_offset = next_call_id, + ) + if not tool_calls and auto_heal_tool_calls: + # Parser found nothing -- surface raw content so any + # literal "" prose is preserved. + if content_accum: + yield {"type": "content", "text": content_accum} + yield {"type": "status", "text": ""} + return + content_text = strip_tool_markup(content_accum, final = True) + + if final_attempt_done: + # Final-answer turn re-called a tool -- stop the loop. + if content_text: + yield {"type": "content", "text": content_text} + yield {"type": "status", "text": ""} + return + + assistant_msg: dict = {"role": "assistant", "content": content_text} + if tool_calls: + assistant_msg["tool_calls"] = tool_calls + next_call_id += len(tool_calls) + conversation.append(assistant_msg) + + for tc in tool_calls or []: + func = tc.get("function", {}) or {} + tool_name = func.get("name", "") or "" + arguments = _coerce_arguments( + func.get("arguments", {}), + heal = auto_heal_tool_calls, + tool_name = tool_name, + ) + + yield {"type": "status", "text": _status_for_tool(tool_name, arguments)} + yield { + "type": "tool_start", + "tool_name": tool_name, + "tool_call_id": tc.get("id", ""), + "arguments": arguments, + } + + tc_key = tool_name + str(arguments) + if allowed_tool_names and tool_name not in allowed_tool_names: + result = ( + f"Error: tool '{tool_name}' is not enabled for this " + "request. Use one of the enabled tools or provide a " + "final answer." + ) + else: + already_ran_ok = any( + k == tc_key and not err for k, err in tool_call_history + ) + if already_ran_ok: + result = DUPLICATE_CALL_NUDGE + else: + eff_timeout = ( + None if tool_call_timeout >= 9999 else tool_call_timeout + ) + try: + result = execute_tool( + tool_name, + arguments, + cancel_event = cancel_event, + timeout = eff_timeout, + session_id = session_id, + ) + except Exception as exc: + logger.exception("Tool %s raised: %s", tool_name, exc) + result = f"Error: tool raised an exception: {exc}" + + yield { + "type": "tool_end", + "tool_name": tool_name, + "tool_call_id": tc.get("id", ""), + "result": result, + } + + is_error = isinstance(result, str) and result.lstrip().startswith( + TOOL_ERROR_PREFIXES + ) + tool_call_history.append((tc_key, is_error)) + + # Strip frontend image sentinel from the model's view. + # Cut at the first occurrence so leading and consecutive + # sentinels are both removed. + result_for_model = result + if isinstance(result_for_model, str) and "__IMAGES__:" in result_for_model: + result_for_model = result_for_model.split("__IMAGES__:", 1)[0].rstrip() + if is_error: + result_for_model = result_for_model + TOOL_ERROR_NUDGE + + tool_msg: dict = { + "role": "tool", + "name": tool_name, + "content": result_for_model, + } + tool_call_id = tc.get("id") + if tool_call_id: + tool_msg["tool_call_id"] = tool_call_id + conversation.append(tool_msg) + + # Clear the status badge before the next turn. + yield {"type": "status", "text": ""} + + if iteration + 1 >= max_tool_iterations and not final_attempt_done: + # Budget exhausted; nudge a final plain answer. + final_attempt_done = True + conversation.append( + { + "role": "user", + "content": BUDGET_EXHAUSTED_NUDGE, + } + ) + + yield {"type": "status", "text": ""} diff --git a/studio/backend/core/inference/tool_call_parser.py b/studio/backend/core/inference/tool_call_parser.py new file mode 100644 index 0000000000..a0ab8a2a53 --- /dev/null +++ b/studio/backend/core/inference/tool_call_parser.py @@ -0,0 +1,204 @@ +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 + +""" +Backend-neutral tool-call XML parser shared by GGUF and safetensors. +Tolerates missing closing tags in either ``{json}`` +or ``v...`` shape. +""" + +import json +import re + + +# _TOOL_CLOSED_PATS: closed pairs only. _TOOL_ALL_PATS: also trailing +# unclosed runs so truncated tails don't leak markup. +_TOOL_CLOSED_PATS = [ + re.compile(r".*?", re.DOTALL), + re.compile(r".*?", re.DOTALL), +] +_TOOL_ALL_PATS = _TOOL_CLOSED_PATS + [ + re.compile(r".*$", re.DOTALL), + re.compile(r".*$", re.DOTALL), +] + + +# Prefixes the streaming buffer watches for to gate in-progress text. +TOOL_XML_SIGNALS = ("", "\s*\{") +_TC_FUNC_START_RE = re.compile(r"\s*") +_TC_END_TAG_RE = re.compile(r"") +_TC_FUNC_CLOSE_RE = re.compile(r"\s*\s*$") +_TC_PARAM_START_RE = re.compile(r"\s*") +_TC_PARAM_CLOSE_RE = re.compile(r"\s*\s*$") + + +def strip_tool_markup(text: str, *, final: bool = False) -> str: + """Strip tool-call XML from streamed text. + + ``final=False`` only removes closed pairs (used during streaming so + in-progress XML stays buffered). ``final=True`` also removes a + trailing unclosed run and trims the result. + """ + pats = _TOOL_ALL_PATS if final else _TOOL_CLOSED_PATS + for pat in pats: + text = pat.sub("", text) + return text.strip() if final else text + + +def parse_tool_calls_from_text(content: str, *, id_offset: int = 0) -> list[dict]: + """Parse OpenAI-format ``tool_calls`` from model text. + + Returns a list of ``{"id", "type", "function": {"name", "arguments"}}`` + dicts. ``arguments`` is always a JSON string so callers can hand it + straight back into an OpenAI-style response. + + Handles two shapes: + + - JSON inside ```` tags: + ``{"name":"web_search","arguments":{"query":"..."}}`` + - XML-style function blocks: + ``v`` + + Closing tags (````, ````, ````) + are all optional since models frequently omit them. + """ + tool_calls: list[dict] = [] + + # Pattern 1: {json}. Balanced-brace scan that skips + # braces inside JSON strings. + for m in _TC_JSON_START_RE.finditer(content): + brace_start = m.end() - 1 # position of the opening { + depth, i = 0, brace_start + in_string = False + while i < len(content): + ch = content[i] + if in_string: + if ch == "\\" and i + 1 < len(content): + i += 2 + continue + if ch == '"': + in_string = False + elif ch == '"': + in_string = True + elif ch == "{": + depth += 1 + elif ch == "}": + depth -= 1 + if depth == 0: + break + i += 1 + if depth == 0: + json_str = content[brace_start : i + 1] + try: + obj = json.loads(json_str) + tc = { + "id": f"call_{id_offset + len(tool_calls)}", + "type": "function", + "function": { + "name": obj.get("name", ""), + "arguments": obj.get("arguments", {}), + }, + } + if isinstance(tc["function"]["arguments"], dict): + tc["function"]["arguments"] = json.dumps( + tc["function"]["arguments"] + ) + tool_calls.append(tc) + except (json.JSONDecodeError, ValueError): + pass + + # Pattern 2: v... -- closing tags + # optional; don't use as body boundary because code + # values can contain that literal. + if not tool_calls: + func_starts = list(_TC_FUNC_START_RE.finditer(content)) + for idx, fm in enumerate(func_starts): + func_name = fm.group(1) + body_start = fm.end() + next_func = ( + func_starts[idx + 1].start() + if idx + 1 < len(func_starts) + else len(content) + ) + end_tag = _TC_END_TAG_RE.search(content[body_start:]) + if end_tag: + body_end = body_start + end_tag.start() + else: + body_end = len(content) + body_end = min(body_end, next_func) + body = content[body_start:body_end] + body = _TC_FUNC_CLOSE_RE.sub("", body) + + arguments: dict = {} + param_starts = list(_TC_PARAM_START_RE.finditer(body)) + if len(param_starts) == 1: + # Single param: take everything to body end so + # embedded in code strings is preserved. + pm = param_starts[0] + val = body[pm.end() :] + val = _TC_PARAM_CLOSE_RE.sub("", val) + arguments[pm.group(1)] = val.strip() + else: + for pidx, pm in enumerate(param_starts): + param_name = pm.group(1) + val_start = pm.end() + next_param = ( + param_starts[pidx + 1].start() + if pidx + 1 < len(param_starts) + else len(body) + ) + val = body[val_start:next_param] + val = _TC_PARAM_CLOSE_RE.sub("", val) + arguments[param_name] = val.strip() + + tc = { + "id": f"call_{id_offset + len(tool_calls)}", + "type": "function", + "function": { + "name": func_name, + "arguments": json.dumps(arguments), + }, + } + tool_calls.append(tc) + + return tool_calls + + +def has_tool_signal(text: str) -> bool: + """Return True if ``text`` contains any tool-call XML signal.""" + return any(s in text for s in TOOL_XML_SIGNALS) diff --git a/studio/backend/core/inference/worker.py b/studio/backend/core/inference/worker.py index cacede2d3e..20a7d2d16c 100644 --- a/studio/backend/core/inference/worker.py +++ b/studio/backend/core/inference/worker.py @@ -346,6 +346,26 @@ def _handle_load(backend, config: dict, resp_queue: Any) -> None: "audio_type": getattr(mc, "audio_type", None), "has_audio_input": getattr(mc, "has_audio_input", False), } + # Forward chat_template_info so the parent can classify + # capabilities without re-entering the subprocess. + try: + _bm = getattr(backend, "models", {}) or {} + _entry = ( + _bm.get(mc.identifier) + or _bm.get(getattr(backend, "active_model_name", None)) + or {} + ) + _tpl_info = _entry.get("chat_template_info") + if isinstance(_tpl_info, dict): + model_info["chat_template_info"] = { + "has_template": bool(_tpl_info.get("has_template", False)), + "template": _tpl_info.get("template"), + "format_type": _tpl_info.get("format_type", "generic"), + "template_name": _tpl_info.get("template_name"), + "special_tokens": _tpl_info.get("special_tokens", {}) or {}, + } + except Exception as _tpl_exc: + logger.warning("chat_template_info forward failed: %s", _tpl_exc) _send_response( resp_queue, { @@ -416,6 +436,18 @@ def _handle_generate( "cancel_event": cancel_event, } + # Optional template/tool plumbing: only forward keys that are + # actually present so the backend signature can evolve without + # breaking older command payloads. + for opt_key in ( + "tools", + "enable_thinking", + "reasoning_effort", + "preserve_thinking", + ): + if opt_key in cmd: + gen_kwargs[opt_key] = cmd[opt_key] + # Choose generation path use_adapter = cmd.get("use_adapter") if use_adapter is not None: @@ -648,36 +680,6 @@ def run_inference_process( os.environ["HF_HUB_DISABLE_XET"] = "1" logger.info("Xet transport disabled (HF_HUB_DISABLE_XET=1)") - # Offline auto-detect: skip 25s of hf_hub_download retries per file - # if DNS is dead; cached files resolve instantly under HF_HUB_OFFLINE=1. - # Scope is this subprocess only -- orchestrator spawns a fresh worker - # per load (see core/inference/orchestrator.py), so the env cannot - # persist across loads. - if "HF_HUB_OFFLINE" not in os.environ: - import socket as _socket - import threading as _threading - - # Probe on a daemon thread so concurrent sockets in the parent - # interpreter are not affected by socket.setdefaulttimeout. - _result: list = [None] - - def _probe() -> None: - try: - _socket.gethostbyname("huggingface.co") - _result[0] = False - except Exception: - _result[0] = True - - _t = _threading.Thread(target = _probe, daemon = True) - _t.start() - _t.join(2.0) - if _result[0] is None or _result[0] is True: - os.environ["HF_HUB_OFFLINE"] = "1" - os.environ.setdefault("TRANSFORMERS_OFFLINE", "1") - logger.warning( - "huggingface.co unreachable; HF_HUB_OFFLINE=1 set for this worker." - ) - import warnings from loggers.config import LogConfig diff --git a/studio/backend/core/tool_healing.py b/studio/backend/core/tool_healing.py new file mode 100644 index 0000000000..bb61965764 --- /dev/null +++ b/studio/backend/core/tool_healing.py @@ -0,0 +1,173 @@ +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 + +"""Tool-call XML parsing and stripping helpers. + +Extracted verbatim from studio/backend/core/inference/llama_cpp.py so that +external inference servers (llama-server wrappers, llama-swap, custom +shims) can reuse the same logic without importing the inference +orchestrator, structlog, httpx, or the rest of the studio backend. + +The regexes and function bodies are byte-for-byte identical to the +original inline implementation in llama_cpp.py. Any change made here must +preserve that equivalence; tests/python/test_tool_healing_extraction_is_exact.py +verifies it with AST comparison. +""" + +import json +import re + +# Pre-compiled patterns for tool XML stripping. +_TOOL_CLOSED_PATS = [ + re.compile(r".*?", re.DOTALL), + re.compile(r".*?", re.DOTALL), +] +_TOOL_ALL_PATS = _TOOL_CLOSED_PATS + [ + re.compile(r".*$", re.DOTALL), + re.compile(r".*$", re.DOTALL), +] + +# Pre-compiled patterns for tool-call XML parsing. +_TC_JSON_START_RE = re.compile(r"\s*\{") +_TC_FUNC_START_RE = re.compile(r"\s*") +_TC_END_TAG_RE = re.compile(r"") +_TC_FUNC_CLOSE_RE = re.compile(r"\s*\s*$") +_TC_PARAM_START_RE = re.compile(r"\s*") +_TC_PARAM_CLOSE_RE = re.compile(r"\s*\s*$") + + +def parse_tool_calls_from_text(content: str) -> list[dict]: + """ + Parse tool calls from XML markup in content text. + + Handles formats like: + {"name":"web_search","arguments":{"query":"..."}} + ... + Closing tags (, , ) are all optional + since models frequently omit them. + """ + tool_calls = [] + + # Pattern 1: JSON inside tags. + # Use balanced-brace extraction that skips braces inside JSON strings. + for m in _TC_JSON_START_RE.finditer(content): + brace_start = m.end() - 1 # position of the opening { + depth, i = 0, brace_start + in_string = False + while i < len(content): + ch = content[i] + if in_string: + if ch == "\\" and i + 1 < len(content): + i += 2 # skip escaped character + continue + if ch == '"': + in_string = False + elif ch == '"': + in_string = True + elif ch == "{": + depth += 1 + elif ch == "}": + depth -= 1 + if depth == 0: + break + i += 1 + if depth == 0: + json_str = content[brace_start : i + 1] + try: + obj = json.loads(json_str) + tc = { + "id": f"call_{len(tool_calls)}", + "type": "function", + "function": { + "name": obj.get("name", ""), + "arguments": obj.get("arguments", {}), + }, + } + if isinstance(tc["function"]["arguments"], dict): + tc["function"]["arguments"] = json.dumps( + tc["function"]["arguments"] + ) + tool_calls.append(tc) + except (json.JSONDecodeError, ValueError): + pass + + # Pattern 2: XML-style value + # All closing tags optional -- models frequently omit , + # , and/or . + if not tool_calls: + # Step 1: Find all positions and extract their bodies. + # Body boundary: use only or next as a boundary because + # code parameter values can contain that literal string. + # After extracting, we trim a trailing if present. + func_starts = list(_TC_FUNC_START_RE.finditer(content)) + for idx, fm in enumerate(func_starts): + func_name = fm.group(1) + body_start = fm.end() + # Hard boundaries: next + next_func = ( + func_starts[idx + 1].start() + if idx + 1 < len(func_starts) + else len(content) + ) + end_tag = _TC_END_TAG_RE.search(content[body_start:]) + if end_tag: + body_end = body_start + end_tag.start() + else: + body_end = len(content) + body_end = min(body_end, next_func) + body = content[body_start:body_end] + # Trim trailing if present (it's the real closing tag) + body = _TC_FUNC_CLOSE_RE.sub("", body) + + # Step 2: Extract parameters from body. + # For single-parameter functions (the common case: code, command, + # query), use body end as the only boundary to avoid false matches + # on inside code strings. + arguments = {} + param_starts = list(_TC_PARAM_START_RE.finditer(body)) + if len(param_starts) == 1: + # Single parameter: value is everything from after the tag + # to end of body, trimming any trailing . + pm = param_starts[0] + val = body[pm.end() :] + val = _TC_PARAM_CLOSE_RE.sub("", val) + arguments[pm.group(1)] = val.strip() + else: + for pidx, pm in enumerate(param_starts): + param_name = pm.group(1) + val_start = pm.end() + # Value ends at next if present + val = _TC_PARAM_CLOSE_RE.sub("", val) + arguments[param_name] = val.strip() + + tc = { + "id": f"call_{len(tool_calls)}", + "type": "function", + "function": { + "name": func_name, + "arguments": json.dumps(arguments), + }, + } + tool_calls.append(tc) + return tool_calls + + +def strip_tool_call_markup(text: str, *, final: bool = False) -> str: + """Strip tool-call XML markup from text. + + When ``final`` is False, only fully closed tool-call blocks are removed. + When ``final`` is True, trailing incomplete tool-call blocks are removed + too, and the result is stripped of surrounding whitespace. + """ + patterns = _TOOL_ALL_PATS if final else _TOOL_CLOSED_PATS + for pat in patterns: + text = pat.sub("", text) + return text.strip() if final else text diff --git a/studio/backend/models/inference.py b/studio/backend/models/inference.py index 99d1df37b6..e32d134628 100644 --- a/studio/backend/models/inference.py +++ b/studio/backend/models/inference.py @@ -70,7 +70,28 @@ class LoadRequest(BaseModel): ) speculative_type: Optional[str] = Field( None, - description = "Speculative decoding mode for GGUF models (e.g. 'ngram-simple', 'ngram-mod'). Ignored for non-GGUF and vision models.", + description = ( + "Speculative decoding mode for GGUF models. Canonical values: " + "'auto' (platform-aware: MTP on MTP GGUFs, ngram-mod fallback " + "for sub-3B), 'mtp' (force draft-mtp only on both GPU and CPU), " + "'ngram' (force ngram-mod only), 'mtp+ngram' (force " + "ngram-mod+draft-mtp chain on both platforms), 'off' (disabled). " + "Legacy values 'default' (-> auto), 'draft-mtp' (-> mtp), " + "'ngram-mod' (-> ngram), and 'ngram-simple' (kept as-is) are " + "still accepted. Ignored for non-GGUF and vision models." + ), + ) + spec_draft_n_max: Optional[int] = Field( + None, + ge = 1, + le = 16, + description = ( + "Max draft tokens per step for MTP speculative decoding " + "(--spec-draft-n-max). Defaults to 2 on GPU and 3 on CPU/Mac " + "when unset (upstream-bench sweet spot for dense Qwen3.6 MTP " + "quants). Only applied when speculative_type resolves to " + "'mtp' or 'mtp+ngram'." + ), ) llama_extra_args: Optional[List[str]] = Field( None, @@ -218,7 +239,19 @@ class LoadResponse(BaseModel): ) speculative_type: Optional[str] = Field( None, - description = "Active speculative decoding mode (e.g. 'ngram-simple', 'ngram-mod'), or None if disabled", + description = ( + "Canonical UI-facing requested speculative decoding mode " + "('auto' / 'mtp' / 'ngram' / 'mtp+ngram' / 'off' / " + "'ngram-simple'), round-tripped from the original LoadRequest " + "via _canonicalize_spec_mode. None when no model is loaded." + ), + ) + spec_draft_n_max: Optional[int] = Field( + None, + description = ( + "Active --spec-draft-n-max for MTP speculative decoding, or " + "None when the platform default is in effect." + ), ) @@ -340,7 +373,19 @@ class InferenceStatusResponse(BaseModel): ) speculative_type: Optional[str] = Field( None, - description = "Active speculative decoding mode (e.g. 'ngram-simple', 'ngram-mod'), or None if disabled", + description = ( + "Canonical UI-facing requested speculative decoding mode " + "('auto' / 'mtp' / 'ngram' / 'mtp+ngram' / 'off' / " + "'ngram-simple'), round-tripped from the original LoadRequest. " + "None when no model is loaded." + ), + ) + spec_draft_n_max: Optional[int] = Field( + None, + description = ( + "Active --spec-draft-n-max for MTP speculative decoding, or " + "None when the platform default is in effect." + ), ) llama_cpp_supports_mtp: bool = Field( True, diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.py index 607245467c..1b4e7051b0 100644 --- a/studio/backend/routes/inference.py +++ b/studio/backend/routes/inference.py @@ -117,6 +117,7 @@ try: LlamaCppBackend, _DEFAULT_MAX_TOKENS_FLOOR, _DEFAULT_T_MAX_PREDICT_MS, + _canonicalize_spec_mode, _hf_offline_if_dns_dead, detect_reasoning_flags, ) @@ -143,6 +144,7 @@ except ImportError: LlamaCppBackend, _DEFAULT_MAX_TOKENS_FLOOR, _DEFAULT_T_MAX_PREDICT_MS, + _canonicalize_spec_mode, _hf_offline_if_dns_dead, detect_reasoning_flags, ) @@ -233,6 +235,57 @@ router = APIRouter() studio_router = APIRouter() +def _detect_safetensors_features(backend, chat_template: Optional[str]) -> dict: + """Classify reasoning/tool capabilities via the GGUF classifier so + flags match across backends. gpt-oss is overridden because Harmony + routes reasoning and tools through tokenizer channels, not template + markup.""" + model_id = getattr(backend, "active_model_name", None) + flags = ( + detect_reasoning_flags( + chat_template, + model_identifier = model_id, + log_source = "safetensors", + ) + if chat_template + else { + "supports_reasoning": False, + "reasoning_style": "enable_thinking", + "reasoning_always_on": False, + "supports_preserve_thinking": False, + "supports_tools": False, + } + ) + # Our safetensors loop only parses {json} + # and .... Llama uses <|python_tag|>, + # Mistral uses [TOOL_CALLS]; advertising tools for those would + # enable a pill the parser cannot honour. GGUF is unaffected -- + # llama-server normalises every format into structured deltas. + if ( + flags.get("supports_tools") + and chat_template + and "" not in chat_template + and " XML this loop parses). + try: + if hasattr(backend, "_is_gpt_oss_model") and backend._is_gpt_oss_model(): + flags["supports_reasoning"] = True + flags["reasoning_style"] = "reasoning_effort" + flags["supports_tools"] = False + except Exception: + logger.debug("gpt_oss_check_failed", exc_info = True) + return flags + + def _effective_enable_tools(payload) -> Optional[bool]: """Resolve `payload.enable_tools` against the process-level tool policy. @@ -441,12 +494,17 @@ def _request_matches_loaded_settings( # spec on ``not is_vision``), so treat the request as ``off`` against # the backend's ``None`` to avoid forcing a redundant reload. if llama_backend.is_vision: - req_spec = "off" + req_mode = "off" else: - req_spec = _normalise_settings_str(request.speculative_type) or "off" - backend_spec = _normalise_settings_str(llama_backend.speculative_type) or "off" - if req_spec != backend_spec: + req_mode = _canonicalize_spec_mode(request.speculative_type) or "auto" + backend_mode = llama_backend.requested_spec_mode or "auto" + if req_mode != backend_mode: return False + # spec_draft_n_max only matters when an MTP variant is engaged; None + # means "platform default" and matches whatever the backend chose. + if backend_mode in ("mtp", "mtp+ngram") and request.spec_draft_n_max is not None: + if int(request.spec_draft_n_max) != (llama_backend.spec_draft_n_max or 0): + return False if (request.chat_template_override or None) != ( llama_backend.chat_template_override or None ): @@ -583,8 +641,10 @@ async def load_model( reasoning_style = llama_backend.reasoning_style, reasoning_always_on = llama_backend.reasoning_always_on, supports_preserve_thinking = llama_backend.supports_preserve_thinking, + supports_tools = llama_backend.supports_tools, chat_template = llama_backend.chat_template, - speculative_type = llama_backend.speculative_type, + speculative_type = llama_backend.requested_spec_mode, + spec_draft_n_max = llama_backend.spec_draft_n_max, ) else: if ( @@ -604,21 +664,10 @@ async def load_model( logger.warning( f"Could not retrieve chat template for {backend.active_model_name}: {e}" ) - # Non-GGUF: only advertise reasoning for gpt-oss Harmony, - # which emits reasoning via channels at the tokenizer level. - # Template-level chat_template_kwargs (enable_thinking / - # preserve_thinking / tools) are not yet forwarded through - # the transformers generation path, so avoid advertising - # controls the server cannot honour outside GGUF. - _sf_supports_reasoning = False - _sf_reasoning_style = "enable_thinking" - if hasattr(backend, "_is_gpt_oss_model"): - try: - if backend._is_gpt_oss_model(): - _sf_supports_reasoning = True - _sf_reasoning_style = "reasoning_effort" - except Exception: - pass + # Classify via the same path as GGUF. + _sf_flags = _detect_safetensors_features(backend, _chat_template) + _sf_supports_reasoning = _sf_flags["supports_reasoning"] + _sf_reasoning_style = _sf_flags["reasoning_style"] return LoadResponse( status = "already_loaded", model = model_log_label @@ -639,9 +688,9 @@ async def load_model( ), supports_reasoning = _sf_supports_reasoning, reasoning_style = _sf_reasoning_style, - reasoning_always_on = False, - supports_preserve_thinking = False, - supports_tools = False, + reasoning_always_on = _sf_flags["reasoning_always_on"], + supports_preserve_thinking = _sf_flags["supports_preserve_thinking"], + supports_tools = _sf_flags["supports_tools"], chat_template = _chat_template, ) @@ -724,7 +773,10 @@ async def load_model( llama_backend.extra_args, strip_context = "max_seq_length" in fields_set, strip_cache = "cache_type_kv" in fields_set, - strip_spec = "speculative_type" in fields_set, + strip_spec = ( + "speculative_type" in fields_set + or "spec_draft_n_max" in fields_set + ), strip_template = "chat_template_override" in fields_set, ) try: @@ -765,6 +817,7 @@ async def load_model( chat_template_override = request.chat_template_override, cache_type_kv = request.cache_type_kv, speculative_type = request.speculative_type, + spec_draft_n_max = request.spec_draft_n_max, n_parallel = _n_parallel, extra_args = extra_llama_args, ) @@ -788,6 +841,7 @@ async def load_model( chat_template_override = request.chat_template_override, cache_type_kv = request.cache_type_kv, speculative_type = request.speculative_type, + spec_draft_n_max = request.spec_draft_n_max, n_parallel = _n_parallel, extra_args = extra_llama_args, ) @@ -846,7 +900,8 @@ async def load_model( supports_tools = llama_backend.supports_tools, cache_type_kv = llama_backend.cache_type_kv, chat_template = llama_backend.chat_template, - speculative_type = llama_backend.speculative_type, + speculative_type = llama_backend.requested_spec_mode, + spec_draft_n_max = llama_backend.spec_draft_n_max, ) # ── Standard path: load via Unsloth/transformers ────────── @@ -968,19 +1023,8 @@ async def load_model( except Exception: pass - # Non-GGUF: gpt-oss Harmony surfaces reasoning via tokenizer-level - # channels; other safetensors reasoning/tools/preserve-thinking - # knobs are not forwarded to tokenizer.apply_chat_template yet, so - # we only advertise support for the Harmony case here. - _sf_supports_reasoning = False - _sf_reasoning_style = "enable_thinking" - if hasattr(backend, "_is_gpt_oss_model"): - try: - if backend._is_gpt_oss_model(): - _sf_supports_reasoning = True - _sf_reasoning_style = "reasoning_effort" - except Exception: - pass + # Classify reasoning/tool flags via the GGUF sniffer. + _sf_flags = _detect_safetensors_features(backend, _chat_template) return LoadResponse( status = "loaded", @@ -998,11 +1042,11 @@ async def load_model( requires_trust_remote_code = bool( inference_config.get("trust_remote_code", False) ), - supports_reasoning = _sf_supports_reasoning, - reasoning_style = _sf_reasoning_style, - reasoning_always_on = False, - supports_preserve_thinking = False, - supports_tools = False, + supports_reasoning = _sf_flags["supports_reasoning"], + reasoning_style = _sf_flags["reasoning_style"], + reasoning_always_on = _sf_flags["reasoning_always_on"], + supports_preserve_thinking = _sf_flags["supports_preserve_thinking"], + supports_tools = _sf_flags["supports_tools"], chat_template = _chat_template, ) @@ -1345,7 +1389,8 @@ async def get_status( native_context_length = llama_backend.native_context_length, cache_type_kv = llama_backend.cache_type_kv, chat_template_override = llama_backend.chat_template_override, - speculative_type = llama_backend.speculative_type, + speculative_type = llama_backend.requested_spec_mode, + spec_draft_n_max = llama_backend.spec_draft_n_max, llama_cpp_supports_mtp = _supports_mtp, llama_cpp_prebuilt_stale = _stale, llama_cpp_installed_tag = _installed_tag, @@ -1373,18 +1418,8 @@ async def get_status( else None ) - # Non-GGUF: only gpt-oss Harmony is wired through the transformers - # generation path. Other template-level reasoning / tool kwargs - # are not yet forwarded, so we do not advertise them here. - supports_reasoning = False - reasoning_style = "enable_thinking" - if backend.active_model_name and hasattr(backend, "_is_gpt_oss_model"): - try: - if backend._is_gpt_oss_model(): - supports_reasoning = True - reasoning_style = "reasoning_effort" - except Exception: - pass + # Non-GGUF: classify from the loaded template. + _sf_flags = _detect_safetensors_features(backend, chat_template) inference_config = ( load_inference_config(backend.active_model_name) if backend.active_model_name @@ -1404,11 +1439,11 @@ async def get_status( requires_trust_remote_code = bool( (inference_config or {}).get("trust_remote_code", False) ), - supports_reasoning = supports_reasoning, - reasoning_style = reasoning_style, - reasoning_always_on = False, - supports_preserve_thinking = False, - supports_tools = False, + supports_reasoning = _sf_flags["supports_reasoning"], + reasoning_style = _sf_flags["reasoning_style"], + reasoning_always_on = _sf_flags["reasoning_always_on"], + supports_preserve_thinking = _sf_flags["supports_preserve_thinking"], + supports_tools = _sf_flags["supports_tools"], chat_template = chat_template, llama_cpp_supports_mtp = _supports_mtp, llama_cpp_prebuilt_stale = _stale, @@ -2254,46 +2289,11 @@ async def openai_chat_completions( detail = "Audio input is not supported for GGUF chat models yet.", ) - # Reject images if this GGUF model doesn't support vision - image_b64 = extracted_image_b64 or payload.image_base64 - if image_b64 and not llama_backend.is_vision: - raise HTTPException( - status_code = 400, - detail = "Image provided but current GGUF model does not support vision.", - ) - - # Convert image to PNG for llama-server (stb_image has limited format support) - if image_b64: - try: - import base64 as _b64 - from io import BytesIO as _BytesIO - from PIL import Image as _Image, UnidentifiedImageError as _UIE - - raw = _b64.b64decode(image_b64) - # Normalize to RGB so PNG encoding succeeds regardless of - # source mode (RGBA, P, L, CMYK, I, F, ...). Previously - # we only converted RGBA, which left CMYK/I/F to raise at - # img.save(PNG). - img = _Image.open(_BytesIO(raw)).convert("RGB") - buf = _BytesIO() - img.save(buf, format = "PNG") - image_b64 = _b64.b64encode(buf.getvalue()).decode("ascii") - except _UIE: - raise HTTPException( - status_code = 400, - detail = "Unsupported or corrupt image format.", - ) - except Exception: - raise HTTPException( - status_code = 400, - detail = "Failed to process image.", - ) - - # Build message list with system prompt prepended - gguf_messages = [] - if system_prompt: - gguf_messages.append({"role": "system", "content": system_prompt}) - gguf_messages.extend(chat_messages) + gguf_messages, has_gguf_image = _openai_messages_for_gguf_chat( + payload, + llama_backend.is_vision, + ) + image_b64 = None cancel_event = threading.Event() @@ -2307,7 +2307,7 @@ async def openai_chat_completions( use_tools = ( _effective_enable_tools(payload) and llama_backend.supports_tools - and not image_b64 + and not has_gguf_image ) if use_tools: @@ -2769,6 +2769,300 @@ async def openai_chat_completions( except Exception as e: raise HTTPException(status_code = 400, detail = f"Failed to decode image: {e}") + # Classify capability flags from the loaded template. + _sf_model_info = backend.models.get(backend.active_model_name, {}) + _sf_tpl = (_sf_model_info.get("chat_template_info") or {}).get("template") + _sf_features = _detect_safetensors_features(backend, _sf_tpl) + + cancel_event = threading.Event() + completion_id = f"chatcmpl-{uuid.uuid4().hex[:12]}" + created = int(time.time()) + + # ── Safetensors tool-calling path ───────────────────────── + # Mirrors the GGUF agentic loop's event shape. Disabled for + # vision turns (untested overlap with image render slot) and + # for gpt-oss (Harmony uses dedicated channels, not + # XML -- gpt-oss tools still work via the GGUF path). + _sf_is_gptoss = False + try: + _sf_is_gptoss = bool( + hasattr(backend, "_is_gpt_oss_model") and backend._is_gpt_oss_model() + ) + except Exception: + _sf_is_gptoss = False + + _sf_tool_budget = ( + payload.max_tool_calls_per_message + if payload.max_tool_calls_per_message is not None + else 25 + ) + + _sf_use_tools = ( + _effective_enable_tools(payload) + and _sf_features.get("supports_tools", False) + and image is None + and not _sf_is_gptoss + and _sf_tool_budget > 0 + ) + + if _sf_use_tools: + from core.inference.tools import ALL_TOOLS + + if payload.enabled_tools is not None: + _sf_tools_to_use = [ + t for t in ALL_TOOLS if t["function"]["name"] in payload.enabled_tools + ] + else: + _sf_tools_to_use = ALL_TOOLS + + _sf_tool_names = {t["function"]["name"] for t in _sf_tools_to_use} + _sf_has_web = "web_search" in _sf_tool_names + _sf_has_code = "python" in _sf_tool_names or "terminal" in _sf_tool_names + + _sf_date_line = f"The current date is {_date.today().isoformat()}." + _sf_model_size_b = _extract_model_size_b(model_name) + _sf_is_small_model = _sf_model_size_b is not None and _sf_model_size_b < 9 + + if _sf_is_small_model: + _sf_web_tips = "Do not repeat the same search query." + else: + _sf_web_tips = ( + "When you search and find a relevant URL in the results, " + "fetch its full content by calling web_search with the url parameter. " + "Do not repeat the same search query. If a search returns " + "no useful results, try rephrasing or fetching a result URL directly." + ) + _sf_code_tips = ( + "Use code execution for math, calculations, data processing, " + "or to parse and analyze information from tool results." + ) + + if _sf_has_web and _sf_has_code: + _sf_nudge = ( + _sf_date_line + " " + "You have access to tools. When appropriate, prefer using " + "tools rather than answering from memory. " + + _sf_web_tips + + " " + + _sf_code_tips + ) + elif _sf_has_code: + _sf_nudge = ( + _sf_date_line + " " + "You have access to tools. When appropriate, prefer using " + "code execution rather than answering from memory. " + _sf_code_tips + ) + elif _sf_has_web: + _sf_nudge = ( + _sf_date_line + " " + "You have access to tools. When appropriate, prefer using " + "web search for up-to-date or uncertain factual " + "information rather than answering from memory. " + _sf_web_tips + ) + else: + _sf_nudge = "" + + _sf_system_prompt = system_prompt + if _sf_nudge: + _sf_nudge += _TOOL_ACTION_NUDGE + if _sf_system_prompt: + _sf_system_prompt = _sf_system_prompt.rstrip() + "\n\n" + _sf_nudge + else: + _sf_system_prompt = _sf_nudge + + # Strip stale tool-call XML from prior assistant turns. + _sf_chat_messages = [] + for _msg in chat_messages: + if _msg.get("role") == "assistant" and isinstance(_msg.get("content"), str): + _sf_chat_messages.append( + { + **_msg, + "content": _TOOL_XML_RE.sub("", _msg["content"]).strip(), + } + ) + else: + _sf_chat_messages.append(_msg) + + def sf_generate_with_tools(): + return backend.generate_chat_completion_with_tools( + messages = _sf_chat_messages, + tools = _sf_tools_to_use, + system_prompt = _sf_system_prompt or "", + temperature = payload.temperature, + top_p = payload.top_p, + top_k = payload.top_k, + min_p = payload.min_p, + max_tokens = payload.max_tokens, + repetition_penalty = payload.repetition_penalty, + cancel_event = cancel_event, + enable_thinking = payload.enable_thinking, + reasoning_effort = payload.reasoning_effort, + preserve_thinking = payload.preserve_thinking, + auto_heal_tool_calls = payload.auto_heal_tool_calls + if payload.auto_heal_tool_calls is not None + else True, + max_tool_iterations = _sf_tool_budget, + tool_call_timeout = payload.tool_call_timeout + if payload.tool_call_timeout is not None + else 300, + session_id = payload.session_id, + use_adapter = payload.use_adapter, + ) + + _sf_tool_sentinel = object() + _sf_cancel_keys = (payload.cancel_id, payload.session_id, completion_id) + _sf_tracker = _TrackedCancel(cancel_event, *_sf_cancel_keys) + _sf_tracker.__enter__() + + async def sf_tool_stream(): + try: + first_chunk = ChatCompletionChunk( + id = completion_id, + created = created, + model = model_name, + choices = [ + ChunkChoice( + delta = ChoiceDelta(role = "assistant"), + finish_reason = None, + ) + ], + ) + yield f"data: {first_chunk.model_dump_json(exclude_none = True)}\n\n" + + gen = sf_generate_with_tools() + prev_text = "" + while True: + if cancel_event.is_set(): + backend.reset_generation_state() + break + if await request.is_disconnected(): + cancel_event.set() + backend.reset_generation_state() + return + + event = await asyncio.to_thread(next, gen, _sf_tool_sentinel) + if event is _sf_tool_sentinel: + break + + if event["type"] == "status": + if not event["text"]: + prev_text = "" + status_data = json.dumps( + { + "type": "tool_status", + "content": event["text"], + } + ) + yield f"data: {status_data}\n\n" + continue + + if event["type"] in ("tool_start", "tool_end"): + if event["type"] == "tool_start": + prev_text = "" + yield f"data: {json.dumps(event)}\n\n" + continue + + # Diff cumulative cleaned text against last snapshot. + raw_cumulative = event.get("text", "") + clean_cumulative = _TOOL_XML_RE.sub("", raw_cumulative) + new_text = clean_cumulative[len(prev_text) :] + prev_text = clean_cumulative + if not new_text: + continue + chunk = ChatCompletionChunk( + id = completion_id, + created = created, + model = model_name, + choices = [ + ChunkChoice( + delta = ChoiceDelta(content = new_text), + finish_reason = None, + ) + ], + ) + yield f"data: {chunk.model_dump_json(exclude_none = True)}\n\n" + + final_chunk = ChatCompletionChunk( + id = completion_id, + created = created, + model = model_name, + choices = [ + ChunkChoice( + delta = ChoiceDelta(), + finish_reason = "stop", + ) + ], + ) + yield f"data: {final_chunk.model_dump_json(exclude_none = True)}\n\n" + yield "data: [DONE]\n\n" + + except asyncio.CancelledError: + cancel_event.set() + backend.reset_generation_state() + raise + except Exception: + backend.reset_generation_state() + # Generic wire message; full trace stays in the log + # (CWE-209: transformers/torch errors may leak paths). + logger.exception("safetensors tool stream error") + error_chunk = { + "error": { + "message": "An internal error occurred.", + "type": "server_error", + }, + } + yield f"data: {json.dumps(error_chunk)}\n\n" + finally: + _sf_tracker.__exit__(None, None, None) + + if payload.stream: + return StreamingResponse( + sf_tool_stream(), + media_type = "text/event-stream", + headers = { + "Cache-Control": "no-cache", + "Connection": "keep-alive", + "X-Accel-Buffering": "no", + }, + ) + + # Non-streaming JSON: drain the loop, build one ChatCompletion. + try: + + def _drain_to_text(): + full_text = "" + gen = sf_generate_with_tools() + for event in gen: + if cancel_event.is_set(): + break + if event.get("type") == "content": + full_text = _TOOL_XML_RE.sub("", event.get("text", "")) + return full_text + + content_text = await asyncio.to_thread(_drain_to_text) + response = ChatCompletion( + id = completion_id, + created = created, + model = model_name, + choices = [ + CompletionChoice( + message = CompletionMessage(content = content_text), + finish_reason = "stop", + ) + ], + ) + return JSONResponse(content = response.model_dump()) + except Exception: + backend.reset_generation_state() + # CWE-209: generic detail; full trace in log. + logger.exception("safetensors tool completion error") + raise HTTPException( + status_code = 500, + detail = "An internal error occurred.", + ) + finally: + _sf_tracker.__exit__(None, None, None) + # Shared generation kwargs gen_kwargs = dict( messages = chat_messages, @@ -2781,9 +3075,14 @@ async def openai_chat_completions( max_new_tokens = payload.max_tokens or 2048, repetition_penalty = payload.repetition_penalty, ) - - # Choose generation path (adapter-controlled or standard) - cancel_event = threading.Event() + # Forward reasoning kwargs; the worker/template wrapper peels off + # any the template doesn't accept. + if payload.enable_thinking is not None: + gen_kwargs["enable_thinking"] = payload.enable_thinking + if payload.reasoning_effort is not None: + gen_kwargs["reasoning_effort"] = payload.reasoning_effort + if payload.preserve_thinking is not None: + gen_kwargs["preserve_thinking"] = payload.preserve_thinking if payload.use_adapter is not None: @@ -2800,9 +3099,6 @@ async def openai_chat_completions( cancel_event = cancel_event, **gen_kwargs ) - completion_id = f"chatcmpl-{uuid.uuid4().hex[:12]}" - created = int(time.time()) - # ── Streaming response ──────────────────────────────────────── if payload.stream: _cancel_keys = (payload.cancel_id, payload.session_id, completion_id) @@ -4804,6 +5100,47 @@ def _openai_messages_for_passthrough(payload) -> list[dict]: return messages +def _openai_messages_for_gguf_chat(payload, is_vision: bool) -> tuple[list[dict], bool]: + """Build llama-server messages for the standard GGUF chat path. + + llama-server accepts OpenAI multimodal content parts directly. Preserve + all per-turn ``image_url`` parts so multi-image chat history keeps each + image attached to its original turn. + """ + messages = _drop_empty_assistant_sentinels( + [m.model_dump(exclude_none = True) for m in payload.messages] + ) + has_message_image = any( + isinstance(msg.get("content"), list) + and any(part.get("type") == "image_url" for part in msg["content"]) + for msg in messages + ) + if payload.image_base64 and not has_message_image: + # Legacy bytes can be any format; the normalizer below sniffs and + # re-encodes to PNG, so the declared mime is rewritten anyway. + image_part = { + "type": "image_url", + "image_url": { + "url": f"data:image/png;base64,{payload.image_base64}", + }, + } + for msg in reversed(messages): + if msg.get("role") != "user": + continue + existing = msg.get("content") + if isinstance(existing, str): + msg["content"] = [{"type": "text", "text": existing}, image_part] + elif isinstance(existing, list): + existing.append(image_part) + else: + msg["content"] = [image_part] + break + else: + messages.append({"role": "user", "content": [image_part]}) + has_image = _normalize_anthropic_openai_images(messages, is_vision) + return messages, has_image + + def _extract_response_format(payload): """Return the ``response_format`` field on an incoming ChatCompletionRequest (or None). The model is declared with ``extra="allow"`` so pydantic stashes diff --git a/studio/backend/tests/test_gguf_reload_inheritance.py b/studio/backend/tests/test_gguf_reload_inheritance.py index 4b0b450cb0..1a663725d9 100644 --- a/studio/backend/tests/test_gguf_reload_inheritance.py +++ b/studio/backend/tests/test_gguf_reload_inheritance.py @@ -78,6 +78,7 @@ def _loaded_backend(**overrides): backend._requested_n_ctx = 8192 backend._cache_type_kv = None backend._speculative_type = None + backend._requested_spec_mode = "auto" backend._chat_template_override = None backend._is_vision = False backend._extra_args = None diff --git a/studio/backend/tests/test_kv_cache_estimation.py b/studio/backend/tests/test_kv_cache_estimation.py index 29d87804ff..d52a58a25c 100644 --- a/studio/backend/tests/test_kv_cache_estimation.py +++ b/studio/backend/tests/test_kv_cache_estimation.py @@ -1558,6 +1558,33 @@ class TestServerFlags: ) assert fitted < 32_768 + def test_fit_mtp_engaged_returns_smaller_or_equal_context(self): + # MTP-engaged budget is 0.85 of available; non-MTP is 0.90. + # On a tight budget the MTP path must yield <= the non-MTP path. + b = self._gqa_backend() + common = dict( + requested_ctx = 32_768, + available_mib = 128, + model_size_bytes = 8 * 1024 * 1024, + cache_type_kv = "f16", + ) + baseline = b._fit_context_to_vram(**common) + mtp = b._fit_context_to_vram(**common, mtp_engaged = True) + assert mtp <= baseline + + def test_fit_mtp_engaged_unchanged_when_kv_off_gpu(self): + # kv_on_gpu=False short-circuits the fit; mtp_engaged is irrelevant. + b = self._gqa_backend() + fitted = b._fit_context_to_vram( + requested_ctx = 32_768, + available_mib = 1, + model_size_bytes = 100, + cache_type_kv = "f16", + kv_on_gpu = False, + mtp_engaged = True, + ) + assert fitted == 32_768 + def test_fit_threads_swa_full_through_estimator(self): # SWA model, generous budget; both should fit but cache size differs. b = self._swa_backend() diff --git a/studio/backend/tests/test_llama_cpp_mtp_detection.py b/studio/backend/tests/test_llama_cpp_mtp_detection.py index 7da633201f..4a8276adc0 100644 --- a/studio/backend/tests/test_llama_cpp_mtp_detection.py +++ b/studio/backend/tests/test_llama_cpp_mtp_detection.py @@ -52,6 +52,9 @@ import pytest from core.inference.llama_cpp import ( LlamaCppBackend, + _backfill_usage_from_timings, + _build_ngram_mod_flags, + _canonicalize_spec_mode, _extra_args_set_spec_type, _is_mtp_model_name, ) @@ -186,6 +189,10 @@ def _mtp_backend(**overrides): backend._requested_n_ctx = 8192 backend._cache_type_kv = None backend._speculative_type = "draft-mtp" + # Default fixture simulates Auto having auto-promoted to draft-mtp. + # Individual tests override _requested_spec_mode when they want a + # forced mode or the user---spec-type-extra-args path. + backend._requested_spec_mode = "auto" backend._chat_template_override = None backend._is_vision = False backend._extra_args = None @@ -233,9 +240,16 @@ def test_already_in_target_state_matches_when_request_uses_default_for_mtp_model ) -def test_already_in_target_state_non_mtp_model_unaffected(): - # Promotion is gated on the name; non-MTP must still mismatch req=None. - backend = _mtp_backend(_model_identifier = "unsloth/Qwen3.6-27B-GGUF") +def test_already_in_target_state_auto_request_matches_auto_backend_for_non_mtp_model(): + # Under the requested-mode round-trip model, Auto requested against an + # Auto-recorded backend matches regardless of model name. The underlying + # resolved emission (--spec-default vs draft-mtp) is handled by the + # backend's own load path and reflected in _speculative_type; the + # short-circuit comparison only cares whether the *intent* changed. + backend = _mtp_backend( + _model_identifier = "unsloth/Qwen3.6-27B-GGUF", + _speculative_type = "default", + ) assert ( backend._already_in_target_state( gguf_path = None, @@ -248,7 +262,7 @@ def test_already_in_target_state_non_mtp_model_unaffected(): extra_args = None, is_vision = False, ) - is False + is True ) @@ -308,6 +322,7 @@ def test_already_in_target_state_user_spec_type_override_matches_clean_backend() # User --spec-type none suppressed auto-MTP; repeat /load must not re-promote. backend = _mtp_backend( _speculative_type = None, + _requested_spec_mode = None, _extra_args = ["--spec-type", "none"], ) assert ( @@ -389,12 +404,17 @@ def test_already_in_target_state_vision_mtp_default_matches(): ) -def test_already_in_target_state_vision_non_mtp_unaffected(): - # Vision non-MTP repo (no -MTP marker) must still mismatch req=None - # against a backend running draft-mtp. +def test_already_in_target_state_vision_off_matches_vision_backend(): + # Vision loads silently drop speculative decoding at the route level + # (_request_matches_loaded_settings overrides req to "off"). At the + # llama_cpp.py level, _already_in_target_state compares canonical + # requested modes; a vision backend recorded with _requested_spec_mode + # = "off" matches a req of "off" or None+vision. backend = _mtp_backend( _model_identifier = "unsloth/Qwen3-VL-4B-Instruct-GGUF", _is_vision = True, + _speculative_type = None, + _requested_spec_mode = "off", ) assert ( backend._already_in_target_state( @@ -403,12 +423,12 @@ def test_already_in_target_state_vision_non_mtp_unaffected(): hf_variant = "Q4_K_M", n_ctx = 8192, cache_type_kv = None, - speculative_type = None, + speculative_type = "off", chat_template_override = None, extra_args = None, is_vision = True, ) - is False + is True ) @@ -482,10 +502,17 @@ def _make_fake_llama_server(path: Path, help_text: str) -> Path: return path +_NEEDS_BASH = pytest.mark.skipif( + sys.platform == "win32", + reason = "fake llama-server is a bash stub; Windows has no direct executor", +) + + def _clear_caps_cache(): LlamaCppBackend._capability_cache.clear() +@_NEEDS_BASH def test_probe_server_capabilities_detects_draft_mtp(tmp_path): # Original naming from llama.cpp #22673. fake = _make_fake_llama_server( @@ -500,6 +527,7 @@ def test_probe_server_capabilities_detects_draft_mtp(tmp_path): assert caps["supports_mtp"] is True +@_NEEDS_BASH def test_probe_server_capabilities_detects_renamed_mtp(tmp_path): # Renamed upstream: draft-mtp -> mtp. fake = _make_fake_llama_server( @@ -513,6 +541,7 @@ def test_probe_server_capabilities_detects_renamed_mtp(tmp_path): assert caps["supports_mtp"] is True +@_NEEDS_BASH def test_probe_server_capabilities_reports_outdated_binary(tmp_path): # Pre-MTP llama.cpp: only ngram variants. fake = _make_fake_llama_server( @@ -533,6 +562,130 @@ def test_probe_server_capabilities_handles_missing_binary(): assert caps["supports_mtp"] is False +# ngram-mod flag flavor detection (new vs legacy llama-server). + +# Help-text fixtures mirror the actual `llama-server --help` block +# layout (flag on its own line; description indented underneath). +_POST_RENAME_HELP = """\ +--spec-draft-n-max N number of tokens to draft for speculative decoding (default: 16) + (env: LLAMA_ARG_SPEC_DRAFT_N_MAX) +--spec-draft-n-min N minimum number of draft tokens to use for speculative decoding (default: 0) + (env: LLAMA_ARG_SPEC_DRAFT_N_MIN) +--spec-draft-p-min, --draft-p-min P minimum speculative decoding probability (greedy) (default: 0.75) + (env: LLAMA_ARG_SPEC_DRAFT_P_MIN) +--spec-ngram-mod-n-min N minimum number of ngram tokens (default: 48) +--spec-ngram-mod-n-max N maximum number of ngram tokens (default: 64) +--spec-ngram-mod-n-match N ngram-mod lookup length (default: 24) +--spec-type none,draft-simple,draft-mtp,ngram-mod comma-separated list of types of speculative decoding to use + (env: LLAMA_ARG_SPEC_TYPE) +--draft, --draft-n, --draft-max N the argument has been removed. use --spec-draft-n-max or --spec-ngram-mod-n-max + (env: LLAMA_ARG_DRAFT_MAX) +--draft-min, --draft-n-min N the argument has been removed. use --spec-draft-n-min or --spec-ngram-mod-n-min + (env: LLAMA_ARG_DRAFT_MIN) +--spec-ngram-size-n N the argument has been removed. use the respective --spec-ngram-*-size-n or --spec-ngram-mod-n-match +""" + +_LEGACY_HELP = """\ +--draft, --draft-n, --draft-max N number of tokens to draft for speculative decoding (default: 8) + (env: LLAMA_ARG_DRAFT_MAX) +--draft-min, --draft-n-min N minimum number of draft tokens to use for speculative decoding (default: 0) + (env: LLAMA_ARG_DRAFT_MIN) +--spec-ngram-size-n N ngram lookup length (default: 24) +--spec-type none,ngram-mod,ngram-simple comma-separated list of types of speculative decoding to use +""" + + +@_NEEDS_BASH +def test_probe_detects_post_rename_ngram_mod_flavor(tmp_path): + fake = _make_fake_llama_server(tmp_path / "llama-server", _POST_RENAME_HELP) + _clear_caps_cache() + caps = LlamaCppBackend.probe_server_capabilities(str(fake)) + assert caps["found"] is True + assert caps["ngram_mod_flavor"] == "new" + assert caps["supports_ngram_mod"] is True + assert caps["spec_draft_n_max_flag"] == "--spec-draft-n-max" + + +@_NEEDS_BASH +def test_probe_detects_legacy_ngram_mod_flavor(tmp_path): + fake = _make_fake_llama_server(tmp_path / "llama-server", _LEGACY_HELP) + _clear_caps_cache() + caps = LlamaCppBackend.probe_server_capabilities(str(fake)) + assert caps["found"] is True + assert caps["ngram_mod_flavor"] == "legacy" + assert caps["supports_ngram_mod"] is True + assert caps["spec_draft_n_max_flag"] == "--draft-max" + + +@_NEEDS_BASH +def test_probe_ignores_removal_stub_descriptions(tmp_path): + # Post-rename binary: legacy flags are present but with + # "argument has been removed" descriptions; must not be detected + # as legacy. + fake = _make_fake_llama_server(tmp_path / "llama-server", _POST_RENAME_HELP) + _clear_caps_cache() + caps = LlamaCppBackend.probe_server_capabilities(str(fake)) + assert caps["ngram_mod_flavor"] == "new" + + +@_NEEDS_BASH +def test_probe_no_ngram_mod_on_minimal_binary(tmp_path): + # Pre-anything: neither set present. + fake = _make_fake_llama_server( + tmp_path / "llama-server", + "--spec-type none\n--threads N\n", + ) + _clear_caps_cache() + caps = LlamaCppBackend.probe_server_capabilities(str(fake)) + assert caps["ngram_mod_flavor"] is None + assert caps["supports_ngram_mod"] is False + + +def test_build_ngram_mod_flags_new(): + flags = _build_ngram_mod_flags({"ngram_mod_flavor": "new"}) + assert flags == [ + "--spec-ngram-mod-n-match", + "24", + "--spec-ngram-mod-n-min", + "48", + "--spec-ngram-mod-n-max", + "64", + ] + + +def test_build_ngram_mod_flags_legacy(): + flags = _build_ngram_mod_flags({"ngram_mod_flavor": "legacy"}) + assert flags == [ + "--spec-ngram-size-n", + "24", + "--draft-min", + "48", + "--draft-max", + "64", + ] + + +def test_build_ngram_mod_flags_empty_when_unsupported(): + assert _build_ngram_mod_flags({"ngram_mod_flavor": None}) == [] + assert _build_ngram_mod_flags(None) == [] + assert _build_ngram_mod_flags({}) == [] + + +def test_build_ngram_mod_flags_respects_custom_values(): + flags = _build_ngram_mod_flags( + {"ngram_mod_flavor": "new"}, n_match = 16, n_min = 24, n_max = 32 + ) + assert flags == [ + "--spec-ngram-mod-n-match", + "16", + "--spec-ngram-mod-n-min", + "24", + "--spec-ngram-mod-n-max", + "32", + ] + + +@_NEEDS_BASH def test_probe_server_capabilities_caches_by_mtime(tmp_path): # Same (path, mtime) -> cache hit. Bumped mtime -> re-probe. fake = _make_fake_llama_server( @@ -555,3 +708,493 @@ def test_probe_server_capabilities_caches_by_mtime(tmp_path): caps2 = LlamaCppBackend.probe_server_capabilities(str(fake)) assert caps2["mtp_token"] == "draft-mtp" assert caps2["supports_mtp"] is True + + +# spec_draft_n_max plumbing (first-class --spec-draft-n-max override). + + +def test_already_in_target_state_matches_when_draft_n_max_unset(): + # None on the request means "platform default"; matches any backend. + backend = _mtp_backend(_spec_draft_n_max = None) + assert ( + backend._already_in_target_state( + gguf_path = None, + model_identifier = "unsloth/Qwen3.6-27B-MTP-GGUF", + hf_variant = "Q4_K_M", + n_ctx = 8192, + cache_type_kv = None, + speculative_type = None, + spec_draft_n_max = None, + chat_template_override = None, + extra_args = None, + is_vision = False, + ) + is True + ) + + +def test_already_in_target_state_matches_when_draft_n_max_equals_backend(): + backend = _mtp_backend(_spec_draft_n_max = 4) + assert ( + backend._already_in_target_state( + gguf_path = None, + model_identifier = "unsloth/Qwen3.6-27B-MTP-GGUF", + hf_variant = "Q4_K_M", + n_ctx = 8192, + cache_type_kv = None, + speculative_type = None, + spec_draft_n_max = 4, + chat_template_override = None, + extra_args = None, + is_vision = False, + ) + is True + ) + + +def test_already_in_target_state_mismatches_when_draft_n_max_differs(): + backend = _mtp_backend(_spec_draft_n_max = 4) + assert ( + backend._already_in_target_state( + gguf_path = None, + model_identifier = "unsloth/Qwen3.6-27B-MTP-GGUF", + hf_variant = "Q4_K_M", + n_ctx = 8192, + cache_type_kv = None, + speculative_type = None, + spec_draft_n_max = 8, + chat_template_override = None, + extra_args = None, + is_vision = False, + ) + is False + ) + + +def test_already_in_target_state_draft_n_max_ignored_when_not_mtp(): + # ngram-mod backend; spec_draft_n_max is MTP-only and must not force + # a reload against a non-MTP active spec. + backend = _mtp_backend( + _speculative_type = "ngram-mod", + _requested_spec_mode = "ngram", + _spec_draft_n_max = None, + ) + assert ( + backend._already_in_target_state( + gguf_path = None, + model_identifier = "unsloth/Qwen3.6-27B-MTP-GGUF", + hf_variant = "Q4_K_M", + n_ctx = 8192, + cache_type_kv = None, + speculative_type = "ngram-mod", + spec_draft_n_max = 8, + chat_template_override = None, + extra_args = None, + is_vision = False, + ) + is True + ) + + +# Sub-3B MTP gate -- tiny dense models regress with the MTP draft +# head, so load_model falls back to ngram-mod (when the binary supports +# it) instead of draft-mtp. The reload-skip mirror must follow the +# same fallback so a sub-3B reload-with-default does not bounce a +# correctly-configured ngram-mod / off backend. + + +def _patch_probe(monkeypatch, ngram_supported): + """Force probe_server_capabilities to a deterministic result so + tests don't depend on whatever llama-server happens to be on PATH.""" + fake = { + "found": True, + "mtp_token": "draft-mtp", + "supports_mtp": True, + "ngram_mod_flavor": "new" if ngram_supported else None, + "supports_ngram_mod": bool(ngram_supported), + "spec_draft_n_max_flag": "--spec-draft-n-max", + } + monkeypatch.setattr( + LlamaCppBackend, + "probe_server_capabilities", + classmethod(lambda cls, binary = None: fake), + ) + monkeypatch.setattr( + LlamaCppBackend, + "_find_llama_server_binary", + classmethod(lambda cls: "/fake/llama-server"), + ) + + +def test_already_in_target_state_sub_3b_falls_back_to_ngram_mod_when_supported( + monkeypatch, +): + # 0.8B MTP request -- load_model would have promoted to ngram-mod + # (no MTP head); reload check must match a ngram-mod backend. + _patch_probe(monkeypatch, ngram_supported = True) + backend = _mtp_backend( + _model_identifier = "unsloth/Qwen3.5-0.8B-MTP-GGUF", + _speculative_type = "ngram-mod", + _spec_draft_n_max = None, + ) + assert ( + backend._already_in_target_state( + gguf_path = None, + model_identifier = "unsloth/Qwen3.5-0.8B-MTP-GGUF", + hf_variant = "Q4_K_M", + n_ctx = 8192, + cache_type_kv = None, + speculative_type = None, + chat_template_override = None, + extra_args = None, + is_vision = False, + ) + is True + ) + + +def test_already_in_target_state_sub_3b_falls_back_to_off_when_no_ngram(monkeypatch): + # 0.8B + binary lacks ngram-mod -> fall back to off. + _patch_probe(monkeypatch, ngram_supported = False) + backend = _mtp_backend( + _model_identifier = "unsloth/Qwen3.5-0.8B-MTP-GGUF", + _speculative_type = None, + _spec_draft_n_max = None, + ) + assert ( + backend._already_in_target_state( + gguf_path = None, + model_identifier = "unsloth/Qwen3.5-0.8B-MTP-GGUF", + hf_variant = "Q4_K_M", + n_ctx = 8192, + cache_type_kv = None, + speculative_type = None, + chat_template_override = None, + extra_args = None, + is_vision = False, + ) + is True + ) + + +def test_already_in_target_state_4b_mtp_request_promotes_as_before(monkeypatch): + # 4B is above the 3B threshold -> auto-promote still applies. + _patch_probe(monkeypatch, ngram_supported = True) + backend = _mtp_backend( + _model_identifier = "unsloth/Qwen3.5-4B-MTP-GGUF", + _speculative_type = "draft-mtp", + _spec_draft_n_max = None, + ) + assert ( + backend._already_in_target_state( + gguf_path = None, + model_identifier = "unsloth/Qwen3.5-4B-MTP-GGUF", + hf_variant = "Q4_K_M", + n_ctx = 8192, + cache_type_kv = None, + speculative_type = None, + chat_template_override = None, + extra_args = None, + is_vision = False, + ) + is True + ) + + +def test_already_in_target_state_2b_falls_back_to_ngram_below_threshold(monkeypatch): + # 2.0B is below the 3B threshold -> ngram-mod fallback, not + # draft-mtp. Clean-bench shows 2B regresses with draft-mtp. + _patch_probe(monkeypatch, ngram_supported = True) + backend = _mtp_backend( + _model_identifier = "unsloth/Qwen3.5-2B-MTP-GGUF", + _speculative_type = "ngram-mod", + _spec_draft_n_max = None, + ) + assert ( + backend._already_in_target_state( + gguf_path = None, + model_identifier = "unsloth/Qwen3.5-2B-MTP-GGUF", + hf_variant = "Q4_K_M", + n_ctx = 8192, + cache_type_kv = None, + speculative_type = None, + chat_template_override = None, + extra_args = None, + is_vision = False, + ) + is True + ) + + +# usage backfill from timings (Studio UI t/s widget fix). + + +def test_backfill_usage_from_timings_fills_when_completion_tokens_zero(): + out = _backfill_usage_from_timings( + {"prompt_tokens": 0, "completion_tokens": 0, "total_tokens": 0}, + {"prompt_n": 42, "predicted_n": 128, "predicted_per_second": 100.0}, + ) + assert out["completion_tokens"] == 128 + assert out["prompt_tokens"] == 42 + assert out["total_tokens"] == 170 + + +def test_backfill_usage_from_timings_fills_when_usage_missing(): + out = _backfill_usage_from_timings( + None, + {"prompt_n": 42, "predicted_n": 128, "predicted_per_second": 100.0}, + ) + assert out["completion_tokens"] == 128 + assert out["prompt_tokens"] == 42 + assert out["total_tokens"] == 170 + + +def test_backfill_usage_from_timings_preserves_real_usage(): + # Non-zero completion_tokens means llama-server reported correctly; + # do not overwrite. + real = {"prompt_tokens": 50, "completion_tokens": 200, "total_tokens": 250} + out = _backfill_usage_from_timings(real, {"predicted_n": 999, "prompt_n": 999}) + assert out is real + assert out["completion_tokens"] == 200 + + +def test_backfill_usage_from_timings_passthrough_when_timings_empty(): + assert _backfill_usage_from_timings(None, None) is None + assert _backfill_usage_from_timings(None, {}) is None + usage = {"completion_tokens": 0} + # No timings.predicted_n -> nothing to fill, return as-is. + assert _backfill_usage_from_timings(usage, {"prompt_ms": 5.0}) is usage + + +# ── _canonicalize_spec_mode (pure) ───────────────────────────────── + + +@pytest.mark.parametrize( + "value, expected", + [ + # New canonical values pass through unchanged. + ("auto", "auto"), + ("mtp", "mtp"), + ("ngram", "ngram"), + ("mtp+ngram", "mtp+ngram"), + ("off", "off"), + ("ngram-simple", "ngram-simple"), + # Legacy wire values map onto the new vocabulary. + ("default", "auto"), + ("draft-mtp", "mtp"), + ("ngram-mod", "ngram"), + # Comma-chained legacy values (e.g. from persisted state) collapse + # to the right canonical mode. + ("ngram-mod,draft-mtp", "mtp+ngram"), + ("draft-mtp,ngram-mod", "mtp+ngram"), + ("draft-mtp,mtp", "mtp"), + ("ngram-mod,ngram", "ngram"), + # Case and whitespace are ignored. + (" AUTO ", "auto"), + ("MTP", "mtp"), + ("MTP+Ngram", "mtp+ngram"), + # None / empty / whitespace pass through as None. + (None, None), + ("", None), + (" ", None), + # Non-string inputs collapse to None. + (42, None), + (True, None), + # Unknown strings fall back to "auto" (safe default). + ("bogus", "auto"), + ], +) +def test_canonicalize_spec_mode(value, expected): + assert _canonicalize_spec_mode(value) == expected + + +# ── _build_speculative_flags resolver matrix ────────────────────── + + +def _resolver_backend(monkeypatch, *, ngram_supported = True, mtp_token = "draft-mtp"): + """Backend with a deterministic probe so the resolver is hermetic.""" + fake = { + "found": True, + "mtp_token": mtp_token, + "supports_mtp": bool(mtp_token), + "ngram_mod_flavor": "new" if ngram_supported else None, + "supports_ngram_mod": bool(ngram_supported), + "spec_draft_n_max_flag": "--spec-draft-n-max", + } + monkeypatch.setattr( + LlamaCppBackend, + "probe_server_capabilities", + classmethod(lambda cls, binary = None: fake), + ) + backend = LlamaCppBackend() + backend._nextn_predict_layers = None + return backend + + +def _flags_dict(flags): + """Parse the spec-flag list into a small {flag: value} dict; collapses + repeated flags by keeping the last (only --spec-type can repeat and + never does in our resolver).""" + out = {} + i = 0 + while i < len(flags): + token = flags[i] + if i + 1 < len(flags) and not flags[i + 1].startswith("--"): + out[token] = flags[i + 1] + i += 2 + else: + out[token] = True + i += 1 + return out + + +_MTP_MODEL = "unsloth/Qwen3.6-27B-MTP-GGUF" +_NON_MTP_MODEL = "unsloth/Qwen3-7B-Instruct-GGUF" +_SUB_3B_MTP_MODEL = "unsloth/Qwen3.5-0.8B-MTP-GGUF" + + +@pytest.mark.parametrize( + "requested, gpus, model, expect_spec_type, expect_n_max, expect_ngram_knobs", + [ + # ── auto + MTP model + 3B+: GPU = mtp only, CPU = chain ── + ("auto", True, _MTP_MODEL, "draft-mtp", "2", False), + ("auto", False, _MTP_MODEL, "ngram-mod,draft-mtp", "3", True), + # ── auto + non-MTP: emit --spec-default ── + ("auto", True, _NON_MTP_MODEL, None, None, False), + ("auto", False, _NON_MTP_MODEL, None, None, False), + # ── auto + sub-3B MTP: fallback to ngram-mod ── + ("auto", True, _SUB_3B_MTP_MODEL, "ngram-mod", None, True), + ("auto", False, _SUB_3B_MTP_MODEL, "ngram-mod", None, True), + # ── mtp forced: MTP-only on BOTH platforms ── + ("mtp", True, _MTP_MODEL, "draft-mtp", "2", False), + ("mtp", False, _MTP_MODEL, "draft-mtp", "3", False), + # ── mtp forced on sub-3B: engage anyway ── + ("mtp", True, _SUB_3B_MTP_MODEL, "draft-mtp", "2", False), + # ── mtp forced on non-MTP: engage anyway ── + ("mtp", True, _NON_MTP_MODEL, "draft-mtp", "2", False), + # ── ngram forced: ngram-mod alone on BOTH platforms ── + ("ngram", True, _MTP_MODEL, "ngram-mod", None, True), + ("ngram", False, _MTP_MODEL, "ngram-mod", None, True), + ("ngram", True, _NON_MTP_MODEL, "ngram-mod", None, True), + # ── mtp+ngram forced: chain on BOTH platforms ── + ("mtp+ngram", True, _MTP_MODEL, "ngram-mod,draft-mtp", "2", True), + ("mtp+ngram", False, _MTP_MODEL, "ngram-mod,draft-mtp", "3", True), + ("mtp+ngram", True, _SUB_3B_MTP_MODEL, "ngram-mod,draft-mtp", "2", True), + # ── off: nothing emitted ── + ("off", True, _MTP_MODEL, None, None, False), + ("off", False, _MTP_MODEL, None, None, False), + # ── legacy values round-trip to the canonical emission ── + ("default", True, _MTP_MODEL, "draft-mtp", "2", False), + ("draft-mtp", True, _MTP_MODEL, "draft-mtp", "2", False), + ("ngram-mod", True, _MTP_MODEL, "ngram-mod", None, True), + ("ngram-mod,draft-mtp", False, _MTP_MODEL, "ngram-mod,draft-mtp", "3", True), + # ── ngram-simple: pass through ── + ("ngram-simple", True, _MTP_MODEL, "ngram-simple", None, False), + ], +) +def test_build_speculative_flags_matrix( + monkeypatch, + requested, + gpus, + model, + expect_spec_type, + expect_n_max, + expect_ngram_knobs, +): + backend = _resolver_backend(monkeypatch) + flags = backend._build_speculative_flags( + speculative_type = requested, + spec_draft_n_max = None, + extra_args = None, + model_identifier = model, + model_path = None, + gpus = gpus, + binary = "/fake/llama-server", + ) + parsed = _flags_dict(flags) + if expect_spec_type is None: + assert "--spec-type" not in parsed + else: + assert parsed.get("--spec-type") == expect_spec_type + if expect_n_max is None: + assert "--spec-draft-n-max" not in parsed + else: + assert parsed.get("--spec-draft-n-max") == expect_n_max + if expect_ngram_knobs: + assert "--spec-ngram-mod-n-match" in parsed + assert "--spec-ngram-mod-n-min" in parsed + assert "--spec-ngram-mod-n-max" in parsed + else: + assert "--spec-ngram-mod-n-match" not in parsed + + +def test_build_speculative_flags_user_extra_args_owns_spec_type(monkeypatch): + # User --spec-type in extra_args bypasses the dropdown entirely. + backend = _resolver_backend(monkeypatch) + flags = backend._build_speculative_flags( + speculative_type = "mtp", # would normally force MTP + spec_draft_n_max = None, + extra_args = ["--spec-type", "ngram-mod"], + model_identifier = _MTP_MODEL, + model_path = None, + gpus = True, + binary = "/fake/llama-server", + ) + # No flags emitted by the resolver -- the user's extra_args carries + # the --spec-type, and the resolver records requested_spec_mode = None. + assert flags == [] + assert backend.requested_spec_mode is None + assert backend.speculative_type is None + + +@pytest.mark.parametrize("mode", ["auto", "mtp", "ngram", "mtp+ngram", "off"]) +def test_build_speculative_flags_round_trips_requested_mode(monkeypatch, mode): + # The status round-trip is the contract that lets the UI dropdown + # restore its picked value after reload / refresh. + backend = _resolver_backend(monkeypatch) + backend._build_speculative_flags( + speculative_type = mode, + spec_draft_n_max = None, + extra_args = None, + model_identifier = _MTP_MODEL, + model_path = None, + gpus = True, + binary = "/fake/llama-server", + ) + assert backend.requested_spec_mode == mode + + +def test_build_speculative_flags_user_draft_n_max_override(monkeypatch): + backend = _resolver_backend(monkeypatch) + flags = backend._build_speculative_flags( + speculative_type = "mtp", + spec_draft_n_max = 5, + extra_args = None, + model_identifier = _MTP_MODEL, + model_path = None, + gpus = True, + binary = "/fake/llama-server", + ) + parsed = _flags_dict(flags) + assert parsed.get("--spec-draft-n-max") == "5" + assert backend.spec_draft_n_max == 5 + + +def test_build_speculative_flags_mtp_token_missing_logs_and_skips(monkeypatch): + # Outdated llama-server with no MTP support: forced MTP must degrade + # to spec-off (warned) rather than emit a bad --spec-type. + backend = _resolver_backend(monkeypatch, mtp_token = None) + flags = backend._build_speculative_flags( + speculative_type = "mtp", + spec_draft_n_max = None, + extra_args = None, + model_identifier = _MTP_MODEL, + model_path = None, + gpus = True, + binary = "/fake/llama-server", + ) + assert "--spec-type" not in flags + # _speculative_type stays None (resolved emission was none), but + # _requested_spec_mode still reflects the user's choice. + assert backend.requested_spec_mode == "mtp" + assert backend.speculative_type is None diff --git a/studio/backend/tests/test_llama_server_args.py b/studio/backend/tests/test_llama_server_args.py index f4dabfcf08..68a1c870fb 100644 --- a/studio/backend/tests/test_llama_server_args.py +++ b/studio/backend/tests/test_llama_server_args.py @@ -47,17 +47,15 @@ from core.inference.llama_server_args import ( ["--spec-type", "draft-mtp", "--spec-draft-n-max", "6"], [ "--spec-type", - "draft-mtp", + "ngram-mod,draft-mtp", "--spec-draft-n-max", "3", - "--spec-type", - "ngram-mod", "--spec-ngram-mod-n-match", "24", "--spec-ngram-mod-n-min", "48", "--spec-ngram-mod-n-max", - "6", + "64", ], # Reasoning controls ["--reasoning-format", "deepseek"], diff --git a/studio/backend/tests/test_mlx_inference_backend.py b/studio/backend/tests/test_mlx_inference_backend.py index ce447bdd1f..16cca7dd40 100644 --- a/studio/backend/tests/test_mlx_inference_backend.py +++ b/studio/backend/tests/test_mlx_inference_backend.py @@ -158,3 +158,97 @@ def test_mlx_inference_vlm_lora_uses_unsloth_loader_without_native_adapter_rewri assert backend._is_vlm is True assert isinstance(backend._processor, _DummyProcessor) assert isinstance(backend._tokenizer, _DummyTokenizer) + + +# Regression: MLXInferenceBackend.generate_chat_response must accept the +# four template kwargs (tools / enable_thinking / reasoning_effort / +# preserve_thinking) so the route layer can forward what the user +# toggled in the UI. The previous signature raised +# "got an unexpected keyword argument 'tools'" on Mac. + + +def test_mlx_generate_chat_response_accepts_template_kwargs(): + import inspect + from core.inference.mlx_inference import MLXInferenceBackend + + sig = inspect.signature(MLXInferenceBackend.generate_chat_response) + params = sig.parameters + for name in ("tools", "enable_thinking", "reasoning_effort", "preserve_thinking"): + assert name in params, ( + f"MLX.generate_chat_response is missing the {name!r} kwarg; " + "the route layer forwards this and a missing kwarg raises " + "TypeError on Mac" + ) + assert ( + params[name].default is None + ), f"{name!r} must default to None so existing callers stay valid" + + +def test_mlx_generate_text_forwards_kwargs_into_template_helper(monkeypatch): + """The Mac text path must route through apply_chat_template_for_ + generation so reasoning / tool kwargs reach the tokenizer.""" + _install_fake_mlx(monkeypatch) + from core.inference.mlx_inference import MLXInferenceBackend + + captured = {} + + def _fake_apply(tokenizer, messages, **kwargs): + captured["tokenizer"] = tokenizer + captured["messages"] = messages + captured["kwargs"] = kwargs + return "" + + monkeypatch.setattr( + "core.inference.chat_template_helpers." "apply_chat_template_for_generation", + _fake_apply, + raising = True, + ) + + # mlx_lm.stream_generate yields response objects with .token; make a + # one-token generator so _generate_text returns without touching the + # real stack. + import types as _types + + mlx_lm_pkg = _types.ModuleType("mlx_lm") + mlx_lm_sample = _types.ModuleType("mlx_lm.sample_utils") + mlx_lm_sample.make_sampler = lambda **_kw: object() + mlx_lm_sample.make_logits_processors = lambda **_kw: None + + class _Resp: + def __init__(self, tok): + self.token = tok + + def _stream_generate(_model, _tokenizer, **_kw): + yield _Resp(1) + + mlx_lm_pkg.stream_generate = _stream_generate + monkeypatch.setitem(sys.modules, "mlx_lm", mlx_lm_pkg) + monkeypatch.setitem(sys.modules, "mlx_lm.sample_utils", mlx_lm_sample) + + class _Tok: + chat_template = "x" + + def decode(self, ids, skip_special_tokens = False): + return "hi" + + backend = MLXInferenceBackend() + backend._model = object() + backend._tokenizer = _Tok() + backend._is_vlm = False + + out = list( + backend.generate_chat_response( + messages = [{"role": "user", "content": "ping"}], + tools = [{"function": {"name": "web_search"}}], + enable_thinking = True, + reasoning_effort = "medium", + preserve_thinking = True, + max_new_tokens = 1, + ) + ) + assert out == ["hi"] + # The kwargs the user toggled must reach the chat-template helper. + assert captured["kwargs"]["tools"] == [{"function": {"name": "web_search"}}] + assert captured["kwargs"]["enable_thinking"] is True + assert captured["kwargs"]["reasoning_effort"] == "medium" + assert captured["kwargs"]["preserve_thinking"] is True diff --git a/studio/backend/tests/test_openai_tool_passthrough.py b/studio/backend/tests/test_openai_tool_passthrough.py index 638cbc12c8..84f3e41998 100644 --- a/studio/backend/tests/test_openai_tool_passthrough.py +++ b/studio/backend/tests/test_openai_tool_passthrough.py @@ -26,6 +26,7 @@ sys.path.insert(0, _backend) import httpx import pytest +from fastapi import HTTPException from pydantic import ValidationError from models.inference import ( @@ -532,6 +533,7 @@ class TestFriendlyErrorHttpx: from routes.inference import ( # noqa: E402 _drop_empty_assistant_sentinels, + _openai_messages_for_gguf_chat, _openai_messages_for_passthrough, ) @@ -616,3 +618,110 @@ class TestDropEmptyAssistantSentinels: assert roles == ["user", "user"] for m in out: assert m.get("content"), m + + +class TestGgufVisionMessages: + _PNG_B64 = ( + "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAIAAACQd1PeAAAADUlEQVR42mNk" + "+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg==" + ) + + def test_preserves_multiturn_image_parts_on_original_turns(self): + req = ChatCompletionRequest( + model = "default", + image_base64 = self._PNG_B64, + messages = [ + { + "role": "user", + "content": [ + {"type": "text", "text": "describe image one"}, + { + "type": "image_url", + "image_url": { + "url": f"data:image/png;base64,{self._PNG_B64}", + }, + }, + ], + }, + {"role": "assistant", "content": "first answer"}, + { + "role": "user", + "content": [ + {"type": "text", "text": "describe image two"}, + { + "type": "image_url", + "image_url": { + "url": f"data:image/png;base64,{self._PNG_B64}", + }, + }, + ], + }, + ], + ) + + messages, has_image = _openai_messages_for_gguf_chat(req, is_vision = True) + + assert has_image is True + assert messages[0]["content"][0] == { + "type": "text", + "text": "describe image one", + } + assert messages[0]["content"][1]["type"] == "image_url" + assert len(messages[0]["content"]) == 2 + assert messages[2]["content"][0] == { + "type": "text", + "text": "describe image two", + } + assert messages[2]["content"][1]["type"] == "image_url" + assert len(messages[2]["content"]) == 2 + assert isinstance(messages[1]["content"], str) + + # Legacy top-level image_base64 must be ignored when any message-level + # image already exists; otherwise turn 2 ends up with two image parts. + for msg in messages: + content = msg.get("content") + if isinstance(content, list): + image_parts = [p for p in content if p.get("type") == "image_url"] + assert len(image_parts) == 1, msg + + def test_legacy_image_base64_is_injected_when_messages_are_text_only(self): + req = ChatCompletionRequest( + model = "default", + image_base64 = self._PNG_B64, + messages = [{"role": "user", "content": "describe this image"}], + ) + + messages, has_image = _openai_messages_for_gguf_chat(req, is_vision = True) + + assert has_image is True + assert messages[0]["content"][0] == { + "type": "text", + "text": "describe this image", + } + assert messages[0]["content"][1]["type"] == "image_url" + assert messages[0]["content"][1]["image_url"]["url"].startswith( + "data:image/png;base64," + ) + + def test_rejects_image_parts_for_text_only_gguf(self): + req = ChatCompletionRequest( + model = "default", + messages = [ + { + "role": "user", + "content": [ + {"type": "text", "text": "look"}, + { + "type": "image_url", + "image_url": { + "url": f"data:image/png;base64,{self._PNG_B64}", + }, + }, + ], + }, + ], + ) + + with pytest.raises(HTTPException) as exc_info: + _openai_messages_for_gguf_chat(req, is_vision = False) + assert "does not support vision" in str(exc_info.value) diff --git a/studio/backend/tests/test_safetensors_capability_advertise.py b/studio/backend/tests/test_safetensors_capability_advertise.py new file mode 100644 index 0000000000..c3ee5b9ff1 --- /dev/null +++ b/studio/backend/tests/test_safetensors_capability_advertise.py @@ -0,0 +1,451 @@ +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 + +""" +Capability advertisement contract: classifier honesty, worker→ +orchestrator IPC hop, and route-layer end-to-end. Pure helpers + fakes; +no torch / transformers import. +""" + +from __future__ import annotations + +import sys +from pathlib import Path +from types import SimpleNamespace +from unittest.mock import MagicMock + +_backend_root = Path(__file__).resolve().parent.parent +if str(_backend_root) not in sys.path: + sys.path.insert(0, str(_backend_root)) + + +# Qwen3 snippet covering tools, enable_thinking, preserve_thinking. +QWEN3_TEMPLATE = """ +{%- if tools %} + {{- '<|im_start|>system\\nFor each function call, return a json object' + ' wrapped inside tags.\\n' }} + {%- for tool in tools %} + {{- tool | tojson }} + {%- endfor %} +{%- endif %} +{%- for message in messages %} + {%- if message.role == 'tool' %} + {{- '<|im_start|>tool\\n' + message.content + '<|im_end|>\\n' }} + {%- endif %} +{%- endfor %} +{%- if enable_thinking is defined and enable_thinking %} + {{- '' }} +{%- endif %} +{%- if preserve_thinking %} + {{- assistant.reasoning_content }} +{%- endif %} +""" + + +GPT_OSS_TEMPLATE = """ +<|start|>system<|message|>You are gpt-oss. +reasoning_effort: {{ reasoning_effort }} +<|end|> +""" + + +PLAIN_TEMPLATE = """ +{%- for message in messages %} + {{- message.role + ': ' + message.content + '\\n' }} +{%- endfor %} +""" + + +# ── Tests: classifier honesty ──────────────────────────────────────── + + +def test_detect_reasoning_flags_qwen3_supports_tools_and_reasoning(): + from core.inference.llama_cpp import detect_reasoning_flags + + flags = detect_reasoning_flags(QWEN3_TEMPLATE, "unsloth/Qwen3-0.6B") + assert flags["supports_tools"] is True + assert flags["supports_reasoning"] is True + assert flags["reasoning_style"] == "enable_thinking" + assert flags["supports_preserve_thinking"] is True + assert flags["reasoning_always_on"] is False + + +def test_detect_reasoning_flags_plain_template_all_false(): + from core.inference.llama_cpp import detect_reasoning_flags + + flags = detect_reasoning_flags(PLAIN_TEMPLATE, "some/PlainChat") + assert flags["supports_tools"] is False + assert flags["supports_reasoning"] is False + assert flags["supports_preserve_thinking"] is False + assert flags["reasoning_always_on"] is False + + +def test_detect_reasoning_flags_none_template_returns_all_false(): + from core.inference.llama_cpp import detect_reasoning_flags + + flags = detect_reasoning_flags(None) + assert flags["supports_tools"] is False + assert flags["supports_reasoning"] is False + assert flags["supports_preserve_thinking"] is False + assert flags["reasoning_always_on"] is False + assert flags["reasoning_style"] == "enable_thinking" + + +def test_detect_safetensors_features_passes_template_through_to_classifier(): + """Route wrapper forwards a real template to the inner classifier.""" + from routes.inference import _detect_safetensors_features + + backend = SimpleNamespace(active_model_name = "unsloth/Qwen3-0.6B") + flags = _detect_safetensors_features(backend, QWEN3_TEMPLATE) + assert flags["supports_tools"] is True + assert flags["supports_reasoning"] is True + + +def test_detect_safetensors_features_none_template_returns_all_false(): + from routes.inference import _detect_safetensors_features + + backend = SimpleNamespace(active_model_name = "unsloth/Qwen3-0.6B") + flags = _detect_safetensors_features(backend, None) + assert flags == { + "supports_reasoning": False, + "reasoning_style": "enable_thinking", + "reasoning_always_on": False, + "supports_preserve_thinking": False, + "supports_tools": False, + } + + +def test_detect_safetensors_features_gptoss_disables_tools(): + """gpt-oss Harmony: tools intentionally off even if template marks it.""" + from routes.inference import _detect_safetensors_features + + backend = MagicMock() + backend.active_model_name = "unsloth/gpt-oss-20b" + backend._is_gpt_oss_model.return_value = True + + flags = _detect_safetensors_features(backend, QWEN3_TEMPLATE) + assert flags["supports_reasoning"] is True + assert flags["reasoning_style"] == "reasoning_effort" + assert flags["supports_tools"] is False + + +# Llama-3 / Mistral templates advertise tool handling but the model emits +# tool calls in <|python_tag|> / [TOOL_CALLS] format -- not the +# / system<|end_header_id|>' }} + {{- 'You have access to the following tools.' }} + {%- for tool in tools %} + {{- tool | tojson }} + {%- endfor %} +{%- endif %} +{%- for message in messages %} + {%- if message.role == 'tool' %} + {{- '<|start_header_id|>ipython<|end_header_id|>' }} + {{- '<|python_tag|>' }} + {{- message.content }} + {%- endif %} +{%- endfor %} +""" + +MISTRAL_TEMPLATE = """ +{%- if tools %} + {%- for tool in tools %} + {{- tool | tojson }} + {%- endfor %} +{%- endif %} +{%- for message in messages %} + {%- if message.role == 'tool' %} + {{- '[TOOL_CALLS]' + message.content + '[/TOOL_CALLS]' }} + {%- endif %} +{%- endfor %} +""" + + +def test_detect_safetensors_features_llama3_template_suppresses_tools(): + """Llama-3 emits <|python_tag|>; safetensors loop cannot parse it.""" + from routes.inference import _detect_safetensors_features + + backend = SimpleNamespace(active_model_name = "unsloth/Llama-3.2-3B-Instruct") + flags = _detect_safetensors_features(backend, LLAMA3_TEMPLATE) + assert flags["supports_tools"] is False + + +def test_detect_safetensors_features_mistral_template_suppresses_tools(): + """Mistral emits [TOOL_CALLS]; safetensors loop cannot parse it.""" + from routes.inference import _detect_safetensors_features + + backend = SimpleNamespace(active_model_name = "unsloth/mistral-7b-instruct-v0.3") + flags = _detect_safetensors_features(backend, MISTRAL_TEMPLATE) + assert flags["supports_tools"] is False + + +def test_detect_safetensors_features_qwen_tool_call_keeps_tools_on(): + """Sanity check: gate only suppresses non-Qwen formats.""" + from routes.inference import _detect_safetensors_features + + backend = SimpleNamespace(active_model_name = "unsloth/Qwen3-0.6B") + flags = _detect_safetensors_features(backend, QWEN3_TEMPLATE) + assert flags["supports_tools"] is True + + +def test_detect_safetensors_features_function_xml_format_keeps_tools_on(): + """Templates emitting XML are parser-compatible.""" + from routes.inference import _detect_safetensors_features + + tpl_with_function_xml = ( + "{%- if tools %}<|im_start|>system\n" + "Tool call format: v" + "<|im_end|>{%- endif %}" + ) + backend = SimpleNamespace(active_model_name = "custom/with-function-xml") + flags = _detect_safetensors_features(backend, tpl_with_function_xml) + assert flags["supports_tools"] is True + + +# Qwen3.5 family pins -- the live GGUF + safetensors templates fetched +# from the unsloth/Qwen3.5-0.8B(-GGUF) repos both wrap tool calls as +# ``\n...``. Capture a faithful slice so the +# classifier never silently regresses for this family. + +QWEN35_TOOL_INSTRUCTION = ( + "{%- if tools %}\n" + " <|im_start|>system\n" + " # Tools\n" + " \n" + " {%- for tool in tools %}{{ tool | tojson }}{%- endfor %}\n" + " \n" + " If you choose to call a function ONLY reply in the following format:\n" + " \n" + " \n" + " \n" + " value_1\n" + " \n" + " \n" + " \n" + " <|im_end|>\n" + "{%- endif %}\n" + "{%- if enable_thinking is defined and enable_thinking %}{{- '' }}{%- endif %}\n" +) + + +def test_detect_safetensors_features_qwen35_keeps_tools_on(): + """unsloth/Qwen3.5-0.8B family must surface tools+reasoning enabled.""" + from routes.inference import _detect_safetensors_features + + backend = SimpleNamespace(active_model_name = "unsloth/Qwen3.5-0.8B") + flags = _detect_safetensors_features(backend, QWEN35_TOOL_INSTRUCTION) + assert flags["supports_tools"] is True + assert flags["supports_reasoning"] is True + assert flags["reasoning_style"] == "enable_thinking" + + +# ── Tests: IPC bridge contract ─────────────────────────────────────── + + +def test_orchestrator_mirrors_chat_template_info_into_models_dict(): + """Worker → orchestrator must copy chat_template_info verbatim.""" + from core.inference.orchestrator import InferenceOrchestrator + + orch = InferenceOrchestrator.__new__(InferenceOrchestrator) + orch.models = {} + orch.active_model_name = None + orch.loading_models = set() + + model_info = { + "identifier": "unsloth/Qwen3-0.6B", + "display_name": "Qwen3-0.6B", + "is_vision": False, + "is_lora": False, + "is_gguf": False, + "is_audio": False, + "audio_type": None, + "has_audio_input": False, + "chat_template_info": { + "has_template": True, + "template": QWEN3_TEMPLATE, + "format_type": "chatml", + "template_name": "qwen3", + "special_tokens": {"bos_token": "<|im_start|>"}, + }, + } + + # Replay orchestrator.load_model's mirror block verbatim. + orch.active_model_name = model_info["identifier"] + orch.models[orch.active_model_name] = { + "is_vision": model_info.get("is_vision", False), + "is_lora": model_info.get("is_lora", False), + "display_name": model_info.get("display_name", "x"), + "is_audio": model_info.get("is_audio", False), + "audio_type": model_info.get("audio_type"), + "has_audio_input": model_info.get("has_audio_input", False), + } + _tpl_info = model_info.get("chat_template_info") + if isinstance(_tpl_info, dict): + orch.models[orch.active_model_name]["chat_template_info"] = _tpl_info + + entry = orch.models[orch.active_model_name] + tpl = entry.get("chat_template_info", {}).get("template") + assert tpl == QWEN3_TEMPLATE + + from routes.inference import _detect_safetensors_features + + flags = _detect_safetensors_features( + SimpleNamespace(active_model_name = orch.active_model_name), tpl + ) + assert flags["supports_tools"] is True + assert flags["supports_reasoning"] is True + + +def test_orchestrator_missing_chat_template_info_falls_back_to_all_false(): + """Old / malformed worker reply: no crash, all flags False.""" + from core.inference.orchestrator import InferenceOrchestrator + from routes.inference import _detect_safetensors_features + + orch = InferenceOrchestrator.__new__(InferenceOrchestrator) + orch.models = {} + orch.active_model_name = "unsloth/Qwen3-0.6B" + + model_info = { + "identifier": "unsloth/Qwen3-0.6B", + "is_vision": False, + "is_lora": False, + # NB: no chat_template_info key + } + orch.models[orch.active_model_name] = { + "is_vision": False, + "is_lora": False, + } + _tpl_info = model_info.get("chat_template_info") + if isinstance(_tpl_info, dict): + orch.models[orch.active_model_name]["chat_template_info"] = _tpl_info + + entry = orch.models[orch.active_model_name] + tpl = entry.get("chat_template_info", {}).get("template") + assert tpl is None + + flags = _detect_safetensors_features( + SimpleNamespace(active_model_name = orch.active_model_name), tpl + ) + assert flags["supports_tools"] is False + + +def test_worker_load_reply_payload_includes_chat_template_info(): + """Worker IPC reply carries chat_template_info dict.""" + + class _StubBackend: + def __init__(self, identifier, template): + self.active_model_name = identifier + self.models = { + identifier: { + "chat_template_info": { + "has_template": True, + "template": template, + "format_type": "chatml", + "template_name": "qwen3", + "special_tokens": {"bos_token": "<|im_start|>"}, + } + } + } + + backend = _StubBackend("unsloth/Qwen3-0.6B", QWEN3_TEMPLATE) + mc = SimpleNamespace( + identifier = "unsloth/Qwen3-0.6B", + display_name = "Qwen3-0.6B", + is_vision = False, + is_lora = False, + ) + + # Replay the worker's payload-build block. + model_info = { + "identifier": mc.identifier, + "display_name": mc.display_name, + "is_vision": mc.is_vision, + "is_lora": mc.is_lora, + "is_gguf": False, + } + _bm = getattr(backend, "models", {}) or {} + _entry = ( + _bm.get(mc.identifier) + or _bm.get(getattr(backend, "active_model_name", None)) + or {} + ) + _tpl_info = _entry.get("chat_template_info") + if isinstance(_tpl_info, dict): + model_info["chat_template_info"] = { + "has_template": bool(_tpl_info.get("has_template", False)), + "template": _tpl_info.get("template"), + "format_type": _tpl_info.get("format_type", "generic"), + "template_name": _tpl_info.get("template_name"), + "special_tokens": _tpl_info.get("special_tokens", {}) or {}, + } + + assert "chat_template_info" in model_info + assert model_info["chat_template_info"]["template"] == QWEN3_TEMPLATE + assert model_info["chat_template_info"]["has_template"] is True + + +def test_worker_load_reply_payload_survives_missing_template(): + """Tokenizer with no chat_template still produces a valid reply.""" + + class _StubBackend: + def __init__(self): + self.active_model_name = "legacy/no-template" + self.models = {"legacy/no-template": {}} # no chat_template_info + + backend = _StubBackend() + mc = SimpleNamespace( + identifier = "legacy/no-template", + display_name = "legacy", + is_vision = False, + is_lora = False, + ) + + model_info = { + "identifier": mc.identifier, + "display_name": mc.display_name, + "is_vision": mc.is_vision, + "is_lora": mc.is_lora, + "is_gguf": False, + } + _bm = getattr(backend, "models", {}) or {} + _entry = _bm.get(mc.identifier) or {} + _tpl_info = _entry.get("chat_template_info") + if isinstance(_tpl_info, dict): + model_info["chat_template_info"] = dict(_tpl_info) + + assert "chat_template_info" not in model_info + + +# ── End-to-end: route layer sees the template, advertises True ─────── + + +def test_route_layer_emits_supports_tools_true_for_qwen3_safetensors(): + """End-to-end: Qwen3 safetensors flips supports_tools=True.""" + from routes.inference import _detect_safetensors_features + + backend = SimpleNamespace( + active_model_name = "unsloth/Qwen3-0.6B", + models = { + "unsloth/Qwen3-0.6B": { + "is_vision": False, + "chat_template_info": { + "has_template": True, + "template": QWEN3_TEMPLATE, + "format_type": "chatml", + }, + } + }, + ) + + _model_info = backend.models.get(backend.active_model_name, {}) + _tpl = _model_info.get("chat_template_info", {}).get("template") + flags = _detect_safetensors_features(backend, _tpl) + + assert flags["supports_tools"] is True + assert flags["supports_reasoning"] is True + assert flags["supports_preserve_thinking"] is True diff --git a/studio/backend/tests/test_safetensors_tool_loop.py b/studio/backend/tests/test_safetensors_tool_loop.py new file mode 100644 index 0000000000..923af87c4f --- /dev/null +++ b/studio/backend/tests/test_safetensors_tool_loop.py @@ -0,0 +1,788 @@ +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 + +""" +Tests for the safetensors agentic tool loop. + +Covers the shared ``tool_call_parser`` helpers and the cumulative-text +state machine inside ``safetensors_agentic.run_safetensors_tool_loop``. +The loop is exercised with hand-crafted fake single-turn generators so +no model load is needed; the tests run in CI under a few seconds. + +Edge cases under coverage: +* Plain answers (no tool calls) flush full content. +* Single ``{json}`` triggers the tool and re-enters. +* Single ``...`` XML form triggers the same path. +* Truncated unclosed ```` is still parsed. +* Tool result is fed back as ``role=tool`` for the next iteration. +* Bad JSON inside ```` does not raise and (when healed) is + routed as a ``{"query": ...}`` web search call. +* Duplicate tool calls produce a synthetic "do not repeat" result the + second time. +* ``__IMAGES__`` sentinel is stripped before the model sees the result. +* Tool execution errors are tagged so the model gets a nudge but the + loop keeps streaming. +* Cancel is honoured between iterations. +* ``max_tool_iterations`` cap is respected and a final-answer attempt + closes the stream cleanly. +""" + +import threading + +import pytest + +from core.inference import safetensors_agentic +from core.inference.safetensors_agentic import ( + _coerce_arguments, + run_safetensors_tool_loop, +) +from core.inference.tool_call_parser import ( + has_tool_signal, + parse_tool_calls_from_text, + strip_tool_markup, +) +from utils.datasets import is_gpt_oss_model_name + + +# ──────────────────────────────────────────────────────────────────── +# parse_tool_calls_from_text +# ──────────────────────────────────────────────────────────────────── + + +class TestParser: + def test_json_tool_call(self): + text = ( + '{"name":"web_search","arguments":{"query":"hello"}}' + ) + result = parse_tool_calls_from_text(text) + assert len(result) == 1 + tc = result[0] + assert tc["type"] == "function" + assert tc["function"]["name"] == "web_search" + # Arguments must always be a JSON string. + assert isinstance(tc["function"]["arguments"], str) + assert "hello" in tc["function"]["arguments"] + + def test_json_tool_call_unclosed(self): + # No ; balanced-brace extractor must still close. + text = '{"name":"python","arguments":{"code":"print(1)"}}' + result = parse_tool_calls_from_text(text) + assert len(result) == 1 + assert result[0]["function"]["name"] == "python" + + def test_xml_function_call(self): + text = "print('hi')" + result = parse_tool_calls_from_text(text) + assert len(result) == 1 + assert result[0]["function"]["name"] == "python" + assert "print('hi')" in result[0]["function"]["arguments"] + + def test_xml_unclosed(self): + # Closing tags omitted; parser must still extract the value. + text = "ls -la" + result = parse_tool_calls_from_text(text) + assert len(result) == 1 + assert result[0]["function"]["name"] == "terminal" + assert "ls -la" in result[0]["function"]["arguments"] + + def test_code_with_embedded_xml(self): + # A code parameter contains the literal . Must not + # truncate the value because the parser uses end-of-body as the + # only boundary for single-parameter calls. + text = ( + "html = ''\n" + "print('hi')" + ) + result = parse_tool_calls_from_text(text) + assert len(result) == 1 + assert "print('hi')" in result[0]["function"]["arguments"] + + def test_multiple_calls(self): + text = ( + '{"name":"web_search","arguments":{"query":"a"}}' + '{"name":"web_search","arguments":{"query":"b"}}' + ) + result = parse_tool_calls_from_text(text) + assert len(result) == 2 + assert result[0]["function"]["name"] == "web_search" + assert result[1]["function"]["name"] == "web_search" + + def test_bad_json_does_not_raise(self): + text = "{not valid json}" + result = parse_tool_calls_from_text(text) + # Bad JSON is silently dropped; caller can fall back to text. + assert result == [] + + def test_has_tool_signal(self): + assert has_tool_signal("blah x") + assert has_tool_signal("hi ...") + assert not has_tool_signal("hello world") + + def test_strip_markup_closed(self): + text = "before {} after" + assert strip_tool_markup(text) == "before after" + + def test_strip_markup_unclosed_final(self): + text = "before {partial" + # With final=True the trailing run is dropped. + assert strip_tool_markup(text, final = True) == "before" + # Without final=True the unclosed run is preserved. + assert "partial" in strip_tool_markup(text) + + +# ──────────────────────────────────────────────────────────────────── +# run_safetensors_tool_loop +# ──────────────────────────────────────────────────────────────────── + + +def _fake_stream(chunks): + """Build a single-turn generator that yields cumulative snapshots.""" + + def _gen(_messages): + acc = "" + for c in chunks: + acc += c + yield acc + + return _gen + + +def _const_stream(text): + """A single-turn generator that yields one cumulative snapshot.""" + + def _gen(_messages): + yield text + + return _gen + + +class FakeExecuteTool: + """Stand-in for ``core.inference.tools.execute_tool``.""" + + def __init__(self, results): + # ``results`` is a list of strings or RuntimeError instances. + self.results = list(results) + self.calls: list[tuple[str, dict]] = [] + + def __call__( + self, + name, + arguments, + *, + cancel_event = None, + timeout = None, + session_id = None, + ): + self.calls.append((name, arguments)) + result = self.results.pop(0) if self.results else "OK" + if isinstance(result, Exception): + raise result + return result + + +def _collect_events(generator, max_events = 200): + events = [] + for ev in generator: + events.append(ev) + if len(events) >= max_events: + break + return events + + +def _make_loop(*, turns, exec_results = None, **kwargs): + """Build a configured loop with a multi-turn fake generator. + + ``turns`` is a list of chunk-lists; iteration N yields chunks from + ``turns[N]``. + """ + turn_iter = iter(turns) + + def _gen(_messages): + try: + chunks = next(turn_iter) + except StopIteration: + return + acc = "" + for c in chunks: + acc += c + yield acc + + exec_fn = FakeExecuteTool(exec_results or []) + return run_safetensors_tool_loop( + single_turn = _gen, + messages = [{"role": "user", "content": "hi"}], + tools = [ + {"type": "function", "function": {"name": "web_search"}}, + {"type": "function", "function": {"name": "python"}}, + {"type": "function", "function": {"name": "terminal"}}, + ], + execute_tool = exec_fn, + **kwargs, + ), exec_fn + + +class TestLoopBasic: + def test_plain_answer(self): + # No tool XML; loop should yield content then status="". + loop, _exec = _make_loop( + turns = [["Hello", " world", "!"]], + exec_results = [], + ) + events = _collect_events(loop) + contents = [e for e in events if e["type"] == "content"] + statuses = [e for e in events if e["type"] == "status"] + assert contents, "expected at least one content event" + # Final cumulative content should contain the answer. + final_text = contents[-1]["text"] + assert "Hello world!" in final_text + assert statuses and statuses[-1]["text"] == "" + + def test_single_tool_then_answer(self): + loop, exec_fn = _make_loop( + turns = [ + # : tool call only. + [ + '{"name":"web_search",', + '"arguments":{"query":"weather"}}', + "", + ], + # : final answer. + ["The ", "weather is ", "sunny."], + ], + exec_results = ["Sunny and 22C"], + ) + events = _collect_events(loop) + kinds = [e["type"] for e in events] + + assert "tool_start" in kinds + assert "tool_end" in kinds + # Tool was actually called with the parsed arguments. + assert exec_fn.calls == [("web_search", {"query": "weather"})] + + tool_start = next(e for e in events if e["type"] == "tool_start") + assert tool_start["tool_name"] == "web_search" + tool_end = next(e for e in events if e["type"] == "tool_end") + assert tool_end["result"] == "Sunny and 22C" + + contents = [e for e in events if e["type"] == "content"] + assert contents and "sunny" in contents[-1]["text"].lower() + + def test_function_xml_form(self): + loop, exec_fn = _make_loop( + turns = [ + ["print(1)"], + ["Result: 1"], + ], + exec_results = ["1\n"], + ) + events = _collect_events(loop) + assert exec_fn.calls == [("python", {"code": "print(1)"})] + contents = [e for e in events if e["type"] == "content"] + assert "Result: 1" in contents[-1]["text"] + + def test_truncated_unclosed_tool_call(self): + loop, exec_fn = _make_loop( + turns = [ + # No ; balanced-brace parser must still + # succeed because the JSON itself is balanced. + ['{"name":"web_search","arguments":{"query":"x"}}'], + ["done"], + ], + exec_results = ["result"], + ) + events = _collect_events(loop) + assert exec_fn.calls == [("web_search", {"query": "x"})] + + def test_bad_json_healed_to_query(self): + # Tool call with non-JSON string arguments. With auto_heal_tool_calls + # the string is routed as {"query": ...}. + loop, exec_fn = _make_loop( + turns = [ + # JSON inside the tool call is well-formed; the + # ``arguments`` is a string that is not itself valid + # JSON for ``_coerce_arguments`` to parse, so the + # heal path runs. + [ + '{"name":"web_search","arguments":"hello world"}' + ], + ["ok"], + ], + exec_results = ["..."], + ) + events = _collect_events(loop) + assert exec_fn.calls and exec_fn.calls[0][0] == "web_search" + assert exec_fn.calls[0][1] == {"query": "hello world"} + + +class TestLoopBehaviour: + def test_duplicate_tool_call_synthetic_result(self): + # Two identical successful calls in a row: the second is short- + # circuited with a "do not repeat" message and execute_tool is + # called only once. + loop, exec_fn = _make_loop( + turns = [ + [ + '{"name":"web_search","arguments":{"query":"x"}}' + ], + [ + '{"name":"web_search","arguments":{"query":"x"}}' + ], + ["final"], + ], + exec_results = ["search-result-1"], + ) + events = _collect_events(loop) + # Only one real call. + assert len(exec_fn.calls) == 1 + tool_end_events = [e for e in events if e["type"] == "tool_end"] + assert len(tool_end_events) == 2 + assert "do not repeat" in tool_end_events[1]["result"].lower() + + def test_image_sentinel_stripped_from_model_feed(self): + # The tool result has a frontend image sentinel that should be + # stripped before being fed back into the next turn, BUT the + # tool_end event still carries the raw result for the UI. + loop, exec_fn = _make_loop( + turns = [ + [ + '{"name":"python","arguments":{"code":"plot()"}}' + ], + ["see chart"], + ], + exec_results = ["chart\n__IMAGES__:/tmp/chart.png"], + ) + events = _collect_events(loop) + tool_end = next(e for e in events if e["type"] == "tool_end") + assert "__IMAGES__" in tool_end["result"] + + def test_image_sentinel_stripped_with_leading_marker(self): + # Sentinel at start (no newline) must not leak to the model. + from core.inference import safetensors_agentic as _sa + + captured: list[list[dict]] = [] + + def fake_single_turn(messages, **_kw): + captured.append([dict(m) for m in messages]) + if len(captured) == 1: + yield '{"name":"python","arguments":{"code":"plot()"}}' + else: + yield "done" + + events = list( + _sa.run_safetensors_tool_loop( + single_turn = fake_single_turn, + messages = [{"role": "user", "content": "plot please"}], + tools = [{"function": {"name": "python"}}], + execute_tool = lambda *_a, **_kw: "__IMAGES__:/tmp/x.png", + cancel_event = threading.Event(), + max_tool_iterations = 3, + auto_heal_tool_calls = True, + ) + ) + # Model's second turn must not see "__IMAGES__". + assert len(captured) >= 2 + tool_msgs = [m for m in captured[1] if m.get("role") == "tool"] + assert tool_msgs, "no tool message reached the model" + for tm in tool_msgs: + assert ( + "__IMAGES__" not in tm["content"] + ), f"sentinel leaked to model: {tm['content']!r}" + + def test_image_sentinel_stripped_with_multiple_markers(self): + # Consecutive sentinels: cut at the first, nothing leaks. + from core.inference import safetensors_agentic as _sa + + captured: list[list[dict]] = [] + + def fake_single_turn(messages, **_kw): + captured.append([dict(m) for m in messages]) + if len(captured) == 1: + yield '{"name":"python","arguments":{"code":"plot()"}}' + else: + yield "done" + + multi = "panel\n__IMAGES__:/tmp/a.png\n__IMAGES__:/tmp/b.png" + events = list( + _sa.run_safetensors_tool_loop( + single_turn = fake_single_turn, + messages = [{"role": "user", "content": "plot please"}], + tools = [{"function": {"name": "python"}}], + execute_tool = lambda *_a, **_kw: multi, + cancel_event = threading.Event(), + max_tool_iterations = 3, + auto_heal_tool_calls = True, + ) + ) + tool_msgs = [m for m in captured[1] if m.get("role") == "tool"] + assert tool_msgs + for tm in tool_msgs: + assert ( + "__IMAGES__" not in tm["content"] + ), f"second sentinel leaked: {tm['content']!r}" + assert ( + tm["content"] == "panel" + ), f"expected payload-only 'panel', got {tm['content']!r}" + + def test_tool_execution_error_is_emitted_but_loop_continues(self): + loop, exec_fn = _make_loop( + turns = [ + [ + '{"name":"web_search","arguments":{"query":"x"}}' + ], + ["sorry, that failed"], + ], + exec_results = ["Error: network unreachable"], + ) + events = _collect_events(loop) + tool_end = next(e for e in events if e["type"] == "tool_end") + assert tool_end["result"].startswith("Error") + # The loop must still produce a content event after the failure. + contents = [e for e in events if e["type"] == "content"] + assert contents + + def test_exception_in_executor_does_not_raise(self): + loop, exec_fn = _make_loop( + turns = [ + [ + '{"name":"web_search","arguments":{"query":"x"}}' + ], + ["recovered"], + ], + exec_results = [RuntimeError("boom")], + ) + events = _collect_events(loop) + tool_end = next(e for e in events if e["type"] == "tool_end") + assert "boom" in tool_end["result"] + + +class TestLoopControl: + def test_cancel_event_breaks_loop(self): + cancel = threading.Event() + cancel.set() + # Even with a fake stream that emits tool calls, the loop must + # bail before invoking execute_tool when cancel is set. + exec_fn = FakeExecuteTool([]) + events = list( + run_safetensors_tool_loop( + single_turn = _const_stream( + '{"name":"web_search",' + '"arguments":{"query":"x"}}' + ), + messages = [{"role": "user", "content": "hi"}], + tools = [], + execute_tool = exec_fn, + cancel_event = cancel, + ) + ) + assert events == [] + assert exec_fn.calls == [] + + def test_max_iterations_caps_loop(self): + # The loop should stop after max_tool_iterations even if the + # model keeps asking for tools, then emit a final-attempt round. + loop, exec_fn = _make_loop( + turns = [ + # : tool call (executes once) + [ + '{"name":"web_search","arguments":{"query":"a"}}' + ], + # : model gives a final answer when nudged. + ["here is the final answer"], + ], + exec_results = ["result"], + max_tool_iterations = 1, + ) + events = _collect_events(loop) + contents = [e for e in events if e["type"] == "content"] + # Final content must include the final answer. + assert contents and "final answer" in contents[-1]["text"] + + +class TestStatusFormatting: + def test_status_for_known_tools(self): + # Use the private helper directly to verify status formatting. + assert ( + safetensors_agentic._status_for_tool("web_search", {"query": "abc"}) + == "Searching: abc" + ) + assert ( + safetensors_agentic._status_for_tool( + "web_search", {"url": "https://www.example.com/x"} + ) + == "Reading: example.com" + ) + assert safetensors_agentic._status_for_tool( + "python", {"code": "x = 1"} + ).startswith("Running Python:") + assert safetensors_agentic._status_for_tool( + "terminal", {"command": "ls"} + ).startswith("Running:") + assert safetensors_agentic._status_for_tool("unknown_tool", {}).startswith( + "Calling:" + ) + + +class TestProseMentioningToolCall: + def test_assistant_prose_with_literal_tool_call_text_survives(self): + # Regression: if the assistant text legitimately mentions + # ```` as a literal string and the parser finds no + # actual call, the loop must surface the full content instead + # of silently stripping everything past the literal marker. + loop, exec_fn = _make_loop( + turns = [ + # : a real tool call so the loop moves to + # . + [ + '{"name":"web_search","arguments":{"query":"x"}}' + ], + # : prose that mentions the literal text. + ["the docs say means an LLM tool call wrapper"], + ], + exec_results = ["result"], + ) + events = _collect_events(loop) + contents = [e for e in events if e["type"] == "content"] + assert contents, "expected at least one content event" + final = contents[-1]["text"] + assert ( + "LLM tool" in final + ), f"prose mentioning should not be truncated; got {final!r}" + + def test_tool_result_with_tool_call_text_does_not_retrigger(self): + # Tool result text contains the literal ```` string. + # The loop must only parse the MODEL output, not the tool + # result, so we should see exactly one call. + loop, exec_fn = _make_loop( + turns = [ + [ + '{"name":"web_search","arguments":{"query":"x"}}' + ], + ["the docs mention wrappers"], + ], + exec_results = ["Page text: appears here in the docs"], + ) + events = _collect_events(loop) + assert len(exec_fn.calls) == 1 + + +class TestChatTemplateHelper: + """Cover the dependency-light helper used by InferenceBackend.""" + + def setup_method(self): + from core.inference.chat_template_helpers import ( + apply_chat_template_for_generation, + ) + + self.apply = apply_chat_template_for_generation + + class _Tok: + def __init__(self, accepted): + self.accepted = accepted + self.call_count = 0 + self.last_kwargs = None + + def apply_chat_template( + self, messages, *, tokenize = False, add_generation_prompt = True, **kw + ): + self.call_count += 1 + unknown = set(kw) - self.accepted + if unknown: + raise TypeError(f"unexpected kwargs: {sorted(unknown)}") + self.last_kwargs = dict(kw) + return "PROMPT" + + def test_richest_call_wins_when_template_supports_all(self): + tok = self._Tok({"tools", "enable_thinking"}) + self.apply(tok, [], tools = [{}], enable_thinking = True) + assert tok.call_count == 1 + assert "tools" in tok.last_kwargs + assert "enable_thinking" in tok.last_kwargs + + def test_falls_back_when_template_rejects_reasoning_kwarg(self): + tok = self._Tok({"tools"}) + self.apply(tok, [], tools = [{}], enable_thinking = True) + assert tok.call_count >= 2 + assert tok.last_kwargs == {"tools": [{}]} + + def test_falls_back_to_bare_call(self): + tok = self._Tok(set()) + self.apply(tok, [], tools = [{}], enable_thinking = True) + assert tok.last_kwargs == {} + + def test_jinja_error_propagates(self): + class Boom: + def apply_chat_template(self, *a, **kw): + raise ValueError("jinja: missing var") + + with pytest.raises(ValueError): + self.apply(Boom(), []) + + def test_no_kwargs_single_call(self): + tok = self._Tok(set()) + self.apply(tok, []) + assert tok.call_count == 1 + + +# ──────────────────────────────────────────────────────────────────── +# Guardrails (allowlist, budget, streaming-leak, dedup, id offset, +# auto_heal=False, canonical healed-arg key) +# ──────────────────────────────────────────────────────────────────── + + +class TestGuardrails: + def test_disabled_tool_is_not_executed(self): + exec_fn = FakeExecuteTool([]) + loop = run_safetensors_tool_loop( + single_turn = _fake_stream( + [ + '{"name":"terminal","arguments":{"command":"echo bypass"}}' + ] + ), + messages = [{"role": "user", "content": "hi"}], + tools = [{"type": "function", "function": {"name": "web_search"}}], + execute_tool = exec_fn, + max_tool_iterations = 2, + ) + events = _collect_events(loop) + assert exec_fn.calls == [] + tool_ends = [e for e in events if e["type"] == "tool_end"] + assert tool_ends and "not enabled" in tool_ends[0]["result"].lower() + + def test_empty_tools_list_does_not_enforce_allowlist(self): + exec_fn = FakeExecuteTool(["OK"]) + loop = run_safetensors_tool_loop( + single_turn = _fake_stream( + [ + '{"name":"python","arguments":{"code":"print(1)"}}' + ] + ), + messages = [{"role": "user", "content": "hi"}], + tools = [], + execute_tool = exec_fn, + max_tool_iterations = 2, + ) + _collect_events(loop) + assert exec_fn.calls == [("python", {"code": "print(1)"})] + + def test_max_iterations_zero_executes_no_tools(self): + loop, exec_fn = _make_loop( + turns = [ + [ + '{"name":"web_search","arguments":{"query":"x"}}' + ] + ], + exec_results = ["OK"], + max_tool_iterations = 0, + ) + events = _collect_events(loop) + assert exec_fn.calls == [] + assert events and events[-1] == {"type": "status", "text": ""} + + def test_streaming_clips_before_tool_signal_no_leak(self): + loop, exec_fn = _make_loop( + turns = [ + [ + "I will look this up. ", + "Some more prose that's long enough to leave the buffer. ", + '{"name":"web_search","arguments":{"query":"x"}}', + ], + ["all done"], + ], + exec_results = ["weather: sunny"], + max_tool_iterations = 2, + ) + events = _collect_events(loop) + assert exec_fn.calls == [("web_search", {"query": "x"})] + for e in events: + if e["type"] == "content": + assert "" not in e["text"] + assert "web_search" not in e["text"] + + def test_auto_heal_disabled_still_parses_valid_tool_call(self): + loop, exec_fn = _make_loop( + turns = [ + [ + '{"name":"web_search","arguments":{"query":"x"}}' + ], + ["done"], + ], + exec_results = ["OK"], + auto_heal_tool_calls = False, + max_tool_iterations = 2, + ) + _collect_events(loop) + assert exec_fn.calls == [("web_search", {"query": "x"})] + + def test_non_consecutive_duplicate_is_short_circuited(self): + loop, exec_fn = _make_loop( + turns = [ + [ + '{"name":"web_search","arguments":{"query":"A"}}' + ], + [ + '{"name":"web_search","arguments":{"query":"B"}}' + ], + [ + '{"name":"web_search","arguments":{"query":"A"}}' + ], + ["final"], + ], + exec_results = ["res-A", "res-B"], + max_tool_iterations = 4, + ) + events = _collect_events(loop) + assert exec_fn.calls == [ + ("web_search", {"query": "A"}), + ("web_search", {"query": "B"}), + ] + tool_ends = [e for e in events if e["type"] == "tool_end"] + assert "already made this exact call" in tool_ends[-1]["result"] + + def test_coerce_string_args_python_uses_code_key(self): + assert _coerce_arguments("print(1)", heal = True, tool_name = "python") == { + "code": "print(1)" + } + + def test_coerce_string_args_terminal_uses_command_key(self): + assert _coerce_arguments("ls -la", heal = True, tool_name = "terminal") == { + "command": "ls -la" + } + + def test_tool_call_ids_unique_across_loop_iterations(self): + loop, _exec = _make_loop( + turns = [ + [ + '{"name":"web_search","arguments":{"query":"A"}}' + ], + [ + '{"name":"web_search","arguments":{"query":"B"}}' + ], + ["done"], + ], + exec_results = ["A", "B"], + max_tool_iterations = 3, + ) + events = _collect_events(loop) + ids = [e["tool_call_id"] for e in events if e["type"] == "tool_start"] + assert len(ids) == 2 and ids[0] != ids[1] + + +# ──────────────────────────────────────────────────────────────────── +# Shared gpt-oss name detector +# ──────────────────────────────────────────────────────────────────── + + +class TestGptOssNameDetection: + def test_substring_match(self): + assert is_gpt_oss_model_name("unsloth/gpt-oss-20b") is True + + def test_negative_known_non_oss_model(self): + assert is_gpt_oss_model_name("meta-llama/Llama-3.1-8B-Instruct") is False + + def test_empty_or_none_returns_false(self): + assert is_gpt_oss_model_name("") is False + assert is_gpt_oss_model_name(None) is False + + +if __name__ == "__main__": + pytest.main([__file__, "-v"]) diff --git a/studio/backend/utils/datasets/__init__.py b/studio/backend/utils/datasets/__init__.py index c9237d83c1..7988b09972 100644 --- a/studio/backend/utils/datasets/__init__.py +++ b/studio/backend/utils/datasets/__init__.py @@ -59,6 +59,7 @@ from .model_mappings import ( TEMPLATE_TO_MODEL_MAPPER, MODEL_TO_TEMPLATE_MAPPER, TEMPLATE_TO_RESPONSES_MAPPER, + is_gpt_oss_model_name, ) # Legacy imports from the original dataset_utils.py for backward compatibility @@ -98,6 +99,7 @@ __all__ = [ "TEMPLATE_TO_MODEL_MAPPER", "MODEL_TO_TEMPLATE_MAPPER", "TEMPLATE_TO_RESPONSES_MAPPER", + "is_gpt_oss_model_name", # Main entry points "check_dataset_format", "format_and_template_dataset", diff --git a/studio/backend/utils/datasets/model_mappings.py b/studio/backend/utils/datasets/model_mappings.py index 21e8566ac5..eb2e5482c9 100644 --- a/studio/backend/utils/datasets/model_mappings.py +++ b/studio/backend/utils/datasets/model_mappings.py @@ -442,6 +442,26 @@ for key, values in TEMPLATE_TO_MODEL_MAPPER.items(): MODEL_TO_TEMPLATE_MAPPER[value.lower()] = lowered_key +def is_gpt_oss_model_name(name: str) -> bool: + """Name-based check for gpt-oss / harmony models. + + Used by both the in-process backend and the parent-process + orchestrator to detect harmony models without an IPC round-trip. + """ + name = (name or "").lower() + if not name: + return False + try: + if MODEL_TO_TEMPLATE_MAPPER.get(name) == "gpt-oss": + return True + for key, tmpl in MODEL_TO_TEMPLATE_MAPPER.items(): + if tmpl == "gpt-oss" and (key in name or name in key): + return True + except Exception: + pass + return "gpt-oss" in name + + TEMPLATE_TO_RESPONSES_MAPPER = { "gemma-4-thinking": { "instruction": "<|turn>user\n", diff --git a/studio/frontend/src/app/auth-guards.ts b/studio/frontend/src/app/auth-guards.ts index 52230f0b6f..6849f380b8 100644 --- a/studio/frontend/src/app/auth-guards.ts +++ b/studio/frontend/src/app/auth-guards.ts @@ -9,6 +9,7 @@ import { hasRefreshToken, mustChangePassword, refreshSession, + setMustChangePassword, } from "@/features/auth"; async function hasActiveSession(): Promise { @@ -26,7 +27,12 @@ async function fetchAuthStatus(): Promise { try { const res = await fetch(apiUrl("/api/auth/status")); if (!res.ok) return { initialized: true, requires_password_change: mustChangePassword() }; - return (await res.json()) as AuthStatus; + const status = (await res.json()) as AuthStatus; + // Server truth wins; keep localStorage in sync both ways. + if (status.requires_password_change !== mustChangePassword()) { + setMustChangePassword(status.requires_password_change); + } + return status; } catch { return { initialized: true, requires_password_change: mustChangePassword() }; } @@ -61,6 +67,8 @@ export async function requireGuest(): Promise { throw redirect({ to: "/chat" }); } if (!(await hasActiveSession())) return; + // Reconcile localStorage before routing. + await fetchAuthStatus(); throw redirect({ to: getPostAuthRoute() }); } diff --git a/studio/frontend/src/app/routes/__root.tsx b/studio/frontend/src/app/routes/__root.tsx index 22d149473c..295f041129 100644 --- a/studio/frontend/src/app/routes/__root.tsx +++ b/studio/frontend/src/app/routes/__root.tsx @@ -15,9 +15,17 @@ import { useRouterState, } from "@tanstack/react-router"; import { AnimatePresence, motion } from "motion/react"; -import { Suspense, useEffect } from "react"; +import { Suspense, useEffect, type ReactNode } from "react"; import { AppProvider } from "../provider"; +// Fallback while a lazy route bundle (Train/Recipes/Export) loads. +// /chat is synchronous and never hits this. +const RouteFallback: ReactNode = ( +
+ Loading... +
+); + const CHAT_ONLY_ALLOWED = new Set([ "/", "/chat", @@ -72,7 +80,7 @@ function RootLayout() { {hideNavbar ? (
- +
@@ -98,7 +106,7 @@ function RootLayout() { transition={{ duration: 0.15 }} className={`flex min-h-0 min-w-0 flex-1 basis-0 flex-col ${isChatRoute ? "overflow-hidden" : "overflow-visible"}`} > - + diff --git a/studio/frontend/src/components/assistant-ui/attachment.tsx b/studio/frontend/src/components/assistant-ui/attachment.tsx index b5b2810008..3ae1c68561 100644 --- a/studio/frontend/src/components/assistant-ui/attachment.tsx +++ b/studio/frontend/src/components/assistant-ui/attachment.tsx @@ -120,12 +120,13 @@ const AttachmentPreviewDialog: FC = ({ children }) => { const AttachmentThumb: FC = () => { const src = useAttachmentSrc(); + const name = useAuiState(({ attachment }) => attachment.name); if (src) { return ( Attachment preview ); @@ -143,6 +144,7 @@ const AttachmentUI: FC = () => { const isComposer = aui.attachment.source === "composer"; const isImage = useAuiState(({ attachment }) => attachment.type === "image"); + const name = useAuiState(({ attachment }) => attachment.name); const typeLabel = useAuiState(({ attachment }) => { const type = attachment.type; switch (type) { @@ -156,6 +158,11 @@ const AttachmentUI: FC = () => { throw new Error(`Unknown attachment type: ${type as string}`); } }); + // Include filename in accessible name so screen readers distinguish + // same-typed attachments. Sighted users get it via the tooltip. + const accessibleName = name + ? `${typeLabel} attachment: ${name}` + : `${typeLabel} attachment`; return ( @@ -175,7 +182,7 @@ const AttachmentUI: FC = () => { "aui-attachment-tile-composer border-foreground/20", )} id="attachment-tile" - aria-label={`${typeLabel} attachment`} + aria-label={accessibleName} type="button" > diff --git a/studio/frontend/src/components/assistant-ui/message-timing.tsx b/studio/frontend/src/components/assistant-ui/message-timing.tsx index 5ad1bdabed..4fb68d4b90 100644 --- a/studio/frontend/src/components/assistant-ui/message-timing.tsx +++ b/studio/frontend/src/components/assistant-ui/message-timing.tsx @@ -38,9 +38,22 @@ export const MessageTiming: FC<{ )?.custom as { serverTimings?: Record } | undefined; const st = serverTimings?.serverTimings; + // Guard unphysical tok/s: llama.cpp emits predicted_ms=0 on no-op + // turns, blowing the rate up to Infinity. Require >=1 token AND a + // non-zero decode window AND a finite rate. Fast cached single-token + // responses (sub-10ms) are legitimate and must stay visible. + const hasPredicted = + (st?.predicted_n ?? 0) >= 1 && (st?.predicted_ms ?? 0) > 0; + const predictedRate = + hasPredicted && + st?.predicted_per_second != null && + Number.isFinite(st.predicted_per_second) + ? st.predicted_per_second + : undefined; + // Badge text: show tok/s if available, otherwise total time - const badgeText = st?.predicted_per_second != null - ? `${st.predicted_per_second.toFixed(1)} tok/s` + const badgeText = predictedRate != null + ? `${predictedRate.toFixed(1)} tok/s` : formatTimingMs(timing.totalStreamTime); return ( @@ -85,7 +98,7 @@ export const MessageTiming: FC<{ )} - {st?.predicted_ms != null && ( + {hasPredicted && st?.predicted_ms != null && (
Generation @@ -93,11 +106,11 @@ export const MessageTiming: FC<{
)} - {st?.predicted_per_second != null && ( + {predictedRate != null && (
Speed - {st.predicted_per_second.toFixed(1)} tok/s + {predictedRate.toFixed(1)} tok/s
)} diff --git a/studio/frontend/src/components/assistant-ui/thread.tsx b/studio/frontend/src/components/assistant-ui/thread.tsx index d72547aa1a..0326b90a97 100644 --- a/studio/frontend/src/components/assistant-ui/thread.tsx +++ b/studio/frontend/src/components/assistant-ui/thread.tsx @@ -916,6 +916,7 @@ const ComposerAction: FC<{ disabled?: boolean; blockSend?: () => boolean }> = ({ @@ -927,6 +928,7 @@ const ComposerAction: FC<{ disabled?: boolean; blockSend?: () => boolean }> = ({ @@ -994,6 +996,24 @@ const GeneratingIndicator: FC = () => { return Generating...; }; +// Placeholder when stop fires before any visible content (e.g. mid-think). +const CancelledIndicator: FC = () => { + const show = useAuiState( + ({ message }) => + message.content.length === 0 && + message.status?.type === "incomplete" && + message.status?.reason === "cancelled", + ); + if (!show) { + return null; + } + return ( + + Cancelled. + + ); +}; + const AssistantMessage: FC = () => { return ( { >
+ { const res = await fetch(apiUrl("/api/auth/status")); if (res.ok) { const data = (await res.json()) as { requires_password_change: boolean }; - if (data.requires_password_change || mustChangePassword()) target = "/change-password"; + // Server truth wins; keep localStorage in sync both ways. + if (data.requires_password_change !== mustChangePassword()) { + setMustChangePassword(data.requires_password_change); + } + if (data.requires_password_change) target = "/change-password"; } } catch { // Fall through to /login on error @@ -155,6 +159,14 @@ export async function authFetch( }); } catch (err) { if (err instanceof TypeError) { + // fetch TypeError = offline | backend down | CORS/DNS. In Tauri + // it's always backend-down; in the web build distinguish offline + // so the user gets the right recovery path. + if (!isTauri && typeof navigator !== "undefined" && navigator.onLine === false) { + throw new Error( + "You appear to be offline. Check your network connection and try again.", + ); + } throw new Error("Studio isn't running -- please relaunch it."); } throw err; diff --git a/studio/frontend/src/features/auth/components/auth-form.tsx b/studio/frontend/src/features/auth/components/auth-form.tsx index a10c77e9fa..3ac97929d8 100644 --- a/studio/frontend/src/features/auth/components/auth-form.tsx +++ b/studio/frontend/src/features/auth/components/auth-form.tsx @@ -105,12 +105,17 @@ export function AuthForm({ mode }: AuthFormProps): ReactElement | null { setInitialized(result.initialized); setRequiresPasswordChange(result.requires_password_change); + // Server truth wins; keep localStorage in sync both ways. + if (result.requires_password_change !== mustChangePassword()) { + setMustChangePassword(result.requires_password_change); + } + // Redirect between login ↔ change-password based on server state if (mode === "login" && result.requires_password_change) { navigate({ to: "/change-password" }); return; } - if (mode === "change-password" && !result.requires_password_change && !mustChangePassword()) { + if (mode === "change-password" && !result.requires_password_change) { navigate({ to: "/login" }); return; } @@ -163,14 +168,14 @@ export function AuthForm({ mode }: AuthFormProps): ReactElement | null { const blockedByState = initialized === false || (mode === "login" && requiresPasswordChange) || - (mode === "change-password" && !requiresPasswordChange && !mustChangePassword()); + (mode === "change-password" && !requiresPasswordChange); let helperText: string | null = null; if (initialized === false) { helperText = "Auth is still bootstrapping the default admin account."; } else if (isLoginMode && requiresPasswordChange) { helperText = "Sign in once with the seeded credentials to change the password."; - } else if (!isLoginMode && !requiresPasswordChange && !mustChangePassword()) { + } else if (!isLoginMode && !requiresPasswordChange) { helperText = "Password already updated. Use the login screen."; } const title = isLoginMode ? "Welcome back" : "Setup your account"; diff --git a/studio/frontend/src/features/chat/api/chat-adapter.ts b/studio/frontend/src/features/chat/api/chat-adapter.ts index 61d71b641a..359099e8b3 100644 --- a/studio/frontend/src/features/chat/api/chat-adapter.ts +++ b/studio/frontend/src/features/chat/api/chat-adapter.ts @@ -437,6 +437,9 @@ function waitForModelReady(abortSignal?: AbortSignal): Promise { * without selecting one. Prefers GGUF (picks smallest cached variant), * falls back to smallest cached safetensors model. */ +// Cap cascade so broken cached repos can't spam /api/inference/load. +const MAX_AUTO_LOAD_ATTEMPTS = 3; + async function autoLoadSmallestModel(): Promise<{ loaded: boolean; blockedByTrustRemoteCode: boolean; @@ -451,6 +454,7 @@ async function autoLoadSmallestModel(): Promise<{ }); let blockedByTrustRemoteCode = false; let hadNonTrustFailure = false; + let loadAttempts = 0; async function canAutoLoad(payload: { model_path: string; @@ -481,6 +485,7 @@ async function autoLoadSmallestModel(): Promise<{ if (ggufRepos.length > 0) { const sorted = [...ggufRepos].sort((a, b) => a.size_bytes - b.size_bytes); for (const repo of sorted) { + if (loadAttempts >= MAX_AUTO_LOAD_ATTEMPTS) break; try { const variants = await listGgufVariants(repo.repo_id); const downloaded = variants.variants @@ -498,6 +503,7 @@ async function autoLoadSmallestModel(): Promise<{ ) { continue; } + loadAttempts += 1; const loadResp = await loadModel({ model_path: repo.repo_id, hf_token: hfToken, @@ -560,6 +566,7 @@ async function autoLoadSmallestModel(): Promise<{ if (modelRepos.length > 0) { const sorted = [...modelRepos].sort((a, b) => a.size_bytes - b.size_bytes); for (const repo of sorted) { + if (loadAttempts >= MAX_AUTO_LOAD_ATTEMPTS) break; try { if ( !(await canAutoLoad({ @@ -571,6 +578,7 @@ async function autoLoadSmallestModel(): Promise<{ ) { continue; } + loadAttempts += 1; const sfLoadResp = await loadModel({ model_path: repo.repo_id, hf_token: hfToken, @@ -593,6 +601,12 @@ async function autoLoadSmallestModel(): Promise<{ reasoningStyle: sfLoadResp.reasoning_style ?? "enable_thinking", supportsPreserveThinking: sfLoadResp.supports_preserve_thinking ?? false, supportsTools: sfLoadResp.supports_tools ?? false, + // Parity with the GGUF branch above. + toolsEnabled: sfLoadResp.supports_tools ?? false, + codeToolsEnabled: sfLoadResp.supports_tools ?? false, + defaultChatTemplate: sfLoadResp.chat_template ?? null, + chatTemplateOverride: null, + loadedChatTemplateOverride: null, }); const sfModel: ChatModelSummary = { id: repo.repo_id, @@ -616,6 +630,17 @@ async function autoLoadSmallestModel(): Promise<{ } } + // Cap also gates the default download so the total /api/inference/load + // budget across cached + fallback is MAX_AUTO_LOAD_ATTEMPTS, not +1. + if (loadAttempts >= MAX_AUTO_LOAD_ATTEMPTS) { + toast.dismiss(toastId); + return { + loaded: false, + blockedByTrustRemoteCode: + blockedByTrustRemoteCode && !hadNonTrustFailure, + }; + } + // No cached models found — try downloading a small default GGUF toast("Downloading a small model…", { id: toastId, @@ -634,6 +659,7 @@ async function autoLoadSmallestModel(): Promise<{ toast.dismiss(toastId); return { loaded: false, blockedByTrustRemoteCode }; } + loadAttempts += 1; const loadResp = await loadModel({ model_path: "unsloth/gemma-4-E2B-it-GGUF", hf_token: hfToken, diff --git a/studio/frontend/src/features/chat/chat-settings-sheet.tsx b/studio/frontend/src/features/chat/chat-settings-sheet.tsx index 3703beff0a..1ed0a88b63 100644 --- a/studio/frontend/src/features/chat/chat-settings-sheet.tsx +++ b/studio/frontend/src/features/chat/chat-settings-sheet.tsx @@ -570,6 +570,11 @@ export function ChatSettingsPanel({ const loadedSpeculativeType = useChatRuntimeStore( (s) => s.loadedSpeculativeType, ); + const specDraftNMax = useChatRuntimeStore((s) => s.specDraftNMax); + const setSpecDraftNMax = useChatRuntimeStore((s) => s.setSpecDraftNMax); + const loadedSpecDraftNMax = useChatRuntimeStore( + (s) => s.loadedSpecDraftNMax, + ); const modelRequiresTrustRemoteCode = useChatRuntimeStore( (s) => s.modelRequiresTrustRemoteCode, ); @@ -608,7 +613,8 @@ export function ChatSettingsPanel({ const kvDirty = kvCacheDtype !== loadedKvCacheDtype; const ctxDirty = customContextLength !== null; const specDirty = speculativeType !== loadedSpeculativeType; - const modelSettingsDirty = kvDirty || ctxDirty || specDirty; + const specDraftDirty = specDraftNMax !== loadedSpecDraftNMax; + const modelSettingsDirty = kvDirty || ctxDirty || specDirty || specDraftDirty; const chatTemplateOverride = useChatRuntimeStore( (s) => s.chatTemplateOverride, ); @@ -985,19 +991,81 @@ export function ChatSettingsPanel({ Speculative Decoding - Faster generation with 0% accuracy hit. + Faster generation with 0% accuracy hit. Auto picks + MTP / ngram-mod based on the model and platform. + Pick MTP, Ngram, or MTP+Ngram to force a specific + strategy on both GPU and CPU.
- { - setSpeculativeType(checked ? "default" : "off"); - }} - /> +
+ +
+ {(speculativeType === "mtp" || + speculativeType === "mtp+ngram") && ( +
+
+ + Draft Tokens + + + Max MTP draft tokens per step + (--spec-draft-n-max). Lower = less wasted + draft decode; higher = bigger speedup when + acceptance stays high. Default: 2 on GPU, + 3 on CPU/Mac. + +
+ { + const raw = e.target.value; + if (raw === "") { + setSpecDraftNMax(null); + return; + } + const parsed = Number.parseInt(raw, 10); + if (Number.isFinite(parsed)) { + const clamped = Math.max(1, Math.min(16, parsed)); + setSpecDraftNMax(clamped); + } + }} + data-test-id="spec-draft-n-max-input" + aria-label="Speculative decoding draft tokens" + className="h-7 w-[72px] rounded-[10px] border-transparent bg-black/[0.04] dark:bg-white/[0.05] hover:bg-black/[0.06] dark:hover:bg-white/[0.07] px-2 py-0 text-[13px] font-medium text-nav-fg outline-none focus-visible:ring-0" + /> +
+ )} )} {!isGguf && params.checkpoint && ( @@ -1051,6 +1119,7 @@ export function ChatSettingsPanel({ setCustomContextLength(null); setKvCacheDtype(loadedKvCacheDtype); setSpeculativeType(loadedSpeculativeType); + setSpecDraftNMax(loadedSpecDraftNMax); setChatTemplateOverride(loadedChatTemplateOverride); }} className="h-7 px-3 text-[12px] font-medium tracking-nav text-muted-foreground" diff --git a/studio/frontend/src/features/chat/hooks/use-chat-model-runtime.ts b/studio/frontend/src/features/chat/hooks/use-chat-model-runtime.ts index 03f80c19f2..3f1060edf7 100644 --- a/studio/frontend/src/features/chat/hooks/use-chat-model-runtime.ts +++ b/studio/frontend/src/features/chat/hooks/use-chat-model-runtime.ts @@ -141,10 +141,30 @@ function getTrustRemoteCodeRequiredMessage(modelName: string): string { return `${modelName} needs custom code enabled to load. Turn on "Enable custom code" in Chat Settings, then try again.`; } +// Canonicalises any value the backend reports (or persisted state holds) +// onto the five UI-facing modes the Speculative Decoding dropdown +// understands: "auto" / "mtp" / "ngram" / "mtp+ngram" / "off" / null. +// Mirrors backend _canonicalize_spec_mode so old persisted "default" / +// "draft-mtp" / "ngram-mod" / chain values round-trip cleanly. function normalizeSpeculativeType(v: string | null | undefined): string | null { if (v == null) return null; - if (v === "default" || v === "off") return v; - return "default"; + const s = String(v).trim().toLowerCase(); + if (!s) return null; + if (s === "auto" || s === "default") return "auto"; + if (s === "off") return "off"; + if (s === "ngram-simple") return "ngram-simple"; + if (s === "mtp" || s === "draft-mtp") return "mtp"; + if (s === "ngram" || s === "ngram-mod") return "ngram"; + if (s === "mtp+ngram") return "mtp+ngram"; + // Comma-chained legacy values (e.g. from older persisted state). + const parts = s.split(",").map((p) => p.trim()).filter(Boolean); + const hasMtp = parts.some((p) => p === "mtp" || p === "draft-mtp"); + const hasNgram = parts.some((p) => p === "ngram" || p === "ngram-mod"); + if (hasMtp && hasNgram) return "mtp+ngram"; + if (hasMtp) return "mtp"; + if (hasNgram) return "ngram"; + // Unknown -> safe fallback to Auto so the dropdown stays controlled. + return "auto"; } type LocalReasoningEffort = Extract; @@ -323,6 +343,12 @@ export function useChatModelRuntime() { speculativeType: currentSpecType, loadedSpeculativeType: currentSpecType, }), + ...(statusRes.spec_draft_n_max !== undefined && + prevState.loadedSpecDraftNMax === null && + prevState.specDraftNMax === null && { + specDraftNMax: statusRes.spec_draft_n_max ?? null, + loadedSpecDraftNMax: statusRes.spec_draft_n_max ?? null, + }), ...(statusRes.cache_type_kv !== undefined && prevState.loadedKvCacheDtype === null && { kvCacheDtype: statusRes.cache_type_kv, @@ -528,12 +554,31 @@ export function useChatModelRuntime() { } if (abortCtrl.signal.aborted) throw new Error("Cancelled"); + // Reset Speculative Decoding to Auto whenever the user + // switches to a different model. Spec strategy is a + // per-model decision: a sub-3B non-MTP GGUF that ran with + // "Off" should not carry that choice into a 27B MTP GGUF + // where Auto would auto-promote to draft-mtp. The user can + // still pick a forced mode on the new model; this just + // clears the stale prior-model choice so the backend's + // platform-aware path runs by default. Same applies to + // spec_draft_n_max which is MTP-only. + if (currentCheckpoint && currentCheckpoint !== modelId) { + useChatRuntimeStore.setState({ + speculativeType: null, + loadedSpeculativeType: null, + specDraftNMax: null, + loadedSpecDraftNMax: null, + }); + } + const { chatTemplateOverride, kvCacheDtype, customContextLength, ggufContextLength, speculativeType, + specDraftNMax, activePresetSource, activeGgufVariant, } = useChatRuntimeStore.getState(); @@ -561,6 +606,7 @@ export function useChatModelRuntime() { chat_template_override: effectiveChatTemplateOverride, cache_type_kv: kvCacheDtype, speculative_type: speculativeType, + spec_draft_n_max: specDraftNMax, }); // If cancelled while loading, don't update UI to show @@ -635,6 +681,8 @@ export function useChatModelRuntime() { loadedKvCacheDtype: loadedKv, speculativeType: loadedSpec, loadedSpeculativeType: loadedSpec, + specDraftNMax: loadResponse.spec_draft_n_max ?? null, + loadedSpecDraftNMax: loadResponse.spec_draft_n_max ?? null, customContextLength: keepCustomCtx, defaultChatTemplate: loadResponse.chat_template ?? null, chatTemplateOverride: effectiveChatTemplateOverride, diff --git a/studio/frontend/src/features/chat/shared-composer.tsx b/studio/frontend/src/features/chat/shared-composer.tsx index aef004e891..11a73ee486 100644 --- a/studio/frontend/src/features/chat/shared-composer.tsx +++ b/studio/frontend/src/features/chat/shared-composer.tsx @@ -523,7 +523,19 @@ export function SharedComposer({ handlesRef.current["model1"] || handlesRef.current["model2"], ); const isGeneralizedCompare = - hasCompareHandles && Boolean(model1?.id || model2?.id); + hasCompareHandles && Boolean(model1?.id && model2?.id); + + // Generalized compare requires both panes to have a model. A + // half-selected send either races to an empty bubble with bogus + // tok/s (#5569) or leaves the empty pane with a dangling prompt. + // hasCompareHandles is true only in GeneralCompareContent, so + // LoraCompare and single-pane chats are unaffected. + if (hasCompareHandles && !isGeneralizedCompare) { + toast.error("Pick a model in each pane to compare", { + description: "Use the model dropdown above each pane, then send your prompt.", + }); + return; + } if (pendingImages.length > 0 && !isGeneralizedCompare && imageUnavailableReason) { // Single mode: the loaded model's runtime capability is known diff --git a/studio/frontend/src/features/chat/stores/chat-runtime-store.ts b/studio/frontend/src/features/chat/stores/chat-runtime-store.ts index 13d3cb533f..e55a25d08d 100644 --- a/studio/frontend/src/features/chat/stores/chat-runtime-store.ts +++ b/studio/frontend/src/features/chat/stores/chat-runtime-store.ts @@ -258,6 +258,9 @@ type ChatRuntimeStore = { loadedKvCacheDtype: string | null; speculativeType: string | null; loadedSpeculativeType: string | null; + /** User --spec-draft-n-max override (null = platform default). */ + specDraftNMax: number | null; + loadedSpecDraftNMax: number | null; loadedIsMultimodal: boolean; customContextLength: number | null; defaultChatTemplate: string | null; @@ -305,6 +308,7 @@ type ChatRuntimeStore = { setToolCallTimeout: (value: number) => void; setKvCacheDtype: (dtype: string | null) => void; setSpeculativeType: (type: string | null) => void; + setSpecDraftNMax: (value: number | null) => void; setCustomContextLength: (v: number | null) => void; setChatTemplateOverride: (template: string | null) => void; setPendingAudio: (base64: string, name: string) => void; @@ -349,8 +353,10 @@ export const useChatRuntimeStore = create((set) => ({ toolCallTimeout: loadInt(TOOL_CALL_TIMEOUT_KEY, 5), kvCacheDtype: null, loadedKvCacheDtype: null, - speculativeType: "default", + speculativeType: "auto", loadedSpeculativeType: null, + specDraftNMax: null, + loadedSpecDraftNMax: null, loadedIsMultimodal: false, customContextLength: null, defaultChatTemplate: null, @@ -457,8 +463,10 @@ export const useChatRuntimeStore = create((set) => ({ toolStatus: null, kvCacheDtype: null, loadedKvCacheDtype: null, - speculativeType: "default", + speculativeType: "auto", loadedSpeculativeType: null, + specDraftNMax: null, + loadedSpecDraftNMax: null, loadedIsMultimodal: false, customContextLength: null, defaultChatTemplate: null, @@ -506,6 +514,7 @@ export const useChatRuntimeStore = create((set) => ({ }), setKvCacheDtype: (kvCacheDtype) => set({ kvCacheDtype }), setSpeculativeType: (speculativeType) => set({ speculativeType }), + setSpecDraftNMax: (specDraftNMax) => set({ specDraftNMax }), setCustomContextLength: (customContextLength) => set({ customContextLength }), setChatTemplateOverride: (chatTemplateOverride) => set({ chatTemplateOverride }), setPendingAudio: (base64, name) => diff --git a/studio/frontend/src/features/chat/types/api.ts b/studio/frontend/src/features/chat/types/api.ts index 3fc5d320df..1e6bcf8b87 100644 --- a/studio/frontend/src/features/chat/types/api.ts +++ b/studio/frontend/src/features/chat/types/api.ts @@ -42,7 +42,21 @@ export interface LoadModelRequest { trust_remote_code?: boolean; chat_template_override?: string | null; cache_type_kv?: string | null; + /** + * Speculative decoding mode for GGUF models. Canonical values: + * "auto" (platform-aware: MTP on MTP GGUFs, ngram-mod fallback for + * sub-3B), "mtp" (force draft-mtp only on both GPU and CPU), "ngram" + * (force ngram-mod only), "mtp+ngram" (force ngram-mod + draft-mtp + * chain on both platforms), or "off". Legacy values "default" / + * "draft-mtp" / "ngram-mod" / "ngram-simple" are still accepted by + * the backend. + */ speculative_type?: string | null; + /** + * Override --spec-draft-n-max for MTP speculative decoding. Only + * applied when speculative_type resolves to "mtp" or "mtp+ngram". + */ + spec_draft_n_max?: number | null; } export interface ValidateModelResponse { @@ -118,7 +132,9 @@ export interface LoadModelResponse { supports_tools?: boolean; cache_type_kv?: string | null; chat_template?: string | null; + /** Canonical UI-facing mode the load request resolved to. See LoadModelRequest. */ speculative_type?: string | null; + spec_draft_n_max?: number | null; } export interface UnloadModelRequest { @@ -155,7 +171,9 @@ export interface InferenceStatusResponse { native_context_length?: number | null; cache_type_kv?: string | null; chat_template_override?: string | null; + /** Canonical UI-facing mode currently active. See LoadModelRequest. */ speculative_type?: string | null; + spec_draft_n_max?: number | null; } export interface AudioGenerationResponse { diff --git a/studio/frontend/src/features/settings/settings-dialog.tsx b/studio/frontend/src/features/settings/settings-dialog.tsx index 1ece504d6f..3b95cb2355 100644 --- a/studio/frontend/src/features/settings/settings-dialog.tsx +++ b/studio/frontend/src/features/settings/settings-dialog.tsx @@ -74,6 +74,7 @@ export function SettingsDialog() { const activeTab = useSettingsDialogStore((s) => s.activeTab); const setActiveTab = useSettingsDialogStore((s) => s.setActiveTab); const closeDialog = useSettingsDialogStore((s) => s.closeDialog); + const opener = useSettingsDialogStore((s) => s.opener); const reduced = useReducedMotion(); const tabButtonRefs = useRef>({ general: null, @@ -98,11 +99,22 @@ export function SettingsDialog() { { + // Restore focus to the element that triggered openDialog(). + // Radix's FocusScope races our rAF-scheduled tab-button focus + // and loses the previous-focus reference, so we restore by hand. + if (opener && opener.isConnected) { + e.preventDefault(); + opener.focus({ preventScroll: true }); + } + }} className={cn( - "!max-w-none h-[560px] w-[820px] p-0 overflow-hidden", + // Cap at 820px but shrink to the viewport so we don't clip + // on iPad-portrait widths (640-820px) where the fixed + // `w-[820px]` overflows by 26px on each side. + "!max-w-[min(820px,calc(100vw-2rem))] h-[560px] w-[min(820px,calc(100vw-2rem))] p-0 overflow-hidden", "shadow-border rounded-xl border-border", - "sm:h-[560px] sm:w-[820px]", - "max-sm:h-dvh max-sm:w-dvw max-sm:rounded-none", + "max-sm:h-dvh max-sm:w-dvw max-sm:!max-w-none max-sm:rounded-none", )} > Settings @@ -110,7 +122,7 @@ export function SettingsDialog() { Manage your Unsloth Studio preferences.
-