From 1c77b4d1496fe5d7f26bf624315acc682ef5cd6e Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Mon, 20 Jul 2026 07:17:31 -0700 Subject: [PATCH 001/128] Bump install.sh / install.ps1 pin to unsloth>=2026.7.4 (#7263) PyPI release unsloth 2026.7.4 is live; bump the pinned floor so fresh installs resolve to the new wheel. --- install.ps1 | 10 +++++----- install.sh | 10 +++++----- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/install.ps1 b/install.ps1 index 6e059ee0dd..a525d4df56 100644 --- a/install.ps1 +++ b/install.ps1 @@ -2266,7 +2266,7 @@ exit 0 if ($SkipTorch) { # No-torch: install unsloth + unsloth-zoo with --no-deps, then # runtime deps (typer, safetensors, transformers, etc.) with --no-deps. - $baseInstallExit = Invoke-InstallCommandRetry -Label "install unsloth (migrated no-torch)" { uv pip install --python $VenvPython --no-deps --reinstall-package unsloth --reinstall-package unsloth-zoo "unsloth>=2026.7.3" "unsloth-zoo>=2026.7.3" } + $baseInstallExit = Invoke-InstallCommandRetry -Label "install unsloth (migrated no-torch)" { uv pip install --python $VenvPython --no-deps --reinstall-package unsloth --reinstall-package unsloth-zoo "unsloth>=2026.7.4" "unsloth-zoo>=2026.7.4" } if ($baseInstallExit -eq 0) { # Resolve pydantic WITH deps so pip pins pydantic-core # to the matching version (no-torch-runtime.txt below @@ -2280,7 +2280,7 @@ exit 0 } } } else { - $baseInstallExit = Invoke-InstallCommandRetry -Label "install unsloth (migrated)" { uv pip install --python $VenvPython --reinstall-package unsloth --reinstall-package unsloth-zoo "unsloth>=2026.7.3" "unsloth-zoo>=2026.7.3" } + $baseInstallExit = Invoke-InstallCommandRetry -Label "install unsloth (migrated)" { uv pip install --python $VenvPython --reinstall-package unsloth --reinstall-package unsloth-zoo "unsloth>=2026.7.4" "unsloth-zoo>=2026.7.4" } } if ($baseInstallExit -ne 0) { Write-Host "[ERROR] Failed to install unsloth (exit code $baseInstallExit)" -ForegroundColor Red @@ -2354,7 +2354,7 @@ exit 0 if ($SkipTorch) { # No-torch: install unsloth + unsloth-zoo with --no-deps, then # runtime deps (typer, safetensors, transformers, etc.) with --no-deps. - $baseInstallExit = Invoke-InstallCommandRetry -Label "install unsloth (no-torch)" { uv pip install --python $VenvPython --no-deps --upgrade-package unsloth --upgrade-package unsloth-zoo "unsloth>=2026.7.3" "unsloth-zoo>=2026.7.3" } + $baseInstallExit = Invoke-InstallCommandRetry -Label "install unsloth (no-torch)" { uv pip install --python $VenvPython --no-deps --upgrade-package unsloth --upgrade-package unsloth-zoo "unsloth>=2026.7.4" "unsloth-zoo>=2026.7.4" } if ($baseInstallExit -eq 0) { # Same pydantic-with-deps trick as the migrated branch. $baseInstallExit = Invoke-InstallCommandRetry -Label "install pydantic" { uv pip install --python $VenvPython pydantic } @@ -2366,7 +2366,7 @@ exit 0 } } } elseif ($StudioLocalInstall) { - $baseInstallExit = Invoke-InstallCommandRetry -Label "install unsloth (local)" { uv pip install --python $VenvPython --upgrade-package unsloth "unsloth>=2026.7.3" "unsloth-zoo>=2026.7.3" } + $baseInstallExit = Invoke-InstallCommandRetry -Label "install unsloth (local)" { uv pip install --python $VenvPython --upgrade-package unsloth "unsloth>=2026.7.4" "unsloth-zoo>=2026.7.4" } } else { $baseInstallExit = Invoke-InstallCommandRetry -Label "install unsloth" { uv pip install --python $VenvPython --upgrade-package unsloth -- "$PackageName" } } @@ -2394,7 +2394,7 @@ exit 0 Write-TauriLog "STEP" "Installing unsloth" substep "installing unsloth (this may take a few minutes)..." if ($StudioLocalInstall) { - $baseInstallExit = Invoke-InstallCommandRetry -Label "install unsloth (auto torch backend)" { uv pip install --python $VenvPython "unsloth-zoo>=2026.7.3" "unsloth>=2026.7.3" --torch-backend=auto } + $baseInstallExit = Invoke-InstallCommandRetry -Label "install unsloth (auto torch backend)" { uv pip install --python $VenvPython "unsloth-zoo>=2026.7.4" "unsloth>=2026.7.4" --torch-backend=auto } if ($baseInstallExit -ne 0) { Write-Host "[ERROR] Failed to install unsloth (exit code $baseInstallExit)" -ForegroundColor Red return (Exit-InstallFailure "Failed to install unsloth (exit code $baseInstallExit)" $baseInstallExit) diff --git a/install.sh b/install.sh index c02552628f..0acccdf049 100755 --- a/install.sh +++ b/install.sh @@ -3096,7 +3096,7 @@ if [ "$_MIGRATED" = true ]; then # to prevent transitive torch resolution. run_install_cmd_retry "install unsloth (migrated no-torch)" uv pip install --python "$_VENV_PY" --no-deps \ --reinstall-package unsloth --reinstall-package unsloth-zoo \ - "unsloth>=2026.7.3" "unsloth-zoo>=2026.7.3" + "unsloth>=2026.7.4" "unsloth-zoo>=2026.7.4" # Resolve pydantic WITH deps so pip pins pydantic-core to the # matching version (no-torch-runtime.txt below is --no-deps). # All transitive deps are torch-free. @@ -3113,7 +3113,7 @@ if [ "$_MIGRATED" = true ]; then run_install_cmd_retry "install unsloth (migrated)" uv pip install --python "$_VENV_PY" \ ${_UNSLOTH_TORCH_OVERRIDES:+--overrides "$_UNSLOTH_TORCH_OVERRIDES"} \ --reinstall-package unsloth --reinstall-package unsloth-zoo \ - "unsloth>=2026.7.3" "unsloth-zoo>=2026.7.3" ${_MLX_LM_EXCLUDE_ARG:-} + "unsloth>=2026.7.4" "unsloth-zoo>=2026.7.4" ${_MLX_LM_EXCLUDE_ARG:-} [ -n "$_UNSLOTH_TORCH_OVERRIDES" ] && rm -f "$_UNSLOTH_TORCH_OVERRIDES" _UNSLOTH_TORCH_OVERRIDES="" fi @@ -3337,7 +3337,7 @@ elif [ -n "$TORCH_INDEX_URL" ]; then # runtime deps (typer, safetensors, transformers, etc.) with --no-deps. run_install_cmd_retry "install unsloth (no-torch)" uv pip install --python "$_VENV_PY" --no-deps \ --upgrade-package unsloth --upgrade-package unsloth-zoo \ - "unsloth>=2026.7.3" "unsloth-zoo>=2026.7.3" + "unsloth>=2026.7.4" "unsloth-zoo>=2026.7.4" # Same pydantic-with-deps trick as the migrated branch. run_install_cmd_retry "install pydantic (with deps for compatible core)" \ uv pip install --python "$_VENV_PY" pydantic @@ -3356,7 +3356,7 @@ elif [ -n "$TORCH_INDEX_URL" ]; then elif [ "$STUDIO_LOCAL_INSTALL" = true ]; then run_install_cmd_retry "install unsloth (local)" uv pip install --python "$_VENV_PY" \ ${_UNSLOTH_TORCH_OVERRIDES:+--overrides "$_UNSLOTH_TORCH_OVERRIDES"} \ - --upgrade-package unsloth "unsloth>=2026.7.3" "unsloth-zoo>=2026.7.3" + --upgrade-package unsloth "unsloth>=2026.7.4" "unsloth-zoo>=2026.7.4" substep "overlaying local repo (editable)..." run_install_cmd "overlay local repo" uv pip install --python "$_VENV_PY" -e "$_REPO_ROOT" --no-deps substep "overlaying unsloth-zoo from git main..." @@ -3384,7 +3384,7 @@ else tauri_log "STEP" "Installing Unsloth" substep "installing unsloth (this may take a few minutes)..." if [ "$STUDIO_LOCAL_INSTALL" = true ]; then - run_install_cmd_retry "install unsloth (auto torch backend)" uv pip install --python "$_VENV_PY" "unsloth-zoo>=2026.7.3" "unsloth>=2026.7.3" --torch-backend=auto + run_install_cmd_retry "install unsloth (auto torch backend)" uv pip install --python "$_VENV_PY" "unsloth-zoo>=2026.7.4" "unsloth>=2026.7.4" --torch-backend=auto substep "overlaying local repo (editable)..." run_install_cmd "overlay local repo" uv pip install --python "$_VENV_PY" -e "$_REPO_ROOT" --no-deps substep "overlaying unsloth-zoo from git main..." From c9d479f9e36103ae17a55b21cb561c10f1a474ba Mon Sep 17 00:00:00 2001 From: oobabooga Date: Mon, 20 Jul 2026 20:55:42 -0300 Subject: [PATCH 002/128] Studio: don't let a malformed HF token empty the model picker's Recommended list (#7266) --- .../src/components/assistant-ui/model-selector/pickers.tsx | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/studio/frontend/src/components/assistant-ui/model-selector/pickers.tsx b/studio/frontend/src/components/assistant-ui/model-selector/pickers.tsx index 766139e2b4..84119cc992 100644 --- a/studio/frontend/src/components/assistant-ui/model-selector/pickers.tsx +++ b/studio/frontend/src/components/assistant-ui/model-selector/pickers.tsx @@ -47,7 +47,7 @@ import { import { useOnlineStatus } from "@/features/hub/hooks/use-online-status"; import { isHiddenModelId } from "@/features/hub/lib/hidden-models"; import { classifyUnslothSupport } from "@/features/hub/lib/unsloth-support"; -import { useHfTokenStore } from "@/features/hub/stores/hf-token-store"; +import { hfApiToken, useHfTokenStore } from "@/features/hub/stores/hf-token-store"; import { downloadManager, jobKeyOf, @@ -1411,7 +1411,8 @@ export function HubModelPicker({ // Shared Hub search stack (the same hooks the Hub page uses) so the picker // and Hub run one implementation. Scoped to unsloth like the old listing. const online = useOnlineStatus(); - const accessToken = hfToken || undefined; + // Sanitize to anonymous on a malformed token, matching the Hub page. + const accessToken = hfApiToken(hfToken); // Recommended section: a live unsloth listing sorted by the dropdown. The // same sort drives the search results so the dropdown works while searching. const [recommendedSort, setRecommendedSort] = From 3d379cdb81ea6b1688eee4812ff5a6e1e85c2c74 Mon Sep 17 00:00:00 2001 From: Long Yixing Date: Tue, 21 Jul 2026 10:14:58 +0800 Subject: [PATCH 003/128] Fix local CLI streamed generation error handling (#7135) --- studio/backend/core/inference/orchestrator.py | 5 +- unsloth_cli/_inference.py | 2 +- unsloth_cli/commands/chat.py | 2 +- unsloth_cli/commands/inference.py | 5 +- unsloth_cli/tests/test_inference_chat.py | 98 +++++++++++++++++++ 5 files changed, 103 insertions(+), 9 deletions(-) diff --git a/studio/backend/core/inference/orchestrator.py b/studio/backend/core/inference/orchestrator.py index eaa474d9b8..75ef9c2399 100644 --- a/studio/backend/core/inference/orchestrator.py +++ b/studio/backend/core/inference/orchestrator.py @@ -54,9 +54,8 @@ class GenStreamError(str): """A stream chunk carrying a real backend/generation error, not model text. Subclasses str so existing display/logging consumers are unaffected, while - callers that must abort a distributed run on error (raise_on_streamed_error) - can distinguish a real error from model output whose visible text starts with - "Error:" by checking isinstance(chunk, GenStreamError). + callers can distinguish a real error from model output whose visible text + starts with "Error:" by checking isinstance(chunk, GenStreamError). """ __slots__ = ("public",) diff --git a/unsloth_cli/_inference.py b/unsloth_cli/_inference.py index 551bef4787..a2b0f9c04f 100644 --- a/unsloth_cli/_inference.py +++ b/unsloth_cli/_inference.py @@ -235,7 +235,7 @@ def collect_stream(stream, show_thinking: bool) -> str: def raise_on_streamed_error(stream): # Match real backend errors by type (GenStreamError), not the "Error:" text # prefix, so a completion whose text opens with "Error:" is not misread as a - # failure that aborts a distributed run. + # backend failure. try: ensure_studio_backend_path() from core.inference.orchestrator import GenStreamError diff --git a/unsloth_cli/commands/chat.py b/unsloth_cli/commands/chat.py index bba5fab08e..fdc5577700 100644 --- a/unsloth_cli/commands/chat.py +++ b/unsloth_cli/commands/chat.py @@ -331,7 +331,7 @@ def chat( enable_thinking = show_thinking, use_adapter = use_adapter, ) - return raise_on_streamed_error(stream) if is_mlx_distributed else stream + return raise_on_streamed_error(stream) if should_print: console.print() diff --git a/unsloth_cli/commands/inference.py b/unsloth_cli/commands/inference.py index 524d8fd015..7df05cef94 100644 --- a/unsloth_cli/commands/inference.py +++ b/unsloth_cli/commands/inference.py @@ -111,15 +111,12 @@ def inference( repetition_penalty = repetition_penalty, enable_thinking = think, ) - if is_mlx_distributed: - stream = raise_on_streamed_error(stream) + stream = raise_on_streamed_error(stream) if rank == 0: typer.echo("Assistant:") try: stream_to_stdout(stream, show_thinking = think) except RuntimeError as exc: - if not is_mlx_distributed: - raise typer.echo(f"Error: {exc}", err = True) raise typer.Exit(code = 1) else: diff --git a/unsloth_cli/tests/test_inference_chat.py b/unsloth_cli/tests/test_inference_chat.py index ae6f8dcfd4..07eed01e97 100644 --- a/unsloth_cli/tests/test_inference_chat.py +++ b/unsloth_cli/tests/test_inference_chat.py @@ -873,6 +873,104 @@ def test_chat_compare_on_mlx_loads_base_model_side_by_side(monkeypatch): assert set(closed) == {"tuned", "base"} +@pytest.mark.parametrize( + ("chunk_kind", "expected_exit"), + [ + ("answer", 0), + ("model_text_error", 0), + ("real_error", 1), + ], +) +def test_inference_local_handles_stream(monkeypatch, chunk_kind, expected_exit): + from unsloth_cli.commands import inference as infermod + from unsloth_cli._inference import ensure_studio_backend_path + + ensure_studio_backend_path() + from core.inference.orchestrator import GenStreamError + + chunks = { + "answer": ["answer"], + "model_text_error": ["Error: printed by the model, not a backend failure"], + "real_error": [GenStreamError("Error: generation failed")], + }[chunk_kind] + closed = [] + + class _FakeBackend: + def stream(self, messages, **kwargs): + return iter(chunks) + + def close(self): + closed.append(True) + + monkeypatch.setattr( + infermod, + "connect_studio_server", + lambda *_a, **_k: (_ for _ in ()).throw(AssertionError("server disabled")), + ) + monkeypatch.setattr(infermod, "load_chat_backend", lambda *a, **k: _FakeBackend()) + + result = CliRunner().invoke( + _inference_app(), + ["fake-model", "hello", "--no-server"], + ) + + assert result.exit_code == expected_exit, result.output + assert closed == [True] + if chunk_kind == "real_error": + assert result.stdout == "Assistant:\n" + assert result.stderr == "Error: generation failed\n" + else: + assert chunks[0] in result.output + + +@pytest.mark.parametrize("chunk_kind", ["answer", "model_text_error", "real_error"]) +def test_chat_local_handles_stream(monkeypatch, chunk_kind): + from unsloth_cli._inference import ensure_studio_backend_path + + ensure_studio_backend_path() + from core.inference.orchestrator import GenStreamError + + first_chunk = { + "answer": "answer", + "model_text_error": "Error: printed by the model, not a backend failure", + "real_error": GenStreamError("Error: generation failed"), + }[chunk_kind] + calls, closed = [], [] + + class _FakeChatBackend: + def stream(self, messages, **kwargs): + calls.append([dict(message) for message in messages]) + return iter([first_chunk if len(calls) == 1 else "second answer"]) + + def close(self): + closed.append(True) + + monkeypatch.setattr(chatmod, "resolve_model_config", lambda *a, **k: _FakeConfig()) + monkeypatch.setattr(chatmod, "connect_studio_server", lambda *a, **k: None) + monkeypatch.setattr(chatmod, "load_chat_backend", lambda *a, **k: _FakeChatBackend()) + monkeypatch.setattr(chatmod, "_compare_needs_second_model", lambda: False) + + result = CliRunner().invoke( + _chat_app(), + ["fake-model"], + input = "first\nsecond\n/exit\n", + ) + + assert result.exit_code == 0, result.output + assert closed == [True] + if chunk_kind == "real_error": + assert calls[1] == [{"role": "user", "content": "second"}] + assert "(error: generation failed)" in result.output + assert "Error: generation failed" not in result.output + else: + assert calls[1] == [ + {"role": "user", "content": "first"}, + {"role": "assistant", "content": first_chunk}, + {"role": "user", "content": "second"}, + ] + assert first_chunk in result.output + + @pytest.mark.parametrize( ("chunk_kind", "expected_exit"), [ From 27f3473c7eb4930c7aadce20945d3d6984029411 Mon Sep 17 00:00:00 2001 From: oobabooga Date: Tue, 21 Jul 2026 02:39:43 -0300 Subject: [PATCH 004/128] Studio: make tab navigation feel immediate (#7271) * Studio: make repeated tab switches feel immediate * Keep cached Studio navigation data fresh * Make first Studio tab visits responsive * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Serve range requests uncompressed for immutable assets (PR #7271) * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --------- Co-authored-by: test Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Co-authored-by: Daniel Han --- studio/backend/main.py | 36 ++++++- studio/backend/tests/test_middleware.py | 66 ++++++++++++ studio/frontend/src/app/auth-guards.ts | 61 ++++++++--- studio/frontend/src/app/routes/__root.tsx | 2 +- .../frontend/src/app/routes/data-recipes.tsx | 10 +- studio/frontend/src/app/routes/export.tsx | 10 +- studio/frontend/src/app/routes/hub.tsx | 16 +-- studio/frontend/src/app/routes/projects.tsx | 10 +- studio/frontend/src/app/routes/studio.tsx | 10 +- .../frontend/src/components/app-sidebar.tsx | 46 +++++++- .../src/features/chat/api/chat-api.ts | 14 ++- .../features/chat/hooks/use-chat-projects.ts | 100 ++++++++++++++---- .../features/data-recipes/data/recipes-db.ts | 46 +++++++- .../src/features/data-recipes/index.ts | 1 + .../export/export-navigation-cache.ts | 61 +++++++++++ .../src/features/export/export-page.tsx | 50 +++++---- .../hub/hooks/use-hub-paginated-search.ts | 48 +++++++-- studio/frontend/src/features/hub/hub-page.tsx | 36 ++++++- 18 files changed, 516 insertions(+), 107 deletions(-) create mode 100644 studio/frontend/src/features/export/export-navigation-cache.ts diff --git a/studio/backend/main.py b/studio/backend/main.py index f686e29bf5..48675b9539 100644 --- a/studio/backend/main.py +++ b/studio/backend/main.py @@ -289,6 +289,7 @@ from fastapi import Depends, FastAPI, HTTPException, Query, Request from fastapi.middleware.cors import CORSMiddleware from fastapi.staticfiles import StaticFiles from fastapi.responses import FileResponse, HTMLResponse, Response +from starlette.middleware.gzip import GZipMiddleware from pathlib import Path from datetime import datetime @@ -1509,6 +1510,34 @@ def _should_inject_bootstrap(request: Request) -> bool: return _is_local_bootstrap_request(request) +_IMMUTABLE_ASSET_CACHE_CONTROL = "public, max-age=31536000, immutable" + + +class ImmutableStaticFiles(StaticFiles): + """Serve Vite's content-hashed assets without browser revalidation.""" + + def file_response( + self, + full_path, + stat_result, + scope, + status_code = 200, + ): + response = super().file_response(full_path, stat_result, scope, status_code) + response.headers["Cache-Control"] = _IMMUTABLE_ASSET_CACHE_CONTROL + return response + + +class _AssetGZipMiddleware(GZipMiddleware): + """Serve range requests uncompressed; gzip + 206 mislabels Content-Range.""" + + async def __call__(self, scope, receive, send): + if scope["type"] == "http" and any(key == b"range" for key, _ in scope["headers"]): + await self.app(scope, receive, send) + return + await super().__call__(scope, receive, send) + + def setup_frontend(app: FastAPI, build_path: Path): """Mount frontend static files (optional)""" if not build_path.exists(): @@ -1516,7 +1545,12 @@ def setup_frontend(app: FastAPI, build_path: Path): assets_dir = build_path / "assets" if assets_dir.exists(): - app.mount("/assets", StaticFiles(directory = assets_dir), name = "assets") + assets_app = _AssetGZipMiddleware( + ImmutableStaticFiles(directory = assets_dir), + minimum_size = 1024, + compresslevel = 6, + ) + app.mount("/assets", assets_app, name = "assets") def _build_index_response(request: Request) -> Response: content = (build_path / "index.html").read_bytes() diff --git a/studio/backend/tests/test_middleware.py b/studio/backend/tests/test_middleware.py index 11aeee6d77..209c6cb90a 100644 --- a/studio/backend/tests/test_middleware.py +++ b/studio/backend/tests/test_middleware.py @@ -14,6 +14,7 @@ import pytest from fastapi import FastAPI, HTTPException, Request from fastapi.responses import Response from fastapi.testclient import TestClient +from starlette.middleware.gzip import GZipMiddleware _BACKEND_ROOT = Path(__file__).resolve().parents[1] @@ -471,6 +472,71 @@ class TestSecurityHeadersMiddleware: assert b"server" in names +class TestFrontendAssets: + def test_hashed_assets_are_compressed_and_cached(self, tmp_path, main_module): + content = b"export const value = 'responsive';\n" * 200 + (tmp_path / "page-abc123.js").write_bytes(content) + app = FastAPI() + assets_app = GZipMiddleware( + main_module.ImmutableStaticFiles(directory = tmp_path), + minimum_size = 1024, + compresslevel = 6, + ) + app.mount("/assets", assets_app, name = "assets") + + response = TestClient(app).get( + "/assets/page-abc123.js", + headers = {"Accept-Encoding": "gzip"}, + ) + + assert response.status_code == 200 + assert response.content == content + assert response.headers["content-encoding"] == "gzip" + assert response.headers["cache-control"] == (main_module._IMMUTABLE_ASSET_CACHE_CONTROL) + assert "accept-encoding" in response.headers["vary"].lower() + + def test_asset_revalidation_keeps_immutable_cache_header(self, tmp_path, main_module): + (tmp_path / "page-abc123.js").write_text("export {};", encoding = "utf-8") + app = FastAPI() + app.mount( + "/assets", + main_module.ImmutableStaticFiles(directory = tmp_path), + name = "assets", + ) + client = TestClient(app) + first = client.get("/assets/page-abc123.js") + + response = client.get( + "/assets/page-abc123.js", + headers = {"If-None-Match": first.headers["etag"]}, + ) + + assert response.status_code == 304 + assert response.headers["cache-control"] == (main_module._IMMUTABLE_ASSET_CACHE_CONTROL) + + def test_range_request_is_not_compressed(self, tmp_path, main_module): + content = b"export const value = 'responsive';\n" * 200 + (tmp_path / "page-abc123.js").write_bytes(content) + app = FastAPI() + assets_app = main_module._AssetGZipMiddleware( + main_module.ImmutableStaticFiles(directory = tmp_path), + minimum_size = 1024, + compresslevel = 6, + ) + app.mount("/assets", assets_app, name = "assets") + + response = TestClient(app).get( + "/assets/page-abc123.js", + headers = {"Accept-Encoding": "gzip", "Range": "bytes=0-99"}, + ) + + assert response.status_code == 206 + assert response.headers.get("content-encoding") != "gzip" + assert response.headers["content-range"] == f"bytes 0-99/{len(content)}" + assert response.content == content[:100] + assert response.headers["cache-control"] == (main_module._IMMUTABLE_ASSET_CACHE_CONTROL) + + # /api/health auth gate diff --git a/studio/frontend/src/app/auth-guards.ts b/studio/frontend/src/app/auth-guards.ts index 6849f380b8..a3523ac580 100644 --- a/studio/frontend/src/app/auth-guards.ts +++ b/studio/frontend/src/app/auth-guards.ts @@ -23,19 +23,47 @@ interface AuthStatus { requires_password_change: boolean; } +const AUTH_STATUS_TTL_MS = 30_000; +let authStatusCheckedAt = 0; +let authStatusRequest: Promise | null = null; + +function hasFreshAuthStatus(): boolean { + return ( + authStatusCheckedAt !== 0 && + Date.now() - authStatusCheckedAt < AUTH_STATUS_TTL_MS + ); +} + async function fetchAuthStatus(): Promise { - try { - const res = await fetch(apiUrl("/api/auth/status")); - if (!res.ok) return { initialized: true, requires_password_change: mustChangePassword() }; - const status = (await res.json()) as AuthStatus; - // Server truth wins; keep localStorage in sync both ways. - if (status.requires_password_change !== mustChangePassword()) { - setMustChangePassword(status.requires_password_change); + if (authStatusRequest) return authStatusRequest; + + const request = (async () => { + try { + const res = await fetch(apiUrl("/api/auth/status")); + if (!res.ok) { + return { + initialized: true, + requires_password_change: mustChangePassword(), + }; + } + const status = (await res.json()) as AuthStatus; + authStatusCheckedAt = Date.now(); + // Server truth wins; keep localStorage in sync both ways. + if (status.requires_password_change !== mustChangePassword()) { + setMustChangePassword(status.requires_password_change); + } + return status; + } catch { + return { + initialized: true, + requires_password_change: mustChangePassword(), + }; } - return status; - } catch { - return { initialized: true, requires_password_change: mustChangePassword() }; - } + })().finally(() => { + authStatusRequest = null; + }); + authStatusRequest = request; + return request; } function authRedirect(to: "/login" | "/change-password"): never { @@ -49,12 +77,17 @@ export async function requireAuth(): Promise { } if (await hasActiveSession()) { - const { requires_password_change } = await fetchAuthStatus(); - if (requires_password_change || mustChangePassword()) { - authRedirect("/change-password"); + // Reconcile periodically so local-only routes cannot outlive a server-side + // password-change requirement, while nearby route switches stay local. + if (mustChangePassword() || !hasFreshAuthStatus()) { + const { requires_password_change } = await fetchAuthStatus(); + if (requires_password_change || mustChangePassword()) { + authRedirect("/change-password"); + } } return; } + const status = await fetchAuthStatus(); if (status.requires_password_change || mustChangePassword()) { authRedirect("/change-password"); diff --git a/studio/frontend/src/app/routes/__root.tsx b/studio/frontend/src/app/routes/__root.tsx index e23892e020..57e890dd5a 100644 --- a/studio/frontend/src/app/routes/__root.tsx +++ b/studio/frontend/src/app/routes/__root.tsx @@ -281,7 +281,7 @@ function RootLayout() { initial={{ opacity: 0 }} animate={{ opacity: 1 }} exit={{ opacity: 0 }} - transition={{ duration: 0.15 }} + transition={{ duration: 0.06 }} className="flex min-h-0 min-w-0 flex-1 basis-0 flex-col overflow-visible" > }> diff --git a/studio/frontend/src/app/routes/data-recipes.tsx b/studio/frontend/src/app/routes/data-recipes.tsx index c35e63da5f..22f87821af 100644 --- a/studio/frontend/src/app/routes/data-recipes.tsx +++ b/studio/frontend/src/app/routes/data-recipes.tsx @@ -1,15 +1,13 @@ // SPDX-License-Identifier: AGPL-3.0-only // Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 -import { createRoute } from "@tanstack/react-router"; -import { lazy } from "react"; +import { createRoute, lazyRouteComponent } from "@tanstack/react-router"; import { requireAuth } from "../auth-guards"; import { Route as rootRoute } from "./__root"; -const DataRecipesPage = lazy(() => - import("@/features/data-recipes").then((m) => ({ - default: m.DataRecipesPage, - })), +const DataRecipesPage = lazyRouteComponent( + () => import("@/features/data-recipes"), + "DataRecipesPage", ); export const Route = createRoute({ diff --git a/studio/frontend/src/app/routes/export.tsx b/studio/frontend/src/app/routes/export.tsx index 40118c6a92..5a7b586f19 100644 --- a/studio/frontend/src/app/routes/export.tsx +++ b/studio/frontend/src/app/routes/export.tsx @@ -1,15 +1,13 @@ // SPDX-License-Identifier: AGPL-3.0-only // Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 -import { createRoute } from "@tanstack/react-router"; -import { lazy } from "react"; +import { createRoute, lazyRouteComponent } from "@tanstack/react-router"; import { requireAuth } from "../auth-guards"; import { Route as rootRoute } from "./__root"; -const ExportPage = lazy(() => - import("@/features/export/export-page").then((m) => ({ - default: m.ExportPage, - })), +const ExportPage = lazyRouteComponent( + () => import("@/features/export/export-page"), + "ExportPage", ); export type ExportSearch = { diff --git a/studio/frontend/src/app/routes/hub.tsx b/studio/frontend/src/app/routes/hub.tsx index c623ef9848..2207490e44 100644 --- a/studio/frontend/src/app/routes/hub.tsx +++ b/studio/frontend/src/app/routes/hub.tsx @@ -1,15 +1,13 @@ // SPDX-License-Identifier: AGPL-3.0-only // Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 -import { createRoute } from "@tanstack/react-router"; -import { lazy } from "react"; +import { createRoute, lazyRouteComponent } from "@tanstack/react-router"; import { requireAuth } from "../auth-guards"; import { Route as rootRoute } from "./__root"; -const ModelsPage = lazy(() => - import("@/features/hub/hub-page").then((m) => ({ - default: m.ModelsPage, - })), +const ModelsPage = lazyRouteComponent( + () => import("@/features/hub/hub-page"), + "ModelsPage", ); export interface ModelsSearch { @@ -31,7 +29,11 @@ export const Route = createRoute({ const model = search.model; if (typeof model === "string" && model.length > 0) next.model = model; const section = search.section; - if (section === "trending" || section === "latest" || section === "finetune") { + if ( + section === "trending" || + section === "latest" || + section === "finetune" + ) { next.section = section; } const kind = search.kind; diff --git a/studio/frontend/src/app/routes/projects.tsx b/studio/frontend/src/app/routes/projects.tsx index c63b1d5838..17f58ef631 100644 --- a/studio/frontend/src/app/routes/projects.tsx +++ b/studio/frontend/src/app/routes/projects.tsx @@ -1,15 +1,13 @@ // SPDX-License-Identifier: AGPL-3.0-only // Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 -import { createRoute } from "@tanstack/react-router"; -import { lazy } from "react"; +import { createRoute, lazyRouteComponent } from "@tanstack/react-router"; import { requireAuth } from "../auth-guards"; import { Route as rootRoute } from "./__root"; -const ProjectsPage = lazy(() => - import("@/features/chat/projects-page").then((m) => ({ - default: m.ProjectsPage, - })), +const ProjectsPage = lazyRouteComponent( + () => import("@/features/chat/projects-page"), + "ProjectsPage", ); export const Route = createRoute({ diff --git a/studio/frontend/src/app/routes/studio.tsx b/studio/frontend/src/app/routes/studio.tsx index ae7f445e94..798044bf64 100644 --- a/studio/frontend/src/app/routes/studio.tsx +++ b/studio/frontend/src/app/routes/studio.tsx @@ -1,15 +1,13 @@ // SPDX-License-Identifier: AGPL-3.0-only // Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 -import { createRoute } from "@tanstack/react-router"; -import { lazy } from "react"; +import { createRoute, lazyRouteComponent } from "@tanstack/react-router"; import { requireAuth } from "../auth-guards"; import { Route as rootRoute } from "./__root"; -const StudioPage = lazy(() => - import("@/features/studio/studio-page").then((m) => ({ - default: m.StudioPage, - })), +const StudioPage = lazyRouteComponent( + () => import("@/features/studio/studio-page"), + "StudioPage", ); export const Route = createRoute({ diff --git a/studio/frontend/src/components/app-sidebar.tsx b/studio/frontend/src/components/app-sidebar.tsx index b8601b00f6..8eab03133b 100644 --- a/studio/frontend/src/components/app-sidebar.tsx +++ b/studio/frontend/src/components/app-sidebar.tsx @@ -93,7 +93,12 @@ import { import { Tooltip as TooltipPrimitive } from "radix-ui"; import { HugeiconsIcon } from "@hugeicons/react"; import { ChevronDown, Moon } from "lucide-react"; -import { Link, useNavigate, useRouterState } from "@tanstack/react-router"; +import { + Link, + useNavigate, + useRouter, + useRouterState, +} from "@tanstack/react-router"; import { archiveChatItem, ChatSearchDialog, @@ -256,6 +261,10 @@ function createNavigationNonce(): string { return `${Date.now()}-${Math.random().toString(36).slice(2, 10)}`; } +function preloadSilently(request: Promise): void { + void request.catch(() => undefined); +} + function NavItem({ icon, label, @@ -267,6 +276,7 @@ function NavItem({ className, spinner, tooltip, + onIntent, }: { icon: typeof ZapIcon; label: string; @@ -277,6 +287,7 @@ function NavItem({ dataTour?: string; className?: string; spinner?: boolean; + onIntent?: () => void; // Overrides the hover tooltip (defaults to `label`). Used to explain why a // disabled item (e.g. Train/Export on a chat-only host) is greyed out. tooltip?: string; @@ -288,6 +299,8 @@ function NavItem({ tooltip={tooltip ?? label} disabled={disabled} onClick={onClick} + onPointerEnter={disabled ? undefined : onIntent} + onFocus={disabled ? undefined : onIntent} isActive={active} data-tour={dataTour} className="sidebar-nav-btn h-[33px] rounded-full gap-[8.5px] pl-3 pr-2.5 font-medium group-data-[collapsible=icon]:px-2.5 group-data-[collapsible=icon]:!w-[32px] group-data-[collapsible=icon]:mx-auto" @@ -324,6 +337,7 @@ export function AppSidebar() { }); const { togglePinned, isMobile, setOpenMobile } = useSidebar(); const navigate = useNavigate(); + const router = useRouter(); // Web update detection: `webUpdate` is non-null only when the installed // (PyPI) version is behind the latest release, so the card is hidden by @@ -1218,6 +1232,9 @@ export function AppSidebar() { navigate({ to: "/projects" }); closeMobileIfOpen(); }} + onIntent={() => { + preloadSilently(router.preloadRoute({ to: "/projects" })); + }} className="group/projects-item relative" > - - - Unpin - - - ) : null} ); } diff --git a/studio/frontend/src/components/assistant-ui/model-selector/remembered-load-settings.ts b/studio/frontend/src/components/assistant-ui/model-selector/remembered-load-settings.ts deleted file mode 100644 index 08492ab480..0000000000 --- a/studio/frontend/src/components/assistant-ui/model-selector/remembered-load-settings.ts +++ /dev/null @@ -1,79 +0,0 @@ -// SPDX-License-Identifier: AGPL-3.0-only -// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 - -// Per-model pre-load inference settings, persisted in localStorage so the load -// dialog can offer "Remember settings for ". GGUF picks only: every -// field is a llama.cpp load knob, so all save/restore call sites gate on -// GGUF-ness (a non-GGUF blob would only snapshot leftover standing values). - -const KEY = "unsloth_load_settings"; - -export interface RememberedLoadSettings { - contextLength: number | null; - kvCacheDtype: string | null; - speculativeType: string | null; - specDraftNMax: number | null; - tensorParallel: boolean; - // GPU Memory controls. Optional so an older blob (which lacked them) still - // parses, leaving the live knobs untouched on apply. The mode is kept with the - // manual knobs (gpuLayers/nCpuMoe are ignored outside Manual mode). A null - // selectedGpuIds is meaningful (all GPUs), so it's distinguished from absent. - // The per-GPU split ratio is deliberately NOT remembered: it's positionally - // bound to the exact GPU set/order and unvalidated, so it would mismatch. - gpuMemoryMode?: "auto" | "manual"; - gpuLayers?: number; - nCpuMoe?: number; - selectedGpuIds?: number[] | null; -} - -// Storage key for a pick's remembered settings, scoped per quant (the VRAM-budget -// knobs differ per quant). An HF repo collapses its GGUF variants into one `id`, -// so fold the variant in. Local .gguf paths are already file-specific; native -// drag-drop files key by display label, so same-named files share an entry. -export function rememberedLoadSettingsKey(selection: { - id: string; - ggufVariant?: string | null; -}): string { - return selection.ggufVariant - ? `${selection.id}::${selection.ggufVariant}` - : selection.id; -} - -function readAll(): Record { - try { - return JSON.parse(localStorage.getItem(KEY) ?? "{}"); - } catch { - return {}; - } -} - -function writeAll(all: Record) { - try { - localStorage.setItem(KEY, JSON.stringify(all)); - } catch { - // Ignore quota / unavailable storage. - } -} - -export function loadRememberedLoadSettings( - key: string, -): RememberedLoadSettings | null { - return readAll()[key] ?? null; -} - -export function saveRememberedLoadSettings( - key: string, - settings: RememberedLoadSettings, -) { - const all = readAll(); - all[key] = settings; - writeAll(all); -} - -export function clearRememberedLoadSettings(key: string) { - const all = readAll(); - if (key in all) { - delete all[key]; - writeAll(all); - } -} diff --git a/studio/frontend/src/features/chat/api/chat-adapter.ts b/studio/frontend/src/features/chat/api/chat-adapter.ts index 7083f02288..b0127b5e40 100644 --- a/studio/frontend/src/features/chat/api/chat-adapter.ts +++ b/studio/frontend/src/features/chat/api/chat-adapter.ts @@ -2,10 +2,7 @@ // Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 import { getAuthToken } from "@/features/auth"; -import { - loadRememberedLoadSettings, - rememberedLoadSettingsKey, -} from "@/components/assistant-ui/model-selector/remembered-load-settings"; +import { resolveInitialConfig } from "@/features/model-picker"; import { projectHasSources } from "@/features/rag/api/rag-api"; import { apiUrl } from "@/lib/api-base"; import { parseParamCountB } from "@/lib/model-size"; @@ -46,7 +43,7 @@ import { type PendingImageEditReference, type RagAutoInject, GPU_LAYERS_AUTO, - loadedGpuMemoryFieldsUnlessStaged, + loadedGpuMemoryFields, reconcilePersistedGpuIds, resolveLoadedSpeculativeSettings, resolveSpeculativeSettingsForLoad, @@ -1533,65 +1530,56 @@ async function autoLoadSmallestModel(): Promise<{ return false; } const currentStore = useChatRuntimeStore.getState(); - // Blobs are saved for GGUF picks only (the sheet gates on it), so don't - // let a legacy non-GGUF blob feed a stale context/spec choice into a - // safetensors auto-load. - const remembered = - candidate.kind === "gguf" - ? loadRememberedLoadSettings( - rememberedLoadSettingsKey({ - id: candidate.id, - ggufVariant: candidate.ggufVariant, - }), - ) - : null; + const { config } = resolveInitialConfig(candidate.id, candidate.ggufVariant); const effectiveMaxSeqLength = resolveLoadMaxSeqLength({ modelId: candidate.id, ggufVariant: candidate.ggufVariant, isGguf: candidate.kind === "gguf", - customContextLength: remembered?.contextLength ?? null, + customContextLength: config.customContextLength, ggufContextLength: null, currentCheckpoint: currentStore.params.checkpoint, activeGgufVariant: currentStore.activeGgufVariant, - maxSeqLength: candidate.maxSeqLength, + maxSeqLength: config.maxSeqLength ?? candidate.maxSeqLength, presetSource: currentStore.activePresetSource, }); - // The GPU knobs are per-model, so read them from the same remembered - // settings that fed effectiveMaxSeqLength -- on a background auto-load the - // live store holds session defaults, not the saved Manual mode / layer pin / - // GPU pick. Absent fields fall back like applyRememberedLoadSettings: the - // mode to the store (a persisted standing preference), the per-model knobs to - // their defaults. The saved GPU pick is reconciled against the GPUs present - // now, like the interactive restore. + // The GPU knobs are per-model, so read them from the same per-model config + // that fed effectiveMaxSeqLength -- on a background auto-load the live store + // holds session defaults, not the saved Manual mode / layer pin / GPU pick. + // Absent fields fall back like the interactive restore: the mode to the store + // (a persisted standing preference), the per-model knobs to their defaults. + // The saved GPU pick is reconciled against the GPUs present now. const effectiveGpuMemoryMode = - remembered?.gpuMemoryMode ?? currentStore.gpuMemoryMode; - const effectiveGpuLayers = remembered?.gpuLayers ?? GPU_LAYERS_AUTO; - const effectiveNCpuMoe = remembered?.nCpuMoe ?? 0; - if (remembered?.selectedGpuIds != null) { + config.gpuMemoryMode ?? currentStore.gpuMemoryMode; + const effectiveGpuLayers = config.gpuLayers ?? GPU_LAYERS_AUTO; + const effectiveNCpuMoe = config.nCpuMoe ?? 0; + if (config.selectedGpuIds != null) { // Warm the device cache first: on a cold cache the reconcile passes the // saved pick through unvalidated, and a stale cross-host pick then fails // the load with the picker hidden. await ensureGpuDeviceCache(); } const effectiveGpuIds = - remembered?.selectedGpuIds !== undefined - ? reconcilePersistedGpuIds(remembered.selectedGpuIds) + config.selectedGpuIds !== undefined + ? reconcilePersistedGpuIds(config.selectedGpuIds) : null; // Under Manual GPU memory + Auto layers, llama.cpp's --fit owns context // sizing, so send 0 (or the pinned length). GGUF-only; a no-op otherwise. - // The context pin is per-model too, so it comes from remembered settings, - // not the live store. + // The context pin is per-model too, so it comes from the saved config, not + // the live store. const fitMaxSeqLength = resolveFitMaxSeqLength( candidate.kind === "gguf", effectiveGpuMemoryMode, effectiveGpuLayers, - remembered?.contextLength ?? null, + config.customContextLength ?? null, effectiveMaxSeqLength, ); const effectiveSpeculativeType = - remembered?.speculativeType ?? specSettings.speculativeType; + config.speculativeType ?? specSettings.speculativeType; const effectiveSpecDraftNMax = - remembered?.specDraftNMax ?? specSettings.specDraftNMax; + config.specDraftNMax ?? specSettings.specDraftNMax; + const effectiveChatTemplateOverride = config.chatTemplateOverride?.trim() + ? config.chatTemplateOverride + : null; if ( !(await canAutoLoad({ model_path: candidate.id, @@ -1621,10 +1609,11 @@ async function autoLoadSmallestModel(): Promise<{ is_lora: false, gguf_variant: candidate.ggufVariant, trust_remote_code: trustRemoteCode, - cache_type_kv: remembered?.kvCacheDtype ?? null, + chat_template_override: effectiveChatTemplateOverride, + cache_type_kv: config.kvCacheDtype, speculative_type: effectiveSpeculativeType, spec_draft_n_max: effectiveSpecDraftNMax, - tensor_parallel: remembered?.tensorParallel ?? false, + tensor_parallel: config.tensorParallel, // GGUF-only: the safetensors fallback loads via HF auto-placement (no // explicit pins). The split ratio is deliberately never remembered // (positionally bound to an exact GPU set), so auto-load leaves llama.cpp's @@ -1638,7 +1627,12 @@ async function autoLoadSmallestModel(): Promise<{ } : {}), }); - saveSpeculativeType(effectiveSpeculativeType); + // Only persist the global preference when the value came from the global + // settings. A per-model config's choice must stay load-local, or autoloading + // a remembered model on startup would rewrite the global default. + if (config.speculativeType == null) { + saveSpeculativeType(effectiveSpeculativeType); + } // Self-gates on is_gguf (skips diffusion), so persists only for a real GGUF load. persistGpuMemoryModeOnLoad(loadResp, effectiveGpuMemoryMode); useChatRuntimeStore @@ -1650,6 +1644,9 @@ async function autoLoadSmallestModel(): Promise<{ ); store.setParams({ ...store.params, + ...(candidate.kind === "gguf" + ? {} + : { maxSeqLength: effectiveMaxSeqLength }), maxTokens: candidate.kind === "gguf" ? loadResp.context_length ?? 131072 @@ -1676,7 +1673,7 @@ async function autoLoadSmallestModel(): Promise<{ const keepCustomCtx = resolveManualAutoCtxPin( effectiveGpuMemoryMode, effectiveGpuLayers, - remembered?.contextLength ?? null, + config.customContextLength ?? null, ); useChatRuntimeStore.setState({ ggufContextLength: loadResp.context_length ?? 131072, @@ -1694,13 +1691,14 @@ async function autoLoadSmallestModel(): Promise<{ loadedKvCacheDtype: loadResp.cache_type_kv ?? null, tensorParallel: loadResp.tensor_parallel ?? false, loadedTensorParallel: loadResp.tensor_parallel ?? false, - ...loadedGpuMemoryFieldsUnlessStaged(loadResp, { - customContextLength: keepCustomCtx, - }), + ...loadedGpuMemoryFields(loadResp), loadedCustomContextLength: keepCustomCtx, defaultChatTemplate: loadResp.chat_template ?? null, - chatTemplateOverride: null, - loadedChatTemplateOverride: null, + chatTemplateOverride: effectiveChatTemplateOverride, + loadedChatTemplateOverride: effectiveChatTemplateOverride, + // Retain the saved requested context so re-saving the config keeps the + // override; null stays null (auto/VRAM-fit). + customContextLength: config.customContextLength, loadedIsMultimodal: isMultimodalResponse(loadResp), loadedIsDiffusion: loadResp.is_diffusion ?? false, ...resolveLoadedSpeculativeSettings(loadResp), @@ -1720,10 +1718,11 @@ async function autoLoadSmallestModel(): Promise<{ loadedTensorParallel: loadResp.tensor_parallel ?? false, // Non-GGUF response: clears any stale GPU baseline a prior manual-GPU // GGUF load left, matching the interactive/status sibling load paths. - ...loadedGpuMemoryFieldsUnlessStaged(loadResp), + ...loadedGpuMemoryFields(loadResp), defaultChatTemplate: loadResp.chat_template ?? null, - chatTemplateOverride: null, - loadedChatTemplateOverride: null, + chatTemplateOverride: effectiveChatTemplateOverride, + loadedChatTemplateOverride: effectiveChatTemplateOverride, + customContextLength: null, ...resolveLoadedSpeculativeSettings(loadResp), loadedIsMultimodal: isMultimodalResponse(loadResp), loadedIsDiffusion: loadResp.is_diffusion ?? false, @@ -1988,7 +1987,7 @@ async function autoLoadSmallestModel(): Promise<{ loadedKvCacheDtype: loadResp.cache_type_kv ?? null, tensorParallel: loadResp.tensor_parallel ?? false, loadedTensorParallel: loadResp.tensor_parallel ?? false, - ...loadedGpuMemoryFieldsUnlessStaged(loadResp), + ...loadedGpuMemoryFields(loadResp), // Drives the GPU Memory controls' diffusion gate; set alongside the // GPU fields on every load path so the gate can't read stale. loadedIsDiffusion: loadResp.is_diffusion ?? false, diff --git a/studio/frontend/src/features/chat/api/chat-api.ts b/studio/frontend/src/features/chat/api/chat-api.ts index 631474c39a..de3e5e370c 100644 --- a/studio/frontend/src/features/chat/api/chat-api.ts +++ b/studio/frontend/src/features/chat/api/chat-api.ts @@ -377,14 +377,33 @@ export async function listCachedModels( return data.cached; } -export async function deleteCachedModel( +export interface CachedModelPath { + path: string; + is_dir: boolean; +} + +/** Absolute on-disk path of a cached repo or one of its GGUF variants. */ +export async function getCachedModelPath( + repoId: string, + variant?: string, +): Promise { + const params = new URLSearchParams({ repo_id: repoId }); + if (variant) params.set("variant", variant); + const response = await authFetch( + `/api/models/cached-model-path?${params.toString()}`, + ); + return parseJsonOrThrow(response); +} + +/** Reveal a cached repo (or one GGUF variant's file) in the OS file manager. */ +export async function revealCachedModel( repoId: string, variant?: string, ): Promise { const payload: Record = { repo_id: repoId }; if (variant) payload.variant = variant; - const response = await authFetch("/api/models/delete-cached", { - method: "DELETE", + const response = await authFetch("/api/models/reveal-cached-model", { + method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify(payload), }); diff --git a/studio/frontend/src/features/chat/chat-page.tsx b/studio/frontend/src/features/chat/chat-page.tsx index 217eaf8b6d..ef018445e0 100644 --- a/studio/frontend/src/features/chat/chat-page.tsx +++ b/studio/frontend/src/features/chat/chat-page.tsx @@ -2,16 +2,19 @@ // Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 import { + applyModelLoadConfigToRuntime, + currentRuntimePerModelConfig, type DeletedModelRef, type ExternalModelOption, type LoraModelOption, type ModelOption, ModelSelector, -} from "@/components/assistant-ui/model-selector"; -import { - loadRememberedLoadSettings, - rememberedLoadSettingsKey, -} from "@/components/assistant-ui/model-selector/remembered-load-settings"; + type ModelSelectorChangeMeta, + type PerModelConfig, + resolveInitialConfig, + SidebarModelConfig, + useActiveModelConfig, +} from "@/features/model-picker"; import { ProjectComposer, Thread } from "@/components/assistant-ui/thread"; import { CopyableErrorChip } from "@/components/ui/copyable-error-chip"; import { @@ -27,10 +30,10 @@ import { } from "@/components/ui/resizable"; import { useSidebar } from "@/components/ui/sidebar"; import { Tooltip, TooltipContent } from "@/components/ui/tooltip"; -import { useLatestRef } from "@/features/hub/hooks/use-latest-ref"; import { DOWNLOAD_KIND, downloadManager, + useRepoDownload, } from "@/features/hub/download-manager"; import { type NativeIntent, @@ -93,7 +96,6 @@ import { renameChatItem, useChatSidebarItems, } from "./hooks/use-chat-sidebar-items"; -import { useStagedModelPreparation } from "./hooks/use-staged-model-preparation"; import { clearTrainingCompareHandoff, getTrainingCompareHandoff, @@ -128,10 +130,8 @@ import { hasGgufSource, isDownloadableHubRepo, loadOptionalBool, - pendingSelectionMatches, useChatRuntimeStore, } from "./stores/chat-runtime-store"; -import type { PendingModelSelection } from "./stores/chat-runtime-store"; import { useChatPreferencesStore } from "./stores/chat-preferences-store"; import { useExternalProvidersStore } from "./stores/external-providers-store"; import { buildChatTourSteps } from "./tour"; @@ -385,6 +385,7 @@ type CompareModelSelection = { id: string; isLora: boolean; ggufVariant?: string; + config?: PerModelConfig; }; function modelMatchesDeleted( @@ -645,6 +646,8 @@ function GeneralCompareHeader({ loraModels, externalModels, value, + selectedConfig, + selectedGgufVariant, onValueChange, onFoldersChange, onModelsChange, @@ -655,9 +658,11 @@ function GeneralCompareHeader({ loraModels: LoraModelOption[]; externalModels: ExternalModelOption[]; value: string; + selectedConfig?: PerModelConfig | null; + selectedGgufVariant?: string | null; onValueChange: ( id: string, - meta: { isLora: boolean; ggufVariant?: string }, + meta: ModelSelectorChangeMeta, ) => void; onFoldersChange?: () => void; onModelsChange?: (deletedModel?: DeletedModelRef) => void; @@ -684,6 +689,8 @@ function GeneralCompareHeader({ loraModels={loraModels} externalModels={externalModels} value={value} + selectedConfig={selectedConfig} + selectedGgufVariant={selectedGgufVariant} onValueChange={onValueChange} onFoldersChange={onFoldersChange} onModelsChange={onModelsChange} @@ -811,11 +818,14 @@ const GeneralCompareContent = memo(function GeneralCompareContent({ loraModels={loraModels} externalModels={externalModels} value={model1.id} + selectedConfig={model1.config} + selectedGgufVariant={model1.ggufVariant} onValueChange={(id, meta) => setModel1({ id, isLora: meta.isLora, ggufVariant: meta.ggufVariant, + config: meta.config, }) } onFoldersChange={onFoldersChange} @@ -838,11 +848,14 @@ const GeneralCompareContent = memo(function GeneralCompareContent({ loraModels={loraModels} externalModels={externalModels} value={model2.id} + selectedConfig={model2.config} + selectedGgufVariant={model2.ggufVariant} onValueChange={(id, meta) => setModel2({ id, isLora: meta.isLora, ggufVariant: meta.ggufVariant, + config: meta.config, }) } onFoldersChange={onFoldersChange} @@ -1236,6 +1249,13 @@ export function validateChatSearch(search: Record): ChatSearch }; } +type PendingHubAutoLoad = { + selection: SelectedModelInput; + contextKey: string; + originCheckpoint: string; + originGgufVariant: string | null; +}; + // `search` comes from RootLayout (not useSearch) so ChatPage stays mounted off-route // (keeping an in-flight generation alive), frozen to the last /chat search. `active` // is false off-route: close body-portaled surfaces and stop route-specific listeners @@ -1248,30 +1268,6 @@ export function ChatPage({ const settingsOpen = useChatRuntimeStore((s) => s.settingsPanelOpen); const setSettingsOpen = useChatRuntimeStore((s) => s.setSettingsPanelOpen); - // Deferred-load staging: downloads a staged GGUF (if needed) and reads its - // header context so the sheet can show the context slider before the load. - // autoLoad picks instead load the cached file as soon as the download ends; - // selectModel is defined below, so the load runs through a ref. - const autoLoadStagedRef = useRef< - ((pending: PendingModelSelection) => void) | null - >(null); - const stagedDownload = useStagedModelPreparation({ - onAutoLoad: (pending) => autoLoadStagedRef.current?.(pending), - }); - // Abandon a staged pick: the store action cancels its in-flight download and - // reverts the edited knobs, so nothing lingers after the user walks away. - const abandonStaged = useCallback(() => { - useChatRuntimeStore.getState().abandonStagedModel(); - }, []); - // Detach a staged pick on navigation without cancelling its download: the - // transfer keeps running in the manager and lands in cache, like Hub. - const detachStaged = useCallback(() => { - useChatRuntimeStore.getState().abandonStagedModel({ keepDownload: true }); - }, []); - // Tracks whether the chat page is still mounted, so a staged-load failure that - // resolves after the user left chat doesn't resurrect the abandoned pick. - const mountedRef = useRef(true); - useEffect(() => () => void (mountedRef.current = false), []); const incognito = useChatRuntimeStore((s) => s.incognito); const setIncognito = useChatRuntimeStore((s) => s.setIncognito); const incognitoLabel = incognito @@ -1363,6 +1359,9 @@ export function ChatPage({ const ggufContextLength = useChatRuntimeStore( (state) => state.ggufContextLength, ); + const ggufNativeContextLength = useChatRuntimeStore( + (state) => state.ggufNativeContextLength, + ); const contextUsage = useChatRuntimeStore((state) => state.contextUsage); const modelsFromStore = useChatRuntimeStore((state) => state.models); const lorasFromStore = useChatRuntimeStore((state) => state.loras); @@ -1440,39 +1439,37 @@ export function ChatPage({ refreshRef.current = refresh; selectModelRef.current = selectModel; }, [refresh, selectModel]); - // Load a cached autoLoad pick once its download finishes. The sheet was never - // opened, so on a load failure just drop the orphaned staged knobs. The knobs - // were already seeded on stage, so keepSpeculative only when a config was - // saved -- otherwise the standing speculative preference should win. - autoLoadStagedRef.current = (pending) => { - // Blobs are saved for GGUF picks only (the sheet gates on it), so don't - // let a legacy non-GGUF blob claim a seeded config here. - const remembered = hasGgufSource(pending) - ? loadRememberedLoadSettings(rememberedLoadSettingsKey(pending)) - : null; - void selectModel({ - ...pending, - isDownloaded: true, - forceReload: true, - keepSpeculative: remembered != null, - throwOnError: true, - }).catch(() => { - const store = useChatRuntimeStore.getState(); - // selectModel only clears pendingSelection on success, so a failed - // auto-load leaves our staged pick (and its edited load knobs) behind. - // Abandon it when it is still the active stage; otherwise just revert the - // settings if the stage was already cleared by something else. - if (pendingSelectionMatches(store.pendingSelection, pending)) { - store.abandonStagedModel(); - } else if (!store.pendingSelection) { - store.resetModelSettingsToLoaded(); - } - }); - }; + const rememberedConfigFor = useCallback( + (selection: { + id: string; + ggufVariant?: string | null; + source?: string; + }) => { + if (selection.source === "external") return null; + const resolved = resolveInitialConfig(selection.id, selection.ggufVariant); + return resolved.remembered ? resolved.config : null; + }, + [], + ); const isExternalModel = useMemo( () => isExternalModelId(inferenceParams.checkpoint), [inferenceParams.checkpoint], ); + const { + checkpoint: runtimeCheckpoint, + isGguf: runtimeModelIsGguf, + config: activeModelConfig, + } = useActiveModelConfig(); + const activeModelIsGguf = + runtimeCheckpoint != null && !isExternalModel && runtimeModelIsGguf; + const activeModelIsLora = useMemo(() => { + const checkpoint = inferenceParams.checkpoint; + if (!checkpoint || isExternalModel) return false; + const model = modelsFromStore.find((entry) => entry.id === checkpoint); + if (model) return model.isLora; + const lora = lorasFromStore.find((entry) => entry.id === checkpoint); + return lora?.exportType === "lora"; + }, [inferenceParams.checkpoint, isExternalModel, modelsFromStore, lorasFromStore]); const reasoningEnabled = useChatRuntimeStore((s) => s.reasoningEnabled); const reasoningStyle = useChatRuntimeStore((s) => s.reasoningStyle); const reasoningEffort = useChatRuntimeStore((s) => s.reasoningEffort); @@ -1783,75 +1780,21 @@ export function ChatPage({ closeArtifactSurface(); }, [activeThreadId, closeArtifactSurface, selectedArtifact, view]); - // Abandon a staged (not-yet-loaded) pick when the chat context actually - // changes — switching threads, leaving single view, or starting a new chat / - // project — so a stale Load button can't resurface in a different context. - // New Chat keeps activeThreadId null and only bumps the `new` search nonce, so - // the key includes the route identity, not just the thread. Mirrors the - // incognito reset pattern. (Route exit is handled in __root.tsx, which runs - // after this unmounts.) Clear only on a real change, never on mount: staging - // from the Hub sets pendingSelection then navigates here, and clearing on - // mount would wipe it. Comparing the previous context (rather than a first-run - // flag) is also safe under StrictMode's double-invoke and component remounts. - const chatContextKey = `${view.mode}|${activeThreadId ?? ""}|${search.new ?? ""}|${search.project ?? ""}`; - const chatContextKeyRef = useLatestRef(chatContextKey); - const prevChatContextRef = useRef(null); - useEffect(() => { - const prev = prevChatContextRef.current; - prevChatContextRef.current = chatContextKey; - if (prev === null || prev === chatContextKey) return; - detachStaged(); - }, [chatContextKey, detachStaged]); - const hasActiveModel = Boolean(inferenceParams.checkpoint); - // Load immediately, or — when "Load on selection" is off — stage the pick so - // its load options can be set first. Shared by the main selector, native - // drag-drop/picker, and the dropped-file chip (the Hub stages via the store). + const chatContextKey = `${view.mode}|${activeThreadId ?? ""}|${search.new ?? ""}|${search.project ?? ""}`; + const [pendingHubAutoLoad, setPendingHubAutoLoad] = + useState(null); const stageOrLoad = useCallback( async (selection: SelectedModelInput) => { const store = useChatRuntimeStore.getState(); - // An un-cached HF repo (GGUF variant or a full non-GGUF snapshot) downloads - // through the manager first (global indicator), then auto-loads. Everything - // else -- cached picks, local/native files, LoRA, external -- loads now. const wantManagerDownload = isDownloadableHubRepo(selection) && !selection.isDownloaded; - if ( - (!hasGgufSource(selection) && !wantManagerDownload) || - (store.loadOnSelection && selection.isDownloaded) - ) { - // Detach any staged pick first so its edited knobs (e.g. a custom - // context length) don't leak into this immediate load -- resolveLoad - // reads customContextLength before checking the target is GGUF. Detach - // (not abandon) keeps its download running. - detachStaged(); - // Load-on-selection skips the sheet, so seed the saved knobs here the - // way the sheet's restore effect would; the switch would otherwise reset - // the remembered speculative choice (keepSpeculative below prevents it). - const remembered = hasGgufSource(selection) - ? loadRememberedLoadSettings(rememberedLoadSettingsKey(selection)) - : null; - if (remembered) store.applyRememberedLoadSettings(remembered); - await selectModel( - remembered ? { ...selection, keepSpeculative: true } : selection, - ); - return; - } - // Loads can't queue behind each other, but a download is independent: if - // the pick needs downloading, start it in the manager so it runs alongside - // the load. Nothing to download (already on device) just waits. if (store.modelLoading) { - // Both an uncached non-GGUF snapshot (wantManagerDownload) and an - // uncached remote GGUF quant download through the manager, so either can - // run in the background while another model loads. wantManagerDownload - // excludes GGUF by design, so the GGUF case is checked separately. const wantBackgroundDownload = wantManagerDownload || (selection.source === "hub" && hasGgufSource(selection) && !selection.isDownloaded); - // The model currently loading already downloads as part of its own load - // (the /load flow fetches before setting the checkpoint), so re-picking - // it must not kick off a second transfer against the same cache. const isLoadingThisPick = !!loadingModel && normalizeModelRef(loadingModel.id) === @@ -1862,11 +1805,6 @@ export function ChatPage({ description: "It's downloading as part of the load in progress.", }); } else if (wantBackgroundDownload) { - // Only claim the download started once a job is actually created. A - // transport conflict records state that is only resolvable from the - // Hub download card, so point the user there instead of showing a - // success toast for a transfer that never began; "busy" and "error" - // already surface their own toasts. const outcome = await downloadManager.requestStart({ kind: DOWNLOAD_KIND.MODEL, repoId: selection.id, @@ -1883,6 +1821,11 @@ export function ChatPage({ description: "An earlier partial download used a different transport. Open the Hub tab to resume or restart it.", }); + } else if (outcome === "busy") { + toast.info("Download already in progress", { + description: + "Another download for this model is still running. Reselect it once that finishes to load it.", + }); } } else { toast.info("Another model is already loading", { @@ -1891,23 +1834,128 @@ export function ChatPage({ } return; } - // Detach the prior staged pick (keeping its download) before rebinding, so - // a second pick downloads alongside the first instead of cancelling it. - detachStaged(); - store.stageModel({ - id: selection.id, - isLora: selection.isLora, - ggufVariant: selection.ggufVariant, - isDownloaded: selection.isDownloaded, - expectedBytes: selection.expectedBytes, - nativePathToken: selection.nativePathToken, - isGguf: selection.isGguf, - isHubRepo: wantManagerDownload || undefined, - autoLoad: store.loadOnSelection, + const wantManagerStage = + wantManagerDownload || + (selection.source === "hub" && + hasGgufSource(selection) && + !selection.isDownloaded); + if (wantManagerStage) { + setPendingHubAutoLoad((current) => + current && + current.selection.id === selection.id && + (current.selection.ggufVariant ?? null) === + (selection.ggufVariant ?? null) && + current.contextKey === chatContextKey && + current.originCheckpoint === store.params.checkpoint && + current.originGgufVariant === store.activeGgufVariant + ? current + : { + selection, + contextKey: chatContextKey, + originCheckpoint: store.params.checkpoint, + originGgufVariant: store.activeGgufVariant, + }, + ); + return; + } + setPendingHubAutoLoad(null); + const previousConfig = currentRuntimePerModelConfig({ + includeMaxSeqLength: true, + }); + const hasAppliedConfig = applyModelLoadConfigToRuntime( + selection.config ?? rememberedConfigFor(selection), + ); + await selectModel({ + ...selection, + ...(hasAppliedConfig ? { keepSpeculative: true } : {}), + previousConfig, }); }, - [detachStaged, selectModel, loadingModel], + [selectModel, loadingModel, rememberedConfigFor, chatContextKey], ); + useRepoDownload({ + kind: DOWNLOAD_KIND.MODEL, + repoId: pendingHubAutoLoad?.selection.id ?? "__hub_autoload_idle__", + activeVariant: pendingHubAutoLoad?.selection.ggufVariant ?? null, + onComplete: (variant) => { + const pending = pendingHubAutoLoad; + if ( + !pending || + (pending.selection.ggufVariant ?? null) !== (variant ?? null) + ) { + return; + } + setPendingHubAutoLoad(null); + const store = useChatRuntimeStore.getState(); + if ( + !active || + pending.contextKey !== chatContextKey || + normalizeModelRef(pending.originCheckpoint) !== + normalizeModelRef(store.params.checkpoint) || + pending.originGgufVariant !== store.activeGgufVariant + ) { + return; + } + void stageOrLoad({ ...pending.selection, isDownloaded: true }); + }, + onError: (variant) => { + if ( + pendingHubAutoLoad && + (pendingHubAutoLoad.selection.ggufVariant ?? null) === (variant ?? null) + ) { + setPendingHubAutoLoad(null); + } + }, + onCancelled: (variant) => { + if ( + pendingHubAutoLoad && + (pendingHubAutoLoad.selection.ggufVariant ?? null) === (variant ?? null) + ) { + setPendingHubAutoLoad(null); + } + }, + }); + useEffect(() => { + const pending = pendingHubAutoLoad; + if (!pending) return; + let active = true; + void (async () => { + const outcome = await downloadManager.requestStart({ + kind: DOWNLOAD_KIND.MODEL, + repoId: pending.selection.id, + variant: pending.selection.ggufVariant ?? null, + expectedBytes: pending.selection.expectedBytes ?? 0, + }); + if (!active) return; + if (outcome === "started") { + toast.info("Downloading model", { + description: "It'll load automatically once the download finishes.", + }); + return; + } + if (outcome === "conflict") { + // Keep pendingHubAutoLoad bound so this surface's cleanup does not wipe + // the conflict just recorded by requestStart (which the toast points the + // user to); resolving it from the Hub completes the download and this + // surface's onComplete auto-loads, mirroring the "started" branch. + toast.info("Resume this download from the Hub", { + description: + "An earlier partial download used a different transport. Open the Hub tab to resume or restart it.", + }); + return; + } + if (outcome === "busy") { + toast.info("Download already in progress", { + description: + "Another download for this model is still running. Reselect it once that finishes to load it.", + }); + } + setPendingHubAutoLoad((current) => (current === pending ? null : current)); + })(); + return () => { + active = false; + }; + }, [pendingHubAutoLoad]); const loadNativeModelIntent = useCallback( async (intent: NativeIntent, loadingDescription: string) => { const label = @@ -1915,6 +1963,7 @@ export function ChatPage({ await stageOrLoad({ id: label, nativePathToken: intent.path.token, + nativePathExpiresAtMs: intent.path.expiresAtMs ?? null, isDownloaded: true, loadingDescription, forceReload: true, @@ -1965,28 +2014,20 @@ export function ChatPage({ const handleCheckpointChange = useCallback( ( value: string, - meta?: { - source?: string; - isLora: boolean; - ggufVariant?: string; - isDownloaded?: boolean; - expectedBytes?: number; - isGguf?: boolean; - }, + meta?: ModelSelectorChangeMeta, ) => { const store = useChatRuntimeStore.getState(); const currentCheckpoint = store.params.checkpoint; const currentVariant = store.activeGgufVariant; - if ( - !value || - (value === currentCheckpoint && - (meta?.ggufVariant ?? null) === (currentVariant ?? null)) - ) + if (!value) return; + setPendingHubAutoLoad(null); + const isSameLoadedModel = + value === currentCheckpoint && + (meta?.ggufVariant ?? null) === (currentVariant ?? null); + if (isSameLoadedModel && !meta?.forceReload) { return; + } if (meta?.source === "external" || isExternalModelId(value)) { - // Switching to an external model abandons any staged local pick: cancel - // its download too (setCheckpoint below only clears the pending + knobs). - abandonStaged(); const selectedExternal = parseExternalModelId(value); const selectedProvider = selectedExternal ? externalProvidersForChat.find( @@ -2087,6 +2128,7 @@ export function ChatPage({ ggufMaxContextLength: null, ggufNativeContextLength: null, activeNativePathToken: null, + activeNativePathExpiresAtMs: null, // Clear previous-model counters, else the relaxed external-provider // render gate shows stale stats until the next completion. contextUsage: null, @@ -2158,19 +2200,18 @@ export function ChatPage({ source: meta?.source, isLora: meta?.isLora, ggufVariant: meta?.ggufVariant, - isDownloaded: meta?.isDownloaded, + isDownloaded: meta?.isDownloaded || isSameLoadedModel, expectedBytes: meta?.expectedBytes, isGguf: meta?.isGguf, + config: meta?.config, + nativePathToken: meta?.nativePathToken, + nativePathExpiresAtMs: meta?.nativePathExpiresAtMs, + forceReload: isSameLoadedModel || undefined, }; - // "Load on selection" off: stage the model and open settings so its - // load knobs (tensor parallel, context length…) can be set, then it - // loads once via the sheet's Load button. The currently loaded model - // stays put until the user commits. await stageOrLoad(selection); })(); }, [ - abandonStaged, activeThreadId, externalProvidersForChat, modelsFromStore, @@ -2178,6 +2219,45 @@ export function ChatPage({ view, ], ); + const handleReloadActiveModel = useCallback( + (config: PerModelConfig) => { + const checkpoint = inferenceParams.checkpoint; + if (!checkpoint) return; + const runtime = useChatRuntimeStore.getState(); + const nativeToken = runtime.activeNativePathToken; + const nativeExpiry = runtime.activeNativePathExpiresAtMs; + // A file-picked GGUF is reachable only via its native path token, which + // the desktop host prunes after a TTL. Reusing an expired token makes the + // reload fail with an opaque error, so prompt the user to re-select the + // file instead. + if (nativeToken && nativeExpiry != null && Date.now() >= nativeExpiry) { + toast.error("This local model file's access has expired.", { + description: "Re-select the model file to reload it.", + }); + return; + } + handleCheckpointChange(checkpoint, { + source: "local", + isLora: activeModelIsLora, + ggufVariant: activeGgufVariant ?? undefined, + // Without the native token the reload validates the display label as a + // repo and fails. + nativePathToken: nativeToken ?? undefined, + nativePathExpiresAtMs: nativeExpiry, + isGguf: activeModelIsGguf, + isDownloaded: true, + config, + forceReload: true, + }); + }, + [ + inferenceParams.checkpoint, + activeGgufVariant, + activeModelIsLora, + activeModelIsGguf, + handleCheckpointChange, + ], + ); const handleEject = useCallback(() => { void (async () => { if (await ejectModel()) { @@ -2446,12 +2526,27 @@ export function ChatPage({ const state = useChatRuntimeStore.getState(); const targetLora = pickBestLoraForBase(state.loras, handoff.baseModel); + const selectWithConfig = async ( + selection: Pick, + ) => { + const previousConfig = currentRuntimePerModelConfig({ + includeMaxSeqLength: true, + }); + const hasAppliedConfig = applyModelLoadConfigToRuntime( + rememberedConfigFor(selection), + ); + await selectModelRef.current({ + ...selection, + ...(hasAppliedConfig ? { keepSpeculative: true } : {}), + previousConfig, + }); + }; if (targetLora) { console.info("[chat-handoff] loading lora", { id: targetLora.id, baseModel: targetLora.baseModel, }); - await selectModelRef.current({ id: targetLora.id, isLora: true }); + await selectWithConfig({ id: targetLora.id, isLora: true }); if (canceled) return; useChatRuntimeStore.getState().setActiveThreadId(null); useChatRuntimeStore.getState().setContextUsage(null); @@ -2468,10 +2563,7 @@ export function ChatPage({ console.info("[chat-handoff] no lora match, loading base", { id: handoff.baseModel, }); - await selectModelRef.current({ - id: handoff.baseModel, - isLora: false, - }); + await selectWithConfig({ id: handoff.baseModel, isLora: false }); if (canceled) return; } else { console.warn("[chat-handoff] no lora/base match found", { @@ -2491,7 +2583,7 @@ export function ChatPage({ return () => { canceled = true; }; - }, [active, navigate]); + }, [active, navigate, rememberedConfigFor]); const tourSteps = useMemo( () => @@ -2580,6 +2672,8 @@ export function ChatPage({ externalModels={externalModels} value={inferenceParams.checkpoint} activeGgufVariant={activeGgufVariant} + activeModelConfig={activeModelConfig} + activeGgufContextLength={ggufContextLength} onValueChange={handleCheckpointChange} onEject={handleEject} onFoldersChange={refreshLocalModels} @@ -2633,7 +2727,12 @@ export function ChatPage({ stageOrLoad(selection)} + onLoad={() => + loadNativeModelIntent( + pendingNativeModelIntent, + "Loading selected local GGUF model.", + ) + } /> ) : null} {loadingModel && loadToastDismissed ? ( @@ -2790,13 +2889,22 @@ export function ChatPage({ open={active && settingsOpen} onOpenChange={(open) => { setSettingsOpen(open); - // Closing the sheet abandons a staged (not-yet-loaded) pick: cancel its - // download and revert the staged knobs so nothing lingers as a dirty - // edit (or a background download) on the loaded model. - if (!open) abandonStaged(); }} params={inferenceParams} onParamsChange={setInferenceParams} + modelConfig={ + view.mode !== "compare" && activeModelConfig && !modelLoading ? ( + + ) : null + } isExternalModel={isExternalModel} providerCapabilities={activeProviderCapabilities} activeExternalProvider={activeExternalProvider} @@ -2808,67 +2916,6 @@ export function ChatPage({ ); }} externalProviderType={activeExternalProviderType} - loadingModel={loadingModel} - onReloadModel={() => { - const state = useChatRuntimeStore.getState(); - if (state.params.checkpoint) { - selectModel({ - id: state.params.checkpoint, - ggufVariant: state.activeGgufVariant ?? undefined, - // A native (drag-drop / picked) GGUF's checkpoint is only a display - // label, so the reload needs its path token to re-mint a lease -- - // else applying the now-exposed GPU/context controls can't resolve - // the file. Null for non-native loads, which reload by id as before. - nativePathToken: state.activeNativePathToken ?? undefined, - forceReload: true, - isDownloaded: true, - loadingDescription: "Reloading with updated chat template.", - }); - } - }} - onLoadPendingModel={() => { - const pending = useChatRuntimeStore.getState().pendingSelection; - if (!pending) return; - const keyAtLoad = chatContextKey; - // forceReload: the staged model isn't loaded yet, so bypass the - // same-checkpoint dedupe. keepSpeculative: honor the speculative mode - // set on the sidebar. - void selectModel({ - ...pending, - forceReload: true, - keepSpeculative: true, - throwOnError: true, - }).catch(() => { - // Recoverable failure (expired token, gated repo, OOM…): the pick is - // cleared only on success, so it normally stays staged with edited - // knobs intact — nothing to restore. - const store = useChatRuntimeStore.getState(); - // Still staged (this pick, or a newer one queued meanwhile): leave it. - if (store.pendingSelection) return; - // Cleared mid-load (sheet closed / switched chats). Re-stage only if - // the staged-load is still wanted: same chat context, sheet still - // open, page still mounted. - const stillWanted = - mountedRef.current && - store.settingsPanelOpen && - chatContextKeyRef.current === keyAtLoad; - if (stillWanted) { - store.setPendingSelection(pending); - } else { - // Abandoned (closed the sheet / switched chats / left chat): drop - // the orphaned staged knob edits so they don't linger as dirty - // settings over the loaded model. - store.resetModelSettingsToLoaded(); - } - }); - }} - stagedDownloadFraction={stagedDownload.progress?.fraction ?? null} - onCancelStagedDownload={() => - stagedDownload.cancelDownload( - useChatRuntimeStore.getState().pendingSelection?.ggufVariant ?? - null, - ) - } /> diff --git a/studio/frontend/src/features/chat/chat-settings-sheet.tsx b/studio/frontend/src/features/chat/chat-settings-sheet.tsx index bd22cc4f55..d4f154882c 100644 --- a/studio/frontend/src/features/chat/chat-settings-sheet.tsx +++ b/studio/frontend/src/features/chat/chat-settings-sheet.tsx @@ -1,19 +1,7 @@ // SPDX-License-Identifier: AGPL-3.0-only // Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 -import { - Alert, - AlertDescription, - AlertTitle, -} from "@/components/ui/alert"; import { Button } from "@/components/ui/button"; -import { Checkbox } from "@/components/ui/checkbox"; -import { - clearRememberedLoadSettings, - loadRememberedLoadSettings, - rememberedLoadSettingsKey, - saveRememberedLoadSettings, -} from "@/components/assistant-ui/model-selector/remembered-load-settings"; import { Dialog, DialogContent, @@ -29,7 +17,7 @@ import { DropdownMenuSeparator, DropdownMenuTrigger, } from "@/components/ui/dropdown-menu"; -import { Input } from "@/components/ui/input"; +import { InfoHint } from "@/components/ui/info-hint"; import { InputGroup, InputGroupAddon, @@ -50,27 +38,22 @@ import { SheetTitle, } from "@/components/ui/sheet"; import { Slider } from "@/components/ui/slider"; -import { Spinner } from "@/components/ui/spinner"; import { Switch } from "@/components/ui/switch"; import { Textarea } from "@/components/ui/textarea"; -import { InfoHint } from "@/components/ui/info-hint"; import { Tooltip, TooltipContent } from "@/components/ui/tooltip"; -import { useGpuDevices } from "@/hooks/use-gpu-info"; -import { useIsMobile } from "@/hooks/use-mobile"; +import { NumericValueInput, snapToStep } from "@/features/model-picker"; +import { RetrievalSettingsSection } from "@/features/rag"; import { useLlamaUpdateCheck } from "@/hooks/use-llama-update-check"; -import { cn } from "@/lib/utils"; -import { - ArrowTurnBackwardIcon, - Edit03Icon, - LayoutAlignRightIcon, -} from "@hugeicons/core-free-icons"; +import { useIsMobile } from "@/hooks/use-mobile"; import { ChevronDownStandardIcon } from "@/lib/chevron-icons"; +import { toast } from "@/lib/toast"; +import { cn } from "@/lib/utils"; +import { Edit03Icon, LayoutAlignRightIcon } from "@hugeicons/core-free-icons"; import { HugeiconsIcon } from "@hugeicons/react"; import { Braces, ChevronDown, ExternalLink } from "lucide-react"; import { Tooltip as TooltipPrimitive } from "radix-ui"; import { Fragment, type ReactNode } from "react"; import { useCallback, useEffect, useMemo, useRef, useState } from "react"; -import { toast } from "@/lib/toast"; import { OpenAICodeExecSection } from "./components/openai-code-exec-section"; import { PermissionModeDropdown } from "./permission-mode-select"; import { resyncInferenceStatusAfterServerModelChange } from "./hooks/use-chat-model-runtime"; @@ -78,8 +61,8 @@ import { type ExternalProviderConfig, getExternalProviderApiKey, parseExternalModelId, - supportsProviderPromptCaching, supportsProviderPromptCacheTtl, + supportsProviderPromptCaching, } from "./external-providers"; import { BUILTIN_PRESETS, @@ -99,15 +82,7 @@ import { providerSupportsBuiltinCodeExecution, providerSupportsFastMode, } from "./provider-capabilities"; -import { - GPU_LAYERS_AUTO, - distributeByWeight, - isPendingGguf, - pendingSelectionMatches, - rebalanceSplit, - useChatRuntimeStore, -} from "./stores/chat-runtime-store"; -import { RetrievalSettingsSection } from "@/features/rag/components/retrieval-settings-section"; +import { useChatRuntimeStore } from "./stores/chat-runtime-store"; import type { InferenceParams } from "./types/runtime"; export { defaultInferenceParams, type Preset } from "./presets/preset-policy"; @@ -130,7 +105,7 @@ function getPromptVariablesError(raw: string): string | null { return null; } } catch { - return "Use valid JSON, for example { \"env\": \"staging\" }."; + return 'Use valid JSON, for example { "env": "staging" }.'; } return "Variables must be a JSON object."; } @@ -139,112 +114,7 @@ function hasPromptVariableSyntax(prompt: string): boolean { return PROMPT_VARIABLE_PATTERN.test(prompt); } -/** - * Editable numeric value display, shared by every slider value and the Context - * Length input. An that looks like text (shows `displayValue ?? value`, - * so "Off"/"Max" labels render) until focus, when it swaps to the raw number, - * selects it, and accepts free text. Commits on blur/Enter, reverts on Escape. - * Clamping happens on commit so typing intermediate values isn't fought. - */ -function snapToStep( - value: number, - step: number, - min?: number, - max?: number, -): number { - const lo = min ?? Number.NEGATIVE_INFINITY; - const hi = max ?? Number.POSITIVE_INFINITY; - const clamped = Math.min(Math.max(value, lo), hi); - const stepStr = String(step); - const decimals = stepStr.includes(".") ? stepStr.split(".")[1].length : 0; - const base = Number.isFinite(lo) ? lo : 0; - const snapped = base + Math.round((clamped - base) / step) * step; - const reclamped = Math.min(Math.max(snapped, lo), hi); - return Number(reclamped.toFixed(decimals)); -} - -function NumericValueInput({ - value, - min, - max, - step, - onChange, - displayValue, - className, - ariaLabel, - size: sizeAttr, - disabled = false, -}: { - value: number; - min?: number; - max?: number; - step: number; - onChange: (v: number) => void; - displayValue?: string; - className?: string; - ariaLabel?: string; - size?: number; - disabled?: boolean; -}) { - const [focused, setFocused] = useState(false); - const [draft, setDraft] = useState(""); - const cancelBlurCommitRef = useRef(false); - - const commit = (raw: string) => { - const parsed = Number.parseFloat(raw); - if (!Number.isFinite(parsed)) { - return; - } - const final = snapToStep(parsed, step, min, max); - if (final !== value) { - onChange(final); - } - }; - - const displayed = focused ? draft : (displayValue ?? String(value)); - - return ( - { - cancelBlurCommitRef.current = false; - setDraft(String(value)); - setFocused(true); - // Defer select() so it runs after the value swap above. - const target = e.currentTarget; - requestAnimationFrame(() => target.select()); - }} - onBlur={() => { - if (cancelBlurCommitRef.current) { - cancelBlurCommitRef.current = false; - } else { - commit(draft); - } - setFocused(false); - }} - onChange={(e) => setDraft(e.target.value)} - onKeyDown={(e) => { - if (e.key === "Enter") { - e.currentTarget.blur(); - } else if (e.key === "Escape") { - cancelBlurCommitRef.current = true; - setDraft(String(value)); - e.currentTarget.blur(); - } - }} - className={cn("panel-number-input", className)} - /> - ); -} - -function ParamSlider({ +export function ParamSlider({ label, value, min, @@ -285,6 +155,7 @@ function ParamSlider({ displayValue={displayValue} ariaLabel={label} size={valueSize ?? 4} + className="panel-number-input" disabled={disabled} /> @@ -385,8 +256,7 @@ function CollapsibleSection({ return (
{labelHref ? ( @@ -458,6 +328,7 @@ interface ChatSettingsPanelProps { onOpenChange?: (open: boolean) => void; params: InferenceParams; onParamsChange: (params: InferenceParams) => void; + modelConfig?: ReactNode; isExternalModel?: boolean; /** * Sampling-param capabilities for the active external provider, or `null` for @@ -472,21 +343,6 @@ interface ChatSettingsPanelProps { * Max Tokens floor in the slider. */ externalProviderType?: string | null; - onReloadModel?: () => void; - /** The in-flight load (id + GGUF variant + native path token), or null when - * idle. Used to show a loading state for the staged pick only — not for an - * unrelated load or a cancel's background unload. */ - loadingModel?: { - id: string; - ggufVariant?: string | null; - nativePathToken?: string | null; - } | null; - /** Loads the staged `pendingSelection` (deferred "Load on selection" flow). */ - onLoadPendingModel?: () => void; - /** Download progress (0–1) for a staged GGUF being fetched, or null when idle. */ - stagedDownloadFraction?: number | null; - /** Cancels the in-flight staged download (paired with abandoning the stage). */ - onCancelStagedDownload?: () => void; } export function ChatSettingsPanel({ @@ -494,16 +350,12 @@ export function ChatSettingsPanel({ onOpenChange, params, onParamsChange, + modelConfig = null, isExternalModel = false, providerCapabilities = null, activeExternalProvider = null, onExternalProviderChange, externalProviderType = null, - onReloadModel, - loadingModel = null, - onLoadPendingModel, - stagedDownloadFraction, - onCancelStagedDownload, }: ChatSettingsPanelProps) { // Local models show every knob; providerCapabilities is only consulted when // isExternalModel. Unknown providers fall back to the OpenAI-compat shape via @@ -518,64 +370,23 @@ export function ChatSettingsPanel({ const showPresencePenalty = !isExternalModel || Boolean(providerCapabilities?.presencePenalty); const isMobile = useIsMobile(); - const pendingSelection = useChatRuntimeStore((s) => s.pendingSelection); - // "Loading" only when the in-flight load IS this staged pick (full id + GGUF - // variant + native token match), not an unrelated load or a cancel's - // background unload. The variant matters: a different quant of the same repo - // staged mid-load must not read as this one loading. - const stagedLoading = - loadingModel != null && - pendingSelectionMatches(pendingSelection, { - id: loadingModel.id, - ggufVariant: loadingModel.ggufVariant, - nativePathToken: loadingModel.nativePathToken, - }); - // Load settings are snapshotted at click time; lock them while loading. - const modelControlsDisabled = stagedLoading; - const abandonStagedModel = useChatRuntimeStore((s) => s.abandonStagedModel); - const resetModelSettingsToLoaded = useChatRuntimeStore( - (s) => s.resetModelSettingsToLoaded, + const isLoadedGguf = useChatRuntimeStore((s) => s.activeGgufVariant) != null; + const currentCheckpoint = params.checkpoint; + const ggufContextLength = useChatRuntimeStore((s) => s.ggufContextLength); + // Direct-file / custom-folder GGUFs load without a variant label but still + // report a GGUF context, so detect them via the context and the checkpoint + // suffix too (mirrors the chat page's activeModelIsGguf). Otherwise Max Tokens + // would fall back to params.maxSeqLength instead of the loaded GGUF context. + const isGguf = + isLoadedGguf || + ggufContextLength != null || + (currentCheckpoint?.toLowerCase().endsWith(".gguf") ?? false); + const ggufMaxContextLength = useChatRuntimeStore( + (s) => s.ggufMaxContextLength, ); - // A staged GGUF pick (deferred load) shows the GGUF load knobs so they can be - // set before the single load. - const pendingIsGguf = isPendingGguf(pendingSelection); - // Short, human-readable name for the staged pick (HF ids carry an org prefix; - // native picks are already a display label). Drives the "staged, not loaded" - // callout so it's obvious the selection hasn't loaded yet. - const stagedLabel = (() => { - const id = pendingSelection?.id ?? ""; - const slash = id.lastIndexOf("/"); - const base = slash >= 0 ? id.slice(slash + 1) : id; - return base || id; - })(); - const activeNativePathToken = useChatRuntimeStore( - (s) => s.activeNativePathToken, - ); - const loadedGgufContextLength = useChatRuntimeStore((s) => s.ggufContextLength); - // A GGUF loaded from a native path / direct .gguf has no HF variant, so key - // off the same signal the status hydration uses -- variant OR native token OR - // a GGUF context -- else the GPU Memory controls hide for a loaded local GGUF. - const isLoadedGguf = - useChatRuntimeStore((s) => s.activeGgufVariant) != null || - activeNativePathToken != null || - loadedGgufContextLength != null; - // While a pick is staged the sheet configures *that* model, so its GGUF-ness - // (not the currently loaded model's) decides whether the GGUF-only controls - // show. Otherwise a staged non-GGUF Hub repo would inherit the loaded GGUF's - // context/KV/speculative controls. - const isGguf = pendingSelection != null ? pendingIsGguf : isLoadedGguf; - // The Model section (and Load button) shows for any staged pick, even when the - // currently active model is external. - const hasModelContent = - pendingSelection != null || - (!isExternalModel && (isGguf || Boolean(params.checkpoint))); + const customContextLength = useChatRuntimeStore((s) => s.customContextLength); const speculativeType = useChatRuntimeStore((s) => s.speculativeType); - const setSpeculativeType = useChatRuntimeStore((s) => s.setSpeculativeType); - const loadedSpeculativeType = useChatRuntimeStore( - (s) => s.loadedSpeculativeType, - ); const specFallbackReason = useChatRuntimeStore((s) => s.specFallbackReason); - // Only binary fallback states are solved by a newer prebuilt. const mtpUpdatable = specFallbackReason === "binary_no_mtp" || specFallbackReason === "binary_outdated"; @@ -597,65 +408,27 @@ export function ChatSettingsPanel({ `llama.cpp updated to ${result.tag ?? "the latest build"}.${reloadHint}`, ); } else { - toast.error(`llama.cpp update failed: ${result.error ?? "unknown error"}`); + toast.error( + `llama.cpp update failed: ${result.error ?? "unknown error"}`, + ); } }, [applyLlamaUpdate]); - const specDraftNMax = useChatRuntimeStore((s) => s.specDraftNMax); - const setSpecDraftNMax = useChatRuntimeStore((s) => s.setSpecDraftNMax); - const loadedSpecDraftNMax = useChatRuntimeStore( - (s) => s.loadedSpecDraftNMax, - ); - const currentCheckpoint = params.checkpoint; - const ggufContextLength = useChatRuntimeStore((s) => s.ggufContextLength); - const ggufMaxContextLength = useChatRuntimeStore( - (s) => s.ggufMaxContextLength, - ); - const ggufNativeContextLength = useChatRuntimeStore( - (s) => s.ggufNativeContextLength, - ); - const kvCacheDtype = useChatRuntimeStore((s) => s.kvCacheDtype); - const setKvCacheDtype = useChatRuntimeStore((s) => s.setKvCacheDtype); - const applyRememberedLoadSettings = useChatRuntimeStore( - (s) => s.applyRememberedLoadSettings, - ); - const loadedKvCacheDtype = useChatRuntimeStore((s) => s.loadedKvCacheDtype); - const tensorParallel = useChatRuntimeStore((s) => s.tensorParallel); - const setTensorParallel = useChatRuntimeStore((s) => s.setTensorParallel); - const loadedTensorParallel = useChatRuntimeStore( - (s) => s.loadedTensorParallel, - ); - const gpuMemoryMode = useChatRuntimeStore((s) => s.gpuMemoryMode); - const setGpuMemoryMode = useChatRuntimeStore((s) => s.setGpuMemoryMode); - const loadedGpuMemoryMode = useChatRuntimeStore((s) => s.loadedGpuMemoryMode); - const loadedIsDiffusion = useChatRuntimeStore((s) => s.loadedIsDiffusion); - const gpuLayers = useChatRuntimeStore((s) => s.gpuLayers); - const setGpuLayers = useChatRuntimeStore((s) => s.setGpuLayers); - const loadedGpuLayers = useChatRuntimeStore((s) => s.loadedGpuLayers); - const nCpuMoe = useChatRuntimeStore((s) => s.nCpuMoe); - const setNCpuMoe = useChatRuntimeStore((s) => s.setNCpuMoe); - const loadedNCpuMoe = useChatRuntimeStore((s) => s.loadedNCpuMoe); - const splitRatio = useChatRuntimeStore((s) => s.splitRatio); - const setSplitRatio = useChatRuntimeStore((s) => s.setSplitRatio); - const loadedSplitRatio = useChatRuntimeStore((s) => s.loadedSplitRatio); - const ggufLayerCount = useChatRuntimeStore((s) => s.ggufLayerCount); - const moeLayerCount = useChatRuntimeStore((s) => s.moeLayerCount); - const selectedGpuIds = useChatRuntimeStore((s) => s.selectedGpuIds); - const setSelectedGpuIds = useChatRuntimeStore((s) => s.setSelectedGpuIds); - const loadedGpuIds = useChatRuntimeStore((s) => s.loadedGpuIds); - const gpuDevices = useGpuDevices(); - const chatTemplateOverride = useChatRuntimeStore( - (s) => s.chatTemplateOverride, - ); - const loadedChatTemplateOverride = useChatRuntimeStore( - (s) => s.loadedChatTemplateOverride, - ); - const customContextLength = useChatRuntimeStore((s) => s.customContextLength); - const loadedCustomContextLength = useChatRuntimeStore( - (s) => s.loadedCustomContextLength, - ); - const setCustomContextLength = useChatRuntimeStore( - (s) => s.setCustomContextLength, - ); + const loadedEffectiveContext = customContextLength ?? ggufContextLength; + const showSpecFallback = + !isExternalModel && + isGguf && + specFallbackReason != null && + (speculativeType === "auto" || + speculativeType === "mtp" || + speculativeType === "mtp+ngram"); + const showContextVramWarning = + !isExternalModel && + isGguf && + ggufMaxContextLength != null && + loadedEffectiveContext != null && + loadedEffectiveContext > ggufMaxContextLength; + const showLoadedDiagnostics = showSpecFallback || showContextVramWarning; + const hasModelContent = showLoadedDiagnostics; const setActivePresetSource = useChatRuntimeStore( (s) => s.setActivePresetSource, ); @@ -666,170 +439,7 @@ export function ChatSettingsPanel({ const setActivePreset = useChatRuntimeStore((s) => s.setActivePreset); const settingsHydrated = useChatRuntimeStore((s) => s.settingsHydrated); - // A staged (not-yet-loaded) GGUF carries its own header context length on - // pendingSelection, so the slider can use the staged model's real ceiling - // without reading the loaded model's `ggufContextLength`. - const stagedContextLength = pendingSelection?.contextLength ?? null; - // "Remember settings next time" tick for a staged model. Seeds the store from - // the saved per-model settings on stage, so the sheet opens with what was used - // last time; the tick reflects whether a saved entry exists. - const [remember, setRemember] = useState(false); - // Keyed per quant: a different variant of the same repo has its own settings. - const pendingKey = pendingSelection - ? rememberedLoadSettingsKey(pendingSelection) - : null; - useEffect(() => { - if (!pendingKey) return; - // GGUF-only, like the stageOrLoad / Hub restore paths: every remembered - // field is a llama.cpp knob, so a non-GGUF pick has nothing to restore -- - // and applying its blob would clobber the standing gpuMemoryMode with a - // stale snapshot (the save on Load below is gated the same way). - const saved = pendingIsGguf ? loadRememberedLoadSettings(pendingKey) : null; - setRemember(saved != null); - if (saved) applyRememberedLoadSettings(saved); - }, [pendingKey, pendingIsGguf, applyRememberedLoadSettings]); - // While staging, the sheet reflects the STAGED model, so its header context - // takes precedence over the loaded model's (which may differ or be larger). - const baseContext = pendingIsGguf ? stagedContextLength : ggufContextLength; - const baseNativeContext = pendingIsGguf - ? stagedContextLength - : ggufNativeContextLength; - // Context controls render once we actually have a ceiling: for a staged GGUF, - // once its header metadata arrives (post-download); otherwise post-load. - const showContextControl = pendingIsGguf - ? stagedContextLength != null - : isLoadedGguf; - const stagedDownloading = - stagedDownloadFraction != null && stagedDownloadFraction < 1; - const ctxDisplayValue = customContextLength ?? baseContext ?? ""; - const ctxMaxValue = baseNativeContext ?? baseContext ?? null; - const kvDirty = kvCacheDtype !== loadedKvCacheDtype; - const ctxDirty = customContextLength !== loadedCustomContextLength; - const specDirty = speculativeType !== loadedSpeculativeType; - const specDraftDirty = specDraftNMax !== loadedSpecDraftNMax; - const tpDirty = tensorParallel !== (loadedTensorParallel ?? false); - // A loaded diffusion GGUF runs mode-agnostic (pins all layers on one GPU, - // ignores --fit/--gpu-layers), so the GPU Memory mode + manual controls don't - // apply -- hide them and don't let the preserved standing mode read as dirty. - // The GPU picker still applies (diffusion pins the chosen device). A staged pick - // keeps the controls (a pending pick's diffusion-ness isn't known until load). - const gpuModeApplies = - isGguf && (pendingSelection != null || !loadedIsDiffusion); - const gpuDirty = - gpuModeApplies && gpuMemoryMode !== (loadedGpuMemoryMode ?? "auto"); - const isManual = gpuModeApplies && gpuMemoryMode === "manual"; - // Manual with the GPU Layers slider at "Auto" (leftmost): --fit owns the whole - // layout, so the offload knobs (MoE, split, TP) don't apply. - const autoLayers = isManual && gpuLayers < 0; - // GPUs actually in use: the picked subset, or all visible when none picked. - const gpusInUse = selectedGpuIds ?? gpuDevices.map((d) => d.index); - // The picker must keep one GPU selected. - const singleGpuInUse = gpusInUse.length <= 1; - // TP needs at least two GPUs because tensor split is a no-op on one and may - // abort. Auto layers hides TP because --fit aborts under --split-mode tensor. - const tpDisabled = singleGpuInUse; - // Manual gpu-layers ceiling = model layer count + 1 (else a safe fallback): - // llama.cpp counts the output layer as one more offloadable layer past the - // repeating blocks ("offloaded 33/33" needs -ngl 33 on a 32-block model), so - // the slider max must reach it or full offload is unreachable. While staging, - // use the staged model's layer count (read from its header). - const stagedLayerCount = pendingSelection?.layerCount ?? null; - const modelLayerCount = pendingIsGguf ? stagedLayerCount : ggufLayerCount; - const gpuLayersMax = modelLayerCount != null ? modelLayerCount + 1 : 256; - // MoE-offload slider: shown only for MoE models, capped at their MoE-layer - // count. While staging, use the staged model's count (read from its header); - // otherwise the loaded model's. - const stagedMoeLayerCount = pendingSelection?.moeLayerCount ?? null; - const moeLayersMax = pendingIsGguf - ? (stagedMoeLayerCount ?? 0) - : (moeLayerCount ?? 0); - const showMoeSlider = isManual && !autoLayers && moeLayersMax > 0; - // gpuLayers always counts; MoE only with an explicit layer count (see above). - const manualDirty = - isManual && - (gpuLayers !== loadedGpuLayers || - (!autoLayers && nCpuMoe !== (loadedNCpuMoe ?? 0))); - // GPU picker: only meaningful on multi-GPU, and only when the reported - // indices are physical (relative ordinals from a parent CUDA_VISIBLE_DEVICES - // mask can't be mapped back to pin a device). null = use all (auto). - const showGpuPicker = - isGguf && - gpuDevices.length > 1 && - gpuDevices.every((d) => d.physicalIndex); - const isGpuChecked = (index: number) => - selectedGpuIds === null || selectedGpuIds.includes(index); - const toggleGpu = (index: number) => { - const all = gpuDevices.map((d) => d.index); - const current = selectedGpuIds ?? all; - const next = current.includes(index) - ? current.filter((i) => i !== index) - : [...current, index].sort((a, b) => a - b); - if (next.length === 0) return; // keep at least one GPU selected - setSelectedGpuIds(next.length === all.length ? null : next); - // The per-GPU split is positional, so any change to the set of GPUs in use - // invalidates it: drop it (the sliders fall back to the VRAM-weighted - // default). TP needs 2+ GPUs, so disable it when only one remains. - setSplitRatio(null); - if (next.length <= 1) { - setTensorParallel(false); - } - }; - const gpuIdsKey = (ids: number[] | null) => (ids === null ? "auto" : ids.join(",")); - const gpuIdsDirty = gpuIdsKey(selectedGpuIds) !== gpuIdsKey(loadedGpuIds); - // Per-GPU layer split (--tensor-split): manual + 2+ GPUs in use. One slider - // per GPU, each a layer count; together they sum to the GPU Layers total. - const showSplitRatio = - isManual && !autoLayers && showGpuPicker && gpusInUse.length > 1; - // The total the per-GPU counts sum to (the GPU Layers slider value); 0 under - // Auto, where the split is hidden. The devices behind the GPUs in use, for - // labels + the VRAM-weighted default. - const splitTotal = Math.max(0, Math.min(gpuLayers, gpuLayersMax)); - const gpusInUseDevices = gpusInUse.map( - (i) => gpuDevices.find((d) => d.index === i) ?? null, - ); - // Displayed per-GPU counts. splitRatio is a stable reference balance (only a - // slider edit changes it), rescaled to the current total; deriving rather than - // mutating it on GPU Layers changes keeps the balance intact when the total - // passes through low values or Auto. No saved split: free-VRAM-weighted default - // (llama.cpp's unset default splits by free VRAM, so the first edit starts from - // the default's placement, not a total-VRAM ratio that can land layers on a - // busy GPU). A genuine 0 (a full GPU) is a real weight, not missing data: the - // probe's no-data case degrades to the total server-side, and an all-zero list - // falls back to an even split in distributeByWeight. Not yet sent. - const splitCounts = - splitRatio && splitRatio.length === gpusInUse.length - ? distributeByWeight(splitTotal, splitRatio) - : distributeByWeight( - splitTotal, - gpusInUseDevices.map((d) => d?.memoryFreeGb ?? d?.memoryTotalGb ?? 1), - ); - const setSplitCount = (k: number, v: number) => - setSplitRatio(rebalanceSplit(splitTotal, splitCounts, k, v)); - const splitRatioDirty = - isManual && - !autoLayers && - JSON.stringify(splitRatio ?? null) !== JSON.stringify(loadedSplitRatio ?? null); - // Auto-fit context (Manual + Auto layers): <= 0 means "Auto" (--fit sizes it); - // a positive value pins it. Surface the length --fit chose once it's loaded. - const fitCtxAuto = autoLayers && (customContextLength ?? 0) <= 0; - const loadedAutoLayers = - loadedGpuMemoryMode === "manual" && (loadedGpuLayers ?? GPU_LAYERS_AUTO) < 0; - const fitResolvedCtx = - fitCtxAuto && loadedAutoLayers ? ggufContextLength : null; - // A saved chat-template override is a reload-time setting too, so surface - // Apply for a template-only edit (otherwise it could never be applied). - const templateDirty = chatTemplateOverride !== loadedChatTemplateOverride; - const modelSettingsDirty = - kvDirty || - ctxDirty || - specDirty || - specDraftDirty || - tpDirty || - gpuDirty || - manualDirty || - gpuIdsDirty || - splitRatioDirty || - templateDirty; + const baseContext = ggufContextLength; const [presetNameInput, setPresetNameInput] = useState(activePreset); const [systemPromptEditorOpen, setSystemPromptEditorOpen] = useState(false); const [systemPromptDraft, setSystemPromptDraft] = useState(""); @@ -855,8 +465,7 @@ export function ChatSettingsPanel({ BUILTIN_PRESETS.find((preset) => preset.name === activePreset) ?? null, [activePreset], ); - const hasUnsavedPresetChanges = useMemo( - () => { + const hasUnsavedPresetChanges = useMemo(() => { if (activePresetDefinition == null) { return false; } @@ -864,9 +473,7 @@ export function ChatSettingsPanel({ return activePresetSource === "modified"; } return !isSamePresetConfig(activePresetDefinition.params, params); - }, - [activePresetDefinition, activePresetSource, params], - ); + }, [activePresetDefinition, activePresetSource, params]); const presetSaveState = useMemo( () => getPresetSaveState({ @@ -895,6 +502,14 @@ export function ChatSettingsPanel({ const externalSelection = currentCheckpoint ? parseExternalModelId(currentCheckpoint) : null; + const maxTokensMax = isExternalModel + ? getExternalMaxOutputTokens( + externalProviderType, + externalSelection?.modelId, + ) + : isGguf && baseContext + ? baseContext + : Math.max(64, params.maxSeqLength); const showOpenAICodeExecSection = activeExternalProvider != null && providerSupportsBuiltinCodeExecution( @@ -977,8 +592,7 @@ export function ChatSettingsPanel({ return; } const fallbackPreset = - BUILTIN_PRESETS.find((preset) => preset.name === "Default") ?? - null; + BUILTIN_PRESETS.find((preset) => preset.name === "Default") ?? null; const next = customPresets.filter((preset) => preset.name !== name); setCustomPresets(next); if (activePreset === name) { @@ -1090,7 +704,7 @@ export function ChatSettingsPanel({ Run settings - + - )} -
- )} - {(speculativeType === "mtp" || - speculativeType === "mtp+ngram") && ( -
-
- - Draft Tokens - - - Max MTP draft tokens per step - (--spec-draft-n-max). Lower = less wasted - draft decode; higher = bigger speedup when - acceptance stays high. Default: 2 on GPU, - 3 on CPU/Mac. - -
- { - const raw = e.target.value; - if (raw === "") { - setSpecDraftNMax(null); - return; - } - const parsed = Number.parseInt(raw, 10); - if (Number.isFinite(parsed)) { - const clamped = Math.max(1, Math.min(16, parsed)); - setSpecDraftNMax(clamped); - } - }} - data-test-id="spec-draft-n-max-input" - aria-label="Speculative decoding draft tokens" - className="h-7 w-[88px] rounded-full border-border bg-background hover:bg-accent/50 dark:border-transparent dark:bg-white/[0.05] dark:hover:bg-white/[0.1] pl-3 py-0 text-[13px] font-medium text-nav-fg outline-none focus-visible:ring-0" - /> -
- )} - - )} - {gpuModeApplies && ( -
-
- - GPU Memory - - -
-
- Default: Unsloth - fits the model and context to your GPUs. -
-
- Manual: set GPU - Layers yourself. Leave it on Auto to let llama.cpp size - the context and offload overflow (including MoE experts) - to RAM. -
-
- -
-
- -
-
- )} - {isManual && ( - <> - - Layers to keep on the GPU (--gpu-layers); the rest run - on CPU. Auto lets llama.cpp size the split (and the - context) to fit VRAM. At the maximum, the whole model - is on the GPU. - - } - /> - {showMoeSlider && ( - - Keep the experts of this many MoE layers on the CPU - (--n-cpu-moe) to save VRAM. 0 = all experts on the - GPU; at the maximum, all are on the CPU. - - } - /> - )} - {showSplitRatio && ( -
-
- - Layers per GPU - - - Splits GPU Layers across GPUs (--tensor-split). - Without Tensor Parallelism each value is the layer - count on that GPU; with it, every GPU holds a slice - of each layer, so the values are only a ratio. - -
- {gpusInUseDevices.map((d, k) => ( - setSplitCount(k, v)} - valueSize={6} - disabled={modelControlsDisabled} - /> - ))} -
- )} - - )} - {showGpuPicker && ( -
-
- - GPUs - - - Which GPUs this model may use. Unchecked GPUs are hidden - from llama.cpp (CUDA_VISIBLE_DEVICES, or - HIP_VISIBLE_DEVICES on ROCm). Leave all checked to use - every GPU. At least one GPU must stay selected. - -
-
- {gpuDevices.map((d) => ( -
- - GPU {d.index}: {d.name} - {d.memoryTotalGb - ? ` · ${Math.round(d.memoryTotalGb)} GB` - : ""} - - toggleGpu(d.index)} - data-test-id={`gpu-pick-${d.index}`} - disabled={ - modelControlsDisabled || - (isGpuChecked(d.index) && singleGpuInUse) - } - /> -
- ))} -
-
- )} - {gpuModeApplies && !autoLayers && ( -
-
- - Tensor Parallelism - - - No effect on a single GPU. On multi-GPU setups, improves - tokens/sec during generation when using dense models. MoE - models don't benefit and can be much slower. - -
- -
- )} - - )} - {/* No persistent "enable custom code" toggle: it is consented per model - via the load-time review dialog. */} - {/* Apply/Reset belongs to the model-reload settings above (context - length, KV cache, speculative decoding). Render it here, before - the Chat Template row, so it never reads as attached to Chat - Template (which is edited via its own dialog). When a model is - staged (deferred load), Load/Cancel takes its place: there's - nothing loaded to "apply" against yet. */} - {pendingSelection ? ( -
- {stagedDownloading && ( -

- Downloading…{" "} - {Math.round((stagedDownloadFraction ?? 0) * 100)}% + : "" + }`}

- )} - {/* GGUF picks only: a non-GGUF pick shows none of the load - knobs the blob captures, so there is nothing to remember. */} - {pendingIsGguf && ( - - )} - {stagedLoading ? ( - // Mid-load: nothing to load or abandon until it settles, so disable. - - ) : ( -
+ {mtpUpdatable && llamaUpdateStatus?.update_available && ( - -
- )} -
- ) : modelSettingsDirty ? ( -
- - -
- ) : null} - {/* The template override is a load-time knob too (applied on the next - reload) and the in-flight load already snapshotted it, so lock its - editors like the sibling controls -- a mid-load save would be - silently clobbered by the load response despite its toast. */} - - - + )} + + )} + {showContextVramWarning && ( +

+ Context length exceeds the estimated VRAM capacity ( + {ggufMaxContextLength?.toLocaleString()} tokens). The + model may use system RAM. +

+ )} + + )}
- +
savePresetWithName(presetNameInput)} disabled={!(settingsHydrated && presetSaveState.canSubmit)} - variant={presetSaveState.isSaveReady ? "default" : "outline"} + variant={ + presetSaveState.isSaveReady ? "default" : "outline" + } size="sm" className={cn( "h-9 w-full rounded-full text-[13px] font-medium tracking-nav", @@ -1850,7 +912,8 @@ export function ChatSettingsPanel({ Prompt caching - Reuse compatible prompt prefixes for lower latency and cost. + Reuse compatible prompt prefixes for lower latency and + cost.
Anthropic exposes a 5 minute and a 1 hour ephemeral - cache pool. The 1 hour pool costs 2x base input on - write vs 1.25x for 5 minute, but reads stay 0.1x for - both, so a single read landing more than 5 minutes - after the write pays off the premium. + cache pool. The 1 hour pool costs 2x base input on write + vs 1.25x for 5 minute, but reads stay 0.1x for both, so + a single read landing more than 5 minutes after the + write pays off the premium.