diff --git a/studio/backend/core/inference/llama_cpp.py b/studio/backend/core/inference/llama_cpp.py index f76afce9f4..4f2d8cd54a 100644 --- a/studio/backend/core/inference/llama_cpp.py +++ b/studio/backend/core/inference/llama_cpp.py @@ -2181,6 +2181,8 @@ class LlamaCppBackend: self._effective_context_length: Optional[int] = None self._max_context_length: Optional[int] = None self._effective_parallel_slots: int = 1 + # --parallel the last load asked for, before any fit-time reduction. + self._requested_n_parallel: int = 1 self._chat_template: Optional[str] = None self._chat_template_override: Optional[str] = None self._supports_reasoning: bool = False @@ -2417,6 +2419,17 @@ class LlamaCppBackend: slots = 1 return max(1, slots) + @property + def requested_parallel_slots(self) -> int: + """--parallel the last load asked for, before any fit-time reduction. + The reload dedupe compares requested-vs-requested (like requested_n_ctx); + the effective count would reload forever after a fitter reduction.""" + try: + slots = int(getattr(self, "_requested_n_parallel", 1)) + except (TypeError, ValueError): + slots = 1 + return max(1, slots) + @property def max_context_length(self) -> Optional[int]: """Return the largest context that fits on this hardware at load time. @@ -2442,6 +2455,8 @@ class LlamaCppBackend: def _reset_effective_parallel_slots(self) -> None: self._effective_parallel_slots = 1 + # Cleared with the effective count so a stale value can't skew the dedupe. + self._requested_n_parallel = 1 @staticmethod def _read_rss_bytes(pid: int) -> Optional[int]: @@ -6787,6 +6802,7 @@ class LlamaCppBackend: chat_template_override = chat_template_override, extra_args = extra_args, is_vision = is_vision, + n_parallel = n_parallel, preserve_multi_gpu_on_layer = preserve_multi_gpu_on_layer, ): logger.info( @@ -9066,6 +9082,8 @@ class LlamaCppBackend: self._extra_args = list(extra_args) self._extra_args_source = (model_identifier, hf_variant) self._requested_n_ctx = int(n_ctx) + # Local n_parallel may have been reduced above; the snapshot has the ask. + self._requested_n_parallel = max(1, int(_pending_load_kwargs["n_parallel"])) # Commit the known-good snapshot + whether MTP+tensor is live, then # watch this load for a mid-generation crash. self._last_load_kwargs = _pending_load_kwargs @@ -9478,6 +9496,7 @@ class LlamaCppBackend: tensor_split: Optional[List[float]] = None, gpu_ids: Optional[List[int]] = None, mtp_draft_path: Optional[str] = None, + n_parallel: int = 1, preserve_multi_gpu_on_layer: bool = False, ) -> bool: """True iff the live server already satisfies these load kwargs. @@ -9542,6 +9561,10 @@ class LlamaCppBackend: # A GPU-memory-mode flip (Unsloth / manual) must always reload. if self._gpu_memory_mode != gpu_memory_mode: return False + # Requested-vs-requested (like n_ctx): comparing the effective count + # would reload forever whenever the fitter launched fewer slots. + if self._requested_n_parallel != max(1, int(n_parallel)): + return False # Manual: a layer-count change always reloads (covers Auto(-1) <-> a # pinned count); MoE/split only matter with an explicit offload. if gpu_memory_mode == "manual" and ( diff --git a/studio/backend/core/inference/llama_server_args.py b/studio/backend/core/inference/llama_server_args.py index 2ecd7e3e2e..7391e62516 100644 --- a/studio/backend/core/inference/llama_server_args.py +++ b/studio/backend/core/inference/llama_server_args.py @@ -16,11 +16,18 @@ from __future__ import annotations import os from typing import Iterable, Mapping, Optional +# Valid llama-server --parallel range, shared with LoadRequest.n_parallel. +# Mirrored by callers that cannot import this: run.py and unsloth_cli/commands/ +# studio.py (_PARALLEL_MIN/MAX), per-model-config.ts (N_PARALLEL_MIN/MAX); +# test_parallel_slots_per_load.py pins them together. +PARALLEL_MIN = 1 +PARALLEL_MAX = 64 + # Each group = every alias (short + long) of one hard-denied flag. # Extend the matching group when llama.cpp adds a new alias. _DENYLIST_GROUPS: tuple[frozenset[str], ...] = ( - # Parallel slots: owned by typer --parallel; a pass-through would desync - # app.state.llama_parallel_slots from llama-server. + # Parallel slots: owned by typer --parallel and LoadRequest.n_parallel; a + # pass-through would desync the slot bookkeeping from llama-server. frozenset({"-np", "--parallel", "--n-parallel"}), # Model identity: Unsloth resolves it from LoadRequest; a second -m would # load a different model than Unsloth thinks it loaded. diff --git a/studio/backend/models/inference.py b/studio/backend/models/inference.py index acd60dd0b9..0edd1aa37f 100644 --- a/studio/backend/models/inference.py +++ b/studio/backend/models/inference.py @@ -18,6 +18,7 @@ from pydantic import ( model_validator, ) +from core.inference.llama_server_args import PARALLEL_MAX, PARALLEL_MIN from picker.schemas import MAX_CHAT_TEMPLATE_BYTES @@ -113,6 +114,18 @@ class LoadRequest(BaseModel): "'mtp' or 'mtp+ngram'." ), ) + n_parallel: Optional[int] = Field( + None, + ge = PARALLEL_MIN, + le = PARALLEL_MAX, + description = ( + "Parallel decode slots for llama-server (--parallel) for this " + f"load ({PARALLEL_MIN}..{PARALLEL_MAX}). Omit for the server-wide " + "default set at launch (the --parallel CLI flag). The VRAM fitter " + "may launch fewer slots to keep the model fully on GPU. Ignored " + "for non-GGUF models." + ), + ) tensor_parallel: bool = Field( False, description = ( @@ -265,6 +278,16 @@ class ValidateModelRequest(BaseModel): "delegate fitting to llama.cpp, while explicit layers are user-owned." ), ) + n_parallel: Optional[int] = Field( + None, + ge = PARALLEL_MIN, + le = PARALLEL_MAX, + description = ( + "Parallel decode slots intended for the follow-up load, so the " + "coexistence estimate sizes the KV cache like /load. Omit for the " + "server-wide --parallel default." + ), + ) include_context_length: bool = Field( False, description = "Also read the native context length from the local GGUF header. " @@ -533,6 +556,23 @@ class LoadResponse(BaseModel): "or None for automatic selection." ), ) + requested_parallel_slots: Optional[int] = Field( + None, + description = ( + "Parallel decode slots the load was invoked with (per-load " + "n_parallel, else the server-wide --parallel default). None for " + "non-GGUF loads and for the diffusion runner, which ignores " + "--parallel." + ), + ) + parallel_slots: Optional[int] = Field( + None, + description = ( + "Serving slots the active llama-server actually runs (--parallel " + "after any fit-time slot reduction). None for non-GGUF loads and " + "for the diffusion runner, which ignores --parallel." + ), + ) class UnloadResponse(BaseModel): @@ -708,6 +748,23 @@ class InferenceStatusResponse(BaseModel): "or None for automatic selection." ), ) + requested_parallel_slots: Optional[int] = Field( + None, + description = ( + "Parallel decode slots the active load was invoked with (per-load " + "n_parallel, else the server-wide --parallel default). None when " + "no GGUF model is loaded and for the diffusion runner, which " + "ignores --parallel." + ), + ) + parallel_slots: Optional[int] = Field( + None, + description = ( + "Serving slots the active llama-server actually runs (--parallel " + "after any fit-time slot reduction). None when no GGUF model is " + "loaded and for the diffusion runner, which ignores --parallel." + ), + ) llama_cpp_supports_mtp: bool = Field( True, description = ( diff --git a/studio/backend/routes/chat_history.py b/studio/backend/routes/chat_history.py index aa59716315..4180518837 100644 --- a/studio/backend/routes/chat_history.py +++ b/studio/backend/routes/chat_history.py @@ -11,6 +11,7 @@ from fastapi import APIRouter, Depends, HTTPException, Query, Request from pydantic import BaseModel, ConfigDict, Field, ValidationError from auth.authentication import get_current_subject +from core.inference.llama_server_args import PARALLEL_MAX, PARALLEL_MIN from loggers import get_logger from utils.utils import safe_curated_detail, log_and_http_error from storage.studio_db import ( @@ -169,6 +170,7 @@ class ChatPresetLoadConfig(BaseModel): kvCacheDtype: Optional[str] = None speculativeType: Optional[str] = None specDraftNMax: Optional[int] = Field(default = None, ge = 1, le = 16) + nParallel: Optional[int] = Field(default = None, ge = PARALLEL_MIN, le = PARALLEL_MAX) tensorParallel: Optional[bool] = None gpuMemoryMode: Optional[Literal["manual"]] = None gpuLayers: Optional[int] = None diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.py index 53b4136e32..12547277f5 100644 --- a/studio/backend/routes/inference.py +++ b/studio/backend/routes/inference.py @@ -3294,10 +3294,25 @@ def _is_explicit_tensor_drop(request: LoadRequest) -> bool: return override is not None and override.strip().lower() != "tensor" +def _parallel_slot_echo(llama_backend: LlamaCppBackend) -> dict: + """requested/effective parallel-slot fields for /load and /status echoes. + + The diffusion runner ignores ``--parallel`` and never commits a count, so it + reports None like the non-GGUF paths; echoing the reset placeholder 1 would + fabricate an "invoked with 1 slot".""" + if llama_backend.is_diffusion: + return {"requested_parallel_slots": None, "parallel_slots": None} + return { + "requested_parallel_slots": llama_backend.requested_parallel_slots, + "parallel_slots": llama_backend.effective_parallel_slots, + } + + def _request_matches_loaded_settings( request: LoadRequest, llama_backend: LlamaCppBackend, effective_chat_template_override: Optional[str] = None, + requested_parallel_slots: Optional[int] = None, ) -> bool: """True iff every runtime setting on the request matches the loaded server. Caller has already checked model+variant+is_loaded. See #5401. @@ -3306,11 +3321,22 @@ def _request_matches_loaded_settings( launched (user override, else a bundled family template such as the gemma-4 override), so the dedup compares against what the backend actually holds rather than the raw request field. Defaults to the request field for - callers that do not resolve a bundled override.""" + callers that do not resolve a bundled override. + + ``requested_parallel_slots`` is the resolved count the load would use + (per-load ``n_parallel``, else the server-wide default); None skips it.""" # Compare requested n_ctx (not effective) so VRAM-cap doesn't mask an # Auto-vs-explicit slider flip. if request.max_seq_length != llama_backend.requested_n_ctx: return False + # Requested-vs-requested for the same reason: the fitter may launch fewer + # slots. Diffusion ignores --parallel, so a change there must not reload. + if ( + requested_parallel_slots is not None + and not llama_backend.is_diffusion + and int(requested_parallel_slots) != llama_backend.requested_parallel_slots + ): + return False if _normalise_settings_str(request.cache_type_kv) != _normalise_settings_str( llama_backend.cache_type_kv ): @@ -4730,6 +4756,20 @@ def _guard_chat_load_against_training( cpu_only = LlamaCppBackend._effective_gpu_count() == 0, ) + # Size with the count that will actually launch, or a load that fits gets a + # 409: diffusion never receives --parallel, and load_model clamps to 1 on an + # llama-server without --kv-unified. An unclassified GGUF keeps the ask. + if is_gguf and n_parallel > 1: + if diffusion_kind is True: + n_parallel = 1 + else: + try: + caps = LlamaCppBackend.probe_server_capabilities() + if caps.get("found") and not caps.get("supports_kv_unified"): + n_parallel = 1 + except Exception as e: + logger.warning("Could not probe llama-server slots for chat-load guard: %s", e) + required_override_gb = ( _estimate_gguf_required_gb( config, @@ -5272,6 +5312,17 @@ async def _load_model_impl( backend = get_inference_backend() llama_backend = get_llama_cpp_backend() + # Resolve the slot count once (per-load field, else the server-wide + # --parallel default) so the dedupe, the training guard and the load + # kwargs all size against what launches. app.state stays the launch + # intent / admission fallback; getattr because direct callers have no app. + _app_state = getattr(getattr(fastapi_request, "app", None), "state", None) + _n_parallel = ( + request.n_parallel + if request.n_parallel is not None + else getattr(_app_state, "llama_parallel_slots", 1) + ) + is_direct_gguf_request = model_identifier.lower().endswith(".gguf") if request.gguf_variant or is_direct_gguf_request: gguf_variant_matches = is_direct_gguf_request or bool( @@ -5289,6 +5340,7 @@ async def _load_model_impl( request, llama_backend, effective_chat_template_override, + requested_parallel_slots = _n_parallel, ) # Skip if a prior audio probe failed -- let load_model retry. and getattr(llama_backend, "_audio_probed", True) @@ -5343,6 +5395,7 @@ async def _load_model_impl( n_moe_layers = llama_backend.n_moe_layers, gpu_ids = llama_backend.gpu_ids, requested_gpu_ids = llama_backend.requested_gpu_ids, + **_parallel_slot_echo(llama_backend), ) else: if ( @@ -5481,7 +5534,7 @@ async def _load_model_impl( max_seq_length = request.max_seq_length, requested_gpu_ids = effective_gpu_ids, llama_extra_args = extra_llama_args, - n_parallel = getattr(fastapi_request.app.state, "llama_parallel_slots", 1), + n_parallel = _n_parallel, cache_type_kv = request.cache_type_kv, tensor_parallel = bool(request.tensor_parallel), gpu_memory_mode = request.gpu_memory_mode, @@ -5558,7 +5611,6 @@ async def _load_model_impl( # Route to HF or local mode based on config. Run in a thread so the # event loop stays free for progress polling and other requests # during the (potentially long) GGUF download + llama-server start. - _n_parallel = getattr(fastapi_request.app.state, "llama_parallel_slots", 1) # Load kwargs common to HF and local modes; the two differ only by # the model-source args (hf_repo/-token vs gguf_path/mmproj). @@ -5756,6 +5808,7 @@ async def _load_model_impl( n_moe_layers = llama_backend.n_moe_layers, gpu_ids = llama_backend.gpu_ids, requested_gpu_ids = llama_backend.requested_gpu_ids, + **_parallel_slot_echo(llama_backend), ) # ── Standard path: load via Unsloth/transformers ────────── @@ -6156,9 +6209,14 @@ async def validate_model( requested_gpu_ids = effective_gpu_ids, llama_extra_args = effective_extra_args, n_parallel = ( - getattr(fastapi_request.app.state, "llama_parallel_slots", 1) - if fastapi_request is not None - else 1 + request.n_parallel + if request.n_parallel is not None + # Same getattr chain as the load path: preflight must size like the load. + else getattr( + getattr(getattr(fastapi_request, "app", None), "state", None), + "llama_parallel_slots", + 1, + ) ), cache_type_kv = request.cache_type_kv, tensor_parallel = request.tensor_parallel, @@ -6987,6 +7045,7 @@ async def get_status(current_subject: str = Depends(get_current_subject)): n_moe_layers = llama_backend.n_moe_layers, gpu_ids = llama_backend.gpu_ids, requested_gpu_ids = llama_backend.requested_gpu_ids, + **_parallel_slot_echo(llama_backend), llama_cpp_supports_mtp = _supports_mtp, spec_fallback_reason = llama_backend.spec_fallback_reason, llama_cpp_prebuilt_stale = _stale, diff --git a/studio/backend/run.py b/studio/backend/run.py index 8ef1ac06b8..08d1c5299e 100644 --- a/studio/backend/run.py +++ b/studio/backend/run.py @@ -1920,7 +1920,8 @@ def _build_arg_parser(): default = _PARALLEL_DEFAULT_PLAIN, help = ( f"llama-server parallel decode slots ({_PARALLEL_MIN}..{_PARALLEL_MAX}). " - f"Default {_PARALLEL_DEFAULT_PLAIN}." + f"Default {_PARALLEL_DEFAULT_PLAIN}. The Studio run settings " + "(Parallel Slots) override it per load." ), ) return parser diff --git a/studio/backend/tests/test_llama_server_args.py b/studio/backend/tests/test_llama_server_args.py index b2ec5034ac..83934e4130 100644 --- a/studio/backend/tests/test_llama_server_args.py +++ b/studio/backend/tests/test_llama_server_args.py @@ -77,8 +77,7 @@ validate_extra_args = _lsa.validate_extra_args ["--reasoning-format", "deepseek"], ["-rea", "auto"], # Soft-managed: user flags last-wins over Unsloth's auto-set version. - # --parallel / -np / --n-parallel are hard-denied (KV-cache + slot - # count would desync); use `unsloth studio run --parallel N` instead. + # --parallel / -np / --n-parallel are hard-denied; use Parallel Slots. ["-c", "131072"], ["--ctx-size", "8192"], ["--flash-attn", "off"], @@ -128,7 +127,7 @@ def test_non_flag_token_passes_through(): @pytest.mark.parametrize( "denied", [ - # Parallel slots -- owned by the typer --parallel flag. + # Parallel slots -- owned by typer --parallel and LoadRequest.n_parallel. "-np", "--parallel", "--n-parallel", @@ -201,9 +200,8 @@ def test_denylist_rejects_all_aliases(denied): @pytest.mark.parametrize( "args,offending", [ - # Pass-through --parallel would last-wins-override the real slot - # count while Unsloth's KV-cache fit + llama_parallel_slots stay at - # the typer value -- plan vs. process disagree. + # Pass-through --parallel would last-wins-override the real slot count + # while the KV-cache fit and slot bookkeeping stay at the resolved value. (["--parallel", "8"], "--parallel"), (["--parallel=8"], "--parallel"), (["--n-parallel", "16"], "--n-parallel"), @@ -213,7 +211,7 @@ def test_denylist_rejects_all_aliases(denied): # `["-np8"]` must still resolve to managed. (["-np8"], "-np"), (["-np64"], "-np"), - # Out-of-range values that would bypass the typer 1..64 guard. + # Out-of-range values that would bypass the PARALLEL_MIN/MAX bounds. (["--parallel", "999"], "--parallel"), (["-np", "0"], "-np"), (["-np999"], "-np"), @@ -300,7 +298,7 @@ def test_is_managed_flag_true_for_denied(): assert is_managed_flag("--api-key") is True assert is_managed_flag("-m") is True assert is_managed_flag("--model") is True - # Parallel slots owned by the typer --parallel flag. + # Parallel slots owned by typer --parallel and LoadRequest.n_parallel. assert is_managed_flag("--parallel") is True assert is_managed_flag("--n-parallel") is True assert is_managed_flag("-np") is True diff --git a/studio/backend/tests/test_parallel_slots_per_load.py b/studio/backend/tests/test_parallel_slots_per_load.py new file mode 100644 index 0000000000..f4f2d31c6f --- /dev/null +++ b/studio/backend/tests/test_parallel_slots_per_load.py @@ -0,0 +1,517 @@ +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 + +"""Backend contract for the per-load parallel-slots knob. + +An optional ``n_parallel`` (llama-server ``--parallel``) rides on LoadRequest; +omitted, the server-wide launch default (``run.py --parallel``) applies. These +tests pin the pydantic contract and the shared PARALLEL_MIN/MAX mirrors, the +``requested_parallel_slots`` lifecycle, the ``_already_in_target_state`` +requested-vs-requested reload branch with its diffusion skip, and the route +wiring behind the /load, /validate and /status echoes. +""" + +from __future__ import annotations + +import inspect +import re +import struct +import sys +import types as _types +from pathlib import Path + +import pytest + +_BACKEND_DIR = str(Path(__file__).resolve().parent.parent) +if _BACKEND_DIR not in sys.path: + sys.path.insert(0, _BACKEND_DIR) + +# Same external-dep stubs as the other llama_cpp unit tests. +_loggers_stub = _types.ModuleType("loggers") +_loggers_stub.get_logger = lambda name: __import__("logging").getLogger(name) +sys.modules.setdefault("loggers", _loggers_stub) + +_structlog_stub = _types.ModuleType("structlog") +_structlog_stub.get_logger = lambda *a, **k: __import__("logging").getLogger("stub") +sys.modules.setdefault("structlog", _structlog_stub) + +# Real httpx: a stub would poison a combined run (routes/inference reads its +# attrs at def time). +import httpx # noqa: F401 + +from core.inference import llama_cpp as llama_cpp_module +from core.inference.llama_server_args import PARALLEL_MAX, PARALLEL_MIN +from core.inference.llama_cpp import LlamaCppBackend +from models.inference import ( + InferenceStatusResponse, + LoadRequest, + LoadResponse, + ValidateModelRequest, +) + + +class _FakeProcess: + def terminate(self): + pass + + def wait(self, timeout = None): + return 0 + + def kill(self): + pass + + def poll(self): + return 0 + + +# ── Pydantic contract ──────────────────────────────────────────────── + + +def test_load_request_defaults_n_parallel_none(): + assert LoadRequest(model_path = "owner/repo").n_parallel is None + + +@pytest.mark.parametrize("value", [PARALLEL_MIN, 4, PARALLEL_MAX]) +def test_load_request_accepts_in_range_n_parallel(value): + assert LoadRequest(model_path = "owner/repo", n_parallel = value).n_parallel == value + + +@pytest.mark.parametrize("value", [0, -1, PARALLEL_MAX + 1]) +def test_load_request_rejects_out_of_range_n_parallel(value): + with pytest.raises(ValueError): + LoadRequest(model_path = "owner/repo", n_parallel = value) + + +def test_load_request_round_trips_json_key(): + req = LoadRequest.model_validate({"model_path": "owner/repo", "n_parallel": 8}) + assert req.n_parallel == 8 + assert req.model_dump()["n_parallel"] == 8 + + +def test_validate_request_n_parallel_contract(): + # /validate sizes like /load, so it carries the same field and bounds. + assert ValidateModelRequest(model_path = "owner/repo").n_parallel is None + assert ( + ValidateModelRequest(model_path = "owner/repo", n_parallel = PARALLEL_MAX).n_parallel + == PARALLEL_MAX + ) + with pytest.raises(ValueError): + ValidateModelRequest(model_path = "owner/repo", n_parallel = PARALLEL_MAX + 1) + + +@pytest.mark.parametrize("model_cls", [LoadResponse, InferenceStatusResponse]) +def test_response_models_emit_parallel_slot_fields(model_cls): + kwargs = ( + dict(status = "loaded", model = "owner/repo", display_name = "repo", inference = {}) + if model_cls is LoadResponse + else {} + ) + empty = model_cls(**kwargs).model_dump() + assert empty["requested_parallel_slots"] is None + assert empty["parallel_slots"] is None + dumped = model_cls(**kwargs, requested_parallel_slots = 8, parallel_slots = 4).model_dump() + assert dumped["requested_parallel_slots"] == 8 + assert dumped["parallel_slots"] == 4 + + +# ── Shared bounds and their deliberate mirrors ─────────────────────── + + +def _mirrored_bounds(source_path: Path) -> tuple[int, int]: + src = source_path.read_text(encoding = "utf-8") + low = re.search(r"^_PARALLEL_MIN\s*=\s*(\d+)$", src, re.MULTILINE) + high = re.search(r"^_PARALLEL_MAX\s*=\s*(\d+)$", src, re.MULTILINE) + assert low and high, f"{source_path} must define _PARALLEL_MIN/_PARALLEL_MAX" + return int(low.group(1)), int(high.group(1)) + + +def test_run_py_mirror_matches_shared_bounds(): + assert _mirrored_bounds(Path(_BACKEND_DIR) / "run.py") == (PARALLEL_MIN, PARALLEL_MAX) + + +def test_cli_mirror_matches_shared_bounds(): + cli = Path(_BACKEND_DIR).parent.parent / "unsloth_cli" / "commands" / "studio.py" + assert _mirrored_bounds(cli) == (PARALLEL_MIN, PARALLEL_MAX) + + +def test_frontend_mirror_matches_shared_bounds(): + # The UI clamps with its own copy; a bumped PARALLEL_MAX that skips it would + # leave the UI silently capping lower. + src = ( + Path(_BACKEND_DIR).parent + / "frontend" + / "src" + / "features" + / "model-picker" + / "model-config" + / "per-model-config.ts" + ).read_text(encoding = "utf-8") + low = re.search(r"^export const N_PARALLEL_MIN = (\d+);$", src, re.MULTILINE) + high = re.search(r"^export const N_PARALLEL_MAX = (\d+);$", src, re.MULTILINE) + assert low and high, "per-model-config.ts must export N_PARALLEL_MIN/MAX" + assert (int(low.group(1)), int(high.group(1))) == (PARALLEL_MIN, PARALLEL_MAX) + + +def test_preset_model_reuses_shared_bounds(): + # Bounds drifting from PARALLEL_MIN/MAX would 422 valid presets on every sync. + from routes.chat_history import ChatPresetLoadConfig + + field = ChatPresetLoadConfig.model_fields["nParallel"] + bounds = {type(m).__name__: getattr(m, "ge", getattr(m, "le", None)) for m in field.metadata} + assert bounds.get("Ge") == PARALLEL_MIN + assert bounds.get("Le") == PARALLEL_MAX + + +# ── requested_parallel_slots lifecycle ─────────────────────────────── + + +@pytest.fixture +def backend(monkeypatch): + monkeypatch.setattr(LlamaCppBackend, "_kill_orphaned_servers", lambda self: 0) + monkeypatch.setattr(llama_cpp_module.atexit, "register", lambda *_args, **_kwargs: None) + return LlamaCppBackend() + + +def test_requested_parallel_slots_initial_value_is_one(backend): + assert backend.requested_parallel_slots == 1 + + +def test_requested_parallel_slots_reflects_field(backend): + backend._requested_n_parallel = 8 + assert backend.requested_parallel_slots == 8 + + +@pytest.mark.parametrize("value", [None, 0, -2, "not-an-int"]) +def test_requested_parallel_slots_invalid_value_falls_back_to_one(backend, value): + backend._requested_n_parallel = value + assert backend.requested_parallel_slots == 1 + + +def test_reset_effective_parallel_slots_also_resets_requested(backend): + backend._requested_n_parallel = 8 + backend._commit_effective_parallel_slots(4) + + backend._reset_effective_parallel_slots() + + assert backend.requested_parallel_slots == 1 + assert backend.effective_parallel_slots == 1 + + +def test_unload_resets_requested_parallel_slots(backend): + backend._process = _FakeProcess() + backend._requested_n_parallel = 8 + + backend.unload_model() + + assert backend.requested_parallel_slots == 1 + + +def test_load_model_commits_requested_from_pending_kwargs(): + # n_parallel may be reduced before the commit, so the requested value must + # come from the pre-reduction pending snapshot. + src = inspect.getsource(LlamaCppBackend.load_model) + commit = src.find( + 'self._requested_n_parallel = max(1, int(_pending_load_kwargs["n_parallel"]))' + ) + healthy = src.find("self._healthy = True\n", 0, commit if commit != -1 else None) + snapshot = src.find("self._last_load_kwargs = _pending_load_kwargs") + assert commit != -1, "load_model must commit the requested slot count" + assert healthy != -1 and healthy < commit < snapshot + + +# ── _already_in_target_state requested-vs-requested branch ─────────── + + +def _loaded_backend() -> LlamaCppBackend: + backend = LlamaCppBackend() + backend._process = _FakeProcess() # is_loaded only checks "is not None" + backend._healthy = True + backend._model_identifier = "owner/repo" + backend._hf_variant = "Q4_K_M" + backend._requested_n_ctx = 8192 + backend._cache_type_kv = None + backend._requested_spec_mode = "auto" + backend._chat_template_override = None + backend._is_vision = False + backend._extra_args = None + backend._gguf_path = None + return backend + + +def _target_state(backend: LlamaCppBackend, n_parallel: int) -> bool: + return backend._already_in_target_state( + gguf_path = None, + model_identifier = "owner/repo", + hf_variant = "Q4_K_M", + n_ctx = 8192, + cache_type_kv = None, + speculative_type = "auto", + chat_template_override = None, + extra_args = None, + is_vision = False, + n_parallel = n_parallel, + ) + + +def test_already_in_target_state_matches_same_slots(): + backend = _loaded_backend() + backend._requested_n_parallel = 4 + assert _target_state(backend, 4) is True + + +def test_already_in_target_state_reloads_on_slots_change(): + backend = _loaded_backend() + backend._requested_n_parallel = 4 + assert _target_state(backend, 8) is False + + +def test_already_in_target_state_compares_requested_not_effective(): + # An identical re-Apply must dedupe even after the fitter reduced the slots. + backend = _loaded_backend() + backend._requested_n_parallel = 8 + backend._commit_effective_parallel_slots(4) + assert _target_state(backend, 8) is True + + +def test_already_in_target_state_ignores_slots_for_diffusion(): + # The diffusion runner ignores --parallel, so a slots change must not reload. + backend = _loaded_backend() + backend._is_diffusion = True + backend._requested_n_parallel = 1 + assert _target_state(backend, 8) is True + + +# ── Route wiring (source contract, mirroring test_gpu_memory_mode) ─── + + +def _route_source() -> str: + return (Path(_BACKEND_DIR) / "routes" / "inference.py").read_text(encoding = "utf-8") + + +def _load_impl_source() -> str: + """Body of _load_model_impl only, so positional assertions can't be + satisfied by a later function in the module.""" + src = _route_source() + body = src[src.index("async def _load_model_impl") :] + return body[: body.index("\n@router.")] + + +def test_route_resolves_slots_once_before_dedupe_guard_and_load(): + load_impl = _load_impl_source() + resolve = load_impl.index("request.n_parallel") + fallback = load_impl.index('getattr(_app_state, "llama_parallel_slots", 1)') + dedupe = load_impl.index("requested_parallel_slots = _n_parallel") + guard = load_impl.index("_guard_chat_load_against_training") + # The GGUF launch kwargs, not the guard's own kwarg (which shares the spelling). + load_kwargs = load_impl.index("_common_load_kwargs = dict(") + assert resolve < dedupe, "resolution must precede the reload dedupe" + assert fallback < dedupe + assert resolve < guard < load_kwargs + # Guard and load kwargs share the resolved value; app.state is read once. + assert load_impl.count("n_parallel = _n_parallel") == 2 + assert "n_parallel = _n_parallel" in load_impl[load_kwargs : load_kwargs + 800] + assert load_impl.count('getattr(_app_state, "llama_parallel_slots", 1)') == 1 + # getattr, so a direct caller without an app cannot raise, and no re-read. + assert "fastapi_request.app.state" not in load_impl + + +def test_route_dedupe_compares_requested_slots_and_skips_diffusion(): + match_impl = _route_source()[_route_source().index("def _request_matches_loaded_settings") :] + match_impl = match_impl[: match_impl.index("\ndef ")] + assert "requested_parallel_slots is not None" in match_impl + assert "not llama_backend.is_diffusion" in match_impl + assert "llama_backend.requested_parallel_slots" in match_impl + + +def test_route_echoes_requested_and_effective_slots(): + route_src = _route_source() + # Both /load returns plus the /status GGUF branch, via the shared helper. + assert route_src.count("**_parallel_slot_echo(llama_backend)") == 3 + + +def test_parallel_slot_echo_reports_none_for_diffusion(): + # Diffusion never commits a count, so echoing the reset placeholder 1 would lie. + from routes.inference import _parallel_slot_echo + + backend = _loaded_backend() + backend._requested_n_parallel = 8 + backend._commit_effective_parallel_slots(4) + assert _parallel_slot_echo(backend) == {"requested_parallel_slots": 8, "parallel_slots": 4} + backend._is_diffusion = True + assert _parallel_slot_echo(backend) == { + "requested_parallel_slots": None, + "parallel_slots": None, + } + + +def test_validate_route_prefers_request_n_parallel(): + validate_impl = _route_source()[_route_source().index("async def validate_model") :] + resolve = validate_impl.index("request.n_parallel") + fallback = validate_impl.index('"llama_parallel_slots",') + guard = validate_impl.index("_guard_chat_load_against_training") + assert guard < resolve and guard < fallback, "the guard call resolves the slots inline" + + +def _load_model_source() -> str: + return inspect.getsource(LlamaCppBackend.load_model) + + +def test_slots_fall_back_to_one_without_kv_unified(): + # Without --kv-unified llama-server gives each slot -c/N, so an explicit + # --parallel N shrinks every context window. + src = _load_model_source() + clamp = src.find("supports_kv_unified") + assert clamp != -1, "load_model must check for --kv-unified before honouring the slots" + block = src[clamp : clamp + 700] + assert ( + "n_parallel > 1" in src[clamp - 300 : clamp] + ), "only an explicit multi-slot load is clamped" + assert "n_parallel = 1" in block + + +def test_clamp_sits_between_the_echo_and_the_fit(): + # The echo reports the ask and the fit uses what launches, so the clamp + # belongs between the two. + src = _load_model_source() + pending = src.index("_pending_load_kwargs") + clamp = src.index("supports_kv_unified") + estimate = src.index("_estimate") + commit = src.index("_commit_effective_parallel_slots") + assert pending < clamp, "the requested count is captured before the clamp" + assert clamp < estimate, "the fit must be estimated from the effective slot count" + assert clamp < commit, "the committed effective count is the clamped one" + + +# ── Training-guard sizing ──────────────────────────────────────────── + + +def _write_swa_gguf(path: Path) -> str: + """Smallest DiffusionGemma-shaped header the KV estimator can size: the + canvas marker routing it to the diffusion runner, plus the sliding-window + dims that make llama.cpp's SWA cache slot-scaled.""" + + def _kv_str(key: str, value: str) -> bytes: + kb, vb = key.encode(), value.encode() + return ( + struct.pack(" bytes: + kb = key.encode() + return struct.pack(" float: + """Run the training guard over a local GGUF and return the size it budgeted.""" + import routes.inference as inf + + seen = {} + + core_training = _types.ModuleType("core.training") + core_training.get_training_backend = lambda: _types.SimpleNamespace( + is_training_active = lambda: True + ) + + def _can_load(**kwargs): + seen.update(kwargs) + return True, {"mode": "single_device"} + + training_vram = _types.ModuleType("routes.training_vram") + training_vram.can_load_chat_during_training = _can_load + monkeypatch.setitem(sys.modules, "core.training", core_training) + monkeypatch.setitem(sys.modules, "routes.training_vram", training_vram) + + monkeypatch.setattr(inf, "_classify_diffusion_gguf", lambda _config: diffusion) + monkeypatch.setattr(LlamaCppBackend, "_is_vulkan_backend", staticmethod(lambda *a, **k: False)) + monkeypatch.setattr(LlamaCppBackend, "_effective_gpu_count", staticmethod(lambda *a, **k: 1)) + monkeypatch.setattr(LlamaCppBackend, "_diffusion_gpu_arg", staticmethod(lambda *a, **k: "0")) + # Pin the --kv-unified probe so the estimate cannot depend on a locally + # installed llama-server. Default "no binary found" leaves the count alone. + monkeypatch.setattr( + LlamaCppBackend, + "probe_server_capabilities", + classmethod(lambda cls, binary = None: dict(caps or {})), + ) + + inf._guard_chat_load_against_training( + _types.SimpleNamespace(is_gguf = True, gguf_file = gguf_path, identifier = "local/model"), + model_identifier = "local/model", + hf_token = None, + load_in_4bit = False, + max_seq_length = 8192, + requested_gpu_ids = None, + n_parallel = n_parallel, + gpu_memory_mode = "auto", + ) + return seen["required_override_gb"] + + +def test_training_guard_sizes_a_diffusion_gguf_at_one_slot(monkeypatch, tmp_path): + # Diffusion ignores --parallel, so slots must not inflate the estimate and 409 + # a load that would have fitted beside training. + gguf = _write_swa_gguf(tmp_path / "diffusion.gguf") + one = _guard_required_gb(monkeypatch, gguf, n_parallel = 1, diffusion = True) + many = _guard_required_gb(monkeypatch, gguf, n_parallel = 8, diffusion = True) + assert one == many + + +def test_training_guard_still_sizes_slots_for_an_ordinary_gguf(monkeypatch, tmp_path): + # llama-server does allocate per-slot SWA cells, so the reduction above must + # be scoped to diffusion and not flatten every GGUF to one slot. + gguf = _write_swa_gguf(tmp_path / "chat.gguf") + one = _guard_required_gb(monkeypatch, gguf, n_parallel = 1, diffusion = False) + many = _guard_required_gb(monkeypatch, gguf, n_parallel = 8, diffusion = False) + assert many > one + + +def test_training_guard_sizes_one_slot_when_the_binary_has_no_kv_unified(monkeypatch, tmp_path): + # load_model clamps a multi-slot request to 1 on such a build, where each slot + # carries its own SWA stream, so sizing the asked count would 409 a load that fits. + gguf = _write_swa_gguf(tmp_path / "chat.gguf") + old = {"found": True, "supports_kv_unified": False} + one = _guard_required_gb(monkeypatch, gguf, n_parallel = 1, diffusion = False, caps = old) + many = _guard_required_gb(monkeypatch, gguf, n_parallel = 8, diffusion = False, caps = old) + assert one == many + + +def test_training_guard_sizes_every_slot_when_kv_unified_exists(monkeypatch, tmp_path): + # The clamp is scoped to binaries that cannot serve the slots; a capable one + # really does allocate the SWA window per slot. + gguf = _write_swa_gguf(tmp_path / "chat.gguf") + new = {"found": True, "supports_kv_unified": True} + one = _guard_required_gb(monkeypatch, gguf, n_parallel = 1, diffusion = False, caps = new) + many = _guard_required_gb(monkeypatch, gguf, n_parallel = 8, diffusion = False, caps = new) + assert many > one + + +def test_training_guard_keeps_slots_for_an_unclassified_gguf(monkeypatch, tmp_path): + # None = inconclusive header, so keep the larger estimate rather than + # under-size against training. + gguf = _write_swa_gguf(tmp_path / "unknown.gguf") + one = _guard_required_gb(monkeypatch, gguf, n_parallel = 1, diffusion = None) + many = _guard_required_gb(monkeypatch, gguf, n_parallel = 8, diffusion = None) + assert many > one diff --git a/studio/frontend/src/features/chat/api/chat-adapter.ts b/studio/frontend/src/features/chat/api/chat-adapter.ts index 08d17f2a65..5f6c6cc589 100644 --- a/studio/frontend/src/features/chat/api/chat-adapter.ts +++ b/studio/frontend/src/features/chat/api/chat-adapter.ts @@ -1604,6 +1604,7 @@ async function autoLoadSmallestModel(): Promise<{ ? { gpu_ids: effectiveGpuIds ?? undefined, gpu_memory_mode: effectiveGpuMemoryMode, + n_parallel: config.nParallel ?? null, } : {}), })) @@ -1637,6 +1638,8 @@ async function autoLoadSmallestModel(): Promise<{ gpu_layers: effectiveGpuLayers, n_cpu_moe: effectiveNCpuMoe, gpu_ids: effectiveGpuIds ?? undefined, + // Per-model too, or the auto-load reverts a remembered override. + n_parallel: config.nParallel ?? null, } : {}), }); @@ -1689,6 +1692,11 @@ async function autoLoadSmallestModel(): Promise<{ effectiveGpuLayers, config.customContextLength ?? null, ); + // Slots this auto-load committed. Diffusion ignores --parallel, so a count + // there would mint a phantom override a saved preset carries onto a GGUF. + const committedSlots = (loadResp.is_diffusion ?? false) + ? null + : (config.nParallel ?? null); useChatRuntimeStore.setState({ ggufContextLength: loadResp.context_length ?? 131072, ggufMaxContextLength: @@ -1703,6 +1711,9 @@ async function autoLoadSmallestModel(): Promise<{ ...resolveToolsEnabledOnLoad(loadResp.supports_tools ?? false), kvCacheDtype: loadResp.cache_type_kv ?? null, loadedKvCacheDtype: loadResp.cache_type_kv ?? null, + // Click-time value, not the resolved backend echo (see performLoad). + nParallel: committedSlots, + loadedNParallel: committedSlots, tensorParallel: loadResp.tensor_parallel ?? false, loadedTensorParallel: loadResp.tensor_parallel ?? false, ...loadedGpuMemoryFields(loadResp), @@ -1728,6 +1739,10 @@ async function autoLoadSmallestModel(): Promise<{ ...resolveToolsEnabledOnLoad(loadResp.supports_tools ?? false), kvCacheDtype: loadResp.cache_type_kv ?? null, loadedKvCacheDtype: loadResp.cache_type_kv ?? null, + // GGUF-only and never sent here: a staged override would be saved for + // a model that cannot use it. + nParallel: null, + loadedNParallel: null, tensorParallel: loadResp.tensor_parallel ?? false, loadedTensorParallel: loadResp.tensor_parallel ?? false, // Non-GGUF response: clears any stale GPU baseline a prior manual-GPU @@ -2001,6 +2016,10 @@ async function autoLoadSmallestModel(): Promise<{ ...resolveToolsEnabledOnLoad(loadResp.supports_tools ?? false), kvCacheDtype: loadResp.cache_type_kv ?? null, loadedKvCacheDtype: loadResp.cache_type_kv ?? null, + // The request above omits n_parallel: a staged override left from a + // preset would read as applied and be re-sent by the next Apply. + nParallel: null, + loadedNParallel: null, tensorParallel: loadResp.tensor_parallel ?? false, loadedTensorParallel: loadResp.tensor_parallel ?? false, ...loadedGpuMemoryFields(loadResp), diff --git a/studio/frontend/src/features/chat/api/chat-api.ts b/studio/frontend/src/features/chat/api/chat-api.ts index 60b737fb68..8ad2691391 100644 --- a/studio/frontend/src/features/chat/api/chat-api.ts +++ b/studio/frontend/src/features/chat/api/chat-api.ts @@ -192,6 +192,8 @@ export async function validateModel( // --fit, while a pinned layer count is owned by the user. Tell validate // so it applies the same training-guard policy as /load. gpu_memory_mode: payload.gpu_memory_mode, + // Slots scale the KV estimate; keep validate sized like the load. + n_parallel: payload.n_parallel, }), }); return parseJsonOrThrow(response); diff --git a/studio/frontend/src/features/chat/chat-settings-sheet.tsx b/studio/frontend/src/features/chat/chat-settings-sheet.tsx index 7b310c50d4..6070bd2e40 100644 --- a/studio/frontend/src/features/chat/chat-settings-sheet.tsx +++ b/studio/frontend/src/features/chat/chat-settings-sheet.tsx @@ -397,6 +397,7 @@ export function ChatSettingsPanel({ const nCpuMoe = useChatRuntimeStore((s) => s.nCpuMoe); const tensorParallel = useChatRuntimeStore((s) => s.tensorParallel); const specDraftNMax = useChatRuntimeStore((s) => s.specDraftNMax); + const nParallel = useChatRuntimeStore((s) => s.nParallel); const speculativeType = useChatRuntimeStore((s) => s.speculativeType); const specFallbackReason = useChatRuntimeStore((s) => s.specFallbackReason); const mtpUpdatable = @@ -504,6 +505,7 @@ export function ChatSettingsPanel({ tensorParallel, speculativeType, specDraftNMax, + nParallel, params.maxSeqLength, ]); const activePresetLoadSummary = useMemo( @@ -522,6 +524,7 @@ export function ChatSettingsPanel({ tensorParallel, speculativeType, specDraftNMax, + nParallel, params.maxSeqLength, ], ); diff --git a/studio/frontend/src/features/chat/hooks/use-chat-model-runtime.ts b/studio/frontend/src/features/chat/hooks/use-chat-model-runtime.ts index bc7227e70d..5f0149d909 100644 --- a/studio/frontend/src/features/chat/hooks/use-chat-model-runtime.ts +++ b/studio/frontend/src/features/chat/hooks/use-chat-model-runtime.ts @@ -567,6 +567,8 @@ export function useChatModelRuntime() { applyActiveModelStatusToStore(residentStatus, { previousCheckpoint: selectedCheckpoint, previousGgufVariant, + // Id and variant matched above: same model, only the tab moved. + readoptingSameModel: true, }); syncModelCapabilities(modelId, residentStatus); return; @@ -669,6 +671,14 @@ export function useChatModelRuntime() { let previousWasUnloaded = false; const pendingLoadConfig = typeof selection !== "string" ? selection.config : undefined; + // The outgoing model's slot INTENT (blank = follow the server + // default), which the resolved baseline cannot express. previousConfig + // is the snapshot the picker took before pre-applying the target's + // config, so the live control is only the outgoing one without it. + const previousNParallel = + typeof selection !== "string" && selection.previousConfig + ? (selection.previousConfig.nParallel ?? null) + : useChatRuntimeStore.getState().nParallel; if (pendingLoadConfig) { applyPerModelConfigToRuntime(pendingLoadConfig); } @@ -761,6 +771,8 @@ export function useChatModelRuntime() { : stateBeforeUnload.speculativeType; let loadSpecDraftNMax = pendingLoadConfig?.specDraftNMax ?? stateBeforeUnload.specDraftNMax; + let loadNParallel = + pendingLoadConfig?.nParallel ?? stateBeforeUnload.nParallel; try { // Lightweight pre-flight validation: avoid unloading a working model // if the new identifier is clearly invalid (e.g. bad HF id / path). @@ -792,6 +804,10 @@ export function useChatModelRuntime() { const validateGpuLayers = resetsPerModelSettings ? GPU_LAYERS_AUTO : loadGpuLayers; + // Per-model: the reset re-baselines to the staged config, like the load. + const validateNParallel = resetsPerModelSettings + ? (pendingLoadConfig?.nParallel ?? null) + : loadNParallel; const validateMaxSeqLength = resolveFitMaxSeqLength( isGguf, loadGpuMemoryMode, @@ -820,7 +836,12 @@ export function useChatModelRuntime() { cache_type_kv: loadKvCacheDtype, tensor_parallel: loadTensorParallel, gpu_ids: validateGpuIds ?? undefined, - ...(isGguf ? { gpu_memory_mode: loadGpuMemoryMode } : {}), + ...(isGguf + ? { + gpu_memory_mode: loadGpuMemoryMode, + n_parallel: validateNParallel, + } + : {}), }); // Upgrade consent runs before the security dialogs; Accept installs and the load continues. if (validation.requires_transformers_upgrade) { @@ -903,6 +924,10 @@ export function useChatModelRuntime() { loadedSpeculativeType: persistedSpeculativeType, specDraftNMax: null, loadedSpecDraftNMax: null, + // Per-model too: a different model follows the server default + // unless its staged config overrides it. + nParallel: null, + loadedNParallel: null, // Per-model GPU knobs must not follow onto a different model // (gpuMemoryMode is a standing preference and is kept). selectedGpuIds: null, @@ -918,6 +943,7 @@ export function useChatModelRuntime() { ? normalizeSpeculativeType(pendingLoadConfig.speculativeType) : persistedSpeculativeType; loadSpecDraftNMax = pendingLoadConfig?.specDraftNMax ?? null; + loadNParallel = pendingLoadConfig?.nParallel ?? null; // Keep the click-time snapshot in lock-step with the store reset so // the load below sizes against the cleared per-model knobs, not the // previous model's (gpuMemoryMode is standing, so left as captured). @@ -984,6 +1010,8 @@ export function useChatModelRuntime() { cache_type_kv: loadKvCacheDtype, speculative_type: loadSpeculativeType, spec_draft_n_max: loadSpecDraftNMax, + // GGUF-only: slots mean nothing for a transformers load. + n_parallel: isGguf ? loadNParallel : null, tensor_parallel: loadTensorParallel, gpu_memory_mode: loadGpuMemoryMode, gpu_layers: loadGpuLayers, @@ -1034,6 +1062,14 @@ export function useChatModelRuntime() { const loadedSpec = normalizeSpeculativeType( loadResponse.speculative_type, ); + // Slots the load actually committed. Non-GGUF never sends them and + // diffusion ignores --parallel, so a click-time count on either + // would mint a phantom override a saved preset carries onto a GGUF. + const committedSlots = + (loadResponse.is_gguf ?? false) && + !(loadResponse.is_diffusion ?? false) + ? (loadNParallel ?? null) + : null; const nativeCtx = loadResponse.is_gguf ? (loadResponse.context_length ?? 131072) : null; @@ -1109,6 +1145,10 @@ export function useChatModelRuntime() { loadedSpeculativeType: loadedSpec, specDraftNMax: loadResponse.spec_draft_n_max ?? null, loadedSpecDraftNMax: loadResponse.spec_draft_n_max ?? null, + // Keep the click-time value: the echo is the resolved count, and + // adopting it would pin a blank "server default" control. + nParallel: committedSlots, + loadedNParallel: committedSlots, customContextLength: keepCustomCtx, loadedCustomContextLength: keepCustomCtx, defaultChatTemplate: loadResponse.chat_template ?? null, @@ -1211,6 +1251,7 @@ export function useChatModelRuntime() { stateBeforeUnload.loadedSpeculativeType, spec_draft_n_max: stateBeforeUnload.loadedSpecDraftNMax, + n_parallel: stateBeforeUnload.loadedNParallel, // Restore the previous model in the split mode it was running, // not the default layer split. tensor_parallel: stateBeforeUnload.loadedTensorParallel ?? false, @@ -1237,6 +1278,9 @@ export function useChatModelRuntime() { // model's; the loaded baselines below come from its reload echo. speculativeType: stateBeforeUnload.loadedSpeculativeType ?? null, specDraftNMax: stateBeforeUnload.loadedSpecDraftNMax ?? null, + // Control keeps its intent; only the baseline takes the echo. + nParallel: previousNParallel, + loadedNParallel: stateBeforeUnload.loadedNParallel ?? null, loadedSpeculativeType: rollbackSpeculativeType, loadedSpecDraftNMax: rollbackResponse.spec_draft_n_max ?? null, diff --git a/studio/frontend/src/features/chat/lib/apply-inference-status-to-store.ts b/studio/frontend/src/features/chat/lib/apply-inference-status-to-store.ts index f85ff3246b..47d40009c8 100644 --- a/studio/frontend/src/features/chat/lib/apply-inference-status-to-store.ts +++ b/studio/frontend/src/features/chat/lib/apply-inference-status-to-store.ts @@ -1,6 +1,9 @@ // SPDX-License-Identifier: AGPL-3.0-only // Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 +// Barrel import (lint rule); the model-picker cycle is fine because the call +// happens at runtime, not module eval. +import { resolveInitialConfig } from "@/features/model-picker"; import { getInferenceStatus } from "../api/chat-api"; import { mergeBackendRecommendedInference, @@ -131,6 +134,9 @@ export type ApplyInferenceStatusOptions = { * status -- without it a variant-only switch underneath the tab reads as * steady state and the hydration reseed keeps the old quant's baselines. */ previousGgufVariant?: string | null; + /** The caller verified the status is the model this tab just picked, so the + * slot control it holds belongs to that model and must survive. */ + readoptingSameModel?: boolean; }; /** Mirror refresh() hydration so adopted CLI models get reasoning/tools flags. */ @@ -201,6 +207,22 @@ export function applyActiveModelStatusToStore( // While a load is in flight, performLoad owns the load params. Seeding them // from a stale poll here would clobber the values the load dialog just set. const seedLoadParams = !prevState.modelLoading; + // A model/variant change underneath this tab, as opposed to re-adopting the + // model the tab just picked, where hydratingExistingModel fires on the stale + // checkpoint. The echo cannot stand in: a new model can report the old count. + const slotsModelChanged = + hydratingExistingModel && !options.readoptingSameModel; + // This model's remembered override, read only on a fresh store or a model + // change, so a steady poll cannot re-pin a control the user just blanked. + const slotsUnseeded = + prevState.loadedNParallel === null && prevState.nParallel === null; + const remembered = + status.is_gguf && (slotsUnseeded || slotsModelChanged) + ? resolveInitialConfig(checkpointId, status.gguf_variant ?? null) + : null; + const rememberedNParallel = remembered?.remembered + ? (remembered.config.nParallel ?? null) + : null; // A Manual + Auto-layers load sent its positive context pin as max_seq_length, // and status only exposes the RESOLVED context; re-seed the pin from the // requested value (parity with the load paths' keepCustomCtx). Baselines @@ -322,6 +344,35 @@ export function applyActiveModelStatusToStore( tensorParallel: status.tensor_parallel, loadedTensorParallel: status.tensor_parallel, }), + // Baseline only, never the control: the echo is the RESOLVED count and would + // pin a blank "server default" control. The rollback re-sends the baseline, + // so without this a rollback after a tab reload loses the override. + ...(seedLoadParams && + status.requested_parallel_slots != null && + (prevState.loadedNParallel === null || hydratingExistingModel) && { + loadedNParallel: status.requested_parallel_slots, + }), + // A slotless model must not keep the previous GGUF's baseline: the rollback + // re-sends it. /status omits the echo for non-GGUF and sends an explicit + // null for diffusion, so an absent field on a GGUF is an older backend. + ...(seedLoadParams && + (status.is_gguf === false || status.requested_parallel_slots === null) && { + loadedNParallel: null, + }), + // Per-model: a change underneath this tab blanks the control like + // performLoad's cross-model reset, or the old count follows onto the new + // model. The baseline above still carries the rollback. + ...(seedLoadParams && slotsModelChanged && { nParallel: null }), + // AFTER that clear, which both a first hydration and a model change trip: + // either would leave the control blank while the model runs on a remembered + // override, so the next Apply would save the blank over it. Adopted only + // when the running count matches, proving it is this model's own. + ...(seedLoadParams && + (slotsUnseeded || slotsModelChanged) && + rememberedNParallel != null && + rememberedNParallel === status.requested_parallel_slots && { + nParallel: rememberedNParallel, + }), // Re-seed on first hydration, model/variant changes, or a same-model backend // placement change. gpuStatusFields preserves dirty local edits in the last // case while advancing their loaded baselines. diff --git a/studio/frontend/src/features/chat/presets/preset-load-config.ts b/studio/frontend/src/features/chat/presets/preset-load-config.ts index 1083655cf2..c0a65c7886 100644 --- a/studio/frontend/src/features/chat/presets/preset-load-config.ts +++ b/studio/frontend/src/features/chat/presets/preset-load-config.ts @@ -12,6 +12,8 @@ import { DEFAULT_MAX_SEQ_LENGTH, KV_CACHE_DTYPES, MTP_SPECULATIVE_TYPES, + N_PARALLEL_MAX, + N_PARALLEL_MIN, SPECULATIVE_TYPES, normalizeMaxSeqLength, type PerModelConfig, @@ -30,6 +32,7 @@ export type PresetLoadConfig = Pick< | "kvCacheDtype" | "speculativeType" | "specDraftNMax" + | "nParallel" | "tensorParallel" | "gpuMemoryMode" | "gpuLayers" @@ -45,6 +48,7 @@ export const EMPTY_PRESET_LOAD_CONFIG: PresetLoadConfig = { kvCacheDtype: null, speculativeType: null, specDraftNMax: null, + nParallel: null, tensorParallel: false, }; @@ -107,6 +111,14 @@ export function normalizePresetLoadConfig( ? speculativeType : null, specDraftNMax, + nParallel: + typeof partial.nParallel === "number" && + Number.isFinite(partial.nParallel) + ? Math.max( + N_PARALLEL_MIN, + Math.min(N_PARALLEL_MAX, Math.round(partial.nParallel)), + ) + : null, tensorParallel: typeof partial.tensorParallel === "boolean" ? partial.tensorParallel @@ -151,6 +163,7 @@ export function capturePresetLoadConfig(): PresetLoadConfig | undefined { kvCacheDtype: snapshot.kvCacheDtype ?? null, speculativeType: normalizeSpeculativeType(snapshot.speculativeType), specDraftNMax: snapshot.specDraftNMax ?? null, + nParallel: snapshot.nParallel ?? null, tensorParallel: snapshot.tensorParallel ?? false, ...(snapshot.gpuMemoryMode === "manual" ? { gpuMemoryMode: "manual" as const } @@ -206,6 +219,7 @@ export function applyPresetLoadConfig( kvCacheDtype: config.kvCacheDtype ?? null, speculativeType: config.speculativeType ?? null, specDraftNMax: config.specDraftNMax ?? null, + nParallel: config.nParallel ?? null, tensorParallel: config.tensorParallel ?? false, chatTemplateOverride: null, gpuMemoryMode: config.gpuMemoryMode, @@ -231,6 +245,9 @@ export function formatPresetLoadConfigSummary( if (config.speculativeType && config.speculativeType !== "auto") { parts.push(`Spec ${config.speculativeType}`); } + if (config.nParallel != null) { + parts.push(`${config.nParallel} slots`); + } if (config.gpuMemoryMode === "manual") { parts.push("GPU manual"); } diff --git a/studio/frontend/src/features/chat/shared-composer.tsx b/studio/frontend/src/features/chat/shared-composer.tsx index 44436b92df..890dd022a0 100644 --- a/studio/frontend/src/features/chat/shared-composer.tsx +++ b/studio/frontend/src/features/chat/shared-composer.tsx @@ -1130,6 +1130,8 @@ export function SharedComposer({ ? { gpu_ids: effectiveSelectedGpuIds ?? undefined, gpu_memory_mode: effectiveGpuMemoryMode, + // Slots scale the KV estimate; keep validate sized like the load. + n_parallel: ownConfig.nParallel ?? null, } : {}), }); @@ -1198,6 +1200,7 @@ export function SharedComposer({ n_cpu_moe: effectiveNCpuMoe, tensor_split: compareLoadKnobs.splitRatio ?? undefined, gpu_ids: effectiveSelectedGpuIds ?? undefined, + n_parallel: ownConfig.nParallel ?? null, } : {}), }); @@ -1229,6 +1232,12 @@ export function SharedComposer({ effectiveCustomContextLength, ) : null; + // Slots this compare load committed. Diffusion ignores --parallel, so a + // count there would mint a phantom override a preset carries onto a GGUF. + const committedSlots = + targetIsGguf && !(resp.is_diffusion ?? false) + ? (ownConfig.nParallel ?? null) + : null; useChatRuntimeStore.setState({ supportsReasoning: resp.supports_reasoning ?? false, reasoningAlwaysOn: resp.reasoning_always_on ?? false, @@ -1237,6 +1246,9 @@ export function SharedComposer({ supportsTools: resp.supports_tools ?? false, kvCacheDtype: resp.cache_type_kv ?? null, loadedKvCacheDtype: resp.cache_type_kv ?? null, + // Click-time value, not the resolved echo (see the single-model load). + nParallel: committedSlots, + loadedNParallel: committedSlots, tensorParallel: resp.tensor_parallel ?? false, loadedTensorParallel: resp.tensor_parallel ?? false, defaultChatTemplate: resp.chat_template ?? null, diff --git a/studio/frontend/src/features/chat/stores/chat-runtime-store.ts b/studio/frontend/src/features/chat/stores/chat-runtime-store.ts index 98b8676c10..2984611780 100644 --- a/studio/frontend/src/features/chat/stores/chat-runtime-store.ts +++ b/studio/frontend/src/features/chat/stores/chat-runtime-store.ts @@ -968,6 +968,12 @@ type ChatRuntimeStore = { /** User --spec-draft-n-max override (null = platform default). */ specDraftNMax: number | null; loadedSpecDraftNMax: number | null; + /** User --parallel slots override for GGUF loads (null = server default). + * Never re-seeded from an echo: the resolved count would pin a blank control. */ + nParallel: number | null; + /** Slots the last successful load sent (null = default); the rollback + * re-sends it so a failed switch can't lose the override. */ + loadedNParallel: number | null; /** Tensor-parallel split (--split-mode tensor) toggle, GGUF multi-GPU only. */ tensorParallel: boolean; /** Backend-reported tensor-parallel state; null until first hydrated. */ @@ -1491,6 +1497,8 @@ export const useChatRuntimeStore = create((set, get) => ({ specFallbackReason: null, specDraftNMax: null, loadedSpecDraftNMax: null, + nParallel: null, + loadedNParallel: null, tensorParallel: false, loadedTensorParallel: null, gpuMemoryMode: readPersistedGpuMemoryMode(), @@ -1874,6 +1882,8 @@ export const useChatRuntimeStore = create((set, get) => ({ specFallbackReason: null, specDraftNMax: null, loadedSpecDraftNMax: null, + nParallel: null, + loadedNParallel: null, tensorParallel: false, loadedTensorParallel: null, // Standing preference: survives unload, unlike the per-model knobs above. diff --git a/studio/frontend/src/features/chat/types/api.ts b/studio/frontend/src/features/chat/types/api.ts index 3681b0f0cb..b67a9eda26 100644 --- a/studio/frontend/src/features/chat/types/api.ts +++ b/studio/frontend/src/features/chat/types/api.ts @@ -65,6 +65,11 @@ export interface LoadModelRequest { * when speculative_type resolves to "mtp" or "mtp+ngram". */ spec_draft_n_max?: number | null; + /** + * Parallel decode slots for llama-server (--parallel), 1..64. Omit/null = + * the launch default. The VRAM fitter may launch fewer to stay on GPU. + */ + n_parallel?: number | null; /** * Split the model across GPUs by tensor (--split-mode tensor) instead * of by layer for GGUF models. Multi-GPU only; no effect on a single GPU. @@ -202,6 +207,12 @@ export interface LoadModelResponse { gpu_ids?: number[] | null; /** User-requested GPU placement pool before fit-time narrowing. */ requested_gpu_ids?: number[] | null; + /** Slots the load was invoked with (else the --parallel default). Null for + * non-GGUF loads. */ + requested_parallel_slots?: number | null; + /** Slots llama-server actually runs, after any fit-time reduction. Null for + * non-GGUF loads. */ + parallel_slots?: number | null; } export interface UnloadModelRequest { @@ -263,6 +274,12 @@ export interface InferenceStatusResponse { gpu_ids?: number[] | null; /** User-requested GPU placement pool before fit-time narrowing. */ requested_gpu_ids?: number[] | null; + /** Slots the active load was invoked with (else the --parallel default). + * Null when no GGUF model is loaded. */ + requested_parallel_slots?: number | null; + /** Slots llama-server actually runs, after any fit-time reduction. Null when + * no GGUF model is loaded. */ + parallel_slots?: number | null; n_layers?: number | null; /** Model's MoE expert-layer count (the n_cpu_moe ceiling); 0 if not MoE. */ n_moe_layers?: number; diff --git a/studio/frontend/src/features/model-picker/components/model-config-page.tsx b/studio/frontend/src/features/model-picker/components/model-config-page.tsx index 90202a2bcf..a753e9016e 100644 --- a/studio/frontend/src/features/model-picker/components/model-config-page.tsx +++ b/studio/frontend/src/features/model-picker/components/model-config-page.tsx @@ -46,6 +46,8 @@ import { MAX_SEQ_LENGTH_MIN, MAX_SEQ_LENGTH_STEP, MTP_SPECULATIVE_TYPES, + N_PARALLEL_MAX, + N_PARALLEL_MIN, type PerModelConfig, SPECULATIVE_TYPES, deletePerModelConfig, @@ -87,6 +89,7 @@ function hasNonDefaultAdvanced(config: PerModelConfig): boolean { config.kvCacheDtype != null || (config.speculativeType ?? "auto") !== "auto" || config.specDraftNMax != null || + config.nParallel != null || config.tensorParallel || config.chatTemplateOverride != null || (config.gpuMemoryMode ?? "auto") !== "auto" || @@ -541,6 +544,44 @@ function GgufAdvancedSettings({ )} +
+
+ Parallel Slots + + llama-server decode slots (--parallel) for concurrent requests. + Leave blank for the server default. More slots share the context + pool and use more VRAM; if they don't fit on GPU, fewer slots are + launched. + +
+ { + const raw = event.target.value; + if (raw === "") { + update({ nParallel: null }); + return; + } + const parsed = Number.parseInt(raw, 10); + if (Number.isFinite(parsed)) { + update({ + nParallel: Math.max( + N_PARALLEL_MIN, + Math.min(N_PARALLEL_MAX, parsed), + ), + }); + } + }} + aria-label="Parallel decode slots" + className={NUMBER_INPUT_CLASS} + /> +
+
Tensor Parallelism diff --git a/studio/frontend/src/features/model-picker/components/sidebar-model-config.tsx b/studio/frontend/src/features/model-picker/components/sidebar-model-config.tsx index 2d12c503a4..0d5f0fd663 100644 --- a/studio/frontend/src/features/model-picker/components/sidebar-model-config.tsx +++ b/studio/frontend/src/features/model-picker/components/sidebar-model-config.tsx @@ -43,6 +43,7 @@ function configSignature(config: PerModelConfig): string { config.kvCacheDtype ?? "", config.speculativeType ?? "", config.specDraftNMax ?? "", + config.nParallel ?? "", config.tensorParallel ? "1" : "0", config.chatTemplateOverride == null ? "" diff --git a/studio/frontend/src/features/model-picker/hooks/use-active-model-config.ts b/studio/frontend/src/features/model-picker/hooks/use-active-model-config.ts index 9d09ee6897..b0a6411019 100644 --- a/studio/frontend/src/features/model-picker/hooks/use-active-model-config.ts +++ b/studio/frontend/src/features/model-picker/hooks/use-active-model-config.ts @@ -20,6 +20,7 @@ export function useActiveModelConfig(): ActiveModelConfigState { const kvCacheDtype = useChatRuntimeStore((s) => s.kvCacheDtype); const speculativeType = useChatRuntimeStore((s) => s.speculativeType); const specDraftNMax = useChatRuntimeStore((s) => s.specDraftNMax); + const nParallel = useChatRuntimeStore((s) => s.nParallel); const tensorParallel = useChatRuntimeStore((s) => s.tensorParallel); const chatTemplateOverride = useChatRuntimeStore( (s) => s.chatTemplateOverride, @@ -44,6 +45,7 @@ export function useActiveModelConfig(): ActiveModelConfigState { kvCacheDtype: kvCacheDtype ?? null, speculativeType: speculativeType ?? "auto", specDraftNMax: specDraftNMax ?? null, + nParallel: nParallel ?? null, tensorParallel: tensorParallel ?? false, chatTemplateOverride: chatTemplateOverride ?? null, }; @@ -65,6 +67,7 @@ export function useActiveModelConfig(): ActiveModelConfigState { kvCacheDtype, speculativeType, specDraftNMax, + nParallel, tensorParallel, chatTemplateOverride, gpuMemoryMode, diff --git a/studio/frontend/src/features/model-picker/model-config/apply-per-model-config.ts b/studio/frontend/src/features/model-picker/model-config/apply-per-model-config.ts index c21d3e164a..829c522cb2 100644 --- a/studio/frontend/src/features/model-picker/model-config/apply-per-model-config.ts +++ b/studio/frontend/src/features/model-picker/model-config/apply-per-model-config.ts @@ -39,6 +39,7 @@ export function applyPerModelConfigToRuntime(config: PerModelConfig): void { normalizeSpeculativeType(config.speculativeType) ?? readPersistedSpeculativeType(), specDraftNMax: config.specDraftNMax ?? null, + nParallel: config.nParallel ?? null, tensorParallel: config.tensorParallel ?? false, chatTemplateOverride: cleanTemplate(config.chatTemplateOverride), // GPU Memory knobs are per-model (GGUF-only). Absent = defaults; the mode is @@ -77,6 +78,7 @@ export function currentRuntimePerModelConfig( kvCacheDtype: s.kvCacheDtype ?? null, speculativeType: normalizeSpeculativeType(s.speculativeType), specDraftNMax: s.specDraftNMax ?? null, + nParallel: s.nParallel ?? null, tensorParallel: s.tensorParallel ?? false, chatTemplateOverride: cleanTemplate(s.chatTemplateOverride), // Snapshot the live GPU knobs too so a failed switch rolls the previous @@ -101,6 +103,7 @@ export function perModelConfigsEqual( normalizeSpeculativeType(a.speculativeType) === normalizeSpeculativeType(b.speculativeType) && (a.specDraftNMax ?? null) === (b.specDraftNMax ?? null) && + (a.nParallel ?? null) === (b.nParallel ?? null) && Boolean(a.tensorParallel) === Boolean(b.tensorParallel) && cleanTemplate(a.chatTemplateOverride) === cleanTemplate(b.chatTemplateOverride) && diff --git a/studio/frontend/src/features/model-picker/model-config/per-model-config.ts b/studio/frontend/src/features/model-picker/model-config/per-model-config.ts index ba6d4cec99..196ac9e5a1 100644 --- a/studio/frontend/src/features/model-picker/model-config/per-model-config.ts +++ b/studio/frontend/src/features/model-picker/model-config/per-model-config.ts @@ -15,6 +15,7 @@ export interface PerModelConfig { kvCacheDtype: string | null; speculativeType: string | null; specDraftNMax: number | null; + nParallel: number | null; tensorParallel: boolean; chatTemplateOverride: string | null; // GPU Memory controls (per-model, GGUF-only), optional so older blobs still @@ -33,10 +34,16 @@ export const DEFAULT_PER_MODEL_CONFIG: PerModelConfig = { kvCacheDtype: null, speculativeType: null, specDraftNMax: null, + nParallel: null, tensorParallel: false, chatTemplateOverride: null, }; +// Mirrors llama_server_args.py PARALLEL_MIN/MAX (LoadRequest.n_parallel +// bounds). null = follow the server-wide default. +export const N_PARALLEL_MIN = 1; +export const N_PARALLEL_MAX = 64; + export const MAX_SEQ_LENGTH_MIN = 128; export const MAX_SEQ_LENGTH_MAX = 1048576; export const MAX_SEQ_LENGTH_STEP = 128; @@ -92,6 +99,7 @@ const STORED_CONFIG_FIELDS = new Set([ "kvCacheDtype", "speculativeType", "specDraftNMax", + "nParallel", "tensorParallel", "chatTemplateOverride", "gpuMemoryMode", @@ -292,6 +300,8 @@ function legacyEntryToConfig(raw: Record): PerModelConfig { typeof raw.speculativeType === "string" ? raw.speculativeType : null, specDraftNMax: typeof raw.specDraftNMax === "number" ? raw.specDraftNMax : null, + // Legacy blobs predate the parallel-slots knob. + nParallel: null, tensorParallel: typeof raw.tensorParallel === "boolean" ? raw.tensorParallel : false, chatTemplateOverride: null, @@ -459,6 +469,10 @@ function normalizeV1(partial: RawConfig): PerModelConfig { : null, speculativeType, specDraftNMax, + nParallel: + typeof partial.nParallel === "number" && Number.isFinite(partial.nParallel) + ? Math.max(N_PARALLEL_MIN, Math.min(N_PARALLEL_MAX, Math.round(partial.nParallel))) + : null, tensorParallel: typeof partial.tensorParallel === "boolean" ? partial.tensorParallel @@ -597,6 +611,7 @@ export function isDefaultConfig(config: PerModelConfig): boolean { (config.kvCacheDtype ?? null) === DEFAULT_PER_MODEL_CONFIG.kvCacheDtype && config.speculativeType === DEFAULT_PER_MODEL_CONFIG.speculativeType && config.specDraftNMax == null && + config.nParallel == null && Boolean(config.tensorParallel) === Boolean(DEFAULT_PER_MODEL_CONFIG.tensorParallel) && (config.chatTemplateOverride ?? null) === null && diff --git a/tests/studio/test_chat_preset_load_config.py b/tests/studio/test_chat_preset_load_config.py index 1588c7d96d..6234ab395c 100644 --- a/tests/studio/test_chat_preset_load_config.py +++ b/tests/studio/test_chat_preset_load_config.py @@ -68,3 +68,18 @@ def test_backend_chat_preset_accepts_load_config(): routes = _read("studio/backend/routes/chat_history.py") assert "class ChatPresetLoadConfig" in routes assert "loadConfig: Optional[ChatPresetLoadConfig]" in routes + + +def test_preset_load_config_carries_parallel_slots(): + # Captured, clamped on read, applied, and accepted by the extra="forbid" + # backend model (a missing backend field would 422 every settings sync). + source = _read("studio/frontend/src/features/chat/presets/preset-load-config.ts") + assert '| "nParallel"' in source + assert "nParallel: snapshot.nParallel ?? null" in source + assert "nParallel: config.nParallel ?? null" in source + assert "N_PARALLEL_MAX, Math.round(partial.nParallel)" in source + routes = _read("studio/backend/routes/chat_history.py") + assert ( + "nParallel: Optional[int] = Field(default = None, ge = PARALLEL_MIN, le = PARALLEL_MAX)" + in routes + ) diff --git a/tests/studio/test_model_picker_contracts.py b/tests/studio/test_model_picker_contracts.py index 00ee83efc7..d9a8efc9a4 100644 --- a/tests/studio/test_model_picker_contracts.py +++ b/tests/studio/test_model_picker_contracts.py @@ -631,6 +631,253 @@ def test_legacy_migration_is_idempotent_and_non_destructive(): assert "if (isDefaultConfig(migrated) || Object.hasOwn(map, key)) {" in src +def test_parallel_slots_setting_wired_end_to_end(): + """The per-load Parallel Slots knob (llama-server --parallel) must flow from + the run-settings form through persistence, every /load builder, the validate + preflight and the cross-model reset; a lost hop silently reverts the model to + the server-wide slot default.""" + config = _read("features/model-picker/model-config/per-model-config.ts") + # Persisted per model, clamped on every read/write, and null (= server + # default) counts as default so blank configs are not stored. + assert '"nParallel",' in config + assert "N_PARALLEL_MAX, Math.round(partial.nParallel)" in config + assert "config.nParallel == null &&" in config + page = _read("features/model-picker/components/model-config-page.tsx") + # Rendered in the GGUF advanced section, which a remembered override reopens. + assert "Parallel Slots" in page + assert "config.nParallel != null ||" in page + assert 'aria-label="Parallel decode slots"' in page + api_types = _read("features/chat/types/api.ts") + assert "n_parallel?: number | null;" in api_types + runtime = _read("features/chat/hooks/use-chat-model-runtime.ts") + # Click-time snapshot, /load body, validate preflight, cross-model reset and + # failed-switch rollback all carry the value. + assert "pendingLoadConfig?.nParallel" in runtime + # GGUF-gated, like the compare pane: a transformers load has no slots. + assert "n_parallel: isGguf ? loadNParallel : null," in runtime + assert "n_parallel: validateNParallel," in runtime + assert "loadNParallel = pendingLoadConfig?.nParallel ?? null;" in runtime + assert "n_parallel: stateBeforeUnload.loadedNParallel," in runtime + chat_api = _read("features/chat/api/chat-api.ts") + assert "n_parallel: payload.n_parallel," in chat_api + composer = _read("features/chat/shared-composer.tsx") + # The compare pane is a second /load builder; its preflight sizes like its load. + assert composer.count("n_parallel: ownConfig.nParallel ?? null,") == 2 + adapter = _read("features/chat/api/chat-adapter.ts") + # The startup auto-load is a third builder reading the remembered config. + assert adapter.count("n_parallel: config.nParallel ?? null,") == 2 + # ... and records it as loaded through the diffusion-gated local below. + assert "loadedNParallel: committedSlots," in adapter + status = _read("features/chat/lib/apply-inference-status-to-store.ts") + # Hydration seeds the rollback BASELINE only; adopting the resolved echo into + # the control would pin a blank "server default" to a number. + assert "loadedNParallel: status.requested_parallel_slots," in status + assert "nParallel: status.requested_parallel_slots," not in status + sidebar = _read("features/model-picker/components/sidebar-model-config.tsx") + # The sidebar form remounts when an external change lands. + assert 'config.nParallel ?? "",' in sidebar + + +def test_parallel_slots_control_cleared_when_the_load_never_sent_them(): + """`nParallel` is the editable control ("blank = follow the server default") + and `loadedNParallel` the rollback baseline. A success path that sends no + slot count must blank the control, or a value staged for another model shows + as applied, is persisted into this model's config (`isDefaultConfig` keys on + nParallel) and is re-sent by the next Apply. Each assertion below is the only + thing pinning one such path.""" + status = " ".join(_read("features/chat/lib/apply-inference-status-to-store.ts").split()) + # A model/variant swap underneath this tab must reset the control like + # performLoad's cross-model reset, or model A's count follows onto model B. + # Narrowly gated -- see test_hydration_keeps_the_slot_control_when_readopting_the_running_model. + assert "...(seedLoadParams && slotsModelChanged && { nParallel: null })," in status + # ... while still never adopting the RESOLVED echo into the control. + assert "nParallel: status.requested_parallel_slots," not in status + + adapter = _read("features/chat/api/chat-adapter.ts") + # Slice the two success branches apart, bounding the second at the shared tail + # so it cannot swallow the fresh-default path below and stay green. + candidate = adapter.split("async function loadAutoLoadCandidate", 1)[1] + gguf_branch, non_gguf_rest = candidate.split('if (candidate.kind === "gguf") {', 1)[1].split( + "\n } else {\n", 1 + ) + non_gguf_branch = non_gguf_rest.split("if (!(loadResp.is_lora ?? false)) {", 1)[0] + # The cached-GGUF branch keeps the remembered override via the gated local... + assert "nParallel: committedSlots," in gguf_branch + assert "nParallel: null," not in gguf_branch + # ... the safetensors fallback sends no slots, so it clears both, or the count + # survives on a model whose form does not even render the field. + assert "nParallel: null," in non_gguf_branch + assert "loadedNParallel: null," in non_gguf_branch + + fresh_default = adapter.split("No downloaded models found. Fetching", 1)[1].split( + 'showAutoLoadSuccess("Loaded Qwen', 1 + )[0] + # The fresh-default download omits the slots, so its success state clears both, + # or the control reads as an unapplied edit against the seeded baseline. + assert "n_parallel" not in fresh_default.split("saveSpeculativeType", 1)[0] + assert "nParallel: null," in fresh_default + assert "loadedNParallel: null," in fresh_default + + +def test_hydration_clears_the_slot_baseline_for_a_slotless_model(): + """The baseline is what a rollback re-sends and what preset capture reads, so + a model that cannot have slots must not inherit the previous GGUF's count. + /status omits the echo for non-GGUF and sends an explicit null for diffusion; + an absent field on a GGUF is an older backend and must NOT wipe it.""" + src = _read("features/chat/lib/apply-inference-status-to-store.ts") + assert ( + "(status.is_gguf === false || status.requested_parallel_slots === null) && {" in src + ), "the slotless clear must key on is_gguf or an explicit null echo" + clear = src.index("status.is_gguf === false || status.requested_parallel_slots === null") + assert "loadedNParallel: null," in src[clear : clear + 200] + # Never `!= null`: that also matches the absent field an older backend sends. + assert "status.requested_parallel_slots !== null && {" not in src + + +def test_hydration_keeps_the_slot_control_when_readopting_the_running_model(): + """`hydratingExistingModel` is true whenever the incoming status disagrees + with what this tab last recorded, which includes RE-ADOPTING a model the tab + never lost: the resident-adopt branch restores the model's own per-model + config and only then hydrates, passing the EXTERNAL id as + `previousCheckpoint`. An ungated clear there wipes the slot count that branch + just restored, and the blank persists into `savePerModelConfig`, so a Save + the user reads as a no-op erases their remembered override. + + Only that branch knows the model is unchanged, so it says so explicitly. + Slot counts cannot stand in: the echo falls back to the server-wide default, + so a genuine A->B swap can echo exactly A's explicit count.""" + status = " ".join(_read("features/chat/lib/apply-inference-status-to-store.ts").split()) + assert ( + "const slotsModelChanged = hydratingExistingModel && !options.readoptingSameModel;" + in status + ) + assert "...(seedLoadParams && slotsModelChanged && { nParallel: null })," in status + # Never a slot-count proxy for "same model". + assert "prevState.loadedNParallel === (status.requested_parallel_slots" not in status + # The baseline seed stays ungated, or a rollback after a tab reload restores + # the model at the server default slots. + assert "loadedNParallel: status.requested_parallel_slots," in status + + runtime = " ".join(_read("features/chat/hooks/use-chat-model-runtime.ts").split()) + resident = runtime.split("if (!forceReload && isExternalModelId(selectedCheckpoint)) {", 1)[ + 1 + ].split("const stopDecision", 1)[0] + # What makes the scenario reachable: the branch restores the model's own + # config, then hydrates against the external id. + assert "applyPerModelConfigToRuntime(selection.previousConfig);" in resident + assert "previousCheckpoint: selectedCheckpoint," in resident + # Only reachable because the branch matched the id AND the variant first. + assert "resolveInferenceCheckpointId(residentStatus) === modelId" in resident + assert "readoptingSameModel: true," in resident + # The refresh() hydrate must NOT claim it: there the model really can change. + poll = runtime.split("setModels(listRes.models.map(toChatModelSummary));", 1)[1].split( + "} else if (!statusRes.active_model", 1 + )[0] + assert "applyActiveModelStatusToStore(statusRes, {" in poll + assert "readoptingSameModel" not in poll + + +def test_parallel_slots_are_never_recorded_for_a_diffusion_load(): + """A DiffusionGemma GGUF answers ``is_gguf: true``, but its runner ignores + ``--parallel``, so ``_parallel_slot_echo`` reports null slots for it. The + three load success paths must gate on ``is_diffusion`` too, or they record a + click-time count the load never committed. + + That phantom does not stay put: ``capturePresetLoadConfig`` snapshots + ``nParallel`` with no model gate and a preset carries no model identity, so + applying it over a TEXT GGUF sends the count as a real ``n_parallel``. + """ + runtime = " ".join(_read("features/chat/hooks/use-chat-model-runtime.ts").split()) + # One gated local feeds the control and the baseline, so they cannot drift. + assert "(loadResponse.is_gguf ?? false) && !(loadResponse.is_diffusion ?? false)" in runtime + assert "nParallel: committedSlots," in runtime + assert "loadedNParallel: committedSlots," in runtime + + adapter = " ".join(_read("features/chat/api/chat-adapter.ts").split()) + assert ( + "const committedSlots = (loadResp.is_diffusion ?? false) ? null " + ": (config.nParallel ?? null);" in adapter + ) + assert "nParallel: committedSlots," in adapter + assert "loadedNParallel: committedSlots," in adapter + + composer = " ".join(_read("features/chat/shared-composer.tsx").split()) + assert "targetIsGguf && !(resp.is_diffusion ?? false)" in composer + assert "nParallel: committedSlots," in composer + assert "loadedNParallel: committedSlots," in composer + + +def test_hydration_restores_a_remembered_slot_override(): + """The control is never seeded from the status echo, so a model running on a + remembered override shows a BLANK slot control after a browser reload or a + tab move to another GGUF. `ModelConfigPage.resolveInitial` prefers the live + store for the active model, so that blank is what the form edits: the next + Apply reloads at the server default and a Save writes the blank over the + remembered count. + + The seed is deliberately narrow: storage is read only on a fresh store or a + model change, never on a steady poll, and the value is adopted only when the + server already runs that exact count, which proves it is this model's own. + """ + src = _read("features/chat/lib/apply-inference-status-to-store.ts") + status = " ".join(src.split()) + assert ( + "resolveInitialConfig(checkpointId, status.gguf_variant ?? null)" in status + ), "the remembered override comes from per-model storage, not the echo" + assert ( + "const slotsUnseeded = prevState.loadedNParallel === null && " + "prevState.nParallel === null;" in status + ) + assert ( + "status.is_gguf && (slotsUnseeded || slotsModelChanged)" in status + ), "storage is read on a fresh store or a model change, never on a steady poll" + assert ( + "...(seedLoadParams && (slotsUnseeded || slotsModelChanged) &&" in status + ), "the seed fires in both cases the clear leaves the control blank" + assert ( + "rememberedNParallel != null && rememberedNParallel === " + "status.requested_parallel_slots && { nParallel: rememberedNParallel, }" in status + ) + # Both cases trip the model-change clear, so the seed only survives by + # being spread after it. + assert src.index("slotsModelChanged && { nParallel: null }") < src.index( + "nParallel: rememberedNParallel," + ) + + +def test_failed_switch_rollback_restores_the_slot_intent_not_the_resolved_count(): + """`loadedNParallel` holds a RESOLVED count even for a load that sent no + slots (the echo falls back to the server-wide default), so it is the right + value to re-send when recreating the previous server and the wrong one to put + back in the control: it turns "follow the server default" into an explicit + override that a later Save or preset capture pins. The outer catch only + repairs that for a staged config, so a plain string pick keeps the phantom. + + The intent comes from the picker's own pre-switch snapshot when there is one: + chat-page pre-applies the TARGET's config before calling selectModel, so the + live control describes the outgoing model only for a bare pick.""" + runtime = " ".join(_read("features/chat/hooks/use-chat-model-runtime.ts").split()) + assert ( + 'const previousNParallel = typeof selection !== "string" && ' + "selection.previousConfig ? (selection.previousConfig.nParallel ?? null) " + ": useChatRuntimeStore.getState().nParallel;" in runtime + ) + assert runtime.index("const previousNParallel") < runtime.index( + "applyPerModelConfigToRuntime(pendingLoadConfig);" + ), "a config staged on the selection must not replace it either" + picker = " ".join(_read("features/chat/chat-page.tsx").split()) + assert ( + "const previousConfig = currentRuntimePerModelConfig({ includeMaxSeqLength: true, }); " + "const hasAppliedConfig = applyModelLoadConfigToRuntime(" in picker + ), "the snapshot must be taken before the target's config is applied" + rollback = runtime.split("const rollbackSpeculativeType", 1)[1] + assert "nParallel: previousNParallel," in rollback + # Baseline and reload payload keep the resolved count, or the rollback + # recreates the previous model at a different slot count. + assert "loadedNParallel: stateBeforeUnload.loadedNParallel ?? null," in rollback + assert "n_parallel: stateBeforeUnload.loadedNParallel," in runtime + + def test_vulkan_inference_devices_are_the_pickable_set(): """GGUF loads run through llama-server, so on a Vulkan build the picker must offer the inference inventory (ggml ordinals, the space `--device Vulkan` diff --git a/unsloth_cli/commands/studio.py b/unsloth_cli/commands/studio.py index 864941a20a..9fd264ddf5 100644 --- a/unsloth_cli/commands/studio.py +++ b/unsloth_cli/commands/studio.py @@ -1263,7 +1263,8 @@ def studio_default( max = _PARALLEL_MAX, help = ( f"llama-server parallel decode slots ({_PARALLEL_MIN}..{_PARALLEL_MAX}). " - f"Default {_PARALLEL_DEFAULT_PLAIN}." + f"Default {_PARALLEL_DEFAULT_PLAIN}. The Studio run settings " + "(Parallel Slots) override it per load." ), ), cloudflare: Optional[bool] = typer.Option( @@ -1880,7 +1881,8 @@ def run( help = ( "llama-server parallel decode slots. N requests share one " "loaded model; each slot gets ctx/N KV cache. Default " - f"{_PARALLEL_DEFAULT_RUN} (pre-PR hardcoded value)." + f"{_PARALLEL_DEFAULT_RUN} (pre-PR hardcoded value). The Studio " + "run settings (Parallel Slots) can override it per load." ), ), cloudflare: Optional[bool] = typer.Option(