From 9ec6c8bb5a341004a9e3aea6e60fe7023024855d Mon Sep 17 00:00:00 2001 From: Unsloth Date: Sun, 26 Jul 2026 06:55:59 -0700 Subject: [PATCH 01/75] studio: apply saved per-model settings on API loads, add API monitor Per-model settings were mirrored to the server as two fields only, llama_extra_args and max_seq_length. Everything else lived in browser localStorage, so a model loaded by an API request came up with app defaults for context length, KV cache dtype, speculative decoding, tensor parallel and GPU placement. Store the full config server side and map it onto the same LoadRequest the picker builds, so a remote load and a picker load of the same model produce the same command line. Entries are keyed per quant, falling back to the bare repo id so existing entries keep resolving. Also moves the API monitor out of the settings tab: a full page at /api-monitor, plus a floating panel that opens itself when traffic arrives, and a settings page per model reachable from the Hub. --- studio/backend/core/inference/api_monitor.py | 15 +- studio/backend/routes/inference.py | 51 +- studio/backend/routes/settings.py | 75 +- studio/backend/tests/test_api_monitor.py | 31 + .../backend/tests/test_openai_auto_switch.py | 174 ++++ .../utils/openai_auto_switch_settings.py | 224 ++++- studio/frontend/src/app/router.tsx | 2 + studio/frontend/src/app/routes/__root.tsx | 28 +- studio/frontend/src/app/routes/api.tsx | 22 + .../api-monitor/api-monitor-overlay.tsx | 384 +++++++++ .../features/api-monitor/api-monitor-page.tsx | 778 ++++++++++++++++++ .../components/saved-model-settings.tsx | 140 ++++ .../src/features/api-monitor/index.ts | 13 + .../src/features/api-monitor/overlay-store.ts | 48 ++ .../features/api-monitor/use-api-monitor.ts | 297 +++++++ .../src/features/chat/api/chat-api.ts | 7 + .../features/hub/catalog/download-card.tsx | 35 + .../hub/catalog/hub-model-settings-view.tsx | 138 ++++ .../hub/catalog/local-on-device-card.tsx | 12 + .../features/hub/catalog/model-inspector.tsx | 4 + .../hub/catalog/models-catalog-lists.tsx | 4 + .../hub/catalog/models-catalog-rows.tsx | 42 +- .../features/hub/catalog/models-catalog.tsx | 5 + studio/frontend/src/features/hub/hub-page.tsx | 174 +++- .../model-picker/api/model-overrides.ts | 174 ++++ .../components/model-config-page.tsx | 25 +- .../model-selector/model-row-menu.tsx | 26 +- .../src/features/model-picker/index.ts | 10 + .../components/api-monitor-console.tsx | 393 --------- .../settings/components/monitor-link.tsx | 97 +++ .../features/settings/tabs/api-keys-tab.tsx | 4 +- 31 files changed, 2988 insertions(+), 444 deletions(-) create mode 100644 studio/frontend/src/app/routes/api.tsx create mode 100644 studio/frontend/src/features/api-monitor/api-monitor-overlay.tsx create mode 100644 studio/frontend/src/features/api-monitor/api-monitor-page.tsx create mode 100644 studio/frontend/src/features/api-monitor/components/saved-model-settings.tsx create mode 100644 studio/frontend/src/features/api-monitor/index.ts create mode 100644 studio/frontend/src/features/api-monitor/overlay-store.ts create mode 100644 studio/frontend/src/features/api-monitor/use-api-monitor.ts create mode 100644 studio/frontend/src/features/hub/catalog/hub-model-settings-view.tsx create mode 100644 studio/frontend/src/features/model-picker/api/model-overrides.ts delete mode 100644 studio/frontend/src/features/settings/components/api-monitor-console.tsx create mode 100644 studio/frontend/src/features/settings/components/monitor-link.tsx diff --git a/studio/backend/core/inference/api_monitor.py b/studio/backend/core/inference/api_monitor.py index f76a38576f..f9922a711a 100644 --- a/studio/backend/core/inference/api_monitor.py +++ b/studio/backend/core/inference/api_monitor.py @@ -269,9 +269,20 @@ class ApiMonitor: if entry.status == "running" and (subject is None or entry.subject == subject) ) - def clear(self) -> None: + def clear(self, *, subject: Optional[str] = None) -> None: + """Drop recorded entries. ``subject`` limits the wipe to one caller's. + + Every other read on this class is subject-scoped, so an unscoped clear + would let one user erase another's history (and zero their active count + mid-generation). Callers that genuinely mean "everything" pass None. + """ with self._lock: - self._entries.clear() + if subject is None: + self._entries.clear() + return + self._entries = deque( + entry for entry in self._entries if entry.subject != subject + ) def _find_locked(self, entry_id: str) -> Optional[ApiMonitorEntry]: for entry in self._entries: diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.py index 9d40650a56..d529a1cd85 100644 --- a/studio/backend/routes/inference.py +++ b/studio/backend/routes/inference.py @@ -3581,6 +3581,7 @@ async def _maybe_auto_switch_model( get_openai_auto_switch_enabled, get_auto_unload_idle_seconds, get_model_override, + model_override_load_kwargs, ) from core.inference.local_model_resolver import resolve_local_gguf from core.inference.llama_keepwarm import ( @@ -3716,13 +3717,35 @@ async def _maybe_auto_switch_model( if _already_serving(): _record_serving_alias() return - # Apply this model's saved launch flags so the swap honors the config. - override = get_model_override(override_id) + # Apply this model's saved launch config so an API-driven swap + # loads it exactly as the picker would: context, KV dtype, + # speculative decoding, chat template and GPU placement, not + # just the two legacy flags. Look the config up under the + # variant-qualified id first (two quants of one repo can carry + # different configs), then the bare advertised id, then the + # concrete load path, so a config saved under any of the names + # this model is known by is found. + override = {} + for override_key in ( + f"{override_id}:{variant}" if variant else None, + override_id, + target_id, + ): + if not override_key: + continue + override = get_model_override(override_key) + if override: + break load_kwargs = {"model_path": target_id, "gguf_variant": variant} - if override.get("llama_extra_args") is not None: - load_kwargs["llama_extra_args"] = override["llama_extra_args"] - if override.get("max_seq_length") is not None: - load_kwargs["max_seq_length"] = override["max_seq_length"] + load_kwargs.update( + model_override_load_kwargs( + override, + # variant is set for every GGUF the resolver returns; the + # reload-stash path carries the quant it froze. + is_gguf = bool(variant) + or target_id.lower().endswith(".gguf"), + ) + ) # Reuse the load impl so its dedup, tensor fallback, and threading # apply. Call the impl directly: we already hold the lifecycle gate # the /load route would otherwise take, so the route would deadlock. @@ -5703,6 +5726,22 @@ async def get_api_monitor(current_subject: str = Depends(get_current_subject)): } +@studio_router.delete("/monitor") +async def clear_api_monitor(current_subject: str = Depends(get_current_subject)): + """Drop this caller's recorded API history so a debugging session starts clean. + + Scoped to the current subject, like every read on the monitor: an unscoped + wipe would erase another user's history and zero their active-request count + while their generation is still streaming. + + The caller's own in-flight requests are dropped from the log too; they keep + streaming to their client, they just stop being reported here (a later append + re-adds nothing, since the entry id no longer resolves). + """ + api_monitor.clear(subject = current_subject) + return {"cleared": True} + + @studio_router.get("/monitor/{entry_id}") async def get_api_monitor_entry(entry_id: str, current_subject: str = Depends(get_current_subject)): """Return full prompt/reply details for one OpenAI-compatible API request.""" diff --git a/studio/backend/routes/settings.py b/studio/backend/routes/settings.py index fef18a9145..2323d33915 100644 --- a/studio/backend/routes/settings.py +++ b/studio/backend/routes/settings.py @@ -34,6 +34,7 @@ from utils.helper_precache_settings import ( helper_model_disabled_by_env, set_helper_precache_enabled, ) +from picker.schemas import MAX_CHAT_TEMPLATE_BYTES from utils.coding_agents import CODING_AGENTS, detect_installed_coding_agents from utils.openai_auto_switch_settings import ( DEFAULT_AUTO_UNLOAD_KEEP_KV, @@ -126,11 +127,51 @@ class OpenAIAutoSwitchResponse(BaseModel): class ModelOverridePayload(BaseModel): - model_id: str = Field(..., min_length = 1) - llama_extra_args: list[str] = Field(default_factory = list) + """One model's saved launch config, applied when the API loads that model. + + Everything past ``model_id`` is optional and omitted means "app default", so a + payload carrying only ``model_id`` clears the entry. The bounds here mirror + ``LoadRequest`` so a bad value is rejected at the boundary instead of being + silently dropped by the normalizer; the enum-ish fields (KV dtype, speculative + mode) are left to it, since their valid sets follow the llama.cpp build. + """ + + model_id: str = Field(..., min_length = 1, max_length = 512) + # None means "leave the stored value alone": the settings UI has no control + # for launch flags, so a save from it must not wipe flags set through this + # API. An explicit [] clears them (that is how "forget this model" arrives). + llama_extra_args: Optional[list[str]] = None # ge=1: 0 is not a valid sequence length, and the setter drops a falsy value, # so reject it at the boundary instead of accepting then silently discarding it. max_seq_length: Optional[int] = Field(default = None, ge = 1, le = 1048576) + custom_context_length: Optional[int] = Field(default = None, ge = 1, le = 1048576) + kv_cache_dtype: Optional[str] = Field(default = None, max_length = 32) + speculative_type: Optional[str] = Field(default = None, max_length = 32) + spec_draft_n_max: Optional[int] = Field(default = None, ge = 1, le = 16) + tensor_parallel: bool = False + # Validated in bytes below, not by max_length: pydantic counts characters, + # so a multi-byte template could pass here and then be silently dropped by + # the normalizer (which measures UTF-8) while the request still returned 200. + chat_template_override: Optional[str] = None + gpu_memory_mode: Optional[Literal["auto", "manual"]] = None + # -1 is Auto (llama.cpp --fit sizes the offload); the normalizer treats it as unset. + gpu_layers: Optional[int] = Field(default = None, ge = -1, le = 1024) + n_cpu_moe: Optional[int] = Field(default = None, ge = 0, le = 1024) + gpu_ids: Optional[list[int]] = None + + + @field_validator("chat_template_override") + @classmethod + def _limit_chat_template_bytes(cls, value: Optional[str]) -> Optional[str]: + # Mirrors LoadRequest.normalize_blank_chat_template_override so the same + # template is accepted or rejected identically on both paths. + if value is None: + return None + if len(value.encode("utf-8")) > MAX_CHAT_TEMPLATE_BYTES: + raise ValueError( + f"Chat template exceeds the {MAX_CHAT_TEMPLATE_BYTES}-byte limit." + ) + return value class ModelOverridesResponse(BaseModel): @@ -289,12 +330,40 @@ def update_openai_auto_switch_override( payload: ModelOverridePayload, current_subject: str = Depends(get_current_subject) ) -> ModelOverridesResponse: from core.inference.llama_server_args import validate_extra_args + from utils.openai_auto_switch_settings import get_model_override try: - extra_args = validate_extra_args(payload.llama_extra_args) + # A payload carrying only model_id is the documented "remove", so it + # wipes everything. Otherwise it is a real save, and omitted launch flags + # are carried over from the stored entry (the settings UI cannot express + # them and must not delete them). + requested_extra_args = payload.llama_extra_args + saved_fields = payload.model_dump( + exclude = {"model_id", "llama_extra_args"}, exclude_none = True + ) + is_removal = not payload.tensor_parallel and not { + key: value + for key, value in saved_fields.items() + if key != "tensor_parallel" + } + if requested_extra_args is None and not is_removal: + requested_extra_args = get_model_override(payload.model_id).get( + "llama_extra_args" + ) + extra_args = validate_extra_args(requested_extra_args) set_model_override( payload.model_id, llama_extra_args = extra_args, max_seq_length = payload.max_seq_length, + custom_context_length = payload.custom_context_length, + kv_cache_dtype = payload.kv_cache_dtype, + speculative_type = payload.speculative_type, + spec_draft_n_max = payload.spec_draft_n_max, + tensor_parallel = payload.tensor_parallel, + chat_template_override = payload.chat_template_override, + gpu_memory_mode = payload.gpu_memory_mode, + gpu_layers = payload.gpu_layers, + n_cpu_moe = payload.n_cpu_moe, + gpu_ids = payload.gpu_ids, ) except ValueError as exc: raise log_and_http_error( diff --git a/studio/backend/tests/test_api_monitor.py b/studio/backend/tests/test_api_monitor.py index 56bc404350..a8d0e1e832 100644 --- a/studio/backend/tests/test_api_monitor.py +++ b/studio/backend/tests/test_api_monitor.py @@ -258,3 +258,34 @@ def test_api_monitor_append_reply_exact_cap_then_more_marks_truncated(): monitor.append_reply(entry_id, "y") reply = monitor.snapshot()[0]["reply"] assert len(reply) == m._MAX_REPLY_CHARS and reply.endswith("...") + + +def test_api_monitor_clear_is_scoped_to_one_subject(): + # Every other read on the monitor is subject-scoped. An unscoped clear from + # the route would let one caller erase another's history and zero their + # active count in the middle of a generation. + monitor = ApiMonitor(max_entries = 4) + alice = monitor.start( + endpoint = "/v1/chat/completions", + method = "POST", + model = "m", + prompt = "alice prompt", + subject = "alice", + ) + bob = monitor.start( + endpoint = "/v1/chat/completions", + method = "POST", + model = "m", + prompt = "bob prompt", + subject = "bob", + ) + + monitor.clear(subject = "alice") + assert monitor.snapshot(subject = "alice") == [] + assert [entry["id"] for entry in monitor.snapshot(subject = "bob")] == [bob] + assert monitor.active_count(subject = "bob") == 1 + assert monitor.get(alice, subject = "alice") is None + + # Passing no subject is the explicit "everything" path. + monitor.clear() + assert monitor.snapshot(subject = "bob") == [] diff --git a/studio/backend/tests/test_openai_auto_switch.py b/studio/backend/tests/test_openai_auto_switch.py index 9c6c20e6b6..3899b1444a 100644 --- a/studio/backend/tests/test_openai_auto_switch.py +++ b/studio/backend/tests/test_openai_auto_switch.py @@ -3819,3 +3819,177 @@ def test_env_idle_below_floor_is_clamped(monkeypatch): assert settings.get_auto_unload_idle_seconds() == 600 monkeypatch.delenv(settings.MODEL_IDLE_TTL_ENV_VAR) assert settings.get_auto_unload_idle_seconds() == 0 + + +# --------------------------------------------------------------------------- +# Per-model launch config: normalization, LoadRequest mapping, key resolution. +# --------------------------------------------------------------------------- + + +def test_normalize_model_override_drops_unusable_fields_and_keeps_the_rest(): + # A stale field must not cost the user the whole config, so bad values are + # dropped one by one rather than rejecting the payload. + entry = settings.normalize_model_override( + { + "max_seq_length": 8192, + "kv_cache_dtype": "not_a_dtype", + "speculative_type": "mtp", + "spec_draft_n_max": 999, # out of range for the MTP draft count + "gpu_memory_mode": "auto", # only "manual" is a real override + "gpu_layers": -1, # -1 is Auto, which is already the default + "n_cpu_moe": 0, + "gpu_ids": [1, 1, 0, "2", -5], + "tensor_parallel": False, + "llama_extra_args": [], + } + ) + assert entry == { + "max_seq_length": 8192, + "speculative_type": "mtp", + "gpu_ids": [1, 0, 2], + } + + +def test_normalize_model_override_rejects_oversized_chat_template(): + small = settings.normalize_model_override({"chat_template_override": "{{ bos }}"}) + assert small["chat_template_override"] == "{{ bos }}" + # The limit is bytes, not characters: a multi-byte template just under the + # character limit can still be over the byte limit. + huge = "é" * settings.MAX_CHAT_TEMPLATE_OVERRIDE_BYTES + assert "chat_template_override" not in settings.normalize_model_override( + {"chat_template_override": huge} + ) + + +def test_spec_draft_n_max_only_stored_for_mtp_modes(): + mtp = settings.normalize_model_override( + {"speculative_type": "mtp", "spec_draft_n_max": 4} + ) + assert mtp["spec_draft_n_max"] == 4 + # A non-MTP mode ignores the draft count at load time, so storing it would + # show the user an edit that never takes effect. + ngram = settings.normalize_model_override( + {"speculative_type": "ngram", "spec_draft_n_max": 4} + ) + assert "spec_draft_n_max" not in ngram + + +def test_resolve_fit_max_seq_length_hands_sizing_to_fit_under_manual_auto_layers(): + # Manual GPU memory with Auto layers means llama.cpp --fit owns the context, + # so the load sends the context pin (or 0), not the stored max seq length. + override = {"gpu_memory_mode": "manual", "max_seq_length": 8192} + assert settings.resolve_fit_max_seq_length(override, is_gguf = True) == 0 + assert ( + settings.resolve_fit_max_seq_length( + {**override, "custom_context_length": 4096}, is_gguf = True + ) + == 4096 + ) + # Pinning the layer count takes --fit back out of the picture. + assert ( + settings.resolve_fit_max_seq_length({**override, "gpu_layers": 20}, is_gguf = True) + == 8192 + ) + # Not a GGUF, so none of this applies. + assert settings.resolve_fit_max_seq_length(override, is_gguf = False) == 8192 + + +def test_model_override_load_kwargs_gates_gpu_placement_on_gguf(): + override = { + "max_seq_length": 4096, + "kv_cache_dtype": "q8_0", + "tensor_parallel": True, + "gpu_memory_mode": "manual", + "gpu_layers": 20, + "n_cpu_moe": 3, + "gpu_ids": [0, 1], + } + gguf = settings.model_override_load_kwargs(override, is_gguf = True) + assert gguf["cache_type_kv"] == "q8_0" + assert gguf["tensor_parallel"] is True + assert gguf["gpu_layers"] == 20 + assert gguf["gpu_ids"] == [0, 1] + + # A safetensors model loads through HF auto-placement; inheriting a GGUF GPU + # pin here would silently change where the weights land. + safetensors = settings.model_override_load_kwargs(override, is_gguf = False) + assert safetensors["max_seq_length"] == 4096 + assert "gpu_layers" not in safetensors + assert "gpu_ids" not in safetensors + assert "n_cpu_moe" not in safetensors + assert "gpu_memory_mode" not in safetensors + + # Every key it produces has to be a real LoadRequest field, or the load call + # raises TypeError at the moment the user's request arrives. + LoadRequest(model_path = "unsloth/B-GGUF", **gguf) + + +def test_auto_switch_prefers_variant_qualified_override(monkeypatch): + # Settings are saved per quant, so Q4_K_M and Q8_0 of the same repo are + # different entries; the bare repo id is only the fallback. + backend = _FakeBackend(None) + rec = _LoadRecorder(backend) + _wire( + monkeypatch, + enabled = True, + resolves_to = ("unsloth/B-GGUF", "Q4_K_M", "unsloth/B-GGUF"), + backend = backend, + recorder = rec, + ) + stored = { + "unsloth/B-GGUF": {"max_seq_length": 1024}, + "unsloth/B-GGUF:Q4_K_M": {"max_seq_length": 8192, "gpu_layers": 20}, + } + monkeypatch.setattr(settings, "get_model_override", lambda mid: stored.get(mid, {})) + + _run_hook("unsloth/B-GGUF") + req = rec.calls[0] + assert req.max_seq_length == 8192 + assert req.gpu_layers == 20 + + +def test_auto_switch_falls_back_to_bare_repo_override(monkeypatch): + backend = _FakeBackend(None) + rec = _LoadRecorder(backend) + _wire( + monkeypatch, + enabled = True, + resolves_to = ("unsloth/B-GGUF", "Q4_K_M", "unsloth/B-GGUF"), + backend = backend, + recorder = rec, + ) + stored = {"unsloth/B-GGUF": {"max_seq_length": 1024}} + monkeypatch.setattr(settings, "get_model_override", lambda mid: stored.get(mid, {})) + + _run_hook("unsloth/B-GGUF") + assert rec.calls[0].max_seq_length == 1024 + + +def test_override_route_preserves_launch_flags_across_a_settings_only_update(monkeypatch): + # The settings page has no control for llama_extra_args, so saving from it + # omits the field. Omitted must mean "leave it alone", or every save from the + # UI would quietly wipe flags set elsewhere. + import routes.settings as settings_route + + _mock_override_store(monkeypatch) + settings_route.update_openai_auto_switch_override( + settings_route.ModelOverridePayload( + model_id = "unsloth/B-GGUF", llama_extra_args = ["--flash-attn"] + ), + "tester", + ) + resp = settings_route.update_openai_auto_switch_override( + settings_route.ModelOverridePayload(model_id = "unsloth/B-GGUF", max_seq_length = 4096), + "tester", + ) + entry = resp.overrides["unsloth/B-GGUF"] + assert entry["llama_extra_args"] == ["--flash-attn"] + assert entry["max_seq_length"] == 4096 + + # An explicit empty list is how the UI says "forget this model", and with no + # other fields left that removes the entry outright. + gone = settings_route.update_openai_auto_switch_override( + settings_route.ModelOverridePayload(model_id = "unsloth/B-GGUF", llama_extra_args = []), + "tester", + ) + assert "unsloth/B-GGUF" not in gone.overrides diff --git a/studio/backend/utils/openai_auto_switch_settings.py b/studio/backend/utils/openai_auto_switch_settings.py index 7007440f4c..413d75ad1e 100644 --- a/studio/backend/utils/openai_auto_switch_settings.py +++ b/studio/backend/utils/openai_auto_switch_settings.py @@ -210,8 +210,211 @@ def set_openai_auto_switch( ) +# --- Per-model launch config ------------------------------------------------- +# +# An override is the server-side twin of the UI's per-model config (the browser +# localStorage map behind features/model-picker/model-config). The UI mirrors +# every save here so a model loaded by an OpenAI-compatible API request gets the +# same launch settings a user would get loading it from the picker; without this +# the API path could only ever apply the two legacy fields below. +# +# Legacy entries hold just {llama_extra_args, max_seq_length}; every field is +# optional and absent means "fall back to the app default", so old entries keep +# loading correctly. A write is a full replace of the fields it expresses, so the +# route carries `llama_extra_args` over when the payload omits it (the settings +# UI has no control for launch flags and must not wipe them). +# +# Known gap: the picker resolves a couple of knobs as "per-model value, else the +# user's global preference" -- GPU memory mode and speculative decoding, whose +# globals live in browser localStorage. An override deliberately stores only an +# explicit per-model choice (so the model keeps following later global changes), +# and the server cannot see the globals at all. So for a model that follows the +# global on one of those two, an API load falls back to the app default rather +# than the user's global. Every other field matches the picker exactly. + +# Mirrors _valid_cache_types in core/inference/llama_cpp.py. +VALID_KV_CACHE_DTYPES = frozenset( + {"f16", "bf16", "q8_0", "q4_0", "q4_1", "q5_0", "q5_1", "iq4_nl", "f32"} +) +# Canonical values plus the legacy spellings LoadRequest still accepts. +VALID_SPECULATIVE_TYPES = frozenset( + { + "auto", + "mtp", + "ngram", + "mtp+ngram", + "off", + "default", + "draft-mtp", + "ngram-mod", + "ngram-simple", + } +) +# Only these two consume spec_draft_n_max (mirrors MTP_SPECULATIVE_TYPES in the UI). +MTP_SPECULATIVE_TYPES = frozenset({"mtp", "mtp+ngram", "draft-mtp"}) +VALID_GPU_MEMORY_MODES = frozenset({"auto", "manual"}) + +MAX_SEQ_LENGTH_CEILING = 1048576 +MAX_CHAT_TEMPLATE_OVERRIDE_BYTES = 65_536 + + +def _clean_str(value: Any, allowed: frozenset[str]) -> Optional[str]: + if not isinstance(value, str): + return None + normalized = value.strip().lower() + return normalized if normalized in allowed else None + + +def _bounded_int(value: Any, *, minimum: int, maximum: int) -> Optional[int]: + try: + parsed = int(value) + except (TypeError, ValueError): + return None + if parsed < minimum or parsed > maximum: + return None + return parsed + + +def normalize_model_override(payload: dict[str, Any]) -> dict[str, Any]: + """Validate one per-model launch config, dropping anything unusable. + + Silently drops rather than raising: an override is a convenience mirror of the + UI's config, so one stale field (a KV dtype this llama.cpp build lost, a GPU id + from another host) must not block persisting the rest or fail the API load that + reads it. ``validate_extra_args`` is the caller's job -- it lives in the + llama_server_args allow-list module, which this one must not import. + """ + entry: dict[str, Any] = {} + + extra_args = payload.get("llama_extra_args") + if isinstance(extra_args, (list, tuple)) and extra_args: + entry["llama_extra_args"] = [str(arg) for arg in extra_args] + + # 0 / negative means "unset"; the loader reads absence as the app default. + for key in ("max_seq_length", "custom_context_length"): + parsed = _bounded_int(payload.get(key), minimum = 1, maximum = MAX_SEQ_LENGTH_CEILING) + if parsed: + entry[key] = parsed + + kv_cache_dtype = _clean_str(payload.get("kv_cache_dtype"), VALID_KV_CACHE_DTYPES) + if kv_cache_dtype: + entry["kv_cache_dtype"] = kv_cache_dtype + + speculative_type = _clean_str(payload.get("speculative_type"), VALID_SPECULATIVE_TYPES) + if speculative_type: + entry["speculative_type"] = speculative_type + # Only meaningful for the MTP modes; storing it otherwise would resurface + # in the UI as an edit the loader silently ignores. + if speculative_type in MTP_SPECULATIVE_TYPES: + spec_draft_n_max = _bounded_int( + payload.get("spec_draft_n_max"), minimum = 1, maximum = 16 + ) + if spec_draft_n_max: + entry["spec_draft_n_max"] = spec_draft_n_max + + if _coerce_bool(payload.get("tensor_parallel")): + entry["tensor_parallel"] = True + + template = payload.get("chat_template_override") + if isinstance(template, str) and template.strip(): + if len(template.encode("utf-8")) <= MAX_CHAT_TEMPLATE_OVERRIDE_BYTES: + entry["chat_template_override"] = template + + # Only "manual" is a real override: persisting "auto" would pin the model and + # stop it following later changes to the global GPU memory preference. + if _clean_str(payload.get("gpu_memory_mode"), VALID_GPU_MEMORY_MODES) == "manual": + entry["gpu_memory_mode"] = "manual" + + # -1 is Auto (llama.cpp --fit owns layer sizing), which is also the default, + # so only a pinned count >= 0 is worth storing. + gpu_layers = _bounded_int(payload.get("gpu_layers"), minimum = 0, maximum = 1024) + if gpu_layers is not None: + entry["gpu_layers"] = gpu_layers + + n_cpu_moe = _bounded_int(payload.get("n_cpu_moe"), minimum = 1, maximum = 1024) + if n_cpu_moe: + entry["n_cpu_moe"] = n_cpu_moe + + gpu_ids = payload.get("gpu_ids") + if isinstance(gpu_ids, (list, tuple)) and gpu_ids: + # De-duplicate, preserving order: resolve_requested_gpu_ids rejects a + # repeated id outright, so storing [0, 0] would make every later API load + # of this model fail with a 400 that the picker never hits. + cleaned_ids: list[int] = [] + for gid in gpu_ids: + parsed = _bounded_int(gid, minimum = 0, maximum = 1024) + if parsed is not None and parsed not in cleaned_ids: + cleaned_ids.append(parsed) + if cleaned_ids: + entry["gpu_ids"] = cleaned_ids + + return entry + + +def resolve_fit_max_seq_length(override: dict[str, Any], *, is_gguf: bool) -> Optional[int]: + """The ``max_seq_length`` an API load should send for this override. + + Mirrors resolveFitMaxSeqLength in the UI (features/chat/presets/preset-policy.ts): + under Manual GPU memory with Auto layers, llama.cpp's ``--fit`` owns context + sizing, so the load sends the explicit context pin (or 0 to hand sizing over) + rather than the stored max sequence length. Returns None to leave the field + at the loader's default. + """ + manual_auto_layers = ( + is_gguf + and override.get("gpu_memory_mode") == "manual" + and override.get("gpu_layers") is None + ) + if manual_auto_layers: + return override.get("custom_context_length") or 0 + # max_seq_length wins where both are set. The UI only ever sends it for a + # non-GGUF model (a GGUF's context is `custom_context_length`), so in + # practice the two never collide from that path; a hand-written or legacy + # entry that sets it on a GGUF is honoured, which is this API's contract. + return override.get("max_seq_length") or override.get("custom_context_length") + + +def model_override_load_kwargs(override: dict[str, Any], *, is_gguf: bool) -> dict[str, Any]: + """Map a stored per-model config onto ``LoadRequest`` keyword arguments. + + Mirrors the UI's load payload (features/chat/api/chat-adapter.ts) so an API + auto-switch load and a picker load of the same model produce the same command + line. GPU placement is GGUF-only there, so it is gated the same way here: a + safetensors model loads through HF auto-placement and must not inherit a + hidden GGUF GPU pin. + """ + if not override: + return {} + kwargs: dict[str, Any] = {} + + max_seq_length = resolve_fit_max_seq_length(override, is_gguf = is_gguf) + if max_seq_length is not None: + kwargs["max_seq_length"] = max_seq_length + for source, target in ( + ("llama_extra_args", "llama_extra_args"), + ("kv_cache_dtype", "cache_type_kv"), + ("speculative_type", "speculative_type"), + ("spec_draft_n_max", "spec_draft_n_max"), + ("tensor_parallel", "tensor_parallel"), + ("chat_template_override", "chat_template_override"), + ): + if override.get(source) is not None: + kwargs[target] = override[source] + + if is_gguf: + if override.get("gpu_memory_mode") is not None: + kwargs["gpu_memory_mode"] = override["gpu_memory_mode"] + if override.get("gpu_layers") is not None: + kwargs["gpu_layers"] = override["gpu_layers"] + if override.get("n_cpu_moe") is not None: + kwargs["n_cpu_moe"] = override["n_cpu_moe"] + if override.get("gpu_ids") is not None: + kwargs["gpu_ids"] = override["gpu_ids"] + return kwargs + + def get_model_overrides() -> dict[str, dict]: - """Per-model launch overrides keyed by model id ({llama_extra_args, max_seq_length}).""" + """Per-model launch configs keyed by model id (see normalize_model_override).""" raw = _cached_setting(MODEL_OVERRIDES_SETTING_KEY, None) return raw if isinstance(raw, dict) else {} @@ -226,15 +429,22 @@ def set_model_override( model_id: str, llama_extra_args: Optional[list[str]] = None, max_seq_length: Optional[int] = None, + **config: Any, ) -> dict: - """Upsert one model's launch override; an override with no fields removes it.""" + """Upsert one model's launch config; a config with no usable fields removes it. + + The two legacy parameters stay positional for existing callers; every other + per-model field is passed by keyword and normalized together. + """ if not model_id or not model_id.strip(): raise ValueError("model_id is required.") - entry: dict[str, Any] = {} - if llama_extra_args: - entry["llama_extra_args"] = [str(arg) for arg in llama_extra_args] - if max_seq_length: - entry["max_seq_length"] = max(0, int(max_seq_length)) + entry = normalize_model_override( + { + **config, + "llama_extra_args": llama_extra_args, + "max_seq_length": max_seq_length, + } + ) from storage.studio_db import upsert_app_setting_map_entry diff --git a/studio/frontend/src/app/router.tsx b/studio/frontend/src/app/router.tsx index 586c03d5df..3dd8d2e8e0 100644 --- a/studio/frontend/src/app/router.tsx +++ b/studio/frontend/src/app/router.tsx @@ -6,6 +6,7 @@ import { Button } from "@/components/ui/button"; import { MascotImg } from "@/components/mascot-img"; import { useT } from "@/i18n"; import { Route as rootRoute } from "./routes/__root"; +import { Route as apiMonitorRoute } from "./routes/api"; import { Route as dataRecipesRoute } from "./routes/data-recipes"; import { Route as dataRecipeRoute } from "./routes/data-recipes.$recipeId"; import { Route as chatRoute } from "./routes/chat"; @@ -32,6 +33,7 @@ const routeTree = rootRoute.addChildren([ exportRoute, dataRecipesRoute, dataRecipeRoute, + apiMonitorRoute, ]); function DefaultNotFound() { diff --git a/studio/frontend/src/app/routes/__root.tsx b/studio/frontend/src/app/routes/__root.tsx index 7137fd6f96..5dbeefa8f8 100644 --- a/studio/frontend/src/app/routes/__root.tsx +++ b/studio/frontend/src/app/routes/__root.tsx @@ -5,16 +5,14 @@ import { AppSidebar } from "@/components/app-sidebar"; import { Navbar } from "@/components/navbar"; import { fetchDeviceType, usePlatformStore } from "@/config/env"; import { SidebarInset, SidebarProvider } from "@/components/ui/sidebar"; -import { - SettingsDialog, - useSettingsDialogStore, -} from "@/features/settings"; +import { SettingsDialog, useSettingsDialogStore } from "@/features/settings"; import { ChatPage, clearNewChatDraft, useChatRuntimeStore, type ChatSearch, } from "@/features/chat"; +import { ApiMonitorOverlay } from "@/features/api-monitor/api-monitor-overlay"; import { RemoteCodeConsentDialog } from "@/features/security"; import { HfTokenWarningDialog } from "@/features/hf-auth"; import { TransformersUpgradeDialog } from "@/features/transformers-upgrade"; @@ -33,13 +31,7 @@ import { useRouterState, } from "@tanstack/react-router"; import { AnimatePresence, motion } from "motion/react"; -import { - Suspense, - useEffect, - useLayoutEffect, - useMemo, - useState, -} from "react"; +import { Suspense, useEffect, useLayoutEffect, useMemo, useState } from "react"; import { AppProvider } from "../provider"; declare module "@tanstack/react-router" { @@ -76,11 +68,17 @@ const CHAT_ONLY_ALLOWED = new Set([ // Export stays reachable on chat-only hosts so the page can show its own grayed-out reason // instead of a silent redirect; it self-gates via export capability, so nothing runs. "/export", + // Chat-only hosts (Intel Macs, Apple Silicon without MLX, no-GPU boxes) serve + // the OpenAI-compatible API exactly like any other host, so the monitor has to + // be reachable there. Without this the floating panel's own "Expand" button + // and the Settings > API card both redirect to /chat. + "/api-monitor", ]); function isChatOnlyAllowed(pathname: string): boolean { if (CHAT_ONLY_ALLOWED.has(pathname)) return true; - if (pathname === "/data-recipes" || pathname.startsWith("/data-recipes/")) return true; + if (pathname === "/data-recipes" || pathname.startsWith("/data-recipes/")) + return true; return false; } @@ -224,6 +222,8 @@ function RootLayout() { {!isAuthFlowRoute && } + {/* Opens itself when API traffic arrives; hides on the full monitor page. */} + {!isAuthFlowRoute && } @@ -241,7 +241,9 @@ function RootLayout() { className="!min-h-0 h-[calc(100dvh-var(--studio-titlebar-height,0px))] overflow-hidden" > - +
import("@/features/api-monitor"), + "ApiMonitorPage", +); + +export const Route = createRoute({ + getParentRoute: () => rootRoute, + // Not "/api": the backend owns that prefix (and "/v1"), and its SPA fallback + // deliberately 404s those paths so API clients get an API-shaped error rather + // than an HTML page. A deep link to /api would never reach the router. + path: "/api-monitor", + staticData: { title: "API" }, + beforeLoad: () => requireAuth(), + component: ApiMonitorPage, +}); diff --git a/studio/frontend/src/features/api-monitor/api-monitor-overlay.tsx b/studio/frontend/src/features/api-monitor/api-monitor-overlay.tsx new file mode 100644 index 0000000000..4411e052ad --- /dev/null +++ b/studio/frontend/src/features/api-monitor/api-monitor-overlay.tsx @@ -0,0 +1,384 @@ +// SPDX-License-Identifier: AGPL-3.0-only +// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 + +// Floating API monitor: opens itself when API traffic arrives, summarises it +// without taking over the window, and links through to the full page. + +import { getApiMonitor } from "@/features/chat/api/chat-api"; +import type { ApiMonitorEntry } from "@/features/chat/types/api"; +import { cn } from "@/lib/utils"; +import { + ArrowExpand01Icon, + DragDropVerticalIcon, + Globe02Icon, +} from "@hugeicons/core-free-icons"; +import { HugeiconsIcon } from "@hugeicons/react"; +import { useNavigate, useRouterState } from "@tanstack/react-router"; +import { XIcon } from "lucide-react"; +import { AnimatePresence, motion, useDragControls } from "motion/react"; +import { + type PointerEvent, + type ReactElement, + useEffect, + useMemo, + useRef, + useState, +} from "react"; +import { useApiMonitorOverlayStore } from "./overlay-store"; +import { computeStats } from "./use-api-monitor"; + +// Live cadence while the panel is on screen. +const OPEN_POLL_MS = 1500; +// While closed the poll only has to notice that traffic started, so it backs off. +const IDLE_POLL_MS = 5000; +// Requests shown in the panel; the rest are one click away on the full page. +const VISIBLE_ENTRIES = 4; +// How long the API must be quiet before a dismissed panel will open itself again. +const REARM_QUIET_MS = 60_000; + +const API_INFERENCE_PREFIX_RE = /^\/api\/inference/; +const V1_PREFIX_RE = /^\/v1\//; + +function compactEndpoint(endpoint: string): string { + return endpoint + .replace(API_INFERENCE_PREFIX_RE, "/api") + .replace(V1_PREFIX_RE, "/"); +} + +function formatDuration(value?: number | null): string { + if (value == null) { + return "live"; + } + if (value < 1000) { + return `${Math.round(value)}ms`; + } + return `${(value / 1000).toFixed(value < 10000 ? 1 : 0)}s`; +} + +function statusDotClass(status: ApiMonitorEntry["status"]): string { + switch (status) { + case "running": + return "bg-blue-500 animate-pulse"; + case "error": + return "bg-red-500"; + case "cancelled": + return "bg-amber-500"; + default: + return "bg-emerald-500"; + } +} + +function StatCell({ + label, + value, + tone, +}: { + label: string; + value: string; + tone?: "error" | "active"; +}): ReactElement { + return ( +
+ + {value} + + {/* Sentence case: Unsloth metric rows read as words, not headers. */} + + {label} + +
+ ); +} + +export function ApiMonitorOverlay(): ReactElement | null { + const { isOpen, suppressed, autoOpen, open, close, setAutoOpen } = + useApiMonitorOverlayStore(); + const navigate = useNavigate(); + const pathname = useRouterState({ select: (s) => s.location.pathname }); + const onFullPage = pathname === "/api-monitor"; + + const [data, setData] = useState + > | null>(null); + + // One loop for both jobs: panel contents while open, traffic watch while + // closed. Stands down on the full page, which polls for itself. + useEffect(() => { + if (onFullPage) { + return; + } + let cancelled = false; + let timer: number | undefined; + const intervalMs = isOpen ? OPEN_POLL_MS : IDLE_POLL_MS; + + function schedule(): void { + timer = window.setTimeout(poll, intervalMs); + } + + function poll(): void { + // A hidden tab has nobody to show the panel to. + if (document.hidden) { + schedule(); + return; + } + getApiMonitor() + .then((next) => { + if (!cancelled) setData(next); + }) + .catch(() => { + // An unreachable server is the full page's story to tell. + if (!cancelled) setData(null); + }) + .finally(() => { + if (!cancelled) schedule(); + }); + } + + poll(); + return () => { + cancelled = true; + if (timer !== undefined) window.clearTimeout(timer); + }; + }, [isOpen, onFullPage]); + + const entries = useMemo(() => data?.entries ?? [], [data]); + const stats = useMemo(() => computeStats(entries), [entries]); + + // Ids already seen. A set, not "the newest id": finishing moves an entry to + // the front, so the head flips without any new traffic. + const seenIdsRef = useRef>(new Set()); + // Seeded on the first response even when empty, so the first request of a + // fresh session is not mistaken for history. + const seededRef = useRef(false); + const lastNewEntryAtRef = useRef(0); + + useEffect(() => { + if (data == null) { + return; + } + const ids = data.entries.map((entry) => entry.id); + if (!seededRef.current) { + seededRef.current = true; + seenIdsRef.current = new Set(ids); + return; + } + const seen = seenIdsRef.current; + const hasNewTraffic = ids.some((id) => !seen.has(id)); + // Re-seed each poll so the set stays bounded by the server's ring buffer. + seenIdsRef.current = new Set(ids); + if (!hasNewTraffic) { + return; + } + const now = Date.now(); + const quietFor = now - lastNewEntryAtRef.current; + lastNewEntryAtRef.current = now; + if (!autoOpen || isOpen) { + return; + } + // A dismissal holds for that burst and re-arms only once the API goes + // quiet, so the next request cannot re-open it a second later. + if (suppressed && quietFor < REARM_QUIET_MS) { + return; + } + open(); + }, [data, autoOpen, suppressed, isOpen, open]); + + // The backlog built up while the poll was stood down is not new traffic. + useEffect(() => { + if (onFullPage) { + seededRef.current = false; + } + }, [onFullPage]); + + const [constraintsElement, setConstraintsElement] = + useState(null); + const constraintsRef = useMemo( + () => ({ current: constraintsElement }), + [constraintsElement], + ); + const dragControls = useDragControls(); + + function startDrag(event: PointerEvent): void { + event.preventDefault(); + dragControls.start(event); + } + + const visible = isOpen && !onFullPage; + const serverStatus = data?.status ?? "idle"; + + return ( + + {visible && ( +
+ +
+
+ + + API monitor + + +
+
+
+ +
+ +
+
+ +

+ {data?.active_model ?? "No model loaded"} +

+ + {/* Metrics on a soft tile, as the Hub and Train pages group readouts. */} +
+ 0 ? "active" : undefined} + /> + + 0 ? "error" : undefined} + /> + +
+ + {/* Borderless rows on 12px hover pills, as in the sidebar. */} +
+ {entries.length === 0 ? ( +

+ No requests yet. +

+ ) : ( + entries.slice(0, VISIBLE_ENTRIES).map((entry) => ( +
+ + + {compactEndpoint(entry.endpoint)} + + + {entry.error ? entry.error : entry.model} + + + {formatDuration(entry.duration_ms)} + +
+ )) + )} +
+ + {/* Through to payloads, filters and per request tokens. */} + + + {/* Closing only silences this burst; this is the permanent off. */} + + +
+ )} + + ); +} diff --git a/studio/frontend/src/features/api-monitor/api-monitor-page.tsx b/studio/frontend/src/features/api-monitor/api-monitor-page.tsx new file mode 100644 index 0000000000..fa5934ab5d --- /dev/null +++ b/studio/frontend/src/features/api-monitor/api-monitor-page.tsx @@ -0,0 +1,778 @@ +// SPDX-License-Identifier: AGPL-3.0-only +// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 + +// Full-page monitor for Unsloth's OpenAI-compatible API server. +// +// This replaces the small console that used to be buried in the API settings +// tab. Settings still owns configuration (keys, auto-switch, examples); this +// page owns observability -- what is being served right now, which requests +// failed and why, and which saved settings a remote load will apply. + +import { Button } from "@/components/ui/button"; +import { Input } from "@/components/ui/input"; +import { + Select, + SelectContent, + SelectItem, + SelectTrigger, + SelectValue, +} from "@/components/ui/select"; +import { Skeleton } from "@/components/ui/skeleton"; +import type { ApiMonitorEntry } from "@/features/chat/types/api"; +import { useSettingsDialogStore } from "@/features/settings"; +import { copyToClipboard } from "@/lib/copy-to-clipboard"; +import { Tick02Icon } from "@/lib/tick-icon"; +import { cn } from "@/lib/utils"; +import { + Copy01Icon, + Delete02Icon, + Globe02Icon, + PauseIcon, + PlayIcon, + RefreshIcon, + Settings02Icon, +} from "@hugeicons/core-free-icons"; +import { HugeiconsIcon } from "@hugeicons/react"; +import { type ReactElement, useEffect, useMemo, useRef, useState } from "react"; +import { SavedModelSettingsPanel } from "./components/saved-model-settings"; +import { + type MonitorStatusFilter, + filterEntries, + useApiMonitor, +} from "./use-api-monitor"; + +const API_INFERENCE_PREFIX_RE = /^\/api\/inference/; +const V1_PREFIX_RE = /^\/v1\//; + +const STATUS_FILTERS: { value: MonitorStatusFilter; label: string }[] = [ + { value: "all", label: "All requests" }, + { value: "running", label: "In flight" }, + { value: "completed", label: "Completed" }, + { value: "error", label: "Errors" }, + { value: "cancelled", label: "Cancelled" }, +]; + +function formatTime(value: number): string { + return new Date(value * 1000).toLocaleTimeString([], { + hour: "2-digit", + minute: "2-digit", + second: "2-digit", + }); +} + +function formatDuration(value?: number | null): string { + if (value == null) { + return "running"; + } + if (value < 1000) { + return `${Math.round(value)} ms`; + } + return `${(value / 1000).toFixed(value < 10000 ? 1 : 0)} s`; +} + +function formatCount(value: number): string { + return value.toLocaleString(); +} + +function compactEndpoint(endpoint: string): string { + return endpoint + .replace(API_INFERENCE_PREFIX_RE, "/api") + .replace(V1_PREFIX_RE, "/"); +} + +function statusDotClass(status: ApiMonitorEntry["status"]): string { + switch (status) { + case "running": + return "bg-blue-500 animate-pulse"; + case "error": + return "bg-red-500"; + case "cancelled": + return "bg-amber-500"; + default: + return "bg-emerald-500"; + } +} + +function statusTextClass(status: ApiMonitorEntry["status"]): string { + switch (status) { + case "running": + return "text-blue-600 dark:text-blue-400"; + case "error": + return "text-red-600 dark:text-red-400"; + case "cancelled": + return "text-amber-600 dark:text-amber-500"; + default: + return "text-emerald-600 dark:text-emerald-500"; + } +} + +function StatCard({ + label, + value, + hint, + tone, +}: { + label: string; + value: string; + hint?: string; + tone?: "default" | "error" | "active"; +}): ReactElement { + return ( +
+ + {label} + + + {value} + + {hint ? ( + + {hint} + + ) : null} +
+ ); +} + +function CopyButton({ + value, + label, +}: { + value: string; + label: string; +}): ReactElement { + const [copied, setCopied] = useState(false); + // Clearing the tick on a timer would set state after unmount if the user + // navigates away mid-flash, so the timer is cancelled on cleanup. + const timerRef = useRef(undefined); + useEffect( + () => () => { + if (timerRef.current !== undefined) window.clearTimeout(timerRef.current); + }, + [], + ); + return ( + + ); +} + +function ContextUsageBar({ + value, +}: { value?: number | null }): ReactElement | null { + if (value == null) { + return null; + } + const pct = Math.max(0, Math.min(100, Math.round(value * 100))); + return ( +
+
+
= 90 + ? "bg-red-500" + : pct >= 75 + ? "bg-amber-500" + : "bg-control-accent", + )} + style={{ width: `${pct}%` }} + /> +
+ + {pct}% + +
+ ); +} + +function RequestRow({ + entry, + selected, + onSelect, +}: { + entry: ApiMonitorEntry; + selected: boolean; + onSelect: () => void; +}): ReactElement { + const preview = + entry.error || + entry.reply_preview || + entry.prompt_preview || + (entry.status === "running" ? "Waiting for output…" : "No preview"); + return ( + + ); +} + +function PayloadBlock({ + title, + body, + truncated, + loading, + tone, +}: { + title: string; + body: string; + truncated?: boolean; + loading?: boolean; + tone?: "error"; +}): ReactElement { + return ( +
+
+

+ {title} +

+
+ {truncated ? ( + + preview only + + ) : null} + {body ? ( + + ) : null} +
+
+ 
+        {loading && !body ? "Loading…" : body || "–"}
+
+
+ ); +} + +function RequestDetail({ + entry, + detail, + loading, +}: { + entry: ApiMonitorEntry; + detail?: ApiMonitorEntry; + loading: boolean; +}): ReactElement { + // The detail fetch is a separate request, so it can describe an older state of + // a still-streaming entry. Prefer it only once it is at least as fresh as the + // list row, otherwise the panel would appear to rewind while tokens arrive. + const detailIsCurrent = + detail != null && + detail.status === entry.status && + detail.updated_at >= entry.updated_at; + const prompt = detail?.prompt ?? entry.prompt_preview; + const reply = detailIsCurrent + ? (detail.reply ?? entry.reply_preview) + : entry.reply_preview; + + return ( +
+
+
+ + + {entry.status} + + + {entry.id} + +
+

+ {entry.method} {entry.endpoint} +

+

+ {entry.model} +

+
+ +
+ {[ + { label: "Started", value: formatTime(entry.started_at) }, + { label: "Duration", value: formatDuration(entry.duration_ms) }, + { + label: "Prompt tokens", + value: + entry.prompt_tokens != null + ? formatCount(entry.prompt_tokens) + : "–", + }, + { + label: "Completion tokens", + value: + entry.completion_tokens != null + ? formatCount(entry.completion_tokens) + : "–", + }, + { + label: "Total tokens", + value: + entry.total_tokens != null + ? formatCount(entry.total_tokens) + : "–", + }, + { + label: "Context", + value: + entry.context_length != null + ? formatCount(entry.context_length) + : "–", + }, + ].map((item) => ( +
+
+ {item.label} +
+
+ {item.value} +
+
+ ))} +
+ + {entry.context_usage != null ? ( +
+ + Context used + + +
+ ) : null} + + {entry.error ? ( + + ) : null} + + + +
+ ); +} + +export function ApiMonitorPage(): ReactElement { + const { + data, + entries, + stats, + error, + loading, + refreshing, + paused, + setPaused, + refresh, + clear, + details, + loadingDetails, + requestDetail, + } = useApiMonitor(); + const [statusFilter, setStatusFilter] = useState("all"); + const [query, setQuery] = useState(""); + const [selectedId, setSelectedId] = useState(null); + + const visible = useMemo( + () => filterEntries(entries, statusFilter, query), + [entries, statusFilter, query], + ); + const selected = useMemo( + () => visible.find((entry) => entry.id === selectedId) ?? null, + [visible, selectedId], + ); + + // Refetch the selected entry while it streams so the payload grows with the + // reply. Keyed on identity and revision, never on `details`: the fetch rewrites + // `details` on every success, so depending on it loops on the detail endpoint, + // which takes the same lock every generated token does. + const selectedId_ = selected?.id ?? null; + const selectedUpdatedAt = selected?.updated_at ?? null; + const selectedIsMissing = selectedId_ != null && details[selectedId_] == null; + const lastFetchedRef = useRef(null); + useEffect(() => { + if (selectedId_ == null) { + return; + } + // `updated_at` advances per poll while streaming and settles when terminal. + // A missing payload always retries, covering a fetch that failed late. + const revision = `${selectedId_}@${selectedUpdatedAt ?? ""}`; + if (!selectedIsMissing && lastFetchedRef.current === revision) { + return; + } + lastFetchedRef.current = revision; + requestDetail(selectedId_); + }, [selectedId_, selectedUpdatedAt, selectedIsMissing, requestDetail]); + + const baseUrl = + typeof window === "undefined" ? "" : `${window.location.origin}/v1`; + const serverStatus = data?.status ?? "idle"; + const statusCopy = + serverStatus === "generating" + ? "Serving requests" + : serverStatus === "ready" + ? "Ready" + : "No model loaded"; + + return ( +
+
+
+

+ API +

+

+ Live traffic through Unsloth's OpenAI-compatible server. +

+
+
+ + + + +
+
+ + {/* Server summary: the two things you check first when a client can't + reach the API -- the base URL to point it at, and what is loaded. */} +
+
+ + + +
+ + Base URL + + + {baseUrl} + +
+ +
+
+ + Status + + + + {statusCopy} + +
+
+ + Loaded model + + + {data?.active_model ?? "None"} + {data?.context_length + ? ` · ${formatCount(data.context_length)} ctx` + : ""} + +
+ {paused ? ( + + Paused + + ) : null} +
+ + {error ? ( +
+ {error} +
+ ) : null} + +
+ 0 ? "active" : "default"} + /> + + + 0 ? "error" : "default"} + hint={ + stats.errorRate != null + ? `${Math.round(stats.errorRate * 100)}% of finished` + : undefined + } + /> + + +
+ +
+
+ setQuery(event.target.value)} + placeholder="Search model, endpoint, preview or error" + aria-label="Search API requests" + className="h-9 w-full min-w-0 flex-1 rounded-full border-none bg-muted shadow-none dark:bg-background sm:w-64 sm:flex-none" + /> + + + {formatCount(visible.length)} of {formatCount(entries.length)} + +
+ +
+
+ {loading ? ( +
+ {[0, 1, 2].map((i) => ( + + ))} +
+ ) : visible.length === 0 ? ( +

+ {entries.length === 0 + ? "No API traffic yet. Point a client at the base URL above to see requests here." + : "No requests match this filter."} +

+ ) : ( + visible.map((entry) => ( + setSelectedId(entry.id)} + /> + )) + )} +
+ +
+ {selected ? ( + + ) : ( +

+ Select a request to inspect its prompt, reply, tokens and + errors. +

+ )} +
+
+
+ + +
+ ); +} diff --git a/studio/frontend/src/features/api-monitor/components/saved-model-settings.tsx b/studio/frontend/src/features/api-monitor/components/saved-model-settings.tsx new file mode 100644 index 0000000000..6a1224472e --- /dev/null +++ b/studio/frontend/src/features/api-monitor/components/saved-model-settings.tsx @@ -0,0 +1,140 @@ +// SPDX-License-Identifier: AGPL-3.0-only +// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 + +// What a remote load will actually apply, which is otherwise unanswerable from +// outside the process. +// +// Read only on purpose: the config lives both here and in the browser's own +// per-model store, and the model's settings page is the only place that owns +// both, so it is the only place that can forget a model completely. + +import { Skeleton } from "@/components/ui/skeleton"; +import { + type ApiModelOverride, + type ApiModelOverrides, + fetchModelOverrides, +} from "@/features/model-picker/api/model-overrides"; +import { type ReactElement, useCallback, useEffect, useState } from "react"; + +/** Human-readable summary of the fields the loader will apply, in load order. */ +function describeOverride(override: ApiModelOverride): string[] { + const parts: string[] = []; + if (override.custom_context_length) { + parts.push(`${override.custom_context_length.toLocaleString()} context`); + } + if (override.max_seq_length) { + parts.push(`${override.max_seq_length.toLocaleString()} max seq`); + } + if (override.kv_cache_dtype) { + parts.push(`KV ${override.kv_cache_dtype}`); + } + if (override.speculative_type) { + parts.push( + override.spec_draft_n_max + ? `spec ${override.speculative_type} ×${override.spec_draft_n_max}` + : `spec ${override.speculative_type}`, + ); + } + if (override.tensor_parallel) { + parts.push("tensor parallel"); + } + if (override.gpu_memory_mode === "manual") { + parts.push("manual GPU memory"); + } + if (override.gpu_layers != null) { + parts.push(`${override.gpu_layers} GPU layers`); + } + if (override.n_cpu_moe) { + parts.push(`${override.n_cpu_moe} MoE layers on CPU`); + } + if (override.gpu_ids?.length) { + parts.push(`GPU ${override.gpu_ids.join(", ")}`); + } + if (override.chat_template_override) { + parts.push("custom chat template"); + } + if (override.llama_extra_args?.length) { + parts.push(override.llama_extra_args.join(" ")); + } + return parts; +} + +export function SavedModelSettingsPanel(): ReactElement { + const [overrides, setOverrides] = useState(null); + const [error, setError] = useState(null); + + const load = useCallback(async () => { + try { + setOverrides(await fetchModelOverrides()); + setError(null); + } catch (err: unknown) { + setError( + err instanceof Error + ? err.message + : "Could not load saved model settings", + ); + } + }, []); + + useEffect(() => { + void load(); + }, [load]); + + const entries = Object.entries(overrides ?? {}); + + return ( +
+
+

+ Settings applied on API load +

+

+ When a request names one of these models, Unsloth loads it with these + settings, the same ones you saved in the model's settings page. + Models without an entry load with app defaults. Edit or forget an + entry from that model's settings, which keeps this list and the + picker in step. +

+
+ + {error ? ( +
+ {error} +
+ ) : overrides == null ? ( +
+ {[0, 1].map((i) => ( + + ))} +
+ ) : entries.length === 0 ? ( +

+ No saved model settings yet. Open a model's settings, turn on + "Remember for this model", and it will be applied to API + loads too. +

+ ) : ( +
    + {entries.map(([modelId, override]) => { + const summary = describeOverride(override); + return ( +
  • +
    + + {modelId} + + + {summary.length > 0 ? summary.join(" · ") : "App defaults"} + +
    +
    • + ); + })} +
+ )} +
+ ); +} diff --git a/studio/frontend/src/features/api-monitor/index.ts b/studio/frontend/src/features/api-monitor/index.ts new file mode 100644 index 0000000000..28b01def8b --- /dev/null +++ b/studio/frontend/src/features/api-monitor/index.ts @@ -0,0 +1,13 @@ +// SPDX-License-Identifier: AGPL-3.0-only +// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 + +export { ApiMonitorPage } from "./api-monitor-page"; +export { ApiMonitorOverlay } from "./api-monitor-overlay"; +export { useApiMonitorOverlayStore } from "./overlay-store"; +export { + computeStats, + filterEntries, + useApiMonitor, + type MonitorStats, + type MonitorStatusFilter, +} from "./use-api-monitor"; diff --git a/studio/frontend/src/features/api-monitor/overlay-store.ts b/studio/frontend/src/features/api-monitor/overlay-store.ts new file mode 100644 index 0000000000..07ad4edd31 --- /dev/null +++ b/studio/frontend/src/features/api-monitor/overlay-store.ts @@ -0,0 +1,48 @@ +// SPDX-License-Identifier: AGPL-3.0-only +// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 + +import { create } from "zustand"; +import { persist } from "zustand/middleware"; + +interface ApiMonitorOverlayState { + /** Whether the floating panel is on screen right now. Session state. */ + isOpen: boolean; + /** Set on close so the panel does not pop back during the same burst. */ + suppressed: boolean; + /** Persisted opt out: when false the panel never opens itself. */ + autoOpen: boolean; + open: () => void; + close: () => void; + setAutoOpen: (autoOpen: boolean) => void; +} + +/** + * Only `autoOpen` persists. Open/closed is session state: a dismissal lasts the + * sitting, not forever. + */ +export const useApiMonitorOverlayStore = create()( + persist( + (set) => ({ + isOpen: false, + suppressed: false, + autoOpen: true, + open: () => set({ isOpen: true, suppressed: false }), + close: () => set({ isOpen: false, suppressed: true }), + setAutoOpen: (autoOpen) => set({ autoOpen }), + }), + { + name: "unsloth_api_monitor_overlay", + version: 1, + partialize: (state) => ({ autoOpen: state.autoOpen }), + // Explicit merge so an older stored payload cannot resurrect `isOpen`. + merge: (persisted, current) => ({ + ...current, + autoOpen: + typeof (persisted as { autoOpen?: unknown } | null)?.autoOpen === + "boolean" + ? (persisted as { autoOpen: boolean }).autoOpen + : current.autoOpen, + }), + }, + ), +); diff --git a/studio/frontend/src/features/api-monitor/use-api-monitor.ts b/studio/frontend/src/features/api-monitor/use-api-monitor.ts new file mode 100644 index 0000000000..20c3915ee3 --- /dev/null +++ b/studio/frontend/src/features/api-monitor/use-api-monitor.ts @@ -0,0 +1,297 @@ +// SPDX-License-Identifier: AGPL-3.0-only +// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 + +import { + clearApiMonitor, + getApiMonitor, + getApiMonitorEntry, +} from "@/features/chat/api/chat-api"; +import type { + ApiMonitorEntry, + ApiMonitorResponse, +} from "@/features/chat/types/api"; +import { useCallback, useEffect, useMemo, useRef, useState } from "react"; + +/** Poll cadence while the monitor is live. Matches the settings console it replaces. */ +const POLL_INTERVAL_MS = 1500; + +export type MonitorStatusFilter = + | "all" + | "running" + | "completed" + | "error" + | "cancelled"; + +export interface MonitorStats { + active: number; + total: number; + completed: number; + errors: number; + cancelled: number; + /** Mean duration over finished requests, or null when none have finished. */ + avgDurationMs: number | null; + /** Slowest finished request, for spotting a single pathological call. */ + maxDurationMs: number | null; + totalTokens: number; + /** Share of finished requests that failed, 0-1. Null when nothing finished. */ + errorRate: number | null; + /** Mean completion tokens per second over requests that reported both. */ + tokensPerSecond: number | null; +} + +function isTerminal(entry: ApiMonitorEntry): boolean { + return entry.status !== "running"; +} + +function completionTokens(entry: ApiMonitorEntry): number | null { + if (entry.completion_tokens != null) { + return entry.completion_tokens; + } + // Some providers only report a total; subtracting the prompt is the best + // available estimate of what was actually generated. + if (entry.total_tokens != null && entry.prompt_tokens != null) { + return Math.max(0, entry.total_tokens - entry.prompt_tokens); + } + return null; +} + +function entryTokens(entry: ApiMonitorEntry): number { + if (entry.total_tokens != null) { + return entry.total_tokens; + } + return (entry.prompt_tokens ?? 0) + (entry.completion_tokens ?? 0); +} + +export function computeStats(entries: ApiMonitorEntry[]): MonitorStats { + let active = 0; + let completed = 0; + let errors = 0; + let cancelled = 0; + let totalTokens = 0; + let durationSum = 0; + let durationCount = 0; + let maxDurationMs: number | null = null; + // Throughput is aggregated as total tokens over total time, not as the mean of + // each request's rate. Averaging rates lets one tiny fast request outweigh a + // long slow one, which is the opposite of what someone debugging wants to see. + let generatedTokens = 0; + let generatedDurationMs = 0; + + for (const entry of entries) { + totalTokens += entryTokens(entry); + if (entry.status === "running") { + active += 1; + } else if (entry.status === "error") { + errors += 1; + } else if (entry.status === "cancelled") { + cancelled += 1; + } else { + completed += 1; + } + const duration = entry.duration_ms; + if (duration != null && isTerminal(entry)) { + durationSum += duration; + durationCount += 1; + maxDurationMs = + maxDurationMs == null ? duration : Math.max(maxDurationMs, duration); + const generated = completionTokens(entry); + // Sub-millisecond durations would divide into a meaningless rate. + if (generated != null && generated > 0 && duration > 0) { + generatedTokens += generated; + generatedDurationMs += duration; + } + } + } + + const finished = completed + errors + cancelled; + return { + active, + total: entries.length, + completed, + errors, + cancelled, + avgDurationMs: durationCount > 0 ? durationSum / durationCount : null, + maxDurationMs, + totalTokens, + errorRate: finished > 0 ? errors / finished : null, + tokensPerSecond: + generatedDurationMs > 0 + ? generatedTokens / (generatedDurationMs / 1000) + : null, + }; +} + +export function filterEntries( + entries: ApiMonitorEntry[], + status: MonitorStatusFilter, + query: string, +): ApiMonitorEntry[] { + const needle = query.trim().toLowerCase(); + return entries.filter((entry) => { + if (status !== "all" && entry.status !== status) { + return false; + } + if (!needle) { + return true; + } + // Search the fields a debugging session actually keys off: which model, + // which endpoint, and the previews/error text visible in the row. + return ( + entry.model.toLowerCase().includes(needle) || + entry.endpoint.toLowerCase().includes(needle) || + entry.prompt_preview.toLowerCase().includes(needle) || + entry.reply_preview.toLowerCase().includes(needle) || + (entry.error ?? "").toLowerCase().includes(needle) + ); + }); +} + +interface UseApiMonitorResult { + data: ApiMonitorResponse | null; + entries: ApiMonitorEntry[]; + stats: MonitorStats; + error: string | null; + /** True until the first response lands, so the page can show skeletons once. */ + loading: boolean; + refreshing: boolean; + paused: boolean; + setPaused: (paused: boolean) => void; + refresh: () => void; + clear: () => Promise; + /** Full prompt/reply for entries the user expanded, keyed by entry id. */ + details: Record; + loadingDetails: ReadonlySet; + requestDetail: (id: string) => void; +} + +/** + * Live view of the server's OpenAI-compatible API traffic. + * + * Polls rather than streams because the backing monitor is an in-memory ring + * buffer with no change feed. Polling is self-rescheduling (never overlapping), + * and pausing stops it entirely so a user reading a stalled request's payload + * isn't fighting a list that reorders under them. + * + * `intervalMs` lets a caller trade freshness for cost: the full page wants the + * default live cadence, while the floating overlay slows right down when it is + * closed and only watching for the traffic that should pop it open. + */ +export function useApiMonitor({ + intervalMs = POLL_INTERVAL_MS, +}: { intervalMs?: number } = {}): UseApiMonitorResult { + const [data, setData] = useState(null); + const [error, setError] = useState(null); + const [loading, setLoading] = useState(true); + const [refreshing, setRefreshing] = useState(false); + const [paused, setPaused] = useState(false); + const [details, setDetails] = useState>({}); + const [loadingDetails, setLoadingDetails] = useState>( + () => new Set(), + ); + // Mirrors `loadingDetails` outside React state so the fetch guard sees writes + // from the same tick (state updates are async and would let duplicates through). + const inFlightDetails = useRef>(new Set()); + + const load = useCallback(async (): Promise => { + setRefreshing(true); + try { + const next = await getApiMonitor(); + setData(next); + setError(null); + } catch (err: unknown) { + setError(err instanceof Error ? err.message : "Monitor unavailable"); + } finally { + setRefreshing(false); + setLoading(false); + } + }, []); + + useEffect(() => { + if (paused) { + return; + } + let cancelled = false; + let timer: number | undefined; + + function poll(): void { + getApiMonitor() + .then((next) => { + if (cancelled) return; + setData(next); + setError(null); + }) + .catch((err: unknown) => { + if (cancelled) return; + setError(err instanceof Error ? err.message : "Monitor unavailable"); + }) + .finally(() => { + if (cancelled) return; + setLoading(false); + timer = window.setTimeout(poll, intervalMs); + }); + } + + poll(); + return () => { + cancelled = true; + if (timer !== undefined) { + window.clearTimeout(timer); + } + }; + }, [paused, intervalMs]); + + const requestDetail = useCallback((id: string): void => { + if (inFlightDetails.current.has(id)) { + return; + } + inFlightDetails.current.add(id); + setLoadingDetails((prev) => new Set(prev).add(id)); + getApiMonitorEntry(id) + .then((entry) => { + setDetails((prev) => ({ ...prev, [id]: entry })); + }) + .catch(() => { + // The entry aged out of the ring buffer; drop any stale copy so the UI + // falls back to the row previews instead of showing a frozen payload. + setDetails((prev) => { + if (!(id in prev)) return prev; + const next = { ...prev }; + delete next[id]; + return next; + }); + }) + .finally(() => { + inFlightDetails.current.delete(id); + setLoadingDetails((prev) => { + const next = new Set(prev); + next.delete(id); + return next; + }); + }); + }, []); + + const clear = useCallback(async (): Promise => { + await clearApiMonitor(); + setDetails({}); + await load(); + }, [load]); + + const entries = useMemo(() => data?.entries ?? [], [data]); + const stats = useMemo(() => computeStats(entries), [entries]); + + return { + data, + entries, + stats, + error, + loading, + refreshing, + paused, + setPaused, + refresh: () => void load(), + clear, + details, + loadingDetails, + requestDetail, + }; +} diff --git a/studio/frontend/src/features/chat/api/chat-api.ts b/studio/frontend/src/features/chat/api/chat-api.ts index 4d123e98ab..822de4fa4f 100644 --- a/studio/frontend/src/features/chat/api/chat-api.ts +++ b/studio/frontend/src/features/chat/api/chat-api.ts @@ -129,6 +129,13 @@ export async function getApiMonitorEntry(id: string): Promise { return parseJsonOrThrow(response); } +export async function clearApiMonitor(): Promise { + const response = await authFetch("/api/inference/monitor", { + method: "DELETE", + }); + await parseJsonOrThrow<{ cleared: boolean }>(response); +} + export async function loadModel( payload: LoadModelRequest, ): Promise { diff --git a/studio/frontend/src/features/hub/catalog/download-card.tsx b/studio/frontend/src/features/hub/catalog/download-card.tsx index 8016a9406d..1618b9cdf1 100644 --- a/studio/frontend/src/features/hub/catalog/download-card.tsx +++ b/studio/frontend/src/features/hub/catalog/download-card.tsx @@ -23,6 +23,7 @@ import { ArrowReloadHorizontalIcon, Delete02Icon, Download01Icon, + Settings02Icon, } from "@hugeicons/core-free-icons"; import { HugeiconsIcon } from "@hugeicons/react"; import { @@ -82,6 +83,40 @@ export function CardDivider() { ); } +/** Gear that opens a downloaded model's full settings page. */ +export function CardSettingsButton({ + label, + onClick, +}: { + label: string; + onClick: () => void; +}) { + return ( + + + + + + {label} + + + ); +} + export function CardDeleteButton({ label, onClick, diff --git a/studio/frontend/src/features/hub/catalog/hub-model-settings-view.tsx b/studio/frontend/src/features/hub/catalog/hub-model-settings-view.tsx new file mode 100644 index 0000000000..a293a0071a --- /dev/null +++ b/studio/frontend/src/features/hub/catalog/hub-model-settings-view.tsx @@ -0,0 +1,138 @@ +// SPDX-License-Identifier: AGPL-3.0-only +// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 + +// Full-page settings for one model, opened from the Hub. +// +// The same controls exist in the chat picker's popover, but a popover is a poor +// place to work through every knob a model has. This gives them a page, and +// states plainly that whatever is saved here is what an API load will use -- +// the settings are mirrored server-side by ModelConfigPage's save. + +import { ModelConfigPage, type ModelPickTarget } from "@/features/model-picker"; +import type { PerModelConfig } from "@/features/model-picker"; +import { ArrowLeft01Icon, Globe02Icon } from "@hugeicons/core-free-icons"; +import { HugeiconsIcon } from "@hugeicons/react"; +import { cn } from "@/lib/utils"; +import { useEffect, useRef, useState } from "react"; + +export function HubModelSettingsView({ + target, + loadedConfig = null, + loadedContextLength = null, + onBack, + onRun, + compact = false, +}: { + target: ModelPickTarget; + /** Non-null when this model is the loaded one, so the page can show live values. */ + loadedConfig?: PerModelConfig | null; + loadedContextLength?: number | null; + onBack: () => void; + /** Apply + load with these settings. */ + onRun: (config: PerModelConfig) => void; + compact?: boolean; +}) { + const scrollRef = useRef(null); + const [scrolled, setScrolled] = useState(false); + // Mirrors HubDetailView so this view sits at the same measure as the rest of + // the Hub rather than introducing a third column width. + const measure = compact + ? "mx-auto w-full max-w-[860px] px-5 sm:px-5" + : "mx-auto w-full max-w-[1100px] px-5 sm:px-8"; + + useEffect(() => { + const el = scrollRef.current; + if (!el) return; + const onScroll = () => { + const next = el.scrollTop > 0; + setScrolled((current) => (current === next ? current : next)); + }; + onScroll(); + el.addEventListener("scroll", onScroll, { passive: true }); + return () => el.removeEventListener("scroll", onScroll); + }, []); + + return ( +
+ + ); +} diff --git a/studio/frontend/src/features/hub/catalog/local-on-device-card.tsx b/studio/frontend/src/features/hub/catalog/local-on-device-card.tsx index 9b508a5413..b432b11f54 100644 --- a/studio/frontend/src/features/hub/catalog/local-on-device-card.tsx +++ b/studio/frontend/src/features/hub/catalog/local-on-device-card.tsx @@ -53,6 +53,7 @@ import { useHfTokenStore } from "../stores/hf-token-store"; import { DotTag } from "./dot-tag"; import { CardDeleteButton, + CardSettingsButton, CardUpdateButton, DeleteConfirmDialog, UpdateConfirmDialog, @@ -96,6 +97,8 @@ interface LocalOnDeviceCardProps { onEject?: () => void; onTrain?: () => void; onChange?: () => void; + /** Open this model's full settings page for the shown quant. */ + onOpenSettings?: (ggufVariant: string | null) => void; } function formatAdapterLabel( @@ -214,6 +217,7 @@ export function LocalOnDeviceCard({ onEject, onTrain, onChange, + onOpenSettings, }: LocalOnDeviceCardProps) { const [deleteOpen, setDeleteOpen] = useState(false); const [updateOpen, setUpdateOpen] = useState(false); @@ -549,6 +553,14 @@ export function LocalOnDeviceCard({ )}
+ {onOpenSettings && ( + onOpenSettings(selectedQuant ?? null)} + /> + )} {canUpdate && ( void; onInventoryChange?: () => void; onSearchHub?: (query: string) => void; + /** Open this model's full settings page, with the quant the card resolved. */ + onOpenSettings?: (ggufVariant: string | null) => void; }; export const ModelInspector = memo(function ModelInspector({ @@ -438,6 +440,7 @@ export const ModelInspector = memo(function ModelInspector({ onTrain, onInventoryChange, onSearchHub, + onOpenSettings, } = actions; const deviceType = usePlatformStore((s) => s.deviceType); const chatOnly = usePlatformStore((s) => s.isChatOnly()); @@ -705,6 +708,7 @@ export const ModelInspector = memo(function ModelInspector({ model.isDownloaded && canTrainModel ? onTrain : undefined } onChange={onInventoryChange} + onOpenSettings={onOpenSettings} /> ) : ( void; + /** Open a downloaded model's full settings page. */ + onOpenModelSettings?: (row: CachedInventoryRow | LocalInventoryRow) => void; }) { // Pinned repos surface first regardless of the active sort; the chosen sort // still orders rows within the pinned and unpinned groups. @@ -383,6 +386,7 @@ export function DownloadedList({ compact={compact} onSelect={onSelect} onChange={onInventoryChange} + onOpenSettings={onOpenModelSettings} /> ); diff --git a/studio/frontend/src/features/hub/catalog/models-catalog-rows.tsx b/studio/frontend/src/features/hub/catalog/models-catalog-rows.tsx index 5236c0be74..1dbf78cff9 100644 --- a/studio/frontend/src/features/hub/catalog/models-catalog-rows.tsx +++ b/studio/frontend/src/features/hub/catalog/models-catalog-rows.tsx @@ -573,6 +573,7 @@ export const InventoryRow = memo(function InventoryRow({ compact = false, onSelect, onChange, + onOpenSettings, }: { row: CachedInventoryRow | LocalInventoryRow; selected: boolean; @@ -585,6 +586,8 @@ export const InventoryRow = memo(function InventoryRow({ compact?: boolean; onSelect: (id: string) => void; onChange?: () => void; + /** Open this model's full settings page. Omitted for datasets. */ + onOpenSettings?: (row: CachedInventoryRow | LocalInventoryRow) => void; }) { const rowModelId = row.kind === "cache" @@ -732,30 +735,41 @@ export const InventoryRow = memo(function InventoryRow({ const rowPinned = cacheDeletableRepoId != null && pinnedKeys.includes(pinKey(cacheDeletableRepoId)); + // Settings is available for any downloaded model, not just deletable ones: a + // model scanned from a local folder is just as configurable as a cached repo. + // The menu therefore renders whenever either action applies, and each item + // gates itself. `deletableRepoId` (rather than the boolean) keeps the non-null + // narrowing the delete closures below rely on. + const settingsAction = + !isDataset && onOpenSettings ? { onOpen: () => onOpenSettings(row) } : undefined; + const deletableRepoId = canDelete ? cacheDeletableRepoId : null; const deleteAction = - canDelete && cacheDeletableRepoId ? ( + deletableRepoId || settingsAction ? ( togglePinned(cacheDeletableRepoId), + onToggle: () => togglePinned(deletableRepoId), } } - cachePath={isDataset ? undefined : { repoId: cacheDeletableRepoId }} - del={{ + cachePath={ + isDataset || !deletableRepoId ? undefined : { repoId: deletableRepoId } + } + del={deletableRepoId ? { title: isDataset ? "Delete cached dataset?" : "Delete cached model?", description: ( <> This will remove{" "} - {cacheDeletableRepoId} + {deletableRepoId} {" "} {isDataset ? "and its downloaded files" @@ -766,17 +780,17 @@ export const InventoryRow = memo(function InventoryRow({ disk. You can re-download it later. ), - successMessage: `Deleted ${cacheDeletableRepoId}`, + successMessage: `Deleted ${deletableRepoId}`, onConfirm: async () => { // Delete only the copy this row shows: cache rows carry the owning // cache path, so pass it through and leave other caches untouched. const rowCachePath = row.kind === "cache" ? (row.cachePath ?? undefined) : undefined; if (isDataset) { - await deleteCachedDataset(cacheDeletableRepoId, rowCachePath); + await deleteCachedDataset(deletableRepoId, rowCachePath); } else { await deleteCachedModel( - cacheDeletableRepoId, + deletableRepoId, undefined, undefined, rowCachePath, @@ -787,11 +801,11 @@ export const InventoryRow = memo(function InventoryRow({ usePinnedModelsStore.getState(); for (const key of pinned) { if ( - key === pinKey(cacheDeletableRepoId) || - key.startsWith(`${cacheDeletableRepoId}::`) + key === pinKey(deletableRepoId) || + key.startsWith(`${deletableRepoId}::`) ) { toggle( - cacheDeletableRepoId, + deletableRepoId, key.includes("::") ? key.slice(key.indexOf("::") + 2) : undefined, @@ -801,7 +815,7 @@ export const InventoryRow = memo(function InventoryRow({ } }, onDeleted: onChange, - }} + } : undefined} /> ) : null; diff --git a/studio/frontend/src/features/hub/catalog/models-catalog.tsx b/studio/frontend/src/features/hub/catalog/models-catalog.tsx index 02b9b62fce..288c525af1 100644 --- a/studio/frontend/src/features/hub/catalog/models-catalog.tsx +++ b/studio/frontend/src/features/hub/catalog/models-catalog.tsx @@ -16,6 +16,7 @@ import { import type { CachedInventoryRow, DiscoverRow, + InventoryRow, LocalInventoryRow, ModelsTab, } from "../types"; @@ -70,6 +71,8 @@ export interface ModelsCatalogHandlers { onRetry: () => void; onInventoryChange?: () => void; onSwitchDevice?: () => void; + /** Open a downloaded model's full settings page. */ + onOpenModelSettings?: (row: InventoryRow) => void; } function assignRef(ref: RefObject, value: T | null) { @@ -128,6 +131,7 @@ export const ModelsCatalog = memo(function ModelsCatalog({ onRetry, onInventoryChange, onSwitchDevice, + onOpenModelSettings, } = handlers; const [scrolled, setScrolled] = useState(false); const [streamingActive, setStreamingActive] = useState(false); @@ -483,6 +487,7 @@ export const ModelsCatalog = memo(function ModelsCatalog({ columns={discoverView === "two" ? 2 : 1} sort={inventorySort} onInventoryChange={onInventoryChange} + onOpenModelSettings={onOpenModelSettings} />
) : ( diff --git a/studio/frontend/src/features/hub/hub-page.tsx b/studio/frontend/src/features/hub/hub-page.tsx index 9fc452b4ed..e853ea0e5c 100644 --- a/studio/frontend/src/features/hub/hub-page.tsx +++ b/studio/frontend/src/features/hub/hub-page.tsx @@ -9,6 +9,7 @@ import { import { getInferenceStatus, isExternalModelId, + listGgufVariants, useChatModelRuntime, useChatRuntimeStore, } from "@/features/chat"; @@ -24,9 +25,13 @@ import { ggufVariantsMatch, modelIdsMatch } from "./lib/model-identity"; import { hfApiToken, useHfTokenStore } from "./stores/hf-token-store"; import { applyModelLoadConfigToRuntime, + applyPerModelConfigToRuntime, currentRuntimePerModelConfig, hfModelFitsDevice, resolveInitialConfig, + useActiveModelConfig, + type ModelPickTarget, + type PerModelConfig, } from "@/features/model-picker"; import { useDebouncedValue } from "@/hooks/use-debounced-value"; import { useGpuInfo } from "@/hooks/use-gpu-info"; @@ -42,6 +47,7 @@ import { } from "react"; import { ExternalLinkConfirmDialog } from "./catalog/external-link-confirm-dialog"; import { HubDetailView } from "./catalog/hub-detail-view"; +import { HubModelSettingsView } from "./catalog/hub-model-settings-view"; import { HubFeed } from "./catalog/hub-feed"; import { HubTopBar } from "./catalog/hub-top-bar"; import { @@ -345,6 +351,12 @@ export function ModelsPage() { const activeCheckpoint = checkpoint && !isExternalModelId(checkpoint) ? checkpoint : null; const activeGgufVariant = useChatRuntimeStore((s) => s.activeGgufVariant); + const activeGgufContextLength = useChatRuntimeStore( + (s) => s.ggufContextLength, + ); + // Live settings of the loaded model, so opening its settings page shows what + // it is actually running with rather than the last saved draft. + const { config: activeModelConfig } = useActiveModelConfig(); // Shared with the chat model selector: list only models sized for this device. const fitOnDeviceOnly = useChatRuntimeStore((s) => s.fitOnDeviceOnly); const setFitOnDeviceOnly = useChatRuntimeStore((s) => s.setFitOnDeviceOnly); @@ -1213,6 +1225,100 @@ export function ModelsPage() { runSelectedModel(opts, selectedModel?.isDownloaded ?? true), [runSelectedModel, selectedModel], ); + + // Full-page per-model settings, opened from a downloaded row's menu. Local + // state rather than a URL param: the page is a transient editor over the + // catalog, and a deep link to it would need the row's identity re-resolved + // against an inventory that may not have loaded yet. + const [settingsTarget, setSettingsTarget] = useState( + null, + ); + const openModelSettings = useCallback( + async (row: CachedInventoryRow | LocalInventoryRow) => { + // loadId is what the loader accepts; repoId is only a display/API alias. + const id = row.loadId; + // Cached repo rows never carry a quant: the inventory emits one row per + // repo with format_variant null (see cache_inventory.py). Opening settings + // with a null variant would key the saved config to `repo::` while the + // loader reads `repo::Q4_K_M`, so the settings would silently never apply + // and the server mirror would be keyed wrong too. Resolve the quant the + // same way the on-device card does before opening. + let ggufVariant = row.formatVariant?.trim() || null; + if (!ggufVariant && row.isGguf && row.capabilities.requiresVariant) { + const repoId = row.kind === "cache" ? row.repoId : (row.repoId ?? null); + if (repoId) { + try { + const res = await listGgufVariants(repoId, hfApiToken(hfToken), { + preferLocalCache: true, + localPath: row.kind === "local" ? row.path : (row.cachePath ?? null), + }); + const downloaded = res.variants.filter((v) => v.downloaded); + ggufVariant = + // Prefer the loaded quant, then the repo default, then whatever is + // on disk, mirroring LocalOnDeviceCard's selectedQuant. + downloaded.find((v) => + ggufVariantsMatch(v.quant, activeGgufVariant), + )?.quant ?? + downloaded.find((v) => + ggufVariantsMatch(v.quant, res.default_variant), + )?.quant ?? + downloaded[0]?.quant ?? + null; + } catch { + // Offline or an unreadable cache: fall through with no variant. The + // settings page still works, it just cannot pin a specific quant. + ggufVariant = null; + } + } + } + const leaf = id.split(/[\\/]/).filter(Boolean).pop() ?? id; + setSettingsTarget({ + id, + displayName: ggufVariant ? `${leaf} · ${ggufVariant}` : leaf, + ggufVariant, + isGguf: row.isGguf, + meta: { + source: "local", + isLora: row.modelFormat === "adapter", + ggufVariant: ggufVariant ?? undefined, + isGguf: row.isGguf, + // Partial downloads still open settings, but must not claim to be + // complete or the loader skips its download-progress reporting. + isDownloaded: !row.partial, + // Not carried on inventory rows; ModelConfigPage reads the GGUF header + // itself to size the context slider. + contextLength: null, + }, + }); + }, + [activeGgufVariant, hfToken], + ); + // Applying from the settings page loads the model with exactly those settings. + // ModelConfigPage has already persisted them (locally and, when "remember" is + // on, to the server), so an API request for this model gets the same load. + const runSettingsTarget = useCallback( + (config: PerModelConfig) => { + const target = settingsTarget; + if (!target) return; + const previousConfig = currentRuntimePerModelConfig({ + includeMaxSeqLength: true, + }); + applyPerModelConfigToRuntime(config); + setSettingsTarget(null); + void selectModel({ + id: target.id, + source: "local", + ggufVariant: target.ggufVariant ?? undefined, + isGguf: target.isGguf, + isDownloaded: true, + isLora: target.meta.isLora, + keepSpeculative: true, + forceReload: true, + previousConfig, + }).catch(() => undefined); + }, + [selectModel, settingsTarget], + ); const handleLoadLocal = useCallback( (opts: ModelLoadOptions = {}) => runSelectedModel(opts, true), [runSelectedModel], @@ -1220,6 +1326,30 @@ export function ModelsPage() { const handleTrain = useCallback(() => { // Hub → train integration ships in a later PR. }, []); + // Settings opened from the detail view's on-device card. The card resolves + // which quant it is showing, so it passes that in rather than re-deriving it. + const openSelectedModelSettings = useCallback( + (ggufVariant: string | null) => { + if (!selectedModel) return; + const id = selectedModel.resource.runId; + const leaf = id.split(/[\\/]/).filter(Boolean).pop() ?? id; + setSettingsTarget({ + id, + displayName: ggufVariant ? `${leaf} · ${ggufVariant}` : leaf, + ggufVariant, + isGguf: selectedModel.isGguf, + meta: { + source: "local", + isLora: selectedModel.modelFormat === "adapter", + ggufVariant: ggufVariant ?? undefined, + isGguf: selectedModel.isGguf, + isDownloaded: selectedModel.isDownloaded, + contextLength: null, + }, + }); + }, + [selectedModel], + ); const handleSearchHub = useCallback( (next: string) => { const trimmed = next.trim(); @@ -1280,6 +1410,7 @@ export function ModelsPage() { onTrain: handleTrain, onInventoryChange: refreshInventory, onSearchHub: handleSearchHub, + onOpenSettings: openSelectedModelSettings, }), [ handleLoad, @@ -1289,6 +1420,7 @@ export function ModelsPage() { handleTrain, handleSearchHub, refreshInventory, + openSelectedModelSettings, ], ); @@ -1370,6 +1502,7 @@ export function ModelsPage() { onRetry: handleRetrySearch, onInventoryChange: refreshInventory, onSwitchDevice: handleSwitchDevice, + onOpenModelSettings: openModelSettings, }), [ handleSelect, @@ -1378,6 +1511,7 @@ export function ModelsPage() { handleRetrySearch, refreshInventory, handleSwitchDevice, + openModelSettings, ], ); @@ -1520,6 +1654,10 @@ export function ModelsPage() { const detailOpen = urlModel !== null; const splitMode = allModelsView === "split"; + // The catalog is unreachable when an opaque overlay sits on top of it: the + // detail view (full-page layout only, since split renders it alongside) or the + // settings page (always full-bleed). + const catalogCovered = (detailOpen && !splitMode) || settingsTarget !== null; return (
@@ -1575,9 +1713,15 @@ export function ModelsPage() { splitMode ? "flex-1 lg:w-[460px] lg:max-w-[44%] lg:flex-none lg:shrink-0 lg:border-r lg:border-border/60" : "flex-1", - detailOpen && !splitMode && "pointer-events-none", + // The settings page is a full-bleed opaque overlay in every layout, + // including split, so it always takes the catalog out of the tab + // order. Without this, tabbing out of the settings form walks into + // the virtualized rows hidden behind it and screen readers announce + // the whole model list underneath. + catalogCovered && "pointer-events-none", )} - aria-hidden={(detailOpen && !splitMode) || undefined} + aria-hidden={catalogCovered || undefined} + inert={catalogCovered || undefined} > ) )} + + {/* Sits above the detail overlay (z-30): opening settings from a row + while a model preview is open should show the settings, not stack + behind it. */} + {settingsTarget && ( +
+ setSettingsTarget(null)} + onRun={runSettingsTarget} + compact={splitMode} + /> +
+ )}
; + +/** + * The key one model's config is stored under. + * + * Uses the `repo:VARIANT` form an OpenAI request names a quant by, so two quants + * of the same repo keep separate configs and the backend can match the requested + * model name directly. Falls back to the bare id when there is no variant. + */ +export function modelOverrideKey( + modelId: string, + ggufVariant?: string | null, +): string { + return ggufVariant ? `${modelId}:${ggufVariant}` : modelId; +} + +export async function fetchModelOverrides(): Promise { + const res = await authFetch(OVERRIDES_URL); + if (!res.ok) { + throw new Error( + await readFastApiError(res, "Failed to load saved model settings"), + ); + } + const body = (await res.json()) as { overrides?: ApiModelOverrides }; + return body.overrides ?? {}; +} + +/** + * Translate the UI's per-model config into the backend's schema. + * + * Only fields the user actually set are sent: the backend reads an absent field + * as "use the app default", so sending nulls would pin defaults and stop the + * model following later changes to the global preferences. `null` config means + * "no saved settings", which clears the entry. + */ +function toApiOverride(config: PerModelConfig | null): ApiModelOverride { + if (!config) { + return {}; + } + const payload: ApiModelOverride = {}; + if (config.maxSeqLength && config.maxSeqLength > 0) { + payload.max_seq_length = config.maxSeqLength; + } + if (config.customContextLength && config.customContextLength > 0) { + payload.custom_context_length = config.customContextLength; + } + if (config.kvCacheDtype) { + payload.kv_cache_dtype = config.kvCacheDtype; + } + if (config.speculativeType) { + payload.speculative_type = config.speculativeType; + } + if (config.specDraftNMax && config.specDraftNMax > 0) { + payload.spec_draft_n_max = config.specDraftNMax; + } + if (config.tensorParallel) { + payload.tensor_parallel = true; + } + if (config.chatTemplateOverride?.trim()) { + payload.chat_template_override = config.chatTemplateOverride; + } + // Only "manual" is a real override; "auto" is the follow-the-global default. + if (config.gpuMemoryMode === "manual") { + payload.gpu_memory_mode = "manual"; + } + // gpuLayers < 0 is Auto, which is also the default. + if (typeof config.gpuLayers === "number" && config.gpuLayers >= 0) { + payload.gpu_layers = config.gpuLayers; + } + if (typeof config.nCpuMoe === "number" && config.nCpuMoe > 0) { + payload.n_cpu_moe = config.nCpuMoe; + } + if (config.selectedGpuIds && config.selectedGpuIds.length > 0) { + payload.gpu_ids = config.selectedGpuIds; + } + return payload; +} + +export async function putModelOverride( + modelId: string, + ggufVariant: string | null | undefined, + config: PerModelConfig | null, +): Promise { + const res = await authFetch(OVERRIDES_URL, { + method: "PUT", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + // biome-ignore lint/style/useNamingConvention: API schema + model_id: modelOverrideKey(modelId, ggufVariant), + // Launch flags have no UI control, so the backend preserves them when the + // field is omitted. Forgetting a model means forgetting all of it, so that + // path sends an explicit empty list to clear them. + // biome-ignore lint/style/useNamingConvention: API schema + ...(config === null ? { llama_extra_args: [] } : {}), + ...toApiOverride(config), + }), + }); + if (!res.ok) { + throw new Error( + await readFastApiError(res, "Failed to save model settings for the API"), + ); + } +} + +/** + * Mirror a per-model config save to the backend without blocking the UI. + * + * Deliberately best-effort: the localStorage write is the source of truth for + * this browser and has already happened by the time this runs, so a failed sync + * must not fail the save or interrupt a model load. It is logged rather than + * toasted -- the only consequence is that an API-triggered load of this model + * falls back to app defaults until the next successful save. + */ +export function syncModelOverride( + modelId: string, + ggufVariant: string | null | undefined, + config: PerModelConfig | null, +): void { + void putModelOverride(modelId, ggufVariant, config).catch( + (error: unknown) => { + console.warn( + "Failed to mirror model settings to the server; an API load of this model will use defaults.", + error, + ); + }, + ); +} diff --git a/studio/frontend/src/features/model-picker/components/model-config-page.tsx b/studio/frontend/src/features/model-picker/components/model-config-page.tsx index 90202a2bcf..edf0cb96ea 100644 --- a/studio/frontend/src/features/model-picker/components/model-config-page.tsx +++ b/studio/frontend/src/features/model-picker/components/model-config-page.tsx @@ -55,6 +55,7 @@ import { resolveInitialConfig, savePerModelConfig, } from "../model-config/per-model-config"; +import { syncModelOverride } from "../api/model-overrides"; import { ChatTemplateEditorDialog } from "./chat-template-editor-dialog"; import type { ModelPickTarget } from "./model-selector/types"; import { @@ -578,6 +579,12 @@ interface ModelConfigPageProps { loadedContextLength?: number | null; initialConfig?: PerModelConfig | null; variant?: "page" | "sidebar"; + /** + * Page variant only: render the built-in "Run settings" title block. A host + * that already shows the model name as its own page heading (the Hub's + * settings page) turns this off so the name is not printed twice. + */ + showHeader?: boolean; } export function ModelConfigPage({ @@ -588,6 +595,7 @@ export function ModelConfigPage({ loadedContextLength = null, initialConfig = null, variant = "page", + showHeader = true, }: ModelConfigPageProps) { const rememberId = useId(); const isActiveModel = loadedConfig != null; @@ -870,6 +878,21 @@ export function ModelConfigPage({ } else { saveFailed = !deletePerModelConfig(target.id, target.ggufVariant); } + // Mirror to the server so an OpenAI-compatible API request that loads this + // model gets these exact settings, not app defaults. Best-effort and + // non-blocking: the localStorage write above already governs this browser. + // Forgetting clears the server entry too, so the two never disagree. + // + // Skipped when the local write failed (quota, a future-schema entry): the + // browser and the server would otherwise permanently disagree about this + // model, with no way for the user to tell which one the next load used. + if (!saveFailed) { + syncModelOverride( + target.id, + target.ggufVariant, + remember ? effectiveRuntimeConfig : null, + ); + } if (effectivePersistenceOnly) { if (saveFailed) { toast.error("Couldn't save settings for this model."); @@ -898,7 +921,7 @@ export function ModelConfigPage({ return (
- {variant === "page" && ( + {variant === "page" && showHeader && (
{onBack && ( - - {expanded ? ( -
-
-
-
- Prompt - {entry.prompt_truncated && !detail ? Preview : null} -
- 
-                {loading && !detail ? "Loading..." : prompt || "No prompt text"}
-
-
-
-
- Reply - {entry.reply_truncated && !detail ? Preview : null} -
- 
-                {loading && !detail ? "Loading..." : reply}
-
-
-
- -
- {formatTokens(entry)} - {entry.context_length ? ( - <> / {entry.context_length.toLocaleString()} context - ) : null} - -
-
- ) : null} - - ); -} - -export function ApiMonitorConsole(): ReactElement { - const [data, setData] = useState(null); - const [error, setError] = useState(null); - const [refreshing, setRefreshing] = useState(false); - const [expandedIds, setExpandedIds] = useState>(() => new Set()); - const [details, setDetails] = useState>({}); - const [loadingDetails, setLoadingDetails] = useState>( - () => new Set(), - ); - const loadingDetailsRef = useRef>(new Set()); - const detailsRef = useRef>({}); - - const loadMonitor = useCallback(async (): Promise => { - setRefreshing(true); - try { - setData(await getApiMonitor()); - setError(null); - } catch (err: unknown) { - setError(err instanceof Error ? err.message : "Monitor unavailable"); - } finally { - setRefreshing(false); - } - }, []); - - useEffect(() => { - let cancelled = false; - let timer: number | undefined; - - function schedule(): void { - timer = window.setTimeout(poll, 1500); - } - - function poll(): void { - getApiMonitor() - .then((next) => { - if (cancelled) { - return; - } - setData(next); - setError(null); - }) - .catch((err: unknown) => { - if (cancelled) { - return; - } - setError(err instanceof Error ? err.message : "Monitor unavailable"); - }) - .finally(() => { - if (!cancelled) { - schedule(); - } - }); - } - - poll(); - return () => { - cancelled = true; - if (timer !== undefined) { - window.clearTimeout(timer); - } - }; - }, []); - - const statusLabel = data?.status ?? "idle"; - const hasActive = (data?.active_requests ?? 0) > 0; - const entries = useMemo(() => data?.entries ?? [], [data]); - const loadDetail = useCallback( - (id: string): void => { - if (loadingDetailsRef.current.has(id)) { - return; - } - loadingDetailsRef.current.add(id); - setLoadingDetails((prev) => new Set(prev).add(id)); - getApiMonitorEntry(id) - .then((entry) => { - setDetails((prev) => { - const next = { ...prev, [id]: entry }; - detailsRef.current = next; - return next; - }); - }) - .catch(() => { - setDetails((prev) => { - const next = { ...prev }; - delete next[id]; - detailsRef.current = next; - return next; - }); - }) - .finally(() => { - loadingDetailsRef.current.delete(id); - setLoadingDetails((prev) => { - const next = new Set(prev); - next.delete(id); - return next; - }); - }); - }, - [], - ); - - const toggleEntry = useCallback( - (entry: ApiMonitorEntry): void => { - setExpandedIds((prev) => { - const next = new Set(prev); - if (next.has(entry.id)) { - next.delete(entry.id); - } else { - next.add(entry.id); - loadDetail(entry.id); - } - return next; - }); - }, - [loadDetail], - ); - - useEffect(() => { - for (const entry of entries) { - if (!expandedIds.has(entry.id)) { - continue; - } - const cached = detailsRef.current[entry.id]; - if (!cached || cached.status !== entry.status || entry.status === "running") { - loadDetail(entry.id); - } - } - }, [entries, expandedIds, loadDetail]); - - return ( -
-
-
-
- - {hasActive ? ( - - ) : null} -
-
-

- API monitor -

-

- {data?.active_model ?? "No model loaded"} -

-
-
-
-
- {statusLabel} -
- -
-
- -
- - {(data?.active_requests ?? 0).toLocaleString()} active /{" "} - {entries.length.toLocaleString()} recent - - {data?.context_length ? ( - {data.context_length.toLocaleString()} context - ) : null} -
- -
- {error ? ( -
- {error} -
- ) : entries.length === 0 ? ( -
- No API traffic yet -
- ) : ( -
- {entries.map((entry) => ( - toggleEntry(entry)} - /> - ))} -
- )} -
-
- ); -} diff --git a/studio/frontend/src/features/settings/components/monitor-link.tsx b/studio/frontend/src/features/settings/components/monitor-link.tsx new file mode 100644 index 0000000000..a0547fdc79 --- /dev/null +++ b/studio/frontend/src/features/settings/components/monitor-link.tsx @@ -0,0 +1,97 @@ +// SPDX-License-Identifier: AGPL-3.0-only +// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 + +// The monitor moved out of this tab and onto its own page. It is normally +// reached from the floating panel; this card is the way in from Settings. + +import { Switch } from "@/components/ui/switch"; +import { useApiMonitorOverlayStore } from "@/features/api-monitor"; +import { getApiMonitor } from "@/features/chat/api/chat-api"; +import type { ApiMonitorResponse } from "@/features/chat/types/api"; +import { cn } from "@/lib/utils"; +import { ActivityIcon, ArrowRight02Icon } from "@hugeicons/core-free-icons"; +import { HugeiconsIcon } from "@hugeicons/react"; +import { useNavigate } from "@tanstack/react-router"; +import { type ReactElement, useEffect, useState } from "react"; +import { useSettingsDialogStore } from "../stores/settings-dialog-store"; + +export function MonitorLink(): ReactElement { + const navigate = useNavigate(); + const [data, setData] = useState(null); + const autoOpen = useApiMonitorOverlayStore((s) => s.autoOpen); + const setAutoOpen = useApiMonitorOverlayStore((s) => s.setAutoOpen); + + // One snapshot, not a poll: the live view is the monitor page. + useEffect(() => { + let cancelled = false; + void getApiMonitor() + .then((next) => { + if (!cancelled) setData(next); + }) + .catch(() => undefined); + return () => { + cancelled = true; + }; + }, []); + + const active = data?.active_requests ?? 0; + const recent = data?.entries.length ?? 0; + + return ( +
+ + + {/* Where the panel's own "stop opening this" gets turned back on. */} +
+ + + Show the floating monitor automatically + + + Opens a small panel when API traffic arrives. + + + +
+
+ ); +} diff --git a/studio/frontend/src/features/settings/tabs/api-keys-tab.tsx b/studio/frontend/src/features/settings/tabs/api-keys-tab.tsx index f1e503f7a3..39d934aacd 100644 --- a/studio/frontend/src/features/settings/tabs/api-keys-tab.tsx +++ b/studio/frontend/src/features/settings/tabs/api-keys-tab.tsx @@ -14,7 +14,7 @@ import { translate, useT } from "@/i18n"; import { AnimatePresence, motion, useReducedMotion } from "motion/react"; import { useCallback, useEffect, useState } from "react"; import { fetchApiKeys, revokeApiKey, type ApiKey } from "../api/api-keys"; -import { ApiMonitorConsole } from "../components/api-monitor-console"; +import { MonitorLink } from "../components/monitor-link"; import { ApiKeyRow } from "../components/api-key-row"; import { CreateKeyForm } from "../components/create-key-form"; import { ModelAutoSwitchSection } from "../components/model-auto-switch-section"; @@ -168,7 +168,7 @@ export function ApiKeysTab() { )} - + From b5cada102848356cee8a7ea3b36c8374a4c00671 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Sun, 26 Jul 2026 13:57:19 +0000 Subject: [PATCH 02/75] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- studio/backend/core/inference/api_monitor.py | 4 +--- studio/backend/routes/inference.py | 3 +-- studio/backend/routes/settings.py | 14 ++++---------- .../backend/tests/test_openai_auto_switch.py | 19 ++++--------------- .../utils/openai_auto_switch_settings.py | 4 +--- 5 files changed, 11 insertions(+), 33 deletions(-) diff --git a/studio/backend/core/inference/api_monitor.py b/studio/backend/core/inference/api_monitor.py index f9922a711a..670f370003 100644 --- a/studio/backend/core/inference/api_monitor.py +++ b/studio/backend/core/inference/api_monitor.py @@ -280,9 +280,7 @@ class ApiMonitor: if subject is None: self._entries.clear() return - self._entries = deque( - entry for entry in self._entries if entry.subject != subject - ) + self._entries = deque(entry for entry in self._entries if entry.subject != subject) def _find_locked(self, entry_id: str) -> Optional[ApiMonitorEntry]: for entry in self._entries: diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.py index d529a1cd85..4bb0edcceb 100644 --- a/studio/backend/routes/inference.py +++ b/studio/backend/routes/inference.py @@ -3742,8 +3742,7 @@ async def _maybe_auto_switch_model( override, # variant is set for every GGUF the resolver returns; the # reload-stash path carries the quant it froze. - is_gguf = bool(variant) - or target_id.lower().endswith(".gguf"), + is_gguf = bool(variant) or target_id.lower().endswith(".gguf"), ) ) # Reuse the load impl so its dedup, tensor fallback, and threading diff --git a/studio/backend/routes/settings.py b/studio/backend/routes/settings.py index 2323d33915..2dd0cabbe7 100644 --- a/studio/backend/routes/settings.py +++ b/studio/backend/routes/settings.py @@ -159,7 +159,6 @@ class ModelOverridePayload(BaseModel): n_cpu_moe: Optional[int] = Field(default = None, ge = 0, le = 1024) gpu_ids: Optional[list[int]] = None - @field_validator("chat_template_override") @classmethod def _limit_chat_template_bytes(cls, value: Optional[str]) -> Optional[str]: @@ -168,9 +167,7 @@ class ModelOverridePayload(BaseModel): if value is None: return None if len(value.encode("utf-8")) > MAX_CHAT_TEMPLATE_BYTES: - raise ValueError( - f"Chat template exceeds the {MAX_CHAT_TEMPLATE_BYTES}-byte limit." - ) + raise ValueError(f"Chat template exceeds the {MAX_CHAT_TEMPLATE_BYTES}-byte limit.") return value @@ -331,6 +328,7 @@ def update_openai_auto_switch_override( ) -> ModelOverridesResponse: from core.inference.llama_server_args import validate_extra_args from utils.openai_auto_switch_settings import get_model_override + try: # A payload carrying only model_id is the documented "remove", so it # wipes everything. Otherwise it is a real save, and omitted launch flags @@ -341,14 +339,10 @@ def update_openai_auto_switch_override( exclude = {"model_id", "llama_extra_args"}, exclude_none = True ) is_removal = not payload.tensor_parallel and not { - key: value - for key, value in saved_fields.items() - if key != "tensor_parallel" + key: value for key, value in saved_fields.items() if key != "tensor_parallel" } if requested_extra_args is None and not is_removal: - requested_extra_args = get_model_override(payload.model_id).get( - "llama_extra_args" - ) + requested_extra_args = get_model_override(payload.model_id).get("llama_extra_args") extra_args = validate_extra_args(requested_extra_args) set_model_override( payload.model_id, diff --git a/studio/backend/tests/test_openai_auto_switch.py b/studio/backend/tests/test_openai_auto_switch.py index 3899b1444a..d1173585b0 100644 --- a/studio/backend/tests/test_openai_auto_switch.py +++ b/studio/backend/tests/test_openai_auto_switch.py @@ -3843,11 +3843,7 @@ def test_normalize_model_override_drops_unusable_fields_and_keeps_the_rest(): "llama_extra_args": [], } ) - assert entry == { - "max_seq_length": 8192, - "speculative_type": "mtp", - "gpu_ids": [1, 0, 2], - } + assert entry == {"max_seq_length": 8192, "speculative_type": "mtp", "gpu_ids": [1, 0, 2]} def test_normalize_model_override_rejects_oversized_chat_template(): @@ -3862,15 +3858,11 @@ def test_normalize_model_override_rejects_oversized_chat_template(): def test_spec_draft_n_max_only_stored_for_mtp_modes(): - mtp = settings.normalize_model_override( - {"speculative_type": "mtp", "spec_draft_n_max": 4} - ) + mtp = settings.normalize_model_override({"speculative_type": "mtp", "spec_draft_n_max": 4}) assert mtp["spec_draft_n_max"] == 4 # A non-MTP mode ignores the draft count at load time, so storing it would # show the user an edit that never takes effect. - ngram = settings.normalize_model_override( - {"speculative_type": "ngram", "spec_draft_n_max": 4} - ) + ngram = settings.normalize_model_override({"speculative_type": "ngram", "spec_draft_n_max": 4}) assert "spec_draft_n_max" not in ngram @@ -3886,10 +3878,7 @@ def test_resolve_fit_max_seq_length_hands_sizing_to_fit_under_manual_auto_layers == 4096 ) # Pinning the layer count takes --fit back out of the picture. - assert ( - settings.resolve_fit_max_seq_length({**override, "gpu_layers": 20}, is_gguf = True) - == 8192 - ) + assert settings.resolve_fit_max_seq_length({**override, "gpu_layers": 20}, is_gguf = True) == 8192 # Not a GGUF, so none of this applies. assert settings.resolve_fit_max_seq_length(override, is_gguf = False) == 8192 diff --git a/studio/backend/utils/openai_auto_switch_settings.py b/studio/backend/utils/openai_auto_switch_settings.py index 413d75ad1e..7f6b64eca4 100644 --- a/studio/backend/utils/openai_auto_switch_settings.py +++ b/studio/backend/utils/openai_auto_switch_settings.py @@ -306,9 +306,7 @@ def normalize_model_override(payload: dict[str, Any]) -> dict[str, Any]: # Only meaningful for the MTP modes; storing it otherwise would resurface # in the UI as an edit the loader silently ignores. if speculative_type in MTP_SPECULATIVE_TYPES: - spec_draft_n_max = _bounded_int( - payload.get("spec_draft_n_max"), minimum = 1, maximum = 16 - ) + spec_draft_n_max = _bounded_int(payload.get("spec_draft_n_max"), minimum = 1, maximum = 16) if spec_draft_n_max: entry["spec_draft_n_max"] = spec_draft_n_max From 79e8c7356e422eb022e1d15043050b2250358c1f Mon Sep 17 00:00:00 2001 From: Unsloth Date: Sun, 26 Jul 2026 07:35:11 -0700 Subject: [PATCH 03/75] studio: harden per-model settings against review findings and fuzzing Review findings (PR #7473): - Look up overrides under the concrete load path with its quant, not just the advertised repo id, so local folders and non-active HF caches are found. - Carry bare-repo launch flags into the first per-quant save. Auto-switch prefers the qualified entry, so without this the flags were silently dropped and no UI could show or restore them. The bare id is only derived when the suffix looks like a quant, so a Windows drive letter is not split. - Drop a saved gpu_ids pin that no longer resolves instead of 400ing the whole load. A pin outlives the machine it was made on. - Build the displayed API base from getApiBase() on desktop; the Tauri webview origin is not the API server. - Keep a partial download's isDownloaded when opening settings, so the loader still reports download progress. - Only prefer the loaded quant when the loaded model is this row. Q4_K_M exists in most repos, so an unguarded match targeted the wrong variant. Found by simulation: - A lone surrogate in a chat template raised UnicodeEncodeError on the byte check, an unhandled 500. Now a validation error, in all three call sites. - _bounded_int accepted bools as GPU ids, truncated fractional floats, and raised OverflowError on Infinity, which json.loads accepts. - api_monitor stored a non-string model verbatim; the monitor page then threw on toLowerCase and rendered nothing. Coerced at the boundary and the filter no longer trusts network data. - The overlay store now uses storage that cannot throw. Safari private mode and blocked-cookie origins make localStorage throw on access, which broke the opt-out toggle. --- studio/backend/core/inference/api_monitor.py | 4 +- studio/backend/picker/schemas.py | 18 +- studio/backend/routes/inference.py | 40 ++++- studio/backend/routes/settings.py | 29 +++- .../backend/tests/test_openai_auto_switch.py | 155 ++++++++++++++++++ .../utils/openai_auto_switch_settings.py | 19 ++- .../features/api-monitor/api-monitor-page.tsx | 9 +- .../src/features/api-monitor/overlay-store.ts | 38 ++++- .../features/api-monitor/use-api-monitor.ts | 19 ++- studio/frontend/src/features/hub/hub-page.tsx | 151 ++++++++--------- 10 files changed, 391 insertions(+), 91 deletions(-) diff --git a/studio/backend/core/inference/api_monitor.py b/studio/backend/core/inference/api_monitor.py index 670f370003..e4aad4ade2 100644 --- a/studio/backend/core/inference/api_monitor.py +++ b/studio/backend/core/inference/api_monitor.py @@ -113,7 +113,9 @@ class ApiMonitor: id = f"apireq_{uuid.uuid4().hex[:12]}", endpoint = endpoint, method = method, - model = model or "default", + # str(): a raw JSON body can carry any type here, and the field is + # rendered in the UI, where a non-string breaks the whole monitor. + model = str(model) if model else "default", prompt = _trim(prompt, _MAX_PROMPT_CHARS), status = "running", started_at = now, diff --git a/studio/backend/picker/schemas.py b/studio/backend/picker/schemas.py index b4f956188f..c43d769817 100644 --- a/studio/backend/picker/schemas.py +++ b/studio/backend/picker/schemas.py @@ -11,13 +11,29 @@ from pydantic import BaseModel, Field, field_validator MAX_CHAT_TEMPLATE_BYTES = 65_536 +def chat_template_byte_length(value: str) -> Optional[int]: + """UTF-8 length, or None if the string cannot be encoded at all. + + JSON can carry an unpaired surrogate, as a truncated emoji paste produces. + json decodes it fine and .encode("utf-8") then raises. Callers treat None as + "reject": such a template can never render. + """ + try: + return len(value.encode("utf-8")) + except UnicodeEncodeError: + return None + + class ValidateChatTemplateRequest(BaseModel): template: str = Field(default = "") @field_validator("template") @classmethod def _enforce_template_size(cls, value: str) -> str: - if len(value.encode("utf-8")) > MAX_CHAT_TEMPLATE_BYTES: + size = chat_template_byte_length(value) + if size is None: + raise ValueError("Chat template contains unpaired surrogate characters.") + if size > MAX_CHAT_TEMPLATE_BYTES: raise ValueError(f"Chat template exceeds the {MAX_CHAT_TEMPLATE_BYTES}-byte limit.") return value diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.py index 4bb0edcceb..eda92b21f1 100644 --- a/studio/backend/routes/inference.py +++ b/studio/backend/routes/inference.py @@ -3722,12 +3722,13 @@ async def _maybe_auto_switch_model( # speculative decoding, chat template and GPU placement, not # just the two legacy flags. Look the config up under the # variant-qualified id first (two quants of one repo can carry - # different configs), then the bare advertised id, then the - # concrete load path, so a config saved under any of the names - # this model is known by is found. + # different configs), then the bare ids. Both the advertised + # repo id and the concrete load path are tried: a local folder + # or a non-active HF cache is configured against its path. override = {} for override_key in ( f"{override_id}:{variant}" if variant else None, + f"{target_id}:{variant}" if variant else None, override_id, target_id, ): @@ -3745,6 +3746,19 @@ async def _maybe_auto_switch_model( is_gguf = bool(variant) or target_id.lower().endswith(".gguf"), ) ) + saved_gpu_ids = load_kwargs.get("gpu_ids") + if saved_gpu_ids and not _override_gpu_ids_still_resolve( + saved_gpu_ids + ): + # A pin saved before a GPU was removed, before a + # visibility-mask change, or on another host. Dropping the + # one dead field beats 400ing the whole load. + load_kwargs.pop("gpu_ids", None) + logger.warning( + "Dropping saved gpu_ids %s for %s: not available here.", + saved_gpu_ids, + override_id, + ) # Reuse the load impl so its dedup, tensor fallback, and threading # apply. Call the impl directly: we already hold the lifecycle gate # the /load route would otherwise take, so the route would deadlock. @@ -3965,6 +3979,26 @@ def _classify_diffusion_gguf(config: ModelConfig) -> Optional[bool]: return True if name_says_diffusion else None +def _override_gpu_ids_still_resolve(gpu_ids: List[int]) -> bool: + """Whether a per-model GPU pin is usable on this machine right now. + + normalize_model_override cannot know the device list, so it stores whatever + was valid where the config was written. This is the load-time reconciliation. + """ + try: + from utils.hardware import DeviceType, get_device + from utils.hardware.hardware import resolve_requested_gpu_ids + + is_vulkan = LlamaCppBackend._is_vulkan_backend() + if get_device() == DeviceType.XPU and not is_vulkan: + # gpu_ids is rejected outright on XPU. + return False + resolve_requested_gpu_ids(gpu_ids, is_vulkan = is_vulkan) + return True + except Exception: + return False + + async def _resolve_gguf_gpu_ids_for_request( config: ModelConfig, gpu_ids: Optional[List[int]] ) -> Optional[List[int]]: diff --git a/studio/backend/routes/settings.py b/studio/backend/routes/settings.py index 2dd0cabbe7..07a751cf61 100644 --- a/studio/backend/routes/settings.py +++ b/studio/backend/routes/settings.py @@ -34,7 +34,7 @@ from utils.helper_precache_settings import ( helper_model_disabled_by_env, set_helper_precache_enabled, ) -from picker.schemas import MAX_CHAT_TEMPLATE_BYTES +from picker.schemas import MAX_CHAT_TEMPLATE_BYTES, chat_template_byte_length from utils.coding_agents import CODING_AGENTS, detect_installed_coding_agents from utils.openai_auto_switch_settings import ( DEFAULT_AUTO_UNLOAD_KEEP_KV, @@ -166,7 +166,10 @@ class ModelOverridePayload(BaseModel): # template is accepted or rejected identically on both paths. if value is None: return None - if len(value.encode("utf-8")) > MAX_CHAT_TEMPLATE_BYTES: + size = chat_template_byte_length(value) + if size is None: + raise ValueError("Chat template contains unpaired surrogate characters.") + if size > MAX_CHAT_TEMPLATE_BYTES: raise ValueError(f"Chat template exceeds the {MAX_CHAT_TEMPLATE_BYTES}-byte limit.") return value @@ -322,6 +325,21 @@ def get_openai_auto_switch_overrides( return ModelOverridesResponse(overrides = get_model_overrides()) +# A quant suffix, as modelOverrideKey builds it: no path separator and short. +# Guards against splitting "C:\\models\\x.gguf", where the colon is a drive letter. +_MAX_VARIANT_SUFFIX_LEN = 64 + + +def _bare_model_id(model_id: str) -> Optional[str]: + """``repo`` for a ``repo:QUANT`` key, or None when there is no quant suffix.""" + head, sep, tail = model_id.rpartition(":") + if not sep or not head or not tail: + return None + if len(tail) > _MAX_VARIANT_SUFFIX_LEN or "/" in tail or "\\" in tail: + return None + return head + + @router.put("/openai-auto-switch/overrides", response_model = ModelOverridesResponse) def update_openai_auto_switch_override( payload: ModelOverridePayload, current_subject: str = Depends(get_current_subject) @@ -343,6 +361,13 @@ def update_openai_auto_switch_override( } if requested_extra_args is None and not is_removal: requested_extra_args = get_model_override(payload.model_id).get("llama_extra_args") + if requested_extra_args is None: + # First per-quant save for a model whose flags were stored under the + # bare repo id. Auto-switch prefers the qualified entry, so without + # this the flags are silently dropped and no UI can restore them. + bare_id = _bare_model_id(payload.model_id) + if bare_id: + requested_extra_args = get_model_override(bare_id).get("llama_extra_args") extra_args = validate_extra_args(requested_extra_args) set_model_override( payload.model_id, diff --git a/studio/backend/tests/test_openai_auto_switch.py b/studio/backend/tests/test_openai_auto_switch.py index d1173585b0..e6ce241d79 100644 --- a/studio/backend/tests/test_openai_auto_switch.py +++ b/studio/backend/tests/test_openai_auto_switch.py @@ -3982,3 +3982,158 @@ def test_override_route_preserves_launch_flags_across_a_settings_only_update(mon "tester", ) assert "unsloth/B-GGUF" not in gone.overrides + + +def test_override_found_under_a_concrete_path_with_variant(monkeypatch): + # A local folder or non-active HF cache resolves to a public repo id plus a + # concrete path. Settings saved against the path must still be found. + backend = _FakeBackend(None) + rec = _LoadRecorder(backend) + _wire( + monkeypatch, + enabled = True, + resolves_to = ("/models/local/Qwen3-8B-Q4_K_M.gguf", "Q4_K_M", "unsloth/Qwen3-8B-GGUF"), + backend = backend, + recorder = rec, + ) + stored = {"/models/local/Qwen3-8B-Q4_K_M.gguf:Q4_K_M": {"max_seq_length": 8192}} + monkeypatch.setattr(settings, "get_model_override", lambda mid: stored.get(mid, {})) + + _run_hook("unsloth/Qwen3-8B-GGUF") + assert rec.calls[0].max_seq_length == 8192 + + +def test_repo_qualified_override_beats_path_qualified(monkeypatch): + # Ordering is most specific first, and the public repo id is the name the + # user configured against in the picker. + backend = _FakeBackend(None) + rec = _LoadRecorder(backend) + _wire( + monkeypatch, + enabled = True, + resolves_to = ("/models/local/x.gguf", "Q4_K_M", "unsloth/B-GGUF"), + backend = backend, + recorder = rec, + ) + stored = { + "unsloth/B-GGUF:Q4_K_M": {"max_seq_length": 8192}, + "/models/local/x.gguf:Q4_K_M": {"max_seq_length": 1024}, + } + monkeypatch.setattr(settings, "get_model_override", lambda mid: stored.get(mid, {})) + + _run_hook("unsloth/B-GGUF") + assert rec.calls[0].max_seq_length == 8192 + + +def test_first_quant_save_keeps_legacy_bare_repo_launch_flags(monkeypatch): + # Flags were stored under the bare repo id before per-quant settings existed. + # The first save from the settings page writes repo:QUANT, and auto-switch + # then prefers that entry, so the flags must come with it or they are + # silently disabled with no UI able to show or restore them. + import routes.settings as settings_route + + _mock_override_store(monkeypatch) + settings.set_model_override("unsloth/B-GGUF", llama_extra_args = ["--flash-attn"]) + + resp = settings_route.update_openai_auto_switch_override( + settings_route.ModelOverridePayload( + model_id = "unsloth/B-GGUF:Q4_K_M", max_seq_length = 4096 + ), + "tester", + ) + entry = resp.overrides["unsloth/B-GGUF:Q4_K_M"] + assert entry["max_seq_length"] == 4096 + assert entry["llama_extra_args"] == ["--flash-attn"] + + +def test_bare_repo_carry_over_does_not_split_a_windows_path(monkeypatch): + # "C:\models\x.gguf" has a colon that is not a variant separator. Splitting + # naively would look up "C" and, worse, could graft another model's flags on. + import routes.settings as settings_route + + _mock_override_store(monkeypatch) + settings.set_model_override("C", llama_extra_args = ["--flash-attn"]) + + resp = settings_route.update_openai_auto_switch_override( + settings_route.ModelOverridePayload( + model_id = r"C:\models\x.gguf", max_seq_length = 4096 + ), + "tester", + ) + assert "llama_extra_args" not in resp.overrides[r"C:\models\x.gguf"] + + +def test_windows_path_with_quant_still_carries_over(monkeypatch): + import routes.settings as settings_route + + _mock_override_store(monkeypatch) + settings.set_model_override(r"C:\models\x.gguf", llama_extra_args = ["--flash-attn"]) + + resp = settings_route.update_openai_auto_switch_override( + settings_route.ModelOverridePayload( + model_id = r"C:\models\x.gguf:Q4_K_M", max_seq_length = 4096 + ), + "tester", + ) + assert resp.overrides[r"C:\models\x.gguf:Q4_K_M"]["llama_extra_args"] == ["--flash-attn"] + + +def test_stale_gpu_ids_are_dropped_not_fatal(monkeypatch): + # A pin saved on a two-GPU box, replayed on a one-GPU box. Before this the + # whole load 400d; the contract is that one dead field degrades to defaults. + backend = _FakeBackend(None) + rec = _LoadRecorder(backend) + _wire( + monkeypatch, + enabled = True, + resolves_to = ("unsloth/B-GGUF", "Q4_K_M", "unsloth/B-GGUF"), + backend = backend, + recorder = rec, + ) + monkeypatch.setattr( + settings, + "get_model_override", + lambda mid: {"gpu_ids": [0, 1], "max_seq_length": 4096}, + ) + monkeypatch.setattr( + inference_route, "_override_gpu_ids_still_resolve", lambda ids: False + ) + + _run_hook("unsloth/B-GGUF") + req = rec.calls[0] + assert not req.gpu_ids + # The rest of the config still applies. + assert req.max_seq_length == 4096 + + +def test_usable_gpu_ids_are_kept(monkeypatch): + backend = _FakeBackend(None) + rec = _LoadRecorder(backend) + _wire( + monkeypatch, + enabled = True, + resolves_to = ("unsloth/B-GGUF", "Q4_K_M", "unsloth/B-GGUF"), + backend = backend, + recorder = rec, + ) + monkeypatch.setattr( + settings, "get_model_override", lambda mid: {"gpu_ids": [0, 1]} + ) + monkeypatch.setattr( + inference_route, "_override_gpu_ids_still_resolve", lambda ids: True + ) + + _run_hook("unsloth/B-GGUF") + assert rec.calls[0].gpu_ids == [0, 1] + + +def test_override_gpu_ids_probe_never_raises(monkeypatch): + # The probe runs on the load path, so any hardware error must read as + # "unusable" rather than escaping as a 500. + import utils.hardware.hardware as hw + + def boom(*args, **kwargs): + raise RuntimeError("driver exploded") + + monkeypatch.setattr(hw, "resolve_requested_gpu_ids", boom) + assert inference_route._override_gpu_ids_still_resolve([0]) is False diff --git a/studio/backend/utils/openai_auto_switch_settings.py b/studio/backend/utils/openai_auto_switch_settings.py index 7f6b64eca4..ba63c47212 100644 --- a/studio/backend/utils/openai_auto_switch_settings.py +++ b/studio/backend/utils/openai_auto_switch_settings.py @@ -266,9 +266,18 @@ def _clean_str(value: Any, allowed: frozenset[str]) -> Optional[str]: def _bounded_int(value: Any, *, minimum: int, maximum: int) -> Optional[int]: + # bool is a subclass of int, so `gpu_ids: [true, false]` would otherwise pin + # the model to GPUs 1 and 0. + if isinstance(value, bool): + return None + # int(1.5) is 1, which would silently turn a fractional context into a + # useless one. Only exact integers count. + if isinstance(value, float) and not value.is_integer(): + return None try: parsed = int(value) - except (TypeError, ValueError): + except (TypeError, ValueError, OverflowError): + # OverflowError is float("inf"), which json.loads accepts as `Infinity`. return None if parsed < minimum or parsed > maximum: return None @@ -315,7 +324,13 @@ def normalize_model_override(payload: dict[str, Any]) -> dict[str, Any]: template = payload.get("chat_template_override") if isinstance(template, str) and template.strip(): - if len(template.encode("utf-8")) <= MAX_CHAT_TEMPLATE_OVERRIDE_BYTES: + # JSON can carry lone surrogates, which encode() rejects outright. Such a + # template can never render, so it is dropped like any other bad field. + try: + template_bytes = len(template.encode("utf-8")) + except UnicodeEncodeError: + template_bytes = MAX_CHAT_TEMPLATE_OVERRIDE_BYTES + 1 + if template_bytes <= MAX_CHAT_TEMPLATE_OVERRIDE_BYTES: entry["chat_template_override"] = template # Only "manual" is a real override: persisting "auto" would pin the model and diff --git a/studio/frontend/src/features/api-monitor/api-monitor-page.tsx b/studio/frontend/src/features/api-monitor/api-monitor-page.tsx index fa5934ab5d..50b27672c5 100644 --- a/studio/frontend/src/features/api-monitor/api-monitor-page.tsx +++ b/studio/frontend/src/features/api-monitor/api-monitor-page.tsx @@ -18,8 +18,10 @@ import { SelectValue, } from "@/components/ui/select"; import { Skeleton } from "@/components/ui/skeleton"; +import { usePlatformStore } from "@/config/env"; import type { ApiMonitorEntry } from "@/features/chat/types/api"; import { useSettingsDialogStore } from "@/features/settings"; +import { getApiBase, isTauri } from "@/lib/api-base"; import { copyToClipboard } from "@/lib/copy-to-clipboard"; import { Tick02Icon } from "@/lib/tick-icon"; import { cn } from "@/lib/utils"; @@ -462,6 +464,7 @@ export function ApiMonitorPage(): ReactElement { loadingDetails, requestDetail, } = useApiMonitor(); + const serverUrl = usePlatformStore((s) => s.serverUrl); const [statusFilter, setStatusFilter] = useState("all"); const [query, setQuery] = useState(""); const [selectedId, setSelectedId] = useState(null); @@ -497,8 +500,10 @@ export function ApiMonitorPage(): ReactElement { requestDetail(selectedId_); }, [selectedId_, selectedUpdatedAt, selectedIsMissing, requestDetail]); - const baseUrl = - typeof window === "undefined" ? "" : `${window.location.origin}/v1`; + // The desktop webview's origin is tauri://, not the API server, and the + // packaged app picks its port dynamically. Same source as the Agents tab. + const origin = typeof window === "undefined" ? "" : window.location.origin; + const baseUrl = `${isTauri ? (serverUrl ?? getApiBase()) : origin}/v1`; const serverStatus = data?.status ?? "idle"; const statusCopy = serverStatus === "generating" diff --git a/studio/frontend/src/features/api-monitor/overlay-store.ts b/studio/frontend/src/features/api-monitor/overlay-store.ts index 07ad4edd31..a2d1bd7631 100644 --- a/studio/frontend/src/features/api-monitor/overlay-store.ts +++ b/studio/frontend/src/features/api-monitor/overlay-store.ts @@ -2,7 +2,39 @@ // Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 import { create } from "zustand"; -import { persist } from "zustand/middleware"; +import { createJSONStorage, persist } from "zustand/middleware"; + +/** + * localStorage that cannot throw. + * + * Safari private browsing, Firefox with the origin's cookies blocked, and an + * opaque origin in a webview all make `window.localStorage` throw on access + * rather than return null. Losing the preference there is fine; taking the + * whole panel down with it is not. + */ +const safeStorage = { + getItem: (name: string): string | null => { + try { + return window.localStorage.getItem(name); + } catch { + return null; + } + }, + setItem: (name: string, value: string): void => { + try { + window.localStorage.setItem(name, value); + } catch { + // Quota exceeded or storage denied. The preference stays session-only. + } + }, + removeItem: (name: string): void => { + try { + window.localStorage.removeItem(name); + } catch { + // Same. + } + }, +}; interface ApiMonitorOverlayState { /** Whether the floating panel is on screen right now. Session state. */ @@ -33,7 +65,11 @@ export const useApiMonitorOverlayStore = create()( { name: "unsloth_api_monitor_overlay", version: 1, + storage: createJSONStorage(() => safeStorage), partialize: (state) => ({ autoOpen: state.autoOpen }), + // Without this a version bump discards the payload, quietly handing the + // popup back to someone who had turned it off. + migrate: (persisted) => persisted, // Explicit merge so an older stored payload cannot resurrect `isOpen`. merge: (persisted, current) => ({ ...current, diff --git a/studio/frontend/src/features/api-monitor/use-api-monitor.ts b/studio/frontend/src/features/api-monitor/use-api-monitor.ts index 20c3915ee3..84be9e3084 100644 --- a/studio/frontend/src/features/api-monitor/use-api-monitor.ts +++ b/studio/frontend/src/features/api-monitor/use-api-monitor.ts @@ -136,12 +136,19 @@ export function filterEntries( } // Search the fields a debugging session actually keys off: which model, // which endpoint, and the previews/error text visible in the row. - return ( - entry.model.toLowerCase().includes(needle) || - entry.endpoint.toLowerCase().includes(needle) || - entry.prompt_preview.toLowerCase().includes(needle) || - entry.reply_preview.toLowerCase().includes(needle) || - (entry.error ?? "").toLowerCase().includes(needle) + // + // Coerced, not trusted: these arrive over the network, and one malformed + // entry throwing here would blank the whole log. + return [ + entry.model, + entry.endpoint, + entry.prompt_preview, + entry.reply_preview, + entry.error, + ].some((field) => + String(field ?? "") + .toLowerCase() + .includes(needle), ); }); } diff --git a/studio/frontend/src/features/hub/hub-page.tsx b/studio/frontend/src/features/hub/hub-page.tsx index e853ea0e5c..8175774689 100644 --- a/studio/frontend/src/features/hub/hub-page.tsx +++ b/studio/frontend/src/features/hub/hub-page.tsx @@ -2,10 +2,6 @@ // Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 import { usePlatformStore } from "@/config/env"; -import { - isChannelEntryFresh, - useHubFeedStore, -} from "./stores/hub-feed-store"; import { getInferenceStatus, isExternalModelId, @@ -13,25 +9,17 @@ import { useChatModelRuntime, useChatRuntimeStore, } from "@/features/chat"; -import { useHubInventory } from "./inventory"; -import type { - HfModelSearchChannel, - HfSortDirection, - HfSortKey, -} from "./hooks/use-hub-model-search"; import { useOnlineStatus } from "@/features/hub"; import { useHubInfiniteScroll } from "@/features/hub"; -import { ggufVariantsMatch, modelIdsMatch } from "./lib/model-identity"; -import { hfApiToken, useHfTokenStore } from "./stores/hf-token-store"; import { + type ModelPickTarget, + type PerModelConfig, applyModelLoadConfigToRuntime, applyPerModelConfigToRuntime, currentRuntimePerModelConfig, hfModelFitsDevice, resolveInitialConfig, useActiveModelConfig, - type ModelPickTarget, - type PerModelConfig, } from "@/features/model-picker"; import { useDebouncedValue } from "@/hooks/use-debounced-value"; import { useGpuInfo } from "@/hooks/use-gpu-info"; @@ -47,8 +35,8 @@ import { } from "react"; import { ExternalLinkConfirmDialog } from "./catalog/external-link-confirm-dialog"; import { HubDetailView } from "./catalog/hub-detail-view"; -import { HubModelSettingsView } from "./catalog/hub-model-settings-view"; import { HubFeed } from "./catalog/hub-feed"; +import { HubModelSettingsView } from "./catalog/hub-model-settings-view"; import { HubTopBar } from "./catalog/hub-top-bar"; import { ModelsCatalog, @@ -72,8 +60,14 @@ import { useDiscoverSearch } from "./hooks/use-discover-search"; import { useFeedWriteBack } from "./hooks/use-feed-write-back"; import { useHiddenEmbeddingModelIds } from "./hooks/use-hidden-embedding-models"; import { useHubFeed } from "./hooks/use-hub-feed"; +import type { + HfModelSearchChannel, + HfSortDirection, + HfSortKey, +} from "./hooks/use-hub-model-search"; import { useHubModelVram } from "./hooks/use-hub-model-vram"; import { useModelsSelection } from "./hooks/use-models-selection"; +import { useHubInventory } from "./inventory"; import { CHANNEL_TO_SECTION, type ChannelId, @@ -88,6 +82,7 @@ import { isHiddenModelId, } from "./lib/hidden-models"; import { inventoryRowMatches, tokenizeQuery } from "./lib/inventory-search"; +import { ggufVariantsMatch, modelIdsMatch } from "./lib/model-identity"; import { type ModelTypeFilter, matchesModelType, @@ -101,6 +96,8 @@ import { matchesCapability, matchesFormat, } from "./lib/view-models"; +import { hfApiToken, useHfTokenStore } from "./stores/hf-token-store"; +import { isChannelEntryFresh, useHubFeedStore } from "./stores/hub-feed-store"; import type { CachedInventoryRow, CapabilityFilter, @@ -669,11 +666,7 @@ export function ModelsPage() { const visibleResults = results.length === 0 && liveListChannel && - isChannelEntryFresh( - cachedListEntry, - liveListChannel.id, - tokenFingerprint, - ) + isChannelEntryFresh(cachedListEntry, liveListChannel.id, tokenFingerprint) ? (cachedListEntry?.results ?? results) : results; @@ -1237,6 +1230,15 @@ export function ModelsPage() { async (row: CachedInventoryRow | LocalInventoryRow) => { // loadId is what the loader accepts; repoId is only a display/API alias. const id = row.loadId; + // Whether the loaded model is this row, under any of the names it goes by. + // Gates the "prefer the loaded quant" hint below. + const rowAliases = + row.kind === "local" + ? [id, row.repoId, row.path] + : [id, row.repoId, row.cachePath]; + const rowIsActive = rowAliases.some((alias) => + modelIdsMatch(alias, activeCheckpoint), + ); // Cached repo rows never carry a quant: the inventory emits one row per // repo with format_variant null (see cache_inventory.py). Opening settings // with a null variant would key the saved config to `repo::` while the @@ -1250,15 +1252,20 @@ export function ModelsPage() { try { const res = await listGgufVariants(repoId, hfApiToken(hfToken), { preferLocalCache: true, - localPath: row.kind === "local" ? row.path : (row.cachePath ?? null), + localPath: + row.kind === "local" ? row.path : (row.cachePath ?? null), }); const downloaded = res.variants.filter((v) => v.downloaded); ggufVariant = // Prefer the loaded quant, then the repo default, then whatever is - // on disk, mirroring LocalOnDeviceCard's selectedQuant. - downloaded.find((v) => - ggufVariantsMatch(v.quant, activeGgufVariant), - )?.quant ?? + // on disk, mirroring LocalOnDeviceCard's selectedQuant. Only when + // this row is the loaded model: Q4_K_M exists in most repos, so an + // unguarded match would target the wrong quant of the wrong model. + (rowIsActive + ? downloaded.find((v) => + ggufVariantsMatch(v.quant, activeGgufVariant), + )?.quant + : undefined) ?? downloaded.find((v) => ggufVariantsMatch(v.quant, res.default_variant), )?.quant ?? @@ -1291,7 +1298,7 @@ export function ModelsPage() { }, }); }, - [activeGgufVariant, hfToken], + [activeCheckpoint, activeGgufVariant, hfToken], ); // Applying from the settings page loads the model with exactly those settings. // ModelConfigPage has already persisted them (locally and, when "remember" is @@ -1310,7 +1317,9 @@ export function ModelsPage() { source: "local", ggufVariant: target.ggufVariant ?? undefined, isGguf: target.isGguf, - isDownloaded: true, + // A partial row opens settings too; claiming complete would skip the + // loader's download-progress reporting. + isDownloaded: target.meta.isDownloaded, isLora: target.meta.isLora, keepSpeculative: true, forceReload: true, @@ -1424,45 +1433,13 @@ export function ModelsPage() { ], ); - const catalogState = useMemo( - () => { - const typeFilterActive = - !isDatasetMode && inventoryTypeFilter !== "all"; - return { - tab, - discoverRows: listRows, - cachedRows: filteredCachedRows, - localRows: filteredLocalRows, - selectedId, - isLoading, - downloadedReady, - inventoryError, - inventoryWarning, - query, - activeCheckpoint, - activeGgufVariant, - searchError, - online, - isDataset: isDatasetMode, - inventoryTokens, - scannedCount, - loadingIntentCount: discoverFetchIntent, - hasMore, - manualFetchAvailable: discoverManualFetchAvailable, - hasActiveFilters: - !isFeedMode && - (deferredFormatFilter !== "all" || - deferredCapabilityFilter !== "all" || - (tab === "downloaded" && typeFilterActive)), - typeFilterActive, - }; - }, - [ + const catalogState = useMemo(() => { + const typeFilterActive = !isDatasetMode && inventoryTypeFilter !== "all"; + return { tab, - isFeedMode, - listRows, - filteredCachedRows, - filteredLocalRows, + discoverRows: listRows, + cachedRows: filteredCachedRows, + localRows: filteredLocalRows, selectedId, isLoading, downloadedReady, @@ -1473,17 +1450,45 @@ export function ModelsPage() { activeGgufVariant, searchError, online, - isDatasetMode, + isDataset: isDatasetMode, inventoryTokens, scannedCount, - discoverFetchIntent, + loadingIntentCount: discoverFetchIntent, hasMore, - discoverManualFetchAvailable, - deferredFormatFilter, - deferredCapabilityFilter, - inventoryTypeFilter, - ], - ); + manualFetchAvailable: discoverManualFetchAvailable, + hasActiveFilters: + !isFeedMode && + (deferredFormatFilter !== "all" || + deferredCapabilityFilter !== "all" || + (tab === "downloaded" && typeFilterActive)), + typeFilterActive, + }; + }, [ + tab, + isFeedMode, + listRows, + filteredCachedRows, + filteredLocalRows, + selectedId, + isLoading, + downloadedReady, + inventoryError, + inventoryWarning, + query, + activeCheckpoint, + activeGgufVariant, + searchError, + online, + isDatasetMode, + inventoryTokens, + scannedCount, + discoverFetchIntent, + hasMore, + discoverManualFetchAvailable, + deferredFormatFilter, + deferredCapabilityFilter, + inventoryTypeFilter, + ]); const catalogPagination = useMemo( () => ({ From 9dee5ac2d0b7e216851e02ae8157b3479647c9b0 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Sun, 26 Jul 2026 14:37:36 +0000 Subject: [PATCH 04/75] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- studio/backend/routes/inference.py | 4 +--- .../backend/tests/test_openai_auto_switch.py | 20 +++++-------------- 2 files changed, 6 insertions(+), 18 deletions(-) diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.py index eda92b21f1..beb9ebdb87 100644 --- a/studio/backend/routes/inference.py +++ b/studio/backend/routes/inference.py @@ -3747,9 +3747,7 @@ async def _maybe_auto_switch_model( ) ) saved_gpu_ids = load_kwargs.get("gpu_ids") - if saved_gpu_ids and not _override_gpu_ids_still_resolve( - saved_gpu_ids - ): + if saved_gpu_ids and not _override_gpu_ids_still_resolve(saved_gpu_ids): # A pin saved before a GPU was removed, before a # visibility-mask change, or on another host. Dropping the # one dead field beats 400ing the whole load. diff --git a/studio/backend/tests/test_openai_auto_switch.py b/studio/backend/tests/test_openai_auto_switch.py index e6ce241d79..503604f03f 100644 --- a/studio/backend/tests/test_openai_auto_switch.py +++ b/studio/backend/tests/test_openai_auto_switch.py @@ -4036,9 +4036,7 @@ def test_first_quant_save_keeps_legacy_bare_repo_launch_flags(monkeypatch): settings.set_model_override("unsloth/B-GGUF", llama_extra_args = ["--flash-attn"]) resp = settings_route.update_openai_auto_switch_override( - settings_route.ModelOverridePayload( - model_id = "unsloth/B-GGUF:Q4_K_M", max_seq_length = 4096 - ), + settings_route.ModelOverridePayload(model_id = "unsloth/B-GGUF:Q4_K_M", max_seq_length = 4096), "tester", ) entry = resp.overrides["unsloth/B-GGUF:Q4_K_M"] @@ -4055,9 +4053,7 @@ def test_bare_repo_carry_over_does_not_split_a_windows_path(monkeypatch): settings.set_model_override("C", llama_extra_args = ["--flash-attn"]) resp = settings_route.update_openai_auto_switch_override( - settings_route.ModelOverridePayload( - model_id = r"C:\models\x.gguf", max_seq_length = 4096 - ), + settings_route.ModelOverridePayload(model_id = r"C:\models\x.gguf", max_seq_length = 4096), "tester", ) assert "llama_extra_args" not in resp.overrides[r"C:\models\x.gguf"] @@ -4095,9 +4091,7 @@ def test_stale_gpu_ids_are_dropped_not_fatal(monkeypatch): "get_model_override", lambda mid: {"gpu_ids": [0, 1], "max_seq_length": 4096}, ) - monkeypatch.setattr( - inference_route, "_override_gpu_ids_still_resolve", lambda ids: False - ) + monkeypatch.setattr(inference_route, "_override_gpu_ids_still_resolve", lambda ids: False) _run_hook("unsloth/B-GGUF") req = rec.calls[0] @@ -4116,12 +4110,8 @@ def test_usable_gpu_ids_are_kept(monkeypatch): backend = backend, recorder = rec, ) - monkeypatch.setattr( - settings, "get_model_override", lambda mid: {"gpu_ids": [0, 1]} - ) - monkeypatch.setattr( - inference_route, "_override_gpu_ids_still_resolve", lambda ids: True - ) + monkeypatch.setattr(settings, "get_model_override", lambda mid: {"gpu_ids": [0, 1]}) + monkeypatch.setattr(inference_route, "_override_gpu_ids_still_resolve", lambda ids: True) _run_hook("unsloth/B-GGUF") assert rec.calls[0].gpu_ids == [0, 1] From 04e8beec62f8df1a6f9b6454271eda8e629970db Mon Sep 17 00:00:00 2001 From: Unsloth Date: Sun, 26 Jul 2026 17:43:30 -0700 Subject: [PATCH 05/75] studio: address second review round on per-model settings - Probe actual Vulkan devices in the GPU reconciliation helper. Vulkan ordinals are their own index space, so resolve_requested_gpu_ids only rejects malformed ones; the helper said "usable" and the load then 400d on the ggml probe it had skipped. - Send explicit save/remove intent. A save whose config is entirely default carries no fields, which is shape-identical to "forget this model", so it was wiping launch flags the UI cannot show or restore. A bare model_id without the flag still removes, keeping the original contract. - Backfill existing per-model settings into the server map once after upgrade. Without it, settings saved before this change never reached the server, so an API load used app defaults while the UI still showed the model as remembered. Never overwrites a server entry and retries until it fully succeeds. - Stop polling the monitor endpoint when auto-open is off and the panel is closed. It cannot open or display anything in that state, so every open Studio window was polling every five seconds for nothing. --- studio/backend/routes/inference.py | 23 ++++- studio/backend/routes/settings.py | 15 ++- .../backend/tests/test_openai_auto_switch.py | 99 ++++++++++++++++++- studio/frontend/src/app/routes/__root.tsx | 31 ++++-- .../api-monitor/api-monitor-overlay.tsx | 6 +- .../api/migrate-model-overrides.ts | 81 +++++++++++++++ .../model-picker/api/model-overrides.ts | 4 + .../model-config/per-model-config.ts | 34 ++++++- 8 files changed, 268 insertions(+), 25 deletions(-) create mode 100644 studio/frontend/src/features/model-picker/api/migrate-model-overrides.ts diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.py index beb9ebdb87..d804b3143d 100644 --- a/studio/backend/routes/inference.py +++ b/studio/backend/routes/inference.py @@ -3747,7 +3747,9 @@ async def _maybe_auto_switch_model( ) ) saved_gpu_ids = load_kwargs.get("gpu_ids") - if saved_gpu_ids and not _override_gpu_ids_still_resolve(saved_gpu_ids): + if saved_gpu_ids and not await _override_gpu_ids_still_resolve( + saved_gpu_ids + ): # A pin saved before a GPU was removed, before a # visibility-mask change, or on another host. Dropping the # one dead field beats 400ing the whole load. @@ -3977,11 +3979,13 @@ def _classify_diffusion_gguf(config: ModelConfig) -> Optional[bool]: return True if name_says_diffusion else None -def _override_gpu_ids_still_resolve(gpu_ids: List[int]) -> bool: +async def _override_gpu_ids_still_resolve(gpu_ids: List[int]) -> bool: """Whether a per-model GPU pin is usable on this machine right now. normalize_model_override cannot know the device list, so it stores whatever - was valid where the config was written. This is the load-time reconciliation. + was valid where the config was written. This is the load-time reconciliation, + and it has to make every check _resolve_gguf_gpu_ids_for_request would later + make, or the load 400s on the check this one skipped. """ try: from utils.hardware import DeviceType, get_device @@ -3991,7 +3995,18 @@ def _override_gpu_ids_still_resolve(gpu_ids: List[int]) -> bool: if get_device() == DeviceType.XPU and not is_vulkan: # gpu_ids is rejected outright on XPU. return False - resolve_requested_gpu_ids(gpu_ids, is_vulkan = is_vulkan) + resolved = resolve_requested_gpu_ids(gpu_ids, is_vulkan = is_vulkan) + if is_vulkan and resolved: + # Vulkan ordinals are their own index space, so resolve() only rejects + # malformed ones. Presence needs the same ggml probe the load does. + binary = LlamaCppBackend._find_llama_server_binary() + if binary: + probed = { + gpu[0] + for gpu in await asyncio.to_thread(LlamaCppBackend._get_gpu_memory, binary) + } + if not {int(gpu_id) for gpu_id in resolved}.issubset(probed): + return False return True except Exception: return False diff --git a/studio/backend/routes/settings.py b/studio/backend/routes/settings.py index 07a751cf61..c6f64f8f16 100644 --- a/studio/backend/routes/settings.py +++ b/studio/backend/routes/settings.py @@ -158,6 +158,10 @@ class ModelOverridePayload(BaseModel): gpu_layers: Optional[int] = Field(default = None, ge = -1, le = 1024) n_cpu_moe: Optional[int] = Field(default = None, ge = 0, le = 1024) gpu_ids: Optional[list[int]] = None + # Explicit intent. A save whose config is entirely default carries no fields + # at all, which is indistinguishable from "forget this model" by shape alone. + # None keeps the original contract: a bare model_id means remove. + remove: Optional[bool] = None @field_validator("chat_template_override") @classmethod @@ -354,11 +358,14 @@ def update_openai_auto_switch_override( # them and must not delete them). requested_extra_args = payload.llama_extra_args saved_fields = payload.model_dump( - exclude = {"model_id", "llama_extra_args"}, exclude_none = True + exclude = {"model_id", "llama_extra_args", "remove"}, exclude_none = True ) - is_removal = not payload.tensor_parallel and not { - key: value for key, value in saved_fields.items() if key != "tensor_parallel" - } + if payload.remove is not None: + is_removal = payload.remove + else: + is_removal = not payload.tensor_parallel and not { + key: value for key, value in saved_fields.items() if key != "tensor_parallel" + } if requested_extra_args is None and not is_removal: requested_extra_args = get_model_override(payload.model_id).get("llama_extra_args") if requested_extra_args is None: diff --git a/studio/backend/tests/test_openai_auto_switch.py b/studio/backend/tests/test_openai_auto_switch.py index 503604f03f..fc02f3f538 100644 --- a/studio/backend/tests/test_openai_auto_switch.py +++ b/studio/backend/tests/test_openai_auto_switch.py @@ -4091,7 +4091,11 @@ def test_stale_gpu_ids_are_dropped_not_fatal(monkeypatch): "get_model_override", lambda mid: {"gpu_ids": [0, 1], "max_seq_length": 4096}, ) - monkeypatch.setattr(inference_route, "_override_gpu_ids_still_resolve", lambda ids: False) + + async def _unusable(ids): + return False + + monkeypatch.setattr(inference_route, "_override_gpu_ids_still_resolve", _unusable) _run_hook("unsloth/B-GGUF") req = rec.calls[0] @@ -4111,7 +4115,11 @@ def test_usable_gpu_ids_are_kept(monkeypatch): recorder = rec, ) monkeypatch.setattr(settings, "get_model_override", lambda mid: {"gpu_ids": [0, 1]}) - monkeypatch.setattr(inference_route, "_override_gpu_ids_still_resolve", lambda ids: True) + + async def _usable(ids): + return True + + monkeypatch.setattr(inference_route, "_override_gpu_ids_still_resolve", _usable) _run_hook("unsloth/B-GGUF") assert rec.calls[0].gpu_ids == [0, 1] @@ -4126,4 +4134,89 @@ def test_override_gpu_ids_probe_never_raises(monkeypatch): raise RuntimeError("driver exploded") monkeypatch.setattr(hw, "resolve_requested_gpu_ids", boom) - assert inference_route._override_gpu_ids_still_resolve([0]) is False + assert asyncio.run(inference_route._override_gpu_ids_still_resolve([0])) is False + + +def test_vulkan_ordinal_absent_from_the_probe_is_unusable(monkeypatch): + # resolve_requested_gpu_ids only rejects malformed Vulkan ordinals, so + # presence needs the same ggml probe the load itself runs. Without it this + # helper says "fine" and the load 400s on the check it skipped. + from core.inference.llama_cpp import LlamaCppBackend + + monkeypatch.setattr(LlamaCppBackend, "_is_vulkan_backend", staticmethod(lambda: True)) + monkeypatch.setattr( + LlamaCppBackend, "_find_llama_server_binary", staticmethod(lambda: "/bin/llama-server") + ) + monkeypatch.setattr( + LlamaCppBackend, "_get_gpu_memory", staticmethod(lambda binary: [(0, 8192)]) + ) + assert asyncio.run(inference_route._override_gpu_ids_still_resolve([0])) is True + assert asyncio.run(inference_route._override_gpu_ids_still_resolve([7])) is False + assert asyncio.run(inference_route._override_gpu_ids_still_resolve([0, 1])) is False + + +def test_vulkan_probe_without_a_binary_does_not_block_the_load(monkeypatch): + # No binary means nothing to probe with. Refusing here would drop a valid + # pin on every load, so the later path stays the authority. + from core.inference.llama_cpp import LlamaCppBackend + + monkeypatch.setattr(LlamaCppBackend, "_is_vulkan_backend", staticmethod(lambda: True)) + monkeypatch.setattr(LlamaCppBackend, "_find_llama_server_binary", staticmethod(lambda: None)) + assert asyncio.run(inference_route._override_gpu_ids_still_resolve([0])) is True + + +def test_default_save_preserves_flags_instead_of_removing(monkeypatch): + # "Remember for this model" is on but every value is default, so the payload + # carries no fields. That is shape-identical to a removal, and guessing wrong + # wipes launch flags no UI can show or restore. + import routes.settings as settings_route + + _mock_override_store(monkeypatch) + settings.set_model_override("unsloth/B-GGUF", llama_extra_args = ["--flash-attn"]) + + resp = settings_route.update_openai_auto_switch_override( + settings_route.ModelOverridePayload(model_id = "unsloth/B-GGUF", remove = False), + "tester", + ) + assert resp.overrides["unsloth/B-GGUF"]["llama_extra_args"] == ["--flash-attn"] + + +def test_explicit_remove_still_clears_everything(monkeypatch): + import routes.settings as settings_route + + _mock_override_store(monkeypatch) + settings.set_model_override( + "unsloth/B-GGUF", llama_extra_args = ["--flash-attn"], max_seq_length = 4096 + ) + resp = settings_route.update_openai_auto_switch_override( + settings_route.ModelOverridePayload( + model_id = "unsloth/B-GGUF", remove = True, llama_extra_args = [] + ), + "tester", + ) + assert "unsloth/B-GGUF" not in resp.overrides + + +def test_bare_payload_without_remove_flag_still_removes(monkeypatch): + # The original contract, kept for any caller that predates the flag. + import routes.settings as settings_route + + _mock_override_store(monkeypatch) + settings.set_model_override("unsloth/B-GGUF", max_seq_length = 4096) + resp = settings_route.update_openai_auto_switch_override( + settings_route.ModelOverridePayload(model_id = "unsloth/B-GGUF"), "tester" + ) + assert "unsloth/B-GGUF" not in resp.overrides + + +def test_remove_false_with_real_fields_saves_normally(monkeypatch): + import routes.settings as settings_route + + _mock_override_store(monkeypatch) + resp = settings_route.update_openai_auto_switch_override( + settings_route.ModelOverridePayload( + model_id = "unsloth/B-GGUF", remove = False, max_seq_length = 8192 + ), + "tester", + ) + assert resp.overrides["unsloth/B-GGUF"]["max_seq_length"] == 8192 diff --git a/studio/frontend/src/app/routes/__root.tsx b/studio/frontend/src/app/routes/__root.tsx index 5dbeefa8f8..ff7de58a4f 100644 --- a/studio/frontend/src/app/routes/__root.tsx +++ b/studio/frontend/src/app/routes/__root.tsx @@ -3,25 +3,26 @@ import { AppSidebar } from "@/components/app-sidebar"; import { Navbar } from "@/components/navbar"; -import { fetchDeviceType, usePlatformStore } from "@/config/env"; import { SidebarInset, SidebarProvider } from "@/components/ui/sidebar"; -import { SettingsDialog, useSettingsDialogStore } from "@/features/settings"; +import { fetchDeviceType, usePlatformStore } from "@/config/env"; +import { ApiMonitorOverlay } from "@/features/api-monitor/api-monitor-overlay"; +import { hasAuthToken } from "@/features/auth"; import { ChatPage, + type ChatSearch, clearNewChatDraft, useChatRuntimeStore, - type ChatSearch, } from "@/features/chat"; -import { ApiMonitorOverlay } from "@/features/api-monitor/api-monitor-overlay"; -import { RemoteCodeConsentDialog } from "@/features/security"; -import { HfTokenWarningDialog } from "@/features/hf-auth"; -import { TransformersUpgradeDialog } from "@/features/transformers-upgrade"; -import { useTrainingUnloadGuard } from "@/features/training"; import { useExportRuntimeLifecycle } from "@/features/export"; -import { hasAuthToken } from "@/features/auth"; +import { HfTokenWarningDialog } from "@/features/hf-auth"; +import { backfillModelOverrides } from "@/features/model-picker/api/migrate-model-overrides"; import { usePersonalizationSync } from "@/features/profile"; +import { RemoteCodeConsentDialog } from "@/features/security"; +import { SettingsDialog, useSettingsDialogStore } from "@/features/settings"; +import { useTrainingUnloadGuard } from "@/features/training"; +import { TransformersUpgradeDialog } from "@/features/transformers-upgrade"; import { useSidebarPin } from "@/hooks/use-sidebar-pin"; -import { useT, type TranslationKey } from "@/i18n"; +import { type TranslationKey, useT } from "@/i18n"; import { Outlet, createRootRoute, @@ -174,6 +175,16 @@ function RootLayout() { : DEFAULT_DOCUMENT_TITLE; }, [documentTitle]); + // Settings saved before the server-side override map existed live only in this + // browser, so an API load would use app defaults while the UI still showed the + // model as remembered. Backfill once, after auth. + useEffect(() => { + if (isAuthFlowRoute) { + return; + } + void backfillModelOverrides(); + }, [isAuthFlowRoute]); + useEffect(() => { if (isAuthFlowRoute) { useSettingsDialogStore.getState().closeDialog(); diff --git a/studio/frontend/src/features/api-monitor/api-monitor-overlay.tsx b/studio/frontend/src/features/api-monitor/api-monitor-overlay.tsx index 4411e052ad..88a4915b93 100644 --- a/studio/frontend/src/features/api-monitor/api-monitor-overlay.tsx +++ b/studio/frontend/src/features/api-monitor/api-monitor-overlay.tsx @@ -111,7 +111,9 @@ export function ApiMonitorOverlay(): ReactElement | null { // One loop for both jobs: panel contents while open, traffic watch while // closed. Stands down on the full page, which polls for itself. useEffect(() => { - if (onFullPage) { + // Opted out and closed: the panel can neither open nor show anything, so + // polling would be pure background load on every open Studio window. + if (onFullPage || (!autoOpen && !isOpen)) { return; } let cancelled = false; @@ -146,7 +148,7 @@ export function ApiMonitorOverlay(): ReactElement | null { cancelled = true; if (timer !== undefined) window.clearTimeout(timer); }; - }, [isOpen, onFullPage]); + }, [isOpen, onFullPage, autoOpen]); const entries = useMemo(() => data?.entries ?? [], [data]); const stats = useMemo(() => computeStats(entries), [entries]); diff --git a/studio/frontend/src/features/model-picker/api/migrate-model-overrides.ts b/studio/frontend/src/features/model-picker/api/migrate-model-overrides.ts new file mode 100644 index 0000000000..945dc1fbdd --- /dev/null +++ b/studio/frontend/src/features/model-picker/api/migrate-model-overrides.ts @@ -0,0 +1,81 @@ +// SPDX-License-Identifier: AGPL-3.0-only +// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 + +// One-time backfill of per-model settings into the server override map. +// +// Settings used to live only in this browser, so on upgrade the server knows +// nothing about models the user already configured. Without this they keep +// showing as remembered in the UI while an API load quietly uses app defaults, +// which is the exact bug the server-side map exists to fix. + +import { + isDefaultConfig, + listPerModelConfigs, +} from "../model-config/per-model-config"; +import { + fetchModelOverrides, + modelOverrideKey, + putModelOverride, +} from "./model-overrides"; + +const DONE_FLAG = "unsloth_model_overrides_backfilled_v1"; + +function alreadyRan(): boolean { + try { + return window.localStorage.getItem(DONE_FLAG) === "1"; + } catch { + // Storage denied: treat as done rather than re-running on every mount. + return true; + } +} + +function markRan(): void { + try { + window.localStorage.setItem(DONE_FLAG, "1"); + } catch { + // Nothing to do; the backfill is idempotent anyway. + } +} + +/** + * Push local configs the server has never seen. Never deletes and never + * overwrites: an entry already on the server is the newer authority, and losing + * a setting here would be worse than leaving one unmigrated. + */ +export async function backfillModelOverrides(): Promise { + if (alreadyRan()) { + return; + } + const local = listPerModelConfigs().filter( + (entry) => !isDefaultConfig(entry.config), + ); + if (local.length === 0) { + markRan(); + return; + } + + let existing: Awaited>; + try { + existing = await fetchModelOverrides(); + } catch { + // Offline or not authenticated yet. Leave the flag unset so the next start + // tries again rather than silently skipping the migration forever. + return; + } + + let failed = false; + for (const entry of local) { + const key = modelOverrideKey(entry.modelId, entry.ggufVariant); + if (existing[key]) { + continue; + } + try { + await putModelOverride(entry.modelId, entry.ggufVariant, entry.config); + } catch { + failed = true; + } + } + if (!failed) { + markRan(); + } +} diff --git a/studio/frontend/src/features/model-picker/api/model-overrides.ts b/studio/frontend/src/features/model-picker/api/model-overrides.ts index e3a735884b..d4e89985b4 100644 --- a/studio/frontend/src/features/model-picker/api/model-overrides.ts +++ b/studio/frontend/src/features/model-picker/api/model-overrides.ts @@ -134,6 +134,10 @@ export async function putModelOverride( body: JSON.stringify({ // biome-ignore lint/style/useNamingConvention: API schema model_id: modelOverrideKey(modelId, ggufVariant), + // Say which operation this is. A save of an all-default config carries no + // fields, which is shape-identical to "forget this model", and guessing + // wrong wipes launch flags the UI cannot show or restore. + remove: config === null, // Launch flags have no UI control, so the backend preserves them when the // field is omitted. Forgetting a model means forgetting all of it, so that // path sends an explicit empty list to clear them. diff --git a/studio/frontend/src/features/model-picker/model-config/per-model-config.ts b/studio/frontend/src/features/model-picker/model-config/per-model-config.ts index ba6d4cec99..8511f3d58e 100644 --- a/studio/frontend/src/features/model-picker/model-config/per-model-config.ts +++ b/studio/frontend/src/features/model-picker/model-config/per-model-config.ts @@ -416,7 +416,10 @@ function writeMap(map: StoredMap): boolean { } } -function warnDroppedFields(raw: Record, version: number): void { +function warnDroppedFields( + raw: Record, + version: number, +): void { if (!import.meta.env?.DEV) { return; } @@ -436,7 +439,8 @@ function normalizeV1(partial: RawConfig): PerModelConfig { typeof partial.speculativeType === "string" ? canonicalizeSpeculativeType(partial.speculativeType) : null; - const speculativeType = rawSpecType ?? DEFAULT_PER_MODEL_CONFIG.speculativeType; + const speculativeType = + rawSpecType ?? DEFAULT_PER_MODEL_CONFIG.speculativeType; const specDraftNMax = speculativeType != null && MTP_SPECULATIVE_TYPES.has(speculativeType) && @@ -648,6 +652,32 @@ export function savePerModelConfig( return writeMap(map); } +/** Every saved per-model config, decoded back to the ids it was keyed by. */ +export function listPerModelConfigs(): { + modelId: string; + ggufVariant: string | null; + config: PerModelConfig; +}[] { + const out: { + modelId: string; + ggufVariant: string | null; + config: PerModelConfig; + }[] = []; + for (const [key, raw] of Object.entries(readMap())) { + const modelId = modelIdFromStorageKey(key); + if (!modelId) { + continue; + } + const variant = ggufVariantFromStorageKey(key); + out.push({ + modelId, + ggufVariant: variant ? variant : null, + config: normalize(raw), + }); + } + return out; +} + export function deletePerModelConfig( modelId: string, ggufVariant?: string | null, From 8f322aa627d191b55cbbc2a57bed539b538b4b94 Mon Sep 17 00:00:00 2001 From: Unsloth Date: Sun, 26 Jul 2026 18:32:50 -0700 Subject: [PATCH 06/75] studio: address third review round on per-model settings - Fall back to a case-insensitive override lookup. The browser normalizes ids to lowercase before storing, so the backfill wrote "org/model:q4_k_m" while the resolver asked for "Org/Model:Q4_K_M" and never matched, which made the whole migration a no-op. Exact match still wins and an ambiguous fallback matches nothing, so two POSIX paths differing only in case stay distinct. - Record a detail revision only when a fetch actually started. requestDetail's in-flight guard can refuse, and recording anyway meant a revision that landed during an earlier fetch was skipped for good once updated_at stopped moving, leaving a terminal request showing a stale running payload. - Discard superseded settings opens. The GGUF variant lookup is async, so opening a second row while the first was pending let whichever finished last win, and the page could then save or load settings for the wrong model. - Raise the model_id cap to PATH_MAX plus a quant suffix. A local model's id is its filesystem path and LoadRequest.model_path is unbounded, so the old 512 limit 422d the server sync while the local save succeeded, leaving the UI showing settings the API would never apply. --- studio/backend/routes/settings.py | 18 ++++++++----- .../backend/tests/test_openai_auto_switch.py | 27 +++++++++++++++++++ .../utils/openai_auto_switch_settings.py | 25 ++++++++++++++--- .../features/api-monitor/api-monitor-page.tsx | 7 +++-- .../features/api-monitor/use-api-monitor.ts | 10 ++++--- studio/frontend/src/features/hub/hub-page.tsx | 9 +++++++ 6 files changed, 82 insertions(+), 14 deletions(-) diff --git a/studio/backend/routes/settings.py b/studio/backend/routes/settings.py index c6f64f8f16..672d9941cc 100644 --- a/studio/backend/routes/settings.py +++ b/studio/backend/routes/settings.py @@ -126,6 +126,17 @@ class OpenAIAutoSwitchResponse(BaseModel): auto_unload_keep_kv: bool = DEFAULT_AUTO_UNLOAD_KEEP_KV +# A quant suffix, as modelOverrideKey builds it: no path separator and short. +# Guards against splitting "C:\\models\\x.gguf", where the colon is a drive letter. +_MAX_VARIANT_SUFFIX_LEN = 64 + +# A local model's id is its filesystem path, optionally with a quant suffix, and +# LoadRequest.model_path is unbounded. A limit under PATH_MAX would 422 the server +# sync while the local save succeeded, leaving the UI showing settings the API +# never applies. +MAX_MODEL_OVERRIDE_KEY_LEN = 4096 + 1 + _MAX_VARIANT_SUFFIX_LEN + + class ModelOverridePayload(BaseModel): """One model's saved launch config, applied when the API loads that model. @@ -136,7 +147,7 @@ class ModelOverridePayload(BaseModel): mode) are left to it, since their valid sets follow the llama.cpp build. """ - model_id: str = Field(..., min_length = 1, max_length = 512) + model_id: str = Field(..., min_length = 1, max_length = MAX_MODEL_OVERRIDE_KEY_LEN) # None means "leave the stored value alone": the settings UI has no control # for launch flags, so a save from it must not wipe flags set through this # API. An explicit [] clears them (that is how "forget this model" arrives). @@ -329,11 +340,6 @@ def get_openai_auto_switch_overrides( return ModelOverridesResponse(overrides = get_model_overrides()) -# A quant suffix, as modelOverrideKey builds it: no path separator and short. -# Guards against splitting "C:\\models\\x.gguf", where the colon is a drive letter. -_MAX_VARIANT_SUFFIX_LEN = 64 - - def _bare_model_id(model_id: str) -> Optional[str]: """``repo`` for a ``repo:QUANT`` key, or None when there is no quant suffix.""" head, sep, tail = model_id.rpartition(":") diff --git a/studio/backend/tests/test_openai_auto_switch.py b/studio/backend/tests/test_openai_auto_switch.py index fc02f3f538..aa20752aee 100644 --- a/studio/backend/tests/test_openai_auto_switch.py +++ b/studio/backend/tests/test_openai_auto_switch.py @@ -4220,3 +4220,30 @@ def test_remove_false_with_real_fields_saves_normally(monkeypatch): "tester", ) assert resp.overrides["unsloth/B-GGUF"]["max_seq_length"] == 8192 + + +def test_override_lookup_falls_back_to_case_insensitive(monkeypatch): + # The browser lowercases ids before storing them, so the backfill writes + # "unsloth/qwen3-8b-gguf:q4_k_m" while the resolver asks for the repo's real + # casing. Without this fallback every migrated entry is invisible. + _mock_override_store(monkeypatch) + settings.set_model_override("unsloth/qwen3-8b-gguf:q4_k_m", max_seq_length = 8192) + got = settings.get_model_override("unsloth/Qwen3-8B-GGUF:Q4_K_M") + assert got["max_seq_length"] == 8192 + + +def test_exact_override_match_beats_a_case_variant(monkeypatch): + _mock_override_store(monkeypatch) + settings.set_model_override("/models/foo.gguf", max_seq_length = 1024) + settings.set_model_override("/models/Foo.gguf", max_seq_length = 8192) + assert settings.get_model_override("/models/Foo.gguf")["max_seq_length"] == 8192 + assert settings.get_model_override("/models/foo.gguf")["max_seq_length"] == 1024 + + +def test_ambiguous_case_fallback_matches_nothing(monkeypatch): + # Two POSIX paths differing only in case are two different files. Guessing + # between them would apply one model's settings to another. + _mock_override_store(monkeypatch) + settings.set_model_override("/models/foo.gguf", max_seq_length = 1024) + settings.set_model_override("/models/FOO.gguf", max_seq_length = 8192) + assert settings.get_model_override("/models/Foo.gguf") == {} diff --git a/studio/backend/utils/openai_auto_switch_settings.py b/studio/backend/utils/openai_auto_switch_settings.py index ba63c47212..ca2a7c5d10 100644 --- a/studio/backend/utils/openai_auto_switch_settings.py +++ b/studio/backend/utils/openai_auto_switch_settings.py @@ -433,9 +433,28 @@ def get_model_overrides() -> dict[str, dict]: def get_model_override(model_id: str) -> dict: - """The launch override applied when auto-switch loads ``model_id`` (or empty).""" - override = get_model_overrides().get(model_id) - return override if isinstance(override, dict) else {} + """The launch override applied when auto-switch loads ``model_id`` (or empty). + + Falls back to a case-insensitive match when nothing matches exactly. Repo ids + and quants are case-insensitive in practice ("Q4_K_M" and "q4_k_m" name one + file), and the browser normalizes them to lowercase before storing, so an + exact-only lookup misses entries written from that side. Exact still wins, and + an ambiguous fallback matches nothing, so two POSIX paths differing only in + case stay distinct. + """ + overrides = get_model_overrides() + override = overrides.get(model_id) + if isinstance(override, dict): + return override + if not isinstance(model_id, str): + return {} + folded = model_id.casefold() + matches = [ + value + for key, value in overrides.items() + if isinstance(key, str) and key.casefold() == folded and isinstance(value, dict) + ] + return matches[0] if len(matches) == 1 else {} def set_model_override( diff --git a/studio/frontend/src/features/api-monitor/api-monitor-page.tsx b/studio/frontend/src/features/api-monitor/api-monitor-page.tsx index 50b27672c5..ad84c77601 100644 --- a/studio/frontend/src/features/api-monitor/api-monitor-page.tsx +++ b/studio/frontend/src/features/api-monitor/api-monitor-page.tsx @@ -496,8 +496,11 @@ export function ApiMonitorPage(): ReactElement { if (!selectedIsMissing && lastFetchedRef.current === revision) { return; } - lastFetchedRef.current = revision; - requestDetail(selectedId_); + // Only remember the revision when a fetch really started; the in-flight + // guard can refuse, and recording it anyway skips that revision for good. + if (requestDetail(selectedId_)) { + lastFetchedRef.current = revision; + } }, [selectedId_, selectedUpdatedAt, selectedIsMissing, requestDetail]); // The desktop webview's origin is tauri://, not the API server, and the diff --git a/studio/frontend/src/features/api-monitor/use-api-monitor.ts b/studio/frontend/src/features/api-monitor/use-api-monitor.ts index 84be9e3084..3a40990120 100644 --- a/studio/frontend/src/features/api-monitor/use-api-monitor.ts +++ b/studio/frontend/src/features/api-monitor/use-api-monitor.ts @@ -168,7 +168,7 @@ interface UseApiMonitorResult { /** Full prompt/reply for entries the user expanded, keyed by entry id. */ details: Record; loadingDetails: ReadonlySet; - requestDetail: (id: string) => void; + requestDetail: (id: string) => boolean; } /** @@ -247,9 +247,12 @@ export function useApiMonitor({ }; }, [paused, intervalMs]); - const requestDetail = useCallback((id: string): void => { + // Returns whether a fetch actually started. Callers that remember "I have + // fetched revision N" must not record it when the in-flight guard turned them + // away, or that revision is skipped for good once updated_at stops moving. + const requestDetail = useCallback((id: string): boolean => { if (inFlightDetails.current.has(id)) { - return; + return false; } inFlightDetails.current.add(id); setLoadingDetails((prev) => new Set(prev).add(id)); @@ -275,6 +278,7 @@ export function useApiMonitor({ return next; }); }); + return true; }, []); const clear = useCallback(async (): Promise => { diff --git a/studio/frontend/src/features/hub/hub-page.tsx b/studio/frontend/src/features/hub/hub-page.tsx index 8175774689..50e35a690b 100644 --- a/studio/frontend/src/features/hub/hub-page.tsx +++ b/studio/frontend/src/features/hub/hub-page.tsx @@ -1226,8 +1226,12 @@ export function ModelsPage() { const [settingsTarget, setSettingsTarget] = useState( null, ); + // Bumped per open so a slow variant lookup for a row the user has moved on + // from cannot land on top of the row they actually chose. + const settingsOpenSeq = useRef(0); const openModelSettings = useCallback( async (row: CachedInventoryRow | LocalInventoryRow) => { + const openSeq = ++settingsOpenSeq.current; // loadId is what the loader accepts; repoId is only a display/API alias. const id = row.loadId; // Whether the loaded model is this row, under any of the names it goes by. @@ -1278,6 +1282,11 @@ export function ModelsPage() { } } } + // The variant lookup above is async, so a second row opened while it was + // pending would otherwise be overwritten by whichever call finished last. + if (settingsOpenSeq.current !== openSeq) { + return; + } const leaf = id.split(/[\\/]/).filter(Boolean).pop() ?? id; setSettingsTarget({ id, From 0a49dfd04704868038cfd26240f4b5c720578254 Mon Sep 17 00:00:00 2001 From: Unsloth Date: Sun, 26 Jul 2026 20:48:59 -0700 Subject: [PATCH 07/75] studio: only open the API monitor for real API clients CI caught this: the Chat UI Playwright run failed because the floating panel opened during an ordinary chat turn and its Expand button covered the composer's Send button. The panel opened because Studio's own chat goes through the same tracked endpoints as the OpenAI-compatible API, so any conversation looked like API traffic. That is wrong regardless of the click interception: this panel exists for when Unsloth is being used as an API server, not when someone is using Unsloth. Record on each monitor entry whether the caller authenticated with an sk-unsloth key rather than a UI session, and auto-open only for those. The discriminator already existed for other routes; this reuses it. --- studio/backend/core/inference/api_monitor.py | 7 +++++ studio/backend/routes/inference.py | 28 ++++++++++++++++++- studio/backend/tests/test_api_monitor.py | 25 +++++++++++++++++ .../backend/tests/test_openai_auto_switch.py | 17 +++++++++++ .../api-monitor/api-monitor-overlay.tsx | 7 ++++- .../frontend/src/features/chat/types/api.ts | 15 ++++++++-- 6 files changed, 94 insertions(+), 5 deletions(-) diff --git a/studio/backend/core/inference/api_monitor.py b/studio/backend/core/inference/api_monitor.py index e4aad4ade2..8b3cd02d85 100644 --- a/studio/backend/core/inference/api_monitor.py +++ b/studio/backend/core/inference/api_monitor.py @@ -41,6 +41,10 @@ class ApiMonitorEntry: started_at: float updated_at: float subject: Optional[str] = None + # True when the caller used an sk-unsloth key rather than a UI session. The + # floating panel only opens itself for these: Studio's own chat goes through + # the same endpoints, and popping the monitor open mid-chat is noise. + via_api_key: bool = False # Monotonic anchors so duration math survives wall-clock steps (NTP). started_monotonic: float = 0.0 finished_monotonic: Optional[float] = None @@ -70,6 +74,7 @@ class ApiMonitorEntry: "endpoint": self.endpoint, "method": self.method, "model": self.model, + "via_api_key": self.via_api_key, "prompt_preview": _trim(self.prompt, _PREVIEW_CHARS), "reply_preview": _trim(self.reply, _PREVIEW_CHARS), "prompt_truncated": len(self.prompt) > _PREVIEW_CHARS, @@ -107,6 +112,7 @@ class ApiMonitor: prompt: str, context_length: Optional[int] = None, subject: Optional[str] = None, + via_api_key: bool = False, ) -> str: now = time.time() entry = ApiMonitorEntry( @@ -121,6 +127,7 @@ class ApiMonitor: started_at = now, updated_at = now, subject = subject, + via_api_key = via_api_key, started_monotonic = time.monotonic(), context_length = context_length, ) diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.py index d804b3143d..fd1d3cc1cb 100644 --- a/studio/backend/routes/inference.py +++ b/studio/backend/routes/inference.py @@ -1754,7 +1754,24 @@ from core.inference.anthropic_compat import ( AnthropicStreamEmitter, AnthropicPassthroughEmitter, ) -from auth.authentication import get_current_subject +from auth.authentication import API_KEY_PREFIX, get_current_subject + + +def _request_used_api_key(request: Any) -> bool: + """True when this request authenticated with an sk-unsloth key. + + Studio's own chat hits these same endpoints with a session JWT, so this is + what separates "someone is using Unsloth as an API server" from "someone is + using Unsloth". + """ + try: + header = request.headers.get("authorization") or "" + except Exception: + return False + scheme, _, token = header.partition(" ") + return scheme.lower() == "bearer" and token.startswith(API_KEY_PREFIX) + + from state.tool_approvals import resolve_tool_decision from core.inference.key_exchange import decrypt_api_key @@ -7238,6 +7255,7 @@ async def _proxy_to_external_provider( if not getattr(request.state, "skip_api_monitor", False): monitor_id = api_monitor.start( endpoint = request.url.path, + via_api_key = _request_used_api_key(request), method = request.method, model = model, prompt = _monitor_prompt_from_messages(payload.messages), @@ -7798,6 +7816,7 @@ async def openai_chat_completions( if not getattr(request.state, "skip_api_monitor", False): tts_monitor_id = api_monitor.start( endpoint = request.url.path, + via_api_key = _request_used_api_key(request), method = request.method, model = model_label, prompt = _monitor_prompt_from_messages(payload.messages), @@ -7865,6 +7884,7 @@ async def openai_chat_completions( if not getattr(request.state, "skip_api_monitor", False): monitor_id = api_monitor.start( endpoint = request.url.path, + via_api_key = _request_used_api_key(request), method = request.method, model = model_name, prompt = _monitor_prompt_from_messages(payload.messages), @@ -8009,6 +8029,7 @@ async def openai_chat_completions( if monitor_id is None and not getattr(request.state, "skip_api_monitor", False): monitor_id = api_monitor.start( endpoint = request.url.path, + via_api_key = _request_used_api_key(request), method = request.method, model = model_name, prompt = _monitor_prompt_from_messages(payload.messages), @@ -10832,6 +10853,7 @@ async def openai_completions(request: Request, current_subject: str = Depends(ge prompt_text = _flatten_monitor_prompt(body.get("prompt", "")) monitor_id = api_monitor.start( endpoint = request.url.path, + via_api_key = _request_used_api_key(request), method = request.method, model = str(body.get("model") or _llama_public_model_id(llama_backend) or "default"), prompt = prompt_text, @@ -11043,6 +11065,7 @@ async def openai_embeddings(request: Request, current_subject: str = Depends(get if not getattr(request.state, "skip_api_monitor", False): monitor_id = api_monitor.start( endpoint = request.url.path, + via_api_key = _request_used_api_key(request), method = request.method, model = str(body.get("model") or _llama_public_model_id(llama_backend) or "default"), prompt = prompt_text, @@ -11637,6 +11660,7 @@ async def _responses_non_streaming( monitor_id = api_monitor.start( endpoint = getattr(getattr(request, "url", None), "path", "/v1/responses"), method = getattr(request, "method", "POST"), + via_api_key = _request_used_api_key(request), model = payload.model, prompt = _monitor_prompt_from_messages(messages), context_length = _monitor_context_length(), @@ -12828,6 +12852,7 @@ async def openai_responses( if not getattr(request.state, "skip_api_monitor", False): monitor_id = api_monitor.start( endpoint = request.url.path, + via_api_key = _request_used_api_key(request), method = request.method, model = payload.model, prompt = _monitor_prompt_from_messages(messages), @@ -13296,6 +13321,7 @@ async def anthropic_messages( monitor_id = api_monitor.start( endpoint = getattr(request_url, "path", "/v1/messages"), method = getattr(request, "method", "POST"), + via_api_key = _request_used_api_key(request), model = model_name, prompt = _monitor_prompt_from_messages(openai_messages), context_length = monitor_context_length, diff --git a/studio/backend/tests/test_api_monitor.py b/studio/backend/tests/test_api_monitor.py index a8d0e1e832..c1da17f7bd 100644 --- a/studio/backend/tests/test_api_monitor.py +++ b/studio/backend/tests/test_api_monitor.py @@ -289,3 +289,28 @@ def test_api_monitor_clear_is_scoped_to_one_subject(): # Passing no subject is the explicit "everything" path. monitor.clear() assert monitor.snapshot(subject = "bob") == [] + + +def test_api_monitor_records_whether_the_caller_used_an_api_key(): + # Studio's own chat hits these endpoints with a session JWT. The floating + # panel keys its auto-open off this flag, so mislabelling in-app chat as API + # traffic pops the panel over the composer mid-conversation. + monitor = ApiMonitor(max_entries = 4) + ui = monitor.start( + endpoint = "/api/inference/chat", + method = "POST", + model = "m", + prompt = "hi", + subject = "u", + ) + api = monitor.start( + endpoint = "/v1/chat/completions", + method = "POST", + model = "m", + prompt = "hi", + subject = "u", + via_api_key = True, + ) + by_id = {entry["id"]: entry for entry in monitor.snapshot(subject = "u")} + assert by_id[ui]["via_api_key"] is False + assert by_id[api]["via_api_key"] is True diff --git a/studio/backend/tests/test_openai_auto_switch.py b/studio/backend/tests/test_openai_auto_switch.py index aa20752aee..b66781375e 100644 --- a/studio/backend/tests/test_openai_auto_switch.py +++ b/studio/backend/tests/test_openai_auto_switch.py @@ -4247,3 +4247,20 @@ def test_ambiguous_case_fallback_matches_nothing(monkeypatch): settings.set_model_override("/models/foo.gguf", max_seq_length = 1024) settings.set_model_override("/models/FOO.gguf", max_seq_length = 8192) assert settings.get_model_override("/models/Foo.gguf") == {} + + +def test_request_used_api_key_distinguishes_key_from_session(): + from auth.authentication import API_KEY_PREFIX + + class _Req: + def __init__(self, header): + self.headers = {"authorization": header} if header else {} + + assert inference_route._request_used_api_key(_Req(f"Bearer {API_KEY_PREFIX}abc")) is True + assert inference_route._request_used_api_key(_Req(f"bearer {API_KEY_PREFIX}abc")) is True + assert inference_route._request_used_api_key(_Req("Bearer eyJhbGciOiJIUzI1NiJ9.x")) is False + assert inference_route._request_used_api_key(_Req("")) is False + assert inference_route._request_used_api_key(_Req(None)) is False + # A malformed request object must read as "not an API key", never raise, since + # this runs on the hot path of every tracked request. + assert inference_route._request_used_api_key(object()) is False diff --git a/studio/frontend/src/features/api-monitor/api-monitor-overlay.tsx b/studio/frontend/src/features/api-monitor/api-monitor-overlay.tsx index 88a4915b93..377694d7f5 100644 --- a/studio/frontend/src/features/api-monitor/api-monitor-overlay.tsx +++ b/studio/frontend/src/features/api-monitor/api-monitor-overlay.tsx @@ -172,7 +172,12 @@ export function ApiMonitorOverlay(): ReactElement | null { return; } const seen = seenIdsRef.current; - const hasNewTraffic = ids.some((id) => !seen.has(id)); + // Only API-key traffic counts. Studio's own chat goes through these same + // endpoints, and this panel is about serving other clients, not about the + // request the user is watching stream in front of them. + const hasNewTraffic = data.entries.some( + (entry) => entry.via_api_key && !seen.has(entry.id), + ); // Re-seed each poll so the set stays bounded by the server's ring buffer. seenIdsRef.current = new Set(ids); if (!hasNewTraffic) { diff --git a/studio/frontend/src/features/chat/types/api.ts b/studio/frontend/src/features/chat/types/api.ts index 6c3e919efe..00f13f87a7 100644 --- a/studio/frontend/src/features/chat/types/api.ts +++ b/studio/frontend/src/features/chat/types/api.ts @@ -169,7 +169,10 @@ export interface LoadModelResponse { max_context_length?: number | null; native_context_length?: number | null; supports_reasoning?: boolean; - reasoning_style?: "enable_thinking" | "reasoning_effort" | "enable_thinking_effort"; + reasoning_style?: + | "enable_thinking" + | "reasoning_effort" + | "enable_thinking_effort"; reasoning_effort_levels?: string[]; reasoning_always_on?: boolean; supports_preserve_thinking?: boolean; @@ -220,7 +223,10 @@ export interface InferenceStatusResponse { } | null; requires_trust_remote_code?: boolean; supports_reasoning?: boolean; - reasoning_style?: "enable_thinking" | "reasoning_effort" | "enable_thinking_effort"; + reasoning_style?: + | "enable_thinking" + | "reasoning_effort" + | "enable_thinking_effort"; reasoning_effort_levels?: string[]; reasoning_always_on?: boolean; supports_preserve_thinking?: boolean; @@ -268,6 +274,9 @@ export interface ApiMonitorEntry { model: string; prompt?: string; reply?: string; + // True when the caller used an API key rather than a UI session. The floating + // panel keys its auto-open off this so Studio's own chat does not pop it. + via_api_key: boolean; prompt_preview: string; reply_preview: string; prompt_truncated: boolean; @@ -389,7 +398,7 @@ export interface OpenAIChatCompletionsRequest { | "xhigh" | null; preserve_thinking?: boolean | null; - thinking?: {type: "disabled" | "enabled";} | null; + thinking?: { type: "disabled" | "enabled" } | null; enable_tools?: boolean | null; enabled_tools?: string[]; /** Local models + enable_tools only. */ From 9c722b9059aa9cd1edc6a7907c85e777b77d9998 Mon Sep 17 00:00:00 2001 From: Unsloth Date: Sun, 26 Jul 2026 23:15:51 -0700 Subject: [PATCH 08/75] studio: address fourth review round on per-model settings - Never case-fold a filesystem path when looking up an override. Two POSIX paths differing only in case are two different files, so a near miss must load defaults rather than replay another model's context and GPU pin. Repo ids still fold, which is what the migration needs. - Match a quant suffix against the loader's own quant pattern instead of a length heuristic. "/models/foo:bar.gguf" is one valid POSIX filename, and splitting it grafted /models/foo's launch flags onto an unrelated model. - Retry a load once without the saved gpu_ids when the loader rejects the pin. The pre-flight check cannot mirror every rule the loader applies (a Vulkan diffusion GGUF refuses GPU selection outright, and the rules move), so this stops chasing them one at a time: a stale placement preference must never be the reason a request cannot be served. - Make an explicit remove win over config fields sent in the same payload. - Seed only finished requests on the monitor's first snapshot. A request still running when Studio loads is traffic the user has not seen, not history. - Nudge the detail effect when the in-flight guard refuses a fetch. Nothing else changes its deps when the older fetch settles, so a terminal reply could stay truncated forever. - Invalidate pending row lookups when opening settings from a detail card, not just from a row. - Mark the covered detail pane inert so it leaves the focus order. - Refuse to open settings for a variant-required GGUF whose quant could not be resolved: the picker matches variants exactly and would never find the saved config, while the API falls back to the bare key and would apply it. - Mirror to the server only for GGUFs. The auto-switch resolver indexes GGUFs, so a safetensors config was being advertised as applied on API load when no API request could ever apply it. --- studio/backend/routes/inference.py | 39 ++++- studio/backend/routes/settings.py | 48 +++--- .../backend/tests/test_openai_auto_switch.py | 140 ++++++++++++++++++ .../utils/openai_auto_switch_settings.py | 13 ++ .../api-monitor/api-monitor-overlay.tsx | 13 +- .../features/api-monitor/api-monitor-page.tsx | 16 +- studio/frontend/src/features/hub/hub-page.tsx | 27 +++- .../api/migrate-model-overrides.ts | 5 +- .../components/model-config-page.tsx | 5 +- 9 files changed, 274 insertions(+), 32 deletions(-) diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.py index fd1d3cc1cb..db593c4bf0 100644 --- a/studio/backend/routes/inference.py +++ b/studio/backend/routes/inference.py @@ -3779,12 +3779,39 @@ async def _maybe_auto_switch_model( # Reuse the load impl so its dedup, tensor fallback, and threading # apply. Call the impl directly: we already hold the lifecycle gate # the /load route would otherwise take, so the route would deadlock. - await _load_model_impl( - LoadRequest(**load_kwargs), - fastapi_request, - current_subject, - current_request_counted = True, - ) + try: + await _load_model_impl( + LoadRequest(**load_kwargs), + fastapi_request, + current_subject, + current_request_counted = True, + ) + except HTTPException as exc: + # The pre-flight check above cannot mirror every rule the + # loader applies to gpu_ids (a Vulkan diffusion GGUF refuses + # GPU selection outright, and the rules move). Rather than + # duplicating them, retry once without the saved pin: a + # stale placement preference must never be the reason an + # API request cannot be served. + if not ( + exc.status_code == 400 + and load_kwargs.get("gpu_ids") + and "gpu" in str(exc.detail).lower() + ): + raise + logger.warning( + "Retrying %s without saved gpu_ids %s: %s", + override_id, + load_kwargs.get("gpu_ids"), + exc.detail, + ) + load_kwargs.pop("gpu_ids", None) + await _load_model_impl( + LoadRequest(**load_kwargs), + fastapi_request, + current_subject, + current_request_counted = True, + ) # Advertise the repo id (not the concrete load path) as the loaded # model's public id and override key for /v1/models and idle stash. get_llama_cpp_backend()._openai_advertised_id = override_id diff --git a/studio/backend/routes/settings.py b/studio/backend/routes/settings.py index 672d9941cc..f3b40bdcdb 100644 --- a/studio/backend/routes/settings.py +++ b/studio/backend/routes/settings.py @@ -126,8 +126,10 @@ class OpenAIAutoSwitchResponse(BaseModel): auto_unload_keep_kv: bool = DEFAULT_AUTO_UNLOAD_KEEP_KV -# A quant suffix, as modelOverrideKey builds it: no path separator and short. -# Guards against splitting "C:\\models\\x.gguf", where the colon is a drive letter. +# A quant suffix, as modelOverrideKey builds it. Matched against the loader's own +# quant pattern rather than a length heuristic: a POSIX path may legitimately +# contain a colon ("/models/foo:bar.gguf"), and treating "bar.gguf" as a quant +# would graft an unrelated model's launch flags onto this one. _MAX_VARIANT_SUFFIX_LEN = 64 # A local model's id is its filesystem path, optionally with a quant suffix, and @@ -342,11 +344,16 @@ def get_openai_auto_switch_overrides( def _bare_model_id(model_id: str) -> Optional[str]: """``repo`` for a ``repo:QUANT`` key, or None when there is no quant suffix.""" + from core.inference.llama_cpp import _GGUF_KNOWN_QUANT_RE + head, sep, tail = model_id.rpartition(":") if not sep or not head or not tail: return None if len(tail) > _MAX_VARIANT_SUFFIX_LEN or "/" in tail or "\\" in tail: return None + # Must actually look like a quant, not just like a short path segment. + if _GGUF_KNOWN_QUANT_RE.fullmatch(tail) is None: + return None return head @@ -382,21 +389,28 @@ def update_openai_auto_switch_override( if bare_id: requested_extra_args = get_model_override(bare_id).get("llama_extra_args") extra_args = validate_extra_args(requested_extra_args) - set_model_override( - payload.model_id, - llama_extra_args = extra_args, - max_seq_length = payload.max_seq_length, - custom_context_length = payload.custom_context_length, - kv_cache_dtype = payload.kv_cache_dtype, - speculative_type = payload.speculative_type, - spec_draft_n_max = payload.spec_draft_n_max, - tensor_parallel = payload.tensor_parallel, - chat_template_override = payload.chat_template_override, - gpu_memory_mode = payload.gpu_memory_mode, - gpu_layers = payload.gpu_layers, - n_cpu_moe = payload.n_cpu_moe, - gpu_ids = payload.gpu_ids, - ) + if payload.remove is True: + # An explicit remove wins over anything else in the payload: a stale + # form field must not turn "forget this model" into an update that + # keeps it. Only the explicit flag short-circuits; the legacy + # inferred path still just gates launch-flag carry-over. + set_model_override(payload.model_id, llama_extra_args = [], max_seq_length = None) + else: + set_model_override( + payload.model_id, + llama_extra_args = extra_args, + max_seq_length = payload.max_seq_length, + custom_context_length = payload.custom_context_length, + kv_cache_dtype = payload.kv_cache_dtype, + speculative_type = payload.speculative_type, + spec_draft_n_max = payload.spec_draft_n_max, + tensor_parallel = payload.tensor_parallel, + chat_template_override = payload.chat_template_override, + gpu_memory_mode = payload.gpu_memory_mode, + gpu_layers = payload.gpu_layers, + n_cpu_moe = payload.n_cpu_moe, + gpu_ids = payload.gpu_ids, + ) except ValueError as exc: raise log_and_http_error( exc, diff --git a/studio/backend/tests/test_openai_auto_switch.py b/studio/backend/tests/test_openai_auto_switch.py index b66781375e..bfd1754301 100644 --- a/studio/backend/tests/test_openai_auto_switch.py +++ b/studio/backend/tests/test_openai_auto_switch.py @@ -4264,3 +4264,143 @@ def test_request_used_api_key_distinguishes_key_from_session(): # A malformed request object must read as "not an API key", never raise, since # this runs on the hot path of every tracked request. assert inference_route._request_used_api_key(object()) is False + + +def test_case_fallback_never_applies_to_a_posix_path(monkeypatch): + # Two files that differ only in case are two different models on Linux, so a + # near miss must load defaults rather than another model's context and GPU pin. + _mock_override_store(monkeypatch) + settings.set_model_override("/models/foo.gguf", max_seq_length = 8192, gpu_ids = [1]) + assert settings.get_model_override("/models/Foo.gguf") == {} + assert settings.get_model_override("/models/foo.gguf")["max_seq_length"] == 8192 + + +def test_case_fallback_never_applies_to_a_windows_path(monkeypatch): + _mock_override_store(monkeypatch) + settings.set_model_override(r"C:\models\foo.gguf", max_seq_length = 8192) + assert settings.get_model_override(r"C:\models\FOO.gguf") == {} + + +def test_case_fallback_still_covers_repo_ids(monkeypatch): + # The migration case this fallback exists for. + _mock_override_store(monkeypatch) + settings.set_model_override("unsloth/qwen3-8b-gguf:q4_k_m", max_seq_length = 8192) + assert settings.get_model_override("unsloth/Qwen3-8B-GGUF:Q4_K_M")["max_seq_length"] == 8192 + + +def test_explicit_remove_wins_over_config_fields_in_the_same_payload(monkeypatch): + # remove is the operation discriminator, so a stale form field alongside it + # must not quietly turn "forget this model" into an update. + import routes.settings as settings_route + + _mock_override_store(monkeypatch) + settings.set_model_override("unsloth/B-GGUF", max_seq_length = 4096) + resp = settings_route.update_openai_auto_switch_override( + settings_route.ModelOverridePayload( + model_id = "unsloth/B-GGUF", remove = True, max_seq_length = 8192, tensor_parallel = True + ), + "tester", + ) + assert "unsloth/B-GGUF" not in resp.overrides + + +def test_posix_colon_in_a_path_is_not_treated_as_a_quant(monkeypatch): + # "/models/foo:bar.gguf" is one valid POSIX filename, not repo + quant. + # Splitting it would graft /models/foo's launch flags onto a different model. + import routes.settings as settings_route + + _mock_override_store(monkeypatch) + settings.set_model_override("/models/foo", llama_extra_args = ["--flash-attn"]) + resp = settings_route.update_openai_auto_switch_override( + settings_route.ModelOverridePayload(model_id = "/models/foo:bar.gguf", max_seq_length = 4096), + "tester", + ) + assert "llama_extra_args" not in resp.overrides["/models/foo:bar.gguf"] + + +def test_real_quant_suffix_on_a_path_still_carries_flags_over(monkeypatch): + import routes.settings as settings_route + + _mock_override_store(monkeypatch) + settings.set_model_override("/models/x.gguf", llama_extra_args = ["--flash-attn"]) + resp = settings_route.update_openai_auto_switch_override( + settings_route.ModelOverridePayload(model_id = "/models/x.gguf:Q4_K_M", max_seq_length = 4096), + "tester", + ) + assert resp.overrides["/models/x.gguf:Q4_K_M"]["llama_extra_args"] == ["--flash-attn"] + + +def test_load_retries_without_gpu_ids_when_the_loader_rejects_the_pin(monkeypatch): + # The pre-flight check cannot mirror every rule the loader applies (a Vulkan + # diffusion GGUF refuses GPU selection outright). A stale placement preference + # must never be the reason a request cannot be served. + from fastapi import HTTPException + + backend = _FakeBackend(None) + rec = _LoadRecorder(backend) + _wire( + monkeypatch, + enabled = True, + resolves_to = ("unsloth/B-GGUF", "Q4_K_M", "unsloth/B-GGUF"), + backend = backend, + recorder = rec, + ) + monkeypatch.setattr( + settings, "get_model_override", lambda mid: {"gpu_ids": [0], "max_seq_length": 4096} + ) + + async def _usable(ids): + return True + + monkeypatch.setattr(inference_route, "_override_gpu_ids_still_resolve", _usable) + + calls = {"n": 0} + + async def _load(request, *args, **kwargs): + calls["n"] += 1 + if calls["n"] == 1: + raise HTTPException( + status_code = 400, + detail = "GPU selection (gpu_ids) is not supported for a DiffusionGemma GGUF", + ) + return await rec(request, *args, **kwargs) + + monkeypatch.setattr(inference_route, "_load_model_impl", _load) + + _run_hook("unsloth/B-GGUF") + assert calls["n"] == 2 + served = rec.calls[-1] + assert not served.gpu_ids + assert served.max_seq_length == 4096 + + +def test_a_non_gpu_load_failure_is_not_retried(monkeypatch): + from fastapi import HTTPException + + backend = _FakeBackend(None) + rec = _LoadRecorder(backend) + _wire( + monkeypatch, + enabled = True, + resolves_to = ("unsloth/B-GGUF", "Q4_K_M", "unsloth/B-GGUF"), + backend = backend, + recorder = rec, + ) + monkeypatch.setattr(settings, "get_model_override", lambda mid: {"gpu_ids": [0]}) + + async def _usable(ids): + return True + + monkeypatch.setattr(inference_route, "_override_gpu_ids_still_resolve", _usable) + + calls = {"n": 0} + + async def _load(request, *args, **kwargs): + calls["n"] += 1 + raise HTTPException(status_code = 400, detail = "Corrupt GGUF header") + + monkeypatch.setattr(inference_route, "_load_model_impl", _load) + + with pytest.raises(HTTPException): + _run_hook("unsloth/B-GGUF") + assert calls["n"] == 1 diff --git a/studio/backend/utils/openai_auto_switch_settings.py b/studio/backend/utils/openai_auto_switch_settings.py index ca2a7c5d10..830d3cd3f7 100644 --- a/studio/backend/utils/openai_auto_switch_settings.py +++ b/studio/backend/utils/openai_auto_switch_settings.py @@ -426,6 +426,14 @@ def model_override_load_kwargs(override: dict[str, Any], *, is_gguf: bool) -> di return kwargs +def _looks_like_filesystem_path(model_id: str) -> bool: + """True for an absolute path id, as the ./models and LM Studio scanners emit.""" + if model_id.startswith(("/", "\\")): + return True + # Windows drive letter, e.g. "C:\models\x.gguf". + return len(model_id) >= 3 and model_id[1] == ":" and model_id[2] in ("\\", "/") + + def get_model_overrides() -> dict[str, dict]: """Per-model launch configs keyed by model id (see normalize_model_override).""" raw = _cached_setting(MODEL_OVERRIDES_SETTING_KEY, None) @@ -448,6 +456,11 @@ def get_model_override(model_id: str) -> dict: return override if not isinstance(model_id, str): return {} + # Only repo-style ids fold. A POSIX path is case-sensitive and names a + # different file, so matching "/models/Foo.gguf" against an entry saved for + # "/models/foo.gguf" would replay another model's context and GPU pin. + if _looks_like_filesystem_path(model_id): + return {} folded = model_id.casefold() matches = [ value diff --git a/studio/frontend/src/features/api-monitor/api-monitor-overlay.tsx b/studio/frontend/src/features/api-monitor/api-monitor-overlay.tsx index 377694d7f5..22128411c0 100644 --- a/studio/frontend/src/features/api-monitor/api-monitor-overlay.tsx +++ b/studio/frontend/src/features/api-monitor/api-monitor-overlay.tsx @@ -168,8 +168,17 @@ export function ApiMonitorOverlay(): ReactElement | null { const ids = data.entries.map((entry) => entry.id); if (!seededRef.current) { seededRef.current = true; - seenIdsRef.current = new Set(ids); - return; + // Seed finished requests only. A request that is still running when the + // first snapshot lands started while Studio was loading, so it is live + // traffic the user has not seen, not history to adopt silently. + seenIdsRef.current = new Set( + data.entries + .filter((entry) => entry.status !== "running") + .map((entry) => entry.id), + ); + if (!data.entries.some((e) => e.via_api_key && e.status === "running")) { + return; + } } const seen = seenIdsRef.current; // Only API-key traffic counts. Studio's own chat goes through these same diff --git a/studio/frontend/src/features/api-monitor/api-monitor-page.tsx b/studio/frontend/src/features/api-monitor/api-monitor-page.tsx index ad84c77601..30d9aad545 100644 --- a/studio/frontend/src/features/api-monitor/api-monitor-page.tsx +++ b/studio/frontend/src/features/api-monitor/api-monitor-page.tsx @@ -486,6 +486,7 @@ export function ApiMonitorPage(): ReactElement { const selectedUpdatedAt = selected?.updated_at ?? null; const selectedIsMissing = selectedId_ != null && details[selectedId_] == null; const lastFetchedRef = useRef(null); + const [retryTick, setRetryTick] = useState(0); useEffect(() => { if (selectedId_ == null) { return; @@ -500,8 +501,21 @@ export function ApiMonitorPage(): ReactElement { // guard can refuse, and recording it anyway skips that revision for good. if (requestDetail(selectedId_)) { lastFetchedRef.current = revision; + setRetryTick(0); + } else { + // Refused because an older fetch is still running. Nothing in this effect's + // deps will change when that one settles, so without a nudge a revision + // rejected here is never fetched, and a terminal reply stays truncated. + const timer = window.setTimeout(() => setRetryTick((n) => n + 1), 250); + return () => window.clearTimeout(timer); } - }, [selectedId_, selectedUpdatedAt, selectedIsMissing, requestDetail]); + }, [ + selectedId_, + selectedUpdatedAt, + selectedIsMissing, + requestDetail, + retryTick, + ]); // The desktop webview's origin is tauri://, not the API server, and the // packaged app picks its port dynamically. Same source as the Agents tab. diff --git a/studio/frontend/src/features/hub/hub-page.tsx b/studio/frontend/src/features/hub/hub-page.tsx index 50e35a690b..1fb98d3d58 100644 --- a/studio/frontend/src/features/hub/hub-page.tsx +++ b/studio/frontend/src/features/hub/hub-page.tsx @@ -23,6 +23,7 @@ import { } from "@/features/model-picker"; import { useDebouncedValue } from "@/hooks/use-debounced-value"; import { useGpuInfo } from "@/hooks/use-gpu-info"; +import { toast } from "@/lib/toast"; import { cn } from "@/lib/utils"; import { useNavigate, useSearch } from "@tanstack/react-router"; import { @@ -1276,11 +1277,20 @@ export function ModelsPage() { downloaded[0]?.quant ?? null; } catch { - // Offline or an unreadable cache: fall through with no variant. The - // settings page still works, it just cannot pin a specific quant. ggufVariant = null; } } + if (!ggufVariant) { + // A model that needs a quant cannot be configured without one: the + // picker matches variants exactly and would never find the saved + // config, while the API falls back to the bare key and would apply it. + // Opening the editor here would quietly create that mismatch. + toast.error("Couldn't determine which quant to configure.", { + description: + "Settings for this model are per quant. Check the connection or the model's cache, then try again.", + }); + return; + } } // The variant lookup above is async, so a second row opened while it was // pending would otherwise be overwritten by whichever call finished last. @@ -1349,6 +1359,9 @@ export function ModelsPage() { const openSelectedModelSettings = useCallback( (ggufVariant: string | null) => { if (!selectedModel) return; + // Share the sequence with openModelSettings: a row's variant lookup may + // still be pending, and it must not land on top of this one. + settingsOpenSeq.current += 1; const id = selectedModel.resource.runId; const leaf = id.split(/[\\/]/).filter(Boolean).pop() ?? id; setSettingsTarget({ @@ -1751,7 +1764,10 @@ export function ModelsPage() { {splitMode ? ( detailOpen ? ( -
+
+
{ return; } const local = listPerModelConfigs().filter( - (entry) => !isDefaultConfig(entry.config), + // A quant means it is a GGUF, which is the only thing API auto-switch + // resolves. Backfilling a safetensors config would claim an API behaviour + // that does not exist. + (entry) => entry.ggufVariant != null && !isDefaultConfig(entry.config), ); if (local.length === 0) { markRan(); diff --git a/studio/frontend/src/features/model-picker/components/model-config-page.tsx b/studio/frontend/src/features/model-picker/components/model-config-page.tsx index edf0cb96ea..91e71d8272 100644 --- a/studio/frontend/src/features/model-picker/components/model-config-page.tsx +++ b/studio/frontend/src/features/model-picker/components/model-config-page.tsx @@ -886,7 +886,10 @@ export function ModelConfigPage({ // Skipped when the local write failed (quota, a future-schema entry): the // browser and the server would otherwise permanently disagree about this // model, with no way for the user to tell which one the next load used. - if (!saveFailed) { + // GGUF only: the API auto-switch resolver indexes GGUFs, so mirroring a + // safetensors config to the server would advertise settings on the monitor's + // "applied on API load" list that no API request can ever apply. + if (!saveFailed && target.isGguf) { syncModelOverride( target.id, target.ggufVariant, From baf11a01f7030c01a761ca119b82a91729f735e9 Mon Sep 17 00:00:00 2001 From: Unsloth Date: Sun, 26 Jul 2026 23:31:51 -0700 Subject: [PATCH 09/75] studio: clear server overrides for models evicted from local storage Saving a config can push the browser's map over its entry or byte budget, at which point older models are silently dropped and the save still reports success. Their server-side overrides survived, so API loads kept applying settings that nothing in the UI showed any more and nothing could forget. savePerModelConfig now reports what it evicted, and the caller clears those server entries alongside the one it saved. Removal also had to resolve the way lookup does. Storage keys are normalized, so an evicted entry comes back lowercased while the server may hold the repo's real casing; deleting only the literal key would leave the entry a load still resolves to. Read and remove now share resolve_model_override_key, so what a load applies and what forgetting clears cannot disagree. Path ids still match exactly, so one file is never forgotten by clearing its case-variant neighbour. --- studio/backend/routes/settings.py | 7 ++++- .../backend/tests/test_openai_auto_switch.py | 31 +++++++++++++++++++ .../utils/openai_auto_switch_settings.py | 28 ++++++++++++----- .../components/model-config-page.tsx | 8 +++++ .../model-config/per-model-config.ts | 30 +++++++++++++++--- 5 files changed, 91 insertions(+), 13 deletions(-) diff --git a/studio/backend/routes/settings.py b/studio/backend/routes/settings.py index f3b40bdcdb..d0ff58d944 100644 --- a/studio/backend/routes/settings.py +++ b/studio/backend/routes/settings.py @@ -43,6 +43,7 @@ from utils.openai_auto_switch_settings import ( get_auto_unload_keep_kv, get_model_overrides, get_openai_auto_switch_enabled, + resolve_model_override_key, get_stored_auto_unload_idle_seconds, set_model_override, set_openai_auto_switch, @@ -394,7 +395,11 @@ def update_openai_auto_switch_override( # form field must not turn "forget this model" into an update that # keeps it. Only the explicit flag short-circuits; the legacy # inferred path still just gates launch-flag carry-over. - set_model_override(payload.model_id, llama_extra_args = [], max_seq_length = None) + # Remove the key a load would actually resolve to, not just the + # literal one sent: the browser normalizes casing before storing, so + # the two can differ and a stale entry would survive forgetting. + target_id = resolve_model_override_key(payload.model_id) or payload.model_id + set_model_override(target_id, llama_extra_args = [], max_seq_length = None) else: set_model_override( payload.model_id, diff --git a/studio/backend/tests/test_openai_auto_switch.py b/studio/backend/tests/test_openai_auto_switch.py index bfd1754301..b8720cb440 100644 --- a/studio/backend/tests/test_openai_auto_switch.py +++ b/studio/backend/tests/test_openai_auto_switch.py @@ -4404,3 +4404,34 @@ def test_a_non_gpu_load_failure_is_not_retried(monkeypatch): with pytest.raises(HTTPException): _run_hook("unsloth/B-GGUF") assert calls["n"] == 1 + + +def test_removal_clears_the_entry_a_load_would_actually_resolve(monkeypatch): + # The browser normalizes casing before storing, so a forget request can carry + # a different casing than the stored key. Removing only the literal key would + # leave the entry a load still resolves to, with no UI able to clear it. + import routes.settings as settings_route + + _mock_override_store(monkeypatch) + settings.set_model_override("unsloth/B-GGUF:Q4_K_M", max_seq_length = 8192) + assert settings.get_model_override("unsloth/b-gguf:q4_k_m")["max_seq_length"] == 8192 + + settings_route.update_openai_auto_switch_override( + settings_route.ModelOverridePayload(model_id = "unsloth/b-gguf:q4_k_m", remove = True), + "tester", + ) + assert settings.get_model_overrides() == {} + assert settings.get_model_override("unsloth/B-GGUF:Q4_K_M") == {} + + +def test_removal_of_a_path_still_only_touches_the_exact_key(monkeypatch): + import routes.settings as settings_route + + _mock_override_store(monkeypatch) + settings.set_model_override("/models/foo.gguf", max_seq_length = 8192) + settings_route.update_openai_auto_switch_override( + settings_route.ModelOverridePayload(model_id = "/models/Foo.gguf", remove = True), + "tester", + ) + # A different file must survive its neighbour being forgotten. + assert settings.get_model_override("/models/foo.gguf")["max_seq_length"] == 8192 diff --git a/studio/backend/utils/openai_auto_switch_settings.py b/studio/backend/utils/openai_auto_switch_settings.py index 830d3cd3f7..2a5844043d 100644 --- a/studio/backend/utils/openai_auto_switch_settings.py +++ b/studio/backend/utils/openai_auto_switch_settings.py @@ -450,24 +450,36 @@ def get_model_override(model_id: str) -> dict: an ambiguous fallback matches nothing, so two POSIX paths differing only in case stay distinct. """ - overrides = get_model_overrides() - override = overrides.get(model_id) - if isinstance(override, dict): - return override - if not isinstance(model_id, str): + key = resolve_model_override_key(model_id) + if key is None: return {} + override = get_model_overrides().get(key) + return override if isinstance(override, dict) else {} + + +def resolve_model_override_key(model_id: str) -> Optional[str]: + """The stored key an override lookup for ``model_id`` would actually hit. + + Shared by read and remove so "what a load applies" and "what forgetting this + model clears" can never disagree. + """ + overrides = get_model_overrides() + if isinstance(overrides.get(model_id), dict): + return model_id + if not isinstance(model_id, str): + return None # Only repo-style ids fold. A POSIX path is case-sensitive and names a # different file, so matching "/models/Foo.gguf" against an entry saved for # "/models/foo.gguf" would replay another model's context and GPU pin. if _looks_like_filesystem_path(model_id): - return {} + return None folded = model_id.casefold() matches = [ - value + key for key, value in overrides.items() if isinstance(key, str) and key.casefold() == folded and isinstance(value, dict) ] - return matches[0] if len(matches) == 1 else {} + return matches[0] if len(matches) == 1 else None def set_model_override( diff --git a/studio/frontend/src/features/model-picker/components/model-config-page.tsx b/studio/frontend/src/features/model-picker/components/model-config-page.tsx index 91e71d8272..ec02532217 100644 --- a/studio/frontend/src/features/model-picker/components/model-config-page.tsx +++ b/studio/frontend/src/features/model-picker/components/model-config-page.tsx @@ -869,11 +869,13 @@ export function ModelConfigPage({ isActiveModel && effectiveAtBaseline && rememberChanged; const defaultConfig = isDefaultConfig(effectiveRuntimeConfig); let saveFailed = false; + const evicted: { modelId: string; ggufVariant: string | null }[] = []; if (remember) { saveFailed = !savePerModelConfig( target.id, target.ggufVariant, effectiveRuntimeConfig, + evicted, ); } else { saveFailed = !deletePerModelConfig(target.id, target.ggufVariant); @@ -896,6 +898,12 @@ export function ModelConfigPage({ remember ? effectiveRuntimeConfig : null, ); } + // Saving can push the local map over budget and silently drop other models. + // Their server entries would otherwise keep being applied by API loads with + // nothing left in the UI showing them or able to forget them. + for (const dropped of evicted) { + syncModelOverride(dropped.modelId, dropped.ggufVariant, null); + } if (effectivePersistenceOnly) { if (saveFailed) { toast.error("Couldn't save settings for this model."); diff --git a/studio/frontend/src/features/model-picker/model-config/per-model-config.ts b/studio/frontend/src/features/model-picker/model-config/per-model-config.ts index 8511f3d58e..acc8efdefa 100644 --- a/studio/frontend/src/features/model-picker/model-config/per-model-config.ts +++ b/studio/frontend/src/features/model-picker/model-config/per-model-config.ts @@ -217,6 +217,7 @@ function serializedMapEntrySize(key: string, value: StoredMap[string]): number { function deleteOldestEvictableEntry( map: StoredMap, protectedKeys?: ReadonlySet, + evicted?: string[], ): { key: string; value: StoredMap[string] } | null { for (const key of Object.keys(map)) { // Never evict a future-schema entry an older client cannot interpret. @@ -228,6 +229,7 @@ function deleteOldestEvictableEntry( } const value = map[key]; delete map[key]; + evicted?.push(key); return { key, value }; } return null; @@ -236,17 +238,18 @@ function deleteOldestEvictableEntry( function enforceStorageBudget( map: StoredMap, protectedKeys?: ReadonlySet, + evicted?: string[], ): boolean { let entryCount = Object.keys(map).length; while (entryCount > MAX_ENTRIES) { - if (!deleteOldestEvictableEntry(map, protectedKeys)) { + if (!deleteOldestEvictableEntry(map, protectedKeys, evicted)) { return false; } entryCount -= 1; } let bytes = serializedMapSize(map); while (bytes > MAX_PER_MODEL_CONFIG_STORAGE_BYTES) { - const removed = deleteOldestEvictableEntry(map, protectedKeys); + const removed = deleteOldestEvictableEntry(map, protectedKeys, evicted); if (!removed) { return false; } @@ -623,6 +626,13 @@ export function savePerModelConfig( modelId: string, ggufVariant: string | null | undefined, config: PerModelConfig, + /** + * Receives models dropped to stay inside the storage budget. Eviction is + * silent and still reports success, so without this their server-side + * overrides would keep being applied by API loads with nothing in the UI + * still showing them or able to forget them. + */ + evicted?: { modelId: string; ggufVariant: string | null }[], ): boolean { if ( typeof config.chatTemplateOverride === "string" && @@ -646,10 +656,22 @@ export function savePerModelConfig( const [key] = storageKeysForModelVariant(modelId, ggufVariant); deleteConfigEntriesForModelVariant(map, modelId, ggufVariant); map[key] = toStoredConfig(normalized); - if (!enforceStorageBudget(map, new Set([key]))) { + const evictedKeys: string[] = []; + if (!enforceStorageBudget(map, new Set([key]), evictedKeys)) { return false; } - return writeMap(map); + const written = writeMap(map); + if (written && evicted) { + for (const evictedKey of evictedKeys) { + const id = modelIdFromStorageKey(evictedKey); + if (!id) { + continue; + } + const variant = ggufVariantFromStorageKey(evictedKey); + evicted.push({ modelId: id, ggufVariant: variant ? variant : null }); + } + } + return written; } /** Every saved per-model config, decoded back to the ids it was keyed by. */ From b6cb7568110a780a2097975e0fd99bb3104704ab Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 27 Jul 2026 12:41:12 +0000 Subject: [PATCH 10/75] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- studio/backend/tests/test_api_monitor.py | 2 ++ studio/backend/tests/test_openai_auto_switch.py | 2 ++ tests/studio/test_settings_compact_overflow_contract.py | 4 +--- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/studio/backend/tests/test_api_monitor.py b/studio/backend/tests/test_api_monitor.py index 93520c70d7..cfbf4bf2f6 100644 --- a/studio/backend/tests/test_api_monitor.py +++ b/studio/backend/tests/test_api_monitor.py @@ -314,6 +314,8 @@ def test_api_monitor_records_whether_the_caller_used_an_api_key(): by_id = {entry["id"]: entry for entry in monitor.snapshot(subject = "u")} assert by_id[ui]["via_api_key"] is False assert by_id[api]["via_api_key"] is True + + # ── model lifecycle rows (load / unload) ──────────────────────────── diff --git a/studio/backend/tests/test_openai_auto_switch.py b/studio/backend/tests/test_openai_auto_switch.py index a456338c56..bf4f41c818 100644 --- a/studio/backend/tests/test_openai_auto_switch.py +++ b/studio/backend/tests/test_openai_auto_switch.py @@ -4759,6 +4759,8 @@ def test_removal_of_a_path_still_only_touches_the_exact_key(monkeypatch): ) # A different file must survive its neighbour being forgotten. assert settings.get_model_override("/models/foo.gguf")["max_seq_length"] == 8192 + + def test_a_tag_that_names_no_quant_resolves_to_the_repo(monkeypatch): # A downloaded but unloaded GGUF asked for as org/model:latest missed the # resolver, so the switch path could not load it: with auto-download on it diff --git a/tests/studio/test_settings_compact_overflow_contract.py b/tests/studio/test_settings_compact_overflow_contract.py index 71b103f5b9..9f68d0b529 100644 --- a/tests/studio/test_settings_compact_overflow_contract.py +++ b/tests/studio/test_settings_compact_overflow_contract.py @@ -26,9 +26,7 @@ def test_api_monitor_entries_and_expanded_text_can_shrink(): assert '
' in source # Prompt and reply are unbounded user text: they must be height-capped, # scrollable, and wrap rather than stretch the pane. - assert ( - "max-h-72 overflow-auto whitespace-pre-wrap break-words rounded-lg bg-muted/50" in source - ) + assert "max-h-72 overflow-auto whitespace-pre-wrap break-words rounded-lg bg-muted/50" in source # A model id or path has no spaces to wrap on, so it needs break-all. assert 'className="min-w-0 break-all font-mono' in source From 6ad8cb6d478a857ecff161226bd053c0ad642529 Mon Sep 17 00:00:00 2001 From: Unsloth Date: Mon, 27 Jul 2026 05:44:13 -0700 Subject: [PATCH 11/75] studio: pin the GGUF mirror gate and eviction cleanup with contract tests Both were flagged in review with no test holding them in place. Also corrects the gpu_ids preflight docstring: it claims to mirror every rule the loader applies, which it deliberately does not, and the retry below is what covers the rest. --- studio/backend/routes/inference.py | 10 ++++-- tests/studio/test_model_picker_contracts.py | 38 +++++++++++++++++++-- 2 files changed, 42 insertions(+), 6 deletions(-) diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.py index 373371fb2f..4bd3774ff7 100644 --- a/studio/backend/routes/inference.py +++ b/studio/backend/routes/inference.py @@ -4578,9 +4578,13 @@ async def _override_gpu_ids_still_resolve(gpu_ids: List[int]) -> bool: """Whether a per-model GPU pin is usable on this machine right now. normalize_model_override cannot know the device list, so it stores whatever - was valid where the config was written. This is the load-time reconciliation, - and it has to make every check _resolve_gguf_gpu_ids_for_request would later - make, or the load 400s on the check this one skipped. + was valid where the config was written. This is the load-time reconciliation + for the device-availability rules, which are the ones that go stale. + + Deliberately not exhaustive: model-dependent rules (a Vulkan diffusion GGUF + refuses gpu_ids outright) need a ModelConfig this has no reason to build. + The caller's retry-without-the-pin covers those, and covers rules added + later, so a check missing here costs one extra attempt, not the load. """ try: from utils.hardware import DeviceType, get_device diff --git a/tests/studio/test_model_picker_contracts.py b/tests/studio/test_model_picker_contracts.py index e1aba66b1b..3370be7e6b 100644 --- a/tests/studio/test_model_picker_contracts.py +++ b/tests/studio/test_model_picker_contracts.py @@ -79,9 +79,9 @@ def test_rollback_restores_native_lease_expiry_with_token(): (which would look non-expiring and skip the expiry guard).""" src = _read("features/chat/hooks/use-chat-model-runtime.ts") assert "previousActiveNativePathExpiresAtMs" in src - assert re.search( - r"activeNativePathExpiresAtMs:\s*previousActiveNativePathToken", src - ), "rollback must restore the expiry alongside the token" + assert re.search(r"activeNativePathExpiresAtMs:\s*previousActiveNativePathToken", src), ( + "rollback must restore the expiry alongside the token" + ) def test_default_caches_keyed_on_inventory_version(): @@ -599,3 +599,35 @@ def test_vulkan_inference_devices_are_the_pickable_set(): # The torch fallback keeps its physical-only gate and the XPU ban. assert 'data?.device_backend !== "xpu" &&' in src assert 'physicalIndex: pinnableBackend && d.index_kind === "physical",' in src + + +def test_only_gguf_configs_are_mirrored_to_the_server(): + """The server override map is read by the OpenAI-compatible auto-switch, and + its resolver indexes GGUFs only. Mirroring a safetensors config there would + advertise settings on the monitor's applied-on-API-load list that no API + request can ever apply. The local write stays unconditional: the picker + loads safetensors models and must honour their config. + """ + src = " ".join(_read("features/model-picker/components/model-config-page.tsx").split()) + assert "if (!saveFailed && target.isGguf) { syncModelOverride(" in src + # The local save is not behind the same gate. + assert "if (remember) { saveFailed = !savePerModelConfig(" in src + + +def test_evicted_local_configs_drop_their_server_overrides(): + """savePerModelConfig evicts older models when the map exceeds its budget. + Those models keep a server override that API loads still apply, with nothing + left in the UI showing it or able to forget it, so eviction has to propagate. + """ + src = " ".join(_read("features/model-picker/components/model-config-page.tsx").split()) + assert "const evicted: { modelId: string; ggufVariant: string | null }[] = [];" in src + assert ( + "for (const dropped of evicted) { syncModelOverride(dropped.modelId, dropped.ggufVariant, null); }" + in src + ) + + # The eviction path has to actually report what it dropped, decoded back into + # a model id and variant rather than the normalized storage key. + store = " ".join(_read("features/model-picker/model-config/per-model-config.ts").split()) + assert "evicted?: { modelId: string; ggufVariant: string | null }[]" in store + assert "modelIdFromStorageKey(" in store and "ggufVariantFromStorageKey(" in store From 33faff1923f3629d2ae844d9f8f659cc0d6e5358 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 27 Jul 2026 12:47:37 +0000 Subject: [PATCH 12/75] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- tests/studio/test_model_picker_contracts.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/studio/test_model_picker_contracts.py b/tests/studio/test_model_picker_contracts.py index 3370be7e6b..e874cbfa5c 100644 --- a/tests/studio/test_model_picker_contracts.py +++ b/tests/studio/test_model_picker_contracts.py @@ -79,9 +79,9 @@ def test_rollback_restores_native_lease_expiry_with_token(): (which would look non-expiring and skip the expiry guard).""" src = _read("features/chat/hooks/use-chat-model-runtime.ts") assert "previousActiveNativePathExpiresAtMs" in src - assert re.search(r"activeNativePathExpiresAtMs:\s*previousActiveNativePathToken", src), ( - "rollback must restore the expiry alongside the token" - ) + assert re.search( + r"activeNativePathExpiresAtMs:\s*previousActiveNativePathToken", src + ), "rollback must restore the expiry alongside the token" def test_default_caches_keyed_on_inventory_version(): From f39fd0e408f7c7cb8bff75198c32cdee15212dad Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Tue, 28 Jul 2026 13:16:39 +0000 Subject: [PATCH 13/75] Resolve the override key on save and match standalone gguf settings The remove branch already resolved the key a load would use, but the save branch wrote payload.model_id literally. The browser normalizes casing before storing, so a backfilled key and a later UI save left two entries for one model; with two equivalent keys present resolve_model_override_key finds no unique match, so any third casing resolved to no override at all and the model silently loaded with defaults. A standalone .gguf gets variants=() from the resolver, so variant is None and only the bare ids were tried. The picker keys the same file by the quant label it derives from the filename, which is never empty, so those settings lived under :LABEL and nothing reached them. Try the filename-derived key after the variant-qualified ones and before the bare ones, so older bare entries still work. --- studio/backend/routes/inference.py | 12 ++++++++++++ studio/backend/routes/settings.py | 8 +++++++- .../backend/tests/test_openai_auto_switch.py | 19 +++++++++++++++++++ 3 files changed, 38 insertions(+), 1 deletion(-) diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.py index 4bd3774ff7..5e56880c3d 100644 --- a/studio/backend/routes/inference.py +++ b/studio/backend/routes/inference.py @@ -4291,10 +4291,22 @@ async def _maybe_auto_switch_model( # different configs), then the bare ids. Both the advertised # repo id and the concrete load path are tried: a local folder # or a non-active HF cache is configured against its path. + # A standalone .gguf needs no quant sub-selection, so the + # resolver reports variant=None for it. The picker still + # keys its config by the quant label it derives from the + # filename (LocalModelInfo.format_variant), which is never + # empty, so those settings live under ":LABEL" and no + # bare key would ever reach them. + file_variant = None + if not variant and target_id.lower().endswith(".gguf"): + from hub.utils.gguf import extract_quant_label + + file_variant = extract_quant_label(os.path.basename(target_id)) override = {} for override_key in ( f"{override_id}:{variant}" if variant else None, f"{target_id}:{variant}" if variant else None, + f"{target_id}:{file_variant}" if file_variant else None, override_id, target_id, ): diff --git a/studio/backend/routes/settings.py b/studio/backend/routes/settings.py index 052c643519..cc8844ad4d 100644 --- a/studio/backend/routes/settings.py +++ b/studio/backend/routes/settings.py @@ -411,8 +411,14 @@ def update_openai_auto_switch_override( target_id = resolve_model_override_key(payload.model_id) or payload.model_id set_model_override(target_id, llama_extra_args = [], max_seq_length = None) else: + # Save under the key a load would resolve to, for the same reason the + # removal branch does. The browser normalizes casing before storing, + # so saving the literal id leaves a second entry for one model, and + # two equivalent keys make every other casing ambiguous: the lookup + # then matches neither and the model silently loses its settings. + target_id = resolve_model_override_key(payload.model_id) or payload.model_id set_model_override( - payload.model_id, + target_id, llama_extra_args = extra_args, max_seq_length = payload.max_seq_length, custom_context_length = payload.custom_context_length, diff --git a/studio/backend/tests/test_openai_auto_switch.py b/studio/backend/tests/test_openai_auto_switch.py index bf4f41c818..55484e67d3 100644 --- a/studio/backend/tests/test_openai_auto_switch.py +++ b/studio/backend/tests/test_openai_auto_switch.py @@ -4748,6 +4748,25 @@ def test_removal_clears_the_entry_a_load_would_actually_resolve(monkeypatch): assert settings.get_model_override("unsloth/B-GGUF:Q4_K_M") == {} +def test_save_updates_the_existing_case_variant_instead_of_forking_it(monkeypatch): + # The backfill stores normalized (lowercase) keys while a later UI save carries + # the catalog's casing. Writing that literally leaves two keys for one model, + # and with two equivalent keys present any third casing resolves ambiguously, + # so the model silently loses every saved setting on the API path. + import routes.settings as settings_route + + _mock_override_store(monkeypatch) + settings.set_model_override("unsloth/b-gguf:q4_k_m", max_seq_length = 8192) + settings_route.update_openai_auto_switch_override( + settings_route.ModelOverridePayload( + model_id = "unsloth/B-GGUF:Q4_K_M", max_seq_length = 4096 + ), + "tester", + ) + assert list(settings.get_model_overrides()) == ["unsloth/b-gguf:q4_k_m"] + assert settings.get_model_override("Unsloth/B-GGUF:Q4_K_M")["max_seq_length"] == 4096 + + def test_removal_of_a_path_still_only_touches_the_exact_key(monkeypatch): import routes.settings as settings_route From 29a0f2ef0802911df53932034b96b66b9267906e Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 28 Jul 2026 13:23:35 +0000 Subject: [PATCH 14/75] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- studio/backend/routes/inference.py | 1 - studio/backend/tests/test_api_monitor.py | 2 ++ studio/backend/tests/test_openai_auto_switch.py | 4 +--- 3 files changed, 3 insertions(+), 4 deletions(-) diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.py index 7e25e2e4c5..bc9b430619 100644 --- a/studio/backend/routes/inference.py +++ b/studio/backend/routes/inference.py @@ -4371,7 +4371,6 @@ async def _maybe_auto_switch_model( file_variant = None if not variant and target_id.lower().endswith(".gguf"): from hub.utils.gguf import extract_quant_label - file_variant = extract_quant_label(os.path.basename(target_id)) override = {} for override_key in ( diff --git a/studio/backend/tests/test_api_monitor.py b/studio/backend/tests/test_api_monitor.py index be959a7dbd..5e5e841263 100644 --- a/studio/backend/tests/test_api_monitor.py +++ b/studio/backend/tests/test_api_monitor.py @@ -314,6 +314,8 @@ def test_api_monitor_records_whether_the_caller_used_an_api_key(): by_id = {entry["id"]: entry for entry in monitor.snapshot(subject = "u")} assert by_id[ui]["via_api_key"] is False assert by_id[api]["via_api_key"] is True + + def test_api_monitor_disabled_is_noop(): monitor = ApiMonitor(max_entries = 3, enabled = False) diff --git a/studio/backend/tests/test_openai_auto_switch.py b/studio/backend/tests/test_openai_auto_switch.py index 9d038aceec..4782816e53 100644 --- a/studio/backend/tests/test_openai_auto_switch.py +++ b/studio/backend/tests/test_openai_auto_switch.py @@ -4763,9 +4763,7 @@ def test_save_updates_the_existing_case_variant_instead_of_forking_it(monkeypatc _mock_override_store(monkeypatch) settings.set_model_override("unsloth/b-gguf:q4_k_m", max_seq_length = 8192) settings_route.update_openai_auto_switch_override( - settings_route.ModelOverridePayload( - model_id = "unsloth/B-GGUF:Q4_K_M", max_seq_length = 4096 - ), + settings_route.ModelOverridePayload(model_id = "unsloth/B-GGUF:Q4_K_M", max_seq_length = 4096), "tester", ) assert list(settings.get_model_overrides()) == ["unsloth/b-gguf:q4_k_m"] From f026c640c54b45348dfd148631413c8a5799088a Mon Sep 17 00:00:00 2001 From: danielhanchen Date: Tue, 28 Jul 2026 14:15:13 +0000 Subject: [PATCH 15/75] Match server overrides by identity during the one-time backfill app_settings carries no schema version, so an install that predates identity normalization holds rows keyed by whatever id was typed, such as Unsloth/Repo-GGUF:Q4_K_M, while this browser only ever stores the folded form. The exact property lookup therefore reported "not on the server" for a row that is, and the backfill PUT over it, replacing server settings the file documents as the newer authority. The migration runs once on every existing profile, so this lands on exactly the upgrades it was written to protect. Fold both sides before comparing, splitting on the last colon because a quant label never contains one. A repo id and a Windows path fold, a POSIX path does not, which is the same rule the backend resolves by. Verified with the real module under node: the legacy-casing, variant-casing and Windows-path cases go from overwriting to skipping, a second run stays clean, and a genuinely new model, a different quant of the same repo, a POSIX path differing only in case, and a bare legacy key all still migrate. --- .../api/migrate-model-overrides.ts | 37 ++++++++++++++++++- tests/studio/test_model_picker_contracts.py | 20 ++++++++++ 2 files changed, 55 insertions(+), 2 deletions(-) diff --git a/studio/frontend/src/features/model-picker/api/migrate-model-overrides.ts b/studio/frontend/src/features/model-picker/api/migrate-model-overrides.ts index 8f6af14e9e..b5f908a109 100644 --- a/studio/frontend/src/features/model-picker/api/migrate-model-overrides.ts +++ b/studio/frontend/src/features/model-picker/api/migrate-model-overrides.ts @@ -8,6 +8,10 @@ // showing as remembered in the UI while an API load quietly uses app defaults, // which is the exact bug the server-side map exists to fix. +import { + normalizeGgufVariantIdentity, + normalizeModelIdentity, +} from "../model-config/model-identity"; import { isDefaultConfig, listPerModelConfigs, @@ -37,6 +41,28 @@ function markRan(): void { } } +/** + * A server key under the same identity this browser stores. + * + * `app_settings` has no schema version and holds whatever id was current when + * the row was written, so an install that predates identity normalization has + * keys like `Unsloth/Repo-GGUF:Q4_K_M` while this browser only ever produces + * the folded form. The backend resolves the two to one model, so an exact + * property lookup would report "not on the server" for a row that is, and the + * backfill would overwrite it. Variants never contain a colon, so the last one + * splits the key; a repo id folds and a POSIX path deliberately does not. + */ +function normalizedOverrideKey(key: string): string { + const separator = key.lastIndexOf(":"); + if (separator < 0) { + return modelOverrideKey(normalizeModelIdentity(key)); + } + return modelOverrideKey( + normalizeModelIdentity(key.slice(0, separator)), + normalizeGgufVariantIdentity(key.slice(separator + 1)), + ); +} + /** * Push local configs the server has never seen. Never deletes and never * overwrites: an entry already on the server is the newer authority, and losing @@ -66,10 +92,17 @@ export async function backfillModelOverrides(): Promise { return; } + const known = new Set(Object.keys(existing).map(normalizedOverrideKey)); + let failed = false; for (const entry of local) { - const key = modelOverrideKey(entry.modelId, entry.ggufVariant); - if (existing[key]) { + // Folded on this side too: a v2 storage key already holds the normalized + // identity, but the older `id::variant` keys this browser still reads back + // hold whatever casing was typed. + const key = normalizedOverrideKey( + modelOverrideKey(entry.modelId, entry.ggufVariant), + ); + if (known.has(key)) { continue; } try { diff --git a/tests/studio/test_model_picker_contracts.py b/tests/studio/test_model_picker_contracts.py index 3634185292..3fa8abb764 100644 --- a/tests/studio/test_model_picker_contracts.py +++ b/tests/studio/test_model_picker_contracts.py @@ -682,3 +682,23 @@ def test_evicted_local_configs_drop_their_server_overrides(): store = " ".join(_read("features/model-picker/model-config/per-model-config.ts").split()) assert "evicted?: { modelId: string; ggufVariant: string | null }[]" in store assert "modelIdFromStorageKey(" in store and "ggufVariantFromStorageKey(" in store + + +def test_backfill_compares_server_keys_by_normalized_identity(): + """app_settings has no schema version, so an install predating identity + normalization holds rows keyed by whatever id was typed, e.g. + "Unsloth/Repo-GGUF:Q4_K_M". This browser only ever stores the folded form, + so an exact property lookup reports "not on the server" for a row that is, + and the one-time backfill then overwrites settings it documents as the newer + authority. The comparison has to fold the same way the backend resolves. + """ + src = " ".join(_read("features/model-picker/api/migrate-model-overrides.ts").split()) + assert "function normalizedOverrideKey(" in src + # Folded on both sides: the older `id::variant` local keys are not folded. + assert "const known = new Set(Object.keys(existing).map(normalizedOverrideKey));" in src + assert "if (known.has(key)) { continue; }" in src + # A variant never holds a colon, so the last one splits the key. Splitting on + # the first would cut a Windows drive letter off every path id. + assert "key.lastIndexOf(\":\")" in src + # Repo ids fold and POSIX paths do not, which is exactly what these do. + assert "normalizeModelIdentity(" in src and "normalizeGgufVariantIdentity(" in src From 082e0dd9d1822a828877b300869301f19a85899e Mon Sep 17 00:00:00 2001 From: danielhanchen Date: Tue, 28 Jul 2026 14:18:15 +0000 Subject: [PATCH 16/75] Fold case-insensitive path ids the way the browser already does resolve_model_override_key refused the case fallback for every filesystem path, but only a POSIX path is case-sensitive. A Windows drive path, a UNC share and a WSL drive path each name one file whatever the casing, and the browser folds exactly those three before storing. A Windows user's migrated entry was therefore keyed lowercase while an API auto-switch resolved the same file with its on-disk casing, so the lookup missed and the saved launch flags silently stopped applying until the settings were saved again. Fold those three shapes here too, normalizing the separator as the browser does so C:/Models/Foo.gguf and c:\models\foo.gguf agree. POSIX stays case-sensitive, /mnt/data stays an ordinary mount rather than a WSL drive, and an ambiguous fold still matches nothing so a load takes defaults instead of guessing. The existing Windows test asserted the opposite. It carried no rationale, unlike its POSIX sibling, and get_model_override's docstring already scopes the rule to POSIX, so it read as an over-generalisation of the POSIX case. --- .../backend/tests/test_openai_auto_switch.py | 37 +++++++++++- .../utils/openai_auto_switch_settings.py | 58 +++++++++++++++++-- 2 files changed, 86 insertions(+), 9 deletions(-) diff --git a/studio/backend/tests/test_openai_auto_switch.py b/studio/backend/tests/test_openai_auto_switch.py index 4782816e53..2119e26164 100644 --- a/studio/backend/tests/test_openai_auto_switch.py +++ b/studio/backend/tests/test_openai_auto_switch.py @@ -4604,10 +4604,41 @@ def test_case_fallback_never_applies_to_a_posix_path(monkeypatch): assert settings.get_model_override("/models/foo.gguf")["max_seq_length"] == 8192 -def test_case_fallback_never_applies_to_a_windows_path(monkeypatch): +def test_case_fallback_does_apply_to_a_windows_path(monkeypatch): + # NTFS is case-insensitive, so these name one file, and the browser folds + # drive paths before storing. Treating them as two models would leave every + # migrated Windows entry unreachable until the user saved it again, which is + # the opposite of the POSIX rule and for the opposite reason. The separator + # is interchangeable there too. _mock_override_store(monkeypatch) - settings.set_model_override(r"C:\models\foo.gguf", max_seq_length = 8192) - assert settings.get_model_override(r"C:\models\FOO.gguf") == {} + settings.set_model_override(r"c:\models\foo.gguf", max_seq_length = 8192) + assert settings.get_model_override(r"C:\models\FOO.gguf")["max_seq_length"] == 8192 + assert settings.get_model_override("C:/Models/Foo.gguf")["max_seq_length"] == 8192 + + +def test_case_fallback_applies_to_unc_and_wsl_drive_paths(monkeypatch): + _mock_override_store(monkeypatch) + settings.set_model_override(r"\\server\share\foo.gguf", max_seq_length = 4096) + settings.set_model_override("/mnt/c/models/bar.gguf", max_seq_length = 2048) + assert settings.get_model_override(r"\\Server\Share\FOO.gguf")["max_seq_length"] == 4096 + assert settings.get_model_override("/mnt/C/Models/Bar.gguf")["max_seq_length"] == 2048 + + +def test_a_plain_posix_path_under_mnt_stays_case_sensitive(monkeypatch): + # Only /mnt/ is a WSL drive mount. /mnt/data is an ordinary Linux + # mount point and stays case-sensitive like any other POSIX path. + _mock_override_store(monkeypatch) + settings.set_model_override("/mnt/data/models/foo.gguf", max_seq_length = 8192) + assert settings.get_model_override("/mnt/data/models/Foo.gguf") == {} + + +def test_an_ambiguous_windows_case_fallback_still_matches_nothing(monkeypatch): + # Two stored keys folding to one leaves no single answer, so the load takes + # defaults rather than guessing between them. + _mock_override_store(monkeypatch) + settings.set_model_override(r"c:\models\foo.gguf", max_seq_length = 1024) + settings.set_model_override("C:/models/FOO.gguf", max_seq_length = 8192) + assert settings.get_model_override(r"C:\Models\Foo.gguf") == {} def test_case_fallback_still_covers_repo_ids(monkeypatch): diff --git a/studio/backend/utils/openai_auto_switch_settings.py b/studio/backend/utils/openai_auto_switch_settings.py index 57d65b98f8..24fa7704f0 100644 --- a/studio/backend/utils/openai_auto_switch_settings.py +++ b/studio/backend/utils/openai_auto_switch_settings.py @@ -27,6 +27,7 @@ per-request hot path; writes invalidate the cache. from __future__ import annotations import os +import re import threading import time from typing import Any, Optional @@ -470,6 +471,37 @@ def _looks_like_filesystem_path(model_id: str) -> bool: return len(model_id) >= 3 and model_id[1] == ":" and model_id[2] in ("\\", "/") +# The three path shapes whose filesystem is case-insensitive, matching the rule +# the browser applies in features/hub/lib/model-identity.ts. Kept in step with +# it: the browser folds these before storing, so the two sides have to agree on +# which paths fold or a stored key becomes unreachable. +_WINDOWS_DRIVE_PATH = re.compile(r"^[A-Za-z]:[\\/]") +_WSL_DRIVE_PATH = re.compile(r"^/mnt/[A-Za-z](?:/|$)") + + +def _fold_case_insensitive_path(model_id: str) -> Optional[str]: + """``model_id`` folded for comparison, or None when the path is case-sensitive. + + A Windows drive path, a UNC share and a WSL drive path all name one file + whatever the casing, and the separator is interchangeable on Windows. A + POSIX path is not: folding "/models/Foo.gguf" onto "/models/foo.gguf" would + replay another model's context and GPU pin. + """ + slashed = model_id.replace("\\", "/") + if _WINDOWS_DRIVE_PATH.match(model_id): + minimum = 3 + elif slashed.startswith("//"): + minimum = 2 + elif _WSL_DRIVE_PATH.match(slashed): + minimum = 6 + else: + return None + trimmed = slashed + while len(trimmed) > minimum and trimmed.endswith("/"): + trimmed = trimmed[:-1] + return trimmed.casefold() + + def get_model_overrides() -> dict[str, dict]: """Per-model launch configs keyed by model id (see normalize_model_override).""" raw = _cached_setting(MODEL_OVERRIDES_SETTING_KEY, None) @@ -504,16 +536,30 @@ def resolve_model_override_key(model_id: str) -> Optional[str]: return model_id if not isinstance(model_id, str): return None - # Only repo-style ids fold. A POSIX path is case-sensitive and names a - # different file, so matching "/models/Foo.gguf" against an entry saved for - # "/models/foo.gguf" would replay another model's context and GPU pin. + # A POSIX path is case-sensitive and names a different file, so matching + # "/models/Foo.gguf" against an entry saved for "/models/foo.gguf" would + # replay another model's context and GPU pin. A Windows drive path, a UNC + # share and a WSL drive path are not case-sensitive, and the browser folds + # exactly those before storing, so refusing to fold them here would leave + # every migrated Windows entry unreachable until the user saved it again. if _looks_like_filesystem_path(model_id): - return None - folded = model_id.casefold() + folded = _fold_case_insensitive_path(model_id) + if folded is None: + return None + + def fold(key: str) -> Optional[str]: + return _fold_case_insensitive_path(key) + else: + folded = model_id.casefold() + + def fold(key: str) -> Optional[str]: + # A path never folds onto a repo id: the shapes cannot collide. + return None if _looks_like_filesystem_path(key) else key.casefold() + matches = [ key for key, value in overrides.items() - if isinstance(key, str) and key.casefold() == folded and isinstance(value, dict) + if isinstance(key, str) and fold(key) == folded and isinstance(value, dict) ] return matches[0] if len(matches) == 1 else None From bf8ef7fd8c24bbb42550aa1ad86743ebbb5dbd34 Mon Sep 17 00:00:00 2001 From: danielhanchen Date: Tue, 28 Jul 2026 14:21:39 +0000 Subject: [PATCH 17/75] Close clearApiMonitor The merge that brought main into this branch dropped the closing brace, so the function body ran straight into the interface declared below it and the frontend did not compile at all: tsc reports TS1005 at the end of the file and vite fails the build. Reproduced against the pushed head and clean with the brace restored. --- studio/frontend/src/features/chat/api/chat-api.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/studio/frontend/src/features/chat/api/chat-api.ts b/studio/frontend/src/features/chat/api/chat-api.ts index cf74779ee7..c36f7a88aa 100644 --- a/studio/frontend/src/features/chat/api/chat-api.ts +++ b/studio/frontend/src/features/chat/api/chat-api.ts @@ -134,6 +134,8 @@ export async function clearApiMonitor(): Promise { method: "DELETE", }); await parseJsonOrThrow<{ cleared: boolean }>(response); +} + export interface ActiveGenerationsResponse { count: number; /** Conversations with a generation in flight. Shorter than `count` when a From f8683d789769d81562670293bd46fdcf8c928fc8 Mon Sep 17 00:00:00 2001 From: danielhanchen Date: Tue, 28 Jul 2026 14:21:50 +0000 Subject: [PATCH 18/75] Carry legacy flags across a scanner-derived GGUF variant label A .gguf whose filename holds no recognizable quant token still gets a label from the scanner, which falls back to the filename stem, so the UI stores keys like "/models/custom.gguf:custom". _bare_model_id accepted only known quant tokens, so the first per-quant save did not carry over llama_extra_args stored under the bare id, and auto-switch prefers the qualified entry, so those flags were silently dropped with no UI able to restore them. Accept a suffix that is exactly the label the scanner derives for that filename. Requiring the head to be a .gguf and the suffix to match exactly is what keeps an arbitrary colon-containing POSIX path out: "/models/foo:bar.gguf" splits to a head that is not a .gguf. The filename is taken by splitting on both separators, since a "C:\\..." key is written on Windows but may be read back by a backend that is not. --- studio/backend/routes/settings.py | 26 +++++++--- .../backend/tests/test_openai_auto_switch.py | 51 +++++++++++++++++++ 2 files changed, 71 insertions(+), 6 deletions(-) diff --git a/studio/backend/routes/settings.py b/studio/backend/routes/settings.py index cc8844ad4d..3e4409ae3b 100644 --- a/studio/backend/routes/settings.py +++ b/studio/backend/routes/settings.py @@ -356,16 +356,30 @@ def get_openai_auto_switch_overrides( def _bare_model_id(model_id: str) -> Optional[str]: """``repo`` for a ``repo:QUANT`` key, or None when there is no quant suffix.""" from core.inference.llama_cpp import _GGUF_KNOWN_QUANT_RE + from hub.utils.gguf import extract_quant_label head, sep, tail = model_id.rpartition(":") if not sep or not head or not tail: return None - if len(tail) > _MAX_VARIANT_SUFFIX_LEN or "/" in tail or "\\" in tail: - return None - # Must actually look like a quant, not just like a short path segment. - if _GGUF_KNOWN_QUANT_RE.fullmatch(tail) is None: - return None - return head + if "/" not in tail and "\\" not in tail and len(tail) <= _MAX_VARIANT_SUFFIX_LEN: + # Must actually look like a quant, not just like a short path segment. + if _GGUF_KNOWN_QUANT_RE.fullmatch(tail) is not None: + return head + # A .gguf with no recognizable quant token is still labelled by the scanner, + # which falls back to the filename stem, so the UI stores keys like + # "/models/custom.gguf:custom". Refusing those dropped the bare entry's + # legacy flags on the first save, and auto-switch then prefers the qualified + # entry, so nothing could restore them. Requiring the suffix to be exactly + # the label the scanner derives for this filename is what keeps an arbitrary + # colon-containing POSIX path out: "/models/foo:bar.gguf" splits to a head + # that is not a .gguf at all. + # Split on both separators rather than os.path.basename: a "C:\..." key is + # written on Windows but may be read back by a backend that is not, and + # there a backslash is an ordinary filename character. + filename = head.replace("\\", "/").rsplit("/", 1)[-1] + if head.lower().endswith(".gguf") and tail == extract_quant_label(filename): + return head + return None @router.put("/openai-auto-switch/overrides", response_model = ModelOverridesResponse) diff --git a/studio/backend/tests/test_openai_auto_switch.py b/studio/backend/tests/test_openai_auto_switch.py index 2119e26164..721c9e2b22 100644 --- a/studio/backend/tests/test_openai_auto_switch.py +++ b/studio/backend/tests/test_openai_auto_switch.py @@ -4678,6 +4678,57 @@ def test_posix_colon_in_a_path_is_not_treated_as_a_quant(monkeypatch): assert "llama_extra_args" not in resp.overrides["/models/foo:bar.gguf"] +def test_unknown_quant_label_on_a_gguf_still_carries_flags_over(monkeypatch): + # A .gguf whose filename holds no recognizable quant token is still labelled + # by the scanner, which falls back to the stem, so the UI saves under + # "/models/custom.gguf:custom". Refusing that suffix dropped the bare entry's + # legacy flags on the first save, and auto-switch then prefers the qualified + # entry, so nothing was left that could restore them. + import routes.settings as settings_route + + _mock_override_store(monkeypatch) + settings.set_model_override("/models/custom.gguf", llama_extra_args = ["--flash-attn"]) + resp = settings_route.update_openai_auto_switch_override( + settings_route.ModelOverridePayload( + model_id = "/models/custom.gguf:custom", max_seq_length = 4096 + ), + "tester", + ) + assert resp.overrides["/models/custom.gguf:custom"]["llama_extra_args"] == ["--flash-attn"] + + +def test_a_suffix_the_scanner_would_not_derive_carries_nothing_over(monkeypatch): + # Only the exact label the scanner derives for this filename is accepted, so + # an unrelated colon suffix cannot reach into another model's flags. + import routes.settings as settings_route + + _mock_override_store(monkeypatch) + settings.set_model_override("/models/custom.gguf", llama_extra_args = ["--flash-attn"]) + resp = settings_route.update_openai_auto_switch_override( + settings_route.ModelOverridePayload( + model_id = "/models/custom.gguf:something-else", max_seq_length = 4096 + ), + "tester", + ) + assert "llama_extra_args" not in resp.overrides["/models/custom.gguf:something-else"] + + +def test_unknown_quant_label_carries_over_for_a_windows_path(monkeypatch): + # The key is written on Windows but may be read back by a backend that is + # not, where a backslash is an ordinary filename character. + import routes.settings as settings_route + + _mock_override_store(monkeypatch) + settings.set_model_override(r"C:\models\custom.gguf", llama_extra_args = ["--flash-attn"]) + resp = settings_route.update_openai_auto_switch_override( + settings_route.ModelOverridePayload( + model_id = r"C:\models\custom.gguf:custom", max_seq_length = 4096 + ), + "tester", + ) + assert resp.overrides[r"C:\models\custom.gguf:custom"]["llama_extra_args"] == ["--flash-attn"] + + def test_real_quant_suffix_on_a_path_still_carries_flags_over(monkeypatch): import routes.settings as settings_route From 64f642982b3c04fbfa9406bf3eadb381cf62500a Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 28 Jul 2026 14:22:41 +0000 Subject: [PATCH 19/75] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- tests/studio/test_model_picker_contracts.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/studio/test_model_picker_contracts.py b/tests/studio/test_model_picker_contracts.py index 3fa8abb764..71a3c7d659 100644 --- a/tests/studio/test_model_picker_contracts.py +++ b/tests/studio/test_model_picker_contracts.py @@ -699,6 +699,6 @@ def test_backfill_compares_server_keys_by_normalized_identity(): assert "if (known.has(key)) { continue; }" in src # A variant never holds a colon, so the last one splits the key. Splitting on # the first would cut a Windows drive letter off every path id. - assert "key.lastIndexOf(\":\")" in src + assert 'key.lastIndexOf(":")' in src # Repo ids fold and POSIX paths do not, which is exactly what these do. assert "normalizeModelIdentity(" in src and "normalizeGgufVariantIdentity(" in src From e27698b3afbf9e1b0b2c2d35c5463c536af37a29 Mon Sep 17 00:00:00 2001 From: danielhanchen Date: Tue, 28 Jul 2026 14:52:39 +0000 Subject: [PATCH 20/75] Keep model lifecycle rows out of the request statistics A load, unload or download is recorded in the monitor but is not an HTTP call. It reads as running for as long as the load takes, so it was counted as an in-flight request with no client waiting, and a multi-minute download was folded into Avg latency and the error rate. The backend already excludes these rows from active_count for the same reason, so the page was also disagreeing with the number the API itself reports. Requests counted them too, so that is now the non-lifecycle count rather than the raw entry count. Also limit the API-reach sentence on the Hub settings page to GGUF models. The Hub opens that page for every downloaded model, but ModelConfigPage mirrors settings to the server only when target.isGguf, because auto-switch indexes GGUFs only, so a safetensors user was told the settings apply to an API request that cannot reach them. --- .../features/api-monitor/use-api-monitor.ts | 14 ++++++++- .../hub/catalog/hub-model-settings-view.tsx | 9 ++++-- tests/studio/test_model_picker_contracts.py | 31 +++++++++++++++++++ 3 files changed, 51 insertions(+), 3 deletions(-) diff --git a/studio/frontend/src/features/api-monitor/use-api-monitor.ts b/studio/frontend/src/features/api-monitor/use-api-monitor.ts index 3a40990120..0b613eb8c6 100644 --- a/studio/frontend/src/features/api-monitor/use-api-monitor.ts +++ b/studio/frontend/src/features/api-monitor/use-api-monitor.ts @@ -77,7 +77,19 @@ export function computeStats(entries: ApiMonitorEntry[]): MonitorStats { let generatedTokens = 0; let generatedDurationMs = 0; + let requests = 0; + for (const entry of entries) { + // A model load, unload or download is not an HTTP call. It shows as + // "running" for as long as the load takes, so counting it would report an + // in-flight request with no client waiting and fold a multi-minute download + // into "Avg latency". The backend already leaves these out of + // active_count for the same reason, so counting them here would also make + // the page disagree with the number the API itself reports. + if (entry.kind === "lifecycle") { + continue; + } + requests += 1; totalTokens += entryTokens(entry); if (entry.status === "running") { active += 1; @@ -106,7 +118,7 @@ export function computeStats(entries: ApiMonitorEntry[]): MonitorStats { const finished = completed + errors + cancelled; return { active, - total: entries.length, + total: requests, completed, errors, cancelled, diff --git a/studio/frontend/src/features/hub/catalog/hub-model-settings-view.tsx b/studio/frontend/src/features/hub/catalog/hub-model-settings-view.tsx index a293a0071a..30176a7d90 100644 --- a/studio/frontend/src/features/hub/catalog/hub-model-settings-view.tsx +++ b/studio/frontend/src/features/hub/catalog/hub-model-settings-view.tsx @@ -109,8 +109,13 @@ export function HubModelSettingsView({ />

- Saved settings apply everywhere this model loads, including when an - OpenAI-compatible API request asks for it. Turn on{" "} + {/* Only a GGUF is mirrored to the server, because API auto-switch + indexes GGUFs only, so promising the API case for anything + else describes a load that cannot happen. */} + {target.isGguf + ? "Saved settings apply everywhere this model loads, including when an OpenAI-compatible API request asks for it." + : "Saved settings apply everywhere Studio loads this model."}{" "} + Turn on{" "} Remember for this model {" "} diff --git a/tests/studio/test_model_picker_contracts.py b/tests/studio/test_model_picker_contracts.py index 71a3c7d659..f3dbd9c545 100644 --- a/tests/studio/test_model_picker_contracts.py +++ b/tests/studio/test_model_picker_contracts.py @@ -702,3 +702,34 @@ def test_backfill_compares_server_keys_by_normalized_identity(): assert 'key.lastIndexOf(":")' in src # Repo ids fold and POSIX paths do not, which is exactly what these do. assert "normalizeModelIdentity(" in src and "normalizeGgufVariantIdentity(" in src + + +def test_monitor_stats_exclude_model_lifecycle_rows(): + """A load, unload or download is recorded as a monitor entry but is not an + HTTP call. It reads as "running" for as long as the load takes, so counting + it reports an in-flight request with no client waiting and folds a + multi-minute download into "Avg latency". The backend already leaves these + out of active_count, so counting them here also makes the page disagree with + the number the API itself reports. + """ + src = " ".join(_read("features/api-monitor/use-api-monitor.ts").split()) + assert 'if (entry.kind === "lifecycle") { continue; }' in src + # "Requests" is a request count too, so it cannot stay entries.length. + assert "total: requests," in src + assert "total: entries.length" not in src + + backend = ( + WORKDIR / "studio" / "backend" / "core" / "inference" / "api_monitor.py" + ).read_text(encoding = "utf-8") + assert 'entry.kind != "lifecycle"' in backend, "the rule this mirrors" + + +def test_api_reach_copy_is_limited_to_gguf_models(): + """The Hub opens this page for every downloaded model, but ModelConfigPage + mirrors settings to the server only when target.isGguf, because API + auto-switch indexes GGUFs only. Telling a safetensors user the settings + apply to an API request describes a load that cannot happen. + """ + src = " ".join(_read("features/hub/catalog/hub-model-settings-view.tsx").split()) + assert "{target.isGguf ?" in src + assert "Saved settings apply everywhere Studio loads this model." in src From c7ecbfe16d39f450b711105829bba4df90022cae Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 28 Jul 2026 14:53:25 +0000 Subject: [PATCH 21/75] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- tests/studio/test_model_picker_contracts.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/studio/test_model_picker_contracts.py b/tests/studio/test_model_picker_contracts.py index f3dbd9c545..0a66f4efe3 100644 --- a/tests/studio/test_model_picker_contracts.py +++ b/tests/studio/test_model_picker_contracts.py @@ -718,9 +718,9 @@ def test_monitor_stats_exclude_model_lifecycle_rows(): assert "total: requests," in src assert "total: entries.length" not in src - backend = ( - WORKDIR / "studio" / "backend" / "core" / "inference" / "api_monitor.py" - ).read_text(encoding = "utf-8") + backend = (WORKDIR / "studio" / "backend" / "core" / "inference" / "api_monitor.py").read_text( + encoding = "utf-8" + ) assert 'entry.kind != "lifecycle"' in backend, "the rule this mirrors" From c8a4fa096193d66ae177e5ac0421e6ac749a0ea8 Mon Sep 17 00:00:00 2001 From: danielhanchen Date: Tue, 28 Jul 2026 15:20:41 +0000 Subject: [PATCH 22/75] Accept bpw variants, fold POSIX quant suffixes, migrate bare GGUF configs Three gaps the previous round left, all in the same key-resolution rule, so the rule now lives in one place as split_quant_suffix. A quant label may carry a bits-per-weight modifier, because two files at the same base quant are kept distinct by it: utils/models/model_config.py preserves IQ4_XS-3.53bpw while hub/utils/gguf.py strips it, and both forms reach the override keys. The known-quant pattern accepted neither, so _bare_model_id missed the bare entry and the first qualified save dropped its launch flags. On POSIX the browser lowercases the quant but keeps the path casing, so a migrated "/models/Foo:q4_k_m" was unreachable from the scanner's "/models/Foo:Q4_K_M". Only the quant suffix folds now, and only when it really is a quant, so "/models/foo:Bar.gguf" stays a distinct filename and the path itself stays case-sensitive. A standalone .gguf picked directly has no quant to choose between and is stored with a null variant. The backfill filter read that as safetensors and skipped it, and the done flag is set on the same pass, so those settings stayed browser-only for good while auto-switch kept loading the model with defaults. --- studio/backend/routes/settings.py | 11 ++-- .../backend/tests/test_openai_auto_switch.py | 38 ++++++++++++ .../utils/openai_auto_switch_settings.py | 60 +++++++++++++++++-- .../api/migrate-model-overrides.ts | 9 ++- tests/studio/test_model_picker_contracts.py | 13 ++++ 5 files changed, 120 insertions(+), 11 deletions(-) diff --git a/studio/backend/routes/settings.py b/studio/backend/routes/settings.py index 3e4409ae3b..139538a3bd 100644 --- a/studio/backend/routes/settings.py +++ b/studio/backend/routes/settings.py @@ -355,16 +355,17 @@ def get_openai_auto_switch_overrides( def _bare_model_id(model_id: str) -> Optional[str]: """``repo`` for a ``repo:QUANT`` key, or None when there is no quant suffix.""" - from core.inference.llama_cpp import _GGUF_KNOWN_QUANT_RE from hub.utils.gguf import extract_quant_label + from utils.openai_auto_switch_settings import split_quant_suffix head, sep, tail = model_id.rpartition(":") if not sep or not head or not tail: return None - if "/" not in tail and "\\" not in tail and len(tail) <= _MAX_VARIANT_SUFFIX_LEN: - # Must actually look like a quant, not just like a short path segment. - if _GGUF_KNOWN_QUANT_RE.fullmatch(tail) is not None: - return head + # Must actually look like a quant, not just like a short path segment. The + # label may carry a bits-per-weight modifier ("IQ4_XS-3.53bpw"), which keeps + # two files at the same base quant distinct, so that form counts too. + if split_quant_suffix(model_id) is not None: + return head # A .gguf with no recognizable quant token is still labelled by the scanner, # which falls back to the filename stem, so the UI stores keys like # "/models/custom.gguf:custom". Refusing those dropped the bare entry's diff --git a/studio/backend/tests/test_openai_auto_switch.py b/studio/backend/tests/test_openai_auto_switch.py index 721c9e2b22..cd77be0c77 100644 --- a/studio/backend/tests/test_openai_auto_switch.py +++ b/studio/backend/tests/test_openai_auto_switch.py @@ -4697,6 +4697,44 @@ def test_unknown_quant_label_on_a_gguf_still_carries_flags_over(monkeypatch): assert resp.overrides["/models/custom.gguf:custom"]["llama_extra_args"] == ["--flash-attn"] +def test_bpw_qualified_variants_still_carry_flags_over(monkeypatch): + # utils/models/model_config.py keeps a bits-per-weight modifier on the label + # so two files at the same base quant stay distinct, and that form reaches + # the override keys. The known-quant pattern does not accept it, so the bare + # entry was missed and the first qualified save dropped its launch flags. + import routes.settings as settings_route + + _mock_override_store(monkeypatch) + settings.set_model_override("unsloth/Repo-GGUF", llama_extra_args = ["--flash-attn"]) + resp = settings_route.update_openai_auto_switch_override( + settings_route.ModelOverridePayload( + model_id = "unsloth/Repo-GGUF:IQ4_XS-3.53bpw", max_seq_length = 4096 + ), + "tester", + ) + assert resp.overrides["unsloth/Repo-GGUF:IQ4_XS-3.53bpw"]["llama_extra_args"] == [ + "--flash-attn" + ] + + +def test_a_posix_path_variant_folds_while_the_path_does_not(monkeypatch): + # The browser lowercases the quant but keeps POSIX path casing, so the + # migrated key is "/models/Foo:q4_k_m" while the scanner asks for + # "/models/Foo:Q4_K_M". The path itself must still be case-sensitive. + _mock_override_store(monkeypatch) + settings.set_model_override("/models/Foo:q4_k_m", max_seq_length = 8192) + assert settings.get_model_override("/models/Foo:Q4_K_M")["max_seq_length"] == 8192 + assert settings.get_model_override("/models/foo:Q4_K_M") == {} + + +def test_a_posix_colon_filename_is_not_folded_as_a_variant(monkeypatch): + # "/models/foo:Bar.gguf" is one filename, not path + quant, so folding its + # tail would let it reach a different file's settings. + _mock_override_store(monkeypatch) + settings.set_model_override("/models/foo:bar.gguf", max_seq_length = 8192) + assert settings.get_model_override("/models/foo:Bar.gguf") == {} + + def test_a_suffix_the_scanner_would_not_derive_carries_nothing_over(monkeypatch): # Only the exact label the scanner derives for this filename is accepted, so # an unrelated colon suffix cannot reach into another model's flags. diff --git a/studio/backend/utils/openai_auto_switch_settings.py b/studio/backend/utils/openai_auto_switch_settings.py index 24fa7704f0..59827121df 100644 --- a/studio/backend/utils/openai_auto_switch_settings.py +++ b/studio/backend/utils/openai_auto_switch_settings.py @@ -502,6 +502,47 @@ def _fold_case_insensitive_path(model_id: str) -> Optional[str]: return trimmed.casefold() +# A quant label may carry a bits-per-weight modifier, because two files at the +# same base quant are kept distinct by it ("IQ4_XS-3.53bpw"). The two label +# helpers disagree on whether to keep it, so anything reading a stored key has +# to accept both forms. +_BPW_SUFFIX = re.compile(r"-[0-9]+(?:\.[0-9]+)?bpw$", re.IGNORECASE) +_MAX_QUANT_SUFFIX_LEN = 64 + + +def split_quant_suffix(value: str) -> Optional[tuple[str, str]]: + """``(head, quant)`` for a ``head:QUANT`` key, or None when there is none. + + The suffix has to be a real quant label, so an ordinary colon inside a POSIX + filename is left alone: "/models/foo:bar.gguf" is one valid filename, and + splitting it would graft /models/foo's launch flags onto a different model. + """ + from core.inference.llama_cpp import _GGUF_KNOWN_QUANT_RE + + head, sep, tail = value.rpartition(":") + if not sep or not head or not tail: + return None + if len(tail) > _MAX_QUANT_SUFFIX_LEN or "/" in tail or "\\" in tail: + return None + if _GGUF_KNOWN_QUANT_RE.fullmatch(_BPW_SUFFIX.sub("", tail)) is None: + return None + return head, tail + + +def _fold_posix_path_variant(value: str) -> str: + """A POSIX path id with only its quant suffix folded. + + The browser lowercases the variant but keeps the path casing, so a stored + "/models/Foo:q4_k_m" has to be reachable from "/models/Foo:Q4_K_M" without + also making "/models/Foo.gguf" reachable from "/models/foo.gguf". + """ + split = split_quant_suffix(value) + if split is None: + return value + head, quant = split + return f"{head}:{quant.casefold()}" + + def get_model_overrides() -> dict[str, dict]: """Per-model launch configs keyed by model id (see normalize_model_override).""" raw = _cached_setting(MODEL_OVERRIDES_SETTING_KEY, None) @@ -544,11 +585,22 @@ def resolve_model_override_key(model_id: str) -> Optional[str]: # every migrated Windows entry unreachable until the user saved it again. if _looks_like_filesystem_path(model_id): folded = _fold_case_insensitive_path(model_id) - if folded is None: - return None + if folded is not None: - def fold(key: str) -> Optional[str]: - return _fold_case_insensitive_path(key) + def fold(key: str) -> Optional[str]: + return _fold_case_insensitive_path(key) + else: + # POSIX: the path itself stays case-sensitive, but the browser + # lowercases the quant suffix while keeping the path casing, so a + # migrated "/models/Foo:q4_k_m" has to stay reachable from the + # scanner's "/models/Foo:Q4_K_M". + folded = _fold_posix_path_variant(model_id) + + def fold(key: str) -> Optional[str]: + # A path only ever folds onto another path. + if not _looks_like_filesystem_path(key): + return None + return None if _fold_case_insensitive_path(key) else _fold_posix_path_variant(key) else: folded = model_id.casefold() diff --git a/studio/frontend/src/features/model-picker/api/migrate-model-overrides.ts b/studio/frontend/src/features/model-picker/api/migrate-model-overrides.ts index b5f908a109..3cfa571db3 100644 --- a/studio/frontend/src/features/model-picker/api/migrate-model-overrides.ts +++ b/studio/frontend/src/features/model-picker/api/migrate-model-overrides.ts @@ -75,8 +75,13 @@ export async function backfillModelOverrides(): Promise { const local = listPerModelConfigs().filter( // A quant means it is a GGUF, which is the only thing API auto-switch // resolves. Backfilling a safetensors config would claim an API behaviour - // that does not exist. - (entry) => entry.ggufVariant != null && !isDefaultConfig(entry.config), + // that does not exist. A standalone .gguf picked directly carries no quant + // to select between, so it is stored with a null variant and would fail + // that test despite being exactly what auto-switch does resolve; the flag + // is then set and its settings stay browser-only for good. + (entry) => + (entry.ggufVariant != null || entry.modelId.toLowerCase().endsWith(".gguf")) && + !isDefaultConfig(entry.config), ); if (local.length === 0) { markRan(); diff --git a/tests/studio/test_model_picker_contracts.py b/tests/studio/test_model_picker_contracts.py index 0a66f4efe3..79efdbf61e 100644 --- a/tests/studio/test_model_picker_contracts.py +++ b/tests/studio/test_model_picker_contracts.py @@ -733,3 +733,16 @@ def test_api_reach_copy_is_limited_to_gguf_models(): src = " ".join(_read("features/hub/catalog/hub-model-settings-view.tsx").split()) assert "{target.isGguf ?" in src assert "Saved settings apply everywhere Studio loads this model." in src + + +def test_backfill_includes_a_standalone_gguf_with_no_variant(): + """A standalone .gguf picked directly has no quant to choose between, so it + is stored with a null variant. The quant filter classified it like + safetensors and skipped it, and since the done flag is set on the same pass + those settings stayed browser-only permanently while API auto-switch, which + does resolve that model, kept loading it with defaults. + """ + src = " ".join(_read("features/model-picker/api/migrate-model-overrides.ts").split()) + assert 'entry.modelId.toLowerCase().endsWith(".gguf")' in src + # Still excluded for safetensors, which auto-switch does not resolve. + assert "entry.ggufVariant != null ||" in src From 88bf2eacfb040d74a72f92ff011fd85d06b76aaa Mon Sep 17 00:00:00 2001 From: danielhanchen Date: Tue, 28 Jul 2026 15:25:04 +0000 Subject: [PATCH 23/75] Let remove win over flag validation, and make Clear log clear shared rows An explicit remove ran the launch-flag validation first, so a form still carrying a rejected flag raised a 400 and left the override in place. Nothing is stored on that path, so there is nothing to validate; remove now short-circuits it, which is what the branch below already claims to do. Clear log dropped only the caller's own rows, but a lifecycle row is shared: it is visible to everyone and owned by no one, so those rows survived and the reload straight after the click brought them back, leaving the button visibly ineffective. Deleting them is not an option either, since that erases another caller's history. They are now hidden per subject, so the clear is true for that caller and harmless to the rest. A shared row that is still running is live state rather than history, so it stays visible, and the hidden ids are pruned against the ring buffer so they cannot accumulate. --- studio/backend/core/inference/api_monitor.py | 32 ++++++++++-- studio/backend/routes/settings.py | 6 ++- studio/backend/tests/test_api_monitor.py | 51 +++++++++++++++++++ .../backend/tests/test_openai_auto_switch.py | 18 +++++++ 4 files changed, 103 insertions(+), 4 deletions(-) diff --git a/studio/backend/core/inference/api_monitor.py b/studio/backend/core/inference/api_monitor.py index 25eff94fff..f18dd449cd 100644 --- a/studio/backend/core/inference/api_monitor.py +++ b/studio/backend/core/inference/api_monitor.py @@ -125,6 +125,12 @@ class ApiMonitor: enabled: bool = True, ): self._entries: deque[ApiMonitorEntry] = deque() + # Shared rows one subject has cleared. A shared row belongs to everyone, + # so dropping it would erase another caller's history, but leaving it + # means "Clear log" visibly does nothing to it: the frontend reloads + # straight after and the row comes back. Hiding it per subject is the + # only thing that is both true for that caller and safe for the others. + self._hidden_shared: dict[str, set[str]] = {} self._max_entries = max(0, max_entries) self._lock = threading.Lock() self._enabled = enabled @@ -403,12 +409,25 @@ class ApiMonitor: with self._lock: if subject is None: self._entries.clear() + self._hidden_shared.clear() return + # A shared row that is still running is a load in progress, not + # history, so it stays visible; clearing the log is about what has + # already happened. + hidden = self._hidden_shared.setdefault(subject, set()) + for entry in self._entries: + if entry.shared and entry.subject != subject and entry.status != "running": + hidden.add(entry.id) self._entries = deque(entry for entry in self._entries if entry.subject != subject) - @staticmethod - def _visible(entry: ApiMonitorEntry, subject: Optional[str]) -> bool: - return subject is None or entry.subject == subject or entry.shared + def _visible(self, entry: ApiMonitorEntry, subject: Optional[str]) -> bool: + if subject is None: + return True + if entry.subject == subject: + return True + if not entry.shared: + return False + return entry.id not in self._hidden_shared.get(subject, ()) def _find_locked(self, entry_id: str) -> Optional[ApiMonitorEntry]: for entry in self._entries: @@ -427,6 +446,13 @@ class ApiMonitor: kept.append(entry) terminal_seen += 1 self._entries = kept + # The hidden sets only ever name rows that exist, so they stay bounded + # by the ring buffer rather than growing for the life of the process. + live = {entry.id for entry in kept} + for subject, hidden in list(self._hidden_shared.items()): + hidden &= live + if not hidden: + del self._hidden_shared[subject] api_monitor = ApiMonitor(enabled = not _api_monitor_disabled()) diff --git a/studio/backend/routes/settings.py b/studio/backend/routes/settings.py index 139538a3bd..49e2f29d39 100644 --- a/studio/backend/routes/settings.py +++ b/studio/backend/routes/settings.py @@ -414,7 +414,11 @@ def update_openai_auto_switch_override( bare_id = _bare_model_id(payload.model_id) if bare_id: requested_extra_args = get_model_override(bare_id).get("llama_extra_args") - extra_args = validate_extra_args(requested_extra_args) + # Not validated on an explicit remove: nothing is stored, so the only + # effect would be a 400 that leaves the override in place, which is the + # opposite of what remove means. A stale form still carrying a rejected + # flag must not be able to block forgetting a model. + extra_args = [] if payload.remove is True else validate_extra_args(requested_extra_args) if payload.remove is True: # An explicit remove wins over anything else in the payload: a stale # form field must not turn "forget this model" into an update that diff --git a/studio/backend/tests/test_api_monitor.py b/studio/backend/tests/test_api_monitor.py index 5e5e841263..821861457c 100644 --- a/studio/backend/tests/test_api_monitor.py +++ b/studio/backend/tests/test_api_monitor.py @@ -468,3 +468,54 @@ def test_request_rows_report_kind_request(): monitor = ApiMonitor(max_entries = 2) monitor.start(endpoint = "/v1/chat/completions", method = "POST", model = "m", prompt = "hi") assert monitor.snapshot()[0]["kind"] == "request" + + +def test_clear_hides_shared_lifecycle_rows_for_that_caller_only(): + """A lifecycle row is shared, so it is visible to every caller but owned by + none. A subject-scoped clear dropped only that subject's own rows, so the + shared ones survived and the reload straight after "Clear log" brought them + back: the button visibly did nothing to them. Dropping them outright is not + an option either, since that erases another caller's history. + """ + monitor = ApiMonitor(max_entries = 10) + mine = monitor.start( + endpoint = "/v1/chat/completions", + method = "POST", + model = "org/A", + prompt = "user: hi", + subject = "alice", + ) + monitor.finish(mine) + shared = monitor.record_lifecycle(event = "unload", model = "org/A") + + assert {e["id"] for e in monitor.snapshot(subject = "alice")} == {mine, shared} + assert {e["id"] for e in monitor.snapshot(subject = "bob")} == {shared} + + monitor.clear(subject = "alice") + + assert monitor.snapshot(subject = "alice") == [] + # Bob's view is untouched: the row is hidden for alice, not deleted. + assert {e["id"] for e in monitor.snapshot(subject = "bob")} == {shared} + assert monitor.get(shared, subject = "alice") is None + assert monitor.get(shared, subject = "bob") is not None + + +def test_clear_leaves_a_running_shared_row_visible(): + """A load still in progress is live state, not history, so clearing the log + must not hide the row that shows it.""" + monitor = ApiMonitor(max_entries = 10) + running = monitor.record_lifecycle(event = "load", model = "org/A", running = True) + monitor.clear(subject = "alice") + assert {e["id"] for e in monitor.snapshot(subject = "alice")} == {running} + + +def test_hidden_shared_ids_do_not_outlive_their_entries(): + """The hidden set names rows that exist, so it stays bounded by the ring + buffer instead of growing for the life of the process.""" + monitor = ApiMonitor(max_entries = 2) + monitor.record_lifecycle(event = "unload", model = "org/A") + monitor.clear(subject = "alice") + assert monitor._hidden_shared.get("alice") + for i in range(5): + monitor.record_lifecycle(event = "unload", model = f"org/M{i}") + assert not monitor._hidden_shared.get("alice") diff --git a/studio/backend/tests/test_openai_auto_switch.py b/studio/backend/tests/test_openai_auto_switch.py index cd77be0c77..c5f454654d 100644 --- a/studio/backend/tests/test_openai_auto_switch.py +++ b/studio/backend/tests/test_openai_auto_switch.py @@ -4648,6 +4648,24 @@ def test_case_fallback_still_covers_repo_ids(monkeypatch): assert settings.get_model_override("unsloth/Qwen3-8B-GGUF:Q4_K_M")["max_seq_length"] == 8192 +def test_explicit_remove_is_not_blocked_by_stale_invalid_flags(monkeypatch): + # remove is the operation discriminator, so a form still carrying a rejected + # launch flag must not turn "forget this model" into a 400 that leaves the + # override in place. Nothing is stored on this path, so there is nothing to + # validate. + import routes.settings as settings_route + + _mock_override_store(monkeypatch) + settings.set_model_override("unsloth/B-GGUF", max_seq_length = 4096) + resp = settings_route.update_openai_auto_switch_override( + settings_route.ModelOverridePayload( + model_id = "unsloth/B-GGUF", remove = True, llama_extra_args = ["--port", "1234"] + ), + "tester", + ) + assert "unsloth/B-GGUF" not in resp.overrides + + def test_explicit_remove_wins_over_config_fields_in_the_same_payload(monkeypatch): # remove is the operation discriminator, so a stale form field alongside it # must not quietly turn "forget this model" into an update. From 221b810d52cb8eb87a2e2b23ca76e97045c46356 Mon Sep 17 00:00:00 2001 From: danielhanchen Date: Tue, 28 Jul 2026 15:27:24 +0000 Subject: [PATCH 24/75] Move the lifecycle labels out of the lazily loaded monitor page The overlay is mounted from __root.tsx and imported two label helpers from the page, so the page and its dependency graph were pulled into the eagerly loaded bundle and the route's lazyRouteComponent bought nothing: every route paid for the monitor page even when it was never opened. Measured on a production vite build, the async api-monitor chunk was 0.20 kB, meaning the implementation had landed in the main bundle. The helpers now live in their own module. The same build gives an 18.83 kB api-monitor chunk and a main bundle 18 kB smaller (3.9 kB gzipped). --- .../api-monitor/api-monitor-overlay.tsx | 2 +- .../features/api-monitor/api-monitor-page.tsx | 35 +------------- .../src/features/api-monitor/lifecycle.ts | 47 +++++++++++++++++++ tests/studio/test_model_picker_contracts.py | 17 +++++++ 4 files changed, 66 insertions(+), 35 deletions(-) create mode 100644 studio/frontend/src/features/api-monitor/lifecycle.ts diff --git a/studio/frontend/src/features/api-monitor/api-monitor-overlay.tsx b/studio/frontend/src/features/api-monitor/api-monitor-overlay.tsx index a9384792d4..93d774cf7e 100644 --- a/studio/frontend/src/features/api-monitor/api-monitor-overlay.tsx +++ b/studio/frontend/src/features/api-monitor/api-monitor-overlay.tsx @@ -24,7 +24,7 @@ import { useRef, useState, } from "react"; -import { isLifecycleEntry, lifecycleLabel } from "./api-monitor-page"; +import { isLifecycleEntry, lifecycleLabel } from "./lifecycle"; import { useApiMonitorOverlayStore } from "./overlay-store"; import { computeStats } from "./use-api-monitor"; diff --git a/studio/frontend/src/features/api-monitor/api-monitor-page.tsx b/studio/frontend/src/features/api-monitor/api-monitor-page.tsx index 502eda0063..4d998c6f88 100644 --- a/studio/frontend/src/features/api-monitor/api-monitor-page.tsx +++ b/studio/frontend/src/features/api-monitor/api-monitor-page.tsx @@ -41,6 +41,7 @@ import { import { HugeiconsIcon } from "@hugeicons/react"; import { type ReactElement, useEffect, useMemo, useRef, useState } from "react"; import { SavedModelSettingsPanel } from "./components/saved-model-settings"; +import { isLifecycleEntry, lifecycleLabel } from "./lifecycle"; import { type MonitorStatusFilter, filterEntries, @@ -86,40 +87,6 @@ function compactEndpoint(endpoint: string): string { .replace(V1_PREFIX_RE, "/"); } -// A lifecycle row is a model load/unload/download, not an HTTP call: it carries -// an event and reason instead of a prompt, so there is no payload to expand. -export function isLifecycleEntry(entry: ApiMonitorEntry): boolean { - return entry.kind === "lifecycle"; -} - -export function lifecycleLabel(entry: ApiMonitorEntry): string { - if (entry.event === "unload") { - return entry.reason === "idle" ? "Model unloaded (idle)" : "Model unloaded"; - } - if (entry.event === "download") { - if (entry.status === "running") { - const pct = entry.progress; - return typeof pct === "number" - ? `Downloading model (${Math.round(pct)}%)` - : "Downloading model"; - } - if (entry.status === "completed") { - return "Model downloaded"; - } - // A cancel is deliberate, so saying it failed misreads the user's own action. - return entry.status === "cancelled" - ? "Model download cancelled" - : "Model download failed"; - } - if (entry.status === "running") { - return "Loading model"; - } - if (entry.status === "completed") { - return "Model loaded"; - } - return "Model load failed"; -} - function statusDotClass(status: ApiMonitorEntry["status"]): string { switch (status) { case "running": diff --git a/studio/frontend/src/features/api-monitor/lifecycle.ts b/studio/frontend/src/features/api-monitor/lifecycle.ts new file mode 100644 index 0000000000..70e31df6fa --- /dev/null +++ b/studio/frontend/src/features/api-monitor/lifecycle.ts @@ -0,0 +1,47 @@ +// SPDX-License-Identifier: AGPL-3.0-only +// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 + +// Labels for model load/unload/download rows, shared by the overlay and the +// full page. +// +// They live here rather than on the page because the overlay is mounted from +// __root.tsx, so importing them from the page would pull the whole page and its +// dependency graph into the eagerly loaded bundle and undo the route's +// lazyRouteComponent: every route would pay for the monitor page even when it +// is never opened. + +import type { ApiMonitorEntry } from "@/features/chat/types/api"; + +// A lifecycle row is a model load/unload/download, not an HTTP call: it carries +// an event and reason instead of a prompt, so there is no payload to expand. +export function isLifecycleEntry(entry: ApiMonitorEntry): boolean { + return entry.kind === "lifecycle"; +} + +export function lifecycleLabel(entry: ApiMonitorEntry): string { + if (entry.event === "unload") { + return entry.reason === "idle" ? "Model unloaded (idle)" : "Model unloaded"; + } + if (entry.event === "download") { + if (entry.status === "running") { + const pct = entry.progress; + return typeof pct === "number" + ? `Downloading model (${Math.round(pct)}%)` + : "Downloading model"; + } + if (entry.status === "completed") { + return "Model downloaded"; + } + // A cancel is deliberate, so saying it failed misreads the user's own action. + return entry.status === "cancelled" + ? "Model download cancelled" + : "Model download failed"; + } + if (entry.status === "running") { + return "Loading model"; + } + if (entry.status === "completed") { + return "Model loaded"; + } + return "Model load failed"; +} diff --git a/tests/studio/test_model_picker_contracts.py b/tests/studio/test_model_picker_contracts.py index 79efdbf61e..f146de5c0f 100644 --- a/tests/studio/test_model_picker_contracts.py +++ b/tests/studio/test_model_picker_contracts.py @@ -746,3 +746,20 @@ def test_backfill_includes_a_standalone_gguf_with_no_variant(): assert 'entry.modelId.toLowerCase().endsWith(".gguf")' in src # Still excluded for safetensors, which auto-switch does not resolve. assert "entry.ggufVariant != null ||" in src + + +def test_monitor_overlay_does_not_pull_in_the_lazy_page(): + """The overlay is mounted from __root.tsx, so a static import of the page + for two label helpers drags the whole 900-line page and its dependency + graph into the eagerly loaded bundle and undoes the route's + lazyRouteComponent. Measured: the async api-monitor chunk was 0.20 kB with + the page in the main bundle, and 18.83 kB after the helpers moved, with the + main bundle 18 kB smaller. + """ + overlay = _read("features/api-monitor/api-monitor-overlay.tsx") + assert 'from "./lifecycle"' in overlay + assert "api-monitor-page" not in overlay, "the overlay must not reach the page" + # The helpers live in their own module, not re-exported through the page. + shared = _read("features/api-monitor/lifecycle.ts") + assert "export function isLifecycleEntry(" in shared + assert "export function lifecycleLabel(" in shared From 423532cbcbac41200b0c7609fc6cfec754f465d0 Mon Sep 17 00:00:00 2001 From: danielhanchen Date: Tue, 28 Jul 2026 15:29:26 +0000 Subject: [PATCH 25/75] Order override writes per model Saving twice quickly, or saving while the one-time backfill is still running, started independent requests with no sequencing, so the older response could commit last and resurrect the entry the newer one meant to replace or remove. An API-driven load then applies context or GPU settings the user has already changed, with nothing in the UI showing it. Writes now chain per override key. The chain hangs off the settled tail, so a failed write cannot cancel the next one, and only the last writer clears the slot so a queue that is still building keeps its order. Different models still overlap. Verified against the real module under node: two saves for one model with the first made slow commit oldest-first and never overlap, where the previous version committed them in the wrong order; a rejected write still lets the next succeed; and two models still run concurrently. --- .../model-picker/api/model-overrides.ts | 31 +++++++++++++++++++ tests/studio/test_model_picker_contracts.py | 17 ++++++++++ 2 files changed, 48 insertions(+) diff --git a/studio/frontend/src/features/model-picker/api/model-overrides.ts b/studio/frontend/src/features/model-picker/api/model-overrides.ts index d4e89985b4..bf8ad1aa58 100644 --- a/studio/frontend/src/features/model-picker/api/model-overrides.ts +++ b/studio/frontend/src/features/model-picker/api/model-overrides.ts @@ -123,10 +123,41 @@ function toApiOverride(config: PerModelConfig | null): ApiModelOverride { return payload; } +// One in-flight write per model, so writes for the same model commit in the +// order they were issued. Saving twice quickly, or saving while the one-time +// backfill is still running, otherwise leaves two independent requests racing: +// the older response can land last and resurrect the entry the newer one meant +// to replace or remove, and an API load then applies settings the user has +// already changed. Different models still overlap. +const writesByKey = new Map>(); + export async function putModelOverride( modelId: string, ggufVariant: string | null | undefined, config: PerModelConfig | null, +): Promise { + const key = modelOverrideKey(modelId, ggufVariant); + // Chain on the settled tail: a failed write must not cancel the next one. + const previous = writesByKey.get(key) ?? Promise.resolve(); + const write = previous + .catch(() => {}) + .then(() => sendModelOverride(modelId, ggufVariant, config)); + writesByKey.set(key, write); + try { + await write; + } finally { + // Only the last writer clears the slot, so a queue that is still building + // keeps its ordering. + if (writesByKey.get(key) === write) { + writesByKey.delete(key); + } + } +} + +async function sendModelOverride( + modelId: string, + ggufVariant: string | null | undefined, + config: PerModelConfig | null, ): Promise { const res = await authFetch(OVERRIDES_URL, { method: "PUT", diff --git a/tests/studio/test_model_picker_contracts.py b/tests/studio/test_model_picker_contracts.py index f146de5c0f..a22ee1f5c3 100644 --- a/tests/studio/test_model_picker_contracts.py +++ b/tests/studio/test_model_picker_contracts.py @@ -763,3 +763,20 @@ def test_monitor_overlay_does_not_pull_in_the_lazy_page(): shared = _read("features/api-monitor/lifecycle.ts") assert "export function isLifecycleEntry(" in shared assert "export function lifecycleLabel(" in shared + + +def test_override_writes_are_ordered_per_model(): + """Two saves for one model, or a save racing the one-time backfill, started + independent requests with no sequencing, so the older response could commit + last and resurrect the entry the newer one meant to replace. An API load + then applies settings the user has already changed. Different models still + overlap, so a slow write for one cannot hold up another. + """ + src = " ".join(_read("features/model-picker/api/model-overrides.ts").split()) + assert "const writesByKey = new Map>();" in src + # Keyed by the same override key the server stores under. + assert "const key = modelOverrideKey(modelId, ggufVariant);" in src + # Chained on the settled tail, so one failed write cannot cancel the next. + assert "previous .catch(() => {}) .then(() => sendModelOverride(" in src + # Only the last writer clears the slot, or a queue still building loses order. + assert "if (writesByKey.get(key) === write) { writesByKey.delete(key); }" in src From 5a15f4d09969f4d31bfa9fd7ebe220c7a0762678 Mon Sep 17 00:00:00 2001 From: danielhanchen Date: Tue, 28 Jul 2026 15:51:36 +0000 Subject: [PATCH 26/75] Key write queues by identity, and reach unknown GGUF labels in either casing Two holes in the previous two commits, both found by the same review round. The per-model write queue keyed on the literal spelling, so the backfill's legacy casing and a UI save's normalized one opened two queues for one model and raced exactly as before. It now keys on the folded identity, which is what the backend resolves by. A .gguf with no recognizable quant token is labelled by its filename stem, and v2 storage lowercases that label while the scanner probes with the filename's own casing. Folding only recognized quant labels therefore left the migrated entry unreachable for precisely the files that need the stem fallback. The suffix rule now also accepts a case-insensitive match against the label the scanner derives for that filename, which keeps an ordinary colon out because the head still has to be a .gguf. _bare_model_id drops onto the same shared rule rather than repeating half of it. --- studio/backend/routes/settings.py | 26 +++---------------- .../backend/tests/test_openai_auto_switch.py | 13 ++++++++++ .../utils/openai_auto_switch_settings.py | 19 +++++++++++--- .../model-picker/api/model-overrides.ts | 13 +++++++++- tests/studio/test_model_picker_contracts.py | 5 +++- 5 files changed, 49 insertions(+), 27 deletions(-) diff --git a/studio/backend/routes/settings.py b/studio/backend/routes/settings.py index 49e2f29d39..9639cee82d 100644 --- a/studio/backend/routes/settings.py +++ b/studio/backend/routes/settings.py @@ -355,32 +355,14 @@ def get_openai_auto_switch_overrides( def _bare_model_id(model_id: str) -> Optional[str]: """``repo`` for a ``repo:QUANT`` key, or None when there is no quant suffix.""" - from hub.utils.gguf import extract_quant_label from utils.openai_auto_switch_settings import split_quant_suffix - head, sep, tail = model_id.rpartition(":") - if not sep or not head or not tail: - return None # Must actually look like a quant, not just like a short path segment. The # label may carry a bits-per-weight modifier ("IQ4_XS-3.53bpw"), which keeps - # two files at the same base quant distinct, so that form counts too. - if split_quant_suffix(model_id) is not None: - return head - # A .gguf with no recognizable quant token is still labelled by the scanner, - # which falls back to the filename stem, so the UI stores keys like - # "/models/custom.gguf:custom". Refusing those dropped the bare entry's - # legacy flags on the first save, and auto-switch then prefers the qualified - # entry, so nothing could restore them. Requiring the suffix to be exactly - # the label the scanner derives for this filename is what keeps an arbitrary - # colon-containing POSIX path out: "/models/foo:bar.gguf" splits to a head - # that is not a .gguf at all. - # Split on both separators rather than os.path.basename: a "C:\..." key is - # written on Windows but may be read back by a backend that is not, and - # there a backslash is an ordinary filename character. - filename = head.replace("\\", "/").rsplit("/", 1)[-1] - if head.lower().endswith(".gguf") and tail == extract_quant_label(filename): - return head - return None + # two files at the same base quant distinct, and a .gguf with no recognized + # token is labelled by its stem, so both forms count. + split = split_quant_suffix(model_id) + return split[0] if split is not None else None @router.put("/openai-auto-switch/overrides", response_model = ModelOverridesResponse) diff --git a/studio/backend/tests/test_openai_auto_switch.py b/studio/backend/tests/test_openai_auto_switch.py index c5f454654d..80d076a865 100644 --- a/studio/backend/tests/test_openai_auto_switch.py +++ b/studio/backend/tests/test_openai_auto_switch.py @@ -4745,6 +4745,19 @@ def test_a_posix_path_variant_folds_while_the_path_does_not(monkeypatch): assert settings.get_model_override("/models/foo:Q4_K_M") == {} +def test_an_unknown_gguf_label_is_reachable_in_either_casing(monkeypatch): + # A .gguf with no recognizable quant token is labelled by its stem, and v2 + # storage lowercases that label while the scanner probes with the filename's + # own casing. Folding only recognized quant labels left the migrated entry + # unreachable for exactly the files that need the fallback. + _mock_override_store(monkeypatch) + settings.set_model_override("/models/CustomModel.gguf:custommodel", max_seq_length = 8192) + got = settings.get_model_override("/models/CustomModel.gguf:CustomModel") + assert got["max_seq_length"] == 8192 + # The path itself is still case-sensitive on POSIX. + assert settings.get_model_override("/models/custommodel.gguf:CustomModel") == {} + + def test_a_posix_colon_filename_is_not_folded_as_a_variant(monkeypatch): # "/models/foo:Bar.gguf" is one filename, not path + quant, so folding its # tail would let it reach a different file's settings. diff --git a/studio/backend/utils/openai_auto_switch_settings.py b/studio/backend/utils/openai_auto_switch_settings.py index 59827121df..3f39525f7e 100644 --- a/studio/backend/utils/openai_auto_switch_settings.py +++ b/studio/backend/utils/openai_auto_switch_settings.py @@ -518,15 +518,28 @@ def split_quant_suffix(value: str) -> Optional[tuple[str, str]]: splitting it would graft /models/foo's launch flags onto a different model. """ from core.inference.llama_cpp import _GGUF_KNOWN_QUANT_RE + from hub.utils.gguf import extract_quant_label head, sep, tail = value.rpartition(":") if not sep or not head or not tail: return None - if len(tail) > _MAX_QUANT_SUFFIX_LEN or "/" in tail or "\\" in tail: + if "/" in tail or "\\" in tail: return None - if _GGUF_KNOWN_QUANT_RE.fullmatch(_BPW_SUFFIX.sub("", tail)) is None: + if len(tail) <= _MAX_QUANT_SUFFIX_LEN and _GGUF_KNOWN_QUANT_RE.fullmatch( + _BPW_SUFFIX.sub("", tail) + ): + return head, tail + # A .gguf whose filename holds no recognizable quant token is still labelled + # by the scanner, which falls back to the stem, so keys like + # "/models/CustomModel.gguf:custommodel" exist. Storage lowercases that + # label while the scanner probes with the filename's own casing, so the + # comparison is case-insensitive. Requiring the suffix to be exactly that + # label is what keeps an ordinary colon out: "/models/foo:bar.gguf" splits + # to a head that is not a .gguf at all. + if not head.lower().endswith(".gguf"): return None - return head, tail + filename = head.replace("\\", "/").rsplit("/", 1)[-1] + return (head, tail) if tail.casefold() == extract_quant_label(filename).casefold() else None def _fold_posix_path_variant(value: str) -> str: diff --git a/studio/frontend/src/features/model-picker/api/model-overrides.ts b/studio/frontend/src/features/model-picker/api/model-overrides.ts index bf8ad1aa58..4c4081acf5 100644 --- a/studio/frontend/src/features/model-picker/api/model-overrides.ts +++ b/studio/frontend/src/features/model-picker/api/model-overrides.ts @@ -13,6 +13,10 @@ import { authFetch } from "@/features/auth"; import { readFastApiError } from "@/lib/format-fastapi-error"; +import { + normalizeGgufVariantIdentity, + normalizeModelIdentity, +} from "../model-config/model-identity"; import type { PerModelConfig } from "../model-config/per-model-config"; const OVERRIDES_URL = "/api/settings/openai-auto-switch/overrides"; @@ -136,7 +140,14 @@ export async function putModelOverride( ggufVariant: string | null | undefined, config: PerModelConfig | null, ): Promise { - const key = modelOverrideKey(modelId, ggufVariant); + // Keyed by the folded identity, not the literal spelling: the backfill sends + // a legacy casing while a UI save sends the normalized one, and the backend + // resolves both to the same row, so raw strings would open two queues for one + // model and let them race again. + const key = modelOverrideKey( + normalizeModelIdentity(modelId), + normalizeGgufVariantIdentity(ggufVariant), + ); // Chain on the settled tail: a failed write must not cancel the next one. const previous = writesByKey.get(key) ?? Promise.resolve(); const write = previous diff --git a/tests/studio/test_model_picker_contracts.py b/tests/studio/test_model_picker_contracts.py index a22ee1f5c3..1141b67003 100644 --- a/tests/studio/test_model_picker_contracts.py +++ b/tests/studio/test_model_picker_contracts.py @@ -775,7 +775,10 @@ def test_override_writes_are_ordered_per_model(): src = " ".join(_read("features/model-picker/api/model-overrides.ts").split()) assert "const writesByKey = new Map>();" in src # Keyed by the same override key the server stores under. - assert "const key = modelOverrideKey(modelId, ggufVariant);" in src + # Folded, not literal: the backfill uses a legacy casing while a UI save + # uses the normalized one, and the backend resolves both to one row, so + # raw strings would open two queues for one model and race again. + assert "const key = modelOverrideKey( normalizeModelIdentity(modelId), normalizeGgufVariantIdentity(ggufVariant), );" in src # Chained on the settled tail, so one failed write cannot cancel the next. assert "previous .catch(() => {}) .then(() => sendModelOverride(" in src # Only the last writer clears the slot, or a queue still building loses order. From fc06185c12d868c6079fcc42f2c28a5d180685d5 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 28 Jul 2026 15:54:28 +0000 Subject: [PATCH 27/75] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- tests/studio/test_model_picker_contracts.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/studio/test_model_picker_contracts.py b/tests/studio/test_model_picker_contracts.py index 1141b67003..83c155a1ce 100644 --- a/tests/studio/test_model_picker_contracts.py +++ b/tests/studio/test_model_picker_contracts.py @@ -778,7 +778,10 @@ def test_override_writes_are_ordered_per_model(): # Folded, not literal: the backfill uses a legacy casing while a UI save # uses the normalized one, and the backend resolves both to one row, so # raw strings would open two queues for one model and race again. - assert "const key = modelOverrideKey( normalizeModelIdentity(modelId), normalizeGgufVariantIdentity(ggufVariant), );" in src + assert ( + "const key = modelOverrideKey( normalizeModelIdentity(modelId), normalizeGgufVariantIdentity(ggufVariant), );" + in src + ) # Chained on the settled tail, so one failed write cannot cancel the next. assert "previous .catch(() => {}) .then(() => sendModelOverride(" in src # Only the last writer clears the slot, or a queue still building loses order. From c2bcb1e6e5b49c0fcf3c04ae4ffe8f670b939218 Mon Sep 17 00:00:00 2001 From: danielhanchen Date: Tue, 28 Jul 2026 16:18:55 +0000 Subject: [PATCH 28/75] Guard the backfill, the detail settings entry point and the detail retry Three separate reports, all confirmed against head. listPerModelConfigs reported future-schema records. loadPerModelConfig refuses to apply one and eviction refuses to drop one, because this client cannot interpret that schema, so handing it to the backfill would persist a partial reading of it server-side and let an API-triggered load apply settings the same client will not apply locally. It is skipped there now, matching the other two paths. The detail view's on-device card passes a null variant while its own lookup is pending or after it failed, and this entry point opened the editor anyway. That saves a bare-model config, which the picker never finds because it matches variants exactly, while the API's bare-key fallback would apply it. openModelSettings already refuses with a toast for exactly that reason; this path now refuses the same way. A failed detail fetch was never retried. The revision is recorded when the fetch starts, and on failure the entry stays missing, so selectedIsMissing does not change and a terminal row's updated_at does not advance: nothing was left to re-run the effect, and the full prompt and reply stayed unavailable until another row was selected. The in-flight flag settling is the trigger now, and the attempt count bounds it, because the usual failure is an entry that has aged out of the ring buffer and will never arrive however often it is asked for. --- .../features/api-monitor/api-monitor-page.tsx | 26 ++++++++++++- studio/frontend/src/features/hub/hub-page.tsx | 16 ++++++++ .../model-config/per-model-config.ts | 7 ++++ tests/studio/test_model_picker_contracts.py | 38 +++++++++++++++++++ 4 files changed, 86 insertions(+), 1 deletion(-) diff --git a/studio/frontend/src/features/api-monitor/api-monitor-page.tsx b/studio/frontend/src/features/api-monitor/api-monitor-page.tsx index 4d998c6f88..1558c9c6eb 100644 --- a/studio/frontend/src/features/api-monitor/api-monitor-page.tsx +++ b/studio/frontend/src/features/api-monitor/api-monitor-page.tsx @@ -50,6 +50,9 @@ import { const API_INFERENCE_PREFIX_RE = /^\/api\/inference/; const V1_PREFIX_RE = /^\/v1\//; +// Tries per revision for a detail payload. Bounded because the usual failure +// is an entry that has aged out of the ring buffer and never comes back. +const DETAIL_FETCH_ATTEMPTS = 3; const STATUS_FILTERS: { value: MonitorStatusFilter; label: string }[] = [ { value: "all", label: "All requests" }, @@ -550,9 +553,16 @@ export function ApiMonitorPage(): ReactElement { const selectedUpdatedAt = selected?.updated_at ?? null; const selectedIsMissing = selectedId_ != null && details[selectedId_] == null; const lastFetchedRef = useRef(null); + const attemptsRef = useRef<{ revision: string; count: number }>({ + revision: "", + count: 0, + }); const [retryTick, setRetryTick] = useState(0); + // Flips as a fetch settles, successfully or not, which is what lets a failed + // one be noticed at all. + const detailInFlight = selectedId_ != null && loadingDetails.has(selectedId_); useEffect(() => { - if (selectedId_ == null) { + if (selectedId_ == null || detailInFlight) { return; } // `updated_at` advances per poll while streaming and settles when terminal. @@ -561,6 +571,19 @@ export function ApiMonitorPage(): ReactElement { if (!selectedIsMissing && lastFetchedRef.current === revision) { return; } + // A terminal row's revision never advances, so a fetch that failed had + // nothing left to re-run this effect and the payload stayed unavailable + // until the user picked another row. `loadingDetails` changing as the failed + // fetch settles is the trigger; the count bounds it, because the usual + // failure is an entry that aged out of the ring buffer and will never + // arrive however often it is asked for. + if (attemptsRef.current.revision !== revision) { + attemptsRef.current = { revision, count: 0 }; + } + if (attemptsRef.current.count >= DETAIL_FETCH_ATTEMPTS) { + return; + } + attemptsRef.current.count += 1; // Only remember the revision when a fetch really started; the in-flight // guard can refuse, and recording it anyway skips that revision for good. if (requestDetail(selectedId_)) { @@ -579,6 +602,7 @@ export function ApiMonitorPage(): ReactElement { selectedIsMissing, requestDetail, retryTick, + detailInFlight, ]); // The desktop webview's origin is tauri://, not the API server, and the diff --git a/studio/frontend/src/features/hub/hub-page.tsx b/studio/frontend/src/features/hub/hub-page.tsx index 2f0dcb677d..7965c6b2da 100644 --- a/studio/frontend/src/features/hub/hub-page.tsx +++ b/studio/frontend/src/features/hub/hub-page.tsx @@ -1365,6 +1365,22 @@ export function ModelsPage() { const openSelectedModelSettings = useCallback( (ggufVariant: string | null) => { if (!selectedModel) return; + // The card passes null while its own variant lookup is pending or after it + // failed, so this needs the same guard openModelSettings applies: a model + // that needs a quant cannot be configured without one, because the picker + // matches variants exactly and would never find the saved config while the + // API falls back to the bare key and would apply it. + if ( + !ggufVariant && + selectedModel.isGguf && + selectedModel.requiresVariant + ) { + toast.error("Couldn't determine which quant to configure.", { + description: + "Settings for this model are per quant. Check the connection or the model's cache, then try again.", + }); + return; + } // Share the sequence with openModelSettings: a row's variant lookup may // still be pending, and it must not land on top of this one. settingsOpenSeq.current += 1; diff --git a/studio/frontend/src/features/model-picker/model-config/per-model-config.ts b/studio/frontend/src/features/model-picker/model-config/per-model-config.ts index acc8efdefa..049b480f80 100644 --- a/studio/frontend/src/features/model-picker/model-config/per-model-config.ts +++ b/studio/frontend/src/features/model-picker/model-config/per-model-config.ts @@ -690,6 +690,13 @@ export function listPerModelConfigs(): { if (!modelId) { continue; } + // Never report a future-schema record. loadPerModelConfig refuses to apply + // one and eviction refuses to drop one, so handing it to the backfill would + // persist this client's partial reading of it server-side and let an + // API-triggered load apply settings the same client will not apply locally. + if (storedConfigVersion(raw) > STORAGE_SCHEMA_VERSION) { + continue; + } const variant = ggufVariantFromStorageKey(key); out.push({ modelId, diff --git a/tests/studio/test_model_picker_contracts.py b/tests/studio/test_model_picker_contracts.py index 83c155a1ce..06d8a00082 100644 --- a/tests/studio/test_model_picker_contracts.py +++ b/tests/studio/test_model_picker_contracts.py @@ -786,3 +786,41 @@ def test_override_writes_are_ordered_per_model(): assert "previous .catch(() => {}) .then(() => sendModelOverride(" in src # Only the last writer clears the slot, or a queue still building loses order. assert "if (writesByKey.get(key) === write) { writesByKey.delete(key); }" in src + + +def test_backfill_skips_future_schema_local_records(): + """loadPerModelConfig refuses to apply a record written by a newer Studio and + eviction refuses to drop one, because this client cannot interpret that + schema. The enumeration the backfill uses had no such guard, so it would + persist this client's partial reading server-side and an API-triggered load + would then apply settings the same client will not apply locally. + """ + src = " ".join( + _read("features/model-picker/model-config/per-model-config.ts").split() + ) + listing = src[src.index("export function listPerModelConfigs()") :] + assert "storedConfigVersion(raw) > STORAGE_SCHEMA_VERSION" in listing[:900] + + +def test_detail_settings_need_a_resolved_quant(): + """The on-device card passes a null variant while its own lookup is pending + or after it failed. Opening the editor then saves a bare-model config, which + the picker never finds because it matches variants exactly, while the API's + bare-key fallback would apply it. openModelSettings already refuses; this + entry point has to refuse the same way.""" + src = " ".join(_read("features/hub/hub-page.tsx").split()) + guard = "if ( !ggufVariant && selectedModel.isGguf && selectedModel.requiresVariant )" + assert guard in src + assert src.count("Couldn't determine which quant to configure.") == 2 + + +def test_a_failed_detail_fetch_is_retried(): + """A terminal row's updated_at never advances and selectedIsMissing stays + true, so a fetch that failed had nothing left to re-run the effect and the + payload stayed unavailable until another row was selected. The retry is + bounded because the usual failure is an entry aged out of the ring buffer, + which never arrives however often it is asked for.""" + src = " ".join(_read("features/api-monitor/api-monitor-page.tsx").split()) + assert "const DETAIL_FETCH_ATTEMPTS = 3;" in src + assert "const detailInFlight = selectedId_ != null && loadingDetails.has(selectedId_);" in src + assert "if (attemptsRef.current.count >= DETAIL_FETCH_ATTEMPTS) { return; }" in src From 65c34fce3929120728bf7ee0f773b5ee9fd0f2a8 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 28 Jul 2026 16:19:39 +0000 Subject: [PATCH 29/75] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- tests/studio/test_model_picker_contracts.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/tests/studio/test_model_picker_contracts.py b/tests/studio/test_model_picker_contracts.py index 06d8a00082..8d0406caca 100644 --- a/tests/studio/test_model_picker_contracts.py +++ b/tests/studio/test_model_picker_contracts.py @@ -795,9 +795,7 @@ def test_backfill_skips_future_schema_local_records(): persist this client's partial reading server-side and an API-triggered load would then apply settings the same client will not apply locally. """ - src = " ".join( - _read("features/model-picker/model-config/per-model-config.ts").split() - ) + src = " ".join(_read("features/model-picker/model-config/per-model-config.ts").split()) listing = src[src.index("export function listPerModelConfigs()") :] assert "storedConfigVersion(raw) > STORAGE_SCHEMA_VERSION" in listing[:900] From 68c2d6caa8e6233759fbdb140ad7c40fe409125f Mon Sep 17 00:00:00 2001 From: danielhanchen Date: Tue, 28 Jul 2026 16:47:28 +0000 Subject: [PATCH 30/75] Point the lifecycle contracts at the module the labels moved to Extracting the labels out of the monitor page left two contracts asserting they were still in it, so the staged run went red on all three platforms. They read the new module now, and the page contract additionally pins that it imports from there rather than redefining them. --- .../test_usage_examples_model_source_contract.py | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/tests/studio/test_usage_examples_model_source_contract.py b/tests/studio/test_usage_examples_model_source_contract.py index 42bbc67cb6..3f83db9b4a 100644 --- a/tests/studio/test_usage_examples_model_source_contract.py +++ b/tests/studio/test_usage_examples_model_source_contract.py @@ -136,6 +136,12 @@ def test_usage_examples_has_no_duplicate_auto_switch_control(): # configuration and links across; these contracts follow the behaviour, not the # old file. API_MONITOR_TSX = REPO / "studio/frontend/src/features/api-monitor/api-monitor-page.tsx" +# The lifecycle labels live in their own module: the overlay is mounted from +# __root.tsx, so importing them from the page pulled the whole page into the +# eager bundle and undid the route's lazyRouteComponent. +API_MONITOR_LIFECYCLE_TS = ( + REPO / "studio/frontend/src/features/api-monitor/lifecycle.ts" +) MONITOR_LINK_TSX = SETTINGS / "components/monitor-link.tsx" @@ -153,12 +159,14 @@ def test_api_monitor_history_does_not_reorder_under_the_reader(): def test_api_monitor_renders_lifecycle_rows(): src = API_MONITOR_TSX.read_text(encoding = "utf-8") - assert "export function isLifecycleEntry(" in src - assert 'entry.kind === "lifecycle"' in src + labels = API_MONITOR_LIFECYCLE_TS.read_text(encoding = "utf-8") + assert "export function isLifecycleEntry(" in labels + assert 'entry.kind === "lifecycle"' in labels for label in ("Loading model", "Model loaded", "Model unloaded"): - assert label in src + assert label in labels # A lifecycle row has no prompt or reply, so it is not selectable for detail. assert "if (isLifecycleEntry(entry)) {" in src + assert 'from "./lifecycle"' in src def test_auto_switch_section_sits_above_the_usage_examples(): @@ -174,7 +182,7 @@ EN_TS = REPO / "studio/frontend/src/i18n/locales/en.ts" def test_api_monitor_renders_download_rows(): - src = API_MONITOR_TSX.read_text(encoding = "utf-8") + src = API_MONITOR_LIFECYCLE_TS.read_text(encoding = "utf-8") assert 'entry.event === "download"' in src for label in ("Downloading model", "Model downloaded", "Model download failed"): assert label in src From 2b2705bcdc4d8f0bab8f25fcd601c7ef478db1f5 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 28 Jul 2026 16:48:09 +0000 Subject: [PATCH 31/75] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- tests/studio/test_usage_examples_model_source_contract.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/tests/studio/test_usage_examples_model_source_contract.py b/tests/studio/test_usage_examples_model_source_contract.py index 3f83db9b4a..5559672f07 100644 --- a/tests/studio/test_usage_examples_model_source_contract.py +++ b/tests/studio/test_usage_examples_model_source_contract.py @@ -139,9 +139,7 @@ API_MONITOR_TSX = REPO / "studio/frontend/src/features/api-monitor/api-monitor-p # The lifecycle labels live in their own module: the overlay is mounted from # __root.tsx, so importing them from the page pulled the whole page into the # eager bundle and undid the route's lazyRouteComponent. -API_MONITOR_LIFECYCLE_TS = ( - REPO / "studio/frontend/src/features/api-monitor/lifecycle.ts" -) +API_MONITOR_LIFECYCLE_TS = REPO / "studio/frontend/src/features/api-monitor/lifecycle.ts" MONITOR_LINK_TSX = SETTINGS / "components/monitor-link.tsx" From 748b5315287a80fdf9e532f1d2e2900bc08cbaf8 Mon Sep 17 00:00:00 2001 From: danielhanchen Date: Tue, 28 Jul 2026 16:57:26 +0000 Subject: [PATCH 32/75] Re-read local state before backfilling, and stop advertising Ollama as API loadable The backfill wrote the snapshot it took before fetchModelOverrides resolved, so a save or a forget during that round trip was undone: the write is queued behind the interactive one and commits last, leaving the browser showing the new settings while an API load applied the old ones. Each write now re-reads the model's current local config and skips it if it has gone or gone back to defaults. Verified against the real module under node: the write carries maxSeqLength 9999 where it previously carried the stale 1000. target.isGguf was also standing in for "an API request can load this". It cannot for an Ollama model: local_model_resolver skips Ollama's scanner on purpose, so those models are never in the auto-switch index, yet the mirror ran and the settings page told the user the API would apply them. The target now carries apiLoadable, set from the inventory source the row already has, and both the mirror and that sentence read it. --- .../hub/catalog/hub-model-settings-view.tsx | 10 +++--- studio/frontend/src/features/hub/hub-page.tsx | 4 +++ .../api/migrate-model-overrides.ts | 23 ++++++++++-- .../components/model-config-page.tsx | 36 ++++++++++--------- .../components/model-selector/types.ts | 9 +++++ tests/studio/test_model_picker_contracts.py | 24 +++++++++++-- 6 files changed, 80 insertions(+), 26 deletions(-) diff --git a/studio/frontend/src/features/hub/catalog/hub-model-settings-view.tsx b/studio/frontend/src/features/hub/catalog/hub-model-settings-view.tsx index 30176a7d90..72686843f4 100644 --- a/studio/frontend/src/features/hub/catalog/hub-model-settings-view.tsx +++ b/studio/frontend/src/features/hub/catalog/hub-model-settings-view.tsx @@ -10,9 +10,9 @@ import { ModelConfigPage, type ModelPickTarget } from "@/features/model-picker"; import type { PerModelConfig } from "@/features/model-picker"; +import { cn } from "@/lib/utils"; import { ArrowLeft01Icon, Globe02Icon } from "@hugeicons/core-free-icons"; import { HugeiconsIcon } from "@hugeicons/react"; -import { cn } from "@/lib/utils"; import { useEffect, useRef, useState } from "react"; export function HubModelSettingsView({ @@ -109,10 +109,10 @@ export function HubModelSettingsView({ />

- {/* Only a GGUF is mirrored to the server, because API auto-switch - indexes GGUFs only, so promising the API case for anything - else describes a load that cannot happen. */} - {target.isGguf + {/* Only what auto-switch can reach is mirrored to the server: it + indexes GGUFs and skips Ollama, so promising the API case for + anything else describes a load that cannot happen. */} + {(target.apiLoadable ?? target.isGguf) ? "Saved settings apply everywhere this model loads, including when an OpenAI-compatible API request asks for it." : "Saved settings apply everywhere Studio loads this model."}{" "} Turn on{" "} diff --git a/studio/frontend/src/features/hub/hub-page.tsx b/studio/frontend/src/features/hub/hub-page.tsx index 7965c6b2da..714a3023ed 100644 --- a/studio/frontend/src/features/hub/hub-page.tsx +++ b/studio/frontend/src/features/hub/hub-page.tsx @@ -69,6 +69,7 @@ import type { import { useHubModelVram } from "./hooks/use-hub-model-vram"; import { useModelsSelection } from "./hooks/use-models-selection"; import { useHubInventory } from "./inventory"; +import { LOCAL_MODEL_SOURCE } from "./inventory/constants"; import { CHANNEL_TO_SECTION, type ChannelId, @@ -1309,6 +1310,9 @@ export function ModelsPage() { displayName: ggufVariant ? `${leaf} · ${ggufVariant}` : leaf, ggufVariant, isGguf: row.isGguf, + apiLoadable: + row.isGguf && + (row.kind !== "local" || row.source !== LOCAL_MODEL_SOURCE.OLLAMA), meta: { source: "local", isLora: row.modelFormat === "adapter", diff --git a/studio/frontend/src/features/model-picker/api/migrate-model-overrides.ts b/studio/frontend/src/features/model-picker/api/migrate-model-overrides.ts index 3cfa571db3..730963cb34 100644 --- a/studio/frontend/src/features/model-picker/api/migrate-model-overrides.ts +++ b/studio/frontend/src/features/model-picker/api/migrate-model-overrides.ts @@ -80,7 +80,8 @@ export async function backfillModelOverrides(): Promise { // that test despite being exactly what auto-switch does resolve; the flag // is then set and its settings stay browser-only for good. (entry) => - (entry.ggufVariant != null || entry.modelId.toLowerCase().endsWith(".gguf")) && + (entry.ggufVariant != null || + entry.modelId.toLowerCase().endsWith(".gguf")) && !isDefaultConfig(entry.config), ); if (local.length === 0) { @@ -110,8 +111,26 @@ export async function backfillModelOverrides(): Promise { if (known.has(key)) { continue; } + // Re-read rather than trusting the snapshot taken before the fetch above. + // A save or a forget during that round trip would otherwise be undone by + // this write, since it is queued behind the interactive one and commits + // last: the browser would show the new settings while an API load applied + // the old ones. + const current = listPerModelConfigs().find( + (candidate) => + normalizedOverrideKey( + modelOverrideKey(candidate.modelId, candidate.ggufVariant), + ) === key, + ); + if (!current || isDefaultConfig(current.config)) { + continue; + } try { - await putModelOverride(entry.modelId, entry.ggufVariant, entry.config); + await putModelOverride( + current.modelId, + current.ggufVariant, + current.config, + ); } catch { failed = true; } diff --git a/studio/frontend/src/features/model-picker/components/model-config-page.tsx b/studio/frontend/src/features/model-picker/components/model-config-page.tsx index ec02532217..adf97553df 100644 --- a/studio/frontend/src/features/model-picker/components/model-config-page.tsx +++ b/studio/frontend/src/features/model-picker/components/model-config-page.tsx @@ -32,6 +32,7 @@ import { useRef, useState, } from "react"; +import { syncModelOverride } from "../api/model-overrides"; import { useDefaultChatTemplate, useModelMaxPositionEmbeddings, @@ -55,7 +56,6 @@ import { resolveInitialConfig, savePerModelConfig, } from "../model-config/per-model-config"; -import { syncModelOverride } from "../api/model-overrides"; import { ChatTemplateEditorDialog } from "./chat-template-editor-dialog"; import type { ModelPickTarget } from "./model-selector/types"; import { @@ -74,14 +74,16 @@ const SELECT_TRIGGER_CLASS = `grid h-8 min-w-0 grid-cols-[minmax(0,1fr)_auto] it const NUMBER_INPUT_CLASS = `h-8 w-[92px] ${CONTROL_SURFACE} pl-3 pr-2 py-0 text-right text-ui-13 font-medium text-nav-fg outline-none focus-visible:ring-0`; const KV_CACHE_DTYPE_DEFAULT = "f16"; -const SPECULATIVE_TYPE_LABELS: Record<(typeof SPECULATIVE_TYPES)[number], string> = - { - auto: "Auto", - mtp: "MTP", - ngram: "Ngram", - "mtp+ngram": "MTP+Ngram", - off: "Off", - }; +const SPECULATIVE_TYPE_LABELS: Record< + (typeof SPECULATIVE_TYPES)[number], + string +> = { + auto: "Auto", + mtp: "MTP", + ngram: "Ngram", + "mtp+ngram": "MTP+Ngram", + off: "Off", +}; function hasNonDefaultAdvanced(config: PerModelConfig): boolean { return ( @@ -352,8 +354,8 @@ function GpuMemorySettings({ info={ <> Layers to keep on the GPU (--gpu-layers); the rest run on CPU. - Auto lets llama.cpp size the split (and the context) to fit VRAM. - At the maximum, the whole model is on the GPU. + Auto lets llama.cpp size the split (and the context) to fit + VRAM. At the maximum, the whole model is on the GPU. } /> @@ -738,8 +740,7 @@ export function ModelConfigPage({ ), maxContext, ); - const setContextLength = (v: number) => - update({ customContextLength: v }); + const setContextLength = (v: number) => update({ customContextLength: v }); const baseline = loadedConfig ?? DEFAULT_PER_MODEL_CONFIG; const atBaseline = perModelConfigsEqual(config, baseline); // An explicit customContextLength equal to the native ceiling is still an @@ -888,10 +889,11 @@ export function ModelConfigPage({ // Skipped when the local write failed (quota, a future-schema entry): the // browser and the server would otherwise permanently disagree about this // model, with no way for the user to tell which one the next load used. - // GGUF only: the API auto-switch resolver indexes GGUFs, so mirroring a - // safetensors config to the server would advertise settings on the monitor's - // "applied on API load" list that no API request can ever apply. - if (!saveFailed && target.isGguf) { + // Auto-switch reach, not just GGUF-ness: the resolver indexes GGUFs and + // skips Ollama's scanner, so mirroring either a safetensors config or an + // Ollama one would advertise settings on the monitor's "applied on API load" + // list that no API request can ever apply. + if (!saveFailed && (target.apiLoadable ?? target.isGguf)) { syncModelOverride( target.id, target.ggufVariant, diff --git a/studio/frontend/src/features/model-picker/components/model-selector/types.ts b/studio/frontend/src/features/model-picker/components/model-selector/types.ts index 9adf2d899e..e92a7f3e57 100644 --- a/studio/frontend/src/features/model-picker/components/model-selector/types.ts +++ b/studio/frontend/src/features/model-picker/components/model-selector/types.ts @@ -49,6 +49,15 @@ export interface ModelPickTarget { displayName: string; ggufVariant?: string | null; isGguf: boolean; + /** + * Whether an OpenAI-compatible request can actually load this model. + * + * Not the same as isGguf: local_model_resolver skips Ollama's scanner, so an + * Ollama GGUF is never in the auto-switch index and no API request can resolve + * it. Mirroring its settings would advertise a load that cannot happen. + * Defaults to isGguf where a caller does not know. + */ + apiLoadable?: boolean; meta: ModelSelectorChangeMeta; } diff --git a/tests/studio/test_model_picker_contracts.py b/tests/studio/test_model_picker_contracts.py index 8d0406caca..bc70549b1d 100644 --- a/tests/studio/test_model_picker_contracts.py +++ b/tests/studio/test_model_picker_contracts.py @@ -660,7 +660,7 @@ def test_only_gguf_configs_are_mirrored_to_the_server(): loads safetensors models and must honour their config. """ src = " ".join(_read("features/model-picker/components/model-config-page.tsx").split()) - assert "if (!saveFailed && target.isGguf) { syncModelOverride(" in src + assert "if (!saveFailed && (target.apiLoadable ?? target.isGguf)) { syncModelOverride(" in src # The local save is not behind the same gate. assert "if (remember) { saveFailed = !savePerModelConfig(" in src @@ -731,7 +731,7 @@ def test_api_reach_copy_is_limited_to_gguf_models(): apply to an API request describes a load that cannot happen. """ src = " ".join(_read("features/hub/catalog/hub-model-settings-view.tsx").split()) - assert "{target.isGguf ?" in src + assert "{(target.apiLoadable ?? target.isGguf)" in src assert "Saved settings apply everywhere Studio loads this model." in src @@ -822,3 +822,23 @@ def test_a_failed_detail_fetch_is_retried(): assert "const DETAIL_FETCH_ATTEMPTS = 3;" in src assert "const detailInFlight = selectedId_ != null && loadingDetails.has(selectedId_);" in src assert "if (attemptsRef.current.count >= DETAIL_FETCH_ATTEMPTS) { return; }" in src + + +def test_ollama_models_are_not_advertised_as_api_loadable(): + """local_model_resolver skips Ollama's scanner, so an Ollama GGUF is never in + the auto-switch index and no OpenAI request can resolve it. target.isGguf is + still true for one, so gating on that alone mirrored settings the API can + never apply and told the user the opposite.""" + types_src = " ".join( + _read("features/model-picker/components/model-selector/types.ts").split() + ) + assert "apiLoadable?: boolean;" in types_src + hub = " ".join(_read("features/hub/hub-page.tsx").split()) + assert 'row.source !== LOCAL_MODEL_SOURCE.OLLAMA' in hub + assert 'apiLoadable:' in hub + backend = ( + WORKDIR / "studio" / "backend" / "core" / "inference" / "local_model_resolver.py" + ).read_text(encoding = "utf-8") + assert "Ollama's\n scanner is skipped" in backend or "scanner is skipped" in backend, ( + "the rule this mirrors" + ) From e57a5a9ce2e35f11340da82b835b40f97a6eb0ee Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 28 Jul 2026 17:00:55 +0000 Subject: [PATCH 33/75] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- tests/studio/test_model_picker_contracts.py | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/tests/studio/test_model_picker_contracts.py b/tests/studio/test_model_picker_contracts.py index bc70549b1d..0387ab7462 100644 --- a/tests/studio/test_model_picker_contracts.py +++ b/tests/studio/test_model_picker_contracts.py @@ -829,16 +829,14 @@ def test_ollama_models_are_not_advertised_as_api_loadable(): the auto-switch index and no OpenAI request can resolve it. target.isGguf is still true for one, so gating on that alone mirrored settings the API can never apply and told the user the opposite.""" - types_src = " ".join( - _read("features/model-picker/components/model-selector/types.ts").split() - ) + types_src = " ".join(_read("features/model-picker/components/model-selector/types.ts").split()) assert "apiLoadable?: boolean;" in types_src hub = " ".join(_read("features/hub/hub-page.tsx").split()) - assert 'row.source !== LOCAL_MODEL_SOURCE.OLLAMA' in hub - assert 'apiLoadable:' in hub + assert "row.source !== LOCAL_MODEL_SOURCE.OLLAMA" in hub + assert "apiLoadable:" in hub backend = ( WORKDIR / "studio" / "backend" / "core" / "inference" / "local_model_resolver.py" ).read_text(encoding = "utf-8") - assert "Ollama's\n scanner is skipped" in backend or "scanner is skipped" in backend, ( - "the rule this mirrors" - ) + assert ( + "Ollama's\n scanner is skipped" in backend or "scanner is skipped" in backend + ), "the rule this mirrors" From 1796ebf3302315c407617a4fef4acc4b710d279a Mon Sep 17 00:00:00 2001 From: danielhanchen Date: Tue, 28 Jul 2026 17:07:11 +0000 Subject: [PATCH 34/75] Key the Hub's per-model settings by the repo id, not by the load path A repo cached outside the active HF cache reports load_id as its snapshot path (cache_inventory), which is what the loader needs, but the chat picker and the auto-switch index both name that repo by repo_id. The new Hub settings page was saving under the load id, so the settings landed on a key no other load reads: the picker, an auto-load and an OpenAI-compatible request all fell back to defaults, and a server override already stored under the repo id could win against the save. ModelPickTarget now carries configId for the case where the storage identity is not the loadable one. Every read, write and server mirror in ModelConfigPage uses it; the chat template and GGUF header probes keep target.id, since they have to open the model. The Hub sets it for cache rows and resolves its own load through the same helper, so a config saved from the settings page is the one a later load finds. Rows whose load id is already their identity, which is every local row and every repo in the active cache, are unaffected. Verified against the real per-model-config module under node: saved under the snapshot path, a picker read reports remembered=false and the default max sequence length; saved under the repo id it reports remembered=true and 8192. --- studio/frontend/src/features/hub/hub-page.tsx | 33 ++++++++++++++++-- .../components/model-config-page.tsx | 12 ++++--- .../components/model-selector/types.ts | 10 ++++++ tests/studio/test_model_picker_contracts.py | 34 +++++++++++++++++++ 4 files changed, 82 insertions(+), 7 deletions(-) diff --git a/studio/frontend/src/features/hub/hub-page.tsx b/studio/frontend/src/features/hub/hub-page.tsx index 714a3023ed..ae152fbf3a 100644 --- a/studio/frontend/src/features/hub/hub-page.tsx +++ b/studio/frontend/src/features/hub/hub-page.tsx @@ -109,8 +109,23 @@ import type { ModelsTab, ResourceTypeFilter, SelectedModelView, + SelectedResourceRef, } from "./types"; +// What per-model settings are keyed by, which is not always what the loader is +// handed: a repo cached outside the active HF cache loads by snapshot path, +// while the chat picker (toCachedModelRepo) and the auto-switch index both key +// it by repo id. Saving under the path would leave the settings where no other +// load looks for them. Local rows are keyed by their load id in both places, so +// they keep it. +function modelConfigIdentity( + kind: SelectedModelView["kind"], + resource: SelectedResourceRef, +): string { + if (kind !== "cache") return resource.runId; + return resource.repoId ?? resource.runId; +} + const MODELS_TAB_STORAGE_KEY = "unsloth.hub.modelsTab"; const ALL_MODELS_VIEW_STORAGE_KEY = "unsloth.hub.allModelsView"; const INVENTORY_SORT_STORAGE_KEY = "unsloth.hub.inventorySort"; @@ -1192,7 +1207,10 @@ export function ModelsPage() { (opts: ModelLoadOptions, isDownloaded: boolean) => { if (!selectedModel) return; const runId = selectedModel.resource.runId; - const resolvedConfig = resolveInitialConfig(runId, opts.ggufVariant); + const resolvedConfig = resolveInitialConfig( + modelConfigIdentity(selectedModel.kind, selectedModel.resource), + opts.ggufVariant, + ); const rememberedConfig = resolvedConfig.remembered ? resolvedConfig.config : null; @@ -1304,9 +1322,13 @@ export function ModelsPage() { if (settingsOpenSeq.current !== openSeq) { return; } - const leaf = id.split(/[\\/]/).filter(Boolean).pop() ?? id; + // A repo in a previous cache loads by snapshot path, so `id` ends in the + // revision hash; name the row by what the user calls it. + const configId = row.kind === "cache" ? row.repoId : id; + const leaf = configId.split(/[\\/]/).filter(Boolean).pop() ?? configId; setSettingsTarget({ id, + configId, displayName: ggufVariant ? `${leaf} · ${ggufVariant}` : leaf, ggufVariant, isGguf: row.isGguf, @@ -1389,9 +1411,14 @@ export function ModelsPage() { // still be pending, and it must not land on top of this one. settingsOpenSeq.current += 1; const id = selectedModel.resource.runId; - const leaf = id.split(/[\\/]/).filter(Boolean).pop() ?? id; + const configId = modelConfigIdentity( + selectedModel.kind, + selectedModel.resource, + ); + const leaf = configId.split(/[\\/]/).filter(Boolean).pop() ?? configId; setSettingsTarget({ id, + configId, displayName: ggufVariant ? `${leaf} · ${ggufVariant}` : leaf, ggufVariant, isGguf: selectedModel.isGguf, diff --git a/studio/frontend/src/features/model-picker/components/model-config-page.tsx b/studio/frontend/src/features/model-picker/components/model-config-page.tsx index adf97553df..635164875b 100644 --- a/studio/frontend/src/features/model-picker/components/model-config-page.tsx +++ b/studio/frontend/src/features/model-picker/components/model-config-page.tsx @@ -611,8 +611,12 @@ export function ModelConfigPage({ const loadedMaxContextLength = useChatRuntimeStore( (s) => s.ggufMaxContextLength, ); + // What the settings are stored under, which is not always what loads: see + // ModelPickTarget.configId. Every read, write and mirror below uses it; the + // probes keep target.id, since they have to open the model. + const configId = target.configId ?? target.id; const resolveInitial = () => { - const resolved = resolveInitialConfig(target.id, target.ggufVariant); + const resolved = resolveInitialConfig(configId, target.ggufVariant); if (loadedConfig) { return { config: loadedConfig, remembered: resolved.remembered }; } @@ -873,13 +877,13 @@ export function ModelConfigPage({ const evicted: { modelId: string; ggufVariant: string | null }[] = []; if (remember) { saveFailed = !savePerModelConfig( - target.id, + configId, target.ggufVariant, effectiveRuntimeConfig, evicted, ); } else { - saveFailed = !deletePerModelConfig(target.id, target.ggufVariant); + saveFailed = !deletePerModelConfig(configId, target.ggufVariant); } // Mirror to the server so an OpenAI-compatible API request that loads this // model gets these exact settings, not app defaults. Best-effort and @@ -895,7 +899,7 @@ export function ModelConfigPage({ // list that no API request can ever apply. if (!saveFailed && (target.apiLoadable ?? target.isGguf)) { syncModelOverride( - target.id, + configId, target.ggufVariant, remember ? effectiveRuntimeConfig : null, ); diff --git a/studio/frontend/src/features/model-picker/components/model-selector/types.ts b/studio/frontend/src/features/model-picker/components/model-selector/types.ts index e92a7f3e57..980d9ed30e 100644 --- a/studio/frontend/src/features/model-picker/components/model-selector/types.ts +++ b/studio/frontend/src/features/model-picker/components/model-selector/types.ts @@ -58,6 +58,16 @@ export interface ModelPickTarget { * Defaults to isGguf where a caller does not know. */ apiLoadable?: boolean; + /** + * Identity the saved settings are keyed by, when that is not what loads. + * + * A repo cached outside the active HF cache loads by snapshot path, while the + * picker and the auto-switch index keep its settings under the repo id. + * Keying the save by the path would strand the settings where no load looks + * for them. Probes that need something openable (the chat template, the GGUF + * header) keep using `id`. Defaults to `id`. + */ + configId?: string; meta: ModelSelectorChangeMeta; } diff --git a/tests/studio/test_model_picker_contracts.py b/tests/studio/test_model_picker_contracts.py index 0387ab7462..b8b7fbb213 100644 --- a/tests/studio/test_model_picker_contracts.py +++ b/tests/studio/test_model_picker_contracts.py @@ -840,3 +840,37 @@ def test_ollama_models_are_not_advertised_as_api_loadable(): assert ( "Ollama's\n scanner is skipped" in backend or "scanner is skipped" in backend ), "the rule this mirrors" + + +def test_cached_repo_settings_are_keyed_by_the_repo_id(): + """A repo cached outside the active HF cache reports load_id = the snapshot + path (hub/services/cache_inventory.py), while the chat picker and the + auto-switch index key it by repo_id. Keying the Hub's settings by the load id + saved them where no other load looks, so they silently never applied.""" + config_page = " ".join( + _read("features/model-picker/components/model-config-page.tsx").split() + ) + assert "const configId = target.configId ?? target.id;" in config_page + for call in ( + "resolveInitialConfig(configId, target.ggufVariant)", + "savePerModelConfig( configId, target.ggufVariant,", + "deletePerModelConfig(configId, target.ggufVariant)", + "syncModelOverride( configId, target.ggufVariant,", + ): + assert call in config_page, call + # The probes have to open the model, so they keep the load id. + assert "useDefaultChatTemplate( target.id," in config_page + assert "model_path: target.id," in config_page + + hub = " ".join(_read("features/hub/hub-page.tsx").split()) + assert 'if (kind !== "cache") return resource.runId;' in hub + assert "return resource.repoId ?? resource.runId;" in hub + # Both openers and the Hub's own load resolve through it. + assert hub.count("modelConfigIdentity(") == 3 + assert 'const configId = row.kind === "cache" ? row.repoId : id;' in hub + assert hub.count("configId,") >= 2 + + backend = ( + WORKDIR / "studio" / "backend" / "hub" / "tests" / "test_model_services.py" + ).read_text(encoding = "utf-8") + assert 'fields["load_id"] == str(snapshot)' in backend, "the rule this mirrors" From 51cfde7b23ba6ad0aaf2bf358932fc01ad55ae71 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 28 Jul 2026 17:08:59 +0000 Subject: [PATCH 35/75] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- tests/studio/test_model_picker_contracts.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/tests/studio/test_model_picker_contracts.py b/tests/studio/test_model_picker_contracts.py index b8b7fbb213..6f926dcec2 100644 --- a/tests/studio/test_model_picker_contracts.py +++ b/tests/studio/test_model_picker_contracts.py @@ -847,9 +847,7 @@ def test_cached_repo_settings_are_keyed_by_the_repo_id(): path (hub/services/cache_inventory.py), while the chat picker and the auto-switch index key it by repo_id. Keying the Hub's settings by the load id saved them where no other load looks, so they silently never applied.""" - config_page = " ".join( - _read("features/model-picker/components/model-config-page.tsx").split() - ) + config_page = " ".join(_read("features/model-picker/components/model-config-page.tsx").split()) assert "const configId = target.configId ?? target.id;" in config_page for call in ( "resolveInitialConfig(configId, target.ggufVariant)", From 58abf4ca503663327bb8456f50c84a88f7ede23a Mon Sep 17 00:00:00 2001 From: danielhanchen Date: Tue, 28 Jul 2026 17:32:46 +0000 Subject: [PATCH 36/75] Tighten the comments across the API monitor and per-model settings work Pass over every comment this branch touches. Collapse the multi-paragraph rationales to the point they were making, drop prop docs that only restated the prop name, and reflow the rest onto fewer lines. No code changes. --- studio/backend/core/inference/api_monitor.py | 23 +-- studio/backend/routes/inference.py | 43 ++--- studio/backend/routes/settings.py | 71 +++---- studio/backend/tests/test_api_monitor.py | 12 +- .../backend/tests/test_openai_auto_switch.py | 178 ++++++++---------- .../utils/openai_auto_switch_settings.py | 102 ++++------ studio/frontend/src/app/routes/__root.tsx | 10 +- studio/frontend/src/app/routes/api.tsx | 5 +- .../api-monitor/api-monitor-overlay.tsx | 44 ++--- .../features/api-monitor/api-monitor-page.tsx | 64 +++---- .../components/saved-model-settings.tsx | 11 +- .../src/features/api-monitor/lifecycle.ts | 17 +- .../src/features/api-monitor/overlay-store.ts | 22 +-- .../features/api-monitor/use-api-monitor.ts | 57 +++--- .../frontend/src/features/chat/types/api.ts | 4 +- .../features/hub/catalog/download-card.tsx | 1 - .../hub/catalog/hub-model-settings-view.tsx | 18 +- .../hub/catalog/local-on-device-card.tsx | 6 +- .../features/hub/catalog/model-inspector.tsx | 2 +- .../hub/catalog/models-catalog-lists.tsx | 1 - .../hub/catalog/models-catalog-rows.tsx | 10 +- .../features/hub/catalog/models-catalog.tsx | 1 - studio/frontend/src/features/hub/hub-page.tsx | 102 +++++----- .../api/migrate-model-overrides.ts | 49 +++-- .../model-picker/api/model-overrides.ts | 67 +++---- .../components/model-config-page.tsx | 34 ++-- .../model-selector/model-row-menu.tsx | 3 +- .../components/model-selector/types.ts | 11 +- .../model-config/per-model-config.ts | 14 +- .../settings/components/monitor-link.tsx | 8 +- tests/studio/test_model_picker_contracts.py | 17 +- ...test_settings_compact_overflow_contract.py | 12 +- ...st_usage_examples_model_source_contract.py | 18 +- 33 files changed, 435 insertions(+), 602 deletions(-) diff --git a/studio/backend/core/inference/api_monitor.py b/studio/backend/core/inference/api_monitor.py index f18dd449cd..92104df688 100644 --- a/studio/backend/core/inference/api_monitor.py +++ b/studio/backend/core/inference/api_monitor.py @@ -50,9 +50,8 @@ class ApiMonitorEntry: started_at: float updated_at: float subject: Optional[str] = None - # True when the caller used an sk-unsloth key rather than a UI session. The - # floating panel only opens itself for these: Studio's own chat goes through - # the same endpoints, and popping the monitor open mid-chat is noise. + # True for sk-unsloth key callers, not UI sessions. The floating panel only + # auto-opens for these, so Studio's own chat does not pop it mid-chat. via_api_key: bool = False # Monotonic anchors so duration math survives wall-clock steps (NTP). started_monotonic: float = 0.0 @@ -125,11 +124,8 @@ class ApiMonitor: enabled: bool = True, ): self._entries: deque[ApiMonitorEntry] = deque() - # Shared rows one subject has cleared. A shared row belongs to everyone, - # so dropping it would erase another caller's history, but leaving it - # means "Clear log" visibly does nothing to it: the frontend reloads - # straight after and the row comes back. Hiding it per subject is the - # only thing that is both true for that caller and safe for the others. + # Shared rows one subject cleared. Deleting them would erase another + # caller's history; keeping them makes "Clear log" look broken on reload. self._hidden_shared: dict[str, set[str]] = {} self._max_entries = max(0, max_entries) self._lock = threading.Lock() @@ -153,8 +149,8 @@ class ApiMonitor: id = f"apireq_{uuid.uuid4().hex[:12]}", endpoint = endpoint, method = method, - # str(): a raw JSON body can carry any type here, and the field is - # rendered in the UI, where a non-string breaks the whole monitor. + # str(): a raw JSON body can carry any type, and a non-string + # breaks the UI that renders it. model = str(model) if model else "default", prompt = _trim(prompt, _MAX_PROMPT_CHARS), status = "running", @@ -411,9 +407,7 @@ class ApiMonitor: self._entries.clear() self._hidden_shared.clear() return - # A shared row that is still running is a load in progress, not - # history, so it stays visible; clearing the log is about what has - # already happened. + # A running shared row is a load in progress, not history, so it stays. hidden = self._hidden_shared.setdefault(subject, set()) for entry in self._entries: if entry.shared and entry.subject != subject and entry.status != "running": @@ -446,8 +440,7 @@ class ApiMonitor: kept.append(entry) terminal_seen += 1 self._entries = kept - # The hidden sets only ever name rows that exist, so they stay bounded - # by the ring buffer rather than growing for the life of the process. + # Keep hidden sets to live rows so they stay bounded by the ring buffer. live = {entry.id for entry in kept} for subject, hidden in list(self._hidden_shared.items()): hidden &= live diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.py index bc9b430619..f951f91c04 100644 --- a/studio/backend/routes/inference.py +++ b/studio/backend/routes/inference.py @@ -4354,20 +4354,14 @@ async def _maybe_auto_switch_model( if _already_serving(): _record_serving_alias() return - # Apply this model's saved launch config so an API-driven swap - # loads it exactly as the picker would: context, KV dtype, - # speculative decoding, chat template and GPU placement, not - # just the two legacy flags. Look the config up under the - # variant-qualified id first (two quants of one repo can carry - # different configs), then the bare ids. Both the advertised - # repo id and the concrete load path are tried: a local folder - # or a non-active HF cache is configured against its path. - # A standalone .gguf needs no quant sub-selection, so the - # resolver reports variant=None for it. The picker still - # keys its config by the quant label it derives from the - # filename (LocalModelInfo.format_variant), which is never - # empty, so those settings live under ":LABEL" and no - # bare key would ever reach them. + # Apply this model's saved launch config so an API swap loads + # it exactly as the picker would. Try variant-qualified keys + # first (two quants of one repo can differ), then bare ids, and + # both the repo id and the load path (a local folder or a + # non-active HF cache is configured against its path). + # A standalone .gguf resolves with variant=None, but the picker + # keys it by the quant label derived from the filename, so its + # settings live under ":LABEL" and no bare key reaches them. file_variant = None if not variant and target_id.lower().endswith(".gguf"): from hub.utils.gguf import extract_quant_label @@ -4389,7 +4383,7 @@ async def _maybe_auto_switch_model( load_kwargs.update( model_override_load_kwargs( override, - # variant is set for every GGUF the resolver returns; the + # Set for every GGUF the resolver returns; the # reload-stash path carries the quant it froze. is_gguf = bool(variant) or target_id.lower().endswith(".gguf"), ) @@ -4398,9 +4392,8 @@ async def _maybe_auto_switch_model( if saved_gpu_ids and not await _override_gpu_ids_still_resolve( saved_gpu_ids ): - # A pin saved before a GPU was removed, before a - # visibility-mask change, or on another host. Dropping the - # one dead field beats 400ing the whole load. + # Stale pin (GPU removed, mask changed, another host). + # Dropping the one dead field beats 400ing the whole load. load_kwargs.pop("gpu_ids", None) logger.warning( "Dropping saved gpu_ids %s for %s: not available here.", @@ -4418,12 +4411,10 @@ async def _maybe_auto_switch_model( current_request_counted = True, ) except HTTPException as exc: - # The pre-flight check above cannot mirror every rule the - # loader applies to gpu_ids (a Vulkan diffusion GGUF refuses - # GPU selection outright, and the rules move). Rather than - # duplicating them, retry once without the saved pin: a - # stale placement preference must never be the reason an - # API request cannot be served. + # The pre-flight check cannot mirror every gpu_ids rule the + # loader applies (a Vulkan diffusion GGUF refuses GPU + # selection outright). Retry once without the saved pin: a + # stale placement preference must never block a request. if not ( exc.status_code == 400 and load_kwargs.get("gpu_ids") @@ -4719,12 +4710,12 @@ async def _override_gpu_ids_still_resolve(gpu_ids: List[int]) -> bool: is_vulkan = LlamaCppBackend._is_vulkan_backend() if get_device() == DeviceType.XPU and not is_vulkan: - # gpu_ids is rejected outright on XPU. + # Rejected outright on XPU. return False resolved = resolve_requested_gpu_ids(gpu_ids, is_vulkan = is_vulkan) if is_vulkan and resolved: # Vulkan ordinals are their own index space, so resolve() only rejects - # malformed ones. Presence needs the same ggml probe the load does. + # malformed ones; presence needs the ggml probe the load runs. binary = LlamaCppBackend._find_llama_server_binary() if binary: probed = { diff --git a/studio/backend/routes/settings.py b/studio/backend/routes/settings.py index 9639cee82d..a5e3cab832 100644 --- a/studio/backend/routes/settings.py +++ b/studio/backend/routes/settings.py @@ -132,16 +132,14 @@ class OpenAIAutoSwitchResponse(BaseModel): auto_download_model: bool = DEFAULT_OPENAI_AUTO_DOWNLOAD_ENABLED -# A quant suffix, as modelOverrideKey builds it. Matched against the loader's own -# quant pattern rather than a length heuristic: a POSIX path may legitimately -# contain a colon ("/models/foo:bar.gguf"), and treating "bar.gguf" as a quant -# would graft an unrelated model's launch flags onto this one. +# A quant suffix, as modelOverrideKey builds it. Matched against the loader's quant +# pattern, not a length heuristic: a POSIX path may hold a colon +# ("/models/foo:bar.gguf") and would otherwise inherit another model's flags. _MAX_VARIANT_SUFFIX_LEN = 64 -# A local model's id is its filesystem path, optionally with a quant suffix, and +# A local model's id is its path plus an optional quant suffix, and # LoadRequest.model_path is unbounded. A limit under PATH_MAX would 422 the server -# sync while the local save succeeded, leaving the UI showing settings the API -# never applies. +# sync while the local save succeeded. MAX_MODEL_OVERRIDE_KEY_LEN = 4096 + 1 + _MAX_VARIANT_SUFFIX_LEN @@ -156,9 +154,8 @@ class ModelOverridePayload(BaseModel): """ model_id: str = Field(..., min_length = 1, max_length = MAX_MODEL_OVERRIDE_KEY_LEN) - # None means "leave the stored value alone": the settings UI has no control - # for launch flags, so a save from it must not wipe flags set through this - # API. An explicit [] clears them (that is how "forget this model" arrives). + # None means "leave the stored value alone": the settings UI has no control for + # launch flags and must not wipe them. An explicit [] clears them (forget). llama_extra_args: Optional[list[str]] = None # ge=1: 0 is not a valid sequence length, and the setter drops a falsy value, # so reject it at the boundary instead of accepting then silently discarding it. @@ -168,25 +165,22 @@ class ModelOverridePayload(BaseModel): speculative_type: Optional[str] = Field(default = None, max_length = 32) spec_draft_n_max: Optional[int] = Field(default = None, ge = 1, le = 16) tensor_parallel: bool = False - # Validated in bytes below, not by max_length: pydantic counts characters, - # so a multi-byte template could pass here and then be silently dropped by - # the normalizer (which measures UTF-8) while the request still returned 200. + # Validated in bytes below, not by max_length: pydantic counts characters, so a + # multi-byte template would pass here and be dropped by the UTF-8 normalizer. chat_template_override: Optional[str] = None gpu_memory_mode: Optional[Literal["auto", "manual"]] = None # -1 is Auto (llama.cpp --fit sizes the offload); the normalizer treats it as unset. gpu_layers: Optional[int] = Field(default = None, ge = -1, le = 1024) n_cpu_moe: Optional[int] = Field(default = None, ge = 0, le = 1024) gpu_ids: Optional[list[int]] = None - # Explicit intent. A save whose config is entirely default carries no fields - # at all, which is indistinguishable from "forget this model" by shape alone. - # None keeps the original contract: a bare model_id means remove. + # Explicit intent: an all-default save carries no fields, which is shape + # identical to "forget this model". None keeps the legacy contract. remove: Optional[bool] = None @field_validator("chat_template_override") @classmethod def _limit_chat_template_bytes(cls, value: Optional[str]) -> Optional[str]: - # Mirrors LoadRequest.normalize_blank_chat_template_override so the same - # template is accepted or rejected identically on both paths. + # Mirrors LoadRequest.normalize_blank_chat_template_override. if value is None: return None size = chat_template_byte_length(value) @@ -357,10 +351,8 @@ def _bare_model_id(model_id: str) -> Optional[str]: """``repo`` for a ``repo:QUANT`` key, or None when there is no quant suffix.""" from utils.openai_auto_switch_settings import split_quant_suffix - # Must actually look like a quant, not just like a short path segment. The - # label may carry a bits-per-weight modifier ("IQ4_XS-3.53bpw"), which keeps - # two files at the same base quant distinct, and a .gguf with no recognized - # token is labelled by its stem, so both forms count. + # Must look like a quant, not just a short path segment. Both a bits-per-weight + # modifier ("IQ4_XS-3.53bpw") and a stem fallback label count. split = split_quant_suffix(model_id) return split[0] if split is not None else None @@ -373,10 +365,8 @@ def update_openai_auto_switch_override( from utils.openai_auto_switch_settings import get_model_override try: - # A payload carrying only model_id is the documented "remove", so it - # wipes everything. Otherwise it is a real save, and omitted launch flags - # are carried over from the stored entry (the settings UI cannot express - # them and must not delete them). + # Only model_id is the documented "remove". Otherwise omitted launch flags + # carry over from the stored entry, since the settings UI cannot express them. requested_extra_args = payload.llama_extra_args saved_fields = payload.model_dump( exclude = {"model_id", "llama_extra_args", "remove"}, exclude_none = True @@ -390,33 +380,24 @@ def update_openai_auto_switch_override( if requested_extra_args is None and not is_removal: requested_extra_args = get_model_override(payload.model_id).get("llama_extra_args") if requested_extra_args is None: - # First per-quant save for a model whose flags were stored under the - # bare repo id. Auto-switch prefers the qualified entry, so without - # this the flags are silently dropped and no UI can restore them. + # First per-quant save for flags stored under the bare repo id. + # Auto-switch prefers the qualified entry, so carry them over. bare_id = _bare_model_id(payload.model_id) if bare_id: requested_extra_args = get_model_override(bare_id).get("llama_extra_args") - # Not validated on an explicit remove: nothing is stored, so the only - # effect would be a 400 that leaves the override in place, which is the - # opposite of what remove means. A stale form still carrying a rejected - # flag must not be able to block forgetting a model. + # Not validated on an explicit remove: nothing is stored, so a 400 would only + # leave the override in place. A stale flag must not block forgetting. extra_args = [] if payload.remove is True else validate_extra_args(requested_extra_args) if payload.remove is True: - # An explicit remove wins over anything else in the payload: a stale - # form field must not turn "forget this model" into an update that - # keeps it. Only the explicit flag short-circuits; the legacy - # inferred path still just gates launch-flag carry-over. - # Remove the key a load would actually resolve to, not just the - # literal one sent: the browser normalizes casing before storing, so - # the two can differ and a stale entry would survive forgetting. + # An explicit remove wins over any other field in the payload. Remove the + # key a load resolves to, not the literal one sent: the browser normalizes + # casing before storing, so a stale entry would survive forgetting. target_id = resolve_model_override_key(payload.model_id) or payload.model_id set_model_override(target_id, llama_extra_args = [], max_seq_length = None) else: - # Save under the key a load would resolve to, for the same reason the - # removal branch does. The browser normalizes casing before storing, - # so saving the literal id leaves a second entry for one model, and - # two equivalent keys make every other casing ambiguous: the lookup - # then matches neither and the model silently loses its settings. + # Save under the key a load resolves to, as the removal branch does. + # Saving the literal id leaves two keys for one model, which makes every + # other casing ambiguous and silently loses the settings. target_id = resolve_model_override_key(payload.model_id) or payload.model_id set_model_override( target_id, diff --git a/studio/backend/tests/test_api_monitor.py b/studio/backend/tests/test_api_monitor.py index 821861457c..2b611928e9 100644 --- a/studio/backend/tests/test_api_monitor.py +++ b/studio/backend/tests/test_api_monitor.py @@ -261,9 +261,8 @@ def test_api_monitor_append_reply_exact_cap_then_more_marks_truncated(): def test_api_monitor_clear_is_scoped_to_one_subject(): - # Every other read on the monitor is subject-scoped. An unscoped clear from - # the route would let one caller erase another's history and zero their - # active count in the middle of a generation. + # Every other read is subject-scoped; an unscoped clear from the route would let + # one caller erase another's history mid-generation. monitor = ApiMonitor(max_entries = 4) alice = monitor.start( endpoint = "/v1/chat/completions", @@ -292,9 +291,8 @@ def test_api_monitor_clear_is_scoped_to_one_subject(): def test_api_monitor_records_whether_the_caller_used_an_api_key(): - # Studio's own chat hits these endpoints with a session JWT. The floating - # panel keys its auto-open off this flag, so mislabelling in-app chat as API - # traffic pops the panel over the composer mid-conversation. + # Studio's own chat hits these endpoints with a session JWT, and the floating + # panel keys its auto-open off this flag, so mislabelling it pops the panel. monitor = ApiMonitor(max_entries = 4) ui = monitor.start( endpoint = "/api/inference/chat", @@ -494,7 +492,7 @@ def test_clear_hides_shared_lifecycle_rows_for_that_caller_only(): monitor.clear(subject = "alice") assert monitor.snapshot(subject = "alice") == [] - # Bob's view is untouched: the row is hidden for alice, not deleted. + # Hidden for alice, not deleted, so bob's view is untouched. assert {e["id"] for e in monitor.snapshot(subject = "bob")} == {shared} assert monitor.get(shared, subject = "alice") is None assert monitor.get(shared, subject = "bob") is not None diff --git a/studio/backend/tests/test_openai_auto_switch.py b/studio/backend/tests/test_openai_auto_switch.py index 80d076a865..73a6e6fb4f 100644 --- a/studio/backend/tests/test_openai_auto_switch.py +++ b/studio/backend/tests/test_openai_auto_switch.py @@ -4150,9 +4150,7 @@ def test_env_idle_below_floor_is_clamped(monkeypatch): assert settings.get_auto_unload_idle_seconds() == 0 -# --------------------------------------------------------------------------- # Per-model launch config: normalization, LoadRequest mapping, key resolution. -# --------------------------------------------------------------------------- def test_normalize_model_override_drops_unusable_fields_and_keeps_the_rest(): @@ -4178,8 +4176,8 @@ def test_normalize_model_override_drops_unusable_fields_and_keeps_the_rest(): def test_normalize_model_override_rejects_oversized_chat_template(): small = settings.normalize_model_override({"chat_template_override": "{{ bos }}"}) assert small["chat_template_override"] == "{{ bos }}" - # The limit is bytes, not characters: a multi-byte template just under the - # character limit can still be over the byte limit. + # The limit is bytes, not characters, so a multi-byte template just under the + # character limit can still be over. huge = "é" * settings.MAX_CHAT_TEMPLATE_OVERRIDE_BYTES assert "chat_template_override" not in settings.normalize_model_override( {"chat_template_override": huge} @@ -4189,15 +4187,15 @@ def test_normalize_model_override_rejects_oversized_chat_template(): def test_spec_draft_n_max_only_stored_for_mtp_modes(): mtp = settings.normalize_model_override({"speculative_type": "mtp", "spec_draft_n_max": 4}) assert mtp["spec_draft_n_max"] == 4 - # A non-MTP mode ignores the draft count at load time, so storing it would - # show the user an edit that never takes effect. + # A non-MTP mode ignores the draft count, so storing it shows an edit that + # never takes effect. ngram = settings.normalize_model_override({"speculative_type": "ngram", "spec_draft_n_max": 4}) assert "spec_draft_n_max" not in ngram def test_resolve_fit_max_seq_length_hands_sizing_to_fit_under_manual_auto_layers(): - # Manual GPU memory with Auto layers means llama.cpp --fit owns the context, - # so the load sends the context pin (or 0), not the stored max seq length. + # Manual GPU memory with Auto layers hands the context to llama.cpp --fit, so + # the load sends the context pin (or 0), not the stored max seq length. override = {"gpu_memory_mode": "manual", "max_seq_length": 8192} assert settings.resolve_fit_max_seq_length(override, is_gguf = True) == 0 assert ( @@ -4228,8 +4226,8 @@ def test_model_override_load_kwargs_gates_gpu_placement_on_gguf(): assert gguf["gpu_layers"] == 20 assert gguf["gpu_ids"] == [0, 1] - # A safetensors model loads through HF auto-placement; inheriting a GGUF GPU - # pin here would silently change where the weights land. + # A safetensors model loads through HF auto-placement, so a GGUF GPU pin would + # silently change where the weights land. safetensors = settings.model_override_load_kwargs(override, is_gguf = False) assert safetensors["max_seq_length"] == 4096 assert "gpu_layers" not in safetensors @@ -4237,14 +4235,14 @@ def test_model_override_load_kwargs_gates_gpu_placement_on_gguf(): assert "n_cpu_moe" not in safetensors assert "gpu_memory_mode" not in safetensors - # Every key it produces has to be a real LoadRequest field, or the load call - # raises TypeError at the moment the user's request arrives. + # Every key must be a real LoadRequest field, or the load raises TypeError when + # the user's request arrives. LoadRequest(model_path = "unsloth/B-GGUF", **gguf) def test_auto_switch_prefers_variant_qualified_override(monkeypatch): - # Settings are saved per quant, so Q4_K_M and Q8_0 of the same repo are - # different entries; the bare repo id is only the fallback. + # Settings are per quant, so Q4_K_M and Q8_0 of one repo are separate entries + # and the bare repo id is only the fallback. backend = _FakeBackend(None) rec = _LoadRecorder(backend) _wire( @@ -4284,9 +4282,8 @@ def test_auto_switch_falls_back_to_bare_repo_override(monkeypatch): def test_override_route_preserves_launch_flags_across_a_settings_only_update(monkeypatch): - # The settings page has no control for llama_extra_args, so saving from it - # omits the field. Omitted must mean "leave it alone", or every save from the - # UI would quietly wipe flags set elsewhere. + # The settings page has no control for llama_extra_args, so it omits the field. + # Omitted must mean "leave it alone", or every UI save wipes flags set elsewhere. import routes.settings as settings_route _mock_override_store(monkeypatch) @@ -4304,8 +4301,8 @@ def test_override_route_preserves_launch_flags_across_a_settings_only_update(mon assert entry["llama_extra_args"] == ["--flash-attn"] assert entry["max_seq_length"] == 4096 - # An explicit empty list is how the UI says "forget this model", and with no - # other fields left that removes the entry outright. + # An explicit empty list is the UI's "forget this model", and with no other + # fields left it removes the entry outright. gone = settings_route.update_openai_auto_switch_override( settings_route.ModelOverridePayload(model_id = "unsloth/B-GGUF", llama_extra_args = []), "tester", @@ -4314,8 +4311,8 @@ def test_override_route_preserves_launch_flags_across_a_settings_only_update(mon def test_override_found_under_a_concrete_path_with_variant(monkeypatch): - # A local folder or non-active HF cache resolves to a public repo id plus a - # concrete path. Settings saved against the path must still be found. + # A local folder or non-active HF cache resolves to a repo id plus a concrete + # path, and settings saved against the path must still be found. backend = _FakeBackend(None) rec = _LoadRecorder(backend) _wire( @@ -4333,8 +4330,8 @@ def test_override_found_under_a_concrete_path_with_variant(monkeypatch): def test_repo_qualified_override_beats_path_qualified(monkeypatch): - # Ordering is most specific first, and the public repo id is the name the - # user configured against in the picker. + # Most specific first, and the public repo id is what the picker configured + # against. backend = _FakeBackend(None) rec = _LoadRecorder(backend) _wire( @@ -4355,10 +4352,9 @@ def test_repo_qualified_override_beats_path_qualified(monkeypatch): def test_first_quant_save_keeps_legacy_bare_repo_launch_flags(monkeypatch): - # Flags were stored under the bare repo id before per-quant settings existed. - # The first save from the settings page writes repo:QUANT, and auto-switch - # then prefers that entry, so the flags must come with it or they are - # silently disabled with no UI able to show or restore them. + # Flags predating per-quant settings live under the bare repo id. The first save + # writes repo:QUANT, which auto-switch prefers, so the flags must come with it + # or they are silently disabled with no UI able to restore them. import routes.settings as settings_route _mock_override_store(monkeypatch) @@ -4374,8 +4370,8 @@ def test_first_quant_save_keeps_legacy_bare_repo_launch_flags(monkeypatch): def test_bare_repo_carry_over_does_not_split_a_windows_path(monkeypatch): - # "C:\models\x.gguf" has a colon that is not a variant separator. Splitting - # naively would look up "C" and, worse, could graft another model's flags on. + # The colon in "C:\models\x.gguf" is not a variant separator: splitting naively + # looks up "C" and could graft another model's flags on. import routes.settings as settings_route _mock_override_store(monkeypatch) @@ -4404,8 +4400,8 @@ def test_windows_path_with_quant_still_carries_over(monkeypatch): def test_stale_gpu_ids_are_dropped_not_fatal(monkeypatch): - # A pin saved on a two-GPU box, replayed on a one-GPU box. Before this the - # whole load 400d; the contract is that one dead field degrades to defaults. + # A two-GPU pin replayed on a one-GPU box used to 400 the whole load; the + # contract is that one dead field degrades to defaults. backend = _FakeBackend(None) rec = _LoadRecorder(backend) _wire( @@ -4455,8 +4451,7 @@ def test_usable_gpu_ids_are_kept(monkeypatch): def test_override_gpu_ids_probe_never_raises(monkeypatch): - # The probe runs on the load path, so any hardware error must read as - # "unusable" rather than escaping as a 500. + # On the load path, so a hardware error must read as "unusable", not a 500. import utils.hardware.hardware as hw def boom(*args, **kwargs): @@ -4467,9 +4462,8 @@ def test_override_gpu_ids_probe_never_raises(monkeypatch): def test_vulkan_ordinal_absent_from_the_probe_is_unusable(monkeypatch): - # resolve_requested_gpu_ids only rejects malformed Vulkan ordinals, so - # presence needs the same ggml probe the load itself runs. Without it this - # helper says "fine" and the load 400s on the check it skipped. + # resolve_requested_gpu_ids only rejects malformed Vulkan ordinals, so presence + # needs the ggml probe the load runs, or the load 400s on the skipped check. from core.inference.llama_cpp import LlamaCppBackend monkeypatch.setattr(LlamaCppBackend, "_is_vulkan_backend", staticmethod(lambda: True)) @@ -4485,8 +4479,8 @@ def test_vulkan_ordinal_absent_from_the_probe_is_unusable(monkeypatch): def test_vulkan_probe_without_a_binary_does_not_block_the_load(monkeypatch): - # No binary means nothing to probe with. Refusing here would drop a valid - # pin on every load, so the later path stays the authority. + # Nothing to probe with, and refusing would drop a valid pin on every load, + # so the later path stays the authority. from core.inference.llama_cpp import LlamaCppBackend monkeypatch.setattr(LlamaCppBackend, "_is_vulkan_backend", staticmethod(lambda: True)) @@ -4495,9 +4489,8 @@ def test_vulkan_probe_without_a_binary_does_not_block_the_load(monkeypatch): def test_default_save_preserves_flags_instead_of_removing(monkeypatch): - # "Remember for this model" is on but every value is default, so the payload - # carries no fields. That is shape-identical to a removal, and guessing wrong - # wipes launch flags no UI can show or restore. + # "Remember for this model" with all-default values sends no fields, which is + # shape-identical to a removal; guessing wrong wipes unrecoverable launch flags. import routes.settings as settings_route _mock_override_store(monkeypatch) @@ -4552,7 +4545,7 @@ def test_remove_false_with_real_fields_saves_normally(monkeypatch): def test_override_lookup_falls_back_to_case_insensitive(monkeypatch): - # The browser lowercases ids before storing them, so the backfill writes + # The browser lowercases ids, so the backfill writes # "unsloth/qwen3-8b-gguf:q4_k_m" while the resolver asks for the repo's real # casing. Without this fallback every migrated entry is invisible. _mock_override_store(monkeypatch) @@ -4570,8 +4563,8 @@ def test_exact_override_match_beats_a_case_variant(monkeypatch): def test_ambiguous_case_fallback_matches_nothing(monkeypatch): - # Two POSIX paths differing only in case are two different files. Guessing - # between them would apply one model's settings to another. + # Two POSIX paths differing only in case are two files, so guessing between + # them applies one model's settings to another. _mock_override_store(monkeypatch) settings.set_model_override("/models/foo.gguf", max_seq_length = 1024) settings.set_model_override("/models/FOO.gguf", max_seq_length = 8192) @@ -4590,14 +4583,14 @@ def test_request_used_api_key_distinguishes_key_from_session(): assert inference_route._request_used_api_key(_Req("Bearer eyJhbGciOiJIUzI1NiJ9.x")) is False assert inference_route._request_used_api_key(_Req("")) is False assert inference_route._request_used_api_key(_Req(None)) is False - # A malformed request object must read as "not an API key", never raise, since - # this runs on the hot path of every tracked request. + # Runs on the hot path of every tracked request, so a malformed request object + # must read as "not an API key" rather than raise. assert inference_route._request_used_api_key(object()) is False def test_case_fallback_never_applies_to_a_posix_path(monkeypatch): - # Two files that differ only in case are two different models on Linux, so a - # near miss must load defaults rather than another model's context and GPU pin. + # Two files differing only in case are two models on Linux, so a near miss must + # load defaults rather than another model's context and GPU pin. _mock_override_store(monkeypatch) settings.set_model_override("/models/foo.gguf", max_seq_length = 8192, gpu_ids = [1]) assert settings.get_model_override("/models/Foo.gguf") == {} @@ -4605,10 +4598,9 @@ def test_case_fallback_never_applies_to_a_posix_path(monkeypatch): def test_case_fallback_does_apply_to_a_windows_path(monkeypatch): - # NTFS is case-insensitive, so these name one file, and the browser folds - # drive paths before storing. Treating them as two models would leave every - # migrated Windows entry unreachable until the user saved it again, which is - # the opposite of the POSIX rule and for the opposite reason. The separator + # NTFS is case-insensitive, so these name one file and the browser folds drive + # paths before storing. Treating them as two would strand every migrated Windows + # entry: the opposite of the POSIX rule, for the opposite reason. The separator # is interchangeable there too. _mock_override_store(monkeypatch) settings.set_model_override(r"c:\models\foo.gguf", max_seq_length = 8192) @@ -4625,16 +4617,16 @@ def test_case_fallback_applies_to_unc_and_wsl_drive_paths(monkeypatch): def test_a_plain_posix_path_under_mnt_stays_case_sensitive(monkeypatch): - # Only /mnt/ is a WSL drive mount. /mnt/data is an ordinary Linux - # mount point and stays case-sensitive like any other POSIX path. + # Only /mnt/ is a WSL drive mount; /mnt/data is an ordinary Linux mount + # point and stays case-sensitive. _mock_override_store(monkeypatch) settings.set_model_override("/mnt/data/models/foo.gguf", max_seq_length = 8192) assert settings.get_model_override("/mnt/data/models/Foo.gguf") == {} def test_an_ambiguous_windows_case_fallback_still_matches_nothing(monkeypatch): - # Two stored keys folding to one leaves no single answer, so the load takes - # defaults rather than guessing between them. + # Two stored keys folding to one has no single answer, so the load takes + # defaults rather than guessing. _mock_override_store(monkeypatch) settings.set_model_override(r"c:\models\foo.gguf", max_seq_length = 1024) settings.set_model_override("C:/models/FOO.gguf", max_seq_length = 8192) @@ -4649,10 +4641,9 @@ def test_case_fallback_still_covers_repo_ids(monkeypatch): def test_explicit_remove_is_not_blocked_by_stale_invalid_flags(monkeypatch): - # remove is the operation discriminator, so a form still carrying a rejected - # launch flag must not turn "forget this model" into a 400 that leaves the - # override in place. Nothing is stored on this path, so there is nothing to - # validate. + # remove is the operation discriminator, so a rejected launch flag must not turn + # "forget this model" into a 400 that leaves the override in place. Nothing is + # stored on this path, so there is nothing to validate. import routes.settings as settings_route _mock_override_store(monkeypatch) @@ -4667,8 +4658,8 @@ def test_explicit_remove_is_not_blocked_by_stale_invalid_flags(monkeypatch): def test_explicit_remove_wins_over_config_fields_in_the_same_payload(monkeypatch): - # remove is the operation discriminator, so a stale form field alongside it - # must not quietly turn "forget this model" into an update. + # remove is the operation discriminator, so a stale field alongside it must not + # turn "forget this model" into an update. import routes.settings as settings_route _mock_override_store(monkeypatch) @@ -4683,8 +4674,8 @@ def test_explicit_remove_wins_over_config_fields_in_the_same_payload(monkeypatch def test_posix_colon_in_a_path_is_not_treated_as_a_quant(monkeypatch): - # "/models/foo:bar.gguf" is one valid POSIX filename, not repo + quant. - # Splitting it would graft /models/foo's launch flags onto a different model. + # "/models/foo:bar.gguf" is one POSIX filename, not repo + quant; splitting it + # grafts /models/foo's launch flags onto a different model. import routes.settings as settings_route _mock_override_store(monkeypatch) @@ -4697,11 +4688,10 @@ def test_posix_colon_in_a_path_is_not_treated_as_a_quant(monkeypatch): def test_unknown_quant_label_on_a_gguf_still_carries_flags_over(monkeypatch): - # A .gguf whose filename holds no recognizable quant token is still labelled - # by the scanner, which falls back to the stem, so the UI saves under - # "/models/custom.gguf:custom". Refusing that suffix dropped the bare entry's - # legacy flags on the first save, and auto-switch then prefers the qualified - # entry, so nothing was left that could restore them. + # A .gguf with no recognizable quant token is labelled by its stem, so the UI + # saves under "/models/custom.gguf:custom". Refusing that suffix dropped the bare + # entry's legacy flags on the first save, and auto-switch prefers the qualified + # entry, so nothing was left to restore them. import routes.settings as settings_route _mock_override_store(monkeypatch) @@ -4716,10 +4706,10 @@ def test_unknown_quant_label_on_a_gguf_still_carries_flags_over(monkeypatch): def test_bpw_qualified_variants_still_carry_flags_over(monkeypatch): - # utils/models/model_config.py keeps a bits-per-weight modifier on the label - # so two files at the same base quant stay distinct, and that form reaches - # the override keys. The known-quant pattern does not accept it, so the bare - # entry was missed and the first qualified save dropped its launch flags. + # utils/models/model_config.py keeps a bits-per-weight modifier on the label to + # keep two files at the same base quant distinct, and that form reaches the + # override keys. The known-quant pattern rejects it, so the bare entry was missed + # and the first qualified save dropped its launch flags. import routes.settings as settings_route _mock_override_store(monkeypatch) @@ -4736,9 +4726,9 @@ def test_bpw_qualified_variants_still_carry_flags_over(monkeypatch): def test_a_posix_path_variant_folds_while_the_path_does_not(monkeypatch): - # The browser lowercases the quant but keeps POSIX path casing, so the - # migrated key is "/models/Foo:q4_k_m" while the scanner asks for - # "/models/Foo:Q4_K_M". The path itself must still be case-sensitive. + # The browser lowercases the quant but keeps POSIX path casing, so the migrated + # "/models/Foo:q4_k_m" must answer the scanner's "/models/Foo:Q4_K_M" while the + # path itself stays case-sensitive. _mock_override_store(monkeypatch) settings.set_model_override("/models/Foo:q4_k_m", max_seq_length = 8192) assert settings.get_model_override("/models/Foo:Q4_K_M")["max_seq_length"] == 8192 @@ -4746,10 +4736,9 @@ def test_a_posix_path_variant_folds_while_the_path_does_not(monkeypatch): def test_an_unknown_gguf_label_is_reachable_in_either_casing(monkeypatch): - # A .gguf with no recognizable quant token is labelled by its stem, and v2 - # storage lowercases that label while the scanner probes with the filename's - # own casing. Folding only recognized quant labels left the migrated entry - # unreachable for exactly the files that need the fallback. + # A .gguf with no recognizable quant token is labelled by its stem, and v2 storage + # lowercases that label while the scanner keeps the filename casing. Folding only + # recognized labels stranded exactly the files that need the fallback. _mock_override_store(monkeypatch) settings.set_model_override("/models/CustomModel.gguf:custommodel", max_seq_length = 8192) got = settings.get_model_override("/models/CustomModel.gguf:CustomModel") @@ -4759,16 +4748,16 @@ def test_an_unknown_gguf_label_is_reachable_in_either_casing(monkeypatch): def test_a_posix_colon_filename_is_not_folded_as_a_variant(monkeypatch): - # "/models/foo:Bar.gguf" is one filename, not path + quant, so folding its - # tail would let it reach a different file's settings. + # "/models/foo:Bar.gguf" is one filename, not path + quant, so folding its tail + # would reach a different file's settings. _mock_override_store(monkeypatch) settings.set_model_override("/models/foo:bar.gguf", max_seq_length = 8192) assert settings.get_model_override("/models/foo:Bar.gguf") == {} def test_a_suffix_the_scanner_would_not_derive_carries_nothing_over(monkeypatch): - # Only the exact label the scanner derives for this filename is accepted, so - # an unrelated colon suffix cannot reach into another model's flags. + # Only the scanner's exact label is accepted, so an unrelated colon suffix cannot + # reach another model's flags. import routes.settings as settings_route _mock_override_store(monkeypatch) @@ -4783,8 +4772,8 @@ def test_a_suffix_the_scanner_would_not_derive_carries_nothing_over(monkeypatch) def test_unknown_quant_label_carries_over_for_a_windows_path(monkeypatch): - # The key is written on Windows but may be read back by a backend that is - # not, where a backslash is an ordinary filename character. + # Written on Windows but read back on a backend where a backslash is an ordinary + # filename character. import routes.settings as settings_route _mock_override_store(monkeypatch) @@ -4811,9 +4800,8 @@ def test_real_quant_suffix_on_a_path_still_carries_flags_over(monkeypatch): def test_load_retries_without_gpu_ids_when_the_loader_rejects_the_pin(monkeypatch): - # The pre-flight check cannot mirror every rule the loader applies (a Vulkan - # diffusion GGUF refuses GPU selection outright). A stale placement preference - # must never be the reason a request cannot be served. + # The pre-flight check cannot mirror every loader rule (a Vulkan diffusion GGUF + # refuses GPU selection), and a stale pin must never block a request. from fastapi import HTTPException backend = _FakeBackend(None) @@ -4887,9 +4875,8 @@ def test_a_non_gpu_load_failure_is_not_retried(monkeypatch): def test_removal_clears_the_entry_a_load_would_actually_resolve(monkeypatch): - # The browser normalizes casing before storing, so a forget request can carry - # a different casing than the stored key. Removing only the literal key would - # leave the entry a load still resolves to, with no UI able to clear it. + # The browser normalizes casing before storing, so a forget can carry a different + # casing; removing only the literal key leaves an entry loads still resolve to. import routes.settings as settings_route _mock_override_store(monkeypatch) @@ -4905,10 +4892,9 @@ def test_removal_clears_the_entry_a_load_would_actually_resolve(monkeypatch): def test_save_updates_the_existing_case_variant_instead_of_forking_it(monkeypatch): - # The backfill stores normalized (lowercase) keys while a later UI save carries - # the catalog's casing. Writing that literally leaves two keys for one model, - # and with two equivalent keys present any third casing resolves ambiguously, - # so the model silently loses every saved setting on the API path. + # The backfill stores lowercase keys while a later UI save carries the catalog's + # casing. Writing that literally leaves two keys for one model, which makes any + # third casing ambiguous and silently loses every setting on the API path. import routes.settings as settings_route _mock_override_store(monkeypatch) diff --git a/studio/backend/utils/openai_auto_switch_settings.py b/studio/backend/utils/openai_auto_switch_settings.py index 3f39525f7e..5cbdbfa2af 100644 --- a/studio/backend/utils/openai_auto_switch_settings.py +++ b/studio/backend/utils/openai_auto_switch_settings.py @@ -250,24 +250,18 @@ def set_openai_auto_switch( # --- Per-model launch config ------------------------------------------------- # # An override is the server-side twin of the UI's per-model config (the browser -# localStorage map behind features/model-picker/model-config). The UI mirrors -# every save here so a model loaded by an OpenAI-compatible API request gets the -# same launch settings a user would get loading it from the picker; without this -# the API path could only ever apply the two legacy fields below. +# localStorage map behind features/model-picker/model-config). The UI mirrors every +# save here so an OpenAI-compatible API load gets the same launch settings the +# picker would apply, rather than only the two legacy fields below. # -# Legacy entries hold just {llama_extra_args, max_seq_length}; every field is -# optional and absent means "fall back to the app default", so old entries keep -# loading correctly. A write is a full replace of the fields it expresses, so the -# route carries `llama_extra_args` over when the payload omits it (the settings -# UI has no control for launch flags and must not wipe them). +# Legacy entries hold just {llama_extra_args, max_seq_length}. Every field is +# optional and absent means "app default". A write replaces the fields it +# expresses, so the route carries `llama_extra_args` over when the payload omits it. # -# Known gap: the picker resolves a couple of knobs as "per-model value, else the -# user's global preference" -- GPU memory mode and speculative decoding, whose -# globals live in browser localStorage. An override deliberately stores only an -# explicit per-model choice (so the model keeps following later global changes), -# and the server cannot see the globals at all. So for a model that follows the -# global on one of those two, an API load falls back to the app default rather -# than the user's global. Every other field matches the picker exactly. +# Known gap: the picker falls back to a global preference for GPU memory mode and +# speculative decoding, and those globals live in browser localStorage. An override +# stores only an explicit per-model choice, so an API load of a model that follows +# the global gets the app default instead. Every other field matches the picker. # Mirrors _valid_cache_types in core/inference/llama_cpp.py. VALID_KV_CACHE_DTYPES = frozenset( @@ -287,7 +281,7 @@ VALID_SPECULATIVE_TYPES = frozenset( "ngram-simple", } ) -# Only these two consume spec_draft_n_max (mirrors MTP_SPECULATIVE_TYPES in the UI). +# Only these consume spec_draft_n_max (mirrors MTP_SPECULATIVE_TYPES in the UI). MTP_SPECULATIVE_TYPES = frozenset({"mtp", "mtp+ngram", "draft-mtp"}) VALID_GPU_MEMORY_MODES = frozenset({"auto", "manual"}) @@ -303,12 +297,10 @@ def _clean_str(value: Any, allowed: frozenset[str]) -> Optional[str]: def _bounded_int(value: Any, *, minimum: int, maximum: int) -> Optional[int]: - # bool is a subclass of int, so `gpu_ids: [true, false]` would otherwise pin - # the model to GPUs 1 and 0. + # bool subclasses int, so `gpu_ids: [true, false]` would pin GPUs 1 and 0. if isinstance(value, bool): return None - # int(1.5) is 1, which would silently turn a fractional context into a - # useless one. Only exact integers count. + # int(1.5) is 1, which would silently mangle a fractional context. if isinstance(value, float) and not value.is_integer(): return None try: @@ -349,8 +341,7 @@ def normalize_model_override(payload: dict[str, Any]) -> dict[str, Any]: speculative_type = _clean_str(payload.get("speculative_type"), VALID_SPECULATIVE_TYPES) if speculative_type: entry["speculative_type"] = speculative_type - # Only meaningful for the MTP modes; storing it otherwise would resurface - # in the UI as an edit the loader silently ignores. + # MTP-only; storing it otherwise shows an edit the loader ignores. if speculative_type in MTP_SPECULATIVE_TYPES: spec_draft_n_max = _bounded_int(payload.get("spec_draft_n_max"), minimum = 1, maximum = 16) if spec_draft_n_max: @@ -361,8 +352,8 @@ def normalize_model_override(payload: dict[str, Any]) -> dict[str, Any]: template = payload.get("chat_template_override") if isinstance(template, str) and template.strip(): - # JSON can carry lone surrogates, which encode() rejects outright. Such a - # template can never render, so it is dropped like any other bad field. + # A lone surrogate from JSON breaks encode(), and such a template can + # never render, so drop it like any other bad field. try: template_bytes = len(template.encode("utf-8")) except UnicodeEncodeError: @@ -370,13 +361,12 @@ def normalize_model_override(payload: dict[str, Any]) -> dict[str, Any]: if template_bytes <= MAX_CHAT_TEMPLATE_OVERRIDE_BYTES: entry["chat_template_override"] = template - # Only "manual" is a real override: persisting "auto" would pin the model and - # stop it following later changes to the global GPU memory preference. + # Only "manual" is a real override: "auto" would pin the model and stop it + # following the global GPU memory preference. if _clean_str(payload.get("gpu_memory_mode"), VALID_GPU_MEMORY_MODES) == "manual": entry["gpu_memory_mode"] = "manual" - # -1 is Auto (llama.cpp --fit owns layer sizing), which is also the default, - # so only a pinned count >= 0 is worth storing. + # -1 is Auto (llama.cpp --fit), which is the default, so only >= 0 is stored. gpu_layers = _bounded_int(payload.get("gpu_layers"), minimum = 0, maximum = 1024) if gpu_layers is not None: entry["gpu_layers"] = gpu_layers @@ -387,9 +377,8 @@ def normalize_model_override(payload: dict[str, Any]) -> dict[str, Any]: gpu_ids = payload.get("gpu_ids") if isinstance(gpu_ids, (list, tuple)) and gpu_ids: - # De-duplicate, preserving order: resolve_requested_gpu_ids rejects a - # repeated id outright, so storing [0, 0] would make every later API load - # of this model fail with a 400 that the picker never hits. + # De-duplicate, preserving order: resolve_requested_gpu_ids rejects a repeat, + # so storing [0, 0] would 400 every later API load of this model. cleaned_ids: list[int] = [] for gid in gpu_ids: parsed = _bounded_int(gid, minimum = 0, maximum = 1024) @@ -417,10 +406,8 @@ def resolve_fit_max_seq_length(override: dict[str, Any], *, is_gguf: bool) -> Op ) if manual_auto_layers: return override.get("custom_context_length") or 0 - # max_seq_length wins where both are set. The UI only ever sends it for a - # non-GGUF model (a GGUF's context is `custom_context_length`), so in - # practice the two never collide from that path; a hand-written or legacy - # entry that sets it on a GGUF is honoured, which is this API's contract. + # max_seq_length wins where both are set. The UI only sends it for non-GGUF + # models, so the two only collide in a hand-written or legacy entry. return override.get("max_seq_length") or override.get("custom_context_length") @@ -471,10 +458,9 @@ def _looks_like_filesystem_path(model_id: str) -> bool: return len(model_id) >= 3 and model_id[1] == ":" and model_id[2] in ("\\", "/") -# The three path shapes whose filesystem is case-insensitive, matching the rule -# the browser applies in features/hub/lib/model-identity.ts. Kept in step with -# it: the browser folds these before storing, so the two sides have to agree on -# which paths fold or a stored key becomes unreachable. +# The three case-insensitive path shapes. Must stay in step with +# features/hub/lib/model-identity.ts, which folds these before storing, or a +# stored key becomes unreachable. _WINDOWS_DRIVE_PATH = re.compile(r"^[A-Za-z]:[\\/]") _WSL_DRIVE_PATH = re.compile(r"^/mnt/[A-Za-z](?:/|$)") @@ -502,10 +488,9 @@ def _fold_case_insensitive_path(model_id: str) -> Optional[str]: return trimmed.casefold() -# A quant label may carry a bits-per-weight modifier, because two files at the -# same base quant are kept distinct by it ("IQ4_XS-3.53bpw"). The two label -# helpers disagree on whether to keep it, so anything reading a stored key has -# to accept both forms. +# A quant label may carry a bits-per-weight modifier ("IQ4_XS-3.53bpw") to keep two +# files at the same base quant distinct. The two label helpers disagree on whether +# to keep it, so readers of a stored key must accept both forms. _BPW_SUFFIX = re.compile(r"-[0-9]+(?:\.[0-9]+)?bpw$", re.IGNORECASE) _MAX_QUANT_SUFFIX_LEN = 64 @@ -529,13 +514,11 @@ def split_quant_suffix(value: str) -> Optional[tuple[str, str]]: _BPW_SUFFIX.sub("", tail) ): return head, tail - # A .gguf whose filename holds no recognizable quant token is still labelled - # by the scanner, which falls back to the stem, so keys like - # "/models/CustomModel.gguf:custommodel" exist. Storage lowercases that - # label while the scanner probes with the filename's own casing, so the - # comparison is case-insensitive. Requiring the suffix to be exactly that - # label is what keeps an ordinary colon out: "/models/foo:bar.gguf" splits - # to a head that is not a .gguf at all. + # A .gguf with no recognizable quant token is labelled by its stem, so keys like + # "/models/CustomModel.gguf:custommodel" exist. Storage lowercases the label + # while the scanner keeps the filename casing, hence the case-insensitive + # compare. Requiring exactly that label keeps an ordinary colon out: + # "/models/foo:bar.gguf" splits to a head that is not a .gguf. if not head.lower().endswith(".gguf"): return None filename = head.replace("\\", "/").rsplit("/", 1)[-1] @@ -590,12 +573,10 @@ def resolve_model_override_key(model_id: str) -> Optional[str]: return model_id if not isinstance(model_id, str): return None - # A POSIX path is case-sensitive and names a different file, so matching - # "/models/Foo.gguf" against an entry saved for "/models/foo.gguf" would - # replay another model's context and GPU pin. A Windows drive path, a UNC - # share and a WSL drive path are not case-sensitive, and the browser folds - # exactly those before storing, so refusing to fold them here would leave - # every migrated Windows entry unreachable until the user saved it again. + # A POSIX path is case-sensitive, so folding "/models/Foo.gguf" onto + # "/models/foo.gguf" would replay another model's settings. Windows drive, UNC + # and WSL paths are not, and the browser folds exactly those before storing, so + # not folding them here would strand every migrated Windows entry. if _looks_like_filesystem_path(model_id): folded = _fold_case_insensitive_path(model_id) if folded is not None: @@ -603,10 +584,9 @@ def resolve_model_override_key(model_id: str) -> Optional[str]: def fold(key: str) -> Optional[str]: return _fold_case_insensitive_path(key) else: - # POSIX: the path itself stays case-sensitive, but the browser - # lowercases the quant suffix while keeping the path casing, so a - # migrated "/models/Foo:q4_k_m" has to stay reachable from the - # scanner's "/models/Foo:Q4_K_M". + # POSIX: the path stays case-sensitive, but the browser lowercases the + # quant suffix, so "/models/Foo:q4_k_m" must be reachable from + # the scanner's "/models/Foo:Q4_K_M". folded = _fold_posix_path_variant(model_id) def fold(key: str) -> Optional[str]: diff --git a/studio/frontend/src/app/routes/__root.tsx b/studio/frontend/src/app/routes/__root.tsx index 7d89c0e83d..ad7845ebf9 100644 --- a/studio/frontend/src/app/routes/__root.tsx +++ b/studio/frontend/src/app/routes/__root.tsx @@ -70,10 +70,9 @@ const CHAT_ONLY_ALLOWED = new Set([ // Export stays reachable on chat-only hosts so the page can show its own grayed-out reason // instead of a silent redirect; it self-gates via export capability, so nothing runs. "/export", - // Chat-only hosts (Intel Macs, Apple Silicon without MLX, no-GPU boxes) serve - // the OpenAI-compatible API exactly like any other host, so the monitor has to - // be reachable there. Without this the floating panel's own "Expand" button - // and the Settings > API card both redirect to /chat. + // Chat-only hosts (Intel Macs, Apple Silicon without MLX, no-GPU boxes) serve the + // OpenAI-compatible API like any other host, so the monitor must be reachable + // there or the overlay's "Expand" and the Settings > API card redirect to /chat. "/api-monitor", ]); @@ -177,8 +176,7 @@ function RootLayout() { }, [documentTitle]); // Settings saved before the server-side override map existed live only in this - // browser, so an API load would use app defaults while the UI still showed the - // model as remembered. Backfill once, after auth. + // browser, so an API load would use app defaults. Backfill once, after auth. useEffect(() => { if (isAuthFlowRoute) { return; diff --git a/studio/frontend/src/app/routes/api.tsx b/studio/frontend/src/app/routes/api.tsx index 3a4da07f4d..d37d739122 100644 --- a/studio/frontend/src/app/routes/api.tsx +++ b/studio/frontend/src/app/routes/api.tsx @@ -12,9 +12,8 @@ const ApiMonitorPage = lazyRouteComponent( export const Route = createRoute({ getParentRoute: () => rootRoute, - // Not "/api": the backend owns that prefix (and "/v1"), and its SPA fallback - // deliberately 404s those paths so API clients get an API-shaped error rather - // than an HTML page. A deep link to /api would never reach the router. + // Not "/api": the backend owns that prefix (and "/v1") and its SPA fallback 404s + // those paths, so a deep link to /api would never reach the router. path: "/api-monitor", staticData: { title: "API" }, beforeLoad: () => requireAuth(), diff --git a/studio/frontend/src/features/api-monitor/api-monitor-overlay.tsx b/studio/frontend/src/features/api-monitor/api-monitor-overlay.tsx index 93d774cf7e..ae2344f4bd 100644 --- a/studio/frontend/src/features/api-monitor/api-monitor-overlay.tsx +++ b/studio/frontend/src/features/api-monitor/api-monitor-overlay.tsx @@ -30,11 +30,11 @@ import { computeStats } from "./use-api-monitor"; // Live cadence while the panel is on screen. const OPEN_POLL_MS = 1500; -// While closed the poll only has to notice that traffic started, so it backs off. +// Closed, the poll only has to notice traffic started, so it backs off. const IDLE_POLL_MS = 5000; // Requests shown in the panel; the rest are one click away on the full page. const VISIBLE_ENTRIES = 4; -// How long the API must be quiet before a dismissed panel will open itself again. +// Quiet time before a dismissed panel re-arms. const REARM_QUIET_MS = 60_000; const API_INFERENCE_PREFIX_RE = /^\/api\/inference/; @@ -90,7 +90,7 @@ function StatCell({ > {value} - {/* Sentence case: Unsloth metric rows read as words, not headers. */} + {/* Sentence case: metric rows read as words, not headers. */} {label} @@ -109,11 +109,10 @@ export function ApiMonitorOverlay(): ReactElement | null { ReturnType > | null>(null); - // One loop for both jobs: panel contents while open, traffic watch while - // closed. Stands down on the full page, which polls for itself. + // One loop for both jobs: panel contents while open, traffic watch while closed. + // Stands down on the full page, which polls for itself. useEffect(() => { - // Opted out and closed: the panel can neither open nor show anything, so - // polling would be pure background load on every open Studio window. + // Opted out and closed: nothing to open or show, so polling is pure load. if (onFullPage || (!autoOpen && !isOpen)) { return; } @@ -154,11 +153,11 @@ export function ApiMonitorOverlay(): ReactElement | null { const entries = useMemo(() => data?.entries ?? [], [data]); const stats = useMemo(() => computeStats(entries), [entries]); - // Ids already seen. A set, not "the newest id": finishing moves an entry to - // the front, so the head flips without any new traffic. + // Ids already seen. A set, not "the newest id": finishing moves an entry to the + // front, so the head flips without any new traffic. const seenIdsRef = useRef>(new Set()); - // Seeded on the first response even when empty, so the first request of a - // fresh session is not mistaken for history. + // Seeded on the first response even when empty, so the first request of a fresh + // session is not mistaken for history. const seededRef = useRef(false); const lastNewEntryAtRef = useRef(0); @@ -169,9 +168,8 @@ export function ApiMonitorOverlay(): ReactElement | null { const ids = data.entries.map((entry) => entry.id); if (!seededRef.current) { seededRef.current = true; - // Seed finished requests only. A request that is still running when the - // first snapshot lands started while Studio was loading, so it is live - // traffic the user has not seen, not history to adopt silently. + // Seed finished requests only: one still running at the first snapshot started + // while Studio was loading, so it is unseen live traffic, not history. seenIdsRef.current = new Set( data.entries .filter((entry) => entry.status !== "running") @@ -182,9 +180,8 @@ export function ApiMonitorOverlay(): ReactElement | null { } } const seen = seenIdsRef.current; - // Only API-key traffic counts. Studio's own chat goes through these same - // endpoints, and this panel is about serving other clients, not about the - // request the user is watching stream in front of them. + // Only API-key traffic counts: Studio's own chat uses these same endpoints, and + // this panel is about serving other clients. const hasNewTraffic = data.entries.some( (entry) => entry.via_api_key && !seen.has(entry.id), ); @@ -199,8 +196,7 @@ export function ApiMonitorOverlay(): ReactElement | null { if (!autoOpen || isOpen) { return; } - // A dismissal holds for that burst and re-arms only once the API goes - // quiet, so the next request cannot re-open it a second later. + // A dismissal holds for the burst and re-arms only once the API goes quiet. if (suppressed && quietFor < REARM_QUIET_MS) { return; } @@ -247,10 +243,8 @@ export function ApiMonitorOverlay(): ReactElement | null { initial={{ opacity: 0, scale: 0.94 }} animate={{ opacity: 1, scale: 1 }} exit={{ opacity: 0, scale: 0.94 }} - /* Panel language borrowed from the sidebar's user menu and the - model selector: no hard border, an inset hairline plus a soft - drop shadow (menu-soft-surface), a 20px corner, and the heading - font throughout. */ + /* Panel language from the sidebar's user menu and the model selector: + menu-soft-surface, a 20px corner, and the heading font throughout. */ className="menu-soft-surface pointer-events-auto fixed bottom-4 right-4 flex w-[400px] max-w-[calc(100vw-2rem)] cursor-default select-none resize flex-col overflow-hidden rounded-[20px] border-0 p-2.5 font-heading ring-0" >

@@ -302,7 +296,7 @@ export function ApiMonitorOverlay(): ReactElement | null { {data?.active_model ?? "No model loaded"}

- {/* Metrics on a soft tile, as the Hub and Train pages group readouts. */} + {/* Soft tile, as the Hub and Train pages group readouts. */}
- {/* Closing only silences this burst; this is the permanent off. */} + {/* Closing silences this burst; this is the permanent off. */}