ceos/unsloth
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Tighten the comments across the API monitor and per-model settings work

Browse source 
Pass over every comment this branch touches. Collapse the multi-paragraph
rationales to the point they were making, drop prop docs that only restated
the prop name, and reflow the rest onto fewer lines. No code changes.
This commit is contained in:
danielhanchen 2026-07-28 17:32:46 +00:00
commit 58abf4ca50
33 changed files with 435 additions and 602 deletions
Download patch file Download diff file Expand all files Collapse all files

23
studio/backend/core/inference/api_monitor.py
View file

@ -50,9 +50,8 @@ class ApiMonitorEntry:
started_at: float
updated_at: float
subject: Optional[str] = None
# True when the caller used an sk-unsloth key rather than a UI session. The
# floating panel only opens itself for these: Studio's own chat goes through
# the same endpoints, and popping the monitor open mid-chat is noise.
# True for sk-unsloth key callers, not UI sessions. The floating panel only
# auto-opens for these, so Studio's own chat does not pop it mid-chat.
via_api_key: bool = False
# Monotonic anchors so duration math survives wall-clock steps (NTP).
started_monotonic: float = 0.0
@ -125,11 +124,8 @@ class ApiMonitor:
enabled: bool = True,
):
self._entries: deque[ApiMonitorEntry] = deque()
# Shared rows one subject has cleared. A shared row belongs to everyone,
# so dropping it would erase another caller's history, but leaving it
# means "Clear log" visibly does nothing to it: the frontend reloads
# straight after and the row comes back. Hiding it per subject is the
# only thing that is both true for that caller and safe for the others.
# Shared rows one subject cleared. Deleting them would erase another
# caller's history; keeping them makes "Clear log" look broken on reload.
self._hidden_shared: dict[str, set[str]] = {}
self._max_entries = max(0, max_entries)
self._lock = threading.Lock()
@ -153,8 +149,8 @@ class ApiMonitor:
id = f"apireq_{uuid.uuid4().hex[:12]}",
endpoint = endpoint,
method = method,
# str(): a raw JSON body can carry any type here, and the field is
# rendered in the UI, where a non-string breaks the whole monitor.
# str(): a raw JSON body can carry any type, and a non-string
# breaks the UI that renders it.
model = str(model) if model else "default",
prompt = _trim(prompt, _MAX_PROMPT_CHARS),
status = "running",
@ -411,9 +407,7 @@ class ApiMonitor:
self._entries.clear()
self._hidden_shared.clear()
return
# A shared row that is still running is a load in progress, not
# history, so it stays visible; clearing the log is about what has
# already happened.
# A running shared row is a load in progress, not history, so it stays.
hidden = self._hidden_shared.setdefault(subject, set())
for entry in self._entries:
if entry.shared and entry.subject != subject and entry.status != "running":
@ -446,8 +440,7 @@ class ApiMonitor:
kept.append(entry)
terminal_seen += 1
self._entries = kept
# The hidden sets only ever name rows that exist, so they stay bounded
# by the ring buffer rather than growing for the life of the process.
# Keep hidden sets to live rows so they stay bounded by the ring buffer.
live = {entry.id for entry in kept}
for subject, hidden in list(self._hidden_shared.items()):
hidden &= live

43
studio/backend/routes/inference.py
View file

@ -4354,20 +4354,14 @@ async def _maybe_auto_switch_model(
if _already_serving():
_record_serving_alias()
return
# Apply this model's saved launch config so an API-driven swap
# loads it exactly as the picker would: context, KV dtype,
# speculative decoding, chat template and GPU placement, not
# just the two legacy flags. Look the config up under the
# variant-qualified id first (two quants of one repo can carry
# different configs), then the bare ids. Both the advertised
# repo id and the concrete load path are tried: a local folder
# or a non-active HF cache is configured against its path.
# A standalone .gguf needs no quant sub-selection, so the
# resolver reports variant=None for it. The picker still
# keys its config by the quant label it derives from the
# filename (LocalModelInfo.format_variant), which is never
# empty, so those settings live under "<path>:LABEL" and no
# bare key would ever reach them.
# Apply this model's saved launch config so an API swap loads
# it exactly as the picker would. Try variant-qualified keys
# first (two quants of one repo can differ), then bare ids, and
# both the repo id and the load path (a local folder or a
# non-active HF cache is configured against its path).
# A standalone .gguf resolves with variant=None, but the picker
# keys it by the quant label derived from the filename, so its
# settings live under "<path>:LABEL" and no bare key reaches them.
file_variant = None
if not variant and target_id.lower().endswith(".gguf"):
from hub.utils.gguf import extract_quant_label
@ -4389,7 +4383,7 @@ async def _maybe_auto_switch_model(
load_kwargs.update(
model_override_load_kwargs(
override,
# variant is set for every GGUF the resolver returns; the
# Set for every GGUF the resolver returns; the
# reload-stash path carries the quant it froze.
is_gguf = bool(variant) or target_id.lower().endswith(".gguf"),
)
@ -4398,9 +4392,8 @@ async def _maybe_auto_switch_model(
if saved_gpu_ids and not await _override_gpu_ids_still_resolve(
saved_gpu_ids
):
# A pin saved before a GPU was removed, before a
# visibility-mask change, or on another host. Dropping the
# one dead field beats 400ing the whole load.
# Stale pin (GPU removed, mask changed, another host).
# Dropping the one dead field beats 400ing the whole load.
load_kwargs.pop("gpu_ids", None)
logger.warning(
"Dropping saved gpu_ids %s for %s: not available here.",
@ -4418,12 +4411,10 @@ async def _maybe_auto_switch_model(
current_request_counted = True,
)
except HTTPException as exc:
# The pre-flight check above cannot mirror every rule the
# loader applies to gpu_ids (a Vulkan diffusion GGUF refuses
# GPU selection outright, and the rules move). Rather than
# duplicating them, retry once without the saved pin: a
# stale placement preference must never be the reason an
# API request cannot be served.
# The pre-flight check cannot mirror every gpu_ids rule the
# loader applies (a Vulkan diffusion GGUF refuses GPU
# selection outright). Retry once without the saved pin: a
# stale placement preference must never block a request.
if not (
exc.status_code == 400
and load_kwargs.get("gpu_ids")
@ -4719,12 +4710,12 @@ async def _override_gpu_ids_still_resolve(gpu_ids: List[int]) -> bool:
is_vulkan = LlamaCppBackend._is_vulkan_backend()
if get_device() == DeviceType.XPU and not is_vulkan:
# gpu_ids is rejected outright on XPU.
# Rejected outright on XPU.
return False
resolved = resolve_requested_gpu_ids(gpu_ids, is_vulkan = is_vulkan)
if is_vulkan and resolved:
# Vulkan ordinals are their own index space, so resolve() only rejects
# malformed ones. Presence needs the same ggml probe the load does.
# malformed ones; presence needs the ggml probe the load runs.
binary = LlamaCppBackend._find_llama_server_binary()
if binary:
probed = {

71
studio/backend/routes/settings.py
View file

@ -132,16 +132,14 @@ class OpenAIAutoSwitchResponse(BaseModel):
auto_download_model: bool = DEFAULT_OPENAI_AUTO_DOWNLOAD_ENABLED
# A quant suffix, as modelOverrideKey builds it. Matched against the loader's own
# quant pattern rather than a length heuristic: a POSIX path may legitimately
# contain a colon ("/models/foo:bar.gguf"), and treating "bar.gguf" as a quant
# would graft an unrelated model's launch flags onto this one.
# A quant suffix, as modelOverrideKey builds it. Matched against the loader's quant
# pattern, not a length heuristic: a POSIX path may hold a colon
# ("/models/foo:bar.gguf") and would otherwise inherit another model's flags.
_MAX_VARIANT_SUFFIX_LEN = 64
# A local model's id is its filesystem path, optionally with a quant suffix, and
# A local model's id is its path plus an optional quant suffix, and
# LoadRequest.model_path is unbounded. A limit under PATH_MAX would 422 the server
# sync while the local save succeeded, leaving the UI showing settings the API
# never applies.
# sync while the local save succeeded.
MAX_MODEL_OVERRIDE_KEY_LEN = 4096 + 1 + _MAX_VARIANT_SUFFIX_LEN
@ -156,9 +154,8 @@ class ModelOverridePayload(BaseModel):
"""
model_id: str = Field(..., min_length = 1, max_length = MAX_MODEL_OVERRIDE_KEY_LEN)
# None means "leave the stored value alone": the settings UI has no control
# for launch flags, so a save from it must not wipe flags set through this
# API. An explicit [] clears them (that is how "forget this model" arrives).
# None means "leave the stored value alone": the settings UI has no control for
# launch flags and must not wipe them. An explicit [] clears them (forget).
llama_extra_args: Optional[list[str]] = None
# ge=1: 0 is not a valid sequence length, and the setter drops a falsy value,
# so reject it at the boundary instead of accepting then silently discarding it.
@ -168,25 +165,22 @@ class ModelOverridePayload(BaseModel):
speculative_type: Optional[str] = Field(default = None, max_length = 32)
spec_draft_n_max: Optional[int] = Field(default = None, ge = 1, le = 16)
tensor_parallel: bool = False
# Validated in bytes below, not by max_length: pydantic counts characters,
# so a multi-byte template could pass here and then be silently dropped by
# the normalizer (which measures UTF-8) while the request still returned 200.
# Validated in bytes below, not by max_length: pydantic counts characters, so a
# multi-byte template would pass here and be dropped by the UTF-8 normalizer.
chat_template_override: Optional[str] = None
gpu_memory_mode: Optional[Literal["auto", "manual"]] = None
# -1 is Auto (llama.cpp --fit sizes the offload); the normalizer treats it as unset.
gpu_layers: Optional[int] = Field(default = None, ge = -1, le = 1024)
n_cpu_moe: Optional[int] = Field(default = None, ge = 0, le = 1024)
gpu_ids: Optional[list[int]] = None
# Explicit intent. A save whose config is entirely default carries no fields
# at all, which is indistinguishable from "forget this model" by shape alone.
# None keeps the original contract: a bare model_id means remove.
# Explicit intent: an all-default save carries no fields, which is shape
# identical to "forget this model". None keeps the legacy contract.
remove: Optional[bool] = None
@field_validator("chat_template_override")
@classmethod
def _limit_chat_template_bytes(cls, value: Optional[str]) -> Optional[str]:
# Mirrors LoadRequest.normalize_blank_chat_template_override so the same
# template is accepted or rejected identically on both paths.
# Mirrors LoadRequest.normalize_blank_chat_template_override.
if value is None:
return None
size = chat_template_byte_length(value)
@ -357,10 +351,8 @@ def _bare_model_id(model_id: str) -> Optional[str]:
"""``repo`` for a ``repo:QUANT`` key, or None when there is no quant suffix."""
from utils.openai_auto_switch_settings import split_quant_suffix
# Must actually look like a quant, not just like a short path segment. The
# label may carry a bits-per-weight modifier ("IQ4_XS-3.53bpw"), which keeps
# two files at the same base quant distinct, and a .gguf with no recognized
# token is labelled by its stem, so both forms count.
# Must look like a quant, not just a short path segment. Both a bits-per-weight
# modifier ("IQ4_XS-3.53bpw") and a stem fallback label count.
split = split_quant_suffix(model_id)
return split[0] if split is not None else None
@ -373,10 +365,8 @@ def update_openai_auto_switch_override(
from utils.openai_auto_switch_settings import get_model_override
try:
# A payload carrying only model_id is the documented "remove", so it
# wipes everything. Otherwise it is a real save, and omitted launch flags
# are carried over from the stored entry (the settings UI cannot express
# them and must not delete them).
# Only model_id is the documented "remove". Otherwise omitted launch flags
# carry over from the stored entry, since the settings UI cannot express them.
requested_extra_args = payload.llama_extra_args
saved_fields = payload.model_dump(
exclude = {"model_id", "llama_extra_args", "remove"}, exclude_none = True
@ -390,33 +380,24 @@ def update_openai_auto_switch_override(
if requested_extra_args is None and not is_removal:
requested_extra_args = get_model_override(payload.model_id).get("llama_extra_args")
if requested_extra_args is None:
# First per-quant save for a model whose flags were stored under the
# bare repo id. Auto-switch prefers the qualified entry, so without
# this the flags are silently dropped and no UI can restore them.
# First per-quant save for flags stored under the bare repo id.
# Auto-switch prefers the qualified entry, so carry them over.
bare_id = _bare_model_id(payload.model_id)
if bare_id:
requested_extra_args = get_model_override(bare_id).get("llama_extra_args")
# Not validated on an explicit remove: nothing is stored, so the only
# effect would be a 400 that leaves the override in place, which is the
# opposite of what remove means. A stale form still carrying a rejected
# flag must not be able to block forgetting a model.
# Not validated on an explicit remove: nothing is stored, so a 400 would only
# leave the override in place. A stale flag must not block forgetting.
extra_args = [] if payload.remove is True else validate_extra_args(requested_extra_args)
if payload.remove is True:
# An explicit remove wins over anything else in the payload: a stale
# form field must not turn "forget this model" into an update that
# keeps it. Only the explicit flag short-circuits; the legacy
# inferred path still just gates launch-flag carry-over.
# Remove the key a load would actually resolve to, not just the
# literal one sent: the browser normalizes casing before storing, so
# the two can differ and a stale entry would survive forgetting.
# An explicit remove wins over any other field in the payload. Remove the
# key a load resolves to, not the literal one sent: the browser normalizes
# casing before storing, so a stale entry would survive forgetting.
target_id = resolve_model_override_key(payload.model_id) or payload.model_id
set_model_override(target_id, llama_extra_args = [], max_seq_length = None)
else:
# Save under the key a load would resolve to, for the same reason the
# removal branch does. The browser normalizes casing before storing,
# so saving the literal id leaves a second entry for one model, and
# two equivalent keys make every other casing ambiguous: the lookup
# then matches neither and the model silently loses its settings.
# Save under the key a load resolves to, as the removal branch does.
# Saving the literal id leaves two keys for one model, which makes every
# other casing ambiguous and silently loses the settings.
target_id = resolve_model_override_key(payload.model_id) or payload.model_id
set_model_override(
target_id,

12
studio/backend/tests/test_api_monitor.py
View file

@ -261,9 +261,8 @@ def test_api_monitor_append_reply_exact_cap_then_more_marks_truncated():
def test_api_monitor_clear_is_scoped_to_one_subject():
# Every other read on the monitor is subject-scoped. An unscoped clear from
# the route would let one caller erase another's history and zero their
# active count in the middle of a generation.
# Every other read is subject-scoped; an unscoped clear from the route would let
# one caller erase another's history mid-generation.
monitor = ApiMonitor(max_entries = 4)
alice = monitor.start(
endpoint = "/v1/chat/completions",
@ -292,9 +291,8 @@ def test_api_monitor_clear_is_scoped_to_one_subject():
def test_api_monitor_records_whether_the_caller_used_an_api_key():
# Studio's own chat hits these endpoints with a session JWT. The floating
# panel keys its auto-open off this flag, so mislabelling in-app chat as API
# traffic pops the panel over the composer mid-conversation.
# Studio's own chat hits these endpoints with a session JWT, and the floating
# panel keys its auto-open off this flag, so mislabelling it pops the panel.
monitor = ApiMonitor(max_entries = 4)
ui = monitor.start(
endpoint = "/api/inference/chat",
@ -494,7 +492,7 @@ def test_clear_hides_shared_lifecycle_rows_for_that_caller_only():
monitor.clear(subject = "alice")
assert monitor.snapshot(subject = "alice") == []
# Bob's view is untouched: the row is hidden for alice, not deleted.
# Hidden for alice, not deleted, so bob's view is untouched.
assert {e["id"] for e in monitor.snapshot(subject = "bob")} == {shared}
assert monitor.get(shared, subject = "alice") is None
assert monitor.get(shared, subject = "bob") is not None

178
studio/backend/tests/test_openai_auto_switch.py
View file

@ -4150,9 +4150,7 @@ def test_env_idle_below_floor_is_clamped(monkeypatch):
assert settings.get_auto_unload_idle_seconds() == 0
# ---------------------------------------------------------------------------
# Per-model launch config: normalization, LoadRequest mapping, key resolution.
# ---------------------------------------------------------------------------
def test_normalize_model_override_drops_unusable_fields_and_keeps_the_rest():
@ -4178,8 +4176,8 @@ def test_normalize_model_override_drops_unusable_fields_and_keeps_the_rest():
def test_normalize_model_override_rejects_oversized_chat_template():
small = settings.normalize_model_override({"chat_template_override": "{{ bos }}"})
assert small["chat_template_override"] == "{{ bos }}"
# The limit is bytes, not characters: a multi-byte template just under the
# character limit can still be over the byte limit.
# The limit is bytes, not characters, so a multi-byte template just under the
# character limit can still be over.
huge = "é" * settings.MAX_CHAT_TEMPLATE_OVERRIDE_BYTES
assert "chat_template_override" not in settings.normalize_model_override(
{"chat_template_override": huge}
@ -4189,15 +4187,15 @@ def test_normalize_model_override_rejects_oversized_chat_template():
def test_spec_draft_n_max_only_stored_for_mtp_modes():
mtp = settings.normalize_model_override({"speculative_type": "mtp", "spec_draft_n_max": 4})
assert mtp["spec_draft_n_max"] == 4
# A non-MTP mode ignores the draft count at load time, so storing it would
# show the user an edit that never takes effect.
# A non-MTP mode ignores the draft count, so storing it shows an edit that
# never takes effect.
ngram = settings.normalize_model_override({"speculative_type": "ngram", "spec_draft_n_max": 4})
assert "spec_draft_n_max" not in ngram
def test_resolve_fit_max_seq_length_hands_sizing_to_fit_under_manual_auto_layers():
# Manual GPU memory with Auto layers means llama.cpp --fit owns the context,
# so the load sends the context pin (or 0), not the stored max seq length.
# Manual GPU memory with Auto layers hands the context to llama.cpp --fit, so
# the load sends the context pin (or 0), not the stored max seq length.
override = {"gpu_memory_mode": "manual", "max_seq_length": 8192}
assert settings.resolve_fit_max_seq_length(override, is_gguf = True) == 0
assert (
@ -4228,8 +4226,8 @@ def test_model_override_load_kwargs_gates_gpu_placement_on_gguf():
assert gguf["gpu_layers"] == 20
assert gguf["gpu_ids"] == [0, 1]
# A safetensors model loads through HF auto-placement; inheriting a GGUF GPU
# pin here would silently change where the weights land.
# A safetensors model loads through HF auto-placement, so a GGUF GPU pin would
# silently change where the weights land.
safetensors = settings.model_override_load_kwargs(override, is_gguf = False)
assert safetensors["max_seq_length"] == 4096
assert "gpu_layers" not in safetensors
@ -4237,14 +4235,14 @@ def test_model_override_load_kwargs_gates_gpu_placement_on_gguf():
assert "n_cpu_moe" not in safetensors
assert "gpu_memory_mode" not in safetensors
# Every key it produces has to be a real LoadRequest field, or the load call
# raises TypeError at the moment the user's request arrives.
# Every key must be a real LoadRequest field, or the load raises TypeError when
# the user's request arrives.
LoadRequest(model_path = "unsloth/B-GGUF", **gguf)
def test_auto_switch_prefers_variant_qualified_override(monkeypatch):
# Settings are saved per quant, so Q4_K_M and Q8_0 of the same repo are
# different entries; the bare repo id is only the fallback.
# Settings are per quant, so Q4_K_M and Q8_0 of one repo are separate entries
# and the bare repo id is only the fallback.
backend = _FakeBackend(None)
rec = _LoadRecorder(backend)
_wire(
@ -4284,9 +4282,8 @@ def test_auto_switch_falls_back_to_bare_repo_override(monkeypatch):
def test_override_route_preserves_launch_flags_across_a_settings_only_update(monkeypatch):
# The settings page has no control for llama_extra_args, so saving from it
# omits the field. Omitted must mean "leave it alone", or every save from the
# UI would quietly wipe flags set elsewhere.
# The settings page has no control for llama_extra_args, so it omits the field.
# Omitted must mean "leave it alone", or every UI save wipes flags set elsewhere.
import routes.settings as settings_route
_mock_override_store(monkeypatch)
@ -4304,8 +4301,8 @@ def test_override_route_preserves_launch_flags_across_a_settings_only_update(mon
assert entry["llama_extra_args"] == ["--flash-attn"]
assert entry["max_seq_length"] == 4096
# An explicit empty list is how the UI says "forget this model", and with no
# other fields left that removes the entry outright.
# An explicit empty list is the UI's "forget this model", and with no other
# fields left it removes the entry outright.
gone = settings_route.update_openai_auto_switch_override(
settings_route.ModelOverridePayload(model_id = "unsloth/B-GGUF", llama_extra_args = []),
"tester",
@ -4314,8 +4311,8 @@ def test_override_route_preserves_launch_flags_across_a_settings_only_update(mon
def test_override_found_under_a_concrete_path_with_variant(monkeypatch):
# A local folder or non-active HF cache resolves to a public repo id plus a
# concrete path. Settings saved against the path must still be found.
# A local folder or non-active HF cache resolves to a repo id plus a concrete
# path, and settings saved against the path must still be found.
backend = _FakeBackend(None)
rec = _LoadRecorder(backend)
_wire(
@ -4333,8 +4330,8 @@ def test_override_found_under_a_concrete_path_with_variant(monkeypatch):
def test_repo_qualified_override_beats_path_qualified(monkeypatch):
# Ordering is most specific first, and the public repo id is the name the
# user configured against in the picker.
# Most specific first, and the public repo id is what the picker configured
# against.
backend = _FakeBackend(None)
rec = _LoadRecorder(backend)
_wire(
@ -4355,10 +4352,9 @@ def test_repo_qualified_override_beats_path_qualified(monkeypatch):
def test_first_quant_save_keeps_legacy_bare_repo_launch_flags(monkeypatch):
# Flags were stored under the bare repo id before per-quant settings existed.
# The first save from the settings page writes repo:QUANT, and auto-switch
# then prefers that entry, so the flags must come with it or they are
# silently disabled with no UI able to show or restore them.
# Flags predating per-quant settings live under the bare repo id. The first save
# writes repo:QUANT, which auto-switch prefers, so the flags must come with it
# or they are silently disabled with no UI able to restore them.
import routes.settings as settings_route
_mock_override_store(monkeypatch)
@ -4374,8 +4370,8 @@ def test_first_quant_save_keeps_legacy_bare_repo_launch_flags(monkeypatch):
def test_bare_repo_carry_over_does_not_split_a_windows_path(monkeypatch):
# "C:\models\x.gguf" has a colon that is not a variant separator. Splitting
# naively would look up "C" and, worse, could graft another model's flags on.
# The colon in "C:\models\x.gguf" is not a variant separator: splitting naively
# looks up "C" and could graft another model's flags on.
import routes.settings as settings_route
_mock_override_store(monkeypatch)
@ -4404,8 +4400,8 @@ def test_windows_path_with_quant_still_carries_over(monkeypatch):
def test_stale_gpu_ids_are_dropped_not_fatal(monkeypatch):
# A pin saved on a two-GPU box, replayed on a one-GPU box. Before this the
# whole load 400d; the contract is that one dead field degrades to defaults.
# A two-GPU pin replayed on a one-GPU box used to 400 the whole load; the
# contract is that one dead field degrades to defaults.
backend = _FakeBackend(None)
rec = _LoadRecorder(backend)
_wire(
@ -4455,8 +4451,7 @@ def test_usable_gpu_ids_are_kept(monkeypatch):
def test_override_gpu_ids_probe_never_raises(monkeypatch):
# The probe runs on the load path, so any hardware error must read as
# "unusable" rather than escaping as a 500.
# On the load path, so a hardware error must read as "unusable", not a 500.
import utils.hardware.hardware as hw
def boom(*args, **kwargs):
@ -4467,9 +4462,8 @@ def test_override_gpu_ids_probe_never_raises(monkeypatch):
def test_vulkan_ordinal_absent_from_the_probe_is_unusable(monkeypatch):
# resolve_requested_gpu_ids only rejects malformed Vulkan ordinals, so
# presence needs the same ggml probe the load itself runs. Without it this
# helper says "fine" and the load 400s on the check it skipped.
# resolve_requested_gpu_ids only rejects malformed Vulkan ordinals, so presence
# needs the ggml probe the load runs, or the load 400s on the skipped check.
from core.inference.llama_cpp import LlamaCppBackend
monkeypatch.setattr(LlamaCppBackend, "_is_vulkan_backend", staticmethod(lambda: True))
@ -4485,8 +4479,8 @@ def test_vulkan_ordinal_absent_from_the_probe_is_unusable(monkeypatch):
def test_vulkan_probe_without_a_binary_does_not_block_the_load(monkeypatch):
# No binary means nothing to probe with. Refusing here would drop a valid
# pin on every load, so the later path stays the authority.
# Nothing to probe with, and refusing would drop a valid pin on every load,
# so the later path stays the authority.
from core.inference.llama_cpp import LlamaCppBackend
monkeypatch.setattr(LlamaCppBackend, "_is_vulkan_backend", staticmethod(lambda: True))
@ -4495,9 +4489,8 @@ def test_vulkan_probe_without_a_binary_does_not_block_the_load(monkeypatch):
def test_default_save_preserves_flags_instead_of_removing(monkeypatch):
# "Remember for this model" is on but every value is default, so the payload
# carries no fields. That is shape-identical to a removal, and guessing wrong
# wipes launch flags no UI can show or restore.
# "Remember for this model" with all-default values sends no fields, which is
# shape-identical to a removal; guessing wrong wipes unrecoverable launch flags.
import routes.settings as settings_route
_mock_override_store(monkeypatch)
@ -4552,7 +4545,7 @@ def test_remove_false_with_real_fields_saves_normally(monkeypatch):
def test_override_lookup_falls_back_to_case_insensitive(monkeypatch):
# The browser lowercases ids before storing them, so the backfill writes
# The browser lowercases ids, so the backfill writes
# "unsloth/qwen3-8b-gguf:q4_k_m" while the resolver asks for the repo's real
# casing. Without this fallback every migrated entry is invisible.
_mock_override_store(monkeypatch)
@ -4570,8 +4563,8 @@ def test_exact_override_match_beats_a_case_variant(monkeypatch):
def test_ambiguous_case_fallback_matches_nothing(monkeypatch):
# Two POSIX paths differing only in case are two different files. Guessing
# between them would apply one model's settings to another.
# Two POSIX paths differing only in case are two files, so guessing between
# them applies one model's settings to another.
_mock_override_store(monkeypatch)
settings.set_model_override("/models/foo.gguf", max_seq_length = 1024)
settings.set_model_override("/models/FOO.gguf", max_seq_length = 8192)
@ -4590,14 +4583,14 @@ def test_request_used_api_key_distinguishes_key_from_session():
assert inference_route._request_used_api_key(_Req("Bearer eyJhbGciOiJIUzI1NiJ9.x")) is False
assert inference_route._request_used_api_key(_Req("")) is False
assert inference_route._request_used_api_key(_Req(None)) is False
# A malformed request object must read as "not an API key", never raise, since
# this runs on the hot path of every tracked request.
# Runs on the hot path of every tracked request, so a malformed request object
# must read as "not an API key" rather than raise.
assert inference_route._request_used_api_key(object()) is False
def test_case_fallback_never_applies_to_a_posix_path(monkeypatch):
# Two files that differ only in case are two different models on Linux, so a
# near miss must load defaults rather than another model's context and GPU pin.
# Two files differing only in case are two models on Linux, so a near miss must
# load defaults rather than another model's context and GPU pin.
_mock_override_store(monkeypatch)
settings.set_model_override("/models/foo.gguf", max_seq_length = 8192, gpu_ids = [1])
assert settings.get_model_override("/models/Foo.gguf") == {}
@ -4605,10 +4598,9 @@ def test_case_fallback_never_applies_to_a_posix_path(monkeypatch):
def test_case_fallback_does_apply_to_a_windows_path(monkeypatch):
# NTFS is case-insensitive, so these name one file, and the browser folds
# drive paths before storing. Treating them as two models would leave every
# migrated Windows entry unreachable until the user saved it again, which is
# the opposite of the POSIX rule and for the opposite reason. The separator
# NTFS is case-insensitive, so these name one file and the browser folds drive
# paths before storing. Treating them as two would strand every migrated Windows
# entry: the opposite of the POSIX rule, for the opposite reason. The separator
# is interchangeable there too.
_mock_override_store(monkeypatch)
settings.set_model_override(r"c:\models\foo.gguf", max_seq_length = 8192)
@ -4625,16 +4617,16 @@ def test_case_fallback_applies_to_unc_and_wsl_drive_paths(monkeypatch):
def test_a_plain_posix_path_under_mnt_stays_case_sensitive(monkeypatch):
# Only /mnt/<letter> is a WSL drive mount. /mnt/data is an ordinary Linux
# mount point and stays case-sensitive like any other POSIX path.
# Only /mnt/<letter> is a WSL drive mount; /mnt/data is an ordinary Linux mount
# point and stays case-sensitive.
_mock_override_store(monkeypatch)
settings.set_model_override("/mnt/data/models/foo.gguf", max_seq_length = 8192)
assert settings.get_model_override("/mnt/data/models/Foo.gguf") == {}
def test_an_ambiguous_windows_case_fallback_still_matches_nothing(monkeypatch):
# Two stored keys folding to one leaves no single answer, so the load takes
# defaults rather than guessing between them.
# Two stored keys folding to one has no single answer, so the load takes
# defaults rather than guessing.
_mock_override_store(monkeypatch)
settings.set_model_override(r"c:\models\foo.gguf", max_seq_length = 1024)
settings.set_model_override("C:/models/FOO.gguf", max_seq_length = 8192)
@ -4649,10 +4641,9 @@ def test_case_fallback_still_covers_repo_ids(monkeypatch):
def test_explicit_remove_is_not_blocked_by_stale_invalid_flags(monkeypatch):
# remove is the operation discriminator, so a form still carrying a rejected
# launch flag must not turn "forget this model" into a 400 that leaves the
# override in place. Nothing is stored on this path, so there is nothing to
# validate.
# remove is the operation discriminator, so a rejected launch flag must not turn
# "forget this model" into a 400 that leaves the override in place. Nothing is
# stored on this path, so there is nothing to validate.
import routes.settings as settings_route
_mock_override_store(monkeypatch)
@ -4667,8 +4658,8 @@ def test_explicit_remove_is_not_blocked_by_stale_invalid_flags(monkeypatch):
def test_explicit_remove_wins_over_config_fields_in_the_same_payload(monkeypatch):
# remove is the operation discriminator, so a stale form field alongside it
# must not quietly turn "forget this model" into an update.
# remove is the operation discriminator, so a stale field alongside it must not
# turn "forget this model" into an update.
import routes.settings as settings_route
_mock_override_store(monkeypatch)
@ -4683,8 +4674,8 @@ def test_explicit_remove_wins_over_config_fields_in_the_same_payload(monkeypatch
def test_posix_colon_in_a_path_is_not_treated_as_a_quant(monkeypatch):
# "/models/foo:bar.gguf" is one valid POSIX filename, not repo + quant.
# Splitting it would graft /models/foo's launch flags onto a different model.
# "/models/foo:bar.gguf" is one POSIX filename, not repo + quant; splitting it
# grafts /models/foo's launch flags onto a different model.
import routes.settings as settings_route
_mock_override_store(monkeypatch)
@ -4697,11 +4688,10 @@ def test_posix_colon_in_a_path_is_not_treated_as_a_quant(monkeypatch):
def test_unknown_quant_label_on_a_gguf_still_carries_flags_over(monkeypatch):
# A .gguf whose filename holds no recognizable quant token is still labelled
# by the scanner, which falls back to the stem, so the UI saves under
# "/models/custom.gguf:custom". Refusing that suffix dropped the bare entry's
# legacy flags on the first save, and auto-switch then prefers the qualified
# entry, so nothing was left that could restore them.
# A .gguf with no recognizable quant token is labelled by its stem, so the UI
# saves under "/models/custom.gguf:custom". Refusing that suffix dropped the bare
# entry's legacy flags on the first save, and auto-switch prefers the qualified
# entry, so nothing was left to restore them.
import routes.settings as settings_route
_mock_override_store(monkeypatch)
@ -4716,10 +4706,10 @@ def test_unknown_quant_label_on_a_gguf_still_carries_flags_over(monkeypatch):
def test_bpw_qualified_variants_still_carry_flags_over(monkeypatch):
# utils/models/model_config.py keeps a bits-per-weight modifier on the label
# so two files at the same base quant stay distinct, and that form reaches
# the override keys. The known-quant pattern does not accept it, so the bare
# entry was missed and the first qualified save dropped its launch flags.
# utils/models/model_config.py keeps a bits-per-weight modifier on the label to
# keep two files at the same base quant distinct, and that form reaches the
# override keys. The known-quant pattern rejects it, so the bare entry was missed
# and the first qualified save dropped its launch flags.
import routes.settings as settings_route
_mock_override_store(monkeypatch)
@ -4736,9 +4726,9 @@ def test_bpw_qualified_variants_still_carry_flags_over(monkeypatch):
def test_a_posix_path_variant_folds_while_the_path_does_not(monkeypatch):
# The browser lowercases the quant but keeps POSIX path casing, so the
# migrated key is "/models/Foo:q4_k_m" while the scanner asks for
# "/models/Foo:Q4_K_M". The path itself must still be case-sensitive.
# The browser lowercases the quant but keeps POSIX path casing, so the migrated
# "/models/Foo:q4_k_m" must answer the scanner's "/models/Foo:Q4_K_M" while the
# path itself stays case-sensitive.
_mock_override_store(monkeypatch)
settings.set_model_override("/models/Foo:q4_k_m", max_seq_length = 8192)
assert settings.get_model_override("/models/Foo:Q4_K_M")["max_seq_length"] == 8192
@ -4746,10 +4736,9 @@ def test_a_posix_path_variant_folds_while_the_path_does_not(monkeypatch):
def test_an_unknown_gguf_label_is_reachable_in_either_casing(monkeypatch):
# A .gguf with no recognizable quant token is labelled by its stem, and v2
# storage lowercases that label while the scanner probes with the filename's
# own casing. Folding only recognized quant labels left the migrated entry
# unreachable for exactly the files that need the fallback.
# A .gguf with no recognizable quant token is labelled by its stem, and v2 storage
# lowercases that label while the scanner keeps the filename casing. Folding only
# recognized labels stranded exactly the files that need the fallback.
_mock_override_store(monkeypatch)
settings.set_model_override("/models/CustomModel.gguf:custommodel", max_seq_length = 8192)
got = settings.get_model_override("/models/CustomModel.gguf:CustomModel")
@ -4759,16 +4748,16 @@ def test_an_unknown_gguf_label_is_reachable_in_either_casing(monkeypatch):
def test_a_posix_colon_filename_is_not_folded_as_a_variant(monkeypatch):
# "/models/foo:Bar.gguf" is one filename, not path + quant, so folding its
# tail would let it reach a different file's settings.
# "/models/foo:Bar.gguf" is one filename, not path + quant, so folding its tail
# would reach a different file's settings.
_mock_override_store(monkeypatch)
settings.set_model_override("/models/foo:bar.gguf", max_seq_length = 8192)
assert settings.get_model_override("/models/foo:Bar.gguf") == {}
def test_a_suffix_the_scanner_would_not_derive_carries_nothing_over(monkeypatch):
# Only the exact label the scanner derives for this filename is accepted, so
# an unrelated colon suffix cannot reach into another model's flags.
# Only the scanner's exact label is accepted, so an unrelated colon suffix cannot
# reach another model's flags.
import routes.settings as settings_route
_mock_override_store(monkeypatch)
@ -4783,8 +4772,8 @@ def test_a_suffix_the_scanner_would_not_derive_carries_nothing_over(monkeypatch)
def test_unknown_quant_label_carries_over_for_a_windows_path(monkeypatch):
# The key is written on Windows but may be read back by a backend that is
# not, where a backslash is an ordinary filename character.
# Written on Windows but read back on a backend where a backslash is an ordinary
# filename character.
import routes.settings as settings_route
_mock_override_store(monkeypatch)
@ -4811,9 +4800,8 @@ def test_real_quant_suffix_on_a_path_still_carries_flags_over(monkeypatch):
def test_load_retries_without_gpu_ids_when_the_loader_rejects_the_pin(monkeypatch):
# The pre-flight check cannot mirror every rule the loader applies (a Vulkan
# diffusion GGUF refuses GPU selection outright). A stale placement preference
# must never be the reason a request cannot be served.
# The pre-flight check cannot mirror every loader rule (a Vulkan diffusion GGUF
# refuses GPU selection), and a stale pin must never block a request.
from fastapi import HTTPException
backend = _FakeBackend(None)
@ -4887,9 +4875,8 @@ def test_a_non_gpu_load_failure_is_not_retried(monkeypatch):
def test_removal_clears_the_entry_a_load_would_actually_resolve(monkeypatch):
# The browser normalizes casing before storing, so a forget request can carry
# a different casing than the stored key. Removing only the literal key would
# leave the entry a load still resolves to, with no UI able to clear it.
# The browser normalizes casing before storing, so a forget can carry a different
# casing; removing only the literal key leaves an entry loads still resolve to.
import routes.settings as settings_route
_mock_override_store(monkeypatch)
@ -4905,10 +4892,9 @@ def test_removal_clears_the_entry_a_load_would_actually_resolve(monkeypatch):
def test_save_updates_the_existing_case_variant_instead_of_forking_it(monkeypatch):
# The backfill stores normalized (lowercase) keys while a later UI save carries
# the catalog's casing. Writing that literally leaves two keys for one model,
# and with two equivalent keys present any third casing resolves ambiguously,
# so the model silently loses every saved setting on the API path.
# The backfill stores lowercase keys while a later UI save carries the catalog's
# casing. Writing that literally leaves two keys for one model, which makes any
# third casing ambiguous and silently loses every setting on the API path.
import routes.settings as settings_route
_mock_override_store(monkeypatch)

102
studio/backend/utils/openai_auto_switch_settings.py
View file

@ -250,24 +250,18 @@ def set_openai_auto_switch(
# --- Per-model launch config -------------------------------------------------
#
# An override is the server-side twin of the UI's per-model config (the browser
# localStorage map behind features/model-picker/model-config). The UI mirrors
# every save here so a model loaded by an OpenAI-compatible API request gets the
# same launch settings a user would get loading it from the picker; without this
# the API path could only ever apply the two legacy fields below.
# localStorage map behind features/model-picker/model-config). The UI mirrors every
# save here so an OpenAI-compatible API load gets the same launch settings the
# picker would apply, rather than only the two legacy fields below.
#
# Legacy entries hold just {llama_extra_args, max_seq_length}; every field is
# optional and absent means "fall back to the app default", so old entries keep
# loading correctly. A write is a full replace of the fields it expresses, so the
# route carries `llama_extra_args` over when the payload omits it (the settings
# UI has no control for launch flags and must not wipe them).
# Legacy entries hold just {llama_extra_args, max_seq_length}. Every field is
# optional and absent means "app default". A write replaces the fields it
# expresses, so the route carries `llama_extra_args` over when the payload omits it.
#
# Known gap: the picker resolves a couple of knobs as "per-model value, else the
# user's global preference" -- GPU memory mode and speculative decoding, whose
# globals live in browser localStorage. An override deliberately stores only an
# explicit per-model choice (so the model keeps following later global changes),
# and the server cannot see the globals at all. So for a model that follows the
# global on one of those two, an API load falls back to the app default rather
# than the user's global. Every other field matches the picker exactly.
# Known gap: the picker falls back to a global preference for GPU memory mode and
# speculative decoding, and those globals live in browser localStorage. An override
# stores only an explicit per-model choice, so an API load of a model that follows
# the global gets the app default instead. Every other field matches the picker.
# Mirrors _valid_cache_types in core/inference/llama_cpp.py.
VALID_KV_CACHE_DTYPES = frozenset(
@ -287,7 +281,7 @@ VALID_SPECULATIVE_TYPES = frozenset(
"ngram-simple",
}
)
# Only these two consume spec_draft_n_max (mirrors MTP_SPECULATIVE_TYPES in the UI).
# Only these consume spec_draft_n_max (mirrors MTP_SPECULATIVE_TYPES in the UI).
MTP_SPECULATIVE_TYPES = frozenset({"mtp", "mtp+ngram", "draft-mtp"})
VALID_GPU_MEMORY_MODES = frozenset({"auto", "manual"})
@ -303,12 +297,10 @@ def _clean_str(value: Any, allowed: frozenset[str]) -> Optional[str]:
def _bounded_int(value: Any, *, minimum: int, maximum: int) -> Optional[int]:
# bool is a subclass of int, so `gpu_ids: [true, false]` would otherwise pin
# the model to GPUs 1 and 0.
# bool subclasses int, so `gpu_ids: [true, false]` would pin GPUs 1 and 0.
if isinstance(value, bool):
return None
# int(1.5) is 1, which would silently turn a fractional context into a
# useless one. Only exact integers count.
# int(1.5) is 1, which would silently mangle a fractional context.
if isinstance(value, float) and not value.is_integer():
return None
try:
@ -349,8 +341,7 @@ def normalize_model_override(payload: dict[str, Any]) -> dict[str, Any]:
speculative_type = _clean_str(payload.get("speculative_type"), VALID_SPECULATIVE_TYPES)
if speculative_type:
entry["speculative_type"] = speculative_type
# Only meaningful for the MTP modes; storing it otherwise would resurface
# in the UI as an edit the loader silently ignores.
# MTP-only; storing it otherwise shows an edit the loader ignores.
if speculative_type in MTP_SPECULATIVE_TYPES:
spec_draft_n_max = _bounded_int(payload.get("spec_draft_n_max"), minimum = 1, maximum = 16)
if spec_draft_n_max:
@ -361,8 +352,8 @@ def normalize_model_override(payload: dict[str, Any]) -> dict[str, Any]:
template = payload.get("chat_template_override")
if isinstance(template, str) and template.strip():
# JSON can carry lone surrogates, which encode() rejects outright. Such a
# template can never render, so it is dropped like any other bad field.
# A lone surrogate from JSON breaks encode(), and such a template can
# never render, so drop it like any other bad field.
try:
template_bytes = len(template.encode("utf-8"))
except UnicodeEncodeError:
@ -370,13 +361,12 @@ def normalize_model_override(payload: dict[str, Any]) -> dict[str, Any]:
if template_bytes <= MAX_CHAT_TEMPLATE_OVERRIDE_BYTES:
entry["chat_template_override"] = template
# Only "manual" is a real override: persisting "auto" would pin the model and
# stop it following later changes to the global GPU memory preference.
# Only "manual" is a real override: "auto" would pin the model and stop it
# following the global GPU memory preference.
if _clean_str(payload.get("gpu_memory_mode"), VALID_GPU_MEMORY_MODES) == "manual":
entry["gpu_memory_mode"] = "manual"
# -1 is Auto (llama.cpp --fit owns layer sizing), which is also the default,
# so only a pinned count >= 0 is worth storing.
# -1 is Auto (llama.cpp --fit), which is the default, so only >= 0 is stored.
gpu_layers = _bounded_int(payload.get("gpu_layers"), minimum = 0, maximum = 1024)
if gpu_layers is not None:
entry["gpu_layers"] = gpu_layers
@ -387,9 +377,8 @@ def normalize_model_override(payload: dict[str, Any]) -> dict[str, Any]:
gpu_ids = payload.get("gpu_ids")
if isinstance(gpu_ids, (list, tuple)) and gpu_ids:
# De-duplicate, preserving order: resolve_requested_gpu_ids rejects a
# repeated id outright, so storing [0, 0] would make every later API load
# of this model fail with a 400 that the picker never hits.
# De-duplicate, preserving order: resolve_requested_gpu_ids rejects a repeat,
# so storing [0, 0] would 400 every later API load of this model.
cleaned_ids: list[int] = []
for gid in gpu_ids:
parsed = _bounded_int(gid, minimum = 0, maximum = 1024)
@ -417,10 +406,8 @@ def resolve_fit_max_seq_length(override: dict[str, Any], *, is_gguf: bool) -> Op
)
if manual_auto_layers:
return override.get("custom_context_length") or 0
# max_seq_length wins where both are set. The UI only ever sends it for a
# non-GGUF model (a GGUF's context is `custom_context_length`), so in
# practice the two never collide from that path; a hand-written or legacy
# entry that sets it on a GGUF is honoured, which is this API's contract.
# max_seq_length wins where both are set. The UI only sends it for non-GGUF
# models, so the two only collide in a hand-written or legacy entry.
return override.get("max_seq_length") or override.get("custom_context_length")
@ -471,10 +458,9 @@ def _looks_like_filesystem_path(model_id: str) -> bool:
return len(model_id) >= 3 and model_id[1] == ":" and model_id[2] in ("\\", "/")
# The three path shapes whose filesystem is case-insensitive, matching the rule
# the browser applies in features/hub/lib/model-identity.ts. Kept in step with
# it: the browser folds these before storing, so the two sides have to agree on
# which paths fold or a stored key becomes unreachable.
# The three case-insensitive path shapes. Must stay in step with
# features/hub/lib/model-identity.ts, which folds these before storing, or a
# stored key becomes unreachable.
_WINDOWS_DRIVE_PATH = re.compile(r"^[A-Za-z]:[\\/]")
_WSL_DRIVE_PATH = re.compile(r"^/mnt/[A-Za-z](?:/|$)")
@ -502,10 +488,9 @@ def _fold_case_insensitive_path(model_id: str) -> Optional[str]:
return trimmed.casefold()
# A quant label may carry a bits-per-weight modifier, because two files at the
# same base quant are kept distinct by it ("IQ4_XS-3.53bpw"). The two label
# helpers disagree on whether to keep it, so anything reading a stored key has
# to accept both forms.
# A quant label may carry a bits-per-weight modifier ("IQ4_XS-3.53bpw") to keep two
# files at the same base quant distinct. The two label helpers disagree on whether
# to keep it, so readers of a stored key must accept both forms.
_BPW_SUFFIX = re.compile(r"-[0-9]+(?:\.[0-9]+)?bpw$", re.IGNORECASE)
_MAX_QUANT_SUFFIX_LEN = 64
@ -529,13 +514,11 @@ def split_quant_suffix(value: str) -> Optional[tuple[str, str]]:
_BPW_SUFFIX.sub("", tail)
):
return head, tail
# A .gguf whose filename holds no recognizable quant token is still labelled
# by the scanner, which falls back to the stem, so keys like
# "/models/CustomModel.gguf:custommodel" exist. Storage lowercases that
# label while the scanner probes with the filename's own casing, so the
# comparison is case-insensitive. Requiring the suffix to be exactly that
# label is what keeps an ordinary colon out: "/models/foo:bar.gguf" splits
# to a head that is not a .gguf at all.
# A .gguf with no recognizable quant token is labelled by its stem, so keys like
# "/models/CustomModel.gguf:custommodel" exist. Storage lowercases the label
# while the scanner keeps the filename casing, hence the case-insensitive
# compare. Requiring exactly that label keeps an ordinary colon out:
# "/models/foo:bar.gguf" splits to a head that is not a .gguf.
if not head.lower().endswith(".gguf"):
return None
filename = head.replace("\\", "/").rsplit("/", 1)[-1]
@ -590,12 +573,10 @@ def resolve_model_override_key(model_id: str) -> Optional[str]:
return model_id
if not isinstance(model_id, str):
return None
# A POSIX path is case-sensitive and names a different file, so matching
# "/models/Foo.gguf" against an entry saved for "/models/foo.gguf" would
# replay another model's context and GPU pin. A Windows drive path, a UNC
# share and a WSL drive path are not case-sensitive, and the browser folds
# exactly those before storing, so refusing to fold them here would leave
# every migrated Windows entry unreachable until the user saved it again.
# A POSIX path is case-sensitive, so folding "/models/Foo.gguf" onto
# "/models/foo.gguf" would replay another model's settings. Windows drive, UNC
# and WSL paths are not, and the browser folds exactly those before storing, so
# not folding them here would strand every migrated Windows entry.
if _looks_like_filesystem_path(model_id):
folded = _fold_case_insensitive_path(model_id)
if folded is not None:
@ -603,10 +584,9 @@ def resolve_model_override_key(model_id: str) -> Optional[str]:
def fold(key: str) -> Optional[str]:
return _fold_case_insensitive_path(key)
else:
# POSIX: the path itself stays case-sensitive, but the browser
# lowercases the quant suffix while keeping the path casing, so a
# migrated "/models/Foo:q4_k_m" has to stay reachable from the
# scanner's "/models/Foo:Q4_K_M".
# POSIX: the path stays case-sensitive, but the browser lowercases the
# quant suffix, so "/models/Foo:q4_k_m" must be reachable from
# the scanner's "/models/Foo:Q4_K_M".
folded = _fold_posix_path_variant(model_id)
def fold(key: str) -> Optional[str]:

10
studio/frontend/src/app/routes/__root.tsx
View file

@ -70,10 +70,9 @@ const CHAT_ONLY_ALLOWED = new Set([
// Export stays reachable on chat-only hosts so the page can show its own grayed-out reason
// instead of a silent redirect; it self-gates via export capability, so nothing runs.
"/export",
// Chat-only hosts (Intel Macs, Apple Silicon without MLX, no-GPU boxes) serve
// the OpenAI-compatible API exactly like any other host, so the monitor has to
// be reachable there. Without this the floating panel's own "Expand" button
// and the Settings > API card both redirect to /chat.
// Chat-only hosts (Intel Macs, Apple Silicon without MLX, no-GPU boxes) serve the
// OpenAI-compatible API like any other host, so the monitor must be reachable
// there or the overlay's "Expand" and the Settings > API card redirect to /chat.
"/api-monitor",
]);
@ -177,8 +176,7 @@ function RootLayout() {
}, [documentTitle]);
// Settings saved before the server-side override map existed live only in this
// browser, so an API load would use app defaults while the UI still showed the
// model as remembered. Backfill once, after auth.
// browser, so an API load would use app defaults. Backfill once, after auth.
useEffect(() => {
if (isAuthFlowRoute) {
return;

5
studio/frontend/src/app/routes/api.tsx
View file

@ -12,9 +12,8 @@ const ApiMonitorPage = lazyRouteComponent(
export const Route = createRoute({
getParentRoute: () => rootRoute,
// Not "/api": the backend owns that prefix (and "/v1"), and its SPA fallback
// deliberately 404s those paths so API clients get an API-shaped error rather
// than an HTML page. A deep link to /api would never reach the router.
// Not "/api": the backend owns that prefix (and "/v1") and its SPA fallback 404s
// those paths, so a deep link to /api would never reach the router.
path: "/api-monitor",
staticData: { title: "API" },
beforeLoad: () => requireAuth(),

44
studio/frontend/src/features/api-monitor/api-monitor-overlay.tsx
View file

@ -30,11 +30,11 @@ import { computeStats } from "./use-api-monitor";
// Live cadence while the panel is on screen.
const OPEN_POLL_MS = 1500;
// While closed the poll only has to notice that traffic started, so it backs off.
// Closed, the poll only has to notice traffic started, so it backs off.
const IDLE_POLL_MS = 5000;
// Requests shown in the panel; the rest are one click away on the full page.
const VISIBLE_ENTRIES = 4;
// How long the API must be quiet before a dismissed panel will open itself again.
// Quiet time before a dismissed panel re-arms.
const REARM_QUIET_MS = 60_000;
const API_INFERENCE_PREFIX_RE = /^\/api\/inference/;
@ -90,7 +90,7 @@ function StatCell({
>
{value}
</span>
{/* Sentence case: Unsloth metric rows read as words, not headers. */}
{/* Sentence case: metric rows read as words, not headers. */}
<span className="truncate text-ui-11 tracking-nav text-muted-foreground">
{label}
</span>
@ -109,11 +109,10 @@ export function ApiMonitorOverlay(): ReactElement | null {
ReturnType<typeof getApiMonitor>
> | null>(null);
// One loop for both jobs: panel contents while open, traffic watch while
// closed. Stands down on the full page, which polls for itself.
// One loop for both jobs: panel contents while open, traffic watch while closed.
// Stands down on the full page, which polls for itself.
useEffect(() => {
// Opted out and closed: the panel can neither open nor show anything, so
// polling would be pure background load on every open Studio window.
// Opted out and closed: nothing to open or show, so polling is pure load.
if (onFullPage || (!autoOpen && !isOpen)) {
return;
}
@ -154,11 +153,11 @@ export function ApiMonitorOverlay(): ReactElement | null {
const entries = useMemo(() => data?.entries ?? [], [data]);
const stats = useMemo(() => computeStats(entries), [entries]);
// Ids already seen. A set, not "the newest id": finishing moves an entry to
// the front, so the head flips without any new traffic.
// Ids already seen. A set, not "the newest id": finishing moves an entry to the
// front, so the head flips without any new traffic.
const seenIdsRef = useRef<Set<string>>(new Set());
// Seeded on the first response even when empty, so the first request of a
// fresh session is not mistaken for history.
// Seeded on the first response even when empty, so the first request of a fresh
// session is not mistaken for history.
const seededRef = useRef(false);
const lastNewEntryAtRef = useRef(0);
@ -169,9 +168,8 @@ export function ApiMonitorOverlay(): ReactElement | null {
const ids = data.entries.map((entry) => entry.id);
if (!seededRef.current) {
seededRef.current = true;
// Seed finished requests only. A request that is still running when the
// first snapshot lands started while Studio was loading, so it is live
// traffic the user has not seen, not history to adopt silently.
// Seed finished requests only: one still running at the first snapshot started
// while Studio was loading, so it is unseen live traffic, not history.
seenIdsRef.current = new Set(
data.entries
.filter((entry) => entry.status !== "running")
@ -182,9 +180,8 @@ export function ApiMonitorOverlay(): ReactElement | null {
}
}
const seen = seenIdsRef.current;
// Only API-key traffic counts. Studio's own chat goes through these same
// endpoints, and this panel is about serving other clients, not about the
// request the user is watching stream in front of them.
// Only API-key traffic counts: Studio's own chat uses these same endpoints, and
// this panel is about serving other clients.
const hasNewTraffic = data.entries.some(
(entry) => entry.via_api_key && !seen.has(entry.id),
);
@ -199,8 +196,7 @@ export function ApiMonitorOverlay(): ReactElement | null {
if (!autoOpen || isOpen) {
return;
}
// A dismissal holds for that burst and re-arms only once the API goes
// quiet, so the next request cannot re-open it a second later.
// A dismissal holds for the burst and re-arms only once the API goes quiet.
if (suppressed && quietFor < REARM_QUIET_MS) {
return;
}
@ -247,10 +243,8 @@ export function ApiMonitorOverlay(): ReactElement | null {
initial={{ opacity: 0, scale: 0.94 }}
animate={{ opacity: 1, scale: 1 }}
exit={{ opacity: 0, scale: 0.94 }}
/* Panel language borrowed from the sidebar's user menu and the
model selector: no hard border, an inset hairline plus a soft
drop shadow (menu-soft-surface), a 20px corner, and the heading
font throughout. */
/* Panel language from the sidebar's user menu and the model selector:
menu-soft-surface, a 20px corner, and the heading font throughout. */
className="menu-soft-surface pointer-events-auto fixed bottom-4 right-4 flex w-[400px] max-w-[calc(100vw-2rem)] cursor-default select-none resize flex-col overflow-hidden rounded-[20px] border-0 p-2.5 font-heading ring-0"
>
<div className="flex items-center justify-between gap-2 px-1.5 pb-2 pt-0.5">
@ -302,7 +296,7 @@ export function ApiMonitorOverlay(): ReactElement | null {
{data?.active_model ?? "No model loaded"}
</p>
{/* Metrics on a soft tile, as the Hub and Train pages group readouts. */}
{/* Soft tile, as the Hub and Train pages group readouts. */}
<div className="grid grid-cols-4 rounded-[14px] bg-muted/45 py-2.5 dark:bg-background/45">
<StatCell
label="Live"
@ -384,7 +378,7 @@ export function ApiMonitorOverlay(): ReactElement | null {
Expand to full monitor
</button>
{/* Closing only silences this burst; this is the permanent off. */}
{/* Closing silences this burst; this is the permanent off. */}
<button
type="button"
onClick={() => {

64
studio/frontend/src/features/api-monitor/api-monitor-page.tsx
View file

@ -3,10 +3,8 @@
// Full-page monitor for Unsloth's OpenAI-compatible API server.
//
// This replaces the small console that used to be buried in the API settings
// tab. Settings still owns configuration (keys, auto-switch, examples); this
// page owns observability -- what is being served right now, which requests
// failed and why, and which saved settings a remote load will apply.
// Replaces the small console buried in the API settings tab. Settings still owns
// configuration (keys, auto-switch, examples); this page owns observability.
import { Button } from "@/components/ui/button";
import { Input } from "@/components/ui/input";
@ -50,8 +48,8 @@ import {
const API_INFERENCE_PREFIX_RE = /^\/api\/inference/;
const V1_PREFIX_RE = /^\/v1\//;
// Tries per revision for a detail payload. Bounded because the usual failure
// is an entry that has aged out of the ring buffer and never comes back.
// Tries per revision for a detail payload. Bounded because the usual failure is
// an entry aged out of the ring buffer, which never comes back.
const DETAIL_FETCH_ATTEMPTS = 3;
const STATUS_FILTERS: { value: MonitorStatusFilter; label: string }[] = [
@ -158,8 +156,7 @@ function CopyButton({
label: string;
}): ReactElement {
const [copied, setCopied] = useState(false);
// Clearing the tick on a timer would set state after unmount if the user
// navigates away mid-flash, so the timer is cancelled on cleanup.
// Cancel on cleanup, or navigating away mid-flash sets state after unmount.
const timerRef = useRef<number | undefined>(undefined);
useEffect(
() => () => {
@ -210,8 +207,7 @@ function ContextUsageBar({
<div
className={cn(
"h-full rounded-full transition-[width]",
// Near-full context is the usual cause of truncated replies, so it
// reads as a warning before it becomes a bug report.
// Near-full context is the usual cause of truncated replies.
pct >= 90
? "bg-red-500"
: pct >= 75
@ -243,7 +239,7 @@ function RequestRow({
entry.prompt_preview ||
(entry.status === "running" ? "Waiting for output…" : "No preview");
// A load, unload or download has no prompt or reply, so it reads as a status
// line rather than a request with a payload behind it.
// line rather than a request with a payload.
if (isLifecycleEntry(entry)) {
return (
<div className="flex w-full min-w-0 flex-col gap-1 border-b border-border/50 bg-muted/25 px-4 py-3 last:border-b-0">
@ -371,9 +367,8 @@ function RequestDetail({
detail?: ApiMonitorEntry;
loading: boolean;
}): ReactElement {
// The detail fetch is a separate request, so it can describe an older state of
// a still-streaming entry. Prefer it only once it is at least as fresh as the
// list row, otherwise the panel would appear to rewind while tokens arrive.
// The detail fetch is separate, so it can describe an older state of a streaming
// entry. Prefer it only once it is as fresh as the list row, or the panel rewinds.
const detailIsCurrent =
detail != null &&
detail.status === entry.status &&
@ -507,9 +502,8 @@ export function ApiMonitorPage(): ReactElement {
const [unloading, setUnloading] = useState(false);
const [unloadError, setUnloadError] = useState<string | null>(null);
// Manual release of the loaded model, so VRAM can be freed without waiting for
// the idle timer. /unload matches on the internal id, which the monitor does
// not carry (it advertises a host path), so read it from status.
// Manual release so VRAM is freed without waiting for the idle timer. /unload
// matches on the internal id, which the monitor does not carry, so read status.
const unloadActiveModel = async (): Promise<void> => {
setUnloading(true);
try {
@ -545,10 +539,9 @@ export function ApiMonitorPage(): ReactElement {
[visible, selectedId],
);
// Refetch the selected entry while it streams so the payload grows with the
// reply. Keyed on identity and revision, never on `details`: the fetch rewrites
// `details` on every success, so depending on it loops on the detail endpoint,
// which takes the same lock every generated token does.
// Refetch the selected entry while it streams so the payload grows with the reply.
// Keyed on identity and revision, never on `details`: the fetch rewrites `details`
// on every success, so depending on it loops on the detail endpoint.
const selectedId_ = selected?.id ?? null;
const selectedUpdatedAt = selected?.updated_at ?? null;
const selectedIsMissing = selectedId_ != null && details[selectedId_] == null;
@ -558,8 +551,7 @@ export function ApiMonitorPage(): ReactElement {
count: 0,
});
const [retryTick, setRetryTick] = useState(0);
// Flips as a fetch settles, successfully or not, which is what lets a failed
// one be noticed at all.
// Flips as a fetch settles either way, which is what lets a failure be noticed.
const detailInFlight = selectedId_ != null && loadingDetails.has(selectedId_);
useEffect(() => {
if (selectedId_ == null || detailInFlight) {
@ -571,12 +563,9 @@ export function ApiMonitorPage(): ReactElement {
if (!selectedIsMissing && lastFetchedRef.current === revision) {
return;
}
// A terminal row's revision never advances, so a fetch that failed had
// nothing left to re-run this effect and the payload stayed unavailable
// until the user picked another row. `loadingDetails` changing as the failed
// fetch settles is the trigger; the count bounds it, because the usual
// failure is an entry that aged out of the ring buffer and will never
// arrive however often it is asked for.
// A terminal row's revision never advances, so a failed fetch had nothing left to
// re-run this effect. `loadingDetails` settling is the trigger; the count bounds
// it, since the usual failure is an entry aged out of the ring buffer.
if (attemptsRef.current.revision !== revision) {
attemptsRef.current = { revision, count: 0 };
}
@ -584,15 +573,14 @@ export function ApiMonitorPage(): ReactElement {
return;
}
attemptsRef.current.count += 1;
// Only remember the revision when a fetch really started; the in-flight
// guard can refuse, and recording it anyway skips that revision for good.
// Only remember the revision when a fetch started: the in-flight guard can
// refuse, and recording it anyway skips that revision for good.
if (requestDetail(selectedId_)) {
lastFetchedRef.current = revision;
setRetryTick(0);
} else {
// Refused because an older fetch is still running. Nothing in this effect's
// deps will change when that one settles, so without a nudge a revision
// rejected here is never fetched, and a terminal reply stays truncated.
// Refused because an older fetch is running. No dep changes when it settles, so
// without this nudge the rejected revision is never fetched.
const timer = window.setTimeout(() => setRetryTick((n) => n + 1), 250);
return () => window.clearTimeout(timer);
}
@ -605,8 +593,8 @@ export function ApiMonitorPage(): ReactElement {
detailInFlight,
]);
// The desktop webview's origin is tauri://, not the API server, and the
// packaged app picks its port dynamically. Same source as the Agents tab.
// The desktop webview's origin is tauri://, not the API server, and the packaged
// app picks its port dynamically. Same source as the Agents tab.
const origin = typeof window === "undefined" ? "" : window.location.origin;
const baseUrl = `${isTauri ? (serverUrl ?? getApiBase()) : origin}/v1`;
const serverStatus = data?.status ?? "idle";
@ -715,8 +703,8 @@ export function ApiMonitorPage(): ReactElement {
</div>
</header>
{/* Server summary: the two things you check first when a client can't
reach the API -- the base URL to point it at, and what is loaded. */}
{/* What you check first when a client can't reach the API: the base URL
to point it at, and what is loaded. */}
<section className="flex flex-wrap items-center gap-x-6 gap-y-3 rounded-xl border border-border/60 bg-card px-4 py-3">
<div className="flex min-w-0 items-center gap-2.5">
<span className="flex size-8 shrink-0 items-center justify-center rounded-lg border border-border/60 bg-muted/40">

11
studio/frontend/src/features/api-monitor/components/saved-model-settings.tsx
View file

@ -1,12 +1,9 @@
// SPDX-License-Identifier: AGPL-3.0-only
// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
// What a remote load will actually apply, which is otherwise unanswerable from
// outside the process.
//
// Read only on purpose: the config lives both here and in the browser's own
// per-model store, and the model's settings page is the only place that owns
// both, so it is the only place that can forget a model completely.
// What a remote load will actually apply, otherwise unanswerable from outside the
// process. Read only: the config also lives in the browser's per-model store, and
// the model's settings page is the only place that owns both.
import { Skeleton } from "@/components/ui/skeleton";
import {
@ -16,7 +13,7 @@ import {
} from "@/features/model-picker/api/model-overrides";
import { type ReactElement, useCallback, useEffect, useState } from "react";
/** Human-readable summary of the fields the loader will apply, in load order. */
/** Summary of the fields the loader will apply, in load order. */
function describeOverride(override: ApiModelOverride): string[] {
const parts: string[] = [];
if (override.custom_context_length) {

17
studio/frontend/src/features/api-monitor/lifecycle.ts
View file

@ -1,19 +1,16 @@
// SPDX-License-Identifier: AGPL-3.0-only
// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
// Labels for model load/unload/download rows, shared by the overlay and the
// full page.
// Labels for model load/unload/download rows, shared by the overlay and the page.
//
// They live here rather than on the page because the overlay is mounted from
// __root.tsx, so importing them from the page would pull the whole page and its
// dependency graph into the eagerly loaded bundle and undo the route's
// lazyRouteComponent: every route would pay for the monitor page even when it
// is never opened.
// Their own module because the overlay is mounted from __root.tsx: importing them
// from the page would pull the whole page into the eager bundle and undo the
// route's lazyRouteComponent.
import type { ApiMonitorEntry } from "@/features/chat/types/api";
// A lifecycle row is a model load/unload/download, not an HTTP call: it carries
// an event and reason instead of a prompt, so there is no payload to expand.
// A lifecycle row is a model load/unload/download, not an HTTP call: it carries an
// event and reason instead of a prompt, so there is no payload to expand.
export function isLifecycleEntry(entry: ApiMonitorEntry): boolean {
return entry.kind === "lifecycle";
}
@ -32,7 +29,7 @@ export function lifecycleLabel(entry: ApiMonitorEntry): string {
if (entry.status === "completed") {
return "Model downloaded";
}
// A cancel is deliberate, so saying it failed misreads the user's own action.
// A cancel is deliberate, so calling it a failure misreads the user's action.
return entry.status === "cancelled"
? "Model download cancelled"
: "Model download failed";

22
studio/frontend/src/features/api-monitor/overlay-store.ts
View file

@ -5,12 +5,9 @@ import { create } from "zustand";
import { createJSONStorage, persist } from "zustand/middleware";
/**
* localStorage that cannot throw.
*
* Safari private browsing, Firefox with the origin's cookies blocked, and an
* opaque origin in a webview all make `window.localStorage` throw on access
* rather than return null. Losing the preference there is fine; taking the
* whole panel down with it is not.
* localStorage that cannot throw: Safari private browsing, blocked cookies and an
* opaque webview origin all make `window.localStorage` throw on access. Losing the
* preference there is fine; taking the whole panel down with it is not.
*/
const safeStorage = {
getItem: (name: string): string | null => {
@ -24,7 +21,7 @@ const safeStorage = {
try {
window.localStorage.setItem(name, value);
} catch {
// Quota exceeded or storage denied. The preference stays session-only.
// Quota exceeded or denied; the preference stays session-only.
}
},
removeItem: (name: string): void => {
@ -39,7 +36,7 @@ const safeStorage = {
interface ApiMonitorOverlayState {
/** Whether the floating panel is on screen right now. Session state. */
isOpen: boolean;
/** Set on close so the panel does not pop back during the same burst. */
/** Set on close so the panel does not pop back in the same burst. */
suppressed: boolean;
/** Persisted opt out: when false the panel never opens itself. */
autoOpen: boolean;
@ -48,10 +45,7 @@ interface ApiMonitorOverlayState {
setAutoOpen: (autoOpen: boolean) => void;
}
/**
* Only `autoOpen` persists. Open/closed is session state: a dismissal lasts the
* sitting, not forever.
*/
/** Only `autoOpen` persists; a dismissal lasts the sitting, not forever. */
export const useApiMonitorOverlayStore = create<ApiMonitorOverlayState>()(
persist(
(set) => ({
@ -67,8 +61,8 @@ export const useApiMonitorOverlayStore = create<ApiMonitorOverlayState>()(
version: 1,
storage: createJSONStorage(() => safeStorage),
partialize: (state) => ({ autoOpen: state.autoOpen }),
// Without this a version bump discards the payload, quietly handing the
// popup back to someone who had turned it off.
// Without this a version bump discards the payload and hands the popup back
// to someone who had turned it off.
migrate: (persisted) => persisted,
// Explicit merge so an older stored payload cannot resurrect `isOpen`.
merge: (persisted, current) => ({

57
studio/frontend/src/features/api-monitor/use-api-monitor.ts
View file

@ -12,7 +12,7 @@ import type {
} from "@/features/chat/types/api";
import { useCallback, useEffect, useMemo, useRef, useState } from "react";
/** Poll cadence while the monitor is live. Matches the settings console it replaces. */
/** Poll cadence while live. Matches the settings console it replaces. */
const POLL_INTERVAL_MS = 1500;
export type MonitorStatusFilter =
@ -30,7 +30,7 @@ export interface MonitorStats {
cancelled: number;
/** Mean duration over finished requests, or null when none have finished. */
avgDurationMs: number | null;
/** Slowest finished request, for spotting a single pathological call. */
/** Slowest finished request, for spotting a pathological call. */
maxDurationMs: number | null;
totalTokens: number;
/** Share of finished requests that failed, 0-1. Null when nothing finished. */
@ -47,8 +47,8 @@ function completionTokens(entry: ApiMonitorEntry): number | null {
if (entry.completion_tokens != null) {
return entry.completion_tokens;
}
// Some providers only report a total; subtracting the prompt is the best
// available estimate of what was actually generated.
// Some providers report only a total, so subtracting the prompt is the best
// estimate of what was generated.
if (entry.total_tokens != null && entry.prompt_tokens != null) {
return Math.max(0, entry.total_tokens - entry.prompt_tokens);
}
@ -71,21 +71,18 @@ export function computeStats(entries: ApiMonitorEntry[]): MonitorStats {
let durationSum = 0;
let durationCount = 0;
let maxDurationMs: number | null = null;
// Throughput is aggregated as total tokens over total time, not as the mean of
// each request's rate. Averaging rates lets one tiny fast request outweigh a
// long slow one, which is the opposite of what someone debugging wants to see.
// Total tokens over total time, not the mean of each request's rate: averaging
// rates lets one tiny fast request outweigh a long slow one.
let generatedTokens = 0;
let generatedDurationMs = 0;
let requests = 0;
for (const entry of entries) {
// A model load, unload or download is not an HTTP call. It shows as
// "running" for as long as the load takes, so counting it would report an
// in-flight request with no client waiting and fold a multi-minute download
// into "Avg latency". The backend already leaves these out of
// active_count for the same reason, so counting them here would also make
// the page disagree with the number the API itself reports.
// A load, unload or download is not an HTTP call. It reads as "running" for the
// whole load, so counting it reports an in-flight request with no client waiting
// and folds a multi-minute download into "Avg latency". The backend leaves these
// out of active_count too, so counting them would also disagree with the API.
if (entry.kind === "lifecycle") {
continue;
}
@ -107,7 +104,7 @@ export function computeStats(entries: ApiMonitorEntry[]): MonitorStats {
maxDurationMs =
maxDurationMs == null ? duration : Math.max(maxDurationMs, duration);
const generated = completionTokens(entry);
// Sub-millisecond durations would divide into a meaningless rate.
// A sub-millisecond duration divides into a meaningless rate.
if (generated != null && generated > 0 && duration > 0) {
generatedTokens += generated;
generatedDurationMs += duration;
@ -146,11 +143,9 @@ export function filterEntries(
if (!needle) {
return true;
}
// Search the fields a debugging session actually keys off: which model,
// which endpoint, and the previews/error text visible in the row.
//
// Coerced, not trusted: these arrive over the network, and one malformed
// entry throwing here would blank the whole log.
// The fields a debugging session keys off: model, endpoint, and the previews and
// error text visible in the row. Coerced, not trusted: these arrive over the
// network, and one malformed entry throwing here would blank the whole log.
return [
entry.model,
entry.endpoint,
@ -170,14 +165,14 @@ interface UseApiMonitorResult {
entries: ApiMonitorEntry[];
stats: MonitorStats;
error: string | null;
/** True until the first response lands, so the page can show skeletons once. */
/** True until the first response lands, so skeletons show once. */
loading: boolean;
refreshing: boolean;
paused: boolean;
setPaused: (paused: boolean) => void;
refresh: () => void;
clear: () => Promise<void>;
/** Full prompt/reply for entries the user expanded, keyed by entry id. */
/** Full prompt/reply for expanded entries, keyed by entry id. */
details: Record<string, ApiMonitorEntry>;
loadingDetails: ReadonlySet<string>;
requestDetail: (id: string) => boolean;
@ -186,10 +181,9 @@ interface UseApiMonitorResult {
/**
* Live view of the server's OpenAI-compatible API traffic.
*
* Polls rather than streams because the backing monitor is an in-memory ring
* buffer with no change feed. Polling is self-rescheduling (never overlapping),
* and pausing stops it entirely so a user reading a stalled request's payload
* isn't fighting a list that reorders under them.
* Polls rather than streams because the backing monitor is an in-memory ring buffer
* with no change feed. Polling self-reschedules (never overlapping), and pausing
* stops it so reading a stalled payload is not fighting a list that reorders.
*
* `intervalMs` lets a caller trade freshness for cost: the full page wants the
* default live cadence, while the floating overlay slows right down when it is
@ -207,8 +201,8 @@ export function useApiMonitor({
const [loadingDetails, setLoadingDetails] = useState<Set<string>>(
() => new Set(),
);
// Mirrors `loadingDetails` outside React state so the fetch guard sees writes
// from the same tick (state updates are async and would let duplicates through).
// Mirrors `loadingDetails` outside React state so the fetch guard sees same-tick
// writes; async state updates would let duplicates through.
const inFlightDetails = useRef<Set<string>>(new Set());
const load = useCallback(async (): Promise<void> => {
@ -259,9 +253,8 @@ export function useApiMonitor({
};
}, [paused, intervalMs]);
// Returns whether a fetch actually started. Callers that remember "I have
// fetched revision N" must not record it when the in-flight guard turned them
// away, or that revision is skipped for good once updated_at stops moving.
// Returns whether a fetch started: a caller must not record "fetched revision N"
// when the guard refused, or that revision is skipped once updated_at settles.
const requestDetail = useCallback((id: string): boolean => {
if (inFlightDetails.current.has(id)) {
return false;
@ -273,8 +266,8 @@ export function useApiMonitor({
setDetails((prev) => ({ ...prev, [id]: entry }));
})
.catch(() => {
// The entry aged out of the ring buffer; drop any stale copy so the UI
// falls back to the row previews instead of showing a frozen payload.
// Aged out of the ring buffer; drop the stale copy so the UI falls back to
// the row previews instead of a frozen payload.
setDetails((prev) => {
if (!(id in prev)) return prev;
const next = { ...prev };

4
studio/frontend/src/features/chat/types/api.ts
View file

@ -284,8 +284,8 @@ export interface ApiMonitorEntry {
model: string;
prompt?: string;
reply?: string;
// True when the caller used an API key rather than a UI session. The floating
// panel keys its auto-open off this so Studio's own chat does not pop it.
// True for API-key callers, not UI sessions. The floating panel keys its
// auto-open off this so Studio's own chat does not pop it.
via_api_key: boolean;
prompt_preview: string;
reply_preview: string;

1
studio/frontend/src/features/hub/catalog/download-card.tsx
View file

@ -83,7 +83,6 @@ export function CardDivider() {
);
}
/** Gear that opens a downloaded model's full settings page. */
export function CardSettingsButton({
label,
onClick,

18
studio/frontend/src/features/hub/catalog/hub-model-settings-view.tsx
View file

@ -4,9 +4,8 @@
// Full-page settings for one model, opened from the Hub.
//
// The same controls exist in the chat picker's popover, but a popover is a poor
// place to work through every knob a model has. This gives them a page, and
// states plainly that whatever is saved here is what an API load will use --
// the settings are mirrored server-side by ModelConfigPage's save.
// place to work through every knob. This gives them a page and says plainly that
// what is saved here is what an API load uses, mirrored by ModelConfigPage.
import { ModelConfigPage, type ModelPickTarget } from "@/features/model-picker";
import type { PerModelConfig } from "@/features/model-picker";
@ -24,7 +23,7 @@ export function HubModelSettingsView({
compact = false,
}: {
target: ModelPickTarget;
/** Non-null when this model is the loaded one, so the page can show live values. */
/** Non-null when this model is loaded, so the page can show live values. */
loadedConfig?: PerModelConfig | null;
loadedContextLength?: number | null;
onBack: () => void;
@ -34,8 +33,7 @@ export function HubModelSettingsView({
}) {
const scrollRef = useRef<HTMLDivElement | null>(null);
const [scrolled, setScrolled] = useState(false);
// Mirrors HubDetailView so this view sits at the same measure as the rest of
// the Hub rather than introducing a third column width.
// Mirrors HubDetailView so this view sits at the Hub's measure.
const measure = compact
? "mx-auto w-full max-w-[860px] px-5 sm:px-5"
: "mx-auto w-full max-w-[1100px] px-5 sm:px-8";
@ -109,9 +107,8 @@ export function HubModelSettingsView({
/>
</span>
<p className="min-w-0 text-ui-12 leading-[1.5] text-muted-foreground">
{/* Only what auto-switch can reach is mirrored to the server: it
indexes GGUFs and skips Ollama, so promising the API case for
anything else describes a load that cannot happen. */}
{/* Only what auto-switch can reach is mirrored: it indexes GGUFs and
skips Ollama, so anything else cannot be loaded by the API. */}
{(target.apiLoadable ?? target.isGguf)
? "Saved settings apply everywhere this model loads, including when an OpenAI-compatible API request asks for it."
: "Saved settings apply everywhere Studio loads this model."}{" "}
@ -131,8 +128,7 @@ export function HubModelSettingsView({
loadedConfig={loadedConfig}
loadedContextLength={loadedContextLength}
variant="page"
// The page heading above already names the model; the built-in
// "Run settings" block would print it a second time.
// The page heading already names the model; "Run settings" would repeat it.
showHeader={false}
/>
</div>

6
studio/frontend/src/features/hub/catalog/local-on-device-card.tsx
View file

@ -97,7 +97,7 @@ interface LocalOnDeviceCardProps {
onEject?: () => void;
onTrain?: () => void;
onChange?: () => void;
/** Open this model's full settings page for the shown quant. */
/** Open settings for the quant this card is showing. */
onOpenSettings?: (ggufVariant: string | null) => void;
}
@ -556,8 +556,8 @@ export function LocalOnDeviceCard({
{onOpenSettings && (
<CardSettingsButton
label={`Settings for ${repoId}`}
// Hand over the quant this card resolved, so the settings page
// edits the variant the user is looking at rather than the repo.
// Pass the quant this card resolved, so the settings page edits the
// variant on screen rather than the repo.
onClick={() => onOpenSettings(selectedQuant ?? null)}
/>
)}

2
studio/frontend/src/features/hub/catalog/model-inspector.tsx
View file

@ -403,7 +403,7 @@ export type ModelInspectorActions = {
onTrain?: () => void;
onInventoryChange?: () => void;
onSearchHub?: (query: string) => void;
/** Open this model's full settings page, with the quant the card resolved. */
/** Open settings with the quant the card resolved. */
onOpenSettings?: (ggufVariant: string | null) => void;
};

1
studio/frontend/src/features/hub/catalog/models-catalog-lists.tsx
View file

@ -285,7 +285,6 @@ export function DownloadedList({
compact?: boolean;
sort: InventorySort;
onInventoryChange?: () => void;
/** Open a downloaded model's full settings page. */
onOpenModelSettings?: (row: CachedInventoryRow | LocalInventoryRow) => void;
}) {
// Pinned repos surface first regardless of the active sort; the chosen sort

10
studio/frontend/src/features/hub/catalog/models-catalog-rows.tsx
View file

@ -586,7 +586,7 @@ export const InventoryRow = memo(function InventoryRow({
compact?: boolean;
onSelect: (id: string) => void;
onChange?: () => void;
/** Open this model's full settings page. Omitted for datasets. */
/** Open this model's settings page. Omitted for datasets. */
onOpenSettings?: (row: CachedInventoryRow | LocalInventoryRow) => void;
}) {
const rowModelId =
@ -735,11 +735,9 @@ export const InventoryRow = memo(function InventoryRow({
const rowPinned =
cacheDeletableRepoId != null &&
pinnedKeys.includes(pinKey(cacheDeletableRepoId));
// Settings is available for any downloaded model, not just deletable ones: a
// model scanned from a local folder is just as configurable as a cached repo.
// The menu therefore renders whenever either action applies, and each item
// gates itself. `deletableRepoId` (rather than the boolean) keeps the non-null
// narrowing the delete closures below rely on.
// Settings applies to any downloaded model, not just deletable ones, so the menu
// renders when either action applies and each item gates itself. `deletableRepoId`
// (not the boolean) keeps the non-null narrowing the delete closures rely on.
const settingsAction =
!isDataset && onOpenSettings ? { onOpen: () => onOpenSettings(row) } : undefined;
const deletableRepoId = canDelete ? cacheDeletableRepoId : null;

1
studio/frontend/src/features/hub/catalog/models-catalog.tsx
View file

@ -71,7 +71,6 @@ export interface ModelsCatalogHandlers {
onRetry: () => void;
onInventoryChange?: () => void;
onSwitchDevice?: () => void;
/** Open a downloaded model's full settings page. */
onOpenModelSettings?: (row: InventoryRow) => void;
}

102
studio/frontend/src/features/hub/hub-page.tsx
View file

@ -113,11 +113,9 @@ import type {
} from "./types";
// What per-model settings are keyed by, which is not always what the loader is
// handed: a repo cached outside the active HF cache loads by snapshot path,
// while the chat picker (toCachedModelRepo) and the auto-switch index both key
// it by repo id. Saving under the path would leave the settings where no other
// load looks for them. Local rows are keyed by their load id in both places, so
// they keep it.
// handed: a repo cached outside the active HF cache loads by snapshot path, while
// the chat picker and the auto-switch index key it by repo id, so saving under the
// path strands the settings. Local rows are keyed by load id in both places.
function modelConfigIdentity(
kind: SelectedModelView["kind"],
resource: SelectedResourceRef,
@ -369,8 +367,8 @@ export function ModelsPage() {
const activeGgufContextLength = useChatRuntimeStore(
(s) => s.ggufContextLength,
);
// Live settings of the loaded model, so opening its settings page shows what
// it is actually running with rather than the last saved draft.
// Live settings of the loaded model, so its settings page shows what it is
// running with rather than the last saved draft.
const { config: activeModelConfig } = useActiveModelConfig();
// Shared with the chat model selector: list only models sized for this device.
const fitOnDeviceOnly = useChatRuntimeStore((s) => s.fitOnDeviceOnly);
@ -1245,23 +1243,23 @@ export function ModelsPage() {
[runSelectedModel, selectedModel],
);
// Full-page per-model settings, opened from a downloaded row's menu. Local
// state rather than a URL param: the page is a transient editor over the
// catalog, and a deep link to it would need the row's identity re-resolved
// against an inventory that may not have loaded yet.
// Full-page per-model settings, opened from a downloaded row's menu. Local state
// rather than a URL param: the page is a transient editor over the catalog, and a
// deep link would need the row re-resolved against an inventory that may not have
// loaded.
const [settingsTarget, setSettingsTarget] = useState<ModelPickTarget | null>(
null,
);
// Bumped per open so a slow variant lookup for a row the user has moved on
// from cannot land on top of the row they actually chose.
// Bumped per open so a slow variant lookup for an abandoned row cannot land
// on top of the row actually chosen.
const settingsOpenSeq = useRef(0);
const openModelSettings = useCallback(
async (row: CachedInventoryRow | LocalInventoryRow) => {
const openSeq = ++settingsOpenSeq.current;
// loadId is what the loader accepts; repoId is only a display/API alias.
const id = row.loadId;
// Whether the loaded model is this row, under any of the names it goes by.
// Gates the "prefer the loaded quant" hint below.
// Whether this row is the loaded model, under any of its names. Gates the
// "prefer the loaded quant" hint below.
const rowAliases =
row.kind === "local"
? [id, row.repoId, row.path]
@ -1269,12 +1267,10 @@ export function ModelsPage() {
const rowIsActive = rowAliases.some((alias) =>
modelIdsMatch(alias, activeCheckpoint),
);
// Cached repo rows never carry a quant: the inventory emits one row per
// repo with format_variant null (see cache_inventory.py). Opening settings
// with a null variant would key the saved config to `repo::` while the
// loader reads `repo::Q4_K_M`, so the settings would silently never apply
// and the server mirror would be keyed wrong too. Resolve the quant the
// same way the on-device card does before opening.
// Cached repo rows never carry a quant (cache_inventory.py emits one row per
// repo with format_variant null). Opening with a null variant keys the config
// to `repo::` while the loader reads `repo::Q4_K_M`, so it never applies and
// the server mirror is wrong too. Resolve it as the on-device card does.
let ggufVariant = row.formatVariant?.trim() || null;
if (!ggufVariant && row.isGguf && row.capabilities.requiresVariant) {
const repoId = row.kind === "cache" ? row.repoId : (row.repoId ?? null);
@ -1287,10 +1283,10 @@ export function ModelsPage() {
});
const downloaded = res.variants.filter((v) => v.downloaded);
ggufVariant =
// Prefer the loaded quant, then the repo default, then whatever is
// on disk, mirroring LocalOnDeviceCard's selectedQuant. Only when
// this row is the loaded model: Q4_K_M exists in most repos, so an
// unguarded match would target the wrong quant of the wrong model.
// Loaded quant, then the repo default, then whatever is on disk,
// mirroring LocalOnDeviceCard's selectedQuant. Only for the loaded
// row: Q4_K_M exists in most repos, so an unguarded match would
// target the wrong quant of the wrong model.
(rowIsActive
? downloaded.find((v) =>
ggufVariantsMatch(v.quant, activeGgufVariant),
@ -1307,9 +1303,8 @@ export function ModelsPage() {
}
if (!ggufVariant) {
// A model that needs a quant cannot be configured without one: the
// picker matches variants exactly and would never find the saved
// config, while the API falls back to the bare key and would apply it.
// Opening the editor here would quietly create that mismatch.
// picker matches variants exactly and would never find the config,
// while the API's bare-key fallback would apply it.
toast.error("Couldn't determine which quant to configure.", {
description:
"Settings for this model are per quant. Check the connection or the model's cache, then try again.",
@ -1317,13 +1312,13 @@ export function ModelsPage() {
return;
}
}
// The variant lookup above is async, so a second row opened while it was
// pending would otherwise be overwritten by whichever call finished last.
// The variant lookup is async, so without this a second row opened while it
// was pending would be overwritten by whichever call finished last.
if (settingsOpenSeq.current !== openSeq) {
return;
}
// A repo in a previous cache loads by snapshot path, so `id` ends in the
// revision hash; name the row by what the user calls it.
// revision hash; name the row by what the user calls it instead.
const configId = row.kind === "cache" ? row.repoId : id;
const leaf = configId.split(/[\\/]/).filter(Boolean).pop() ?? configId;
setSettingsTarget({
@ -1340,20 +1335,19 @@ export function ModelsPage() {
isLora: row.modelFormat === "adapter",
ggufVariant: ggufVariant ?? undefined,
isGguf: row.isGguf,
// Partial downloads still open settings, but must not claim to be
// complete or the loader skips its download-progress reporting.
// A partial download opens settings too, but claiming complete would skip
// the loader's download-progress reporting.
isDownloaded: !row.partial,
// Not carried on inventory rows; ModelConfigPage reads the GGUF header
// itself to size the context slider.
// Not on inventory rows; ModelConfigPage reads the GGUF header itself.
contextLength: null,
},
});
},
[activeCheckpoint, activeGgufVariant, hfToken],
);
// Applying from the settings page loads the model with exactly those settings.
// ModelConfigPage has already persisted them (locally and, when "remember" is
// on, to the server), so an API request for this model gets the same load.
// Applying loads the model with exactly these settings. ModelConfigPage has
// already persisted them locally and, when "remember" is on, to the server, so
// an API request for this model gets the same load.
const runSettingsTarget = useCallback(
(config: PerModelConfig) => {
const target = settingsTarget;
@ -1386,16 +1380,14 @@ export function ModelsPage() {
const handleTrain = useCallback(() => {
// Hub → train integration ships in a later PR.
}, []);
// Settings opened from the detail view's on-device card. The card resolves
// which quant it is showing, so it passes that in rather than re-deriving it.
// Opened from the detail view's on-device card, which passes in the quant it
// resolved rather than making this re-derive it.
const openSelectedModelSettings = useCallback(
(ggufVariant: string | null) => {
if (!selectedModel) return;
// The card passes null while its own variant lookup is pending or after it
// failed, so this needs the same guard openModelSettings applies: a model
// that needs a quant cannot be configured without one, because the picker
// matches variants exactly and would never find the saved config while the
// API falls back to the bare key and would apply it.
// The card passes null while its variant lookup is pending or after it failed,
// so this needs the same guard openModelSettings applies: a model that needs a
// quant cannot be configured without one.
if (
!ggufVariant &&
selectedModel.isGguf &&
@ -1407,8 +1399,8 @@ export function ModelsPage() {
});
return;
}
// Share the sequence with openModelSettings: a row's variant lookup may
// still be pending, and it must not land on top of this one.
// Share the sequence with openModelSettings: a pending variant lookup for
// another row must not land on top of this one.
settingsOpenSeq.current += 1;
const id = selectedModel.resource.runId;
const configId = modelConfigIdentity(
@ -1736,9 +1728,8 @@ export function ModelsPage() {
const detailOpen = urlModel !== null;
const splitMode = allModelsView === "split";
// The catalog is unreachable when an opaque overlay sits on top of it: the
// detail view (full-page layout only, since split renders it alongside) or the
// settings page (always full-bleed).
// The catalog is unreachable under an opaque overlay: the detail view (full-page
// layout only, since split renders it alongside) or the settings page.
const catalogCovered = (detailOpen && !splitMode) || settingsTarget !== null;
return (
@ -1796,10 +1787,8 @@ export function ModelsPage() {
? "flex-1 lg:w-[460px] lg:max-w-[44%] lg:flex-none lg:shrink-0 lg:border-r lg:border-border/60"
: "flex-1",
// The settings page is a full-bleed opaque overlay in every layout,
// including split, so it always takes the catalog out of the tab
// order. Without this, tabbing out of the settings form walks into
// the virtualized rows hidden behind it and screen readers announce
// the whole model list underneath.
// so it always takes the catalog out of the tab order. Without this,
// tabbing out of the form walks into the virtualized rows behind it.
catalogCovered && "pointer-events-none",
)}
aria-hidden={catalogCovered || undefined}
@ -1858,9 +1847,8 @@ export function ModelsPage() {
)
)}
{/* Sits above the detail overlay (z-30): opening settings from a row
while a model preview is open should show the settings, not stack
behind it. */}
{/* Above the detail overlay (z-30): opening settings while a model
preview is open should show the settings, not stack behind it. */}
{settingsTarget && (
<div className="hub-canvas absolute inset-0 z-30 flex min-h-0 flex-col">
<HubModelSettingsView

49
studio/frontend/src/features/model-picker/api/migrate-model-overrides.ts
View file

@ -4,9 +4,8 @@
// One-time backfill of per-model settings into the server override map.
//
// Settings used to live only in this browser, so on upgrade the server knows
// nothing about models the user already configured. Without this they keep
// showing as remembered in the UI while an API load quietly uses app defaults,
// which is the exact bug the server-side map exists to fix.
// nothing about models already configured: they still show as remembered while an
// API load uses app defaults, the exact bug the server-side map exists to fix.
import {
normalizeGgufVariantIdentity,
@ -44,13 +43,12 @@ function markRan(): void {
/**
* A server key under the same identity this browser stores.
*
* `app_settings` has no schema version and holds whatever id was current when
* the row was written, so an install that predates identity normalization has
* keys like `Unsloth/Repo-GGUF:Q4_K_M` while this browser only ever produces
* the folded form. The backend resolves the two to one model, so an exact
* property lookup would report "not on the server" for a row that is, and the
* backfill would overwrite it. Variants never contain a colon, so the last one
* splits the key; a repo id folds and a POSIX path deliberately does not.
* `app_settings` has no schema version and holds whatever id was current when the
* row was written, so an old install has keys like `Unsloth/Repo-GGUF:Q4_K_M` while
* this browser only produces the folded form. The backend resolves both to one
* model, so an exact lookup would report "not on the server" and let the backfill
* overwrite it. Variants never contain a colon, so the last one splits the key; a
* repo id folds and a POSIX path deliberately does not.
*/
function normalizedOverrideKey(key: string): string {
const separator = key.lastIndexOf(":");
@ -64,21 +62,19 @@ function normalizedOverrideKey(key: string): string {
}
/**
* Push local configs the server has never seen. Never deletes and never
* overwrites: an entry already on the server is the newer authority, and losing
* a setting here would be worse than leaving one unmigrated.
* Push local configs the server has never seen. Never deletes and never overwrites:
* an entry already there is the newer authority, and losing a setting would be
* worse than leaving one unmigrated.
*/
export async function backfillModelOverrides(): Promise<void> {
if (alreadyRan()) {
return;
}
const local = listPerModelConfigs().filter(
// A quant means it is a GGUF, which is the only thing API auto-switch
// resolves. Backfilling a safetensors config would claim an API behaviour
// that does not exist. A standalone .gguf picked directly carries no quant
// to select between, so it is stored with a null variant and would fail
// that test despite being exactly what auto-switch does resolve; the flag
// is then set and its settings stay browser-only for good.
// A quant means GGUF, the only thing API auto-switch resolves, so backfilling a
// safetensors config would claim behaviour that does not exist. A standalone
// .gguf has no quant to select between and is stored with a null variant, so it
// needs the extra test or its settings stay browser-only for good.
(entry) =>
(entry.ggufVariant != null ||
entry.modelId.toLowerCase().endsWith(".gguf")) &&
@ -94,7 +90,7 @@ export async function backfillModelOverrides(): Promise<void> {
existing = await fetchModelOverrides();
} catch {
// Offline or not authenticated yet. Leave the flag unset so the next start
// tries again rather than silently skipping the migration forever.
// retries rather than skipping the migration forever.
return;
}
@ -102,20 +98,17 @@ export async function backfillModelOverrides(): Promise<void> {
let failed = false;
for (const entry of local) {
// Folded on this side too: a v2 storage key already holds the normalized
// identity, but the older `id::variant` keys this browser still reads back
// hold whatever casing was typed.
// Folded here too: a v2 storage key holds the normalized identity, but the
// older `id::variant` keys this browser still reads hold the typed casing.
const key = normalizedOverrideKey(
modelOverrideKey(entry.modelId, entry.ggufVariant),
);
if (known.has(key)) {
continue;
}
// Re-read rather than trusting the snapshot taken before the fetch above.
// A save or a forget during that round trip would otherwise be undone by
// this write, since it is queued behind the interactive one and commits
// last: the browser would show the new settings while an API load applied
// the old ones.
// Re-read rather than trusting the snapshot from before the fetch: this write
// is queued behind the interactive one and commits last, so a save or forget
// during the round trip would be undone by it.
const current = listPerModelConfigs().find(
(candidate) =>
normalizedOverrideKey(

67
studio/frontend/src/features/model-picker/api/model-overrides.ts
View file

@ -3,13 +3,11 @@
// Server-side mirror of the per-model config.
//
// The per-model config in ../model-config/per-model-config.ts lives in browser
// localStorage, so it only ever applied to loads the browser made. A model loaded
// by an OpenAI-compatible API request (Model auto-switch) is loaded by the backend
// with no browser in the loop, which is why an API load used to come up with none
// of the settings the user had configured. Mirroring every save to the backend's
// override map closes that gap: routes/inference.py reads it on the auto-switch
// path and rebuilds the same LoadRequest the picker would have sent.
// The config in ../model-config/per-model-config.ts lives in browser localStorage,
// so it only applied to loads the browser made, and an API auto-switch load (no
// browser in the loop) came up with none of the user's settings. Mirroring every
// save to the backend's override map closes that gap: routes/inference.py reads it
// and rebuilds the same LoadRequest the picker would have sent.
import { authFetch } from "@/features/auth";
import { readFastApiError } from "@/lib/format-fastapi-error";
@ -52,11 +50,9 @@ export interface ApiModelOverride {
export type ApiModelOverrides = Record<string, ApiModelOverride>;
/**
* The key one model's config is stored under.
*
* Uses the `repo:VARIANT` form an OpenAI request names a quant by, so two quants
* of the same repo keep separate configs and the backend can match the requested
* model name directly. Falls back to the bare id when there is no variant.
* The key one model's config is stored under: the `repo:VARIANT` form an OpenAI
* request names a quant by, so two quants of one repo keep separate configs and the
* backend matches the requested name directly. Bare id when there is no variant.
*/
export function modelOverrideKey(
modelId: string,
@ -79,10 +75,9 @@ export async function fetchModelOverrides(): Promise<ApiModelOverrides> {
/**
* Translate the UI's per-model config into the backend's schema.
*
* Only fields the user actually set are sent: the backend reads an absent field
* as "use the app default", so sending nulls would pin defaults and stop the
* model following later changes to the global preferences. `null` config means
* "no saved settings", which clears the entry.
* Only fields the user set are sent: the backend reads an absent field as "app
* default", so nulls would pin defaults and stop the model following later global
* changes. A `null` config means "no saved settings", which clears the entry.
*/
function toApiOverride(config: PerModelConfig | null): ApiModelOverride {
if (!config) {
@ -127,12 +122,10 @@ function toApiOverride(config: PerModelConfig | null): ApiModelOverride {
return payload;
}
// One in-flight write per model, so writes for the same model commit in the
// order they were issued. Saving twice quickly, or saving while the one-time
// backfill is still running, otherwise leaves two independent requests racing:
// the older response can land last and resurrect the entry the newer one meant
// to replace or remove, and an API load then applies settings the user has
// already changed. Different models still overlap.
// One in-flight write per model, so writes for a model commit in issue order.
// Otherwise saving twice quickly, or saving during the one-time backfill, races:
// the older response can land last and resurrect the entry the newer one meant to
// replace. Different models still overlap.
const writesByKey = new Map<string, Promise<void>>();
export async function putModelOverride(
@ -140,10 +133,9 @@ export async function putModelOverride(
ggufVariant: string | null | undefined,
config: PerModelConfig | null,
): Promise<void> {
// Keyed by the folded identity, not the literal spelling: the backfill sends
// a legacy casing while a UI save sends the normalized one, and the backend
// resolves both to the same row, so raw strings would open two queues for one
// model and let them race again.
// Keyed by the folded identity, not the literal spelling: the backfill sends a
// legacy casing and a UI save the normalized one, and the backend resolves both
// to one row, so raw strings would open two queues and race again.
const key = modelOverrideKey(
normalizeModelIdentity(modelId),
normalizeGgufVariantIdentity(ggufVariant),
@ -157,8 +149,7 @@ export async function putModelOverride(
try {
await write;
} finally {
// Only the last writer clears the slot, so a queue that is still building
// keeps its ordering.
// Only the last writer clears the slot, so a queue still building keeps order.
if (writesByKey.get(key) === write) {
writesByKey.delete(key);
}
@ -176,13 +167,12 @@ async function sendModelOverride(
body: JSON.stringify({
// biome-ignore lint/style/useNamingConvention: API schema
model_id: modelOverrideKey(modelId, ggufVariant),
// Say which operation this is. A save of an all-default config carries no
// fields, which is shape-identical to "forget this model", and guessing
// wrong wipes launch flags the UI cannot show or restore.
// Say which operation this is: an all-default save carries no fields, which is
// shape-identical to "forget this model", and guessing wrong wipes launch flags
// the UI cannot show or restore.
remove: config === null,
// Launch flags have no UI control, so the backend preserves them when the
// field is omitted. Forgetting a model means forgetting all of it, so that
// path sends an explicit empty list to clear them.
// Launch flags have no UI control, so the backend preserves them when omitted.
// Forgetting means forgetting all of it, so that path sends an explicit [].
// biome-ignore lint/style/useNamingConvention: API schema
...(config === null ? { llama_extra_args: [] } : {}),
...toApiOverride(config),
@ -198,11 +188,10 @@ async function sendModelOverride(
/**
* Mirror a per-model config save to the backend without blocking the UI.
*
* Deliberately best-effort: the localStorage write is the source of truth for
* this browser and has already happened by the time this runs, so a failed sync
* must not fail the save or interrupt a model load. It is logged rather than
* toasted -- the only consequence is that an API-triggered load of this model
* falls back to app defaults until the next successful save.
* Best-effort: the localStorage write is this browser's source of truth and has
* already happened, so a failed sync must not fail the save or interrupt a load.
* Logged rather than toasted: an API load of this model just falls back to app
* defaults until the next successful save.
*/
export function syncModelOverride(
modelId: string,

34
studio/frontend/src/features/model-picker/components/model-config-page.tsx
View file

@ -582,9 +582,8 @@ interface ModelConfigPageProps {
initialConfig?: PerModelConfig | null;
variant?: "page" | "sidebar";
/**
* Page variant only: render the built-in "Run settings" title block. A host
* that already shows the model name as its own page heading (the Hub's
* settings page) turns this off so the name is not printed twice.
* Page variant only: render the built-in "Run settings" title block. A host that
* already shows the model name as its page heading turns this off.
*/
showHeader?: boolean;
}
@ -611,9 +610,9 @@ export function ModelConfigPage({
const loadedMaxContextLength = useChatRuntimeStore(
(s) => s.ggufMaxContextLength,
);
// What the settings are stored under, which is not always what loads: see
// ModelPickTarget.configId. Every read, write and mirror below uses it; the
// probes keep target.id, since they have to open the model.
// What the settings are stored under, which is not always what loads (see
// ModelPickTarget.configId). Every read, write and mirror uses it; the probes
// keep target.id, since they have to open the model.
const configId = target.configId ?? target.id;
const resolveInitial = () => {
const resolved = resolveInitialConfig(configId, target.ggufVariant);
@ -885,18 +884,14 @@ export function ModelConfigPage({
} else {
saveFailed = !deletePerModelConfig(configId, target.ggufVariant);
}
// Mirror to the server so an OpenAI-compatible API request that loads this
// model gets these exact settings, not app defaults. Best-effort and
// non-blocking: the localStorage write above already governs this browser.
// Forgetting clears the server entry too, so the two never disagree.
// Mirror to the server so an API request that loads this model gets these exact
// settings, not app defaults. Best-effort and non-blocking: the localStorage
// write above already governs this browser, and forgetting clears both.
//
// Skipped when the local write failed (quota, a future-schema entry): the
// browser and the server would otherwise permanently disagree about this
// model, with no way for the user to tell which one the next load used.
// Auto-switch reach, not just GGUF-ness: the resolver indexes GGUFs and
// skips Ollama's scanner, so mirroring either a safetensors config or an
// Ollama one would advertise settings on the monitor's "applied on API load"
// list that no API request can ever apply.
// Skipped when the local write failed (quota, a future-schema entry), or the
// two would permanently disagree with no way to tell which the next load used.
// Gated on auto-switch reach, not just GGUF-ness: the resolver indexes GGUFs and
// skips Ollama, so mirroring either would advertise a load that cannot happen.
if (!saveFailed && (target.apiLoadable ?? target.isGguf)) {
syncModelOverride(
configId,
@ -904,9 +899,8 @@ export function ModelConfigPage({
remember ? effectiveRuntimeConfig : null,
);
}
// Saving can push the local map over budget and silently drop other models.
// Their server entries would otherwise keep being applied by API loads with
// nothing left in the UI showing them or able to forget them.
// Saving can push the local map over budget and drop other models, whose server
// entries would keep being applied with nothing in the UI able to forget them.
for (const dropped of evicted) {
syncModelOverride(dropped.modelId, dropped.ggufVariant, null);
}

3
studio/frontend/src/features/model-picker/components/model-selector/model-row-menu.tsx
View file

@ -74,7 +74,7 @@ interface ModelRowMenuCachePath {
variant?: string;
}
/** Opens the model's own settings page (load config + what the API will apply). */
/** The model's settings page: load config plus what the API will apply. */
interface ModelRowMenuSettings {
onOpen: () => void;
}
@ -94,7 +94,6 @@ export function ModelRowMenu({
iconClassName?: string;
/** Enables "Reveal in Finder" for cached repos. */
cachePath?: ModelRowMenuCachePath;
/** Opens this model's full settings page. */
settings?: ModelRowMenuSettings;
pin?: ModelRowMenuPin;
update?: ModelRowMenuUpdate;

11
studio/frontend/src/features/model-picker/components/model-selector/types.ts
View file

@ -53,19 +53,16 @@ export interface ModelPickTarget {
* Whether an OpenAI-compatible request can actually load this model.
*
* Not the same as isGguf: local_model_resolver skips Ollama's scanner, so an
* Ollama GGUF is never in the auto-switch index and no API request can resolve
* it. Mirroring its settings would advertise a load that cannot happen.
* Defaults to isGguf where a caller does not know.
* Ollama GGUF is never in the auto-switch index and mirroring its settings would
* advertise a load that cannot happen. Defaults to isGguf when unknown.
*/
apiLoadable?: boolean;
/**
* Identity the saved settings are keyed by, when that is not what loads.
*
* A repo cached outside the active HF cache loads by snapshot path, while the
* picker and the auto-switch index keep its settings under the repo id.
* Keying the save by the path would strand the settings where no load looks
* for them. Probes that need something openable (the chat template, the GGUF
* header) keep using `id`. Defaults to `id`.
* picker and the auto-switch index key its settings by repo id, so saving by the
* path would strand them. Probes that must open the model keep using `id`.
*/
configId?: string;
meta: ModelSelectorChangeMeta;

14
studio/frontend/src/features/model-picker/model-config/per-model-config.ts
View file

@ -627,10 +627,9 @@ export function savePerModelConfig(
ggufVariant: string | null | undefined,
config: PerModelConfig,
/**
* Receives models dropped to stay inside the storage budget. Eviction is
* silent and still reports success, so without this their server-side
* overrides would keep being applied by API loads with nothing in the UI
* still showing them or able to forget them.
* Receives models dropped to stay inside the storage budget. Eviction is silent
* and still reports success, so without this their server-side overrides would
* keep being applied with nothing in the UI able to forget them.
*/
evicted?: { modelId: string; ggufVariant: string | null }[],
): boolean {
@ -690,10 +689,9 @@ export function listPerModelConfigs(): {
if (!modelId) {
continue;
}
// Never report a future-schema record. loadPerModelConfig refuses to apply
// one and eviction refuses to drop one, so handing it to the backfill would
// persist this client's partial reading of it server-side and let an
// API-triggered load apply settings the same client will not apply locally.
// Never report a future-schema record: loadPerModelConfig refuses to apply one
// and eviction refuses to drop one, so the backfill would persist this client's
// partial reading and let an API load apply what it will not apply locally.
if (storedConfigVersion(raw) > STORAGE_SCHEMA_VERSION) {
continue;
}

8
studio/frontend/src/features/settings/components/monitor-link.tsx
View file

@ -1,12 +1,12 @@
// SPDX-License-Identifier: AGPL-3.0-only
// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
// The monitor moved out of this tab and onto its own page. It is normally
// reached from the floating panel; this card is the way in from Settings.
// The monitor moved onto its own page, normally reached from the floating
// panel; this card is the way in from Settings.
import { Switch } from "@/components/ui/switch";
// Direct path, not the barrel: the barrel re-exports the page, which would
// pull it into this chunk and defeat the route's dynamic import.
// Direct path, not the barrel: the barrel re-exports the page, which would pull
// it into this chunk and defeat the route's dynamic import.
import { useApiMonitorOverlayStore } from "@/features/api-monitor/overlay-store";
import { getApiMonitor } from "@/features/chat/api/chat-api";
import type { ApiMonitorResponse } from "@/features/chat/types/api";

17
tests/studio/test_model_picker_contracts.py
View file

@ -677,8 +677,8 @@ def test_evicted_local_configs_drop_their_server_overrides():
in src
)
# The eviction path has to actually report what it dropped, decoded back into
# a model id and variant rather than the normalized storage key.
# The eviction path must report what it dropped, decoded back into a model id
# and variant rather than the normalized storage key.
store = " ".join(_read("features/model-picker/model-config/per-model-config.ts").split())
assert "evicted?: { modelId: string; ggufVariant: string | null }[]" in store
assert "modelIdFromStorageKey(" in store and "ggufVariantFromStorageKey(" in store
@ -694,13 +694,13 @@ def test_backfill_compares_server_keys_by_normalized_identity():
"""
src = " ".join(_read("features/model-picker/api/migrate-model-overrides.ts").split())
assert "function normalizedOverrideKey(" in src
# Folded on both sides: the older `id::variant` local keys are not folded.
# Folded on both sides: the older `id::variant` local keys are not.
assert "const known = new Set(Object.keys(existing).map(normalizedOverrideKey));" in src
assert "if (known.has(key)) { continue; }" in src
# A variant never holds a colon, so the last one splits the key. Splitting on
# the first would cut a Windows drive letter off every path id.
# A variant never holds a colon, so the last one splits the key; the first would
# cut the drive letter off every Windows path id.
assert 'key.lastIndexOf(":")' in src
# Repo ids fold and POSIX paths do not, which is exactly what these do.
# Repo ids fold and POSIX paths do not, which is what these do.
assert "normalizeModelIdentity(" in src and "normalizeGgufVariantIdentity(" in src
@ -775,9 +775,8 @@ def test_override_writes_are_ordered_per_model():
src = " ".join(_read("features/model-picker/api/model-overrides.ts").split())
assert "const writesByKey = new Map<string, Promise<void>>();" in src
# Keyed by the same override key the server stores under.
# Folded, not literal: the backfill uses a legacy casing while a UI save
# uses the normalized one, and the backend resolves both to one row, so
# raw strings would open two queues for one model and race again.
# Folded, not literal: the backfill uses a legacy casing and a UI save the
# normalized one, and the backend resolves both to one row.
assert (
"const key = modelOverrideKey( normalizeModelIdentity(modelId), normalizeGgufVariantIdentity(ggufVariant), );"
in src

12
tests/studio/test_settings_compact_overflow_contract.py
View file

@ -5,8 +5,8 @@ from pathlib import Path
REPO = Path(__file__).resolve().parents[2]
SETTINGS_DIALOG = REPO / "studio/frontend/src/features/settings/settings-dialog.tsx"
# The monitor moved out of the settings dialog and onto its own page; Settings
# now links to it. The shrink contract still applies to both surfaces.
# The monitor moved onto its own page and Settings links to it; the shrink
# contract still applies to both surfaces.
API_MONITOR_PAGE = REPO / "studio/frontend/src/features/api-monitor/api-monitor-page.tsx"
MONITOR_LINK = REPO / "studio/frontend/src/features/settings/components/monitor-link.tsx"
GENERAL_TAB = REPO / "studio/frontend/src/features/settings/tabs/general-tab.tsx"
@ -21,11 +21,10 @@ def test_dialog_content_can_shrink_inside_the_dialog_grid():
def test_api_monitor_entries_and_expanded_text_can_shrink():
source = API_MONITOR_PAGE.read_text(encoding = "utf-8")
# Rows and the detail pane sit in flex parents, so they need min-w-0 or a long
# model id or endpoint pushes the layout wider than the viewport.
# model id pushes the layout wider than the viewport.
assert '"flex w-full min-w-0 flex-col gap-1 border-b border-border/50' in source
assert '<section className="flex min-w-0 flex-col gap-1.5">' in source
# Prompt and reply are unbounded user text: they must be height-capped,
# scrollable, and wrap rather than stretch the pane.
# Prompt and reply are unbounded user text: height-capped, scrollable, wrapped.
assert "max-h-72 overflow-auto whitespace-pre-wrap break-words rounded-lg bg-muted/50" in source
# A model id or path has no spaces to wrap on, so it needs break-all.
assert 'className="min-w-0 break-all font-mono' in source
@ -34,8 +33,7 @@ def test_api_monitor_entries_and_expanded_text_can_shrink():
def test_settings_monitor_link_can_shrink():
source = MONITOR_LINK.read_text(encoding = "utf-8")
assert "flex w-full min-w-0 items-center gap-3" in source
# The summary line carries a model id, so it truncates instead of widening
# the settings dialog.
# The summary line carries a model id, so it truncates instead of widening.
assert '<span className="truncate text-xs text-muted-foreground">' in source

18
tests/studio/test_usage_examples_model_source_contract.py
View file

@ -132,21 +132,19 @@ def test_usage_examples_has_no_duplicate_auto_switch_control():
assert "<ModelAutoSwitchSection />" in tab
# The monitor moved out of the settings dialog onto its own page. Settings keeps
# configuration and links across; these contracts follow the behaviour, not the
# old file.
# The monitor moved onto its own page; Settings keeps configuration and links
# across. These contracts follow the behaviour, not the old file.
API_MONITOR_TSX = REPO / "studio/frontend/src/features/api-monitor/api-monitor-page.tsx"
# The lifecycle labels live in their own module: the overlay is mounted from
# __root.tsx, so importing them from the page pulled the whole page into the
# eager bundle and undid the route's lazyRouteComponent.
# __root.tsx, so importing them from the page pulled it into the eager bundle.
API_MONITOR_LIFECYCLE_TS = REPO / "studio/frontend/src/features/api-monitor/lifecycle.ts"
MONITOR_LINK_TSX = SETTINGS / "components/monitor-link.tsx"
def test_api_monitor_history_does_not_reorder_under_the_reader():
# The backend retains 50 terminal entries and moves an entry to the front when
# it finishes. The console froze ids while paging; the full page pauses the
# poll instead, which holds the whole list still while a payload is read.
# The backend keeps 50 terminal entries and moves one to the front as it
# finishes. The console froze ids while paging; the page pauses the poll instead,
# holding the whole list still while a payload is read.
src = API_MONITOR_TSX.read_text(encoding = "utf-8")
assert "paused" in src
assert "setPaused" in src
@ -169,8 +167,8 @@ def test_api_monitor_renders_lifecycle_rows():
def test_auto_switch_section_sits_above_the_usage_examples():
tab = API_KEYS_TAB_TSX.read_text(encoding = "utf-8")
# The console became a link out to the monitor page; ordering still puts
# configuration ahead of the examples that depend on it.
# The console became a link out; ordering still puts configuration ahead of the
# examples that depend on it.
assert tab.index("<MonitorLink />") < tab.index("<ModelAutoSwitchSection />")
assert tab.index("<ModelAutoSwitchSection />") < tab.index("<UsageExamples")