From 9ec6c8bb5a341004a9e3aea6e60fe7023024855d Mon Sep 17 00:00:00 2001 From: Unsloth Date: Sun, 26 Jul 2026 06:55:59 -0700 Subject: [PATCH] studio: apply saved per-model settings on API loads, add API monitor Per-model settings were mirrored to the server as two fields only, llama_extra_args and max_seq_length. Everything else lived in browser localStorage, so a model loaded by an API request came up with app defaults for context length, KV cache dtype, speculative decoding, tensor parallel and GPU placement. Store the full config server side and map it onto the same LoadRequest the picker builds, so a remote load and a picker load of the same model produce the same command line. Entries are keyed per quant, falling back to the bare repo id so existing entries keep resolving. Also moves the API monitor out of the settings tab: a full page at /api-monitor, plus a floating panel that opens itself when traffic arrives, and a settings page per model reachable from the Hub. --- studio/backend/core/inference/api_monitor.py | 15 +- studio/backend/routes/inference.py | 51 +- studio/backend/routes/settings.py | 75 +- studio/backend/tests/test_api_monitor.py | 31 + .../backend/tests/test_openai_auto_switch.py | 174 ++++ .../utils/openai_auto_switch_settings.py | 224 ++++- studio/frontend/src/app/router.tsx | 2 + studio/frontend/src/app/routes/__root.tsx | 28 +- studio/frontend/src/app/routes/api.tsx | 22 + .../api-monitor/api-monitor-overlay.tsx | 384 +++++++++ .../features/api-monitor/api-monitor-page.tsx | 778 ++++++++++++++++++ .../components/saved-model-settings.tsx | 140 ++++ .../src/features/api-monitor/index.ts | 13 + .../src/features/api-monitor/overlay-store.ts | 48 ++ .../features/api-monitor/use-api-monitor.ts | 297 +++++++ .../src/features/chat/api/chat-api.ts | 7 + .../features/hub/catalog/download-card.tsx | 35 + .../hub/catalog/hub-model-settings-view.tsx | 138 ++++ .../hub/catalog/local-on-device-card.tsx | 12 + .../features/hub/catalog/model-inspector.tsx | 4 + .../hub/catalog/models-catalog-lists.tsx | 4 + .../hub/catalog/models-catalog-rows.tsx | 42 +- .../features/hub/catalog/models-catalog.tsx | 5 + studio/frontend/src/features/hub/hub-page.tsx | 174 +++- .../model-picker/api/model-overrides.ts | 174 ++++ .../components/model-config-page.tsx | 25 +- .../model-selector/model-row-menu.tsx | 26 +- .../src/features/model-picker/index.ts | 10 + .../components/api-monitor-console.tsx | 393 --------- .../settings/components/monitor-link.tsx | 97 +++ .../features/settings/tabs/api-keys-tab.tsx | 4 +- 31 files changed, 2988 insertions(+), 444 deletions(-) create mode 100644 studio/frontend/src/app/routes/api.tsx create mode 100644 studio/frontend/src/features/api-monitor/api-monitor-overlay.tsx create mode 100644 studio/frontend/src/features/api-monitor/api-monitor-page.tsx create mode 100644 studio/frontend/src/features/api-monitor/components/saved-model-settings.tsx create mode 100644 studio/frontend/src/features/api-monitor/index.ts create mode 100644 studio/frontend/src/features/api-monitor/overlay-store.ts create mode 100644 studio/frontend/src/features/api-monitor/use-api-monitor.ts create mode 100644 studio/frontend/src/features/hub/catalog/hub-model-settings-view.tsx create mode 100644 studio/frontend/src/features/model-picker/api/model-overrides.ts delete mode 100644 studio/frontend/src/features/settings/components/api-monitor-console.tsx create mode 100644 studio/frontend/src/features/settings/components/monitor-link.tsx diff --git a/studio/backend/core/inference/api_monitor.py b/studio/backend/core/inference/api_monitor.py index f76a38576f..f9922a711a 100644 --- a/studio/backend/core/inference/api_monitor.py +++ b/studio/backend/core/inference/api_monitor.py @@ -269,9 +269,20 @@ class ApiMonitor: if entry.status == "running" and (subject is None or entry.subject == subject) ) - def clear(self) -> None: + def clear(self, *, subject: Optional[str] = None) -> None: + """Drop recorded entries. ``subject`` limits the wipe to one caller's. + + Every other read on this class is subject-scoped, so an unscoped clear + would let one user erase another's history (and zero their active count + mid-generation). Callers that genuinely mean "everything" pass None. + """ with self._lock: - self._entries.clear() + if subject is None: + self._entries.clear() + return + self._entries = deque( + entry for entry in self._entries if entry.subject != subject + ) def _find_locked(self, entry_id: str) -> Optional[ApiMonitorEntry]: for entry in self._entries: diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.py index 9d40650a56..d529a1cd85 100644 --- a/studio/backend/routes/inference.py +++ b/studio/backend/routes/inference.py @@ -3581,6 +3581,7 @@ async def _maybe_auto_switch_model( get_openai_auto_switch_enabled, get_auto_unload_idle_seconds, get_model_override, + model_override_load_kwargs, ) from core.inference.local_model_resolver import resolve_local_gguf from core.inference.llama_keepwarm import ( @@ -3716,13 +3717,35 @@ async def _maybe_auto_switch_model( if _already_serving(): _record_serving_alias() return - # Apply this model's saved launch flags so the swap honors the config. - override = get_model_override(override_id) + # Apply this model's saved launch config so an API-driven swap + # loads it exactly as the picker would: context, KV dtype, + # speculative decoding, chat template and GPU placement, not + # just the two legacy flags. Look the config up under the + # variant-qualified id first (two quants of one repo can carry + # different configs), then the bare advertised id, then the + # concrete load path, so a config saved under any of the names + # this model is known by is found. + override = {} + for override_key in ( + f"{override_id}:{variant}" if variant else None, + override_id, + target_id, + ): + if not override_key: + continue + override = get_model_override(override_key) + if override: + break load_kwargs = {"model_path": target_id, "gguf_variant": variant} - if override.get("llama_extra_args") is not None: - load_kwargs["llama_extra_args"] = override["llama_extra_args"] - if override.get("max_seq_length") is not None: - load_kwargs["max_seq_length"] = override["max_seq_length"] + load_kwargs.update( + model_override_load_kwargs( + override, + # variant is set for every GGUF the resolver returns; the + # reload-stash path carries the quant it froze. + is_gguf = bool(variant) + or target_id.lower().endswith(".gguf"), + ) + ) # Reuse the load impl so its dedup, tensor fallback, and threading # apply. Call the impl directly: we already hold the lifecycle gate # the /load route would otherwise take, so the route would deadlock. @@ -5703,6 +5726,22 @@ async def get_api_monitor(current_subject: str = Depends(get_current_subject)): } +@studio_router.delete("/monitor") +async def clear_api_monitor(current_subject: str = Depends(get_current_subject)): + """Drop this caller's recorded API history so a debugging session starts clean. + + Scoped to the current subject, like every read on the monitor: an unscoped + wipe would erase another user's history and zero their active-request count + while their generation is still streaming. + + The caller's own in-flight requests are dropped from the log too; they keep + streaming to their client, they just stop being reported here (a later append + re-adds nothing, since the entry id no longer resolves). + """ + api_monitor.clear(subject = current_subject) + return {"cleared": True} + + @studio_router.get("/monitor/{entry_id}") async def get_api_monitor_entry(entry_id: str, current_subject: str = Depends(get_current_subject)): """Return full prompt/reply details for one OpenAI-compatible API request.""" diff --git a/studio/backend/routes/settings.py b/studio/backend/routes/settings.py index fef18a9145..2323d33915 100644 --- a/studio/backend/routes/settings.py +++ b/studio/backend/routes/settings.py @@ -34,6 +34,7 @@ from utils.helper_precache_settings import ( helper_model_disabled_by_env, set_helper_precache_enabled, ) +from picker.schemas import MAX_CHAT_TEMPLATE_BYTES from utils.coding_agents import CODING_AGENTS, detect_installed_coding_agents from utils.openai_auto_switch_settings import ( DEFAULT_AUTO_UNLOAD_KEEP_KV, @@ -126,11 +127,51 @@ class OpenAIAutoSwitchResponse(BaseModel): class ModelOverridePayload(BaseModel): - model_id: str = Field(..., min_length = 1) - llama_extra_args: list[str] = Field(default_factory = list) + """One model's saved launch config, applied when the API loads that model. + + Everything past ``model_id`` is optional and omitted means "app default", so a + payload carrying only ``model_id`` clears the entry. The bounds here mirror + ``LoadRequest`` so a bad value is rejected at the boundary instead of being + silently dropped by the normalizer; the enum-ish fields (KV dtype, speculative + mode) are left to it, since their valid sets follow the llama.cpp build. + """ + + model_id: str = Field(..., min_length = 1, max_length = 512) + # None means "leave the stored value alone": the settings UI has no control + # for launch flags, so a save from it must not wipe flags set through this + # API. An explicit [] clears them (that is how "forget this model" arrives). + llama_extra_args: Optional[list[str]] = None # ge=1: 0 is not a valid sequence length, and the setter drops a falsy value, # so reject it at the boundary instead of accepting then silently discarding it. max_seq_length: Optional[int] = Field(default = None, ge = 1, le = 1048576) + custom_context_length: Optional[int] = Field(default = None, ge = 1, le = 1048576) + kv_cache_dtype: Optional[str] = Field(default = None, max_length = 32) + speculative_type: Optional[str] = Field(default = None, max_length = 32) + spec_draft_n_max: Optional[int] = Field(default = None, ge = 1, le = 16) + tensor_parallel: bool = False + # Validated in bytes below, not by max_length: pydantic counts characters, + # so a multi-byte template could pass here and then be silently dropped by + # the normalizer (which measures UTF-8) while the request still returned 200. + chat_template_override: Optional[str] = None + gpu_memory_mode: Optional[Literal["auto", "manual"]] = None + # -1 is Auto (llama.cpp --fit sizes the offload); the normalizer treats it as unset. + gpu_layers: Optional[int] = Field(default = None, ge = -1, le = 1024) + n_cpu_moe: Optional[int] = Field(default = None, ge = 0, le = 1024) + gpu_ids: Optional[list[int]] = None + + + @field_validator("chat_template_override") + @classmethod + def _limit_chat_template_bytes(cls, value: Optional[str]) -> Optional[str]: + # Mirrors LoadRequest.normalize_blank_chat_template_override so the same + # template is accepted or rejected identically on both paths. + if value is None: + return None + if len(value.encode("utf-8")) > MAX_CHAT_TEMPLATE_BYTES: + raise ValueError( + f"Chat template exceeds the {MAX_CHAT_TEMPLATE_BYTES}-byte limit." + ) + return value class ModelOverridesResponse(BaseModel): @@ -289,12 +330,40 @@ def update_openai_auto_switch_override( payload: ModelOverridePayload, current_subject: str = Depends(get_current_subject) ) -> ModelOverridesResponse: from core.inference.llama_server_args import validate_extra_args + from utils.openai_auto_switch_settings import get_model_override try: - extra_args = validate_extra_args(payload.llama_extra_args) + # A payload carrying only model_id is the documented "remove", so it + # wipes everything. Otherwise it is a real save, and omitted launch flags + # are carried over from the stored entry (the settings UI cannot express + # them and must not delete them). + requested_extra_args = payload.llama_extra_args + saved_fields = payload.model_dump( + exclude = {"model_id", "llama_extra_args"}, exclude_none = True + ) + is_removal = not payload.tensor_parallel and not { + key: value + for key, value in saved_fields.items() + if key != "tensor_parallel" + } + if requested_extra_args is None and not is_removal: + requested_extra_args = get_model_override(payload.model_id).get( + "llama_extra_args" + ) + extra_args = validate_extra_args(requested_extra_args) set_model_override( payload.model_id, llama_extra_args = extra_args, max_seq_length = payload.max_seq_length, + custom_context_length = payload.custom_context_length, + kv_cache_dtype = payload.kv_cache_dtype, + speculative_type = payload.speculative_type, + spec_draft_n_max = payload.spec_draft_n_max, + tensor_parallel = payload.tensor_parallel, + chat_template_override = payload.chat_template_override, + gpu_memory_mode = payload.gpu_memory_mode, + gpu_layers = payload.gpu_layers, + n_cpu_moe = payload.n_cpu_moe, + gpu_ids = payload.gpu_ids, ) except ValueError as exc: raise log_and_http_error( diff --git a/studio/backend/tests/test_api_monitor.py b/studio/backend/tests/test_api_monitor.py index 56bc404350..a8d0e1e832 100644 --- a/studio/backend/tests/test_api_monitor.py +++ b/studio/backend/tests/test_api_monitor.py @@ -258,3 +258,34 @@ def test_api_monitor_append_reply_exact_cap_then_more_marks_truncated(): monitor.append_reply(entry_id, "y") reply = monitor.snapshot()[0]["reply"] assert len(reply) == m._MAX_REPLY_CHARS and reply.endswith("...") + + +def test_api_monitor_clear_is_scoped_to_one_subject(): + # Every other read on the monitor is subject-scoped. An unscoped clear from + # the route would let one caller erase another's history and zero their + # active count in the middle of a generation. + monitor = ApiMonitor(max_entries = 4) + alice = monitor.start( + endpoint = "/v1/chat/completions", + method = "POST", + model = "m", + prompt = "alice prompt", + subject = "alice", + ) + bob = monitor.start( + endpoint = "/v1/chat/completions", + method = "POST", + model = "m", + prompt = "bob prompt", + subject = "bob", + ) + + monitor.clear(subject = "alice") + assert monitor.snapshot(subject = "alice") == [] + assert [entry["id"] for entry in monitor.snapshot(subject = "bob")] == [bob] + assert monitor.active_count(subject = "bob") == 1 + assert monitor.get(alice, subject = "alice") is None + + # Passing no subject is the explicit "everything" path. + monitor.clear() + assert monitor.snapshot(subject = "bob") == [] diff --git a/studio/backend/tests/test_openai_auto_switch.py b/studio/backend/tests/test_openai_auto_switch.py index 9c6c20e6b6..3899b1444a 100644 --- a/studio/backend/tests/test_openai_auto_switch.py +++ b/studio/backend/tests/test_openai_auto_switch.py @@ -3819,3 +3819,177 @@ def test_env_idle_below_floor_is_clamped(monkeypatch): assert settings.get_auto_unload_idle_seconds() == 600 monkeypatch.delenv(settings.MODEL_IDLE_TTL_ENV_VAR) assert settings.get_auto_unload_idle_seconds() == 0 + + +# --------------------------------------------------------------------------- +# Per-model launch config: normalization, LoadRequest mapping, key resolution. +# --------------------------------------------------------------------------- + + +def test_normalize_model_override_drops_unusable_fields_and_keeps_the_rest(): + # A stale field must not cost the user the whole config, so bad values are + # dropped one by one rather than rejecting the payload. + entry = settings.normalize_model_override( + { + "max_seq_length": 8192, + "kv_cache_dtype": "not_a_dtype", + "speculative_type": "mtp", + "spec_draft_n_max": 999, # out of range for the MTP draft count + "gpu_memory_mode": "auto", # only "manual" is a real override + "gpu_layers": -1, # -1 is Auto, which is already the default + "n_cpu_moe": 0, + "gpu_ids": [1, 1, 0, "2", -5], + "tensor_parallel": False, + "llama_extra_args": [], + } + ) + assert entry == { + "max_seq_length": 8192, + "speculative_type": "mtp", + "gpu_ids": [1, 0, 2], + } + + +def test_normalize_model_override_rejects_oversized_chat_template(): + small = settings.normalize_model_override({"chat_template_override": "{{ bos }}"}) + assert small["chat_template_override"] == "{{ bos }}" + # The limit is bytes, not characters: a multi-byte template just under the + # character limit can still be over the byte limit. + huge = "é" * settings.MAX_CHAT_TEMPLATE_OVERRIDE_BYTES + assert "chat_template_override" not in settings.normalize_model_override( + {"chat_template_override": huge} + ) + + +def test_spec_draft_n_max_only_stored_for_mtp_modes(): + mtp = settings.normalize_model_override( + {"speculative_type": "mtp", "spec_draft_n_max": 4} + ) + assert mtp["spec_draft_n_max"] == 4 + # A non-MTP mode ignores the draft count at load time, so storing it would + # show the user an edit that never takes effect. + ngram = settings.normalize_model_override( + {"speculative_type": "ngram", "spec_draft_n_max": 4} + ) + assert "spec_draft_n_max" not in ngram + + +def test_resolve_fit_max_seq_length_hands_sizing_to_fit_under_manual_auto_layers(): + # Manual GPU memory with Auto layers means llama.cpp --fit owns the context, + # so the load sends the context pin (or 0), not the stored max seq length. + override = {"gpu_memory_mode": "manual", "max_seq_length": 8192} + assert settings.resolve_fit_max_seq_length(override, is_gguf = True) == 0 + assert ( + settings.resolve_fit_max_seq_length( + {**override, "custom_context_length": 4096}, is_gguf = True + ) + == 4096 + ) + # Pinning the layer count takes --fit back out of the picture. + assert ( + settings.resolve_fit_max_seq_length({**override, "gpu_layers": 20}, is_gguf = True) + == 8192 + ) + # Not a GGUF, so none of this applies. + assert settings.resolve_fit_max_seq_length(override, is_gguf = False) == 8192 + + +def test_model_override_load_kwargs_gates_gpu_placement_on_gguf(): + override = { + "max_seq_length": 4096, + "kv_cache_dtype": "q8_0", + "tensor_parallel": True, + "gpu_memory_mode": "manual", + "gpu_layers": 20, + "n_cpu_moe": 3, + "gpu_ids": [0, 1], + } + gguf = settings.model_override_load_kwargs(override, is_gguf = True) + assert gguf["cache_type_kv"] == "q8_0" + assert gguf["tensor_parallel"] is True + assert gguf["gpu_layers"] == 20 + assert gguf["gpu_ids"] == [0, 1] + + # A safetensors model loads through HF auto-placement; inheriting a GGUF GPU + # pin here would silently change where the weights land. + safetensors = settings.model_override_load_kwargs(override, is_gguf = False) + assert safetensors["max_seq_length"] == 4096 + assert "gpu_layers" not in safetensors + assert "gpu_ids" not in safetensors + assert "n_cpu_moe" not in safetensors + assert "gpu_memory_mode" not in safetensors + + # Every key it produces has to be a real LoadRequest field, or the load call + # raises TypeError at the moment the user's request arrives. + LoadRequest(model_path = "unsloth/B-GGUF", **gguf) + + +def test_auto_switch_prefers_variant_qualified_override(monkeypatch): + # Settings are saved per quant, so Q4_K_M and Q8_0 of the same repo are + # different entries; the bare repo id is only the fallback. + backend = _FakeBackend(None) + rec = _LoadRecorder(backend) + _wire( + monkeypatch, + enabled = True, + resolves_to = ("unsloth/B-GGUF", "Q4_K_M", "unsloth/B-GGUF"), + backend = backend, + recorder = rec, + ) + stored = { + "unsloth/B-GGUF": {"max_seq_length": 1024}, + "unsloth/B-GGUF:Q4_K_M": {"max_seq_length": 8192, "gpu_layers": 20}, + } + monkeypatch.setattr(settings, "get_model_override", lambda mid: stored.get(mid, {})) + + _run_hook("unsloth/B-GGUF") + req = rec.calls[0] + assert req.max_seq_length == 8192 + assert req.gpu_layers == 20 + + +def test_auto_switch_falls_back_to_bare_repo_override(monkeypatch): + backend = _FakeBackend(None) + rec = _LoadRecorder(backend) + _wire( + monkeypatch, + enabled = True, + resolves_to = ("unsloth/B-GGUF", "Q4_K_M", "unsloth/B-GGUF"), + backend = backend, + recorder = rec, + ) + stored = {"unsloth/B-GGUF": {"max_seq_length": 1024}} + monkeypatch.setattr(settings, "get_model_override", lambda mid: stored.get(mid, {})) + + _run_hook("unsloth/B-GGUF") + assert rec.calls[0].max_seq_length == 1024 + + +def test_override_route_preserves_launch_flags_across_a_settings_only_update(monkeypatch): + # The settings page has no control for llama_extra_args, so saving from it + # omits the field. Omitted must mean "leave it alone", or every save from the + # UI would quietly wipe flags set elsewhere. + import routes.settings as settings_route + + _mock_override_store(monkeypatch) + settings_route.update_openai_auto_switch_override( + settings_route.ModelOverridePayload( + model_id = "unsloth/B-GGUF", llama_extra_args = ["--flash-attn"] + ), + "tester", + ) + resp = settings_route.update_openai_auto_switch_override( + settings_route.ModelOverridePayload(model_id = "unsloth/B-GGUF", max_seq_length = 4096), + "tester", + ) + entry = resp.overrides["unsloth/B-GGUF"] + assert entry["llama_extra_args"] == ["--flash-attn"] + assert entry["max_seq_length"] == 4096 + + # An explicit empty list is how the UI says "forget this model", and with no + # other fields left that removes the entry outright. + gone = settings_route.update_openai_auto_switch_override( + settings_route.ModelOverridePayload(model_id = "unsloth/B-GGUF", llama_extra_args = []), + "tester", + ) + assert "unsloth/B-GGUF" not in gone.overrides diff --git a/studio/backend/utils/openai_auto_switch_settings.py b/studio/backend/utils/openai_auto_switch_settings.py index 7007440f4c..413d75ad1e 100644 --- a/studio/backend/utils/openai_auto_switch_settings.py +++ b/studio/backend/utils/openai_auto_switch_settings.py @@ -210,8 +210,211 @@ def set_openai_auto_switch( ) +# --- Per-model launch config ------------------------------------------------- +# +# An override is the server-side twin of the UI's per-model config (the browser +# localStorage map behind features/model-picker/model-config). The UI mirrors +# every save here so a model loaded by an OpenAI-compatible API request gets the +# same launch settings a user would get loading it from the picker; without this +# the API path could only ever apply the two legacy fields below. +# +# Legacy entries hold just {llama_extra_args, max_seq_length}; every field is +# optional and absent means "fall back to the app default", so old entries keep +# loading correctly. A write is a full replace of the fields it expresses, so the +# route carries `llama_extra_args` over when the payload omits it (the settings +# UI has no control for launch flags and must not wipe them). +# +# Known gap: the picker resolves a couple of knobs as "per-model value, else the +# user's global preference" -- GPU memory mode and speculative decoding, whose +# globals live in browser localStorage. An override deliberately stores only an +# explicit per-model choice (so the model keeps following later global changes), +# and the server cannot see the globals at all. So for a model that follows the +# global on one of those two, an API load falls back to the app default rather +# than the user's global. Every other field matches the picker exactly. + +# Mirrors _valid_cache_types in core/inference/llama_cpp.py. +VALID_KV_CACHE_DTYPES = frozenset( + {"f16", "bf16", "q8_0", "q4_0", "q4_1", "q5_0", "q5_1", "iq4_nl", "f32"} +) +# Canonical values plus the legacy spellings LoadRequest still accepts. +VALID_SPECULATIVE_TYPES = frozenset( + { + "auto", + "mtp", + "ngram", + "mtp+ngram", + "off", + "default", + "draft-mtp", + "ngram-mod", + "ngram-simple", + } +) +# Only these two consume spec_draft_n_max (mirrors MTP_SPECULATIVE_TYPES in the UI). +MTP_SPECULATIVE_TYPES = frozenset({"mtp", "mtp+ngram", "draft-mtp"}) +VALID_GPU_MEMORY_MODES = frozenset({"auto", "manual"}) + +MAX_SEQ_LENGTH_CEILING = 1048576 +MAX_CHAT_TEMPLATE_OVERRIDE_BYTES = 65_536 + + +def _clean_str(value: Any, allowed: frozenset[str]) -> Optional[str]: + if not isinstance(value, str): + return None + normalized = value.strip().lower() + return normalized if normalized in allowed else None + + +def _bounded_int(value: Any, *, minimum: int, maximum: int) -> Optional[int]: + try: + parsed = int(value) + except (TypeError, ValueError): + return None + if parsed < minimum or parsed > maximum: + return None + return parsed + + +def normalize_model_override(payload: dict[str, Any]) -> dict[str, Any]: + """Validate one per-model launch config, dropping anything unusable. + + Silently drops rather than raising: an override is a convenience mirror of the + UI's config, so one stale field (a KV dtype this llama.cpp build lost, a GPU id + from another host) must not block persisting the rest or fail the API load that + reads it. ``validate_extra_args`` is the caller's job -- it lives in the + llama_server_args allow-list module, which this one must not import. + """ + entry: dict[str, Any] = {} + + extra_args = payload.get("llama_extra_args") + if isinstance(extra_args, (list, tuple)) and extra_args: + entry["llama_extra_args"] = [str(arg) for arg in extra_args] + + # 0 / negative means "unset"; the loader reads absence as the app default. + for key in ("max_seq_length", "custom_context_length"): + parsed = _bounded_int(payload.get(key), minimum = 1, maximum = MAX_SEQ_LENGTH_CEILING) + if parsed: + entry[key] = parsed + + kv_cache_dtype = _clean_str(payload.get("kv_cache_dtype"), VALID_KV_CACHE_DTYPES) + if kv_cache_dtype: + entry["kv_cache_dtype"] = kv_cache_dtype + + speculative_type = _clean_str(payload.get("speculative_type"), VALID_SPECULATIVE_TYPES) + if speculative_type: + entry["speculative_type"] = speculative_type + # Only meaningful for the MTP modes; storing it otherwise would resurface + # in the UI as an edit the loader silently ignores. + if speculative_type in MTP_SPECULATIVE_TYPES: + spec_draft_n_max = _bounded_int( + payload.get("spec_draft_n_max"), minimum = 1, maximum = 16 + ) + if spec_draft_n_max: + entry["spec_draft_n_max"] = spec_draft_n_max + + if _coerce_bool(payload.get("tensor_parallel")): + entry["tensor_parallel"] = True + + template = payload.get("chat_template_override") + if isinstance(template, str) and template.strip(): + if len(template.encode("utf-8")) <= MAX_CHAT_TEMPLATE_OVERRIDE_BYTES: + entry["chat_template_override"] = template + + # Only "manual" is a real override: persisting "auto" would pin the model and + # stop it following later changes to the global GPU memory preference. + if _clean_str(payload.get("gpu_memory_mode"), VALID_GPU_MEMORY_MODES) == "manual": + entry["gpu_memory_mode"] = "manual" + + # -1 is Auto (llama.cpp --fit owns layer sizing), which is also the default, + # so only a pinned count >= 0 is worth storing. + gpu_layers = _bounded_int(payload.get("gpu_layers"), minimum = 0, maximum = 1024) + if gpu_layers is not None: + entry["gpu_layers"] = gpu_layers + + n_cpu_moe = _bounded_int(payload.get("n_cpu_moe"), minimum = 1, maximum = 1024) + if n_cpu_moe: + entry["n_cpu_moe"] = n_cpu_moe + + gpu_ids = payload.get("gpu_ids") + if isinstance(gpu_ids, (list, tuple)) and gpu_ids: + # De-duplicate, preserving order: resolve_requested_gpu_ids rejects a + # repeated id outright, so storing [0, 0] would make every later API load + # of this model fail with a 400 that the picker never hits. + cleaned_ids: list[int] = [] + for gid in gpu_ids: + parsed = _bounded_int(gid, minimum = 0, maximum = 1024) + if parsed is not None and parsed not in cleaned_ids: + cleaned_ids.append(parsed) + if cleaned_ids: + entry["gpu_ids"] = cleaned_ids + + return entry + + +def resolve_fit_max_seq_length(override: dict[str, Any], *, is_gguf: bool) -> Optional[int]: + """The ``max_seq_length`` an API load should send for this override. + + Mirrors resolveFitMaxSeqLength in the UI (features/chat/presets/preset-policy.ts): + under Manual GPU memory with Auto layers, llama.cpp's ``--fit`` owns context + sizing, so the load sends the explicit context pin (or 0 to hand sizing over) + rather than the stored max sequence length. Returns None to leave the field + at the loader's default. + """ + manual_auto_layers = ( + is_gguf + and override.get("gpu_memory_mode") == "manual" + and override.get("gpu_layers") is None + ) + if manual_auto_layers: + return override.get("custom_context_length") or 0 + # max_seq_length wins where both are set. The UI only ever sends it for a + # non-GGUF model (a GGUF's context is `custom_context_length`), so in + # practice the two never collide from that path; a hand-written or legacy + # entry that sets it on a GGUF is honoured, which is this API's contract. + return override.get("max_seq_length") or override.get("custom_context_length") + + +def model_override_load_kwargs(override: dict[str, Any], *, is_gguf: bool) -> dict[str, Any]: + """Map a stored per-model config onto ``LoadRequest`` keyword arguments. + + Mirrors the UI's load payload (features/chat/api/chat-adapter.ts) so an API + auto-switch load and a picker load of the same model produce the same command + line. GPU placement is GGUF-only there, so it is gated the same way here: a + safetensors model loads through HF auto-placement and must not inherit a + hidden GGUF GPU pin. + """ + if not override: + return {} + kwargs: dict[str, Any] = {} + + max_seq_length = resolve_fit_max_seq_length(override, is_gguf = is_gguf) + if max_seq_length is not None: + kwargs["max_seq_length"] = max_seq_length + for source, target in ( + ("llama_extra_args", "llama_extra_args"), + ("kv_cache_dtype", "cache_type_kv"), + ("speculative_type", "speculative_type"), + ("spec_draft_n_max", "spec_draft_n_max"), + ("tensor_parallel", "tensor_parallel"), + ("chat_template_override", "chat_template_override"), + ): + if override.get(source) is not None: + kwargs[target] = override[source] + + if is_gguf: + if override.get("gpu_memory_mode") is not None: + kwargs["gpu_memory_mode"] = override["gpu_memory_mode"] + if override.get("gpu_layers") is not None: + kwargs["gpu_layers"] = override["gpu_layers"] + if override.get("n_cpu_moe") is not None: + kwargs["n_cpu_moe"] = override["n_cpu_moe"] + if override.get("gpu_ids") is not None: + kwargs["gpu_ids"] = override["gpu_ids"] + return kwargs + + def get_model_overrides() -> dict[str, dict]: - """Per-model launch overrides keyed by model id ({llama_extra_args, max_seq_length}).""" + """Per-model launch configs keyed by model id (see normalize_model_override).""" raw = _cached_setting(MODEL_OVERRIDES_SETTING_KEY, None) return raw if isinstance(raw, dict) else {} @@ -226,15 +429,22 @@ def set_model_override( model_id: str, llama_extra_args: Optional[list[str]] = None, max_seq_length: Optional[int] = None, + **config: Any, ) -> dict: - """Upsert one model's launch override; an override with no fields removes it.""" + """Upsert one model's launch config; a config with no usable fields removes it. + + The two legacy parameters stay positional for existing callers; every other + per-model field is passed by keyword and normalized together. + """ if not model_id or not model_id.strip(): raise ValueError("model_id is required.") - entry: dict[str, Any] = {} - if llama_extra_args: - entry["llama_extra_args"] = [str(arg) for arg in llama_extra_args] - if max_seq_length: - entry["max_seq_length"] = max(0, int(max_seq_length)) + entry = normalize_model_override( + { + **config, + "llama_extra_args": llama_extra_args, + "max_seq_length": max_seq_length, + } + ) from storage.studio_db import upsert_app_setting_map_entry diff --git a/studio/frontend/src/app/router.tsx b/studio/frontend/src/app/router.tsx index 586c03d5df..3dd8d2e8e0 100644 --- a/studio/frontend/src/app/router.tsx +++ b/studio/frontend/src/app/router.tsx @@ -6,6 +6,7 @@ import { Button } from "@/components/ui/button"; import { MascotImg } from "@/components/mascot-img"; import { useT } from "@/i18n"; import { Route as rootRoute } from "./routes/__root"; +import { Route as apiMonitorRoute } from "./routes/api"; import { Route as dataRecipesRoute } from "./routes/data-recipes"; import { Route as dataRecipeRoute } from "./routes/data-recipes.$recipeId"; import { Route as chatRoute } from "./routes/chat"; @@ -32,6 +33,7 @@ const routeTree = rootRoute.addChildren([ exportRoute, dataRecipesRoute, dataRecipeRoute, + apiMonitorRoute, ]); function DefaultNotFound() { diff --git a/studio/frontend/src/app/routes/__root.tsx b/studio/frontend/src/app/routes/__root.tsx index 7137fd6f96..5dbeefa8f8 100644 --- a/studio/frontend/src/app/routes/__root.tsx +++ b/studio/frontend/src/app/routes/__root.tsx @@ -5,16 +5,14 @@ import { AppSidebar } from "@/components/app-sidebar"; import { Navbar } from "@/components/navbar"; import { fetchDeviceType, usePlatformStore } from "@/config/env"; import { SidebarInset, SidebarProvider } from "@/components/ui/sidebar"; -import { - SettingsDialog, - useSettingsDialogStore, -} from "@/features/settings"; +import { SettingsDialog, useSettingsDialogStore } from "@/features/settings"; import { ChatPage, clearNewChatDraft, useChatRuntimeStore, type ChatSearch, } from "@/features/chat"; +import { ApiMonitorOverlay } from "@/features/api-monitor/api-monitor-overlay"; import { RemoteCodeConsentDialog } from "@/features/security"; import { HfTokenWarningDialog } from "@/features/hf-auth"; import { TransformersUpgradeDialog } from "@/features/transformers-upgrade"; @@ -33,13 +31,7 @@ import { useRouterState, } from "@tanstack/react-router"; import { AnimatePresence, motion } from "motion/react"; -import { - Suspense, - useEffect, - useLayoutEffect, - useMemo, - useState, -} from "react"; +import { Suspense, useEffect, useLayoutEffect, useMemo, useState } from "react"; import { AppProvider } from "../provider"; declare module "@tanstack/react-router" { @@ -76,11 +68,17 @@ const CHAT_ONLY_ALLOWED = new Set([ // Export stays reachable on chat-only hosts so the page can show its own grayed-out reason // instead of a silent redirect; it self-gates via export capability, so nothing runs. "/export", + // Chat-only hosts (Intel Macs, Apple Silicon without MLX, no-GPU boxes) serve + // the OpenAI-compatible API exactly like any other host, so the monitor has to + // be reachable there. Without this the floating panel's own "Expand" button + // and the Settings > API card both redirect to /chat. + "/api-monitor", ]); function isChatOnlyAllowed(pathname: string): boolean { if (CHAT_ONLY_ALLOWED.has(pathname)) return true; - if (pathname === "/data-recipes" || pathname.startsWith("/data-recipes/")) return true; + if (pathname === "/data-recipes" || pathname.startsWith("/data-recipes/")) + return true; return false; } @@ -224,6 +222,8 @@ function RootLayout() { {!isAuthFlowRoute && } + {/* Opens itself when API traffic arrives; hides on the full monitor page. */} + {!isAuthFlowRoute && } @@ -241,7 +241,9 @@ function RootLayout() { className="!min-h-0 h-[calc(100dvh-var(--studio-titlebar-height,0px))] overflow-hidden" > - +
import("@/features/api-monitor"), + "ApiMonitorPage", +); + +export const Route = createRoute({ + getParentRoute: () => rootRoute, + // Not "/api": the backend owns that prefix (and "/v1"), and its SPA fallback + // deliberately 404s those paths so API clients get an API-shaped error rather + // than an HTML page. A deep link to /api would never reach the router. + path: "/api-monitor", + staticData: { title: "API" }, + beforeLoad: () => requireAuth(), + component: ApiMonitorPage, +}); diff --git a/studio/frontend/src/features/api-monitor/api-monitor-overlay.tsx b/studio/frontend/src/features/api-monitor/api-monitor-overlay.tsx new file mode 100644 index 0000000000..4411e052ad --- /dev/null +++ b/studio/frontend/src/features/api-monitor/api-monitor-overlay.tsx @@ -0,0 +1,384 @@ +// SPDX-License-Identifier: AGPL-3.0-only +// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 + +// Floating API monitor: opens itself when API traffic arrives, summarises it +// without taking over the window, and links through to the full page. + +import { getApiMonitor } from "@/features/chat/api/chat-api"; +import type { ApiMonitorEntry } from "@/features/chat/types/api"; +import { cn } from "@/lib/utils"; +import { + ArrowExpand01Icon, + DragDropVerticalIcon, + Globe02Icon, +} from "@hugeicons/core-free-icons"; +import { HugeiconsIcon } from "@hugeicons/react"; +import { useNavigate, useRouterState } from "@tanstack/react-router"; +import { XIcon } from "lucide-react"; +import { AnimatePresence, motion, useDragControls } from "motion/react"; +import { + type PointerEvent, + type ReactElement, + useEffect, + useMemo, + useRef, + useState, +} from "react"; +import { useApiMonitorOverlayStore } from "./overlay-store"; +import { computeStats } from "./use-api-monitor"; + +// Live cadence while the panel is on screen. +const OPEN_POLL_MS = 1500; +// While closed the poll only has to notice that traffic started, so it backs off. +const IDLE_POLL_MS = 5000; +// Requests shown in the panel; the rest are one click away on the full page. +const VISIBLE_ENTRIES = 4; +// How long the API must be quiet before a dismissed panel will open itself again. +const REARM_QUIET_MS = 60_000; + +const API_INFERENCE_PREFIX_RE = /^\/api\/inference/; +const V1_PREFIX_RE = /^\/v1\//; + +function compactEndpoint(endpoint: string): string { + return endpoint + .replace(API_INFERENCE_PREFIX_RE, "/api") + .replace(V1_PREFIX_RE, "/"); +} + +function formatDuration(value?: number | null): string { + if (value == null) { + return "live"; + } + if (value < 1000) { + return `${Math.round(value)}ms`; + } + return `${(value / 1000).toFixed(value < 10000 ? 1 : 0)}s`; +} + +function statusDotClass(status: ApiMonitorEntry["status"]): string { + switch (status) { + case "running": + return "bg-blue-500 animate-pulse"; + case "error": + return "bg-red-500"; + case "cancelled": + return "bg-amber-500"; + default: + return "bg-emerald-500"; + } +} + +function StatCell({ + label, + value, + tone, +}: { + label: string; + value: string; + tone?: "error" | "active"; +}): ReactElement { + return ( +
+ + {value} + + {/* Sentence case: Unsloth metric rows read as words, not headers. */} + + {label} + +
+ ); +} + +export function ApiMonitorOverlay(): ReactElement | null { + const { isOpen, suppressed, autoOpen, open, close, setAutoOpen } = + useApiMonitorOverlayStore(); + const navigate = useNavigate(); + const pathname = useRouterState({ select: (s) => s.location.pathname }); + const onFullPage = pathname === "/api-monitor"; + + const [data, setData] = useState + > | null>(null); + + // One loop for both jobs: panel contents while open, traffic watch while + // closed. Stands down on the full page, which polls for itself. + useEffect(() => { + if (onFullPage) { + return; + } + let cancelled = false; + let timer: number | undefined; + const intervalMs = isOpen ? OPEN_POLL_MS : IDLE_POLL_MS; + + function schedule(): void { + timer = window.setTimeout(poll, intervalMs); + } + + function poll(): void { + // A hidden tab has nobody to show the panel to. + if (document.hidden) { + schedule(); + return; + } + getApiMonitor() + .then((next) => { + if (!cancelled) setData(next); + }) + .catch(() => { + // An unreachable server is the full page's story to tell. + if (!cancelled) setData(null); + }) + .finally(() => { + if (!cancelled) schedule(); + }); + } + + poll(); + return () => { + cancelled = true; + if (timer !== undefined) window.clearTimeout(timer); + }; + }, [isOpen, onFullPage]); + + const entries = useMemo(() => data?.entries ?? [], [data]); + const stats = useMemo(() => computeStats(entries), [entries]); + + // Ids already seen. A set, not "the newest id": finishing moves an entry to + // the front, so the head flips without any new traffic. + const seenIdsRef = useRef>(new Set()); + // Seeded on the first response even when empty, so the first request of a + // fresh session is not mistaken for history. + const seededRef = useRef(false); + const lastNewEntryAtRef = useRef(0); + + useEffect(() => { + if (data == null) { + return; + } + const ids = data.entries.map((entry) => entry.id); + if (!seededRef.current) { + seededRef.current = true; + seenIdsRef.current = new Set(ids); + return; + } + const seen = seenIdsRef.current; + const hasNewTraffic = ids.some((id) => !seen.has(id)); + // Re-seed each poll so the set stays bounded by the server's ring buffer. + seenIdsRef.current = new Set(ids); + if (!hasNewTraffic) { + return; + } + const now = Date.now(); + const quietFor = now - lastNewEntryAtRef.current; + lastNewEntryAtRef.current = now; + if (!autoOpen || isOpen) { + return; + } + // A dismissal holds for that burst and re-arms only once the API goes + // quiet, so the next request cannot re-open it a second later. + if (suppressed && quietFor < REARM_QUIET_MS) { + return; + } + open(); + }, [data, autoOpen, suppressed, isOpen, open]); + + // The backlog built up while the poll was stood down is not new traffic. + useEffect(() => { + if (onFullPage) { + seededRef.current = false; + } + }, [onFullPage]); + + const [constraintsElement, setConstraintsElement] = + useState(null); + const constraintsRef = useMemo( + () => ({ current: constraintsElement }), + [constraintsElement], + ); + const dragControls = useDragControls(); + + function startDrag(event: PointerEvent): void { + event.preventDefault(); + dragControls.start(event); + } + + const visible = isOpen && !onFullPage; + const serverStatus = data?.status ?? "idle"; + + return ( + + {visible && ( +
+ +
+
+ + + API monitor + + +
+
+
+ +
+ +
+
+ +

+ {data?.active_model ?? "No model loaded"} +

+ + {/* Metrics on a soft tile, as the Hub and Train pages group readouts. */} +
+ 0 ? "active" : undefined} + /> + + 0 ? "error" : undefined} + /> + +
+ + {/* Borderless rows on 12px hover pills, as in the sidebar. */} +
+ {entries.length === 0 ? ( +

+ No requests yet. +

+ ) : ( + entries.slice(0, VISIBLE_ENTRIES).map((entry) => ( +
+ + + {compactEndpoint(entry.endpoint)} + + + {entry.error ? entry.error : entry.model} + + + {formatDuration(entry.duration_ms)} + +
+ )) + )} +
+ + {/* Through to payloads, filters and per request tokens. */} + + + {/* Closing only silences this burst; this is the permanent off. */} + + +
+ )} + + ); +} diff --git a/studio/frontend/src/features/api-monitor/api-monitor-page.tsx b/studio/frontend/src/features/api-monitor/api-monitor-page.tsx new file mode 100644 index 0000000000..fa5934ab5d --- /dev/null +++ b/studio/frontend/src/features/api-monitor/api-monitor-page.tsx @@ -0,0 +1,778 @@ +// SPDX-License-Identifier: AGPL-3.0-only +// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 + +// Full-page monitor for Unsloth's OpenAI-compatible API server. +// +// This replaces the small console that used to be buried in the API settings +// tab. Settings still owns configuration (keys, auto-switch, examples); this +// page owns observability -- what is being served right now, which requests +// failed and why, and which saved settings a remote load will apply. + +import { Button } from "@/components/ui/button"; +import { Input } from "@/components/ui/input"; +import { + Select, + SelectContent, + SelectItem, + SelectTrigger, + SelectValue, +} from "@/components/ui/select"; +import { Skeleton } from "@/components/ui/skeleton"; +import type { ApiMonitorEntry } from "@/features/chat/types/api"; +import { useSettingsDialogStore } from "@/features/settings"; +import { copyToClipboard } from "@/lib/copy-to-clipboard"; +import { Tick02Icon } from "@/lib/tick-icon"; +import { cn } from "@/lib/utils"; +import { + Copy01Icon, + Delete02Icon, + Globe02Icon, + PauseIcon, + PlayIcon, + RefreshIcon, + Settings02Icon, +} from "@hugeicons/core-free-icons"; +import { HugeiconsIcon } from "@hugeicons/react"; +import { type ReactElement, useEffect, useMemo, useRef, useState } from "react"; +import { SavedModelSettingsPanel } from "./components/saved-model-settings"; +import { + type MonitorStatusFilter, + filterEntries, + useApiMonitor, +} from "./use-api-monitor"; + +const API_INFERENCE_PREFIX_RE = /^\/api\/inference/; +const V1_PREFIX_RE = /^\/v1\//; + +const STATUS_FILTERS: { value: MonitorStatusFilter; label: string }[] = [ + { value: "all", label: "All requests" }, + { value: "running", label: "In flight" }, + { value: "completed", label: "Completed" }, + { value: "error", label: "Errors" }, + { value: "cancelled", label: "Cancelled" }, +]; + +function formatTime(value: number): string { + return new Date(value * 1000).toLocaleTimeString([], { + hour: "2-digit", + minute: "2-digit", + second: "2-digit", + }); +} + +function formatDuration(value?: number | null): string { + if (value == null) { + return "running"; + } + if (value < 1000) { + return `${Math.round(value)} ms`; + } + return `${(value / 1000).toFixed(value < 10000 ? 1 : 0)} s`; +} + +function formatCount(value: number): string { + return value.toLocaleString(); +} + +function compactEndpoint(endpoint: string): string { + return endpoint + .replace(API_INFERENCE_PREFIX_RE, "/api") + .replace(V1_PREFIX_RE, "/"); +} + +function statusDotClass(status: ApiMonitorEntry["status"]): string { + switch (status) { + case "running": + return "bg-blue-500 animate-pulse"; + case "error": + return "bg-red-500"; + case "cancelled": + return "bg-amber-500"; + default: + return "bg-emerald-500"; + } +} + +function statusTextClass(status: ApiMonitorEntry["status"]): string { + switch (status) { + case "running": + return "text-blue-600 dark:text-blue-400"; + case "error": + return "text-red-600 dark:text-red-400"; + case "cancelled": + return "text-amber-600 dark:text-amber-500"; + default: + return "text-emerald-600 dark:text-emerald-500"; + } +} + +function StatCard({ + label, + value, + hint, + tone, +}: { + label: string; + value: string; + hint?: string; + tone?: "default" | "error" | "active"; +}): ReactElement { + return ( +
+ + {label} + + + {value} + + {hint ? ( + + {hint} + + ) : null} +
+ ); +} + +function CopyButton({ + value, + label, +}: { + value: string; + label: string; +}): ReactElement { + const [copied, setCopied] = useState(false); + // Clearing the tick on a timer would set state after unmount if the user + // navigates away mid-flash, so the timer is cancelled on cleanup. + const timerRef = useRef(undefined); + useEffect( + () => () => { + if (timerRef.current !== undefined) window.clearTimeout(timerRef.current); + }, + [], + ); + return ( + + ); +} + +function ContextUsageBar({ + value, +}: { value?: number | null }): ReactElement | null { + if (value == null) { + return null; + } + const pct = Math.max(0, Math.min(100, Math.round(value * 100))); + return ( +
+
+
= 90 + ? "bg-red-500" + : pct >= 75 + ? "bg-amber-500" + : "bg-control-accent", + )} + style={{ width: `${pct}%` }} + /> +
+ + {pct}% + +
+ ); +} + +function RequestRow({ + entry, + selected, + onSelect, +}: { + entry: ApiMonitorEntry; + selected: boolean; + onSelect: () => void; +}): ReactElement { + const preview = + entry.error || + entry.reply_preview || + entry.prompt_preview || + (entry.status === "running" ? "Waiting for output…" : "No preview"); + return ( + + ); +} + +function PayloadBlock({ + title, + body, + truncated, + loading, + tone, +}: { + title: string; + body: string; + truncated?: boolean; + loading?: boolean; + tone?: "error"; +}): ReactElement { + return ( +
+
+

+ {title} +

+
+ {truncated ? ( + + preview only + + ) : null} + {body ? ( + + ) : null} +
+
+ 
+        {loading && !body ? "Loading…" : body || "–"}
+
+
+ ); +} + +function RequestDetail({ + entry, + detail, + loading, +}: { + entry: ApiMonitorEntry; + detail?: ApiMonitorEntry; + loading: boolean; +}): ReactElement { + // The detail fetch is a separate request, so it can describe an older state of + // a still-streaming entry. Prefer it only once it is at least as fresh as the + // list row, otherwise the panel would appear to rewind while tokens arrive. + const detailIsCurrent = + detail != null && + detail.status === entry.status && + detail.updated_at >= entry.updated_at; + const prompt = detail?.prompt ?? entry.prompt_preview; + const reply = detailIsCurrent + ? (detail.reply ?? entry.reply_preview) + : entry.reply_preview; + + return ( +
+
+
+ + + {entry.status} + + + {entry.id} + +
+

+ {entry.method} {entry.endpoint} +

+

+ {entry.model} +

+
+ +
+ {[ + { label: "Started", value: formatTime(entry.started_at) }, + { label: "Duration", value: formatDuration(entry.duration_ms) }, + { + label: "Prompt tokens", + value: + entry.prompt_tokens != null + ? formatCount(entry.prompt_tokens) + : "–", + }, + { + label: "Completion tokens", + value: + entry.completion_tokens != null + ? formatCount(entry.completion_tokens) + : "–", + }, + { + label: "Total tokens", + value: + entry.total_tokens != null + ? formatCount(entry.total_tokens) + : "–", + }, + { + label: "Context", + value: + entry.context_length != null + ? formatCount(entry.context_length) + : "–", + }, + ].map((item) => ( +
+
+ {item.label} +
+
+ {item.value} +
+
+ ))} +
+ + {entry.context_usage != null ? ( +
+ + Context used + + +
+ ) : null} + + {entry.error ? ( + + ) : null} + + + +
+ ); +} + +export function ApiMonitorPage(): ReactElement { + const { + data, + entries, + stats, + error, + loading, + refreshing, + paused, + setPaused, + refresh, + clear, + details, + loadingDetails, + requestDetail, + } = useApiMonitor(); + const [statusFilter, setStatusFilter] = useState("all"); + const [query, setQuery] = useState(""); + const [selectedId, setSelectedId] = useState(null); + + const visible = useMemo( + () => filterEntries(entries, statusFilter, query), + [entries, statusFilter, query], + ); + const selected = useMemo( + () => visible.find((entry) => entry.id === selectedId) ?? null, + [visible, selectedId], + ); + + // Refetch the selected entry while it streams so the payload grows with the + // reply. Keyed on identity and revision, never on `details`: the fetch rewrites + // `details` on every success, so depending on it loops on the detail endpoint, + // which takes the same lock every generated token does. + const selectedId_ = selected?.id ?? null; + const selectedUpdatedAt = selected?.updated_at ?? null; + const selectedIsMissing = selectedId_ != null && details[selectedId_] == null; + const lastFetchedRef = useRef(null); + useEffect(() => { + if (selectedId_ == null) { + return; + } + // `updated_at` advances per poll while streaming and settles when terminal. + // A missing payload always retries, covering a fetch that failed late. + const revision = `${selectedId_}@${selectedUpdatedAt ?? ""}`; + if (!selectedIsMissing && lastFetchedRef.current === revision) { + return; + } + lastFetchedRef.current = revision; + requestDetail(selectedId_); + }, [selectedId_, selectedUpdatedAt, selectedIsMissing, requestDetail]); + + const baseUrl = + typeof window === "undefined" ? "" : `${window.location.origin}/v1`; + const serverStatus = data?.status ?? "idle"; + const statusCopy = + serverStatus === "generating" + ? "Serving requests" + : serverStatus === "ready" + ? "Ready" + : "No model loaded"; + + return ( +
+
+
+

+ API +

+

+ Live traffic through Unsloth's OpenAI-compatible server. +

+
+
+ + + + +
+
+ + {/* Server summary: the two things you check first when a client can't + reach the API -- the base URL to point it at, and what is loaded. */} +
+
+ + + +
+ + Base URL + + + {baseUrl} + +
+ +
+
+ + Status + + + + {statusCopy} + +
+
+ + Loaded model + + + {data?.active_model ?? "None"} + {data?.context_length + ? ` · ${formatCount(data.context_length)} ctx` + : ""} + +
+ {paused ? ( + + Paused + + ) : null} +
+ + {error ? ( +
+ {error} +
+ ) : null} + +
+ 0 ? "active" : "default"} + /> + + + 0 ? "error" : "default"} + hint={ + stats.errorRate != null + ? `${Math.round(stats.errorRate * 100)}% of finished` + : undefined + } + /> + + +
+ +
+
+ setQuery(event.target.value)} + placeholder="Search model, endpoint, preview or error" + aria-label="Search API requests" + className="h-9 w-full min-w-0 flex-1 rounded-full border-none bg-muted shadow-none dark:bg-background sm:w-64 sm:flex-none" + /> + + + {formatCount(visible.length)} of {formatCount(entries.length)} + +
+ +
+
+ {loading ? ( +
+ {[0, 1, 2].map((i) => ( + + ))} +
+ ) : visible.length === 0 ? ( +

+ {entries.length === 0 + ? "No API traffic yet. Point a client at the base URL above to see requests here." + : "No requests match this filter."} +

+ ) : ( + visible.map((entry) => ( + setSelectedId(entry.id)} + /> + )) + )} +
+ +
+ {selected ? ( + + ) : ( +

+ Select a request to inspect its prompt, reply, tokens and + errors. +

+ )} +
+
+
+ + +
+ ); +} diff --git a/studio/frontend/src/features/api-monitor/components/saved-model-settings.tsx b/studio/frontend/src/features/api-monitor/components/saved-model-settings.tsx new file mode 100644 index 0000000000..6a1224472e --- /dev/null +++ b/studio/frontend/src/features/api-monitor/components/saved-model-settings.tsx @@ -0,0 +1,140 @@ +// SPDX-License-Identifier: AGPL-3.0-only +// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 + +// What a remote load will actually apply, which is otherwise unanswerable from +// outside the process. +// +// Read only on purpose: the config lives both here and in the browser's own +// per-model store, and the model's settings page is the only place that owns +// both, so it is the only place that can forget a model completely. + +import { Skeleton } from "@/components/ui/skeleton"; +import { + type ApiModelOverride, + type ApiModelOverrides, + fetchModelOverrides, +} from "@/features/model-picker/api/model-overrides"; +import { type ReactElement, useCallback, useEffect, useState } from "react"; + +/** Human-readable summary of the fields the loader will apply, in load order. */ +function describeOverride(override: ApiModelOverride): string[] { + const parts: string[] = []; + if (override.custom_context_length) { + parts.push(`${override.custom_context_length.toLocaleString()} context`); + } + if (override.max_seq_length) { + parts.push(`${override.max_seq_length.toLocaleString()} max seq`); + } + if (override.kv_cache_dtype) { + parts.push(`KV ${override.kv_cache_dtype}`); + } + if (override.speculative_type) { + parts.push( + override.spec_draft_n_max + ? `spec ${override.speculative_type} ×${override.spec_draft_n_max}` + : `spec ${override.speculative_type}`, + ); + } + if (override.tensor_parallel) { + parts.push("tensor parallel"); + } + if (override.gpu_memory_mode === "manual") { + parts.push("manual GPU memory"); + } + if (override.gpu_layers != null) { + parts.push(`${override.gpu_layers} GPU layers`); + } + if (override.n_cpu_moe) { + parts.push(`${override.n_cpu_moe} MoE layers on CPU`); + } + if (override.gpu_ids?.length) { + parts.push(`GPU ${override.gpu_ids.join(", ")}`); + } + if (override.chat_template_override) { + parts.push("custom chat template"); + } + if (override.llama_extra_args?.length) { + parts.push(override.llama_extra_args.join(" ")); + } + return parts; +} + +export function SavedModelSettingsPanel(): ReactElement { + const [overrides, setOverrides] = useState(null); + const [error, setError] = useState(null); + + const load = useCallback(async () => { + try { + setOverrides(await fetchModelOverrides()); + setError(null); + } catch (err: unknown) { + setError( + err instanceof Error + ? err.message + : "Could not load saved model settings", + ); + } + }, []); + + useEffect(() => { + void load(); + }, [load]); + + const entries = Object.entries(overrides ?? {}); + + return ( +
+
+

+ Settings applied on API load +

+

+ When a request names one of these models, Unsloth loads it with these + settings, the same ones you saved in the model's settings page. + Models without an entry load with app defaults. Edit or forget an + entry from that model's settings, which keeps this list and the + picker in step. +

+
+ + {error ? ( +
+ {error} +
+ ) : overrides == null ? ( +
+ {[0, 1].map((i) => ( + + ))} +
+ ) : entries.length === 0 ? ( +

+ No saved model settings yet. Open a model's settings, turn on + "Remember for this model", and it will be applied to API + loads too. +

+ ) : ( +
    + {entries.map(([modelId, override]) => { + const summary = describeOverride(override); + return ( +
  • +
    + + {modelId} + + + {summary.length > 0 ? summary.join(" · ") : "App defaults"} + +
    +
    • + ); + })} +
+ )} +
+ ); +} diff --git a/studio/frontend/src/features/api-monitor/index.ts b/studio/frontend/src/features/api-monitor/index.ts new file mode 100644 index 0000000000..28b01def8b --- /dev/null +++ b/studio/frontend/src/features/api-monitor/index.ts @@ -0,0 +1,13 @@ +// SPDX-License-Identifier: AGPL-3.0-only +// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 + +export { ApiMonitorPage } from "./api-monitor-page"; +export { ApiMonitorOverlay } from "./api-monitor-overlay"; +export { useApiMonitorOverlayStore } from "./overlay-store"; +export { + computeStats, + filterEntries, + useApiMonitor, + type MonitorStats, + type MonitorStatusFilter, +} from "./use-api-monitor"; diff --git a/studio/frontend/src/features/api-monitor/overlay-store.ts b/studio/frontend/src/features/api-monitor/overlay-store.ts new file mode 100644 index 0000000000..07ad4edd31 --- /dev/null +++ b/studio/frontend/src/features/api-monitor/overlay-store.ts @@ -0,0 +1,48 @@ +// SPDX-License-Identifier: AGPL-3.0-only +// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 + +import { create } from "zustand"; +import { persist } from "zustand/middleware"; + +interface ApiMonitorOverlayState { + /** Whether the floating panel is on screen right now. Session state. */ + isOpen: boolean; + /** Set on close so the panel does not pop back during the same burst. */ + suppressed: boolean; + /** Persisted opt out: when false the panel never opens itself. */ + autoOpen: boolean; + open: () => void; + close: () => void; + setAutoOpen: (autoOpen: boolean) => void; +} + +/** + * Only `autoOpen` persists. Open/closed is session state: a dismissal lasts the + * sitting, not forever. + */ +export const useApiMonitorOverlayStore = create()( + persist( + (set) => ({ + isOpen: false, + suppressed: false, + autoOpen: true, + open: () => set({ isOpen: true, suppressed: false }), + close: () => set({ isOpen: false, suppressed: true }), + setAutoOpen: (autoOpen) => set({ autoOpen }), + }), + { + name: "unsloth_api_monitor_overlay", + version: 1, + partialize: (state) => ({ autoOpen: state.autoOpen }), + // Explicit merge so an older stored payload cannot resurrect `isOpen`. + merge: (persisted, current) => ({ + ...current, + autoOpen: + typeof (persisted as { autoOpen?: unknown } | null)?.autoOpen === + "boolean" + ? (persisted as { autoOpen: boolean }).autoOpen + : current.autoOpen, + }), + }, + ), +); diff --git a/studio/frontend/src/features/api-monitor/use-api-monitor.ts b/studio/frontend/src/features/api-monitor/use-api-monitor.ts new file mode 100644 index 0000000000..20c3915ee3 --- /dev/null +++ b/studio/frontend/src/features/api-monitor/use-api-monitor.ts @@ -0,0 +1,297 @@ +// SPDX-License-Identifier: AGPL-3.0-only +// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 + +import { + clearApiMonitor, + getApiMonitor, + getApiMonitorEntry, +} from "@/features/chat/api/chat-api"; +import type { + ApiMonitorEntry, + ApiMonitorResponse, +} from "@/features/chat/types/api"; +import { useCallback, useEffect, useMemo, useRef, useState } from "react"; + +/** Poll cadence while the monitor is live. Matches the settings console it replaces. */ +const POLL_INTERVAL_MS = 1500; + +export type MonitorStatusFilter = + | "all" + | "running" + | "completed" + | "error" + | "cancelled"; + +export interface MonitorStats { + active: number; + total: number; + completed: number; + errors: number; + cancelled: number; + /** Mean duration over finished requests, or null when none have finished. */ + avgDurationMs: number | null; + /** Slowest finished request, for spotting a single pathological call. */ + maxDurationMs: number | null; + totalTokens: number; + /** Share of finished requests that failed, 0-1. Null when nothing finished. */ + errorRate: number | null; + /** Mean completion tokens per second over requests that reported both. */ + tokensPerSecond: number | null; +} + +function isTerminal(entry: ApiMonitorEntry): boolean { + return entry.status !== "running"; +} + +function completionTokens(entry: ApiMonitorEntry): number | null { + if (entry.completion_tokens != null) { + return entry.completion_tokens; + } + // Some providers only report a total; subtracting the prompt is the best + // available estimate of what was actually generated. + if (entry.total_tokens != null && entry.prompt_tokens != null) { + return Math.max(0, entry.total_tokens - entry.prompt_tokens); + } + return null; +} + +function entryTokens(entry: ApiMonitorEntry): number { + if (entry.total_tokens != null) { + return entry.total_tokens; + } + return (entry.prompt_tokens ?? 0) + (entry.completion_tokens ?? 0); +} + +export function computeStats(entries: ApiMonitorEntry[]): MonitorStats { + let active = 0; + let completed = 0; + let errors = 0; + let cancelled = 0; + let totalTokens = 0; + let durationSum = 0; + let durationCount = 0; + let maxDurationMs: number | null = null; + // Throughput is aggregated as total tokens over total time, not as the mean of + // each request's rate. Averaging rates lets one tiny fast request outweigh a + // long slow one, which is the opposite of what someone debugging wants to see. + let generatedTokens = 0; + let generatedDurationMs = 0; + + for (const entry of entries) { + totalTokens += entryTokens(entry); + if (entry.status === "running") { + active += 1; + } else if (entry.status === "error") { + errors += 1; + } else if (entry.status === "cancelled") { + cancelled += 1; + } else { + completed += 1; + } + const duration = entry.duration_ms; + if (duration != null && isTerminal(entry)) { + durationSum += duration; + durationCount += 1; + maxDurationMs = + maxDurationMs == null ? duration : Math.max(maxDurationMs, duration); + const generated = completionTokens(entry); + // Sub-millisecond durations would divide into a meaningless rate. + if (generated != null && generated > 0 && duration > 0) { + generatedTokens += generated; + generatedDurationMs += duration; + } + } + } + + const finished = completed + errors + cancelled; + return { + active, + total: entries.length, + completed, + errors, + cancelled, + avgDurationMs: durationCount > 0 ? durationSum / durationCount : null, + maxDurationMs, + totalTokens, + errorRate: finished > 0 ? errors / finished : null, + tokensPerSecond: + generatedDurationMs > 0 + ? generatedTokens / (generatedDurationMs / 1000) + : null, + }; +} + +export function filterEntries( + entries: ApiMonitorEntry[], + status: MonitorStatusFilter, + query: string, +): ApiMonitorEntry[] { + const needle = query.trim().toLowerCase(); + return entries.filter((entry) => { + if (status !== "all" && entry.status !== status) { + return false; + } + if (!needle) { + return true; + } + // Search the fields a debugging session actually keys off: which model, + // which endpoint, and the previews/error text visible in the row. + return ( + entry.model.toLowerCase().includes(needle) || + entry.endpoint.toLowerCase().includes(needle) || + entry.prompt_preview.toLowerCase().includes(needle) || + entry.reply_preview.toLowerCase().includes(needle) || + (entry.error ?? "").toLowerCase().includes(needle) + ); + }); +} + +interface UseApiMonitorResult { + data: ApiMonitorResponse | null; + entries: ApiMonitorEntry[]; + stats: MonitorStats; + error: string | null; + /** True until the first response lands, so the page can show skeletons once. */ + loading: boolean; + refreshing: boolean; + paused: boolean; + setPaused: (paused: boolean) => void; + refresh: () => void; + clear: () => Promise; + /** Full prompt/reply for entries the user expanded, keyed by entry id. */ + details: Record; + loadingDetails: ReadonlySet; + requestDetail: (id: string) => void; +} + +/** + * Live view of the server's OpenAI-compatible API traffic. + * + * Polls rather than streams because the backing monitor is an in-memory ring + * buffer with no change feed. Polling is self-rescheduling (never overlapping), + * and pausing stops it entirely so a user reading a stalled request's payload + * isn't fighting a list that reorders under them. + * + * `intervalMs` lets a caller trade freshness for cost: the full page wants the + * default live cadence, while the floating overlay slows right down when it is + * closed and only watching for the traffic that should pop it open. + */ +export function useApiMonitor({ + intervalMs = POLL_INTERVAL_MS, +}: { intervalMs?: number } = {}): UseApiMonitorResult { + const [data, setData] = useState(null); + const [error, setError] = useState(null); + const [loading, setLoading] = useState(true); + const [refreshing, setRefreshing] = useState(false); + const [paused, setPaused] = useState(false); + const [details, setDetails] = useState>({}); + const [loadingDetails, setLoadingDetails] = useState>( + () => new Set(), + ); + // Mirrors `loadingDetails` outside React state so the fetch guard sees writes + // from the same tick (state updates are async and would let duplicates through). + const inFlightDetails = useRef>(new Set()); + + const load = useCallback(async (): Promise => { + setRefreshing(true); + try { + const next = await getApiMonitor(); + setData(next); + setError(null); + } catch (err: unknown) { + setError(err instanceof Error ? err.message : "Monitor unavailable"); + } finally { + setRefreshing(false); + setLoading(false); + } + }, []); + + useEffect(() => { + if (paused) { + return; + } + let cancelled = false; + let timer: number | undefined; + + function poll(): void { + getApiMonitor() + .then((next) => { + if (cancelled) return; + setData(next); + setError(null); + }) + .catch((err: unknown) => { + if (cancelled) return; + setError(err instanceof Error ? err.message : "Monitor unavailable"); + }) + .finally(() => { + if (cancelled) return; + setLoading(false); + timer = window.setTimeout(poll, intervalMs); + }); + } + + poll(); + return () => { + cancelled = true; + if (timer !== undefined) { + window.clearTimeout(timer); + } + }; + }, [paused, intervalMs]); + + const requestDetail = useCallback((id: string): void => { + if (inFlightDetails.current.has(id)) { + return; + } + inFlightDetails.current.add(id); + setLoadingDetails((prev) => new Set(prev).add(id)); + getApiMonitorEntry(id) + .then((entry) => { + setDetails((prev) => ({ ...prev, [id]: entry })); + }) + .catch(() => { + // The entry aged out of the ring buffer; drop any stale copy so the UI + // falls back to the row previews instead of showing a frozen payload. + setDetails((prev) => { + if (!(id in prev)) return prev; + const next = { ...prev }; + delete next[id]; + return next; + }); + }) + .finally(() => { + inFlightDetails.current.delete(id); + setLoadingDetails((prev) => { + const next = new Set(prev); + next.delete(id); + return next; + }); + }); + }, []); + + const clear = useCallback(async (): Promise => { + await clearApiMonitor(); + setDetails({}); + await load(); + }, [load]); + + const entries = useMemo(() => data?.entries ?? [], [data]); + const stats = useMemo(() => computeStats(entries), [entries]); + + return { + data, + entries, + stats, + error, + loading, + refreshing, + paused, + setPaused, + refresh: () => void load(), + clear, + details, + loadingDetails, + requestDetail, + }; +} diff --git a/studio/frontend/src/features/chat/api/chat-api.ts b/studio/frontend/src/features/chat/api/chat-api.ts index 4d123e98ab..822de4fa4f 100644 --- a/studio/frontend/src/features/chat/api/chat-api.ts +++ b/studio/frontend/src/features/chat/api/chat-api.ts @@ -129,6 +129,13 @@ export async function getApiMonitorEntry(id: string): Promise { return parseJsonOrThrow(response); } +export async function clearApiMonitor(): Promise { + const response = await authFetch("/api/inference/monitor", { + method: "DELETE", + }); + await parseJsonOrThrow<{ cleared: boolean }>(response); +} + export async function loadModel( payload: LoadModelRequest, ): Promise { diff --git a/studio/frontend/src/features/hub/catalog/download-card.tsx b/studio/frontend/src/features/hub/catalog/download-card.tsx index 8016a9406d..1618b9cdf1 100644 --- a/studio/frontend/src/features/hub/catalog/download-card.tsx +++ b/studio/frontend/src/features/hub/catalog/download-card.tsx @@ -23,6 +23,7 @@ import { ArrowReloadHorizontalIcon, Delete02Icon, Download01Icon, + Settings02Icon, } from "@hugeicons/core-free-icons"; import { HugeiconsIcon } from "@hugeicons/react"; import { @@ -82,6 +83,40 @@ export function CardDivider() { ); } +/** Gear that opens a downloaded model's full settings page. */ +export function CardSettingsButton({ + label, + onClick, +}: { + label: string; + onClick: () => void; +}) { + return ( + + + + + + {label} + + + ); +} + export function CardDeleteButton({ label, onClick, diff --git a/studio/frontend/src/features/hub/catalog/hub-model-settings-view.tsx b/studio/frontend/src/features/hub/catalog/hub-model-settings-view.tsx new file mode 100644 index 0000000000..a293a0071a --- /dev/null +++ b/studio/frontend/src/features/hub/catalog/hub-model-settings-view.tsx @@ -0,0 +1,138 @@ +// SPDX-License-Identifier: AGPL-3.0-only +// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 + +// Full-page settings for one model, opened from the Hub. +// +// The same controls exist in the chat picker's popover, but a popover is a poor +// place to work through every knob a model has. This gives them a page, and +// states plainly that whatever is saved here is what an API load will use -- +// the settings are mirrored server-side by ModelConfigPage's save. + +import { ModelConfigPage, type ModelPickTarget } from "@/features/model-picker"; +import type { PerModelConfig } from "@/features/model-picker"; +import { ArrowLeft01Icon, Globe02Icon } from "@hugeicons/core-free-icons"; +import { HugeiconsIcon } from "@hugeicons/react"; +import { cn } from "@/lib/utils"; +import { useEffect, useRef, useState } from "react"; + +export function HubModelSettingsView({ + target, + loadedConfig = null, + loadedContextLength = null, + onBack, + onRun, + compact = false, +}: { + target: ModelPickTarget; + /** Non-null when this model is the loaded one, so the page can show live values. */ + loadedConfig?: PerModelConfig | null; + loadedContextLength?: number | null; + onBack: () => void; + /** Apply + load with these settings. */ + onRun: (config: PerModelConfig) => void; + compact?: boolean; +}) { + const scrollRef = useRef(null); + const [scrolled, setScrolled] = useState(false); + // Mirrors HubDetailView so this view sits at the same measure as the rest of + // the Hub rather than introducing a third column width. + const measure = compact + ? "mx-auto w-full max-w-[860px] px-5 sm:px-5" + : "mx-auto w-full max-w-[1100px] px-5 sm:px-8"; + + useEffect(() => { + const el = scrollRef.current; + if (!el) return; + const onScroll = () => { + const next = el.scrollTop > 0; + setScrolled((current) => (current === next ? current : next)); + }; + onScroll(); + el.addEventListener("scroll", onScroll, { passive: true }); + return () => el.removeEventListener("scroll", onScroll); + }, []); + + return ( +
+ + ); +} diff --git a/studio/frontend/src/features/hub/catalog/local-on-device-card.tsx b/studio/frontend/src/features/hub/catalog/local-on-device-card.tsx index 9b508a5413..b432b11f54 100644 --- a/studio/frontend/src/features/hub/catalog/local-on-device-card.tsx +++ b/studio/frontend/src/features/hub/catalog/local-on-device-card.tsx @@ -53,6 +53,7 @@ import { useHfTokenStore } from "../stores/hf-token-store"; import { DotTag } from "./dot-tag"; import { CardDeleteButton, + CardSettingsButton, CardUpdateButton, DeleteConfirmDialog, UpdateConfirmDialog, @@ -96,6 +97,8 @@ interface LocalOnDeviceCardProps { onEject?: () => void; onTrain?: () => void; onChange?: () => void; + /** Open this model's full settings page for the shown quant. */ + onOpenSettings?: (ggufVariant: string | null) => void; } function formatAdapterLabel( @@ -214,6 +217,7 @@ export function LocalOnDeviceCard({ onEject, onTrain, onChange, + onOpenSettings, }: LocalOnDeviceCardProps) { const [deleteOpen, setDeleteOpen] = useState(false); const [updateOpen, setUpdateOpen] = useState(false); @@ -549,6 +553,14 @@ export function LocalOnDeviceCard({ )}
+ {onOpenSettings && ( + onOpenSettings(selectedQuant ?? null)} + /> + )} {canUpdate && ( void; onInventoryChange?: () => void; onSearchHub?: (query: string) => void; + /** Open this model's full settings page, with the quant the card resolved. */ + onOpenSettings?: (ggufVariant: string | null) => void; }; export const ModelInspector = memo(function ModelInspector({ @@ -438,6 +440,7 @@ export const ModelInspector = memo(function ModelInspector({ onTrain, onInventoryChange, onSearchHub, + onOpenSettings, } = actions; const deviceType = usePlatformStore((s) => s.deviceType); const chatOnly = usePlatformStore((s) => s.isChatOnly()); @@ -705,6 +708,7 @@ export const ModelInspector = memo(function ModelInspector({ model.isDownloaded && canTrainModel ? onTrain : undefined } onChange={onInventoryChange} + onOpenSettings={onOpenSettings} /> ) : ( void; + /** Open a downloaded model's full settings page. */ + onOpenModelSettings?: (row: CachedInventoryRow | LocalInventoryRow) => void; }) { // Pinned repos surface first regardless of the active sort; the chosen sort // still orders rows within the pinned and unpinned groups. @@ -383,6 +386,7 @@ export function DownloadedList({ compact={compact} onSelect={onSelect} onChange={onInventoryChange} + onOpenSettings={onOpenModelSettings} /> ); diff --git a/studio/frontend/src/features/hub/catalog/models-catalog-rows.tsx b/studio/frontend/src/features/hub/catalog/models-catalog-rows.tsx index 5236c0be74..1dbf78cff9 100644 --- a/studio/frontend/src/features/hub/catalog/models-catalog-rows.tsx +++ b/studio/frontend/src/features/hub/catalog/models-catalog-rows.tsx @@ -573,6 +573,7 @@ export const InventoryRow = memo(function InventoryRow({ compact = false, onSelect, onChange, + onOpenSettings, }: { row: CachedInventoryRow | LocalInventoryRow; selected: boolean; @@ -585,6 +586,8 @@ export const InventoryRow = memo(function InventoryRow({ compact?: boolean; onSelect: (id: string) => void; onChange?: () => void; + /** Open this model's full settings page. Omitted for datasets. */ + onOpenSettings?: (row: CachedInventoryRow | LocalInventoryRow) => void; }) { const rowModelId = row.kind === "cache" @@ -732,30 +735,41 @@ export const InventoryRow = memo(function InventoryRow({ const rowPinned = cacheDeletableRepoId != null && pinnedKeys.includes(pinKey(cacheDeletableRepoId)); + // Settings is available for any downloaded model, not just deletable ones: a + // model scanned from a local folder is just as configurable as a cached repo. + // The menu therefore renders whenever either action applies, and each item + // gates itself. `deletableRepoId` (rather than the boolean) keeps the non-null + // narrowing the delete closures below rely on. + const settingsAction = + !isDataset && onOpenSettings ? { onOpen: () => onOpenSettings(row) } : undefined; + const deletableRepoId = canDelete ? cacheDeletableRepoId : null; const deleteAction = - canDelete && cacheDeletableRepoId ? ( + deletableRepoId || settingsAction ? ( togglePinned(cacheDeletableRepoId), + onToggle: () => togglePinned(deletableRepoId), } } - cachePath={isDataset ? undefined : { repoId: cacheDeletableRepoId }} - del={{ + cachePath={ + isDataset || !deletableRepoId ? undefined : { repoId: deletableRepoId } + } + del={deletableRepoId ? { title: isDataset ? "Delete cached dataset?" : "Delete cached model?", description: ( <> This will remove{" "} - {cacheDeletableRepoId} + {deletableRepoId} {" "} {isDataset ? "and its downloaded files" @@ -766,17 +780,17 @@ export const InventoryRow = memo(function InventoryRow({ disk. You can re-download it later. ), - successMessage: `Deleted ${cacheDeletableRepoId}`, + successMessage: `Deleted ${deletableRepoId}`, onConfirm: async () => { // Delete only the copy this row shows: cache rows carry the owning // cache path, so pass it through and leave other caches untouched. const rowCachePath = row.kind === "cache" ? (row.cachePath ?? undefined) : undefined; if (isDataset) { - await deleteCachedDataset(cacheDeletableRepoId, rowCachePath); + await deleteCachedDataset(deletableRepoId, rowCachePath); } else { await deleteCachedModel( - cacheDeletableRepoId, + deletableRepoId, undefined, undefined, rowCachePath, @@ -787,11 +801,11 @@ export const InventoryRow = memo(function InventoryRow({ usePinnedModelsStore.getState(); for (const key of pinned) { if ( - key === pinKey(cacheDeletableRepoId) || - key.startsWith(`${cacheDeletableRepoId}::`) + key === pinKey(deletableRepoId) || + key.startsWith(`${deletableRepoId}::`) ) { toggle( - cacheDeletableRepoId, + deletableRepoId, key.includes("::") ? key.slice(key.indexOf("::") + 2) : undefined, @@ -801,7 +815,7 @@ export const InventoryRow = memo(function InventoryRow({ } }, onDeleted: onChange, - }} + } : undefined} /> ) : null; diff --git a/studio/frontend/src/features/hub/catalog/models-catalog.tsx b/studio/frontend/src/features/hub/catalog/models-catalog.tsx index 02b9b62fce..288c525af1 100644 --- a/studio/frontend/src/features/hub/catalog/models-catalog.tsx +++ b/studio/frontend/src/features/hub/catalog/models-catalog.tsx @@ -16,6 +16,7 @@ import { import type { CachedInventoryRow, DiscoverRow, + InventoryRow, LocalInventoryRow, ModelsTab, } from "../types"; @@ -70,6 +71,8 @@ export interface ModelsCatalogHandlers { onRetry: () => void; onInventoryChange?: () => void; onSwitchDevice?: () => void; + /** Open a downloaded model's full settings page. */ + onOpenModelSettings?: (row: InventoryRow) => void; } function assignRef(ref: RefObject, value: T | null) { @@ -128,6 +131,7 @@ export const ModelsCatalog = memo(function ModelsCatalog({ onRetry, onInventoryChange, onSwitchDevice, + onOpenModelSettings, } = handlers; const [scrolled, setScrolled] = useState(false); const [streamingActive, setStreamingActive] = useState(false); @@ -483,6 +487,7 @@ export const ModelsCatalog = memo(function ModelsCatalog({ columns={discoverView === "two" ? 2 : 1} sort={inventorySort} onInventoryChange={onInventoryChange} + onOpenModelSettings={onOpenModelSettings} />
) : ( diff --git a/studio/frontend/src/features/hub/hub-page.tsx b/studio/frontend/src/features/hub/hub-page.tsx index 9fc452b4ed..e853ea0e5c 100644 --- a/studio/frontend/src/features/hub/hub-page.tsx +++ b/studio/frontend/src/features/hub/hub-page.tsx @@ -9,6 +9,7 @@ import { import { getInferenceStatus, isExternalModelId, + listGgufVariants, useChatModelRuntime, useChatRuntimeStore, } from "@/features/chat"; @@ -24,9 +25,13 @@ import { ggufVariantsMatch, modelIdsMatch } from "./lib/model-identity"; import { hfApiToken, useHfTokenStore } from "./stores/hf-token-store"; import { applyModelLoadConfigToRuntime, + applyPerModelConfigToRuntime, currentRuntimePerModelConfig, hfModelFitsDevice, resolveInitialConfig, + useActiveModelConfig, + type ModelPickTarget, + type PerModelConfig, } from "@/features/model-picker"; import { useDebouncedValue } from "@/hooks/use-debounced-value"; import { useGpuInfo } from "@/hooks/use-gpu-info"; @@ -42,6 +47,7 @@ import { } from "react"; import { ExternalLinkConfirmDialog } from "./catalog/external-link-confirm-dialog"; import { HubDetailView } from "./catalog/hub-detail-view"; +import { HubModelSettingsView } from "./catalog/hub-model-settings-view"; import { HubFeed } from "./catalog/hub-feed"; import { HubTopBar } from "./catalog/hub-top-bar"; import { @@ -345,6 +351,12 @@ export function ModelsPage() { const activeCheckpoint = checkpoint && !isExternalModelId(checkpoint) ? checkpoint : null; const activeGgufVariant = useChatRuntimeStore((s) => s.activeGgufVariant); + const activeGgufContextLength = useChatRuntimeStore( + (s) => s.ggufContextLength, + ); + // Live settings of the loaded model, so opening its settings page shows what + // it is actually running with rather than the last saved draft. + const { config: activeModelConfig } = useActiveModelConfig(); // Shared with the chat model selector: list only models sized for this device. const fitOnDeviceOnly = useChatRuntimeStore((s) => s.fitOnDeviceOnly); const setFitOnDeviceOnly = useChatRuntimeStore((s) => s.setFitOnDeviceOnly); @@ -1213,6 +1225,100 @@ export function ModelsPage() { runSelectedModel(opts, selectedModel?.isDownloaded ?? true), [runSelectedModel, selectedModel], ); + + // Full-page per-model settings, opened from a downloaded row's menu. Local + // state rather than a URL param: the page is a transient editor over the + // catalog, and a deep link to it would need the row's identity re-resolved + // against an inventory that may not have loaded yet. + const [settingsTarget, setSettingsTarget] = useState( + null, + ); + const openModelSettings = useCallback( + async (row: CachedInventoryRow | LocalInventoryRow) => { + // loadId is what the loader accepts; repoId is only a display/API alias. + const id = row.loadId; + // Cached repo rows never carry a quant: the inventory emits one row per + // repo with format_variant null (see cache_inventory.py). Opening settings + // with a null variant would key the saved config to `repo::` while the + // loader reads `repo::Q4_K_M`, so the settings would silently never apply + // and the server mirror would be keyed wrong too. Resolve the quant the + // same way the on-device card does before opening. + let ggufVariant = row.formatVariant?.trim() || null; + if (!ggufVariant && row.isGguf && row.capabilities.requiresVariant) { + const repoId = row.kind === "cache" ? row.repoId : (row.repoId ?? null); + if (repoId) { + try { + const res = await listGgufVariants(repoId, hfApiToken(hfToken), { + preferLocalCache: true, + localPath: row.kind === "local" ? row.path : (row.cachePath ?? null), + }); + const downloaded = res.variants.filter((v) => v.downloaded); + ggufVariant = + // Prefer the loaded quant, then the repo default, then whatever is + // on disk, mirroring LocalOnDeviceCard's selectedQuant. + downloaded.find((v) => + ggufVariantsMatch(v.quant, activeGgufVariant), + )?.quant ?? + downloaded.find((v) => + ggufVariantsMatch(v.quant, res.default_variant), + )?.quant ?? + downloaded[0]?.quant ?? + null; + } catch { + // Offline or an unreadable cache: fall through with no variant. The + // settings page still works, it just cannot pin a specific quant. + ggufVariant = null; + } + } + } + const leaf = id.split(/[\\/]/).filter(Boolean).pop() ?? id; + setSettingsTarget({ + id, + displayName: ggufVariant ? `${leaf} · ${ggufVariant}` : leaf, + ggufVariant, + isGguf: row.isGguf, + meta: { + source: "local", + isLora: row.modelFormat === "adapter", + ggufVariant: ggufVariant ?? undefined, + isGguf: row.isGguf, + // Partial downloads still open settings, but must not claim to be + // complete or the loader skips its download-progress reporting. + isDownloaded: !row.partial, + // Not carried on inventory rows; ModelConfigPage reads the GGUF header + // itself to size the context slider. + contextLength: null, + }, + }); + }, + [activeGgufVariant, hfToken], + ); + // Applying from the settings page loads the model with exactly those settings. + // ModelConfigPage has already persisted them (locally and, when "remember" is + // on, to the server), so an API request for this model gets the same load. + const runSettingsTarget = useCallback( + (config: PerModelConfig) => { + const target = settingsTarget; + if (!target) return; + const previousConfig = currentRuntimePerModelConfig({ + includeMaxSeqLength: true, + }); + applyPerModelConfigToRuntime(config); + setSettingsTarget(null); + void selectModel({ + id: target.id, + source: "local", + ggufVariant: target.ggufVariant ?? undefined, + isGguf: target.isGguf, + isDownloaded: true, + isLora: target.meta.isLora, + keepSpeculative: true, + forceReload: true, + previousConfig, + }).catch(() => undefined); + }, + [selectModel, settingsTarget], + ); const handleLoadLocal = useCallback( (opts: ModelLoadOptions = {}) => runSelectedModel(opts, true), [runSelectedModel], @@ -1220,6 +1326,30 @@ export function ModelsPage() { const handleTrain = useCallback(() => { // Hub → train integration ships in a later PR. }, []); + // Settings opened from the detail view's on-device card. The card resolves + // which quant it is showing, so it passes that in rather than re-deriving it. + const openSelectedModelSettings = useCallback( + (ggufVariant: string | null) => { + if (!selectedModel) return; + const id = selectedModel.resource.runId; + const leaf = id.split(/[\\/]/).filter(Boolean).pop() ?? id; + setSettingsTarget({ + id, + displayName: ggufVariant ? `${leaf} · ${ggufVariant}` : leaf, + ggufVariant, + isGguf: selectedModel.isGguf, + meta: { + source: "local", + isLora: selectedModel.modelFormat === "adapter", + ggufVariant: ggufVariant ?? undefined, + isGguf: selectedModel.isGguf, + isDownloaded: selectedModel.isDownloaded, + contextLength: null, + }, + }); + }, + [selectedModel], + ); const handleSearchHub = useCallback( (next: string) => { const trimmed = next.trim(); @@ -1280,6 +1410,7 @@ export function ModelsPage() { onTrain: handleTrain, onInventoryChange: refreshInventory, onSearchHub: handleSearchHub, + onOpenSettings: openSelectedModelSettings, }), [ handleLoad, @@ -1289,6 +1420,7 @@ export function ModelsPage() { handleTrain, handleSearchHub, refreshInventory, + openSelectedModelSettings, ], ); @@ -1370,6 +1502,7 @@ export function ModelsPage() { onRetry: handleRetrySearch, onInventoryChange: refreshInventory, onSwitchDevice: handleSwitchDevice, + onOpenModelSettings: openModelSettings, }), [ handleSelect, @@ -1378,6 +1511,7 @@ export function ModelsPage() { handleRetrySearch, refreshInventory, handleSwitchDevice, + openModelSettings, ], ); @@ -1520,6 +1654,10 @@ export function ModelsPage() { const detailOpen = urlModel !== null; const splitMode = allModelsView === "split"; + // The catalog is unreachable when an opaque overlay sits on top of it: the + // detail view (full-page layout only, since split renders it alongside) or the + // settings page (always full-bleed). + const catalogCovered = (detailOpen && !splitMode) || settingsTarget !== null; return (
@@ -1575,9 +1713,15 @@ export function ModelsPage() { splitMode ? "flex-1 lg:w-[460px] lg:max-w-[44%] lg:flex-none lg:shrink-0 lg:border-r lg:border-border/60" : "flex-1", - detailOpen && !splitMode && "pointer-events-none", + // The settings page is a full-bleed opaque overlay in every layout, + // including split, so it always takes the catalog out of the tab + // order. Without this, tabbing out of the settings form walks into + // the virtualized rows hidden behind it and screen readers announce + // the whole model list underneath. + catalogCovered && "pointer-events-none", )} - aria-hidden={(detailOpen && !splitMode) || undefined} + aria-hidden={catalogCovered || undefined} + inert={catalogCovered || undefined} > ) )} + + {/* Sits above the detail overlay (z-30): opening settings from a row + while a model preview is open should show the settings, not stack + behind it. */} + {settingsTarget && ( +
+ setSettingsTarget(null)} + onRun={runSettingsTarget} + compact={splitMode} + /> +
+ )}
; + +/** + * The key one model's config is stored under. + * + * Uses the `repo:VARIANT` form an OpenAI request names a quant by, so two quants + * of the same repo keep separate configs and the backend can match the requested + * model name directly. Falls back to the bare id when there is no variant. + */ +export function modelOverrideKey( + modelId: string, + ggufVariant?: string | null, +): string { + return ggufVariant ? `${modelId}:${ggufVariant}` : modelId; +} + +export async function fetchModelOverrides(): Promise { + const res = await authFetch(OVERRIDES_URL); + if (!res.ok) { + throw new Error( + await readFastApiError(res, "Failed to load saved model settings"), + ); + } + const body = (await res.json()) as { overrides?: ApiModelOverrides }; + return body.overrides ?? {}; +} + +/** + * Translate the UI's per-model config into the backend's schema. + * + * Only fields the user actually set are sent: the backend reads an absent field + * as "use the app default", so sending nulls would pin defaults and stop the + * model following later changes to the global preferences. `null` config means + * "no saved settings", which clears the entry. + */ +function toApiOverride(config: PerModelConfig | null): ApiModelOverride { + if (!config) { + return {}; + } + const payload: ApiModelOverride = {}; + if (config.maxSeqLength && config.maxSeqLength > 0) { + payload.max_seq_length = config.maxSeqLength; + } + if (config.customContextLength && config.customContextLength > 0) { + payload.custom_context_length = config.customContextLength; + } + if (config.kvCacheDtype) { + payload.kv_cache_dtype = config.kvCacheDtype; + } + if (config.speculativeType) { + payload.speculative_type = config.speculativeType; + } + if (config.specDraftNMax && config.specDraftNMax > 0) { + payload.spec_draft_n_max = config.specDraftNMax; + } + if (config.tensorParallel) { + payload.tensor_parallel = true; + } + if (config.chatTemplateOverride?.trim()) { + payload.chat_template_override = config.chatTemplateOverride; + } + // Only "manual" is a real override; "auto" is the follow-the-global default. + if (config.gpuMemoryMode === "manual") { + payload.gpu_memory_mode = "manual"; + } + // gpuLayers < 0 is Auto, which is also the default. + if (typeof config.gpuLayers === "number" && config.gpuLayers >= 0) { + payload.gpu_layers = config.gpuLayers; + } + if (typeof config.nCpuMoe === "number" && config.nCpuMoe > 0) { + payload.n_cpu_moe = config.nCpuMoe; + } + if (config.selectedGpuIds && config.selectedGpuIds.length > 0) { + payload.gpu_ids = config.selectedGpuIds; + } + return payload; +} + +export async function putModelOverride( + modelId: string, + ggufVariant: string | null | undefined, + config: PerModelConfig | null, +): Promise { + const res = await authFetch(OVERRIDES_URL, { + method: "PUT", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + // biome-ignore lint/style/useNamingConvention: API schema + model_id: modelOverrideKey(modelId, ggufVariant), + // Launch flags have no UI control, so the backend preserves them when the + // field is omitted. Forgetting a model means forgetting all of it, so that + // path sends an explicit empty list to clear them. + // biome-ignore lint/style/useNamingConvention: API schema + ...(config === null ? { llama_extra_args: [] } : {}), + ...toApiOverride(config), + }), + }); + if (!res.ok) { + throw new Error( + await readFastApiError(res, "Failed to save model settings for the API"), + ); + } +} + +/** + * Mirror a per-model config save to the backend without blocking the UI. + * + * Deliberately best-effort: the localStorage write is the source of truth for + * this browser and has already happened by the time this runs, so a failed sync + * must not fail the save or interrupt a model load. It is logged rather than + * toasted -- the only consequence is that an API-triggered load of this model + * falls back to app defaults until the next successful save. + */ +export function syncModelOverride( + modelId: string, + ggufVariant: string | null | undefined, + config: PerModelConfig | null, +): void { + void putModelOverride(modelId, ggufVariant, config).catch( + (error: unknown) => { + console.warn( + "Failed to mirror model settings to the server; an API load of this model will use defaults.", + error, + ); + }, + ); +} diff --git a/studio/frontend/src/features/model-picker/components/model-config-page.tsx b/studio/frontend/src/features/model-picker/components/model-config-page.tsx index 90202a2bcf..edf0cb96ea 100644 --- a/studio/frontend/src/features/model-picker/components/model-config-page.tsx +++ b/studio/frontend/src/features/model-picker/components/model-config-page.tsx @@ -55,6 +55,7 @@ import { resolveInitialConfig, savePerModelConfig, } from "../model-config/per-model-config"; +import { syncModelOverride } from "../api/model-overrides"; import { ChatTemplateEditorDialog } from "./chat-template-editor-dialog"; import type { ModelPickTarget } from "./model-selector/types"; import { @@ -578,6 +579,12 @@ interface ModelConfigPageProps { loadedContextLength?: number | null; initialConfig?: PerModelConfig | null; variant?: "page" | "sidebar"; + /** + * Page variant only: render the built-in "Run settings" title block. A host + * that already shows the model name as its own page heading (the Hub's + * settings page) turns this off so the name is not printed twice. + */ + showHeader?: boolean; } export function ModelConfigPage({ @@ -588,6 +595,7 @@ export function ModelConfigPage({ loadedContextLength = null, initialConfig = null, variant = "page", + showHeader = true, }: ModelConfigPageProps) { const rememberId = useId(); const isActiveModel = loadedConfig != null; @@ -870,6 +878,21 @@ export function ModelConfigPage({ } else { saveFailed = !deletePerModelConfig(target.id, target.ggufVariant); } + // Mirror to the server so an OpenAI-compatible API request that loads this + // model gets these exact settings, not app defaults. Best-effort and + // non-blocking: the localStorage write above already governs this browser. + // Forgetting clears the server entry too, so the two never disagree. + // + // Skipped when the local write failed (quota, a future-schema entry): the + // browser and the server would otherwise permanently disagree about this + // model, with no way for the user to tell which one the next load used. + if (!saveFailed) { + syncModelOverride( + target.id, + target.ggufVariant, + remember ? effectiveRuntimeConfig : null, + ); + } if (effectivePersistenceOnly) { if (saveFailed) { toast.error("Couldn't save settings for this model."); @@ -898,7 +921,7 @@ export function ModelConfigPage({ return (
- {variant === "page" && ( + {variant === "page" && showHeader && (
{onBack && ( - - {expanded ? ( -
-
-
-
- Prompt - {entry.prompt_truncated && !detail ? Preview : null} -
- 
-                {loading && !detail ? "Loading..." : prompt || "No prompt text"}
-
-
-
-
- Reply - {entry.reply_truncated && !detail ? Preview : null} -
- 
-                {loading && !detail ? "Loading..." : reply}
-
-
-
- -
- {formatTokens(entry)} - {entry.context_length ? ( - <> / {entry.context_length.toLocaleString()} context - ) : null} - -
-
- ) : null} - - ); -} - -export function ApiMonitorConsole(): ReactElement { - const [data, setData] = useState(null); - const [error, setError] = useState(null); - const [refreshing, setRefreshing] = useState(false); - const [expandedIds, setExpandedIds] = useState>(() => new Set()); - const [details, setDetails] = useState>({}); - const [loadingDetails, setLoadingDetails] = useState>( - () => new Set(), - ); - const loadingDetailsRef = useRef>(new Set()); - const detailsRef = useRef>({}); - - const loadMonitor = useCallback(async (): Promise => { - setRefreshing(true); - try { - setData(await getApiMonitor()); - setError(null); - } catch (err: unknown) { - setError(err instanceof Error ? err.message : "Monitor unavailable"); - } finally { - setRefreshing(false); - } - }, []); - - useEffect(() => { - let cancelled = false; - let timer: number | undefined; - - function schedule(): void { - timer = window.setTimeout(poll, 1500); - } - - function poll(): void { - getApiMonitor() - .then((next) => { - if (cancelled) { - return; - } - setData(next); - setError(null); - }) - .catch((err: unknown) => { - if (cancelled) { - return; - } - setError(err instanceof Error ? err.message : "Monitor unavailable"); - }) - .finally(() => { - if (!cancelled) { - schedule(); - } - }); - } - - poll(); - return () => { - cancelled = true; - if (timer !== undefined) { - window.clearTimeout(timer); - } - }; - }, []); - - const statusLabel = data?.status ?? "idle"; - const hasActive = (data?.active_requests ?? 0) > 0; - const entries = useMemo(() => data?.entries ?? [], [data]); - const loadDetail = useCallback( - (id: string): void => { - if (loadingDetailsRef.current.has(id)) { - return; - } - loadingDetailsRef.current.add(id); - setLoadingDetails((prev) => new Set(prev).add(id)); - getApiMonitorEntry(id) - .then((entry) => { - setDetails((prev) => { - const next = { ...prev, [id]: entry }; - detailsRef.current = next; - return next; - }); - }) - .catch(() => { - setDetails((prev) => { - const next = { ...prev }; - delete next[id]; - detailsRef.current = next; - return next; - }); - }) - .finally(() => { - loadingDetailsRef.current.delete(id); - setLoadingDetails((prev) => { - const next = new Set(prev); - next.delete(id); - return next; - }); - }); - }, - [], - ); - - const toggleEntry = useCallback( - (entry: ApiMonitorEntry): void => { - setExpandedIds((prev) => { - const next = new Set(prev); - if (next.has(entry.id)) { - next.delete(entry.id); - } else { - next.add(entry.id); - loadDetail(entry.id); - } - return next; - }); - }, - [loadDetail], - ); - - useEffect(() => { - for (const entry of entries) { - if (!expandedIds.has(entry.id)) { - continue; - } - const cached = detailsRef.current[entry.id]; - if (!cached || cached.status !== entry.status || entry.status === "running") { - loadDetail(entry.id); - } - } - }, [entries, expandedIds, loadDetail]); - - return ( -
-
-
-
- - {hasActive ? ( - - ) : null} -
-
-

- API monitor -

-

- {data?.active_model ?? "No model loaded"} -

-
-
-
-
- {statusLabel} -
- -
-
- -
- - {(data?.active_requests ?? 0).toLocaleString()} active /{" "} - {entries.length.toLocaleString()} recent - - {data?.context_length ? ( - {data.context_length.toLocaleString()} context - ) : null} -
- -
- {error ? ( -
- {error} -
- ) : entries.length === 0 ? ( -
- No API traffic yet -
- ) : ( -
- {entries.map((entry) => ( - toggleEntry(entry)} - /> - ))} -
- )} -
-
- ); -} diff --git a/studio/frontend/src/features/settings/components/monitor-link.tsx b/studio/frontend/src/features/settings/components/monitor-link.tsx new file mode 100644 index 0000000000..a0547fdc79 --- /dev/null +++ b/studio/frontend/src/features/settings/components/monitor-link.tsx @@ -0,0 +1,97 @@ +// SPDX-License-Identifier: AGPL-3.0-only +// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 + +// The monitor moved out of this tab and onto its own page. It is normally +// reached from the floating panel; this card is the way in from Settings. + +import { Switch } from "@/components/ui/switch"; +import { useApiMonitorOverlayStore } from "@/features/api-monitor"; +import { getApiMonitor } from "@/features/chat/api/chat-api"; +import type { ApiMonitorResponse } from "@/features/chat/types/api"; +import { cn } from "@/lib/utils"; +import { ActivityIcon, ArrowRight02Icon } from "@hugeicons/core-free-icons"; +import { HugeiconsIcon } from "@hugeicons/react"; +import { useNavigate } from "@tanstack/react-router"; +import { type ReactElement, useEffect, useState } from "react"; +import { useSettingsDialogStore } from "../stores/settings-dialog-store"; + +export function MonitorLink(): ReactElement { + const navigate = useNavigate(); + const [data, setData] = useState(null); + const autoOpen = useApiMonitorOverlayStore((s) => s.autoOpen); + const setAutoOpen = useApiMonitorOverlayStore((s) => s.setAutoOpen); + + // One snapshot, not a poll: the live view is the monitor page. + useEffect(() => { + let cancelled = false; + void getApiMonitor() + .then((next) => { + if (!cancelled) setData(next); + }) + .catch(() => undefined); + return () => { + cancelled = true; + }; + }, []); + + const active = data?.active_requests ?? 0; + const recent = data?.entries.length ?? 0; + + return ( +
+ + + {/* Where the panel's own "stop opening this" gets turned back on. */} +
+ + + Show the floating monitor automatically + + + Opens a small panel when API traffic arrives. + + + +
+
+ ); +} diff --git a/studio/frontend/src/features/settings/tabs/api-keys-tab.tsx b/studio/frontend/src/features/settings/tabs/api-keys-tab.tsx index f1e503f7a3..39d934aacd 100644 --- a/studio/frontend/src/features/settings/tabs/api-keys-tab.tsx +++ b/studio/frontend/src/features/settings/tabs/api-keys-tab.tsx @@ -14,7 +14,7 @@ import { translate, useT } from "@/i18n"; import { AnimatePresence, motion, useReducedMotion } from "motion/react"; import { useCallback, useEffect, useState } from "react"; import { fetchApiKeys, revokeApiKey, type ApiKey } from "../api/api-keys"; -import { ApiMonitorConsole } from "../components/api-monitor-console"; +import { MonitorLink } from "../components/monitor-link"; import { ApiKeyRow } from "../components/api-key-row"; import { CreateKeyForm } from "../components/create-key-form"; import { ModelAutoSwitchSection } from "../components/model-auto-switch-section"; @@ -168,7 +168,7 @@ export function ApiKeysTab() { )} - +