diff --git a/studio/backend/core/inference/api_monitor.py b/studio/backend/core/inference/api_monitor.py index f76a38576f..f9922a711a 100644 --- a/studio/backend/core/inference/api_monitor.py +++ b/studio/backend/core/inference/api_monitor.py @@ -269,9 +269,20 @@ class ApiMonitor: if entry.status == "running" and (subject is None or entry.subject == subject) ) - def clear(self) -> None: + def clear(self, *, subject: Optional[str] = None) -> None: + """Drop recorded entries. ``subject`` limits the wipe to one caller's. + + Every other read on this class is subject-scoped, so an unscoped clear + would let one user erase another's history (and zero their active count + mid-generation). Callers that genuinely mean "everything" pass None. + """ with self._lock: - self._entries.clear() + if subject is None: + self._entries.clear() + return + self._entries = deque( + entry for entry in self._entries if entry.subject != subject + ) def _find_locked(self, entry_id: str) -> Optional[ApiMonitorEntry]: for entry in self._entries: diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.py index 9d40650a56..d529a1cd85 100644 --- a/studio/backend/routes/inference.py +++ b/studio/backend/routes/inference.py @@ -3581,6 +3581,7 @@ async def _maybe_auto_switch_model( get_openai_auto_switch_enabled, get_auto_unload_idle_seconds, get_model_override, + model_override_load_kwargs, ) from core.inference.local_model_resolver import resolve_local_gguf from core.inference.llama_keepwarm import ( @@ -3716,13 +3717,35 @@ async def _maybe_auto_switch_model( if _already_serving(): _record_serving_alias() return - # Apply this model's saved launch flags so the swap honors the config. - override = get_model_override(override_id) + # Apply this model's saved launch config so an API-driven swap + # loads it exactly as the picker would: context, KV dtype, + # speculative decoding, chat template and GPU placement, not + # just the two legacy flags. Look the config up under the + # variant-qualified id first (two quants of one repo can carry + # different configs), then the bare advertised id, then the + # concrete load path, so a config saved under any of the names + # this model is known by is found. + override = {} + for override_key in ( + f"{override_id}:{variant}" if variant else None, + override_id, + target_id, + ): + if not override_key: + continue + override = get_model_override(override_key) + if override: + break load_kwargs = {"model_path": target_id, "gguf_variant": variant} - if override.get("llama_extra_args") is not None: - load_kwargs["llama_extra_args"] = override["llama_extra_args"] - if override.get("max_seq_length") is not None: - load_kwargs["max_seq_length"] = override["max_seq_length"] + load_kwargs.update( + model_override_load_kwargs( + override, + # variant is set for every GGUF the resolver returns; the + # reload-stash path carries the quant it froze. + is_gguf = bool(variant) + or target_id.lower().endswith(".gguf"), + ) + ) # Reuse the load impl so its dedup, tensor fallback, and threading # apply. Call the impl directly: we already hold the lifecycle gate # the /load route would otherwise take, so the route would deadlock. @@ -5703,6 +5726,22 @@ async def get_api_monitor(current_subject: str = Depends(get_current_subject)): } +@studio_router.delete("/monitor") +async def clear_api_monitor(current_subject: str = Depends(get_current_subject)): + """Drop this caller's recorded API history so a debugging session starts clean. + + Scoped to the current subject, like every read on the monitor: an unscoped + wipe would erase another user's history and zero their active-request count + while their generation is still streaming. + + The caller's own in-flight requests are dropped from the log too; they keep + streaming to their client, they just stop being reported here (a later append + re-adds nothing, since the entry id no longer resolves). + """ + api_monitor.clear(subject = current_subject) + return {"cleared": True} + + @studio_router.get("/monitor/{entry_id}") async def get_api_monitor_entry(entry_id: str, current_subject: str = Depends(get_current_subject)): """Return full prompt/reply details for one OpenAI-compatible API request.""" diff --git a/studio/backend/routes/settings.py b/studio/backend/routes/settings.py index fef18a9145..2323d33915 100644 --- a/studio/backend/routes/settings.py +++ b/studio/backend/routes/settings.py @@ -34,6 +34,7 @@ from utils.helper_precache_settings import ( helper_model_disabled_by_env, set_helper_precache_enabled, ) +from picker.schemas import MAX_CHAT_TEMPLATE_BYTES from utils.coding_agents import CODING_AGENTS, detect_installed_coding_agents from utils.openai_auto_switch_settings import ( DEFAULT_AUTO_UNLOAD_KEEP_KV, @@ -126,11 +127,51 @@ class OpenAIAutoSwitchResponse(BaseModel): class ModelOverridePayload(BaseModel): - model_id: str = Field(..., min_length = 1) - llama_extra_args: list[str] = Field(default_factory = list) + """One model's saved launch config, applied when the API loads that model. + + Everything past ``model_id`` is optional and omitted means "app default", so a + payload carrying only ``model_id`` clears the entry. The bounds here mirror + ``LoadRequest`` so a bad value is rejected at the boundary instead of being + silently dropped by the normalizer; the enum-ish fields (KV dtype, speculative + mode) are left to it, since their valid sets follow the llama.cpp build. + """ + + model_id: str = Field(..., min_length = 1, max_length = 512) + # None means "leave the stored value alone": the settings UI has no control + # for launch flags, so a save from it must not wipe flags set through this + # API. An explicit [] clears them (that is how "forget this model" arrives). + llama_extra_args: Optional[list[str]] = None # ge=1: 0 is not a valid sequence length, and the setter drops a falsy value, # so reject it at the boundary instead of accepting then silently discarding it. max_seq_length: Optional[int] = Field(default = None, ge = 1, le = 1048576) + custom_context_length: Optional[int] = Field(default = None, ge = 1, le = 1048576) + kv_cache_dtype: Optional[str] = Field(default = None, max_length = 32) + speculative_type: Optional[str] = Field(default = None, max_length = 32) + spec_draft_n_max: Optional[int] = Field(default = None, ge = 1, le = 16) + tensor_parallel: bool = False + # Validated in bytes below, not by max_length: pydantic counts characters, + # so a multi-byte template could pass here and then be silently dropped by + # the normalizer (which measures UTF-8) while the request still returned 200. + chat_template_override: Optional[str] = None + gpu_memory_mode: Optional[Literal["auto", "manual"]] = None + # -1 is Auto (llama.cpp --fit sizes the offload); the normalizer treats it as unset. + gpu_layers: Optional[int] = Field(default = None, ge = -1, le = 1024) + n_cpu_moe: Optional[int] = Field(default = None, ge = 0, le = 1024) + gpu_ids: Optional[list[int]] = None + + + @field_validator("chat_template_override") + @classmethod + def _limit_chat_template_bytes(cls, value: Optional[str]) -> Optional[str]: + # Mirrors LoadRequest.normalize_blank_chat_template_override so the same + # template is accepted or rejected identically on both paths. + if value is None: + return None + if len(value.encode("utf-8")) > MAX_CHAT_TEMPLATE_BYTES: + raise ValueError( + f"Chat template exceeds the {MAX_CHAT_TEMPLATE_BYTES}-byte limit." + ) + return value class ModelOverridesResponse(BaseModel): @@ -289,12 +330,40 @@ def update_openai_auto_switch_override( payload: ModelOverridePayload, current_subject: str = Depends(get_current_subject) ) -> ModelOverridesResponse: from core.inference.llama_server_args import validate_extra_args + from utils.openai_auto_switch_settings import get_model_override try: - extra_args = validate_extra_args(payload.llama_extra_args) + # A payload carrying only model_id is the documented "remove", so it + # wipes everything. Otherwise it is a real save, and omitted launch flags + # are carried over from the stored entry (the settings UI cannot express + # them and must not delete them). + requested_extra_args = payload.llama_extra_args + saved_fields = payload.model_dump( + exclude = {"model_id", "llama_extra_args"}, exclude_none = True + ) + is_removal = not payload.tensor_parallel and not { + key: value + for key, value in saved_fields.items() + if key != "tensor_parallel" + } + if requested_extra_args is None and not is_removal: + requested_extra_args = get_model_override(payload.model_id).get( + "llama_extra_args" + ) + extra_args = validate_extra_args(requested_extra_args) set_model_override( payload.model_id, llama_extra_args = extra_args, max_seq_length = payload.max_seq_length, + custom_context_length = payload.custom_context_length, + kv_cache_dtype = payload.kv_cache_dtype, + speculative_type = payload.speculative_type, + spec_draft_n_max = payload.spec_draft_n_max, + tensor_parallel = payload.tensor_parallel, + chat_template_override = payload.chat_template_override, + gpu_memory_mode = payload.gpu_memory_mode, + gpu_layers = payload.gpu_layers, + n_cpu_moe = payload.n_cpu_moe, + gpu_ids = payload.gpu_ids, ) except ValueError as exc: raise log_and_http_error( diff --git a/studio/backend/tests/test_api_monitor.py b/studio/backend/tests/test_api_monitor.py index 56bc404350..a8d0e1e832 100644 --- a/studio/backend/tests/test_api_monitor.py +++ b/studio/backend/tests/test_api_monitor.py @@ -258,3 +258,34 @@ def test_api_monitor_append_reply_exact_cap_then_more_marks_truncated(): monitor.append_reply(entry_id, "y") reply = monitor.snapshot()[0]["reply"] assert len(reply) == m._MAX_REPLY_CHARS and reply.endswith("...") + + +def test_api_monitor_clear_is_scoped_to_one_subject(): + # Every other read on the monitor is subject-scoped. An unscoped clear from + # the route would let one caller erase another's history and zero their + # active count in the middle of a generation. + monitor = ApiMonitor(max_entries = 4) + alice = monitor.start( + endpoint = "/v1/chat/completions", + method = "POST", + model = "m", + prompt = "alice prompt", + subject = "alice", + ) + bob = monitor.start( + endpoint = "/v1/chat/completions", + method = "POST", + model = "m", + prompt = "bob prompt", + subject = "bob", + ) + + monitor.clear(subject = "alice") + assert monitor.snapshot(subject = "alice") == [] + assert [entry["id"] for entry in monitor.snapshot(subject = "bob")] == [bob] + assert monitor.active_count(subject = "bob") == 1 + assert monitor.get(alice, subject = "alice") is None + + # Passing no subject is the explicit "everything" path. + monitor.clear() + assert monitor.snapshot(subject = "bob") == [] diff --git a/studio/backend/tests/test_openai_auto_switch.py b/studio/backend/tests/test_openai_auto_switch.py index 9c6c20e6b6..3899b1444a 100644 --- a/studio/backend/tests/test_openai_auto_switch.py +++ b/studio/backend/tests/test_openai_auto_switch.py @@ -3819,3 +3819,177 @@ def test_env_idle_below_floor_is_clamped(monkeypatch): assert settings.get_auto_unload_idle_seconds() == 600 monkeypatch.delenv(settings.MODEL_IDLE_TTL_ENV_VAR) assert settings.get_auto_unload_idle_seconds() == 0 + + +# --------------------------------------------------------------------------- +# Per-model launch config: normalization, LoadRequest mapping, key resolution. +# --------------------------------------------------------------------------- + + +def test_normalize_model_override_drops_unusable_fields_and_keeps_the_rest(): + # A stale field must not cost the user the whole config, so bad values are + # dropped one by one rather than rejecting the payload. + entry = settings.normalize_model_override( + { + "max_seq_length": 8192, + "kv_cache_dtype": "not_a_dtype", + "speculative_type": "mtp", + "spec_draft_n_max": 999, # out of range for the MTP draft count + "gpu_memory_mode": "auto", # only "manual" is a real override + "gpu_layers": -1, # -1 is Auto, which is already the default + "n_cpu_moe": 0, + "gpu_ids": [1, 1, 0, "2", -5], + "tensor_parallel": False, + "llama_extra_args": [], + } + ) + assert entry == { + "max_seq_length": 8192, + "speculative_type": "mtp", + "gpu_ids": [1, 0, 2], + } + + +def test_normalize_model_override_rejects_oversized_chat_template(): + small = settings.normalize_model_override({"chat_template_override": "{{ bos }}"}) + assert small["chat_template_override"] == "{{ bos }}" + # The limit is bytes, not characters: a multi-byte template just under the + # character limit can still be over the byte limit. + huge = "é" * settings.MAX_CHAT_TEMPLATE_OVERRIDE_BYTES + assert "chat_template_override" not in settings.normalize_model_override( + {"chat_template_override": huge} + ) + + +def test_spec_draft_n_max_only_stored_for_mtp_modes(): + mtp = settings.normalize_model_override( + {"speculative_type": "mtp", "spec_draft_n_max": 4} + ) + assert mtp["spec_draft_n_max"] == 4 + # A non-MTP mode ignores the draft count at load time, so storing it would + # show the user an edit that never takes effect. + ngram = settings.normalize_model_override( + {"speculative_type": "ngram", "spec_draft_n_max": 4} + ) + assert "spec_draft_n_max" not in ngram + + +def test_resolve_fit_max_seq_length_hands_sizing_to_fit_under_manual_auto_layers(): + # Manual GPU memory with Auto layers means llama.cpp --fit owns the context, + # so the load sends the context pin (or 0), not the stored max seq length. + override = {"gpu_memory_mode": "manual", "max_seq_length": 8192} + assert settings.resolve_fit_max_seq_length(override, is_gguf = True) == 0 + assert ( + settings.resolve_fit_max_seq_length( + {**override, "custom_context_length": 4096}, is_gguf = True + ) + == 4096 + ) + # Pinning the layer count takes --fit back out of the picture. + assert ( + settings.resolve_fit_max_seq_length({**override, "gpu_layers": 20}, is_gguf = True) + == 8192 + ) + # Not a GGUF, so none of this applies. + assert settings.resolve_fit_max_seq_length(override, is_gguf = False) == 8192 + + +def test_model_override_load_kwargs_gates_gpu_placement_on_gguf(): + override = { + "max_seq_length": 4096, + "kv_cache_dtype": "q8_0", + "tensor_parallel": True, + "gpu_memory_mode": "manual", + "gpu_layers": 20, + "n_cpu_moe": 3, + "gpu_ids": [0, 1], + } + gguf = settings.model_override_load_kwargs(override, is_gguf = True) + assert gguf["cache_type_kv"] == "q8_0" + assert gguf["tensor_parallel"] is True + assert gguf["gpu_layers"] == 20 + assert gguf["gpu_ids"] == [0, 1] + + # A safetensors model loads through HF auto-placement; inheriting a GGUF GPU + # pin here would silently change where the weights land. + safetensors = settings.model_override_load_kwargs(override, is_gguf = False) + assert safetensors["max_seq_length"] == 4096 + assert "gpu_layers" not in safetensors + assert "gpu_ids" not in safetensors + assert "n_cpu_moe" not in safetensors + assert "gpu_memory_mode" not in safetensors + + # Every key it produces has to be a real LoadRequest field, or the load call + # raises TypeError at the moment the user's request arrives. + LoadRequest(model_path = "unsloth/B-GGUF", **gguf) + + +def test_auto_switch_prefers_variant_qualified_override(monkeypatch): + # Settings are saved per quant, so Q4_K_M and Q8_0 of the same repo are + # different entries; the bare repo id is only the fallback. + backend = _FakeBackend(None) + rec = _LoadRecorder(backend) + _wire( + monkeypatch, + enabled = True, + resolves_to = ("unsloth/B-GGUF", "Q4_K_M", "unsloth/B-GGUF"), + backend = backend, + recorder = rec, + ) + stored = { + "unsloth/B-GGUF": {"max_seq_length": 1024}, + "unsloth/B-GGUF:Q4_K_M": {"max_seq_length": 8192, "gpu_layers": 20}, + } + monkeypatch.setattr(settings, "get_model_override", lambda mid: stored.get(mid, {})) + + _run_hook("unsloth/B-GGUF") + req = rec.calls[0] + assert req.max_seq_length == 8192 + assert req.gpu_layers == 20 + + +def test_auto_switch_falls_back_to_bare_repo_override(monkeypatch): + backend = _FakeBackend(None) + rec = _LoadRecorder(backend) + _wire( + monkeypatch, + enabled = True, + resolves_to = ("unsloth/B-GGUF", "Q4_K_M", "unsloth/B-GGUF"), + backend = backend, + recorder = rec, + ) + stored = {"unsloth/B-GGUF": {"max_seq_length": 1024}} + monkeypatch.setattr(settings, "get_model_override", lambda mid: stored.get(mid, {})) + + _run_hook("unsloth/B-GGUF") + assert rec.calls[0].max_seq_length == 1024 + + +def test_override_route_preserves_launch_flags_across_a_settings_only_update(monkeypatch): + # The settings page has no control for llama_extra_args, so saving from it + # omits the field. Omitted must mean "leave it alone", or every save from the + # UI would quietly wipe flags set elsewhere. + import routes.settings as settings_route + + _mock_override_store(monkeypatch) + settings_route.update_openai_auto_switch_override( + settings_route.ModelOverridePayload( + model_id = "unsloth/B-GGUF", llama_extra_args = ["--flash-attn"] + ), + "tester", + ) + resp = settings_route.update_openai_auto_switch_override( + settings_route.ModelOverridePayload(model_id = "unsloth/B-GGUF", max_seq_length = 4096), + "tester", + ) + entry = resp.overrides["unsloth/B-GGUF"] + assert entry["llama_extra_args"] == ["--flash-attn"] + assert entry["max_seq_length"] == 4096 + + # An explicit empty list is how the UI says "forget this model", and with no + # other fields left that removes the entry outright. + gone = settings_route.update_openai_auto_switch_override( + settings_route.ModelOverridePayload(model_id = "unsloth/B-GGUF", llama_extra_args = []), + "tester", + ) + assert "unsloth/B-GGUF" not in gone.overrides diff --git a/studio/backend/utils/openai_auto_switch_settings.py b/studio/backend/utils/openai_auto_switch_settings.py index 7007440f4c..413d75ad1e 100644 --- a/studio/backend/utils/openai_auto_switch_settings.py +++ b/studio/backend/utils/openai_auto_switch_settings.py @@ -210,8 +210,211 @@ def set_openai_auto_switch( ) +# --- Per-model launch config ------------------------------------------------- +# +# An override is the server-side twin of the UI's per-model config (the browser +# localStorage map behind features/model-picker/model-config). The UI mirrors +# every save here so a model loaded by an OpenAI-compatible API request gets the +# same launch settings a user would get loading it from the picker; without this +# the API path could only ever apply the two legacy fields below. +# +# Legacy entries hold just {llama_extra_args, max_seq_length}; every field is +# optional and absent means "fall back to the app default", so old entries keep +# loading correctly. A write is a full replace of the fields it expresses, so the +# route carries `llama_extra_args` over when the payload omits it (the settings +# UI has no control for launch flags and must not wipe them). +# +# Known gap: the picker resolves a couple of knobs as "per-model value, else the +# user's global preference" -- GPU memory mode and speculative decoding, whose +# globals live in browser localStorage. An override deliberately stores only an +# explicit per-model choice (so the model keeps following later global changes), +# and the server cannot see the globals at all. So for a model that follows the +# global on one of those two, an API load falls back to the app default rather +# than the user's global. Every other field matches the picker exactly. + +# Mirrors _valid_cache_types in core/inference/llama_cpp.py. +VALID_KV_CACHE_DTYPES = frozenset( + {"f16", "bf16", "q8_0", "q4_0", "q4_1", "q5_0", "q5_1", "iq4_nl", "f32"} +) +# Canonical values plus the legacy spellings LoadRequest still accepts. +VALID_SPECULATIVE_TYPES = frozenset( + { + "auto", + "mtp", + "ngram", + "mtp+ngram", + "off", + "default", + "draft-mtp", + "ngram-mod", + "ngram-simple", + } +) +# Only these two consume spec_draft_n_max (mirrors MTP_SPECULATIVE_TYPES in the UI). +MTP_SPECULATIVE_TYPES = frozenset({"mtp", "mtp+ngram", "draft-mtp"}) +VALID_GPU_MEMORY_MODES = frozenset({"auto", "manual"}) + +MAX_SEQ_LENGTH_CEILING = 1048576 +MAX_CHAT_TEMPLATE_OVERRIDE_BYTES = 65_536 + + +def _clean_str(value: Any, allowed: frozenset[str]) -> Optional[str]: + if not isinstance(value, str): + return None + normalized = value.strip().lower() + return normalized if normalized in allowed else None + + +def _bounded_int(value: Any, *, minimum: int, maximum: int) -> Optional[int]: + try: + parsed = int(value) + except (TypeError, ValueError): + return None + if parsed < minimum or parsed > maximum: + return None + return parsed + + +def normalize_model_override(payload: dict[str, Any]) -> dict[str, Any]: + """Validate one per-model launch config, dropping anything unusable. + + Silently drops rather than raising: an override is a convenience mirror of the + UI's config, so one stale field (a KV dtype this llama.cpp build lost, a GPU id + from another host) must not block persisting the rest or fail the API load that + reads it. ``validate_extra_args`` is the caller's job -- it lives in the + llama_server_args allow-list module, which this one must not import. + """ + entry: dict[str, Any] = {} + + extra_args = payload.get("llama_extra_args") + if isinstance(extra_args, (list, tuple)) and extra_args: + entry["llama_extra_args"] = [str(arg) for arg in extra_args] + + # 0 / negative means "unset"; the loader reads absence as the app default. + for key in ("max_seq_length", "custom_context_length"): + parsed = _bounded_int(payload.get(key), minimum = 1, maximum = MAX_SEQ_LENGTH_CEILING) + if parsed: + entry[key] = parsed + + kv_cache_dtype = _clean_str(payload.get("kv_cache_dtype"), VALID_KV_CACHE_DTYPES) + if kv_cache_dtype: + entry["kv_cache_dtype"] = kv_cache_dtype + + speculative_type = _clean_str(payload.get("speculative_type"), VALID_SPECULATIVE_TYPES) + if speculative_type: + entry["speculative_type"] = speculative_type + # Only meaningful for the MTP modes; storing it otherwise would resurface + # in the UI as an edit the loader silently ignores. + if speculative_type in MTP_SPECULATIVE_TYPES: + spec_draft_n_max = _bounded_int( + payload.get("spec_draft_n_max"), minimum = 1, maximum = 16 + ) + if spec_draft_n_max: + entry["spec_draft_n_max"] = spec_draft_n_max + + if _coerce_bool(payload.get("tensor_parallel")): + entry["tensor_parallel"] = True + + template = payload.get("chat_template_override") + if isinstance(template, str) and template.strip(): + if len(template.encode("utf-8")) <= MAX_CHAT_TEMPLATE_OVERRIDE_BYTES: + entry["chat_template_override"] = template + + # Only "manual" is a real override: persisting "auto" would pin the model and + # stop it following later changes to the global GPU memory preference. + if _clean_str(payload.get("gpu_memory_mode"), VALID_GPU_MEMORY_MODES) == "manual": + entry["gpu_memory_mode"] = "manual" + + # -1 is Auto (llama.cpp --fit owns layer sizing), which is also the default, + # so only a pinned count >= 0 is worth storing. + gpu_layers = _bounded_int(payload.get("gpu_layers"), minimum = 0, maximum = 1024) + if gpu_layers is not None: + entry["gpu_layers"] = gpu_layers + + n_cpu_moe = _bounded_int(payload.get("n_cpu_moe"), minimum = 1, maximum = 1024) + if n_cpu_moe: + entry["n_cpu_moe"] = n_cpu_moe + + gpu_ids = payload.get("gpu_ids") + if isinstance(gpu_ids, (list, tuple)) and gpu_ids: + # De-duplicate, preserving order: resolve_requested_gpu_ids rejects a + # repeated id outright, so storing [0, 0] would make every later API load + # of this model fail with a 400 that the picker never hits. + cleaned_ids: list[int] = [] + for gid in gpu_ids: + parsed = _bounded_int(gid, minimum = 0, maximum = 1024) + if parsed is not None and parsed not in cleaned_ids: + cleaned_ids.append(parsed) + if cleaned_ids: + entry["gpu_ids"] = cleaned_ids + + return entry + + +def resolve_fit_max_seq_length(override: dict[str, Any], *, is_gguf: bool) -> Optional[int]: + """The ``max_seq_length`` an API load should send for this override. + + Mirrors resolveFitMaxSeqLength in the UI (features/chat/presets/preset-policy.ts): + under Manual GPU memory with Auto layers, llama.cpp's ``--fit`` owns context + sizing, so the load sends the explicit context pin (or 0 to hand sizing over) + rather than the stored max sequence length. Returns None to leave the field + at the loader's default. + """ + manual_auto_layers = ( + is_gguf + and override.get("gpu_memory_mode") == "manual" + and override.get("gpu_layers") is None + ) + if manual_auto_layers: + return override.get("custom_context_length") or 0 + # max_seq_length wins where both are set. The UI only ever sends it for a + # non-GGUF model (a GGUF's context is `custom_context_length`), so in + # practice the two never collide from that path; a hand-written or legacy + # entry that sets it on a GGUF is honoured, which is this API's contract. + return override.get("max_seq_length") or override.get("custom_context_length") + + +def model_override_load_kwargs(override: dict[str, Any], *, is_gguf: bool) -> dict[str, Any]: + """Map a stored per-model config onto ``LoadRequest`` keyword arguments. + + Mirrors the UI's load payload (features/chat/api/chat-adapter.ts) so an API + auto-switch load and a picker load of the same model produce the same command + line. GPU placement is GGUF-only there, so it is gated the same way here: a + safetensors model loads through HF auto-placement and must not inherit a + hidden GGUF GPU pin. + """ + if not override: + return {} + kwargs: dict[str, Any] = {} + + max_seq_length = resolve_fit_max_seq_length(override, is_gguf = is_gguf) + if max_seq_length is not None: + kwargs["max_seq_length"] = max_seq_length + for source, target in ( + ("llama_extra_args", "llama_extra_args"), + ("kv_cache_dtype", "cache_type_kv"), + ("speculative_type", "speculative_type"), + ("spec_draft_n_max", "spec_draft_n_max"), + ("tensor_parallel", "tensor_parallel"), + ("chat_template_override", "chat_template_override"), + ): + if override.get(source) is not None: + kwargs[target] = override[source] + + if is_gguf: + if override.get("gpu_memory_mode") is not None: + kwargs["gpu_memory_mode"] = override["gpu_memory_mode"] + if override.get("gpu_layers") is not None: + kwargs["gpu_layers"] = override["gpu_layers"] + if override.get("n_cpu_moe") is not None: + kwargs["n_cpu_moe"] = override["n_cpu_moe"] + if override.get("gpu_ids") is not None: + kwargs["gpu_ids"] = override["gpu_ids"] + return kwargs + + def get_model_overrides() -> dict[str, dict]: - """Per-model launch overrides keyed by model id ({llama_extra_args, max_seq_length}).""" + """Per-model launch configs keyed by model id (see normalize_model_override).""" raw = _cached_setting(MODEL_OVERRIDES_SETTING_KEY, None) return raw if isinstance(raw, dict) else {} @@ -226,15 +429,22 @@ def set_model_override( model_id: str, llama_extra_args: Optional[list[str]] = None, max_seq_length: Optional[int] = None, + **config: Any, ) -> dict: - """Upsert one model's launch override; an override with no fields removes it.""" + """Upsert one model's launch config; a config with no usable fields removes it. + + The two legacy parameters stay positional for existing callers; every other + per-model field is passed by keyword and normalized together. + """ if not model_id or not model_id.strip(): raise ValueError("model_id is required.") - entry: dict[str, Any] = {} - if llama_extra_args: - entry["llama_extra_args"] = [str(arg) for arg in llama_extra_args] - if max_seq_length: - entry["max_seq_length"] = max(0, int(max_seq_length)) + entry = normalize_model_override( + { + **config, + "llama_extra_args": llama_extra_args, + "max_seq_length": max_seq_length, + } + ) from storage.studio_db import upsert_app_setting_map_entry diff --git a/studio/frontend/src/app/router.tsx b/studio/frontend/src/app/router.tsx index 586c03d5df..3dd8d2e8e0 100644 --- a/studio/frontend/src/app/router.tsx +++ b/studio/frontend/src/app/router.tsx @@ -6,6 +6,7 @@ import { Button } from "@/components/ui/button"; import { MascotImg } from "@/components/mascot-img"; import { useT } from "@/i18n"; import { Route as rootRoute } from "./routes/__root"; +import { Route as apiMonitorRoute } from "./routes/api"; import { Route as dataRecipesRoute } from "./routes/data-recipes"; import { Route as dataRecipeRoute } from "./routes/data-recipes.$recipeId"; import { Route as chatRoute } from "./routes/chat"; @@ -32,6 +33,7 @@ const routeTree = rootRoute.addChildren([ exportRoute, dataRecipesRoute, dataRecipeRoute, + apiMonitorRoute, ]); function DefaultNotFound() { diff --git a/studio/frontend/src/app/routes/__root.tsx b/studio/frontend/src/app/routes/__root.tsx index 7137fd6f96..5dbeefa8f8 100644 --- a/studio/frontend/src/app/routes/__root.tsx +++ b/studio/frontend/src/app/routes/__root.tsx @@ -5,16 +5,14 @@ import { AppSidebar } from "@/components/app-sidebar"; import { Navbar } from "@/components/navbar"; import { fetchDeviceType, usePlatformStore } from "@/config/env"; import { SidebarInset, SidebarProvider } from "@/components/ui/sidebar"; -import { - SettingsDialog, - useSettingsDialogStore, -} from "@/features/settings"; +import { SettingsDialog, useSettingsDialogStore } from "@/features/settings"; import { ChatPage, clearNewChatDraft, useChatRuntimeStore, type ChatSearch, } from "@/features/chat"; +import { ApiMonitorOverlay } from "@/features/api-monitor/api-monitor-overlay"; import { RemoteCodeConsentDialog } from "@/features/security"; import { HfTokenWarningDialog } from "@/features/hf-auth"; import { TransformersUpgradeDialog } from "@/features/transformers-upgrade"; @@ -33,13 +31,7 @@ import { useRouterState, } from "@tanstack/react-router"; import { AnimatePresence, motion } from "motion/react"; -import { - Suspense, - useEffect, - useLayoutEffect, - useMemo, - useState, -} from "react"; +import { Suspense, useEffect, useLayoutEffect, useMemo, useState } from "react"; import { AppProvider } from "../provider"; declare module "@tanstack/react-router" { @@ -76,11 +68,17 @@ const CHAT_ONLY_ALLOWED = new Set([ // Export stays reachable on chat-only hosts so the page can show its own grayed-out reason // instead of a silent redirect; it self-gates via export capability, so nothing runs. "/export", + // Chat-only hosts (Intel Macs, Apple Silicon without MLX, no-GPU boxes) serve + // the OpenAI-compatible API exactly like any other host, so the monitor has to + // be reachable there. Without this the floating panel's own "Expand" button + // and the Settings > API card both redirect to /chat. + "/api-monitor", ]); function isChatOnlyAllowed(pathname: string): boolean { if (CHAT_ONLY_ALLOWED.has(pathname)) return true; - if (pathname === "/data-recipes" || pathname.startsWith("/data-recipes/")) return true; + if (pathname === "/data-recipes" || pathname.startsWith("/data-recipes/")) + return true; return false; } @@ -224,6 +222,8 @@ function RootLayout() { {!isAuthFlowRoute && } + {/* Opens itself when API traffic arrives; hides on the full monitor page. */} + {!isAuthFlowRoute && } @@ -241,7 +241,9 @@ function RootLayout() { className="!min-h-0 h-[calc(100dvh-var(--studio-titlebar-height,0px))] overflow-hidden" > - +
import("@/features/api-monitor"), + "ApiMonitorPage", +); + +export const Route = createRoute({ + getParentRoute: () => rootRoute, + // Not "/api": the backend owns that prefix (and "/v1"), and its SPA fallback + // deliberately 404s those paths so API clients get an API-shaped error rather + // than an HTML page. A deep link to /api would never reach the router. + path: "/api-monitor", + staticData: { title: "API" }, + beforeLoad: () => requireAuth(), + component: ApiMonitorPage, +}); diff --git a/studio/frontend/src/features/api-monitor/api-monitor-overlay.tsx b/studio/frontend/src/features/api-monitor/api-monitor-overlay.tsx new file mode 100644 index 0000000000..4411e052ad --- /dev/null +++ b/studio/frontend/src/features/api-monitor/api-monitor-overlay.tsx @@ -0,0 +1,384 @@ +// SPDX-License-Identifier: AGPL-3.0-only +// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 + +// Floating API monitor: opens itself when API traffic arrives, summarises it +// without taking over the window, and links through to the full page. + +import { getApiMonitor } from "@/features/chat/api/chat-api"; +import type { ApiMonitorEntry } from "@/features/chat/types/api"; +import { cn } from "@/lib/utils"; +import { + ArrowExpand01Icon, + DragDropVerticalIcon, + Globe02Icon, +} from "@hugeicons/core-free-icons"; +import { HugeiconsIcon } from "@hugeicons/react"; +import { useNavigate, useRouterState } from "@tanstack/react-router"; +import { XIcon } from "lucide-react"; +import { AnimatePresence, motion, useDragControls } from "motion/react"; +import { + type PointerEvent, + type ReactElement, + useEffect, + useMemo, + useRef, + useState, +} from "react"; +import { useApiMonitorOverlayStore } from "./overlay-store"; +import { computeStats } from "./use-api-monitor"; + +// Live cadence while the panel is on screen. +const OPEN_POLL_MS = 1500; +// While closed the poll only has to notice that traffic started, so it backs off. +const IDLE_POLL_MS = 5000; +// Requests shown in the panel; the rest are one click away on the full page. +const VISIBLE_ENTRIES = 4; +// How long the API must be quiet before a dismissed panel will open itself again. +const REARM_QUIET_MS = 60_000; + +const API_INFERENCE_PREFIX_RE = /^\/api\/inference/; +const V1_PREFIX_RE = /^\/v1\//; + +function compactEndpoint(endpoint: string): string { + return endpoint + .replace(API_INFERENCE_PREFIX_RE, "/api") + .replace(V1_PREFIX_RE, "/"); +} + +function formatDuration(value?: number | null): string { + if (value == null) { + return "live"; + } + if (value < 1000) { + return `${Math.round(value)}ms`; + } + return `${(value / 1000).toFixed(value < 10000 ? 1 : 0)}s`; +} + +function statusDotClass(status: ApiMonitorEntry["status"]): string { + switch (status) { + case "running": + return "bg-blue-500 animate-pulse"; + case "error": + return "bg-red-500"; + case "cancelled": + return "bg-amber-500"; + default: + return "bg-emerald-500"; + } +} + +function StatCell({ + label, + value, + tone, +}: { + label: string; + value: string; + tone?: "error" | "active"; +}): ReactElement { + return ( +
+ + {value} + + {/* Sentence case: Unsloth metric rows read as words, not headers. */} + + {label} + +
+ ); +} + +export function ApiMonitorOverlay(): ReactElement | null { + const { isOpen, suppressed, autoOpen, open, close, setAutoOpen } = + useApiMonitorOverlayStore(); + const navigate = useNavigate(); + const pathname = useRouterState({ select: (s) => s.location.pathname }); + const onFullPage = pathname === "/api-monitor"; + + const [data, setData] = useState + > | null>(null); + + // One loop for both jobs: panel contents while open, traffic watch while + // closed. Stands down on the full page, which polls for itself. + useEffect(() => { + if (onFullPage) { + return; + } + let cancelled = false; + let timer: number | undefined; + const intervalMs = isOpen ? OPEN_POLL_MS : IDLE_POLL_MS; + + function schedule(): void { + timer = window.setTimeout(poll, intervalMs); + } + + function poll(): void { + // A hidden tab has nobody to show the panel to. + if (document.hidden) { + schedule(); + return; + } + getApiMonitor() + .then((next) => { + if (!cancelled) setData(next); + }) + .catch(() => { + // An unreachable server is the full page's story to tell. + if (!cancelled) setData(null); + }) + .finally(() => { + if (!cancelled) schedule(); + }); + } + + poll(); + return () => { + cancelled = true; + if (timer !== undefined) window.clearTimeout(timer); + }; + }, [isOpen, onFullPage]); + + const entries = useMemo(() => data?.entries ?? [], [data]); + const stats = useMemo(() => computeStats(entries), [entries]); + + // Ids already seen. A set, not "the newest id": finishing moves an entry to + // the front, so the head flips without any new traffic. + const seenIdsRef = useRef>(new Set()); + // Seeded on the first response even when empty, so the first request of a + // fresh session is not mistaken for history. + const seededRef = useRef(false); + const lastNewEntryAtRef = useRef(0); + + useEffect(() => { + if (data == null) { + return; + } + const ids = data.entries.map((entry) => entry.id); + if (!seededRef.current) { + seededRef.current = true; + seenIdsRef.current = new Set(ids); + return; + } + const seen = seenIdsRef.current; + const hasNewTraffic = ids.some((id) => !seen.has(id)); + // Re-seed each poll so the set stays bounded by the server's ring buffer. + seenIdsRef.current = new Set(ids); + if (!hasNewTraffic) { + return; + } + const now = Date.now(); + const quietFor = now - lastNewEntryAtRef.current; + lastNewEntryAtRef.current = now; + if (!autoOpen || isOpen) { + return; + } + // A dismissal holds for that burst and re-arms only once the API goes + // quiet, so the next request cannot re-open it a second later. + if (suppressed && quietFor < REARM_QUIET_MS) { + return; + } + open(); + }, [data, autoOpen, suppressed, isOpen, open]); + + // The backlog built up while the poll was stood down is not new traffic. + useEffect(() => { + if (onFullPage) { + seededRef.current = false; + } + }, [onFullPage]); + + const [constraintsElement, setConstraintsElement] = + useState(null); + const constraintsRef = useMemo( + () => ({ current: constraintsElement }), + [constraintsElement], + ); + const dragControls = useDragControls(); + + function startDrag(event: PointerEvent): void { + event.preventDefault(); + dragControls.start(event); + } + + const visible = isOpen && !onFullPage; + const serverStatus = data?.status ?? "idle"; + + return ( + + {visible && ( +
+ +
+
+ + + API monitor + + +
+
+
+ +
+ +
+
+ +

+ {data?.active_model ?? "No model loaded"} +

+ + {/* Metrics on a soft tile, as the Hub and Train pages group readouts. */} +
+ 0 ? "active" : undefined} + /> + + 0 ? "error" : undefined} + /> + +
+ + {/* Borderless rows on 12px hover pills, as in the sidebar. */} +
+ {entries.length === 0 ? ( +

+ No requests yet. +

+ ) : ( + entries.slice(0, VISIBLE_ENTRIES).map((entry) => ( +
+ + + {compactEndpoint(entry.endpoint)} + + + {entry.error ? entry.error : entry.model} + + + {formatDuration(entry.duration_ms)} + +
+ )) + )} +
+ + {/* Through to payloads, filters and per request tokens. */} + + + {/* Closing only silences this burst; this is the permanent off. */} + + +
+ )} + + ); +} diff --git a/studio/frontend/src/features/api-monitor/api-monitor-page.tsx b/studio/frontend/src/features/api-monitor/api-monitor-page.tsx new file mode 100644 index 0000000000..fa5934ab5d --- /dev/null +++ b/studio/frontend/src/features/api-monitor/api-monitor-page.tsx @@ -0,0 +1,778 @@ +// SPDX-License-Identifier: AGPL-3.0-only +// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 + +// Full-page monitor for Unsloth's OpenAI-compatible API server. +// +// This replaces the small console that used to be buried in the API settings +// tab. Settings still owns configuration (keys, auto-switch, examples); this +// page owns observability -- what is being served right now, which requests +// failed and why, and which saved settings a remote load will apply. + +import { Button } from "@/components/ui/button"; +import { Input } from "@/components/ui/input"; +import { + Select, + SelectContent, + SelectItem, + SelectTrigger, + SelectValue, +} from "@/components/ui/select"; +import { Skeleton } from "@/components/ui/skeleton"; +import type { ApiMonitorEntry } from "@/features/chat/types/api"; +import { useSettingsDialogStore } from "@/features/settings"; +import { copyToClipboard } from "@/lib/copy-to-clipboard"; +import { Tick02Icon } from "@/lib/tick-icon"; +import { cn } from "@/lib/utils"; +import { + Copy01Icon, + Delete02Icon, + Globe02Icon, + PauseIcon, + PlayIcon, + RefreshIcon, + Settings02Icon, +} from "@hugeicons/core-free-icons"; +import { HugeiconsIcon } from "@hugeicons/react"; +import { type ReactElement, useEffect, useMemo, useRef, useState } from "react"; +import { SavedModelSettingsPanel } from "./components/saved-model-settings"; +import { + type MonitorStatusFilter, + filterEntries, + useApiMonitor, +} from "./use-api-monitor"; + +const API_INFERENCE_PREFIX_RE = /^\/api\/inference/; +const V1_PREFIX_RE = /^\/v1\//; + +const STATUS_FILTERS: { value: MonitorStatusFilter; label: string }[] = [ + { value: "all", label: "All requests" }, + { value: "running", label: "In flight" }, + { value: "completed", label: "Completed" }, + { value: "error", label: "Errors" }, + { value: "cancelled", label: "Cancelled" }, +]; + +function formatTime(value: number): string { + return new Date(value * 1000).toLocaleTimeString([], { + hour: "2-digit", + minute: "2-digit", + second: "2-digit", + }); +} + +function formatDuration(value?: number | null): string { + if (value == null) { + return "running"; + } + if (value < 1000) { + return `${Math.round(value)} ms`; + } + return `${(value / 1000).toFixed(value < 10000 ? 1 : 0)} s`; +} + +function formatCount(value: number): string { + return value.toLocaleString(); +} + +function compactEndpoint(endpoint: string): string { + return endpoint + .replace(API_INFERENCE_PREFIX_RE, "/api") + .replace(V1_PREFIX_RE, "/"); +} + +function statusDotClass(status: ApiMonitorEntry["status"]): string { + switch (status) { + case "running": + return "bg-blue-500 animate-pulse"; + case "error": + return "bg-red-500"; + case "cancelled": + return "bg-amber-500"; + default: + return "bg-emerald-500"; + } +} + +function statusTextClass(status: ApiMonitorEntry["status"]): string { + switch (status) { + case "running": + return "text-blue-600 dark:text-blue-400"; + case "error": + return "text-red-600 dark:text-red-400"; + case "cancelled": + return "text-amber-600 dark:text-amber-500"; + default: + return "text-emerald-600 dark:text-emerald-500"; + } +} + +function StatCard({ + label, + value, + hint, + tone, +}: { + label: string; + value: string; + hint?: string; + tone?: "default" | "error" | "active"; +}): ReactElement { + return ( +
+ + {label} + + + {value} + + {hint ? ( + + {hint} + + ) : null} +
+ ); +} + +function CopyButton({ + value, + label, +}: { + value: string; + label: string; +}): ReactElement { + const [copied, setCopied] = useState(false); + // Clearing the tick on a timer would set state after unmount if the user + // navigates away mid-flash, so the timer is cancelled on cleanup. + const timerRef = useRef(undefined); + useEffect( + () => () => { + if (timerRef.current !== undefined) window.clearTimeout(timerRef.current); + }, + [], + ); + return ( + + ); +} + +function ContextUsageBar({ + value, +}: { value?: number | null }): ReactElement | null { + if (value == null) { + return null; + } + const pct = Math.max(0, Math.min(100, Math.round(value * 100))); + return ( +
+
+
= 90 + ? "bg-red-500" + : pct >= 75 + ? "bg-amber-500" + : "bg-control-accent", + )} + style={{ width: `${pct}%` }} + /> +
+ + {pct}% + +
+ ); +} + +function RequestRow({ + entry, + selected, + onSelect, +}: { + entry: ApiMonitorEntry; + selected: boolean; + onSelect: () => void; +}): ReactElement { + const preview = + entry.error || + entry.reply_preview || + entry.prompt_preview || + (entry.status === "running" ? "Waiting for output…" : "No preview"); + return ( + + ); +} + +function PayloadBlock({ + title, + body, + truncated, + loading, + tone, +}: { + title: string; + body: string; + truncated?: boolean; + loading?: boolean; + tone?: "error"; +}): ReactElement { + return ( +
+
+

+ {title} +

+
+ {truncated ? ( + + preview only + + ) : null} + {body ? ( + + ) : null} +
+
+ 
+        {loading && !body ? "Loading…" : body || "–"}
+
+
+ ); +} + +function RequestDetail({ + entry, + detail, + loading, +}: { + entry: ApiMonitorEntry; + detail?: ApiMonitorEntry; + loading: boolean; +}): ReactElement { + // The detail fetch is a separate request, so it can describe an older state of + // a still-streaming entry. Prefer it only once it is at least as fresh as the + // list row, otherwise the panel would appear to rewind while tokens arrive. + const detailIsCurrent = + detail != null && + detail.status === entry.status && + detail.updated_at >= entry.updated_at; + const prompt = detail?.prompt ?? entry.prompt_preview; + const reply = detailIsCurrent + ? (detail.reply ?? entry.reply_preview) + : entry.reply_preview; + + return ( +
+
+
+ + + {entry.status} + + + {entry.id} + +
+

+ {entry.method} {entry.endpoint} +

+

+ {entry.model} +

+
+ +
+ {[ + { label: "Started", value: formatTime(entry.started_at) }, + { label: "Duration", value: formatDuration(entry.duration_ms) }, + { + label: "Prompt tokens", + value: + entry.prompt_tokens != null + ? formatCount(entry.prompt_tokens) + : "–", + }, + { + label: "Completion tokens", + value: + entry.completion_tokens != null + ? formatCount(entry.completion_tokens) + : "–", + }, + { + label: "Total tokens", + value: + entry.total_tokens != null + ? formatCount(entry.total_tokens) + : "–", + }, + { + label: "Context", + value: + entry.context_length != null + ? formatCount(entry.context_length) + : "–", + }, + ].map((item) => ( +
+
+ {item.label} +
+
+ {item.value} +
+
+ ))} +
+ + {entry.context_usage != null ? ( +
+ + Context used + + +
+ ) : null} + + {entry.error ? ( + + ) : null} + + + +
+ ); +} + +export function ApiMonitorPage(): ReactElement { + const { + data, + entries, + stats, + error, + loading, + refreshing, + paused, + setPaused, + refresh, + clear, + details, + loadingDetails, + requestDetail, + } = useApiMonitor(); + const [statusFilter, setStatusFilter] = useState("all"); + const [query, setQuery] = useState(""); + const [selectedId, setSelectedId] = useState(null); + + const visible = useMemo( + () => filterEntries(entries, statusFilter, query), + [entries, statusFilter, query], + ); + const selected = useMemo( + () => visible.find((entry) => entry.id === selectedId) ?? null, + [visible, selectedId], + ); + + // Refetch the selected entry while it streams so the payload grows with the + // reply. Keyed on identity and revision, never on `details`: the fetch rewrites + // `details` on every success, so depending on it loops on the detail endpoint, + // which takes the same lock every generated token does. + const selectedId_ = selected?.id ?? null; + const selectedUpdatedAt = selected?.updated_at ?? null; + const selectedIsMissing = selectedId_ != null && details[selectedId_] == null; + const lastFetchedRef = useRef(null); + useEffect(() => { + if (selectedId_ == null) { + return; + } + // `updated_at` advances per poll while streaming and settles when terminal. + // A missing payload always retries, covering a fetch that failed late. + const revision = `${selectedId_}@${selectedUpdatedAt ?? ""}`; + if (!selectedIsMissing && lastFetchedRef.current === revision) { + return; + } + lastFetchedRef.current = revision; + requestDetail(selectedId_); + }, [selectedId_, selectedUpdatedAt, selectedIsMissing, requestDetail]); + + const baseUrl = + typeof window === "undefined" ? "" : `${window.location.origin}/v1`; + const serverStatus = data?.status ?? "idle"; + const statusCopy = + serverStatus === "generating" + ? "Serving requests" + : serverStatus === "ready" + ? "Ready" + : "No model loaded"; + + return ( +
+
+
+

+ API +

+

+ Live traffic through Unsloth's OpenAI-compatible server. +

+
+
+ + + + +
+
+ + {/* Server summary: the two things you check first when a client can't + reach the API -- the base URL to point it at, and what is loaded. */} +
+
+ + + +
+ + Base URL + + + {baseUrl} + +
+ +
+
+ + Status + + + + {statusCopy} + +
+
+ + Loaded model + + + {data?.active_model ?? "None"} + {data?.context_length + ? ` · ${formatCount(data.context_length)} ctx` + : ""} + +
+ {paused ? ( + + Paused + + ) : null} +
+ + {error ? ( +
+ {error} +
+ ) : null} + +
+ 0 ? "active" : "default"} + /> + + + 0 ? "error" : "default"} + hint={ + stats.errorRate != null + ? `${Math.round(stats.errorRate * 100)}% of finished` + : undefined + } + /> + + +
+ +
+
+ setQuery(event.target.value)} + placeholder="Search model, endpoint, preview or error" + aria-label="Search API requests" + className="h-9 w-full min-w-0 flex-1 rounded-full border-none bg-muted shadow-none dark:bg-background sm:w-64 sm:flex-none" + /> + + + {formatCount(visible.length)} of {formatCount(entries.length)} + +
+ +
+
+ {loading ? ( +
+ {[0, 1, 2].map((i) => ( + + ))} +
+ ) : visible.length === 0 ? ( +

+ {entries.length === 0 + ? "No API traffic yet. Point a client at the base URL above to see requests here." + : "No requests match this filter."} +

+ ) : ( + visible.map((entry) => ( + setSelectedId(entry.id)} + /> + )) + )} +
+ +
+ {selected ? ( + + ) : ( +

+ Select a request to inspect its prompt, reply, tokens and + errors. +

+ )} +
+
+
+ + +
+ ); +} diff --git a/studio/frontend/src/features/api-monitor/components/saved-model-settings.tsx b/studio/frontend/src/features/api-monitor/components/saved-model-settings.tsx new file mode 100644 index 0000000000..6a1224472e --- /dev/null +++ b/studio/frontend/src/features/api-monitor/components/saved-model-settings.tsx @@ -0,0 +1,140 @@ +// SPDX-License-Identifier: AGPL-3.0-only +// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 + +// What a remote load will actually apply, which is otherwise unanswerable from +// outside the process. +// +// Read only on purpose: the config lives both here and in the browser's own +// per-model store, and the model's settings page is the only place that owns +// both, so it is the only place that can forget a model completely. + +import { Skeleton } from "@/components/ui/skeleton"; +import { + type ApiModelOverride, + type ApiModelOverrides, + fetchModelOverrides, +} from "@/features/model-picker/api/model-overrides"; +import { type ReactElement, useCallback, useEffect, useState } from "react"; + +/** Human-readable summary of the fields the loader will apply, in load order. */ +function describeOverride(override: ApiModelOverride): string[] { + const parts: string[] = []; + if (override.custom_context_length) { + parts.push(`${override.custom_context_length.toLocaleString()} context`); + } + if (override.max_seq_length) { + parts.push(`${override.max_seq_length.toLocaleString()} max seq`); + } + if (override.kv_cache_dtype) { + parts.push(`KV ${override.kv_cache_dtype}`); + } + if (override.speculative_type) { + parts.push( + override.spec_draft_n_max + ? `spec ${override.speculative_type} ×${override.spec_draft_n_max}` + : `spec ${override.speculative_type}`, + ); + } + if (override.tensor_parallel) { + parts.push("tensor parallel"); + } + if (override.gpu_memory_mode === "manual") { + parts.push("manual GPU memory"); + } + if (override.gpu_layers != null) { + parts.push(`${override.gpu_layers} GPU layers`); + } + if (override.n_cpu_moe) { + parts.push(`${override.n_cpu_moe} MoE layers on CPU`); + } + if (override.gpu_ids?.length) { + parts.push(`GPU ${override.gpu_ids.join(", ")}`); + } + if (override.chat_template_override) { + parts.push("custom chat template"); + } + if (override.llama_extra_args?.length) { + parts.push(override.llama_extra_args.join(" ")); + } + return parts; +} + +export function SavedModelSettingsPanel(): ReactElement { + const [overrides, setOverrides] = useState(null); + const [error, setError] = useState(null); + + const load = useCallback(async () => { + try { + setOverrides(await fetchModelOverrides()); + setError(null); + } catch (err: unknown) { + setError( + err instanceof Error + ? err.message + : "Could not load saved model settings", + ); + } + }, []); + + useEffect(() => { + void load(); + }, [load]); + + const entries = Object.entries(overrides ?? {}); + + return ( +
+
+

+ Settings applied on API load +

+

+ When a request names one of these models, Unsloth loads it with these + settings, the same ones you saved in the model's settings page. + Models without an entry load with app defaults. Edit or forget an + entry from that model's settings, which keeps this list and the + picker in step. +

+
+ + {error ? ( +
+ {error} +
+ ) : overrides == null ? ( +
+ {[0, 1].map((i) => ( + + ))} +
+ ) : entries.length === 0 ? ( +

+ No saved model settings yet. Open a model's settings, turn on + "Remember for this model", and it will be applied to API + loads too. +

+ ) : ( +
    + {entries.map(([modelId, override]) => { + const summary = describeOverride(override); + return ( +
  • +
    + + {modelId} + + + {summary.length > 0 ? summary.join(" · ") : "App defaults"} + +
    +
    • + ); + })} +
+ )} +
+ ); +} diff --git a/studio/frontend/src/features/api-monitor/index.ts b/studio/frontend/src/features/api-monitor/index.ts new file mode 100644 index 0000000000..28b01def8b --- /dev/null +++ b/studio/frontend/src/features/api-monitor/index.ts @@ -0,0 +1,13 @@ +// SPDX-License-Identifier: AGPL-3.0-only +// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 + +export { ApiMonitorPage } from "./api-monitor-page"; +export { ApiMonitorOverlay } from "./api-monitor-overlay"; +export { useApiMonitorOverlayStore } from "./overlay-store"; +export { + computeStats, + filterEntries, + useApiMonitor, + type MonitorStats, + type MonitorStatusFilter, +} from "./use-api-monitor"; diff --git a/studio/frontend/src/features/api-monitor/overlay-store.ts b/studio/frontend/src/features/api-monitor/overlay-store.ts new file mode 100644 index 0000000000..07ad4edd31 --- /dev/null +++ b/studio/frontend/src/features/api-monitor/overlay-store.ts @@ -0,0 +1,48 @@ +// SPDX-License-Identifier: AGPL-3.0-only +// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 + +import { create } from "zustand"; +import { persist } from "zustand/middleware"; + +interface ApiMonitorOverlayState { + /** Whether the floating panel is on screen right now. Session state. */ + isOpen: boolean; + /** Set on close so the panel does not pop back during the same burst. */ + suppressed: boolean; + /** Persisted opt out: when false the panel never opens itself. */ + autoOpen: boolean; + open: () => void; + close: () => void; + setAutoOpen: (autoOpen: boolean) => void; +} + +/** + * Only `autoOpen` persists. Open/closed is session state: a dismissal lasts the + * sitting, not forever. + */ +export const useApiMonitorOverlayStore = create()( + persist( + (set) => ({ + isOpen: false, + suppressed: false, + autoOpen: true, + open: () => set({ isOpen: true, suppressed: false }), + close: () => set({ isOpen: false, suppressed: true }), + setAutoOpen: (autoOpen) => set({ autoOpen }), + }), + { + name: "unsloth_api_monitor_overlay", + version: 1, + partialize: (state) => ({ autoOpen: state.autoOpen }), + // Explicit merge so an older stored payload cannot resurrect `isOpen`. + merge: (persisted, current) => ({ + ...current, + autoOpen: + typeof (persisted as { autoOpen?: unknown } | null)?.autoOpen === + "boolean" + ? (persisted as { autoOpen: boolean }).autoOpen + : current.autoOpen, + }), + }, + ), +); diff --git a/studio/frontend/src/features/api-monitor/use-api-monitor.ts b/studio/frontend/src/features/api-monitor/use-api-monitor.ts new file mode 100644 index 0000000000..20c3915ee3 --- /dev/null +++ b/studio/frontend/src/features/api-monitor/use-api-monitor.ts @@ -0,0 +1,297 @@ +// SPDX-License-Identifier: AGPL-3.0-only +// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 + +import { + clearApiMonitor, + getApiMonitor, + getApiMonitorEntry, +} from "@/features/chat/api/chat-api"; +import type { + ApiMonitorEntry, + ApiMonitorResponse, +} from "@/features/chat/types/api"; +import { useCallback, useEffect, useMemo, useRef, useState } from "react"; + +/** Poll cadence while the monitor is live. Matches the settings console it replaces. */ +const POLL_INTERVAL_MS = 1500; + +export type MonitorStatusFilter = + | "all" + | "running" + | "completed" + | "error" + | "cancelled"; + +export interface MonitorStats { + active: number; + total: number; + completed: number; + errors: number; + cancelled: number; + /** Mean duration over finished requests, or null when none have finished. */ + avgDurationMs: number | null; + /** Slowest finished request, for spotting a single pathological call. */ + maxDurationMs: number | null; + totalTokens: number; + /** Share of finished requests that failed, 0-1. Null when nothing finished. */ + errorRate: number | null; + /** Mean completion tokens per second over requests that reported both. */ + tokensPerSecond: number | null; +} + +function isTerminal(entry: ApiMonitorEntry): boolean { + return entry.status !== "running"; +} + +function completionTokens(entry: ApiMonitorEntry): number | null { + if (entry.completion_tokens != null) { + return entry.completion_tokens; + } + // Some providers only report a total; subtracting the prompt is the best + // available estimate of what was actually generated. + if (entry.total_tokens != null && entry.prompt_tokens != null) { + return Math.max(0, entry.total_tokens - entry.prompt_tokens); + } + return null; +} + +function entryTokens(entry: ApiMonitorEntry): number { + if (entry.total_tokens != null) { + return entry.total_tokens; + } + return (entry.prompt_tokens ?? 0) + (entry.completion_tokens ?? 0); +} + +export function computeStats(entries: ApiMonitorEntry[]): MonitorStats { + let active = 0; + let completed = 0; + let errors = 0; + let cancelled = 0; + let totalTokens = 0; + let durationSum = 0; + let durationCount = 0; + let maxDurationMs: number | null = null; + // Throughput is aggregated as total tokens over total time, not as the mean of + // each request's rate. Averaging rates lets one tiny fast request outweigh a + // long slow one, which is the opposite of what someone debugging wants to see. + let generatedTokens = 0; + let generatedDurationMs = 0; + + for (const entry of entries) { + totalTokens += entryTokens(entry); + if (entry.status === "running") { + active += 1; + } else if (entry.status === "error") { + errors += 1; + } else if (entry.status === "cancelled") { + cancelled += 1; + } else { + completed += 1; + } + const duration = entry.duration_ms; + if (duration != null && isTerminal(entry)) { + durationSum += duration; + durationCount += 1; + maxDurationMs = + maxDurationMs == null ? duration : Math.max(maxDurationMs, duration); + const generated = completionTokens(entry); + // Sub-millisecond durations would divide into a meaningless rate. + if (generated != null && generated > 0 && duration > 0) { + generatedTokens += generated; + generatedDurationMs += duration; + } + } + } + + const finished = completed + errors + cancelled; + return { + active, + total: entries.length, + completed, + errors, + cancelled, + avgDurationMs: durationCount > 0 ? durationSum / durationCount : null, + maxDurationMs, + totalTokens, + errorRate: finished > 0 ? errors / finished : null, + tokensPerSecond: + generatedDurationMs > 0 + ? generatedTokens / (generatedDurationMs / 1000) + : null, + }; +} + +export function filterEntries( + entries: ApiMonitorEntry[], + status: MonitorStatusFilter, + query: string, +): ApiMonitorEntry[] { + const needle = query.trim().toLowerCase(); + return entries.filter((entry) => { + if (status !== "all" && entry.status !== status) { + return false; + } + if (!needle) { + return true; + } + // Search the fields a debugging session actually keys off: which model, + // which endpoint, and the previews/error text visible in the row. + return ( + entry.model.toLowerCase().includes(needle) || + entry.endpoint.toLowerCase().includes(needle) || + entry.prompt_preview.toLowerCase().includes(needle) || + entry.reply_preview.toLowerCase().includes(needle) || + (entry.error ?? "").toLowerCase().includes(needle) + ); + }); +} + +interface UseApiMonitorResult { + data: ApiMonitorResponse | null; + entries: ApiMonitorEntry[]; + stats: MonitorStats; + error: string | null; + /** True until the first response lands, so the page can show skeletons once. */ + loading: boolean; + refreshing: boolean; + paused: boolean; + setPaused: (paused: boolean) => void; + refresh: () => void; + clear: () => Promise; + /** Full prompt/reply for entries the user expanded, keyed by entry id. */ + details: Record; + loadingDetails: ReadonlySet; + requestDetail: (id: string) => void; +} + +/** + * Live view of the server's OpenAI-compatible API traffic. + * + * Polls rather than streams because the backing monitor is an in-memory ring + * buffer with no change feed. Polling is self-rescheduling (never overlapping), + * and pausing stops it entirely so a user reading a stalled request's payload + * isn't fighting a list that reorders under them. + * + * `intervalMs` lets a caller trade freshness for cost: the full page wants the + * default live cadence, while the floating overlay slows right down when it is + * closed and only watching for the traffic that should pop it open. + */ +export function useApiMonitor({ + intervalMs = POLL_INTERVAL_MS, +}: { intervalMs?: number } = {}): UseApiMonitorResult { + const [data, setData] = useState(null); + const [error, setError] = useState(null); + const [loading, setLoading] = useState(true); + const [refreshing, setRefreshing] = useState(false); + const [paused, setPaused] = useState(false); + const [details, setDetails] = useState>({}); + const [loadingDetails, setLoadingDetails] = useState>( + () => new Set(), + ); + // Mirrors `loadingDetails` outside React state so the fetch guard sees writes + // from the same tick (state updates are async and would let duplicates through). + const inFlightDetails = useRef>(new Set()); + + const load = useCallback(async (): Promise => { + setRefreshing(true); + try { + const next = await getApiMonitor(); + setData(next); + setError(null); + } catch (err: unknown) { + setError(err instanceof Error ? err.message : "Monitor unavailable"); + } finally { + setRefreshing(false); + setLoading(false); + } + }, []); + + useEffect(() => { + if (paused) { + return; + } + let cancelled = false; + let timer: number | undefined; + + function poll(): void { + getApiMonitor() + .then((next) => { + if (cancelled) return; + setData(next); + setError(null); + }) + .catch((err: unknown) => { + if (cancelled) return; + setError(err instanceof Error ? err.message : "Monitor unavailable"); + }) + .finally(() => { + if (cancelled) return; + setLoading(false); + timer = window.setTimeout(poll, intervalMs); + }); + } + + poll(); + return () => { + cancelled = true; + if (timer !== undefined) { + window.clearTimeout(timer); + } + }; + }, [paused, intervalMs]); + + const requestDetail = useCallback((id: string): void => { + if (inFlightDetails.current.has(id)) { + return; + } + inFlightDetails.current.add(id); + setLoadingDetails((prev) => new Set(prev).add(id)); + getApiMonitorEntry(id) + .then((entry) => { + setDetails((prev) => ({ ...prev, [id]: entry })); + }) + .catch(() => { + // The entry aged out of the ring buffer; drop any stale copy so the UI + // falls back to the row previews instead of showing a frozen payload. + setDetails((prev) => { + if (!(id in prev)) return prev; + const next = { ...prev }; + delete next[id]; + return next; + }); + }) + .finally(() => { + inFlightDetails.current.delete(id); + setLoadingDetails((prev) => { + const next = new Set(prev); + next.delete(id); + return next; + }); + }); + }, []); + + const clear = useCallback(async (): Promise => { + await clearApiMonitor(); + setDetails({}); + await load(); + }, [load]); + + const entries = useMemo(() => data?.entries ?? [], [data]); + const stats = useMemo(() => computeStats(entries), [entries]); + + return { + data, + entries, + stats, + error, + loading, + refreshing, + paused, + setPaused, + refresh: () => void load(), + clear, + details, + loadingDetails, + requestDetail, + }; +} diff --git a/studio/frontend/src/features/chat/api/chat-api.ts b/studio/frontend/src/features/chat/api/chat-api.ts index 4d123e98ab..822de4fa4f 100644 --- a/studio/frontend/src/features/chat/api/chat-api.ts +++ b/studio/frontend/src/features/chat/api/chat-api.ts @@ -129,6 +129,13 @@ export async function getApiMonitorEntry(id: string): Promise { return parseJsonOrThrow(response); } +export async function clearApiMonitor(): Promise { + const response = await authFetch("/api/inference/monitor", { + method: "DELETE", + }); + await parseJsonOrThrow<{ cleared: boolean }>(response); +} + export async function loadModel( payload: LoadModelRequest, ): Promise { diff --git a/studio/frontend/src/features/hub/catalog/download-card.tsx b/studio/frontend/src/features/hub/catalog/download-card.tsx index 8016a9406d..1618b9cdf1 100644 --- a/studio/frontend/src/features/hub/catalog/download-card.tsx +++ b/studio/frontend/src/features/hub/catalog/download-card.tsx @@ -23,6 +23,7 @@ import { ArrowReloadHorizontalIcon, Delete02Icon, Download01Icon, + Settings02Icon, } from "@hugeicons/core-free-icons"; import { HugeiconsIcon } from "@hugeicons/react"; import { @@ -82,6 +83,40 @@ export function CardDivider() { ); } +/** Gear that opens a downloaded model's full settings page. */ +export function CardSettingsButton({ + label, + onClick, +}: { + label: string; + onClick: () => void; +}) { + return ( + + + + + + {label} + + + ); +} + export function CardDeleteButton({ label, onClick, diff --git a/studio/frontend/src/features/hub/catalog/hub-model-settings-view.tsx b/studio/frontend/src/features/hub/catalog/hub-model-settings-view.tsx new file mode 100644 index 0000000000..a293a0071a --- /dev/null +++ b/studio/frontend/src/features/hub/catalog/hub-model-settings-view.tsx @@ -0,0 +1,138 @@ +// SPDX-License-Identifier: AGPL-3.0-only +// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 + +// Full-page settings for one model, opened from the Hub. +// +// The same controls exist in the chat picker's popover, but a popover is a poor +// place to work through every knob a model has. This gives them a page, and +// states plainly that whatever is saved here is what an API load will use -- +// the settings are mirrored server-side by ModelConfigPage's save. + +import { ModelConfigPage, type ModelPickTarget } from "@/features/model-picker"; +import type { PerModelConfig } from "@/features/model-picker"; +import { ArrowLeft01Icon, Globe02Icon } from "@hugeicons/core-free-icons"; +import { HugeiconsIcon } from "@hugeicons/react"; +import { cn } from "@/lib/utils"; +import { useEffect, useRef, useState } from "react"; + +export function HubModelSettingsView({ + target, + loadedConfig = null, + loadedContextLength = null, + onBack, + onRun, + compact = false, +}: { + target: ModelPickTarget; + /** Non-null when this model is the loaded one, so the page can show live values. */ + loadedConfig?: PerModelConfig | null; + loadedContextLength?: number | null; + onBack: () => void; + /** Apply + load with these settings. */ + onRun: (config: PerModelConfig) => void; + compact?: boolean; +}) { + const scrollRef = useRef(null); + const [scrolled, setScrolled] = useState(false); + // Mirrors HubDetailView so this view sits at the same measure as the rest of + // the Hub rather than introducing a third column width. + const measure = compact + ? "mx-auto w-full max-w-[860px] px-5 sm:px-5" + : "mx-auto w-full max-w-[1100px] px-5 sm:px-8"; + + useEffect(() => { + const el = scrollRef.current; + if (!el) return; + const onScroll = () => { + const next = el.scrollTop > 0; + setScrolled((current) => (current === next ? current : next)); + }; + onScroll(); + el.addEventListener("scroll", onScroll, { passive: true }); + return () => el.removeEventListener("scroll", onScroll); + }, []); + + return ( +
+ + ); +} diff --git a/studio/frontend/src/features/hub/catalog/local-on-device-card.tsx b/studio/frontend/src/features/hub/catalog/local-on-device-card.tsx index 9b508a5413..b432b11f54 100644 --- a/studio/frontend/src/features/hub/catalog/local-on-device-card.tsx +++ b/studio/frontend/src/features/hub/catalog/local-on-device-card.tsx @@ -53,6 +53,7 @@ import { useHfTokenStore } from "../stores/hf-token-store"; import { DotTag } from "./dot-tag"; import { CardDeleteButton, + CardSettingsButton, CardUpdateButton, DeleteConfirmDialog, UpdateConfirmDialog, @@ -96,6 +97,8 @@ interface LocalOnDeviceCardProps { onEject?: () => void; onTrain?: () => void; onChange?: () => void; + /** Open this model's full settings page for the shown quant. */ + onOpenSettings?: (ggufVariant: string | null) => void; } function formatAdapterLabel( @@ -214,6 +217,7 @@ export function LocalOnDeviceCard({ onEject, onTrain, onChange, + onOpenSettings, }: LocalOnDeviceCardProps) { const [deleteOpen, setDeleteOpen] = useState(false); const [updateOpen, setUpdateOpen] = useState(false); @@ -549,6 +553,14 @@ export function LocalOnDeviceCard({ )}
+ {onOpenSettings && ( + onOpenSettings(selectedQuant ?? null)} + /> + )} {canUpdate && ( void; onInventoryChange?: () => void; onSearchHub?: (query: string) => void; + /** Open this model's full settings page, with the quant the card resolved. */ + onOpenSettings?: (ggufVariant: string | null) => void; }; export const ModelInspector = memo(function ModelInspector({ @@ -438,6 +440,7 @@ export const ModelInspector = memo(function ModelInspector({ onTrain, onInventoryChange, onSearchHub, + onOpenSettings, } = actions; const deviceType = usePlatformStore((s) => s.deviceType); const chatOnly = usePlatformStore((s) => s.isChatOnly()); @@ -705,6 +708,7 @@ export const ModelInspector = memo(function ModelInspector({ model.isDownloaded && canTrainModel ? onTrain : undefined } onChange={onInventoryChange} + onOpenSettings={onOpenSettings} /> ) : ( void; + /** Open a downloaded model's full settings page. */ + onOpenModelSettings?: (row: CachedInventoryRow | LocalInventoryRow) => void; }) { // Pinned repos surface first regardless of the active sort; the chosen sort // still orders rows within the pinned and unpinned groups. @@ -383,6 +386,7 @@ export function DownloadedList({ compact={compact} onSelect={onSelect} onChange={onInventoryChange} + onOpenSettings={onOpenModelSettings} /> ); diff --git a/studio/frontend/src/features/hub/catalog/models-catalog-rows.tsx b/studio/frontend/src/features/hub/catalog/models-catalog-rows.tsx index 5236c0be74..1dbf78cff9 100644 --- a/studio/frontend/src/features/hub/catalog/models-catalog-rows.tsx +++ b/studio/frontend/src/features/hub/catalog/models-catalog-rows.tsx @@ -573,6 +573,7 @@ export const InventoryRow = memo(function InventoryRow({ compact = false, onSelect, onChange, + onOpenSettings, }: { row: CachedInventoryRow | LocalInventoryRow; selected: boolean; @@ -585,6 +586,8 @@ export const InventoryRow = memo(function InventoryRow({ compact?: boolean; onSelect: (id: string) => void; onChange?: () => void; + /** Open this model's full settings page. Omitted for datasets. */ + onOpenSettings?: (row: CachedInventoryRow | LocalInventoryRow) => void; }) { const rowModelId = row.kind === "cache" @@ -732,30 +735,41 @@ export const InventoryRow = memo(function InventoryRow({ const rowPinned = cacheDeletableRepoId != null && pinnedKeys.includes(pinKey(cacheDeletableRepoId)); + // Settings is available for any downloaded model, not just deletable ones: a + // model scanned from a local folder is just as configurable as a cached repo. + // The menu therefore renders whenever either action applies, and each item + // gates itself. `deletableRepoId` (rather than the boolean) keeps the non-null + // narrowing the delete closures below rely on. + const settingsAction = + !isDataset && onOpenSettings ? { onOpen: () => onOpenSettings(row) } : undefined; + const deletableRepoId = canDelete ? cacheDeletableRepoId : null; const deleteAction = - canDelete && cacheDeletableRepoId ? ( + deletableRepoId || settingsAction ? ( togglePinned(cacheDeletableRepoId), + onToggle: () => togglePinned(deletableRepoId), } } - cachePath={isDataset ? undefined : { repoId: cacheDeletableRepoId }} - del={{ + cachePath={ + isDataset || !deletableRepoId ? undefined : { repoId: deletableRepoId } + } + del={deletableRepoId ? { title: isDataset ? "Delete cached dataset?" : "Delete cached model?", description: ( <> This will remove{" "} - {cacheDeletableRepoId} + {deletableRepoId} {" "} {isDataset ? "and its downloaded files" @@ -766,17 +780,17 @@ export const InventoryRow = memo(function InventoryRow({ disk. You can re-download it later. ), - successMessage: `Deleted ${cacheDeletableRepoId}`, + successMessage: `Deleted ${deletableRepoId}`, onConfirm: async () => { // Delete only the copy this row shows: cache rows carry the owning // cache path, so pass it through and leave other caches untouched. const rowCachePath = row.kind === "cache" ? (row.cachePath ?? undefined) : undefined; if (isDataset) { - await deleteCachedDataset(cacheDeletableRepoId, rowCachePath); + await deleteCachedDataset(deletableRepoId, rowCachePath); } else { await deleteCachedModel( - cacheDeletableRepoId, + deletableRepoId, undefined, undefined, rowCachePath, @@ -787,11 +801,11 @@ export const InventoryRow = memo(function InventoryRow({ usePinnedModelsStore.getState(); for (const key of pinned) { if ( - key === pinKey(cacheDeletableRepoId) || - key.startsWith(`${cacheDeletableRepoId}::`) + key === pinKey(deletableRepoId) || + key.startsWith(`${deletableRepoId}::`) ) { toggle( - cacheDeletableRepoId, + deletableRepoId, key.includes("::") ? key.slice(key.indexOf("::") + 2) : undefined, @@ -801,7 +815,7 @@ export const InventoryRow = memo(function InventoryRow({ } }, onDeleted: onChange, - }} + } : undefined} /> ) : null; diff --git a/studio/frontend/src/features/hub/catalog/models-catalog.tsx b/studio/frontend/src/features/hub/catalog/models-catalog.tsx index 02b9b62fce..288c525af1 100644 --- a/studio/frontend/src/features/hub/catalog/models-catalog.tsx +++ b/studio/frontend/src/features/hub/catalog/models-catalog.tsx @@ -16,6 +16,7 @@ import { import type { CachedInventoryRow, DiscoverRow, + InventoryRow, LocalInventoryRow, ModelsTab, } from "../types"; @@ -70,6 +71,8 @@ export interface ModelsCatalogHandlers { onRetry: () => void; onInventoryChange?: () => void; onSwitchDevice?: () => void; + /** Open a downloaded model's full settings page. */ + onOpenModelSettings?: (row: InventoryRow) => void; } function assignRef(ref: RefObject, value: T | null) { @@ -128,6 +131,7 @@ export const ModelsCatalog = memo(function ModelsCatalog({ onRetry, onInventoryChange, onSwitchDevice, + onOpenModelSettings, } = handlers; const [scrolled, setScrolled] = useState(false); const [streamingActive, setStreamingActive] = useState(false); @@ -483,6 +487,7 @@ export const ModelsCatalog = memo(function ModelsCatalog({ columns={discoverView === "two" ? 2 : 1} sort={inventorySort} onInventoryChange={onInventoryChange} + onOpenModelSettings={onOpenModelSettings} />
) : ( diff --git a/studio/frontend/src/features/hub/hub-page.tsx b/studio/frontend/src/features/hub/hub-page.tsx index 9fc452b4ed..e853ea0e5c 100644 --- a/studio/frontend/src/features/hub/hub-page.tsx +++ b/studio/frontend/src/features/hub/hub-page.tsx @@ -9,6 +9,7 @@ import { import { getInferenceStatus, isExternalModelId, + listGgufVariants, useChatModelRuntime, useChatRuntimeStore, } from "@/features/chat"; @@ -24,9 +25,13 @@ import { ggufVariantsMatch, modelIdsMatch } from "./lib/model-identity"; import { hfApiToken, useHfTokenStore } from "./stores/hf-token-store"; import { applyModelLoadConfigToRuntime, + applyPerModelConfigToRuntime, currentRuntimePerModelConfig, hfModelFitsDevice, resolveInitialConfig, + useActiveModelConfig, + type ModelPickTarget, + type PerModelConfig, } from "@/features/model-picker"; import { useDebouncedValue } from "@/hooks/use-debounced-value"; import { useGpuInfo } from "@/hooks/use-gpu-info"; @@ -42,6 +47,7 @@ import { } from "react"; import { ExternalLinkConfirmDialog } from "./catalog/external-link-confirm-dialog"; import { HubDetailView } from "./catalog/hub-detail-view"; +import { HubModelSettingsView } from "./catalog/hub-model-settings-view"; import { HubFeed } from "./catalog/hub-feed"; import { HubTopBar } from "./catalog/hub-top-bar"; import { @@ -345,6 +351,12 @@ export function ModelsPage() { const activeCheckpoint = checkpoint && !isExternalModelId(checkpoint) ? checkpoint : null; const activeGgufVariant = useChatRuntimeStore((s) => s.activeGgufVariant); + const activeGgufContextLength = useChatRuntimeStore( + (s) => s.ggufContextLength, + ); + // Live settings of the loaded model, so opening its settings page shows what + // it is actually running with rather than the last saved draft. + const { config: activeModelConfig } = useActiveModelConfig(); // Shared with the chat model selector: list only models sized for this device. const fitOnDeviceOnly = useChatRuntimeStore((s) => s.fitOnDeviceOnly); const setFitOnDeviceOnly = useChatRuntimeStore((s) => s.setFitOnDeviceOnly); @@ -1213,6 +1225,100 @@ export function ModelsPage() { runSelectedModel(opts, selectedModel?.isDownloaded ?? true), [runSelectedModel, selectedModel], ); + + // Full-page per-model settings, opened from a downloaded row's menu. Local + // state rather than a URL param: the page is a transient editor over the + // catalog, and a deep link to it would need the row's identity re-resolved + // against an inventory that may not have loaded yet. + const [settingsTarget, setSettingsTarget] = useState( + null, + ); + const openModelSettings = useCallback( + async (row: CachedInventoryRow | LocalInventoryRow) => { + // loadId is what the loader accepts; repoId is only a display/API alias. + const id = row.loadId; + // Cached repo rows never carry a quant: the inventory emits one row per + // repo with format_variant null (see cache_inventory.py). Opening settings + // with a null variant would key the saved config to `repo::` while the + // loader reads `repo::Q4_K_M`, so the settings would silently never apply + // and the server mirror would be keyed wrong too. Resolve the quant the + // same way the on-device card does before opening. + let ggufVariant = row.formatVariant?.trim() || null; + if (!ggufVariant && row.isGguf && row.capabilities.requiresVariant) { + const repoId = row.kind === "cache" ? row.repoId : (row.repoId ?? null); + if (repoId) { + try { + const res = await listGgufVariants(repoId, hfApiToken(hfToken), { + preferLocalCache: true, + localPath: row.kind === "local" ? row.path : (row.cachePath ?? null), + }); + const downloaded = res.variants.filter((v) => v.downloaded); + ggufVariant = + // Prefer the loaded quant, then the repo default, then whatever is + // on disk, mirroring LocalOnDeviceCard's selectedQuant. + downloaded.find((v) => + ggufVariantsMatch(v.quant, activeGgufVariant), + )?.quant ?? + downloaded.find((v) => + ggufVariantsMatch(v.quant, res.default_variant), + )?.quant ?? + downloaded[0]?.quant ?? + null; + } catch { + // Offline or an unreadable cache: fall through with no variant. The + // settings page still works, it just cannot pin a specific quant. + ggufVariant = null; + } + } + } + const leaf = id.split(/[\\/]/).filter(Boolean).pop() ?? id; + setSettingsTarget({ + id, + displayName: ggufVariant ? `${leaf} · ${ggufVariant}` : leaf, + ggufVariant, + isGguf: row.isGguf, + meta: { + source: "local", + isLora: row.modelFormat === "adapter", + ggufVariant: ggufVariant ?? undefined, + isGguf: row.isGguf, + // Partial downloads still open settings, but must not claim to be + // complete or the loader skips its download-progress reporting. + isDownloaded: !row.partial, + // Not carried on inventory rows; ModelConfigPage reads the GGUF header + // itself to size the context slider. + contextLength: null, + }, + }); + }, + [activeGgufVariant, hfToken], + ); + // Applying from the settings page loads the model with exactly those settings. + // ModelConfigPage has already persisted them (locally and, when "remember" is + // on, to the server), so an API request for this model gets the same load. + const runSettingsTarget = useCallback( + (config: PerModelConfig) => { + const target = settingsTarget; + if (!target) return; + const previousConfig = currentRuntimePerModelConfig({ + includeMaxSeqLength: true, + }); + applyPerModelConfigToRuntime(config); + setSettingsTarget(null); + void selectModel({ + id: target.id, + source: "local", + ggufVariant: target.ggufVariant ?? undefined, + isGguf: target.isGguf, + isDownloaded: true, + isLora: target.meta.isLora, + keepSpeculative: true, + forceReload: true, + previousConfig, + }).catch(() => undefined); + }, + [selectModel, settingsTarget], + ); const handleLoadLocal = useCallback( (opts: ModelLoadOptions = {}) => runSelectedModel(opts, true), [runSelectedModel], @@ -1220,6 +1326,30 @@ export function ModelsPage() { const handleTrain = useCallback(() => { // Hub → train integration ships in a later PR. }, []); + // Settings opened from the detail view's on-device card. The card resolves + // which quant it is showing, so it passes that in rather than re-deriving it. + const openSelectedModelSettings = useCallback( + (ggufVariant: string | null) => { + if (!selectedModel) return; + const id = selectedModel.resource.runId; + const leaf = id.split(/[\\/]/).filter(Boolean).pop() ?? id; + setSettingsTarget({ + id, + displayName: ggufVariant ? `${leaf} · ${ggufVariant}` : leaf, + ggufVariant, + isGguf: selectedModel.isGguf, + meta: { + source: "local", + isLora: selectedModel.modelFormat === "adapter", + ggufVariant: ggufVariant ?? undefined, + isGguf: selectedModel.isGguf, + isDownloaded: selectedModel.isDownloaded, + contextLength: null, + }, + }); + }, + [selectedModel], + ); const handleSearchHub = useCallback( (next: string) => { const trimmed = next.trim(); @@ -1280,6 +1410,7 @@ export function ModelsPage() { onTrain: handleTrain, onInventoryChange: refreshInventory, onSearchHub: handleSearchHub, + onOpenSettings: openSelectedModelSettings, }), [ handleLoad, @@ -1289,6 +1420,7 @@ export function ModelsPage() { handleTrain, handleSearchHub, refreshInventory, + openSelectedModelSettings, ], ); @@ -1370,6 +1502,7 @@ export function ModelsPage() { onRetry: handleRetrySearch, onInventoryChange: refreshInventory, onSwitchDevice: handleSwitchDevice, + onOpenModelSettings: openModelSettings, }), [ handleSelect, @@ -1378,6 +1511,7 @@ export function ModelsPage() { handleRetrySearch, refreshInventory, handleSwitchDevice, + openModelSettings, ], ); @@ -1520,6 +1654,10 @@ export function ModelsPage() { const detailOpen = urlModel !== null; const splitMode = allModelsView === "split"; + // The catalog is unreachable when an opaque overlay sits on top of it: the + // detail view (full-page layout only, since split renders it alongside) or the + // settings page (always full-bleed). + const catalogCovered = (detailOpen && !splitMode) || settingsTarget !== null; return (
@@ -1575,9 +1713,15 @@ export function ModelsPage() { splitMode ? "flex-1 lg:w-[460px] lg:max-w-[44%] lg:flex-none lg:shrink-0 lg:border-r lg:border-border/60" : "flex-1", - detailOpen && !splitMode && "pointer-events-none", + // The settings page is a full-bleed opaque overlay in every layout, + // including split, so it always takes the catalog out of the tab + // order. Without this, tabbing out of the settings form walks into + // the virtualized rows hidden behind it and screen readers announce + // the whole model list underneath. + catalogCovered && "pointer-events-none", )} - aria-hidden={(detailOpen && !splitMode) || undefined} + aria-hidden={catalogCovered || undefined} + inert={catalogCovered || undefined} > ) )} + + {/* Sits above the detail overlay (z-30): opening settings from a row + while a model preview is open should show the settings, not stack + behind it. */} + {settingsTarget && ( +
+ setSettingsTarget(null)} + onRun={runSettingsTarget} + compact={splitMode} + /> +
+ )}
; + +/** + * The key one model's config is stored under. + * + * Uses the `repo:VARIANT` form an OpenAI request names a quant by, so two quants + * of the same repo keep separate configs and the backend can match the requested + * model name directly. Falls back to the bare id when there is no variant. + */ +export function modelOverrideKey( + modelId: string, + ggufVariant?: string | null, +): string { + return ggufVariant ? `${modelId}:${ggufVariant}` : modelId; +} + +export async function fetchModelOverrides(): Promise { + const res = await authFetch(OVERRIDES_URL); + if (!res.ok) { + throw new Error( + await readFastApiError(res, "Failed to load saved model settings"), + ); + } + const body = (await res.json()) as { overrides?: ApiModelOverrides }; + return body.overrides ?? {}; +} + +/** + * Translate the UI's per-model config into the backend's schema. + * + * Only fields the user actually set are sent: the backend reads an absent field + * as "use the app default", so sending nulls would pin defaults and stop the + * model following later changes to the global preferences. `null` config means + * "no saved settings", which clears the entry. + */ +function toApiOverride(config: PerModelConfig | null): ApiModelOverride { + if (!config) { + return {}; + } + const payload: ApiModelOverride = {}; + if (config.maxSeqLength && config.maxSeqLength > 0) { + payload.max_seq_length = config.maxSeqLength; + } + if (config.customContextLength && config.customContextLength > 0) { + payload.custom_context_length = config.customContextLength; + } + if (config.kvCacheDtype) { + payload.kv_cache_dtype = config.kvCacheDtype; + } + if (config.speculativeType) { + payload.speculative_type = config.speculativeType; + } + if (config.specDraftNMax && config.specDraftNMax > 0) { + payload.spec_draft_n_max = config.specDraftNMax; + } + if (config.tensorParallel) { + payload.tensor_parallel = true; + } + if (config.chatTemplateOverride?.trim()) { + payload.chat_template_override = config.chatTemplateOverride; + } + // Only "manual" is a real override; "auto" is the follow-the-global default. + if (config.gpuMemoryMode === "manual") { + payload.gpu_memory_mode = "manual"; + } + // gpuLayers < 0 is Auto, which is also the default. + if (typeof config.gpuLayers === "number" && config.gpuLayers >= 0) { + payload.gpu_layers = config.gpuLayers; + } + if (typeof config.nCpuMoe === "number" && config.nCpuMoe > 0) { + payload.n_cpu_moe = config.nCpuMoe; + } + if (config.selectedGpuIds && config.selectedGpuIds.length > 0) { + payload.gpu_ids = config.selectedGpuIds; + } + return payload; +} + +export async function putModelOverride( + modelId: string, + ggufVariant: string | null | undefined, + config: PerModelConfig | null, +): Promise { + const res = await authFetch(OVERRIDES_URL, { + method: "PUT", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + // biome-ignore lint/style/useNamingConvention: API schema + model_id: modelOverrideKey(modelId, ggufVariant), + // Launch flags have no UI control, so the backend preserves them when the + // field is omitted. Forgetting a model means forgetting all of it, so that + // path sends an explicit empty list to clear them. + // biome-ignore lint/style/useNamingConvention: API schema + ...(config === null ? { llama_extra_args: [] } : {}), + ...toApiOverride(config), + }), + }); + if (!res.ok) { + throw new Error( + await readFastApiError(res, "Failed to save model settings for the API"), + ); + } +} + +/** + * Mirror a per-model config save to the backend without blocking the UI. + * + * Deliberately best-effort: the localStorage write is the source of truth for + * this browser and has already happened by the time this runs, so a failed sync + * must not fail the save or interrupt a model load. It is logged rather than + * toasted -- the only consequence is that an API-triggered load of this model + * falls back to app defaults until the next successful save. + */ +export function syncModelOverride( + modelId: string, + ggufVariant: string | null | undefined, + config: PerModelConfig | null, +): void { + void putModelOverride(modelId, ggufVariant, config).catch( + (error: unknown) => { + console.warn( + "Failed to mirror model settings to the server; an API load of this model will use defaults.", + error, + ); + }, + ); +} diff --git a/studio/frontend/src/features/model-picker/components/model-config-page.tsx b/studio/frontend/src/features/model-picker/components/model-config-page.tsx index 90202a2bcf..edf0cb96ea 100644 --- a/studio/frontend/src/features/model-picker/components/model-config-page.tsx +++ b/studio/frontend/src/features/model-picker/components/model-config-page.tsx @@ -55,6 +55,7 @@ import { resolveInitialConfig, savePerModelConfig, } from "../model-config/per-model-config"; +import { syncModelOverride } from "../api/model-overrides"; import { ChatTemplateEditorDialog } from "./chat-template-editor-dialog"; import type { ModelPickTarget } from "./model-selector/types"; import { @@ -578,6 +579,12 @@ interface ModelConfigPageProps { loadedContextLength?: number | null; initialConfig?: PerModelConfig | null; variant?: "page" | "sidebar"; + /** + * Page variant only: render the built-in "Run settings" title block. A host + * that already shows the model name as its own page heading (the Hub's + * settings page) turns this off so the name is not printed twice. + */ + showHeader?: boolean; } export function ModelConfigPage({ @@ -588,6 +595,7 @@ export function ModelConfigPage({ loadedContextLength = null, initialConfig = null, variant = "page", + showHeader = true, }: ModelConfigPageProps) { const rememberId = useId(); const isActiveModel = loadedConfig != null; @@ -870,6 +878,21 @@ export function ModelConfigPage({ } else { saveFailed = !deletePerModelConfig(target.id, target.ggufVariant); } + // Mirror to the server so an OpenAI-compatible API request that loads this + // model gets these exact settings, not app defaults. Best-effort and + // non-blocking: the localStorage write above already governs this browser. + // Forgetting clears the server entry too, so the two never disagree. + // + // Skipped when the local write failed (quota, a future-schema entry): the + // browser and the server would otherwise permanently disagree about this + // model, with no way for the user to tell which one the next load used. + if (!saveFailed) { + syncModelOverride( + target.id, + target.ggufVariant, + remember ? effectiveRuntimeConfig : null, + ); + } if (effectivePersistenceOnly) { if (saveFailed) { toast.error("Couldn't save settings for this model."); @@ -898,7 +921,7 @@ export function ModelConfigPage({ return (
- {variant === "page" && ( + {variant === "page" && showHeader && (
{onBack && ( - - {expanded ? ( -
-
-
-
- Prompt - {entry.prompt_truncated && !detail ? Preview : null} -
- 
-                {loading && !detail ? "Loading..." : prompt || "No prompt text"}
-
-
-
-
- Reply - {entry.reply_truncated && !detail ? Preview : null} -
- 
-                {loading && !detail ? "Loading..." : reply}
-
-
-
- -
- {formatTokens(entry)} - {entry.context_length ? ( - <> / {entry.context_length.toLocaleString()} context - ) : null} - -
-
- ) : null} - - ); -} - -export function ApiMonitorConsole(): ReactElement { - const [data, setData] = useState(null); - const [error, setError] = useState(null); - const [refreshing, setRefreshing] = useState(false); - const [expandedIds, setExpandedIds] = useState>(() => new Set()); - const [details, setDetails] = useState>({}); - const [loadingDetails, setLoadingDetails] = useState>( - () => new Set(), - ); - const loadingDetailsRef = useRef>(new Set()); - const detailsRef = useRef>({}); - - const loadMonitor = useCallback(async (): Promise => { - setRefreshing(true); - try { - setData(await getApiMonitor()); - setError(null); - } catch (err: unknown) { - setError(err instanceof Error ? err.message : "Monitor unavailable"); - } finally { - setRefreshing(false); - } - }, []); - - useEffect(() => { - let cancelled = false; - let timer: number | undefined; - - function schedule(): void { - timer = window.setTimeout(poll, 1500); - } - - function poll(): void { - getApiMonitor() - .then((next) => { - if (cancelled) { - return; - } - setData(next); - setError(null); - }) - .catch((err: unknown) => { - if (cancelled) { - return; - } - setError(err instanceof Error ? err.message : "Monitor unavailable"); - }) - .finally(() => { - if (!cancelled) { - schedule(); - } - }); - } - - poll(); - return () => { - cancelled = true; - if (timer !== undefined) { - window.clearTimeout(timer); - } - }; - }, []); - - const statusLabel = data?.status ?? "idle"; - const hasActive = (data?.active_requests ?? 0) > 0; - const entries = useMemo(() => data?.entries ?? [], [data]); - const loadDetail = useCallback( - (id: string): void => { - if (loadingDetailsRef.current.has(id)) { - return; - } - loadingDetailsRef.current.add(id); - setLoadingDetails((prev) => new Set(prev).add(id)); - getApiMonitorEntry(id) - .then((entry) => { - setDetails((prev) => { - const next = { ...prev, [id]: entry }; - detailsRef.current = next; - return next; - }); - }) - .catch(() => { - setDetails((prev) => { - const next = { ...prev }; - delete next[id]; - detailsRef.current = next; - return next; - }); - }) - .finally(() => { - loadingDetailsRef.current.delete(id); - setLoadingDetails((prev) => { - const next = new Set(prev); - next.delete(id); - return next; - }); - }); - }, - [], - ); - - const toggleEntry = useCallback( - (entry: ApiMonitorEntry): void => { - setExpandedIds((prev) => { - const next = new Set(prev); - if (next.has(entry.id)) { - next.delete(entry.id); - } else { - next.add(entry.id); - loadDetail(entry.id); - } - return next; - }); - }, - [loadDetail], - ); - - useEffect(() => { - for (const entry of entries) { - if (!expandedIds.has(entry.id)) { - continue; - } - const cached = detailsRef.current[entry.id]; - if (!cached || cached.status !== entry.status || entry.status === "running") { - loadDetail(entry.id); - } - } - }, [entries, expandedIds, loadDetail]); - - return ( -
-
-
-
- - {hasActive ? ( - - ) : null} -
-
-

- API monitor -

-

- {data?.active_model ?? "No model loaded"} -

-
-
-
-
- {statusLabel} -
- -
-
- -
- - {(data?.active_requests ?? 0).toLocaleString()} active /{" "} - {entries.length.toLocaleString()} recent - - {data?.context_length ? ( - {data.context_length.toLocaleString()} context - ) : null} -
- -
- {error ? ( -
- {error} -
- ) : entries.length === 0 ? ( -
- No API traffic yet -
- ) : ( -
- {entries.map((entry) => ( - toggleEntry(entry)} - /> - ))} -
- )} -
-
- ); -} diff --git a/studio/frontend/src/features/settings/components/monitor-link.tsx b/studio/frontend/src/features/settings/components/monitor-link.tsx new file mode 100644 index 0000000000..a0547fdc79 --- /dev/null +++ b/studio/frontend/src/features/settings/components/monitor-link.tsx @@ -0,0 +1,97 @@ +// SPDX-License-Identifier: AGPL-3.0-only +// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 + +// The monitor moved out of this tab and onto its own page. It is normally +// reached from the floating panel; this card is the way in from Settings. + +import { Switch } from "@/components/ui/switch"; +import { useApiMonitorOverlayStore } from "@/features/api-monitor"; +import { getApiMonitor } from "@/features/chat/api/chat-api"; +import type { ApiMonitorResponse } from "@/features/chat/types/api"; +import { cn } from "@/lib/utils"; +import { ActivityIcon, ArrowRight02Icon } from "@hugeicons/core-free-icons"; +import { HugeiconsIcon } from "@hugeicons/react"; +import { useNavigate } from "@tanstack/react-router"; +import { type ReactElement, useEffect, useState } from "react"; +import { useSettingsDialogStore } from "../stores/settings-dialog-store"; + +export function MonitorLink(): ReactElement { + const navigate = useNavigate(); + const [data, setData] = useState(null); + const autoOpen = useApiMonitorOverlayStore((s) => s.autoOpen); + const setAutoOpen = useApiMonitorOverlayStore((s) => s.setAutoOpen); + + // One snapshot, not a poll: the live view is the monitor page. + useEffect(() => { + let cancelled = false; + void getApiMonitor() + .then((next) => { + if (!cancelled) setData(next); + }) + .catch(() => undefined); + return () => { + cancelled = true; + }; + }, []); + + const active = data?.active_requests ?? 0; + const recent = data?.entries.length ?? 0; + + return ( +
+ + + {/* Where the panel's own "stop opening this" gets turned back on. */} +
+ + + Show the floating monitor automatically + + + Opens a small panel when API traffic arrives. + + + +
+
+ ); +} diff --git a/studio/frontend/src/features/settings/tabs/api-keys-tab.tsx b/studio/frontend/src/features/settings/tabs/api-keys-tab.tsx index f1e503f7a3..39d934aacd 100644 --- a/studio/frontend/src/features/settings/tabs/api-keys-tab.tsx +++ b/studio/frontend/src/features/settings/tabs/api-keys-tab.tsx @@ -14,7 +14,7 @@ import { translate, useT } from "@/i18n"; import { AnimatePresence, motion, useReducedMotion } from "motion/react"; import { useCallback, useEffect, useState } from "react"; import { fetchApiKeys, revokeApiKey, type ApiKey } from "../api/api-keys"; -import { ApiMonitorConsole } from "../components/api-monitor-console"; +import { MonitorLink } from "../components/monitor-link"; import { ApiKeyRow } from "../components/api-key-row"; import { CreateKeyForm } from "../components/create-key-form"; import { ModelAutoSwitchSection } from "../components/model-auto-switch-section"; @@ -168,7 +168,7 @@ export function ApiKeysTab() { )} - +