diff --git a/studio/backend/core/inference/api_monitor.py b/studio/backend/core/inference/api_monitor.py index b637ba56d1..6239361dcb 100644 --- a/studio/backend/core/inference/api_monitor.py +++ b/studio/backend/core/inference/api_monitor.py @@ -49,7 +49,12 @@ class ApiMonitorEntry: status: str started_at: float updated_at: float + # Who this row belongs to. On a shared lifecycle row it does not restrict + # visibility (see _visible); it names the caller the row is attributed to. subject: Optional[str] = None + # True for sk-unsloth key callers, not UI sessions. The floating panel only + # auto-opens for these, so Studio's own chat does not pop it mid-chat. + via_api_key: bool = False # Monotonic anchors so duration math survives wall-clock steps (NTP). started_monotonic: float = 0.0 finished_monotonic: Optional[float] = None @@ -69,7 +74,12 @@ class ApiMonitorEntry: # 0-100 for a running download row; None when not applicable. progress: Optional[float] = None - def snapshot(self, *, include_details: bool = True) -> dict[str, Any]: + def snapshot( + self, + *, + include_details: bool = True, + attributed: bool = True, + ) -> dict[str, Any]: duration_ms = None if self.finished_monotonic is not None: duration_ms = max( @@ -86,6 +96,10 @@ class ApiMonitorEntry: "endpoint": self.endpoint, "method": self.method, "model": self.model, + # A lifecycle row is shared, so it reaches subjects that had nothing to + # do with it. The overlay auto-opens on this flag, so report it only to + # the caller the row is attributed to; everyone else still sees the row. + "via_api_key": self.via_api_key and attributed, "prompt_preview": _trim(self.prompt, _PREVIEW_CHARS), "reply_preview": _trim(self.reply, _PREVIEW_CHARS), "prompt_truncated": len(self.prompt) > _PREVIEW_CHARS, @@ -120,6 +134,9 @@ class ApiMonitor: enabled: bool = True, ): self._entries: deque[ApiMonitorEntry] = deque() + # Shared rows one subject cleared. Deleting them would erase another + # caller's history; keeping them makes "Clear log" look broken on reload. + self._hidden_shared: dict[str, set[str]] = {} self._max_entries = max(0, max_entries) self._lock = threading.Lock() self._enabled = enabled @@ -133,6 +150,7 @@ class ApiMonitor: prompt: str, context_length: Optional[int] = None, subject: Optional[str] = None, + via_api_key: bool = False, ) -> str: if not self._enabled: return "" @@ -141,12 +159,15 @@ class ApiMonitor: id = f"apireq_{uuid.uuid4().hex[:12]}", endpoint = endpoint, method = method, - model = model or "default", + # str(): a raw JSON body can carry any type, and a non-string + # breaks the UI that renders it. + model = str(model) if model else "default", prompt = _trim(prompt, _MAX_PROMPT_CHARS), status = "running", started_at = now, updated_at = now, subject = subject, + via_api_key = via_api_key, started_monotonic = time.monotonic(), context_length = context_length, ) @@ -162,12 +183,18 @@ class ApiMonitor: model: str, reason: Optional[str] = None, running: bool = False, + via_api_key: bool = False, + subject: Optional[str] = None, ) -> str: """Record a model load/unload alongside the request traffic that caused it. ``running=True`` opens the row for the caller to close with :meth:`finish` / :meth:`fail`; an unload is terminal on arrival. Rows are shared (visible to every subject) and share the request retention budget. + + ``subject`` names the caller whose request drove this, which is what + ``via_api_key`` is reported to; it does not narrow who sees the row. Pass it + whenever ``via_api_key`` is set, or the attribution reaches nobody. """ if not self._enabled: return "" @@ -188,6 +215,14 @@ class ApiMonitor: event = event, reason = reason, shared = True, + # The overlay opens on API-key traffic only. A switch or download that + # is refused never reaches api_monitor.start, so this row is the whole + # trace of it, and without the attribution the monitor stayed shut on + # exactly the failures it exists to surface. + via_api_key = via_api_key, + # Shared rows are read by every subject, so the attribution needs an + # owner or the pop-open lands in browsers that did not cause it. + subject = subject, ) with self._lock: self._entries.appendleft(entry) @@ -354,7 +389,10 @@ class ApiMonitor: ) -> list[dict[str, Any]]: with self._lock: return [ - entry.snapshot(include_details = include_details) + entry.snapshot( + include_details = include_details, + attributed = self._attributed(entry, subject), + ) for entry in self._entries if self._visible(entry, subject) ] @@ -371,7 +409,10 @@ class ApiMonitor: return None if not self._visible(entry, subject): return None - return entry.snapshot(include_details = True) + return entry.snapshot( + include_details = True, + attributed = self._attributed(entry, subject), + ) def active_count(self, *, subject: Optional[str] = None) -> int: # Lifecycle rows show as "running" while loading but are not in-flight API requests. @@ -384,13 +425,48 @@ class ApiMonitor: and (subject is None or entry.subject == subject) ) - def clear(self) -> None: - with self._lock: - self._entries.clear() + def clear(self, *, subject: Optional[str] = None) -> None: + """Drop recorded entries. ``subject`` limits the wipe to one caller's. - @staticmethod - def _visible(entry: ApiMonitorEntry, subject: Optional[str]) -> bool: - return subject is None or entry.subject == subject or entry.shared + Every other read on this class is subject-scoped, so an unscoped clear + would let one user erase another's history (and zero their active count + mid-generation). Callers that genuinely mean "everything" pass None. + """ + with self._lock: + if subject is None: + self._entries.clear() + self._hidden_shared.clear() + return + # A running shared row is a load in progress, not history, so it stays. + hidden = self._hidden_shared.setdefault(subject, set()) + for entry in self._entries: + if entry.shared and entry.status != "running": + hidden.add(entry.id) + # Shared rows are hidden, never dropped, even when this subject owns + # one: they are another caller's history too, and an owned shared row + # is exactly what an API-key load produces. + self._entries = deque( + entry for entry in self._entries if entry.shared or entry.subject != subject + ) + + def _visible(self, entry: ApiMonitorEntry, subject: Optional[str]) -> bool: + if subject is None: + return True + if entry.shared: + # Shared rows reach every subject, minus the ones this one cleared. + # Checked before ownership so clearing hides a subject's own rows too. + return entry.id not in self._hidden_shared.get(subject, ()) + return entry.subject == subject + + def _attributed(self, entry: ApiMonitorEntry, subject: Optional[str]) -> bool: + """Whether *subject* is the caller this row's API traffic belongs to. + + Only they should have the overlay pop open for it. An unscoped read (no + subject: internal callers and tests) sees the row's own flag. + """ + if subject is None: + return True + return entry.subject == subject def _find_locked(self, entry_id: str) -> Optional[ApiMonitorEntry]: for entry in self._entries: @@ -409,6 +485,12 @@ class ApiMonitor: kept.append(entry) terminal_seen += 1 self._entries = kept + # Keep hidden sets to live rows so they stay bounded by the ring buffer. + live = {entry.id for entry in kept} + for subject, hidden in list(self._hidden_shared.items()): + hidden &= live + if not hidden: + del self._hidden_shared[subject] api_monitor = ApiMonitor(enabled = not _api_monitor_disabled()) diff --git a/studio/backend/core/inference/local_model_resolver.py b/studio/backend/core/inference/local_model_resolver.py index 5d2a9e9c87..07d00f0374 100644 --- a/studio/backend/core/inference/local_model_resolver.py +++ b/studio/backend/core/inference/local_model_resolver.py @@ -47,10 +47,18 @@ _WARM_DUTY = 10.0 def _is_abs_path_id(value: str) -> bool: """True when an id is an absolute filesystem path (the ./models and LM Studio - scanners use the on-disk path as the id) rather than a repo id like org/name.""" - from pathlib import Path + scanners use the on-disk path as the id) rather than a repo id like org/name. + + Both spellings count on every host. Path() follows the running OS, so a + Windows backend read "/home/me/x.gguf" as relative and a POSIX one read + "C:\\models\\x.gguf" the same way, and either then reached /v1/models as a + published id. Ids outlive the machine that wrote them: settings sync, a WSL + session and a copied config all carry the other platform's spelling, and the + model-override identity already folds both. Neither reading can misfire on a + repo id, which has no leading separator, drive or UNC prefix.""" + from pathlib import PurePosixPath, PureWindowsPath try: - return Path(value).is_absolute() + return PurePosixPath(value).is_absolute() or PureWindowsPath(value).is_absolute() except Exception: return False diff --git a/studio/backend/core/inference/openai_auto_download.py b/studio/backend/core/inference/openai_auto_download.py index cad5e40d14..7252d3382d 100644 --- a/studio/backend/core/inference/openai_auto_download.py +++ b/studio/backend/core/inference/openai_auto_download.py @@ -448,6 +448,8 @@ async def maybe_auto_download( *, hf_token: Optional[str] = None, require_vision: bool = False, + subject: Optional[str] = None, + via_api_key: bool = False, ) -> Optional[AutoDownloadRefusal]: """Start (or report on) a background fetch of *requested_model*. @@ -457,6 +459,10 @@ async def maybe_auto_download( ``require_vision`` refuses a target with no mmproj companion rather than spend gigabytes on weights that cannot answer the request; the local capability guard only ever sees an already-downloaded model. + + ``subject`` and ``via_api_key`` describe the caller for the monitor row this + opens: the same /v1 endpoints serve Studio's own chat on a session JWT, so the + download is not API-key traffic unless the request that asked for it was. """ global _active @@ -529,7 +535,14 @@ async def maybe_auto_download( try: return await _admit_and_start( - repo_id, wanted_variant, requested_model, hf_token, provisional, require_vision + repo_id, + wanted_variant, + requested_model, + hf_token, + provisional, + require_vision, + subject = subject, + via_api_key = via_api_key, ) except BaseException: # Not `except Exception`: a cancel mid-probe would otherwise wedge the provisional slot. @@ -544,6 +557,9 @@ async def _admit_and_start( hf_token: Optional[str], active: _Active, require_vision: bool = False, + *, + subject: Optional[str] = None, + via_api_key: bool = False, ) -> Optional[AutoDownloadRefusal]: from hub.utils.hf_errors import hf_error_status @@ -690,7 +706,16 @@ async def _admit_and_start( ), ) - return await _dispatch(repo_id, variant, expected_bytes, requested_model, hf_token, active) + return await _dispatch( + repo_id, + variant, + expected_bytes, + requested_model, + hf_token, + active, + subject = subject, + via_api_key = via_api_key, + ) def preferred_quant(labels) -> Optional[str]: @@ -737,6 +762,9 @@ async def _dispatch( requested_model: str, hf_token: Optional[str], active: _Active, + *, + subject: Optional[str] = None, + via_api_key: bool = False, ) -> AutoDownloadRefusal: global _active @@ -777,7 +805,18 @@ async def _dispatch( return busy monitor_id = api_monitor.record_lifecycle( - event = "download", model = label, reason = "api", running = True + # Reason "api" because only a /v1 request reaches auto-download. That is not + # the same as API-key traffic though: Studio's own chat calls those same + # endpoints with a session JWT, and marking its download as API traffic pops + # the overlay mid-chat, which via_api_key exists to prevent. So take the + # attribution from the request, and name the caller it belongs to, since the + # row is shared with every other subject. + event = "download", + model = label, + reason = "api", + running = True, + via_api_key = via_api_key, + subject = subject, ) with _lock: if _active is active: diff --git a/studio/backend/picker/schemas.py b/studio/backend/picker/schemas.py index b4f956188f..c43d769817 100644 --- a/studio/backend/picker/schemas.py +++ b/studio/backend/picker/schemas.py @@ -11,13 +11,29 @@ from pydantic import BaseModel, Field, field_validator MAX_CHAT_TEMPLATE_BYTES = 65_536 +def chat_template_byte_length(value: str) -> Optional[int]: + """UTF-8 length, or None if the string cannot be encoded at all. + + JSON can carry an unpaired surrogate, as a truncated emoji paste produces. + json decodes it fine and .encode("utf-8") then raises. Callers treat None as + "reject": such a template can never render. + """ + try: + return len(value.encode("utf-8")) + except UnicodeEncodeError: + return None + + class ValidateChatTemplateRequest(BaseModel): template: str = Field(default = "") @field_validator("template") @classmethod def _enforce_template_size(cls, value: str) -> str: - if len(value.encode("utf-8")) > MAX_CHAT_TEMPLATE_BYTES: + size = chat_template_byte_length(value) + if size is None: + raise ValueError("Chat template contains unpaired surrogate characters.") + if size > MAX_CHAT_TEMPLATE_BYTES: raise ValueError(f"Chat template exceeds the {MAX_CHAT_TEMPLATE_BYTES}-byte limit.") return value diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.py index 12547277f5..378d36788a 100644 --- a/studio/backend/routes/inference.py +++ b/studio/backend/routes/inference.py @@ -1805,8 +1805,25 @@ from core.inference.anthropic_compat import ( AnthropicStreamEmitter, AnthropicPassthroughEmitter, ) -from auth.authentication import get_current_subject +from auth.authentication import API_KEY_PREFIX, get_current_subject from state import active_generations + + +def _request_used_api_key(request: Any) -> bool: + """True when this request authenticated with an sk-unsloth key. + + Studio's own chat hits these same endpoints with a session JWT, so this is + what separates "someone is using Unsloth as an API server" from "someone is + using Unsloth". + """ + try: + header = request.headers.get("authorization") or "" + except Exception: + return False + scheme, _, token = header.partition(" ") + return scheme.lower() == "bearer" and token.startswith(API_KEY_PREFIX) + + from state.tool_approvals import resolve_tool_decision from core.inference.key_exchange import decrypt_api_key @@ -3864,6 +3881,7 @@ async def _maybe_auto_download_model( fastapi_request: Optional[Request], *, require_vision: bool = False, + current_subject: Optional[str] = None, ) -> None: """Opt-in: start fetching a named GGUF this server doesn't have. @@ -3886,6 +3904,10 @@ async def _maybe_auto_download_model( requested_model, hf_token = _auto_download_hf_token(fastapi_request), require_vision = require_vision, + subject = current_subject, + # These /v1 endpoints also serve Studio's own chat on a session JWT, + # so only mark the download row as API traffic when it really is. + via_api_key = _request_used_api_key(fastapi_request), ) except Exception as exc: # Never turn a servable request into a 500 over the download attempt. @@ -3905,6 +3927,7 @@ async def _maybe_auto_download_model( if isinstance(path, str) else refusal.message ) + _record_refused_request(fastapi_request, requested_model, refusal, current_subject) raise HTTPException( status_code = refusal.status, detail = detail, @@ -3912,6 +3935,35 @@ async def _maybe_auto_download_model( ) +def _record_refused_request( + fastapi_request: Optional[Request], + requested_model: str, + refusal: Any, + current_subject: Optional[str], +) -> None: + """Log the refused call itself, not just the download it is waiting on. + + The refusal replaces the request, so the handler's own ``api_monitor.start`` + never runs. Only the caller that dispatched a download gets a row from + ``record_lifecycle``; anyone refused while it runs left no trace at all, and a + download some other caller started carries their attribution, so an API-key + client waiting on it never opened the overlay and read as Studio's own traffic. + """ + state = getattr(fastapi_request, "state", None) + if getattr(state, "skip_api_monitor", False): + return + path = getattr(getattr(fastapi_request, "url", None), "path", None) + entry_id = api_monitor.start( + endpoint = path if isinstance(path, str) else "/v1", + method = str(getattr(fastapi_request, "method", "") or "POST"), + model = requested_model, + prompt = "", + subject = current_subject, + via_api_key = _request_used_api_key(fastapi_request), + ) + api_monitor.fail(entry_id, refusal.message) + + def _loaded_satisfies(requested: str) -> bool: """Whether what is serving right now actually answers to *requested*. @@ -4214,6 +4266,7 @@ async def _maybe_auto_switch_model( get_openai_auto_switch_enabled, get_auto_unload_idle_seconds, get_model_override, + model_override_load_kwargs, ) from core.inference.local_model_resolver import resolve_local_gguf from core.inference.llama_keepwarm import ( @@ -4254,7 +4307,10 @@ async def _maybe_auto_switch_model( # Not on disk. Opt-in: fetch in the background and ask the caller to retry. if auto_switch_on and not reload_only: await _maybe_auto_download_model( - requested_model, fastapi_request, require_vision = require_vision + requested_model, + fastapi_request, + require_vision = require_vision, + current_subject = current_subject, ) # Idle-unload may have freed the model; reload exactly what it freed # (path + quant + advertised id) so an alias/unknown name stays servable @@ -4362,22 +4418,94 @@ async def _maybe_auto_switch_model( if _already_serving(): _record_serving_alias() return - # Apply this model's saved launch flags so the swap honors the config. - override = get_model_override(override_id) + # Apply this model's saved launch config so an API swap loads + # it exactly as the picker would. Try variant-qualified keys + # first (two quants of one repo can differ), then bare ids, and + # within each pair the concrete load path before the advertised + # id. The settings UI keys every local row (a folder, an LM + # Studio dir, a non-active HF cache, a loose .gguf) by that path, + # while override_id is a derived alias -- the /v1/models name a + # hand-written overrides PUT uses, and for a loose file only its + # filename stem. Reading the alias first let an older entry under + # it shadow the settings the user just saved, for good. A cached + # repo is keyed by its repo id, which is override_id, and no path + # entry exists for it, so it still resolves on the second try. + # A standalone .gguf resolves with variant=None; an early build + # of this feature keyed it by the quant label derived from the + # filename, so read ":LABEL" too, after the bare path the + # picker writes today. + file_variant = None + if not variant and target_id.lower().endswith(".gguf"): + from hub.utils.gguf import extract_quant_label + file_variant = extract_quant_label(os.path.basename(target_id)) + override = {} + for override_key in ( + f"{target_id}:{variant}" if variant else None, + f"{override_id}:{variant}" if variant else None, + target_id, + f"{target_id}:{file_variant}" if file_variant else None, + override_id, + ): + if not override_key: + continue + override = get_model_override(override_key) + if override: + break load_kwargs = {"model_path": target_id, "gguf_variant": variant} - if override.get("llama_extra_args") is not None: - load_kwargs["llama_extra_args"] = override["llama_extra_args"] - if override.get("max_seq_length") is not None: - load_kwargs["max_seq_length"] = override["max_seq_length"] + load_kwargs.update( + model_override_load_kwargs( + override, + # Set for every GGUF the resolver returns; the + # reload-stash path carries the quant it froze. + is_gguf = bool(variant) or target_id.lower().endswith(".gguf"), + ) + ) + saved_gpu_ids = load_kwargs.get("gpu_ids") + if saved_gpu_ids and not await _override_gpu_ids_still_resolve( + saved_gpu_ids + ): + # Stale pin (GPU removed, mask changed, another host). + # Dropping the one dead field beats 400ing the whole load. + load_kwargs.pop("gpu_ids", None) + logger.warning( + "Dropping saved gpu_ids %s for %s: not available here.", + saved_gpu_ids, + override_id, + ) # Reuse the load impl so its dedup, tensor fallback, and threading # apply. Call the impl directly: we already hold the lifecycle gate # the /load route would otherwise take, so the route would deadlock. - await _load_model_impl( - LoadRequest(**load_kwargs), - fastapi_request, - current_subject, - current_request_counted = True, - ) + try: + await _load_model_impl( + LoadRequest(**load_kwargs), + fastapi_request, + current_subject, + current_request_counted = True, + ) + except HTTPException as exc: + # The pre-flight check cannot mirror every gpu_ids rule the + # loader applies (a Vulkan diffusion GGUF refuses GPU + # selection outright). Retry once without the saved pin: a + # stale placement preference must never block a request. + if not ( + exc.status_code == 400 + and load_kwargs.get("gpu_ids") + and "gpu" in str(exc.detail).lower() + ): + raise + logger.warning( + "Retrying %s without saved gpu_ids %s: %s", + override_id, + load_kwargs.get("gpu_ids"), + exc.detail, + ) + load_kwargs.pop("gpu_ids", None) + await _load_model_impl( + LoadRequest(**load_kwargs), + fastapi_request, + current_subject, + current_request_counted = True, + ) # Advertise the repo id (not the concrete load path) as the loaded # model's public id and override key for /v1/models and idle stash. get_llama_cpp_backend()._openai_advertised_id = override_id @@ -4636,6 +4764,43 @@ def _classify_diffusion_gguf(config: ModelConfig) -> Optional[bool]: return True if name_says_diffusion else None +async def _override_gpu_ids_still_resolve(gpu_ids: List[int]) -> bool: + """Whether a per-model GPU pin is usable on this machine right now. + + normalize_model_override cannot know the device list, so it stores whatever + was valid where the config was written. This is the load-time reconciliation + for the device-availability rules, which are the ones that go stale. + + Deliberately not exhaustive: model-dependent rules (a Vulkan diffusion GGUF + refuses gpu_ids outright) need a ModelConfig this has no reason to build. + The caller's retry-without-the-pin covers those, and covers rules added + later, so a check missing here costs one extra attempt, not the load. + """ + try: + from utils.hardware import DeviceType, get_device + from utils.hardware.hardware import resolve_requested_gpu_ids + + is_vulkan = LlamaCppBackend._is_vulkan_backend() + if get_device() == DeviceType.XPU and not is_vulkan: + # Rejected outright on XPU. + return False + resolved = resolve_requested_gpu_ids(gpu_ids, is_vulkan = is_vulkan) + if is_vulkan and resolved: + # Vulkan ordinals are their own index space, so resolve() only rejects + # malformed ones; presence needs the ggml probe the load runs. + binary = LlamaCppBackend._find_llama_server_binary() + if binary: + probed = { + gpu[0] + for gpu in await asyncio.to_thread(LlamaCppBackend._get_gpu_memory, binary) + } + if not {int(gpu_id) for gpu_id in resolved}.issubset(probed): + return False + return True + except Exception: + return False + + async def _resolve_gguf_gpu_ids_for_request( config: ModelConfig, gpu_ids: Optional[List[int]] ) -> Optional[List[int]]: @@ -5234,6 +5399,12 @@ async def _load_model_impl( event = "load", model = _lifecycle_model_label(request.model_path, request.gguf_variant), running = True, + # Auto-switch loads run before the endpoint opens its request row, so a + # load that fails there leaves this as the only trace of API traffic. + via_api_key = _request_used_api_key(fastapi_request), + # The row is shared, so every subject reads it. Name the caller it belongs + # to or the overlay pops open in browsers that did not cause the traffic. + subject = current_subject, ) native_grant_backed = False @@ -6796,6 +6967,10 @@ async def get_api_monitor(current_subject: str = Depends(get_current_subject)): operating_status = "idle" return { "status": operating_status, + # The clock every entry's started_at is on. The floating monitor dates its + # first snapshot against this instead of the browser's clock, which need not + # agree with ours over a tunnel or from a container. + "server_time": time.time(), "active_model": active_model, "context_length": _monitor_context_length(), "active_requests": active_requests, @@ -6803,6 +6978,22 @@ async def get_api_monitor(current_subject: str = Depends(get_current_subject)): } +@studio_router.delete("/monitor") +async def clear_api_monitor(current_subject: str = Depends(get_current_subject)): + """Drop this caller's recorded API history so a debugging session starts clean. + + Scoped to the current subject, like every read on the monitor: an unscoped + wipe would erase another user's history and zero their active-request count + while their generation is still streaming. + + The caller's own in-flight requests are dropped from the log too; they keep + streaming to their client, they just stop being reported here (a later append + re-adds nothing, since the entry id no longer resolves). + """ + api_monitor.clear(subject = current_subject) + return {"cleared": True} + + @studio_router.get("/monitor/{entry_id}") async def get_api_monitor_entry(entry_id: str, current_subject: str = Depends(get_current_subject)): """Return full prompt/reply details for one OpenAI-compatible API request.""" @@ -8291,6 +8482,7 @@ async def _proxy_to_external_provider( if not getattr(request.state, "skip_api_monitor", False): monitor_id = api_monitor.start( endpoint = request.url.path, + via_api_key = _request_used_api_key(request), method = request.method, model = model, prompt = _monitor_prompt_from_messages(payload.messages), @@ -8851,6 +9043,7 @@ async def openai_chat_completions( if not getattr(request.state, "skip_api_monitor", False): tts_monitor_id = api_monitor.start( endpoint = request.url.path, + via_api_key = _request_used_api_key(request), method = request.method, model = model_label, prompt = _monitor_prompt_from_messages(payload.messages), @@ -8921,6 +9114,7 @@ async def openai_chat_completions( if not getattr(request.state, "skip_api_monitor", False): monitor_id = api_monitor.start( endpoint = request.url.path, + via_api_key = _request_used_api_key(request), method = request.method, model = model_name, prompt = _monitor_prompt_from_messages(payload.messages), @@ -9074,6 +9268,7 @@ async def openai_chat_completions( if monitor_id is None and not getattr(request.state, "skip_api_monitor", False): monitor_id = api_monitor.start( endpoint = request.url.path, + via_api_key = _request_used_api_key(request), method = request.method, model = model_name, prompt = _monitor_prompt_from_messages(payload.messages), @@ -12010,6 +12205,7 @@ async def openai_completions(request: Request, current_subject: str = Depends(ge monitor_model = str(body.get("model") or _llama_public_model_id(llama_backend) or "default") monitor_id = api_monitor.start( endpoint = request.url.path, + via_api_key = _request_used_api_key(request), method = request.method, model = monitor_model, prompt = prompt_text, @@ -12272,6 +12468,7 @@ async def openai_embeddings(request: Request, current_subject: str = Depends(get if not getattr(request.state, "skip_api_monitor", False): monitor_id = api_monitor.start( endpoint = request.url.path, + via_api_key = _request_used_api_key(request), method = request.method, model = str(body.get("model") or _llama_public_model_id(llama_backend) or "default"), prompt = prompt_text, @@ -12902,6 +13099,7 @@ async def _responses_non_streaming( monitor_id = api_monitor.start( endpoint = getattr(getattr(request, "url", None), "path", "/v1/responses"), method = getattr(request, "method", "POST"), + via_api_key = _request_used_api_key(request), model = payload.model, prompt = _monitor_prompt_from_messages(messages), context_length = _monitor_context_length(), @@ -14112,6 +14310,7 @@ async def openai_responses( if not getattr(request.state, "skip_api_monitor", False): monitor_id = api_monitor.start( endpoint = request.url.path, + via_api_key = _request_used_api_key(request), method = request.method, model = payload.model, prompt = _monitor_prompt_from_messages(messages), @@ -14597,6 +14796,7 @@ async def anthropic_messages( monitor_id = api_monitor.start( endpoint = getattr(request_url, "path", "/v1/messages"), method = getattr(request, "method", "POST"), + via_api_key = _request_used_api_key(request), model = model_name, prompt = _monitor_prompt_from_messages(openai_messages), context_length = monitor_context_length, diff --git a/studio/backend/routes/settings.py b/studio/backend/routes/settings.py index 7770c12a8a..0081864db1 100644 --- a/studio/backend/routes/settings.py +++ b/studio/backend/routes/settings.py @@ -2,7 +2,7 @@ # Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 import re -from typing import Literal, Optional +from typing import Any, Literal, Optional from urllib.parse import unquote, urlsplit from fastapi import APIRouter, Depends, HTTPException @@ -34,15 +34,20 @@ from utils.helper_precache_settings import ( helper_model_disabled_by_env, set_helper_precache_enabled, ) +from picker.schemas import MAX_CHAT_TEMPLATE_BYTES, chat_template_byte_length from utils.coding_agents import CODING_AGENTS, detect_installed_coding_agents from utils.openai_auto_switch_settings import ( DEFAULT_AUTO_UNLOAD_KEEP_KV, DEFAULT_OPENAI_AUTO_DOWNLOAD_ENABLED, DEFAULT_OPENAI_AUTO_SWITCH_ENABLED, + MAX_GPU_ID, + PARALLEL_SLOTS_MAX, + PARALLEL_SLOTS_MIN, get_auto_unload_idle_seconds, get_auto_unload_keep_kv, get_model_overrides, get_openai_auto_switch_enabled, + resolve_model_override_key, get_stored_auto_unload_idle_seconds, get_stored_openai_auto_download_enabled, set_model_override, @@ -130,12 +135,102 @@ class OpenAIAutoSwitchResponse(BaseModel): auto_download_model: bool = DEFAULT_OPENAI_AUTO_DOWNLOAD_ENABLED +# A quant suffix, as modelOverrideKey builds it. Matched against the loader's quant +# pattern, not a length heuristic: a POSIX path may hold a colon +# ("/models/foo:bar.gguf") and would otherwise inherit another model's flags. +_MAX_VARIANT_SUFFIX_LEN = 64 + +# A local model's id is its path plus an optional quant suffix, and +# LoadRequest.model_path is unbounded. A limit under PATH_MAX would 422 the server +# sync while the local save succeeded. +MAX_MODEL_OVERRIDE_KEY_LEN = 4096 + 1 + _MAX_VARIANT_SUFFIX_LEN + +# normalize_model_override keeps ids 0..MAX_GPU_ID, so a longer list cannot name a +# device the normalizer would store; it only makes it walk more duplicates. Bound +# it here so an oversized array is rejected at the boundary instead of costing CPU. +MAX_GPU_IDS = MAX_GPU_ID + 1 + + class ModelOverridePayload(BaseModel): - model_id: str = Field(..., min_length = 1) - llama_extra_args: list[str] = Field(default_factory = list) + """One model's saved launch config, applied when the API loads that model. + + Everything past ``model_id`` is optional and omitted means "app default", so a + payload carrying only ``model_id`` clears the entry. The bounds here mirror + ``LoadRequest`` so a bad value is rejected at the boundary instead of being + silently dropped by the normalizer; the enum-ish fields (KV dtype, speculative + mode) are left to it, since their valid sets follow the llama.cpp build. + """ + + model_id: str = Field(..., min_length = 1, max_length = MAX_MODEL_OVERRIDE_KEY_LEN) + # None means "leave the stored value alone": the settings UI has no control for + # launch flags and must not wipe them. An explicit [] clears them (forget). + llama_extra_args: Optional[list[str]] = None # ge=1: 0 is not a valid sequence length, and the setter drops a falsy value, # so reject it at the boundary instead of accepting then silently discarding it. max_seq_length: Optional[int] = Field(default = None, ge = 1, le = 1048576) + custom_context_length: Optional[int] = Field(default = None, ge = 1, le = 1048576) + kv_cache_dtype: Optional[str] = Field(default = None, max_length = 32) + speculative_type: Optional[str] = Field(default = None, max_length = 32) + spec_draft_n_max: Optional[int] = Field(default = None, ge = 1, le = 16) + # Parallel decode slots (llama-server --parallel), GGUF-only like the picker. + # None follows the server-wide default set at launch. + n_parallel: Optional[int] = Field(default = None, ge = PARALLEL_SLOTS_MIN, le = PARALLEL_SLOTS_MAX) + tensor_parallel: bool = False + # Validated in bytes below, not by max_length: pydantic counts characters, so a + # multi-byte template would pass here and be dropped by the UTF-8 normalizer. + chat_template_override: Optional[str] = None + gpu_memory_mode: Optional[Literal["auto", "manual"]] = None + # -1 is Auto (llama.cpp --fit sizes the offload); the normalizer treats it as unset. + gpu_layers: Optional[int] = Field(default = None, ge = -1, le = 1024) + n_cpu_moe: Optional[int] = Field(default = None, ge = 0, le = 1024) + gpu_ids: Optional[list[int]] = Field(default = None, max_length = MAX_GPU_IDS) + # Explicit intent: an all-default save carries no fields, which is shape + # identical to "forget this model". None keeps the legacy contract. + remove: Optional[bool] = None + # Fill in, don't replace: the one-time localStorage backfill reads the map once + # and then writes each model in turn, so another tab saving during that pass + # would be overwritten by this browser's older copy. The server reads and writes + # under one transaction, which costs no extra round trip. Field level, because an + # install upgraded from a release that stored only llama_extra_args and + # max_seq_length holds an entry the browser has the rest of, and skipping the + # whole entry would strand exactly the settings this migration exists to carry. + fill_absent_fields: bool = False + + @field_validator("chat_template_override") + @classmethod + def _limit_chat_template_bytes(cls, value: Optional[str]) -> Optional[str]: + # Mirrors LoadRequest.normalize_blank_chat_template_override. + if value is None: + return None + size = chat_template_byte_length(value) + if size is None: + raise ValueError("Chat template contains unpaired surrogate characters.") + if size > MAX_CHAT_TEMPLATE_BYTES: + raise ValueError(f"Chat template exceeds the {MAX_CHAT_TEMPLATE_BYTES}-byte limit.") + return value + + @field_validator( + "max_seq_length", + "custom_context_length", + "spec_draft_n_max", + "n_parallel", + "gpu_layers", + "n_cpu_moe", + "gpu_ids", + mode = "before", + ) + @classmethod + def _no_booleans(cls, value: Any) -> Any: + # bool subclasses int and pydantic parses non-strictly, so `true` arrives + # as 1 and `false` as 0: a payload could pin GPU 1 or set a one-token + # context. _bounded_int in the normalizer rejects bools for exactly that + # reason, but never sees one, because coercion happens here first. Reject + # only bools, so every other lax conversion the field relies on still runs. + if isinstance(value, bool): + raise ValueError("Expected a number, got a boolean.") + if isinstance(value, list) and any(isinstance(item, bool) for item in value): + raise ValueError("Expected numbers, got a boolean.") + return value class ModelOverridesResponse(BaseModel): @@ -294,18 +389,130 @@ def get_openai_auto_switch_overrides( return ModelOverridesResponse(overrides = get_model_overrides()) +def _bare_model_id(model_id: str) -> Optional[str]: + """``repo`` for a ``repo:QUANT`` key, or None when there is no quant suffix.""" + from utils.openai_auto_switch_settings import split_quant_suffix + + # Must look like a quant, not just a short path segment. Both a bits-per-weight + # modifier ("IQ4_XS-3.53bpw") and a stem fallback label count. + split = split_quant_suffix(model_id) + return split[0] if split is not None else None + + +def _legacy_standalone_gguf_key(model_id: str) -> Optional[str]: + """The stored ``:LABEL`` entry for a bare standalone .gguf path, if any. + + A loose file has no quant to choose between, so it is keyed by the bare path, + but the label derived from its filename is never empty and that is how the + picker keyed the same file before, so an upgraded install carries entries + under it. The auto-switch loader reads that spelling after the bare path + misses; resolve_model_override_key does not, since folding a POSIX path only + touches an existing suffix. None for an id that already names a quant, for a + repo id, and when nothing is stored under the derived key. + """ + import os + + if not model_id.lower().endswith(".gguf"): + return None + # Already qualified, so the caller named the entry it meant. Mirrors the + # loader, which derives a label only when the resolver gave it no variant. + if _bare_model_id(model_id) is not None: + return None + from hub.utils.gguf import extract_quant_label + + label = extract_quant_label(os.path.basename(model_id)) + if not label: + return None + # Through the resolver rather than a raw lookup: the browser lowercases the + # variant, and an ambiguous fold resolves to nothing rather than guessing. + return resolve_model_override_key(f"{model_id}:{label}") + + @router.put("/openai-auto-switch/overrides", response_model = ModelOverridesResponse) def update_openai_auto_switch_override( payload: ModelOverridePayload, current_subject: str = Depends(get_current_subject) ) -> ModelOverridesResponse: from core.inference.llama_server_args import validate_extra_args + from utils.openai_auto_switch_settings import get_model_override + try: - extra_args = validate_extra_args(payload.llama_extra_args) - set_model_override( - payload.model_id, - llama_extra_args = extra_args, - max_seq_length = payload.max_seq_length, + if payload.fill_absent_fields and payload.remove is True: + # A fill that is also a delete has no meaning, and silently picking one + # would either lose settings or resurrect them. + raise ValueError("fill_absent_fields cannot be combined with remove.") + # Only model_id is the documented "remove". Otherwise omitted launch flags + # carry over from the stored entry, since the settings UI cannot express them. + requested_extra_args = payload.llama_extra_args + # fill_absent_fields is a write mode, not a saved field: leaving it in would + # make every payload look non-empty and break the legacy "no fields means + # remove". + saved_fields = payload.model_dump( + exclude = {"model_id", "llama_extra_args", "remove", "fill_absent_fields"}, + exclude_none = True, ) + if payload.remove is not None: + is_removal = payload.remove + else: + is_removal = not payload.tensor_parallel and not { + key: value for key, value in saved_fields.items() if key != "tensor_parallel" + } + if requested_extra_args is None and not is_removal: + stored = get_model_override(payload.model_id) + # A fill leaves every stored value alone, so an entry that is already + # there keeps its flags without this echoing them back through + # validation: one accepted when it was saved but denylisted since would + # 400 the one-time migration, which then retries on every start. + if not (payload.fill_absent_fields and stored): + requested_extra_args = stored.get("llama_extra_args") + if requested_extra_args is None: + # First per-quant save for flags stored under the bare repo id. + # Auto-switch prefers the qualified entry, so carry them over. + bare_id = _bare_model_id(payload.model_id) + if bare_id: + requested_extra_args = get_model_override(bare_id).get("llama_extra_args") + # Not validated on an explicit remove: nothing is stored, so a 400 would only + # leave the override in place. A stale flag must not block forgetting. + extra_args = [] if payload.remove is True else validate_extra_args(requested_extra_args) + if payload.remove is True: + # An explicit remove wins over any other field in the payload. Remove the + # key a load resolves to, not the literal one sent: the browser normalizes + # casing before storing, so a stale entry would survive forgetting. + target_id = resolve_model_override_key(payload.model_id) or payload.model_id + set_model_override(target_id, llama_extra_args = [], max_seq_length = None) + # A standalone .gguf is keyed by its bare path now, but a load also + # reads the filename-derived :LABEL entry an upgraded install + # still holds. Clearing only what the resolver sees leaves that one + # applying to every later API load, with the settings gone from the + # UI and no way left to reach them. + legacy_id = _legacy_standalone_gguf_key(payload.model_id) + if legacy_id and legacy_id != target_id: + set_model_override( + legacy_id, + llama_extra_args = [], + max_seq_length = None, + ) + else: + # Save under the key a load resolves to, as the removal branch does. + # Saving the literal id leaves two keys for one model, which makes every + # other casing ambiguous and silently loses the settings. + target_id = resolve_model_override_key(payload.model_id) or payload.model_id + set_model_override( + target_id, + llama_extra_args = extra_args, + max_seq_length = payload.max_seq_length, + custom_context_length = payload.custom_context_length, + kv_cache_dtype = payload.kv_cache_dtype, + speculative_type = payload.speculative_type, + spec_draft_n_max = payload.spec_draft_n_max, + n_parallel = payload.n_parallel, + tensor_parallel = payload.tensor_parallel, + chat_template_override = payload.chat_template_override, + gpu_memory_mode = payload.gpu_memory_mode, + gpu_layers = payload.gpu_layers, + n_cpu_moe = payload.n_cpu_moe, + gpu_ids = payload.gpu_ids, + fill_absent_fields = payload.fill_absent_fields, + ) except ValueError as exc: raise log_and_http_error( exc, diff --git a/studio/backend/storage/studio_db.py b/studio/backend/storage/studio_db.py index e1e2953fe7..46488698cd 100644 --- a/studio/backend/storage/studio_db.py +++ b/studio/backend/storage/studio_db.py @@ -2959,11 +2959,25 @@ def upsert_app_settings(settings: dict[str, Any]) -> dict[str, Any]: def upsert_app_setting_map_entry( - key: str, entry_key: str, entry_value: dict[str, Any] | None + key: str, + entry_key: str, + entry_value: dict[str, Any] | None, + *, + fill_absent_fields: bool = False, ) -> dict[str, Any]: """Set (or delete, when entry_value is falsy) one sub-entry of a dict-valued app setting, atomically under BEGIN IMMEDIATE so concurrent writers to other - sub-entries cannot drop each other's updates.""" + sub-entries cannot drop each other's updates. + + ``fill_absent_fields`` writes only what is missing: the entry is created when + it is not there, and otherwise gains the fields it does not already hold while + every stored value is left exactly as it is. Nothing is ever deleted. The read + and the write share this transaction, so a caller that read the map earlier + cannot replace a value written since. Used by the one-time localStorage + backfill, whose contract is that the server copy is the newer authority: an + upgraded install can hold an entry with only the fields an older release knew, + while this browser holds the rest, and entry-level skipping would strand them. + """ conn = get_connection() try: conn.execute("BEGIN IMMEDIATE") @@ -2971,7 +2985,21 @@ def upsert_app_setting_map_entry( current = _json_loads(row["value_json"], {}) if row else {} if not isinstance(current, dict): current = {} - if entry_value: + if fill_absent_fields: + if not entry_value: + conn.rollback() + return current + stored = current.get(entry_key) + if isinstance(stored, dict): + # Stored values win field by field, so this only ever adds. + merged = {**entry_value, **stored} + if merged == stored: + conn.rollback() + return current + current[entry_key] = merged + else: + current[entry_key] = entry_value + elif entry_value: current[entry_key] = entry_value else: current.pop(entry_key, None) diff --git a/studio/backend/tests/test_api_monitor.py b/studio/backend/tests/test_api_monitor.py index 4602b5cf62..2ad5eb2397 100644 --- a/studio/backend/tests/test_api_monitor.py +++ b/studio/backend/tests/test_api_monitor.py @@ -260,6 +260,60 @@ def test_api_monitor_append_reply_exact_cap_then_more_marks_truncated(): assert len(reply) == m._MAX_REPLY_CHARS and reply.endswith("...") +def test_api_monitor_clear_is_scoped_to_one_subject(): + # Every other read is subject-scoped; an unscoped clear from the route would let + # one caller erase another's history mid-generation. + monitor = ApiMonitor(max_entries = 4) + alice = monitor.start( + endpoint = "/v1/chat/completions", + method = "POST", + model = "m", + prompt = "alice prompt", + subject = "alice", + ) + bob = monitor.start( + endpoint = "/v1/chat/completions", + method = "POST", + model = "m", + prompt = "bob prompt", + subject = "bob", + ) + + monitor.clear(subject = "alice") + assert monitor.snapshot(subject = "alice") == [] + assert [entry["id"] for entry in monitor.snapshot(subject = "bob")] == [bob] + assert monitor.active_count(subject = "bob") == 1 + assert monitor.get(alice, subject = "alice") is None + + # Passing no subject is the explicit "everything" path. + monitor.clear() + assert monitor.snapshot(subject = "bob") == [] + + +def test_api_monitor_records_whether_the_caller_used_an_api_key(): + # Studio's own chat hits these endpoints with a session JWT, and the floating + # panel keys its auto-open off this flag, so mislabelling it pops the panel. + monitor = ApiMonitor(max_entries = 4) + ui = monitor.start( + endpoint = "/api/inference/chat", + method = "POST", + model = "m", + prompt = "hi", + subject = "u", + ) + api = monitor.start( + endpoint = "/v1/chat/completions", + method = "POST", + model = "m", + prompt = "hi", + subject = "u", + via_api_key = True, + ) + by_id = {entry["id"]: entry for entry in monitor.snapshot(subject = "u")} + assert by_id[ui]["via_api_key"] is False + assert by_id[api]["via_api_key"] is True + + def test_api_monitor_disabled_is_noop(): monitor = ApiMonitor(max_entries = 3, enabled = False) @@ -412,3 +466,124 @@ def test_request_rows_report_kind_request(): monitor = ApiMonitor(max_entries = 2) monitor.start(endpoint = "/v1/chat/completions", method = "POST", model = "m", prompt = "hi") assert monitor.snapshot()[0]["kind"] == "request" + + +def test_clear_hides_shared_lifecycle_rows_for_that_caller_only(): + """A lifecycle row is shared, so it is visible to every caller but owned by + none. A subject-scoped clear dropped only that subject's own rows, so the + shared ones survived and the reload straight after "Clear log" brought them + back: the button visibly did nothing to them. Dropping them outright is not + an option either, since that erases another caller's history. + """ + monitor = ApiMonitor(max_entries = 10) + mine = monitor.start( + endpoint = "/v1/chat/completions", + method = "POST", + model = "org/A", + prompt = "user: hi", + subject = "alice", + ) + monitor.finish(mine) + shared = monitor.record_lifecycle(event = "unload", model = "org/A") + + assert {e["id"] for e in monitor.snapshot(subject = "alice")} == {mine, shared} + assert {e["id"] for e in monitor.snapshot(subject = "bob")} == {shared} + + monitor.clear(subject = "alice") + + assert monitor.snapshot(subject = "alice") == [] + # Hidden for alice, not deleted, so bob's view is untouched. + assert {e["id"] for e in monitor.snapshot(subject = "bob")} == {shared} + assert monitor.get(shared, subject = "alice") is None + assert monitor.get(shared, subject = "bob") is not None + + +def test_clear_leaves_a_running_shared_row_visible(): + """A load still in progress is live state, not history, so clearing the log + must not hide the row that shows it.""" + monitor = ApiMonitor(max_entries = 10) + running = monitor.record_lifecycle(event = "load", model = "org/A", running = True) + monitor.clear(subject = "alice") + assert {e["id"] for e in monitor.snapshot(subject = "alice")} == {running} + + +def test_hidden_shared_ids_do_not_outlive_their_entries(): + """The hidden set names rows that exist, so it stays bounded by the ring + buffer instead of growing for the life of the process.""" + monitor = ApiMonitor(max_entries = 2) + monitor.record_lifecycle(event = "unload", model = "org/A") + monitor.clear(subject = "alice") + assert monitor._hidden_shared.get("alice") + for i in range(5): + monitor.record_lifecycle(event = "unload", model = f"org/M{i}") + assert not monitor._hidden_shared.get("alice") + + +def test_an_api_triggered_lifecycle_row_carries_the_attribution(): + """The overlay opens on API-key traffic only. An auto-switch or auto-download + that is refused never reaches api_monitor.start, so the lifecycle row is the + whole trace of that request; without the attribution the monitor stayed shut + on exactly the failures it exists to surface.""" + monitor = ApiMonitor(max_entries = 5) + + api_load = monitor.record_lifecycle( + event = "load", model = "org/Repo-GGUF", running = True, via_api_key = True + ) + monitor.record_lifecycle(event = "unload", model = "org/Repo-GGUF", reason = "idle") + + rows = {e["id"]: e for e in monitor.snapshot()} + assert rows[api_load]["via_api_key"] is True + # A background unload is not API traffic and must not pop the overlay. + idle = [e for e in rows.values() if e["event"] == "unload"] + assert idle and all(e["via_api_key"] is False for e in idle) + + # The failure path keeps it: failing the row must not drop the attribution. + monitor.fail(api_load, error = "auto-switch refused") + after = {e["id"]: e for e in monitor.snapshot()} + assert after[api_load]["via_api_key"] is True + assert after[api_load]["status"] == "error" + + +def test_an_api_lifecycle_row_pops_the_overlay_only_for_its_own_caller(): + """A lifecycle row is shared so it appears in every monitor list, and it also + carries via_api_key, which is what the floating panel auto-opens on. Reported + to everyone, the panel springs open in a browser that had nothing to do with + the traffic. The row stays visible to all; only the attribution is scoped.""" + monitor = ApiMonitor(max_entries = 5) + + row = monitor.record_lifecycle( + event = "load", + model = "org/Repo-GGUF", + running = True, + via_api_key = True, + subject = "alice", + ) + + mine = {e["id"]: e for e in monitor.snapshot(subject = "alice")} + theirs = {e["id"]: e for e in monitor.snapshot(subject = "bob")} + # Shared visibility is deliberate and must survive: bob still sees the load. + assert row in mine and row in theirs + assert mine[row]["via_api_key"] is True + assert theirs[row]["via_api_key"] is False + + # The details read is scoped the same way, so the panel cannot re-derive it. + assert monitor.get(row, subject = "alice")["via_api_key"] is True + assert monitor.get(row, subject = "bob")["via_api_key"] is False + # An unscoped read (internal callers) still sees the row's own flag. + assert monitor.get(row)["via_api_key"] is True + + +def test_clearing_hides_a_shared_row_this_caller_owns_rather_than_deleting_it(): + """An API-key load now owns its shared row. A subject-scoped clear drops that + subject's rows, so without this the owner's Clear would delete a row every + other caller can still see and wipe it out of their history too.""" + monitor = ApiMonitor(max_entries = 10) + row = monitor.record_lifecycle( + event = "unload", model = "org/Repo-GGUF", via_api_key = True, subject = "alice" + ) + + monitor.clear(subject = "alice") + + assert monitor.snapshot(subject = "alice") == [] + assert {e["id"] for e in monitor.snapshot(subject = "bob")} == {row} + assert monitor.get(row, subject = "bob") is not None diff --git a/studio/backend/tests/test_openai_auto_download.py b/studio/backend/tests/test_openai_auto_download.py index b1dde175da..9653a5842d 100644 --- a/studio/backend/tests/test_openai_auto_download.py +++ b/studio/backend/tests/test_openai_auto_download.py @@ -112,8 +112,7 @@ def _hub_error(error_type, status_code: int, message: str): def test_the_hub_error_helper_carries_a_status_on_both_majors(): # CI runs huggingface_hub 1.x and this box 0.x, and each takes only one of the - # constructor shapes. A helper that silently dropped the response would make an - # error-mapping test pass here and fail there. + # constructor shapes. from hub.utils.hf_errors import hf_error_status class _Legacy(Exception): @@ -607,8 +606,7 @@ def test_a_hanging_auth_check_falls_through_to_the_download(hub, monkeypatch): def test_a_companion_only_repo_is_not_held_at_busy(hub): # mmproj and MTP files are companions, not quants, so such a repo is non-servable - # and falls through to the resident model. The busy probe accepted any .gguf, which - # stranded that ordinary traffic behind an unrelated multi-hour download. + # and falls through to the resident model. assert _run("unsloth/x-GGUF:UD-Q4_K_XL").code == "model_downloading" gb = 1024**3 hub["info"] = _Info([_Sibling("mmproj-F16.gguf", gb), _Sibling("mtp-model.gguf", gb)]) @@ -684,13 +682,22 @@ class _Req: self.headers = headers or {} -def _hook(model, request, enabled): +def _hook( + model, + request, + enabled, + current_subject = None, +): import utils.openai_auto_switch_settings as s original = s.get_openai_auto_download_enabled s.get_openai_auto_download_enabled = lambda: enabled try: - return asyncio.run(inference_route._maybe_auto_download_model(model, request)) + return asyncio.run( + inference_route._maybe_auto_download_model( + model, request, current_subject = current_subject + ) + ) finally: s.get_openai_auto_download_enabled = original @@ -726,13 +733,98 @@ def test_hook_uses_the_anthropic_envelope_on_messages(hub): def test_hook_swallows_unexpected_failures(hub, monkeypatch): # A broken download path must not turn a servable request into a 500. - async def _boom(model, hf_token = None): + async def _boom( + model, + hf_token = None, + **kwargs, + ): raise RuntimeError("boom") monkeypatch.setattr(auto_dl, "maybe_auto_download", _boom) assert _hook("unsloth/x-GGUF", _Req(), enabled = True) is None +def _download_rows(): + from core.inference.api_monitor import api_monitor + return [e for e in api_monitor.snapshot() if e["event"] == "download"] + + +def test_a_ui_session_download_is_not_marked_as_api_traffic(hub): + """The monitor overlay auto-opens on via_api_key, which exists to separate + "someone is serving other clients" from "someone is using Unsloth". Studio's + own chat hits these same /v1 endpoints with a session JWT, so hardcoding the + flag on the download row popped the panel open mid-chat.""" + from fastapi import HTTPException + from core.inference.api_monitor import api_monitor + + api_monitor.clear() + with pytest.raises(HTTPException): + # No Authorization header: the UI's session-JWT path. + _hook("unsloth/x-GGUF", _Req(), enabled = True, current_subject = "unsloth") + rows = _download_rows() + assert rows and all(row["via_api_key"] is False for row in rows) + + +def test_an_api_key_download_keeps_the_attribution_and_names_its_caller(hub): + """The row is shared, so it needs the subject as well: without one the + attribution is reported to every logged-in browser instead of the caller.""" + from fastapi import HTTPException + from auth.authentication import API_KEY_PREFIX + from core.inference.api_monitor import api_monitor + + api_monitor.clear() + with pytest.raises(HTTPException): + _hook( + "unsloth/x-GGUF", + _Req(headers = {"authorization": f"Bearer {API_KEY_PREFIX}abc123"}), + enabled = True, + current_subject = "unsloth", + ) + rows = _download_rows() + assert rows and all(row["via_api_key"] is True for row in rows) + # Still shared: another subject sees the row, just not the attribution. + others = [e for e in api_monitor.snapshot(subject = "someone-else") if e["event"] == "download"] + assert len(others) == len(rows) + assert all(row["via_api_key"] is False for row in others) + + +def test_an_api_key_caller_waiting_on_someone_elses_download_gets_a_row(hub): + """A download started by Studio's own chat is attributed to the session, so an + API-key client that asks for the same repo while it runs is refused before the + handler's own api_monitor.start. Without a row of its own that call is invisible: + the only row is the session's via_api_key=False download, so the overlay stays + shut and the monitor presents API traffic as Studio's own.""" + from fastapi import HTTPException + from auth.authentication import API_KEY_PREFIX + from core.inference.api_monitor import api_monitor + + api_monitor.clear() + with pytest.raises(HTTPException): + # Studio's chat (session JWT) starts the download and takes the slot. + _hook("unsloth/x-GGUF", _Req(), enabled = True, current_subject = "unsloth") + seeded = {row["id"] for row in api_monitor.snapshot(subject = "unsloth")} + + with pytest.raises(HTTPException) as excinfo: + # The adopted-download branch: same repo, an sk-unsloth key this time. + _hook( + "unsloth/x-GGUF", + _Req(headers = {"authorization": f"Bearer {API_KEY_PREFIX}abc123"}), + enabled = True, + current_subject = "unsloth", + ) + assert excinfo.value.status_code == 503 + + fresh = [e for e in api_monitor.snapshot(subject = "unsloth") if e["id"] not in seeded] + # New (so the overlay counts it as unseen traffic) and attributed to this caller. + assert [e for e in fresh if e["via_api_key"]], "the refused API-key call left no row" + row = next(e for e in fresh if e["via_api_key"]) + assert row["endpoint"] == "/v1/chat/completions" + assert row["status"] == "error" + # Shared rows aside, another subject must not inherit the attribution. + others = [e for e in api_monitor.snapshot(subject = "someone-else") if e["id"] == row["id"]] + assert others == [] + + def test_hook_prefers_the_hub_header_token(hub): from fastapi import HTTPException from hub.dependencies import HUB_HF_TOKEN_HEADER @@ -1305,8 +1397,7 @@ def test_a_cold_scan_that_never_finishes_says_so_instead_of_guessing(monkeypatch def test_a_refusal_is_never_swallowed_by_the_cannot_verify_handler(monkeypatch): # The checks run inside a broad `except Exception` that turns a failure to decide - # into a fallthrough. An HTTPException there is a decision, but was logged as a - # failure and answered by the resident model. + # into a fallthrough. loaded = _Loaded("unsloth/A-GGUF", "UD-Q4_K_XL") monkeypatch.setattr(inference_route, "get_llama_cpp_backend", lambda: loaded) monkeypatch.setattr( @@ -1372,8 +1463,7 @@ def test_a_stale_index_is_refreshed_so_a_hub_download_becomes_visible(monkeypatc def test_an_id_v1_models_advertised_is_refused_before_the_resolver_warms(monkeypatch): - # /v1/models can advertise an unloaded local GGUF while the resolver index is cold. A bare - # id has no quant to refuse on, so without that evidence the resident model would answer. + # /v1/models can advertise an unloaded local GGUF while the resolver index is cold. from core.inference import local_model_resolver as resolver monkeypatch.setattr(resolver, "_scan", (0.0, {})) @@ -1428,8 +1518,7 @@ def test_an_advertised_alias_for_the_resident_weights_is_still_served(monkeypatc def test_a_rejected_token_says_so_instead_of_asking_for_a_retry(hub): - # Hugging Face 401s an expired X-Unsloth-HF-Token. Only 403/404 were handled, so it - # fell through to a 503 telling the caller to retry something that cannot work. + # Hugging Face 401s an expired X-Unsloth-HF-Token. from huggingface_hub.utils import HfHubHTTPError hub["raise"] = _hub_error(HfHubHTTPError, 401, "unauthorized") @@ -1504,8 +1593,7 @@ def test_a_quant_request_is_not_satisfied_by_transformers_weights(monkeypatch): def test_a_timed_out_download_keeps_the_slot_while_it_is_still_running(monkeypatch): - # The watch window only bounds progress reporting. Releasing on the clock while - # the worker is alive would admit a second multi-GB download beside it. + # The watch window only bounds progress reporting. monkeypatch.setattr(auto_dl, "_MAX_WATCH_S", 0.0) monkeypatch.setattr(auto_dl, "_WATCH_POLL_S", 0.001) monkeypatch.setattr(auto_dl, "_TIMED_OUT_POLL_S", 0.001) @@ -1597,8 +1685,7 @@ def test_a_remote_tag_that_names_no_quant_picks_the_preferred_one(hub): def test_a_generic_gguf_advertises_the_label_the_worker_resolves(hub): # With no recognized quant token the extractors part ways: one takes the last - # hyphenated segment, the plan and worker key the whole stem. Dispatching ours - # made the worker exit with "No GGUF shards matching variant". + # hyphenated segment, the plan and worker key the whole stem. from hub.utils.gguf import extract_quant_label as canonical from hub.utils.gguf_plan import build_gguf_variant_plans @@ -1686,8 +1773,7 @@ def test_a_non_quant_tag_does_not_tear_down_a_serving_quant(monkeypatch): def test_the_trust_probe_never_falls_back_to_the_server_identity(hub, monkeypatch): # huggingface_hub treats None as "use the cached login", so only an explicit False - # is anonymous. This probe passed None, so a caller-named repo was read with the - # server's identity. + # is anonymous. seen: list = [] def _probe(model_name, hf_token = None): @@ -1757,8 +1843,7 @@ def test_the_retry_after_a_failure_is_told_instead_of_restarting_it(hub, monkeyp def test_a_completed_download_does_not_restage_the_scan_it_just_warmed(monkeypatch): - # finalize_worker_exit invalidates and warms. A second invalidation here marks - # that fresh scan stale and pushes a synchronous rescan onto the client's retry. + # finalize_worker_exit invalidates and warms. import inspect src = inspect.getsource(auto_dl._watch) diff --git a/studio/backend/tests/test_openai_auto_switch.py b/studio/backend/tests/test_openai_auto_switch.py index e29fc07a95..5ffac3ead3 100644 --- a/studio/backend/tests/test_openai_auto_switch.py +++ b/studio/backend/tests/test_openai_auto_switch.py @@ -248,8 +248,7 @@ def test_same_repo_same_variant_does_not_reload(monkeypatch): def test_responses_endpoint_wires_auto_switch_before_dispatch(): # The /v1/responses endpoint must invoke the auto-switch hook before either - # dispatcher so streaming requests switch too. Asserted on the source, which - # is immune to test-ordering effects on the shared inference module. + # dispatcher so streaming requests switch too. import inspect src = inspect.getsource(inference_route.openai_responses) @@ -281,9 +280,7 @@ def test_count_tokens_endpoint_wires_auto_switch_before_loaded_check(): def test_openai_compat_routes_bound_to_handlers_with_auth(): # Inserting a helper between a @router.post decorator and its handler silently # rebinds the route to the helper and drops its auth dependency (this happened to - # /messages/count_tokens). The source-inspection tests above miss it because they - # call the handler directly. Lock the path -> (handler, auth) mapping at the route - # level so any decorator/handler split is caught. + # /messages/count_tokens). expected = { ("POST", "/chat/completions"): "openai_chat_completions", ("POST", "/completions"): "openai_completions", @@ -342,10 +339,8 @@ def test_local_gguf_entry_filters_non_gguf_and_recurses(tmp_path): def test_local_gguf_entry_rejects_standalone_mmproj(tmp_path): - # Codex P2: _scan_models_dir's standalone-.gguf pass emits an entry for a - # bare mmproj projector (it only filters mmproj inside directory scans). A - # projector is not a servable model, so the resolver must reject it or - # /v1/models advertises it and a switch could load it over the real weights. + # Codex P2: _scan_models_dir's standalone-.gguf pass emits an entry for a bare + # mmproj projector (it only filters mmproj inside directory scans). from types import SimpleNamespace proj = tmp_path / "mmproj-F16.gguf" @@ -385,9 +380,8 @@ def test_resolver_matches_and_splits_variant(monkeypatch): def test_resolver_failsafe_on_internal_error(monkeypatch): - # Resolution is best-effort: any internal failure must fall through to None - # so the request still serves the loaded model instead of 500-ing. The hook - # calls resolve_local_gguf without its own guard, so the guard lives here. + # Resolution is best-effort: any internal failure must fall through to None so the + # request still serves the loaded model instead of 500-ing. def boom(): raise RuntimeError("scan blew up") @@ -444,8 +438,7 @@ def test_describe_local_miss_is_failsafe(monkeypatch): def test_resolver_exact_id_with_colon_wins(monkeypatch): - # A local id that itself contains a colon (e.g. a Windows path) must match - # exactly rather than being split at the drive-letter colon. + # A local id that itself contains a colon (e.g. win = r"C:\models\foo.gguf" monkeypatch.setattr(resolver, "_build_index", lambda: {win.lower(): _entry(win)}) resolver._scan = (0.0, {}) @@ -566,7 +559,10 @@ def test_idle_loop_deletes_saved_kv_when_unload_fails(monkeypatch, tmp_path): async def _drive(): task = asyncio.create_task(kw.idle_unload_loop(poll_seconds = 0.01)) - for _ in range(200): + # Wall clock, not an iteration count: Windows rounds a 10 ms sleep up to the + # ~15.6 ms tick on both loops, so a fixed count failed for being slow. + deadline = time.monotonic() + 15.0 + while time.monotonic() < deadline: await asyncio.sleep(0.01) if manifests and not saved.exists(): break @@ -698,9 +694,24 @@ def _mock_override_store(monkeypatch): store = {} - def _merge_entry(key, entry_key, entry_value): + def _merge_entry( + key, + entry_key, + entry_value, + *, + fill_absent_fields = False, + ): current = dict(store.get(key) or {}) - if entry_value: + if fill_absent_fields: + # Fill only: every stored value wins and nothing is deleted. + if not entry_value: + return current + stored = current.get(entry_key) + if isinstance(stored, dict): + current[entry_key] = {**entry_value, **stored} + else: + current[entry_key] = entry_value + elif entry_value: current[entry_key] = entry_value else: current.pop(entry_key, None) @@ -713,6 +724,21 @@ def _mock_override_store(monkeypatch): return store +@pytest.fixture +def override_store(monkeypatch): + """The in-memory override store, for a test that needs nothing else mocked.""" + _mock_override_store(monkeypatch) + + +def _put(model_id, **fields): + """One override PUT through the route, spelled the way the UI sends it.""" + import routes.settings as settings_route + return settings_route.update_openai_auto_switch_override( + settings_route.ModelOverridePayload(model_id = model_id, **fields), + "tester", + ) + + def test_model_override_roundtrip(monkeypatch): _mock_override_store(monkeypatch) @@ -1051,9 +1077,7 @@ def test_embeddings_malformed_body_503_not_500_when_unloaded(monkeypatch): def test_non_string_model_falls_through_without_error(monkeypatch): - # A non-string model (e.g. {"model": 123} on a raw-body endpoint) must be - # treated as absent, never raising in the membership checks, even when a stash - # exists from idle-unload. + # A non-string model (e.g. from core.inference import llama_keepwarm as kw backend = _FakeBackend(None) @@ -1212,8 +1236,7 @@ def _json_body_request(payload): def test_completions_list_body_is_400_not_500(monkeypatch): - # A valid JSON non-dict body (e.g. a list) on a loaded backend is a clean 400, - # not a 500 from body.get(...). + # A valid JSON non-dict body (e.g. from fastapi import HTTPException backend = _FakeBackend("unsloth/A-GGUF") # loaded @@ -1409,8 +1432,7 @@ def _revision_pair(root, complete: bool): def test_sibling_revision_resolves_to_its_own_weights(tmp_path): # /v1/models advertises only the snapshot dir name, so a durable pin holds one - # revision hash. A newer snapshot must not strand it, and the old revision must - # resolve to ITS OWN directory rather than be redirected onto the newest. + # revision hash. old, new = _revision_pair(tmp_path, complete = True) found = dict(resolver._sibling_revision_entries(str(new), "org/Repo")) @@ -1446,9 +1468,8 @@ def test_sibling_revisions_skip_plain_repo_ids(): def test_already_loaded_by_repo_id_is_not_reswapped(monkeypatch): - # A model loaded normally has model_identifier == repo id, but the resolver - # returns the concrete load path. A request for that repo must count as already - # serving (no reload, no 409) even with another inference active. + # A model loaded normally has model_identifier == repo id, but the resolver returns + # the concrete load path. from core.inference import llama_keepwarm as kw backend = _FakeBackend("org/Repo-GGUF", hf_variant = "Q4_K_M") @@ -1508,10 +1529,7 @@ def test_already_serving_by_path_records_advertised_alias(monkeypatch): def test_streaming_responses_uses_advertised_id_helper(): # Codex P2: streamed /v1/responses envelopes must derive the model id from # _llama_public_model_id (which prefers _openai_advertised_id), not the raw - # model_identifier. After an auto-switch to a cached HF GGUF the identifier is - # the snapshot path while the repo id lives in _openai_advertised_id, so the raw - # form would stream a snapshot basename while /v1/models, chat, and non-streaming - # responses report the repo id. + # model_identifier. import inspect src = inspect.getsource(inference_route._responses_stream) @@ -1520,9 +1538,8 @@ def test_streaming_responses_uses_advertised_id_helper(): def test_concurrent_same_target_requests_load_once(monkeypatch): - # Two concurrent requests for the same unloaded model must load once, not each - # 409 the other. Simulate the second request already waiting (registered) while - # the first runs the hook with _inflight counting both. + # Two concurrent requests for the same unloaded model must load once, not each 409 + # the other. from core.inference import llama_keepwarm as kw backend = _FakeBackend("org/A-GGUF") @@ -1577,9 +1594,7 @@ def test_v1_models_advertises_repo_id_not_load_path(monkeypatch): def test_idle_alias_reload_preserves_override_via_advertised_id(monkeypatch): - # The idle stash carries (load_path, quant, advertised_id). An alias reload must - # look up the override by the advertised repo id, not the concrete load path, - # so the user's saved launch flags survive the unload/reload. + # The idle stash carries (load_path, quant, advertised_id). from core.inference import llama_keepwarm as kw backend = _FakeBackend(None) # idle-unload emptied the slot @@ -1606,9 +1621,8 @@ def test_load_route_holds_lifecycle_gate(monkeypatch): def test_model_replacements_recheck_sidecar_swap_before_either_backend_is_unloaded(): - # Both replacement directions drain, then recheck whether a sidecar install reserved the - # gate meanwhile. That recheck is the last thing that can reject the load, so the - # destructive cancel must follow it. Exact-model reuse exits earlier and never waits. + # Both replacement directions drain, then recheck whether a sidecar install reserved + # the gate meanwhile. import inspect src = inspect.getsource(inference_route._load_model_impl) @@ -1701,9 +1715,8 @@ def test_pending_same_target_request_does_not_block_swap(monkeypatch): def test_swap_waits_until_concurrent_request_finishes_resolving(monkeypatch): - # The real middleware counts a concurrent same-model request as in-flight - # before it resolves and registers a target waiter. Treat it as active until - # its target is known, then recognize it as another queued switch request. + # The real middleware counts a concurrent same-model request as in-flight before it + # resolves and registers a target waiter. from core.inference import llama_keepwarm as kw backend = _FakeBackend("org/A-GGUF") @@ -1768,7 +1781,6 @@ def test_manual_unload_interrupts_even_while_inference_active(monkeypatch): def test_auto_switch_waits_when_unsloth_stream_active(monkeypatch): # The GGUF slot is empty but an Unsloth model is streaming (counted in-flight). - # The replacement waits for it just as it does for a GGUF generation. from core.inference import llama_keepwarm as kw backend = _FakeBackend(None) # no GGUF loaded @@ -2053,8 +2065,7 @@ def test_index_advertises_alias_not_filesystem_path(tmp_path, monkeypatch): def test_build_index_survives_a_failing_scanner(tmp_path, monkeypatch): - # gemini: one bad scanner (e.g. a permission error on ./models) must drop only - # that source, not abort the whole index and lose what the others found. + # gemini: one bad scanner (e.g. from types import SimpleNamespace import routes.models as models_route import utils.paths as paths @@ -2085,9 +2096,8 @@ def test_build_index_survives_a_failing_scanner(tmp_path, monkeypatch): def test_info_has_local_gguf_reads_files_not_model_format(tmp_path): - # Codex: HF-cache GGUF snapshots leave model_format unset, so /v1/models must - # decide GGUF-ness from the on-disk files. A standalone .gguf (no model_format) - # is servable; a safetensors-only dir is not. + # Codex: HF-cache GGUF snapshots leave model_format unset, so /v1/models must decide + # GGUF-ness from the on-disk files. from types import SimpleNamespace gguf = tmp_path / "model-Q4_K_M.gguf" @@ -2169,10 +2179,8 @@ def test_retrieve_model_tolerates_non_string_id(monkeypatch): def test_retrieve_model_resolves_raw_path_to_advertised_id(monkeypatch): - # Codex P2: a client caching the legacy absolute .gguf path must still retrieve - # a loaded auto-switch model. Its /v1/models entry is keyed by the advertised - # repo id (identifier = snapshot path), so the raw-path fallback must map the raw - # id to that advertised id, not public_model_id(path), or a loaded model 404s. + # Codex P2: a client caching the legacy absolute .gguf path must still retrieve a + # loaded auto-switch model. from types import SimpleNamespace raw_path = "/cache/models--org--B-GGUF/snapshots/abc/model.gguf" @@ -2219,8 +2227,7 @@ def test_chat_streaming_n_gt_1_rejected_before_switch(monkeypatch): def test_resolver_cache_stamped_after_slow_build(monkeypatch): - # Codex P2: the cache must be stamped AFTER _build_index. A scan slower than the - # TTL would otherwise store an already-expired cache and rebuild every request. + # Codex P2: the cache must be stamped AFTER _build_index. import core.inference.local_model_resolver as r clock = {"t": 1000.0} @@ -2354,8 +2361,7 @@ def test_embeddings_missing_input_rejected_before_idle_reload(monkeypatch): def test_messages_does_not_503_before_reload_hook_when_idle_on(monkeypatch): # #3: /v1/messages 503'd before the reload hook when auto-switch was off, so a - # standalone idle TTL could never restore the freed model. The early 503 now - # defers to any automatic-load trigger, so the reload hook runs. + # standalone idle TTL could never restore the freed model. backend = _FakeBackend(None) rec = _LoadRecorder(backend) _wire(monkeypatch, enabled = False, resolves_to = None, backend = backend, recorder = rec) @@ -2383,8 +2389,7 @@ def test_messages_503_gated_on_automatic_load_predicate(): def test_raw_body_without_model_reloads_freed_model(monkeypatch): # #6: a raw completions/embeddings body that omits `model` passed None, which - # skipped the idle-stash reload and 503'd. A non-empty sentinel now lets the - # reload run while still resolving as unknown. + # skipped the idle-stash reload and 503'd. backend = _FakeBackend(None) rec = _LoadRecorder(backend) _wire(monkeypatch, enabled = False, resolves_to = None, backend = backend, recorder = rec) @@ -2439,8 +2444,7 @@ def test_audio_generate_does_not_reload_on_invalid_request(monkeypatch): def test_preview_scope_disables_auto_switch(monkeypatch): # #7: the public preview route delegates to the chat handler; a caller-supplied - # model must not switch away from the pinned checkpoint. The scope opt-out flag - # makes the hook a no-op. + # model must not switch away from the pinned checkpoint. backend = _FakeBackend("org/A-GGUF") rec = _LoadRecorder(backend) _wire( @@ -2515,9 +2519,8 @@ def test_note_start_does_not_reset_idle_timer(): def test_omitted_model_does_not_resolve_to_a_named_gguf(monkeypatch): - # Codex P2: a raw-body request that omits `model` must never run the resolver, - # so a downloaded GGUF literally named "default" can't be switched to. The - # resolver here would switch to B if it ran; it must not. + # Codex P2: a raw-body request that omits `model` must never run the resolver, so a + # downloaded GGUF literally named "default" can't be switched to. backend = _FakeBackend("org/A-GGUF") # a model is already loaded rec = _LoadRecorder(backend) _wire( @@ -2835,9 +2838,8 @@ def test_chat_confirm_with_bypass_permissions_reaches_hook(monkeypatch): def test_chat_audio_input_guards_target_before_switch(monkeypatch): # Codex P2: a chat request carrying audio_base64 must guard the target before the - # switch -- audio rides the same companion mmproj as vision -- so a text-only - # target can't be loaded and evict the working audio model. Assert the handler - # flags require_vision so the hook's multimodal probe runs. + # switch -- audio rides the same companion mmproj as vision -- so a text-only target + # can't be loaded and evict the working audio model. class _Reached(Exception): pass @@ -2863,8 +2865,7 @@ def test_chat_audio_input_guards_target_before_switch(monkeypatch): def test_completions_rejects_object_prompt_before_switch(monkeypatch): # Codex P2: an object prompt like {"prompt": {}} is a deterministic client error - # (only a string or array is valid). It must 400 before the switch so a bad shape - # can't load the named GGUF only to be rejected by llama-server after eviction. + # (only a string or array is valid). from fastapi import HTTPException backend = _FakeBackend("org/A-GGUF") @@ -3071,8 +3072,7 @@ def test_anthropic_request_has_image_helper(): def test_responses_and_anthropic_wire_require_vision_from_images(): # P2: the modality guard must fire on /v1/responses and /v1/messages too, so an - # image request can't evict a vision model for a text-only target. Lock the wiring - # at the source: each hook derives require_vision from the request's images. + # image request can't evict a vision model for a text-only target. import inspect responses_src = inspect.getsource(inference_route.openai_responses) @@ -3137,11 +3137,7 @@ def test_count_tokens_forwards_vision_guard_to_switch(monkeypatch): def test_audio_generate_is_reload_only(monkeypatch): - # Codex P2: /audio/generate must not switch to a client-named GGUF. A local - # GGUF's audio-input capability is not a cheap pre-load probe (the mmproj signal - # can't tell an audio projector from a vision one), so resolving the client model - # could evict the working audio model for a target that then fails the audio - # check. Only the idle-stash restore runs: the hook gets the reload-only sentinel. + # Codex P2: /audio/generate must not switch to a client-named GGUF. from models.inference import ChatCompletionRequest class _Reached(Exception): @@ -3170,8 +3166,7 @@ def test_audio_generate_is_reload_only(monkeypatch): def test_note_model_unloaded_clears_reload_stash(monkeypatch): # Codex P2: a deliberate unload must drop the idle reload stash so the next /v1 - # request can't resurrect the just-unloaded model. (The idle loop unloads via the - # backend directly, so clearing on the route never fights keep-warm.) + # request can't resurrect the just-unloaded model. import core.inference.llama_keepwarm as kw kw._set_last_unloaded(("org/A-GGUF", "Q4_K_M")) @@ -3267,9 +3262,8 @@ def test_lifecycle_gate_serializes_across_loops(): def test_auto_switch_serializes_across_event_loops(monkeypatch): - # Codex P2: the per-loop asyncio lock can't serialize two swaps on different - # event loops in one process. The process-wide gate must, so the two slow loads - # never overlap on the single model slot. + # Codex P2: the per-loop asyncio lock can't serialize two swaps on different event + # loops in one process. import threading backend = _FakeBackend("org/A-GGUF") @@ -3321,10 +3315,8 @@ def test_auto_switch_serializes_across_event_loops(monkeypatch): def test_acquire_swap_gate_is_cancellation_safe(): - # A waiter cancelled while waiting for the gate (client disconnect mid-swap) - # must not leak it: after the holder releases, a fresh acquire still succeeds. - # The to_thread(acquire) approach would leak here -- its worker thread keeps - # acquiring after cancel, so the gate is taken but never released. + # A waiter cancelled while waiting for the gate (client disconnect mid-swap) must + # not leak it: after the holder releases, a fresh acquire still succeeds. async def main(): await inference_route._acquire_swap_gate() # this loop holds the gate try: @@ -3347,9 +3339,8 @@ def test_acquire_swap_gate_is_cancellation_safe(): def test_no_model_loaded_detail_appends_hint_only_when_off(monkeypatch): - # The "no model loaded" errors point at the opt-in auto-switch toggle so a - # request naming a listed-but-unloaded model is self-explanatory -- but only - # when it's off. With it on the name simply didn't resolve, so no hint. + # The "no model loaded" errors point at the opt-in auto-switch toggle so a request + # naming a listed-but-unloaded model is self-explanatory -- but only when it's off. base = "No GGUF model loaded. Load a GGUF model first." monkeypatch.setattr(settings, "get_openai_auto_switch_enabled", lambda: False) @@ -3390,8 +3381,7 @@ def _run_responses_stream_no_model( def test_responses_stream_hint_matches_toggle_regardless_of_active_model(monkeypatch): - # The hint attaches whenever the toggle is off, whatever is active. With it on the name - # resolved to nothing local, so 404 rather than 400. + # The hint attaches whenever the toggle is off, whatever is active. off_status, hinted = _run_responses_stream_no_model( monkeypatch, enabled = False, active_model_name = None ) @@ -4150,11 +4140,703 @@ def test_env_idle_below_floor_is_clamped(monkeypatch): assert settings.get_auto_unload_idle_seconds() == 0 +# Per-model launch config: normalization, LoadRequest mapping, key resolution. + + +def test_normalize_model_override_drops_unusable_fields_and_keeps_the_rest(): + # A stale field must not cost the user the whole config, so bad values are + # dropped one by one rather than rejecting the payload. + entry = settings.normalize_model_override( + { + "max_seq_length": 8192, + "kv_cache_dtype": "not_a_dtype", + "speculative_type": "mtp", + "spec_draft_n_max": 999, # out of range for the MTP draft count + "gpu_memory_mode": "auto", # only "manual" is a real override + "gpu_layers": -1, # -1 is Auto, which is already the default + "n_cpu_moe": 0, + "gpu_ids": [1, 1, 0, "2", -5], + "tensor_parallel": False, + "llama_extra_args": [], + } + ) + assert entry == {"max_seq_length": 8192, "speculative_type": "mtp", "gpu_ids": [1, 0, 2]} + + +def test_normalize_model_override_rejects_oversized_chat_template(): + small = settings.normalize_model_override({"chat_template_override": "{{ bos }}"}) + assert small["chat_template_override"] == "{{ bos }}" + # The limit is bytes, not characters, so a multi-byte template just under the + # character limit can still be over. + huge = "รฉ" * settings.MAX_CHAT_TEMPLATE_OVERRIDE_BYTES + assert "chat_template_override" not in settings.normalize_model_override( + {"chat_template_override": huge} + ) + + +def test_spec_draft_n_max_only_stored_for_mtp_modes(): + mtp = settings.normalize_model_override({"speculative_type": "mtp", "spec_draft_n_max": 4}) + assert mtp["spec_draft_n_max"] == 4 + # A non-MTP mode ignores the draft count, so storing it shows an edit that + # never takes effect. + ngram = settings.normalize_model_override({"speculative_type": "ngram", "spec_draft_n_max": 4}) + assert "spec_draft_n_max" not in ngram + + +def test_resolve_fit_max_seq_length_hands_sizing_to_fit_under_manual_auto_layers(): + # Manual GPU memory with Auto layers hands the context to llama.cpp --fit, so + # the load sends the context pin (or 0), not the stored max seq length. + override = {"gpu_memory_mode": "manual", "max_seq_length": 8192} + assert settings.resolve_fit_max_seq_length(override, is_gguf = True) == 0 + assert ( + settings.resolve_fit_max_seq_length( + {**override, "custom_context_length": 4096}, is_gguf = True + ) + == 4096 + ) + # Pinning the layer count takes --fit back out of the picture. + assert settings.resolve_fit_max_seq_length({**override, "gpu_layers": 20}, is_gguf = True) == 8192 + # Not a GGUF, so none of this applies. + assert settings.resolve_fit_max_seq_length(override, is_gguf = False) == 8192 + + +def test_model_override_load_kwargs_gates_gpu_placement_on_gguf(): + override = { + "max_seq_length": 4096, + "kv_cache_dtype": "q8_0", + "tensor_parallel": True, + "gpu_memory_mode": "manual", + "gpu_layers": 20, + "n_cpu_moe": 3, + "gpu_ids": [0, 1], + } + gguf = settings.model_override_load_kwargs(override, is_gguf = True) + assert gguf["cache_type_kv"] == "q8_0" + assert gguf["tensor_parallel"] is True + assert gguf["gpu_layers"] == 20 + assert gguf["gpu_ids"] == [0, 1] + + # A safetensors model loads through HF auto-placement, so a GGUF GPU pin would + # silently change where the weights land. + safetensors = settings.model_override_load_kwargs(override, is_gguf = False) + assert safetensors["max_seq_length"] == 4096 + assert "gpu_layers" not in safetensors + assert "gpu_ids" not in safetensors + assert "n_cpu_moe" not in safetensors + assert "gpu_memory_mode" not in safetensors + + # Every key must be a real LoadRequest field, or the load raises TypeError when + # the user's request arrives. + LoadRequest(model_path = "unsloth/B-GGUF", **gguf) + + +def test_saved_parallel_slots_reach_an_api_load(monkeypatch): + # Parallel decode slots are a per-model setting the picker sends on every GGUF load. + backend = _FakeBackend(None) + rec = _LoadRecorder(backend) + _wire( + monkeypatch, + enabled = True, + resolves_to = ("unsloth/B-GGUF", "Q4_K_M", "unsloth/B-GGUF"), + backend = backend, + recorder = rec, + ) + monkeypatch.setattr(settings, "get_model_override", lambda mid: {"n_parallel": 8}) + + _run_hook("unsloth/B-GGUF") + assert rec.calls[0].n_parallel == 8 + + +def test_parallel_slots_are_stored_and_gated_on_gguf(): + override = settings.normalize_model_override({"n_parallel": 8}) + assert override == {"n_parallel": 8} + # Blank, out of range and non-integer all mean "follow the server-wide default". + for bad in (None, 0, -1, settings.PARALLEL_SLOTS_MAX + 1, "many", True): + assert "n_parallel" not in settings.normalize_model_override({"n_parallel": bad}) + + gguf = settings.model_override_load_kwargs(override, is_gguf = True) + assert gguf["n_parallel"] == 8 + # A safetensors load has no llama-server slots, exactly as the picker gates it. + assert "n_parallel" not in settings.model_override_load_kwargs(override, is_gguf = False) + LoadRequest(model_path = "unsloth/B-GGUF", **gguf) + + +def test_override_route_persists_parallel_slots(override_store): + # The mirror the picker writes has to carry the field, or a config whose only + # change is the slot count saves as an empty entry. + resp = _put("unsloth/B-GGUF:Q4_K_M", n_parallel = 8) + assert resp.overrides["unsloth/B-GGUF:Q4_K_M"] == {"n_parallel": 8} + + +def test_eviction_cleanup_clears_mirrored_fields_but_keeps_launch_flags(override_store): + # Dropping a local entry to stay inside the browser's storage budget is not the user + # forgetting the model, so the cleanup sends remove=false with no fields. + settings.set_model_override( + "unsloth/B-GGUF:Q4_K_M", + llama_extra_args = ["--flash-attn"], + custom_context_length = 32768, + kv_cache_dtype = "q8_0", + ) + resp = _put("unsloth/B-GGUF:Q4_K_M", remove = False) + assert resp.overrides["unsloth/B-GGUF:Q4_K_M"] == {"llama_extra_args": ["--flash-attn"]} + + # Nothing server-owned left, so the row goes rather than lingering empty. + settings.set_model_override("unsloth/C-GGUF:Q4_K_M", custom_context_length = 32768) + gone = _put("unsloth/C-GGUF:Q4_K_M", remove = False) + assert "unsloth/C-GGUF:Q4_K_M" not in gone.overrides + + +def test_auto_switch_prefers_variant_qualified_override(monkeypatch): + # Settings are per quant, so Q4_K_M and Q8_0 of one repo are separate entries + # and the bare repo id is only the fallback. + backend = _FakeBackend(None) + rec = _LoadRecorder(backend) + _wire( + monkeypatch, + enabled = True, + resolves_to = ("unsloth/B-GGUF", "Q4_K_M", "unsloth/B-GGUF"), + backend = backend, + recorder = rec, + ) + stored = { + "unsloth/B-GGUF": {"max_seq_length": 1024}, + "unsloth/B-GGUF:Q4_K_M": {"max_seq_length": 8192, "gpu_layers": 20}, + } + monkeypatch.setattr(settings, "get_model_override", lambda mid: stored.get(mid, {})) + + _run_hook("unsloth/B-GGUF") + req = rec.calls[0] + assert req.max_seq_length == 8192 + assert req.gpu_layers == 20 + + +def test_auto_switch_falls_back_to_bare_repo_override(monkeypatch): + backend = _FakeBackend(None) + rec = _LoadRecorder(backend) + _wire( + monkeypatch, + enabled = True, + resolves_to = ("unsloth/B-GGUF", "Q4_K_M", "unsloth/B-GGUF"), + backend = backend, + recorder = rec, + ) + stored = {"unsloth/B-GGUF": {"max_seq_length": 1024}} + monkeypatch.setattr(settings, "get_model_override", lambda mid: stored.get(mid, {})) + + _run_hook("unsloth/B-GGUF") + assert rec.calls[0].max_seq_length == 1024 + + +def test_override_route_preserves_launch_flags_across_a_settings_only_update(override_store): + # The settings page has no control for llama_extra_args, so it omits the field. + _put("unsloth/B-GGUF", llama_extra_args = ["--flash-attn"]) + resp = _put("unsloth/B-GGUF", max_seq_length = 4096) + entry = resp.overrides["unsloth/B-GGUF"] + assert entry["llama_extra_args"] == ["--flash-attn"] + assert entry["max_seq_length"] == 4096 + + # An explicit empty list is the UI's "forget this model", and with no other + # fields left it removes the entry outright. + gone = _put("unsloth/B-GGUF", llama_extra_args = []) + assert "unsloth/B-GGUF" not in gone.overrides + + +def test_override_found_under_a_concrete_path_with_variant(monkeypatch): + # A local folder or non-active HF cache resolves to a repo id plus a concrete + # path, and settings saved against the path must still be found. + backend = _FakeBackend(None) + rec = _LoadRecorder(backend) + _wire( + monkeypatch, + enabled = True, + resolves_to = ("/models/local/Qwen3-8B-Q4_K_M.gguf", "Q4_K_M", "unsloth/Qwen3-8B-GGUF"), + backend = backend, + recorder = rec, + ) + stored = {"/models/local/Qwen3-8B-Q4_K_M.gguf:Q4_K_M": {"max_seq_length": 8192}} + monkeypatch.setattr(settings, "get_model_override", lambda mid: stored.get(mid, {})) + + _run_hook("unsloth/Qwen3-8B-GGUF") + assert rec.calls[0].max_seq_length == 8192 + + +def test_path_qualified_override_beats_repo_qualified(monkeypatch): + # Most specific first: the settings page keys a local row (a folder, an LM Studio + # dir, a loose file) by the path being loaded, while the repo id is the advertised + # alias that a second copy of the same repo, or a hand-written overrides PUT, is + # configured under. The row the user actually edited wins. + backend = _FakeBackend(None) + rec = _LoadRecorder(backend) + _wire( + monkeypatch, + enabled = True, + resolves_to = ("/models/local/x.gguf", "Q4_K_M", "unsloth/B-GGUF"), + backend = backend, + recorder = rec, + ) + stored = { + "unsloth/B-GGUF:Q4_K_M": {"max_seq_length": 8192}, + "/models/local/x.gguf:Q4_K_M": {"max_seq_length": 1024}, + } + monkeypatch.setattr(settings, "get_model_override", lambda mid: stored.get(mid, {})) + + _run_hook("unsloth/B-GGUF") + assert rec.calls[0].max_seq_length == 1024 + + +def test_first_quant_save_keeps_legacy_bare_repo_launch_flags(override_store): + # Flags predating per-quant settings live under the bare repo id. + settings.set_model_override("unsloth/B-GGUF", llama_extra_args = ["--flash-attn"]) + + resp = _put("unsloth/B-GGUF:Q4_K_M", max_seq_length = 4096) + entry = resp.overrides["unsloth/B-GGUF:Q4_K_M"] + assert entry["max_seq_length"] == 4096 + assert entry["llama_extra_args"] == ["--flash-attn"] + + +def test_bare_repo_carry_over_does_not_split_a_windows_path(override_store): + # The colon in "C:\models\x.gguf" is not a variant separator: splitting naively + # looks up "C" and could graft another model's flags on. + settings.set_model_override("C", llama_extra_args = ["--flash-attn"]) + + resp = _put(r"C:\models\x.gguf", max_seq_length = 4096) + assert "llama_extra_args" not in resp.overrides[r"C:\models\x.gguf"] + + +def test_windows_path_with_quant_still_carries_over(override_store): + settings.set_model_override(r"C:\models\x.gguf", llama_extra_args = ["--flash-attn"]) + + resp = _put(r"C:\models\x.gguf:Q4_K_M", max_seq_length = 4096) + assert resp.overrides[r"C:\models\x.gguf:Q4_K_M"]["llama_extra_args"] == ["--flash-attn"] + + +def test_stale_gpu_ids_are_dropped_not_fatal(monkeypatch): + # A two-GPU pin replayed on a one-GPU box used to 400 the whole load; the + # contract is that one dead field degrades to defaults. + backend = _FakeBackend(None) + rec = _LoadRecorder(backend) + _wire( + monkeypatch, + enabled = True, + resolves_to = ("unsloth/B-GGUF", "Q4_K_M", "unsloth/B-GGUF"), + backend = backend, + recorder = rec, + ) + monkeypatch.setattr( + settings, + "get_model_override", + lambda mid: {"gpu_ids": [0, 1], "max_seq_length": 4096}, + ) + + async def _unusable(ids): + return False + + monkeypatch.setattr(inference_route, "_override_gpu_ids_still_resolve", _unusable) + + _run_hook("unsloth/B-GGUF") + req = rec.calls[0] + assert not req.gpu_ids + # The rest of the config still applies. + assert req.max_seq_length == 4096 + + +def test_usable_gpu_ids_are_kept(monkeypatch): + backend = _FakeBackend(None) + rec = _LoadRecorder(backend) + _wire( + monkeypatch, + enabled = True, + resolves_to = ("unsloth/B-GGUF", "Q4_K_M", "unsloth/B-GGUF"), + backend = backend, + recorder = rec, + ) + monkeypatch.setattr(settings, "get_model_override", lambda mid: {"gpu_ids": [0, 1]}) + + async def _usable(ids): + return True + + monkeypatch.setattr(inference_route, "_override_gpu_ids_still_resolve", _usable) + + _run_hook("unsloth/B-GGUF") + assert rec.calls[0].gpu_ids == [0, 1] + + +def test_override_gpu_ids_probe_never_raises(monkeypatch): + # On the load path, so a hardware error must read as "unusable", not a 500. + import utils.hardware.hardware as hw + + def boom(*args, **kwargs): + raise RuntimeError("driver exploded") + + monkeypatch.setattr(hw, "resolve_requested_gpu_ids", boom) + assert asyncio.run(inference_route._override_gpu_ids_still_resolve([0])) is False + + +def test_vulkan_ordinal_absent_from_the_probe_is_unusable(monkeypatch): + # resolve_requested_gpu_ids only rejects malformed Vulkan ordinals, so presence + # needs the ggml probe the load runs, or the load 400s on the skipped check. + from core.inference.llama_cpp import LlamaCppBackend + + monkeypatch.setattr(LlamaCppBackend, "_is_vulkan_backend", staticmethod(lambda: True)) + monkeypatch.setattr( + LlamaCppBackend, "_find_llama_server_binary", staticmethod(lambda: "/bin/llama-server") + ) + monkeypatch.setattr( + LlamaCppBackend, "_get_gpu_memory", staticmethod(lambda binary: [(0, 8192)]) + ) + assert asyncio.run(inference_route._override_gpu_ids_still_resolve([0])) is True + assert asyncio.run(inference_route._override_gpu_ids_still_resolve([7])) is False + assert asyncio.run(inference_route._override_gpu_ids_still_resolve([0, 1])) is False + + +def test_vulkan_probe_without_a_binary_does_not_block_the_load(monkeypatch): + # Nothing to probe with, and refusing would drop a valid pin on every load, + # so the later path stays the authority. + from core.inference.llama_cpp import LlamaCppBackend + + monkeypatch.setattr(LlamaCppBackend, "_is_vulkan_backend", staticmethod(lambda: True)) + monkeypatch.setattr(LlamaCppBackend, "_find_llama_server_binary", staticmethod(lambda: None)) + assert asyncio.run(inference_route._override_gpu_ids_still_resolve([0])) is True + + +def test_default_save_preserves_flags_instead_of_removing(override_store): + # "Remember for this model" with all-default values sends no fields, which is + # shape-identical to a removal; guessing wrong wipes unrecoverable launch flags. + settings.set_model_override("unsloth/B-GGUF", llama_extra_args = ["--flash-attn"]) + + resp = _put("unsloth/B-GGUF", remove = False) + assert resp.overrides["unsloth/B-GGUF"]["llama_extra_args"] == ["--flash-attn"] + + +def test_explicit_remove_still_clears_everything(override_store): + settings.set_model_override( + "unsloth/B-GGUF", llama_extra_args = ["--flash-attn"], max_seq_length = 4096 + ) + resp = _put("unsloth/B-GGUF", remove = True, llama_extra_args = []) + assert "unsloth/B-GGUF" not in resp.overrides + + +def test_bare_payload_without_remove_flag_still_removes(override_store): + # The original contract, kept for any caller that predates the flag. + settings.set_model_override("unsloth/B-GGUF", max_seq_length = 4096) + resp = _put("unsloth/B-GGUF") + assert "unsloth/B-GGUF" not in resp.overrides + + +def test_remove_false_with_real_fields_saves_normally(override_store): + resp = _put("unsloth/B-GGUF", remove = False, max_seq_length = 8192) + assert resp.overrides["unsloth/B-GGUF"]["max_seq_length"] == 8192 + + +def test_override_lookup_falls_back_to_case_insensitive(override_store): + # The browser lowercases ids, so the backfill writes "unsloth/qwen3-8b-gguf:q4_k_m" + # while the resolver asks for the repo's real casing. + settings.set_model_override("unsloth/qwen3-8b-gguf:q4_k_m", max_seq_length = 8192) + got = settings.get_model_override("unsloth/Qwen3-8B-GGUF:Q4_K_M") + assert got["max_seq_length"] == 8192 + + +def test_exact_override_match_beats_a_case_variant(override_store): + settings.set_model_override("/models/foo.gguf", max_seq_length = 1024) + settings.set_model_override("/models/Foo.gguf", max_seq_length = 8192) + assert settings.get_model_override("/models/Foo.gguf")["max_seq_length"] == 8192 + assert settings.get_model_override("/models/foo.gguf")["max_seq_length"] == 1024 + + +def test_ambiguous_case_fallback_matches_nothing(override_store): + # Two POSIX paths differing only in case are two files, so guessing between + # them applies one model's settings to another. + settings.set_model_override("/models/foo.gguf", max_seq_length = 1024) + settings.set_model_override("/models/FOO.gguf", max_seq_length = 8192) + assert settings.get_model_override("/models/Foo.gguf") == {} + + +def test_request_used_api_key_distinguishes_key_from_session(): + from auth.authentication import API_KEY_PREFIX + + class _Req: + def __init__(self, header): + self.headers = {"authorization": header} if header else {} + + assert inference_route._request_used_api_key(_Req(f"Bearer {API_KEY_PREFIX}abc")) is True + assert inference_route._request_used_api_key(_Req(f"bearer {API_KEY_PREFIX}abc")) is True + assert inference_route._request_used_api_key(_Req("Bearer eyJhbGciOiJIUzI1NiJ9.x")) is False + assert inference_route._request_used_api_key(_Req("")) is False + assert inference_route._request_used_api_key(_Req(None)) is False + # Runs on the hot path of every tracked request, so a malformed request object + # must read as "not an API key" rather than raise. + assert inference_route._request_used_api_key(object()) is False + + +def test_case_fallback_never_applies_to_a_posix_path(override_store): + # Two files differing only in case are two models on Linux, so a near miss must + # load defaults rather than another model's context and GPU pin. + settings.set_model_override("/models/foo.gguf", max_seq_length = 8192, gpu_ids = [1]) + assert settings.get_model_override("/models/Foo.gguf") == {} + assert settings.get_model_override("/models/foo.gguf")["max_seq_length"] == 8192 + + +def test_case_fallback_does_apply_to_a_windows_path(override_store): + # NTFS is case-insensitive, so these name one file and the browser folds drive paths + # before storing. + settings.set_model_override(r"c:\models\foo.gguf", max_seq_length = 8192) + assert settings.get_model_override(r"C:\models\FOO.gguf")["max_seq_length"] == 8192 + assert settings.get_model_override("C:/Models/Foo.gguf")["max_seq_length"] == 8192 + + +def test_case_fallback_applies_to_unc_and_wsl_drive_paths(override_store): + settings.set_model_override(r"\\server\share\foo.gguf", max_seq_length = 4096) + settings.set_model_override("/mnt/c/models/bar.gguf", max_seq_length = 2048) + assert settings.get_model_override(r"\\Server\Share\FOO.gguf")["max_seq_length"] == 4096 + assert settings.get_model_override("/mnt/C/Models/Bar.gguf")["max_seq_length"] == 2048 + + +def test_a_plain_posix_path_under_mnt_stays_case_sensitive(override_store): + # Only /mnt/ is a WSL drive mount; /mnt/data is an ordinary Linux mount + # point and stays case-sensitive. + settings.set_model_override("/mnt/data/models/foo.gguf", max_seq_length = 8192) + assert settings.get_model_override("/mnt/data/models/Foo.gguf") == {} + + +def test_an_ambiguous_windows_case_fallback_still_matches_nothing(override_store): + # Two stored keys folding to one has no single answer, so the load takes + # defaults rather than guessing. + settings.set_model_override(r"c:\models\foo.gguf", max_seq_length = 1024) + settings.set_model_override("C:/models/FOO.gguf", max_seq_length = 8192) + assert settings.get_model_override(r"C:\Models\Foo.gguf") == {} + + +def test_case_fallback_still_covers_repo_ids(override_store): + # The migration case this fallback exists for. + settings.set_model_override("unsloth/qwen3-8b-gguf:q4_k_m", max_seq_length = 8192) + assert settings.get_model_override("unsloth/Qwen3-8B-GGUF:Q4_K_M")["max_seq_length"] == 8192 + + +def test_explicit_remove_is_not_blocked_by_stale_invalid_flags(override_store): + # remove is the operation discriminator, so a rejected launch flag must not turn + # "forget this model" into a 400 that leaves the override in place. + settings.set_model_override("unsloth/B-GGUF", max_seq_length = 4096) + resp = _put("unsloth/B-GGUF", remove = True, llama_extra_args = ["--port", "1234"]) + assert "unsloth/B-GGUF" not in resp.overrides + + +def test_explicit_remove_wins_over_config_fields_in_the_same_payload(override_store): + # remove is the operation discriminator, so a stale field alongside it must not + # turn "forget this model" into an update. + settings.set_model_override("unsloth/B-GGUF", max_seq_length = 4096) + resp = _put("unsloth/B-GGUF", remove = True, max_seq_length = 8192, tensor_parallel = True) + assert "unsloth/B-GGUF" not in resp.overrides + + +def test_posix_colon_in_a_path_is_not_treated_as_a_quant(override_store): + # "/models/foo:bar.gguf" is one POSIX filename, not repo + quant; splitting it + # grafts /models/foo's launch flags onto a different model. + settings.set_model_override("/models/foo", llama_extra_args = ["--flash-attn"]) + resp = _put("/models/foo:bar.gguf", max_seq_length = 4096) + assert "llama_extra_args" not in resp.overrides["/models/foo:bar.gguf"] + + +def test_unknown_quant_label_on_a_gguf_still_carries_flags_over(override_store): + # A .gguf with no recognizable quant token is labelled by its stem, so the UI saves + # under "/models/custom.gguf:custom". + settings.set_model_override("/models/custom.gguf", llama_extra_args = ["--flash-attn"]) + resp = _put("/models/custom.gguf:custom", max_seq_length = 4096) + assert resp.overrides["/models/custom.gguf:custom"]["llama_extra_args"] == ["--flash-attn"] + + +def test_bpw_qualified_variants_still_carry_flags_over(override_store): + # utils/models/model_config.py keeps a bits-per-weight modifier on the label to keep + # two files at the same base quant distinct, and that form reaches the override + # keys. + settings.set_model_override("unsloth/Repo-GGUF", llama_extra_args = ["--flash-attn"]) + resp = _put("unsloth/Repo-GGUF:IQ4_XS-3.53bpw", max_seq_length = 4096) + assert resp.overrides["unsloth/Repo-GGUF:IQ4_XS-3.53bpw"]["llama_extra_args"] == [ + "--flash-attn" + ] + + +def test_a_posix_path_variant_folds_while_the_path_does_not(override_store): + # The browser lowercases the quant but keeps POSIX path casing, so the migrated + # "/models/Foo:q4_k_m" must answer the scanner's "/models/Foo:Q4_K_M" while the + # path itself stays case-sensitive. + settings.set_model_override("/models/Foo:q4_k_m", max_seq_length = 8192) + assert settings.get_model_override("/models/Foo:Q4_K_M")["max_seq_length"] == 8192 + assert settings.get_model_override("/models/foo:Q4_K_M") == {} + + +def test_an_unknown_gguf_label_is_reachable_in_either_casing(override_store): + # A .gguf with no recognizable quant token is labelled by its stem, and v2 storage + # lowercases that label while the scanner keeps the filename casing. + settings.set_model_override("/models/CustomModel.gguf:custommodel", max_seq_length = 8192) + got = settings.get_model_override("/models/CustomModel.gguf:CustomModel") + assert got["max_seq_length"] == 8192 + # The path itself is still case-sensitive on POSIX. + assert settings.get_model_override("/models/custommodel.gguf:CustomModel") == {} + + +def test_a_posix_colon_filename_is_not_folded_as_a_variant(override_store): + # "/models/foo:Bar.gguf" is one filename, not path + quant, so folding its tail + # would reach a different file's settings. + settings.set_model_override("/models/foo:bar.gguf", max_seq_length = 8192) + assert settings.get_model_override("/models/foo:Bar.gguf") == {} + + +def test_a_suffix_the_scanner_would_not_derive_carries_nothing_over(override_store): + # Only the scanner's exact label is accepted, so an unrelated colon suffix cannot + # reach another model's flags. + settings.set_model_override("/models/custom.gguf", llama_extra_args = ["--flash-attn"]) + resp = _put("/models/custom.gguf:something-else", max_seq_length = 4096) + assert "llama_extra_args" not in resp.overrides["/models/custom.gguf:something-else"] + + +def test_unknown_quant_label_carries_over_for_a_windows_path(override_store): + # Written on Windows but read back on a backend where a backslash is an ordinary + # filename character. + settings.set_model_override(r"C:\models\custom.gguf", llama_extra_args = ["--flash-attn"]) + resp = _put(r"C:\models\custom.gguf:custom", max_seq_length = 4096) + assert resp.overrides[r"C:\models\custom.gguf:custom"]["llama_extra_args"] == ["--flash-attn"] + + +def test_real_quant_suffix_on_a_path_still_carries_flags_over(override_store): + settings.set_model_override("/models/x.gguf", llama_extra_args = ["--flash-attn"]) + resp = _put("/models/x.gguf:Q4_K_M", max_seq_length = 4096) + assert resp.overrides["/models/x.gguf:Q4_K_M"]["llama_extra_args"] == ["--flash-attn"] + + +def test_load_retries_without_gpu_ids_when_the_loader_rejects_the_pin(monkeypatch): + # The pre-flight check cannot mirror every loader rule (a Vulkan diffusion GGUF + # refuses GPU selection), and a stale pin must never block a request. + from fastapi import HTTPException + + backend = _FakeBackend(None) + rec = _LoadRecorder(backend) + _wire( + monkeypatch, + enabled = True, + resolves_to = ("unsloth/B-GGUF", "Q4_K_M", "unsloth/B-GGUF"), + backend = backend, + recorder = rec, + ) + monkeypatch.setattr( + settings, "get_model_override", lambda mid: {"gpu_ids": [0], "max_seq_length": 4096} + ) + + async def _usable(ids): + return True + + monkeypatch.setattr(inference_route, "_override_gpu_ids_still_resolve", _usable) + + calls = {"n": 0} + + async def _load(request, *args, **kwargs): + calls["n"] += 1 + if calls["n"] == 1: + raise HTTPException( + status_code = 400, + detail = "GPU selection (gpu_ids) is not supported for a DiffusionGemma GGUF", + ) + return await rec(request, *args, **kwargs) + + monkeypatch.setattr(inference_route, "_load_model_impl", _load) + + _run_hook("unsloth/B-GGUF") + assert calls["n"] == 2 + served = rec.calls[-1] + assert not served.gpu_ids + assert served.max_seq_length == 4096 + + +def test_a_non_gpu_load_failure_is_not_retried(monkeypatch): + from fastapi import HTTPException + + backend = _FakeBackend(None) + rec = _LoadRecorder(backend) + _wire( + monkeypatch, + enabled = True, + resolves_to = ("unsloth/B-GGUF", "Q4_K_M", "unsloth/B-GGUF"), + backend = backend, + recorder = rec, + ) + monkeypatch.setattr(settings, "get_model_override", lambda mid: {"gpu_ids": [0]}) + + async def _usable(ids): + return True + + monkeypatch.setattr(inference_route, "_override_gpu_ids_still_resolve", _usable) + + calls = {"n": 0} + + async def _load(request, *args, **kwargs): + calls["n"] += 1 + raise HTTPException(status_code = 400, detail = "Corrupt GGUF header") + + monkeypatch.setattr(inference_route, "_load_model_impl", _load) + + with pytest.raises(HTTPException): + _run_hook("unsloth/B-GGUF") + assert calls["n"] == 1 + + +def test_removal_clears_the_entry_a_load_would_actually_resolve(override_store): + # The browser normalizes casing before storing, so a forget can carry a different + # casing; removing only the literal key leaves an entry loads still resolve to. + settings.set_model_override("unsloth/B-GGUF:Q4_K_M", max_seq_length = 8192) + assert settings.get_model_override("unsloth/b-gguf:q4_k_m")["max_seq_length"] == 8192 + + _put("unsloth/b-gguf:q4_k_m", remove = True) + assert settings.get_model_overrides() == {} + assert settings.get_model_override("unsloth/B-GGUF:Q4_K_M") == {} + + +def test_save_updates_the_existing_case_variant_instead_of_forking_it(override_store): + # The backfill stores lowercase keys while a later UI save carries the catalog's + # casing. + settings.set_model_override("unsloth/b-gguf:q4_k_m", max_seq_length = 8192) + _put("unsloth/B-GGUF:Q4_K_M", max_seq_length = 4096) + assert list(settings.get_model_overrides()) == ["unsloth/b-gguf:q4_k_m"] + assert settings.get_model_override("Unsloth/B-GGUF:Q4_K_M")["max_seq_length"] == 4096 + + +def test_removal_of_a_path_still_only_touches_the_exact_key(override_store): + settings.set_model_override("/models/foo.gguf", max_seq_length = 8192) + _put("/models/Foo.gguf", remove = True) + # A different file must survive its neighbour being forgotten. + assert settings.get_model_override("/models/foo.gguf")["max_seq_length"] == 8192 + + +def test_forget_clears_the_filename_derived_key_a_load_still_reads(override_store): + # The picker keyed a standalone .gguf by the quant label from its filename before it + # settled on the bare path, and the backfill carries those entries over from an + # upgraded browser. + settings.set_model_override( + "/models/Qwen3-8B-Q4_K_M.gguf:q4_k_m", + max_seq_length = 8192, + ) + _put("/models/Qwen3-8B-Q4_K_M.gguf", remove = True) + assert settings.get_model_override("/models/Qwen3-8B-Q4_K_M.gguf:Q4_K_M") == {} + assert settings.get_model_overrides() == {} + + +def test_forget_leaves_another_file_own_derived_key_alone(override_store): + # The derived key is built from the forgotten file's own path and its own + # label, so a neighbour that happens to share a quant keeps its settings. + settings.set_model_override("/models/Other-Q4_K_M.gguf:q4_k_m", max_seq_length = 4096) + _put("/models/Qwen3-8B-Q4_K_M.gguf", remove = True) + assert settings.get_model_override("/models/Other-Q4_K_M.gguf:Q4_K_M")["max_seq_length"] == 4096 + + +def test_forget_of_a_repo_quant_key_derives_nothing(override_store): + # Only a bare .gguf path derives a label. + settings.set_model_override("unsloth/b-gguf:q4_k_m", max_seq_length = 8192) + _put("unsloth/B-GGUF", remove = True) + assert settings.get_model_override("unsloth/b-gguf:q4_k_m")["max_seq_length"] == 8192 + + def test_a_tag_that_names_no_quant_resolves_to_the_repo(monkeypatch): # A downloaded but unloaded GGUF asked for as org/model:latest missed the resolver, # so the switch could not load it (404ing on a quant that was never a quant with - # auto-download on, refusing with it off). A real quant that is not on disk must - # still miss, or a swap would serve the wrong weights under the right name. + # auto-download on, refusing with it off). from core.inference.local_model_resolver import _LocalGgufEntry import time @@ -4179,7 +4861,6 @@ def test_a_tag_that_names_no_quant_resolves_to_the_repo(monkeypatch): def test_any_finished_download_drops_the_resolver_cache(monkeypatch): # Only the API auto-download watcher invalidated, so a GGUF fetched in the Hub UI # stayed absent to the cache-only request path and the resident model answered. - # Every worker exits through here. import logging from hub.services import download_lifecycle @@ -4227,8 +4908,7 @@ def test_any_finished_download_drops_the_resolver_cache(monkeypatch): def test_invalidating_keeps_the_entries_it_already_had(monkeypatch): # The request path reads this cache without scanning, so emptying it leaves no - # evidence until the rebuild lands. Only a completed download invalidates, and - # that only adds, so the entries stay true. + # evidence until the rebuild lands. import time entry = resolver._LocalGgufEntry("org/old", "/srv/models/org--old", ("Q4_K_M",)) @@ -4244,8 +4924,7 @@ def test_invalidating_keeps_the_entries_it_already_had(monkeypatch): def test_a_bare_local_id_takes_the_quant_a_plain_load_would(monkeypatch, tmp_path): # list_local_gguf_variants orders by descending size, so the head is the biggest - # quant. Resolving a bare id to that could evict a working model and then OOM on an - # F16 next to a fitting Q4, and /v1/models advertised the same head for pinning. + # quant. from core.inference.local_model_resolver import _local_gguf_entry for name, size in (("model-F16.gguf", 900), ("model-Q4_K_M.gguf", 100)): @@ -4314,9 +4993,7 @@ def test_a_just_downloaded_model_is_evidence_before_the_scan_indexes_it(monkeypa def test_a_finished_dataset_is_not_recorded_as_a_local_model(monkeypatch): - # finalize_worker_exit is shared with dataset downloads. Noting one as a local model - # would refuse a bare /v1 request naming that id instead of letting a foreign id - # fall through, and would kick off a multi-directory scan for nothing. + # finalize_worker_exit is shared with dataset downloads. import logging import time @@ -4378,3 +5055,412 @@ def test_two_local_paths_differing_only_in_case_are_not_the_same_model(monkeypat alias = _FakeBackend(loaded_id = "unsloth/Qwen3-4B-GGUF") monkeypatch.setattr(inference_route, "get_llama_cpp_backend", lambda: alias) assert inference_route._loaded_satisfies("unsloth/qwen3-4b-gguf") is True + + +def test_abs_path_ids_are_recognised_in_either_platform_spelling(): + """Path() follows the running OS, so a Windows host read "/home/me/x.gguf" as + relative and a POSIX host read "C:\\models\\x.gguf" the same way, and either + then reached /v1/models as a published host path. Ids outlive the machine + that wrote them (settings sync, WSL, a copied config), and the model-override + identity already folds both spellings.""" + from types import SimpleNamespace + + for spelling in ("/home/me/models/x.gguf", "C:\\models\\x.gguf", "//host/share/x.gguf"): + assert resolver._is_abs_path_id(spelling) is True, spelling + # An alias wins, and with none the path is stripped to a public id. + assert ( + resolver._advertised_loader_id( + SimpleNamespace(id = spelling, model_id = "org/X-GGUF", display_name = "X") + ) + == "org/X-GGUF" + ) + advertised = resolver._advertised_loader_id( + SimpleNamespace(id = spelling, model_id = None, display_name = None) + ) + assert advertised is not None + assert "/" not in advertised and "\\" not in advertised, advertised + + # A repo id has no leading separator, drive or UNC prefix under either reading. + for repo_id in ("org/Repo-GGUF", "Repo", "org/Repo-GGUF:Q4_K_M"): + assert resolver._is_abs_path_id(repo_id) is False, repo_id + + +def test_fill_absent_fields_put_never_replaces_a_newer_server_value(override_store): + """The one-time localStorage backfill reads the override map once and then + writes each model in turn, so a save by another tab during that pass was + overwritten by this browser's older copy. fill_absent_fields writes only what + the entry lacks, so every value already on the server wins.""" + import routes.settings as settings_route + + # The other tab's save lands first. + newer = settings_route.ModelOverridePayload(model_id = "unsloth/B-GGUF", max_seq_length = 8192) + settings_route.update_openai_auto_switch_override(newer, "tester") + + # The backfill's write, carrying this browser's older localStorage value. + backfill = settings_route.ModelOverridePayload( + model_id = "unsloth/B-GGUF", max_seq_length = 2048, fill_absent_fields = True + ) + resp = settings_route.update_openai_auto_switch_override(backfill, "tester") + assert resp.overrides["unsloth/B-GGUF"]["max_seq_length"] == 8192 + + # With nothing stored it still creates, or the migration would never run. + fresh = settings_route.ModelOverridePayload( + model_id = "unsloth/C-GGUF", max_seq_length = 2048, fill_absent_fields = True + ) + resp2 = settings_route.update_openai_auto_switch_override(fresh, "tester") + assert resp2.overrides["unsloth/C-GGUF"]["max_seq_length"] == 2048 + + +def test_fill_absent_fields_carries_the_browser_only_settings_into_a_legacy_entry(monkeypatch): + """Codex P1: the override map shipped before the browser mirror did, storing only + llama_extra_args and max_seq_length. An upgraded install holds such an entry while + localStorage holds the context, KV cache, speculative and GPU settings, and an + entry-level skip would strand exactly what the migration exists to carry.""" + import routes.settings as settings_route + + store = _mock_override_store(monkeypatch) + + legacy = settings_route.ModelOverridePayload( + model_id = "unsloth/B-GGUF:Q4_K_M", + llama_extra_args = ["--flash-attn"], + max_seq_length = 8192, + ) + settings_route.update_openai_auto_switch_override(legacy, "tester") + + backfill = settings_route.ModelOverridePayload( + model_id = "unsloth/B-GGUF:Q4_K_M", + # The browser's own copy of a field the server already has, plus the ones + # only it holds. + max_seq_length = 2048, + custom_context_length = 32768, + kv_cache_dtype = "q8_0", + speculative_type = "ngram", + gpu_ids = [0, 1], + fill_absent_fields = True, + ) + resp = settings_route.update_openai_auto_switch_override(backfill, "tester") + entry = resp.overrides["unsloth/B-GGUF:Q4_K_M"] + # The server's own values survive untouched. + assert entry["max_seq_length"] == 8192 + assert entry["llama_extra_args"] == ["--flash-attn"] + # The browser-only settings are now there, so an API load applies them. + assert entry["custom_context_length"] == 32768 + assert entry["kv_cache_dtype"] == "q8_0" + assert entry["speculative_type"] == "ngram" + assert entry["gpu_ids"] == [0, 1] + # One entry, not two: the fill resolves onto the key a load reads. + assert list(store[settings.MODEL_OVERRIDES_SETTING_KEY]) == ["unsloth/B-GGUF:Q4_K_M"] + + # An ordinary save is still a replacement, or a settings edit could never + # clear a field. + edit = settings_route.ModelOverridePayload( + model_id = "unsloth/B-GGUF:Q4_K_M", max_seq_length = 4096 + ) + resp2 = settings_route.update_openai_auto_switch_override(edit, "tester") + assert resp2.overrides["unsloth/B-GGUF:Q4_K_M"]["max_seq_length"] == 4096 + assert "kv_cache_dtype" not in resp2.overrides["unsloth/B-GGUF:Q4_K_M"] + + +def test_fill_absent_fields_matches_a_legacy_casing_and_never_deletes(override_store): + """The stored key can carry the casing an older install typed, and it must not + be duplicated or emptied by a fill for the folded spelling.""" + import routes.settings as settings_route + + stored = settings_route.ModelOverridePayload( + model_id = "Unsloth/B-GGUF:Q4_K_M", max_seq_length = 8192 + ) + settings_route.update_openai_auto_switch_override(stored, "tester") + + folded = settings_route.ModelOverridePayload( + model_id = "unsloth/b-gguf:q4_k_m", max_seq_length = 2048, fill_absent_fields = True + ) + resp = settings_route.update_openai_auto_switch_override(folded, "tester") + assert list(resp.overrides) == ["Unsloth/B-GGUF:Q4_K_M"] + assert resp.overrides["Unsloth/B-GGUF:Q4_K_M"]["max_seq_length"] == 8192 + + # An all-default fill is a no-op, not the "empty payload means forget" path. + empty = settings_route.ModelOverridePayload( + model_id = "Unsloth/B-GGUF:Q4_K_M", fill_absent_fields = True + ) + resp2 = settings_route.update_openai_auto_switch_override(empty, "tester") + assert resp2.overrides["Unsloth/B-GGUF:Q4_K_M"]["max_seq_length"] == 8192 + + # A fill that is also a delete has no meaning. + with pytest.raises(HTTPException) as excinfo: + _put("Unsloth/B-GGUF:Q4_K_M", remove = True, fill_absent_fields = True) + assert excinfo.value.status_code == 400 + + +def test_fill_absent_fields_does_not_break_the_empty_payload_removal(override_store): + """fill_absent_fields is a write mode, not a saved field: leaving it in the dumped + payload would make every request look non-empty and silently retire the legacy + "a payload carrying only model_id forgets this model" contract.""" + import routes.settings as settings_route + + stored = settings_route.ModelOverridePayload(model_id = "unsloth/B-GGUF", max_seq_length = 4096) + settings_route.update_openai_auto_switch_override(stored, "tester") + empty = settings_route.ModelOverridePayload(model_id = "unsloth/B-GGUF") + resp = settings_route.update_openai_auto_switch_override(empty, "tester") + assert "unsloth/B-GGUF" not in resp.overrides + + +def test_map_entry_fill_reads_and_writes_in_one_transaction(tmp_path, monkeypatch): + """The real store, not the in-memory stand-in: the read has to share the write's + transaction, or a concurrent writer still slips between them.""" + import storage.studio_db as db + + monkeypatch.setenv("UNSLOTH_STUDIO_HOME", str(tmp_path)) + monkeypatch.setattr(db, "_schema_ready", False) + + key = "test_map_entry_create" + assert db.upsert_app_setting_map_entry(key, "a", {"v": 1}) == {"a": {"v": 1}} + # Present: the stored value stays, and a field it lacks is added. + assert db.upsert_app_setting_map_entry(key, "a", {"v": 2}, fill_absent_fields = True) == { + "a": {"v": 1} + } + assert db.get_app_setting(key) == {"a": {"v": 1}} + assert db.upsert_app_setting_map_entry(key, "a", {"v": 2, "w": 7}, fill_absent_fields = True) == { + "a": {"v": 1, "w": 7} + } + assert db.get_app_setting(key) == {"a": {"v": 1, "w": 7}} + # Absent: created. + assert db.upsert_app_setting_map_entry(key, "b", {"v": 3}, fill_absent_fields = True) == { + "a": {"v": 1, "w": 7}, + "b": {"v": 3}, + } + # A fill never deletes, even with nothing to store. + assert db.upsert_app_setting_map_entry(key, "a", None, fill_absent_fields = True) == { + "a": {"v": 1, "w": 7}, + "b": {"v": 3}, + } + # The ordinary write still replaces and still removes. + db.upsert_app_setting_map_entry(key, "a", {"v": 9}) + db.upsert_app_setting_map_entry(key, "b", None) + assert db.get_app_setting(key) == {"a": {"v": 9}} + + +def test_gpu_ids_dedupe_is_not_a_scan_of_the_list_being_built(): + """gpu_ids arrives from an authenticated client and normalize_model_override + de-duplicates it. Testing membership against the growing list walks up to + MAX_GPU_ID entries per element; a set keeps the pass linear. Order, bounds and + the bool rejection all have to survive the change.""" + import time + + from utils.openai_auto_switch_settings import MAX_GPU_ID, normalize_model_override + + assert normalize_model_override({"gpu_ids": [3, 1, 3, 0, 1, 2]})["gpu_ids"] == [3, 1, 0, 2] + # bool is an int subclass; [True, False] must not pin GPUs 1 and 0. + assert normalize_model_override({"gpu_ids": [True, False]}) == {} + assert normalize_model_override({"gpu_ids": [MAX_GPU_ID + 1, -1, 2]})["gpu_ids"] == [2] + + ids = [index % (MAX_GPU_ID + 1) for index in range(200_000)] + started = time.perf_counter() + normalized = normalize_model_override({"gpu_ids": ids}) + elapsed = time.perf_counter() - started + assert len(normalized["gpu_ids"]) == MAX_GPU_ID + 1 + # The scan version took ~1s for this input on a dev box; a linear pass is ~50ms. + assert elapsed < 0.5, elapsed + + +def test_gpu_ids_payload_is_bounded(): + """A list longer than the number of ids the normalizer can store adds nothing + but work, so it is rejected at the boundary. A real device list is tiny.""" + import pydantic + + import routes.settings as settings_route + from utils.openai_auto_switch_settings import MAX_GPU_ID + + assert settings_route.MAX_GPU_IDS == MAX_GPU_ID + 1 + at_limit = settings_route.ModelOverridePayload( + model_id = "x", gpu_ids = list(range(settings_route.MAX_GPU_IDS)) + ) + assert len(at_limit.gpu_ids) == settings_route.MAX_GPU_IDS + with pytest.raises(pydantic.ValidationError): + settings_route.ModelOverridePayload( + model_id = "x", gpu_ids = [0] * (settings_route.MAX_GPU_IDS + 1) + ) + # The ordinary case is untouched. + assert settings_route.ModelOverridePayload(model_id = "x", gpu_ids = [0, 1]).gpu_ids == [0, 1] + + +# โ”€โ”€ codex round: the concrete key beats the advertised alias โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ + + +def _switch_with_overrides(monkeypatch, resolves_to, stored, requested): + """Run the auto-switch hook against a real override map and return the load.""" + backend = _FakeBackend(None) + rec = _LoadRecorder(backend) + _wire(monkeypatch, enabled = True, resolves_to = resolves_to, backend = backend, recorder = rec) + _mock_override_store(monkeypatch) + for key, max_seq_length in stored.items(): + settings.set_model_override(key, max_seq_length = max_seq_length) + _run_hook(requested) + assert len(rec.calls) == 1 + return rec.calls[0] + + +def test_a_loose_gguf_prefers_its_path_keyed_settings_over_the_alias(monkeypatch): + """Codex: the settings UI keys a standalone .gguf by its bare path, while + override_id is the filename stem /v1/models advertises and an overrides PUT can + be written against. Reading the alias first let it shadow the saved settings for + good, so an API load kept applying the old flags.""" + path = "/srv/models/Qwen3-8B-Q4_K_M.gguf" + alias = "Qwen3-8B-Q4_K_M" + req = _switch_with_overrides( + monkeypatch, + resolves_to = (path, None, alias), + stored = {alias: 2048, path: 32768}, + requested = alias, + ) + assert req.max_seq_length == 32768 + + # The alias is still read when it is the only key, so an entry written against + # the advertised id keeps working. + req2 = _switch_with_overrides( + monkeypatch, + resolves_to = (path, None, alias), + stored = {alias: 2048}, + requested = alias, + ) + assert req2.max_seq_length == 2048 + + +def test_the_filename_label_key_no_longer_shadows_the_bare_path(monkeypatch): + """An early build of this feature keyed a standalone .gguf by the quant label + derived from its filename. Those entries stay readable, but the bare path the + picker writes today comes first.""" + path = "/srv/models/Qwen3-8B-Q4_K_M.gguf" + alias = "Qwen3-8B-Q4_K_M" + req = _switch_with_overrides( + monkeypatch, + resolves_to = (path, None, alias), + stored = {f"{path}:Q4_K_M": 2048, path: 32768}, + requested = alias, + ) + assert req.max_seq_length == 32768 + + req2 = _switch_with_overrides( + monkeypatch, + resolves_to = (path, None, alias), + stored = {f"{path}:Q4_K_M": 2048}, + requested = alias, + ) + assert req2.max_seq_length == 2048 + + +def test_a_variant_qualified_path_key_beats_the_same_quant_under_the_alias(monkeypatch): + """An LM Studio dir or a non-active HF cache is configured against its path, so + a same-quant entry under the repo id (another copy of the same repo, or a + hand-written PUT) must not win over the row the user actually edited.""" + path = "/srv/lmstudio/publisher/Qwen3-8B-GGUF" + repo = "publisher/Qwen3-8B-GGUF" + req = _switch_with_overrides( + monkeypatch, + resolves_to = (path, "Q4_K_M", repo), + stored = {f"{repo}:Q4_K_M": 2048, f"{path}:Q4_K_M": 32768}, + requested = f"{repo}:Q4_K_M", + ) + assert req.max_seq_length == 32768 + + +def test_a_cached_repo_still_resolves_by_its_repo_id(monkeypatch): + """The Hub keys a cached repo row by its repo id, which is the advertised id, + and no path entry exists for it, so it still resolves on the second try.""" + snapshot = "/mnt/old-cache/models--unsloth--Qwen3-8B-GGUF/snapshots/abc123" + repo = "unsloth/Qwen3-8B-GGUF" + req = _switch_with_overrides( + monkeypatch, + resolves_to = (snapshot, "Q4_K_M", repo), + stored = {f"{repo}:Q4_K_M": 32768}, + requested = f"{repo}:Q4_K_M", + ) + assert req.max_seq_length == 32768 + + # A bare entry under the repo id keeps working too. + req2 = _switch_with_overrides( + monkeypatch, + resolves_to = (snapshot, "Q4_K_M", repo), + stored = {repo: 16384}, + requested = f"{repo}:Q4_K_M", + ) + assert req2.max_seq_length == 16384 + + +def test_a_fill_does_not_replay_a_stored_flag_through_validation(monkeypatch): + """The migration now writes for entries it used to skip, and an omitted + llama_extra_args is normally carried over from the stored entry. Replaying a + flag that has been denylisted since it was saved would 400 the one-time + migration, which then retries on every start. A fill keeps the stored flags + without sending them back.""" + import routes.settings as settings_route + from core.inference import llama_server_args + + store = _mock_override_store(monkeypatch) + settings.set_model_override("unsloth/B-GGUF:Q4_K_M", llama_extra_args = ["--flash-attn"]) + + # The flag is refused from now on, as a later release's denylist would. + real_validate = llama_server_args.validate_extra_args + + def _reject_flash_attn(args): + if args and "--flash-attn" in args: + raise ValueError("--flash-attn is managed by the server.") + return real_validate(args) + + monkeypatch.setattr(llama_server_args, "validate_extra_args", _reject_flash_attn) + + fill = settings_route.ModelOverridePayload( + model_id = "unsloth/B-GGUF:Q4_K_M", custom_context_length = 32768, fill_absent_fields = True + ) + resp = settings_route.update_openai_auto_switch_override(fill, "tester") + entry = resp.overrides["unsloth/B-GGUF:Q4_K_M"] + assert entry["llama_extra_args"] == ["--flash-attn"] + assert entry["custom_context_length"] == 32768 + assert list(store[settings.MODEL_OVERRIDES_SETTING_KEY]) == ["unsloth/B-GGUF:Q4_K_M"] + + # An ordinary save still validates what it is handed. + with pytest.raises(HTTPException) as excinfo: + _put("unsloth/C-GGUF", llama_extra_args = ["--flash-attn"]) + assert excinfo.value.status_code == 400 + + +def test_override_payload_rejects_booleans_for_numeric_fields(): + """bool subclasses int and pydantic parses non-strictly, so `true` would + arrive as 1: `max_seq_length: true` becomes a one-token context and + `gpu_ids: [true]` pins GPU 1. _bounded_int rejects bools for exactly that + reason, but never sees one, because coercion happens at the route boundary + first. Reject them there so that guard is reachable through this path.""" + import pytest + from pydantic import ValidationError + from routes.settings import ModelOverridePayload + + for field, value in ( + ("max_seq_length", True), + ("custom_context_length", True), + ("spec_draft_n_max", True), + ("n_parallel", True), + ("gpu_layers", False), + ("n_cpu_moe", True), + ("gpu_ids", [True]), + ("gpu_ids", [0, False, 2]), + ): + with pytest.raises(ValidationError): + ModelOverridePayload(model_id = "unsloth/x-GGUF:Q4_K_M", **{field: value}) + + # Only bools: every real value the picker sends still validates, and the + # fields that ARE booleans keep working. + ok = ModelOverridePayload( + model_id = "unsloth/x-GGUF:Q4_K_M", + max_seq_length = 4096, + gpu_layers = -1, + n_cpu_moe = 0, + gpu_ids = [0, 1], + tensor_parallel = True, + remove = True, + fill_absent_fields = True, + ) + assert ok.max_seq_length == 4096 + assert ok.gpu_ids == [0, 1] + assert ok.gpu_layers == -1 + assert ok.tensor_parallel is True + assert ok.remove is True + assert ok.fill_absent_fields is True diff --git a/studio/backend/tests/test_parallel_slots_per_load.py b/studio/backend/tests/test_parallel_slots_per_load.py index f4f2d31c6f..a3da4ea24b 100644 --- a/studio/backend/tests/test_parallel_slots_per_load.py +++ b/studio/backend/tests/test_parallel_slots_per_load.py @@ -152,6 +152,13 @@ def test_frontend_mirror_matches_shared_bounds(): assert (int(low.group(1)), int(high.group(1))) == (PARALLEL_MIN, PARALLEL_MAX) +def test_override_mirror_matches_shared_bounds(): + # The API auto-switch override map mirrors the bounds rather than importing them: + # llama_server_args owns the extra-args allow-list that module stays out of. + from utils.openai_auto_switch_settings import PARALLEL_SLOTS_MAX, PARALLEL_SLOTS_MIN + assert (PARALLEL_SLOTS_MIN, PARALLEL_SLOTS_MAX) == (PARALLEL_MIN, PARALLEL_MAX) + + def test_preset_model_reuses_shared_bounds(): # Bounds drifting from PARALLEL_MIN/MAX would 422 valid presets on every sync. from routes.chat_history import ChatPresetLoadConfig diff --git a/studio/backend/utils/openai_auto_switch_settings.py b/studio/backend/utils/openai_auto_switch_settings.py index 9520846283..07df0a584d 100644 --- a/studio/backend/utils/openai_auto_switch_settings.py +++ b/studio/backend/utils/openai_auto_switch_settings.py @@ -27,6 +27,7 @@ per-request hot path; writes invalidate the cache. from __future__ import annotations import os +import re import threading import time from typing import Any, Optional @@ -246,35 +247,427 @@ def set_openai_auto_switch( ) +# --- Per-model launch config ------------------------------------------------- +# +# An override is the server-side twin of the UI's per-model config (the browser +# localStorage map behind features/model-picker/model-config). The UI mirrors every +# save here so an OpenAI-compatible API load gets the same launch settings the +# picker would apply, rather than only the two legacy fields below. +# +# Legacy entries hold just {llama_extra_args, max_seq_length}. Every field is +# optional and absent means "app default". A write replaces the fields it +# expresses, so the route carries `llama_extra_args` over when the payload omits it. +# +# Known gap: the picker falls back to a global preference for GPU memory mode and +# speculative decoding, and those globals live in browser localStorage. An override +# stores only an explicit per-model choice, so an API load of a model that follows +# the global gets the app default instead. Every other field matches the picker. + +# Mirrors _valid_cache_types in core/inference/llama_cpp.py. +VALID_KV_CACHE_DTYPES = frozenset( + {"f16", "bf16", "q8_0", "q4_0", "q4_1", "q5_0", "q5_1", "iq4_nl", "f32"} +) +# Canonical values plus the legacy spellings LoadRequest still accepts. +VALID_SPECULATIVE_TYPES = frozenset( + { + "auto", + "mtp", + "ngram", + "mtp+ngram", + "off", + "default", + "draft-mtp", + "ngram-mod", + "ngram-simple", + } +) +# Only these consume spec_draft_n_max (mirrors MTP_SPECULATIVE_TYPES in the UI). +MTP_SPECULATIVE_TYPES = frozenset({"mtp", "mtp+ngram", "draft-mtp"}) +VALID_GPU_MEMORY_MODES = frozenset({"auto", "manual"}) + +# Mirrors PARALLEL_MIN/MAX in core/inference/llama_server_args.py (the LoadRequest +# n_parallel bounds). Mirrored rather than imported: that module owns the extra-args +# allow-list this one must stay out of. test_parallel_slots_per_load.py pins them together. +PARALLEL_SLOTS_MIN = 1 +PARALLEL_SLOTS_MAX = 64 + +MAX_SEQ_LENGTH_CEILING = 1048576 +MAX_CHAT_TEMPLATE_OVERRIDE_BYTES = 65_536 +# Highest device index a stored gpu_ids entry may name. Also bounds how many +# distinct ids one entry can hold, which is what the payload limit is built from. +MAX_GPU_ID = 1024 + + +def _clean_str(value: Any, allowed: frozenset[str]) -> Optional[str]: + if not isinstance(value, str): + return None + normalized = value.strip().lower() + return normalized if normalized in allowed else None + + +def _bounded_int(value: Any, *, minimum: int, maximum: int) -> Optional[int]: + # bool subclasses int, so `gpu_ids: [true, false]` would pin GPUs 1 and 0. + if isinstance(value, bool): + return None + # int(1.5) is 1, which would silently mangle a fractional context. + if isinstance(value, float) and not value.is_integer(): + return None + try: + parsed = int(value) + except (TypeError, ValueError, OverflowError): + # OverflowError is float("inf"), which json.loads accepts as `Infinity`. + return None + if parsed < minimum or parsed > maximum: + return None + return parsed + + +def normalize_model_override(payload: dict[str, Any]) -> dict[str, Any]: + """Validate one per-model launch config, dropping anything unusable. + + Silently drops rather than raising: an override is a convenience mirror of the + UI's config, so one stale field (a KV dtype this llama.cpp build lost, a GPU id + from another host) must not block persisting the rest or fail the API load that + reads it. ``validate_extra_args`` is the caller's job -- it lives in the + llama_server_args allow-list module, which this one must not import. + """ + entry: dict[str, Any] = {} + + extra_args = payload.get("llama_extra_args") + if isinstance(extra_args, (list, tuple)) and extra_args: + entry["llama_extra_args"] = [str(arg) for arg in extra_args] + + # 0 / negative means "unset"; the loader reads absence as the app default. + for key in ("max_seq_length", "custom_context_length"): + parsed = _bounded_int(payload.get(key), minimum = 1, maximum = MAX_SEQ_LENGTH_CEILING) + if parsed: + entry[key] = parsed + + kv_cache_dtype = _clean_str(payload.get("kv_cache_dtype"), VALID_KV_CACHE_DTYPES) + if kv_cache_dtype: + entry["kv_cache_dtype"] = kv_cache_dtype + + speculative_type = _clean_str(payload.get("speculative_type"), VALID_SPECULATIVE_TYPES) + if speculative_type: + entry["speculative_type"] = speculative_type + # MTP-only; storing it otherwise shows an edit the loader ignores. + if speculative_type in MTP_SPECULATIVE_TYPES: + spec_draft_n_max = _bounded_int(payload.get("spec_draft_n_max"), minimum = 1, maximum = 16) + if spec_draft_n_max: + entry["spec_draft_n_max"] = spec_draft_n_max + + # Blank means "follow the server-wide --parallel default", which is also what an + # out-of-range value falls back to. + n_parallel = _bounded_int( + payload.get("n_parallel"), minimum = PARALLEL_SLOTS_MIN, maximum = PARALLEL_SLOTS_MAX + ) + if n_parallel: + entry["n_parallel"] = n_parallel + + if _coerce_bool(payload.get("tensor_parallel")): + entry["tensor_parallel"] = True + + template = payload.get("chat_template_override") + if isinstance(template, str) and template.strip(): + # A lone surrogate from JSON breaks encode(), and such a template can + # never render, so drop it like any other bad field. + try: + template_bytes = len(template.encode("utf-8")) + except UnicodeEncodeError: + template_bytes = MAX_CHAT_TEMPLATE_OVERRIDE_BYTES + 1 + if template_bytes <= MAX_CHAT_TEMPLATE_OVERRIDE_BYTES: + entry["chat_template_override"] = template + + # Only "manual" is a real override: "auto" would pin the model and stop it + # following the global GPU memory preference. + if _clean_str(payload.get("gpu_memory_mode"), VALID_GPU_MEMORY_MODES) == "manual": + entry["gpu_memory_mode"] = "manual" + + # -1 is Auto (llama.cpp --fit), which is the default, so only >= 0 is stored. + gpu_layers = _bounded_int(payload.get("gpu_layers"), minimum = 0, maximum = 1024) + if gpu_layers is not None: + entry["gpu_layers"] = gpu_layers + + n_cpu_moe = _bounded_int(payload.get("n_cpu_moe"), minimum = 1, maximum = 1024) + if n_cpu_moe: + entry["n_cpu_moe"] = n_cpu_moe + + gpu_ids = payload.get("gpu_ids") + if isinstance(gpu_ids, (list, tuple)) and gpu_ids: + # De-duplicate, preserving order: resolve_requested_gpu_ids rejects a repeat, + # so storing [0, 0] would 400 every later API load of this model. Membership + # is a set, not a scan of the list being built: an id only has to be in + # 0..MAX_GPU_ID, so a long array walks that scan once per element. + cleaned_ids: list[int] = [] + seen_ids: set[int] = set() + for gid in gpu_ids: + parsed = _bounded_int(gid, minimum = 0, maximum = MAX_GPU_ID) + if parsed is not None and parsed not in seen_ids: + seen_ids.add(parsed) + cleaned_ids.append(parsed) + if cleaned_ids: + entry["gpu_ids"] = cleaned_ids + + return entry + + +def resolve_fit_max_seq_length(override: dict[str, Any], *, is_gguf: bool) -> Optional[int]: + """The ``max_seq_length`` an API load should send for this override. + + Mirrors resolveFitMaxSeqLength in the UI (features/chat/presets/preset-policy.ts): + under Manual GPU memory with Auto layers, llama.cpp's ``--fit`` owns context + sizing, so the load sends the explicit context pin (or 0 to hand sizing over) + rather than the stored max sequence length. Returns None to leave the field + at the loader's default. + """ + manual_auto_layers = ( + is_gguf + and override.get("gpu_memory_mode") == "manual" + and override.get("gpu_layers") is None + ) + if manual_auto_layers: + return override.get("custom_context_length") or 0 + # max_seq_length wins where both are set. The UI only sends it for non-GGUF + # models, so the two only collide in a hand-written or legacy entry. + return override.get("max_seq_length") or override.get("custom_context_length") + + +def model_override_load_kwargs(override: dict[str, Any], *, is_gguf: bool) -> dict[str, Any]: + """Map a stored per-model config onto ``LoadRequest`` keyword arguments. + + Mirrors the UI's load payload (features/chat/api/chat-adapter.ts) so an API + auto-switch load and a picker load of the same model produce the same command + line. GPU placement is GGUF-only there, so it is gated the same way here: a + safetensors model loads through HF auto-placement and must not inherit a + hidden GGUF GPU pin. + """ + if not override: + return {} + kwargs: dict[str, Any] = {} + + max_seq_length = resolve_fit_max_seq_length(override, is_gguf = is_gguf) + if max_seq_length is not None: + kwargs["max_seq_length"] = max_seq_length + for source, target in ( + ("llama_extra_args", "llama_extra_args"), + ("kv_cache_dtype", "cache_type_kv"), + ("speculative_type", "speculative_type"), + ("spec_draft_n_max", "spec_draft_n_max"), + ("tensor_parallel", "tensor_parallel"), + ("chat_template_override", "chat_template_override"), + ): + if override.get(source) is not None: + kwargs[target] = override[source] + + if is_gguf: + # Slots are a llama-server flag, and the picker sends them for GGUF only. + if override.get("n_parallel") is not None: + kwargs["n_parallel"] = override["n_parallel"] + if override.get("gpu_memory_mode") is not None: + kwargs["gpu_memory_mode"] = override["gpu_memory_mode"] + if override.get("gpu_layers") is not None: + kwargs["gpu_layers"] = override["gpu_layers"] + if override.get("n_cpu_moe") is not None: + kwargs["n_cpu_moe"] = override["n_cpu_moe"] + if override.get("gpu_ids") is not None: + kwargs["gpu_ids"] = override["gpu_ids"] + return kwargs + + +def _looks_like_filesystem_path(model_id: str) -> bool: + """True for an absolute path id, as the ./models and LM Studio scanners emit.""" + if model_id.startswith(("/", "\\")): + return True + # Windows drive letter, e.g. "C:\models\x.gguf". + return len(model_id) >= 3 and model_id[1] == ":" and model_id[2] in ("\\", "/") + + +# The three case-insensitive path shapes. Must stay in step with +# features/hub/lib/model-identity.ts, which folds these before storing, or a +# stored key becomes unreachable. +_WINDOWS_DRIVE_PATH = re.compile(r"^[A-Za-z]:[\\/]") +_WSL_DRIVE_PATH = re.compile(r"^/mnt/[A-Za-z](?:/|$)") + + +def _fold_case_insensitive_path(model_id: str) -> Optional[str]: + """``model_id`` folded for comparison, or None when the path is case-sensitive. + + A Windows drive path, a UNC share and a WSL drive path all name one file + whatever the casing, and the separator is interchangeable on Windows. A + POSIX path is not: folding "/models/Foo.gguf" onto "/models/foo.gguf" would + replay another model's context and GPU pin. + """ + slashed = model_id.replace("\\", "/") + if _WINDOWS_DRIVE_PATH.match(model_id): + minimum = 3 + elif slashed.startswith("//"): + minimum = 2 + elif _WSL_DRIVE_PATH.match(slashed): + minimum = 6 + else: + return None + trimmed = slashed + while len(trimmed) > minimum and trimmed.endswith("/"): + trimmed = trimmed[:-1] + return trimmed.casefold() + + +# A quant label may carry a bits-per-weight modifier ("IQ4_XS-3.53bpw") to keep two +# files at the same base quant distinct. The two label helpers disagree on whether +# to keep it, so readers of a stored key must accept both forms. +_BPW_SUFFIX = re.compile(r"-[0-9]+(?:\.[0-9]+)?bpw$", re.IGNORECASE) +_MAX_QUANT_SUFFIX_LEN = 64 + + +def split_quant_suffix(value: str) -> Optional[tuple[str, str]]: + """``(head, quant)`` for a ``head:QUANT`` key, or None when there is none. + + The suffix has to be a real quant label, so an ordinary colon inside a POSIX + filename is left alone: "/models/foo:bar.gguf" is one valid filename, and + splitting it would graft /models/foo's launch flags onto a different model. + """ + from core.inference.llama_cpp import _GGUF_KNOWN_QUANT_RE + from hub.utils.gguf import extract_quant_label + + head, sep, tail = value.rpartition(":") + if not sep or not head or not tail: + return None + if "/" in tail or "\\" in tail: + return None + if len(tail) <= _MAX_QUANT_SUFFIX_LEN and _GGUF_KNOWN_QUANT_RE.fullmatch( + _BPW_SUFFIX.sub("", tail) + ): + return head, tail + # A .gguf with no recognizable quant token is labelled by its stem, so keys like + # "/models/CustomModel.gguf:custommodel" exist. Storage lowercases the label + # while the scanner keeps the filename casing, hence the case-insensitive + # compare. Requiring exactly that label keeps an ordinary colon out: + # "/models/foo:bar.gguf" splits to a head that is not a .gguf. + if not head.lower().endswith(".gguf"): + return None + filename = head.replace("\\", "/").rsplit("/", 1)[-1] + return (head, tail) if tail.casefold() == extract_quant_label(filename).casefold() else None + + +def _fold_posix_path_variant(value: str) -> str: + """A POSIX path id with only its quant suffix folded. + + The browser lowercases the variant but keeps the path casing, so a stored + "/models/Foo:q4_k_m" has to be reachable from "/models/Foo:Q4_K_M" without + also making "/models/Foo.gguf" reachable from "/models/foo.gguf". + """ + split = split_quant_suffix(value) + if split is None: + return value + head, quant = split + return f"{head}:{quant.casefold()}" + + def get_model_overrides() -> dict[str, dict]: - """Per-model launch overrides keyed by model id ({llama_extra_args, max_seq_length}).""" + """Per-model launch configs keyed by model id (see normalize_model_override).""" raw = _cached_setting(MODEL_OVERRIDES_SETTING_KEY, None) return raw if isinstance(raw, dict) else {} def get_model_override(model_id: str) -> dict: - """The launch override applied when auto-switch loads ``model_id`` (or empty).""" - override = get_model_overrides().get(model_id) + """The launch override applied when auto-switch loads ``model_id`` (or empty). + + Falls back to a case-insensitive match when nothing matches exactly. Repo ids + and quants are case-insensitive in practice ("Q4_K_M" and "q4_k_m" name one + file), and the browser normalizes them to lowercase before storing, so an + exact-only lookup misses entries written from that side. Exact still wins, and + an ambiguous fallback matches nothing, so two POSIX paths differing only in + case stay distinct. + """ + key = resolve_model_override_key(model_id) + if key is None: + return {} + override = get_model_overrides().get(key) return override if isinstance(override, dict) else {} +def resolve_model_override_key(model_id: str) -> Optional[str]: + """The stored key an override lookup for ``model_id`` would actually hit. + + Shared by read and remove so "what a load applies" and "what forgetting this + model clears" can never disagree. + """ + overrides = get_model_overrides() + if isinstance(overrides.get(model_id), dict): + return model_id + if not isinstance(model_id, str): + return None + # A POSIX path is case-sensitive, so folding "/models/Foo.gguf" onto + # "/models/foo.gguf" would replay another model's settings. Windows drive, UNC + # and WSL paths are not, and the browser folds exactly those before storing, so + # not folding them here would strand every migrated Windows entry. + if _looks_like_filesystem_path(model_id): + folded = _fold_case_insensitive_path(model_id) + if folded is not None: + + def fold(key: str) -> Optional[str]: + return _fold_case_insensitive_path(key) + else: + # POSIX: the path stays case-sensitive, but the browser lowercases the + # quant suffix, so "/models/Foo:q4_k_m" must be reachable from + # the scanner's "/models/Foo:Q4_K_M". + folded = _fold_posix_path_variant(model_id) + + def fold(key: str) -> Optional[str]: + # A path only ever folds onto another path. + if not _looks_like_filesystem_path(key): + return None + return None if _fold_case_insensitive_path(key) else _fold_posix_path_variant(key) + else: + folded = model_id.casefold() + + def fold(key: str) -> Optional[str]: + # A path never folds onto a repo id: the shapes cannot collide. + return None if _looks_like_filesystem_path(key) else key.casefold() + + matches = [ + key + for key, value in overrides.items() + if isinstance(key, str) and fold(key) == folded and isinstance(value, dict) + ] + return matches[0] if len(matches) == 1 else None + + def set_model_override( model_id: str, llama_extra_args: Optional[list[str]] = None, max_seq_length: Optional[int] = None, + *, + fill_absent_fields: bool = False, + **config: Any, ) -> dict: - """Upsert one model's launch override; an override with no fields removes it.""" + """Upsert one model's launch config; a config with no usable fields removes it. + + The two legacy parameters stay positional for existing callers; every other + per-model field is passed by keyword and normalized together. + + ``fill_absent_fields`` writes only what is missing: an entry already stored + keeps every field it holds and gains only the ones it lacks. Returns the + normalized entry either way; read the map back to see what is actually stored. + """ if not model_id or not model_id.strip(): raise ValueError("model_id is required.") - entry: dict[str, Any] = {} - if llama_extra_args: - entry["llama_extra_args"] = [str(arg) for arg in llama_extra_args] - if max_seq_length: - entry["max_seq_length"] = max(0, int(max_seq_length)) + entry = normalize_model_override( + { + **config, + "llama_extra_args": llama_extra_args, + "max_seq_length": max_seq_length, + } + ) from storage.studio_db import upsert_app_setting_map_entry # Atomic per-entry merge so two PUTs for different models can't drop each other. - upsert_app_setting_map_entry(MODEL_OVERRIDES_SETTING_KEY, model_id.strip(), entry or None) + upsert_app_setting_map_entry( + MODEL_OVERRIDES_SETTING_KEY, + model_id.strip(), + entry or None, + fill_absent_fields = fill_absent_fields, + ) _invalidate(MODEL_OVERRIDES_SETTING_KEY) return entry diff --git a/studio/frontend/src/app/router.tsx b/studio/frontend/src/app/router.tsx index 586c03d5df..3dd8d2e8e0 100644 --- a/studio/frontend/src/app/router.tsx +++ b/studio/frontend/src/app/router.tsx @@ -6,6 +6,7 @@ import { Button } from "@/components/ui/button"; import { MascotImg } from "@/components/mascot-img"; import { useT } from "@/i18n"; import { Route as rootRoute } from "./routes/__root"; +import { Route as apiMonitorRoute } from "./routes/api"; import { Route as dataRecipesRoute } from "./routes/data-recipes"; import { Route as dataRecipeRoute } from "./routes/data-recipes.$recipeId"; import { Route as chatRoute } from "./routes/chat"; @@ -32,6 +33,7 @@ const routeTree = rootRoute.addChildren([ exportRoute, dataRecipesRoute, dataRecipeRoute, + apiMonitorRoute, ]); function DefaultNotFound() { diff --git a/studio/frontend/src/app/routes/__root.tsx b/studio/frontend/src/app/routes/__root.tsx index 9e72135bdd..ad7845ebf9 100644 --- a/studio/frontend/src/app/routes/__root.tsx +++ b/studio/frontend/src/app/routes/__root.tsx @@ -3,28 +3,27 @@ import { AppSidebar } from "@/components/app-sidebar"; import { Navbar } from "@/components/navbar"; -import { fetchDeviceType, usePlatformStore } from "@/config/env"; import { SidebarInset, SidebarProvider } from "@/components/ui/sidebar"; -import { - SettingsDialog, - useSettingsDialogStore, -} from "@/features/settings"; +import { fetchDeviceType, usePlatformStore } from "@/config/env"; +import { ApiMonitorOverlay } from "@/features/api-monitor/api-monitor-overlay"; +import { hasAuthToken } from "@/features/auth"; import { ChatPage, + type ChatSearch, clearNewChatDraft, StopRunningChatsDialog, useChatRuntimeStore, - type ChatSearch, } from "@/features/chat"; -import { RemoteCodeConsentDialog } from "@/features/security"; -import { HfTokenWarningDialog } from "@/features/hf-auth"; -import { TransformersUpgradeDialog } from "@/features/transformers-upgrade"; -import { useTrainingUnloadGuard } from "@/features/training"; import { useExportRuntimeLifecycle } from "@/features/export"; -import { hasAuthToken } from "@/features/auth"; +import { HfTokenWarningDialog } from "@/features/hf-auth"; +import { backfillModelOverrides } from "@/features/model-picker/api/migrate-model-overrides"; import { usePersonalizationSync } from "@/features/profile"; +import { RemoteCodeConsentDialog } from "@/features/security"; +import { SettingsDialog, useSettingsDialogStore } from "@/features/settings"; +import { useTrainingUnloadGuard } from "@/features/training"; +import { TransformersUpgradeDialog } from "@/features/transformers-upgrade"; import { useSidebarPin } from "@/hooks/use-sidebar-pin"; -import { useT, type TranslationKey } from "@/i18n"; +import { type TranslationKey, useT } from "@/i18n"; import { Outlet, createRootRoute, @@ -34,13 +33,7 @@ import { useRouterState, } from "@tanstack/react-router"; import { AnimatePresence, motion } from "motion/react"; -import { - Suspense, - useEffect, - useLayoutEffect, - useMemo, - useState, -} from "react"; +import { Suspense, useEffect, useLayoutEffect, useMemo, useState } from "react"; import { AppProvider } from "../provider"; declare module "@tanstack/react-router" { @@ -77,11 +70,16 @@ const CHAT_ONLY_ALLOWED = new Set([ // Export stays reachable on chat-only hosts so the page can show its own grayed-out reason // instead of a silent redirect; it self-gates via export capability, so nothing runs. "/export", + // Chat-only hosts (Intel Macs, Apple Silicon without MLX, no-GPU boxes) serve the + // OpenAI-compatible API like any other host, so the monitor must be reachable + // there or the overlay's "Expand" and the Settings > API card redirect to /chat. + "/api-monitor", ]); function isChatOnlyAllowed(pathname: string): boolean { if (CHAT_ONLY_ALLOWED.has(pathname)) return true; - if (pathname === "/data-recipes" || pathname.startsWith("/data-recipes/")) return true; + if (pathname === "/data-recipes" || pathname.startsWith("/data-recipes/")) + return true; return false; } @@ -177,6 +175,15 @@ function RootLayout() { : DEFAULT_DOCUMENT_TITLE; }, [documentTitle]); + // Settings saved before the server-side override map existed live only in this + // browser, so an API load would use app defaults. Backfill once, after auth. + useEffect(() => { + if (isAuthFlowRoute) { + return; + } + void backfillModelOverrides(); + }, [isAuthFlowRoute]); + useEffect(() => { if (isAuthFlowRoute) { useSettingsDialogStore.getState().closeDialog(); @@ -225,6 +232,8 @@ function RootLayout() { {!isAuthFlowRoute && } + {/* Opens itself when API traffic arrives; hides on the full monitor page. */} + {!isAuthFlowRoute && } @@ -244,7 +253,9 @@ function RootLayout() { className="!min-h-0 h-[calc(100dvh-var(--studio-titlebar-height,0px))] overflow-hidden" > - +
import("@/features/api-monitor"), + "ApiMonitorPage", +); + +export const Route = createRoute({ + getParentRoute: () => rootRoute, + // Not "/api": the backend owns that prefix (and "/v1") and its SPA fallback 404s + // those paths, so a deep link to /api would never reach the router. + path: "/api-monitor", + staticData: { title: "API" }, + beforeLoad: () => requireAuth(), + component: ApiMonitorPage, +}); diff --git a/studio/frontend/src/features/api-monitor/api-monitor-overlay.tsx b/studio/frontend/src/features/api-monitor/api-monitor-overlay.tsx new file mode 100644 index 0000000000..d846edbb6f --- /dev/null +++ b/studio/frontend/src/features/api-monitor/api-monitor-overlay.tsx @@ -0,0 +1,391 @@ +// SPDX-License-Identifier: AGPL-3.0-only +// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 + +// Floating API monitor: opens itself when API traffic arrives, summarises it +// without taking over the window, and links through to the full page. + +import { getApiMonitor } from "@/features/chat/api/chat-api"; +import type { ApiMonitorEntry } from "@/features/chat/types/api"; +import { cn } from "@/lib/utils"; +import { + ArrowExpand01Icon, + DragDropVerticalIcon, + Globe02Icon, +} from "@hugeicons/core-free-icons"; +import { HugeiconsIcon } from "@hugeicons/react"; +import { useNavigate, useRouterState } from "@tanstack/react-router"; +import { XIcon } from "lucide-react"; +import { AnimatePresence, motion, useDragControls } from "motion/react"; +import { + type PointerEvent, + type ReactElement, + useEffect, + useMemo, + useRef, + useState, +} from "react"; +import { isLifecycleEntry, lifecycleLabel } from "./lifecycle"; +import { + type ApiMonitorWatch, + createWatch, + observeResponse, + rearmWatch, + startWatching, +} from "./new-traffic"; +import { useApiMonitorOverlayStore } from "./overlay-store"; +import { computeStats } from "./use-api-monitor"; + +// Live cadence while the panel is on screen. +const OPEN_POLL_MS = 1500; +// Closed, the poll only has to notice traffic started, so it backs off. +const IDLE_POLL_MS = 5000; +// Requests shown in the panel; the rest are one click away on the full page. +const VISIBLE_ENTRIES = 4; +// Quiet time before a dismissed panel re-arms. +const REARM_QUIET_MS = 60_000; + +const API_INFERENCE_PREFIX_RE = /^\/api\/inference/; +const V1_PREFIX_RE = /^\/v1\//; + +function compactEndpoint(endpoint: string): string { + return endpoint + .replace(API_INFERENCE_PREFIX_RE, "/api") + .replace(V1_PREFIX_RE, "/"); +} + +function formatDuration(value?: number | null): string { + if (value == null) { + return "live"; + } + if (value < 1000) { + return `${Math.round(value)}ms`; + } + return `${(value / 1000).toFixed(value < 10000 ? 1 : 0)}s`; +} + +function statusDotClass(status: ApiMonitorEntry["status"]): string { + switch (status) { + case "running": + return "bg-blue-500 animate-pulse"; + case "error": + return "bg-red-500"; + case "cancelled": + return "bg-amber-500"; + default: + return "bg-emerald-500"; + } +} + +function StatCell({ + label, + value, + tone, +}: { + label: string; + value: string; + tone?: "error" | "active"; +}): ReactElement { + return ( +
+ + {value} + + {/* Sentence case: metric rows read as words, not headers. */} + + {label} + +
+ ); +} + +export function ApiMonitorOverlay(): ReactElement | null { + const { isOpen, suppressed, autoOpen, open, close, setAutoOpen } = + useApiMonitorOverlayStore(); + const navigate = useNavigate(); + const pathname = useRouterState({ select: (s) => s.location.pathname }); + const onFullPage = pathname === "/api-monitor"; + + const [data, setData] = useState + > | null>(null); + + // What this session has already shown, and when it started watching. + const watchRef = useRef(createWatch(0)); + const lastNewEntryAtRef = useRef(0); + + // One loop for both jobs: panel contents while open, traffic watch while closed. + // Stands down on the full page, which polls for itself. + useEffect(() => { + // Opted out and closed: nothing to open or show, so polling is pure load. + if (onFullPage || (!autoOpen && !isOpen)) { + return; + } + let cancelled = false; + let timer: number | undefined; + const intervalMs = isOpen ? OPEN_POLL_MS : IDLE_POLL_MS; + + // Anchor the watch here, not at mount: the first snapshot can arrive much + // later (a hidden tab skips its poll entirely, an unreachable backend fails + // one), and everything terminal in it would otherwise read as history. + startWatching(watchRef.current, performance.now()); + + function schedule(): void { + timer = window.setTimeout(poll, intervalMs); + } + + function poll(): void { + // A hidden tab has nobody to show the panel to. + if (document.hidden) { + schedule(); + return; + } + getApiMonitor() + .then((next) => { + if (!cancelled) setData(next); + }) + .catch(() => { + // An unreachable server is the full page's story to tell. + if (!cancelled) setData(null); + }) + .finally(() => { + if (!cancelled) schedule(); + }); + } + + poll(); + return () => { + cancelled = true; + if (timer !== undefined) window.clearTimeout(timer); + }; + }, [isOpen, onFullPage, autoOpen]); + + const entries = useMemo(() => data?.entries ?? [], [data]); + const stats = useMemo(() => computeStats(entries), [entries]); + + useEffect(() => { + if (data == null) { + return; + } + const hasNewTraffic = observeResponse( + watchRef.current, + data, + performance.now(), + ); + if (!hasNewTraffic) { + return; + } + const now = Date.now(); + const quietFor = now - lastNewEntryAtRef.current; + lastNewEntryAtRef.current = now; + if (!autoOpen || isOpen) { + return; + } + // A dismissal holds for the burst and re-arms only once the API goes quiet. + if (suppressed && quietFor < REARM_QUIET_MS) { + return; + } + open(); + }, [data, autoOpen, suppressed, isOpen, open]); + + // The backlog built up while the poll was stood down is not new traffic. The + // watch re-anchors when the poll stands back up, not here, or the whole stay on + // the full page would read as unwatched and the panel would pop with rows the + // user has just read. + useEffect(() => { + if (onFullPage) { + rearmWatch(watchRef.current); + } + }, [onFullPage]); + + const [constraintsElement, setConstraintsElement] = + useState(null); + const constraintsRef = useMemo( + () => ({ current: constraintsElement }), + [constraintsElement], + ); + const dragControls = useDragControls(); + + function startDrag(event: PointerEvent): void { + event.preventDefault(); + dragControls.start(event); + } + + const visible = isOpen && !onFullPage; + const serverStatus = data?.status ?? "idle"; + + return ( + + {visible && ( +
+ +
+
+ + + API monitor + + +
+
+
+ +
+ +
+
+ +

+ {data?.active_model ?? "No model loaded"} +

+ + {/* Soft tile, as the Hub and Train pages group readouts. */} +
+ 0 ? "active" : undefined} + /> + + 0 ? "error" : undefined} + /> + +
+ + {/* Borderless rows on 12px hover pills, as in the sidebar. */} +
+ {entries.length === 0 ? ( +

+ No requests yet. +

+ ) : ( + entries.slice(0, VISIBLE_ENTRIES).map((entry) => ( +
+ + + {isLifecycleEntry(entry) + ? lifecycleLabel(entry) + : compactEndpoint(entry.endpoint)} + + + {entry.error ? entry.error : entry.model} + + + {formatDuration(entry.duration_ms)} + +
+ )) + )} +
+ + {/* Through to payloads, filters and per request tokens. */} + + + {/* Closing silences this burst; this is the permanent off. */} + +
+
+ )} +
+ ); +} diff --git a/studio/frontend/src/features/api-monitor/api-monitor-page.tsx b/studio/frontend/src/features/api-monitor/api-monitor-page.tsx new file mode 100644 index 0000000000..590348c347 --- /dev/null +++ b/studio/frontend/src/features/api-monitor/api-monitor-page.tsx @@ -0,0 +1,914 @@ +// SPDX-License-Identifier: AGPL-3.0-only +// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 + +// Full-page monitor for Unsloth's OpenAI-compatible API server. +// +// Replaces the small console buried in the API settings tab. Settings still owns +// configuration (keys, auto-switch, examples); this page owns observability. + +import { Button } from "@/components/ui/button"; +import { Input } from "@/components/ui/input"; +import { + Select, + SelectContent, + SelectItem, + SelectTrigger, + SelectValue, +} from "@/components/ui/select"; +import { Skeleton } from "@/components/ui/skeleton"; +import { usePlatformStore } from "@/config/env"; +import { getInferenceStatus, unloadModel } from "@/features/chat/api/chat-api"; +import { resolveInferenceCheckpointId } from "@/features/chat/lib/apply-inference-status-to-store"; +import { useChatRuntimeStore } from "@/features/chat/stores/chat-runtime-store"; +import type { ApiMonitorEntry } from "@/features/chat/types/api"; +import { isExternalModelId } from "@/features/chat/external-providers"; +import { modelIdsMatch } from "@/features/hub/lib/model-identity"; +import { useSettingsDialogStore } from "@/features/settings"; +import { getApiBase, isTauri } from "@/lib/api-base"; +import { copyToClipboard } from "@/lib/copy-to-clipboard"; +import { Tick02Icon } from "@/lib/tick-icon"; +import { cn } from "@/lib/utils"; +import { + Copy01Icon, + Delete02Icon, + Globe02Icon, + PauseIcon, + PlayIcon, + PowerSocket01Icon, + RefreshIcon, + Settings02Icon, +} from "@hugeicons/core-free-icons"; +import { HugeiconsIcon } from "@hugeicons/react"; +import { type ReactElement, useEffect, useMemo, useRef, useState } from "react"; +import { SavedModelSettingsPanel } from "./components/saved-model-settings"; +import { isLifecycleEntry, lifecycleLabel } from "./lifecycle"; +import { + type MonitorStatusFilter, + filterEntries, + useApiMonitor, +} from "./use-api-monitor"; + +const API_INFERENCE_PREFIX_RE = /^\/api\/inference/; +const V1_PREFIX_RE = /^\/v1\//; +// Tries per revision for a detail payload. Bounded because the usual failure is +// an entry aged out of the ring buffer, which never comes back. +const DETAIL_FETCH_ATTEMPTS = 3; + +const STATUS_FILTERS: { value: MonitorStatusFilter; label: string }[] = [ + { value: "all", label: "All requests" }, + { value: "running", label: "In flight" }, + { value: "completed", label: "Completed" }, + { value: "error", label: "Errors" }, + { value: "cancelled", label: "Cancelled" }, +]; + +function formatTime(value: number): string { + return new Date(value * 1000).toLocaleTimeString([], { + hour: "2-digit", + minute: "2-digit", + second: "2-digit", + }); +} + +function formatDuration(value?: number | null): string { + if (value == null) { + return "running"; + } + if (value < 1000) { + return `${Math.round(value)} ms`; + } + return `${(value / 1000).toFixed(value < 10000 ? 1 : 0)} s`; +} + +function formatCount(value: number): string { + return value.toLocaleString(); +} + +function compactEndpoint(endpoint: string): string { + return endpoint + .replace(API_INFERENCE_PREFIX_RE, "/api") + .replace(V1_PREFIX_RE, "/"); +} + +function statusDotClass(status: ApiMonitorEntry["status"]): string { + switch (status) { + case "running": + return "bg-blue-500 animate-pulse"; + case "error": + return "bg-red-500"; + case "cancelled": + return "bg-amber-500"; + default: + return "bg-emerald-500"; + } +} + +function statusTextClass(status: ApiMonitorEntry["status"]): string { + switch (status) { + case "running": + return "text-blue-600 dark:text-blue-400"; + case "error": + return "text-red-600 dark:text-red-400"; + case "cancelled": + return "text-amber-600 dark:text-amber-500"; + default: + return "text-emerald-600 dark:text-emerald-500"; + } +} + +function StatCard({ + label, + value, + hint, + tone, +}: { + label: string; + value: string; + hint?: string; + tone?: "default" | "error" | "active"; +}): ReactElement { + return ( +
+ + {label} + + + {value} + + {hint ? ( + + {hint} + + ) : null} +
+ ); +} + +function CopyButton({ + value, + label, +}: { + value: string; + label: string; +}): ReactElement { + const [copied, setCopied] = useState(false); + // Cancel on cleanup, or navigating away mid-flash sets state after unmount. + const timerRef = useRef(undefined); + useEffect( + () => () => { + if (timerRef.current !== undefined) window.clearTimeout(timerRef.current); + }, + [], + ); + return ( + + ); +} + +function ContextUsageBar({ + value, +}: { value?: number | null }): ReactElement | null { + if (value == null) { + return null; + } + const pct = Math.max(0, Math.min(100, Math.round(value * 100))); + return ( +
+
+
= 90 + ? "bg-red-500" + : pct >= 75 + ? "bg-amber-500" + : "bg-control-accent", + )} + style={{ width: `${pct}%` }} + /> +
+ + {pct}% + +
+ ); +} + +function RequestRow({ + entry, + selected, + onSelect, +}: { + entry: ApiMonitorEntry; + selected: boolean; + onSelect: () => void; +}): ReactElement { + const preview = + entry.error || + entry.reply_preview || + entry.prompt_preview || + (entry.status === "running" ? "Waiting for outputโ€ฆ" : "No preview"); + // A load, unload or download has no prompt or reply, so it reads as a status + // line rather than a request with a payload. + if (isLifecycleEntry(entry)) { + return ( +
+
+ + + {lifecycleLabel(entry)} + + + {formatTime(entry.started_at)} + +
+
+ {entry.model} +
+ {entry.error ? ( +
+ {entry.error} +
+ ) : null} +
+ ); + } + return ( + + ); +} + +function PayloadBlock({ + title, + body, + truncated, + loading, + tone, +}: { + title: string; + body: string; + truncated?: boolean; + loading?: boolean; + tone?: "error"; +}): ReactElement { + return ( +
+
+

+ {title} +

+
+ {truncated ? ( + + preview only + + ) : null} + {body ? ( + + ) : null} +
+
+
+        {loading && !body ? "Loadingโ€ฆ" : body || "โ€“"}
+      
+
+ ); +} + +function RequestDetail({ + entry, + detail, + loading, +}: { + entry: ApiMonitorEntry; + detail?: ApiMonitorEntry; + loading: boolean; +}): ReactElement { + // The detail fetch is separate, so it can describe an older state of a streaming + // entry. Prefer it only once it is as fresh as the list row, or the panel rewinds. + const detailIsCurrent = + detail != null && + detail.status === entry.status && + detail.updated_at >= entry.updated_at; + const prompt = detail?.prompt ?? entry.prompt_preview; + const reply = detailIsCurrent + ? (detail.reply ?? entry.reply_preview) + : entry.reply_preview; + + return ( +
+
+
+ + + {entry.status} + + + {entry.id} + +
+

+ {entry.method} {entry.endpoint} +

+

+ {entry.model} +

+
+ +
+ {[ + { label: "Started", value: formatTime(entry.started_at) }, + { label: "Duration", value: formatDuration(entry.duration_ms) }, + { + label: "Prompt tokens", + value: + entry.prompt_tokens != null + ? formatCount(entry.prompt_tokens) + : "โ€“", + }, + { + label: "Completion tokens", + value: + entry.completion_tokens != null + ? formatCount(entry.completion_tokens) + : "โ€“", + }, + { + label: "Total tokens", + value: + entry.total_tokens != null + ? formatCount(entry.total_tokens) + : "โ€“", + }, + { + label: "Context", + value: + entry.context_length != null + ? formatCount(entry.context_length) + : "โ€“", + }, + ].map((item) => ( +
+
+ {item.label} +
+
+ {item.value} +
+
+ ))} +
+ + {entry.context_usage != null ? ( +
+ + Context used + + +
+ ) : null} + + {entry.error ? ( + + ) : null} + + + +
+ ); +} + +export function ApiMonitorPage(): ReactElement { + const { + data, + entries, + stats, + error, + loading, + refreshing, + paused, + setPaused, + refresh, + clear, + details, + loadingDetails, + requestDetail, + } = useApiMonitor(); + const serverUrl = usePlatformStore((s) => s.serverUrl); + const [unloading, setUnloading] = useState(false); + const [unloadError, setUnloadError] = useState(null); + + // Manual release so VRAM is freed without waiting for the idle timer. /unload + // matches on the internal id, which the monitor does not carry, so read status. + const unloadActiveModel = async (): Promise => { + setUnloading(true); + try { + const status = await getInferenceStatus(); + const checkpoint = resolveInferenceCheckpointId(status); + if (!checkpoint) { + setUnloadError(null); + return; + } + await unloadModel({ model_path: checkpoint }); + // Same as the chat eject flow: the store still holds the freed checkpoint. + // Only when that IS the model just unloaded, though. Chat can have an + // external provider selected while a local model stays resident, and + // clearCheckpoint calls saveLastExternalCheckpoint(null), so clearing + // unconditionally would delete a selection this button never touched. + // Both spellings: status reports the concrete load path as the identifier + // while the store may hold the advertised repo id, so matching only the + // path leaves the store pinned to a model this button just freed. + const store = useChatRuntimeStore.getState(); + const selected = store.params.checkpoint; + const unloadedAliases = [checkpoint, status.active_model]; + if ( + selected && + !isExternalModelId(selected) && + unloadedAliases.some((alias) => modelIdsMatch(selected, alias)) + ) { + store.clearCheckpoint(); + } + setUnloadError(null); + refresh(); + } catch (err: unknown) { + setUnloadError( + err instanceof Error ? err.message : "Failed to unload the model", + ); + } finally { + setUnloading(false); + } + }; + const [statusFilter, setStatusFilter] = useState("all"); + const [query, setQuery] = useState(""); + const [selectedId, setSelectedId] = useState(null); + + const visible = useMemo( + () => filterEntries(entries, statusFilter, query), + [entries, statusFilter, query], + ); + const selected = useMemo( + () => visible.find((entry) => entry.id === selectedId) ?? null, + [visible, selectedId], + ); + + // Refetch the selected entry while it streams so the payload grows with the reply. + // Keyed on identity and revision, never on `details`: the fetch rewrites `details` + // on every success, so depending on it loops on the detail endpoint. + const selectedId_ = selected?.id ?? null; + const selectedUpdatedAt = selected?.updated_at ?? null; + const selectedIsMissing = selectedId_ != null && details[selectedId_] == null; + const lastFetchedRef = useRef(null); + const attemptsRef = useRef<{ revision: string; count: number }>({ + revision: "", + count: 0, + }); + const [retryTick, setRetryTick] = useState(0); + // Flips as a fetch settles either way, which is what lets a failure be noticed. + const detailInFlight = selectedId_ != null && loadingDetails.has(selectedId_); + useEffect(() => { + if (selectedId_ == null || detailInFlight) { + return; + } + // `updated_at` advances per poll while streaming and settles when terminal. + // A missing payload always retries, covering a fetch that failed late. + const revision = `${selectedId_}@${selectedUpdatedAt ?? ""}`; + if (!selectedIsMissing && lastFetchedRef.current === revision) { + return; + } + // A terminal row's revision never advances, so a failed fetch had nothing left to + // re-run this effect. `loadingDetails` settling is the trigger; the count bounds + // it, since the usual failure is an entry aged out of the ring buffer. + if (attemptsRef.current.revision !== revision) { + attemptsRef.current = { revision, count: 0 }; + } + if (attemptsRef.current.count >= DETAIL_FETCH_ATTEMPTS) { + return; + } + attemptsRef.current.count += 1; + // Only remember the revision when a fetch started: the in-flight guard can + // refuse, and recording it anyway skips that revision for good. + if (requestDetail(selectedId_)) { + lastFetchedRef.current = revision; + setRetryTick(0); + } else { + // Refused because an older fetch is running. No dep changes when it settles, so + // without this nudge the rejected revision is never fetched. + const timer = window.setTimeout(() => setRetryTick((n) => n + 1), 250); + return () => window.clearTimeout(timer); + } + }, [ + selectedId_, + selectedUpdatedAt, + selectedIsMissing, + requestDetail, + retryTick, + detailInFlight, + ]); + + // The desktop webview's origin is tauri://, not the API server, and the packaged + // app picks its port dynamically. Same source as the Agents tab. + const origin = typeof window === "undefined" ? "" : window.location.origin; + const baseUrl = `${isTauri ? (serverUrl ?? getApiBase()) : origin}/v1`; + const serverStatus = data?.status ?? "idle"; + const statusCopy = + serverStatus === "generating" + ? "Serving requests" + : serverStatus === "ready" + ? "Ready" + : "No model loaded"; + + return ( +
+
+
+

+ API +

+

+ Live traffic through Unsloth's OpenAI-compatible server. +

+
+
+ + + + + +
+
+ + {/* What you check first when a client can't reach the API: the base URL + to point it at, and what is loaded. */} +
+
+ + + +
+ + Base URL + + + {baseUrl} + +
+ +
+
+ + Status + + + + {statusCopy} + +
+
+ + Loaded model + + + {data?.active_model ?? "None"} + {data?.context_length + ? ` ยท ${formatCount(data.context_length)} ctx` + : ""} + +
+ {paused ? ( + + Paused + + ) : null} +
+ + {error || unloadError ? ( +
+ {error || unloadError} +
+ ) : null} + +
+ 0 ? "active" : "default"} + /> + + + 0 ? "error" : "default"} + hint={ + stats.errorRate != null + ? `${Math.round(stats.errorRate * 100)}% of finished` + : undefined + } + /> + + +
+ +
+
+ setQuery(event.target.value)} + placeholder="Search model, endpoint, preview or error" + aria-label="Search API requests" + className="h-9 w-full min-w-0 flex-1 rounded-full border-none bg-muted shadow-none dark:bg-background sm:w-64 sm:flex-none" + /> + + + {formatCount(visible.length)} of {formatCount(entries.length)} + +
+ +
+
+ {loading ? ( +
+ {[0, 1, 2].map((i) => ( + + ))} +
+ ) : visible.length === 0 ? ( +

+ {entries.length === 0 + ? "No API traffic yet. Point a client at the base URL above to see requests here." + : "No requests match this filter."} +

+ ) : ( + visible.map((entry) => ( + setSelectedId(entry.id)} + /> + )) + )} +
+ +
+ {selected ? ( + + ) : ( +

+ Select a request to inspect its prompt, reply, tokens and + errors. +

+ )} +
+
+
+ + +
+ ); +} diff --git a/studio/frontend/src/features/api-monitor/components/saved-model-settings.tsx b/studio/frontend/src/features/api-monitor/components/saved-model-settings.tsx new file mode 100644 index 0000000000..1ede884013 --- /dev/null +++ b/studio/frontend/src/features/api-monitor/components/saved-model-settings.tsx @@ -0,0 +1,140 @@ +// SPDX-License-Identifier: AGPL-3.0-only +// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 + +// What a remote load will actually apply, otherwise unanswerable from outside the +// process. Read only: the config also lives in the browser's per-model store, and +// the model's settings page is the only place that owns both. + +import { Skeleton } from "@/components/ui/skeleton"; +import { + type ApiModelOverride, + type ApiModelOverrides, + fetchModelOverrides, +} from "@/features/model-picker/api/model-overrides"; +import { type ReactElement, useCallback, useEffect, useState } from "react"; + +/** Summary of the fields the loader will apply, in load order. */ +function describeOverride(override: ApiModelOverride): string[] { + const parts: string[] = []; + if (override.custom_context_length) { + parts.push(`${override.custom_context_length.toLocaleString()} context`); + } + if (override.max_seq_length) { + parts.push(`${override.max_seq_length.toLocaleString()} max seq`); + } + if (override.kv_cache_dtype) { + parts.push(`KV ${override.kv_cache_dtype}`); + } + if (override.speculative_type) { + parts.push( + override.spec_draft_n_max + ? `spec ${override.speculative_type} ร—${override.spec_draft_n_max}` + : `spec ${override.speculative_type}`, + ); + } + if (override.n_parallel) { + parts.push(`${override.n_parallel} parallel slots`); + } + if (override.tensor_parallel) { + parts.push("tensor parallel"); + } + if (override.gpu_memory_mode === "manual") { + parts.push("manual GPU memory"); + } + if (override.gpu_layers != null) { + parts.push(`${override.gpu_layers} GPU layers`); + } + if (override.n_cpu_moe) { + parts.push(`${override.n_cpu_moe} MoE layers on CPU`); + } + if (override.gpu_ids?.length) { + parts.push(`GPU ${override.gpu_ids.join(", ")}`); + } + if (override.chat_template_override) { + parts.push("custom chat template"); + } + if (override.llama_extra_args?.length) { + parts.push(override.llama_extra_args.join(" ")); + } + return parts; +} + +export function SavedModelSettingsPanel(): ReactElement { + const [overrides, setOverrides] = useState(null); + const [error, setError] = useState(null); + + const load = useCallback(async () => { + try { + setOverrides(await fetchModelOverrides()); + setError(null); + } catch (err: unknown) { + setError( + err instanceof Error + ? err.message + : "Could not load saved model settings", + ); + } + }, []); + + useEffect(() => { + void load(); + }, [load]); + + const entries = Object.entries(overrides ?? {}); + + return ( +
+
+

+ Settings applied on API load +

+

+ When a request names one of these models, Unsloth loads it with these + settings, the same ones you saved in the model's settings page. + Models without an entry load with app defaults. Edit or forget an + entry from that model's settings, which keeps this list and the + picker in step. +

+
+ + {error ? ( +
+ {error} +
+ ) : overrides == null ? ( +
+ {[0, 1].map((i) => ( + + ))} +
+ ) : entries.length === 0 ? ( +

+ No saved model settings yet. Open a model's settings, turn on + "Remember for this model", and it will be applied to API + loads too. +

+ ) : ( +
    + {entries.map(([modelId, override]) => { + const summary = describeOverride(override); + return ( +
  • +
    + + {modelId} + + + {summary.length > 0 ? summary.join(" ยท ") : "App defaults"} + +
    +
  • + ); + })} +
+ )} +
+ ); +} diff --git a/studio/frontend/src/features/api-monitor/index.ts b/studio/frontend/src/features/api-monitor/index.ts new file mode 100644 index 0000000000..28b01def8b --- /dev/null +++ b/studio/frontend/src/features/api-monitor/index.ts @@ -0,0 +1,13 @@ +// SPDX-License-Identifier: AGPL-3.0-only +// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 + +export { ApiMonitorPage } from "./api-monitor-page"; +export { ApiMonitorOverlay } from "./api-monitor-overlay"; +export { useApiMonitorOverlayStore } from "./overlay-store"; +export { + computeStats, + filterEntries, + useApiMonitor, + type MonitorStats, + type MonitorStatusFilter, +} from "./use-api-monitor"; diff --git a/studio/frontend/src/features/api-monitor/lifecycle.ts b/studio/frontend/src/features/api-monitor/lifecycle.ts new file mode 100644 index 0000000000..ba59c5fce4 --- /dev/null +++ b/studio/frontend/src/features/api-monitor/lifecycle.ts @@ -0,0 +1,44 @@ +// SPDX-License-Identifier: AGPL-3.0-only +// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 + +// Labels for model load/unload/download rows, shared by the overlay and the page. +// +// Their own module because the overlay is mounted from __root.tsx: importing them +// from the page would pull the whole page into the eager bundle and undo the +// route's lazyRouteComponent. + +import type { ApiMonitorEntry } from "@/features/chat/types/api"; + +// A lifecycle row is a model load/unload/download, not an HTTP call: it carries an +// event and reason instead of a prompt, so there is no payload to expand. +export function isLifecycleEntry(entry: ApiMonitorEntry): boolean { + return entry.kind === "lifecycle"; +} + +export function lifecycleLabel(entry: ApiMonitorEntry): string { + if (entry.event === "unload") { + return entry.reason === "idle" ? "Model unloaded (idle)" : "Model unloaded"; + } + if (entry.event === "download") { + if (entry.status === "running") { + const pct = entry.progress; + return typeof pct === "number" + ? `Downloading model (${Math.round(pct)}%)` + : "Downloading model"; + } + if (entry.status === "completed") { + return "Model downloaded"; + } + // A cancel is deliberate, so calling it a failure misreads the user's action. + return entry.status === "cancelled" + ? "Model download cancelled" + : "Model download failed"; + } + if (entry.status === "running") { + return "Loading model"; + } + if (entry.status === "completed") { + return "Model loaded"; + } + return "Model load failed"; +} diff --git a/studio/frontend/src/features/api-monitor/new-traffic.ts b/studio/frontend/src/features/api-monitor/new-traffic.ts new file mode 100644 index 0000000000..b430565ae9 --- /dev/null +++ b/studio/frontend/src/features/api-monitor/new-traffic.ts @@ -0,0 +1,131 @@ +// SPDX-License-Identifier: AGPL-3.0-only +// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 + +// Which rows of a monitor snapshot are traffic this session has not shown yet. +// Split out of the overlay so it can be driven without a browser. + +import type { ApiMonitorEntry } from "@/features/chat/types/api"; + +export type WatchedEntry = Pick< + ApiMonitorEntry, + "id" | "status" | "via_api_key" | "started_at" +>; + +export interface WatchedResponse { + entries: readonly WatchedEntry[]; + // biome-ignore lint/style/useNamingConvention: API schema + server_time?: number | null; +} + +export interface ApiMonitorWatch { + /** The first snapshot has been folded in and its backlog written off. */ + seeded: boolean; + /** Ids already shown. A set, not "the newest id": finishing moves an entry to + * the front, so the head flips without any new traffic. */ + seenIds: Set; + /** performance.now() when this watch began; monotonic, so a client clock step + * mid-session cannot move it. */ + watchStartedAt: number; + /** This seed follows a stay on the full page rather than starting a session. */ + resumed: boolean; +} + +export function createWatch(nowMs: number): ApiMonitorWatch { + return { + seeded: false, + seenIds: new Set(), + watchStartedAt: nowMs, + resumed: false, + }; +} + +/** + * Re-anchor as the poll stands up, and only while still unseeded: the first + * snapshot can land long after the overlay mounted (a hidden tab issues no + * fetch, a backend still coming up fails one), and dating the backlog from + * mount would write that whole gap off as history. + */ +export function startWatching(watch: ApiMonitorWatch, nowMs: number): void { + if (!watch.seeded) { + watch.watchStartedAt = nowMs; + } +} + +/** The full page took over; what it showed is not new traffic on the way back. */ +export function rearmWatch(watch: ApiMonitorWatch): void { + watch.seeded = false; + watch.resumed = true; +} + +/** + * When this watch began, on the server's clock. + * + * The server's own ``time.time()`` minus a browser *duration*, never minus a + * browser timestamp, so a browser clock that disagrees with the server's -- a + * Studio behind a tunnel, in a container, on a host that has not run NTP -- + * cancels instead of skewing the answer. Null on a backend with no clock field, + * which keeps the old behaviour. + */ +function historyCutoff( + watch: ApiMonitorWatch, + response: WatchedResponse, + nowMs: number, +): number | null { + const serverTime = response.server_time; + if (typeof serverTime !== "number" || !Number.isFinite(serverTime)) { + return null; + } + return serverTime - Math.max(0, nowMs - watch.watchStartedAt) / 1000; +} + +function isHistory(entry: WatchedEntry, cutoff: number | null): boolean { + // Still running at the first snapshot: it started while Studio was loading, so + // it is unseen live traffic. + if (entry.status === "running") { + return false; + } + if (cutoff == null || !Number.isFinite(entry.started_at)) { + return true; + } + // Finished before the first snapshot is not the same as started before we did: + // a call made while the tab was hidden is already terminal when the poll + // finally runs, and writing it off is how the panel misses the first request. + return entry.started_at <= cutoff; +} + +/** + * Fold a snapshot in and report whether it holds API-key traffic not shown yet. + */ +export function observeResponse( + watch: ApiMonitorWatch, + response: WatchedResponse, + nowMs: number, +): boolean { + const { entries } = response; + if (!watch.seeded) { + watch.seeded = true; + const { resumed } = watch; + watch.resumed = false; + const cutoff = historyCutoff(watch, response, nowMs); + // A rearm is not a fresh watch. isHistory holds a running row back on + // purpose: at a session's first snapshot it started while Studio was still + // loading and nobody has seen it. On the way off the full page the opposite + // is true -- that page was showing this same feed, running rows included -- + // so seeding from isHistory alone reopens the overlay on the request the + // user was reading when they left. Everything the page could show is read. + watch.seenIds = new Set( + entries + .filter((entry) => resumed || isHistory(entry, cutoff)) + .map((e) => e.id), + ); + } + const seen = watch.seenIds; + // Only API-key traffic counts: Studio's own chat uses these same endpoints, and + // this panel is about serving other clients. + const hasNewTraffic = entries.some( + (entry) => entry.via_api_key && !seen.has(entry.id), + ); + // Re-seed each poll so the set stays bounded by the server's ring buffer. + watch.seenIds = new Set(entries.map((entry) => entry.id)); + return hasNewTraffic; +} diff --git a/studio/frontend/src/features/api-monitor/overlay-store.ts b/studio/frontend/src/features/api-monitor/overlay-store.ts new file mode 100644 index 0000000000..071314b93f --- /dev/null +++ b/studio/frontend/src/features/api-monitor/overlay-store.ts @@ -0,0 +1,78 @@ +// SPDX-License-Identifier: AGPL-3.0-only +// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 + +import { create } from "zustand"; +import { createJSONStorage, persist } from "zustand/middleware"; + +/** + * localStorage that cannot throw: Safari private browsing, blocked cookies and an + * opaque webview origin all make `window.localStorage` throw on access. Losing the + * preference there is fine; taking the whole panel down with it is not. + */ +const safeStorage = { + getItem: (name: string): string | null => { + try { + return window.localStorage.getItem(name); + } catch { + return null; + } + }, + setItem: (name: string, value: string): void => { + try { + window.localStorage.setItem(name, value); + } catch { + // Quota exceeded or denied; the preference stays session-only. + } + }, + removeItem: (name: string): void => { + try { + window.localStorage.removeItem(name); + } catch { + // Same. + } + }, +}; + +interface ApiMonitorOverlayState { + /** Whether the floating panel is on screen right now. Session state. */ + isOpen: boolean; + /** Set on close so the panel does not pop back in the same burst. */ + suppressed: boolean; + /** Persisted opt out: when false the panel never opens itself. */ + autoOpen: boolean; + open: () => void; + close: () => void; + setAutoOpen: (autoOpen: boolean) => void; +} + +/** Only `autoOpen` persists; a dismissal lasts the sitting, not forever. */ +export const useApiMonitorOverlayStore = create()( + persist( + (set) => ({ + isOpen: false, + suppressed: false, + autoOpen: true, + open: () => set({ isOpen: true, suppressed: false }), + close: () => set({ isOpen: false, suppressed: true }), + setAutoOpen: (autoOpen) => set({ autoOpen }), + }), + { + name: "unsloth_api_monitor_overlay", + version: 1, + storage: createJSONStorage(() => safeStorage), + partialize: (state) => ({ autoOpen: state.autoOpen }), + // Without this a version bump discards the payload and hands the popup back + // to someone who had turned it off. + migrate: (persisted) => persisted, + // Explicit merge so an older stored payload cannot resurrect `isOpen`. + merge: (persisted, current) => ({ + ...current, + autoOpen: + typeof (persisted as { autoOpen?: unknown } | null)?.autoOpen === + "boolean" + ? (persisted as { autoOpen: boolean }).autoOpen + : current.autoOpen, + }), + }, + ), +); diff --git a/studio/frontend/src/features/api-monitor/use-api-monitor.ts b/studio/frontend/src/features/api-monitor/use-api-monitor.ts new file mode 100644 index 0000000000..78a2877a42 --- /dev/null +++ b/studio/frontend/src/features/api-monitor/use-api-monitor.ts @@ -0,0 +1,313 @@ +// SPDX-License-Identifier: AGPL-3.0-only +// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 + +import { + clearApiMonitor, + getApiMonitor, + getApiMonitorEntry, +} from "@/features/chat/api/chat-api"; +import type { + ApiMonitorEntry, + ApiMonitorResponse, +} from "@/features/chat/types/api"; +import { useCallback, useEffect, useMemo, useRef, useState } from "react"; + +/** Poll cadence while live. Matches the settings console it replaces. */ +const POLL_INTERVAL_MS = 1500; + +export type MonitorStatusFilter = + | "all" + | "running" + | "completed" + | "error" + | "cancelled"; + +export interface MonitorStats { + active: number; + total: number; + completed: number; + errors: number; + cancelled: number; + /** Mean duration over finished requests, or null when none have finished. */ + avgDurationMs: number | null; + /** Slowest finished request, for spotting a pathological call. */ + maxDurationMs: number | null; + totalTokens: number; + /** Share of finished requests that failed, 0-1. Null when nothing finished. */ + errorRate: number | null; + /** Mean completion tokens per second over requests that reported both. */ + tokensPerSecond: number | null; +} + +function isTerminal(entry: ApiMonitorEntry): boolean { + return entry.status !== "running"; +} + +function completionTokens(entry: ApiMonitorEntry): number | null { + if (entry.completion_tokens != null) { + return entry.completion_tokens; + } + // Some providers report only a total, so subtracting the prompt is the best + // estimate of what was generated. + if (entry.total_tokens != null && entry.prompt_tokens != null) { + return Math.max(0, entry.total_tokens - entry.prompt_tokens); + } + return null; +} + +function entryTokens(entry: ApiMonitorEntry): number { + if (entry.total_tokens != null) { + return entry.total_tokens; + } + return (entry.prompt_tokens ?? 0) + (entry.completion_tokens ?? 0); +} + +export function computeStats(entries: ApiMonitorEntry[]): MonitorStats { + let active = 0; + let completed = 0; + let errors = 0; + let cancelled = 0; + let totalTokens = 0; + let durationSum = 0; + let durationCount = 0; + let maxDurationMs: number | null = null; + // Total tokens over total time, not the mean of each request's rate: averaging + // rates lets one tiny fast request outweigh a long slow one. + let generatedTokens = 0; + let generatedDurationMs = 0; + + let requests = 0; + + for (const entry of entries) { + // A load, unload or download is not an HTTP call. It reads as "running" for the + // whole load, so counting it reports an in-flight request with no client waiting + // and folds a multi-minute download into "Avg latency". The backend leaves these + // out of active_count too, so counting them would also disagree with the API. + if (entry.kind === "lifecycle") { + continue; + } + requests += 1; + totalTokens += entryTokens(entry); + if (entry.status === "running") { + active += 1; + } else if (entry.status === "error") { + errors += 1; + } else if (entry.status === "cancelled") { + cancelled += 1; + } else { + completed += 1; + } + const duration = entry.duration_ms; + if (duration != null && isTerminal(entry)) { + durationSum += duration; + durationCount += 1; + maxDurationMs = + maxDurationMs == null ? duration : Math.max(maxDurationMs, duration); + const generated = completionTokens(entry); + // A sub-millisecond duration divides into a meaningless rate. + if (generated != null && generated > 0 && duration > 0) { + generatedTokens += generated; + generatedDurationMs += duration; + } + } + } + + const finished = completed + errors + cancelled; + return { + active, + total: requests, + completed, + errors, + cancelled, + avgDurationMs: durationCount > 0 ? durationSum / durationCount : null, + maxDurationMs, + totalTokens, + errorRate: finished > 0 ? errors / finished : null, + tokensPerSecond: + generatedDurationMs > 0 + ? generatedTokens / (generatedDurationMs / 1000) + : null, + }; +} + +export function filterEntries( + entries: ApiMonitorEntry[], + status: MonitorStatusFilter, + query: string, +): ApiMonitorEntry[] { + const needle = query.trim().toLowerCase(); + return entries.filter((entry) => { + if (status !== "all" && entry.status !== status) { + return false; + } + if (!needle) { + return true; + } + // The fields a debugging session keys off: model, endpoint, and the previews and + // error text visible in the row. Coerced, not trusted: these arrive over the + // network, and one malformed entry throwing here would blank the whole log. + return [ + entry.model, + entry.endpoint, + entry.prompt_preview, + entry.reply_preview, + entry.error, + ].some((field) => + String(field ?? "") + .toLowerCase() + .includes(needle), + ); + }); +} + +interface UseApiMonitorResult { + data: ApiMonitorResponse | null; + entries: ApiMonitorEntry[]; + stats: MonitorStats; + error: string | null; + /** True until the first response lands, so skeletons show once. */ + loading: boolean; + refreshing: boolean; + paused: boolean; + setPaused: (paused: boolean) => void; + refresh: () => void; + clear: () => Promise; + /** Full prompt/reply for expanded entries, keyed by entry id. */ + details: Record; + loadingDetails: ReadonlySet; + requestDetail: (id: string) => boolean; +} + +/** + * Live view of the server's OpenAI-compatible API traffic. + * + * Polls rather than streams because the backing monitor is an in-memory ring buffer + * with no change feed. Polling self-reschedules (never overlapping), and pausing + * stops it so reading a stalled payload is not fighting a list that reorders. + * + * `intervalMs` lets a caller trade freshness for cost: the full page wants the + * default live cadence, while the floating overlay slows right down when it is + * closed and only watching for the traffic that should pop it open. + */ +export function useApiMonitor({ + intervalMs = POLL_INTERVAL_MS, +}: { intervalMs?: number } = {}): UseApiMonitorResult { + const [data, setData] = useState(null); + const [error, setError] = useState(null); + const [loading, setLoading] = useState(true); + const [refreshing, setRefreshing] = useState(false); + const [paused, setPaused] = useState(false); + const [details, setDetails] = useState>({}); + const [loadingDetails, setLoadingDetails] = useState>( + () => new Set(), + ); + // Mirrors `loadingDetails` outside React state so the fetch guard sees same-tick + // writes; async state updates would let duplicates through. + const inFlightDetails = useRef>(new Set()); + + const load = useCallback(async (): Promise => { + setRefreshing(true); + try { + const next = await getApiMonitor(); + setData(next); + setError(null); + } catch (err: unknown) { + setError(err instanceof Error ? err.message : "Monitor unavailable"); + } finally { + setRefreshing(false); + setLoading(false); + } + }, []); + + useEffect(() => { + if (paused) { + return; + } + let cancelled = false; + let timer: number | undefined; + + function poll(): void { + getApiMonitor() + .then((next) => { + if (cancelled) return; + setData(next); + setError(null); + }) + .catch((err: unknown) => { + if (cancelled) return; + setError(err instanceof Error ? err.message : "Monitor unavailable"); + }) + .finally(() => { + if (cancelled) return; + setLoading(false); + timer = window.setTimeout(poll, intervalMs); + }); + } + + poll(); + return () => { + cancelled = true; + if (timer !== undefined) { + window.clearTimeout(timer); + } + }; + }, [paused, intervalMs]); + + // Returns whether a fetch started: a caller must not record "fetched revision N" + // when the guard refused, or that revision is skipped once updated_at settles. + const requestDetail = useCallback((id: string): boolean => { + if (inFlightDetails.current.has(id)) { + return false; + } + inFlightDetails.current.add(id); + setLoadingDetails((prev) => new Set(prev).add(id)); + getApiMonitorEntry(id) + .then((entry) => { + setDetails((prev) => ({ ...prev, [id]: entry })); + }) + .catch(() => { + // Aged out of the ring buffer; drop the stale copy so the UI falls back to + // the row previews instead of a frozen payload. + setDetails((prev) => { + if (!(id in prev)) return prev; + const next = { ...prev }; + delete next[id]; + return next; + }); + }) + .finally(() => { + inFlightDetails.current.delete(id); + setLoadingDetails((prev) => { + const next = new Set(prev); + next.delete(id); + return next; + }); + }); + return true; + }, []); + + const clear = useCallback(async (): Promise => { + await clearApiMonitor(); + setDetails({}); + await load(); + }, [load]); + + const entries = useMemo(() => data?.entries ?? [], [data]); + const stats = useMemo(() => computeStats(entries), [entries]); + + return { + data, + entries, + stats, + error, + loading, + refreshing, + paused, + setPaused, + refresh: () => void load(), + clear, + details, + loadingDetails, + requestDetail, + }; +} diff --git a/studio/frontend/src/features/chat/api/chat-api.ts b/studio/frontend/src/features/chat/api/chat-api.ts index 8ad2691391..5431608c46 100644 --- a/studio/frontend/src/features/chat/api/chat-api.ts +++ b/studio/frontend/src/features/chat/api/chat-api.ts @@ -129,6 +129,13 @@ export async function getApiMonitorEntry(id: string): Promise { return parseJsonOrThrow(response); } +export async function clearApiMonitor(): Promise { + const response = await authFetch("/api/inference/monitor", { + method: "DELETE", + }); + await parseJsonOrThrow<{ cleared: boolean }>(response); +} + export interface ActiveGenerationsResponse { count: number; /** Conversations with a generation in flight. Shorter than `count` when a diff --git a/studio/frontend/src/features/chat/index.ts b/studio/frontend/src/features/chat/index.ts index cfeb8fff0f..036736930d 100644 --- a/studio/frontend/src/features/chat/index.ts +++ b/studio/frontend/src/features/chat/index.ts @@ -35,6 +35,10 @@ export type { GgufVariantDetail, InferenceStatusResponse, } from "./types/api"; +export { + applyActiveModelStatusToStore, + resolveInferenceCheckpointId, +} from "./lib/apply-inference-status-to-store"; export { ChatSettingsPanel, ParamSlider, diff --git a/studio/frontend/src/features/chat/types/api.ts b/studio/frontend/src/features/chat/types/api.ts index b67a9eda26..669d8fbac0 100644 --- a/studio/frontend/src/features/chat/types/api.ts +++ b/studio/frontend/src/features/chat/types/api.ts @@ -301,6 +301,9 @@ export interface ApiMonitorEntry { model: string; prompt?: string; reply?: string; + // True for API-key callers, not UI sessions. The floating panel keys its + // auto-open off this so Studio's own chat does not pop it. + via_api_key: boolean; prompt_preview: string; reply_preview: string; prompt_truncated: boolean; @@ -326,6 +329,10 @@ export interface ApiMonitorEntry { export interface ApiMonitorResponse { status: "idle" | "ready" | "generating"; + // Server wall clock (seconds) when the snapshot was taken, so an entry's + // started_at can be dated without trusting the browser's clock to agree. + // Absent on a backend older than the field. + server_time?: number; active_model?: string | null; context_length?: number | null; active_requests: number; diff --git a/studio/frontend/src/features/hub/catalog/download-card.tsx b/studio/frontend/src/features/hub/catalog/download-card.tsx index 8016a9406d..cf767c7903 100644 --- a/studio/frontend/src/features/hub/catalog/download-card.tsx +++ b/studio/frontend/src/features/hub/catalog/download-card.tsx @@ -23,6 +23,7 @@ import { ArrowReloadHorizontalIcon, Delete02Icon, Download01Icon, + Settings02Icon, } from "@hugeicons/core-free-icons"; import { HugeiconsIcon } from "@hugeicons/react"; import { @@ -82,6 +83,39 @@ export function CardDivider() { ); } +export function CardSettingsButton({ + label, + onClick, +}: { + label: string; + onClick: () => void; +}) { + return ( + + + + + + {label} + + + ); +} + export function CardDeleteButton({ label, onClick, diff --git a/studio/frontend/src/features/hub/catalog/hub-model-settings-view.tsx b/studio/frontend/src/features/hub/catalog/hub-model-settings-view.tsx new file mode 100644 index 0000000000..31cf3b31c8 --- /dev/null +++ b/studio/frontend/src/features/hub/catalog/hub-model-settings-view.tsx @@ -0,0 +1,154 @@ +// SPDX-License-Identifier: AGPL-3.0-only +// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 + +// Full-page settings for one model, opened from the Hub. +// +// The same controls exist in the chat picker's popover, but a popover is a poor +// place to work through every knob. This gives them a page and says plainly that +// what is saved here is what an API load uses, mirrored by ModelConfigPage. + +import { + ModelConfigPage, + type ModelPickTarget, + modelConfigInstanceKey, +} from "@/features/model-picker"; +import type { PerModelConfig } from "@/features/model-picker"; +import { cn } from "@/lib/utils"; +import { ArrowLeft01Icon, Globe02Icon } from "@hugeicons/core-free-icons"; +import { HugeiconsIcon } from "@hugeicons/react"; +import { useEffect, useRef, useState } from "react"; + +export function HubModelSettingsView({ + target, + loadedConfig = null, + loadedContextLength = null, + onBack, + onRun, + compact = false, +}: { + target: ModelPickTarget; + /** Non-null when this model is loaded, so the page can show live values. */ + loadedConfig?: PerModelConfig | null; + loadedContextLength?: number | null; + onBack: () => void; + /** Apply + load with these settings. */ + onRun: (config: PerModelConfig) => void; + compact?: boolean; +}) { + const scrollRef = useRef(null); + const [scrolled, setScrolled] = useState(false); + // Mirrors HubDetailView so this view sits at the Hub's measure. + const measure = compact + ? "mx-auto w-full max-w-[860px] px-5 sm:px-5" + : "mx-auto w-full max-w-[1100px] px-5 sm:px-8"; + + useEffect(() => { + const el = scrollRef.current; + if (!el) return; + const onScroll = () => { + const next = el.scrollTop > 0; + setScrolled((current) => (current === next ? current : next)); + }; + onScroll(); + el.addEventListener("scroll", onScroll, { passive: true }); + return () => el.removeEventListener("scroll", onScroll); + }, []); + + return ( +
+ + ); +} diff --git a/studio/frontend/src/features/hub/catalog/local-on-device-card.tsx b/studio/frontend/src/features/hub/catalog/local-on-device-card.tsx index 8f2672a706..1a4bb50369 100644 --- a/studio/frontend/src/features/hub/catalog/local-on-device-card.tsx +++ b/studio/frontend/src/features/hub/catalog/local-on-device-card.tsx @@ -58,6 +58,7 @@ import { useHfTokenStore } from "../stores/hf-token-store"; import { DotTag } from "./dot-tag"; import { CardDeleteButton, + CardSettingsButton, CardUpdateButton, DeleteConfirmDialog, UpdateConfirmDialog, @@ -104,6 +105,14 @@ interface LocalOnDeviceCardProps { onEject?: () => void; onTrain?: () => void; onChange?: () => void; + /** + * Open settings for the quant this card is showing. + * + * ``quantIsUserPicked`` says whether that quant came from a pick in this + * card's selector or was derived from the resident model, which decides + * whether a fresher status read may override it. + */ + onOpenSettings?: (ggufVariant: string | null, quantIsUserPicked: boolean) => void; } function formatAdapterLabel( @@ -225,6 +234,7 @@ export function LocalOnDeviceCard({ onEject, onTrain, onChange, + onOpenSettings, }: LocalOnDeviceCardProps) { const [deleteOpen, setDeleteOpen] = useState(false); const [updateOpen, setUpdateOpen] = useState(false); @@ -367,6 +377,14 @@ export function LocalOnDeviceCard({ )?.quant ?? sortedVariants?.[0]?.quant ?? null); + // Only the first branch above is a choice; the rest are fallbacks that read the + // store, and the store can be stale for as long as this window keeps focus. + const quantIsUserPicked = Boolean( + selectedVariantOverride && + sortedVariants?.some((variant) => + ggufVariantsMatch(variant.quant, selectedVariantOverride), + ), + ); const selectedVariant = sortedVariants?.find((variant) => ggufVariantsMatch(variant.quant, selectedQuant), @@ -578,6 +596,16 @@ export function LocalOnDeviceCard({ )}
+ {onOpenSettings && ( + + onOpenSettings(selectedQuant ?? null, quantIsUserPicked) + } + /> + )} {canUpdate && ( void; onInventoryChange?: () => void; onSearchHub?: (query: string) => void; + /** Open settings with the quant the card resolved. */ + onOpenSettings?: (ggufVariant: string | null) => void; }; export const ModelInspector = memo(function ModelInspector({ @@ -444,6 +446,7 @@ export const ModelInspector = memo(function ModelInspector({ onTrain, onInventoryChange, onSearchHub, + onOpenSettings, } = actions; const deviceType = usePlatformStore((s) => s.deviceType); const chatOnly = usePlatformStore((s) => s.isChatOnly()); @@ -714,6 +717,7 @@ export const ModelInspector = memo(function ModelInspector({ model.isDownloaded && canTrainModel ? onTrain : undefined } onChange={onInventoryChange} + onOpenSettings={onOpenSettings} /> ) : ( void; + onOpenModelSettings?: (row: CachedInventoryRow | LocalInventoryRow) => void; }) { // Pinned repos surface first regardless of the active sort; the chosen sort // still orders rows within the pinned and unpinned groups. @@ -383,6 +385,7 @@ export function DownloadedList({ compact={compact} onSelect={onSelect} onChange={onInventoryChange} + onOpenSettings={onOpenModelSettings} /> ); diff --git a/studio/frontend/src/features/hub/catalog/models-catalog-rows.tsx b/studio/frontend/src/features/hub/catalog/models-catalog-rows.tsx index 5236c0be74..3a9aaa064b 100644 --- a/studio/frontend/src/features/hub/catalog/models-catalog-rows.tsx +++ b/studio/frontend/src/features/hub/catalog/models-catalog-rows.tsx @@ -573,6 +573,7 @@ export const InventoryRow = memo(function InventoryRow({ compact = false, onSelect, onChange, + onOpenSettings, }: { row: CachedInventoryRow | LocalInventoryRow; selected: boolean; @@ -585,6 +586,8 @@ export const InventoryRow = memo(function InventoryRow({ compact?: boolean; onSelect: (id: string) => void; onChange?: () => void; + /** Open this model's settings page. Omitted for datasets. */ + onOpenSettings?: (row: CachedInventoryRow | LocalInventoryRow) => void; }) { const rowModelId = row.kind === "cache" @@ -732,30 +735,39 @@ export const InventoryRow = memo(function InventoryRow({ const rowPinned = cacheDeletableRepoId != null && pinnedKeys.includes(pinKey(cacheDeletableRepoId)); + // Settings applies to any downloaded model, not just deletable ones, so the menu + // renders when either action applies and each item gates itself. `deletableRepoId` + // (not the boolean) keeps the non-null narrowing the delete closures rely on. + const settingsAction = + !isDataset && onOpenSettings ? { onOpen: () => onOpenSettings(row) } : undefined; + const deletableRepoId = canDelete ? cacheDeletableRepoId : null; const deleteAction = - canDelete && cacheDeletableRepoId ? ( + deletableRepoId || settingsAction ? ( togglePinned(cacheDeletableRepoId), + onToggle: () => togglePinned(deletableRepoId), } } - cachePath={isDataset ? undefined : { repoId: cacheDeletableRepoId }} - del={{ + cachePath={ + isDataset || !deletableRepoId ? undefined : { repoId: deletableRepoId } + } + del={deletableRepoId ? { title: isDataset ? "Delete cached dataset?" : "Delete cached model?", description: ( <> This will remove{" "} - {cacheDeletableRepoId} + {deletableRepoId} {" "} {isDataset ? "and its downloaded files" @@ -766,17 +778,17 @@ export const InventoryRow = memo(function InventoryRow({ disk. You can re-download it later. ), - successMessage: `Deleted ${cacheDeletableRepoId}`, + successMessage: `Deleted ${deletableRepoId}`, onConfirm: async () => { // Delete only the copy this row shows: cache rows carry the owning // cache path, so pass it through and leave other caches untouched. const rowCachePath = row.kind === "cache" ? (row.cachePath ?? undefined) : undefined; if (isDataset) { - await deleteCachedDataset(cacheDeletableRepoId, rowCachePath); + await deleteCachedDataset(deletableRepoId, rowCachePath); } else { await deleteCachedModel( - cacheDeletableRepoId, + deletableRepoId, undefined, undefined, rowCachePath, @@ -787,11 +799,11 @@ export const InventoryRow = memo(function InventoryRow({ usePinnedModelsStore.getState(); for (const key of pinned) { if ( - key === pinKey(cacheDeletableRepoId) || - key.startsWith(`${cacheDeletableRepoId}::`) + key === pinKey(deletableRepoId) || + key.startsWith(`${deletableRepoId}::`) ) { toggle( - cacheDeletableRepoId, + deletableRepoId, key.includes("::") ? key.slice(key.indexOf("::") + 2) : undefined, @@ -801,7 +813,7 @@ export const InventoryRow = memo(function InventoryRow({ } }, onDeleted: onChange, - }} + } : undefined} /> ) : null; diff --git a/studio/frontend/src/features/hub/catalog/models-catalog.tsx b/studio/frontend/src/features/hub/catalog/models-catalog.tsx index 02b9b62fce..e67954d267 100644 --- a/studio/frontend/src/features/hub/catalog/models-catalog.tsx +++ b/studio/frontend/src/features/hub/catalog/models-catalog.tsx @@ -16,6 +16,7 @@ import { import type { CachedInventoryRow, DiscoverRow, + InventoryRow, LocalInventoryRow, ModelsTab, } from "../types"; @@ -70,6 +71,7 @@ export interface ModelsCatalogHandlers { onRetry: () => void; onInventoryChange?: () => void; onSwitchDevice?: () => void; + onOpenModelSettings?: (row: InventoryRow) => void; } function assignRef(ref: RefObject, value: T | null) { @@ -128,6 +130,7 @@ export const ModelsCatalog = memo(function ModelsCatalog({ onRetry, onInventoryChange, onSwitchDevice, + onOpenModelSettings, } = handlers; const [scrolled, setScrolled] = useState(false); const [streamingActive, setStreamingActive] = useState(false); @@ -483,6 +486,7 @@ export const ModelsCatalog = memo(function ModelsCatalog({ columns={discoverView === "two" ? 2 : 1} sort={inventorySort} onInventoryChange={onInventoryChange} + onOpenModelSettings={onOpenModelSettings} />
) : ( diff --git a/studio/frontend/src/features/hub/hub-page.tsx b/studio/frontend/src/features/hub/hub-page.tsx index 36879097d0..a67c6cab50 100644 --- a/studio/frontend/src/features/hub/hub-page.tsx +++ b/studio/frontend/src/features/hub/hub-page.tsx @@ -3,33 +3,29 @@ import { usePlatformStore } from "@/config/env"; import { - isChannelEntryFresh, - useHubFeedStore, -} from "./stores/hub-feed-store"; -import { + applyActiveModelStatusToStore, getInferenceStatus, isExternalModelId, + listGgufVariants, + resolveInferenceCheckpointId, useChatModelRuntime, useChatRuntimeStore, } from "@/features/chat"; -import { useHubInventory } from "./inventory"; -import type { - HfModelSearchChannel, - HfSortDirection, - HfSortKey, -} from "./hooks/use-hub-model-search"; import { useOnlineStatus } from "@/features/hub"; import { useHubInfiniteScroll } from "@/features/hub"; -import { ggufVariantsMatch, modelIdsMatch } from "./lib/model-identity"; -import { hfApiToken, useHfTokenStore } from "./stores/hf-token-store"; import { + type ModelPickTarget, + type PerModelConfig, applyModelLoadConfigToRuntime, + applyPerModelConfigToRuntime, currentRuntimePerModelConfig, hfModelFitsDevice, resolveInitialConfig, + useActiveModelConfig, } from "@/features/model-picker"; import { useDebouncedValue } from "@/hooks/use-debounced-value"; import { useGpuInfo, useInferenceGpuInfo } from "@/hooks/use-gpu-info"; +import { toast } from "@/lib/toast"; import { cn } from "@/lib/utils"; import { useNavigate, useSearch } from "@tanstack/react-router"; import { @@ -43,6 +39,7 @@ import { import { ExternalLinkConfirmDialog } from "./catalog/external-link-confirm-dialog"; import { HubDetailView } from "./catalog/hub-detail-view"; import { HubFeed } from "./catalog/hub-feed"; +import { HubModelSettingsView } from "./catalog/hub-model-settings-view"; import { HubTopBar } from "./catalog/hub-top-bar"; import { ModelsCatalog, @@ -66,8 +63,18 @@ import { useDiscoverSearch } from "./hooks/use-discover-search"; import { useFeedWriteBack } from "./hooks/use-feed-write-back"; import { useHiddenEmbeddingModelIds } from "./hooks/use-hidden-embedding-models"; import { useHubFeed } from "./hooks/use-hub-feed"; +import type { + HfModelSearchChannel, + HfSortDirection, + HfSortKey, +} from "./hooks/use-hub-model-search"; import { useHubModelVram } from "./hooks/use-hub-model-vram"; import { useModelsSelection } from "./hooks/use-models-selection"; +import { useHubInventory } from "./inventory"; +import { LOCAL_MODEL_SOURCE } from "./inventory/constants"; +import { settingsGgufVariantForRow } from "./inventory/settings-identity"; +import { adoptResidentModelStatus } from "./lib/adopt-inference-status"; +import { subscribeResidentStatusRefresh } from "./lib/resident-status-refresh"; import { CHANNEL_TO_SECTION, type ChannelId, @@ -82,6 +89,11 @@ import { isHiddenModelId, } from "./lib/hidden-models"; import { inventoryRowMatches, tokenizeQuery } from "./lib/inventory-search"; +import { + ggufVariantsMatch, + modelIdsMatch, + residentModelIdMatches, +} from "./lib/model-identity"; import { type ModelTypeFilter, matchesModelType, @@ -95,6 +107,8 @@ import { matchesCapability, matchesFormat, } from "./lib/view-models"; +import { hfApiToken, useHfTokenStore } from "./stores/hf-token-store"; +import { isChannelEntryFresh, useHubFeedStore } from "./stores/hub-feed-store"; import type { CachedInventoryRow, CapabilityFilter, @@ -104,8 +118,29 @@ import type { ModelsTab, ResourceTypeFilter, SelectedModelView, + SelectedResourceRef, } from "./types"; +// What per-model settings are keyed by, which is not always what the loader is +// handed: a repo cached outside the active HF cache loads by snapshot path, while +// the chat picker and the auto-switch index key it by repo id, so saving under the +// path strands the settings. Local rows are keyed by load id in both places. +// The row decides that, not the view it is being shown in: the same cached repo +// is reachable from Discover, where the kind is "discover" while the resource is +// still the cache row with its snapshot-path run id. Keying that off the kind +// stranded the settings for exactly the case this exists to handle. `hub_cache` +// is set only for a cached repo; a local row in the active HF cache is `hf_cache` +// and stays on its run id, as the Downloaded row keys it. +function modelConfigIdentity( + kind: SelectedModelView["kind"], + resource: SelectedResourceRef, +): string { + if (kind !== "cache" && resource.source !== "hub_cache") { + return resource.runId; + } + return resource.repoId ?? resource.runId; +} + const MODELS_TAB_STORAGE_KEY = "unsloth.hub.modelsTab"; const ALL_MODELS_VIEW_STORAGE_KEY = "unsloth.hub.allModelsView"; const INVENTORY_SORT_STORAGE_KEY = "unsloth.hub.inventorySort"; @@ -348,33 +383,81 @@ export function ModelsPage() { const activeCheckpoint = checkpoint && !isExternalModelId(checkpoint) ? checkpoint : null; const activeGgufVariant = useChatRuntimeStore((s) => s.activeGgufVariant); + const activeGgufContextLength = useChatRuntimeStore( + (s) => s.ggufContextLength, + ); + // Live settings of the loaded model, so its settings page shows what it is + // running with rather than the last saved draft. + const { config: activeModelConfig } = useActiveModelConfig(); // Shared with the chat model selector: list only models sized for this device. const fitOnDeviceOnly = useChatRuntimeStore((s) => s.fitOnDeviceOnly); const setFitOnDeviceOnly = useChatRuntimeStore((s) => s.setFitOnDeviceOnly); - useEffect(() => { - let cancelled = false; - void getInferenceStatus() + // Drops a response that lands after a newer read started, or after unmount. + const residentStatusSeq = useRef(0); + // Returns the read so a caller that needs the answer before it decides + // anything can wait for it; fire-and-forget callers just drop it. + const refreshResidentModelStatus = useCallback((): Promise => { + const seq = ++residentStatusSeq.current; + return getInferenceStatus() .then((status) => { - if (cancelled || !status.active_model) return; + if (seq !== residentStatusSeq.current) return; const store = useChatRuntimeStore.getState(); - if ( - !isExternalModelId(store.params.checkpoint) && - (!modelIdsMatch(store.params.checkpoint, status.active_model) || - !ggufVariantsMatch( - store.activeGgufVariant, - status.gguf_variant ?? null, - )) - ) { - store.setCheckpoint(status.active_model, status.gguf_variant ?? null); - } + adoptResidentModelStatus( + { + // The loadable identifier, as every other status reader records it: a + // GGUF from a non-active HF cache or straight off disk loads by path, + // while active_model is the clean public id (an HF snapshot's repo id, + // any other file's filename stem). Two files that share a stem collapse + // onto one id, so storing that would make the catalog row for one of + // them look loaded. + checkpointId: resolveInferenceCheckpointId(status), + ggufVariant: status.gguf_variant ?? null, + }, + { + checkpoint: store.params.checkpoint, + checkpointIsExternal: isExternalModelId(store.params.checkpoint), + activeGgufVariant: store.activeGgufVariant, + modelLoading: store.modelLoading, + }, + { + setCheckpoint: (checkpointId, ggufVariant) => { + store.setCheckpoint(checkpointId, ggufVariant); + }, + clearCheckpoint: () => { + store.clearCheckpoint(); + }, + // Landing here is the one entry point that has applied no status yet, + // so the settings page would read this model's live config off a store + // still holding defaults. Same call the chat runtime's refresh makes. + applyStatus: (previous) => { + applyActiveModelStatusToStore(status, { + previousCheckpoint: previous.checkpoint ?? undefined, + previousGgufVariant: previous.ggufVariant, + }); + }, + }, + ); }) .catch(() => undefined); - return () => { - cancelled = true; - }; }, []); + // Mount, then again whenever this tab could have missed an API-driven switch. + // adoptResidentModelStatus is what makes re-reading safe: it stands down for an + // external selection and for a load this tab started, so a refresh never fights + // the model the user is switching to. + useEffect(() => { + void refreshResidentModelStatus(); + const unsubscribe = subscribeResidentStatusRefresh( + refreshResidentModelStatus, + ); + return () => { + // A response still in flight adopts nothing once the Hub is gone. + residentStatusSeq.current += 1; + unsubscribe(); + }; + }, [refreshResidentModelStatus]); + const { tab, setTab: setModelsTab } = useModelsTabState(); const [query, setQuery] = useState(""); const [sortBy, setSortBy] = useState( @@ -660,11 +743,7 @@ export function ModelsPage() { const visibleResults = results.length === 0 && liveListChannel && - isChannelEntryFresh( - cachedListEntry, - liveListChannel.id, - tokenFingerprint, - ) + isChannelEntryFresh(cachedListEntry, liveListChannel.id, tokenFingerprint) ? (cachedListEntry?.results ?? results) : results; @@ -1187,7 +1266,10 @@ export function ModelsPage() { (opts: ModelLoadOptions, isDownloaded: boolean) => { if (!selectedModel) return; const runId = selectedModel.resource.runId; - const resolvedConfig = resolveInitialConfig(runId, opts.ggufVariant); + const resolvedConfig = resolveInitialConfig( + modelConfigIdentity(selectedModel.kind, selectedModel.resource), + opts.ggufVariant, + ); const rememberedConfig = resolvedConfig.remembered ? resolvedConfig.config : null; @@ -1221,6 +1303,177 @@ export function ModelsPage() { runSelectedModel(opts, selectedModel?.isDownloaded ?? true), [runSelectedModel, selectedModel], ); + + // Full-page per-model settings, opened from a downloaded row's menu. Local state + // rather than a URL param: the page is a transient editor over the catalog, and a + // deep link would need the row re-resolved against an inventory that may not have + // loaded. + const [settingsTarget, setSettingsTarget] = useState( + null, + ); + // Opening a model's settings is where a stale read costs something: the editor + // is seeded once, from saved/default values when the store says this model is + // not the resident one, and Apply reloads with them. Reading status here covers + // a switch that landed while this window kept focus the whole time. + // + // It cannot cover the resolution of the target itself, though: a target has to + // exist before this runs. openModelSettings therefore does its own read; this + // one is for the handlers that hand the target over ready-made. + useEffect(() => { + if (settingsTarget) void refreshResidentModelStatus(); + }, [settingsTarget, refreshResidentModelStatus]); + // Bumped per open so a slow variant lookup for an abandoned row cannot land + // on top of the row actually chosen. + const settingsOpenSeq = useRef(0); + const openModelSettings = useCallback( + async (row: CachedInventoryRow | LocalInventoryRow) => { + const openSeq = ++settingsOpenSeq.current; + // loadId is what the loader accepts; repoId is only a display/API alias. + const id = row.loadId; + // Every name this row answers to. Residency is judged against the store + // AFTER the variant lookup settles, not here, because that lookup and the + // status refresh this same click starts are in flight together. + const rowAliases = + row.kind === "local" + ? [id, row.repoId, row.path] + : [id, row.repoId, row.cachePath]; + // Cached repo rows never carry a quant (cache_inventory.py emits one row per + // repo with format_variant null). Opening with a null variant keys the config + // to `repo::` while the loader reads `repo::Q4_K_M`, so it never applies and + // the server mirror is wrong too. Resolve it as the on-device card does. + let ggufVariant = settingsGgufVariantForRow(row); + if (!ggufVariant && row.isGguf && row.capabilities.requiresVariant) { + // A local row only carries a repo id when it sits in the HF cache, so a + // plain folder of quants (the models dir, a custom folder, LM Studio) has + // none while still being marked as needing one. The listing takes a path + // in the same position and scans it, which is exactly what the on-device + // card already does for these rows, so without the fallback this menu + // entry could only ever reach the "couldn't determine which quant" toast. + const repoId = + row.kind === "cache" ? row.repoId : (row.repoId ?? row.path ?? null); + if (repoId) { + try { + const [res] = await Promise.all([ + listGgufVariants(repoId, hfApiToken(hfToken), { + preferLocalCache: true, + localPath: + row.kind === "local" ? row.path : (row.cachePath ?? null), + }), + // Read status as part of this click. The effect above cannot help + // here, because it does not run until the target it watches exists, + // and the Hub has no polling timer: it re-reads on focus and + // visibility only. So a window that has kept focus since the last + // read holds a checkpoint from before any API-driven switch, for + // however long the user has been sitting on the page. Alongside the + // variant lookup rather than before it, since that one usually hits + // the network while this is a loopback call. + refreshResidentModelStatus(), + ]); + const downloaded = res.variants.filter((v) => v.downloaded); + // Read residency after the awaits, never from the values closed over + // above: the status read that just settled is what knows which model + // is resident now, and the loaded-quant branch below would otherwise + // silently pick the quant of whichever model it displaced. + const settled = useChatRuntimeStore.getState(); + const settledCheckpoint = + settled.params.checkpoint && + !isExternalModelId(settled.params.checkpoint) + ? settled.params.checkpoint + : null; + const settledIsActive = rowAliases.some((alias) => + modelIdsMatch(alias, settledCheckpoint), + ); + ggufVariant = + // Loaded quant, then the repo default, then whatever is on disk, + // mirroring LocalOnDeviceCard's selectedQuant. Only for the loaded + // row: Q4_K_M exists in most repos, so an unguarded match would + // target the wrong quant of the wrong model. + (settledIsActive + ? downloaded.find((v) => + ggufVariantsMatch(v.quant, settled.activeGgufVariant), + )?.quant + : undefined) ?? + downloaded.find((v) => + ggufVariantsMatch(v.quant, res.default_variant), + )?.quant ?? + downloaded[0]?.quant ?? + null; + } catch { + ggufVariant = null; + } + } + if (!ggufVariant) { + // A model that needs a quant cannot be configured without one: the + // picker matches variants exactly and would never find the config, + // while the API's bare-key fallback would apply it. + toast.error("Couldn't determine which quant to configure.", { + description: + "Settings for this model are per quant. Check the connection or the model's cache, then try again.", + }); + return; + } + } + // The variant lookup is async, so without this a second row opened while it + // was pending would be overwritten by whichever call finished last. + if (settingsOpenSeq.current !== openSeq) { + return; + } + // A repo in a previous cache loads by snapshot path, so `id` ends in the + // revision hash; name the row by what the user calls it instead. + const configId = row.kind === "cache" ? row.repoId : id; + const leaf = configId.split(/[\\/]/).filter(Boolean).pop() ?? configId; + setSettingsTarget({ + id, + configId, + displayName: ggufVariant ? `${leaf} ยท ${ggufVariant}` : leaf, + ggufVariant, + isGguf: row.isGguf, + apiLoadable: + row.isGguf && + (row.kind !== "local" || row.source !== LOCAL_MODEL_SOURCE.OLLAMA), + meta: { + source: "local", + isLora: row.modelFormat === "adapter", + ggufVariant: ggufVariant ?? undefined, + isGguf: row.isGguf, + // A partial download opens settings too, but claiming complete would skip + // the loader's download-progress reporting. + isDownloaded: !row.partial, + // Not on inventory rows; ModelConfigPage reads the GGUF header itself. + contextLength: null, + }, + }); + }, + [activeCheckpoint, activeGgufVariant, hfToken, refreshResidentModelStatus], + ); + // Applying loads the model with exactly these settings. ModelConfigPage has + // already persisted them locally and, when "remember" is on, to the server, so + // an API request for this model gets the same load. + const runSettingsTarget = useCallback( + (config: PerModelConfig) => { + const target = settingsTarget; + if (!target) return; + const previousConfig = currentRuntimePerModelConfig({ + includeMaxSeqLength: true, + }); + applyPerModelConfigToRuntime(config); + setSettingsTarget(null); + void selectModel({ + id: target.id, + source: "local", + ggufVariant: target.ggufVariant ?? undefined, + isGguf: target.isGguf, + // A partial row opens settings too; claiming complete would skip the + // loader's download-progress reporting. + isDownloaded: target.meta.isDownloaded, + isLora: target.meta.isLora, + keepSpeculative: true, + forceReload: true, + previousConfig, + }).catch(() => undefined); + }, + [selectModel, settingsTarget], + ); const handleLoadLocal = useCallback( (opts: ModelLoadOptions = {}) => runSelectedModel(opts, true), [runSelectedModel], @@ -1228,6 +1481,100 @@ export function ModelsPage() { const handleTrain = useCallback(() => { // Hub โ†’ train integration ships in a later PR. }, []); + // Opened from the detail view's on-device card, which passes in the quant it + // resolved rather than making this re-derive it. + const openSelectedModelSettings = useCallback( + async (ggufVariant: string | null, quantIsUserPicked = false) => { + if (!selectedModel) return; + // Share the sequence with openModelSettings: a pending variant lookup for + // another row must not land on top of this one. + const openSeq = ++settingsOpenSeq.current; + let variant = ggufVariant; + // The card derived this quant from the store's active variant, and nothing + // re-reads status while this window keeps focus, so an API-driven switch + // since the last focus event leaves it naming the model that switch + // displaced. A quant the user chose in the card is theirs and stands; a + // derived one defers to whatever a fresh read says is actually loaded. + if (!quantIsUserPicked) { + await refreshResidentModelStatus(); + if (settingsOpenSeq.current !== openSeq) return; + const settled = useChatRuntimeStore.getState(); + const settledCheckpoint = + settled.params.checkpoint && + !isExternalModelId(settled.params.checkpoint) + ? settled.params.checkpoint + : null; + // Every name this model answers to, as the row menu path matches them. + const aliases = [ + selectedModel.resource.runId, + selectedModel.resource.repoId, + selectedModel.resource.localPath, + ]; + if ( + settled.activeGgufVariant && + aliases.some((alias) => modelIdsMatch(alias, settledCheckpoint)) + ) { + variant = settled.activeGgufVariant; + } + } + // The card passes null while its variant lookup is pending or after it failed, + // so this needs the same guard openModelSettings applies: a model that needs a + // quant cannot be configured without one. + if (!variant && selectedModel.isGguf && selectedModel.requiresVariant) { + toast.error("Couldn't determine which quant to configure.", { + description: + "Settings for this model are per quant. Check the connection or the model's cache, then try again.", + }); + return; + } + const id = selectedModel.resource.runId; + const configId = modelConfigIdentity( + selectedModel.kind, + selectedModel.resource, + ); + const leaf = configId.split(/[\\/]/).filter(Boolean).pop() ?? configId; + setSettingsTarget({ + id, + configId, + displayName: variant ? `${leaf} ยท ${variant}` : leaf, + ggufVariant: variant, + isGguf: selectedModel.isGguf, + apiLoadable: + selectedModel.isGguf && + selectedModel.localSource !== LOCAL_MODEL_SOURCE.OLLAMA, + meta: { + source: "local", + isLora: selectedModel.modelFormat === "adapter", + ggufVariant: variant ?? undefined, + isGguf: selectedModel.isGguf, + isDownloaded: selectedModel.isDownloaded, + contextLength: null, + }, + }); + }, + [selectedModel, refreshResidentModelStatus], + ); + // Whether the settings page is open on the model that is actually loaded, so it + // can show the live launch config. A GGUF loaded from an inactive HF cache or + // straight off disk loads by path but is reported by its clean public id, so the + // row's path and its settings identity both have to be offered as aliases. + // A loose .gguf is one file, so its path already names the quant and its + // settings deliberately carry no variant. The loader still derives one from the + // filename and /status reports it, so requiring the two to agree would never + // hold and the page would withhold the live config from the resident file. + const settingsTargetIsStandaloneFile = + settingsTarget !== null && + settingsTarget.ggufVariant == null && + settingsTarget.id.toLowerCase().endsWith(".gguf"); + const settingsTargetIsResident = + settingsTarget !== null && + residentModelIdMatches( + activeCheckpoint, + settingsTarget.id, + settingsTarget.configId, + ) && + (settingsTargetIsStandaloneFile || + ggufVariantsMatch(activeGgufVariant, settingsTarget.ggufVariant)); const handleSearchHub = useCallback( (next: string) => { const trimmed = next.trim(); @@ -1290,6 +1637,7 @@ export function ModelsPage() { onTrain: handleTrain, onInventoryChange: refreshInventory, onSearchHub: handleSearchHub, + onOpenSettings: openSelectedModelSettings, }), [ handleLoad, @@ -1299,48 +1647,17 @@ export function ModelsPage() { handleTrain, handleSearchHub, refreshInventory, + openSelectedModelSettings, ], ); - const catalogState = useMemo( - () => { - const typeFilterActive = - !isDatasetMode && inventoryTypeFilter !== "all"; - return { - tab, - discoverRows: listRows, - cachedRows: filteredCachedRows, - localRows: filteredLocalRows, - selectedId, - isLoading, - downloadedReady, - inventoryError, - inventoryWarning, - query, - activeCheckpoint, - activeGgufVariant, - searchError, - online, - isDataset: isDatasetMode, - inventoryTokens, - scannedCount, - loadingIntentCount: discoverFetchIntent, - hasMore, - manualFetchAvailable: discoverManualFetchAvailable, - hasActiveFilters: - !isFeedMode && - (deferredFormatFilter !== "all" || - deferredCapabilityFilter !== "all" || - (tab === "downloaded" && typeFilterActive)), - typeFilterActive, - }; - }, - [ + const catalogState = useMemo(() => { + const typeFilterActive = !isDatasetMode && inventoryTypeFilter !== "all"; + return { tab, - isFeedMode, - listRows, - filteredCachedRows, - filteredLocalRows, + discoverRows: listRows, + cachedRows: filteredCachedRows, + localRows: filteredLocalRows, selectedId, isLoading, downloadedReady, @@ -1351,17 +1668,45 @@ export function ModelsPage() { activeGgufVariant, searchError, online, - isDatasetMode, + isDataset: isDatasetMode, inventoryTokens, scannedCount, - discoverFetchIntent, + loadingIntentCount: discoverFetchIntent, hasMore, - discoverManualFetchAvailable, - deferredFormatFilter, - deferredCapabilityFilter, - inventoryTypeFilter, - ], - ); + manualFetchAvailable: discoverManualFetchAvailable, + hasActiveFilters: + !isFeedMode && + (deferredFormatFilter !== "all" || + deferredCapabilityFilter !== "all" || + (tab === "downloaded" && typeFilterActive)), + typeFilterActive, + }; + }, [ + tab, + isFeedMode, + listRows, + filteredCachedRows, + filteredLocalRows, + selectedId, + isLoading, + downloadedReady, + inventoryError, + inventoryWarning, + query, + activeCheckpoint, + activeGgufVariant, + searchError, + online, + isDatasetMode, + inventoryTokens, + scannedCount, + discoverFetchIntent, + hasMore, + discoverManualFetchAvailable, + deferredFormatFilter, + deferredCapabilityFilter, + inventoryTypeFilter, + ]); const catalogPagination = useMemo( () => ({ @@ -1380,6 +1725,7 @@ export function ModelsPage() { onRetry: handleRetrySearch, onInventoryChange: refreshInventory, onSwitchDevice: handleSwitchDevice, + onOpenModelSettings: openModelSettings, }), [ handleSelect, @@ -1388,6 +1734,7 @@ export function ModelsPage() { handleRetrySearch, refreshInventory, handleSwitchDevice, + openModelSettings, ], ); @@ -1530,6 +1877,9 @@ export function ModelsPage() { const detailOpen = urlModel !== null; const splitMode = allModelsView === "split"; + // The catalog is unreachable under an opaque overlay: the detail view (full-page + // layout only, since split renders it alongside) or the settings page. + const catalogCovered = (detailOpen && !splitMode) || settingsTarget !== null; return (
@@ -1585,9 +1935,13 @@ export function ModelsPage() { splitMode ? "flex-1 lg:w-[460px] lg:max-w-[44%] lg:flex-none lg:shrink-0 lg:border-r lg:border-border/60" : "flex-1", - detailOpen && !splitMode && "pointer-events-none", + // The settings page is a full-bleed opaque overlay in every layout, + // so it always takes the catalog out of the tab order. Without this, + // tabbing out of the form walks into the virtualized rows behind it. + catalogCovered && "pointer-events-none", )} - aria-hidden={(detailOpen && !splitMode) || undefined} + aria-hidden={catalogCovered || undefined} + inert={catalogCovered || undefined} > +
+
) )} + + {/* Above the detail overlay (z-30): opening settings while a model + preview is open should show the settings, not stack behind it. */} + {settingsTarget && ( +
+ setSettingsTarget(null)} + onRun={runSettingsTarget} + compact={splitMode} + /> +
+ )}
:Q4_K_M` while the Chat model picker, the + * detail view's on-device card and the one-time backfill all use the bare path, + * leaving two surfaces editing two different configs for one file. + */ +export function settingsGgufVariantForRow( + row: CachedInventoryRow | LocalInventoryRow, +): string | null { + if (row.kind === "local" && row.path.toLowerCase().endsWith(".gguf")) { + return null; + } + return row.formatVariant?.trim() || null; +} diff --git a/studio/frontend/src/features/hub/lib/adopt-inference-status.ts b/studio/frontend/src/features/hub/lib/adopt-inference-status.ts new file mode 100644 index 0000000000..16c28f086f --- /dev/null +++ b/studio/frontend/src/features/hub/lib/adopt-inference-status.ts @@ -0,0 +1,107 @@ +// SPDX-License-Identifier: AGPL-3.0-only +// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 + +// Adopting the resident model into the chat runtime store from the Hub. +// +// Landing straight on /hub (a reload, or a deep link after an OpenAI-compatible +// auto-switch loaded something else) is the one entry point where nothing has +// applied /api/inference/status yet: useChatModelRuntime has no mount sync and +// the chat page is a different route. Pinning only the checkpoint leaves every +// other field useActiveModelConfig reads at its default, so the Hub's settings +// page passes those defaults on as the resident model's live config and Apply +// reloads the model with them. Adoption therefore has to apply the whole status, +// exactly as the chat runtime's refresh does. + +import { ggufVariantsMatch, modelIdsMatch } from "./model-identity.ts"; + +/** The parts of the chat runtime store adoption has to look at. */ +export interface ResidentAdoptionState { + /** ``params.checkpoint``. */ + checkpoint: string | null; + /** Whether that checkpoint names an external provider's model. */ + checkpointIsExternal: boolean; + /** ``activeGgufVariant``. */ + activeGgufVariant: string | null; + /** ``modelLoading``: a load this tab started still owns the store. */ + modelLoading: boolean; +} + +/** What ``/api/inference/status`` says is resident, already resolved. */ +export interface ResidentStatusFacts { + /** ``resolveInferenceCheckpointId(status)``; null when nothing is loaded. */ + checkpointId: string | null; + /** ``status.gguf_variant``. */ + ggufVariant: string | null; +} + +export interface ResidentAdoptionActions { + /** Re-pin ``params.checkpoint`` onto the resident model. */ + setCheckpoint: (checkpointId: string, ggufVariant: string | null) => void; + /** + * Drop a local checkpoint the server no longer has. + * + * Optional so a caller that only wants the pinning half can leave it out. + */ + clearCheckpoint?: () => void; + /** + * Apply the rest of the status. Receives the store values from BEFORE + * ``setCheckpoint`` ran, which is what applyActiveModelStatusToStore needs to + * tell a hydration from steady state. + */ + applyStatus: (previous: { + checkpoint: string | null; + ggufVariant: string | null; + }) => void; +} + +/** + * Adopt the resident model reported by ``/api/inference/status``. + * + * Returns whether anything was adopted. Never loads or unloads a model: it only + * mirrors what the server already has. + */ +export function adoptResidentModelStatus( + status: ResidentStatusFacts, + state: ResidentAdoptionState, + actions: ResidentAdoptionActions, +): boolean { + const { checkpointId } = status; + // An external-provider selection has no local mirror, so stamping the resident + // GGUF's capabilities and launch settings onto it would describe a model the + // user is not talking to. It also owns the store, so an empty status must not + // clear it: clearCheckpoint drops the persisted external pick as well. + if (state.checkpointIsExternal) { + return false; + } + // A load this tab started applies its own status when it settles, and the load + // dialog owns the params meanwhile. Adopting underneath it would fight both. + if (state.modelLoading) { + return false; + } + if (!checkpointId) { + // The server has nothing loaded, so neither should we. Unloading from another + // tab, from the monitor or over the API leaves this store pinned otherwise, + // and the settings page goes on treating that row as resident and seeding the + // editor from a launch config nothing is running. + if (state.checkpoint) { + actions.clearCheckpoint?.(); + return true; + } + return false; + } + const previous = { + checkpoint: state.checkpoint, + ggufVariant: state.activeGgufVariant, + }; + const alreadyPinned = + modelIdsMatch(previous.checkpoint, checkpointId) && + ggufVariantsMatch(previous.ggufVariant, status.ggufVariant); + if (!alreadyPinned) { + actions.setCheckpoint(checkpointId, status.ggufVariant); + } + // Unconditional, even when the checkpoint already matched: a persisted + // checkpoint rehydrates from localStorage on its own, with none of the fields + // that say how the model was actually launched. + actions.applyStatus(previous); + return true; +} diff --git a/studio/frontend/src/features/hub/lib/model-identity.ts b/studio/frontend/src/features/hub/lib/model-identity.ts index 488502ae28..890b321127 100644 --- a/studio/frontend/src/features/hub/lib/model-identity.ts +++ b/studio/frontend/src/features/hub/lib/model-identity.ts @@ -63,3 +63,138 @@ export function ggufVariantsMatch( normalizeGgufVariantIdentity(left) === normalizeGgufVariantIdentity(right) ); } + +// Mirrors core/inference/model_ids.py _looks_like_path. +const PUBLIC_ID_PATH_PREFIX_RE = /^(?:[/\\]|\.{1,2}[\\/]|~)/; +const GGUF_SUFFIX_RE = /\.gguf$/i; +const BACKSLASHES_RE = /\\/g; +const TRAILING_SLASHES_RE = /\/+$/; + +function looksLikeModelPath(identifier: string): boolean { + if (GGUF_SUFFIX_RE.test(identifier)) { + return true; + } + if (PUBLIC_ID_PATH_PREFIX_RE.test(identifier)) { + return true; + } + if (identifier.length >= 2 && identifier[1] === ":") { + return true; + } + return identifier.split("/").length - 1 >= 2 || identifier.includes("\\"); +} + +/** `.../models--org--name/snapshots/` -> `org/name`, else null. */ +function hfCacheRepoId(path: string): string | null { + const parts = path.replace(BACKSLASHES_RE, "/").split("/"); + for (let index = 0; index < parts.length; index += 1) { + const part = parts[index]; + if (part.startsWith("models--") && parts[index + 1] === "snapshots") { + return part.slice("models--".length).replaceAll("--", "/"); + } + } + return null; +} + +/** + * The clean id the backend reports for a model loaded by path. + * + * Mirrors ``public_model_id`` in studio/backend/core/inference/model_ids.py, which + * is what ``/api/inference/status`` puts in ``active_model``: an HF cache snapshot + * becomes its repo id and any other local GGUF becomes its filename stem. Repo ids + * and already-clean names come back unchanged. + */ +export function publicModelId(identifier: string): string { + const trimmed = identifier.trim(); + if (!(trimmed && looksLikeModelPath(trimmed))) { + return trimmed; + } + const repoId = hfCacheRepoId(trimmed); + if (repoId) { + return repoId; + } + const slashPath = trimmed + .replace(BACKSLASHES_RE, "/") + .replace(TRAILING_SLASHES_RE, ""); + const name = slashPath.slice(slashPath.lastIndexOf("/") + 1); + return name.replace(GGUF_SUFFIX_RE, "") || trimmed; +} + +/** + * Whether the model the backend reports as loaded is one of *candidates*. + * + * A GGUF loaded from an inactive HF cache is loaded by path, but a caller holding + * only the public id would read an exact comparison against the catalog row's path + * as "not loaded" and fall back to saved or default values instead of the live + * launch config. Candidates are compared literally first, then by the public id. + * + * That second pass only accepts an identity that can name one model: an HF cache + * snapshot collapses onto its repo id, which is globally unique, while every other + * path collapses onto a filename or directory stem that two models can share + * (`/models/alpha/model.gguf` and `/models/beta/model.gguf` are both "model"). + * Accepting a stem would mark the wrong row resident, seeding its editor with + * another model's live config and saving it under this model's key. Callers with + * the loadable identifier (`/status`'s `model_identifier`) pass it as the active + * id, and the literal pass answers exactly. + */ +export function residentModelIdMatches( + activeModelId: string | null | undefined, + ...candidates: (string | null | undefined)[] +): boolean { + if (candidates.some((candidate) => modelIdsMatch(activeModelId, candidate))) { + return true; + } + const active = activeModelId?.trim(); + // A path-shaped active id is the raw identifier, which the literal pass covered. + if (!active || looksLikeModelPath(active)) { + return false; + } + return candidates.some((candidate) => { + const trimmed = candidate?.trim(); + if (!trimmed) { + return false; + } + const publicId = publicModelId(trimmed); + // Unambiguous only when the collapse produced a namespaced repo id. + return publicId.includes("/") && modelIdsMatch(active, publicId); + }); +} + +// Ollama's blobs reach the picker through a ".studio_links"/"ollama_links" symlink +// directory. core/inference/local_model_resolver.py refuses to index anything under +// those (the scanner that creates them runs off the request path), so the API can +// never load one and mirroring its settings would advertise a load that cannot happen. +const OLLAMA_LINK_SEGMENTS = new Set([".studio_links", "ollama_links"]); + +export function isOllamaLinkPath(modelId: string | null | undefined): boolean { + if (!modelId) { + return false; + } + return modelId + .replace(BACKSLASHES_RE, "/") + .split("/") + .some((segment) => OLLAMA_LINK_SEGMENTS.has(segment)); +} + +// A drag-dropped or file-picked GGUF is the API's second unreachable identity. +// /api/inference/status reports model_identifier as null for a lease-backed load +// (routes/inference.py withholds the host path), so the checkpoint the browser +// keys settings by is the bare file name the backend echoes back. _build_index +// keys a standalone GGUF by its on-disk path and by its .gguf-stripped stem, so +// that name is never an index key and no auto-switch load can read an override +// stored under it. Anything the API can load is keyed by a path or a repo id, +// both of which carry a separator. +const NATIVE_FILE_LABEL_RE = /^[^/\\]+\.gguf$/i; + +export function isNativeFileLabel(modelId: string | null | undefined): boolean { + return modelId != null && NATIVE_FILE_LABEL_RE.test(modelId); +} + +// A scanned standalone .gguf, keyed by its on-disk path. Its settings identity +// carries no variant: it has no quant to choose between, while the loader and the +// inventory both label it from its filename, so adopting that label would key one +// file's config two ways. settings-identity.ts applies the same rule to a Hub row. +export function isStandaloneGgufPath( + modelId: string | null | undefined, +): boolean { + return modelId != null && modelId.toLowerCase().endsWith(".gguf"); +} diff --git a/studio/frontend/src/features/hub/lib/resident-status-refresh.ts b/studio/frontend/src/features/hub/lib/resident-status-refresh.ts new file mode 100644 index 0000000000..37ce910f10 --- /dev/null +++ b/studio/frontend/src/features/hub/lib/resident-status-refresh.ts @@ -0,0 +1,50 @@ +// SPDX-License-Identifier: AGPL-3.0-only +// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 + +// When the Hub has to re-read /api/inference/status. +// +// An OpenAI-compatible request can auto-switch the resident model at any moment, +// and nothing else on /hub reads status: the chat runtime hook has no mount sync +// and the chat page is a different route. A mount-only read therefore leaves +// every "loaded" marker -- and, worse, the settings page's live config -- pinned +// to whatever was resident when the Hub opened, so the newly loaded model's +// editor seeds from saved/default values and Apply reloads it with them. +// +// A background timer would keep asking a server that has usually not changed, so +// re-read only on the moments this tab could have missed a switch instead. + +/** The event targets to listen on; injected so this is testable off a browser. */ +export interface ResidentStatusRefreshTargets { + window: Pick; + document: Pick & { + readonly hidden: boolean; + }; +} + +function browserTargets(): ResidentStatusRefreshTargets { + return { window, document }; +} + +/** + * Call ``refresh`` whenever this tab comes back to the foreground: returning from + * the terminal or client that made the API call is exactly when what is resident + * may have moved. Returns the unsubscribe. + */ +export function subscribeResidentStatusRefresh( + refresh: () => void, + targets: ResidentStatusRefreshTargets = browserTargets(), +): () => void { + const onFocus = () => refresh(); + // Focus alone misses a tab that was merely backgrounded, and visibility alone + // misses a window that never went hidden; a redundant pair of reads is cheaper + // than a missed switch. + const onVisibility = () => { + if (!targets.document.hidden) refresh(); + }; + targets.window.addEventListener("focus", onFocus); + targets.document.addEventListener("visibilitychange", onVisibility); + return () => { + targets.window.removeEventListener("focus", onFocus); + targets.document.removeEventListener("visibilitychange", onVisibility); + }; +} diff --git a/studio/frontend/src/features/model-picker/api/migrate-model-overrides.ts b/studio/frontend/src/features/model-picker/api/migrate-model-overrides.ts new file mode 100644 index 0000000000..8479d95001 --- /dev/null +++ b/studio/frontend/src/features/model-picker/api/migrate-model-overrides.ts @@ -0,0 +1,174 @@ +// SPDX-License-Identifier: AGPL-3.0-only +// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 + +// One-time backfill of per-model settings into the server override map. +// +// Settings used to live only in this browser, so on upgrade the server knows +// nothing about models already configured: they still show as remembered while an +// API load uses app defaults, the exact bug the server-side map exists to fix. + +import { + isNativeFileLabel, + isOllamaLinkPath, + normalizeGgufVariantIdentity, + normalizeModelIdentity, + splitQuantSuffix, +} from "../model-config/model-identity"; +import { + isDefaultConfig, + listPerModelConfigs, +} from "../model-config/per-model-config"; +import type { ApiModelOverride } from "./model-overrides"; +import { + fetchModelOverrides, + modelOverrideKey, + putModelOverride, + toApiOverride, +} from "./model-overrides"; + +const DONE_FLAG = "unsloth_model_overrides_backfilled_v1"; + +function alreadyRan(): boolean { + try { + return window.localStorage.getItem(DONE_FLAG) === "1"; + } catch { + // Storage denied: treat as done rather than re-running on every mount. + return true; + } +} + +function markRan(): void { + try { + window.localStorage.setItem(DONE_FLAG, "1"); + } catch { + // Nothing to do; the backfill is idempotent anyway. + } +} + +/** + * A server key under the same identity this browser stores. + * + * `app_settings` has no schema version, so an old install holds keys like + * `Unsloth/Repo-GGUF:Q4_K_M` that the backend resolves to the same model as this + * browser's folded form; an exact lookup would call it missing and let the backfill + * overwrite it. The quant-aware split folds repo ids and leaves POSIX paths alone. + */ +function normalizedOverrideKey(key: string): string { + const split = splitQuantSuffix(key); + if (!split) { + return modelOverrideKey(normalizeModelIdentity(key)); + } + return modelOverrideKey( + normalizeModelIdentity(split[0]), + normalizeGgufVariantIdentity(split[1]), + ); +} + +/** + * The fields *config* would contribute that the stored entry does not hold. + * + * A malformed entry (nothing constrains what an older install wrote into + * app_settings) counts as holding nothing, so the migration still runs. + */ +function absentFields( + stored: ApiModelOverride, + config: Parameters[0], +): string[] { + const fields = Object.keys(toApiOverride(config)); + if (typeof stored !== "object" || stored === null) { + return fields; + } + return fields.filter((field) => !(field in stored)); +} + +/** + * Push local settings the server does not hold. Never deletes and never overwrites: + * a value already there is the newer authority, and losing a setting would be worse + * than leaving one unmigrated. + * + * Field by field, not entry by entry. The override map shipped before this browser + * mirror did, storing only llama_extra_args and max_seq_length, so an upgraded + * install can hold an entry for a model whose context, KV cache, speculative and GPU + * settings live only here. Treating the key as done would skip exactly the settings + * this migration exists to carry and then mark it complete. + */ +export async function backfillModelOverrides(): Promise { + if (alreadyRan()) { + return; + } + const local = listPerModelConfigs().filter( + // A quant means GGUF, the only thing API auto-switch resolves, so backfilling a + // safetensors config would claim behaviour that does not exist. A standalone + // .gguf has no quant to select between and is stored with a null variant, so it + // needs the extra test or its settings stay browser-only for good. An Ollama + // blob is GGUF but reached through a link dir the resolver skips, so it is not + // auto-switchable either, and a bare file name is a dropped/picked file's label, + // which the resolver never keys. + (entry) => + (entry.ggufVariant != null || + entry.modelId.toLowerCase().endsWith(".gguf")) && + !isOllamaLinkPath(entry.modelId) && + !isNativeFileLabel(entry.modelId) && + !isDefaultConfig(entry.config), + ); + if (local.length === 0) { + markRan(); + return; + } + + let existing: Awaited>; + try { + existing = await fetchModelOverrides(); + } catch { + // Offline or not authenticated yet. Leave the flag unset so the next start + // retries rather than skipping the migration forever. + return; + } + + const known = new Map(); + for (const [storedKey, storedEntry] of Object.entries(existing)) { + known.set(normalizedOverrideKey(storedKey), storedEntry); + } + + let failed = false; + for (const entry of local) { + // Folded here too: a v2 storage key holds the normalized identity, but the + // older `id::variant` keys this browser still reads hold the typed casing. + const key = normalizedOverrideKey( + modelOverrideKey(entry.modelId, entry.ggufVariant), + ); + // Re-read rather than trusting the snapshot from before the fetch: this write + // is queued behind the interactive one and commits last, so a save or forget + // during the round trip would be undone by it. + const current = listPerModelConfigs().find( + (candidate) => + normalizedOverrideKey( + modelOverrideKey(candidate.modelId, candidate.ggufVariant), + ) === key, + ); + if (!current || isDefaultConfig(current.config)) { + continue; + } + const stored = known.get(key); + // Nothing this browser could add, so skip the round trip entirely. + if (stored && absentFields(stored, current.config).length === 0) { + continue; + } + try { + // Fills the gaps only. `known` is a snapshot from before this loop started, so + // a save by another tab during the pass is invisible here; the server reads and + // writes together rather than this re-fetching once per model. + await putModelOverride( + current.modelId, + current.ggufVariant, + current.config, + { fillAbsentFields: true }, + ); + } catch { + failed = true; + } + } + if (!failed) { + markRan(); + } +} diff --git a/studio/frontend/src/features/model-picker/api/model-overrides.ts b/studio/frontend/src/features/model-picker/api/model-overrides.ts new file mode 100644 index 0000000000..f7535718d7 --- /dev/null +++ b/studio/frontend/src/features/model-picker/api/model-overrides.ts @@ -0,0 +1,252 @@ +// SPDX-License-Identifier: AGPL-3.0-only +// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 + +// Server-side mirror of the per-model config. +// +// The config in ../model-config/per-model-config.ts lives in browser localStorage, +// so it only applied to loads the browser made, and an API auto-switch load (no +// browser in the loop) came up with none of the user's settings. Mirroring every +// save to the backend's override map closes that gap: routes/inference.py reads it +// and rebuilds the same LoadRequest the picker would have sent. + +import { authFetch } from "@/features/auth"; +import { readFastApiError } from "@/lib/format-fastapi-error"; +import { + normalizeGgufVariantIdentity, + normalizeModelIdentity, +} from "../model-config/model-identity"; +import type { PerModelConfig } from "../model-config/per-model-config"; + +const OVERRIDES_URL = "/api/settings/openai-auto-switch/overrides"; + +/** One model's stored launch config, as the backend persists it. */ +export interface ApiModelOverride { + // biome-ignore lint/style/useNamingConvention: API schema + llama_extra_args?: string[]; + // biome-ignore lint/style/useNamingConvention: API schema + max_seq_length?: number; + // biome-ignore lint/style/useNamingConvention: API schema + custom_context_length?: number; + // biome-ignore lint/style/useNamingConvention: API schema + kv_cache_dtype?: string; + // biome-ignore lint/style/useNamingConvention: API schema + speculative_type?: string; + // biome-ignore lint/style/useNamingConvention: API schema + spec_draft_n_max?: number; + // biome-ignore lint/style/useNamingConvention: API schema + n_parallel?: number; + // biome-ignore lint/style/useNamingConvention: API schema + tensor_parallel?: boolean; + // biome-ignore lint/style/useNamingConvention: API schema + chat_template_override?: string; + // biome-ignore lint/style/useNamingConvention: API schema + gpu_memory_mode?: "auto" | "manual"; + // biome-ignore lint/style/useNamingConvention: API schema + gpu_layers?: number; + // biome-ignore lint/style/useNamingConvention: API schema + n_cpu_moe?: number; + // biome-ignore lint/style/useNamingConvention: API schema + gpu_ids?: number[]; +} + +export type ApiModelOverrides = Record; + +/** + * The key one model's config is stored under: the `repo:VARIANT` form an OpenAI + * request names a quant by, so two quants of one repo keep separate configs and the + * backend matches the requested name directly. Bare id when there is no variant. + */ +export function modelOverrideKey( + modelId: string, + ggufVariant?: string | null, +): string { + return ggufVariant ? `${modelId}:${ggufVariant}` : modelId; +} + +export async function fetchModelOverrides(): Promise { + const res = await authFetch(OVERRIDES_URL); + if (!res.ok) { + throw new Error( + await readFastApiError(res, "Failed to load saved model settings"), + ); + } + const body = (await res.json()) as { overrides?: ApiModelOverrides }; + return body.overrides ?? {}; +} + +/** + * Translate the UI's per-model config into the backend's schema. + * + * Only fields the user set are sent: the backend reads an absent field as "app + * default", so nulls would pin defaults and stop the model following later global + * changes. A `null` config means "no saved settings", which clears the entry. + * + * Exported so the one-time backfill can ask what this config would contribute and + * compare it against the entry already on the server, field by field. + */ +export function toApiOverride(config: PerModelConfig | null): ApiModelOverride { + if (!config) { + return {}; + } + const payload: ApiModelOverride = {}; + if (config.maxSeqLength && config.maxSeqLength > 0) { + payload.max_seq_length = config.maxSeqLength; + } + if (config.customContextLength && config.customContextLength > 0) { + payload.custom_context_length = config.customContextLength; + } + if (config.kvCacheDtype) { + payload.kv_cache_dtype = config.kvCacheDtype; + } + if (config.speculativeType) { + payload.speculative_type = config.speculativeType; + } + if (config.specDraftNMax && config.specDraftNMax > 0) { + payload.spec_draft_n_max = config.specDraftNMax; + } + // Blank follows the server-wide --parallel default, which is the app default here. + if (config.nParallel && config.nParallel > 0) { + payload.n_parallel = config.nParallel; + } + if (config.tensorParallel) { + payload.tensor_parallel = true; + } + if (config.chatTemplateOverride?.trim()) { + payload.chat_template_override = config.chatTemplateOverride; + } + // Only "manual" is a real override; "auto" is the follow-the-global default. + if (config.gpuMemoryMode === "manual") { + payload.gpu_memory_mode = "manual"; + } + // gpuLayers < 0 is Auto, which is also the default. + if (typeof config.gpuLayers === "number" && config.gpuLayers >= 0) { + payload.gpu_layers = config.gpuLayers; + } + if (typeof config.nCpuMoe === "number" && config.nCpuMoe > 0) { + payload.n_cpu_moe = config.nCpuMoe; + } + if (config.selectedGpuIds && config.selectedGpuIds.length > 0) { + payload.gpu_ids = config.selectedGpuIds; + } + return payload; +} + +// One in-flight write per model, so writes for a model commit in issue order. +// Otherwise saving twice quickly, or saving during the one-time backfill, races: +// the older response can land last and resurrect the entry the newer one meant to +// replace. Different models still overlap. +const writesByKey = new Map>(); + +export interface PutModelOverrideOptions { + /** + * Fill in only what is missing: every value already on the server stays as it is. + * + * The one-time backfill reads the map once and then writes each model in turn, so + * another tab saving during that pass would be overwritten by this browser's older + * localStorage copy. The server reads and writes under one transaction, which + * closes the window without a round trip per model. Field level, so an entry an + * older release stored with only its two fields still gains the browser-only ones + * without the server losing anything it holds. + */ + fillAbsentFields?: boolean; + /** + * Clear the fields this UI mirrors but leave the server's own launch flags alone. + * + * Dropping a local entry to stay inside the storage budget is not the user asking + * to forget that model, so it must not take `llama_extra_args` the settings API + * set and the settings page can neither show nor restore. The route still drops + * the row outright once nothing is left in it. + */ + keepLaunchFlags?: boolean; +} + +export async function putModelOverride( + modelId: string, + ggufVariant: string | null | undefined, + config: PerModelConfig | null, + options?: PutModelOverrideOptions, +): Promise { + // Keyed by the folded identity, not the literal spelling: the backfill sends a + // legacy casing and a UI save the normalized one, and the backend resolves both + // to one row, so raw strings would open two queues and race again. + const key = modelOverrideKey( + normalizeModelIdentity(modelId), + normalizeGgufVariantIdentity(ggufVariant), + ); + // Chain on the settled tail: a failed write must not cancel the next one. + const previous = writesByKey.get(key) ?? Promise.resolve(); + const write = previous + .catch(() => {}) + .then(() => sendModelOverride(modelId, ggufVariant, config, options)); + writesByKey.set(key, write); + try { + await write; + } finally { + // Only the last writer clears the slot, so a queue still building keeps order. + if (writesByKey.get(key) === write) { + writesByKey.delete(key); + } + } +} + +async function sendModelOverride( + modelId: string, + ggufVariant: string | null | undefined, + config: PerModelConfig | null, + options?: PutModelOverrideOptions, +): Promise { + const res = await authFetch(OVERRIDES_URL, { + method: "PUT", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + // biome-ignore lint/style/useNamingConvention: API schema + model_id: modelOverrideKey(modelId, ggufVariant), + // Only sent when set, so an older backend that does not know the field is + // not handed an unexpected key by every ordinary save. + ...(options?.fillAbsentFields + ? // biome-ignore lint/style/useNamingConvention: API schema + { fill_absent_fields: true } + : {}), + // Say which operation this is: an all-default save carries no fields, which is + // shape-identical to "forget this model", and guessing wrong wipes launch flags + // the UI cannot show or restore. + remove: config === null && !options?.keepLaunchFlags, + // Launch flags have no UI control, so the backend preserves them when omitted. + // Forgetting means forgetting all of it, so that path sends an explicit []. + ...(config === null && !options?.keepLaunchFlags + ? // biome-ignore lint/style/useNamingConvention: API schema + { llama_extra_args: [] } + : {}), + ...toApiOverride(config), + }), + }); + if (!res.ok) { + throw new Error( + await readFastApiError(res, "Failed to save model settings for the API"), + ); + } +} + +/** + * Mirror a per-model config save to the backend without blocking the UI. + * + * Best-effort: the localStorage write is this browser's source of truth and has + * already happened, so a failed sync must not fail the save or interrupt a load. + * Logged rather than toasted: an API load of this model just falls back to app + * defaults until the next successful save. + */ +export function syncModelOverride( + modelId: string, + ggufVariant: string | null | undefined, + config: PerModelConfig | null, + options?: PutModelOverrideOptions, +): void { + void putModelOverride(modelId, ggufVariant, config, options).catch( + (error: unknown) => { + console.warn( + "Failed to mirror model settings to the server; an API load of this model will use defaults.", + error, + ); + }, + ); +} diff --git a/studio/frontend/src/features/model-picker/components/model-config-page.tsx b/studio/frontend/src/features/model-picker/components/model-config-page.tsx index a753e9016e..e2b5f270d9 100644 --- a/studio/frontend/src/features/model-picker/components/model-config-page.tsx +++ b/studio/frontend/src/features/model-picker/components/model-config-page.tsx @@ -32,6 +32,7 @@ import { useRef, useState, } from "react"; +import { syncModelOverride } from "../api/model-overrides"; import { useDefaultChatTemplate, useModelMaxPositionEmbeddings, @@ -54,6 +55,7 @@ import { floorMaxSeqLength, isDefaultConfig, normalizeMaxSeqLength, + normalizePerModelConfig, resolveInitialConfig, savePerModelConfig, } from "../model-config/per-model-config"; @@ -75,14 +77,16 @@ const SELECT_TRIGGER_CLASS = `grid h-8 min-w-0 grid-cols-[minmax(0,1fr)_auto] it const NUMBER_INPUT_CLASS = `h-8 w-[92px] ${CONTROL_SURFACE} pl-3 pr-2 py-0 text-right text-ui-13 font-medium text-nav-fg outline-none focus-visible:ring-0`; const KV_CACHE_DTYPE_DEFAULT = "f16"; -const SPECULATIVE_TYPE_LABELS: Record<(typeof SPECULATIVE_TYPES)[number], string> = - { - auto: "Auto", - mtp: "MTP", - ngram: "Ngram", - "mtp+ngram": "MTP+Ngram", - off: "Off", - }; +const SPECULATIVE_TYPE_LABELS: Record< + (typeof SPECULATIVE_TYPES)[number], + string +> = { + auto: "Auto", + mtp: "MTP", + ngram: "Ngram", + "mtp+ngram": "MTP+Ngram", + off: "Off", +}; function hasNonDefaultAdvanced(config: PerModelConfig): boolean { return ( @@ -354,8 +358,8 @@ function GpuMemorySettings({ info={ <> Layers to keep on the GPU (--gpu-layers); the rest run on CPU. - Auto lets llama.cpp size the split (and the context) to fit VRAM. - At the maximum, the whole model is on the GPU. + Auto lets llama.cpp size the split (and the context) to fit + VRAM. At the maximum, the whole model is on the GPU. } /> @@ -619,6 +623,11 @@ interface ModelConfigPageProps { loadedContextLength?: number | null; initialConfig?: PerModelConfig | null; variant?: "page" | "sidebar"; + /** + * Page variant only: render the built-in "Run settings" title block. A host that + * already shows the model name as its page heading turns this off. + */ + showHeader?: boolean; } export function ModelConfigPage({ @@ -629,6 +638,7 @@ export function ModelConfigPage({ loadedContextLength = null, initialConfig = null, variant = "page", + showHeader = true, }: ModelConfigPageProps) { const rememberId = useId(); const isActiveModel = loadedConfig != null; @@ -642,8 +652,12 @@ export function ModelConfigPage({ const loadedMaxContextLength = useChatRuntimeStore( (s) => s.ggufMaxContextLength, ); + // What the settings are stored under, which is not always what loads (see + // ModelPickTarget.configId). Every read, write and mirror uses it; the probes + // keep target.id, since they have to open the model. + const configId = target.configId ?? target.id; const resolveInitial = () => { - const resolved = resolveInitialConfig(target.id, target.ggufVariant); + const resolved = resolveInitialConfig(configId, target.ggufVariant); if (loadedConfig) { return { config: loadedConfig, remembered: resolved.remembered }; } @@ -771,8 +785,7 @@ export function ModelConfigPage({ ), maxContext, ); - const setContextLength = (v: number) => - update({ customContextLength: v }); + const setContextLength = (v: number) => update({ customContextLength: v }); const baseline = loadedConfig ?? DEFAULT_PER_MODEL_CONFIG; const atBaseline = perModelConfigsEqual(config, baseline); // An explicit customContextLength equal to the native ceiling is still an @@ -900,16 +913,56 @@ export function ModelConfigPage({ const effectiveAtBaseline = perModelConfigsEqual(effectiveConfig, baseline); const effectivePersistenceOnly = isActiveModel && effectiveAtBaseline && rememberChanged; - const defaultConfig = isDefaultConfig(effectiveRuntimeConfig); + // Judge what storage keeps: savePerModelConfig normalizes first, and the runtime's + // Speculative Decoding "auto" canonicalizes to null, so judging the raw object + // reported saved while the write dropped it, and mirrored an override nothing held. + const normalizedRuntimeConfig = normalizePerModelConfig( + effectiveRuntimeConfig, + ); + const defaultConfig = isDefaultConfig(normalizedRuntimeConfig); let saveFailed = false; + const evicted: { modelId: string; ggufVariant: string | null }[] = []; if (remember) { saveFailed = !savePerModelConfig( - target.id, + configId, target.ggufVariant, - effectiveRuntimeConfig, + normalizedRuntimeConfig, + evicted, ); } else { - saveFailed = !deletePerModelConfig(target.id, target.ggufVariant); + saveFailed = !deletePerModelConfig(configId, target.ggufVariant); + } + // Mirror to the server so an API request that loads this model gets these exact + // settings, not app defaults. Best-effort and non-blocking: the localStorage + // write above already governs this browser, and forgetting clears both. + // + // Skipped when the local write failed (quota, a future-schema entry), or the + // two would permanently disagree with no way to tell which the next load used. + // Gated on auto-switch reach, not just GGUF-ness: the resolver indexes GGUFs and + // skips Ollama, so mirroring either would advertise a load that cannot happen. + // A native-path lease is the same case: /status withholds model_identifier for a + // dropped or file-picked GGUF, so this id is only the file's display name, which + // the resolver never keys, and reopening the file needs a lease the API cannot + // mint. The token, not the name, decides: the fallback label carries no suffix. + if ( + !saveFailed && + (target.apiLoadable ?? target.isGguf) && + !nativePathToken + ) { + syncModelOverride( + configId, + target.ggufVariant, + remember ? normalizedRuntimeConfig : null, + ); + } + // Saving can push the local map over budget and drop other models, whose server + // entries would keep being applied with nothing in the UI able to forget them. + // Not a Forget though: the user never asked to drop these, so only the mirrored + // fields go and launch flags set through the API stay. + for (const dropped of evicted) { + syncModelOverride(dropped.modelId, dropped.ggufVariant, null, { + keepLaunchFlags: true, + }); } if (effectivePersistenceOnly) { if (saveFailed) { @@ -939,7 +992,7 @@ export function ModelConfigPage({ return (
- {variant === "page" && ( + {variant === "page" && showHeader && (
{onBack && ( - - {expanded ? ( -
-
-
-
- Prompt - {entry.prompt_truncated && !detail ? Preview : null} -
-
-                {loading && !detail ? "Loading..." : prompt || "No prompt text"}
-              
-
-
-
- Reply - {entry.reply_truncated && !detail ? Preview : null} -
-
-                {loading && !detail ? "Loading..." : reply}
-              
-
-
- -
- {formatTokens(entry)} - {entry.context_length ? ( - <> / {entry.context_length.toLocaleString()} context - ) : null} - -
-
- ) : null} - - ); -} - -export function ApiMonitorConsole(): ReactElement { - const [data, setData] = useState(null); - const [error, setError] = useState(null); - const [refreshing, setRefreshing] = useState(false); - const [unloading, setUnloading] = useState(false); - const [expandedIds, setExpandedIds] = useState>(() => new Set()); - const [details, setDetails] = useState>({}); - const [loadingDetails, setLoadingDetails] = useState>( - () => new Set(), - ); - const loadingDetailsRef = useRef>(new Set()); - const detailsRef = useRef>({}); - - const loadMonitor = useCallback(async (): Promise => { - setRefreshing(true); - try { - setData(await getApiMonitor()); - setError(null); - } catch (err: unknown) { - setError(err instanceof Error ? err.message : "Monitor unavailable"); - } finally { - setRefreshing(false); - } - }, []); - - // /unload matches on the internal id, which the monitor omits, so read it from status. - const unloadActiveModel = useCallback(async (): Promise => { - setUnloading(true); - try { - const status = await getInferenceStatus(); - const checkpoint = resolveInferenceCheckpointId(status); - if (!checkpoint) { - setError(null); - return; - } - await unloadModel({ model_path: checkpoint }); - // Same as the chat eject flow: the store still holds the freed checkpoint. - useChatRuntimeStore.getState().clearCheckpoint(); - setError(null); - await loadMonitor(); - } catch (err: unknown) { - setError(err instanceof Error ? err.message : "Failed to unload the model"); - } finally { - setUnloading(false); - } - }, [loadMonitor]); - - useEffect(() => { - let cancelled = false; - let timer: number | undefined; - - function schedule(): void { - timer = window.setTimeout(poll, 1500); - } - - function poll(): void { - getApiMonitor() - .then((next) => { - if (cancelled) { - return; - } - setData(next); - setError(null); - }) - .catch((err: unknown) => { - if (cancelled) { - return; - } - setError(err instanceof Error ? err.message : "Monitor unavailable"); - }) - .finally(() => { - if (!cancelled) { - schedule(); - } - }); - } - - poll(); - return () => { - cancelled = true; - if (timer !== undefined) { - window.clearTimeout(timer); - } - }; - }, []); - - const statusLabel = data?.status ?? "idle"; - const hasActive = (data?.active_requests ?? 0) > 0; - const entries = useMemo(() => data?.entries ?? [], [data]); - - // Page 1 tracks the live list; paging back freezes the id order so history holds still. - const [page, setPage] = useState(0); - const [frozenIds, setFrozenIds] = useState(null); - const byId = useMemo( - () => new Map(entries.map((entry) => [entry.id, entry])), - [entries], - ); - const ordered = useMemo(() => { - if (frozenIds === null) { - return entries; - } - return frozenIds.flatMap((id) => { - const entry = byId.get(id); - return entry ? [entry] : []; - }); - }, [byId, entries, frozenIds]); - const pageCount = Math.max(1, Math.ceil(ordered.length / PAGE_SIZE)); - const pageIndex = Math.min(page, pageCount - 1); - const visible = ordered.slice( - pageIndex * PAGE_SIZE, - pageIndex * PAGE_SIZE + PAGE_SIZE, - ); - const newerCount = - frozenIds === null - ? 0 - : entries.filter((entry) => !frozenIds.includes(entry.id)).length; - - const goToPage = useCallback( - (next: number): void => { - if (next <= 0) { - setFrozenIds(null); - setPage(0); - return; - } - // Freeze on the way off page 1 so the history under the cursor holds still. - setFrozenIds((prev) => prev ?? entries.map((entry) => entry.id)); - setPage(next); - }, - [entries], - ); - - const loadDetail = useCallback( - (id: string): void => { - if (loadingDetailsRef.current.has(id)) { - return; - } - loadingDetailsRef.current.add(id); - setLoadingDetails((prev) => new Set(prev).add(id)); - getApiMonitorEntry(id) - .then((entry) => { - setDetails((prev) => { - const next = { ...prev, [id]: entry }; - detailsRef.current = next; - return next; - }); - }) - .catch(() => { - setDetails((prev) => { - const next = { ...prev }; - delete next[id]; - detailsRef.current = next; - return next; - }); - }) - .finally(() => { - loadingDetailsRef.current.delete(id); - setLoadingDetails((prev) => { - const next = new Set(prev); - next.delete(id); - return next; - }); - }); - }, - [], - ); - - const toggleEntry = useCallback( - (entry: ApiMonitorEntry): void => { - setExpandedIds((prev) => { - const next = new Set(prev); - if (next.has(entry.id)) { - next.delete(entry.id); - } else { - next.add(entry.id); - loadDetail(entry.id); - } - return next; - }); - }, - [loadDetail], - ); - - useEffect(() => { - // Only rows on screen: an expanded row on another page would keep polling. - for (const entry of visible) { - if (isLifecycle(entry) || !expandedIds.has(entry.id)) { - continue; - } - const cached = detailsRef.current[entry.id]; - if (!cached || cached.status !== entry.status || entry.status === "running") { - loadDetail(entry.id); - } - } - }, [visible, expandedIds, loadDetail]); - - return ( -
-
-
-
- - {hasActive ? ( - - ) : null} -
-
-

- API monitor -

-

- {data?.active_model ?? "No model loaded"} -

-
-
-
-
- {statusLabel} -
- {/* Always rendered, disabled when idle: the only manual release must stay visible. */} - - -
-
- -
- - {(data?.active_requests ?? 0).toLocaleString()} active /{" "} - {entries.length.toLocaleString()} recent - - {data?.context_length ? ( - {data.context_length.toLocaleString()} context - ) : null} -
- -
- {error ? ( -
- {error} -
- ) : entries.length === 0 ? ( -
- No API traffic yet -
- ) : ( -
- {visible.map((entry) => - isLifecycle(entry) ? ( - - ) : ( - toggleEntry(entry)} - /> - ), - )} -
- )} -
- - {/* Also while frozen: retention can shrink that list below one page, and hiding the - pager would strand the console on a stale snapshot. */} - {ordered.length > PAGE_SIZE || frozenIds !== null ? ( -
- - Page {pageIndex + 1} of {pageCount} - {newerCount > 0 ? ` (${newerCount.toLocaleString()} new)` : ""} - -
- - -
-
- ) : null} -
- ); -} diff --git a/studio/frontend/src/features/settings/components/monitor-link.tsx b/studio/frontend/src/features/settings/components/monitor-link.tsx new file mode 100644 index 0000000000..a2849ac69f --- /dev/null +++ b/studio/frontend/src/features/settings/components/monitor-link.tsx @@ -0,0 +1,99 @@ +// SPDX-License-Identifier: AGPL-3.0-only +// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 + +// The monitor moved onto its own page, normally reached from the floating +// panel; this card is the way in from Settings. + +import { Switch } from "@/components/ui/switch"; +// Direct path, not the barrel: the barrel re-exports the page, which would pull +// it into this chunk and defeat the route's dynamic import. +import { useApiMonitorOverlayStore } from "@/features/api-monitor/overlay-store"; +import { getApiMonitor } from "@/features/chat/api/chat-api"; +import type { ApiMonitorResponse } from "@/features/chat/types/api"; +import { cn } from "@/lib/utils"; +import { ActivityIcon, ArrowRight02Icon } from "@hugeicons/core-free-icons"; +import { HugeiconsIcon } from "@hugeicons/react"; +import { useNavigate } from "@tanstack/react-router"; +import { type ReactElement, useEffect, useState } from "react"; +import { useSettingsDialogStore } from "../stores/settings-dialog-store"; + +export function MonitorLink(): ReactElement { + const navigate = useNavigate(); + const [data, setData] = useState(null); + const autoOpen = useApiMonitorOverlayStore((s) => s.autoOpen); + const setAutoOpen = useApiMonitorOverlayStore((s) => s.setAutoOpen); + + // One snapshot, not a poll: the live view is the monitor page. + useEffect(() => { + let cancelled = false; + void getApiMonitor() + .then((next) => { + if (!cancelled) setData(next); + }) + .catch(() => undefined); + return () => { + cancelled = true; + }; + }, []); + + const active = data?.active_requests ?? 0; + const recent = data?.entries.length ?? 0; + + return ( +
+ + + {/* Where the panel's own "stop opening this" gets turned back on. */} +
+ + + Show the floating monitor automatically + + + Opens a small panel when API traffic arrives. + + + +
+
+ ); +} diff --git a/studio/frontend/src/features/settings/tabs/api-keys-tab.tsx b/studio/frontend/src/features/settings/tabs/api-keys-tab.tsx index 7a1fdc1fb2..b2ca822bb0 100644 --- a/studio/frontend/src/features/settings/tabs/api-keys-tab.tsx +++ b/studio/frontend/src/features/settings/tabs/api-keys-tab.tsx @@ -14,7 +14,7 @@ import { translate, useT } from "@/i18n"; import { AnimatePresence, motion, useReducedMotion } from "motion/react"; import { useCallback, useEffect, useState } from "react"; import { fetchApiKeys, revokeApiKey, type ApiKey } from "../api/api-keys"; -import { ApiMonitorConsole } from "../components/api-monitor-console"; +import { MonitorLink } from "../components/monitor-link"; import { ApiKeyRow } from "../components/api-key-row"; import { CreateKeyForm } from "../components/create-key-form"; import { ModelAutoSwitchSection } from "../components/model-auto-switch-section"; @@ -168,9 +168,9 @@ export function ApiKeysTab() { )} - + - + diff --git a/studio/frontend/tests/api-monitor-new-traffic.test.ts b/studio/frontend/tests/api-monitor-new-traffic.test.ts new file mode 100644 index 0000000000..fba8a002ff --- /dev/null +++ b/studio/frontend/tests/api-monitor-new-traffic.test.ts @@ -0,0 +1,187 @@ +// SPDX-License-Identifier: AGPL-3.0-only +// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 + +import assert from "node:assert/strict"; +import test from "node:test"; + +// The overlay is a .tsx pulling in motion, hugeicons and the router, so it cannot +// be imported here. Its new-traffic decision lives in a plain module for exactly +// that reason, and this drives the real one the overlay calls. +import { + type WatchedEntry, + type WatchedResponse, + createWatch, + observeResponse, + rearmWatch, + startWatching, +} from "../src/features/api-monitor/new-traffic.ts"; + +// The server's clock. Entry timestamps are its time.time(), so the tests keep them +// in those units and never mix in a browser instant. +const SERVER_NOW = 1_000_000; +// performance.now() when the poll stood up. +const WATCH_AT = 1_000; + +function entry( + id: string, + status: WatchedEntry["status"], + startedAt: number, + viaApiKey = true, +): WatchedEntry { + // biome-ignore lint/style/useNamingConvention: API schema + return { id, status, via_api_key: viaApiKey, started_at: startedAt }; +} + +function snapshot( + entries: WatchedEntry[], + serverTime: number | null = SERVER_NOW, +): WatchedResponse { + // biome-ignore lint/style/useNamingConvention: API schema + return { entries, server_time: serverTime }; +} + +function watchFrom(startedAtMs: number) { + const watch = createWatch(0); + startWatching(watch, startedAtMs); + return watch; +} + +test("a call that finished before the first snapshot arrived is new traffic", () => { + // The tab was hidden for 4s after the poll stood up, so poll() issued no fetch. + // The user's first curl ran 2s into that gap and was already done when the + // snapshot finally landed. Terminal, but not history. + const opened = observeResponse( + watchFrom(WATCH_AT), + snapshot([entry("apireq_new", "completed", SERVER_NOW - 2)]), + WATCH_AT + 4_000, + ); + assert.equal(opened, true); +}); + +test("traffic from before the watch began stays history", () => { + const opened = observeResponse( + watchFrom(WATCH_AT), + snapshot([entry("apireq_old", "completed", SERVER_NOW - 90)]), + WATCH_AT + 4_000, + ); + assert.equal(opened, false); +}); + +test("a request still running at the first snapshot is live traffic", () => { + const opened = observeResponse( + watchFrom(WATCH_AT), + snapshot([entry("apireq_live", "running", SERVER_NOW - 90)]), + WATCH_AT + 10, + ); + assert.equal(opened, true); +}); + +test("a fresh id in a later snapshot opens the panel", () => { + const watch = watchFrom(WATCH_AT); + const backlog = [entry("apireq_old", "completed", SERVER_NOW - 90)]; + assert.equal(observeResponse(watch, snapshot(backlog), WATCH_AT + 10), false); + const opened = observeResponse( + watch, + snapshot( + [entry("apireq_next", "completed", SERVER_NOW + 4), ...backlog], + SERVER_NOW + 5, + ), + WATCH_AT + 5_010, + ); + assert.equal(opened, true); +}); + +test("Studio's own chat never opens the panel", () => { + const opened = observeResponse( + watchFrom(WATCH_AT), + snapshot([entry("uireq", "completed", SERVER_NOW - 2, false)]), + WATCH_AT + 4_000, + ); + assert.equal(opened, false); +}); + +test("a backend with no clock field keeps the old terminal-is-history seed", () => { + const opened = observeResponse( + watchFrom(WATCH_AT), + snapshot([entry("apireq_new", "completed", SERVER_NOW - 2)], null), + WATCH_AT + 4_000, + ); + assert.equal(opened, false); +}); + +test("a browser clock disagreeing with the server's does not replay the backlog", () => { + // The cutoff is the server's own clock minus a browser DURATION, never minus a + // browser timestamp, so a browser whose wall clock is minutes off still dates + // the backlog correctly. + const opened = observeResponse( + watchFrom(WATCH_AT), + snapshot([ + entry("apireq_a", "completed", SERVER_NOW - 300), + entry("apireq_b", "completed", SERVER_NOW - 120), + ]), + WATCH_AT + 20, + ); + assert.equal(opened, false); +}); + +test("coming back from the full page does not replay the rows it showed", () => { + const watch = watchFrom(WATCH_AT); + const backlog = [entry("apireq_old", "completed", SERVER_NOW - 90)]; + observeResponse(watch, snapshot(backlog), WATCH_AT + 10); + // 60s on /api-monitor reading those rows, then back to chat. + rearmWatch(watch); + startWatching(watch, WATCH_AT + 60_000); + const opened = observeResponse( + watch, + snapshot(backlog, SERVER_NOW + 60), + WATCH_AT + 60_010, + ); + assert.equal(opened, false); +}); + +test("a request still running when the full page is left does not reopen the overlay", () => { + // The user opened /api-monitor to watch a long generation, then went back to + // chat while it was still running. That row was on screen the whole time. + const watch = watchFrom(WATCH_AT); + const live = entry("apireq_live", "running", SERVER_NOW - 5); + observeResponse(watch, snapshot([live]), WATCH_AT + 10); + rearmWatch(watch); + startWatching(watch, WATCH_AT + 60_000); + const opened = observeResponse( + watch, + snapshot([live], SERVER_NOW + 60), + WATCH_AT + 60_010, + ); + assert.equal(opened, false); +}); + +test("a rearm writes off only the snapshot it comes back to", () => { + // The write-off is one seed, not a mode: a call that arrives after the return + // is still new traffic. + const watch = watchFrom(WATCH_AT); + const live = entry("apireq_live", "running", SERVER_NOW - 5); + observeResponse(watch, snapshot([live]), WATCH_AT + 10); + rearmWatch(watch); + startWatching(watch, WATCH_AT + 60_000); + observeResponse(watch, snapshot([live], SERVER_NOW + 60), WATCH_AT + 60_010); + const opened = observeResponse( + watch, + snapshot( + [entry("apireq_next", "running", SERVER_NOW + 61), live], + SERVER_NOW + 62, + ), + WATCH_AT + 62_010, + ); + assert.equal(opened, true); +}); + +test("a fresh watch still reports a request that was already running", () => { + // The rearm write-off must not become the default seed for a session that + // never saw the full page. + const opened = observeResponse( + watchFrom(WATCH_AT), + snapshot([entry("apireq_live", "running", SERVER_NOW - 90)]), + WATCH_AT + 10, + ); + assert.equal(opened, true); +}); diff --git a/studio/frontend/tests/bundler-resolver.mjs b/studio/frontend/tests/bundler-resolver.mjs new file mode 100644 index 0000000000..7987b8544c --- /dev/null +++ b/studio/frontend/tests/bundler-resolver.mjs @@ -0,0 +1,36 @@ +// SPDX-License-Identifier: AGPL-3.0-only +// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 + +// The two resolution rules vite and tsconfig's "bundler" mode give the app that +// bare node does not have: the "@/*" path alias, and a relative import written +// without its extension. Register this from a test that needs to import a src +// module using either, which is otherwise unreachable from the test runner. +import { existsSync } from "node:fs"; +import { fileURLToPath, pathToFileURL } from "node:url"; + +const SRC = fileURLToPath(new URL("../src/", import.meta.url)); + +function firstExisting(base) { + for (const candidate of [`${base}.ts`, `${base}/index.ts`, base]) { + if (existsSync(candidate)) { + return pathToFileURL(candidate).href; + } + } + return null; +} + +export function resolve(specifier, context, next) { + if (specifier.startsWith("@/")) { + const resolved = firstExisting(SRC + specifier.slice(2)); + return next(resolved ?? specifier, context); + } + if (specifier.startsWith(".") && context.parentURL?.startsWith("file:")) { + const resolved = firstExisting( + fileURLToPath(new URL(specifier, context.parentURL)), + ); + if (resolved) { + return next(resolved, context); + } + } + return next(specifier, context); +} diff --git a/studio/frontend/tests/helpers/kit.ts b/studio/frontend/tests/helpers/kit.ts new file mode 100644 index 0000000000..d3d8bf80e4 --- /dev/null +++ b/studio/frontend/tests/helpers/kit.ts @@ -0,0 +1,131 @@ +// SPDX-License-Identifier: AGPL-3.0-only +// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 + +import { register } from "node:module"; + +import type { ResidentAdoptionState } from "../../src/features/hub/lib/adopt-inference-status.ts"; +import type { ResidentStatusRefreshTargets } from "../../src/features/hub/lib/resident-status-refresh.ts"; + +/** + * Teach the loader the two resolution rules vite and tsconfig's "bundler" mode + * give the app. Call this before the dynamic import of any src module that + * resolves the way vite and tsconfig resolve, not the way bare node does. + */ +export function registerBundlerResolver(): void { + register("../bundler-resolver.mjs", import.meta.url); +} + +export type StorageFake = { + getItem: (key: string) => string | null; + setItem: (key: string, value: string) => void; + removeItem: (key: string) => void; +}; + +/** + * An in-memory localStorage, installed on globalThis under both the names the + * app reads it by. The returned map is the backing store, so a test can stage + * records before the module under test is imported. + */ +export function installLocalStorageFake(): { + store: Map; + storage: StorageFake; +} { + const store = new Map(); + const storage: StorageFake = { + getItem: (key: string) => store.get(key) ?? null, + setItem: (key: string, value: string) => { + store.set(key, value); + }, + removeItem: (key: string) => { + store.delete(key); + }, + }; + Object.assign(globalThis, { + window: { localStorage: storage }, + localStorage: storage, + }); + return { store, storage }; +} + +/** The chat-runtime store as it stands before anything has hydrated it. */ +export function emptyStore( + overrides: Partial = {}, +): ResidentAdoptionState { + return { + checkpoint: null, + checkpointIsExternal: false, + activeGgufVariant: null, + modelLoading: false, + ...overrides, + }; +} + +/** Records the store actions adoptResidentModelStatus takes, in order. */ +export function spies() { + const calls: string[] = []; + const previouslySeen: { checkpoint: string | null; ggufVariant: string | null }[] = + []; + return { + calls, + previouslySeen, + actions: { + setCheckpoint(checkpointId: string, ggufVariant: string | null) { + calls.push(`setCheckpoint:${checkpointId}:${ggufVariant ?? ""}`); + }, + applyStatus(previous: { + checkpoint: string | null; + ggufVariant: string | null; + }) { + calls.push("applyStatus"); + previouslySeen.push(previous); + }, + }, + }; +} + +/** A window/document pair whose events and visibility a test drives by hand. */ +export function fakeTargets(): ResidentStatusRefreshTargets & { + hidden: boolean; + fire: (target: "window" | "document", type: string) => void; + listenerCount: () => number; +} { + const listeners = new Map>(); + const key = (target: string, type: string) => `${target}:${type}`; + const make = (target: "window" | "document") => ({ + addEventListener(type: string, fn: EventListenerOrEventListenerObject) { + const set = listeners.get(key(target, type)) ?? new Set(); + set.add(fn); + listeners.set(key(target, type), set); + }, + removeEventListener(type: string, fn: EventListenerOrEventListenerObject) { + listeners.get(key(target, type))?.delete(fn); + }, + }); + const visibility = { hidden: false }; + const state = { + get hidden() { + return visibility.hidden; + }, + set hidden(next: boolean) { + visibility.hidden = next; + }, + window: make("window"), + document: { + ...make("document"), + get hidden() { + return visibility.hidden; + }, + }, + fire(target: "window" | "document", type: string) { + for (const fn of listeners.get(key(target, type)) ?? []) { + (fn as EventListener)(new Event(type)); + } + }, + listenerCount() { + let total = 0; + for (const set of listeners.values()) total += set.size; + return total; + }, + }; + return state as never; +} diff --git a/studio/frontend/tests/hub-resident-status.test.ts b/studio/frontend/tests/hub-resident-status.test.ts new file mode 100644 index 0000000000..2b1813010d --- /dev/null +++ b/studio/frontend/tests/hub-resident-status.test.ts @@ -0,0 +1,311 @@ +// SPDX-License-Identifier: AGPL-3.0-only +// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 + +import assert from "node:assert/strict"; +import test from "node:test"; + +import { adoptResidentModelStatus } from "../src/features/hub/lib/adopt-inference-status.ts"; +import { + ggufVariantsMatch, + residentModelIdMatches, +} from "../src/features/hub/lib/model-identity.ts"; +import { subscribeResidentStatusRefresh } from "../src/features/hub/lib/resident-status-refresh.ts"; +import { emptyStore, fakeTargets, spies } from "./helpers/kit.ts"; + +const RESIDENT = { + checkpointId: "unsloth/Qwen3-8B-GGUF", + ggufVariant: "Q4_K_M", +}; + +/** + * Store actions that refuse to be called. The message on each names what must + * not happen, so a test states its rule by the action it declines to forbid. + */ +function refusing(messages: { + setCheckpoint?: string; + clearCheckpoint?: string; + applyStatus?: string; +}) { + const refuse = (message = "unreachable") => { + return () => { + throw new Error(message); + }; + }; + return { + setCheckpoint: refuse(messages.setCheckpoint), + clearCheckpoint: refuse(messages.clearCheckpoint), + applyStatus: refuse(messages.applyStatus), + }; +} + +test("landing on the Hub applies the whole status, not just the checkpoint", () => { + // Nothing else on /hub hydrates the runtime store: useChatModelRuntime has no + // mount sync and the chat page is a different route. Pinning only the + // checkpoint leaves every field useActiveModelConfig reads at its default, so + // the settings page offers those defaults as the resident model's live config. + const { calls, actions } = spies(); + const adopted = adoptResidentModelStatus(RESIDENT, emptyStore(), actions); + assert.equal(adopted, true); + assert.deepEqual(calls, [ + "setCheckpoint:unsloth/Qwen3-8B-GGUF:Q4_K_M", + "applyStatus", + ]); +}); + +test("a checkpoint that already matches is still hydrated", () => { + // A reload rehydrates params.checkpoint from localStorage on its own, with + // none of the fields that say how the model was actually launched. + const { calls, actions } = spies(); + adoptResidentModelStatus( + RESIDENT, + emptyStore({ + checkpoint: "unsloth/Qwen3-8B-GGUF", + activeGgufVariant: "Q4_K_M", + }), + actions, + ); + assert.deepEqual(calls, ["applyStatus"]); +}); + +test("an API auto-switch under the tab re-pins the model and the quant", () => { + for (const stale of [ + { checkpoint: "unsloth/Llama-3.1-8B-GGUF", activeGgufVariant: "Q4_K_M" }, + { checkpoint: "unsloth/Qwen3-8B-GGUF", activeGgufVariant: "Q8_0" }, + ]) { + const { calls, actions } = spies(); + adoptResidentModelStatus(RESIDENT, emptyStore(stale), actions); + assert.deepEqual(calls, [ + "setCheckpoint:unsloth/Qwen3-8B-GGUF:Q4_K_M", + "applyStatus", + ]); + } +}); + +test("the status applied is the one from before the checkpoint moved", () => { + // applyActiveModelStatusToStore tells a hydration from steady state by the + // previous checkpoint/quant, so it has to be read before setCheckpoint syncs + // them, or a variant-only switch reads as steady state and keeps the old + // quant's baselines. + const { previouslySeen, actions } = spies(); + adoptResidentModelStatus( + RESIDENT, + emptyStore({ + checkpoint: "unsloth/Qwen3-8B-GGUF", + activeGgufVariant: "Q8_0", + }), + actions, + ); + assert.deepEqual(previouslySeen, [ + { checkpoint: "unsloth/Qwen3-8B-GGUF", ggufVariant: "Q8_0" }, + ]); +}); + +test("nothing is adopted when no model is loaded", () => { + const { calls, actions } = spies(); + const adopted = adoptResidentModelStatus( + { checkpointId: null, ggufVariant: null }, + emptyStore(), + actions, + ); + assert.equal(adopted, false); + assert.deepEqual(calls, []); +}); + +test("an external-provider selection is left alone", () => { + // It has no local mirror, so stamping the resident GGUF's launch settings onto + // it would describe a model the user is not talking to. + const { calls, actions } = spies(); + const adopted = adoptResidentModelStatus( + RESIDENT, + emptyStore({ + checkpoint: "openai/gpt-5", + checkpointIsExternal: true, + }), + actions, + ); + assert.equal(adopted, false); + assert.deepEqual(calls, []); +}); + +test("a load in flight is not fought", () => { + // The load applies its own status when it settles, and the load dialog owns + // the params meanwhile. + const { calls, actions } = spies(); + const adopted = adoptResidentModelStatus( + RESIDENT, + emptyStore({ modelLoading: true }), + actions, + ); + assert.equal(adopted, false); + assert.deepEqual(calls, []); +}); + +test("an empty status drops a local checkpoint the server no longer has", () => { + // Unloading from another tab, from the API monitor or over the API leaves this + // store pinned; the settings page then treats the row as resident and seeds the + // editor from a launch config nothing is running. + const cleared: string[] = []; + const adopted = adoptResidentModelStatus( + { checkpointId: null, ggufVariant: null }, + emptyStore({ + checkpoint: "/models/llama.gguf", + activeGgufVariant: "Q4_K_M", + }), + { + ...refusing({ + setCheckpoint: "nothing is resident, so nothing may be pinned", + applyStatus: "there is no status to apply", + }), + clearCheckpoint: () => { + cleared.push("cleared"); + }, + }, + ); + assert.equal(adopted, true); + assert.deepEqual(cleared, ["cleared"]); +}); + +test("an empty status leaves an external pick alone", () => { + // clearCheckpoint also drops the persisted external selection, so an empty + // status must not reach it: the local model is not what the user is talking to. + const adopted = adoptResidentModelStatus( + { checkpointId: null, ggufVariant: null }, + emptyStore({ + checkpoint: "gemini/gemini-2.5-pro", + checkpointIsExternal: true, + }), + refusing({ + clearCheckpoint: "an external pick must survive an empty status", + }), + ); + assert.equal(adopted, false); +}); + +test("an empty status does not fight a load this tab started", () => { + const adopted = adoptResidentModelStatus( + { checkpointId: null, ggufVariant: null }, + emptyStore({ + checkpoint: "/models/llama.gguf", + modelLoading: true, + }), + refusing({ clearCheckpoint: "the load owns the store until it settles" }), + ); + assert.equal(adopted, false); +}); + +test("an empty status on an already empty store changes nothing", () => { + const adopted = adoptResidentModelStatus( + { checkpointId: null, ggufVariant: null }, + emptyStore(), + refusing({ clearCheckpoint: "there is nothing to clear" }), + ); + assert.equal(adopted, false); +}); + +test("coming back to the window re-reads inference status", () => { + // An OpenAI-compatible request auto-switches the resident model whenever it + // likes. The Hub's only other status read is its mount effect, so without this + // the catalog and the settings page keep describing the previous model for as + // long as the Hub stays mounted. + const targets = fakeTargets(); + let reads = 0; + subscribeResidentStatusRefresh(() => { + reads += 1; + }, targets); + + assert.equal(reads, 0, "subscribing must not read on its own"); + targets.fire("window", "focus"); + assert.equal(reads, 1); + targets.fire("document", "visibilitychange"); + assert.equal(reads, 2); +}); + +test("a tab going hidden does not read", () => { + // visibilitychange fires on the way out too, and a hidden tab has no settings + // page to correct. + const targets = fakeTargets(); + let reads = 0; + subscribeResidentStatusRefresh(() => { + reads += 1; + }, targets); + + targets.hidden = true; + targets.fire("document", "visibilitychange"); + assert.equal(reads, 0); + + targets.hidden = false; + targets.fire("document", "visibilitychange"); + assert.equal(reads, 1); +}); + +test("an auto-switch under a mounted Hub stops hiding the live config", () => { + // The whole point, end to end: while the Hub is mounted an OpenAI-compatible + // request swaps the resident model. Without a second read the store still names + // the old one, so hub-page's settingsTargetIsResident says the newly loaded + // model is not resident, its settings page is handed loadedConfig=null, and + // ModelConfigPage seeds the editor from saved/default values -- which Apply then + // reloads the model with, over what the API actually selected. + const store = emptyStore({ + checkpoint: "unsloth/Qwen3-8B-GGUF", + activeGgufVariant: "Q4_K_M", + }); + // What the server reports once the API request has switched it. + let serverStatus = { + checkpointId: "unsloth/Llama-3.1-8B-Instruct-GGUF", + ggufVariant: "Q8_0", + }; + const readStatusAndAdopt = () => { + adoptResidentModelStatus( + serverStatus, + { ...store }, + { + setCheckpoint: (checkpointId, ggufVariant) => { + store.checkpoint = checkpointId; + store.activeGgufVariant = ggufVariant; + }, + applyStatus: () => undefined, + }, + ); + }; + + // hub-page.tsx's settingsTargetIsResident, for the model the API just loaded. + const settingsTargetIsResident = () => + residentModelIdMatches(store.checkpoint, serverStatus.checkpointId) && + ggufVariantsMatch(store.activeGgufVariant, serverStatus.ggufVariant); + + const targets = fakeTargets(); + subscribeResidentStatusRefresh(readStatusAndAdopt, targets); + + assert.equal( + settingsTargetIsResident(), + false, + "precondition: the mount-time read predates the switch", + ); + targets.fire("window", "focus"); + assert.equal(settingsTargetIsResident(), true); + + // A load this tab started owns the store until it settles, so a refresh landing + // mid-switch must not re-pin the model the user is moving away from. + store.modelLoading = true; + serverStatus = { + checkpointId: "unsloth/Qwen3-8B-GGUF", + ggufVariant: "Q4_K_M", + }; + targets.fire("window", "focus"); + assert.equal(store.checkpoint, "unsloth/Llama-3.1-8B-Instruct-GGUF"); +}); + +test("unsubscribing stops the reads and leaves no listener behind", () => { + const targets = fakeTargets(); + let reads = 0; + const unsubscribe = subscribeResidentStatusRefresh(() => { + reads += 1; + }, targets); + + assert.equal(targets.listenerCount(), 2); + unsubscribe(); + assert.equal(targets.listenerCount(), 0); + targets.fire("window", "focus"); + targets.fire("document", "visibilitychange"); + assert.equal(reads, 0); +}); diff --git a/studio/frontend/tests/model-config-instance-key.test.ts b/studio/frontend/tests/model-config-instance-key.test.ts new file mode 100644 index 0000000000..d5832d9656 --- /dev/null +++ b/studio/frontend/tests/model-config-instance-key.test.ts @@ -0,0 +1,132 @@ +// SPDX-License-Identifier: AGPL-3.0-only +// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 + +import assert from "node:assert/strict"; +import test from "node:test"; + +import { modelConfigInstanceKey } from "../src/features/model-picker/model-config/config-signature.ts"; +import type { PerModelConfig } from "../src/features/model-picker/model-config/per-model-config.ts"; + +const MODEL = "unsloth/Qwen3-8B-GGUF"; +const VARIANT = "Q4_K_M"; + +// What the model is actually running with, as useActiveModelConfig reports it. +const LIVE: PerModelConfig = { + customContextLength: 16384, + maxSeqLength: null, + kvCacheDtype: "q8_0", + speculativeType: "ngram", + specDraftNMax: 6, + nParallel: 4, + tensorParallel: true, + chatTemplateOverride: null, + gpuMemoryMode: "manual", + gpuLayers: 24, + nCpuMoe: 3, + selectedGpuIds: [0, 1], +}; + +// What ModelConfigPage would fall back to before the live config lands. +const SAVED: PerModelConfig = { + customContextLength: null, + maxSeqLength: null, + kvCacheDtype: null, + speculativeType: "auto", + specDraftNMax: null, + nParallel: null, + tensorParallel: false, + chatTemplateOverride: null, + gpuMemoryMode: "auto", + gpuLayers: -1, + nCpuMoe: 0, + selectedGpuIds: null, +}; + +/** + * ModelConfigPage reads `loadedConfig` in a useState initializer, so it seeds its + * editable state once per MOUNTED instance; React keeps that instance for as long + * as the key is unchanged. This is that rule, and nothing else. + */ +function renderEditor( + previous: { key: string; editing: PerModelConfig } | null, + key: string, + loadedConfig: PerModelConfig | null, +): { key: string; editing: PerModelConfig } { + if (previous && previous.key === key) { + return previous; + } + return { key, editing: loadedConfig ?? SAVED }; +} + +test("the settings editor re-seeds when the live config arrives after mount", () => { + // Opened before /api/inference/status answered, or while the target was still + // loading: loadedConfig is null on the first render and live on the next. + let editor = renderEditor( + null, + modelConfigInstanceKey(MODEL, VARIANT, null), + null, + ); + assert.deepEqual(editor.editing, SAVED); + + editor = renderEditor( + editor, + modelConfigInstanceKey(MODEL, VARIANT, LIVE), + LIVE, + ); + // Without the live config in the key the editor would still hold SAVED, and + // Apply would reload the model with it over what it is running with. + assert.deepEqual(editor.editing, LIVE); +}); + +test("a repeated status poll keeps the same editor instance", () => { + const first = renderEditor( + null, + modelConfigInstanceKey(MODEL, VARIANT, LIVE), + LIVE, + ); + // A structurally equal config from the next poll must not remount and throw + // away whatever the user has typed since. + const again = renderEditor( + first, + modelConfigInstanceKey(MODEL, VARIANT, { ...LIVE }), + LIVE, + ); + assert.equal(again, first); +}); + +test("every mirrored setting moves the instance key", () => { + const base = modelConfigInstanceKey(MODEL, VARIANT, LIVE); + const changes: PerModelConfig[] = [ + { ...LIVE, customContextLength: 8192 }, + { ...LIVE, maxSeqLength: 4096 }, + { ...LIVE, kvCacheDtype: "f16" }, + { ...LIVE, speculativeType: "off" }, + { ...LIVE, specDraftNMax: 4 }, + { ...LIVE, nParallel: 1 }, + { ...LIVE, tensorParallel: false }, + { ...LIVE, chatTemplateOverride: "{{ bos_token }}" }, + { ...LIVE, gpuMemoryMode: "auto" }, + { ...LIVE, gpuLayers: 20 }, + { ...LIVE, nCpuMoe: 0 }, + { ...LIVE, selectedGpuIds: [0] }, + ]; + for (const changed of changes) { + assert.notEqual(modelConfigInstanceKey(MODEL, VARIANT, changed), base); + } + // The GPU pick is a set, not an order. + assert.equal( + modelConfigInstanceKey(MODEL, VARIANT, { ...LIVE, selectedGpuIds: [1, 0] }), + base, + ); +}); + +test("the model and its quant still key the editor", () => { + const base = modelConfigInstanceKey(MODEL, VARIANT, LIVE); + assert.notEqual(modelConfigInstanceKey("unsloth/Other-GGUF", VARIANT, LIVE), base); + assert.notEqual(modelConfigInstanceKey(MODEL, "Q8_0", LIVE), base); + // A loose .gguf carries no quant; null and undefined are the same absence. + assert.equal( + modelConfigInstanceKey(MODEL, null, LIVE), + modelConfigInstanceKey(MODEL, undefined, LIVE), + ); +}); diff --git a/studio/frontend/tests/model-identity.test.ts b/studio/frontend/tests/model-identity.test.ts new file mode 100644 index 0000000000..d1da35ecc1 --- /dev/null +++ b/studio/frontend/tests/model-identity.test.ts @@ -0,0 +1,344 @@ +// SPDX-License-Identifier: AGPL-3.0-only +// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 + +import assert from "node:assert/strict"; +import test from "node:test"; + +import { settingsGgufVariantForRow } from "../src/features/hub/inventory/settings-identity.ts"; +import type { + CachedInventoryRow, + LocalInventoryRow, +} from "../src/features/hub/inventory/types.ts"; +import { + isOllamaLinkPath, + modelIdsMatch, + publicModelId, + residentModelIdMatches, +} from "../src/features/hub/lib/model-identity.ts"; +import { + installLocalStorageFake, + registerBundlerResolver, +} from "./helpers/kit.ts"; + +registerBundlerResolver(); +const { store, storage } = installLocalStorageFake(); + +const REPO_KEY = 'v2:["unsloth/repo-gguf","q4_k_m"]'; + +// The legacy import of unsloth_load_settings runs once, on the first read after +// load, so it has to be staged before the module is imported. +store.set( + "unsloth_model_configs", + JSON.stringify({ [REPO_KEY]: { version: 1, maxSeqLength: 32768 } }), +); +store.set( + "unsloth_load_settings", + JSON.stringify({ "Unsloth/Repo-GGUF::Q4_K_M": { contextLength: 8192 } }), +); + +const { listPerModelConfigs, resolveInitialConfig, savePerModelConfig } = + await import("../src/features/model-picker/model-config/per-model-config.ts"); +const { modelStorageKey, splitQuantSuffix } = await import( + "../src/features/model-picker/model-config/model-identity.ts" +); + +function config(maxSeqLength: number, kvCacheDtype: string | null = null) { + return { + customContextLength: null, + maxSeqLength, + kvCacheDtype, + speculativeType: null, + specDraftNMax: null, + nParallel: null, + tensorParallel: false, + chatTemplateOverride: null, + }; +} + +function storedKeys(): string[] { + return Object.keys( + JSON.parse(storage.getItem("unsloth_model_configs") ?? "{}"), + ); +} + +test("publicModelId mirrors what /status reports for a path-loaded model", () => { + // Mirrors public_model_id in studio/backend/core/inference/model_ids.py. + assert.equal( + publicModelId("/srv/models/Qwen3-8B-Q4_K_M.gguf"), + "Qwen3-8B-Q4_K_M", + ); + assert.equal( + publicModelId( + "/home/u/.cache/huggingface/hub/models--unsloth--Qwen3-8B-GGUF/snapshots/abc123", + ), + "unsloth/Qwen3-8B-GGUF", + ); + assert.equal(publicModelId("C:\\models\\Foo-Q4_K_M.gguf"), "Foo-Q4_K_M"); + assert.equal(publicModelId("~/models/Foo.gguf"), "Foo"); + assert.equal(publicModelId("/srv/models/repo/"), "repo"); + // A repo id and an already-clean name come back untouched. + assert.equal(publicModelId("unsloth/Qwen3-8B-GGUF"), "unsloth/Qwen3-8B-GGUF"); + assert.equal(publicModelId("Qwen3-8B-Q4_K_M"), "Qwen3-8B-Q4_K_M"); + // "models--" alone is not the cache layout; only the snapshots sibling is. + assert.equal(publicModelId("models--only--nosnapshots/blobs/x"), "x"); +}); + +test("a resident path-loaded model is matched by the id /status reports", () => { + // A loose .gguf: the catalog row is keyed by the path, and the Hub page records + // the loadable identifier (status.model_identifier), so the literal pass answers. + assert.equal( + modelIdsMatch("Qwen3-8B-Q4_K_M", "/srv/models/Qwen3-8B-Q4_K_M.gguf"), + false, + ); + assert.equal( + residentModelIdMatches( + "/srv/models/Qwen3-8B-Q4_K_M.gguf", + "/srv/models/Qwen3-8B-Q4_K_M.gguf", + "/srv/models/Qwen3-8B-Q4_K_M.gguf", + ), + true, + ); + // A repo in an inactive HF cache loads by snapshot path but keeps the repo id + // as its settings identity, so the configId alias already covers it. + assert.equal( + residentModelIdMatches( + "unsloth/Qwen3-8B-GGUF", + "/mnt/old-cache/models--unsloth--Qwen3-8B-GGUF/snapshots/abc123", + "unsloth/Qwen3-8B-GGUF", + ), + true, + ); + // The raw identifier is still matched literally. + assert.equal( + residentModelIdMatches( + "/srv/models/Qwen3-8B-Q4_K_M.gguf", + "/srv/models/Qwen3-8B-Q4_K_M.gguf", + null, + ), + true, + ); + // Another model is still not the loaded one. + assert.equal( + residentModelIdMatches( + "Qwen3-8B-Q4_K_M", + "/srv/models/Llama-3-8B-Q4_K_M.gguf", + null, + ), + false, + ); + assert.equal( + residentModelIdMatches( + "unsloth/Qwen3-8B-GGUF", + "/mnt/old-cache/models--unsloth--Llama-3-GGUF/snapshots/abc123", + "unsloth/Llama-3-GGUF", + ), + false, + ); + assert.equal(residentModelIdMatches(null, "/srv/models/x.gguf"), false); + assert.equal(residentModelIdMatches("Qwen3-8B-Q4_K_M"), false); +}); + +test("a shared filename or folder name never marks a row resident", () => { + // Two loose GGUFs with the same filename in different folders collapse onto one + // public id, so a stem can only say "one of these", never which. + const loaded = "/srv/models/alpha/model.gguf"; + const other = "/srv/models/beta/model.gguf"; + assert.equal(publicModelId(loaded), publicModelId(other)); + assert.equal(residentModelIdMatches(publicModelId(loaded), other, other), false); + // The loadable identifier names exactly one of them. + assert.equal(residentModelIdMatches(loaded, loaded, loaded), true); + assert.equal(residentModelIdMatches(loaded, other, other), false); + + // Same collapse one level up: two model directories sharing a basename. + const loadedDir = "/srv/lmstudio/publisher-a/Llama-3-8B-GGUF"; + const otherDir = "/srv/models/publisher-b/Llama-3-8B-GGUF"; + assert.equal(publicModelId(loadedDir), publicModelId(otherDir)); + assert.equal( + residentModelIdMatches(publicModelId(loadedDir), otherDir, otherDir), + false, + ); + + // A cache snapshot still collapses onto its repo id, which names one model. + assert.equal( + residentModelIdMatches( + "unsloth/Qwen3-8B-GGUF", + "/mnt/old-cache/models--unsloth--Qwen3-8B-GGUF/snapshots/abc123", + null, + ), + true, + ); +}); + +test("Ollama link paths are recognised the way the resolver excludes them", () => { + // core/inference/local_model_resolver.py refuses any path with these segments. + assert.equal( + isOllamaLinkPath("/home/u/.ollama/models/.studio_links/q/qwen3-Q4_K_M.gguf"), + true, + ); + assert.equal( + isOllamaLinkPath("/home/u/.cache/unsloth/ollama_links/ab12/llama3.gguf"), + true, + ); + assert.equal( + isOllamaLinkPath("C:\\Users\\u\\.ollama\\models\\.studio_links\\q\\a.gguf"), + true, + ); + // Only those exact segments, not a directory that merely contains the name. + assert.equal(isOllamaLinkPath("/srv/studio_links_backup/a.gguf"), false); + assert.equal(isOllamaLinkPath("/srv/models/Qwen3-8B-Q4_K_M.gguf"), false); + assert.equal(isOllamaLinkPath("unsloth/Qwen3-8B-GGUF"), false); + assert.equal(isOllamaLinkPath(null), false); +}); + +test("a standalone gguf keeps one settings identity across surfaces", () => { + const loose = { + kind: "local", + path: "/srv/models/Qwen3-8B-Q4_K_M.gguf", + // What hub/services/models/common.py emits for a single scanned file. + formatVariant: "Q4_K_M", + } as LocalInventoryRow; + // The Chat picker opens the same file with no variant, so the Hub row must not + // adopt the filename-derived label or the two edit different configs. + assert.equal(settingsGgufVariantForRow(loose), null); + + // A GGUF directory still has a variant slot for the quant lookup to fill. + const repoDir = { + kind: "local", + path: "/srv/models/Qwen3-8B-GGUF", + formatVariant: null, + } as LocalInventoryRow; + assert.equal(settingsGgufVariantForRow(repoDir), null); + const lmStudioDir = { + kind: "local", + path: "/srv/lmstudio/Qwen3-8B-GGUF", + formatVariant: "Q8_0", + } as LocalInventoryRow; + assert.equal(settingsGgufVariantForRow(lmStudioDir), "Q8_0"); + + // Cached repo rows are unaffected (cache_inventory.py never sets one). + const cached = { kind: "cache", formatVariant: null } as CachedInventoryRow; + assert.equal(settingsGgufVariantForRow(cached), null); +}); + +// The one-time backfill re-reads listPerModelConfigs() to pick up a save that +// landed while the override fetch was in flight, and matches on the folded +// identity. That is only unambiguous because storage holds one record per model, +// so these pin that rule rather than the backfill. +test("importing the legacy load settings never doubles up a model", () => { + // The typed casing in unsloth_load_settings names the model the v2 record + // already holds, so the import has to leave it alone rather than add a second + // record the picker would prefer and the backfill would not. + assert.deepEqual(listPerModelConfigs().length, 1); + assert.deepEqual(storedKeys(), [REPO_KEY]); + assert.equal( + resolveInitialConfig("unsloth/repo-gguf", "q4_k_m").config + .customContextLength, + null, + ); +}); + +test("two spellings of one model id keep a single stored record", () => { + store.clear(); + savePerModelConfig("Unsloth/Repo-GGUF", "Q4_K_M", config(4096)); + savePerModelConfig("unsloth/repo-gguf", "q4_k_m", config(32768, "q8_0")); + + assert.deepEqual(storedKeys(), [REPO_KEY]); + const listed = listPerModelConfigs(); + assert.equal(listed.length, 1); + assert.equal(listed[0]?.config.maxSeqLength, 32768); + // What the picker applies and the only thing the backfill can see agree. + assert.equal( + resolveInitialConfig("Unsloth/Repo-GGUF", "Q4_K_M").config.maxSeqLength, + 32768, + ); +}); + +test("two spellings of one Windows path keep a single stored record", () => { + store.clear(); + savePerModelConfig("C:\\Models\\Foo.gguf", null, config(4096)); + savePerModelConfig("c:/models/foo.gguf", null, config(32768, "q8_0")); + + assert.deepEqual(storedKeys(), ['v2:["c:/models/foo.gguf",""]']); + assert.equal(listPerModelConfigs().length, 1); +}); + +test("a POSIX path is case sensitive, so its two spellings stay separate", () => { + store.clear(); + savePerModelConfig("/models/Foo.gguf", null, config(4096)); + savePerModelConfig("/models/foo.gguf", null, config(32768, "q8_0")); + + assert.equal(storedKeys().length, 2); + assert.equal( + resolveInitialConfig("/models/Foo.gguf", null).config.maxSeqLength, + 4096, + ); +}); + +// Every answer below is the one split_quant_suffix in +// studio/backend/utils/openai_auto_switch_settings.py gives for the same key. The +// backfill folds a stored key with this before comparing it against the server's, +// so a suffix this splits and the backend does not collapses two models onto one +// key on the browser side only. +const CASES: [string, [string, string] | null][] = [ + // A known quant label, with and without the optional bpw modifier. + ["org/Repo-GGUF:Q4_K_M", ["org/Repo-GGUF", "Q4_K_M"]], + ["org/Repo-GGUF:IQ4_XS-3.53bpw", ["org/Repo-GGUF", "IQ4_XS-3.53bpw"]], + ["org/Repo-GGUF:UD-Q4_K_XL", ["org/Repo-GGUF", "UD-Q4_K_XL"]], + // A .gguf with no quant token in its name is labelled by its stem, and storage + // lowercases the label while the scanner keeps the filename's casing. + ["/models/CustomModel.gguf:custommodel", ["/models/CustomModel.gguf", "custommodel"]], + ["/models/CustomModel.gguf:CustomModel", ["/models/CustomModel.gguf", "CustomModel"]], + ["C:\\models\\CustomModel.gguf:custommodel", ["C:\\models\\CustomModel.gguf", "custommodel"]], + // A shard suffix is not part of the label. + [ + "/models/Custom-00001-of-00003.gguf:custom", + ["/models/Custom-00001-of-00003.gguf", "custom"], + ], + ["/models/Custom-00001-of-00003.gguf:custom-00001-of-00003", null], + // An extensionless .gguf still has a label. + ["/models/.gguf:gguf", ["/models/.gguf", "gguf"]], + // A quant token inside the filename wins over the stem. + ["/models/tinyllama-Q4_K_M.gguf:q4_k_m", ["/models/tinyllama-Q4_K_M.gguf", "q4_k_m"]], + ["/models/tinyllama-Q4_K_M.gguf:tinyllama-q4_k_m", null], + // Only the basename is labelled, never the directories above it. + [ + "/models/dir/CustomModel.gguf:custommodel", + ["/models/dir/CustomModel.gguf", "custommodel"], + ], + ["/models/dir/CustomModel.gguf:dir/custommodel", null], + // A colon is legal in a POSIX filename. Neither of these is a variant, and + // reading them as one folds two real files onto a single key. + ["/models/foo:Bar.gguf", null], + ["/models/foo:bar.gguf", null], + ["/models/llama.gguf:Bar.gguf", null], + ["/models/llama.gguf:bar.gguf", null], + ["/models/CustomModel.gguf:othermodel", null], + ["/models/model.gguf:notalabel", null], + ["/models/plain.gguf:plain:extra", null], + // A Windows drive letter is not a separator either. + ["C:\\models\\foo.gguf", null], + ["C:/models/foo.gguf", null], + // Nothing to split. + ["org/Repo-GGUF", null], + ["/models/foo.gguf", null], + ["org/Repo:", null], + [":Q4_K_M", null], +]; + +test("splitQuantSuffix answers exactly as the backend's split_quant_suffix", () => { + for (const [value, expected] of CASES) { + assert.deepEqual(splitQuantSuffix(value), expected, value); + } +}); + +test("a .gguf filename carrying a colon is not folded into a variant", () => { + // Two real, distinct files: POSIX allows a colon in a name and is case + // sensitive, so the one-time backfill has to keep their settings apart. The + // variant half of an override key is stored lowercased, so folding these makes + // one key and strands whichever file the backfill reaches second. + const upper = "/models/llama.gguf:Bar.gguf"; + const lower = "/models/llama.gguf:bar.gguf"; + assert.equal(splitQuantSuffix(upper), null); + assert.equal(splitQuantSuffix(lower), null); + assert.notEqual(modelStorageKey(upper, null), modelStorageKey(lower, null)); +}); diff --git a/studio/frontend/tsconfig.test.json b/studio/frontend/tsconfig.test.json index da6cdcc9ba..be53907fc7 100644 --- a/studio/frontend/tsconfig.test.json +++ b/studio/frontend/tsconfig.test.json @@ -3,7 +3,9 @@ "target": "ES2022", "lib": ["ES2022", "DOM", "DOM.Iterable"], "module": "ESNext", - "types": ["node"], + // vite/client so a test may import a src module that (transitively) reads + // import.meta.env; without it those reads fail to typecheck here only. + "types": ["node", "vite/client"], "skipLibCheck": true, "moduleResolution": "bundler", diff --git a/tests/studio/test_model_picker_contracts.py b/tests/studio/test_model_picker_contracts.py index d9a8efc9a4..bbb782f5c2 100644 --- a/tests/studio/test_model_picker_contracts.py +++ b/tests/studio/test_model_picker_contracts.py @@ -27,9 +27,9 @@ def _read(rel: str) -> str: def test_models_api_sends_token_via_header_not_query(): - """getModelConfig / checkVisionModel / checkEmbeddingModel must pass the HF - token through hubTokenHeader, never as a ?hf_token= query param (which leaks - the credential into server/proxy access logs).""" + """getModelConfig / checkVisionModel / checkEmbeddingModel must pass the HF token + through hubTokenHeader, never as a ?hf_token= query param (which leaks the + credential into server/proxy access logs).""" src = _read("features/training/api/models-api.ts") assert src.count("hubTokenHeader(") >= 3 assert "hf_token=" not in src @@ -43,27 +43,27 @@ def test_model_metadata_probe_never_puts_token_in_query(): def test_model_config_page_floors_the_context_ceiling(): - """The model's native max-context must be FLOORED to the step grid, never - rounded up (rounding up can offer/persist a length above the model's real - ceiling and break loading).""" + """The model's native max-context must be FLOORED to the step grid, never rounded up + (rounding up can offer/persist a length above the model's real ceiling and break + loading).""" src = _read("features/model-picker/components/model-config-page.tsx") assert "floorMaxSeqLength(modelMaxPosition.maxPositionEmbeddings)" in src assert "normalizeMaxSeqLength(modelMaxPosition.maxPositionEmbeddings)" not in src def test_compare_load_clears_stale_native_lease(): - """A compare-pane load never comes from the desktop file picker, so it must - clear any prior picked file's lease token + expiry, otherwise a reload can - send a stale lease for the now-active model.""" + """A compare-pane load never comes from the desktop file picker, so it must clear any + prior picked file's lease token + expiry, otherwise a reload can send a stale lease + for the now-active model.""" src = _read("features/chat/shared-composer.tsx") assert "activeNativePathToken: null" in src assert "activeNativePathExpiresAtMs: null" in src def test_autoload_records_backend_loaded_model_identity(): - """An inactive-cache inventory row loads by local path, so startup autoload - must key both the active checkpoint and its summary by the backend's loaded - model identity instead of the catalog repo id.""" + """An inactive-cache inventory row loads by local path, so startup autoload must key + both the active checkpoint and its summary by the backend's loaded model identity + instead of the catalog repo id.""" src = _read("features/chat/api/chat-adapter.ts") autoload = src.split("async function loadAutoLoadCandidate", 1)[1] autoload = autoload.split("\n try {", 1)[0] @@ -74,8 +74,8 @@ def test_autoload_records_backend_loaded_model_identity(): def test_chat_autoload_toast_is_persistent_and_dismissible(): - """Send-triggered autoload stays visible until it settles but remains - dismissible, matching the explicit model-loading toast's lifetime.""" + """Send-triggered autoload stays visible until it settles but remains dismissible, + matching the explicit model-loading toast's lifetime.""" src = _read("features/chat/api/chat-adapter.ts") auto_load = src.split("async function autoLoadSmallestModel", 1)[1] auto_load = auto_load.split("export function createOpenAIStreamAdapter", 1)[0] @@ -101,8 +101,8 @@ def test_chat_autoload_toast_is_persistent_and_dismissible(): def test_recipe_model_load_toast_is_persistent_and_dismissible(): - """Recipe model loading uses the same dismissible persistent lifecycle as - chat loading because both call the non-abortable loadModel API.""" + """Recipe model loading uses the same dismissible persistent lifecycle as chat loading + because both call the non-abortable loadModel API.""" src = _read("features/recipe-studio/hooks/use-recipe-executions.ts") model_load = src.split("async function loadLocalModelSelection", 1)[1] model_load = model_load.split("function getLocalModelLoadPlanForPayload", 1)[0] @@ -125,9 +125,9 @@ def test_recipe_model_load_toast_is_persistent_and_dismissible(): def test_rollback_restores_native_lease_expiry_with_token(): - """A failed model switch that rolls back to a previously loaded picked GGUF - must restore the lease expiry paired with the token, never the token alone - (which would look non-expiring and skip the expiry guard).""" + """A failed model switch that rolls back to a previously loaded picked GGUF must + restore the lease expiry paired with the token, never the token alone (which would + look non-expiring and skip the expiry guard).""" src = _read("features/chat/hooks/use-chat-model-runtime.ts") assert "previousActiveNativePathExpiresAtMs" in src assert re.search( @@ -136,17 +136,17 @@ def test_rollback_restores_native_lease_expiry_with_token(): def test_default_caches_keyed_on_inventory_version(): - """The chat-template and max-position caches must key on the inventory - version so a model update in the same session invalidates the cached value - instead of showing the stale revision.""" + """The chat-template and max-position caches must key on the inventory version so a + model update in the same session invalidates the cached value instead of showing the + stale revision.""" src = _read("features/model-picker/hooks/use-model-defaults.ts") # Both cache keys (template + max-position) end with the inventory version. assert src.count("${inventoryVersion}") >= 2 def test_hidden_infra_model_needles_present(): - """The frontend static needle list must keep hiding the RAG embedder and the - llama.cpp validation probe.""" + """The frontend static needle list must keep hiding the RAG embedder and the llama.cpp + validation probe.""" src = _read("features/hub/lib/hidden-models.ts") assert '"bge-small-en-v1.5"' in src assert '"ggml-org/models"' in src @@ -154,9 +154,9 @@ def test_hidden_infra_model_needles_present(): def test_hidden_models_dynamic_exact_ids_wired(): - """The configured embedder arrives from /api/hub/hidden-models as exact - repo ids; a substring needle would let a generic basename like "model" - hide unrelated chat models.""" + """The configured embedder arrives from /api/hub/hidden-models as exact repo ids; a + substring needle would let a generic basename like "model" hide unrelated chat + models.""" src = _read("features/hub/lib/hidden-models.ts") assert "toLowerStrings(data.exact_ids)" in src assert "dynamicExactIds.includes(lower)" in src @@ -170,9 +170,9 @@ def test_hidden_model_matchers_refresh_with_inventory_version(): def test_diffusion_capability_labeled_image_generation(): - """The diffusion capability detects image GENERATORS (FLUX, SDXL, - text-to-image tags); labeling it "Image to text" showed generators when - users asked for captioning models.""" + """The diffusion capability detects image GENERATORS (FLUX, SDXL, text-to-image tags); + labeling it "Image to text" showed generators when users asked for captioning + models.""" for rel in ( "features/hub/lib/model-capabilities.ts", "features/hub/lib/model-type-filter.ts", @@ -184,9 +184,9 @@ def test_diffusion_capability_labeled_image_generation(): def test_active_model_config_round_trips_gpu_fields(): - """The active model's config must carry the GPU Memory knobs (GGUF only) so - a sidebar/hub-gear reload cannot silently reset manual GPU settings, and - "Remember settings" cannot persist a GPU-less config over a saved one.""" + """The active model's config must carry the GPU Memory knobs (GGUF only) so a + sidebar/hub-gear reload cannot silently reset manual GPU settings, and "Remember + settings" cannot persist a GPU-less config over a saved one.""" src = _read("features/model-picker/hooks/use-active-model-config.ts") for field in ("gpuMemoryMode", "gpuLayers", "nCpuMoe", "selectedGpuIds"): assert field in src, field @@ -196,15 +196,25 @@ def test_active_model_config_round_trips_gpu_fields(): "features/hub/catalog/sampling-settings-dialog.tsx", ): assert "useActiveModelConfig(" in _read(rel), rel - signature = _read("features/model-picker/components/sidebar-model-config.tsx") - assert "gpuFieldsSignature(config)" in signature - shared = _read("features/model-picker/model-config/apply-per-model-config.ts") + # The GPU knobs are part of the editor's instance key, so a reload that lands on + # different placement re-seeds the editor instead of leaving it on the old values. + shared = _read("features/model-picker/model-config/config-signature.ts") assert "export function gpuFieldsSignature" in shared + assert "gpuFieldsSignature(config)," in shared + assert "export function modelConfigInstanceKey" in shared + for rel in ( + "features/model-picker/components/sidebar-model-config.tsx", + "features/hub/catalog/hub-model-settings-view.tsx", + ): + assert "modelConfigInstanceKey(" in _read(rel), rel + # apply-per-model-config re-exports it, so its own callers are unchanged. + reexport = _read("features/model-picker/model-config/apply-per-model-config.ts") + assert "export { gpuFieldsSignature };" in reexport def test_gpu_picker_round_trips_requested_pool_not_fitted_subset(): - """A GGUF fit may narrow [0, 1] to [0], but load/status hydration must keep - [0, 1] as the editable pool so a later reload can grow back onto GPU 1.""" + """A GGUF fit may narrow [0, 1] to [0], but load/status hydration must keep [0, 1] as + the editable pool so a later reload can grow back onto GPU 1.""" types = _read("features/chat/types/api.ts") assert types.count("requested_gpu_ids?: number[] | null") >= 2 @@ -245,9 +255,9 @@ def test_model_default_hooks_do_not_reset_state_in_effect(): def test_variant_expander_refreshes_after_delete(): - """Deleting a downloaded quant from an expanded repo that still has other - cached quants must bump the expander refresh key, or the deleted quant stays - shown as downloaded and clickable and tries to reload the removed file.""" + """Deleting a downloaded quant from an expanded repo that still has other cached quants + must bump the expander refresh key, or the deleted quant stays shown as downloaded + and clickable and tries to reload the removed file.""" src = _read("features/model-picker/components/model-selector/pickers.tsx") del_confirm = re.search( r"await onDeleteVariant\(v\.quant\);.*?setRefreshKey\(\(key\) => key \+ 1\)", @@ -258,10 +268,7 @@ def test_variant_expander_refreshes_after_delete(): def test_local_picker_rows_require_chat_capability(): - """Local inventory rows can be classified non-chat (canChat false, e.g. a - folder with only config.json). The picker must filter those out, or selecting - one loads a weightless path; toLocalModelInfo drops capabilities so the memo - is the only place the guard can live.""" + """Local inventory rows can be classified non-chat (canChat false, e.g.""" src = _read("features/model-picker/inventory/use-chat-picker-inventory.ts") memo = re.search(r"const localModels = useMemo\(.*?\[inventory\.localRows\]", src, re.S) assert memo, "localModels memo not found" @@ -269,8 +276,8 @@ def test_local_picker_rows_require_chat_capability(): def test_model_picker_toolbar_reflows_before_crossing_picker_edge(): - """The content-sized section tabs and fixed-width dropdowns must reflow, - while an oversized tab group must shrink labels but preserve its icons.""" + """The content-sized section tabs and fixed-width dropdowns must reflow, while an + oversized tab group must shrink labels but preserve its icons.""" picker = _read("features/model-picker/components/model-selector/pickers.tsx") assert '"flex flex-wrap items-center gap-2"' in picker assert 'hasConnected ? "-mr-4" : "-mr-2"' in picker @@ -287,12 +294,10 @@ def test_model_picker_toolbar_reflows_before_crossing_picker_edge(): def test_native_picked_gguf_template_read_through_lease(): - """A native (picked / drag-drop) GGUF's path lives only in its signed lease, - and the picker chat-template GET has no lease plumbing, so the default - template must be read through the lease-aware validate probe: mint a - validate-model lease and post include_chat_template. The native token also - has to reach the fetch (threaded through the hook) and be part of the cache - key so two picks of the same basename don't share a template.""" + """A native (picked / drag-drop) GGUF's path lives only in its signed lease, and the + picker chat-template GET has no lease plumbing, so the default template must be read + through the lease-aware validate probe: mint a validate-model lease and post + include_chat_template.""" api = _read("features/model-picker/api/templates.ts") assert 'consumeNativePathToken(nativePathToken, "validate-model")' in api assert "include_chat_template: true" in api @@ -303,10 +308,9 @@ def test_native_picked_gguf_template_read_through_lease(): def test_model_load_guard_is_cross_instance(): - """The in-flight load guard must consult the shared store pick (not only the - per-hook ref) and ejectModel must refuse while any instance is loading: - three live useChatModelRuntime instances exist (chat page, hub page, hub - gear dialog).""" + """The in-flight load guard must consult the shared store pick (not only the per-hook + ref) and ejectModel must refuse while any instance is loading: three live + useChatModelRuntime instances exist (chat page, hub page, hub gear dialog).""" src = _read("features/chat/hooks/use-chat-model-runtime.ts") assert "useChatRuntimeStore.getState().loadingModelPick" in src assert "clearLoadingModelPick" in src @@ -315,23 +319,17 @@ def test_model_load_guard_is_cross_instance(): def test_partial_safetensors_download_keeps_delete_menu(): - """A stopped partial safetensors download must keep its options menu (the - Delete affordance) like the GGUF card does, or partial downloads can only - be cleaned up by finishing or leaving them. During an ACTIVE download the - menu stays hidden (every item would be disabled: no Copy path while not - downloaded, no Delete while downloading, pin suppressed in the run bar).""" + """A stopped partial safetensors download must keep its options menu (the Delete + affordance) like the GGUF card does, or partial downloads can only be cleaned up by + finishing or leaving them.""" src = _read("features/hub/catalog/safetensors-download-card.tsx") assert "(isDownloaded || (isPartial && !downloading))" in src def test_pinned_validation_uses_cached_local_variant_listing(): - """Pinned-quant validation must use the TTL-cached hub client with - preferLocalCache (downloaded-ness is local state) instead of one uncached - round-trip per pinned repo on every picker open. Picker deletes must go - through the hub inventory client, whose delete invalidates both the - variants TTL cache and the server-side HF cache scan (the legacy - /api/models/delete-cached route invalidates neither, so a post-delete - inventory refresh would resurrect the deleted row until the scan TTL).""" + """Pinned-quant validation must use the TTL-cached hub client with preferLocalCache + (downloaded-ness is local state) instead of one uncached round-trip per pinned repo + on every picker open.""" src = _read("features/model-picker/components/model-selector/pickers.tsx") assert "listGgufVariantsCached(" in src assert "preferLocalCache: true" in src @@ -344,8 +342,8 @@ def test_pinned_validation_uses_cached_local_variant_listing(): def test_chat_autoload_scopes_variant_lookup_to_cached_repo_path(): - """Autoload must probe the exact cache row it will load, including rows - retained from a previously selected Hugging Face cache.""" + """Autoload must probe the exact cache row it will load, including rows retained from a + previously selected Hugging Face cache.""" src = _read("features/chat/api/chat-adapter.ts") auto_load = src.split("async function autoLoadSmallestModel", 1)[1] assert auto_load.count("preferLocalCache: true") >= 2 @@ -359,8 +357,8 @@ def test_chat_autoload_scopes_variant_lookup_to_cached_repo_path(): def test_cache_location_update_invalidates_frontend_inventory(): - """A successful cache switch must refresh both inventory rows and cached - GGUF variant results before any stale active-cache identity can be reused.""" + """A successful cache switch must refresh both inventory rows and cached GGUF variant + results before any stale active-cache identity can be reused.""" src = _read("features/settings/api/hugging-face-cache.ts") update_fn = src.split("export async function updateHuggingFaceCacheSettings", 1)[1] assert "bumpInventoryVersion();" in update_fn @@ -368,18 +366,17 @@ def test_cache_location_update_invalidates_frontend_inventory(): def test_downloaded_list_offsets_virtual_rows(): - """The On Device virtualized list sits below the Pinned block in the same - scroll element, so it must pass its measured offset as scrollMargin or rows - past the overscan render blank.""" + """The On Device virtualized list sits below the Pinned block in the same scroll + element, so it must pass its measured offset as scrollMargin or rows past the + overscan render blank.""" src = _read("features/hub/catalog/models-catalog-lists.tsx") assert "scrollMargin={scrollMargin}" in src def test_local_gguf_diagnostics_gate_on_broad_is_gguf(): - """The MTP fallback note and the context/VRAM warning must gate on the broad - isGguf (variant, loaded gguf context, or .gguf suffix), not the variant-only - isLoadedGguf, so direct-file and custom-folder GGUF loads keep those - diagnostics.""" + """The MTP fallback note and the context/VRAM warning must gate on the broad isGguf + (variant, loaded gguf context, or .gguf suffix), not the variant-only isLoadedGguf, + so direct-file and custom-folder GGUF loads keep those diagnostics.""" src = _read("features/chat/chat-settings-sheet.tsx") spec = re.search(r"const showSpecFallback =.*?;", src, re.S) vram = re.search(r"const showContextVramWarning =.*?;", src, re.S) @@ -388,9 +385,9 @@ def test_local_gguf_diagnostics_gate_on_broad_is_gguf(): def test_fixed_layer_gguf_pins_displayed_context(): - """An already-loaded auto-fit GGUF saved with Manual fixed GPU layers must - pin the shown context, so a later fresh load keeps the fitted placement - instead of sending native/0 and recreating the OOM.""" + """An already-loaded auto-fit GGUF saved with Manual fixed GPU layers must pin the + shown context, so a later fresh load keeps the fitted placement instead of sending + native/0 and recreating the OOM.""" src = _read("features/model-picker/components/model-config-page.tsx") assert "const pinFixedLayerContext =" in src assert 'config.gpuMemoryMode === "manual"' in src @@ -398,11 +395,8 @@ def test_fixed_layer_gguf_pins_displayed_context(): def test_fixed_layer_pin_recomputed_after_committing_gpu_layers(): - """pinFixedLayerContext is computed from the render-time config, before a - same-click GPU Layers draft is committed. handleRun must recompute it from the - committed effectiveConfig; otherwise typing a positive GPU Layers value on an - auto-fit GGUF and clicking Reload saves customContextLength: null, so a later - fresh load sends the native context with fixed layers (the OOM the pin avoids).""" + """pinFixedLayerContext is computed from the render-time config, before a same-click + GPU Layers draft is committed.""" src = _read("features/model-picker/components/model-config-page.tsx") assert "const effectivePinFixedLayerContext =" in src assert 'effectiveConfig.gpuMemoryMode === "manual"' in src @@ -413,11 +407,7 @@ def test_fixed_layer_pin_recomputed_after_committing_gpu_layers(): def test_blur_cache_cleared_on_every_settled_render(): """The lastBlurCommittedRef bridge is valid only across the single synchronous - same-click gesture that set it. Keying its clear on [value] missed a Reset (or - external edit) that restores the shown value unchanged after the blur dispatched - onChange: value nets back to its prior number, the effect never re-ran, and a - later Load/Save replayed the override Reset removed. Clear it on every settled - render instead.""" + same-click gesture that set it.""" src = _read("features/model-picker/components/numeric-value-input.tsx") # The clearing effect must run on every commit, not be gated on [value] alone. assert not re.search(r"lastBlurCommittedRef\.current = null;\s*\}, \[value\]\);", src) @@ -428,9 +418,9 @@ def test_blur_cache_cleared_on_every_settled_render(): def test_auto_defaults_not_persisted_as_overrides(): - """Auto GPU memory mode and Auto/default speculative type are follow-global - defaults; normalization must not persist them as per-model overrides, else a - model stops following later changes to the global preference.""" + """Auto GPU memory mode and Auto/default speculative type are follow-global defaults; + normalization must not persist them as per-model overrides, else a model stops + following later changes to the global preference.""" src = _read("features/model-picker/model-config/per-model-config.ts") assert 'if (partial.gpuMemoryMode === "manual") {' in src assert 'partial.gpuMemoryMode === "auto" || partial.gpuMemoryMode === "manual"' not in src @@ -439,18 +429,18 @@ def test_auto_defaults_not_persisted_as_overrides(): def test_compare_pane_context_from_own_config_only(): - """A compare pane's context comes from its own config only (a saved pin, else - null for Auto/native); it must not inherit the active model's shared snapshot, - which resolveFitMaxSeqLength would treat as an explicit pin (VRAM/OOM).""" + """A compare pane's context comes from its own config only (a saved pin, else null for + Auto/native); it must not inherit the active model's shared snapshot, which + resolveFitMaxSeqLength would treat as an explicit pin (VRAM/OOM).""" src = _read("features/chat/shared-composer.tsx") assert "const effectiveCustomContextLength = ownConfig.customContextLength;" in src assert "compareLoadKnobs.customContextLength" not in src def test_reset_max_seq_length_falls_back_to_app_default(): - """After Reset clears maxSeqLength (null), a non-GGUF active model's shown - max sequence length must fall back to the app default, never the loaded - runtime snapshot, or a remembered/active override can never be cleared.""" + """After Reset clears maxSeqLength (null), a non-GGUF active model's shown max sequence + length must fall back to the app default, never the loaded runtime snapshot, or a + remembered/active override can never be cleared.""" src = _read("features/model-picker/components/model-config-page.tsx") # The null fallback resolves to the app-default constant, not a runtime value. assert "clampMaxSeqLength(DEFAULT_MAX_SEQ_LENGTH, nativeMaxSeqLength)" in src @@ -459,9 +449,9 @@ def test_reset_max_seq_length_falls_back_to_app_default(): def test_reset_persists_null_max_length_and_substitutes_only_for_load(): - """The persisted per-model record must keep config.maxSeqLength (null after - Reset) so isDefaultConfig can clear a remembered override; the concrete - fallback is substituted only into the load request, not the saved record.""" + """The persisted per-model record must keep config.maxSeqLength (null after Reset) so + isDefaultConfig can clear a remembered override; the concrete fallback is + substituted only into the load request, not the saved record.""" src = _read("features/model-picker/components/model-config-page.tsx") # Load-only substitution of the resolved value (recomputed from any committed # same-click Max Seq Length draft, so it is never dropped). @@ -474,8 +464,8 @@ def test_reset_persists_null_max_length_and_substitutes_only_for_load(): def test_initial_load_uses_staged_config_payload(): - """Run-settings Load must pass the staged config through to /load even when - React has not flushed NumericValueInput blur commits into the store yet.""" + """Run-settings Load must pass the staged config through to /load even when React has + not flushed NumericValueInput blur commits into the store yet.""" runtime = _read("features/chat/hooks/use-chat-model-runtime.ts") assert "const pendingLoadConfig =" in runtime assert "pendingLoadConfig?.kvCacheDtype" in runtime @@ -511,11 +501,8 @@ def test_initial_load_uses_staged_config_payload(): def test_same_click_commit_covers_all_numeric_inputs(): - """The same-click blur bridge must flush every NumericValueInput-backed - setting, not just Context Length. Max Seq Length (non-GGUF), GPU Layers and - MoE Layers (GGUF) also stage their draft only on blur, so handleRun must - imperatively commit each and fold the value into the staged load config; - otherwise a value the user typed right before clicking Load/Reload is lost.""" + """The same-click blur bridge must flush every NumericValueInput-backed setting, not + just Context Length.""" page = _read("features/model-picker/components/model-config-page.tsx") # Each numeric input owns an imperative handle that handleRun commits, and the # handle is forwarded down to the actual NumericValueInput. @@ -548,10 +535,8 @@ def test_context_commit_rechecks_persistence_only_shortcut(): def test_reset_enabled_for_explicit_context_pin_at_native(): - """An explicit customContextLength that equals the native ceiling is still a - user override, so contextAtDefault must require customContextLength == null. - The buggy form treated `contextValue === native` alone as default, wedging - the Reset button disabled for a deliberate pin-to-native.""" + """An explicit customContextLength that equals the native ceiling is still a user + override, so contextAtDefault must require customContextLength == null.""" src = " ".join(_read("features/model-picker/components/model-config-page.tsx").split()) assert ( "const contextAtDefault = !target.isGguf || " @@ -569,9 +554,9 @@ def test_reset_enabled_for_explicit_context_pin_at_native(): def test_compare_pane_non_gguf_falls_back_to_app_default(): - """A non-GGUF compare pane with no saved maxSeqLength must fall back to the - shared app default, not the active model's runtime snapshot; otherwise an - unconfigured pane inherits a saved 128K neighbor's context and can OOM.""" + """A non-GGUF compare pane with no saved maxSeqLength must fall back to the shared app + default, not the active model's runtime snapshot; otherwise an unconfigured pane + inherits a saved 128K neighbor's context and can OOM.""" per_model = _read("features/model-picker/model-config/per-model-config.ts") assert "export const DEFAULT_MAX_SEQ_LENGTH = 4096;" in per_model barrel = _read("features/model-picker/index.ts") @@ -590,8 +575,8 @@ def test_compare_pane_non_gguf_falls_back_to_app_default(): def test_default_gpu_mode_clears_manual_knobs(): """Switching GPU Memory back to Default must clear the Manual-only knobs - (gpuLayers/nCpuMoe/selectedGpuIds); otherwise a remembered config keeps stale - pins that a later load re-applies when the global preference is Manual.""" + (gpuLayers/nCpuMoe/selectedGpuIds); otherwise a remembered config keeps stale pins + that a later load re-applies when the global preference is Manual.""" src = _read("features/model-picker/components/model-config-page.tsx") assert 'gpuMemoryMode: "auto",' in src assert "gpuLayers: undefined," in src @@ -600,13 +585,10 @@ def test_default_gpu_mode_clears_manual_knobs(): def test_legacy_migration_is_idempotent_and_non_destructive(): - """The v1->v2 localStorage migration (unsloth_load_settings -> - unsloth_model_configs) is invoked on every store read, so it must be - idempotent: repeated reads, browser reloads, and Studio restarts must never - re-migrate, duplicate records, or overwrite a newer per-model config. This - was the class of regression that reverted the predecessor PR, so pin all - three idempotency layers at source level; dropping any of them reddens here. - """ + """The v1->v2 localStorage migration (unsloth_load_settings -> unsloth_model_configs) + is invoked on every store read, so it must be idempotent: repeated reads, browser + reloads, and Studio restarts must never re-migrate, duplicate records, or overwrite + a newer per-model config.""" raw = _read("features/model-picker/model-config/per-model-config.ts") src = " ".join(raw.split()) # Migration runs from readMap (every store read), so it must be safe to repeat. @@ -619,10 +601,7 @@ def test_legacy_migration_is_idempotent_and_non_destructive(): assert "let legacyMigrationChecked = false;" in src assert "if (legacyMigrationChecked || !canUseStorage()) {" in src assert "legacyMigrationChecked = true;" in src - # Layer 2: persistent cross-session flag so a completed migration is never - # redone. Set in every terminal branch (malformed data, nothing to migrate, - # successful write); a failed quota write leaves it unset so the next session - # retries. Three set-sites encode exactly that. + # Layer 2: persistent cross-session flag so a completed migration is never redone. assert 'const LEGACY_MIGRATION_FLAG = "unsloth_model_configs_migrated";' in src assert "if (localStorage.getItem(LEGACY_MIGRATION_FLAG)) {" in src assert src.count('localStorage.setItem(LEGACY_MIGRATION_FLAG, "1");') >= 3 @@ -632,10 +611,10 @@ def test_legacy_migration_is_idempotent_and_non_destructive(): def test_parallel_slots_setting_wired_end_to_end(): - """The per-load Parallel Slots knob (llama-server --parallel) must flow from - the run-settings form through persistence, every /load builder, the validate - preflight and the cross-model reset; a lost hop silently reverts the model to - the server-wide slot default.""" + """The per-load Parallel Slots knob (llama-server --parallel) must flow from the + run-settings form through persistence, every /load builder, the validate preflight + and the cross-model reset; a lost hop silently reverts the model to the server-wide + slot default.""" config = _read("features/model-picker/model-config/per-model-config.ts") # Persisted per model, clamped on every read/write, and null (= server # default) counts as default so blank configs are not stored. @@ -673,22 +652,45 @@ def test_parallel_slots_setting_wired_end_to_end(): # the control would pin a blank "server default" to a number. assert "loadedNParallel: status.requested_parallel_slots," in status assert "nParallel: status.requested_parallel_slots," not in status - sidebar = _read("features/model-picker/components/sidebar-model-config.tsx") # The sidebar form remounts when an external change lands. - assert 'config.nParallel ?? "",' in sidebar + signature = _read("features/model-picker/model-config/config-signature.ts") + assert 'config.nParallel ?? "",' in signature + sidebar = " ".join(_read("features/model-picker/components/sidebar-model-config.tsx").split()) + assert "key={modelConfigInstanceKey(modelId, settingsGgufVariant, loadedConfig)}" in sidebar + + +def test_parallel_slots_reach_an_api_load_through_the_server_mirror(): + """The server mirror is the hop an OpenAI-compatible auto-switch load reads, and it is + the browser's only way to express a per-model setting to a load no browser makes.""" + api = " ".join(_read("features/model-picker/api/model-overrides.ts").split()) + assert "n_parallel?: number;" in api + assert ( + "if (config.nParallel && config.nParallel > 0) { payload.n_parallel = config.nParallel; }" + in api + ) + # The monitor lists what a remote load applies, so an entry holding only slots + # must not read as "App defaults". + monitor = " ".join(_read("features/api-monitor/components/saved-model-settings.tsx").split()) + assert "if (override.n_parallel) {" in monitor + + route = (WORKDIR / "studio" / "backend" / "routes" / "settings.py").read_text(encoding = "utf-8") + assert "n_parallel: Optional[int] = Field(" in route + assert "n_parallel = payload.n_parallel," in route + store = (WORKDIR / "studio" / "backend" / "utils" / "openai_auto_switch_settings.py").read_text( + encoding = "utf-8" + ) + assert 'entry["n_parallel"] = n_parallel' in store + # GGUF-only, like the picker: a safetensors load has no llama-server slots. + gguf_block = store.split(" if is_gguf:", 1)[1] + assert 'kwargs["n_parallel"] = override["n_parallel"]' in gguf_block def test_parallel_slots_control_cleared_when_the_load_never_sent_them(): - """`nParallel` is the editable control ("blank = follow the server default") - and `loadedNParallel` the rollback baseline. A success path that sends no - slot count must blank the control, or a value staged for another model shows - as applied, is persisted into this model's config (`isDefaultConfig` keys on - nParallel) and is re-sent by the next Apply. Each assertion below is the only - thing pinning one such path.""" + """`nParallel` is the editable control ("blank = follow the server default") and + `loadedNParallel` the rollback baseline.""" status = " ".join(_read("features/chat/lib/apply-inference-status-to-store.ts").split()) - # A model/variant swap underneath this tab must reset the control like - # performLoad's cross-model reset, or model A's count follows onto model B. - # Narrowly gated -- see test_hydration_keeps_the_slot_control_when_readopting_the_running_model. + # A model/variant swap underneath this tab must reset the control like performLoad's + # cross-model reset, or model A's count follows onto model B. assert "...(seedLoadParams && slotsModelChanged && { nParallel: null })," in status # ... while still never adopting the RESOLVED echo into the control. assert "nParallel: status.requested_parallel_slots," not in status @@ -704,8 +706,7 @@ def test_parallel_slots_control_cleared_when_the_load_never_sent_them(): # The cached-GGUF branch keeps the remembered override via the gated local... assert "nParallel: committedSlots," in gguf_branch assert "nParallel: null," not in gguf_branch - # ... the safetensors fallback sends no slots, so it clears both, or the count - # survives on a model whose form does not even render the field. + # ... assert "nParallel: null," in non_gguf_branch assert "loadedNParallel: null," in non_gguf_branch @@ -720,10 +721,8 @@ def test_parallel_slots_control_cleared_when_the_load_never_sent_them(): def test_hydration_clears_the_slot_baseline_for_a_slotless_model(): - """The baseline is what a rollback re-sends and what preset capture reads, so - a model that cannot have slots must not inherit the previous GGUF's count. - /status omits the echo for non-GGUF and sends an explicit null for diffusion; - an absent field on a GGUF is an older backend and must NOT wipe it.""" + """The baseline is what a rollback re-sends and what preset capture reads, so a model + that cannot have slots must not inherit the previous GGUF's count.""" src = _read("features/chat/lib/apply-inference-status-to-store.ts") assert ( "(status.is_gguf === false || status.requested_parallel_slots === null) && {" in src @@ -735,17 +734,10 @@ def test_hydration_clears_the_slot_baseline_for_a_slotless_model(): def test_hydration_keeps_the_slot_control_when_readopting_the_running_model(): - """`hydratingExistingModel` is true whenever the incoming status disagrees - with what this tab last recorded, which includes RE-ADOPTING a model the tab - never lost: the resident-adopt branch restores the model's own per-model - config and only then hydrates, passing the EXTERNAL id as - `previousCheckpoint`. An ungated clear there wipes the slot count that branch - just restored, and the blank persists into `savePerModelConfig`, so a Save - the user reads as a no-op erases their remembered override. - - Only that branch knows the model is unchanged, so it says so explicitly. - Slot counts cannot stand in: the echo falls back to the server-wide default, - so a genuine A->B swap can echo exactly A's explicit count.""" + """`hydratingExistingModel` is true whenever the incoming status disagrees with what + this tab last recorded, which includes RE-ADOPTING a model the tab never lost: the + resident-adopt branch restores the model's own per-model config and only then + hydrates, passing the EXTERNAL id as `previousCheckpoint`.""" status = " ".join(_read("features/chat/lib/apply-inference-status-to-store.ts").split()) assert ( "const slotsModelChanged = hydratingExistingModel && !options.readoptingSameModel;" @@ -779,14 +771,7 @@ def test_hydration_keeps_the_slot_control_when_readopting_the_running_model(): def test_parallel_slots_are_never_recorded_for_a_diffusion_load(): """A DiffusionGemma GGUF answers ``is_gguf: true``, but its runner ignores - ``--parallel``, so ``_parallel_slot_echo`` reports null slots for it. The - three load success paths must gate on ``is_diffusion`` too, or they record a - click-time count the load never committed. - - That phantom does not stay put: ``capturePresetLoadConfig`` snapshots - ``nParallel`` with no model gate and a preset carries no model identity, so - applying it over a TEXT GGUF sends the count as a real ``n_parallel``. - """ + ``--parallel``, so ``_parallel_slot_echo`` reports null slots for it.""" runtime = " ".join(_read("features/chat/hooks/use-chat-model-runtime.ts").split()) # One gated local feeds the control and the baseline, so they cannot drift. assert "(loadResponse.is_gguf ?? false) && !(loadResponse.is_diffusion ?? false)" in runtime @@ -808,17 +793,9 @@ def test_parallel_slots_are_never_recorded_for_a_diffusion_load(): def test_hydration_restores_a_remembered_slot_override(): - """The control is never seeded from the status echo, so a model running on a - remembered override shows a BLANK slot control after a browser reload or a - tab move to another GGUF. `ModelConfigPage.resolveInitial` prefers the live - store for the active model, so that blank is what the form edits: the next - Apply reloads at the server default and a Save writes the blank over the - remembered count. - - The seed is deliberately narrow: storage is read only on a fresh store or a - model change, never on a steady poll, and the value is adopted only when the - server already runs that exact count, which proves it is this model's own. - """ + """The control is never seeded from the status echo, so a model running on a remembered + override shows a BLANK slot control after a browser reload or a tab move to another + GGUF.""" src = _read("features/chat/lib/apply-inference-status-to-store.ts") status = " ".join(src.split()) assert ( @@ -846,16 +823,11 @@ def test_hydration_restores_a_remembered_slot_override(): def test_failed_switch_rollback_restores_the_slot_intent_not_the_resolved_count(): - """`loadedNParallel` holds a RESOLVED count even for a load that sent no - slots (the echo falls back to the server-wide default), so it is the right - value to re-send when recreating the previous server and the wrong one to put - back in the control: it turns "follow the server default" into an explicit - override that a later Save or preset capture pins. The outer catch only - repairs that for a staged config, so a plain string pick keeps the phantom. - - The intent comes from the picker's own pre-switch snapshot when there is one: - chat-page pre-applies the TARGET's config before calling selectModel, so the - live control describes the outgoing model only for a bare pick.""" + """`loadedNParallel` holds a RESOLVED count even for a load that sent no slots (the + echo falls back to the server-wide default), so it is the right value to re-send + when recreating the previous server and the wrong one to put back in the control: it + turns "follow the server default" into an explicit override that a later Save or + preset capture pins.""" runtime = " ".join(_read("features/chat/hooks/use-chat-model-runtime.ts").split()) assert ( 'const previousNParallel = typeof selection !== "string" && ' @@ -879,12 +851,9 @@ def test_failed_switch_rollback_restores_the_slot_intent_not_the_resolved_count( def test_vulkan_inference_devices_are_the_pickable_set(): - """GGUF loads run through llama-server, so on a Vulkan build the picker must - offer the inference inventory (ggml ordinals, the space `--device Vulkan` - pins) rather than the torch view, which can miss cards llama-server drives. - The XPU ban must not apply there: it is about torch-xpu ordinals no - applicator speaks, and a Vulkan pick does not use them. - """ + """GGUF loads run through llama-server, so on a Vulkan build the picker must offer the + inference inventory (ggml ordinals, the space `--device Vulkan` pins) rather than + the torch view, which can miss cards llama-server drives.""" src = " ".join(_read("hooks/use-gpu-info.ts").split()) # The Vulkan inventory is consulted first, and only when it has devices. assert ( @@ -897,3 +866,565 @@ def test_vulkan_inference_devices_are_the_pickable_set(): # The torch fallback keeps its physical-only gate and the XPU ban. assert 'data?.device_backend !== "xpu" &&' in src assert 'physicalIndex: pinnableBackend && d.index_kind === "physical",' in src + + +def test_only_gguf_configs_are_mirrored_to_the_server(): + """The server override map is read by the OpenAI-compatible auto-switch, and its + resolver indexes GGUFs only.""" + src = " ".join(_read("features/model-picker/components/model-config-page.tsx").split()) + assert ( + "if ( !saveFailed && (target.apiLoadable ?? target.isGguf) && !nativePathToken ) " + "{ syncModelOverride(" in src + ) + # The local save is not behind the same gate. + assert "if (remember) { saveFailed = !savePerModelConfig(" in src + + +def test_a_native_leased_gguf_is_not_mirrored_to_the_server(): + """A dropped or file-picked GGUF loads through a signed native-path lease, and + /api/inference/status reports model_identifier as null for it, so the checkpoint the + browser keys settings by is the bare file name the backend echoes back.""" + page = " ".join(_read("features/model-picker/components/model-config-page.tsx").split()) + assert "&& !nativePathToken ) { syncModelOverride(" in page + assert ( + "const nativePathToken = target.meta.nativePathToken ?? " + "(isActiveModel ? activeNativePathToken : null);" in page + ), "the token this gate reads" + # The one-time backfill has no token to read, so it goes by the identity shape. + backfill = " ".join(_read("features/model-picker/api/migrate-model-overrides.ts").split()) + assert "!isNativeFileLabel(entry.modelId) &&" in backfill + identity = " ".join(_read("features/hub/lib/model-identity.ts").split()) + assert "export function isNativeFileLabel(" in identity + # A bare file name: no separator, and the .gguf the resolver's keys never carry. + assert "const NATIVE_FILE_LABEL_RE = /^[^/\\\\]+\\.gguf$/i;" in identity + + backend = WORKDIR / "studio" / "backend" + inference = (backend / "routes" / "inference.py").read_text(encoding = "utf-8") + assert ( + "model_identifier = None if _native_grant_backed else _model_id" in inference + ), "why the checkpoint is only a display name" + models = (backend / "routes" / "models.py").read_text(encoding = "utf-8") + assert "display_name = gguf_file.stem," in models, "why the name is never an index key" + + +def test_evicted_local_configs_drop_their_server_overrides(): + """savePerModelConfig evicts older models when the map exceeds its budget.""" + src = " ".join(_read("features/model-picker/components/model-config-page.tsx").split()) + assert "const evicted: { modelId: string; ggufVariant: string | null }[] = [];" in src + assert ( + "for (const dropped of evicted) { syncModelOverride(dropped.modelId, " + "dropped.ggufVariant, null, { keepLaunchFlags: true, }); }" in src + ) + api = " ".join(_read("features/model-picker/api/model-overrides.ts").split()) + assert "keepLaunchFlags?: boolean;" in api + # remove=false with no fields: the route re-supplies the stored flags and drops + # the row outright once nothing server-owned is left in it. + assert "remove: config === null && !options?.keepLaunchFlags," in api + assert ( + "...(config === null && !options?.keepLaunchFlags ? { llama_extra_args: [] } : {})," + in api.replace("// biome-ignore lint/style/useNamingConvention: API schema ", "") + ) + route = (WORKDIR / "studio" / "backend" / "routes" / "settings.py").read_text(encoding = "utf-8") + assert 'requested_extra_args = stored.get("llama_extra_args")' in route, "the rule this mirrors" + + # The eviction path must report what it dropped, decoded back into a model id + # and variant rather than the normalized storage key. + store = " ".join(_read("features/model-picker/model-config/per-model-config.ts").split()) + assert "evicted?: { modelId: string; ggufVariant: string | null }[]" in store + assert "modelIdFromStorageKey(" in store and "ggufVariantFromStorageKey(" in store + + +def test_backfill_compares_server_keys_by_normalized_identity(): + """app_settings has no schema version, so an install predating identity normalization + holds rows keyed by whatever id was typed, e.g.""" + src = " ".join(_read("features/model-picker/api/migrate-model-overrides.ts").split()) + assert "function normalizedOverrideKey(" in src + # Folded on both sides: the older `id::variant` local keys are not. + assert "known.set(normalizedOverrideKey(storedKey), storedEntry);" in src + assert "const stored = known.get(key);" in src + # A quant-aware split, so a Windows drive letter is not read as a separator. + assert "const split = splitQuantSuffix(key);" in src + # Repo ids fold and POSIX paths do not, which is what these do. + assert "normalizeModelIdentity(" in src and "normalizeGgufVariantIdentity(" in src + + +def test_monitor_stats_exclude_model_lifecycle_rows(): + """A load, unload or download is recorded as a monitor entry but is not an HTTP call.""" + src = " ".join(_read("features/api-monitor/use-api-monitor.ts").split()) + assert 'if (entry.kind === "lifecycle") { continue; }' in src + # "Requests" is a request count too, so it cannot stay entries.length. + assert "total: requests," in src + assert "total: entries.length" not in src + + backend = (WORKDIR / "studio" / "backend" / "core" / "inference" / "api_monitor.py").read_text( + encoding = "utf-8" + ) + assert 'entry.kind != "lifecycle"' in backend, "the rule this mirrors" + + +def test_api_reach_copy_is_limited_to_gguf_models(): + """The Hub opens this page for every downloaded model, but ModelConfigPage mirrors + settings to the server only when target.isGguf, because API auto-switch indexes + GGUFs only.""" + src = " ".join(_read("features/hub/catalog/hub-model-settings-view.tsx").split()) + assert "{(target.apiLoadable ?? target.isGguf)" in src + assert "Saved settings apply everywhere Studio loads this model." in src + + +def test_backfill_includes_a_standalone_gguf_with_no_variant(): + """A standalone .gguf picked directly has no quant to choose between, so it is stored + with a null variant.""" + src = " ".join(_read("features/model-picker/api/migrate-model-overrides.ts").split()) + assert 'entry.modelId.toLowerCase().endsWith(".gguf")' in src + # Still excluded for safetensors, which auto-switch does not resolve. + assert "entry.ggufVariant != null ||" in src + + +def test_monitor_overlay_does_not_pull_in_the_lazy_page(): + """The overlay is mounted from __root.tsx, so a static import of the page for two label + helpers drags the whole 900-line page and its dependency graph into the eagerly + loaded bundle and undoes the route's lazyRouteComponent.""" + overlay = _read("features/api-monitor/api-monitor-overlay.tsx") + assert 'from "./lifecycle"' in overlay + assert "api-monitor-page" not in overlay, "the overlay must not reach the page" + # The helpers live in their own module, not re-exported through the page. + shared = _read("features/api-monitor/lifecycle.ts") + assert "export function isLifecycleEntry(" in shared + assert "export function lifecycleLabel(" in shared + + +def test_override_writes_are_ordered_per_model(): + """Two saves for one model, or a save racing the one-time backfill, started independent + requests with no sequencing, so the older response could commit last and resurrect + the entry the newer one meant to replace.""" + src = " ".join(_read("features/model-picker/api/model-overrides.ts").split()) + assert "const writesByKey = new Map>();" in src + # Keyed by the same override key the server stores under. + assert ( + "const key = modelOverrideKey( normalizeModelIdentity(modelId), normalizeGgufVariantIdentity(ggufVariant), );" + in src + ) + # Chained on the settled tail, so one failed write cannot cancel the next. + assert "previous .catch(() => {}) .then(() => sendModelOverride(" in src + # Only the last writer clears the slot, or a queue still building loses order. + assert "if (writesByKey.get(key) === write) { writesByKey.delete(key); }" in src + + +def test_backfill_skips_future_schema_local_records(): + """loadPerModelConfig refuses to apply a record written by a newer Studio and eviction + refuses to drop one, because this client cannot interpret that schema.""" + src = " ".join(_read("features/model-picker/model-config/per-model-config.ts").split()) + listing = src[src.index("export function listPerModelConfigs()") :] + assert "storedConfigVersion(raw) > STORAGE_SCHEMA_VERSION" in listing[:900] + + +def test_detail_settings_need_a_resolved_quant(): + """The on-device card passes a null variant while its own lookup is pending or after it + failed.""" + src = " ".join(_read("features/hub/hub-page.tsx").split()) + # `variant`, not the argument: a derived quant may have been replaced by the + # resident one first, and the guard has to judge what will actually be saved. + guard = "if (!variant && selectedModel.isGguf && selectedModel.requiresVariant) {" + assert guard in src + assert src.count("Couldn't determine which quant to configure.") == 2 + + +def test_a_failed_detail_fetch_is_retried(): + """A terminal row's updated_at never advances and selectedIsMissing stays true, so a + fetch that failed had nothing left to re-run the effect and the payload stayed + unavailable until another row was selected.""" + src = " ".join(_read("features/api-monitor/api-monitor-page.tsx").split()) + assert "const DETAIL_FETCH_ATTEMPTS = 3;" in src + assert "const detailInFlight = selectedId_ != null && loadingDetails.has(selectedId_);" in src + assert "if (attemptsRef.current.count >= DETAIL_FETCH_ATTEMPTS) { return; }" in src + + +def test_ollama_models_are_not_advertised_as_api_loadable(): + """local_model_resolver skips Ollama's scanner, so an Ollama GGUF is never in the + auto-switch index and no OpenAI request can resolve it.""" + types_src = " ".join(_read("features/model-picker/components/model-selector/types.ts").split()) + assert "apiLoadable?: boolean;" in types_src + hub = " ".join(_read("features/hub/hub-page.tsx").split()) + assert "row.source !== LOCAL_MODEL_SOURCE.OLLAMA" in hub + assert "apiLoadable:" in hub + backend = ( + WORKDIR / "studio" / "backend" / "core" / "inference" / "local_model_resolver.py" + ).read_text(encoding = "utf-8") + assert ( + "Ollama's\n scanner is skipped" in backend or "scanner is skipped" in backend + ), "the rule this mirrors" + + +def test_cached_repo_settings_are_keyed_by_the_repo_id(): + """A repo cached outside the active HF cache reports load_id = the snapshot path + (hub/services/cache_inventory.py), while the chat picker and the auto-switch index + key it by repo_id.""" + config_page = " ".join(_read("features/model-picker/components/model-config-page.tsx").split()) + assert "const configId = target.configId ?? target.id;" in config_page + for call in ( + "resolveInitialConfig(configId, target.ggufVariant)", + "savePerModelConfig( configId, target.ggufVariant,", + "deletePerModelConfig(configId, target.ggufVariant)", + "syncModelOverride( configId, target.ggufVariant,", + ): + assert call in config_page, call + # The probes have to open the model, so they keep the load id. + assert "useDefaultChatTemplate( target.id," in config_page + assert "model_path: target.id," in config_page + + hub = " ".join(_read("features/hub/hub-page.tsx").split()) + assert 'if (kind !== "cache" && resource.source !== "hub_cache") {' in hub + assert "return resource.repoId ?? resource.runId;" in hub + # Both openers and the Hub's own load resolve through it. + assert hub.count("modelConfigIdentity(") == 3 + assert 'const configId = row.kind === "cache" ? row.repoId : id;' in hub + assert hub.count("configId,") >= 2 + + backend = ( + WORKDIR / "studio" / "backend" / "hub" / "tests" / "test_model_services.py" + ).read_text(encoding = "utf-8") + assert 'fields["load_id"] == str(snapshot)' in backend, "the rule this mirrors" + + +def test_backfill_splits_a_quant_suffix_the_way_the_backend_does(): + """The backfill compared server keys under an identity taken by splitting on the last + colon, so a Windows drive letter and an ordinary colon inside a POSIX filename were + read as quant separators: `/models/foo:Bar.gguf` and `/models/foo:bar.gguf` folded + to one key, and whichever was already on the server made the other look migrated.""" + identity = " ".join(_read("features/model-picker/model-config/model-identity.ts").split()) + assert "export function splitQuantSuffix(" in identity + # The two rules that keep a path out: no separator in the tail, and a head + # that is not a .gguf cannot carry a free-form label. + assert 'if (tail.includes("/") || tail.includes("\\\\"))' in identity + assert 'if (!head.toLowerCase().endsWith(".gguf")) { return null; }' in identity + # A .gguf head is not enough on its own: the suffix has to be the label the + # scanner derives from that filename, or a name that itself contains ".gguf:" + # ("/models/llama.gguf:Bar.gguf" and its lowercase sibling, two real POSIX + # files) folds onto one key and one file's settings never migrate. + assert "tail.toLowerCase() === ggufQuantLabel(filename).toLowerCase()" in identity + + migrate = " ".join(_read("features/model-picker/api/migrate-model-overrides.ts").split()) + assert "const split = splitQuantSuffix(key);" in migrate + assert 'key.lastIndexOf(":")' not in migrate, "the unconditional split is gone" + + backend = ( + WORKDIR / "studio" / "backend" / "utils" / "openai_auto_switch_settings.py" + ).read_text(encoding = "utf-8") + assert "def split_quant_suffix(" in backend, "the rule this mirrors" + assert "_BPW_SUFFIX" in backend and "bpw" in identity + assert "extract_quant_label(filename).casefold()" in backend, "the label rule" + # Both sides accept the same quant vocabulary; the regex lives with the loader. + quants = (WORKDIR / "studio" / "backend" / "core" / "inference" / "llama_cpp.py").read_text( + encoding = "utf-8" + ) + for token in ("MXFP", "IQ", "TQ", "BF16", "F16", "F32"): + assert token in quants and token in identity, token + # The label helpers the .gguf branch leans on are ported too, shard suffix and + # float-precision fallback included, or the two sides label a filename apart. + gguf = (WORKDIR / "studio" / "backend" / "hub" / "utils" / "gguf.py").read_text( + encoding = "utf-8" + ) + assert "def extract_quant_label(" in gguf and "def _gguf_stem(" in gguf + assert "function ggufQuantLabel(" in identity and "function ggufStem(" in identity + assert "_GGUF_SPLIT_SUFFIX_RE" in gguf and "GGUF_SPLIT_SUFFIX" in identity + assert "_FLOAT_PRECISION_QUANTS" in gguf and "FLOAT_PRECISION_QUANTS" in identity + # The executable half of this contract, checked case by case against the + # answers split_quant_suffix gives. + assert (WORKDIR / "studio" / "frontend" / "tests" / "model-identity.test.ts").is_file() + + +def test_the_detail_card_also_gates_ollama_out_of_the_api_promise(): + """Settings opens from two places in the Hub.""" + hub = " ".join(_read("features/hub/hub-page.tsx").split()) + assert hub.count("LOCAL_MODEL_SOURCE.OLLAMA") == 2 + assert "selectedModel.localSource !== LOCAL_MODEL_SOURCE.OLLAMA" in hub + assert "row.source !== LOCAL_MODEL_SOURCE.OLLAMA" in hub + + +def test_the_settings_page_judges_the_config_storage_actually_keeps(): + """savePerModelConfig normalizes before deciding, and the runtime hands this page + Speculative Decoding "auto", which canonicalizes to null.""" + src = " ".join(_read("features/model-picker/components/model-config-page.tsx").split()) + assert ( + "const normalizedRuntimeConfig = normalizePerModelConfig( effectiveRuntimeConfig, );" in src + ) + assert "const defaultConfig = isDefaultConfig(normalizedRuntimeConfig);" in src + # The same object goes to storage and to the server, or they disagree again. + assert "target.ggufVariant, normalizedRuntimeConfig, evicted," in src + assert "remember ? normalizedRuntimeConfig : null," in src + assert "isDefaultConfig(effectiveRuntimeConfig)" not in src + + store = " ".join(_read("features/model-picker/model-config/per-model-config.ts").split()) + assert "export function normalizePerModelConfig(" in store + assert "const normalized = normalize(config);" in store + assert "if (isDefaultConfig(normalized)) {" in store, "the rule this mirrors" + + +def test_the_chat_picker_marks_ollama_targets_unloadable_by_the_api(): + """A settings target opened from the Chat model picker carried no apiLoadable, so the + `??""" + picker = " ".join(_read("features/model-picker/components/model-selector.tsx").split()) + assert "apiLoadable: isGguf && !isOllamaLinkPath(id)," in picker + sidebar = " ".join(_read("features/model-picker/components/sidebar-model-config.tsx").split()) + assert "apiLoadable: isGguf && !isOllamaLinkPath(modelId)," in sidebar + # The same classification gates the one-time backfill, or a config saved before + # the upgrade still reaches the server on the next start. + backfill = " ".join(_read("features/model-picker/api/migrate-model-overrides.ts").split()) + assert "!isOllamaLinkPath(entry.modelId) &&" in backfill + + identity = _read("features/hub/lib/model-identity.ts") + assert 'new Set([".studio_links", "ollama_links"])' in identity + resolver = ( + WORKDIR / "studio" / "backend" / "core" / "inference" / "local_model_resolver.py" + ).read_text(encoding = "utf-8") + assert 'seg in (".studio_links", "ollama_links")' in resolver, "the rule this mirrors" + + +def test_the_backfill_fills_in_fields_rather_than_skipping_known_keys(): + """The backfill reads the override map once and then writes each model in turn, so a + save by another tab during that pass was overwritten by this browser's older + localStorage copy.""" + backfill = " ".join(_read("features/model-picker/api/migrate-model-overrides.ts").split()) + assert "{ fillAbsentFields: true }," in backfill + # Key presence alone is not "done": what the server lacks decides. + assert "const stored = known.get(key);" in backfill + assert ( + "if (stored && absentFields(stored, current.config).length === 0) { continue; }" in backfill + ) + assert "const fields = Object.keys(toApiOverride(config));" in backfill + assert "return fields.filter((field) => !(field in stored));" in backfill + + api = " ".join(_read("features/model-picker/api/model-overrides.ts").split()) + assert "fillAbsentFields?: boolean;" in api + assert "options?.fillAbsentFields ? { fill_absent_fields: true } : {}" in api.replace( + "// biome-ignore lint/style/useNamingConvention: API schema ", "" + ) + # Ordinary saves must stay unconditional, or a settings edit would never land. + assert "syncModelOverride" in api and "fill_absent_fields: true" in api + + route = (WORKDIR / "studio" / "backend" / "routes" / "settings.py").read_text(encoding = "utf-8") + assert "fill_absent_fields: bool = False" in route, "the rule this mirrors" + assert "fill_absent_fields = payload.fill_absent_fields," in route + # A write mode must not leak into the saved fields, or "only model_id means + # forget this model" stops working. + assert '"remove", "fill_absent_fields"' in route + + # The merge is the server's, under the write's own transaction: a client-side + # read-modify-write would reopen the race the conditional write closed. + db = (WORKDIR / "studio" / "backend" / "storage" / "studio_db.py").read_text(encoding = "utf-8") + assert "merged = {**entry_value, **stored}" in db + assert "BEGIN IMMEDIATE" in db + + +def test_the_hub_settings_page_matches_a_resident_path_loaded_model(): + """A GGUF loaded from an inactive HF cache or straight off disk loads by path, but + /status reports the clean public id, so comparing it to settingsTarget.id said "not + loaded" and the page showed saved or default values instead of the live launch + config.""" + hub = " ".join(_read("features/hub/hub-page.tsx").split()) + assert ( + "residentModelIdMatches( activeCheckpoint, settingsTarget.id, settingsTarget.configId, )" + in hub + ) + assert "loadedConfig={settingsTargetIsResident ? activeModelConfig : null}" in hub + assert "settingsTargetIsResident ? activeGgufContextLength : null" in hub + # The loadable identifier, as every other status reader records it. + assert "checkpointId: resolveInferenceCheckpointId(status)," in hub + assert "setCheckpoint(status.active_model" not in hub + chat = " ".join(_read("features/chat/lib/apply-inference-status-to-store.ts").split()) + assert "return status.model_identifier ?? status.active_model;" in chat, "the rule this mirrors" + # The alias is the backend's own public id rule, not a private heuristic. + identity = _read("features/hub/lib/model-identity.ts") + assert "export function publicModelId(" in identity + assert "models--" in identity and "snapshots" in identity + # Only a namespaced repo id names one model; a filename or directory stem does + # not, so it must never stand in for the loaded model's identity. + assert 'return publicId.includes("/") && modelIdsMatch(active, publicId);' in identity + backend = (WORKDIR / "studio" / "backend" / "core" / "inference" / "model_ids.py").read_text( + encoding = "utf-8" + ) + assert "def public_model_id(" in backend, "the rule this mirrors" + + +def test_the_hub_hydrates_the_live_settings_before_it_offers_them(): + """The Hub builds activeModelConfig out of the chat runtime store, and landing straight + on /hub is the one entry point where nothing has applied /api/inference/status yet: + useChatModelRuntime has no mount sync and the chat page is a different route.""" + hub = " ".join(_read("features/hub/hub-page.tsx").split()) + assert "adoptResidentModelStatus(" in hub + assert "applyActiveModelStatusToStore(status, {" in hub + assert "previousCheckpoint: previous.checkpoint ?? undefined," in hub + assert "previousGgufVariant: previous.ggufVariant," in hub + assert "modelLoading: store.modelLoading," in hub + + adopt = " ".join(_read("features/hub/lib/adopt-inference-status.ts").split()) + # Unconditional: a persisted checkpoint rehydrates from localStorage on its + # own, carrying none of the fields that say how the model was launched. + assert "actions.applyStatus(previous); return true;" in adopt + # Never fight the load that owns the store, and never describe an external + # provider's model with the resident GGUF's launch settings. + assert "if (state.checkpointIsExternal) { return false; }" in adopt + assert "if (state.modelLoading) { return false; }" in adopt + + # Same call the chat runtime's own refresh makes, which is the rule this mirrors. + runtime = " ".join(_read("features/chat/hooks/use-chat-model-runtime.ts").split()) + assert "applyActiveModelStatusToStore(statusRes, {" in runtime + + +def test_the_hub_settings_editor_reseeds_when_the_live_config_lands(): + """ModelConfigPage reads loadedConfig in a useState initializer, so it seeds once per + mounted instance.""" + view = " ".join(_read("features/hub/catalog/hub-model-settings-view.tsx").split()) + assert "key={modelConfigInstanceKey( target.id, target.ggufVariant, loadedConfig, )}" in view + # Same key the sidebar entry uses; that parity is the point. + sidebar = " ".join(_read("features/model-picker/components/sidebar-model-config.tsx").split()) + assert "key={modelConfigInstanceKey(modelId, settingsGgufVariant, loadedConfig)}" in sidebar + + signature = " ".join(_read("features/model-picker/model-config/config-signature.ts").split()) + # "No live config yet" has to be its own value: that transition is exactly + # the one that must remount. + assert 'if (!config) { return "none"; }' in signature + for field in ( + "config.customContextLength", + "config.maxSeqLength", + "config.kvCacheDtype", + "config.speculativeType", + "config.specDraftNMax", + "config.tensorParallel", + "config.chatTemplateOverride", + ): + assert field in signature, field + + page = " ".join(_read("features/model-picker/components/model-config-page.tsx").split()) + assert "const [initial] = useState(resolveInitial);" in page, "the rule this mirrors" + + +def test_a_standalone_gguf_has_one_settings_key(): + """The inventory labels a single scanned .gguf from its filename, so the Hub row menu + keyed its settings to `:Q4_K_M` while the Chat picker, the detail card and the + backfill all use the bare path: two surfaces, two configs.""" + hub = " ".join(_read("features/hub/hub-page.tsx").split()) + assert "let ggufVariant = settingsGgufVariantForRow(row);" in hub + assert "row.formatVariant" not in hub, "the row's raw label is not a settings key" + + helper = " ".join(_read("features/hub/inventory/settings-identity.ts").split()) + assert 'row.kind === "local" && row.path.toLowerCase().endsWith(".gguf")' in helper + + common = ( + WORKDIR / "studio" / "backend" / "hub" / "services" / "models" / "common.py" + ).read_text(encoding = "utf-8") + # The rule this mirrors: a variant is derived only for a single scanned file. + assert "extract_quant_label(gguf_files[0].name)" in common + assert "if scan_path.is_file() and len(gguf_files) == 1" in common + + +def test_a_standalone_gguf_is_resident_despite_its_derived_quant(): + """A loose .gguf keys its settings by the bare path with no variant, but the loader + derives one from the filename (llama_cpp sets _hf_variant from _extract_quant_label) + and /status reports it, so an equality between the two could never hold and the + settings page withheld the live launch config from the very file that was loaded.""" + hub = " ".join(_read("features/hub/hub-page.tsx").split()) + assert "const settingsTargetIsStandaloneFile =" in hub + assert 'settingsTarget.id.toLowerCase().endsWith(".gguf")' in hub + assert ( + "(settingsTargetIsStandaloneFile || ggufVariantsMatch(activeGgufVariant, settingsTarget.ggufVariant))" + in hub + ) + backend = (WORKDIR / "studio" / "backend" / "core" / "inference" / "llama_cpp.py").read_text( + encoding = "utf-8" + ) + assert ( + "self._hf_variant = _extract_quant_label(gguf_path)" in backend + ), "the derived label this accounts for" + + +def test_a_standalone_gguf_has_one_settings_identity_everywhere(): + """A loose .gguf has no quant to choose between, but llama_cpp falls back to + _extract_quant_label(gguf_path) when a load names no variant, and /status echoes + that as gguf_variant.""" + sidebar = " ".join(_read("features/model-picker/components/sidebar-model-config.tsx").split()) + # Nulled for the settings identity, and used for every field that keys it. + assert ( + "const settingsGgufVariant = isStandaloneGgufPath(modelId) ? null : ggufVariant;" in sidebar + ) + assert "ggufVariant: settingsGgufVariant," in sidebar + assert "ggufVariant: settingsGgufVariant ?? undefined," in sidebar + # The label still shows the quant; only the identity drops it. + assert "displayName: ggufVariant ? `${leaf} ยท ${ggufVariant}` : leaf," in sidebar + + # One rule, one definition: the Hub row applies the same test. + identity = _read("features/hub/lib/model-identity.ts") + assert "export function isStandaloneGgufPath(" in identity + assert 'modelId.toLowerCase().endsWith(".gguf")' in identity + row_identity = _read("features/hub/inventory/settings-identity.ts") + assert 'row.path.toLowerCase().endsWith(".gguf")' in row_identity + + # The precedence that makes the bare path the one that wins. + route = (WORKDIR / "studio" / "backend" / "routes" / "inference.py").read_text( + encoding = "utf-8", + ) + assert 'f"{target_id}:{file_variant}" if file_variant else None,' in route + bare = route.index("\n target_id,\n") + labelled = route.index('f"{target_id}:{file_variant}"') + assert bare < labelled, "the bare path must be read before the filename label" + + +def test_monitor_unload_clears_only_the_model_it_freed(): + """Unload targets the resident local model from /status, but the store may hold either + spelling: status reports the concrete load path while the store can hold the + advertised repo id.""" + page = " ".join(_read("features/api-monitor/api-monitor-page.tsx").split()) + assert "const unloadedAliases = [checkpoint, status.active_model];" in page + assert "!isExternalModelId(selected)" in page + assert "unloadedAliases.some((alias) => modelIdsMatch(selected, alias))" in page + assert "store.clearCheckpoint();" in page + + +def test_settings_open_reads_status_before_resolving_the_quant(): + """A cache row carries no quant, so opening its settings resolves one from the store's + active variant.""" + page = " ".join(_read("features/hub/hub-page.tsx").split()) + assert "const refreshResidentModelStatus = useCallback((): Promise => {" in page + assert "const [res] = await Promise.all([ listGgufVariants(" in page + assert "refreshResidentModelStatus(), ]);" in page + + +def test_a_local_quant_folder_resolves_its_variants_by_path(): + """A local row carries a repo id only inside the HF cache, so a plain folder of + quants has none while still being marked as needing one, and the row menu's + Settings could then only reach the error toast.""" + hub = " ".join(_read("features/hub/hub-page.tsx").split()) + assert ( + 'const repoId = row.kind === "cache" ? row.repoId : (row.repoId ?? row.path ?? null);' + in hub + ) + # The on-device card already lists by path for the same rows; this is the + # request it makes, so both surfaces choose from one set of quants. + card = " ".join(_read("features/hub/catalog/local-on-device-card.tsx").split()) + assert "repoId: modelId, hfToken, preferLocalCache: true, localPath: localGgufPath," in card + # The backend takes a path in the repo_id position and scans it, before the + # repo-id validation that would otherwise 400 on a path. + variants = ( + WORKDIR / "studio" / "backend" / "hub" / "services" / "models" / "gguf_variants.py" + ).read_text(encoding = "utf-8") + scan = variants.split("if is_local_path(repo_id):", 1) + assert len(scan) == 2, "the local-path branch this leans on" + assert "_is_valid_repo_id(repo_id)" in scan[1], "the branch has to come first" + + +def test_cached_repo_settings_key_follows_the_row_not_the_view(): + """A repo in an inactive HF cache loads by snapshot path while its settings are keyed + by repo id.""" + page = " ".join(_read("features/hub/hub-page.tsx").split()) + assert 'if (kind !== "cache" && resource.source !== "hub_cache") {' in page + + +def test_detail_settings_defers_a_derived_quant_to_a_fresh_status_read(): + """The on-device card resolves the quant it shows from the store's active variant, and + nothing re-reads status while the window keeps focus, so an API-driven switch leaves + that quant naming the model it displaced.""" + page = " ".join(_read("features/hub/hub-page.tsx").split()) + card = " ".join(_read("features/hub/catalog/local-on-device-card.tsx").split()) + assert "if (!quantIsUserPicked) { await refreshResidentModelStatus();" in page + assert "variant = settled.activeGgufVariant;" in page + assert "onOpenSettings(selectedQuant ?? null, quantIsUserPicked)" in card diff --git a/tests/studio/test_settings_compact_overflow_contract.py b/tests/studio/test_settings_compact_overflow_contract.py index b8547975d2..58a522b23b 100644 --- a/tests/studio/test_settings_compact_overflow_contract.py +++ b/tests/studio/test_settings_compact_overflow_contract.py @@ -5,7 +5,10 @@ from pathlib import Path REPO = Path(__file__).resolve().parents[2] SETTINGS_DIALOG = REPO / "studio/frontend/src/features/settings/settings-dialog.tsx" -API_MONITOR = REPO / "studio/frontend/src/features/settings/components/api-monitor-console.tsx" +# The monitor moved onto its own page and Settings links to it; the shrink +# contract still applies to both surfaces. +API_MONITOR_PAGE = REPO / "studio/frontend/src/features/api-monitor/api-monitor-page.tsx" +MONITOR_LINK = REPO / "studio/frontend/src/features/settings/components/monitor-link.tsx" GENERAL_TAB = REPO / "studio/frontend/src/features/settings/tabs/general-tab.tsx" @@ -16,15 +19,22 @@ def test_dialog_content_can_shrink_inside_the_dialog_grid(): def test_api_monitor_entries_and_expanded_text_can_shrink(): - source = API_MONITOR.read_text(encoding = "utf-8") - assert ( - '
' in source - ) - assert ( - '
' - in source - ) - assert source.count('className="max-h-44 overflow-auto whitespace-pre-wrap break-words') == 2 + source = API_MONITOR_PAGE.read_text(encoding = "utf-8") + # Rows and the detail pane sit in flex parents, so they need min-w-0 or a long + # model id pushes the layout wider than the viewport. + assert '"flex w-full min-w-0 flex-col gap-1 border-b border-border/50' in source + assert '
' in source + # Prompt and reply are unbounded user text: height-capped, scrollable, wrapped. + assert "max-h-72 overflow-auto whitespace-pre-wrap break-words rounded-lg bg-muted/50" in source + # A model id or path has no spaces to wrap on, so it needs break-all. + assert 'className="min-w-0 break-all font-mono' in source + + +def test_settings_monitor_link_can_shrink(): + source = MONITOR_LINK.read_text(encoding = "utf-8") + assert "flex w-full min-w-0 items-center gap-3" in source + # The summary line carries a model id, so it truncates instead of widening. + assert '' in source def test_embedding_model_controls_stack_on_the_narrowest_viewports(): diff --git a/tests/studio/test_usage_examples_model_source_contract.py b/tests/studio/test_usage_examples_model_source_contract.py index a0c0d04020..6a594edeb1 100644 --- a/tests/studio/test_usage_examples_model_source_contract.py +++ b/tests/studio/test_usage_examples_model_source_contract.py @@ -132,33 +132,45 @@ def test_usage_examples_has_no_duplicate_auto_switch_control(): assert "" in tab -API_MONITOR_TSX = SETTINGS / "components/api-monitor-console.tsx" +# The monitor moved onto its own page; Settings keeps configuration and links +# across. These contracts follow the behaviour, not the old file. +API_MONITOR_TSX = REPO / "studio/frontend/src/features/api-monitor/api-monitor-page.tsx" +# The lifecycle labels live in their own module: the overlay is mounted from +# __root.tsx, so importing them from the page pulled it into the eager bundle. +API_MONITOR_LIFECYCLE_TS = REPO / "studio/frontend/src/features/api-monitor/lifecycle.ts" +MONITOR_LINK_TSX = SETTINGS / "components/monitor-link.tsx" -def test_api_monitor_pages_five_at_a_time(): - # The backend retains 50 terminal entries; the console used to dump them all at once. +def test_api_monitor_history_does_not_reorder_under_the_reader(): + # The backend keeps 50 terminal entries and moves one to the front as it + # finishes. The console froze ids while paging; the page pauses the poll instead, + # holding the whole list still while a payload is read. src = API_MONITOR_TSX.read_text(encoding = "utf-8") - assert "const PAGE_SIZE = 5;" in src - assert "ordered.slice(" in src - # Paging back must freeze the id order, or live traffic reorders history under it. - assert "frozenIds" in src - assert "setFrozenIds((prev) => prev ?? entries.map((entry) => entry.id))" in src + assert "paused" in src + assert "setPaused" in src + # Filters and search are what keep 50 rows usable without paging. + assert "filterEntries(" in src + assert "STATUS_FILTERS" in src def test_api_monitor_renders_lifecycle_rows(): src = API_MONITOR_TSX.read_text(encoding = "utf-8") - assert "function LifecycleEntry(" in src - assert 'entry.kind === "lifecycle"' in src + labels = API_MONITOR_LIFECYCLE_TS.read_text(encoding = "utf-8") + assert "export function isLifecycleEntry(" in labels + assert 'entry.kind === "lifecycle"' in labels for label in ("Loading model", "Model loaded", "Model unloaded"): - assert label in src - # Lifecycle rows have no prompt/reply to fetch. - assert "isLifecycle(entry) || !expandedIds.has(entry.id)" in src + assert label in labels + # A lifecycle row has no prompt or reply, so it is not selectable for detail. + assert "if (isLifecycleEntry(entry)) {" in src + assert 'from "./lifecycle"' in src -def test_auto_switch_section_sits_above_the_monitor(): +def test_auto_switch_section_sits_above_the_usage_examples(): tab = API_KEYS_TAB_TSX.read_text(encoding = "utf-8") - assert tab.index("") < tab.index("") - assert tab.index("") < tab.index("") < tab.index("") + assert tab.index("") < tab.index("